initz.h

网页抓取程序

int InitzSersOoi(int argc, char* argv[], OOI_NODE** lexicon, RANKS** rankList)
{
DWORD gtc;
char* sqlQuery;
char* sqlQueryCount;

MYSQL_RES gRes;
MYSQL_RES** tmpRes=NULL;
MYSQL_ROW row;

unsigned int counter;


/* create index */
    gtc = GetTickCount();

/*ricezione dell indice*/
    printf("Connecting to MySQL...");
    if(sqlConnect(glMysqlHost,glMysqlUser,glMysqlPass,glMysqlDB, atoi(glMysqlPort), &glMysql)==0)
	{
		printf( "ERROR\r\nFailed to connect to database: Error: %s\r\n",mysql_error(&glMysql));
		return 0;
	}
    printf("OK\n");

    sqlQuery = malloc(150);
    sqlQueryCount = malloc(150);
    switch(argc)
    {
        /* porta e limite inferiore */
    case 3:
        sprintf(sqlQuery, "SELECT wordID, pageID, position FROM ii WHERE pageid >= %i ",atoi(argv[2]));
        sprintf(sqlQueryCount, "SELECT idpage,rank FROM pagelist WHERE idpage >= %i ",atoi(argv[2]));
        break;
        /* porta e limite inferiore e superiore */
    case 4:
        sprintf(sqlQuery, "SELECT wordID, pageID, position FROM ii WHERE pageid BETWEEN %i AND %i ",atoi(argv[2]) ,atoi(argv[3]) );
        sprintf(sqlQueryCount, "SELECT idpage,rank FROM pagelist WHERE idpage BETWEEN %i AND %i ",atoi(argv[2]) ,atoi(argv[3]) );
        break;
    default:
        sprintf(sqlQuery, "SELECT wordID, pageID, position FROM ii");
        sprintf(sqlQueryCount, "SELECT id,rank FROM pagelist");
        break;
    }

    printf("Sending SQL-Query...");
    /* =============================================================================== */
    /* contatore documenti da indicizzare */
    tmpRes=(MYSQL_RES**)malloc(sizeof(MYSQL_RES));

    my_mysql_query_and_store_results(&glMysql,sqlQueryCount,tmpRes,&gRes);

	printf( "ERROR\r\nFailed to connect to database: Error: %s\r\n",mysql_error(&glMysql));

    FREE(sqlQueryCount);

    nRanks = mysql_affected_rows(&glMysql);
    
    *rankList = InitRankList(nRanks);

    while( (row = mysql_fetch_row(&gRes)) )
	{
		InsertDocRank(*rankList, atoi(row[0]), atoi(row[1]),nRanks);
	}

    /* ordina per page */
    /* ordine crescente */
    insertion_sortRanks(*rankList, nRanks, 0, 0);

    //i=ranksLookForPage(rankList,55,nRanks);

    if(*tmpRes)
	{
		mysql_free_result(*tmpRes);
	}
		
	FREE(tmpRes);


    

    /* =============================================================================== */

    /* =============================================================================== */
    /* riceve l'indice completo e lo mette in memoria */
    tmpRes=(MYSQL_RES**)malloc(sizeof(MYSQL_RES));

    my_mysql_query_and_store_results(&glMysql,sqlQuery,tmpRes,&gRes);
    
    counter = 0;

    FREE(sqlQuery);
    

    printf("OK\n");

    printf("Generating the index...\n");
	while( (row = mysql_fetch_row(&gRes)) )
	{
		UpdateInvertedIndex(lexicon,atoi(row[0]),atoi(row[1]),atoi(row[2]));
        if( (++counter) % 1000 == 0)
            printf("\rElements indexed: %d            ",counter);
	}

    if(*tmpRes)
	{
		mysql_free_result(*tmpRes);
	}
		
	FREE(tmpRes);

	
    /* =============================================================================== */

    printf("\n\nIndex(%d documents / %d words) created in ms: %d\n",nRanks ,number_of_elements , (int)(GetTickCount()-gtc) );
    
    fflush(stdout);

	/************************************************************************************/

return 1;
}


int InitzSersMysql()
{

	bIsMySQLConnected = 0;

	/* nessun mysql server */
	if( glMysqlHost[0] == 0 )
		return 1;

    printf("Connecting to MySQL...");
    if(sqlConnect(glMysqlHost,glMysqlUser,glMysqlPass,glMysqlDB, atoi(glMysqlPort), &glMysql)==0)
	{
		printf( "ERROR\r\nFailed to connect to database: Error: %s\r\n",mysql_error(&glMysql));
		return 0;
	}
    printf("OK\n");

	bIsMySQLConnected = 1;

return 1;
}