core.c

a meta serach engine

			break;
		case 2:
			n = rpp * (page - 1);
			break;
		case 3:
			n = (rpp * (page - 1)) + 1;
			break;
	}
	return itoa(n);
}
// ----------------------------------------------------------------------------






// createHTTPRequest ----------------------------------------------------------
// This  function make the HTTP REQUEST using the given parameters. Then return
// a pointer to it.
// ----------------------------------------------------------------------------
void createHTTPRequest(struct engineWork *e) {
	char *t1, *t2, *t3, *npage, *rpp;																		// temporary buffers
	extern char *query;
	
	npage = enginePageCalc(e->info->npc, e->rpp, e->startpage);										// calculate page number
	rpp = itoa(e->rpp);																							// convert page number into integer
	t1 = strrpl(e->info->query, "%QUERYSTR", query);													// replace the query string
	t2 = strrpl(t1, "%RESULTS", rpp);																		// replace the desidered result number
	t3 = strrpl(t2, "%PAGE", npage);																			// replace the desidered page number
	
	clearOutputBuffer(e);																						// clear the output buffer
	strcpy(e->ob, t3);																							// put maked HTTP REQUEST into buffer

	if (config.verbose) printf("HTTP REQUEST: socket=%d, engine=%s\n%s",e->socket, e->info->host, t3);
	
	free(t1); free(t2); free(t3); free(npage); free(rpp);
}
// ----------------------------------------------------------------------------






// bufferFreeMemory -----------------------------------------------------------
// Calculate free memory left in the engine's input-buffer
// ----------------------------------------------------------------------------
int bufferFreeMemory(struct engineWork *e) {
	return (e->ib + ibsize) - e->ib_free;
}
// ----------------------------------------------------------------------------






// tx -------------------------------------------------------------------------
// If  there  still  is  something to send in the output-buffer, send it, else,
// prepare to receive data
// ----------------------------------------------------------------------------
int tx(struct engineWork *e) {
	short k=strlen(e->ob_next);

	bench(0);
	if (e->ob_next[0]!='\0') {																					// if there is something ready to send
		k=write(e->socket, e->ob_next, k);																	// try to send it on the socket
		
		if (k==-1) return engineError(e, "write");														// if error in WRITE, report and stop
		
		e->ob_next += k;																							// move ptr according to how much sent
		stats.tx_bytes += k;																						// increment sent bytes counter
	
	} else {
		clearInputBuffer(e);																						// prepare INPUT buffer to receive data
		clearParserInformations(e);																			// reset the parse script controller
		setReadInterest(e);																						// set READING interest
	}
	
	if (config.verbose) printf("TX: socket=%d, write=%d\n", e->socket, k);
	stats.tx_time += bench(0);
	return 1;
}
// ----------------------------------------------------------------------------






// requestNewPage -------------------------------------------------------------
// Close current TCP connection with the server, increase the page counter, and
// ask for the new page, opening a new connection with the server.
// ----------------------------------------------------------------------------
int requestNewPage(struct engineWork *e) {
	short k = e->socket;

	e->startpage++;																								// choose new page
	e->socket = createTCPSocket();																			// create new socket
	if (e->socket==-1) return engineError(e, "createTCPSocket");									// if error, report it and stop engine
	
	engineConnect(e);																								// connect the engine to web server
	createHTTPRequest(e);																						// create new request in output buffer
	clearFoundCounter(e);																						// clear FOUND counter
	setWriteInterest(e);																							// set WRITING interest
	
	if (config.socket_trace) printf("SK: engine=%s, old_socket=%d, new_socket=%d\n",e->info->name, k, e->socket);
	return 1;
}
// ----------------------------------------------------------------------------






// rx -------------------------------------------------------------------------
// Receive  as  much  data  is  possible  from  the  server and store it in the
// engine's input-buffer
// ----------------------------------------------------------------------------
int rx(struct engineWork *e) {
	int k=0;
	int free = bufferFreeMemory(e);

	bench(0);
	k = read(e->socket, e->ib_free, free);																	// read as much data is possible
	
	if (k==-1) return engineError(e, "read");																// if error in READ, report it and stop
	
	stats.rx_bytes += k;																							// increment the receive bytes counter
	if (config.dump_to_disk) dumpToDisk(e);																// if requested, dump to disk
	e->ib_free += k;																								// move the FREE-pointer forward

	if (k==0) {																										// if READ returned 0, data finished
		socketClose(e->socket);																					// close the engine's socket
		if (config.socket_trace) printf("SK: engine=%s, closing_socket=%d\n", e->info->name, e->socket);
				
		if (e->startpage < e->endpage)																		// if it's not last page, ask next
			requestNewPage(e);
		else
			setNoInterest(e);																						// if it's the last page, stop reading
	}
		
	if (config.verbose) printf("RX: socket=%d, free=%d, read=%d\n", e->socket, free, k);
			
	e->rxTime += bench(0);																						// save the RX time for engine
	return k;
}
// ----------------------------------------------------------------------------






// decreaseRetry --------------------------------------------------------------
// Scroll  all working-engine's structures and decrease the retry counter after
// a SELECT timeout
// ----------------------------------------------------------------------------
void decreaseRetry (struct engineWork e[], short n) {
	short i;
	for (i=0; i<n; i++) {
		if (e[i].readretry>0) e[i].readretry--;
		if (e[i].writeretry>0) e[i].writeretry--;
	}
}
// ----------------------------------------------------------------------------






// setSocketSet ---------------------------------------------------------------
// After  clearing  the  FD_SET structures, set their bits checking the engines
// retry counters.
// ----------------------------------------------------------------------------
void setSocketSet(struct engineWork e[], short n, fd_set *rset, fd_set *wset) {
	short i;
	
	FD_ZERO(rset); FD_ZERO(wset);
	for (i=0; i<n; i++) {
		if (e[i].readretry>0) {
			FD_SET(e[i].socket, rset);
			continue;
		}
		if (e[i].writeretry>0) {
			FD_SET(e[i].socket, wset);
			continue;
		}
	}
}
// ----------------------------------------------------------------------------






// setTimeout -----------------------------------------------------------------
// Set the TIMEVAL structure to the specified timeout value
// ----------------------------------------------------------------------------
void setTimeout(short ms, struct timeval *t) {
	t->tv_sec = ms / 1000;
	t->tv_usec = ms % 1000;
}
// ----------------------------------------------------------------------------






// maxEngineSocket ------------------------------------------------------------
// Find maximum socket value in the given engineWork array
// ----------------------------------------------------------------------------
short maxEngineSocket(struct engineWork e[], short n) {
	short i, max=0;
	for (i=0; i<n; i++)
		if (max<e[i].socket) max=e[i].socket;
	return max;
}
// ----------------------------------------------------------------------------






// stillAny -------------------------------------------------------------------
// Return TRUE if there is at least one engine still at work
// ----------------------------------------------------------------------------
char stillAny(struct engineWork e[], short n) {
	short i;
	
	for (i=0; i<n; i++)
		if (e[i].readretry>0 || e[i].writeretry>0) return 1;
	return 0;
}
// ----------------------------------------------------------------------------






// engineSetup ----------------------------------------------------------------
// Fill  the  engine  work structure with parameter's data, or with the default
// engine preferences if no data were given
// ----------------------------------------------------------------------------
void engineSetup(char *arg, struct engineWork *e) {
	char *name, *results, *pages, *startpage;

	name			= strsep2(&arg, '=');																		// look for '=' to grab the name
	results		= strsep2(&arg, ',');																		// look for ',' to grab results4page
	pages 		= strsep2(&arg, ',');																		// look for ',' to grab pages number
	startpage	= arg;																							// last num, if exist, is start-page

	e->info = ptrToEngineInformations(name);																// set the pointer to engine infos
	
	if (results!=NULL)	e->rpp = atoi(results);															// if results4page specified, use it
		else					e->rpp = e->info->rpp;															// if not specified, use default
	
	if (startpage!=NULL)	e->startpage = atoi(startpage);												// if start page specified, use it
		else					e->startpage = 1;																	// if not specified, start from first
	
	if (pages!=NULL)		e->endpage = atoi(pages) + e->startpage - 1;								// if number of pages specified, use
		else					e->endpage = e->info->pag + e->startpage - 1;							// if not specified, use default
	
	e->requested = e->rpp * (e->endpage - e->startpage + 1);											// calc number of requested results
	stats.req_result += e->requested;																		// calc total requested results
	e->entry.rank = ((e->startpage-1)*e->rpp) + 1;														// calculate first entry rank
	
	createHTTPRequest(e);
}
// ----------------------------------------------------------------------------






// engineConnect --------------------------------------------------------------
// Connect the engine with it's socket, using default engine informations
// ----------------------------------------------------------------------------
int engineConnect(struct engineWork *e) {
	return socketConnect(e->socket, e->info->sa);
}
// ----------------------------------------------------------------------------





// printEngineInfo ------------------------------------------------------------
// Print to screen the avaiable engine's informations
// ----------------------------------------------------------------------------
void printEngineInfo(short i, struct engineWork e, fd_set rset, fd_set wset) {
	printf("\n");
	printf("=== Engine %d ===\n",i);
	printf("host\t\t: %s\n",e.info->host);
	printf("port\t\t: %d\n",e.info->port);
	printf("rpp\t\t: %d\n", e.rpp);
	printf("startpage\t: %d\n",e.startpage);
	printf("endpage\t\t: %d\n",e.endpage);
	printf("readretry\t: %d\n",e.readretry);
	printf("writeretry\t: %d\n",e.writeretry);
	printf("socket\t\t: %d\n",e.socket);
	printf("wset\t\t: %d\n",FD_ISSET(e.socket, &wset));
	printf("rset\t\t: %d\n",FD_ISSET(e.socket, &rset));
}
// ----------------------------------------------------------------------------