experiment.c

C编写的用来实现search engine的推荐功能

/**
 * experiment.c, get the experiment results
 * zhiyong zhang
 * louisville May 23th
 **/
#include <stdio.h>
#include <string.h>
#include <strings.h>
#include <math.h>
#include <unistd.h>
#include <sys/time.h>

#include "c_http.h"
#include "NonbSocket2.h"
#include "getUrlTermFreqs.h"

#define MAX_TERMS_PER_VECTOR 20
#define ONE_SEC 1000000L
#define TEST_NUM 1000

static char* extraHeaders = NULL;
static REQUEST_METHOD_T method = M_GET;
static int timeout = 100;
static int show1stLine = 0;
static int showAllLine = 0;
static int processResponseBody(int hSocket, char* buf, int size);
static void addTime(const struct timeval* endTime, const struct timeval* startTime, struct timeval* totalTime);

static char search_root[256] = "http://webmining.spd.louisville.edu:8090/cgi-bin/recommendation";
static char search_params[50] = "?query=";
static char markov_bflag[20] = "Markov_Begin";
static char markov_eflag[20] = "Markov_End";
static char content_bflag[20] = "Content_Begin";
static char content_eflag[20] = "Content_End";
static char terms_gdbm[256] = "/home/zhiyong/software/apache2/cgi-bin/terms.gdbm";


static int processResponseBody(int hSocket, char* buf, int size)
{
	int readByte = 0;
	
	while (1) {
		int ret = Receive(hSocket, buf, size, 1);
		switch (ret) {
		case SOCKET_ERROR:
		case SOCKET_CLOSED:
			return readByte;
		case SOCKET_TIMEOUT:
			break;
		default:
			if (show1stLine) {
				buf[ret] = '\0';
				//writeLog(buf);
				//fwrite(buf, 1, ret, stdout);
				show1stLine = 0;
			} else if (showAllLine) {
				//fwrite(buf, 1, ret, stdout);
				buf[ret] = '\0';
				//writeLog(buf);
			}
			readByte += ret;
			break;
		}
	}
}
/*add time  */
static void addTime(const struct timeval* endTime, const struct timeval* startTime, struct timeval* totalTime)
{
	if (endTime->tv_sec == startTime->tv_sec)
	{
		totalTime->tv_usec += (endTime->tv_usec-startTime->tv_usec);
	}
	else if (endTime->tv_usec >= startTime->tv_usec)
	{
		totalTime->tv_sec += (endTime->tv_sec-startTime->tv_sec);
		totalTime->tv_usec += (endTime->tv_usec-startTime->tv_usec);
	}
	else
	{
		totalTime->tv_sec += (endTime->tv_sec-startTime->tv_sec-1);
		totalTime->tv_usec += (ONE_SEC+endTime->tv_usec-startTime->tv_usec);
	}
	
	if (totalTime->tv_usec >= ONE_SEC)
	{
		totalTime->tv_sec += (totalTime->tv_usec / ONE_SEC);
		totalTime->tv_usec %= ONE_SEC;
	}
}

int main(int argc, char** argv)
{
	int i,j,k,iRet;
	char line[1024];
	char buf[20481];
	char para[10240];
	char temp[256];
	HTTP_RESPONSE_HEADER_T headers;
	char request_str[1024];
	char search_query[1024];
	char next_visit[1024];
	int len;
	char *ptr = NULL;
	char *pptr = NULL;
	char *ptr_tmp = NULL;
	FILE *fp_in = NULL;
	FILE *fp_out = NULL;
	int num;
	int match_flag = 0;
	int norec_num = 0;
	double sim;
	static struct timeval start_time, end_time, total_time = {0};
	if(argc < 4)
	{
		fprintf(stdout, "Usage:%s <input> <output> <#len>\n", argv[0]);
		return 0;
	}
	fp_in = fopen(argv[1], "r");
	if(fp_in == NULL)
	{
		fprintf(stdout, "failed to open input file for read.\n");
		return -1;
	} 
	fp_out = fopen(argv[2], "w");
	if(fp_out == NULL)
	{
		fprintf(stdout, "failed to create length two file.\n");
		fclose(fp_in);
		return -1;
	} 
	num = atoi(argv[3]);
	initResponseHeader(&headers);
	headers.processResponseBody = processResponseBody;
	k = 0;
	norec_num = 0;
	while(fgets(line, 1024, fp_in) != NULL)
	{
		bzero(&total_time,sizeof(total_time));
		ptr = strchr(line, '\n');
		if(ptr != NULL) *ptr = '\0';
		ptr = strchr(line, ' ');
		if(num == 3)
		{
			if(ptr != NULL) *ptr = '+';
			pptr = ptr + 1;
			ptr = strchr(pptr, ' ');
		}
		if(ptr != NULL) *ptr = '\0';
		strcpy(search_query, line);
		pptr = ptr + 1;
		ptr = strchr(pptr, ' ');
		if(ptr != NULL) *ptr = '\0';
		strcpy(next_visit, pptr);
		sprintf(request_str, "%s%s%s", search_root, search_params, search_query);
		gettimeofday(&start_time, NULL);
		len = HTTPRequest(request_str, method, extraHeaders, &headers, timeout, buf, 20481);
		gettimeofday(&end_time, NULL);
		if(len <= 0) 
		{
			fprintf(stdout, "HTTPRequest error:%s", request_str);
			continue;
		}
		addTime(&end_time, &start_time, &total_time);
#if defined(MARKOV_TEST)
		ptr = strstr(buf, markov_bflag);
		if(ptr == NULL) 
		{
			fprintf(stdout, "strange no markov begin flag\n");
			fprintf(stdout, "request = %s \n result = %s\n", request_str, buf);
			break;
		}
		pptr = ptr + strlen(markov_bflag);
		ptr = strstr(pptr, markov_eflag);
		if(ptr == NULL)
		{
			fprintf(stdout, "strange no markov end flag\n");
			break;
		}
#elif defined(CONTENT_TEST)
		ptr = strstr(buf, content_bflag);
		if(ptr == NULL) 
		{
			fprintf(stdout, "no content begin flag\n");
			fprintf(stdout, "request = %s \n", request_str);
			continue;
		}
		pptr = ptr + strlen(content_bflag);
		ptr = strstr(pptr, content_eflag);
		if(ptr == NULL)
		{
			fprintf(stdout, "no content end flag\n");
			fprintf(stdout, "request = %s \n", request_str);
			continue;
		}
#else
		fprintf(stdout, "please define a testing item in Make file\n");
		return 0;
#endif
		*ptr = '\0';
		strcpy(para, pptr);
		/*if(strstr(para, "href=") == NULL)
		{
			norec_num++;
			continue;
		}*/
		i = 0;
		match_flag = 0;
		while((ptr=strstr(pptr, "href=")) != NULL)
		{
			sim = 0.0;
			ptr_tmp = ptr + strlen("href=");
			ptr = strchr(ptr_tmp, '>');
			if(ptr != NULL) *ptr = '\0';
			strcpy(temp, ptr_tmp);
			//the following for content evaluation
#if defined(CONTENT_TEST)
			sim = getPageSimlarity(next_visit, temp, terms_gdbm);
			if(sim < 0)
			{
				fprintf(stdout, "getPageSimlarity failed:%.2f\n", sim);
				fprintf(fp_out, "N/A ");
				i++;
				pptr = ptr + 1;
				continue;
			}
			fprintf(fp_out, "%.3f ", sim);
			//the following for markov evaluation.
#elif defined(MARKOV_TEST)
			if(strlen(next_visit) > 1 && next_visit[strlen(next_visit)-1] == '/') next_visit[strlen(next_visit)-1] = '\0';
			if(strlen(temp) > 1 && temp[strlen(temp)-1] == '/') temp[strlen(temp)-1] = '\0';
			if(!strcmp(next_visit, temp))
			{
				fprintf(fp_out, "1 ");
				//match_flag = 1;
				//break;
			}
			else
			{
				fprintf(fp_out, "0 ");
			}
#else
		fprintf(stdout, "please define a testing item in Make file\n");
		return 0;
#endif
			i++;
			pptr = ptr + 1;
		}
#if defined(MARKOV_TEST)
		if(i < 10) for(j=i;j<10;j++) fprintf(fp_out, "0 ");
		fprintf(fp_out, "%d\n", i);
#elif defined(CONTENT_TEST)
		if(i < 10) for(j=i;j<10;j++) fprintf(fp_out, "N/A ");
		fprintf(fp_out, "%u.%06u\n", total_time.tv_sec, total_time.tv_usec);
#else
		fprintf(stdout, "please define a testing item in Make file\n");
		return 0;
#endif
		/*if(match_flag == 0)
		{
			fprintf(fp_out, "0 nomatch %u.%06us\n", total_time.tv_sec, total_time.tv_usec);
		}*/
		fflush(NULL);
		k++;
		if(k > TEST_NUM) break;
		//sleep(2);
	}
	fprintf(stdout, "no rec num = %d\n", norec_num);
	fprintf(stdout, "experiement finished!\n");
	fclose(fp_in);
	fclose(fp_out);
	return 0;
}