visitors.c

web服务器访问统计

		"FreeBSD", NULL,
		"OpenBSD", NULL,
		"NetBSD", NULL,
		"BEOS", NULL,
		"", "Unknown",
		NULL, NULL,
	};
	return vi_counter_incr_matchtable(&vih->os, agent, oslist);
}

/* Process browsers information. */
int vi_process_browsers(struct vih *vih, char *agent)
{
	/* Note that the order matters. For example Safari
	 * send an user agent where there is the string "Gecko"
	 * so it must be before Gecko. */
	char *browserslist[] = {
		"Opera", NULL,
		"MSIE 4", "Explorer 4.x",
		"MSIE 5", "Explorer 5.x",
		"MSIE 6", "Explorer 6.x",
		"MSIE", "Explorer unknown version",
		"Safari", NULL,
		"Konqueror", NULL,
		"Galeon", NULL,
		"Firefox", NULL,
		"MultiZilla", NULL,
		"Gecko", "Other Mozilla based",
		"Wget", NULL,
		"Lynx", NULL,
		"Links ", "Links",
		"ELinks ", "Links",
		"Elinks ", "Links",
		"w3m", "W3M",
		"NATSU-MICAN", NULL,
		"msnbot", "MSNbot",
		"Slurp", "Yahoo Slurp",
		"Jeeves", "Ask Jeeves",
		"ZyBorg", NULL,
		"asteria", NULL,
		"contype", "Explorer",
		"Gigabot", NULL,
		"Windows-Media-Player", "Windows-MP",
		"NSPlayer", NULL,
		"Googlebot", "GoogleBot",
		"googlebot", "GoogleBot",
		"", "Unknown",
		NULL, NULL,
	};
	return vi_counter_incr_matchtable(&vih->browsers, agent, browserslist);
}

/* Process req/agents to get information about pages retrivied by Google.
 * Return non-zero on out of memory. */
int vi_process_googled(struct vih *vih, char *req, char *agent, time_t age)
{
        if (vi_is_googlebot_agent(agent)) {
	    return vi_replace_if_newer(&vih->googled, req, age);
        } else if (vi_is_adsensebot_agent(agent)) {
	    return vi_replace_if_newer(&vih->adsensed, req, age);
        }
        return 0;
}

/* Process screen resolution and color depth info, if the javascript
 * code needed was inserted in the pages (see the README file). */
int vi_process_screen_info(struct vih *vih, char *req) {
    char *p;

    if ((p = strstr(req, "visitors-screen-res-check.jpg?"))) {
        char buf[64];

        p += 30;
        if (p[0] == '\0' || strstr(p, "undefined")) goto parseerror;
        vi_strlcpy(buf, p, 64);
        /* The string is somethink like: 1024x768x32, so we
         * search for the second 'x' char. */
        p = strchr(buf,'x'); if (!p) goto parseerror;
        p = strchr(p+1,'x'); if (!p) goto parseerror;
        *p = '\0'; p++;
        /* Populate the screen resolution hash table */
        if (vi_counter_incr(&vih->screenres, buf) == 0)
            return 1;
        /* ... and the screen color depth one. */
        if (vi_counter_incr(&vih->screendepth, p) == 0)
            return 1;
    }
parseerror:
    return 0;
}

/* Process accesses with the referer from google.
 * This is used to populate the keyphrases hashtable.
 * TODO: url decoding */
int vi_process_google_keyphrases(struct vih *vih, char *ref, time_t age)
{
	char *s, *p, *e;
	int res, page;
	char urldecoded[VI_LINE_MAX];
	char buf[64];

	if (!vi_is_google_link(ref)) return 0;
        /* Try to process gogoe human language info first. */
        if (Config_process_google_human_language) {
            s = strstr(ref+18, "&hl=");
            if (s == NULL) s = strstr(ref+18, "?hl=");
            if (s && s[4] && s[5]) {
                buf[0] = s[4];
                buf[1] = s[5];
                buf[2] = '\0';
	        if (vi_counter_incr(&vih->googlehumanlanguage, buf) == 0)
                    return 1;
            }
        }

	/* It's possible to start the search for the query 18 chars
	 * after the start of the referer because all the
	 * google links will start with "http://www.google.". */
	if ((s = strstr(ref+18, "?q=")) == NULL &&
	    (s = strstr(ref+18, "&q=")) == NULL) return 0;
	if ((p = strstr(ref+18, "&start=")) == NULL)
		p = strstr(ref+18, "?start=");
	if ((e = strchr(s+3, '&')) != NULL)
		*e = '\0';
	if (p && (e = strchr(p+7, '&')) != NULL)
		*e = '\0';
	if (!strncmp(s+3, "cache:", 6))
		return !vi_counter_incr(&vih->googlekeyphrases, "Google Cache Access");
	vi_urldecode(urldecoded, s+3, VI_LINE_MAX);
	vi_strtolower(urldecoded);
	page = p ? (1+(atoi(p+7)/10)) : 1;
	snprintf(buf, 64, " (page %d)", page);
	buf[63] = '\0';
	vi_strlcat(urldecoded, buf, VI_LINE_MAX);
	res = vi_counter_incr(&vih->googlekeyphrases, urldecoded);
	if (e) *e = '&';
	if (res == 0) return 1;
	/* Process keyphrases by first time */
	if (Config_process_google_keyphrases_age) {
		if (vi_replace_if_older(&vih->googlekeyphrasesage,
					urldecoded, age)) return 1;
	}
	return 0;
}

/* Process robots information. For visitors every client accessing
 * to robots.txt is considered a robot.
 * Returns 1 on out of memory, otherwise zero is returned. */
int vi_process_robots(struct vih *vih, char *req, char *agent)
{
	if (strncmp(req, "/robots.txt", 11) != 0) return 0;
	if (strstr(agent, "MSIECrawler")) return 0;
	return !vi_counter_incr(&vih->robots, agent);
}

/* Process referer -> request pairs for web trails */
int vi_process_web_trails(struct vih *vih, char *ref, char *req)
{
	int res, plen, google;
	char buf[VI_LINE_MAX];
	char *src;

	if (vi_is_image(req)) return 0;
	plen = vi_is_internal_link(ref);
	google = vi_is_google_link(ref);
	if (plen) {
		src = (ref[plen] == '\0') ? "/" : ref+plen;
	} else if (google) {
		if (Config_graphviz_ignorenode_google) return 0;
		src = "Google";
	} else if (ref[0] != '\0') {
		if (Config_graphviz_ignorenode_external) return 0;
		src = "External Link";
	} else {
		if (Config_graphviz_ignorenode_noreferer) return 0;
		src = "No Referer";
	}
	if (!strcmp(src, req)) return 0; /* avoid self references */

	snprintf(buf, VI_LINE_MAX, "%s -> %s", src, req);
	buf[VI_LINE_MAX-1] = '\0';
	res = vi_counter_incr(&vih->trails, buf);
	if (res == 0) return 1;
	return 0;
}

/* Process Top Level Domains.
 * Returns zero on success. Non zero is returned on out of memory. */
int vi_process_tld(struct vih *vih, char *hostname)
{
	char *tld;
	int res;

	if (vi_is_numeric_address(hostname)) {
		tld = "numeric IP";
	} else {
		tld = strrchr(hostname, '.');
		if (!tld) return 0;
		tld++;
	}
	res = vi_counter_incr(&vih->tld, tld);
	if (res == 0) return 1;
	return 0;
}

/* Match a log line against --grep and --exclude patters to check
 * if the line must be processed or not. */
int vi_match_line(char *line)
{
    int i;

    for (i = 0; i < Config_grep_pattern_num; i++) {
        char *pattern = Config_grep_pattern[i].pattern;
        int nocase = 1;

        /* Patterns starting with 'cs:' are matched in a case-sensitive
         * way after the 'cs:' prefix is discarded. */
        if (pattern[0] == 'c' && pattern[1] == 's' && pattern[2] == ':') {
            nocase = 0;
            pattern += 3;
        }
        if (vi_match(Config_grep_pattern[i].pattern, line, nocase)) {
            if (Config_grep_pattern[i].type == VI_PATTERNTYPE_EXCLUDE)
                return 0;
        } else {
            if (Config_grep_pattern[i].type == VI_PATTERNTYPE_GREP)
                return 0;
        }
    }
    return 1;
}

/* Process a line of log. Returns non-zero on error. */
int vi_process_line(struct vih *vih, char *l)
{
	struct logline ll;
	char origline[VI_LINE_MAX];

        /* Test the line against --grep --exclude patterns before
         * to process it. */
        if (Config_grep_pattern_num) {
            if (vi_match_line(l) == 0)
                return 0; /* No match? skip. */
        }

	vih->processed++;
	/* Take a copy of the original log line before to
	 * copy it. Will be useful for some processing.
	 * Do it only if required in order to speedup. */
	if (Config_process_error404 || Config_debug)
		vi_strlcpy(origline, l, VI_LINE_MAX);
	/* Split the line and run all the selected processing. */
	if (vi_parse_line(&ll, l) == 0) {
		int seen, is404;

                /* We process 404 errors first, in order to skip
                 * all the other reports if --ignore-404 option is active. */
		if (Config_process_error404 &&
		    vi_process_error404(vih, origline, ll.req, &is404))
                        goto oom;
                /* Process screen info if needed. */
                if (Config_process_screen_info && is404)
                    if (vi_process_screen_info(vih, ll.req)) goto oom;
                /* 404 error AND --ignore-404? Stop processing of this line. */
                if (Config_ignore_404 && is404)
                    return 0;

                /* Now it's time to process unique visitors. The 'save'
                 * local var saves if this log line is about a new visit
                 * or not. Some report is generated only against the first
                 * line of every visitor, other reports are generated
                 * for every single log line. */
		if (vi_process_visitors_per_day(vih, ll.host, ll.agent,
					ll.date, ll.ref, ll.req, &seen))
			goto oom;

		/* The following are processed for every log line */
		if (vi_process_page_request(vih, ll.req)) goto oom;
		if (Config_process_google &&
		    vi_process_googled(vih, ll.req, ll.agent, ll.time))
			goto oom;
		if (Config_process_web_trails &&
		    vi_process_web_trails(vih, ll.ref, ll.req)) goto oom;
		if (Config_process_google_keyphrases &&
		    vi_process_google_keyphrases(vih, ll.ref, ll.time))
			goto oom;

		/* The following are processed only for new visits */
		if (seen) return 0;
		vi_process_date_and_hour(vih, (ll.tm.tm_wday+6)%7,
				ll.tm.tm_hour);
		vi_process_month_and_day(vih, ll.tm.tm_mon, ll.tm.tm_mday-1);
		if (vi_process_referer(vih, ll.ref, ll.time)) goto oom;
		if (Config_process_agents &&
		    vi_process_agents(vih, ll.agent)) goto oom;
		if (Config_process_os &&
		    vi_process_os(vih, ll.agent)) goto oom;
		if (Config_process_browsers &&
		    vi_process_browsers(vih, ll.agent)) goto oom;
		if (Config_process_tld &&
		    vi_process_tld(vih, ll.host)) goto oom;
		if (Config_process_robots &&
		    vi_process_robots(vih, ll.req, ll.agent)) goto oom;
		return 0;
	} else {
		vih->invalid++;
                if (Config_debug)
                    fprintf(stderr, "Invalid line: %s\n", origline);
		return 0;
	}
oom:
	vi_set_error(vih, "Out of memory processing data");
	return 1;
}

/* Process the specified log file. Returns zero on success.
 * On error non zero is returned and an error is set in the handle. */
int vi_scan(struct vih *vih, char *filename)
{
	FILE *fp;
	char buf[VI_LINE_MAX];
	int use_stdin = 0;

	if (filename[0] == '-' && filename[1] == '\0') {
		/* If we are in stream mode, just return. Stdin
		 * is implicit in this mode and will be read
		 * after all the other files are processed. */
		if (Config_stream_mode) return 0;
		fp = stdin;
		use_stdin = 1;
	} else {
		if ((fp = fopen(filename, "r")) == NULL) {
			vi_set_error(vih, "Unable to open '%s': '%s'", filename, strerror(errno));
			return 1;
		}
	}
	while (fgets(buf, VI_LINE_MAX, fp) != NULL) {
		if (vi_process_line(vih, buf)) {
			fclose(fp);
			fprintf(stderr, "%s: %s\n", filename, vi_get_error(vih));
			return 1;
		}
	}
	if (!use_stdin)
		fclose(fp);
	vih->endt = time(NULL);
	return 0;
}

/* Postprocessing of pageviews per visit data.
 * The source hashtable entries are in the form: uniqe-visitor -> pageviews.
 * After the postprocessing we obtain another hashtable in the form:
 * pageviews-range -> quantity. This hashtable can be used directly
 * with generic output functions to generate the output. */
int vi_postprocess_pageviews(struct vih *vih)
{
	void **table;
	int len = ht_used(&vih->pageviews), i;

	if ((table = ht_get_array(&vih->pageviews)) == NULL) {
		fprintf(stderr, "Out of memory in vi_postprocess_pageviews()\n");
		return 1;
	}
	/* Run the hashtable in order to populate 'pageviews_grouped' */
	for (i = 0; i < len; i++) {
		int pv = (long) table[(i*2)+1]; /* pageviews of visit */
		int res;
		char *key;

		if (pv == 1) key = "1";
		else if (pv == 2) key = "2";
		else if (pv == 3) key = "3";
		else if (pv == 4) key = "4";
		else if (pv == 5) key = "5";
		else if (pv == 6) key = "6";
		else if (pv == 7) key = "7";
		else if (pv == 8) key = "8";
		else if (pv == 9) key = "9";
		else if (pv == 10) key = "10";
		else if (pv >= 11 && pv <= 20) key = "11-20";
		else if (pv >= 21 && pv <= 30) key = "21-30";
		else key = "> 30";

		res = vi_counter_incr(&vih->pageviews_grouped, key);
		if (res == 0) {
			free(table);
			return 1; /* out of memory */
		}
	}
	free(table);
	return 0;
}

/* This function is called from vi_print_report() in order to
 * run some postprocessing to raw data collected needed to generate reports. */
int vi_postprocess(struct vih *vih)
{
	if (vi_postprocess_pageviews(vih)) goto oom;
	return 0;
oom:
	vi_set_error(vih, "Out of memory");
	return 1;
}

/* ---------------------------- text output module -------------------------- */
void om_text_print_header(FILE *fp)
{
	fp = fp;
	return;
}

void om_text_print_footer(FILE *fp)
{
	fp = fp;
	return;
}

void om_text_print_title(FILE *fp, char *title)
{
	fprintf(fp, "=== %s ===\n", title);
}

void om_text_print_subtitle(FILE *fp, char *subtitle)
{
	fprintf(fp, "--- %s\n", subtitle);
}

void om_text_print_numkey_info(FILE *fp, char *key, int val)
{
	fprintf(fp, "* %s: %d\n", key, val);
}

void om_text_print_keykey_entry(FILE *fp, char *key1, char *key2, int num)
{
	fprintf(fp, "%d)    %s: %s\n", num, key1, key2);
}

void om_text_print_numkey_entry(FILE *fp, char *key, int val, char *link,
		int num)
{
	link = link; /* avoid warning. Text output don't use this argument. */
	fprintf(fp, "%d)    %s: %d\n", num, key, val);
}

/* Print a bar, c1 and c2 are the colors of the left and right parts.
 * Max is the maximum value of the bar, the bar length is printed
 * to be porportional to max. tot is the "total" needed to compute
 * the precentage value. */
void om_text_print_bar(FILE *fp, int max, int tot, int this, int cols,
		char c1, char c2)
{
	int l;
	float p;
	char *bar;
	if (tot == 0) tot++;
	if (max == 0) max++;
	l = ((float)(cols*this))/max;
	p = ((float)(100*this))/tot;
	bar = malloc(cols+1);
	if (!bar) return;
	memset(bar, c2, cols+1);
	memset(bar, c1, l);
	bar[cols] = '\0';
	fprintf(fp, "%s %02.1f%%", bar, p);
	free(bar);
}

void om_text_print_numkeybar_entry(FILE *fp, char *key, int max, int tot, int this)
{
	fprintf(fp, "   %-12s: %-9d |", key, this);
	om_text_print_bar(fp, max, tot, this, 44, '#', ' ');
	fprintf(fp, "\n");