registry.c

harvest是一个下载html网页得机器人

/* -----------------------------------------------------------------
   RG_Object_Search_ByURL() -- approx. search for an entry in the registry
   which matches the given URL.  Note that this search is only approx.
   since may objects may exist with the same URL, but different GIDs.
   ----------------------------------------------------------------- */
reg_t *RG_Object_Search_ByURL(url)
char *url;
{
	static hash_link *tmp;

	if (url == NULL || strlen(url) < 1)
		return (NULL);

	for (tmp = RG_hash_url_bucket(url); tmp != NULL; tmp = tmp->next) {
		Debug(70,9,("RG_Object_Search_ByURL: URL bucket: Looking at %d, %s\n", tmp->item->FD, tmp->item->url));
		if (!strcmp(url, tmp->item->url)) {
			return(tmp->item);
		}
	}

	return (NULL);
}

/* -----------------------------------------------------------------
    RG_Uniqify_by_URL() - Remove duplicate URLs from the broker
   ----------------------------------------------------------------- */
int RG_Uniqify_by_URL()
{
	reg_t		*entry = NULL;
	hash_link	*tmp = NULL;
	hash_link	*next = NULL;
	int		n = 0;

	for (entry=Registry; entry; entry=entry->next) {

		for (tmp=RG_hash_url_bucket(entry->url); tmp; tmp=next) {
			next = tmp->next;
			if (tmp->item == entry) continue;
			if (!RG_Object_URL_Match(entry,tmp->item)) continue;
			if (tmp->item->update_time <= entry->update_time) {
				Debug (70,1,("RG_Uniqify: Removing duplicate: %s\n", tmp->item->url));
				RG_Clean_Entry(tmp->item);
				n++;
			}
		}
	}

	Log("RG_Uniqify: %d objects removed.\n", n);
	RG_Compress ();
	return n;
}
#undef safe_memmatch
#undef RG_Object_Match
#undef RG_Object_URL_Match


/***********************************************************************
 		Registry hashing support
 ***********************************************************************/

/*
 *  hash_url() - Returns a well-distributed hash function for URLs.
 *  The best way is to sum up the last half of the string.
 *  Adapted from code written by Mic Bowman.  -Darren
 *  Generates a standard deviation = 15.73
 */
static int hash_url(s)
char *s;
{
        unsigned int i, j, n;

        j = strlen(s);
        for (i = j/2, n = 0; i < j; i++)
                n ^= 271 * (unsigned)s[i];
	i = n ^ (j * 271);
	return(uhash(i));
}

/*
 *  hash_md5() - Returns a well-distributed hash function for MD5s
 *  Do a quickie based on first 8 characters.  Convert MD5 into an int.
 *  This is very fast and providers a great even distribution.
 *  Generates a standard deviation =  8.70
 */
static int hash_md5(s)
char *s;
{
        unsigned int n = 0;
	char tmp[9];

	strncpy(tmp, s, 8);
	tmp[8] = '\0';
	sscanf(tmp, "%x", &n);
	return(uhash(n));
}


static void RG_hash_init()
{
	Debug(73,1,("RG_hash_init: HASH_SIZE %d, hash_link %d, reg_t %d.\n",
		HASH_SIZE, sizeof(hash_link), sizeof(reg_t)));
	memset(fdhtable, '\0',  HASH_SIZE * sizeof(hash_link *));
	memset(urlhtable, '\0', HASH_SIZE * sizeof(hash_link *));
	memset(md5htable, '\0', HASH_SIZE * sizeof(hash_link *));
}

/*
 *  RG_hash_build() - Builds all of the hash tables for each Registry entry.
 */
static void RG_hash_build()
{
	reg_t *tmp;
	int cnt = 0;

	Log("Building fast search structure for the Registry...\n");
	for (tmp = Registry; tmp != NULL; tmp = tmp->next, cnt++) {
		RG_hash_insert(tmp);
	}
	Log("Search structure contains %d objects.\n", cnt);
	if (debug_ok(70,9))
		RG_hash_print();
}

/*
 *  RG_hash_insert() - Inserts the Registry item into the hash tables.
 *  One based on the item's FD, one based on the URL, and one based
 *  on the MD5.
 */
static void RG_hash_insert(item)
reg_t *item;
{
	int i;
	hash_link *new;

	/* Add to the FD hash table */
	new = (hash_link *) xmalloc(sizeof(hash_link));
	new->item = item;
	i = hash(item->FD);
	Debug(73,5,("RG_hash_insert: Inserting %p %d in FD bucket %d\n", item, item->FD, i));
	if (fdhtable[i] == NULL) {		/* first item */
		fdhtable[i] = new;
		fdhtable[i]->next = NULL;
	} else {				/* prepend to list */
		new->next = fdhtable[i];
		fdhtable[i] = new;
	}

	/* Add to the URL hash table */
	if (item->url != NULL && item->urls > 1) {	/* not 0 length */
		new = (hash_link *) xmalloc(sizeof(hash_link));
		new->item = item;
		i = hash_url(item->url);
		Debug(73,5,("RG_hash_insert: Inserting %p %d in URL bucket %d\n", item, item->FD, i));
		if (urlhtable[i] == NULL) {		/* first item */
			urlhtable[i] = new;
			urlhtable[i]->next = NULL;
		} else {				/* prepend to list */
			new->next = urlhtable[i];
			urlhtable[i] = new;
		}
	}

	/* Add to the MD5 hash table */
	if (item->md5 != NULL && item->md5s > 8) {
		new = (hash_link *) xmalloc(sizeof(hash_link));
		new->item = item;
		i = hash_md5(item->md5);
		Debug(73,5,("RG_hash_insert: Inserting %p %d in MD5 bucket %d\n", item, item->FD, i));
		if (md5htable[i] == NULL) {		/* first item */
			md5htable[i] = new;
			md5htable[i]->next = NULL;
		} else {				/* prepend to list */
			new->next = md5htable[i];
			md5htable[i] = new;
		}
	}
}

/*
 *  RG_hash_destroy() - frees all memory associated with the hash
 *  structure; and nulls the hash tables.
 */
static void RG_hash_destroy()
{
	hash_link *walker, *t;
	int i;

	Debug(73,1,("RG_hash_destroy: Deleting the entire hash table.\n"));
	for (i = 0; i < HASH_SIZE; i++) {
		/* Delete fd hash */
		walker = fdhtable[i];
		while (walker != NULL) {
			t = walker;
			walker = walker->next;
			xfree(t);
		}
		fdhtable[i] = NULL;

		/* Delete URL hash */
		walker = urlhtable[i];
		while (walker != NULL) {
			t = walker;
			walker = walker->next;
			xfree(t);
		}
		urlhtable[i] = NULL;

		/* Delete MD5 hash */
		walker = md5htable[i];
		while (walker != NULL) {
			t = walker;
			walker = walker->next;
			xfree(t);
		}
		md5htable[i] = NULL;
	}
}


/*
 *  RG_hash_search_byfd - Returns the Registry item that matches FD.
 */
static reg_t *RG_hash_search_byfd(FD)
fd_t FD;
{
	int i = hash(FD);
	static hash_link *walker;

	Debug(73,9,("RG_hash_search_byfd: Looking up %d in bucket %d\n",FD,i));
	for (walker = fdhtable[i]; walker != NULL; walker = walker->next) {
		if (walker->item->FD == FD)
			return (walker->item);
	}
	return (NULL);
}

static hash_link *RG_hash_url_bucket(url)
char *url;
{
	int i = hash_url(url);
	Debug(73,9,("RG_hash_url_bucket: Looking up %s in bucket %d\n",url,i));
	return(urlhtable[i]);
}

static hash_link *RG_hash_md5_bucket(md5)
char *md5;
{
	int i = hash_md5(md5);
	Debug(73,9,("RG_hash_md5_bucket: Looking up %s in bucket %d\n",md5,i));
	return(md5htable[i]);
}

/*
 *  RG_hash_delete() - Deletes the Registry item from the Hash tables.
 */
static void RG_hash_delete(item)
reg_t *item;
{
	int i;
	hash_link *walker, *prev;

	Debug(73,5,("RG_hash_delete: Deleting item %p, %d, %p, %p\n",
		item, item->FD, item->url, item->md5));
	i = hash(item->FD);
	Debug(73,9,("RG_hash_delete: Deleting %d: FD bucket %d\n",item->FD,i));
	prev = walker = fdhtable[i];
	while (walker != NULL) {
		if (walker->item->FD == item->FD) {
			if (prev == fdhtable[i])
				fdhtable[i] = walker->next;
			else
				prev->next = walker->next;
			xfree(walker);
			break;
		}
		prev = walker;
		walker = walker->next;
	}

	if (item->url != NULL && item->urls > 1) {
		i = hash_url(item->url);
		Debug(73,9,("RG_hash_delete: Deleting %d: URL bucket %d\n",item->FD,i));
		prev = walker = urlhtable[i];
		while (walker != NULL) {
			if (walker->item->FD == item->FD) {
				if (prev == urlhtable[i])
					urlhtable[i] = walker->next;
				else
					prev->next = walker->next;
				xfree(walker);
				break;
			}
			prev = walker;
			walker = walker->next;
		}
	}

	if (item->md5 != NULL && item->md5s > 8) {
		i = hash_md5(item->md5);
		Debug(73,9,("RG_hash_delete: Deleting %d: MD5 bucket %d\n",item->FD,i));
		prev = walker = md5htable[i];
		while (walker != NULL) {
			if (walker->item->FD == item->FD) {
				if (prev == md5htable[i])
					md5htable[i] = walker->next;
				else
					prev->next = walker->next;
				xfree(walker);
				break;
			}
			prev = walker;
			walker = walker->next;
		}
	}
}

/* RG_hash_print - debugging output; prints the entire hash table */
static void RG_hash_print()
{
	hash_link *walker;
	int i, cnt;

	for (i = 0; i < HASH_SIZE; i++) {
		cnt = 0;
		walker = fdhtable[i];
		while (walker != NULL) {
			cnt++;
			printf("FD  Hash: %5d: %10p: FD %10d.\n", i,
				walker->item, walker->item->FD);
			walker = walker->next;
		}
		printf("FD  Hash: %5d: Total %d\n", i, cnt);

		cnt = 0;
		walker = urlhtable[i];
		while (walker != NULL) {
			cnt++;
			printf("URL Hash: %5d: %10p: FD %10d.\n", i,
				walker->item, walker->item->FD);
			walker = walker->next;
		}
		printf("URL Hash: %5d: Total %d\n", i, cnt);

		cnt = 0;
		walker = md5htable[i];
		while (walker != NULL) {
			cnt++;
			printf("MD5 Hash: %5d: %10p: FD %10d.\n", i,
				walker->item, walker->item->FD);
			walker = walker->next;
		}
		printf("MD5 Hash: %5d: Total %d\n", i, cnt);
	}
}

/*
 *  RG_print_reg_t - For debugging only.  prints a Registry entry.
 */
static void RG_print_reg_t(r)
reg_t *r;
{
	char *s;

	printf("reg_t: %p\n", r);
	s = r->url;
	printf("reg_t: url: %p, %s\n", s, s ? s : "(null)");
	printf("reg_t: urls: %d\n", r->urls);
	s = r->md5;
	printf("reg_t: md5: %p, %s\n", s, s ? s : "(null)");
	printf("reg_t: md5s: %d\n", r->md5s);
	printf("reg_t: ttl: %d\n", r->ttl);
	printf("reg_t: lmt: %d\n", r->lmt);
	printf("reg_t: refresh_rate: %d\n", r->refresh_rate);
	printf("reg_t: update_time: %d\n", r->update_time);
	printf("reg_t: FD: %d\n", r->FD);
	printf("reg_t: GID: %d\n", r->GID);
	s = r->desc;
	printf("reg_t: desc: %p, %s\n", s, s ? s : "(null)");
	printf("reg_t: descs: %d\n", r->descs);
	printf("reg_t: flag: %d\n", r->flag);
	printf("reg_t: rec_off: %ld\n", (long) r->rec_off);
	printf("reg_t: next: %p\n", r->next);
	printf("reg_t: prev: %p\n", r->prev);
	fflush(stdout);
}