stor_reg.c

harvest是一个下载html网页得机器人

	 *      n bytes of Gatherer-Host
	 *      4 network-order bytes of Gatherer-Version size
	 *      n bytes of Gatherer-Version
	 *      4 network-order bytes of MD5
	 *      n bytes of MD5
	 *      4 network-order bytes of Description size
	 *      n bytes of Description
	 *      4 network-order bytes of FD
	 *      4 network-order bytes of TTL
	 *      4 network-order bytes of L-M-T
	 *      4 network-order bytes of Refresh-Rate
	 *      4 network-order bytes of Update-Time
	 */
	rp = record;
	memcpy(&x, rp, NUM32LEN);	/* Grab URL size and URL */
	rp += NUM32LEN;
	registry_entry->urls = ntohl(x);
	registry_entry->url = (char *) xmalloc(registry_entry->urls + 1);
	memcpy(registry_entry->url, rp, registry_entry->urls);
	registry_entry->url[registry_entry->urls] = '\0';
	rp += registry_entry->urls;

	memcpy(&x, rp, NUM32LEN);	/* Grab Gname size and Gname */
	rp += NUM32LEN;
	MyGid.gns = ntohl(x);
	MyGid.gn = (char *) xmalloc(MyGid.gns + 1);
	memcpy(MyGid.gn, rp, MyGid.gns);
	MyGid.gn[MyGid.gns] = '\0';
	rp += MyGid.gns;

	memcpy(&x, rp, NUM32LEN);	/* Grab Ghost size and Ghost */
	rp += NUM32LEN;
	MyGid.ghs = ntohl(x);
	MyGid.gh = (char *) xmalloc(MyGid.ghs + 1);
	memcpy(MyGid.gh, rp, MyGid.ghs);
	MyGid.gh[MyGid.ghs] = '\0';
	rp += MyGid.ghs;

	memcpy(&x, rp, NUM32LEN);	/* Grab Gver size and Gver */
	rp += NUM32LEN;
	MyGid.gvs = ntohl(x);
	MyGid.gv = (char *) xmalloc(MyGid.gvs + 1);
	memcpy(MyGid.gv, rp, MyGid.gvs);
	MyGid.gv[MyGid.gvs] = '\0';
	rp += MyGid.gvs;

	/* Set the GathererID, and clean up */
	MyGid.GID = -1;
	registry_entry->GID = RG_gid_register(&MyGid);
	xfree(MyGid.gn);
	xfree(MyGid.gh);
	xfree(MyGid.gv);

	memcpy(&x, rp, NUM32LEN);	/* Grab MD5 size and MD5 */
	rp += NUM32LEN;
	registry_entry->md5s = ntohl(x);
	registry_entry->md5 = (char *) xmalloc(registry_entry->md5s + 1);
	memcpy(registry_entry->md5, rp, registry_entry->md5s);
	registry_entry->md5[registry_entry->md5s] = '\0';
	rp += registry_entry->md5s;

	memcpy(&x, rp, NUM32LEN);	/* Grab Desc size and Desc */
	rp += NUM32LEN;
	registry_entry->descs = ntohl(x);
	registry_entry->desc = (char *) xmalloc(registry_entry->descs + 1);
	memcpy(registry_entry->desc, rp, registry_entry->descs);
	registry_entry->desc[registry_entry->descs] = '\0';
	rp += registry_entry->descs;

	memcpy(&x, rp, NUM32LEN);	/* Grab FD */
	rp += NUM32LEN;
	registry_entry->FD = ntohl(x);

	memcpy(&x, rp, NUM32LEN);	/* Grab TTL */
	rp += NUM32LEN;
	registry_entry->ttl = ntohl(x);

	memcpy(&x, rp, NUM32LEN);	/* Grab L-M-T */
	rp += NUM32LEN;
	registry_entry->lmt = ntohl(x);

	memcpy(&x, rp, NUM32LEN);	/* Grab Refresh-Rate */
	rp += NUM32LEN;
	registry_entry->refresh_rate = ntohl(x);

	memcpy(&x, rp, NUM32LEN);	/* Grab Update-Time */
	rp += NUM32LEN;
	registry_entry->update_time = ntohl(x);

	xfree(record);
	return SUCCESS;
}

/* ----------------------------------------------------------------- *
   write_record() - Write the record to the registry.  To reset
   to the beginning of the registry file, use read_header().  MUST
   call read_header() (or write_header) before the first read_record().  
   For example,

   read_header()
   while (write_record(r) == SUCCESS) { r = something_new(); };

   Will write the next record to the registry, then returns SUCCESS.  
   On error, it returns ERROR.
   * ----------------------------------------------------------------- */
int write_record(registry_entry)
reg_t *registry_entry;
{
	RECORD_HEADER rhdr;
	char *record, *rp;
	GathererID *MyGid;
	int n;
	num32 x;

	/* Grab a pointer to the GathererID */
	MyGid = RG_gid(registry_entry->GID);
	if (MyGid == NULL) {
		errorlog("write_record: Internal error with Gatherer ID!\n");
		return ERROR;
	}

	n = registry_entry->urls + NUM32LEN +	/* url + size */
	    MyGid->gns + NUM32LEN +		/* gatherer name + size */
	    MyGid->ghs + NUM32LEN +		/* gatherer host + size */
	    MyGid->gvs + NUM32LEN +		/* gatherer ver + size */
	    registry_entry->md5s + NUM32LEN +	/* MD5 + size */
	    registry_entry->descs + NUM32LEN +	/* description + size */
	    (5 * NUM32LEN);	/* 5 numbers */

	rhdr.record_size = htonl(n);
	rhdr.magic = htonl(REGISTRY_MAGIC);
	rhdr.flag = htonl(registry_entry->flag);

	if (write(rfd, &rhdr, sizeof(RECORD_HEADER)) < 0) {
		log_errno(registry_file);
		return ERROR;
	}
	record = (char *) xmalloc(n);
	rp = record;

	x = htonl(registry_entry->urls);	/* Copy over URL and size */
	memcpy(rp, &x, NUM32LEN);
	rp += NUM32LEN;
	memcpy(rp, registry_entry->url, registry_entry->urls);
	rp += registry_entry->urls;

	x = htonl(MyGid->gns);			/* Copy over Gname and size */
	memcpy(rp, &x, NUM32LEN);
	rp += NUM32LEN;
	memcpy(rp, MyGid->gn, MyGid->gns);
	rp += MyGid->gns;

	x = htonl(MyGid->ghs);			/* Copy over Ghost and size */
	memcpy(rp, &x, NUM32LEN);
	rp += NUM32LEN;
	memcpy(rp, MyGid->gh, MyGid->ghs);
	rp += MyGid->ghs;

	x = htonl(MyGid->gvs);			/* Copy over Gver and size */
	memcpy(rp, &x, NUM32LEN);
	rp += NUM32LEN;
	memcpy(rp, MyGid->gv, MyGid->gvs);
	rp += MyGid->gvs;

	x = htonl(registry_entry->md5s);	/* Copy over MD5 and size */
	memcpy(rp, &x, NUM32LEN);
	rp += NUM32LEN;
	memcpy(rp, registry_entry->md5, registry_entry->md5s);
	rp += registry_entry->md5s;

	x = htonl(registry_entry->descs);	/* Copy over Desc and size */
	memcpy(rp, &x, NUM32LEN);
	rp += NUM32LEN;
	memcpy(rp, registry_entry->desc, registry_entry->descs);
	rp += registry_entry->descs;

	x = htonl(registry_entry->FD);	/* Copy over FD */
	memcpy(rp, &x, NUM32LEN);
	rp += NUM32LEN;

	x = htonl(registry_entry->ttl);		/* Copy over TTL */
	memcpy(rp, &x, NUM32LEN);
	rp += NUM32LEN;

	x = htonl(registry_entry->lmt);		/* Copy over L-M-T */
	memcpy(rp, &x, NUM32LEN);
	rp += NUM32LEN;

	x = htonl(registry_entry->refresh_rate);	/* Copy over Refresh-Rate */
	memcpy(rp, &x, NUM32LEN);
	rp += NUM32LEN;

	x = htonl(registry_entry->update_time);		/* Copy over Update-Time */
	memcpy(rp, &x, NUM32LEN);
	rp += NUM32LEN;

	if (write(rfd, record, n) < 0) {
		log_errno(registry_file);
		xfree(record);
		return ERROR;
	}
	xfree(record);
	return SUCCESS;
}

/* ----------------------------------------------------------------- *
   replace_record() -- replace a record with one that is known to be
   the same size; this happens when the expiration time is updated.
   * ----------------------------------------------------------------- */
int replace_record(regent)
reg_t *regent;
{
	if (seek_registry((off_t)regent->rec_off, SEEK_SET) < 0)
		return ERROR;

	return (write_record(regent));
}

/* ----------------------------------------------------------------- *
   get_record() -- 
   * ----------------------------------------------------------------- */
int get_record(regent)
reg_t *regent;
{
	off_t x;

	if ((x = seek_registry((off_t)0, SEEK_CUR)) < 0)
		return ERROR;

	if (regent->rec_off == 0)
		regent->rec_off = x;
	else if (regent->rec_off != x) {
		errorlog("RECORD OFFSET MISMATCH!!!\n");
		return ERROR;
	}
	return (read_record(regent));
}

/* ----------------------------------------------------------------- *
   append_new_record()
   * ----------------------------------------------------------------- */
int append_new_record(regent)
reg_t *regent;
{

	if ((regent->rec_off = seek_registry((off_t)0, SEEK_END)) < 0)
		return ERROR;

	return (write_record(regent));
}

/* ----------------------------------------------------------------- *
   remove_record()
   * ----------------------------------------------------------------- */
int remove_record(regent)
reg_t *regent;
{
	RECORD_HEADER r;
	int status;

	if (seek_registry((off_t)regent->rec_off, SEEK_SET) < 0)
		return ERROR;

	status = read_record_hdr(&r);
	if (status == SUCCESS) {
		SET_DELETED(r.flag);
		if (seek_registry((off_t)regent->rec_off, SEEK_SET) < 0)
			return ERROR;
		return (write_record_hdr(&r));
	} else if (status == ENTRY_DELETED)
		return SUCCESS;

	return ERROR;
}

/* ----------------------------------------------------------------- *
   read_record_hdr()
   * ----------------------------------------------------------------- */
int read_record_hdr(rhdr)
RECORD_HEADER *rhdr;
{
	int n;

	/* Read and canonicalize the record header */
	if ((n = read(rfd, rhdr, sizeof(RECORD_HEADER))) < 0) {
		log_errno(registry_file);
		return ERROR;
	}
	if (n == 0)
		return REGISTRY_EOF;

	rhdr->record_size = ntohl(rhdr->record_size);
	rhdr->magic = ntohl(rhdr->magic);
	rhdr->flag = ntohl(rhdr->flag);

	/* See if the header is corrupt */
	if (rhdr->magic != REGISTRY_MAGIC) {
		errorlog("Record Header is corrupt at offset %d: 0x%08x 0x%08x 0x%08x\n",
			seek_registry((off_t)0, SEEK_CUR),
			rhdr->record_size, rhdr->magic, rhdr->flag);
		return ERROR;
	}
	/* Check to see if the record is deleted.  If so, skip nbytes */
	if (IS_DELETED(rhdr->flag)) {
		if (seek_registry((off_t)rhdr->record_size, SEEK_CUR) < 0) {
			log_errno(registry_file);
			return ERROR;
		}
		return ENTRY_DELETED;
	}
	return SUCCESS;
}

/* ----------------------------------------------------------------- *
   write_record_hdr()
   * ----------------------------------------------------------------- */
int write_record_hdr(rhdr)
RECORD_HEADER *rhdr;
{
	RECORD_HEADER tmp;

	tmp.record_size = htonl(rhdr->record_size);
	tmp.magic = htonl(rhdr->magic);
	tmp.flag = htonl(rhdr->flag);
	if (write(rfd, &tmp, sizeof(RECORD_HEADER)) < 0) {
		log_errno(registry_file);
		return ERROR;
	}
	return SUCCESS;
}

/* Place holder routines: set_registry_mark, restore_registry_mark */
static off_t rmark = 0;
off_t set_registry_mark()
{
	rmark = seek_registry((off_t)0, SEEK_CUR);
	return(rmark);
}

off_t restore_registry_mark()
{
	return(seek_registry((off_t)rmark, SEEK_SET));
}