gopherenum-depth.c

harvest是一个下载html网页得机器人

    char *gopher_port = NULL;
    int y;

    if (url_in_db(up->url)) {	/* Have we been here? */
	Debug(43, 1, ("Already Visited URL: %s\n", up->url));
	return (NULL);
    }
    if ((y = filter_selection(up))) {	/* Match the URL based on REs */
	Debug(43, 1, ("Removing Candidate: [%s] %s\n",
		Filter_Type_Name[y], up->url));
	return (NULL);
    }

    if (!visit_server(up)) {	/* Can we visit this server? */
	Debug(43, 1, ("Disallowed to Visit Server: %s\n", up->url));
	return (NULL);
    }

    if (!RobotsTxtCheck(up)) {
        Debug(43, 1, ("Disallowed by robots.txt: %s\n", up->url));
	return (NULL);
    }

    if (url_retrieve(up)) {	/* Grab the URL; success? */
	Debug(43, 1, ("Cannot Retrieve URL: %s\n", up->url));

#ifdef DONT_RETRY_FAILS
	mark_failed(up);
#endif

	return (NULL);
    }
    if (up->md5 && md5_in_db(up->md5)) {	/* Have we been here? */
	Debug(43, 1, ("Already Visited MD5: %s\n", up->url));
	return (NULL);
    }
    /* Remember that we've been here before */
    if (up->md5 != NULL)
	mark_retrieved(up);
    if (up->gophertype == 0)
	return (NULL);

    /*
     *  For each pointer, convert it to a URL, and add it to
     *  the list of URLs to return.
     */
    if ((fp = fopen(up->filename, "r")) == NULL) {
	log_errno2(__FILE__, __LINE__, up->filename);
	return (NULL);
    }
    Tail = &head;
    while (fgets(buf, BUFSIZ, fp)) {
	if (buf[0] == '.' || buf[0] == '\n')
	    break;
	urlbuf = xstrdup(buf);
	if ((q = strrchr(buf, '\n')))
	    *q = (char) '\0';

	p = urlbuf;
	if ((q = strchr(p, '\t')) == NULL) {
	    errorlog("Illegal Gopher format: No Name: %s\n", buf);
	    goto gopher_enum_cont;
	}
	*q = (char) '\0';
	gopher_name = xstrdup(p);

	p = q + 1;
	if ((q = strchr(p, '\t')) == NULL) {
	    errorlog("Illegal Gopher format: No Path: %s\n", buf);
	    goto gopher_enum_cont;
	}
	*q = (char) '\0';
	gopher_path = xstrdup(rfc1738_escape(p));

	p = q + 1;
	if ((q = strchr(p, '\t')) == NULL) {
	    errorlog("Illegal Gopher format: No Host: %s\n", buf);
	    goto gopher_enum_cont;
	}
	*q = (char) '\0';
	gopher_host = xstrdup(p);

	p = q + 1;
	if ((q = strchr(p, '\n')) == NULL) {
	    errorlog("Illegal Gopher format: No Port: %s\n", buf);
	    goto gopher_enum_cont;
	}
	*q = (char) '\0';
	gopher_port = xstrdup(p);

	/* Fix for wierd cross-site Gopher links - wessels */
	if (!strncasecmp(gopher_path, "ftp%3a", 6))
	    goto gopher_enum_cont;
	if (!strncasecmp(gopher_path, "ftp:", 4))
	    goto gopher_enum_cont;
	if (!strncasecmp(gopher_path, "exec%3a", 7))
	    goto gopher_enum_cont;
	if (!strncasecmp(gopher_path, "exec:", 5))
	    goto gopher_enum_cont;

	sprintf(newurl, "gopher://%s:%d/%c%s\n", gopher_host,
	    atoi(gopher_port), gopher_name[0], gopher_path);
	l = (list_t *) xmalloc(sizeof(list_t));
	l->ptr = (void *) xstrdup(newurl);
	l->next = (list_t *) NULL;
	*Tail = l;
	Tail = &(l->next);

      gopher_enum_cont:
	if (gopher_name)
	    xfree(gopher_name);
	gopher_name = NULL;
	if (gopher_path)
	    xfree(gopher_path);
	gopher_path = NULL;
	if (gopher_host)
	    xfree(gopher_host);
	gopher_host = NULL;
	if (gopher_port)
	    xfree(gopher_port);
	gopher_port = NULL;
	if (urlbuf)
	    xfree(urlbuf);
	gopher_name = NULL;
    }
    fclose(fp);
    return (head);
}

/*
 *  process_url() - Retrieves the given URL, computes an MD5,
 *  and extracts the list of menu pointers within the documents.
 */
static void process_url(up, depth)
     URL *up;
     int depth;
{
    list_t *head = 0;
    list_t *l = 0;
    list_t *next_l = 0;
    char *url;
    URL *tup;

    if (max_depth > 0 && depth > max_depth) {
	Debug(43, 1, ("Maximum Depth of %d Reached: %s\n",
		max_depth, up->url));
	url_close(up);
	return;
    }
    Debug(43, 1, ("Processing: [%2d] %s\n", depth, up->url));

    if ((head = gopher_enum(up)) == NULL) {
	url_close(up);
	return;
    }

    url_close(up);

    /*
     *  Now, for each URL in the list, call process_url() if
     *  the URL is a Gopher url and it is on the same host
     */
    for (l = head; l; l = next_l) {
	next_l = l->next;
	url = (char *) l->ptr;
	if (url == (char *) NULL)
	    goto free_list_entry;
	if ((tup = url_open(url)) == NULL)
	    goto free_list_entry;
	if ((tup->type != URL_GOPHER)) {
	    url_close(tup);
	    goto free_list_entry;
	}
	if (tup->gophertype >= 2) {	/* ignore everything 2 or higher */
	    url_close(tup);
	    goto free_list_entry;
	}
	process_url(tup, depth + 1);	/* should be a 1 - menu */

      free_list_entry:
	xfree(l->ptr);
	xfree(l);
    }
}

/* ---------------------------------------------------------------------- */

/*
 *  initialize() - Basic init routines
 */
static void initialize()
{
    char *s;
    FILE *logfp = NULL;

#ifdef USE_HOST_CACHE
    host_cache_init();
#endif

    max_depth = url_max = host_max = 0;
    if ((s = getenv("HARVEST_URL_MAX")) != NULL)
	url_max = atoi(s);
    if ((s = getenv("HARVEST_HOST_MAX")) != NULL)
	host_max = atoi(s);
    if ((s = getenv("HARVEST_DEPTH_MAX")) != NULL)
	max_depth = atoi(s);
    if ((s = getenv("HARVEST_DEPTH_CUR")) != NULL)
	start_depth = atoi(s);
    Debug(43, 9, ("HARVEST_DEPTH_CUR=%d\n", s ? s : "NULL"));
    if (url_max < 1)
	url_max = 250;		/* hard-coded maximum */
    if (host_max < 1)
	host_max = 1;		/* hard-coded maximum */
    if (max_depth < 1)
	max_depth = 0;		/* hard-coded maximum */
    host_filterfile = getenv("HARVEST_HOST_FILTER");
    url_filterfile = getenv("HARVEST_URL_FILTER");

    if (getenv("HARVEST_GATHERER_LOGFILE") != (char *) NULL)
	logfp = fopen(getenv("HARVEST_GATHERER_LOGFILE"), "a+");
    if (logfp == (FILE *) NULL)
	logfp = stderr;
    init_log3("gopherenum-depth", logfp, stderr);
    init_url();
    filter_initialize();

    /* Open GDBM databases to keep track of where we've been */
    urldb_filename = xstrdup(tempnam(NULL, "Gurl"));
    urldbf = gdbm_open(urldb_filename, 0, GDBM_NEWDB, 0644, NULL);
    if (urldbf == NULL) {
	log_errno(urldb_filename);
	fatal("gdbm_open: %s: %s", urldb_filename,
	    gdbm_strerror(gdbm_errno));
    }
    hostdb_filename = xstrdup(tempnam(NULL, "Ghost"));
    hostdbf = gdbm_open(hostdb_filename, 0, GDBM_NEWDB, 0644, NULL);
    if (hostdbf == NULL) {
	log_errno(hostdb_filename);
	fatal("gdbm_open: %s: %s", hostdb_filename,
	    gdbm_strerror(gdbm_errno));
    }
    md5db_filename = xstrdup(tempnam(NULL, "Gmd5"));
    md5dbf = gdbm_open(md5db_filename, 0, GDBM_NEWDB, 0644, NULL);
    if (md5dbf == NULL) {
	log_errno(md5db_filename);
	fatal("gdbm_open: %s: %s", md5db_filename,
	    gdbm_strerror(gdbm_errno));
    }
}

/* Die gracefully */
static void sigdie()
{
    if (urldbf != NULL)
	gdbm_close(urldbf);
    if (hostdbf != NULL)
	gdbm_close(hostdbf);
    if (md5dbf != NULL)
	gdbm_close(md5dbf);

    /*
     * (void) unlink(urldb_filename);
     * (void) unlink(hostdb_filename);
     * (void) unlink(md5db_filename);
     */

    crremove(urldb_filename);
    crremove(hostdb_filename);
    crremove(md5db_filename);

    exit(0);
}

/* ---------------------------------------------------------------------- */

static void usage()
{
    fprintf(stderr, "Usage: gopherenum-depth gopher-URL\n");
    exit(1);
}

int main(argc, argv)
     int argc;
     char **argv;
{
    URL *up;

    debug_init();
    for (argc--, argv++; argc > 0 && **argv == '-'; argc--, argv++) {
	if (strncmp(*argv, "-D", 2) == 0) {
	    debug_flag(*argv);
	}
    }

    if (argc != 1)
	usage();

    signal(SIGTERM, sigdie);	/* Die gracefully */
    signal(SIGINT, sigdie);
    signal(SIGPIPE, sigdie);	/* Clean up on broken pipe */

    initialize();		/* Initialize */

    /* Grab the RootNode URL from the command line */
    if ((up = url_open(*argv)) == NULL || up->type != URL_GOPHER)
	usage();

    /* Mark the RootNode */
    tree_root = xstrdup(up->url);
    printf("%s\n", up->url);	/* Print tree root */

    process_url(up, start_depth);	/* Do the Enumeration recursively */

    url_close(up);		/* Clean up */
    finish_url();
    sigdie();
    /* NOTREACHED */
}