ftp.c

harvest是一个下载html网页得机器人

static char rcsid[] = "ftp.c,v 1.22 1996/01/06 08:07:08 duane Exp";
/*
 *  ftp.c - URL processing for ftp-specific URLs
 *
 *  Darren Hardy, [e-mail], April 1994
 *
 *  DEBUG: section  23, level 1         Common liburl FTP routines
 *
 *  ----------------------------------------------------------------------
 *  Copyright (c) 1994, 1995.  All rights reserved.
 *  
 *    The Harvest software was developed by the Internet Research Task
 *    Force Research Group on Resource Discovery (IRTF-RD):
 *  
 *          Mic Bowman of Transarc Corporation.
 *          Peter Danzig of the University of Southern California.
 *          Darren R. Hardy of the University of Colorado at Boulder.
 *          Udi Manber of the University of Arizona.
 *          Michael F. Schwartz of the University of Colorado at Boulder.
 *          Duane Wessels of the University of Colorado at Boulder.
 *  
 *    This copyright notice applies to software in the Harvest
 *    ``src/'' directory only.  Users should consult the individual
 *    copyright notices in the ``components/'' subdirectories for
 *    copyright information about other software bundled with the
 *    Harvest source code distribution.
 *  
 *  TERMS OF USE
 *    
 *    The Harvest software may be used and re-distributed without
 *    charge, provided that the software origin and research team are
 *    cited in any use of the system.  Most commonly this is
 *    accomplished by including a link to the Harvest Home Page
 *    (http://harvest.cs.colorado.edu/) from the query page of any
 *    Broker you deploy, as well as in the query result pages.  These
 *    links are generated automatically by the standard Broker
 *    software distribution.
 *    
 *    The Harvest software is provided ``as is'', without express or
 *    implied warranty, and with no support nor obligation to assist
 *    in its use, correction, modification or enhancement.  We assume
 *    no liability with respect to the infringement of copyrights,
 *    trade secrets, or any patents, and are not responsible for
 *    consequential damages.  Proper use of the Harvest software is
 *    entirely the responsibility of the user.
 *  
 *  DERIVATIVE WORKS
 *  
 *    Users may make derivative works from the Harvest software, subject 
 *    to the following constraints:
 *  
 *      - You must include the above copyright notice and these 
 *        accompanying paragraphs in all forms of derivative works, 
 *        and any documentation and other materials related to such 
 *        distribution and use acknowledge that the software was 
 *        developed at the above institutions.
 *  
 *      - You must notify IRTF-RD regarding your distribution of 
 *        the derivative work.
 *  
 *      - You must clearly notify users that your are distributing 
 *        a modified version and not the original Harvest software.
 *  
 *      - Any derivative product is also subject to these copyright 
 *        and use restrictions.
 *  
 *    Note that the Harvest software is NOT in the public domain.  We
 *    retain copyright, as specified above.
 *  
 *  HISTORY OF FREE SOFTWARE STATUS
 *  
 *    Originally we required sites to license the software in cases
 *    where they were going to build commercial products/services
 *    around Harvest.  In June 1995 we changed this policy.  We now
 *    allow people to use the core Harvest software (the code found in
 *    the Harvest ``src/'' directory) for free.  We made this change
 *    in the interest of encouraging the widest possible deployment of
 *    the technology.  The Harvest software is really a reference
 *    implementation of a set of protocols and formats, some of which
 *    we intend to standardize.  We encourage commercial
 *    re-implementations of code complying to this set of standards.  
 *  
 */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include "util.h"
#include "url.h"

struct ftp_auth {
	char *host;
	int port;
	char *user;
	char *password;
	struct ftp_auth *next;
};

static struct ftp_auth *FTPAuth = NULL;

/*
 *  ftp_get() - retrieves the URL and prints into the file up->fp.
 *  Returns non-zero on error; 0 on success.
 */
int ftp_get(up)
     URL *up;
{
	static char cmd[BUFSIZ];
	char *pathname = NULL;

	/* Remove leading slash from FTP url-path so that we can        */
	/* handle ftp://user:pw@host/path objects where path and /path  */
	/* are quite different.         -DW                             */

	pathname = up->pathname;
	if (!strcmp(pathname, "/"))
		*pathname = '.';
	if (*pathname == '/')
		pathname++;

	sprintf(cmd, "ftpget \"%s\" \"%s\" \"%s\" %c \"%s\" \"%s\"",
	    up->filename, up->host, pathname, 'I', up->user,
	    up->password);

	return (run_cmd(cmd));
}

static void ftp_init_auth()
{
	static char buf[256];
	struct ftp_auth *tmp = NULL;
	struct ftp_auth *tail = NULL;
	char *filename = NULL;
	FILE *fp = NULL;
	char *t, *s;

	filename = getenv("HARVEST_FTP_AUTHENTICATIONS");
	if (filename == (char *) NULL)
		return;

	Debug(23, 1, ("ftp_init_auth: Open FTP Auth file: %s\n", filename));

	if ((fp = fopen(filename, "r")) == (FILE *) NULL) {
		log_errno(filename);
		return;
	}
	while (fgets(buf, 255, fp)) {
		Debug(23, 1, ("ftp_init_auth: Input line: %s", buf));
		tmp = (struct ftp_auth *) xmalloc(sizeof(struct ftp_auth));
		tmp->port = url_table[URL_FTP].port;
		if ((t = strtok(buf, " \t\n")) == (char *) NULL)
			continue;
		tmp->host = xstrdup(t);
		if ((s = strrchr(tmp->host, ':'))) {
			*s = (char) '\0';
			tmp->port = atoi(s + 1);
		}
		if ((t = strtok(NULL, " \t\n")) == (char *) NULL)
			continue;
		tmp->user = xstrdup(t);
		if ((t = strtok(NULL, " \t\n")) == (char *) NULL)
			continue;
		tmp->password = xstrdup(t);
		tmp->next = (struct ftp_auth *) NULL;
		if (tail)
			tail->next = tmp;
		tail = tmp;
		if (FTPAuth == (struct ftp_auth *) NULL)
			FTPAuth = tmp;
	}
	fclose(fp);
}



int ftp_get_auth(up)
     URL *up;
{
	static int inited = 0;
	struct ftp_auth *t = NULL;


	Debug(23, 1, ("ftp_get_auth: Looking for: %s:%d\n", up->host, up->port));
	if (!inited) {
		ftp_init_auth();
		inited = 1;
	}
	if (FTPAuth == (struct ftp_auth *) NULL)
		return 0;

	for (t = FTPAuth; t; t = t->next) {
		if (!strcasecmp(up->host, t->host) && up->port == t->port) {
			Debug(23, 1, ("ftp_get_auth: Match found: %s:%s\n", t->user, t->password));
			xfree(up->user);
			up->user = xstrdup(t->user);
			xfree(up->password);
			up->password = xstrdup(t->password);
			URL_FLAG_CLR(up->flags, URL_FLAG_PASS_USERINFO);
			return 1;
		}
	}
	Debug(23, 1, ("ftp_get_auth: Match NOT found.\n"));
	return 0;
}