hget.c

这是一个同样来自贝尔实验室的和UNIX有着渊源的操作系统, 其简洁的设计和实现易于我们学习和理解

#include <u.h>
#include <libc.h>
#include <ctype.h>
#include <bio.h>
#include <ip.h>
#include <libsec.h>
#include <auth.h>

typedef struct URL URL;
struct URL
{
	int	method;
	char	*host;
	char	*port;
	char	*page;
	char	*etag;
	char	*redirect;
	char	*postbody;
	char	*cred;
	long	mtime;
};

typedef struct Range Range;
struct Range
{
	long	start;	/* only 2 gig supported, tdb */
	long	end;
};

typedef struct Out Out;
struct Out
{
	int fd;
	int offset;				/* notional current offset in output */
	int written;			/* number of bytes successfully transferred to output */
	DigestState *curr;		/* digest state up to offset (if known) */
	DigestState *hiwat;		/* digest state of all bytes written */
};

enum
{
	Other,
	Http,
	Https,
	Ftp,
};

enum
{
	Eof = 0,
	Error = -1,
	Server = -2,
	Changed = -3,
};

int debug;
char *ofile;


int	doftp(URL*, URL*, Range*, Out*, long);
int	dohttp(URL*, URL*,  Range*, Out*, long);
int	crackurl(URL*, char*);
Range*	crackrange(char*);
int	getheader(int, char*, int);
int	httpheaders(int, int, URL*, Range*);
int	httprcode(int);
int	cistrncmp(char*, char*, int);
int	cistrcmp(char*, char*);
void	initibuf(void);
int	readline(int, char*, int);
int	readibuf(int, char*, int);
int	dfprint(int, char*, ...);
void	unreadline(char*);
int	output(Out*, char*, int);
void	setoffset(Out*, int);

int	verbose;
char	*net;
char	tcpdir[NETPATHLEN];
int	headerprint;

struct {
	char	*name;
	int	(*f)(URL*, URL*, Range*, Out*, long);
} method[] = {
	[Http]	{ "http",	dohttp },
	[Https]	{ "https",	dohttp },
	[Ftp]	{ "ftp",	doftp },
	[Other]	{ "_______",	nil },
};

void
usage(void)
{
	fprint(2, "usage: %s [-dhv] [-o outfile] [-p body] [-x netmtpt] url\n", argv0);
	exits("usage");
}

void
main(int argc, char **argv)
{
	URL u;
	Range r;
	int errs, n;
	ulong mtime;
	Dir *d;
	char postbody[4096], *p, *e, *t, *hpx;
	URL px; // Proxy
	Out out;

	ofile = nil;
	p = postbody;
	e = p + sizeof(postbody);
	r.start = 0;
	r.end = -1;
	mtime = 0;
	memset(&u, 0, sizeof(u));
	memset(&px, 0, sizeof(px));
	hpx = getenv("httpproxy");

	ARGBEGIN {
	case 'o':
		ofile = EARGF(usage());
		break;
	case 'd':
		debug = 1;
		break;
	case 'h':
		headerprint = 1;
		break;
	case 'v':
		verbose = 1;
		break;
	case 'x':
		net = EARGF(usage());
		break;
	case 'p':
		t = EARGF(usage());
		if(p != postbody)
			p = seprint(p, e, "&%s", t);
		else
			p = seprint(p, e, "%s", t);
		u.postbody = postbody;
		
		break;
	default:
		usage();
	} ARGEND;

	if(net != nil){
		if(strlen(net) > sizeof(tcpdir)-5)
			sysfatal("network mount point too long");
		snprint(tcpdir, sizeof(tcpdir), "%s/tcp", net);
	} else
		snprint(tcpdir, sizeof(tcpdir), "tcp");

	if(argc != 1)
		usage();

	
	out.fd = 1;
	out.written = 0;
	out.offset = 0;
	out.curr = nil;
	out.hiwat = nil;
	if(ofile != nil){
		d = dirstat(ofile);
		if(d == nil){
			out.fd = create(ofile, OWRITE, 0664);
			if(out.fd < 0)
				sysfatal("creating %s: %r", ofile);
		} else {
			out.fd = open(ofile, OWRITE);
			if(out.fd < 0)
				sysfatal("can't open %s: %r", ofile);
			r.start = d->length;
			mtime = d->mtime;
			free(d);
		}
	}

	errs = 0;

	if(crackurl(&u, argv[0]) < 0)
		sysfatal("%r");
	if(hpx && crackurl(&px, hpx) < 0)
		sysfatal("%r");

	for(;;){
		setoffset(&out, 0);
		/* transfer data */
		werrstr("");
		n = (*method[u.method].f)(&u, &px, &r, &out, mtime);

		switch(n){
		case Eof:
			exits(0);
			break;
		case Error:
			if(errs++ < 10)
				continue;
			sysfatal("too many errors with no progress %r");
			break;
		case Server:
			sysfatal("server returned: %r");
			break;
		}

		/* forward progress */
		errs = 0;
		r.start += n;
		if(r.start >= r.end)
			break;
	}

	exits(0);
}

int
crackurl(URL *u, char *s)
{
	char *p;
	int i;

	if(u->page != nil){
		free(u->page);
		u->page = nil;
	}

	/* get type */ 
	for(p = s; *p; p++){
		if(*p == '/'){
			p = s;
			if(u->method == Other){
				werrstr("missing method");
				return -1;
			}
			if(u->host == nil){
				werrstr("missing host");
				return -1;
			}
			u->page = strdup(p);
			return 0;
		}
		if(*p == ':' && *(p+1)=='/' && *(p+2)=='/'){
			*p = 0;
			p += 3;
			for(i = 0; i < nelem(method); i++){
				if(cistrcmp(s, method[i].name) == 0){
					u->method = i;
					break;
				}
			}
			break;
		}
	}

	if(u->method == Other){
		werrstr("unsupported URL type %s", s);
		return -1;
	}

	/* get system */
	free(u->host);
	s = p;
	p = strchr(s, '/');
	if(p == nil){
		u->host = strdup(s);
		u->page = strdup("/");
	} else {
		u->page = strdup(p);
		*p = 0;
		u->host = strdup(s);
		*p = '/';
	}

	if(p = strchr(u->host, ':')) {
		*p++ = 0;
		u->port = p;
	} else 
		u->port = method[u->method].name;

	if(*(u->host) == 0){
		werrstr("bad url, null host");
		return -1;
	}

	return 0;
}

char *day[] = {
	"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"
};

char *month[] = {
	"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
};

struct
{
	int	fd;
	long	mtime;
} note;

void
catch(void*, char*)
{
	Dir d;

	nulldir(&d);
	d.mtime = note.mtime;
	if(dirfwstat(note.fd, &d) < 0)
		sysfatal("catch: can't dirfwstat: %r");
	noted(NDFLT);
}

int
dohttp(URL *u, URL *px, Range *r, Out *out, long mtime)
{
	int fd, cfd;
	int redirect, auth, loop;
	int n, rv, code;
	long tot, vtime;
	Tm *tm;
	char buf[1024];
	char err[ERRMAX];


	/*  always move back to a previous 512 byte bound because some
	 *  servers can't seem to deal with requests that start at the
	 *  end of the file
	 */
	if(r->start)
		r->start = ((r->start-1)/512)*512;

	/* loop for redirects, requires reading both response code and headers */
	fd = -1;
	for(loop = 0; loop < 32; loop++){
		if(px->host == nil){
			fd = dial(netmkaddr(u->host, tcpdir, u->port), 0, 0, 0);
		} else {
			fd = dial(netmkaddr(px->host, tcpdir, px->port), 0, 0, 0);
		}
		if(fd < 0)
			return Error;

		if(u->method == Https){
			int tfd;
			TLSconn conn;

			memset(&conn, 0, sizeof conn);
			tfd = tlsClient(fd, &conn);
			if(tfd < 0){
				fprint(2, "tlsClient: %r\n");
				close(fd);
				return Error;
			}
			/* BUG: check cert here? */
			if(conn.cert)
				free(conn.cert);
			close(fd);
			fd = tfd;
		}

		/* write request, use range if not start of file */
		if(u->postbody == nil){
			if(px->host == nil){
				dfprint(fd,	"GET %s HTTP/1.0\r\n"
						"Host: %s\r\n"
						"User-agent: Plan9/hget\r\n"
						"Cache-Control: no-cache\r\n"
						"Pragma: no-cache\r\n",
						u->page, u->host);
			} else {
				dfprint(fd,	"GET http://%s%s HTTP/1.0\r\n"
						"Host: %s\r\n"
						"User-agent: Plan9/hget\r\n"
						"Cache-Control: no-cache\r\n"
						"Pragma: no-cache\r\n",
						u->host, u->page, u->host);
			}
			if(u->cred)
				dfprint(fd,	"Authorization: Basic %s\r\n",
						u->cred);
		} else {
			dfprint(fd,	"POST %s HTTP/1.0\r\n"
					"Host: %s\r\n"
					"Content-type: application/x-www-form-urlencoded\r\n"
					"Content-length: %d\r\n"
					"User-agent: Plan9/hget\r\n",
					u->page, u->host, strlen(u->postbody));
			if(u->cred)
				dfprint(fd, "Authorization: Basic %s\r\n", u->cred);
		}
		if(r->start != 0){
			dfprint(fd, "Range: bytes=%d-\n", r->start);
			if(u->etag != nil){
				dfprint(fd, "If-range: %s\n", u->etag);
			} else {
				tm = gmtime(mtime);
				dfprint(fd, "If-range: %s, %d %s %d %2d:%2.2d:%2.2d GMT\n",
					day[tm->wday], tm->mday, month[tm->mon],
					tm->year+1900, tm->hour, tm->min, tm->sec);
			}
		}
		if((cfd = open("/mnt/webcookies/http", ORDWR)) >= 0){
			if(fprint(cfd, "http://%s%s", u->host, u->page) > 0){
				while((n = read(cfd, buf, sizeof buf)) > 0){
					if(debug)
						write(2, buf, n);
					write(fd, buf, n);
				}
			}else{
				close(cfd);
				cfd = -1;
			}
		}
			
		dfprint(fd, "\r\n", u->host);
		if(u->postbody)
			dfprint(fd,	"%s", u->postbody);

		auth = 0;
		redirect = 0;
		initibuf();
		code = httprcode(fd);
		switch(code){
		case Error:	/* connection timed out */
		case Eof:
			close(fd);
			close(cfd);
			return code;

		case 200:	/* OK */
		case 201:	/* Created */
		case 202:	/* Accepted */
			if(ofile == nil && r->start != 0)
				sysfatal("page changed underfoot");
			break;

		case 204:	/* No Content */
			sysfatal("No Content");

		case 206:	/* Partial Content */
			setoffset(out, r->start);
			break;

		case 301:	/* Moved Permanently */
		case 302:	/* Moved Temporarily */
			redirect = 1;
			u->postbody = nil;
			break;

		case 304:	/* Not Modified */
			break;

		case 400:	/* Bad Request */
			sysfatal("Bad Request");

		case 401:	/* Unauthorized */
			if (auth)
				sysfatal("Authentication failed");
			auth = 1;
			break;

		case 402:	/* ??? */
			sysfatal("Unauthorized");

		case 403:	/* Forbidden */
			sysfatal("Forbidden by server");

		case 404:	/* Not Found */
			sysfatal("Not found on server");

		case 407:	/* Proxy Authentication */
			sysfatal("Proxy authentication required");

		case 500:	/* Internal server error */
			sysfatal("Server choked");

		case 501:	/* Not implemented */
			sysfatal("Server can't do it!");

		case 502:	/* Bad gateway */
			sysfatal("Bad gateway");

		case 503:	/* Service unavailable */
			sysfatal("Service unavailable");
		
		default:
			sysfatal("Unknown response code %d", code);
		}

		if(u->redirect != nil){
			free(u->redirect);
			u->redirect = nil;
		}

		rv = httpheaders(fd, cfd, u, r);
		close(cfd);
		if(rv != 0){
			close(fd);
			return rv;
		}

		if(!redirect && !auth)
			break;

		if (redirect){
			if(u->redirect == nil)
				sysfatal("redirect: no URL");
			if(crackurl(u, u->redirect) < 0)
				sysfatal("redirect: %r");
		}
	}

	/* transfer whatever you get */
	if(ofile != nil && u->mtime != 0){
		note.fd = out->fd;
		note.mtime = u->mtime;
		notify(catch);
	}

	tot = 0;
	vtime = 0;
	for(;;){
		n = readibuf(fd, buf, sizeof(buf));
		if(n <= 0)
			break;
		if(output(out, buf, n) != n)
			break;
		tot += n;
		if(verbose && (vtime != time(0) || r->start == r->end)) {
			vtime = time(0);
			fprint(2, "%ld %ld\n", r->start+tot, r->end);
		}
	}
	notify(nil);
	close(fd);

	if(ofile != nil && u->mtime != 0){
		Dir d;

		rerrstr(err, sizeof err);
		nulldir(&d);
		d.mtime = u->mtime;
		if(dirfwstat(out->fd, &d) < 0)
			fprint(2, "couldn't set mtime: %r\n");
		errstr(err, sizeof err);
	}

	return tot;
}

/* get the http response code */
int
httprcode(int fd)
{
	int n;
	char *p;
	char buf[256];

	n = readline(fd, buf, sizeof(buf)-1);
	if(n <= 0)
		return n;
	if(debug)
		fprint(2, "%d <- %s\n", fd, buf);
	p = strchr(buf, ' ');
	if(strncmp(buf, "HTTP/", 5) != 0 || p == nil){
		werrstr("bad response from server");
		return -1;
	}
	buf[n] = 0;
	return atoi(p+1);
}

/* read in and crack the http headers, update u and r */
void	hhetag(char*, URL*, Range*);
void	hhmtime(char*, URL*, Range*);
void	hhclen(char*, URL*, Range*);
void	hhcrange(char*, URL*, Range*);
void	hhuri(char*, URL*, Range*);
void	hhlocation(char*, URL*, Range*);
void	hhauth(char*, URL*, Range*);

struct {
	char *name;
	void (*f)(char*, URL*, Range*);
} headers[] = {
	{ "etag:", hhetag },
	{ "last-modified:", hhmtime },
	{ "content-length:", hhclen },
	{ "content-range:", hhcrange },
	{ "uri:", hhuri },
	{ "location:", hhlocation },
	{ "WWW-Authenticate:", hhauth },
};
int
httpheaders(int fd, int cfd, URL *u, Range *r)
{
	char buf[2048];
	char *p;
	int i, n;

	for(;;){
		n = getheader(fd, buf, sizeof(buf));
		if(n <= 0)
			break;
		if(cfd >= 0)
			fprint(cfd, "%s\n", buf);
		for(i = 0; i < nelem(headers); i++){
			n = strlen(headers[i].name);
			if(cistrncmp(buf, headers[i].name, n) == 0){
				/* skip field name and leading white */
				p = buf + n;
				while(*p == ' ' || *p == '\t')
					p++;

				(*headers[i].f)(p, u, r);
				break;
			}
		}
	}
	return n;
}

/*
 *  read a single mime header, collect continuations.
 *
 *  this routine assumes that there is a blank line twixt
 *  the header and the message body, otherwise bytes will
 *  be lost.
 */
int
getheader(int fd, char *buf, int n)
{
	char *p, *e;
	int i;

	n--;
	p = buf;
	for(e = p + n; ; p += i){
		i = readline(fd, p, e-p);
		if(i < 0)
			return i;

		if(p == buf){
			/* first line */
			if(strchr(buf, ':') == nil)
				break;		/* end of headers */
		} else {
			/* continuation line */
			if(*p != ' ' && *p != '\t'){
				unreadline(p);
				*p = 0;
				break;		/* end of this header */
			}
		}
	}
	if(headerprint)
		print("%s\n", buf);

	if(debug)
		fprint(2, "%d <- %s\n", fd, buf);
	return p-buf;
}

void
hhetag(char *p, URL *u, Range*)
{
	if(u->etag != nil){
		if(strcmp(u->etag, p) != 0)
			sysfatal("file changed underfoot");
	} else
		u->etag = strdup(p);
}

char*	monthchars = "janfebmaraprmayjunjulaugsepoctnovdec";

void
hhmtime(char *p, URL *u, Range*)
{
	char *month, *day, *yr, *hms;
	char *fields[6];
	Tm tm, now;
	int i;

	i = getfields(p, fields, 6, 1, " \t");
	if(i < 5)
		return;

	day = fields[1];
	month = fields[2];
	yr = fields[3];
	hms = fields[4];

	/* default time */
	now = *gmtime(time(0));
	tm = now;
	tm.yday = 0;

	/* convert ascii month to a number twixt 1 and 12 */
	if(*month >= '0' && *month <= '9'){
		tm.mon = atoi(month) - 1;
		if(tm.mon < 0 || tm.mon > 11)
			tm.mon = 5;
	} else {
		for(p = month; *p; p++)
			*p = tolower(*p);
		for(i = 0; i < 12; i++)
			if(strncmp(&monthchars[i*3], month, 3) == 0){
				tm.mon = i;
				break;
			}
	}

	tm.mday = atoi(day);

	if(hms) {
		tm.hour = strtoul(hms, &p, 10);
		if(*p == ':') {
			p++;
			tm.min = strtoul(p, &p, 10);
			if(*p == ':') {
				p++;
				tm.sec = strtoul(p, &p, 10);
			}
		}
		if(tolower(*p) == 'p')
			tm.hour += 12;
	}

	if(yr) {
		tm.year = atoi(yr);
		if(tm.year >= 1900)
			tm.year -= 1900;
	} else {
		if(tm.mon > now.mon || (tm.mon == now.mon && tm.mday > now.mday+1))