http.c

一个很有名的浏览器

/* Internal "http" protocol implementation */
/* $Id: http.c,v 1.387.2.5 2005/05/01 22:47:55 jonas Exp $ */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_FCNTL_H
#include <fcntl.h> /* OS/2 needs this after sys/types.h */
#endif
#ifdef HAVE_LIMITS_H
#include <limits.h>
#endif

#include "elinks.h"

#include "cache/cache.h"
#include "config/options.h"
#include "cookies/cookies.h"
#include "intl/charsets.h"
#include "intl/gettext/libintl.h"
#include "lowlevel/connect.h"
#include "lowlevel/sysname.h"
#include "modules/module.h"
#include "osdep/ascii.h"
#include "osdep/osdep.h"
#include "protocol/auth/auth.h"
#include "protocol/auth/digest.h"
#include "protocol/date.h"
#include "protocol/header.h"
#include "protocol/http/blacklist.h"
#include "protocol/http/codes.h"
#include "protocol/http/http.h"
#include "protocol/uri.h"
#include "sched/connection.h"
#include "sched/session.h"
#include "ssl/connect.h"
#include "ssl/ssl.h"
#include "terminal/terminal.h"
#include "util/base64.h"
#include "util/conv.h"
#include "util/memory.h"
#include "util/string.h"


static struct auth_entry proxy_auth;

static unsigned char *accept_charset = NULL;


static struct option_info http_options[] = {
	INIT_OPT_TREE("protocol", N_("HTTP"),
		"http", 0,
		N_("HTTP-specific options.")),


	INIT_OPT_TREE("protocol.http", N_("Server bug workarounds"),
		"bugs", 0,
		N_("Server-side HTTP bugs workarounds.")),

	INIT_OPT_BOOL("protocol.http.bugs", N_("Do not send Accept-Charset"),
		"accept_charset", 0, 1,
		N_("The Accept-Charset header is quite long and sending it can trigger\n"
		"bugs in some rarely found servers.")),

	INIT_OPT_BOOL("protocol.http.bugs", N_("Allow blacklisting"),
		"allow_blacklist", 0, 1,
		N_("Allow blacklisting of buggy servers.")),

	INIT_OPT_BOOL("protocol.http.bugs", N_("Broken 302 redirects"),
		"broken_302_redirect", 0, 1,
		N_("Broken 302 redirect (violates RFC but compatible with Netscape).\n"
		"This is a problem for a lot of web discussion boards and the like.\n"
		"If they will do strange things to you, try to play with this.")),

	INIT_OPT_BOOL("protocol.http.bugs", N_("No keepalive after POST requests"),
		"post_no_keepalive", 0, 0,
		N_("Disable keepalive connection after POST request.")),

	INIT_OPT_BOOL("protocol.http.bugs", N_("Use HTTP/1.0"),
		"http10", 0, 0,
		N_("Use HTTP/1.0 protocol instead of HTTP/1.1.")),

	INIT_OPT_TREE("protocol.http", N_("Proxy configuration"),
		"proxy", 0,
		N_("HTTP proxy configuration.")),

	INIT_OPT_STRING("protocol.http.proxy", N_("Host and port-number"),
		"host", 0, "",
		N_("Host and port-number (host:port) of the HTTP proxy, or blank.\n"
		"If it's blank, HTTP_PROXY environment variable is checked as well.")),

	INIT_OPT_STRING("protocol.http.proxy", N_("Username"),
		"user", 0, "",
		N_("Proxy authentication username.")),

	INIT_OPT_STRING("protocol.http.proxy", N_("Password"),
		"passwd", 0, "",
		N_("Proxy authentication password.")),


	INIT_OPT_TREE("protocol.http", N_("Referer sending"),
		"referer", 0,
		N_("HTTP referer sending options. HTTP referer is a special header\n"
		"sent in the HTTP requests, which is supposed to contain the previous\n"
		"page visited by the browser. This way, the server can know what link\n"
		"did you follow when accessing that page. However, this behaviour\n"
		"can unfortunately considerably affect privacy and can lead even to a\n"
		"security problem on some badly designed web pages.")),

	INIT_OPT_INT("protocol.http.referer", N_("Policy"),
		"policy", 0,
		REFERER_NONE, REFERER_TRUE, REFERER_TRUE,
		N_("Mode of sending HTTP referer:\n"
		"0 is send no referer\n"
		"1 is send current URL as referer\n"
		"2 is send fixed fake referer\n"
		"3 is send previous URL as referer (correct, but insecure)")),

	INIT_OPT_STRING("protocol.http.referer", N_("Fake referer URL"),
		"fake", 0, "",
		N_("Fake referer to be sent when policy is 2.")),


	INIT_OPT_STRING("protocol.http", N_("Send Accept-Language header"),
		"accept_language", 0, "",
		N_("Send Accept-Language header.")),

	INIT_OPT_BOOL("protocol.http", N_("Use UI language as Accept-Language"),
		"accept_ui_language", 0, 1,
		N_("Request localised versions of documents from web-servers (using the\n"
		"Accept-Language header) using the language you have configured for\n"
		"ELinks' user-interface (this also affects navigator.language ECMAScript\n"
		"value available to scripts). Note that some see this as a potential\n"
		"security risk because it tells web-masters and the FBI sniffers about\n"
		"your language preference.")),

	INIT_OPT_BOOL("protocol.http", N_("Activate HTTP TRACE debugging"),
		"trace", 0, 0,
		N_("If active, all HTTP requests are sent with TRACE as their method\n"
		"rather than GET or POST. This is useful for debugging of both ELinks\n"
		"and various server-side scripts --- the server only returns the client's\n"
		"request back to the client verbatim. Note that this type of request may\n"
		"not be enabled on all servers.")),

	/* OSNews.com is supposed to be relying on the textmode token, at least. */
	INIT_OPT_STRING("protocol.http", N_("User-agent identification"),
		"user_agent", 0, "ELinks/%v (textmode; %s; %t-%b)",
		N_("Change the User Agent ID. That means identification string, which\n"
		"is sent to HTTP server when a document is requested. The 'textmode'\n"
		"token in the first field is our silent attempt to establish this as\n"
		"a standard for new textmode user agents, so that the webmasters can\n"
		"have just a single uniform test for these if they are ie. pushing\n"
		"some lite version to them automagically.\n"
		"%v in the string means ELinks version\n"
		"%s in the string means system identification\n"
		"%t in the string means size of the terminal\n"
		"%b in the string means number of bars displayed by ELinks\n"
		"Use \" \" if you don't want any User-Agent header to be sent at all.")),


	INIT_OPT_TREE("protocol", N_("HTTPS"),
  		"https", 0,
		N_("HTTPS-specific options.")),

	INIT_OPT_TREE("protocol.https", N_("Proxy configuration"),
	  	"proxy", 0,
		N_("HTTPS proxy configuration.")),

	INIT_OPT_STRING("protocol.https.proxy", N_("Host and port-number"),
	  	"host", 0, "",
		N_("Host and port-number (host:port) of the HTTPS CONNECT proxy, or blank.\n"
		"If it's blank, HTTPS_PROXY environment variable is checked as well.")),
	NULL_OPTION_INFO,
};

static void done_http();

struct module http_protocol_module = struct_module(
	/* name: */		N_("HTTP"),
	/* options: */		http_options,
	/* hooks: */		NULL,
	/* submodules: */	NULL,
	/* data: */		NULL,
	/* init: */		NULL,
	/* done: */		done_http
);


static void decompress_shutdown(struct connection *);

static void
done_http()
{
	mem_free_if(proxy_auth.realm);
	mem_free_if(proxy_auth.nonce);
	mem_free_if(proxy_auth.opaque);

	free_blacklist();

	if (accept_charset)
		mem_free(accept_charset);
}

static void
init_accept_charset()
{
	struct string ac;
	unsigned char *cs;
	int i;

	if (!init_string(&ac)) return;

	for (i = 0; (cs = get_cp_mime_name(i)); i++) {
		if (ac.length) {
			add_to_string(&ac, ", ");
		} else {
			add_to_string(&ac, "Accept-Charset: ");
		}
		add_to_string(&ac, cs);
	}

	if (ac.length) {
		add_crlf_to_string(&ac);
	}

	accept_charset = squeezastring(&ac);

	done_string(&ac);
}


unsigned char *
subst_user_agent(unsigned char *fmt, unsigned char *version,
		 unsigned char *sysname, unsigned char *termsize)
{
	struct string agent;

	if (!init_string(&agent)) return NULL;

	while (*fmt) {
		int p;

		for (p = 0; fmt[p] && fmt[p] != '%'; p++);

		add_bytes_to_string(&agent, fmt, p);
		fmt += p;

		if (*fmt != '%') continue;

		fmt++;
		switch (*fmt) {
			case 'b':
				if (!list_empty(sessions)) {
					unsigned char bs[4] = "";
					int blen = 0;
					struct session *ses = sessions.prev;
					int bars = ses->status.show_status_bar
						+ ses->status.show_tabs_bar
						+ ses->status.show_title_bar;

					ulongcat(bs, &blen, bars, 2, 0);
					add_to_string(&agent, bs);
				}
				break;
			case 'v':
				add_to_string(&agent, version);
				break;
			case 's':
				add_to_string(&agent, sysname);
				break;
			case 't':
				if (termsize)
					add_to_string(&agent, termsize);
				break;
			default:
				add_bytes_to_string(&agent, fmt - 1, 2);
				break;
		}
		if (*fmt) fmt++;
	}

	return agent.source;
}

static void
add_url_to_http_string(struct string *header, struct uri *uri, int components)
{
	/* This block substitues spaces in URL by %20s. This is
	 * certainly not the right place where to do it, but now the
	 * behaviour is at least improved compared to what we had
	 * before. We should probably encode all URLs as early as
	 * possible, and possibly decode them back in protocol
	 * backends. --pasky */
	unsigned char *string = get_uri_string(uri, components);
	unsigned char *data = string;

	if (!string) return;

	while (*data) {
		int len = strcspn(data, " \t\r\n\\");

		add_bytes_to_string(header, data, len);

		if (!data[len]) break;

		if (data[len++] == '\\')
			add_char_to_string(header, '/');
		else
			add_to_string(header, "%20");

		data	+= len;
	}

	mem_free(string);
}

/* Parse from @end - 1 to @start and set *@value to integer found.
 * It returns -1 if not a number, 0 otherwise.
 * @end should be > @start. */
static int
revstr2num(unsigned char *start, unsigned char *end, int *value)
{
	int q = 1, val = 0;

	do {
		--end;
		if (!isdigit(*end)) return -1; /* NaN */
		val += (*end - '0') * q;
		q *= 10;
	} while (end > start);

	*value = val;
	return 0;
}

/* This function extracts code, major and minor version from string
 * "\s*HTTP/\d+.\d+\s+\d\d\d..."
 * It returns a negative value on error, 0 on success.
 */
static int
get_http_code(unsigned char *head, int *code, struct http_version *version)
{
	unsigned char *start;

	*code = 0;
	version->major = 0;
	version->minor = 0;

	/* Ignore spaces. */
	while (*head == ' ') head++;

	/* HTTP/ */
	if (toupper(*head) != 'H' || toupper(*++head) != 'T' ||
	    toupper(*++head) != 'T' || toupper(*++head) != 'P'
	    || *++head != '/')
		return -1;

	/* Version */
	start = ++head;
	/* Find next '.' */
	while (*head && *head != '.') head++;
	/* Sanity check. */
	if (!*head || !(head - start)
	    || (head - start) > 4
	    || !isdigit(*(head + 1)))
		return -2;

	/* Extract major version number. */
	if (revstr2num(start, head, &version->major)) return -3; /* NaN */

	start = head + 1;

	/* Find next ' '. */
	while (*head && *head != ' ') head++;
	/* Sanity check. */
	if (!*head || !(head - start) || (head - start) > 4) return -4;

	/* Extract minor version number. */
	if (revstr2num(start, head, &version->minor)) return -5; /* NaN */

	/* Ignore spaces. */
	while (*head == ' ') head++;

	/* Sanity check for code. */
	if (head[0] < '1' || head[0] > '9' ||
	    !isdigit(head[1]) ||
	    !isdigit(head[2]))
		return -6; /* Invalid code. */

	/* Extract code. */
	*code = (head[0] - '0') * 100 + (head[1] - '0') * 10 + head[2] - '0';

	return 0;
}

static int
check_http_server_bugs(struct uri *uri, struct http_connection_info *info,
		       unsigned char *head)
{
	unsigned char *server, **s;
	static unsigned char *buggy_servers[] = {
		"mod_czech/3.1.0",
		"Purveyor",
		"Netscape-Enterprise",
		NULL
	};

	if (!get_opt_bool("protocol.http.bugs.allow_blacklist")
	    || HTTP_1_0(info->sent_version))
		return 0;

	server = parse_header(head, "Server", NULL);
	if (!server)
		return 0;

	for (s = buggy_servers; *s; s++) {
		if (strstr(server, *s)) {
			add_blacklist_entry(uri, SERVER_BLACKLIST_HTTP10);
			break;
		}
	}

	mem_free(server);
	return (*s != NULL);
}

static void
http_end_request(struct connection *conn, enum connection_state state,
		 int notrunc)
{
	set_connection_state(conn, state);
	decompress_shutdown(conn);

	if (conn->state == S_OK && conn->cached) {
		if (!notrunc) truncate_entry(conn->cached, conn->from, 1);
		conn->cached->incomplete = 0;
		conn->cached->preformatted = 0;
	}

	if (conn->info && !((struct http_connection_info *) conn->info)->close
	    && (!conn->socket.ssl) /* We won't keep alive ssl connections */
	    && (!get_opt_bool("protocol.http.bugs.post_no_keepalive")
		|| !conn->uri->post)) {
		add_keepalive_connection(conn, HTTP_KEEPALIVE_TIMEOUT, NULL);
	} else {