http.c

wget (command line browser) source code

	{
	  proxy_user = opt.proxy_user;
	  proxy_passwd = opt.proxy_passwd;
	}
      else
	{
	  proxy_user = proxy->user;
	  proxy_passwd = proxy->passwd;
	}
      /* #### This does not appear right.  Can't the proxy request,
         say, `Digest' authentication?  */
      if (proxy_user && proxy_passwd)
	proxyauth = basic_authentication_encode (proxy_user, proxy_passwd,
						 "Proxy-Authorization");
    }

  /* String of the form :PORT.  Used only for non-standard ports. */
  port_maybe = NULL;
  if (u->port != scheme_default_port (u->scheme))
    {
      port_maybe = (char *)alloca (numdigit (u->port) + 2);
      sprintf (port_maybe, ":%d", u->port);
    }

  if (!inhibit_keep_alive)
    request_keep_alive = "Connection: Keep-Alive\r\n";
  else
    request_keep_alive = NULL;

  if (opt.cookies)
    cookies = cookie_jar_generate_cookie_header (wget_cookie_jar, u->host,
						 u->port, u->path,
#ifdef HAVE_SSL
						 u->scheme == SCHEME_HTTPS
#else
						 0
#endif
				 );

  if (opt.post_data || opt.post_file_name)
    {
      post_content_type = "Content-Type: application/x-www-form-urlencoded\r\n";
      if (opt.post_data)
	post_data_size = strlen (opt.post_data);
      else
	{
	  post_data_size = file_size (opt.post_file_name);
	  if (post_data_size == -1)
	    {
	      logprintf (LOG_NOTQUIET, "POST data file missing: %s\n",
			 opt.post_file_name);
	      post_data_size = 0;
	    }
	}
      post_content_length = xmalloc (16 + numdigit (post_data_size) + 2 + 1);
      sprintf (post_content_length,
	       "Content-Length: %ld\r\n", post_data_size);
    }

  if (proxy)
    full_path = xstrdup (u->url);
  else
    /* Use the full path, i.e. one that includes the leading slash and
       the query string.  E.g. if u->path is "foo/bar" and u->query is
       "param=value", full_path will be "/foo/bar?param=value".  */
    full_path = url_full_path (u);

  if (strchr (u->host, ':'))
    squares_around_host = 1;

  /* Allocate the memory for the request.  */
  request = (char *)alloca (strlen (command)
			    + strlen (full_path)
			    + strlen (useragent)
			    + strlen (u->host)
			    + (port_maybe ? strlen (port_maybe) : 0)
			    + strlen (HTTP_ACCEPT)
			    + (request_keep_alive
			       ? strlen (request_keep_alive) : 0)
			    + (referer ? strlen (referer) : 0)
			    + (cookies ? strlen (cookies) : 0)
			    + (wwwauth ? strlen (wwwauth) : 0)
			    + (proxyauth ? strlen (proxyauth) : 0)
			    + (range ? strlen (range) : 0)
			    + strlen (pragma_h)
			    + (post_content_type
			       ? strlen (post_content_type) : 0)
			    + (post_content_length
			       ? strlen (post_content_length) : 0)
			    + (opt.user_header ? strlen (opt.user_header) : 0)
			    + 64);
  /* Construct the request.  */
  sprintf (request, "\
%s %s HTTP/1.0\r\n\
User-Agent: %s\r\n\
Host: %s%s%s%s\r\n\
Accept: %s\r\n\
%s%s%s%s%s%s%s%s%s%s\r\n",
	   command, full_path,
	   useragent,
	   squares_around_host ? "[" : "", u->host, squares_around_host ? "]" : "",
	   port_maybe ? port_maybe : "",
	   HTTP_ACCEPT,
	   request_keep_alive ? request_keep_alive : "",
	   referer ? referer : "",
	   cookies ? cookies : "", 
	   wwwauth ? wwwauth : "", 
	   proxyauth ? proxyauth : "", 
	   range ? range : "",
	   pragma_h,
	   post_content_type ? post_content_type : "",
	   post_content_length ? post_content_length : "",
	   opt.user_header ? opt.user_header : "");
  DEBUGP (("---request begin---\n%s", request));

  /* Free the temporary memory.  */
  FREE_MAYBE (wwwauth);
  FREE_MAYBE (proxyauth);
  FREE_MAYBE (cookies);
  xfree (full_path);

  /* Send the request to server.  */
#ifdef HAVE_SSL
  if (conn->scheme == SCHEME_HTTPS)
    write_error = ssl_iwrite (ssl, request, strlen (request));
  else
#endif
    write_error = iwrite (sock, request, strlen (request));

  if (write_error >= 0)
    {
      if (opt.post_data)
	{
	  DEBUGP (("[POST data: %s]\n", opt.post_data));
#ifdef HAVE_SSL
	  if (conn->scheme == SCHEME_HTTPS)
	    write_error = ssl_iwrite (ssl, opt.post_data, post_data_size);
	  else
#endif
	    write_error = iwrite (sock, opt.post_data, post_data_size);
	}
      else if (opt.post_file_name && post_data_size != 0)
	{
#ifdef HAVE_SSL
	  if (conn->scheme == SCHEME_HTTPS)
	    write_error = post_file (-1, ssl, opt.post_file_name,
				     post_data_size);
	  else
#endif
	    write_error = post_file (sock, NULL, opt.post_file_name,
				     post_data_size);
	}
    }
  DEBUGP (("---request end---\n"));

  if (write_error < 0)
    {
      logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"),
		 strerror (errno));
      CLOSE_INVALIDATE (sock);
      return WRITEFAILED;
    }
  logprintf (LOG_VERBOSE, _("%s request sent, awaiting response... "),
	     proxy ? "Proxy" : "HTTP");
  contlen = contrange = -1;
  type = NULL;
  statcode = -1;
  *dt &= ~RETROKF;

  /* Before reading anything, initialize the rbuf.  */
  rbuf_initialize (&rbuf, sock);
#ifdef HAVE_SSL
  if (conn->scheme == SCHEME_HTTPS)
    rbuf.ssl = ssl;
  else
    rbuf.ssl = NULL;
#endif /* HAVE_SSL */
  all_headers = NULL;
  all_length = 0;
  /* Header-fetching loop.  */
  hcount = 0;
  while (1)
    {
      char *hdr;
      int status;

      ++hcount;
      /* Get the header.  */
      status = header_get (&rbuf, &hdr,
			   /* Disallow continuations for status line.  */
			   (hcount == 1 ? HG_NO_CONTINUATIONS : HG_NONE));

      /* Check for errors.  */
      if (status == HG_EOF && *hdr)
	{
	  /* This used to be an unconditional error, but that was
             somewhat controversial, because of a large number of
             broken CGI's that happily "forget" to send the second EOL
             before closing the connection of a HEAD request.

	     So, the deal is to check whether the header is empty
	     (*hdr is zero if it is); if yes, it means that the
	     previous header was fully retrieved, and that -- most
	     probably -- the request is complete.  "...be liberal in
	     what you accept."  Oh boy.  */
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("End of file while parsing headers.\n"));
	  xfree (hdr);
	  FREE_MAYBE (type);
	  FREE_MAYBE (all_headers);
	  CLOSE_INVALIDATE (sock);
	  return HEOF;
	}
      else if (status == HG_ERROR)
	{
	  logputs (LOG_VERBOSE, "\n");
	  logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"),
		     strerror (errno));
	  xfree (hdr);
	  FREE_MAYBE (type);
	  FREE_MAYBE (all_headers);
	  CLOSE_INVALIDATE (sock);
	  return HERR;
	}

      /* If the headers are to be saved to a file later, save them to
	 memory now.  */
      if (opt.save_headers)
	{
	  int lh = strlen (hdr);
	  all_headers = (char *)xrealloc (all_headers, all_length + lh + 2);
	  memcpy (all_headers + all_length, hdr, lh);
	  all_length += lh;
	  all_headers[all_length++] = '\n';
	  all_headers[all_length] = '\0';
	}

      /* Check for status line.  */
      if (hcount == 1)
	{
	  const char *error;
	  /* Parse the first line of server response.  */
	  statcode = parse_http_status_line (hdr, &error);
	  hs->statcode = statcode;
	  /* Store the descriptive response.  */
	  if (statcode == -1) /* malformed response */
	    {
	      /* A common reason for "malformed response" error is the
                 case when no data was actually received.  Handle this
                 special case.  */
	      if (!*hdr)
		hs->error = xstrdup (_("No data received"));
	      else
		hs->error = xstrdup (_("Malformed status line"));
	      xfree (hdr);
	      break;
	    }
	  else if (!*error)
	    hs->error = xstrdup (_("(no description)"));
	  else
	    hs->error = xstrdup (error);

	  if ((statcode != -1)
#ifdef ENABLE_DEBUG
	      && !opt.debug
#endif
	      )
	    {
             if (opt.server_response)
	       logprintf (LOG_VERBOSE, "\n%2d %s", hcount, hdr);
             else
	       logprintf (LOG_VERBOSE, "%2d %s", statcode, error);
	    }

	  goto done_header;
	}

      /* Exit on empty header.  */
      if (!*hdr)
	{
	  xfree (hdr);
	  break;
	}

      /* Print the header if requested.  */
      if (opt.server_response && hcount != 1)
	logprintf (LOG_VERBOSE, "\n%2d %s", hcount, hdr);

      /* Try getting content-length.  */
      if (contlen == -1 && !opt.ignore_length)
	if (header_process (hdr, "Content-Length", header_extract_number,
			    &contlen))
	  goto done_header;
      /* Try getting content-type.  */
      if (!type)
	if (header_process (hdr, "Content-Type", http_process_type, &type))
	  goto done_header;
      /* Try getting location.  */
      if (!hs->newloc)
	if (header_process (hdr, "Location", header_strdup, &hs->newloc))
	  goto done_header;
      /* Try getting last-modified.  */
      if (!hs->remote_time)
	if (header_process (hdr, "Last-Modified", header_strdup,
			    &hs->remote_time))
	  goto done_header;
      /* Try getting cookies. */
      if (opt.cookies)
	if (header_process (hdr, "Set-Cookie", http_process_set_cookie, u))
	  goto done_header;
      /* Try getting www-authentication.  */
      if (!authenticate_h)
	if (header_process (hdr, "WWW-Authenticate", header_strdup,
			    &authenticate_h))
	  goto done_header;
      /* Check for accept-ranges header.  If it contains the word
	 `none', disable the ranges.  */
      if (*dt & ACCEPTRANGES)
	{
	  int nonep;
	  if (header_process (hdr, "Accept-Ranges", http_process_none, &nonep))
	    {
	      if (nonep)
		*dt &= ~ACCEPTRANGES;
	      goto done_header;
	    }
	}
      /* Try getting content-range.  */
      if (contrange == -1)
	{
	  struct http_process_range_closure closure;
	  if (header_process (hdr, "Content-Range", http_process_range, &closure))
	    {
	      contrange = closure.first_byte_pos;
	      goto done_header;
	    }
	}
      /* Check for keep-alive related responses. */
      if (!inhibit_keep_alive)
	{
	  /* Check for the `Keep-Alive' header. */
	  if (!http_keep_alive_1)
	    {
	      if (header_process (hdr, "Keep-Alive", header_exists,
				  &http_keep_alive_1))
		goto done_header;
	    }
	  /* Check for `Connection: Keep-Alive'. */
	  if (!http_keep_alive_2)
	    {
	      if (header_process (hdr, "Connection", http_process_connection,
				  &http_keep_alive_2))
		goto done_header;
	    }
	}
    done_header:
      xfree (hdr);
    }

  logputs (LOG_VERBOSE, "\n");

  if (contlen != -1
      && (http_keep_alive_1 || http_keep_alive_2))
    {
      assert (inhibit_keep_alive == 0);
      keep_alive = 1;
    }
  if (keep_alive)
    /* The server has promised that it will not close the connection
       when we're done.  This means that we can register it.  */
#ifndef HAVE_SSL
    register_persistent (conn->host, conn->port, sock);
#else
    register_persistent (conn->host, conn->port, sock, ssl);
#endif /* HAVE_SSL */

  if ((statcode == HTTP_STATUS_UNAUTHORIZED)
      && authenticate_h)
    {
      /* Authorization is required.  */
      FREE_MAYBE (type);
      type = NULL;
      free_hstat (hs);
      CLOSE_INVALIDATE (sock);	/* would be CLOSE_FINISH, but there
				   might be more bytes in the body. */
      if (auth_tried_already)
	{
	  /* If we have tried it already, then there is not point
	     retrying it.  */
	failed:
	  logputs (LOG_NOTQUIET, _("Authorization failed.\n"));
	  xfree (authenticate_h);
	  return AUTHFAILED;
	}
      else if (!known_authentication_scheme_p (authenticate_h))
	{
	  xfree (authenticate_h);
	  logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
	  return AUTHFAILED;
	}
      else if (BEGINS_WITH (authenticate_h, "Basic"))
	{
	  /* The authentication scheme is basic, the one we try by
             default, and it failed.  There's no sense in trying
             again.  */
	  goto failed;
	}
      else
	{
	  auth_tried_already = 1;
	  goto again;
	}
    }
  /* We do not need this anymore.  */
  if (authenticate_h)
    {
      xfree (authenticate_h);
      authenticate_h = NULL;
    }

  /* 20x responses are counted among successful by default.  */
  if (H_20X (statcode))
    *dt |= RETROKF;

  /* Return if redirected.  */
  if (H_REDIRECTED (statcode) || statcode == HTTP_STATUS_MULTIPLE_CHOICES)
    {
      /* RFC2068 says that in case of the 300 (multiple choices)
	 response, the server can output a preferred URL through
	 `Location' header; otherwise, the request should be treated
	 like GET.  So, if the location is set, it will be a
	 redirection; otherwise, just proceed normally.  */
      if (statcode == HTTP_STATUS_MULTIPLE_CHOICES && !hs->newloc)
	*dt |= RETROKF;
      else
	{
	  logprintf (LOG_VERBOSE,
		     _("Location: %s%s\n"),
		     hs->newloc ? hs->newloc : _("unspecified"),