http.c

Wget很好的处理了http和ftp的下载,很值得学习的经典代码

      else if (host_lookup_failed)
        {
          request_free (req);
          logprintf(LOG_NOTQUIET,
                    _("%s: unable to resolve host address `%s'\n"),
                    exec_name, relevant->host);
          return HOSTERR;
        }
    }

  if (sock < 0)
    {
      sock = connect_to_host (conn->host, conn->port);
      if (sock == E_HOST)
        {
          request_free (req);
          return HOSTERR;
        }
      else if (sock < 0)
        {
          request_free (req);
          return (retryable_socket_connect_error (errno)
                  ? CONERROR : CONIMPOSSIBLE);
        }

#ifdef HAVE_SSL
      if (proxy && u->scheme == SCHEME_HTTPS)
        {
          /* When requesting SSL URLs through proxies, use the
             CONNECT method to request passthrough.  */
          struct request *connreq = request_new ();
          request_set_method (connreq, "CONNECT",
                              aprintf ("%s:%d", u->host, u->port));
          SET_USER_AGENT (connreq);
          if (proxyauth)
            {
              request_set_header (connreq, "Proxy-Authorization",
                                  proxyauth, rel_value);
              /* Now that PROXYAUTH is part of the CONNECT request,
                 zero it out so we don't send proxy authorization with
                 the regular request below.  */
              proxyauth = NULL;
            }
          /* Examples in rfc2817 use the Host header in CONNECT
             requests.  I don't see how that gains anything, given
             that the contents of Host would be exactly the same as
             the contents of CONNECT.  */

          write_error = request_send (connreq, sock);
          request_free (connreq);
          if (write_error < 0)
            {
              CLOSE_INVALIDATE (sock);
              return WRITEFAILED;
            }

          head = read_http_response_head (sock);
          if (!head)
            {
              logprintf (LOG_VERBOSE, _("Failed reading proxy response: %s\n"),
                         fd_errstr (sock));
              CLOSE_INVALIDATE (sock);
              return HERR;
            }
          message = NULL;
          if (!*head)
            {
              xfree (head);
              goto failed_tunnel;
            }
          DEBUGP (("proxy responded with: [%s]\n", head));

          resp = resp_new (head);
          statcode = resp_status (resp, &message);
          resp_free (resp);
          xfree (head);
          if (statcode != 200)
            {
            failed_tunnel:
              logprintf (LOG_NOTQUIET, _("Proxy tunneling failed: %s"),
                         message ? escnonprint (message) : "?");
              xfree_null (message);
              return CONSSLERR;
            }
          xfree_null (message);

          /* SOCK is now *really* connected to u->host, so update CONN
             to reflect this.  That way register_persistent will
             register SOCK as being connected to u->host:u->port.  */
          conn = u;
        }

      if (conn->scheme == SCHEME_HTTPS)
        {
          if (!ssl_connect (sock) || !ssl_check_certificate (sock, u->host))
            {
              fd_close (sock);
              return CONSSLERR;
            }
          using_ssl = true;
        }
#endif /* HAVE_SSL */
    }

  /* Send the request to server.  */
  write_error = request_send (req, sock);

  if (write_error >= 0)
    {
      if (opt.post_data)
        {
          DEBUGP (("[POST data: %s]\n", opt.post_data));
          write_error = fd_write (sock, opt.post_data, post_data_size, -1);
        }
      else if (opt.post_file_name && post_data_size != 0)
        write_error = post_file (sock, opt.post_file_name, post_data_size);
    }

  if (write_error < 0)
    {
      CLOSE_INVALIDATE (sock);
      request_free (req);
      return WRITEFAILED;
    }
  logprintf (LOG_VERBOSE, _("%s request sent, awaiting response... "),
             proxy ? "Proxy" : "HTTP");
  contlen = -1;
  contrange = 0;
  *dt &= ~RETROKF;

  head = read_http_response_head (sock);
  if (!head)
    {
      if (errno == 0)
        {
          logputs (LOG_NOTQUIET, _("No data received.\n"));
          CLOSE_INVALIDATE (sock);
          request_free (req);
          return HEOF;
        }
      else
        {
          logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"),
                     fd_errstr (sock));
          CLOSE_INVALIDATE (sock);
          request_free (req);
          return HERR;
        }
    }
  DEBUGP (("\n---response begin---\n%s---response end---\n", head));

  resp = resp_new (head);

  /* Check for status line.  */
  message = NULL;
  statcode = resp_status (resp, &message);
  if (!opt.server_response)
    logprintf (LOG_VERBOSE, "%2d %s\n", statcode,
               message ? escnonprint (message) : "");
  else
    {
      logprintf (LOG_VERBOSE, "\n");
      print_server_response (resp, "  ");
    }

  /* Determine the local filename if needed. Notice that if -O is used 
   * hstat.local_file is set by http_loop to the argument of -O. */
  if (!hs->local_file)
    {
      /* Honor Content-Disposition whether possible. */
      if (!opt.content_disposition
          || !resp_header_copy (resp, "Content-Disposition", 
                                hdrval, sizeof (hdrval))
          || !parse_content_disposition (hdrval, &hs->local_file))
        {
          /* The Content-Disposition header is missing or broken. 
           * Choose unique file name according to given URL. */
          hs->local_file = url_file_name (u);
        }
    }
  
  /* TODO: perform this check only once. */
  if (!hs->existence_checked && file_exists_p (hs->local_file))
    {
      if (opt.noclobber)
        {
          /* If opt.noclobber is turned on and file already exists, do not
             retrieve the file */
          logprintf (LOG_VERBOSE, _("\
File `%s' already there; not retrieving.\n\n"), hs->local_file);
          /* If the file is there, we suppose it's retrieved OK.  */
          *dt |= RETROKF;

          /* #### Bogusness alert.  */
          /* If its suffix is "html" or "htm" or similar, assume text/html.  */
          if (has_html_suffix_p (hs->local_file))
            *dt |= TEXTHTML;

          return RETRUNNEEDED;
        }
      else if (!ALLOW_CLOBBER)
        {
          char *unique = unique_name (hs->local_file, true);
          if (unique != hs->local_file)
            xfree (hs->local_file);
          hs->local_file = unique;
        }
    }
  hs->existence_checked = true;

  /* Support timestamping */
  /* TODO: move this code out of gethttp. */
  if (opt.timestamping && !hs->timestamp_checked)
    {
      size_t filename_len = strlen (hs->local_file);
      char *filename_plus_orig_suffix = alloca (filename_len + sizeof (".orig"));
      bool local_dot_orig_file_exists = false;
      char *local_filename = NULL;
      struct_stat st;

      if (opt.backup_converted)
        /* If -K is specified, we'll act on the assumption that it was specified
           last time these files were downloaded as well, and instead of just
           comparing local file X against server file X, we'll compare local
           file X.orig (if extant, else X) against server file X.  If -K
           _wasn't_ specified last time, or the server contains files called
           *.orig, -N will be back to not operating correctly with -k. */
        {
          /* Would a single s[n]printf() call be faster?  --dan

             Definitely not.  sprintf() is horribly slow.  It's a
             different question whether the difference between the two
             affects a program.  Usually I'd say "no", but at one
             point I profiled Wget, and found that a measurable and
             non-negligible amount of time was lost calling sprintf()
             in url.c.  Replacing sprintf with inline calls to
             strcpy() and number_to_string() made a difference.
             --hniksic */
          memcpy (filename_plus_orig_suffix, hs->local_file, filename_len);
          memcpy (filename_plus_orig_suffix + filename_len,
                  ".orig", sizeof (".orig"));

          /* Try to stat() the .orig file. */
          if (stat (filename_plus_orig_suffix, &st) == 0)
            {
              local_dot_orig_file_exists = true;
              local_filename = filename_plus_orig_suffix;
            }
        }      

      if (!local_dot_orig_file_exists)
        /* Couldn't stat() <file>.orig, so try to stat() <file>. */
        if (stat (hs->local_file, &st) == 0)
          local_filename = hs->local_file;

      if (local_filename != NULL)
        /* There was a local file, so we'll check later to see if the version
           the server has is the same version we already have, allowing us to
           skip a download. */
        {
          hs->orig_file_name = xstrdup (local_filename);
          hs->orig_file_size = st.st_size;
          hs->orig_file_tstamp = st.st_mtime;
#ifdef WINDOWS
          /* Modification time granularity is 2 seconds for Windows, so
             increase local time by 1 second for later comparison. */
          ++hs->orig_file_tstamp;
#endif
        }
    }

  if (!opt.ignore_length
      && resp_header_copy (resp, "Content-Length", hdrval, sizeof (hdrval)))
    {
      wgint parsed;
      errno = 0;
      parsed = str_to_wgint (hdrval, NULL, 10);
      if (parsed == WGINT_MAX && errno == ERANGE)
        {
          /* Out of range.
             #### If Content-Length is out of range, it most likely
             means that the file is larger than 2G and that we're
             compiled without LFS.  In that case we should probably
             refuse to even attempt to download the file.  */
          contlen = -1;
        }
      else if (parsed < 0)
        {
          /* Negative Content-Length; nonsensical, so we can't
             assume any information about the content to receive. */
          contlen = -1;
        }
      else
        contlen = parsed;
    }

  /* Check for keep-alive related responses. */
  if (!inhibit_keep_alive && contlen != -1)
    {
      if (resp_header_copy (resp, "Keep-Alive", NULL, 0))
        keep_alive = true;
      else if (resp_header_copy (resp, "Connection", hdrval, sizeof (hdrval)))
        {
          if (0 == strcasecmp (hdrval, "Keep-Alive"))
            keep_alive = true;
        }
    }
  if (keep_alive)
    /* The server has promised that it will not close the connection
       when we're done.  This means that we can register it.  */
    register_persistent (conn->host, conn->port, sock, using_ssl);

  if (statcode == HTTP_STATUS_UNAUTHORIZED)
    {
      /* Authorization is required.  */
      if (keep_alive && !head_only && skip_short_body (sock, contlen))
        CLOSE_FINISH (sock);
      else
        CLOSE_INVALIDATE (sock);
      pconn.authorized = false;
      if (!auth_finished && (user && passwd))
        {
          /* IIS sends multiple copies of WWW-Authenticate, one with
             the value "negotiate", and other(s) with data.  Loop over
             all the occurrences and pick the one we recognize.  */
          int wapos;
          const char *wabeg, *waend;
          char *www_authenticate = NULL;
          for (wapos = 0;
               (wapos = resp_header_locate (resp, "WWW-Authenticate", wapos,
                                            &wabeg, &waend)) != -1;
               ++wapos)
            if (known_authentication_scheme_p (wabeg, waend))
              {
                BOUNDED_TO_ALLOCA (wabeg, waend, www_authenticate);
                break;
              }

          if (!www_authenticate)
            {
              /* If the authentication header is missing or
                 unrecognized, there's no sense in retrying.  */
              logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
            }
          else if (!basic_auth_finished
                   || !BEGINS_WITH (www_authenticate, "Basic"))
            {
              char *pth;
              pth = url_full_path (u);
              request_set_header (req, "Authorization",
                                  create_authorization_line (www_authenticate,
                                                             user, passwd,
                                                             request_method (req),
                                                             pth,
                                                             &auth_finished),
                                  rel_value);
              if (BEGINS_WITH (www_authenticate, "NTLM"))
                ntlm_seen = true;
              else if (!u->user && BEGINS_WITH (www_authenticate, "Basic"))
                {
                  /* Need to register this host as using basic auth,
                   * so we automatically send creds next time. */
                  register_basic_auth_host (u->host);
                }
              xfree (pth);
              goto retry_with_auth;
            }
          else
            {
              /* We already did Basic auth, and it failed. Gotta
               * give up. */
            }
        }
      logputs (LOG_NOTQUIET, _("Authorization failed.\n"));
      request_free (req);
      return AUTHFAILED;
    }
  else /* statcode != HTTP_STATUS_UNAUTHORIZED */
    {
      /* Kludge: if NTLM is used, mark the TCP connection as authorized. */
      if (ntlm_seen)
        pconn.authorized = true;
    }
  request_free (req);

  hs->statcode = statcode;
  if (statcode == -1)
    hs->error = xstrdup (_("Malformed status line"));
  else if (!*message)
    hs->error = xstrdup (_("(no description)"));
  else
    hs->error = xstrdup (message);
  xfree_null (message);

  type = resp_header_strdup (resp, "Content-Type");
  if (type)
    {
      char *tmp = strchr (type, ';');
      if (tmp)
        {
          while (tmp > type && ISSPACE (tmp[-1]))
            --tmp;
          *tmp = '\0';
        }
    }
  hs->newloc = resp_header_strdup (resp, "Location");
  hs->remote_time = resp_header_strdup (resp, "Last-Modified");

  /* Handle (possibly multiple instances of) the Set-Cookie header. */
  if