http.c

harvest是一个下载html网页得机器人

/***************************************************************************
 *                                  _   _ ____  _     
 *  Project                     ___| | | |  _ \| |    
 *                             / __| | | | |_) | |    
 *                            | (__| |_| |  _ <| |___ 
 *                             \___|\___/|_| \_\_____|
 *
 * Copyright (C) 1998 - 2003, Daniel Stenberg, <[e-mail]>, et al.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution. The terms
 * are also available at http://curl.haxx.se/docs/copyright.html.
 * 
 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
 * copies of the Software, and permit persons to whom the Software is
 * furnished to do so, under the terms of the COPYING file.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 * KIND, either express or implied.
 *
 * $Id: http.c,v 1.172 2003/10/31 21:43:22 bagder Exp $
 ***************************************************************************/

#include "setup.h"

#ifndef CURL_DISABLE_HTTP
/* -- WIN32 approved -- */
#include <stdio.h>
#include <string.h>
#include <stdarg.h>
#include <stdlib.h>
#include <ctype.h>
#include <sys/types.h>
#include <sys/stat.h>

#include <errno.h>

#if defined(WIN32) && !defined(__GNUC__) || defined(__MINGW32__)
#include <time.h>
#include <io.h>
#else
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h>
#endif
#include <sys/time.h>

#ifdef HAVE_TIME_H
#ifdef TIME_WITH_SYS_TIME
#include <time.h>
#endif
#endif

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <netdb.h>
#ifdef HAVE_ARPA_INET_H
#include <arpa/inet.h>
#endif
#ifdef HAVE_NET_IF_H
#include <net/if.h>
#endif
#include <sys/ioctl.h>
#include <signal.h>

#ifdef HAVE_SYS_PARAM_H
#include <sys/param.h>
#endif

#ifdef HAVE_SYS_SELECT_H
#include <sys/select.h>
#endif


#endif

#include "urldata.h"
#include <curl/curl.h>
#include "transfer.h"
#include "sendf.h"
#include "formdata.h"
#include "progress.h"
#include "base64.h"
#include "cookie.h"
#include "strequal.h"
#include "ssluse.h"
#include "http_digest.h"
#include "http_ntlm.h"
#include "http_negotiate.h"
#include "url.h"
#include "share.h"
#include "http.h"

#define _MPRINTF_REPLACE /* use our functions only */
#include <curl/mprintf.h>

/* The last #include file should be: */
#ifdef CURLDEBUG
#include "memdebug.h"
#endif

static CURLcode Curl_output_basic_proxy(struct connectdata *conn);

/*
 * This function checks the linked list of custom HTTP headers for a particular
 * header (prefix).
 */
static char *checkheaders(struct SessionHandle *data, const char *thisheader)
{
  struct curl_slist *head;
  size_t thislen = strlen(thisheader);

  for(head = data->set.headers; head; head=head->next) {
    if(strnequal(head->data, thisheader, thislen))
      return head->data;
  }
  return NULL;
}

static CURLcode Curl_output_basic(struct connectdata *conn)
{
  char *authorization;
  struct SessionHandle *data=conn->data;

  sprintf(data->state.buffer, "%s:%s", conn->user, conn->passwd);
  if(Curl_base64_encode(data->state.buffer, strlen(data->state.buffer),
                        &authorization) >= 0) {
    if(conn->allocptr.userpwd)
      free(conn->allocptr.userpwd);
    conn->allocptr.userpwd = aprintf( "Authorization: Basic %s\015\012",
                                      authorization);
    free(authorization);
  }
  else
    return CURLE_OUT_OF_MEMORY;
  return CURLE_OK;
}

static CURLcode Curl_output_basic_proxy(struct connectdata *conn)
{
  char *authorization;
  struct SessionHandle *data=conn->data;

  sprintf(data->state.buffer, "%s:%s",
          conn->proxyuser, conn->proxypasswd);
  if(Curl_base64_encode(data->state.buffer, strlen(data->state.buffer),
                        &authorization) >= 0) {
    Curl_safefree(conn->allocptr.proxyuserpwd);
    conn->allocptr.proxyuserpwd =
      aprintf("Proxy-authorization: Basic %s\015\012", authorization);
    free(authorization);
  }
  else
    return CURLE_OUT_OF_MEMORY;
  return CURLE_OK;
}

void Curl_http_auth_act(struct connectdata *conn)
{
  struct SessionHandle *data = conn->data;

  if(data->state.authavail) {
    if(data->state.authavail & CURLAUTH_GSSNEGOTIATE)
      data->state.authwant = CURLAUTH_GSSNEGOTIATE;
    else if(data->state.authavail & CURLAUTH_DIGEST)
      data->state.authwant = CURLAUTH_DIGEST;
    else if(data->state.authavail & CURLAUTH_NTLM)
      data->state.authwant = CURLAUTH_NTLM;
    else if(data->state.authavail & CURLAUTH_BASIC)
      data->state.authwant = CURLAUTH_BASIC;
    else
      data->state.authwant = CURLAUTH_NONE; /* clear it */

    if(data->state.authwant)
      conn->newurl = strdup(data->change.url); /* clone URL */
    data->state.authavail = CURLAUTH_NONE; /* clear it here */
  }
}

/*
 * Setup the authentication headers for the host/proxy and the correct
 * authentication method.
 */

CURLcode http_auth_headers(struct connectdata *conn,
                           char *request,
                           char *path,
                           bool *ready) /* set TRUE when the auth phase is
                                           done and ready to do the *actual*
                                           request */
{
  CURLcode result = CURLE_OK;
  struct SessionHandle *data = conn->data;
  char *auth=NULL;

  *ready = FALSE; /* default is no */

  if(!data->state.authstage) {
    if(conn->bits.httpproxy && conn->bits.proxy_user_passwd)
      Curl_http_auth_stage(data, 407);
    else if(conn->bits.user_passwd)
      Curl_http_auth_stage(data, 401);
    else {
      *ready = TRUE;
      return CURLE_OK; /* no authentication with no user or password */
    }
  }

  /* To prevent the user+password to get sent to other than the original
     host due to a location-follow, we do some weirdo checks here */
  if(!data->state.this_is_a_follow ||
     !data->state.auth_host ||
     curl_strequal(data->state.auth_host, conn->hostname) ||
     data->set.http_disable_hostname_check_before_authentication) {

  /* Send proxy authentication header if needed */
    if (data->state.authstage == 407) {
#ifdef USE_SSLEAY
      if(data->state.authwant == CURLAUTH_NTLM) {
        auth=(char *)"NTLM";
        result = Curl_output_ntlm(conn, TRUE, ready);
        if(result)
          return result;
      }
      else
#endif
      if((data->state.authwant == CURLAUTH_BASIC) && /* Basic */
         conn->bits.proxy_user_passwd &&
         !checkheaders(data, "Proxy-authorization:")) {
        auth=(char *)"Basic";
        result = Curl_output_basic_proxy(conn);
        if(result)
          return result;
        *ready = TRUE;
        /* Switch to web authentication after proxy authentication is done */
        Curl_http_auth_stage(data, 401);
      }
      infof(data, "Proxy auth using %s with user '%s'\n",
            auth, conn->proxyuser?conn->proxyuser:"");
    }
    /* Send web authentication header if needed */
    if (data->state.authstage == 401) {
      auth = NULL;
#ifdef HAVE_GSSAPI
      if((data->state.authwant == CURLAUTH_GSSNEGOTIATE) &&
         data->state.negotiate.context && 
         !GSS_ERROR(data->state.negotiate.status)) {
        auth=(char *)"GSS-Negotiate";
        result = Curl_output_negotiate(conn);
        if (result)
          return result;
        *ready = TRUE;
      }
      else
#endif
#ifdef USE_SSLEAY
      if(data->state.authwant == CURLAUTH_NTLM) {
        auth=(char *)"NTLM";
        result = Curl_output_ntlm(conn, FALSE, ready);
        if(result)
          return result;
      }
      else
#endif
      {
        if((data->state.authwant == CURLAUTH_DIGEST) &&
           data->state.digest.nonce) {
          auth=(char *)"Digest";
          result = Curl_output_digest(conn,
                                      (unsigned char *)request,
                                      (unsigned char *)path);
          if(result)
            return result;
          *ready = TRUE;
        }
        else if((data->state.authwant == CURLAUTH_BASIC) && /* Basic */
                conn->bits.user_passwd &&
                !checkheaders(data, "Authorization:")) {
          auth=(char *)"Basic";
          result = Curl_output_basic(conn);
          if(result)
            return result;
          *ready = TRUE;
        }
      }
      if(auth)
        infof(data, "Server auth using %s with user '%s'\n",
              auth, conn->user);
    }
  }
  else
    *ready = TRUE;

  return result;
}


/*
 * Curl_http_auth() deals with Proxy-Authenticate: and WWW-Authenticate:
 * headers. They are dealt with both in the transfer.c main loop and in the
 * proxy CONNECT loop.
 */

CURLcode Curl_http_auth(struct connectdata *conn,
                        int httpcode,
                        char *header) /* pointing to the first non-space */
{
  /*
   * This resource requires authentication
   */
  struct SessionHandle *data = conn->data;

  long *availp;
  char *start;

  if (httpcode == 407) {
    start = header+strlen("Proxy-authenticate:");
    availp = &data->info.proxyauthavail;
  }
  else {
    start = header+strlen("WWW-Authenticate:");
    availp = &data->info.httpauthavail;
  }
  /*
   * Switch from proxy to web authentication and back if needed
   */
  if (httpcode == 407 && data->state.authstage != 407)
    Curl_http_auth_stage(data, 407);
              
  else if (httpcode == 401 && data->state.authstage != 401)
    Curl_http_auth_stage(data, 401);

  /* pass all white spaces */
  while(*start && isspace((int)*start))
    start++;

#ifdef HAVE_GSSAPI
  if (checkprefix("GSS-Negotiate", start) ||
      checkprefix("Negotiate", start)) {
    *availp |= CURLAUTH_GSSNEGOTIATE;
    if(data->state.authwant == CURLAUTH_GSSNEGOTIATE) {
      /* if exactly this is wanted, go */
      int neg = Curl_input_negotiate(conn, start);
      if (neg == 0)
        conn->newurl = strdup(data->change.url);
    }
    else
      if(data->state.authwant & CURLAUTH_GSSNEGOTIATE)
        data->state.authavail |= CURLAUTH_GSSNEGOTIATE;
  }
  else
#endif
#ifdef USE_SSLEAY
    /* NTLM support requires the SSL crypto libs */
    if(checkprefix("NTLM", start)) {
      *availp |= CURLAUTH_NTLM;
      if(data->state.authwant == CURLAUTH_NTLM) {
        /* NTLM authentication is activated */
        CURLntlm ntlm =
          Curl_input_ntlm(conn, (bool)(httpcode == 407), start);
                  
        if(CURLNTLM_BAD != ntlm)
          conn->newurl = strdup(data->change.url); /* clone string */
        else
          infof(data, "Authentication problem. Ignoring this.\n");
      }
      else
        if(data->state.authwant & CURLAUTH_NTLM)
          data->state.authavail |= CURLAUTH_NTLM;
    }
    else
#endif
      if(checkprefix("Digest", start)) {
        *availp |= CURLAUTH_DIGEST;
        if(data->state.authwant == CURLAUTH_DIGEST) {
          /* Digest authentication is activated */
          CURLdigest dig = CURLDIGEST_BAD;

          if(data->state.digest.nonce)
            infof(data, "Authentication problem. Ignoring this.\n");
          else
            dig = Curl_input_digest(conn, start);
          
          if(CURLDIGEST_FINE == dig)
            /* We act on it. Store our new url, which happens to be
               the same one we already use! */
            conn->newurl = strdup(data->change.url); /* clone string */
        }
        else
          if(data->state.authwant & CURLAUTH_DIGEST) {
            /* We don't know if Digest is what we're gonna use, but we
               call this function anyway to store the digest data that
               is provided on this line, to skip the extra round-trip
               we need to do otherwise. We must sure to free this
               data! */
            Curl_input_digest(conn, start);
            data->state.authavail |= CURLAUTH_DIGEST;
          }
      }
      else if(checkprefix("Basic", start)) {
        *availp |= CURLAUTH_BASIC;
        if((data->state.authwant == CURLAUTH_BASIC) && (httpcode == 401)) {
          /* We asked for Basic authentication but got a 401 back
             anyway, which basicly means our name+password isn't
             valid. */
          data->state.authavail = CURLAUTH_NONE;
          infof(data, "Authentication problem. Ignoring this.\n");