httpclient.cc

本人收集整理的一份c/c++跨平台网络库

#include <time.h>

#include "httpcommon-inl.h"

#include "asyncsocket.h"
#include "common.h"
#include "diskcache.h"
#include "httpclient.h"
#include "logging.h"
#include "pathutils.h"
#include "socketstream.h"
#include "stringencode.h"
#include "stringutils.h"
#include "basicdefs.h"

namespace utils_base {

//////////////////////////////////////////////////////////////////////
// Helpers
//////////////////////////////////////////////////////////////////////

namespace {

const size_t kCacheHeader = 0;
const size_t kCacheBody = 1;

std::string HttpAddress(const SocketAddress& address) {
  return (address.port() == HTTP_DEFAULT_PORT)
          ? address.hostname() : address.ToString();
}

// Convert decimal string to integer
bool HttpStringToInt(const std::string& str, unsigned long* val) {
  ASSERT(NULL != val);
  char* eos = NULL;
  *val = strtoul(str.c_str(), &eos, 10);
  return (*eos == '\0');
}

bool HttpShouldCache(const HttpRequestData& request, 
                     const HttpResponseData& response) {
  bool verb_allows_cache = (request.verb == HV_GET)
                           || (request.verb == HV_HEAD);
  bool is_range_response = response.hasHeader(HH_CONTENT_RANGE, NULL);
  bool has_expires = response.hasHeader(HH_EXPIRES, NULL);
  bool request_allows_cache =
    has_expires || (std::string::npos != request.path.find('?'));
  bool response_allows_cache =
    has_expires || HttpCodeIsCacheable(response.scode);

  bool may_cache = verb_allows_cache
                   && request_allows_cache
                   && response_allows_cache
                   && !is_range_response;

  std::string value;
  if (response.hasHeader(HH_CACHE_CONTROL, &value)) {
    HttpAttributeList directives;
    HttpParseAttributes(value.data(), value.size(), directives);
    // Response Directives Summary:
    // public - always cacheable
    // private - do not cache in a shared cache
    // no-cache - may cache, but must revalidate whether fresh or stale
    // no-store - sensitive information, do not cache or store in any way
    // max-age - supplants Expires for staleness
    // s-maxage - use as max-age for shared caches, ignore otherwise
    // must-revalidate - may cache, but must revalidate after stale
    // proxy-revalidate - shared cache must revalidate
    if (HttpHasAttribute(directives, "no-store", NULL)) {
      may_cache = false;
    } else if (HttpHasAttribute(directives, "public", NULL)) {
      may_cache = true;
    }
  }
  return may_cache;
}

enum HttpCacheState {
  HCS_FRESH,  // In cache, may use
  HCS_STALE,  // In cache, must revalidate
  HCS_NONE    // Not in cache
};

HttpCacheState HttpGetCacheState(const HttpRequestData& request, 
                                 const HttpResponseData& response) {
  // Temporaries
  std::string s_temp;
  unsigned long i_temp;

  // Current time
  unsigned long now = (unsigned long)time(0);

  HttpAttributeList cache_control;
  if (response.hasHeader(HH_CACHE_CONTROL, &s_temp)) {
    HttpParseAttributes(s_temp.data(), s_temp.size(), cache_control);
  }

  // Compute age of cache document
  unsigned long date;
  if (!response.hasHeader(HH_DATE, &s_temp)
      || !HttpDateToSeconds(s_temp, &date))
    return HCS_NONE;

  // TODO: Timestamp when cache request sent and response received?
  unsigned long request_time = date;
  unsigned long response_time = date;

  unsigned long apparent_age = 0;
  if (response_time > date) {
    apparent_age = response_time - date;
  }

  unsigned long corrected_received_age = apparent_age;
  if (response.hasHeader(HH_AGE, &s_temp)
      && HttpStringToInt(s_temp, &i_temp)) {
    corrected_received_age = stdmax(apparent_age, i_temp);
  }

  unsigned long response_delay = response_time - request_time;
  unsigned long corrected_initial_age = corrected_received_age + response_delay;
  unsigned long resident_time = now - response_time;
  unsigned long current_age = corrected_initial_age + resident_time;

  // Compute lifetime of document
  unsigned long lifetime;
  if (HttpHasAttribute(cache_control, "max-age", &s_temp)) {
    lifetime = atoi(s_temp.c_str());
  } else if (response.hasHeader(HH_EXPIRES, &s_temp)
             && HttpDateToSeconds(s_temp, &i_temp)) {
    lifetime = i_temp - date;
  } else if (response.hasHeader(HH_LAST_MODIFIED, &s_temp)
             && HttpDateToSeconds(s_temp, &i_temp)) {
    // TODO: Issue warning 113 if age > 24 hours
    lifetime = (now - i_temp) / 10;
  } else {
    return HCS_STALE;
  }

  return (lifetime > current_age) ? HCS_FRESH : HCS_STALE;
}

enum HttpValidatorStrength {
  HVS_NONE,
  HVS_WEAK,
  HVS_STRONG
};

HttpValidatorStrength
HttpRequestValidatorLevel(const HttpRequestData& request) {
  if (HV_GET != request.verb)
    return HVS_STRONG;
  return request.hasHeader(HH_RANGE, NULL) ? HVS_STRONG : HVS_WEAK;
}

HttpValidatorStrength
HttpResponseValidatorLevel(const HttpResponseData& response) {
  std::string value;
  if (response.hasHeader(HH_ETAG, &value)) {
    bool is_weak = (strnicmp(value.c_str(), "W/", 2) == 0);
    return is_weak ? HVS_WEAK : HVS_STRONG;
  }
  if (response.hasHeader(HH_LAST_MODIFIED, &value)) {
    unsigned long last_modified, date;
    if (HttpDateToSeconds(value, &last_modified) 
        && response.hasHeader(HH_DATE, &value)
        && HttpDateToSeconds(value, &date)
        && (last_modified + 60 < date)) {
      return HVS_STRONG;
    }
    return HVS_WEAK;
  }
  return HVS_NONE;
}

std::string GetCacheID(const SocketAddress& server,
                       const HttpRequestData& request) {
  std::string url;
  url.append(ToString(request.verb));
  url.append("_");
  if ((_strnicmp(request.path.c_str(), "http://", 7) == 0)
      || (_strnicmp(request.path.c_str(), "https://", 8) == 0)) {
    url.append(request.path);
  } else {
    url.append("http://");
    url.append(HttpAddress(server));
    url.append(request.path);
  }
  return url;
}

}  // anonymous namespace

//////////////////////////////////////////////////////////////////////
// HttpClient
//////////////////////////////////////////////////////////////////////

HttpClient::HttpClient(const std::string& agent, StreamPool* pool)
: agent_(agent), pool_(pool), fail_redirect_(false), absolute_uri_(false),
  cache_(NULL), cache_state_(CS_READY)
{
  base_.notify(this);
}

HttpClient::~HttpClient() {
  base_.notify(NULL);
  base_.abort(HE_SHUTDOWN);
  release();
}

void HttpClient::reset() {
  server_.Clear();
  request_.clear(true);
  response_.clear(true);
  context_.reset();
  base_.abort(HE_OPERATION_CANCELLED);
}

void HttpClient::set_server(const SocketAddress& address) {
  server_ = address;
  // Setting 'Host' here allows it to be overridden before starting the request,
  // if necessary.
  request_.setHeader(HH_HOST, HttpAddress(server_), true);
}

void HttpClient::start() {
  if (base_.mode() != HM_NONE) {
    // call reset() to abort an in-progress request
    ASSERT(false);
    return;
  }

  ASSERT(!IsCacheActive());

  if (request_.hasHeader(HH_TRANSFER_ENCODING, NULL)) {
    // Exact size must be known on the client.  Instead of using chunked
    // encoding, wrap data with auto-caching file or memory stream.
    ASSERT(false);
    return;
  }

  // If no content has been specified, using length of 0.
  request_.setHeader(HH_CONTENT_LENGTH, "0", false);

  request_.setHeader(HH_USER_AGENT, agent_, false);
  request_.setHeader(HH_CONNECTION, "Keep-Alive", false);
  if (_strnicmp(request_.path.c_str(), "http", 4) == 0) {
    request_.setHeader(HH_PROXY_CONNECTION, "Keep-Alive", false);
  }

  bool absolute_uri = absolute_uri_;
  if (PROXY_HTTPS == proxy_.type) {
    request().version = HVER_1_0;
    // Proxies require canonical form
    absolute_uri = true;
  }

  // Convert to canonical form (if not already)
  if (absolute_uri && (_strnicmp(request().path.c_str(), "http://", 7) != 0)) {
    std::string canonical_path("http://");
    canonical_path.append(HttpAddress(server_));
    canonical_path.append(request().path);
    request().path = canonical_path;
  }

  if ((NULL != cache_) && CheckCache()) {
    return;
  }

  int stream_err;
  StreamInterface* stream = pool_->RequestConnectedStream(server_, &stream_err);
  if (stream == NULL) {
    if (stream_err)
      LOG(LS_ERROR) << "RequestConnectedStream returned: " << stream_err;
    onHttpComplete(HM_CONNECT, (stream_err == 0) ? HE_NONE : HE_SOCKET);
  } else {
    base_.attach(stream);
    if (stream->GetState() == SS_OPEN) {
      base_.send(&request_);
    }
  }
}

void HttpClient::prepare_get(const std::string& url) {
  reset();
  Url<char> purl(url);
  set_server(SocketAddress(purl.server(), purl.port(), false));
  request().verb = HV_GET;
  request().path = purl.full_path();
}

void HttpClient::prepare_post(const std::string& url,
                              const std::string& content_type,
                              StreamInterface* request_doc) {
  reset();
  Url<char> purl(url);
  set_server(SocketAddress(purl.server(), purl.port(), false));
  request().verb = HV_POST;
  request().path = purl.full_path();
  request().setContent(content_type, request_doc);
}

void HttpClient::release() {
  if (StreamInterface* stream = base_.detach()) {
    pool_->ReturnConnectedStream(stream);
  }
}

bool HttpClient::BeginCacheFile() {
  ASSERT(NULL != cache_);
  ASSERT(CS_READY == cache_state_);

  std::string id = GetCacheID(server_, request_);
  CacheLock lock(cache_, id, true);
  if (!lock.IsLocked()) {
    LOG_F(LS_WARNING) << "Couldn't lock cache";
    return false;
  }

  if (HE_NONE != WriteCacheHeaders(id)) {
    return false;
  }

  scoped_ptr<StreamInterface> stream(cache_->WriteResource(id, kCacheBody));
  if (!stream.get()) {
    LOG_F(LS_ERROR) << "Couldn't open body cache";
    return false;
  }
  lock.Commit();

  // Let's secretly replace the response document with Folgers Crystals,
  // er, StreamTap, so that we can mirror the data to our cache.
  StreamInterface* output = response_.document.release();
  if (!output) {
    output = new NullStream;
  }
  StreamTap* tap = new StreamTap(output, stream.release());
  response_.document.reset(tap);
  return true;
}

HttpError HttpClient::WriteCacheHeaders(const std::string& id) {
  scoped_ptr<StreamInterface> stream(cache_->WriteResource(id, kCacheHeader));
  if (!stream.get()) {
    LOG_F(LS_ERROR) << "Couldn't open header cache";