s3.c

开源备份软件源码 AMANDA, the Advanced Maryland Automatic Network Disk Archiver, is a backup system that a

/*
 * Copyright (c) 2005 Zmanda, Inc.  All Rights Reserved.
 * 
 * This library is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License version 2.1 as 
 * published by the Free Software Foundation.
 * 
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
 * License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with this library; if not, write to the Free Software Foundation,
 * Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA.
 * 
 * Contact information: Zmanda Inc., 505 N Mathlida Ave, Suite 120
 * Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
 */

/* TODO
 * - Compute and send Content-MD5 header
 * - check SSL certificate
 * - collect speed statistics
 * - debugging mode
 */

#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h>
#include <regex.h>
#include <time.h>
#include "util.h"
#include "amanda.h"
#include "s3.h"
#include "base64.h"
#include <curl/curl.h>

/* Constant renamed after version 7.10.7 */
#ifndef CURLINFO_RESPONSE_CODE
#define CURLINFO_RESPONSE_CODE CURLINFO_HTTP_CODE
#endif

/* We don't need OpenSSL's kerberos support, and it's broken in
 * RHEL 3 anyway. */
#define OPENSSL_NO_KRB5

#ifdef HAVE_OPENSSL_HMAC_H
# include <openssl/hmac.h>
#else
# ifdef HAVE_CRYPTO_HMAC_H
#  include <crypto/hmac.h>
# else
#  ifdef HAVE_HMAC_H
#   include <hmac.h>
#  endif
# endif
#endif

#include <openssl/err.h>
#include <openssl/ssl.h>

/*
 * Constants / definitions
 */

/* Maximum key length as specified in the S3 documentation
 * (*excluding* null terminator) */
#define S3_MAX_KEY_LENGTH 1024

#if defined(LIBCURL_FEATURE_SSL) && defined(LIBCURL_PROTOCOL_HTTPS)
# define S3_URL "https://s3.amazonaws.com"
#else
# define S3_URL "http://s3.amazonaws.com"
#endif

#define AMAZON_SECURITY_HEADER "x-amz-security-token"

/* parameters for exponential backoff in the face of retriable errors */

/* start at 0.01s */
#define EXPONENTIAL_BACKOFF_START_USEC 10000
/* double at each retry */
#define EXPONENTIAL_BACKOFF_BASE 2
/* retry 15 times (for a total of about 5 minutes spent waiting) */
#define EXPONENTIAL_BACKOFF_MAX_RETRIES 5

/* general "reasonable size" parameters */
#define MAX_ERROR_RESPONSE_LEN (100*1024)

/* Results which should always be retried */
#define RESULT_HANDLING_ALWAYS_RETRY \
        { 400,  S3_ERROR_RequestTimeout,     0,                         S3_RESULT_RETRY }, \
        { 409,  S3_ERROR_OperationAborted,   0,                         S3_RESULT_RETRY }, \
        { 412,  S3_ERROR_PreconditionFailed, 0,                         S3_RESULT_RETRY }, \
        { 500,  S3_ERROR_InternalError,      0,                         S3_RESULT_RETRY }, \
        { 501,  S3_ERROR_NotImplemented,     0,                         S3_RESULT_RETRY }, \
        { 0,    0,                           CURLE_COULDNT_CONNECT,     S3_RESULT_RETRY }, \
        { 0,    0,                           CURLE_PARTIAL_FILE,        S3_RESULT_RETRY }, \
        { 0,    0,                           CURLE_OPERATION_TIMEOUTED, S3_RESULT_RETRY }, \
        { 0,    0,                           CURLE_SEND_ERROR,          S3_RESULT_RETRY }, \
        { 0,    0,                           CURLE_RECV_ERROR,          S3_RESULT_RETRY }

/*
 * Data structures and associated functions
 */

struct S3Handle {
    /* (all strings in this struct are freed by s3_free()) */

    char *access_key;
    char *secret_key;
#ifdef WANT_DEVPAY
    char *user_token;
#endif

    CURL *curl;

    gboolean verbose;

    /* information from the last request */
    char *last_message;
    guint last_response_code;
    s3_error_code_t last_s3_error_code;
    CURLcode last_curl_code;
    guint last_num_retries;
    void *last_response_body;
    guint last_response_body_size;
};

/*
 * S3 errors */

/* (see preprocessor magic in s3.h) */

static char * s3_error_code_names[] = {
#define S3_ERROR(NAME) #NAME
    S3_ERROR_LIST
#undef S3_ERROR
};

/* Convert an s3 error name to an error code.  This function
 * matches strings case-insensitively, and is appropriate for use
 * on data from the network.
 *
 * @param s3_error_code: the error name
 * @returns: the error code (see constants in s3.h)
 */
static s3_error_code_t
s3_error_code_from_name(char *s3_error_name);

/* Convert an s3 error code to a string
 *
 * @param s3_error_code: the error code to convert
 * @returns: statically allocated string
 */
static const char *
s3_error_name_from_code(s3_error_code_t s3_error_code);

/*
 * result handling */

/* result handling is specified by a static array of result_handling structs,
 * which match based on response_code (from HTTP) and S3 error code.  The result
 * given for the first match is used.  0 acts as a wildcard for both response_code
 * and s3_error_code.  The list is terminated with a struct containing 0 for both
 * response_code and s3_error_code; the result for that struct is the default
 * result.
 *
 * See RESULT_HANDLING_ALWAYS_RETRY for an example.
 */
typedef enum {
    S3_RESULT_RETRY = -1,
    S3_RESULT_FAIL = 0,
    S3_RESULT_OK = 1
} s3_result_t;

typedef struct result_handling {
    guint response_code;
    s3_error_code_t s3_error_code;
    CURLcode curl_code;
    s3_result_t result;
} result_handling_t;

/* Lookup a result in C{result_handling}.
 *
 * @param result_handling: array of handling specifications
 * @param response_code: response code from operation
 * @param s3_error_code: s3 error code from operation, if any
 * @param curl_code: the CURL error, if any
 * @returns: the matching result
 */
static s3_result_t
lookup_result(const result_handling_t *result_handling,
              guint response_code,
              s3_error_code_t s3_error_code,
              CURLcode curl_code);

/*
 * Precompiled regular expressions */

static const char *error_name_regex_string = "<Code>[:space:]*([^<]*)[:space:]*</Code>";
static const char *message_regex_string = "<Message>[:space:]*([^<]*)[:space:]*</Message>";
static regex_t error_name_regex, message_regex;

/*
 * Utility functions
 */

/* Build a resource URI as /[bucket[/key]], with proper URL
 * escaping.
 *
 * The caller is responsible for freeing the resulting string.
 *
 * @param bucket: the bucket, or NULL if none is involved
 * @param key: the key within the bucket, or NULL if none is involved
 * @returns: completed URI
 */
static char *
build_resource(const char *bucket,
               const char *key);

/* Create proper authorization headers for an Amazon S3 REST
 * request to C{headers}.
 *
 * @note: C{X-Amz} headers (in C{headers}) must
 *  - be in lower-case
 *  - be in alphabetical order
 *  - have no spaces around the colon
 * (don't yell at me -- see the Amazon Developer Guide)
 *
 * @param hdl: the S3Handle object
 * @param verb: capitalized verb for this request ('PUT', 'GET', etc.)
 * @param resource: the resource being accessed
 */
static struct curl_slist *
authenticate_request(S3Handle *hdl,
                     const char *verb,
                     const char *resource);

/* Interpret the response to an S3 operation, assuming CURL completed its request
 * successfully.  This function fills in the relevant C{hdl->last*} members.
 *
 * @param hdl: The S3Handle object
 * @param body: the response body
 * @param body_len: the length of the response body
 * @returns: TRUE if the response should be retried (e.g., network error)
 */
static gboolean
interpret_response(S3Handle *hdl,
                   CURLcode curl_code,
                   char *curl_error_buffer,
                   void *body,
                   guint body_len);

/* Perform an S3 operation.  This function handles all of the details
 * of retryig requests and so on.
 * 
 * @param hdl: the S3Handle object
 * @param resource: the UTF-8 encoded resource to access
                    (without query parameters)
 * @param uri: the urlencoded URI to access at Amazon (may be identical to resource)
 * @param verb: the HTTP request method
 * @param request_body: the request body, or NULL if none should be sent
 * @param request_body_size: the length of the request body
 * @param max_response_size: the maximum number of bytes to accept in the
 * response, or 0 for no limit.
 * @param preallocate_response_size: for more efficient operation, preallocate
 * a buffer of this size for the response body.  Addition space will be allocated
 * if the response exceeds this size.
 * @param result_handling: instructions for handling the results; see above.
 * @returns: the result specified by result_handling; details of the response
 * are then available in C{hdl->last*}
 */
static s3_result_t
perform_request(S3Handle *hdl,
                const char *resource,
                const char *uri,
                const char *verb,
                const void *request_body,
                guint request_body_size,
                guint max_response_size,
                guint preallocate_response_size,
                const result_handling_t *result_handling);

/*
 * Static function implementations
 */

/* {{{ s3_error_code_from_name */
static s3_error_code_t
s3_error_code_from_name(char *s3_error_name)
{
    int i;

    if (!s3_error_name) return S3_ERROR_Unknown;

    /* do a brute-force search through the list, since it's not sorted */
    for (i = 0; i < S3_ERROR_END; i++) {
        if (strcasecmp(s3_error_name, s3_error_code_names[i]) == 0)
            return i;
    }

    return S3_ERROR_Unknown;
}
/* }}} */

/* {{{ s3_error_name_from_code */
static const char *
s3_error_name_from_code(s3_error_code_t s3_error_code)
{
    if (s3_error_code >= S3_ERROR_END)
        s3_error_code = S3_ERROR_Unknown;

    if (s3_error_code == 0)
        return NULL;

    return s3_error_code_names[s3_error_code];
}
/* }}} */

/* {{{ lookup_result */
static s3_result_t
lookup_result(const result_handling_t *result_handling,
              guint response_code,
              s3_error_code_t s3_error_code,
              CURLcode curl_code)
{
    g_return_val_if_fail(result_handling != NULL, S3_RESULT_FAIL);

    while (result_handling->response_code
        || result_handling->s3_error_code 
        || result_handling->curl_code) {
        if ((result_handling->response_code && result_handling->response_code != response_code)
         || (result_handling->s3_error_code && result_handling->s3_error_code != s3_error_code)
         || (result_handling->curl_code && result_handling->curl_code != curl_code)) {
            result_handling++;
            continue;
        }

        return result_handling->result;
    }

    /* return the result for the terminator, as the default */
    return result_handling->result;
}
/* }}} */

/* {{{ build_resource */
static char *
build_resource(const char *bucket,
               const char *key)
{
    char *esc_bucket = NULL, *esc_key = NULL;
    char *resource = NULL;

    if (bucket)
        if (!(esc_bucket = curl_escape(bucket, 0)))
            goto cleanup;

    if (key)
        if (!(esc_key = curl_escape(key, 0)))
            goto cleanup;

    if (esc_bucket) {
        if (esc_key) {
            resource = g_strdup_printf("/%s/%s", esc_bucket, esc_key);
        } else {
            resource = g_strdup_printf("/%s", esc_bucket);
        }
    } else {
        resource = g_strdup("/");
    }
cleanup:
    if (esc_bucket) curl_free(esc_bucket);
    if (esc_key) curl_free(esc_key);

    return resource;
}
/* }}} */

/* {{{ authenticate_request */
static struct curl_slist *
authenticate_request(S3Handle *hdl,
                     const char *verb,
                     const char *resource) 
{
    time_t t;
    struct tm tmp;
    char date[100];
    char * buf;
    HMAC_CTX ctx;
    char md_value[EVP_MAX_MD_SIZE+1];
    char auth_base64[40];
    unsigned int md_len;
    struct curl_slist *headers = NULL;
    char * auth_string;

    /* calculate the date */
    t = time(NULL);
    if (!localtime_r(&t, &tmp)) perror("localtime");
    if (!strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S %Z", &tmp)) 
        perror("strftime");

    /* run HMAC-SHA1 on the canonicalized string */
    HMAC_CTX_init(&ctx);
    HMAC_Init_ex(&ctx, hdl->secret_key, strlen(hdl->secret_key), EVP_sha1(), NULL);
    auth_string = g_strconcat(verb, "\n\n\n", date, "\n",
#ifdef WANT_DEVPAY
                              AMAZON_SECURITY_HEADER, ":",
                              hdl->user_token, ",",
                              STS_PRODUCT_TOKEN, "\n",
#endif
                              resource, NULL);
    HMAC_Update(&ctx, (unsigned char*) auth_string, strlen(auth_string));
    g_free(auth_string);
    md_len = EVP_MAX_MD_SIZE;
    HMAC_Final(&ctx, (unsigned char*)md_value, &md_len);
    HMAC_CTX_cleanup(&ctx);
    base64_encode(md_value, md_len, auth_base64, sizeof(auth_base64));

    /* append the new headers */
#ifdef WANT_DEVPAY
    /* Devpay headers are included in hash. */
    buf = g_strdup_printf(AMAZON_SECURITY_HEADER ": %s", hdl->user_token);
    headers = curl_slist_append(headers, buf);
    amfree(buf);

    buf = g_strdup_printf(AMAZON_SECURITY_HEADER ": %s", STS_PRODUCT_TOKEN);
    headers = curl_slist_append(headers, buf);
    amfree(buf);
#endif

    buf = g_strdup_printf("Authorization: AWS %s:%s",
                          hdl->access_key, auth_base64);
    headers = curl_slist_append(headers, buf);
    amfree(buf);
    
    buf = g_strdup_printf("Date: %s", date);
    headers = curl_slist_append(headers, buf);
    amfree(buf);

    return headers;
}
/* }}} */

/* {{{ interpret_response */
static void
regex_error(regex_t *regex, int reg_result)
{
    char *message;
    int size;

    size = regerror(reg_result, regex, NULL, 0);
    message = g_malloc(size);
    if (!message) abort(); /* we're really out of luck */