s3.c

开源备份软件源码 AMANDA, the Advanced Maryland Automatic Network Disk Archiver, is a backup system that a

    regerror(reg_result, regex, message, size);

    /* this is programmer error (bad regexp), so just log
     * and abort().  There's no good way to signal a
     * permanaent error from interpret_response. */
    g_error(_("Regex error: %s"), message);
    g_assert_not_reached();
}

static gboolean
interpret_response(S3Handle *hdl,
                   CURLcode curl_code,
                   char *curl_error_buffer,
                   void *body,
                   guint body_len)
{
    long response_code = 0;
    regmatch_t pmatch[2];
    int reg_result;
    char *error_name = NULL, *message = NULL;
    char *body_copy = NULL;

    if (!hdl) return FALSE;

    if (hdl->last_message) g_free(hdl->last_message);
    hdl->last_message = NULL;

    /* bail out from a CURL error */
    if (curl_code != CURLE_OK) {
        hdl->last_curl_code = curl_code;
        hdl->last_message = g_strdup_printf("CURL error: %s", curl_error_buffer);
        return FALSE;
    }

    /* CURL seems to think things were OK, so get its response code */
    curl_easy_getinfo(hdl->curl, CURLINFO_RESPONSE_CODE, &response_code);
    hdl->last_response_code = response_code;

    /* 2xx and 3xx codes won't have a response body*/
    if (200 <= response_code && response_code < 400) {
        hdl->last_s3_error_code = S3_ERROR_None;
        return FALSE;
    }

    /* Now look at the body to try to get the actual Amazon error message. Rather
     * than parse out the XML, just use some regexes. */

    /* impose a reasonable limit on body size */
    if (body_len > MAX_ERROR_RESPONSE_LEN) {
        hdl->last_message = g_strdup("S3 Error: Unknown (response body too large to parse)");
        return FALSE;
    } else if (!body || body_len == 0) {
        hdl->last_message = g_strdup("S3 Error: Unknown (empty response body)");
        return TRUE; /* perhaps a network error; retry the request */
    }

    /* use strndup to get a zero-terminated string */
    body_copy = g_strndup(body, body_len);
    if (!body_copy) goto cleanup;

    reg_result = regexec(&error_name_regex, body_copy, 2, pmatch, 0);
    if (reg_result != 0) {
        if (reg_result == REG_NOMATCH) {
            error_name = NULL;
        } else {
            regex_error(&error_name_regex, reg_result);
            g_assert_not_reached();
        }
    } else {
        error_name = find_regex_substring(body_copy, pmatch[1]);
    }

    reg_result = regexec(&message_regex, body_copy, 2, pmatch, 0);
    if (reg_result != 0) {
        if (reg_result == REG_NOMATCH) {
            message = NULL;
        } else {
            regex_error(&message_regex, reg_result);
            g_assert_not_reached();
        }
    } else {
        message = find_regex_substring(body_copy, pmatch[1]);
    }

    if (error_name) {
        hdl->last_s3_error_code = s3_error_code_from_name(error_name);
    }

    if (message) {
        hdl->last_message = message;
        message = NULL; /* steal the reference to the string */
    }

cleanup:
    if (body_copy) g_free(body_copy);
    if (message) g_free(message);
    if (error_name) g_free(error_name);

    return FALSE;
}
/* }}} */

/* {{{ perform_request */
size_t buffer_readfunction(void *ptr, size_t size,
                           size_t nmemb, void * stream) {
    CurlBuffer *data = stream;
    guint bytes_desired = size * nmemb;

    /* check the number of bytes remaining, just to be safe */
    if (bytes_desired > data->buffer_len - data->buffer_pos)
        bytes_desired = data->buffer_len - data->buffer_pos;

    memcpy((char *)ptr, data->buffer + data->buffer_pos, bytes_desired);
    data->buffer_pos += bytes_desired;

    return bytes_desired;
}

size_t
buffer_writefunction(void *ptr, size_t size, size_t nmemb, void *stream)
{
    CurlBuffer * data = stream;
    guint new_bytes = size * nmemb;
    guint bytes_needed = data->buffer_pos + new_bytes;

    /* error out if the new size is greater than the maximum allowed */
    if (data->max_buffer_size && bytes_needed > data->max_buffer_size)
        return 0;

    /* reallocate if necessary. We use exponential sizing to make this
     * happen less often. */
    if (bytes_needed > data->buffer_len) {
        guint new_size = MAX(bytes_needed, data->buffer_len * 2);
        if (data->max_buffer_size) {
            new_size = MIN(new_size, data->max_buffer_size);
        }
        data->buffer = g_realloc(data->buffer, new_size);
        data->buffer_len = new_size;
    }
    g_return_val_if_fail(data->buffer, 0); /* returning zero signals an error to libcurl */

    /* actually copy the data to the buffer */
    memcpy(data->buffer + data->buffer_pos, ptr, new_bytes);
    data->buffer_pos += new_bytes;

    /* signal success to curl */
    return new_bytes;
}

static int 
curl_debug_message(CURL *curl G_GNUC_UNUSED, 
		   curl_infotype type, 
		   char *s, 
		   size_t len, 
		   void *unused G_GNUC_UNUSED)
{
    char *lineprefix;
    char *message;
    char **lines, **line;

    switch (type) {
	case CURLINFO_TEXT:
	    lineprefix="";
	    break;

	case CURLINFO_HEADER_IN:
	    lineprefix="Hdr In: ";
	    break;

	case CURLINFO_HEADER_OUT:
	    lineprefix="Hdr Out: ";
	    break;

	default:
	    /* ignore data in/out -- nobody wants to see that in the
	     * debug logs! */
	    return 0;
    }

    /* split the input into lines */
    message = g_strndup(s, len);
    lines = g_strsplit(message, "\n", -1);
    g_free(message);

    for (line = lines; *line; line++) {
	if (**line == '\0') continue; /* skip blank lines */
	g_debug("%s%s", lineprefix, *line);
    }
    g_strfreev(lines);

    return 0;
}

static s3_result_t
perform_request(S3Handle *hdl,
                const char *resource,
                const char *uri,
                const char *verb,
                const void *request_body,
                guint request_body_size,
                guint max_response_size,
                guint preallocate_response_size,
                const result_handling_t *result_handling)
{
    char *url = NULL;
    s3_result_t result = S3_RESULT_FAIL; /* assume the worst.. */
    CURLcode curl_code = CURLE_OK;
    char curl_error_buffer[CURL_ERROR_SIZE] = "";
    struct curl_slist *headers = NULL;
    CurlBuffer readdata = { (void*)request_body, request_body_size, 0, 0 };
    CurlBuffer writedata = { NULL, 0, 0, max_response_size };
    gboolean should_retry;
    guint retries = 0;
    gulong backoff = EXPONENTIAL_BACKOFF_START_USEC;

    g_return_val_if_fail(hdl != NULL && hdl->curl != NULL, S3_RESULT_FAIL);

    s3_reset(hdl);

    url = g_strconcat(S3_URL, uri, NULL);
    if (!url) goto cleanup;

    if (preallocate_response_size) {
        writedata.buffer = g_malloc(preallocate_response_size);
        if (!writedata.buffer) goto cleanup;
        writedata.buffer_len = preallocate_response_size;
    }

    while (1) {
        /* reset things */
        if (headers) {
            curl_slist_free_all(headers);
        }
        readdata.buffer_pos = 0;
        writedata.buffer_pos = 0;
	curl_error_buffer[0] = '\0';

        /* set up the request */
        headers = authenticate_request(hdl, verb, resource);

        if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_VERBOSE, hdl->verbose)))
            goto curl_error;
	if (hdl->verbose)
	    if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_DEBUGFUNCTION, 
					      curl_debug_message)))
		goto curl_error;
        if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_ERRORBUFFER,
                                          curl_error_buffer)))
            goto curl_error;
        if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_NOPROGRESS, 1)))
            goto curl_error;
        if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_URL, url)))
            goto curl_error;
        if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_HTTPHEADER,
                                          headers)))
            goto curl_error;
        if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_CUSTOMREQUEST,
                                          verb)))
            goto curl_error;
        if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_WRITEFUNCTION, buffer_writefunction))) 
            goto curl_error;
        if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_WRITEDATA, &writedata))) 
            goto curl_error;
        if (max_response_size) {
#ifdef CURLOPT_MAXFILESIZE_LARGE
            if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_MAXFILESIZE_LARGE, (curl_off_t)max_response_size))) 
                goto curl_error;
#else
# ifdef CURLOPT_MAXFILESIZE
            if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_MAXFILESIZE, (long)max_response_size))) 
                goto curl_error;
# else
	    /* no MAXFILESIZE option -- that's OK */
# endif
#endif
	}

        if (request_body) {
            if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_UPLOAD, 1))) 
                goto curl_error;
#ifdef CURLOPT_INFILESIZE_LARGE
	    if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)request_body_size))) 
                goto curl_error;
#else
	    if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_INFILESIZE, (long)request_body_size))) 
                goto curl_error;
#endif
            if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READFUNCTION, buffer_readfunction))) 
                goto curl_error;
            if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READDATA, &readdata))) 
                goto curl_error;
        } else {
            /* Clear request_body options. */
            if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_UPLOAD, 0))) 
                goto curl_error;
            if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READFUNCTION,
                                              NULL)))
                goto curl_error;
            if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READDATA, 
                                              NULL)))
                goto curl_error;
        }

        /* Perform the request */
        curl_code = curl_easy_perform(hdl->curl);


        /* interpret the response into hdl->last* */
    curl_error: /* (label for short-circuiting the curl_easy_perform call) */
        should_retry = interpret_response(hdl, curl_code, curl_error_buffer, 
                            writedata.buffer, writedata.buffer_pos);
        
        /* and, unless we know we need to retry, see what we're to do now */
        if (!should_retry) {
            result = lookup_result(result_handling, hdl->last_response_code, 
                                   hdl->last_s3_error_code, hdl->last_curl_code);

            /* break out of the while(1) unless we're retrying */
            if (result != S3_RESULT_RETRY)
                break;
        }

        if (retries >= EXPONENTIAL_BACKOFF_MAX_RETRIES) {
            /* we're out of retries, so annotate hdl->last_message appropriately and bail
             * out. */
            char *m = g_strdup_printf("Too many retries; last message was '%s'", hdl->last_message);
            if (hdl->last_message) g_free(hdl->last_message);
            hdl->last_message = m;
            result = S3_RESULT_FAIL;
            break;
        }

        g_usleep(backoff);
        retries++;
        backoff *= EXPONENTIAL_BACKOFF_BASE;
    }

    if (result != S3_RESULT_OK) {
        g_debug(_("%s %s failed with %d/%s"), verb, url,
                hdl->last_response_code,
                s3_error_name_from_code(hdl->last_s3_error_code)); 
    }

cleanup:
    if (url) g_free(url);
    if (headers) curl_slist_free_all(headers);
    
    /* we don't deallocate the response body -- we keep it for later */
    hdl->last_response_body = writedata.buffer;
    hdl->last_response_body_size = writedata.buffer_pos;
    hdl->last_num_retries = retries;

    return result;
}
/* }}} */

/*
 * Public function implementations
 */

/* {{{ s3_init */
gboolean
s3_init(void)
{
    char regmessage[1024];
    int size;
    int reg_result;

    reg_result = regcomp(&error_name_regex, error_name_regex_string, REG_EXTENDED | REG_ICASE);
    if (reg_result != 0) {
        size = regerror(reg_result, &error_name_regex, regmessage, sizeof(regmessage));
        g_error(_("Regex error: %s"), regmessage);
        return FALSE;
    }

    reg_result = regcomp(&message_regex, message_regex_string, REG_EXTENDED | REG_ICASE);
    if (reg_result != 0) {
        size = regerror(reg_result, &message_regex, regmessage, sizeof(regmessage));
        g_error(_("Regex error: %s"), regmessage);
        return FALSE;
    }

    return TRUE;
}
/* }}} */

/* {{{ s3_open */
S3Handle *
s3_open(const char *access_key,
        const char *secret_key
#ifdef WANT_DEVPAY
        ,
        const char *user_token
#endif
        ) {
    S3Handle *hdl;

    hdl = g_new0(S3Handle, 1);
    if (!hdl) goto error;

    hdl->verbose = FALSE;

    hdl->access_key = g_strdup(access_key);
    if (!hdl->access_key) goto error;

    hdl->secret_key = g_strdup(secret_key);
    if (!hdl->secret_key) goto error;

#ifdef WANT_DEVPAY
    hdl->user_token = g_strdup(user_token);
    if (!hdl->user_token) goto error;
#endif

    hdl->curl = curl_easy_init();
    if (!hdl->curl) goto error;

    return hdl;

error:
    s3_free(hdl);
    return NULL;
}
/* }}} */

/* {{{ s3_free */
void
s3_free(S3Handle *hdl)
{
    s3_reset(hdl);

    if (hdl) {
        if (hdl->access_key) g_free(hdl->access_key);
        if (hdl->secret_key) g_free(hdl->secret_key);
#ifdef WANT_DEVPAY
        if (hdl->user_token) g_free(hdl->user_token);
#endif
        if (hdl->curl) curl_easy_cleanup(hdl->curl);

        g_free(hdl);
    }
}
/* }}} */

/* {{{ s3_reset */
void
s3_reset(S3Handle *hdl)
{
    if (hdl) {
        /* We don't call curl_easy_reset here, because doing that in curl
         * < 7.16 blanks the default CA certificate path, and there's no way
         * to get it back. */
        if (hdl->last_message) {
            g_free(hdl->last_message);
            hdl->last_message = NULL;
        }

        hdl->last_response_code = 0;
        hdl->last_curl_code = 0;