s3-device.c

开源备份软件源码 AMANDA, the Advanced Maryland Automatic Network Disk Archiver, is a backup system that a

/*
 * Copyright (c) 2005 Zmanda, Inc.  All Rights Reserved.
 * 
 * This library is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License version 2.1 as 
 * published by the Free Software Foundation.
 * 
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
 * License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with this library; if not, write to the Free Software Foundation,
 * Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA.
 * 
 * Contact information: Zmanda Inc., 505 N Mathlida Ave, Suite 120
 * Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
 */

/* An S3 device uses Amazon's S3 service (http://www.amazon.com/s3) to store 
 * data.  It stores data in keys named with a user-specified prefix, inside a
 * user-specified bucket.  Data is stored in the form of numbered (large) 
 * blocks. 
 */

#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h>
#include <regex.h>
#include <time.h>
#include "util.h"
#include "amanda.h"
#include "conffile.h"
#include "device.h"
#include "s3-device.h"
#include <curl/curl.h>
#ifdef HAVE_OPENSSL_HMAC_H
# include <openssl/hmac.h>
#else
# ifdef HAVE_CRYPTO_HMAC_H
#  include <crypto/hmac.h>
# else
#  ifdef HAVE_HMAC_H
#   include <hmac.h>
#  endif
# endif
#endif

/*
 * Constants and static data
 */

/* Maximum key length as specified in the S3 documentation
 * (*excluding* null terminator) */
#define S3_MAX_KEY_LENGTH 1024

#define S3_DEVICE_MIN_BLOCK_SIZE 1024
#define S3_DEVICE_MAX_BLOCK_SIZE (10*1024*1024)

/* This goes in lieu of file number for metadata. */
#define SPECIAL_INFIX "special-"

/* pointer to the class of our parent */
static DeviceClass *parent_class = NULL;

/*
 * prototypes
 */

/* 
 * utility functions */

/* Given file and block numbers, return an S3 key.
 * 
 * @param self: the S3Device object
 * @param file: the file number
 * @param block: the block within that file
 * @returns: a newly allocated string containing an S3 key.
 */
static char * 
file_and_block_to_key(S3Device *self, 
                      int file, 
                      guint64 block);

/* Given the name of a special file (such as 'tapestart'), generate
 * the S3 key to use for that file.
 *
 * @param self: the S3Device object
 * @param special_name: name of the special file
 * @param file: a file number to include; omitted if -1
 * @returns: a newly alocated string containing an S3 key.
 */
static char * 
special_file_to_key(S3Device *self, 
                    char *special_name, 
                    int file);
/* Write an amanda header file to S3.
 *
 * @param self: the S3Device object
 * @param label: the volume label
 * @param timestamp: the volume timestamp
 */
static gboolean 
write_amanda_header(S3Device *self, 
                    char *label, 
                    char * timestamp);

/* "Fast forward" this device to the end by looking up the largest file number
 * present and setting the current file number one greater.
 *
 * @param self: the S3Device object
 */
static gboolean 
seek_to_end(S3Device *self);

/* Find the number of the last file that contains any data (even just a header). 
 *
 * @param self: the S3Device object
 * @returns: the last file, or -1 in event of an error
 */
static int 
find_last_file(S3Device *self);

/* Delete all blocks in the given file, including the filestart block
 *
 * @param self: the S3Device object
 * @param file: the file to delete
 */
static gboolean 
delete_file(S3Device *self, 
            int file);

/* Set up self->s3 as best as possible.  Unless SILENT is TRUE,
 * any problems will generate warnings (with g_warning).  Regardless,
 * the return value is TRUE iff self->s3 is useable.
 *
 * @param self: the S3Device object
 * @param silent: silence warnings
 * @returns: TRUE if the handle is set up
 */
static gboolean 
setup_handle(S3Device * self, 
	     gboolean ignore_problems);

/* 
 * class mechanics */

static void
s3_device_init(S3Device * o);

static void
s3_device_class_init(S3DeviceClass * c);

static void
s3_device_finalize(GObject * o);

static Device*
s3_device_factory(char * device_type,
                  char * device_name);

/* 
 * virtual functions */

static gboolean
s3_device_open_device(Device *pself, 
                      char *device_name);

static ReadLabelStatusFlags s3_device_read_label(Device * self);

static gboolean 
s3_device_start(Device * self, 
                DeviceAccessMode mode, 
                char * label, 
                char * timestamp);

static gboolean 
s3_device_start_file(Device * self,
                     const dumpfile_t * jobInfo);

static gboolean 
s3_device_write_block(Device * self, 
                      guint size, 
                      gpointer data, 
                      gboolean last);

static gboolean 
s3_device_finish_file(Device * self);

static dumpfile_t* 
s3_device_seek_file(Device *pself, 
                    guint file);

static gboolean 
s3_device_seek_block(Device *pself, 
                     guint64 block);

static gboolean 
s3_device_read_block(Device * pself, 
                     gpointer data, 
                     int *size_req);

static gboolean 
s3_device_recycle_file(Device *pself, 
                       guint file);

static gboolean s3_device_property_set(Device * p_self, DevicePropertyId id,
                                       GValue * val);
static gboolean s3_device_property_get(Device * p_self, DevicePropertyId id,
                                       GValue * val);
/*
 * Private functions
 */

/* {{{ file_and_block_to_key */
static char *
file_and_block_to_key(S3Device *self, 
                      int file, 
                      guint64 block)
{
    char *s3_key = g_strdup_printf("%sf%08x-b%016llx.data",
                                   self->prefix, file, (long long unsigned int)block);
    g_assert(strlen(s3_key) <= S3_MAX_KEY_LENGTH);
    return s3_key;
}
/* }}} */

/* {{{ special_file_to_key */
static char *
special_file_to_key(S3Device *self, 
                    char *special_name, 
                    int file)
{
    if (file == -1)
        return g_strdup_printf("%s" SPECIAL_INFIX "%s", self->prefix, special_name);
    else
        return g_strdup_printf("%sf%08x-%s", self->prefix, file, special_name);
}
/* }}} */

/* {{{ write_amanda_header */
static gboolean
write_amanda_header(S3Device *self, 
                    char *label, 
                    char * timestamp)
{
    char * amanda_header = NULL;
    char * key = NULL;
    int header_size;
    gboolean header_fits, result;
    dumpfile_t * dumpinfo = NULL;

    /* build the header */
    dumpinfo = make_tapestart_header(DEVICE(self), label, timestamp);
    amanda_header = device_build_amanda_header(DEVICE(self), dumpinfo, 
                                               &header_size, &header_fits);
    if (!header_fits) {
        fprintf(stderr,
                _("Amanda tapestart header won't fit in a single block!\n"));
	g_free(amanda_header);
	return FALSE;
    }

    /* write out the header and flush the uploads. */
    key = special_file_to_key(self, "tapestart", -1);
    result = s3_upload(self->s3, self->bucket, key, amanda_header, header_size);
    g_free(amanda_header);
    g_free(key);

    if (!result) {
        fprintf(stderr, _("While writing amanda header: %s\n"),
                s3_strerror(self->s3));
    }
    return result;
}
/* }}} */

/* {{{ seek_to_end */
static gboolean
seek_to_end(S3Device *self) {
    int last_file;

    Device *pself = DEVICE(self);

    last_file = find_last_file(self);
    if (last_file < 0)
        return FALSE;

    pself->file = last_file;

    return TRUE;
}
/* }}} */

/* Convert an object name into a file number, assuming the given prefix
 * length. Returns -1 if the object name is invalid, or 0 if the object name
 * is a "special" key. */
static int key_to_file(guint prefix_len, const char * key) {
    int file;
    int i;
    
    /* skip the prefix */
    g_return_val_if_fail(strlen(key) > prefix_len, -1);

    key += prefix_len;

    if (strncmp(key, SPECIAL_INFIX, strlen(SPECIAL_INFIX)) == 0) {
        return 0;
    }
    
    /* check that key starts with 'f' */
    g_return_val_if_fail(key[0] == 'f', -1);
    key++;
    
    /* check that key is of the form "%08x-" */
    for (i = 0; i < 8; i++) {
        if (!(key[i] >= '0' && key[i] <= '9') &&
            !(key[i] >= 'a' && key[i] <= 'f') &&
            !(key[i] >= 'A' && key[i] <= 'F')) break;
    }
    if (key[i] != '-') return -1;
    if (i < 8) return -1;

    /* convert the file number */
    errno = 0;
    file = strtoul(key, NULL, 16);
    if (errno != 0) {
        g_warning(_("unparseable file number '%s'"), key);
        return -1;
    }
    
    return file;
}

/* {{{ find_last_file */
/* Find the number of the last file that contains any data (even just a header). 
 * Returns -1 in event of an error
 */
static int
find_last_file(S3Device *self) {
    gboolean result;
    GSList *keys;
    unsigned int prefix_len = strlen(self->prefix);
    int last_file = 0;

    /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
    result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
    if (!result) {
        fprintf(stderr, _("While listing S3 keys: %s\n"),
                s3_strerror(self->s3));
        return -1;
    }

    for (; keys; keys = g_slist_remove(keys, keys->data)) {
        int file = key_to_file(prefix_len, keys->data);

        /* and if it's the last, keep it */
        if (file > last_file)
            last_file = file;
    }

    return last_file;
}
/* }}} */

/* {{{ find_next_file */
/* Find the number of the file following the requested one, if any. 
 * Returns 0 if there is no such file or -1 in event of an error
 */
static int
find_next_file(S3Device *self, int last_file) {
    gboolean result;
    GSList *keys;
    unsigned int prefix_len = strlen(self->prefix);
    int next_file = 0;

    /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
    result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
    if (!result) {
        fprintf(stderr, _("While listing S3 keys: %s\n"),
                s3_strerror(self->s3));
        return -1;
    }

    for (; keys; keys = g_slist_remove(keys, keys->data)) {
        int file;

        file = key_to_file(prefix_len, (char*)keys->data);

        if (file < 0) {
            /* Set this in case we don't find a next file; this is not a
             * hard error, so if we can find a next file we'll return that
             * instead. */