s3_glue.cpp

sleuthit-2.09 一个磁盘的工具集

#include "config.h"

#include <stdio.h>
#include <errno.h>
#include <stdlib.h>

#ifdef USE_S3

#ifdef HAVE_SYS_PARAM_H
#include <sys/param.h>
#endif

#if (defined(__FreeBSD_version)) && (__FreeBSD_version<500000) && (!defined(BAD_STL))
#define BAD_STL
#endif


#include "s3_glue.h"
#include "curl/curl.h"
#include "base64.h"

#include <expat.h>
#include <time.h>
#include <netinet/in.h>

#ifdef HAVE_OPENSSL_MD5_H
#include <openssl/md5.h>
#endif

#ifdef HAVE_OPENSSL_HMAC_H
#include <openssl/hmac.h>
#endif

#ifdef HAVE_ERR_H
#include <err.h>
#endif

#ifdef HAVE_CTYPE_H
#include <ctype.h>
#endif

#if !defined(HAVE_OPENSSL_MD5_H)
#error S3 support requires MD5 support
#endif



int s3_debug = 0;
int s3_retry_max   = 5;			// read by the code

/* debug levels:
 *  1 - print retries
 *  2- print queries
 *  3- print full results
 */

/* Counters that are used; they aren't threadsafe, but they are never referenced */
int s3_request_retry_count = 0;
int s3_object_put_retry_count = 0;
long long s3_bytes_written=0;
long long s3_bytes_read=0;

using namespace std;
using namespace s3;

const char *aws_access_key_id;
const char *aws_secret_access_key;
const char *aws_base_url = "http://s3.amazonaws.com/";

/* Simson's S3 implementation in C++.
 * Note that libcurl and expat will both handle data in chunks, so
 * technically we don't need to create a single buffer with the entire response
 * from AWS. For AFFLIB, though, we want to work on data as buffers.
 * As a result, we create and use a buffer for all work.
 */

namespace s3 {

static string itos(int i)
{
    char buf[64];
    snprintf(buf,sizeof(buf),"%d",i);
    return string(buf);
}

size_t buffer::write(const char *b,size_t count){
    if(!writable) return false;
    base = (char *)realloc(base,len+count);
    if(base){
	memcpy(base+len,b,count);	// copy the memory over
	len += count;
	return count;
    }
    return 0;
}
size_t buffer::read(char *b,size_t count){
    if(base){
	if(count>len-ptr) count=len-ptr;
	memcpy(b,base+ptr,count);
	ptr += count;
	return count;
    }
    return 0;
}
	
void buffer::print() {
    fwrite(base,1,len,stdout);
}

void buffer::clear(){
    if(base){
	free(base);
	base = 0;
    }
    len = 0;
}


static size_t  buffer_write(void  *buffer,  size_t  size,  size_t  nmemb,  void *userp)
{
    return ((class buffer *)userp)->write((const char *)buffer,size * nmemb);
}

static size_t  buffer_read(void  *buffer,  size_t  size,  size_t  nmemb,  void *userp)
{
    return ((class buffer *)userp)->read((char *)buffer,size * nmemb);
}


static void startElement(void *userData, const char *name, const char **atts)
{
    class s3_result *einfo = (class s3_result *)userData;
    einfo->depth++;
    switch(einfo->depth){
    case 1:
	if(!strcmp(name,"ListBucketResult")) {einfo->lbr = new ListBucketResult();break;}
	if(!strcmp(name,"ListAllMyBucketsResult")) {einfo->lambr = new ListAllMyBucketsResult();break;}
	fprintf(stderr,"\ns3 buffer:\n%s",einfo->buf->base);
	errx(1,"Unknown XML element from S3: '%s'",name);
	break;
    case 2:
	if(einfo->lbr && !strcmp(name,"Contents")){ einfo->lbr->contents.push_back(new Contents());break;}
	break;
    case 3:
	if(einfo->lambr && !strcmp(name,"Bucket")){ einfo->lambr->Buckets.push_back(new Bucket());break;}
	break;
    }
}

static void endElement(void *userData, const char *name)
{
    class s3_result *einfo = (class s3_result *)userData;
    if(einfo->lambr){
	switch(einfo->depth){
	case 3:
	    if(!strcmp(name,"ID")){ einfo->lambr->OwnerID = einfo->cbuf;break;}
	    if(!strcmp(name,"DisplayName")){ einfo->lambr->OwnerDisplayName = einfo->cbuf;break;}
	    break;
	case 4:
	    if(!strcmp(name,"Name")) { einfo->lambr->Buckets.back()->Name = einfo->cbuf;break;}
	    if(!strcmp(name,"CreationDate")) { einfo->lambr->Buckets.back()->CreationDate = einfo->cbuf;break;}
	}
    }
    if(einfo->lbr){
	switch(einfo->depth){
	case 2:
	    if(!strcmp(name,"Name")){	     einfo->lbr->Name = einfo->cbuf; break;}
	    if(!strcmp(name,"Prefix")){      einfo->lbr->Prefix = einfo->cbuf;break;}
	    if(!strcmp(name,"Marker")){      einfo->lbr->Marker = einfo->cbuf;break;}
	    if(!strcmp(name,"MaxKeys")){     einfo->lbr->MaxKeys = atoi(einfo->cbuf.c_str());break;}
	    if(!strcmp(name,"IsTruncated")){ einfo->lbr->IsTruncated = tolower(einfo->cbuf[0]) == 't';break;}
	    break;
	case 3:
	    if(!strcmp(name,"Key")){         einfo->lbr->contents.back()->Key = einfo->cbuf; break;}
	    if(!strcmp(name,"LastModified")){einfo->lbr->contents.back()->LastModified = einfo->cbuf;break;}
	    if(!strcmp(name,"ETag")){        einfo->lbr->contents.back()->ETag = einfo->cbuf;break;}
	    if(!strcmp(name,"Size")){        einfo->lbr->contents.back()->Size = atoi(einfo->cbuf.c_str());break;}
	    break;
	case 4:
	    if(!strcmp(name,"ID")){          einfo->lbr->contents.back()->OwnerID = einfo->cbuf;break;}
	    if(!strcmp(name,"DisplayName")){ einfo->lbr->contents.back()->OwnerDisplayName = einfo->cbuf;break;}
	    break;
	default:;
	}
    }
#ifdef BAD_STL
    einfo->cbuf = "";
#else    
    einfo->cbuf.clear();
#endif
    einfo->depth--;
}

static void characterDataHandler(void *userData,const XML_Char *s,int len)
{
    class s3_result *einfo = (class s3_result *)userData;
    einfo->cbuf.append((const char *)s,len);
}


static class s3_result *xml_extract_response(const class buffer *buf) 
{
    class s3_result *e = new s3_result();

    e->buf = buf;
    XML_Parser parser = XML_ParserCreate(NULL);
    XML_SetUserData(parser, e);
    XML_SetElementHandler(parser, startElement, endElement);
    XML_SetCharacterDataHandler(parser,characterDataHandler);

    if (!XML_Parse(parser, (const char *)buf->base, buf->len, 1)) {
	char buf2[2048];
	snprintf(buf2,sizeof(buf2),
		 "XML Error: %s at line %d",
		 XML_ErrorString(XML_GetErrorCode(parser)),(int)XML_GetCurrentLineNumber(parser));
	fprintf(stderr,"%s:\n",buf2);
	XML_ParserFree(parser);
	return 0;
    }
    XML_ParserFree(parser);
    return e;
}



/* Create the cannonical string for the headers */
static string canonical_string(string method,string path,curl_slist *headers, time_t expires)
{
    /* Iterate through the headers a line at a time */
    map<string,string> interesting_headers;
    
    for(;headers;headers = headers->next){
	char *line = strdup(headers->data);
	char *word;
	char *brk2;
	word = strtok_r(line,": ",&brk2);
	if(word){
	    if(strcasecmp(word,"Date")==0 ||
	       strcasecmp(word,"Range")==0 ||
	       strncmp(word,AMAZON_METADATA_PREFIX,strlen(AMAZON_METADATA_PREFIX))==0){
		char *value = strtok_r(NULL,"",&brk2);
		while(value && isspace(*value)) value++;
		interesting_headers[word] = value;
	    }
	}
	free(line);
    }
    /* Add the headers that we don't have */

    /* handle the date */

    /* Get the sorted headers */
    vector<string> sorted_header_keys;
    for(map<string,string>::const_iterator i = interesting_headers.begin();
	i!=interesting_headers.end();
	i++){
	sorted_header_keys.push_back(i->first);
    }

#ifndef BAD_STL
    sort(sorted_header_keys.begin(),sorted_header_keys.end());
#endif

    string buf = method + "\n";
    buf += "\n";			// content-md5 value
    buf += "\n";			// content-type value

    /* Either put in a date header or else do the expires */
    if(expires){
	char b[64];
	snprintf(b,sizeof(b),"%d\n",(int)expires);
	buf += b;
    }
    else {
	buf += interesting_headers["Date"] + "\n"; //  date
    }

    /* AMAON_HEADER_PREFIX headers only... */
    for(vector<string>::const_iterator i = sorted_header_keys.begin();
	i != sorted_header_keys.end();
	i++){
	if(i->substr(0,strlen(AMAZON_METADATA_PREFIX))==AMAZON_METADATA_PREFIX){
	    buf += *i + ":" + interesting_headers[*i] + "\n";
	}
    }
    buf += "/" + path;			// the resource

    //printf("canonical: \n===========\n%s\n=========\n",buf.c_str());

    return buf;

}

static string encode(const char *aws_secret_access_key,string str)
{
    unsigned char md[20];
    unsigned int md_len = sizeof(md);

    HMAC(EVP_sha1(),aws_secret_access_key,strlen(aws_secret_access_key),
	 (const unsigned char *)str.c_str(),str.size(),
	 md,&md_len);
    /* Now encode this to base64 */
    char b64str[64];
    memset(b64str,sizeof(b64str),0);
    b64_ntop(md,md_len,b64str,sizeof(b64str));
    return string(b64str);
}



static string quote_plus(string &url)
{
    /* encode the URL */
    string eurl;
    char buf[6];
    for(string::const_iterator c = url.begin(); c != url.end(); c++){
	switch(*c){
	case '%':
	case ';':
	case '/':
	case '?':
	case '@':
	case '&':
	case '=':
	case '+':
	case '$':
	case ',':
	    sprintf(buf,"%%%02X",*c);
	    eurl += buf;
	    continue;
	case ' ':
	    eurl += "+";
	    continue;
	default:
	    eurl += *c;
	    continue;
	}
    }
    return eurl;
}

#ifndef HAVE_ISDIGIT
static int isdigit(char ch)
{
    return ch>='0' && ch<='9';
}
#endif

static int hexval(int ch) { return (isdigit(ch) ? ch-'0' : ch-'a'+10);}

/*
 * Execute an S3 request:
 * method  - method to execute.
 * path    - path for the object.
 * query   - anything optional after the "?" in the path
 * expires - When the authorization URL should expire.
 * sendbuf - if we are sending something ,this is what is being sent.
 * sendbuflen - how long that buffer is
 * extraheaders - any additional headers that should be sent; useful for metadata
 * 
 * Returns a response buffer
 */

/* CURLINFO_RESPONSE_CODE is the new name for the option previously known as 
 * CURLINFO_HTTP_CODE.
 */


#ifndef CURLINFO_RESPONSE_CODE