aftest.cpp

sleuthit-2.09 一个磁盘的工具集

/*
 * atest.cpp:
 * test suite for the AFF Library.
 */

#include "config.h"
#include "afflib.h"
#include "afflib_i.h"

#include "LzmaRam.h"
extern "C" {
#include "LzmaRamDecode.h"
}

#define MAX_FMTS 10000			// how many formats we should write

char *fmt = "%8d Another format string.\n"; // must be constant size
const char *progname = 0;

char *opt_ext = "aff";
int   opt_compression_level = AF_COMPRESSION_DEFAULT;// default compression level
int   opt_compression_type  = AF_COMPRESSION_ALG_ZLIB;	// 


/* Create the segment that we need */

#ifndef MIN
#define MIN(x,y) ((x)<(y)?(x):(y))
#endif


const char *filename(char *buf,int buflen,const char *base)
{
    snprintf(buf,buflen,"%s.%s",base,opt_ext);
    return buf;
}

void enable_writing(AFFILE *af)
{
    af_enable_writing(af,1);
    af_enable_compression(af,opt_compression_type,opt_compression_level);
    af_set_pagesize(af,1024);
    af_set_maxsize(af,(int64)65536);		// force splitting of raw and afd files
}

int sequential_test()
{
    char buf[1024];

    printf("Sequential test...\n");

    char fn[1024];

    filename(fn,sizeof(fn),"test_sequential");
    unlink(fn);				// make sure it is gone
    AFFILE *af = af_open(fn,O_CREAT|O_RDWR|O_TRUNC,0666);
    if(!af) err(1,"af_open");
    enable_writing(af);
    for(int i=0;i<MAX_FMTS;i++){
	printf("\rwriting %d/%d...",i,MAX_FMTS);
	sprintf(buf,fmt,i);
	if(af_write(af,(unsigned char *)buf,strlen(buf))!=(int)strlen(buf)){
	    err(1,"Attempt to write buffer %d failed\n",i);
	}
    }
    af_close(af);

    printf("\nSequential file written.\n");
    printf("\n");
    printf("Now verifying the string...\n");
    af = af_open(fn,O_RDONLY,0666);
    if(!af) err(1,"af_open");
    for(int i=0;i<MAX_FMTS;i++){
	char rbuf[1024];
	sprintf(buf,fmt,i);
	int len = strlen(buf);
	if(af_read(af,(unsigned char *)rbuf,len)!=len){
	    err(1,"Attempt to read entry %d failed\n",i);
	}
	rbuf[len] = 0;			// terminate the string
	if(strcmp(buf,rbuf)!=0){
	    err(1,"Attempt to verify entry %d failed. Expected: %s. Got: %s\n",i,buf,rbuf);
	}
    }
    af_close(af);

    printf("===========================\n\n");
    return 0;
}

int reverse_test()
{
    char buf[1024];
    char fn[1024];

    printf("Reverse write test...\n");
    filename(fn,sizeof(fn),"test_reverse");
    unlink(fn);
    AFFILE *af = af_open(fn,O_CREAT|O_RDWR|O_TRUNC,0666);
    if(!af) err(1,"af_open");
    enable_writing(af);
    for(int i=MAX_FMTS-1;i>=0;i--){
	sprintf(buf,fmt,i);
	af_seek(af,strlen(buf)*i,SEEK_SET);
	if(af_write(af,(unsigned char *)buf,strlen(buf))!=(int)strlen(buf)){
	    err(1,"Attempt to write buffer %d failed\n",i);
	}
	printf("\r%d  ",i);
	fflush(stdout);
    }
    af_close(af);
    printf("\nReverse test passes.\n");
    printf("======================\n\n");
    return 0;
}


int random_write_test()
{
    char fn[1024];
    char buf[1024];
    char tally[MAX_FMTS];
    int i;

    memset(tally,0,sizeof(tally));

    /* Create the AFF file */
    sprintf(buf,fmt,0);		// figure out how big fmt string is
    int fmt_size = strlen(buf);

    printf("Random write test...\n");
    printf("Creating test file with  %d byte records.\n", fmt_size);

    filename(fn,sizeof(fn),"test_random");
    unlink(fn);				// make sure it is gone
    AFFILE *af = af_open(fn,O_CREAT|O_RDWR|O_TRUNC,0666);
    if(!af) err(1,"af_open");
    enable_writing(af);


    if(af_write(af,(unsigned char *)buf,fmt_size)!=fmt_size){
	err(1,"af_write");
    }
    for(i=0;i<MAX_FMTS;i++){
	/* Find a random spot that's available */
	int pos = rand() % MAX_FMTS;
	while(tally[pos]==1){		//  if this one is used, find next
	    pos = (pos + 1) % MAX_FMTS;
	}
	tally[pos] = 1;
	sprintf(buf,fmt,pos);
	assert((int)strlen(buf)==fmt_size);	// make sure
	af_seek(af,fmt_size*pos,SEEK_SET);
	int wrote = af_write(af,(unsigned char *)buf,fmt_size);
	if(wrote !=fmt_size){
	    fprintf(stderr,"Attempt to write buffer #%d \n",pos);
	    fprintf(stderr,"wrote %d bytes instead of %d bytes\n",wrote,fmt_size);
	    exit(1);
	}
	if(i%25==0) printf("\r%d ...",i);
	fflush(stdout);
    }
    af_close(af);
    
    /* Now verify what was written */
    printf("Verifying write test...\n");
    af = af_open(fn,O_RDONLY,0);
    if(!af) err(1,"af_open");
    
    for(i=0;i<MAX_FMTS;i++){
	char should[256];		// what we should get
	sprintf(should,fmt,i);
	int got = af_read(af,(unsigned char *)buf,fmt_size);
	if(got != fmt_size){
	    fprintf(stderr,"Attempt to read %d bytes; got %d\n",fmt_size,got);
	    exit(1);
	}
	if(i%25==24) printf("\r%d .. %d okay",i-24,i);
    }
    af_close(af);
    printf("\n");
    printf("\nRandom write test passes.\n");
    printf("======================\n");
    return 0;
}

int random_read_test(int total_bytes,int data_page_size)
{
    printf("\n\n\nrandom read test. filesize=%d, page_size=%d\n",
	   total_bytes,data_page_size);

    /* Create a regular file and an AFF file */

    printf("Creating random_contents.img and random_contents.%s, "
	   "both with %d bytes of user data...\n",
	   opt_ext,total_bytes);

    int    fd = open("test_random_contents.img",
		     O_CREAT|O_RDWR|O_TRUNC|O_BINARY,0666);
    if(fd<0) err(1,"fopen");

    char fn[1024];
    AFFILE *af = af_open(filename(fn,sizeof(fn),"test_random_contents"),
			 O_CREAT|O_RDWR|O_TRUNC,0666);
    if(!af) err(1,"af_open");
    enable_writing(af);

    /* Just write it out as one big write */

    unsigned char *buf = (unsigned char *)malloc(total_bytes);
    unsigned char *buf2 = (unsigned char *)malloc(total_bytes);

    /* First half is random */
    RAND_pseudo_bytes(buf,total_bytes/2);

    /* Second half is a bit more predictable */
    for(int i=total_bytes/2;i<total_bytes;i++){
	buf[i] = ((i % 256) + (i / 256)) % 256;
    }

    if(write(fd,buf,total_bytes)!=total_bytes) err(1,"fwrite");
    if(af_write(af,buf,total_bytes)!=(int)total_bytes) err(1,"af_write");

    /* Now try lots of seeks and reads */
    for(int i=0;i<MAX_FMTS;i++){
	unsigned int loc = rand() % total_bytes;
	unsigned int len = rand() % total_bytes;
	memset(buf,0,total_bytes);
	memset(buf2,0,total_bytes);

	printf("\r#%d  reading %u bytes at %u    ...",i,loc,len);
	fflush(stdout);

	unsigned long l1 = (unsigned long)lseek(fd,loc,SEEK_SET);
	unsigned long l2 = (unsigned long)af_seek(af,loc,SEEK_SET);


	if(l1!=l2){
	    err(1,"l1 (%lu) != l2 (%lu)",l1,l2);
	}

	int r1 = read(fd,buf,len);
	int r2 = af_read(af,buf2,len);

	if(r1!=r2){
	    err(1,"r1 (%d) != r2 (%d)",r1,r2);
	}
    }
    af_close(af);
    close(fd);
    printf("\nRandom read test passes\n");
    return 0;
}

void large_file_test()
{
    int pagesize = 1024*1024;		// megabyte sized segments
    int64 num_segments = 5000;
    int64 i;
    char fn[1024];

    printf("Large file test... Creating a %"I64d"MB file...\n",pagesize*num_segments/(1024*1024));
    filename(fn,sizeof(fn),"large_file");
    AFFILE *af = af_open(fn,O_CREAT|O_RDWR|O_TRUNC,0666);

    unsigned char *buf = (unsigned char *)malloc(pagesize);

    memset(buf,'E', pagesize);
    af_enable_writing(af,1);
    af_enable_compression(af,opt_compression_type,opt_compression_level);
    af_set_pagesize(af,pagesize);
    af_set_maxsize(af,(int64)pagesize * 600);

    for(i=0;i<num_segments;i++){
	sprintf((char *)buf,"%"I64d" page is put here",i);
	printf("\rWriting page %"I64d"\r",i);
	if(af_write(af,buf,pagesize)!=pagesize){
	    err(1,"Can't write page %"I64d,i);
	}
    }
    printf("\n\n");
    /* Now let's just read some test locations */
    for(i=0;i<num_segments;i+=num_segments/25){	// check a few places
	int r;
	af_seek(af,pagesize*i,SEEK_SET);
	r = af_read(af,buf,1024);		// just read a bit
	if(r!=1024){
	    err(1,"Tried to read 1024 bytes; got %d\n",r);
	}
	if(atoi((char *)buf)!=i){
	    err(1,"at page %"I64d", expected %"I64d", got %s\n",i,i,buf);
	}
	printf("Page %"I64d" validates\n",i);
    }

    af_close(af);
    if(unlink("large_file.aff")){
	err(1,"Can't delete large_file.aff");
    }
    printf("Large file test passes\n");
}

void maxsize_test()
{
    printf("Maxsize test. This test is designed to test creation of files\n");
    printf("Larger than 4GB. Currently it's disabled, though.\n");
#if 0
    char segname[16];
    char buf[1024];
    char fn[1024];
    int numpages = 1000;

    AFFILE *af = af_open(filename(fn,sizeof(fn),"maxsize"),O_CREAT|O_RDWR|O_TRUNC,0666);
    af_enable_writing(af,1);		// just take the segment defaults
    memset(buf,0,sizeof(buf));
    for(int64 i=0;i<numpages;i++){
	sprintf(buf,"This is page %"I64d". ****************************************************\n",i);
	sprintf(segname,AF_PAGE,i);
	af_update_seg(af,segname,0,buf,sizeof(buf));
    }
    af_close(af);
    printf("\nMaxsize test passes.\n");
#endif
    printf("\n====================\n");
}

void sparse_test()
{
    printf("Sparse test...\n");

    char buf[1024];
    char fn[1024];

    uint64 loc = (uint64)3 * (uint64)1000000000;		// 3GB in

    AFFILE *af = af_open(filename(fn,sizeof(fn),"sparse"),O_CREAT|O_RDWR|O_TRUNC,0666);
    af_enable_writing(af,1);
    af_enable_compression(af,opt_compression_type,opt_compression_level);
    af_set_maxsize(af,(int64)1024*1024*256);
    af_set_pagesize(af,1024*1024*16);

    for(uint i=0;i<10;i++){
	uint64 pos = loc*i;
	memset(buf,0,sizeof(buf));
	snprintf(buf,sizeof(buf),"This is at location=%"I64u"\n",pos);
	af_seek(af,pos,SEEK_SET);
	af_write(af,(unsigned char *)buf,sizeof(buf));
    }

    /* Now verify */
    for(uint i=0;i<10;i++){
	uint64 pos = loc*i;
	uint64 q;
	af_seek(af,pos,SEEK_SET);
	af_read(af,(unsigned char *)buf,sizeof(buf));
	char *cc = strchr(buf,'=');
	if(!cc){
	    printf("Garbage read at location %"I64u"\n.",pos);
	    exit(1);
	}
	fputs(buf,stdout);
	if(sscanf(cc+1,"%"I64u,&q)!=1){
	    printf("Could not decode value at location %"I64u"(%s)\n",pos,cc+1);
	    exit(1);
	}
	if(pos!=q){
	    printf("Wrong value at location %"I64u"; read %"I64u" in error.\n", loc,q);
	    exit(1);
	}
    }

    af_close(af);
    printf("\nSprase test passes.\n");
    printf("=====================\n\n");
}

void usage()
{
    printf("usage: %s [options]\n",progname);
    printf("    -e ext = use ext for extension (default is %s)\n",opt_ext);
    printf("    -a = do all tests (except -L)\n");
    printf("    -1 = do sequential test\n");
    printf("    -2 = do reverse test\n");
    printf("    -3 = do random write test\n");
    printf("    -4 = do random read test\n");
    printf("    -5 = do maxsize multi-file test\n");
    printf("    -6 = sparse file test\n");
    printf("    -B = run large file test (needs 5GB of disk)\n");
    printf("    -L = use LZMA compression\n");
    printf("    -r# = Repeat the random tests # times.\n");
    printf("    -d<dir> = use <dir> as the working dir for files\n");
    printf("    -f<dev> = run af_figure_media on dev and print the results\n");
    printf("    -c filename = compress filename and output to stdout\n");
    printf("    -T = just test the LZMA compression\n");
}



void figure(const char *fn)
{
    struct af_figure_media_buf afb;

    int fd = open(fn,O_RDONLY);
    if(fd<0) err(1,"%s",fn);
    if(af_figure_media(fd,&afb)){
	err(1,"af_figure_media");
    }
    printf("sector size: %d\n",afb.sector_size);
    printf("total sectors: %qd\n",afb.total_sectors);
    printf("max read blocks: %d\n",afb.max_read_blocks);
    exit(0);
}


void compress(const char *fname)
{
    int fd = open(fname,O_RDONLY,0666);
    if(fd<0) err(1,"%s",fname);
    
    struct stat st;
    if(fstat(fd,&st)) err(1,"stat");

    /* Allocate memory */
    char *buf = (char *)malloc(st.st_size);
    if(buf==0) errx(1,"malloc");

    if(read(fd,buf,st.st_size)!=st.st_size) err(1,"read");
    //size_t outSize = (int)((double)st.st_size * 1.05);
    //char *outBuffer = (char *)malloc(outSize);
    //size_t outSizeProcessed = 0;
}

void lzma_test()
{
#ifdef __FreeBSD__
    _malloc_options = "XARV";
#endif
    //char *fn = "/usr/share/dict/web2";
    char *fn = "/etc/motd";

    printf("starting up\n");
    FILE *f = fopen(fn,"r");
    if(!f) err(1,"%s",fn);
    
    struct stat st;
    if(fstat(fileno(f),&st)) err(1,"stat");

    /* Allocate memory */
    size_t buflen = st.st_size;
    printf("size=%qd\n",(long long)buflen);
    unsigned char *buf = (unsigned char *)malloc(buflen);
    if(buf==0) errx(1,"malloc");
    if(fread(buf,1,st.st_size,f)!=(unsigned long long)st.st_size) err(1,"read");

    /* Allocate memory for the compressed buffer */
    size_t cbufsize = (int)(buflen*1.05);
    size_t cbuf_actual=0;
    unsigned char *cbuf = (unsigned char *)malloc(cbufsize);


#ifdef USE_LZMA
    lzma_compress(cbuf,&cbufsize,buf,st.st_size,9);
#endif
    printf("cbuf_actual=%d\n",(int)cbuf_actual);

    /* Now try to decompress */
    size_t outbuf_size = buflen*2;
    unsigned char *outbuf = (unsigned char *)malloc(outbuf_size);

#ifdef USE_LZMA
    lzma_uncompress(outbuf,&outbuf_size,cbuf,cbufsize);
#endif
    printf("cbuf[0]=%d\n",cbuf[0]);

    if(memcmp(buf,outbuf,outbuf_size)==0){
	printf("Decompression works!\n");
    }
}


int main(int argc,char **argv)
{
    progname = argv[0];
    int do_sequential = 0;
    int do_reverse    = 0;
    int do_random_write_test = 0;
    int do_random_read_test  = 0;
    int do_large_file = 0;
    int do_maxsize_test = 0;
    int random_repeat = 1;
    int do_sparse_test = 0;
    int do_all=0;
    int ch;
    const char *dir = getenv(AFFLIB_BIGTMP);	// use by default

    setvbuf(stdout,0,_IONBF,0);
    putenv(AFFLIB_CACHE_STATS"=1");

    while ((ch = getopt(argc, argv, "123456aBr:Ld:h?f:e:c:T")) != -1) {
	switch(ch){
	case '1':
	    do_sequential = 1;
	    break;
	case '2':
	    do_reverse = 1;
	    break;
	case '3':
	    do_random_write_test = 1;
	    break;
	case '4':
	    do_random_read_test = 1;
	    break;
	case '5':
	    do_maxsize_test = 1;
	    break;
	case '6':
	    do_sparse_test = 1;
	    break;
	case 'l':
	    random_repeat = atoi(optarg);
	    break;
	case 'B':
	    do_large_file = 1;
	    break;
	case 'L':
	    opt_compression_type = AF_COMPRESSION_ALG_LZMA;
	    break;
	case 'T':
	    lzma_test();
	    break;

	case 'a':
	    do_all = 1;
	    break;
	case 'd':
	    dir = optarg;
	    break;
	case 'f':
	    figure(optarg);
	    break;
	case 'e':
	    opt_ext = optarg;
	    break;
	case 'c':
	    compress(optarg);
	    break;
	case 'h':
	case '?':
	default:
	    usage();
	}
    }
	
    if(dir){
	fprintf(stderr,"Changing to %s\n",dir);
	if(chdir(dir)) err(1,"Can't change directory to %s",optarg);
    }

    if(do_sequential || do_all) sequential_test();
    if(do_reverse || do_all ) reverse_test();
    if(do_maxsize_test || do_all) maxsize_test();
    if(do_sparse_test || do_all) sparse_test();

    for(int i=0;i<random_repeat;i++){
	if(do_random_read_test  || do_all) random_read_test(256*1024,rand() % 65536); 
	if(do_random_write_test || do_all) random_write_test();
    }

    if(do_large_file) large_file_test();

    return 0;
}