tiotest.c

Linux Distributed Replication Block Device

/*
 *    Threaded io test
 *
 *  Copyright (C) 1999-2000 Mika Kuoppala <[e-mail]>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2, or (at your option)
 *  any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software Foundation,
 *  Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  
 *
 *
 */

#include "tiotest.h"
#include "crc32.h"

static const char* versionStr = "tiotest v0.3.3 (C) 1999-2000 Mika Kuoppala <[e-mail]>";

/* 
   This is global for easier usage. If you put changing data
   in here from threads, be sure to protect it with mutexes.
*/
ArgumentOptions args;

static void * aligned_alloc(ssize_t size)
{
	caddr_t a;
	a = mmap((caddr_t )0, size, 
	         PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
	if (a == MAP_FAILED)
		return NULL;
	return a;
}

static int aligned_free(caddr_t a, ssize_t size)
{
	return munmap(a, size);
}

int main(int argc, char *argv[])
{
	ThreadTest test;
	int i;

	strcpy(args.path[0], DEFAULT_DIRECTORY );
	args.pathsCount = 1;
	args.fileSizeInMBytes = DEFAULT_FILESIZE;
	args.blockSize = DEFAULT_BLOCKSIZE;
	args.numThreads = DEFAULT_THREADS;
	args.numRandomOps = DEFAULT_RANDOM_OPS;
	args.debugLevel = DEFAULT_DEBUG_LEVEL;
	args.verbose = 0;
	args.terse = 0;
	args.consistencyCheckData = 0;
	args.syncWriting = 0;
	args.rawDrives = 0;
	args.showLatency = 1;
	args.threadOffset = DEFAULT_RAW_OFFSET;
	args.useThreadOffsetForFirstThread = 0;
	
	for(i = 0; i < TESTS_COUNT; i++)
		args.testsToRun[i] = 1;
	
#if (LARGEFILES && USE_MMAP)
	printf("warning: LARGEFILES with MMAP needs mmap64 support which is not working yet in tiotest!\n");
#endif

	parse_args( &args, argc, argv );
    
	initialize_test( &test );
	
	do_tests( &test );

	print_results( &test );

	cleanup_test( &test );

	return 0;
}

inline void checkIntZero(int value, char *mess)
{
	if (value <= 0) 
	{
		printf(mess);
		printf("Try 'tiotest -h' for more information.\n");
		exit(1);
	}
}

inline void checkLong(long value, char *mess)
{
	if (value < 0) 
	{
		printf(mess);
		printf("Try 'tiotest -h' for more information\n");
		exit(1);
	}
}

void parse_args( ArgumentOptions* args, int argc, char *argv[] )
{
	int c;
	int once = 0;

	while (1)
	{
		c = getopt( argc, argv, "f:b:d:t:r:D:k:o:hLRTWSOc");

		if (c == -1)
			break;
	
		switch (c)
		{
		case 'f':
			args->fileSizeInMBytes = atoi(optarg);
			checkIntZero(args->fileSizeInMBytes, "Wrong file size\n");
			break;
	    
		case 'b':
			args->blockSize = atoi(optarg);
			checkIntZero(args->blockSize, "Wrong block size\n");
			break;
	    
		case 'd':
			if (args->pathsCount < MAX_PATHS) 
			{
				if (!once) 
				{
					args->pathsCount = 0;           
					once = 1;
				}
				strcpy(args->path[args->pathsCount++], optarg);
			}
			break;
	    
		case 't':
			args->numThreads = atoi(optarg);
			checkIntZero(args->numThreads, "Wrong number of threads\n");
			break;
	    
		case 'r':
			args->numRandomOps = atoi(optarg);
			checkIntZero(args->numRandomOps, "Wrong number of random I/O operations\n");
			break;
	    
	    	case 'L':
			args->showLatency = FALSE;
			break;
	    
		case 'T':
			args->terse = TRUE;
			break;

		case 'W':
			args->sequentialWriting = TRUE;
			break;
			
		case 'S':
			args->syncWriting = TRUE;
			break;
			
		case 'R':
			args->rawDrives = TRUE;
			break;

		case 'c':
			args->consistencyCheckData = TRUE;
			break;

		case 'h':
			print_help_and_exit();
			break;
		
		case 'D':
			args->debugLevel = atoi(optarg);
			break;
		
		case 'o':
			args->threadOffset = atol(optarg);
			checkLong(args->threadOffset, "Wrong offset between threads\n");
			break;
			
		case 'O':
			args->useThreadOffsetForFirstThread = TRUE;
			break;
			
		case 'k':
		{
			int i = atoi(optarg);
			if (i < TESTS_COUNT) 
			{
				args->testsToRun[i] = 0;
				break;
			}
			else
				printf("Wrong test number %d\n", i);
			/* Go through */
		}
		case '?':
		default:
			printf("Try 'tiotest -h' for more information\n");
			exit(1);
			break;
		}
	}
}

void initialize_test( ThreadTest *d )
{
	int i;
	int pathLoadBalIdx = 0;
	toff_t offs, cur_offs[KBYTE] = {0};

	memset( d, 0, sizeof(ThreadTest) );
    
	d->numThreads = args.numThreads; 

	for(i = 0; i < d->numThreads; i++)
	{
		d->threads = calloc( d->numThreads, sizeof(ThreadData) );
		if( d->threads == NULL )
		{
			perror("Error allocating memory");
			exit(-1);
		}
	}

	/* Initializing thread data */
	if (args.rawDrives) 
	{
		if (args.threadOffset != 0) 
		{
			offs = (args.threadOffset + args.fileSizeInMBytes) * MBYTE;
			if (args.useThreadOffsetForFirstThread) 
			{
				int k;
				for(k = 0; k < KBYTE; k++)
					cur_offs[k] = args.threadOffset * MBYTE;
			}
		}
		else
			offs = args.fileSizeInMBytes * MBYTE;
	}
	else
		offs = 0;
	for(i = 0; i < d->numThreads; i++)
	{
		d->threads[i].myNumber = i;
		d->threads[i].blockSize = args.blockSize;
		d->threads[i].numRandomOps = args.numRandomOps;
		d->threads[i].fileSizeInMBytes = args.fileSizeInMBytes;
		if (args.rawDrives)
		{
			d->threads[i].fileOffset = cur_offs[pathLoadBalIdx];
			cur_offs[pathLoadBalIdx] += offs;
			sprintf(d->threads[i].fileName, "%s",
				args.path[pathLoadBalIdx++]);
		}
		else
		{
			d->threads[i].fileOffset = 0;
			sprintf(d->threads[i].fileName, "%s/_%d_tiotest.%d",
				args.path[pathLoadBalIdx++], getpid(), i);
		}
		
		if( pathLoadBalIdx >= args.pathsCount )
			pathLoadBalIdx = 0;

		pthread_attr_init( &(d->threads[i].thread_attr) );

		pthread_attr_setscope(&(d->threads[i].thread_attr),
				      PTHREAD_SCOPE_SYSTEM);

		d->threads[i].buffer = aligned_alloc( d->threads[i].blockSize );
		if( d->threads[i].buffer == NULL )
		{
			perror("Error allocating memory");
			exit(-1);
		}

		if( args.consistencyCheckData )
		{
			int j;
			const unsigned long bsize = d->threads[i].blockSize;
			unsigned char *b = d->threads[i].buffer;

			for(j = 0; j < bsize; j++)
				b[j] = rand() & 0xFF;

			d->threads[i].bufferCrc = crc32(b, bsize, 0);
		}
	}
}

void print_option(const char* s, 
		  const char* desc, 
		  const char* def)
{
	printf("  %s          %s", s, desc);
    
	if(def)
		printf(" (default: %s)", def);

	printf("\n");
   
}

char *my_int_to_string(int a)
{
	static char tempBuffer[128];

	sprintf(tempBuffer, "%d", a);

	return tempBuffer;
}

void print_help_and_exit()
{
	printf("%s\n", versionStr);

	printf("Usage: tiotest [options]\n");

	print_option("-f", "Filesize per thread in MBytes",
		     my_int_to_string(DEFAULT_FILESIZE));

	print_option("-b", "Blocksize to use in bytes",
		     my_int_to_string(DEFAULT_BLOCKSIZE));

	print_option("-d", "Directory for test files", 
		     DEFAULT_DIRECTORY);

	print_option("-t", "Number of concurrent test threads",
		     my_int_to_string(DEFAULT_THREADS));

	print_option("-r", "Random I/O operations per thread", 
		     my_int_to_string(DEFAULT_RANDOM_OPS));
		     
	print_option("-o", "Offset in Mb on disk between threads. Use with -R option",
		     0);
	
	print_option("-k", "Skip test number n. Could be used several times.", 0);	  
	
	print_option("-L", "Hide latency output", 0);	  
	
	print_option("-R", "Use raw devices. Set device name with -d option", 0);

	print_option("-T", "More terse output", 0);

	print_option("-W", "Do writing phase sequentially", 0);
	
	print_option("-S", "Do writing synchronously", 0);
	
	print_option("-O", "Use offset from -o option for first thread. Use with -R option",
		     0);

	print_option("-c", 
		     "Consistency check data (will slow io and raise cpu%)",
		     0);
	
	print_option("-D", "Debug level",
		     my_int_to_string(DEFAULT_DEBUG_LEVEL));

	print_option("-h", "Print this help and exit", 0);

	exit(1);
}

void cleanup_test( ThreadTest *d )
{
	int i;

	for(i = 0; i < d->numThreads; i++)
	{
		if (!args.rawDrives)
		unlink(d->threads[i].fileName);
		aligned_free( d->threads[i].buffer, d->threads[i].blockSize );
		d->threads[i].buffer = 0;
	
		pthread_attr_destroy( &(d->threads[i].thread_attr) );
	}

	free(d->threads);
    
	d->threads = 0;
}

void wait_for_threads( ThreadTest *d )
{
	int i;

	for(i = 0; i < d->numThreads; i++)
		pthread_join(d->threads[i].thread, NULL);	
}

void do_tests( ThreadTest *thisTest )
{
	Timings *timeWrite       = &(thisTest->totalTimeWrite);
	Timings *timeRandomWrite = &(thisTest->totalTimeRandomWrite);
	Timings *timeRead        = &(thisTest->totalTimeRead);
	Timings *timeRandomRead  = &(thisTest->totalTimeRandomRead);

	timer_init( timeWrite );
	timer_init( timeRandomWrite );
	timer_init( timeRead );
	timer_init( timeRandomRead );

	/*
	  Write testing 
	*/
    	if (args.testsToRun[WRITE_TEST])
		do_test( thisTest, WRITE_TEST, args.sequentialWriting,
			timeWrite,  "Waiting write threads to finish...");

	/*
	  RandomWrite testing 
	*/
    	if (args.testsToRun[RANDOM_WRITE_TEST])
		do_test( thisTest, RANDOM_WRITE_TEST, FALSE, timeRandomWrite,
			"Waiting random write threads to finish...");

	/*
	  Read testing 
	*/
    	if (args.testsToRun[READ_TEST])
		do_test( thisTest, READ_TEST, FALSE, timeRead,
			"Waiting read threads to finish..." );

	/*
	  RandomRead testing 
	*/
	if (args.testsToRun[RANDOM_READ_TEST])
		do_test( thisTest, RANDOM_READ_TEST, FALSE, timeRandomRead,
			"Waiting random read threads to finish...");
}

typedef struct 
{
	volatile int *child_status;
	TestFunc fn;
	ThreadData *d;
	volatile int *pstart;
} StartData;

void* start_proc( void *data )
{
	StartData *sd = (StartData*)data;
	*sd->child_status = getpid();
	if (sd->pstart != NULL)
		while (*sd->pstart == 0) sleep(0);
	return sd->fn(sd->d);
}

void do_test( ThreadTest *test, int testCase, int sequential,
	Timings *t, char *debugMessage )
{
	int i;
	volatile int *child_status;
	StartData *sd;
	int synccount;
	volatile int start = 0;
	
	child_status = (volatile int *)calloc(test->numThreads, sizeof(int));
	if (child_status == NULL) 
	{
		perror("Error allocating memory");
		return;
	}
	
	sd = (StartData*)calloc(test->numThreads, sizeof(StartData));
	if (sd == NULL) 
	{
		perror("Error allocating memory");
		free((int*)child_status);
		return;
	}