tiotest.c

Linux Distributed Replication Block Device

	
	if (sequential)
		timer_start(t);
	
	for(i = 0; i < test->numThreads; i++)
	{
		sd[i].child_status = &child_status[i];
		sd[i].fn = Tests[testCase];
		sd[i].d = &test->threads[i];
		if (sequential)
			sd[i].pstart = NULL;
		else
			sd[i].pstart = &start;
		if( pthread_create(
			&(test->threads[i].thread), 
			&(test->threads[i].thread_attr), 
			start_proc, 
			(void *)&sd[i]))
		{
			perror("Error creating threads");
			free((int*)child_status);
			free(sd);
			exit(-1);
		}

		if(sequential)
		{
			if(args.debugLevel > 2)
				fprintf(stderr, 
					"Waiting previous thread "
					"to finish before starting "
					"a new one\n" );
	    
			pthread_join(test->threads[i].thread, NULL);
		}
	}
	
	if(sequential) 
		timer_stop(t);
	else 
	{
		struct timeval tv1, tv2;
		gettimeofday(&tv1, NULL);
		do 
		{
			synccount = 0;
			for(i = 0; i < test->numThreads; i++) 
				if (child_status[i]) 
					synccount++;
			if (synccount == test->numThreads) 
				break;
			sleep(1);
			gettimeofday(&tv2, NULL);
		} while ((tv2.tv_sec - tv1.tv_sec) < 30);

		if (synccount != test->numThreads) 
		{
			printf("Unable to start %d threads (started %d)\n", 
				test->numThreads, synccount);
			start = 1;
			wait_for_threads(test);
			free((int*)child_status);
			free(sd);
			return;
		}

		if(args.debugLevel > 4)
		{
			printf("Created %d threads\n", i);
			fprintf(stderr, debugMessage);
			fflush(stderr);
		}
	
		timer_start(t);

		start = 1;
    
		wait_for_threads(test);
    
		timer_stop(t);
	}
	free((int*)child_status);
	free(sd);
    
	if(args.debugLevel > 4)
	{
		fprintf(stderr, "Done!\n");
		fflush(stderr);
	}
}

void print_results( ThreadTest *d )
{
/*
  This is messy and should be rewritten but some of unixes, didn't
  understand all printf options and long long formats.
*/
	int i;    
	double totalBlocksWrite = 0, totalBlocksRead = 0, 
	    totalBlocksRandomWrite = 0, totalBlocksRandomRead = 0;

	double read_rate,write_rate,random_read_rate,random_write_rate;
	double realtime_write,usrtime_write = 0, systime_write = 0;
	double realtime_rwrite = 0, usrtime_rwrite = 0, systime_rwrite = 0;
	double realtime_read, usrtime_read = 0, systime_read = 0;
	double realtime_rread = 0, usrtime_rread= 0, systime_rread = 0;

	double mbytesWrite, mbytesRandomWrite, mbytesRead, mbytesRandomRead;
	
	double avgWriteLat=0, avgRWriteLat=0, avgReadLat=0, avgRReadLat=0;
	double maxWriteLat=0, maxRWriteLat=0, maxReadLat=0, maxRReadLat=0;
	double countWriteLat=0, countRWriteLat=0, countReadLat=0, countRReadLat=0;
	double count1WriteLat=0, count1RWriteLat=0, count1ReadLat=0, 
		count1RReadLat=0;
	double count2WriteLat=0, count2RWriteLat=0, count2ReadLat=0, 
		count2RReadLat=0;
	double perc1WriteLat=0, perc1RWriteLat=0, perc1ReadLat=0, 
		perc1RReadLat=0;
	double perc2WriteLat=0, perc2RWriteLat=0, perc2ReadLat=0, 
		perc2RReadLat=0;
	double avgLat=0, maxLat=0, countLat=0, count1Lat=0, count2Lat=0,
		perc1Lat=0, perc2Lat=0;

	for(i = 0; i < d->numThreads; i++)
	{
		usrtime_write += 
		    timer_usertime( &(d->threads[i].writeTimings) );
		systime_write += 
		    timer_systime( &(d->threads[i].writeTimings) );

		usrtime_rwrite += 
		    timer_usertime( &(d->threads[i].randomWriteTimings) );
		systime_rwrite += 
		    timer_systime( &(d->threads[i].randomWriteTimings) );

		usrtime_read += timer_usertime( &(d->threads[i].readTimings) );
		systime_read += timer_systime( &(d->threads[i].readTimings) );

		usrtime_rread += 
		    timer_usertime( &(d->threads[i].randomReadTimings) );
		systime_rread += 
		    timer_systime( &(d->threads[i].randomReadTimings) );

		totalBlocksWrite       += d->threads[i].blocksWritten;
		totalBlocksRandomWrite += d->threads[i].blocksRandomWritten;
		totalBlocksRead        += d->threads[i].blocksRead; 
		totalBlocksRandomRead  += d->threads[i].blocksRandomRead;
		
		avgWriteLat += d->threads[i].writeLatency.avg;
		avgRWriteLat += d->threads[i].randomWriteLatency.avg;
		avgReadLat += d->threads[i].readLatency.avg;
		avgRReadLat += d->threads[i].randomReadLatency.avg;
		
		avgLat += d->threads[i].writeLatency.avg;
		avgLat += d->threads[i].randomWriteLatency.avg;
		avgLat += d->threads[i].readLatency.avg;
		avgLat += d->threads[i].randomReadLatency.avg;
		
		countWriteLat += d->threads[i].writeLatency.count;
		countRWriteLat += d->threads[i].randomWriteLatency.count;
		countReadLat += d->threads[i].readLatency.count;
		countRReadLat += d->threads[i].randomReadLatency.count;
		
		count1WriteLat += d->threads[i].writeLatency.count1;
		count1RWriteLat += d->threads[i].randomWriteLatency.count1;
		count1ReadLat += d->threads[i].readLatency.count1;
		count1RReadLat += d->threads[i].randomReadLatency.count1;
		
		count2WriteLat += d->threads[i].writeLatency.count2;
		count2RWriteLat += d->threads[i].randomWriteLatency.count2;
		count2ReadLat += d->threads[i].readLatency.count2;
		count2RReadLat += d->threads[i].randomReadLatency.count2;
		
		countLat += d->threads[i].writeLatency.count;
		countLat += d->threads[i].randomWriteLatency.count;
		countLat += d->threads[i].readLatency.count;
		countLat += d->threads[i].randomReadLatency.count;
		
		count1Lat += d->threads[i].writeLatency.count1;
		count1Lat += d->threads[i].randomWriteLatency.count1;
		count1Lat += d->threads[i].readLatency.count1;
		count1Lat += d->threads[i].randomReadLatency.count1;
		
		count2Lat += d->threads[i].writeLatency.count2;
		count2Lat += d->threads[i].randomWriteLatency.count2;
		count2Lat += d->threads[i].readLatency.count2;
		count2Lat += d->threads[i].randomReadLatency.count2;
		
		if (maxWriteLat < d->threads[i].writeLatency.max)
			maxWriteLat = d->threads[i].writeLatency.max;
		if (maxRWriteLat < d->threads[i].randomWriteLatency.max)
			maxRWriteLat = d->threads[i].randomWriteLatency.max;
		if (maxReadLat < d->threads[i].readLatency.max)
			maxReadLat = d->threads[i].readLatency.max;
		if (maxRReadLat < d->threads[i].randomReadLatency.max)
			maxRReadLat = d->threads[i].randomReadLatency.max;
			
		if (maxLat < maxWriteLat)
			maxLat = maxWriteLat;
		if (maxLat < maxRWriteLat)
			maxLat = maxRWriteLat;
		if (maxLat < maxReadLat)
			maxLat = maxReadLat;
		if (maxLat <maxRReadLat)
			maxLat = maxRReadLat;
	}

	if (countWriteLat > 0) 
	{	
		avgWriteLat /= countWriteLat;
		perc1WriteLat = count1WriteLat*100.0/countWriteLat;
		perc2WriteLat = count2WriteLat*100.0/countWriteLat;
	}
	else
		avgWriteLat = 0;

	if (countRWriteLat > 0)
	{
		avgRWriteLat /= countRWriteLat;
		perc1RWriteLat = count1RWriteLat*100.0/countRWriteLat;
		perc2RWriteLat = count2RWriteLat*100.0/countRWriteLat;
	}
	else
		avgRWriteLat = 0;

	if (countReadLat > 0)
	{
		avgReadLat /= countReadLat;
		perc1ReadLat = count1ReadLat*100.0/countReadLat;
		perc2ReadLat = count2ReadLat*100.0/countReadLat;
	}
	else
		avgReadLat = 0;

	if (countRReadLat > 0)
	{
		avgRReadLat /= countRReadLat;
		perc1RReadLat = count1RReadLat*100.0/countRReadLat;
		perc2RReadLat = count2RReadLat*100.0/countRReadLat;
	}
	else
		avgRReadLat = 0;

	if (countLat > 0)
	{
		avgLat /= countLat;
		perc1Lat = count1Lat*100.0/countLat;
		perc2Lat = count2Lat*100.0/countLat;
	}
	else
		avgLat = 0;
		
	mbytesWrite = totalBlocksWrite / 
	    ((double)MBYTE/(double)(d->threads[0].blockSize));
	mbytesRandomWrite = totalBlocksRandomWrite /
	    ((double)MBYTE/(double)(d->threads[0].blockSize));

	mbytesRead = totalBlocksRead / 
	    ((double)MBYTE/(double)(d->threads[0].blockSize));
	mbytesRandomRead = totalBlocksRandomRead / 
	    ((double)MBYTE/(double)(d->threads[0].blockSize));

	realtime_write  = timer_realtime( &(d->totalTimeWrite) );
	realtime_rwrite = timer_realtime( &(d->totalTimeRandomWrite) );
	realtime_read   = timer_realtime( &(d->totalTimeRead) );
	realtime_rread  = timer_realtime( &(d->totalTimeRandomRead) );

	if(args.terse)
	{
		printf("write:%.5f,%.5f,%.5f,%.5f,%.5f,%.5f,%.5f,%.5f\n",
		       mbytesWrite, 
		       realtime_write, usrtime_write, systime_write,
		       avgWriteLat*1000, maxWriteLat*1000,
		       perc1WriteLat, perc2WriteLat );

		printf("rwrite:%.5f,%.5f,%.5f,%.5f,%.5f,%.5f,%.5f,%.5f\n",
		       mbytesRandomWrite, 
		       realtime_rwrite, usrtime_rwrite, systime_rwrite,
		       avgRWriteLat*1000, maxRWriteLat*1000,
		       perc1RWriteLat, perc2RWriteLat );

		printf("read:%.5f,%.5f,%.5f,%.5f,%.5f,%.5f,%.5f,%.5f\n",
		       mbytesRead, 
		       realtime_read, usrtime_read, systime_read,
		       avgReadLat*1000, maxReadLat*1000,
		       perc1ReadLat, perc2ReadLat );

		printf("rread:%.5f,%.5f,%.5f,%.5f,%.5f,%.5f,%.5f,%.5f\n",
		       mbytesRandomRead, 
		       realtime_rread, usrtime_rread, systime_rread,
		       avgRReadLat*1000, maxRReadLat*1000,
		       perc1RReadLat, perc2RReadLat );

		printf("total:%.5f,%.5f,%.5f,%.5f\n", 
			avgLat*1000, maxLat*1000, perc1Lat, perc2Lat );

		return;
	}

	write_rate = mbytesWrite / realtime_write;
	random_write_rate = mbytesRandomWrite / realtime_rwrite;
 
	read_rate  = mbytesRead / realtime_read;
	random_read_rate  = mbytesRandomRead / realtime_rread;

	printf("Tiotest results for %d concurrent io threads:\n", 
	       d->numThreads);
	
	printf(",----------------------------------------------------------------------.\n");
	printf("| Item                  | Time     | Rate         | Usr CPU  | Sys CPU |\n");
	printf("+-----------------------+----------+--------------+----------+---------+\n");
    
	if(totalBlocksWrite)
		printf("| Write %11.0f MBs | %6.1f s | %7.3f MB/s | %5.1f %%  | %5.1f %% |\n",
		       mbytesWrite,
		       realtime_write,write_rate,
		       usrtime_write*100.0/realtime_write,
		       systime_write*100.0/realtime_write );

	if(totalBlocksRandomWrite)
		printf("| Random Write %4.0f MBs | %6.1f s | %7.3f MB/s | %5.1f %%  | %5.1f %% |\n",
		       mbytesRandomWrite,
		       realtime_rwrite,random_write_rate,
		       usrtime_rwrite*100.0/realtime_rwrite,
		       systime_rwrite*100.0/realtime_rwrite );

    
	if(totalBlocksRead)
		printf("| Read %12.0f MBs | %6.1f s | %7.3f MB/s | %5.1f %%  | %5.1f %% |\n",
		       mbytesRead,
		       realtime_read,read_rate,
		       usrtime_read*100.0/realtime_read,
		       systime_read*100.0/realtime_read );

    
	if(totalBlocksRandomRead)
		printf("| Random Read %5.0f MBs | %6.1f s | %7.3f MB/s | %5.1f %%  | %5.1f %% |\n",
		       mbytesRandomRead,
		       realtime_rread,random_read_rate,
		       usrtime_rread*100.0/realtime_rread,
		       systime_rread*100.0/realtime_rread );

	printf("`----------------------------------------------------------------------'\n");
	
	if (args.showLatency)
	{
		printf("Tiotest latency results:\n");
	
		printf(",-------------------------------------------------------------------------.\n");
		printf("| Item         | Average latency | Maximum latency | %% >%d sec | %% >%d sec |\n", 
			LATENCY_STAT1, LATENCY_STAT2);
		printf("+--------------+-----------------+-----------------+----------+-----------+\n");
    
		if(totalBlocksWrite)
			printf("| Write        | %12.3f ms | %12.3f ms | %8.5f | %9.5f |\n",
			       avgWriteLat*1000, maxWriteLat*1000, perc1WriteLat,
			       perc2WriteLat);

		if(totalBlocksRandomWrite)
			printf("| Random Write | %12.3f ms | %12.3f ms | %8.5f | %9.5f |\n",
			       avgRWriteLat*1000, maxRWriteLat*1000, perc1RWriteLat,
			       perc2RWriteLat);
    
		if(totalBlocksRead)
			printf("| Read         | %12.3f ms | %12.3f ms | %8.5f | %9.5f |\n",
			       avgReadLat*1000, maxReadLat*1000, perc1ReadLat,
			       perc2ReadLat);
    
		if(totalBlocksRandomRead)
			printf("| Random Read  | %12.3f ms | %12.3f ms | %8.5f | %9.5f |\n",
			       avgRReadLat*1000, maxRReadLat*1000, perc1RReadLat,
			       perc2RReadLat);

		printf("|--------------+-----------------+-----------------+----------+-----------|\n");

		printf("| Total        | %12.3f ms | %12.3f ms | %8.5f | %9.5f |\n",
		       avgLat*1000, maxLat*1000, perc1Lat, perc2Lat);

		printf("`--------------+-----------------+-----------------+----------+-----------'\n\n");
	}
}

void report_seek_error(toff_t offset, unsigned long wr)
{
	char buf[1024];
	sprintf(buf, 
#ifdef LARGEFILES			
		"Error in seek, offset= %Ld, seeks = %ld: ", 
#else				
		"Error in seek, offset = %ld, seeks = %ld:",
#endif				
		offset, wr );
	perror(buf);
}

void report_random_write_error(toff_t offset, ssize_t bytesWritten, unsigned long wr)
{
	char buf[1024];
	sprintf(buf, 
#ifdef LARGEFILES
		"Error in randomwrite, off=%Ld, read=%d, seeks=%ld : ", 
#else
		"Error in randomwrite, off=%ld, read=%d, seeks=%ld : ",
#endif
		offset, bytesWritten, wr );
		    
		perror(buf);
}

void report_read_error(toff_t offset, ssize_t bytesRead, unsigned long rd)
{
	char buf[1024];
	sprintf(buf, 
#ifdef LARGEFILES
		"Error in seek/read, off=%Ld, read=%d, seeks=%ld : ", 
#else
		"Error in seek/read, off=%ld, read=%d, seeks=%ld : ",
#endif
		offset, bytesRead, rd );
		    
	perror(buf);
}

void* do_write_test( ThreadData *d )
{
	int     fd;
	char    *buf = d->buffer;
	toff_t  blocks=(d->fileSizeInMBytes*MBYTE)/d->blockSize;
	toff_t  i;
	int     openFlags;
	
#ifdef USE_MMAP
	toff_t  bytesize=blocks*d->blockSize; /* truncates down to BS multiple */
	void *file_loc;
#endif

	if (args.rawDrives) 
		openFlags = O_RDWR;
	else
		openFlags = O_RDWR | O_CREAT | O_TRUNC;

	if( args.syncWriting )
		openFlags |= O_SYNC;

#ifdef LARGEFILES
	openFlags |= O_LARGEFILE;
#endif
    
	fd = open(d->fileName, openFlags, 0600 );
	if(fd == -1) 
	{
		fprintf(stderr, "%s: %s\n", strerror(errno), d->fileName);
		return 0;
	}

	if (args.debugLevel > 1)
	{
		fprintf(stderr, "do_write_test: initial seek %lu\n", d->fileOffset);
		fflush(stderr);
	}
	
#ifdef USE_MMAP
	if (!args.rawDrives) 
		ftruncate(fd,bytesize); /* pre-allocate space */
	file_loc=mmap(NULL,bytesize,PROT_READ|PROT_WRITE,MAP_SHARED,fd,
		d->fileOffset);
	if(file_loc == MAP_FAILED) 
	{
		perror("Error mmap()ing file");
		close(fd);
		return 0;
	}
#  ifdef USE_MADVISE
	/* madvise(file_loc,bytesize,MADV_DONTNEED); */
	madvise(file_loc,bytesize,MADV_RANDOM);
#  endif
#else
	if( tlseek( fd, d->fileOffset, SEEK_SET ) != d->fileOffset )
	{
		report_seek_error(d->fileOffset, d->blocksRandomWritten);
		close(fd);
		return 0;
	}
#endif