marsds.h

GPU实现的MapReduce framework,对于学习并行编程和cuda平台的编程方面有着极好的参考价值

/**
 *This is the source code for Mars, a MapReduce framework on graphics
 *processors.
 *Author: Wenbin Fang (HKUST), Bingsheng He (HKUST)
 *Mentor: Naga K. Govindaraju (Microsoft Corp.), Qiong Luo (HKUST), Tuyong
 *Wang (Sina.com).
 *If you have any question on the code, please contact us at {saven,
 *wenbin, luo}@cse.ust.hk.
 *The copyright is held by HKUST. Mars is provided "as is" without any 
 *guarantees of any kind.
 */

//Data structures
#ifndef __MARSDS_H__
#define __MARSDS_H__

//-----------------------------------------------
//used in MarsSort, for gpu bitonic sort
//defined by saven
//-----------------------------------------------
typedef int4 cmp_type_t;

//-----------------------------------------
//describe big chunk and small chunk
//-----------------------------------------
typedef struct
{
	//buffers
	char	*keys;
	char	*vals;
	int4	*index;
	int2	*keyListRange;

	//size info
	size_t	keySize;
	size_t	valSize;
	size_t	indexSize;
	size_t	rangeSize;

	//count
	size_t	recCount;
	size_t	diffKeyCount;

	//position info
	size_t	fileCursor;
	size_t	keyOffset;
	size_t	valOffset;
} ChunkInfo_t;

//--------------------------------------------------
//for external merge sort
//1, maintain the big chunk info
//2, maintain the small chunk info
//--------------------------------------------------
typedef struct
{
	char *keys;
	size_t keyOffset;
	size_t keySize;

	char *vals;
	size_t valOffset;
	size_t valSize;

	int4 *index;
	size_t recCount;
	size_t indexOffset;
	size_t indexSize;

	int2 *keyListRange;
	size_t diffKeyCount;
	size_t rangeOffset;
	size_t rangeSize;

	size_t smallCursor;
	size_t cursor;
} SortChunk_t;

typedef struct
{
	char *key;
	char *val;
	size_t keySize;
	size_t valCount;
	size_t allKeySize;
	size_t allValSize;
	int4 *index;
	int2 keyListRange;
} SortRec_t;

typedef struct
{
	SortChunk_t *chunks;
	size_t fullChunkCount;
	size_t realChunkCount;
} SortInfo_t;

//----------------------------------------
//for scheduler
//----------------------------------------
typedef struct
{
	ChunkInfo_t	inputSmallChunk;
	ChunkInfo_t	outputSmallChunk;

	size_t cpuMapThreadNum;
	size_t cpuReduceThreadNum;

	size_t gpuMapGridDim;
	size_t gpuMapBlockDim;
	size_t gpuReduceGridDim;
	size_t gpuReduceBlockDim;

	size_t gpuMapSharedMemSize;
	size_t gpuReduceSharedMemSize;
}Schedule_t;

//----------------------------------------
//for filename
//----------------------------------------
typedef struct
{
	char	*keyFile;
	char	*valFile;
	char	*indexFile;
	char	*rangeFile;
} FileName_t;

//----------------------------------------
//runtime configuration
//Note:
//Used in the only one global varible g_spec.
//----------------------------------------
typedef struct
{
	//------------------------------------
	//Usage: if (mode & state) ... else ...
	//state applicable:
	//USE_FILE /USE_MEM
	//GPU / CPU
	//MAP_ONLY / MAP_SORT / MAP_SORT_REDUCE
	//------------------------------------
	char mode;

	//used in gpu mode
	char cpuSort;

	//---------------------------------------
	//if (mode & USE_FILE) then use file
	//---------------------------------------
	FileName_t inputFile;
	FileName_t interFile;
	FileName_t outputFile;
	FileName_t tmpFile;
	FileName_t gpuInterFile;
	FileName_t cpuInterFile;
	FileName_t gpuOutputFile;
	FileName_t cpuOutputFile;
	size_t flushThreshhold;

	//----------------------------------------
	//thread number
	//----------------------------------------
	size_t cpuMapThreadNum;
	size_t cpuReduceThreadNum;

	size_t gpuMapGridDim;
	size_t gpuMapBlockDim;
	size_t gpuReduceGridDim;
	size_t gpuReduceBlockDim;

	//-----------------------------------------
	//gpu shared memory size
	//-----------------------------------------
	size_t gpuMapSharedMemSize;
	size_t gpuReduceSharedMemSize;

	//----------------------------------------
	//ratio of gpu input
	//(0.0, 1.0)
	//----------------------------------------
	float gpuInputRatio;

	//----------------------------------------
	//buffer
	//----------------------------------------
	ChunkInfo_t *inputChunk;
	ChunkInfo_t *interChunk;
	ChunkInfo_t *outputChunk;

	SortInfo_t *sortInfo;

	//----------------------------------------
	//scheduler
	//----------------------------------------
	Schedule_t *gpuSched;
	Schedule_t *cpuSched;

	//----------------------------------------
	//for iterators
	//----------------------------------------
	size_t totalInputRecCount;
	size_t totalInterRecCount;
	size_t totalDiffKeyCount;
	size_t totalOutputRecCount;

} Spec_t;

//record iterator
typedef struct
{
	ChunkInfo_t *chunk;
	size_t cursor;
	
	size_t fileCursor; //used when read from files
	size_t totalRecCount; //used when read from files
	size_t chunkSize; //used when read from files
	FileName_t *file;
} RecIterator_t;

//a record
typedef struct
{
	char *key;
	char *val;
	size_t keySize;
	size_t valSize;
	size_t keyOffset;
	size_t valOffset;
} Record_t;

//group iterator
typedef struct
{
	ChunkInfo_t *chunk;
	size_t cursor;
	
	size_t fileCursor; //used when read from files
	size_t totalGroup; //used when read from files
	size_t chunkSize; //used when read from files
	FileName_t *file;
} GroupIterator_t;

//a group
typedef struct
{
	char *key;
	char *vals;
	int4 *index;
	size_t keySize;
	size_t valCount;
	size_t startIndex;
	size_t endIndex;
	size_t cursor;
} Group_t;

//used in MapCount and ReduceCount
typedef struct
{
	char*	inKeys;
	char*	inVals;
	int4*	inIndex;
	int2*	inKeyListRange;
	size_t*	interKeySizePerThread;
	size_t*	interValSizePerThread;
	size_t*	interCountPerThread;
	size_t recCount; 
	size_t recPerThread;
	size_t threadNum;
	size_t keyOffset;
	size_t valOffset;
} CountArg_t;

//used in Map and Reduce
typedef struct
{
	char*	inKeys;
	char*	inVals;
	int4*	inIndex;
	int2*	inKeyListRange;
	size_t*	psKeySizes;
	size_t*	psValSizes;
	size_t*	psCounts;
	int2*	keyValOffsets;
	char*	outKeys;
	char*	outVals;
	int4*	outIndex;
	size_t*	curIndex;
	size_t	recCount; 
	size_t	recPerThread;
	size_t	threadNum;
} WorkerArg_t;

//for timer
extern double map_time;
extern double merge_inter_time;
extern double reduce_time;
extern double merge_output_time;
extern double group_time;
extern double io_time;

typedef struct  
{
	int x;
	int y;
	int z;
	int w;
	int v;
} int5;
#endif //__MARSDS_H__