common.cpp

x-wrt is the GUI config tool for openwrt,which is a open project about wireless Router

		file=fopen((const char*)dictName,"rb");
		if (file==NULL)
		{
			printf("Can't open dictionary %s\n",dictName);
			return false;
		}

		fileLen=flen(file);
		fscanf(file,"%d",&dict123size);


		do { c=getc(file); } while (c>=32); if (c==13) c=getc(file); // skip CR+LF or LF

		for (i=0; i<CHARSET_COUNT; i++)
		{
			loadCharset(file);
			loadCharset(file);
		}

#ifdef DYNAMIC_DICTIONARY
		dict123size+=240000;
#else
		dict123size+=sortedDictSize;
#endif

		initializeCodeWords(dict123size);
		int dicsize=fileLen*2+dict123size*WORD_AVG_SIZE*2;
		dictmem=(unsigned char*)calloc(dicsize,1);
		dictmem_end=dictmem+dicsize-256;
		PRINT_DICT(("allocated memory=%d\n",dicsize));

		if (!dictmem)
			OUT_OF_MEMORY();

		sizeDict=1;
		mem=loadDynamicDictionary(dictmem,dictmem_end);
		mem=loadDictionary(file,mem,dictionary);

		fclose(file);
	}

	if (WRT_verbose)
		printf(" + loaded dictionary %d/%d words\n",sizeDict,dictionary);

	return true;
}


void XWRT_Common::WRT_deinitialize()
{
	if (dict)
	{
		free(dict);
		dict=NULL;
	}
	if (dictlen)
	{
		free(dictlen);
		dictlen=NULL;
	}
	if (dictmem)
	{
		free(dictmem);
		dictmem=NULL;
	}
	if (dictfreq)
	{
		free(dictfreq);
		dictfreq=NULL;
	}

	sizeDict=0;
}

void XWRT_Common::WRT_print_options()
{
	if (IF_OPTION(OPTION_UNICODE_LE) || IF_OPTION(OPTION_UNICODE_BE)) PRINT_DICT(("UNICODE "));
	if (IF_OPTION(OPTION_BINARY_DATA)) PRINT_DICT(("BINARY_DATA "));
	if (IF_OPTION(OPTION_USE_DICTIONARY)) PRINT_DICT(("USE_DICTIONARY "));
	if (IF_OPTION(OPTION_SPACELESS_WORDS)) PRINT_DICT(("SPACELESS "));
	if (IF_OPTION(OPTION_SPACES_MODELING)) PRINT_DICT(("SPACES_MODELING "));
	if (IF_OPTION(OPTION_TRY_SHORTER_WORD)) PRINT_DICT(("TRY_SHORTER "));
	if (IF_OPTION(OPTION_NUMBER_CONTAINER)) PRINT_DICT(("NUMBER_CONTAINER "));
	if (IF_OPTION(OPTION_ADD_SYMBOLS_14_31)) PRINT_DICT(("ADD_SYMBOLS_14_31 "));
	if (IF_OPTION(OPTION_ADD_SYMBOLS_0_5)) PRINT_DICT(("ADD_SYMBOLS_0_5 "));
	if (IF_OPTION(OPTION_SPACELESS_WORDS)) PRINT_DICT(("SPACELESS_WORDS "));
	if (IF_OPTION(OPTION_ZLIB)) PRINT_DICT(("ZLIB "));
	PRINT_DICT(("prepType=%d\n",preprocType));
}

void XWRT_Common::init_PPMVC(int fileSizeInMB,int mode)
{
#ifdef USE_PPMVC_LIBRARY

	if (IF_OPTION(OPTION_PPMVC)) // preprocType==PPM
	{
		PPMVClib_Deinit();

		if (additionalParam>=64) 
			PPMVClib_Init(additionalParam,4); // 8=nasa
		else
			PPMVClib_Init(additionalParam,2);


		if (fileSizeInMB>=23)
			PPMVClib_order=6;
		else
		if (fileSizeInMB>=7)
			PPMVClib_order=8;
		else
			PPMVClib_order=10;
	}
#endif

#ifdef USE_LZMA_LIBRARY
	if (IF_OPTION(OPTION_LZMA)) // preprocType==LZMA
		LZMAlib_Init(additionalParam);
#endif

#ifdef USE_PAQ_LIBRARY
	if (IF_OPTION(OPTION_PAQ)) // preprocType==PAQ
	{
		set_PAQ_level(additionalParam);
		if (mode==COMPRESS)
			PAQ_encoder=new Encoder(mode,XWRT_fileout);
		else
			PAQ_encoder=new Encoder(mode,XWRT_file);
	}

#endif
}

void XWRT_Common::getAlgName(std::string& compName)
{
	if (IF_OPTION(OPTION_PAQ))
	{
		switch (additionalParam)
		{
			case 5:
				compName="lpaq6 104 MB";
				break;
			case 6:
				compName="lpaq6 198 MB"; 
				break;
			case 7:
				compName="lpaq6 390 MB";
				break;
			case 8:
				compName="lpaq6 774 MB";
				break;
			case 9:
				compName="lpaq6 1542 MB";
				break;
		}

#ifndef USE_PAQ_LIBRARY
		printf("lpaq6 compression not supported (in this compilation of XWRT)!\n");
		exit(0);
#endif
	}
	else
	if (IF_OPTION(OPTION_PPMVC))
	{
		if (additionalParam==16)
			compName="PPMVC 16MB";
		else
		if (additionalParam==32)
			compName="PPMVC 32MB";
		else
			compName="PPMVC 64MB";

#ifndef USE_PPMVC_LIBRARY
		printf("PPMVC compression not supported (in this compilation of XWRT)!\n");
		exit(0);
#endif
	}
	else
	if (IF_OPTION(OPTION_LZMA))
	{
		if (additionalParam==0)	
			compName="LZMA 64KB";
		else
		if (additionalParam==1)	
			compName="LZMA 1MB";
		else
			compName="LZMA 8MB";

#ifndef USE_LZMA_LIBRARY
		printf("LZMA compression not supported (in this compilation of XWRT)!\n");
		exit(0);
#endif
	}
	else
	if (IF_OPTION(OPTION_ZLIB))
	{
		if (additionalParam==1)	
			compName="zlib fast";
		else
		if (additionalParam==6)	
			compName="zlib normal";
		else
			compName="zlib best";

#ifndef USE_ZLIB_LIBRARY
		printf("Zlib compression not supported (in this compilation of XWRT)!\n");
		exit(0);
#endif
	}
	else
		compName="store";
}

int XWRT_Common::defaultSettings(int argc, char* argv[])
{
	static bool firstTime=true;
	bool minWordFreqChanged=false;

	RESET_OPTIONS;
	codewordType=preprocType;

	TURN_ON(OPTION_TRY_SHORTER_WORD);
	TURN_ON(OPTION_SPACES_MODELING);
#ifdef USE_ZLIB_LIBRARY
	TURN_ON(OPTION_ZLIB);
#endif

	tryShorterBound=2;
	maxMemSize=8*1024*1024;
	maxDynDictBuf=8;
	additionalParam=6; // zlib normal
	maxDictSize=65535*32700;
	compName="zlib normal";

	switch (preprocType)
	{
		case LZ77: // for LZ77 there are different codeWords
			TURN_ON(OPTION_SPACELESS_WORDS);
			TURN_ON(OPTION_ADD_SYMBOLS_0_5);
			TURN_ON(OPTION_ADD_SYMBOLS_14_31);
			TURN_ON(OPTION_ADD_SYMBOLS_MISC);
			TURN_ON(OPTION_USE_CONTAINERS);
			minWordFreq=6;
			TURN_ON(OPTION_NUMBER_CONTAINER);
			TURN_ON(OPTION_LETTER_CONTAINER);
			break;
		case LZMA: // for LZMA there are different codeWords 
			TURN_ON(OPTION_USE_CONTAINERS);
			TURN_ON(OPTION_SPACELESS_WORDS);
			minWordFreq=6;
			TURN_ON(OPTION_NUMBER_CONTAINER);
			break;
		case PPM:
			TURN_ON(OPTION_SPACE_AFTER_CC_FLAG);
			TURN_ON(OPTION_ADD_SYMBOLS_MISC);
			TURN_ON(OPTION_ADD_SYMBOLS_0_5);
			TURN_ON(OPTION_ADD_SYMBOLS_14_31);
		case PAQ:
			TURN_ON(OPTION_NUMBER_CONTAINER);
			tryShorterBound=4;
			minWordFreq=64;
			break;
	}

	int optCount=0;
	int backArgc=argc;
	char** backArgv=argv;
	EPreprocessType newPreprocType=preprocType;

	while (argc>1 && (argv[1][0]=='-' || argv[1][0]=='+')) 
	{
		switch (argv[1][1])
		{
			case 'o':
				if (argv[1][0]=='-')
				{
					YesToAll=true;
					if (firstTime)
						printf("* Force overwrite of output files is turned on\n");
				}
				break;
			case 'i':
				if (argv[1][0]=='-')
				{
					deleteInputFiles=true;
					if (firstTime)
						printf("* Delete input files is turned on\n");
				}
				break;
			case 'e':
				if (argv[1][0]=='-')
				{
					maxDictSize=CLAMP(atoi(argv[1]+2),0,65535*32700);
					if (firstTime)
						printf("* Maximum dictionary size is %d\n",maxDictSize);
				}
				break;
			case 'f':
				if (argv[1][0]=='-')
				{
					minWordFreq=CLAMP(atoi(argv[1]+2),1,65535);
					minWordFreqChanged=true;
					if (firstTime)
						printf("* Minimal word frequency is %d\n",minWordFreq);
				}
				break;
			case 'm':
				if (argv[1][0]=='-')
				{
					maxMemSize=CLAMP(atoi(argv[1]+2),1,255);

					if (firstTime)
						printf("* Maximum memory buffer size is %d MB\n",maxMemSize);

					maxMemSize*=1024*1024;
				}
				break;

			case 'b':
				if (argv[1][0]=='-')
				{
					maxDynDictBuf=CLAMP(atoi(argv[1]+2),1,255);

					if (firstTime)
						printf("* Maximum buffer for creating dynamic dictionary size is %d MB\n",maxDynDictBuf);
				}
				break;

			case 'l':
				if (argv[1][0]=='-')
				{
					compLevel=CLAMP(atoi(argv[1]+2),0,14);

					if (firstTime)
						printf("* Compression level=%d\n",compLevel);

					TURN_OFF(OPTION_ZLIB);
					TURN_OFF(OPTION_LZMA);
					TURN_OFF(OPTION_PPMVC);
					TURN_OFF(OPTION_PAQ);

					switch (compLevel)
					{
						case 0:
							break;
#ifdef USE_ZLIB_LIBRARY
						case 1:
							TURN_ON(OPTION_ZLIB);
							newPreprocType=LZ77;
							additionalParam=1;
							break;
						case 2:
							TURN_ON(OPTION_ZLIB);
							newPreprocType=LZ77;
							additionalParam=6;
							break;
						case 3:
							TURN_ON(OPTION_ZLIB);
							newPreprocType=LZ77;
							additionalParam=9;
							break;
#else
						case 1:
						case 2:
						case 3:
							printf("warning: ZLIB compression not supported in this compilation!\n");
							break;
#endif
#ifdef USE_LZMA_LIBRARY
						case 4:
						case 5:
						case 6:
							TURN_ON(OPTION_LZMA);
							newPreprocType=LZMA;

							if (compLevel==4)
								additionalParam=0;
							else
							if (compLevel==5)
								additionalParam=1;
							else
								additionalParam=8;
							break;
#else
						case 4:
						case 5:
						case 6:
							printf("warning: LZMA compression not supported in this compilation!\n");
							break;
#endif
#ifdef USE_PPMVC_LIBRARY
						case 7:
						case 8:
						case 9:
							TURN_ON(OPTION_PPMVC);
							newPreprocType=PPM;

							if (compLevel==7)
							{
								additionalParam=16;
								if (!minWordFreqChanged)
									minWordFreq=6;
							}
							else
							if (compLevel==8)
							{
								additionalParam=32;
								if (!minWordFreqChanged)
									minWordFreq=16;
							}
							else
							{
								additionalParam=64;
								if (!minWordFreqChanged)
									minWordFreq=64;
							}
							break;
#else
						case 7:
						case 8:
						case 9:
							printf("warning: PPMVC compression not supported in this compilation!\n");
							break;
#endif
#ifdef USE_PAQ_LIBRARY
						case 10:
							TURN_ON(OPTION_PAQ);
							additionalParam=5;
							newPreprocType=PAQ;
							break;
						case 11:
							TURN_ON(OPTION_PAQ);
							additionalParam=6;
							newPreprocType=PAQ;
							break;
						case 12:
							TURN_ON(OPTION_PAQ);
							additionalParam=7;
							newPreprocType=PAQ;
							break;
						case 13:
							TURN_ON(OPTION_PAQ);
							additionalParam=8;
							newPreprocType=PAQ;
							break;
						case 14:
							TURN_ON(OPTION_PAQ);
							additionalParam=9;
							newPreprocType=PAQ;
							break;
#else
						case 10:
						case 11:
						case 12:
						case 13:
						case 14:
							printf("warning: lpaq6 compression not supported in this compilation!\n");
							break;
#endif
					}

				}
				break;

			case 's':
				if (argv[1][0]=='-')
				{
					if (firstTime)
						printf("* Spaces modeling is off\n");
					TURN_OFF(OPTION_SPACES_MODELING);
				}
				break;
			case 'd':
				if (argv[1][0]=='+')
				{
					if (firstTime)
						printf("* Use static dictionary option is on\n");
					TURN_ON(OPTION_USE_DICTIONARY);
				}
				break;
			case 't':
				if (argv[1][0]=='-')
				{
					if (firstTime)
						printf("* Try shoter word option is off\n");
					TURN_OFF(OPTION_TRY_SHORTER_WORD);
				}
				break;
			case 'c':
				if (argv[1][0]=='-')
				{
					if (firstTime)
						printf("* Use containers is off\n");
					TURN_OFF(OPTION_USE_CONTAINERS);
				}
				break;
			case 'n':
				if (argv[1][0]=='-')
				{
					if (firstTime)
						printf("* Number encoding is off\n");
					TURN_OFF(OPTION_NUMBER_CONTAINER);
				}
				break;
			case 'w':
				if (argv[1][0]=='-')
				{
					if (firstTime)
						printf("* Word conntainers are off\n");
					TURN_OFF(OPTION_LETTER_CONTAINER);
				}
				break;
			case 'p':
				if (argv[1][0]=='-')
				{
					firstPassBlock=CLAMP(atoi(argv[1]+2),1,255);
					if (firstTime)
						printf("* Preprocess only (file_size/%d) bytes in a first pass\n",firstPassBlock);
				}
				break;
			case '0':
			case '1':
			case '2':
			case '3':
				TURN_OFF(OPTION_ZLIB);
				TURN_OFF(OPTION_LZMA);
				TURN_OFF(OPTION_PPMVC);
				TURN_OFF(OPTION_PAQ);

				if (firstTime)
				switch (argv[1][1])
				{
					case '0': printf("* LZ77 optimized preprocessing\n"); break;
					case '1': printf("* LZMA/BWT optimized preprocessing\n"); break;
					case '2': printf("* PPM optimized preprocessing\n"); break;
					case '3': printf("* PAQ optimized preprocessing\n"); break;
				}

				newPreprocType=(EPreprocessType)(argv[1][1]-'0');
				break;
			default:
				if (firstTime)
					printf("* Option %s ignored\n", argv[1]);
		}
		argc--;
		optCount++;
		argv++;
	}

	firstTime=false;

	if (newPreprocType!=preprocType)
	{
		preprocType=newPreprocType;
		return defaultSettings(backArgc,backArgv);
	}

	return optCount;
}


int XWRT_Common::getSourcePath(char* buf, int buf_size)
{
#if defined WIN32 || defined WIN64
	int pos;

	pos=GetModuleFileName(NULL,buf,buf_size);

	if (pos>0)
	{	
		for (int i=pos-1; i>=0; i--)
			if (buf[i]=='\\')
			{
				buf[i+1]=0;
				pos=i+1;
				break;
			}
	}

	return pos;
#else
	return 0;
#endif
}

size_t fread_fast(unsigned char* dst, int len, FILE* file)
{
	return fread(dst,1,len,file);

	int rd;
	size_t sum=0;

	while (len > 1<<17) // 128 kb
	{
		rd=fread(dst,1,1<<17,file);
		dst+=rd;
		len-=rd;
		sum+=rd;
	}

	sum+=fread(dst,1,len,file);

	return sum;
}

size_t fwrite_fast(unsigned char* dst, int len, FILE* file)
{
	return fwrite(dst,1,len,file);

	int wt;
	size_t sum=0;

	while (len > 1<<17) // 128 kb
	{
		wt=fwrite(dst,1,1<<17,file);
		dst+=wt;
		len-=wt;
		sum+=wt;
	}

	sum+=fwrite(dst,1,len,file);

	return sum;
}