1.h

基于最短路径的中文分词

int predeal(char* predealfilename){
	chineseflag=0;
	toolongflag=0;
	outbuflen=0;
	errbuflen=0;	 
	sentencelen=0;
	wordkeep[0]='\0';
	wordkeep[1]='\0';
	wordkeep[2]='\0';
	fseek(hdeal,0L,SEEK_END);
	filelen=ftell(hdeal);
	fseek(hdeal,0L,SEEK_SET);
	do{ 	
		filecurpos=ftell(hdeal);
		if (filecurpos>=filelen){
			break;
		}
		if ((filelen-filecurpos)>MAXFILEBUFLEN)
			temp=MAXFILEBUFLEN;
		else
			temp=filelen-filecurpos;	
		inbuflen=fread(inbuf,1,temp,hdeal);
		for(i=0;i<inbuflen;i++){
			temp=( int)inbuf[i];
			if(temp<0){ 		 //可能是汉字的部分
				sentencelen++;
				sentence[sentencelen-1]=inbuf[i];
				if (sentencelen==MAXSENTENCELEN){
					for(j=0;j<sentencelen;j++){
						errbuflen++;
						errbuf[errbuflen-1]=sentence[j];
						if(errbuflen==MAXFILEBUFLEN){
							fwrite(errbuf,1,MAXFILEBUFLEN,herr);
							errbuflen=0;
						}
					}
					sentencelen=0;
					toolongflag=1;
					//	 dealok("single sentence too long!\n");
				}
				if(chineseflag==0)
				{
					chineseflag=1;
				}
				else{
					chineseflag=0;	 
				}
				wordkeep[0]=wordkeep[1];
				wordkeep[1]=inbuf[i];
				if((strcmp(wordkeep,symbol1)==0)||(strcmp(wordkeep,symbol2)==0)||(strcmp(wordkeep,symbol3)==0)||(strcmp(wordkeep,symbol4)==0)){
					if(chineseflag==0) 
					{if (toolongflag==0){
						for(j=0;j<sentencelen;j++){
							outbuflen++;
							outbuf[outbuflen]=sentence[j];//???-1
							if(outbuflen==MAXFILEBUFLEN){
								fwrite(outbuf,1,MAXFILEBUFLEN,hout);
								outbuflen=0;
							}
						}
					}
					else{
						for(j=0;j<sentencelen;j++){
							errbuflen++;
							errbuf[errbuflen-1]=sentence[j];
							if(errbuflen==MAXFILEBUFLEN){
								fwrite(errbuf,1,MAXFILEBUFLEN,herr);
								errbuflen=0;
							}
						}
						toolongflag=0;
					}
					sentencelen=0;
					}
					else{
						for(j=0;j<sentencelen;j++){
							errbuflen++;
							errbuf[er
							
								
								rbuflen-1]=sentence[j];
							if(errbuflen==MAXFILEBUFLEN){
								fwrite(errbuf,1,MAXFILEBUFLEN,herr);
								errbuflen=0;
							}
						}
						sentencelen=0;
						chineseflag=0;
						;
					}
					;
				}
			}
			else{  //非汉字部分
				if(((temp>32)||(temp==10)||(temp==13))&&(temp!=255)){
					sentencelen++;
					sentence[sentencelen-1]=inbuf[i]; 
					if (sentencelen==MAXSENTENCELEN){
						errbuflen++;
						errbuf[errbuflen-1]='d';
						errbuflen++;
						errbuf[errbuflen-1]='f';
						for(j=0;j<sentencelen;j++){
							errbuflen++;
							errbuf[errbuflen-1]=sentence[j];
							if(errbuflen==MAXFILEBUFLEN){
								fwrite(errbuf,1,MAXFILEBUFLEN,herr);
								errbuflen=0;
							}
						}
						sentencelen=0; 
						toolongflag=1;
						//	dealok("single sentence too long!\n");
					}
				}
				if ((chineseflag!=0 )||(temp==32))
				{
					for(j=0;j<sentencelen;j++){
						errbuflen++;
						errbuf[errbuflen-1]=sentence[j];
						if(errbuflen==MAXFILEBUFLEN){
							fwrite(errbuf,1,MAXFILEBUFLEN,herr);
							errbuflen=0;
						}
					}
					sentencelen=0;
					chineseflag=0;
					if(temp==32)
					{					
					   toolongflag=0;
					}
					
				};
				
			}
		}
	 }while(1);
	 fwrite(outbuf,1,outbuflen,hout);
	 fwrite(errbuf,1,errbuflen,herr);
	 if(chineseflag==0){
		 fwrite(sentence,1,sentencelen,hout);  
	 }
	 else{
		 fwrite(sentence,1,sentencelen,herr);
	 }
	 fclose(hdeal);
	 fclose(hout);
	 fclose(herr);
	 okreturn("predeal ok");		
}