diction.cpp

计算机英汉机器翻译系统中的英语词性标注方法实现

		ASSERT(FALSE);

	long lDataSite = WriteIndexData(fpTranDat,m_pszWordInfoBuff,
					nWordInfoLen);

	// 各行的格式: 英文单词 该单词的词条内容在ECDICT.TMP中的位置 该单词的词条内容的长度
	// 上面三项内容之间用 ' ,'隔开
	fprintf(fpEnglishInfo,"%s ,%d ,%d\r\n",pOneWord->m_pszEnglish,lDataSite,nWordInfoLen);
	
	//fprintf(fpEnglishInfo,"%2d,%s\n",strlen(pOneWord->m_pszEnglish),pOneWord->m_pszEnglish);
	return TRUE;
}

BOOL BuildSingleFileIndexStepOne(LPSTR szDicName,COneWord *pOneWord,
								 FILE *fpEnglishInfo,FILE *fpTempDat,
								 CDictIndex *pDictIndex)
{
	BOOL bIsNoError;
	BOOL bIsEof;
	
	FILE *fpSouFile;
	fpSouFile = fopen(szDicName,"rb");
	if ( fpSouFile == NULL ) {
		AfxMessageBox("Cann't Open Dictictionary!");
		return FALSE;
	}
	
	pOneWord->m_nCurrReadLineNum = 0;
	do {
		bIsNoError = pOneWord->ReadWordFromSouDict(fpSouFile,szDicName,
							NULL,bIsEof);
		if ( !bIsNoError ) {
			pOneWord->GotoNextWordBegin(fpSouFile);
			AfxMessageBox("There are errors in Dictionary,\nPlease \
correct them and retry!");
				return FALSE;
		} else {
			if ( bIsEof == FALSE )
				pDictIndex->OutputOneWordToTempDatFile(fpEnglishInfo,
										fpTempDat,pOneWord);
		}
		pOneWord->EmptyWordContent();
	} while ( !bIsEof );
	fclose(fpSouFile);
	return TRUE;
}

// yys 98.5.14 Bgn
// 加入一个形式参数 szDicName  
BOOL BuildIndexStepOne(LPSTR pszInDefineName,LPSTR pszOutDefineName,
				LPSTR pszDicIndexName,LPSTR pszEngInfoName,
				LPSTR pszCodeTable,char *szDicName)
// yys 98.5.14 End
// 对英汉词典建索引的第一步 : 
// 将单词的词条内容输出到中间文件pszDicIndexName.tmp中,
// 同时将对应的单词的英文和词条内容在pszDicIndexName.tmp中的位置
// 输出到pszEngInfoName中
{
	FILE *fpTempDat;
	const int MAXLINELEN = 400;

	char szDictIndexName[MAX_FILENAME_LEN]; // 索引文件名
	char szDictDtaName[MAX_FILENAME_LEN]; // 二级索引文件名
	char szDictDatName[MAX_FILENAME_LEN]; // 数据文件名
	char szTempDatName[MAX_FILENAME_LEN]; // 中间数据文件名
	
	strcpy(szDictIndexName,pszDicIndexName);
	strcat(szDictIndexName,".idx");
	
	strcpy(szDictDtaName,pszDicIndexName);
	strcat(szDictDtaName,".dta");

	strcpy(szDictDatName,pszDicIndexName);
	strcat(szDictDatName,".dat");

	strcpy(szTempDatName,pszDicIndexName);
	strcat(szTempDatName,".tmp");

	remove(szDictIndexName);
	remove(szDictDtaName);
	remove(szDictDatName);
	remove(szTempDatName);
	
	// 
	if ( (fpTempDat = fopen(szTempDatName,"wb") ) == NULL ) {
		CString strMsg;
		strMsg.Format("无法创建文件 %s !",szTempDatName);
		AfxMessageBox(strMsg);
		return FALSE;
	}

	DictHeader phHead;

	strcpy(phHead.szMagic,szDictDataMagic);
	fwrite(&phHead,sizeof(DictHeader),1,fpTempDat);

	int nIndex = 1;

	COneWord OneWord;
	//char szDicName[100] = "Ecdic\\Ecdic.dic";  //作为一个形式参数述传入
	
	OneWord.m_fpErrLogFile = fopen("DictRes\\Error.txt","wb");
	if ( OneWord.m_fpErrLogFile == NULL ) {
		AfxMessageBox("无法创建文件!");
		return FALSE;
	}
	OneWord.m_bIsWriteLog = TRUE;
	FILE *fpEnglishInfo;
	fpEnglishInfo = fopen(pszEngInfoName,"wb");
	if ( fpEnglishInfo == NULL ) {
		CString strMsg;
		strMsg.Format("无法创建文件 %s !",pszEngInfoName);
		AfxMessageBox(strMsg);
		return FALSE;
	}
	
	OneWord.m_bIsWriteErrToDic = FALSE;
	// 初始化
	if ( OneWord.Init(pszInDefineName) == FALSE )
		return FALSE;
	
	CDictIndex DictIndex;
	DictIndex.m_nTotalWordsNum = 0;

	if ( DictIndex.BuildIndexInit(pszInDefineName,
						pszOutDefineName,pszCodeTable) == FALSE )
		return FALSE;

	//FILE *fpTarDic;

	//if ( OneWord.m_bIsWriteErrToDic )
	//	OneWord.m_fpTarDic = fpTarDic;

	BuildSingleFileIndexStepOne(szDicName,&OneWord,fpEnglishInfo,
							fpTempDat,&DictIndex);

	if ( OneWord.m_bIsWriteLog )
		fclose(OneWord.m_fpErrLogFile);

	// 释放保留字数组占用的空间
	OneWord.ExitInit();

	DictIndex.BuildIndexExitInit();

	fclose(OneWord.m_fpErrLogFile);

	fclose(fpTempDat);
	fclose(fpEnglishInfo);

	fprintf(fpLog,"本索引共有 %d 个词条.\n",
		DictIndex.m_nTotalWordsNum);

	return TRUE;
}

BOOL ReadALineFromEngInfo(FILE *fpEnglishInfo,LPSTR pszEnglish,
						 long &lDataSite,int &nWordInfoLen)
// 从ENGINFO.TXT中读入一行,并将行中的内容分解开
// 各行的格式: 英文单词 该单词的词条内容在ECDICT.TMP中的位置 该单词的词条内容的长度
// 上面三项内容之间用 ' ,'隔开
{
	char szLine[500];
	char szDataSite[20];
	char szWordInfoLen[20];
	CString strTempA;
	CString strTempB;

	myfgets(szLine,500,fpEnglishInfo);
	if ( feof(fpEnglishInfo) ) return FALSE;
	strTempA = szLine;
	strTempB = strTempA.Left(strTempA.Find(" ,"));
	strcpy(pszEnglish,strTempB);

	strTempA = strTempA.Right(strTempA.GetLength()-strTempA.Find(" ,")-2);
	strTempB = strTempA.Left(strTempA.Find(" ,"));
	strTempA = strTempA.Right(strTempA.GetLength()-strTempA.Find(" ,")-2);
	strcpy(szDataSite,strTempB);
	strcpy(szWordInfoLen,strTempA);
	nWordInfoLen = atol(szWordInfoLen);
	lDataSite = atol(szDataSite);
	return TRUE;
}

int CalculateWordNum(LPSTR pszEnglish)
// 计算包含单词的个数
{
	int nWordNum = 1;
	while ( *pszEnglish != '\0' ) {
		if ( *pszEnglish == ' ' ) nWordNum ++;
		pszEnglish ++;
	};
	return nWordNum;
}

BOOL BuildIndexStepTwo(LPSTR pszEngInfoName,LPSTR pszHuffmanFreqDataName,
					   LPSTR pszEngHuffName)
// 对英汉词典建索引的第二步:
// 生成HUFFMAN频率文件及pszEngHuffName文件

// pszEngHuffName 将pszEngInfoName中的英文词条用HUFFMAN压缩,
// 同时对每个单词增加词典中包括该单词的个数的信息
{
	/*
	// 根据ENGINFO.TXT生成HUFFMAN压缩所需的频率文件HUFFFREQ.DAT
	CHuffman huffman;
	BOOL bResult = huffman.CreateHuffmanFreqDataEx(pszEngInfoName,
					pszHuffmanFreqDataName);
	if ( !bResult ) {
		return FALSE;
	}
	
	bResult = huffman.ReadHuffmanFreqDate(pszHuffmanFreqDataName);
	if ( !bResult ) {
		return FALSE;
	}

	huffman.BuildHufTree();
	*/

	FILE *fpEnglishInfo;
	fpEnglishInfo = fopen(pszEngInfoName,"rb");
	if ( fpEnglishInfo == NULL ) {
		CString strMsg;
		strMsg.Format("无法打开文件 %s !",pszEngInfoName);
		AfxMessageBox(strMsg);
		return FALSE;
	}
	
	FILE *fpEnglishHuff;
	fpEnglishHuff = fopen(pszEngHuffName,"wb");
	if ( fpEnglishInfo == NULL ) {
		CString strMsg;
		strMsg.Format("无法创建文件 %s !",pszEngHuffName);
		AfxMessageBox(strMsg);
		return FALSE;
	}
	
	long lDataSite;
	int nWordInfoLen;

	char szEnglish[200];
	int nEnglishLen;
	char szCompEnglish[200];
	int nCompEnglishLen;
	
	char szLastFirstWord[100]; // 词组的首词
	int nMaxWordsNum; // 以该词为首词的词组包含单词的最大个数
	BOOL bLastIsPhrase = FALSE; // 上一个词是否是词组
	int nCurrMaxWordsNum;
	char *pszTep;
	char szCurrFirstWord[100];

	do {
		if ( ReadALineFromEngInfo(fpEnglishInfo,szEnglish,
							lDataSite,nWordInfoLen) == FALSE )
			break;
		nEnglishLen = strlen(szEnglish);
		
		//huffman.CompressString(szEnglish,nEnglishLen,szCompEnglish,nCompEnglishLen);
		strcpy(szCompEnglish,szEnglish);
		nCompEnglishLen = nEnglishLen;

		pszTep = strchr(szEnglish,' ');
		if ( pszTep != NULL ) { // 当前单词是词组
			nCurrMaxWordsNum = 1;
			strncpy(szCurrFirstWord,szEnglish,pszTep-szEnglish);
			szCurrFirstWord[pszTep-szEnglish] = '\0';
			int nNowWordNum = CalculateWordNum(szEnglish);
			if ( bLastIsPhrase ) { // 上一个单词也是词组
				if ( strcmp(szCurrFirstWord,szLastFirstWord) == 0 ) {
					if ( nNowWordNum > nMaxWordsNum )
						nMaxWordsNum = nNowWordNum;
				} else {
					nMaxWordsNum = nNowWordNum;
					strcpy(szLastFirstWord,szCurrFirstWord);
				}
			} else { // // 上一个单词不是词组
				bLastIsPhrase = TRUE;
				nMaxWordsNum = nNowWordNum;
				strcpy(szLastFirstWord,szCurrFirstWord);
			}
		} else { // 当前单词不是词组
			if ( strcmp(szEnglish,szLastFirstWord) == 0 )
				nCurrMaxWordsNum = nMaxWordsNum;
			else
				nCurrMaxWordsNum = 1;
		}
		
		//for ( int Loop=0;Loop<nCompEnglishLen;Loop++ ) {
		//	ch = szCompEnglish[Loop];
		//	fprintf(fpEnglishHuff,"%0.2X",ch);
		//}

		fprintf(fpEnglishHuff,"%s ,%d ,%d ,%d ,%d\r\n",szCompEnglish,
					nEnglishLen,nCurrMaxWordsNum,lDataSite,nWordInfoLen);
	} while ( TRUE );
	
	fclose(fpEnglishInfo);
	fclose(fpEnglishHuff);
	
	remove(pszEngInfoName);
	return TRUE;
}

void HexToAscString(LPSTR pszHexLine,LPSTR pszEngHuff,int &nEngHuffLen)
// 将16进制表示的字符串转换成ASC码表示的字符串
{
	UCHAR ch1,ch2;
	int nHexSite = 0;
	int nEngHuffSite = 0;
	while ( pszHexLine[nHexSite] != '\0' ) {
		ch1 = pszHexLine[nHexSite];
		if ( ch1 >= 'A' )
			ch1 = ch1 - 'A'+ 0xA;
		else
			ch1 -= '0';

		nHexSite ++;
		ch2 = pszHexLine[nHexSite];
		if ( ch2 >= 'A' )
			ch2 = ch2 -'A'+ 0xA;
		else
			ch2 -= '0';

		nHexSite ++;
		pszEngHuff[nEngHuffSite] = (UCHAR)(ch1*0x10+ch2);
		nEngHuffSite ++;
	}
	//pszEngHuff[nEngHuffSite] = '\0';
	nEngHuffLen = nEngHuffSite;
}

BOOL ReadALineFromEngHuff(FILE *fpEnglishHuff,LPSTR pszEngHuff,
						  int &nEngHuffLen,
						  int &nEngOrgLen,int &nWordsNum,
						  long &lDataSite,int &nWordInfoLen)
// 从ENGHUFF.TXT中读入一行,并将行中的内容分解开
// 各行的格式: 经HUFFMAN压缩后的英文单词 压缩前的英文单词长度 
// 以该词为首词的词组包含单词的最大个数
// 该单词的词条内容在ECDICT.TMP中的位置 该单词的词条内容的长度
// 上面五项内容之间用 ' ,'隔开
// 如果文件结束返回FALSE,否则返回TRUE
{
	char szLine[500];
	char szEngOrgLen[10];
	char szWordsNum[10];
	char szDataSite[20];
	char szWordInfoLen[20];
	//char szHexEngHuff[200];
	CString strTempA;
	CString strTempB;

	myfgets(szLine,500,fpEnglishHuff);
	if ( feof(fpEnglishHuff) )
		return FALSE;

	strTempA = szLine;
	strTempB = strTempA.Left(strTempA.Find(" ,"));
	//strcpy(szHexEngHuff,strTempB);
	strcpy(pszEngHuff,strTempB);
	nEngHuffLen = strTempB.GetLength();
	//HexToAscString(szHexEngHuff,pszEngHuff,nEngHuffLen);

	strTempA = strTempA.Right(strTempA.GetLength()-strTempA.Find(" ,")-2);
	strTempB = strTempA.Left(strTempA.Find(" ,"));
	strcpy(szEngOrgLen,strTempB);

	strTempA = strTempA.Right(strTempA.GetLength()-strTempA.Find(" ,")-2);
	strTempB = strTempA.Left(strTempA.Find(" ,"));
	strcpy(szWordsNum,strTempB);

	strTempA = strTempA.Right(strTempA.GetLength()-strTempA.Find(" ,")-2);
	strTempB = strTempA.Left(strTempA.Find(" ,"));
	strcpy(szDataSite,strTempB);

	strTempA = strTempA.Right(strTempA.GetLength()-strTempA.Find(" ,")-2);
	strcpy(szWordInfoLen,strTempA);

	nEngOrgLen = atol(szEngOrgLen);
	nWordsNum = atol(szWordsNum);
	lDataSite = atol(szDataSite);
	nWordInfoLen = atol(szWordInfoLen);
	return TRUE;
}

//yys 98.5.14 Bgn
//添加一个形参szDicName
BOOL BuildIndexStepThree(LPSTR pszEngHuffName,LPSTR pszDicIndexName,char *szDicName)
//yys 98.5.14 End
// 对英汉词典建索引的第三步:
{
	int nMaxKeyLen = 0; // 字典中的最大的关键字的长度
	int nExceedMaxNum = 0; // 词典中超过允许最大关键字的词的个数

	FILE *fpEnglishHuff;
	fpEnglishHuff = fopen(pszEngHuffName,"rb");
	if ( fpEnglishHuff == NULL ) {
		CString strMsg;
		strMsg.Format("Cann't open %s !",pszEngHuffName);
		AfxMessageBox(strMsg);
		return FALSE;
	}

	FILE *fpIndexDat;
	FILE *fpTempDat;
	const int MAXLINELEN = 400;

	char szDictIndexName[MAX_FILENAME_LEN]; // 索引文件名
	char szDictDtaName[MAX_FILENAME_LEN]; // 二级索引文件名
	char szDictDatName[MAX_FILENAME_LEN]; // 数据文件名
	char szTempDatName[MAX_FILENAME_LEN]; // 中间数据文件名
	
	strcpy(szDictIndexName,pszDicIndexName);
	strcat(szDictIndexName,".idx");
	
	strcpy(szDictDtaName,pszDicIndexName);
	strcat(szDictDtaName,".dta");

	strcpy(szDictDatName,pszDicIndexName);
	strcat(szDictDatName,".dat");

	strcpy(szTempDatName,pszDicIndexName);
	strcat(szTempDatName,".tmp");

	remove(szDictIndexName);
	remove(szDictDtaName);
	remove(szDictDatName);
	// 
	if ( (fpTempDat = fopen(szTempDatName,"rb") ) == NULL ) {
		CString strMsg;
		strMsg.Format("无法打开文件 %s !",szTempDatName);
		AfxMessageBox(strMsg);
		return FALSE;
	}

	if ( (fpIndexDat = fopen(szDictDatName,"wb") ) == NULL ) {
		CString strMsg;
		strMsg.Format("无法创建文件%s !",szDictDatName);
		AfxMessageBox(strMsg);
		return FALSE;
	}

	Dictionary obDiction(szDictDtaName,szDictIndexName,0);

	DictHeader phHead;

	strcpy(phHead.szMagic,szDictDataMagic);
	fwrite(&phHead,sizeof(DictHeader),1,fpIndexDat);

	// Make Hash Table and Write to fpIndexDat Begin
	DictSearch dictsearch;
	//yys 98.5.14 Bgn
	
	dictsearch.CreateHashTableAndAddToDat(fpIndexDat,phHead,szDicName);
	//yys 98.5.14 End
	// Make Hash Table and Write to fpIndexDat End

	// 申请缓冲区
	int nWordInfoBuffSize = MAX_RECORD_LEN;
    LPSTR pszWordInfoBuff = (LPSTR)GlobalLock(GlobalAlloc(GMEM_MOVEABLE|GMEM_SHARE,
                        nWordInfoBuffSize));

	char szEngHuff[100];