diction.cpp

计算机英汉机器翻译系统中的英语词性标注方法实现

		pszOrgPtr ++;

		ucLen = *pszCodePtr; // 规则左部
		pszCodePtr ++;
		strncpy(pszOrgPtr,pszCodePtr,ucLen);
		pszOrgPtr[ucLen] = '\0';
		pszOrgPtr += ucLen;
		pszCodePtr += ucLen;

		ucLen = *pszCodePtr; // 规则右部
		pszCodePtr ++;
		strncpy(pszOrgPtr,pszCodePtr,ucLen);
		pszOrgPtr[ucLen] = '\0';
		pszOrgPtr += ucLen;
		pszCodePtr += ucLen;
	}
}

void EncodeWordRule(LPCSTR pszSouCode,LPSTR pszTarCode,int &nTarCodeLen)
// 对词典规则的编码
// pszTarCode 编码后的规则
// pszSouCode 源词典中的原始规则
// （根据xuned于1997.10.25的要求）
// 原始规则的格式：形如：
// @1:W=chark->sdafasdf@2:W=chark->sdafasdf
// 编码后的规则的格式：
// 规则个数(1 byte),规则左部的长度(1 byte),规则左部,
// 规则右部的长度(1 byte),规则右部,...
{
	LPSTR pszRuleRightPtr = NULL;
	LPSTR pszNextRulePtr = NULL;
	LPCSTR pszSouPtr = pszSouCode;
	nTarCodeLen = 0;
	int nRuleNum = 0; // 规则个数
	nTarCodeLen ++;// 跳过第一个字节，用于保存规则的个数
	do {
		nRuleNum ++;
		pszSouPtr ++;
		pszRuleRightPtr = strstr(pszSouPtr,"->");
		pszNextRulePtr = strchr(pszSouPtr,'@');
		if ( pszRuleRightPtr != NULL && pszNextRulePtr != NULL ) {
			if ( pszRuleRightPtr > pszNextRulePtr )
				pszRuleRightPtr = NULL;
		}
		
		if ( pszRuleRightPtr != NULL ) { // 如果有规则右部
			*(pszTarCode+nTarCodeLen) = pszRuleRightPtr - pszSouPtr;
			nTarCodeLen ++;
			strncpy(pszTarCode+nTarCodeLen,pszSouPtr,
						pszRuleRightPtr - pszSouPtr);
			nTarCodeLen += pszRuleRightPtr - pszSouPtr;
			
			pszRuleRightPtr += 2;
			if ( pszNextRulePtr != NULL ) { // 如果有下一条规则
				*(pszTarCode+nTarCodeLen) = pszNextRulePtr - pszRuleRightPtr;
				nTarCodeLen ++;
				strncpy(pszTarCode+nTarCodeLen,pszRuleRightPtr,
							pszNextRulePtr - pszRuleRightPtr);
				nTarCodeLen += pszNextRulePtr - pszRuleRightPtr;
			} else { // 如果没有下一条规则
				*(pszTarCode+nTarCodeLen) = strlen(pszRuleRightPtr);
				nTarCodeLen ++;
				strcpy(pszTarCode+nTarCodeLen,pszRuleRightPtr);
				nTarCodeLen += strlen(pszRuleRightPtr);
			}
		} else { // 如果没有规则右部
			if ( pszNextRulePtr != NULL ) {// 如果有下一条规则
				*(pszTarCode+nTarCodeLen) = pszNextRulePtr - pszSouPtr;
				nTarCodeLen ++;
				strncpy(pszTarCode+nTarCodeLen,pszSouPtr,
							pszNextRulePtr - pszSouPtr);
				nTarCodeLen += pszNextRulePtr - pszSouPtr;
			} else {// 如果没有下一条规则
				*(pszTarCode+nTarCodeLen) = strlen(pszSouPtr);
				nTarCodeLen ++;
				strcpy(pszTarCode+nTarCodeLen,pszSouPtr);
				nTarCodeLen += strlen(pszSouPtr);
			}

			*(pszTarCode+nTarCodeLen) = 0;
			nTarCodeLen ++;
		}
		pszSouPtr = pszNextRulePtr;
	} while ( pszSouPtr != NULL );
	
	*pszTarCode = (UCHAR)nRuleNum;
}

int CDictIndex::EncodeSingleSlot(Slot *pSlot,LPSTR pszSingleSlot)
{
	ObWord* pObject;
	int nSingleSlotLen = 0;

	if ( pSlot->m_pszSlotValue != NULL ) {
		if ( m_mapSlotName.Lookup(pSlot->m_pszSlotName,( CObject*& )pObject) == FALSE )
			ASSERT( FALSE ); // 没有找到Slot Name
	
		if ( pObject->m_nIndex == m_nQualfrCode ) { // 当前槽是量词
			// 当前的槽名是AddQualfr,槽值是汉字的量词,
			// 在SLOT的链表中的保存格式是:
			// offset 0: AddQualfr的编码的低位字节
			// offset 1: AddQualfr的编码的高位字节
			// offset 2: 量词的长度
			// offset 3--: 量词
			pszSingleSlot[nSingleSlotLen] = LOBYTE(ADDQUALFR); //(WORD)pObject->m_nIndex);
			nSingleSlotLen ++;
			pszSingleSlot[nSingleSlotLen] = HIBYTE(ADDQUALFR); //(WORD)pObject->m_nIndex);
			nSingleSlotLen ++;
			
			pszSingleSlot[nSingleSlotLen] = (BYTE)strlen(pSlot->m_pszSlotValue);
			nSingleSlotLen ++;

			strcpy(pszSingleSlot+nSingleSlotLen,pSlot->m_pszSlotValue);
			
			nSingleSlotLen += strlen(pSlot->m_pszSlotValue);
		} else {
			if ( m_mapSlotValue[pObject->m_nIndex].Lookup(pSlot->m_pszSlotValue,
								( CObject*& )pObject) == FALSE ) {
				ASSERT( FALSE ); // 没有找到Slot Value
			}
			pszSingleSlot[nSingleSlotLen] = LOBYTE((WORD)pObject->m_nIndex);
			nSingleSlotLen ++;
			pszSingleSlot[nSingleSlotLen] = HIBYTE((WORD)pObject->m_nIndex);
			nSingleSlotLen ++;
		}
	} else { // 没有槽值的槽
		if ( pSlot->m_pszSlotName[0] != '@' ) { // 不是规则
			if ( m_mapNoValueSlot.Lookup(pSlot->m_pszSlotName,( CObject*& )pObject) == FALSE )
				ASSERT( FALSE ); // 没有找到Slot Name
		
			pszSingleSlot[nSingleSlotLen] = LOBYTE((WORD)pObject->m_nIndex);
			nSingleSlotLen ++;
			pszSingleSlot[nSingleSlotLen] = HIBYTE((WORD)pObject->m_nIndex);
			nSingleSlotLen ++;
		} else { // 是规则
			pszSingleSlot[nSingleSlotLen] = LOBYTE(RULE_CODE);
			nSingleSlotLen ++;
			pszSingleSlot[nSingleSlotLen] = HIBYTE(RULE_CODE);
			nSingleSlotLen ++;
			
			nSingleSlotLen ++;

			int nRuleCodeLen;
			EncodeWordRule(pSlot->m_pszSlotName,
					pszSingleSlot+nSingleSlotLen,nRuleCodeLen);
			
			pszSingleSlot[nSingleSlotLen-1] = (BYTE)nRuleCodeLen;

			nSingleSlotLen += nRuleCodeLen;
		}
	}

	return nSingleSlotLen;
}

int CDictIndex::EncodeSlotLink(Slot *pFirstSlot,LPSTR pszSlotLink)
// 对槽链表编码
{
	Slot *pCurrSlot = pFirstSlot;
	int nSlotLinkLen = 0;
	do {
		nSlotLinkLen += EncodeSingleSlot(pCurrSlot,
								pszSlotLink+nSlotLinkLen);

		pCurrSlot = pCurrSlot->m_pNextSlot;
	} while ( pCurrSlot != NULL );

	return nSlotLinkLen;
}

int CDictIndex::CalculateSlotNum(Slot *pFirstSlot)
// 计算链表中槽的数量,返回数量
{
	Slot *pCurrSlot = pFirstSlot;
	int nSlotNum = 0;				
	do {
		pCurrSlot = pCurrSlot->m_pNextSlot;
		if ( pCurrSlot != NULL )
			//if ( pCurrSlot->m_bIsTranRule == FALSE ) // 忽略翻译规则
			nSlotNum ++;
	} while ( pCurrSlot != NULL );

	return nSlotNum+1;
}

int CDictIndex::EncodeSingleChinesePart(ChinesePart *pCurrChinese,
										LPSTR pszSinglePartInfo)
{
	int nSingleChinPartLen = 0;

	pszSinglePartInfo[nSingleChinPartLen] = pCurrChinese->m_nChineseLen;//中文译文长度
	nSingleChinPartLen ++;
	
	memcpy(pszSinglePartInfo+nSingleChinPartLen,
			pCurrChinese->m_pszChinese,pCurrChinese->m_nChineseLen);// 中文译文
	nSingleChinPartLen += pCurrChinese->m_nChineseLen;

	ObWord* pObject;
	if ( pCurrChinese->m_pszCate != NULL ) {
		if ( m_mapCate.Lookup(pCurrChinese->m_pszCate,( CObject*& )pObject) == FALSE ) {
			ASSERT( FALSE ); // 没有找到Cate
		}
		pszSinglePartInfo[nSingleChinPartLen] = (UCHAR)pObject->m_nIndex;//词性
		nSingleChinPartLen ++;
	} else {
		pszSinglePartInfo[nSingleChinPartLen] = (UCHAR)0; // No Cate
		nSingleChinPartLen ++;
	}
	
	if ( pCurrChinese->m_pszHead != NULL ) {
		if ( m_mapHead.Lookup(pCurrChinese->m_pszHead,( CObject*& )pObject) == FALSE ) {
			ASSERT( FALSE ); // 没有找到Head
		}
		pszSinglePartInfo[nSingleChinPartLen] = (UCHAR)pObject->m_nIndex;//语义大类
		nSingleChinPartLen ++;
	} else {
		pszSinglePartInfo[nSingleChinPartLen] = (UCHAR)0; // No Head
		nSingleChinPartLen ++;
	}
	
	// Add for Debug Begin
	//if ( strcmp(pCurrChinese->m_pszChinese,"几") == 0 ) {
	//	ASSERT(FALSE);
	//}
	// Add for Debug End

	if ( pCurrChinese->m_pFirstSlot != NULL ) {
		pszSinglePartInfo[nSingleChinPartLen] = CalculateSlotNum(pCurrChinese->m_pFirstSlot);//槽的数量
		nSingleChinPartLen ++;

		nSingleChinPartLen += EncodeSlotLink(pCurrChinese->m_pFirstSlot,
								pszSinglePartInfo+nSingleChinPartLen);
	} else {
		pszSinglePartInfo[nSingleChinPartLen] = (UCHAR)0;//槽的数量
		nSingleChinPartLen ++;
	}

	return nSingleChinPartLen;
}

int CDictIndex::EncodeChinesePartInfo(COneWord *pOneWord,
									  LPSTR pszChinesePartInfo)
// 编码中文译文及相关内容部分
{
	int nChinesePartInfoLen = 0;

	if ( pOneWord->m_pFirstChinese != NULL ) {
		ChinesePart *pCurrChinese = pOneWord->m_pFirstChinese;
		do {
			nChinesePartInfoLen += EncodeSingleChinesePart(pCurrChinese,
						pszChinesePartInfo+nChinesePartInfoLen);

			pCurrChinese = pCurrChinese->m_pNextPart;
		} while ( pCurrChinese != NULL );
	}
	return nChinesePartInfoLen;
}

int CDictIndex::CalcuChinPartNum(ChinesePart *pFirstChinese)
// 计算中文译文的个数
{	
	ChinesePart *pCurrChinese = pFirstChinese;
	int nTotalNum = 0;
	do {
		nTotalNum ++;
		pCurrChinese = pCurrChinese->m_pNextPart;
	} while ( pCurrChinese != NULL );
	return nTotalNum;
}

int CDictIndex::EncodeDictRecord(COneWord *pOneWord,LPSTR pszWordInfo)
{
	int nNowSite = 0;
	// 是否存在原形 ( 1 Byte )
	pszWordInfo[nNowSite] = (UCHAR)pOneWord->m_bIsExistOrig;
	nNowSite ++;

	// 是否存在兼类
	pszWordInfo[nNowSite] = (UCHAR)pOneWord->m_bIsExistAmbig;
	nNowSite ++;

	// 下面紧跟一个标志和标志相关的其他内容,参考下面说明
	// 原形的存储
	if ( pOneWord->m_bIsExistOrig ) {
		pszWordInfo[nNowSite] = (UCHAR)DICT_FLAG_ORIGIN; // 存储内容的标志
		nNowSite ++;

		pszWordInfo[nNowSite] = (UCHAR)pOneWord->m_nWordStyle;// 词的类型:过去式,进行式...(1 Byte) m_nWordStyle
		nNowSite ++;
		
		pszWordInfo[nNowSite] = (UCHAR)pOneWord->m_nOrigLen; // 原形长度 ( 1 Byte )
		nNowSite ++;

		memcpy(pszWordInfo+nNowSite,pOneWord->m_pszOrig,pOneWord->m_nOrigLen);// 原形
		nNowSite += pOneWord->m_nOrigLen;
	}

	// 兼类的存储
	if ( pOneWord->m_bIsExistAmbig ) {
		pszWordInfo[nNowSite] = (UCHAR)DICT_FLAG_AMBIG; // 存储内容的标志
		nNowSite ++;

		pszWordInfo[nNowSite] = (UCHAR)pOneWord->m_nAmbigLen; // 兼类长度 ( 1 Byte )
		nNowSite ++;

		memcpy(pszWordInfo+nNowSite,pOneWord->m_pszAmbig,pOneWord->m_nAmbigLen);// 原形
		nNowSite += pOneWord->m_nAmbigLen;
	}
	if ( pOneWord->m_pFirstChinese != NULL ) { // 有中文
		pszWordInfo[nNowSite] = (UCHAR)DICT_FLAG_CHINESE;
		nNowSite ++;
		pszWordInfo[nNowSite] = (UCHAR)CalcuChinPartNum(pOneWord->m_pFirstChinese);
		nNowSite ++;
	} else { // 无中文
		pszWordInfo[nNowSite] = (UCHAR)DICT_FLAG_CHINESE;
		nNowSite ++;
		pszWordInfo[nNowSite] = 0; // // 中文译文的个数
		nNowSite ++;
	}

	nNowSite += EncodeChinesePartInfo(pOneWord,pszWordInfo+nNowSite);
	return nNowSite;
}

BOOL CDictIndex::InsertOneWordToIndex(Dictionary *pobDiction,
									  FILE *fpIndexDat,
									  CHuffman *pHuffman,
									  COneWord *pOneWord)
// 旧函数,现不使用
{
	// 如果该词条为空,则跳过该词条
	if ( pOneWord->m_pszEnglish == NULL )
		return TRUE;

	int  nTarBuffSize = DIC_WORD_LEN*10;
	LPSTR pszTarBuff = new char[nTarBuffSize];
	memset(pszTarBuff,nTarBuffSize,0);

	int nTarLen;
	pHuffman->CompressString(pOneWord->m_pszEnglish,
					pOneWord->m_nEnglishLen,
					pszTarBuff,nTarLen);
	if ( nTarLen > DIC_WORD_LEN ) {
		//ASSERT( FALSE );// 词典中英文词条太长
		// 跳过这个词
		delete pszTarBuff;
		return TRUE;
	} else {
		m_nTotalWordsNum ++;

		int nWordInfoLen;
		nWordInfoLen = EncodeDictRecord(pOneWord,m_pszWordInfoBuff);

		long lDataSite = WriteIndexData(fpIndexDat,m_pszWordInfoBuff,
						nWordInfoLen);
		CompressIndexOffsetInfo(lDataSite,nWordInfoLen,
						pobDiction->m_pszOffset);

		memset(pobDiction->m_pszWord,0x0,DIC_WORD_LEN);
		memcpy(pobDiction->m_pszWord,pszTarBuff,nTarLen);

		pobDiction->insert();
	}

	delete pszTarBuff;
	return TRUE;
}

int EncodeChinFormatCheck(LPSTR pszWordInfo)
{
	int nNowSite = 0;

	int nChineseLen = (UCHAR)pszWordInfo[nNowSite]; // 中文译文长度
	nNowSite ++;

	nNowSite += nChineseLen;
	// 词性
	nNowSite ++;
	// 语义大类
	nNowSite ++;
	
	int nSlotNum = (UCHAR)pszWordInfo[nNowSite];
	nNowSite ++;
	
	WORD wTep;
	int nQualfrLen;
	int nDicRule;
	for ( int Loop=0;Loop<nSlotNum;Loop++ ) {
		wTep = MAKEWORD(pszWordInfo[nNowSite],pszWordInfo[nNowSite+1]);
		nNowSite += 2;
		if ( wTep == ADDQUALFR ) { // 量词的解码
			nQualfrLen = (BYTE) *(pszWordInfo+nNowSite);
			nNowSite ++;
			nNowSite += nQualfrLen;
		} else if ( wTep == RULE_CODE ) { // 规则的解码
			nDicRule = (BYTE) *(pszWordInfo+nNowSite);
			nNowSite ++;
			nNowSite += nDicRule;
		} else { // 普通槽
		}
	}
	return nNowSite;
}

BOOL EncodeResultFormatCheck(LPSTR pszWordInfo,int nWordInfoLen)
// 编码格式检查
{
	int nNowSite = 0;
	nNowSite ++;
	nNowSite ++;
	
	int Loop;
	DictChin *pLastChin = NULL;
	int nOrigLen;
	int nAmbigLen;
	int nChinNum;
	do {
		switch ( (UCHAR)pszWordInfo[nNowSite] ) {
		case DICT_FLAG_ORIGIN:
			nNowSite ++;
			nNowSite ++;
			nOrigLen = (UCHAR)pszWordInfo[nNowSite];
			nNowSite ++;
			nNowSite += nOrigLen;
			break;
		case DICT_FLAG_AMBIG:
			nNowSite ++;	
			nAmbigLen = (UCHAR)pszWordInfo[nNowSite];
			nNowSite ++;
			// 兼类内容
			nNowSite += nAmbigLen;
		break;
		case DICT_FLAG_CHINESE:
			nNowSite ++;
			nChinNum = (UCHAR)pszWordInfo[nNowSite];
			nNowSite ++;
			for ( Loop=0;Loop<nChinNum;Loop++ )
				nNowSite += EncodeChinFormatCheck(pszWordInfo+nNowSite);
			break;
		default: 
			return FALSE; // 编码格式错误
		}
	} while ( nNowSite < nWordInfoLen );
	return TRUE;
}

BOOL CDictIndex::OutputOneWordToTempDatFile(FILE *fpEnglishInfo,
											FILE *fpTranDat,
											COneWord *pOneWord)
// 将单词的词条内容输出到中间文件fpTranDat中,
// 同时将对应的单词的英文和词条内容在fpTranDat中的位置输出到fpEnglishInfo中
{
	// 如果该词条为空,则跳过该词条
	if ( pOneWord->m_pszEnglish == NULL )
		return TRUE;

	m_nTotalWordsNum ++;

	int nWordInfoLen;
	nWordInfoLen = EncodeDictRecord(pOneWord,m_pszWordInfoBuff);

	// 编码格式检查
	if ( EncodeResultFormatCheck(m_pszWordInfoBuff,nWordInfoLen) == FALSE )