mainfrm.cpp

汉字字频统计程序

// MainFrm.cpp : implementation of the CMainFrame class
//

#include "stdafx.h"
#include "BiHZFreq.h"
#include <stdlib.h> // qsort()
#include <search.h> 

#include "MainFrm.h"

#ifdef _DEBUG
#define new DEBUG_NEW
#undef THIS_FILE
static char THIS_FILE[] = __FILE__;
#endif

# define HZ_NUM 6768
# define HZ_ID(c1,c2) ((c1)-176) *94 +((c2)-161)

int HZFreq[HZ_NUM];

BOOL ADD_HANZI=FALSE; // 判断是否对语料进行了字频统计

/////////////////////////////////////////////////////////////////////////////
// CMainFrame

IMPLEMENT_DYNAMIC(CMainFrame, CMDIFrameWnd)

BEGIN_MESSAGE_MAP(CMainFrame, CMDIFrameWnd)
	//{{AFX_MSG_MAP(CMainFrame)
	ON_WM_CREATE()
	ON_COMMAND(ID_LoadHZFreq, OnLoadHZFreq)
	ON_COMMAND(ID_StoreHZFreq, OnStoreHZFreq)
	ON_COMMAND(ID_HZPairs, OnHZPairs)
	//}}AFX_MSG_MAP
END_MESSAGE_MAP()

static UINT indicators[] =
{
	ID_SEPARATOR,           // status line indicator
	ID_INDICATOR_CAPS,
	ID_INDICATOR_NUM,
	ID_INDICATOR_SCRL,
};

/////////////////////////////////////////////////////////////////////////////
// CMainFrame construction/destruction

CMainFrame::CMainFrame()
{
	// TODO: add member initialization code here
	
}

CMainFrame::~CMainFrame()
{
}

int CMainFrame::OnCreate(LPCREATESTRUCT lpCreateStruct)
{
	if (CMDIFrameWnd::OnCreate(lpCreateStruct) == -1)
		return -1;
	
	if (!m_wndToolBar.CreateEx(this, TBSTYLE_FLAT, WS_CHILD | WS_VISIBLE | CBRS_TOP
		| CBRS_GRIPPER | CBRS_TOOLTIPS | CBRS_FLYBY | CBRS_SIZE_DYNAMIC) ||
		!m_wndToolBar.LoadToolBar(IDR_MAINFRAME))
	{
		TRACE0("Failed to create toolbar\n");
		return -1;      // fail to create
	}

	if (!m_wndStatusBar.Create(this) ||
		!m_wndStatusBar.SetIndicators(indicators,
		  sizeof(indicators)/sizeof(UINT)))
	{
		TRACE0("Failed to create status bar\n");
		return -1;      // fail to create
	}

	// TODO: Delete these three lines if you don't want the toolbar to
	//  be dockable
	m_wndToolBar.EnableDocking(CBRS_ALIGN_ANY);
	EnableDocking(CBRS_ALIGN_ANY);
	DockControlBar(&m_wndToolBar);

	return 0;
}

BOOL CMainFrame::PreCreateWindow(CREATESTRUCT& cs)
{
	if( !CMDIFrameWnd::PreCreateWindow(cs) )
		return FALSE;
	// TODO: Modify the Window class or styles here by modifying
	//  the CREATESTRUCT cs

	return TRUE;
}

/////////////////////////////////////////////////////////////////////////////
// CMainFrame diagnostics

#ifdef _DEBUG
void CMainFrame::AssertValid() const
{
	CMDIFrameWnd::AssertValid();
}

void CMainFrame::Dump(CDumpContext& dc) const
{
	CMDIFrameWnd::Dump(dc);
}

#endif //_DEBUG

/////////////////////////////////////////////////////////////////////////////
// CMainFrame message handlers

CSortedPairs hzPairs; // 定义一个排序动态数组类的对象

void HZPairInFile(CString FileName) 
{ // 统计一个文件中汉字对出现次数函数
	FILE * in;
	unsigned char c1,c2;
	char zz[5];
	int id;

	in=fopen((const char *)FileName,"rt");
	if(in==NULL) {
		AfxMessageBox("无法打开文件");
		return;
	}

	zz[0]='\0';
	
	while(!feof(in)) {
		c1=(unsigned char)fgetc(in);
		
		if(c1<128 || c1==255) {
			zz[0] = '\0';
			continue;
		}
		else { 
			if(c1<176) {
				c2=fgetc(in);
				zz[0]='\0';
				continue;
			}
		}

		c2=(unsigned char)fgetc(in);
		
		if(c2==255 || c2<161) {
			break; // 如果碰到非法字符，跳出循环
		}
		else {
			id=HZ_ID(c1,c2);
			HZFreq[id]++; // 加入单字频率
		}

		if(zz[0]!='\0') {
			zz[2]=c1;
			zz[3]=c2;
			zz[4]='\0';
			hzPairs.Insert(zz); // 加入汉字对频率
		}

 		zz[0]=c1;
		zz[1]=c2;
		zz[2]='\0';
	}
	fclose(in);
	return;
}

void CMainFrame::OnLoadHZFreq() 
{
	// TODO: Add your command handler code here
	char buf[512];
	FILE * in;
	CFile inFile;
	
	long int hzArrayLength=hzPairs.GetSize();
	hzPairs.SetSize(hzArrayLength,1);

	// 用文件对话框取得存放字频统计信息的文件路径和文件名
	
	CFileDialog dlg(TRUE, "dat","hzpairs.dat",OFN_HIDEREADONLY | OFN_OVERWRITEPROMPT,"*.dat|所有文件||");
	if(dlg.DoModal()!=IDOK) 
		return;

	if(inFile.Open((const char *) (dlg.GetPathName()),CFile::modeRead))
	{
		CArchive ar(&inFile, CArchive::load,512,buf); // 设置为装载模式
		hzPairs.Serialize(ar); // 从文件中装载双字字表
		inFile.Close();
		AfxMessageBox("装载双字字表完毕！");
	}
	else 
		AfxMessageBox("无法打开双字字频文件");
	
	CFileDialog dlg2(TRUE, "dat","hzsingle.dat",OFN_HIDEREADONLY | OFN_OVERWRITEPROMPT,"*.dat|所有文件||");
	if(dlg2.DoModal()!=IDOK) 
		return;

	in=fopen((const char *) (dlg2.GetPathName()),"rb");
	
	if(in)
	{
		fread(HZFreq,sizeof(int),HZ_NUM,in); //从文件中装载单字字频
		fclose(in);
		AfxMessageBox("装载单字字表完毕！");
	}
	else {
		AfxMessageBox("无法打开单字字频文件");
		for(int i=0;i<HZ_NUM;i++) 
			HZFreq[i]=0;
	}
}

void CMainFrame::OnStoreHZFreq() 
{
	// TODO: Add your command handler code here
	if(ADD_HANZI) {
		char buf[512];
		FILE *out;
		CFile outFile;

/*		CFileDialog dlg(FALSE, "dat","hzpairs.dat",OFN_HIDEREADONLY | OFN_OVERWRITEPROMPT,"*.dat|所有文件||");
		if(dlg.DoModal()!=IDOK) 
			return;

		if(outFile.Open((const char *) (dlg.GetPathName()),CFile::modeCreate| CFile::modeWrite))
*/
		if(outFile.Open("hzpairs.dat",CFile::modeCreate| CFile::modeWrite))
		{
			CArchive ar(&outFile,CArchive::store,512,buf); // 设置为存放模式
			hzPairs.Serialize(ar); // 将双字字频写回文件
		}
		else {
			AfxMessageBox("无法写双字字频文件");
			return;
		}
/*
		CFileDialog dlg2(FALSE, "dat","hzsingle.dat",OFN_HIDEREADONLY | OFN_OVERWRITEPROMPT,"*.dat|所有文件||");
		if(dlg2.DoModal()!=IDOK) 
			return;

		out=fopen((const char *) (dlg2.GetPathName()),"wb");
*/
		out=fopen("hzsingle.dat","wb");
		
		if(out) {
			fwrite(HZFreq,sizeof(int),HZ_NUM,out);
			fclose(out);
		}
		else{
			AfxMessageBox("无法写单字字频文件");
			return;
		}

		AfxMessageBox("存贮双字字频和单字字频完毕");
	}
}

struct myHzPair 
{// 定义一个结构，存贮hzPairs中的内容
	char BiHz[5];
	int BiHzFreq;
};

int compare( const void *arg1, const void *arg2 )
{// 比较双字频度大小
   
	if (((struct myHzPair *)arg1)->BiHzFreq >= ((struct myHzPair *) arg2)->BiHzFreq)
		return -1;
	else 
		return 1;
}

void CMainFrame::OnHZPairs() 
{
	// TODO: Add your command handler code here
	int n=ProcessFiles("","*.*",HZPairInFile); // 统计成批文件中的双字

	if(n>0) {
		ADD_HANZI = TRUE;
		
		int hzTypeCounter=0,hzTokenCounter=0;
		for(int id=0;id<HZ_NUM;id++) {
			if(HZFreq[id]>0) {
				hzTypeCounter++;
				hzTokenCounter+=HZFreq[id];
			}
		}
		CString msg;
		msg.Format("已统计语料累计共%d字；\n其中不同单字%d个,\n不同双字%d个",hzTokenCounter,hzTypeCounter,hzPairs.GetSize());
		AfxMessageBox(msg);
	}
	else 
		return;

	// 将统计结果输出到文本文件

	FILE *outFile;
	outFile=fopen("bihzfreq_alphabet_order.txt","wt"); // 创建双字字频输出文件，按双字拼音序

	if(outFile==NULL) {
		AfxMessageBox("can't write the file");
		return;
	}
	
	long int BiHzArrayLength = hzPairs.GetSize();
	int id;
	
	// 定义两个指针，然后分配一定空间
	// 用来放myHzPair结构的数据，这两
	// 个指针指向这片空间
	
	myHzPair * hzp, *hzp1; 

	hzp = (myHzPair *) malloc(BiHzArrayLength * sizeof(myHzPair));
	hzp1 = hzp;

	HZPair * hp ;

	for (id=0;id<BiHzArrayLength;id++) {
		hp = (HZPair *)hzPairs.GetAt(id);
		hzp->BiHz[0]=hp->zz[0];
		hzp->BiHz[1]=hp->zz[1];
		hzp->BiHz[2]=hp->zz[2];
		hzp->BiHz[3]=hp->zz[3];
		hp->zz[4]='\0'; // 将数组最后一位置零，避免出现乱字符
		hzp->BiHz[4]='\0';
		hzp->BiHzFreq = hp->freq;
		hzp ++;
		fprintf(outFile,"%s:\t%d\n",hp->zz,hp->freq);
	}

	fprintf(outFile,"\n双字总数为:%d",BiHzArrayLength);
	fclose(outFile);

	outFile=fopen("bihzfreq_frequency_order.txt","wt");
	
	qsort(hzp1, BiHzArrayLength, sizeof(myHzPair), compare);

	for (id=0;id<BiHzArrayLength;id++) {		
		fprintf(outFile,"%s:\t%d\n",hzp1->BiHz,hzp1->BiHzFreq);
		hzp1++;
	}

	fprintf(outFile,"\n双字总数为:%d",BiHzArrayLength);
	fclose(outFile);
	
}