word_pos.prg

中文词性标注子程序第一步

set talk off
close all
clear all
clear
set exact on


&&创建三个数据表，并建立索引
select 1
create table 词性标记集(pos C(4),freq N(10),is_open L(1))
index on pos tag sy1
select 2
create table 标记同现集(pos1 C(4),pos2 C(4),freq N(10))
index on alltrim(pos1)+alltrim(pos2) tag sy2
select 3
create table 词语标记集(word C(10),pos C(4),freq N(10))
index on alltrim(word)+alltrim(pos) tag sy3

new_dir=getdir("txt","请选择训练用的标注语料")
if empty(new_dir)
	?"未选择训练语料！"
	return
endif
cd (new_dir)
t1=second()
do process_files with "train_a_file","*.txt"
t2=second()
?"程序运行时间:",(t2-t1)
return
***************************************************************************
procedure process_files
 && 成批处理文件的子程序
parameters proc_name,file_names
&& 欲调用的子程序名字、文件名
private all
fname=sys(2000,file_names)
do while !empty(fname)
	do &proc_name with fname
	fname=sys(2000,file_names,1)
enddo
return
*****************************************************************
procedure  train_a_file
&&处理每一个标注文件
parameters fname
private all
&&?"训练语料:"fname
fp=fopen(fname)
if fp<0
	? "无法打开输入文件："+fname
	return
else
	?"    语料 "+fname+" 处理完毕"
endif
t0=""
open_pos="A l n ns Nt t V"
do while !feof(fp)
	s=alltrim(fread(fp,100000))+"  "
	do while !empty(s)
		do process_lines with s
	enddo
enddo
=fclose(fp)
return 
*********************************************************************************************
procedure process_lines
parameters s
		i=at_c("/",s)
		w=leftc(s,i-1)                                 &&词串
		s=ltrim(stuffc(s,1,i,""))
		j=at_c("  ",s)
		t=leftc(s,j-1)                                 &&词性标志
		s=ltrim(stuffc(s,1,j,""))
		
		select 1 
		do put_pos with t
		select 2                                    
		do put_pos_pos with t0,t
		select 3                                      
		do put_word_pos with t,w
		t0=t
return 
******************************************************************************************************
procedure put_pos
 &&向词性标志集中添加词性及其频率
parameters t
	if !empty(t)
		if seek(t) 
			replace freq with freq+1
		else
			append blank
			replace pos with t,freq with 1
			if atc(t,open_pos)>0
				replace is_open with .t.	
			else
				replace is_open with .f.
			endif
		endif
	endif
return 	
**********************************************************************************************************
procedure put_pos_pos
 &&为标记同现集中添加t0和t及其频率
parameters t0,t				
	if !empty(t0) and !empty(t)
		if seek(alltrim(t0)+alltrim(t))
			replace freq with freq+1
		else
			append blank
			replace pos1 with t0,pos2 with t,freq with 1
		endif
	endif
return 	
****************************************************************************************************
procedure put_word_pos
&&在词语标记表中为w和t添加频率
parameters t,w
	if !empty(t) and !empty(w)
		if seek(alltrim(w)+alltrim(t))
			replace freq with freq+1
		else
			append blank
			replace word with w,pos with t,freq with 1
		endif
	endif
return