wf.py

中文自然語言處理相關程式

import codecs, re

f1=codecs.open(r"./T0263_utf8.txt", "r", "utf8")
f2=codecs.open(r"./T0263_wfrq.txt", "w", "utf8")

l=f1.read()
f1.close()

d={}
pattern=re.compile(r"(\[[^]]*\]|.)")
sl=pattern.findall(l)
i=0
for k in sl:
	i=i+1
	if k in d:
		d[k]=d[k]+1
	else:
		d[k]=1

wf=d.items()
wf.sort(lambda x, y : cmp(x[1],y[1]))    
wf.reverse()

print>>f2, 'num=',i
for w in wf:
	word=w[0]
	freq=w[1]
	print>>f2, word, freq
	
f2.close()