kodak.pl

一个简单的中文分词

#!/usr/bin/perl
use strict;
use warnings;

print "Success!!!\n";

my %thesaurus;
my $thesaurus=".\\thesaurus.txt";#词库
my $test=".\\test.txt";#测试文本	
my $result=".\\result.txt";#结果文本
my $maxlen = 8;
my $p0;#要切分的部分	
my $p1 ;#取词
my $p2;#切分结果
open(FILE1,"$thesaurus")||die "cannot open the file:$!";
open(FILE3,">$result")||die "cannot open the file:$!";
while(<FILE1>)#读入词库，建立哈希表
{
chomp;
$thesaurus{$_}='1';
}
open(FILE2,"$test")||die "cannot open the file:$!";
while(<FILE2>)
{
 $p1 = "";
 $p2 = "";
 chomp;
 $p0 = $_; 
 while (length($p0) > 0 &&length($p0)>=$maxlen ) 
	 {
	
	 $p1 = substr($p0,0,$maxlen);
	 if($p1=~m/(.*?)(\w*).*?/g)#处理数字、英文单词的特殊情况
		 {
		if(length($1)!=0)
			 {
			$p1 = $1;
			 $maxlen = length($p1);
			 }
	   else 
			 {
			$p2 = "$p2$2";			
			$p0 = substr($p0,length($2));
		    $p1 = substr($p0,0,$maxlen);
			 }
		 }
	 while(!exists $thesaurus{$p1}&&length($p1) > 2)#查找词库
		 {
	  $maxlen = $maxlen - 2;
      $p1 = substr($p1,0,$maxlen);#左移一字继续查找        
     			 }           
		   
		   $p2 = "$p2$p1/";
		   $p0 = substr($p0,$maxlen);   #从p0中去掉p1
		   $maxlen = 8;
     }
     print FILE3 "$p2/";
	 print FILE3 "$p0\n";
}
close(FILE1);
close(FILE2);
close(FILE3);