dicttreeanalyzer.java

这是关于中文分词的有关程序

/*
 * Copyright 2002-2005 the original author or authors.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/*
 * Created on 2005-12-28
 * author 谢骋超
 * 
 */
package cn.edu.zju.dartsplitter.analysis;

import java.io.Reader;
import java.util.ArrayList;
import java.util.List;

import junit.framework.Assert;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LetterTokenizer;
import org.apache.lucene.analysis.TokenStream;

import cn.edu.zju.dartsplitter.Splitter;

/**
 * 分词系统的Analyzer，也是整个包里唯一需要与外部打交道的类， 给lucene用的，
 * 由于该Analyzer用了Spring里的配置，因此调用它的类必须也配置在Spring里，而不能用 new DictTreeAnalyzer()
 * 否则我们只能用ApplicationContext.getBean来得到了。
 * 
 * @author xiecc
 * @email [e-mail] [e-mail]
 * homepage：  http://blog.itpub.net/xiecc
 * project page: http://ccnt.zju.edu.cn/projects
 *
 */
public class DictTreeAnalyzer extends Analyzer{
    private List<String> filterTagList = new ArrayList<String>();
    private Splitter splitter;
    
    /**
     * @return Returns the splitter.
     */
    public Splitter getSplitter() {
        return splitter;
    }

    /**
     * @param splitter The splitter to set.
     */
    public void setSplitter(Splitter splitter) {
        this.splitter = splitter;
    }

    /**
     * @return Returns the filterTagList.
     */
    public List<String> getFilterTagList() {
        return filterTagList;
    }

    /**
     * @param filterTagList The filterTagList to set.
     */
    public void setFilterTagList(List<String> filterTagList) {
        this.filterTagList = filterTagList;
    }


/**
 * 调用DictTreeTokenFilter来完成Analyzer的核心任务
 */
    public TokenStream tokenStream(String fieldName, Reader reader) {
        Assert.assertNotNull(splitter);
        return new DictTreeTokenFilter(new LetterTokenizer(reader),splitter,filterTagList);
      }
}