dicttreetokenfilter.java

这是关于中文分词的有关程序

/*
 * Copyright 2002-2005 the original author or authors.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/*
 * Created on 2005-12-28
 * author 谢骋超
 * 
 */
package cn.edu.zju.dartsplitter.analysis;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;

import org.apache.log4j.Logger;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;

import cn.edu.zju.dartsplitter.Splitter;

/**
 * 将一个SimpleAnalyzer分下来的Token(一个句子)分解成词,存放在一个队列中
 * 同时还有过滤功能,例如可配置: <span>...</span>中间的内容不被索引
 * @author xiecc
 * @email [e-mail] [e-mail]
 * homepage：  http://blog.itpub.net/xiecc
 * projectpage: http://ccnt.zju.edu.cn/projects
 */
public class DictTreeTokenFilter extends TokenFilter {
    
    /**
     * Logger for this class
     */
    private static final Logger logger = Logger
            .getLogger(DictTreeTokenFilter.class);

    private boolean tagFiltered;

    private List<String> filterTagList = new ArrayList<String>();
    
    private Splitter splitter;
    
    private Queue<Token> tokenQueue=new LinkedBlockingQueue<Token>();//分词时,将一个句子的分完后保存在queue中,下一个next只在这个queue中取,而不用input.next

    /**
     * @return Returns the tagFiltered.
     */
    public boolean isTagFiltered() {
        return tagFiltered;
    }

    /**
     * @param tagFiltered
     *            The tagFiltered to set.
     */
    public void setTagFiltered(boolean tagFiltered) {
        this.tagFiltered = tagFiltered;
    }

    public DictTreeTokenFilter(TokenStream in,Splitter splitter,List<String> filterTagList) {
        super(in); 
        this.filterTagList=filterTagList;
        this.splitter=splitter;
    }

    private boolean isFilterTag(Token t) {
        if (null == t) {
            return false;
        }
        return filterTagList.contains(t.termText());
    }

    @Override
    public Token next() throws IOException {
        Token t=this.tokenQueue.poll();
        //logger.debug(" tokenQueue size is: "+this.tokenQueue.size());
        if (null!=t){
            //logger.debug(" chinese token is: "+t.termText()+" "+t.startOffset()+"->"+t.endOffset());
            return t;
        }        
        
        while (true){
          t = getNextAvailableToken();
          if (t==null){
              return null;
          }
          this.tokenQueue=this.splitter.split(t);
          if (this.tokenQueue.size()>0){
              break;
          }
        }
        
        return this.tokenQueue.poll();
    }

    private Token getNextAvailableToken() throws IOException {        
        Token t = input.next();
        if (null == t) {
            return null;
        }

        int times = 0;
        if (isFilterTag(t)) {
            do {
                t = input.next();
                times++;
            } while (!isFilterTag(t) && times < 3);// 滤掉<span></span>中间的内容,直到遇到下一个span,否则一直滤掉
            return getNextAvailableToken();
        }
        else{
            return t;
        }
    }


}