doctokenizer.java

来自「本人自己写的机遇lucene的简单搜速引擎」· Java 代码 · 共 62 行

/*
 * DocTokenizer.java
 *
 * Created on 2007年1月17日, 下午10:29
 *
 * To change this template, choose Tools | Template Manager
 * and open the template in the editor.
 */

package util.word;

import java.io.*;

/**
 *
 * @author JinfengLee
 */
public class DocTokenizer {
    /** Stores the produced tokens */
    protected TokenStream ts;
    /** Stores the analyzer */
    protected MMChineseAnalyzer analyzer;
    
    /** Creates a new instance of DocTokenizer */
    
    public DocTokenizer(String dictionaryPath) {
        analyzer = new MMChineseAnalyzer(dictionaryPath);
    }
 
    /**
     * Get the next Token
     *@return nextToken
     */
    public String nextToken() {
        try{
            Token token = ts.next();
            if(token != null)
                return token.termText();
            else
                return null;
        }
        catch (IOException ioe){
            return null;
        }
    }
    
    /**
     * Set the file to be analysised
     *@param file
     *@return true we consider set file always true
     */
    public boolean setReader(Reader rd) {
        try{
            ts = analyzer.tokenStream(rd);
            return true;
        }
        catch (Exception ex){
            return false;
        }
    }
}