doctokenizer.java

一个简单的中文分词器,java语言描述

/*
 * DocTokenizer.java
 *
 * Created on 2007年1月17日, 下午10:29
 *
 * To change this template, choose Tools | Template Manager
 * and open the template in the editor.
 */

package util.word;

import java.io.*;


/**
 *
 * @author JinfengLee
 */
public class DocTokenizer implements Scanner {
    
    /** Creates a new instance of DocTokenizer */
    
    public DocTokenizer(String dictionaryPath) {
        this.dictionaryPath = dictionaryPath;
    }
    
    /** Stores the path of the input file */
    private File docPath;
    
    private String dictionaryPath;
    
    /** Stores the produced tokens */
    protected TokenStream ts;
    
    /** Stores the next token */
    protected Token token;
    
    /** Stores the next token returned */
    protected String nextToken =  null;
      
    /**
     * Get the file path 
     *@return pathname
     */
    public String getDocPath(){
        return this.docPath.toString();
    }
    
    /**
     *
     */
    private void docTokenizer() throws Exception {
        InputStreamReader in = new InputStreamReader(new FileInputStream(docPath),"UTF-8");
        Analyzer analyzer = new MMChineseAnalyzer();
        ts = analyzer.tokenStream(dictionaryPath,in);
        //return analyzer.tokenStream("",in);
    }
    /**
     * Get the next Token
     *@return nextToken
     */
    public String nextToken() {
        try{
            token = ts.next();
            if(token != null)
                return token.termText();
            else
                return null;
        }
        catch (IOException ioe){
            return null;
        }
    }
    
    /**
     * Set the file to be analysised
     *@param file
     *@return true we consider set file always true
     */
    public boolean setFile(File file) {        
        this.docPath = file ;
        try{
            this.docTokenizer();
            return true;
        }
        catch (Exception ignore){
            return false;
        }
    }
}