cwordtokenizer.java~

「我是中國人」

package org.apache.lucene.analysis.cw;

import java.io.Reader;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.standard.ParseException;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import java.lang.*;
import java.io.*;
import java.util.*;
import java.util.logging.*;

/**
 * Title: CWordTokenizer
 * Description: 
 * Company:
 *
 * @author 
 * @version 1.0
 *
 */

public final class CWordTokenizer extends Tokenizer {
    public static Segmenter mainsegmenter;
    private static Logger logger = Logger.getLogger(CWordTokenizer.class.getName());
    public TokenStream tokenizer;
    
    private Reader input;
    
    public CWordTokenizer(Reader in) {
        logger.info("Initializing sementer ... ");
        mainsegmenter = Segmenter.getSegmenter("zword.obj");
        logger.info("sementer initialized. ");
        
        BufferedReader bin = new BufferedReader(in);
        StringBuffer buffer = new StringBuffer();
        String dataline;
        String processed;
        logger.info("reading src inputstream ");
        try{
            while ((dataline = bin.readLine()) != null){
                
                processed = mainsegmenter.segmentLine(dataline, " ");
                logger.info("process line: " + dataline);
                logger.info("processed line: " + processed);
                buffer.append(processed + "\n");
            }
        }catch(IOException ioe){
            // ignored
        }finally{
            try{
                bin.close();
                in.close();
            }catch(Exception e){}
        }
        
        logger.info("create StandardAnalyzer and pass the job to it ");
        input = new StringReader(buffer.toString());
        
        tokenizer = new CStandardTokenizer(input);
    }
    
    public final void close() throws IOException{
        tokenizer.close();
    }

    public final Token next() throws IOException {
        return tokenizer.next();
    }
}