ktdictsegtokenizer.cs

KTDictSeg 简介: KTDictSeg 是由KaiToo搜索开发的一款基于字典的简单中英文分词算法 * 主要功能: 中英文分词

/**********************************************
 * 采用KTDictSeg的Lucene.Net 中文分词分析器
 * 参考 suyuan 的开源代码修改
 * suyuan 的开源代码出处 http://www.cnblogs.com/suyuan/archive/2008/03/25/1120827.html
 *********************************************/

using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using KTDictSeg;
using FTAlgorithm;
using Lucene.Net.Analysis;

namespace Lucene.Net.Analysis.KTDictSeg
{
    public class KTDictSegTokenizer : Tokenizer
    {
        private static CSimpleDictSeg m_SimpleDictSeg;

        private List<T_WordInfo> m_WordList = new List<T_WordInfo>();
        private int m_Position = -1; //词汇在缓冲中的位置.


        public KTDictSegTokenizer(System.IO.TextReader input)
            : base(input)
        {

            //这里用了一个第三方的中文分词组件.
            if (m_SimpleDictSeg == null)
            {
                try
                {
                    m_SimpleDictSeg = new CSimpleDictSeg();
                    m_SimpleDictSeg.LoadConfig("KTDictSeg.xml");

                    m_SimpleDictSeg.LoadDict();
                }
                catch (Exception e)
                {
                    m_SimpleDictSeg = null;
                    throw e;
                }
            }

            m_WordList = m_SimpleDictSeg.SegmentToWordInfos(input.ReadToEnd());
        }

        //DotLucene的分词器简单来说，就是实现Tokenizer的Next方法，把分解出来的每一个词构造为一个Token，因为Token是DotLucene分词的基本单位。
        public override Token Next()
        {
            int length = 0;    //词汇的长度.
            int start = 0;     //开始偏移量.

            m_Position++;
            if (m_Position < m_WordList.Count)
            {
                length = m_WordList[m_Position].Word.Length;
                start = m_WordList[m_Position].Position;
                return new Token(m_WordList[m_Position].Word, start, start + length);
            }

            return null;
        }
    }

}