tokeniser.cs

用java语言实现文本聚类

/*
Tokenization
Author: Thanh Ngoc Dao - [e-mail]
Copyright (c) 2005 by Thanh Ngoc Dao.
*/

using System;
using System.Collections;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using WawaSoft.Search.Common;


namespace WawaSoft.Search.Common
{
	/// <summary>
	/// Summary description for Tokeniser.
	/// Partition string into SUBwords
	/// </summary>
	internal class Tokeniser : ITokeniser
	{

        /// <summary>
        /// 以空白字符进行简单分词，并忽略大小写，
        /// 实际情况中可以用其它中文分词算法
        /// </summary>
        /// <param name="input"></param>
        /// <returns></returns>
		public IList<string> Partition(string input)
		{
			Regex r=new Regex("([ \\t{}():;. \n])"); 	
			input=input.ToLower() ;

			String [] tokens=r.Split(input); 									

			List<string> filter=new  List<string>() ;

			for (int i=0; i < tokens.Length ; i++)
			{
				MatchCollection mc=r.Matches(tokens[i]);
				if (mc.Count <= 0 && tokens[i].Trim().Length > 0 					 
					&& !StopWordsHandler.IsStopword (tokens[i]) )								
					filter.Add(tokens[i]) ;
            }
			
			return filter.ToArray();
		}


		public Tokeniser()
		{
		}

    }
}