tokeniser.cs

In the previous article, we presented an approach for capturing similarity between words that was co

/*
*/

using System;
using System.Collections;
using System.Text.RegularExpressions;


namespace ServiceRanking
{
	/// <summary>
	/// Summary description for Tokeniser.
	/// Partition string off into subwords
	/// </summary>
	public class Tokeniser
	{

        public static string[] ArrayListToArray(ArrayList arraylist)
		{
			return (string[])arraylist.ToArray (typeof(string));
		}

		void Normalize_Casing(ref string input)
		{
			for(int i=0; i < input.Length; i++)
			{
				if (Char.IsSeparator(input[i]))
					input=input.Replace(input[i].ToString() , " ") ;
			}
			int idx=1;
			while (idx < input.Length - 2)
			{
				++idx;								
				if (
					(Char.IsUpper(input[idx])   
					&& Char.IsLower(input[idx + 1]))
					&& 
					(!Char.IsWhiteSpace(input[idx - 1]) && !Char.IsSeparator(input[idx - 1]) )
					)
				{
					input=input.Insert(idx, " ") ; 
					++idx;
				}
			}
		}

		public string[] Partition(string input)
		{
			Regex r=new Regex("([ \\t{}():;.,!?_/ \n])");
			
			Normalize_Casing(ref input);		
			input=input.ToLower() ;

			String [] tokens=r.Split(input); 									

			ArrayList filter=new ArrayList() ;
   

			for (int i=0; i < tokens.Length ; i++)
			{
				MatchCollection mc=r.Matches(tokens[i]);
				if (mc.Count <= 0 && tokens[i].Trim().Length > 0 					 
					&& !StopWordsHandler.IsStopword (tokens[i]) )								
					filter.Add(tokens[i]) ;
				
				
			}

            StemmerInterface istem = new PorterStemmer();
			tokens=new string[filter.Count] ;

            for (int i = 0; i < filter.Count; i++)
            {                
                string stem=istem.stemTerm ((string)filter[i]);
                tokens[i] = stem;
            }
			
			return tokens;
		}


		public Tokeniser()
		{
            StopWordsHandler stop = new StopWordsHandler();
		}
	}
}