analyzerutils.cs

DotLucentet,用来做撬

using System;
using System.Collections;
using System.IO;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
using NUnit.Framework;
using Token = Lucene.Net.Analysis.Token;
namespace dotLucene.inAction.Analyse
{
	

	public class AnalyzerUtils
	{
		public static Lucene.Net.Analysis.Token[] TokensFromAnalysis(Analyzer analyzer,
			String text)
		{
			TokenStream stream =
				analyzer.TokenStream("contents", new StringReader(text));
			ArrayList tokenList = new ArrayList();
			while (true)
			{
				Token token = stream.Next();
				if (token == null)
				{
					break;
				}

				tokenList.Add(token);
			}

			return (Token[]) tokenList.ToArray(typeof (Token));
		}

		public static void DisplayTokens(Analyzer analyzer,
			String text)
		{
			Token[] tokens = TokensFromAnalysis(analyzer, text);

			for (int i = 0; i < tokens.Length; i++)
			{
				Token token = tokens[i];

				Console.Write("[" + token.TermText() + "] ");
			}

			Console.WriteLine();
		}

		public static void DisplayTokensWithPositions(Analyzer analyzer,
			String text)
		{
			Token[] tokens = TokensFromAnalysis(analyzer, text);

			int position = 0;

			for (int i = 0; i < tokens.Length; i++)
			{
				Token token = tokens[i];

				int increment = token.GetPositionIncrement();

				if (increment > 0)
				{
					position = position + increment;
					Console.WriteLine();
					Console.Write(position + ": ");
				}

				Console.Write("[" + token.TermText() + "] ");
			}
			Console.WriteLine();
		}

		public static void DisplayTokensWithFullDetails(
			Analyzer analyzer, String text)
		{
			Token[] tokens = TokensFromAnalysis(analyzer, text);

			int position = 0;

			for (int i = 0; i < tokens.Length; i++)
			{
				Token token = tokens[i];

				int increment = token.GetPositionIncrement();

				if (increment > 0)
				{
					position = position + increment;
					Console.WriteLine();
					Console.Write(position + ": ");
				}

				Console.Write("[" + token.TermText() + ":" +
					token.StartOffset() + "->" +
					token.EndOffset() + ":" +
					token.Type() + "] ");
			}
			Console.WriteLine();
		}

		public static void assertTokensEqual(Token[] tokens,
			String[] strings)
		{
			Assert.AreEqual(strings.Length, tokens.Length);

			for (int i = 0; i < tokens.Length; i++)
			{
				Assert.AreEqual("index " + i, strings[i], tokens[i].TermText());
			}
		}

		public static void Main(String[] args)
		{
			string text="The quick brown fox jumped over the lazy dogs XY&Z Corporation - [e-mail]";
			if (args.Length >0)
			{
				text = args[0];
			}

		
			Console.Out.WriteLine("The Text to be Analysed : {0}", text);
			Console.WriteLine("SimpleAnalyzer");
			DisplayTokens(new SimpleAnalyzer(), text);

			Console.WriteLine("WhiteSpaceAnalyzer");
			DisplayTokens(new WhitespaceAnalyzer(), text);

			Console.WriteLine("StopAnalyzer");
			DisplayTokens(new StopAnalyzer(), text);

			Console.WriteLine("StandardAnalyzer");
			DisplayTokensWithFullDetails(new StandardAnalyzer(), text);

			Console.ReadLine();
		}
	}
}