indexerqueue.cs

小型搜索软件的源代码

using System;
using System.Collections;
using System.IO;
using System.Threading;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.Search;
using ShootSearch.Plugin;
using ShootSearch.Indexing;
using ShootSearch.Logging;
using Directory = Lucene.Net.Store.Directory;

namespace ShootSearch.Indexing
{
	/// <summary>
	/// IndexerQueue 的摘要说明。
	/// </summary>
	public class IndexerQueue
	{

		private static ArrayList items = ArrayList.Synchronized(new ArrayList());
		
		private static bool isRunning = false;
		private static object runningLock = new object();

		private static bool shouldStop = false;
		private static bool dirty = false;

		private static Directory instanceDirectory;
		private static Analyzer defaultAnalyzer;

		private static Thread indexerThread;
		
		private static Hashtable plugins = new Hashtable();

		public static Hashtable Plugins
		{
			get
			{
				return plugins;
			}
		}
//
//		public static void Add(string path)
//		{
//			Add(path, true);
//		}

//		public static void Add(string path, bool startIndexing)
//		{
//			path = path.ToLower();
//
//			lock (items.SyncRoot)
//			{
//				if (!items.Contains(path))
//					items.Add(path);
//			}
//
//			//			Log.Echo(path + " added to the queue");
//			//			Log.Echo(items.Count);
//
//			if (startIndexing)
//				Start();
//
//		}

		public static void Stop()
		{
			shouldStop = true;
		}

		public static void WaitUntilStopped()
		{
			while (isRunning)
				Thread.Sleep(10);
		}

//		public static void Start()
//		{
//			if (instanceDirectory == null)
//				throw new ApplicationException("You must initialize the queue first by calling Init().");
//
//			lock (runningLock)
//			{
//				if (!isRunning)
//				{
//					indexerThread = new Thread(new ThreadStart(Run));
//					indexerThread.Name = "Indexer";
//					indexerThread.Start();
//				}
//			}
//		}

		public static bool IsRunning
		{
			get
			{
				return isRunning;
			}
		}

//		public static void Run()
//		{
//			// only one thread will process the tasks queue
//			lock (runningLock)
//			{
//				if (isRunning)
//					return;
//				isRunning = true;
//			}
//
//			Log.Echo("indexer queue started");
//
//			shouldStop = false;
//			dirty = false;
//
//			string nextPath = next();
//			while (!shouldStop)
//			{
//				//				Log.Echo("NEXT taken from the queue");
//
//				if (nextPath != null)
//				{
//
//					string normalizedPath = nextPath.Replace("/", "\\").ToLower();
//					try
//					{
//						// it's a directory - that means we need to expand it into the queue
//						if (System.IO.Directory.Exists(normalizedPath))
//						{
//							updateDirectoryRecursively(new DirectoryInfo(normalizedPath), 0);
//						}
//						else if (System.IO.File.Exists(normalizedPath))
//						{
//							updateDocument(normalizedPath);
//						}
//							// deleted file or directory
//						else
//						{
//							// delete all existing document with this name
//							deleteDocuments(normalizedPath);
//
//							// it might have been a directory
//							deleteDirectory(normalizedPath); 
//						}
//					}
//					catch (Exception e)
//					{
//						Log.Echo("FAIL " + nextPath);
//						Log.Debug(e);
//					}
//
//
//					// remove it from the list
//					lock (items.SyncRoot) 
//					{
//						items.Remove(nextPath);
//					}
//
//				}
//					// nothing to do, let the processor do something else
//				else
//				{
//					if (dirty)
//					{
//						optimizeIndex();
//						dirty = false;						
//					}
//					else
//					{
//						Thread.Sleep(100);
//					}
//				}
//
//				// try to take a next item
//				nextPath = next();
//			}
//
//			isRunning = false;
//		}

//		private static void updateDocument(string path)
//		{
//			FileInfo fi = new FileInfo(path);
//
//			// updates are expensive - proceed only if the file is not up-to-date
//			if (isInIndex(fi))
//				return;
//
//			// delete all existing document with this name
//			deleteDocuments(fi.FullName);
//
//			// add the document again
//			addDocument(fi);
//
//			// the index has been updated
//			dirty = true;
//
//		}

//		private static void addDocument(FileInfo fi)
//		{
//			SeekafileDocument ld = new SeekafileDocument(fi);
//
//			IndexWriter writer = new IndexWriter(instanceDirectory, defaultAnalyzer, false);
//			writer.SetUseCompoundFile(true);
//
//			try 
//			{
//				if (ld.Analyzer != null)
//					writer.AddDocument(ld.LuceneDocument, ld.Analyzer);
//				else
//					writer.AddDocument(ld.LuceneDocument);
//			}
//			finally
//			{
//				if (writer != null)
//					writer.Close();
//			}
//
//			Log.Echo("OK " + fi.FullName);
//		}
//
//		private static void addDirectory(DirectoryInfo di)
//		{
//			SeekafileDirectory ld = new SeekafileDirectory(di);
//
//			IndexWriter writer = new IndexWriter(instanceDirectory, defaultAnalyzer, false);
//			writer.SetUseCompoundFile(true);
//
//			try 
//			{
//				writer.AddDocument(ld.LuceneDocument);
//			}
//			finally
//			{
//				if (writer != null)
//					writer.Close();
//			}
//
//			Log.Echo("OK (dir) " + di.FullName);
//		}


		private static bool isInIndex(FileSystemInfo fsi)
		{
			IndexSearcher searcher = new IndexSearcher(instanceDirectory);

			try 
			{

				BooleanQuery bq = new BooleanQuery();
				bq.Add(new TermQuery(new Term("fullname", fsi.FullName)), true, false);
				
				FileInfo fi = fsi as FileInfo;
				if (fi != null)
					bq.Add(new TermQuery(new Term("length", fi.Length.ToString())), true, false);

				bq.Add(new TermQuery(new Term("created", DateField.DateToString(fsi.CreationTime))), true, false);
				bq.Add(new TermQuery(new Term("modified", DateField.DateToString(fsi.LastWriteTime))), true, false);

				Hits hits = searcher.Search(bq);
				int count = hits.Length();

				//				Log.Echo("isInIndex(" + fsi.FullName + "): " + count);

				return count == 1;
			}
			finally
			{
				searcher.Close();
			}
		}

//		private static void updateDirectoryRecursively(DirectoryInfo di, int level)
//		{
//			updateDirectory(di);
//
//			updateDeletedContent(di);
//
//			foreach (FileInfo f in di.GetFiles())
//			{
//				Add(f.FullName, false);
//			}
//
//			foreach (DirectoryInfo d in di.GetDirectories())
//			{
//				updateDirectoryRecursively(d, level + 1);
//			}
//
//		}
//
//		private static void updateDirectory(DirectoryInfo di)
//		{
//			// updates are expensive - proceed only if the file is not up-to-date
//			if (isInIndex(di))
//				return;
//
//			//			Log.Echo("UPDATING (dir) " + di.FullName);
//	
//			// delete all existing document with this name
//			deleteDocuments(di.FullName);
//	
//			// add the document again
//			addDirectory(di);
//	
//			// the index has been updated
//			dirty = true;
//		}

		private static void updateDeletedContent(DirectoryInfo di)
		{
			IndexSearcher s = new IndexSearcher(instanceDirectory);
			Query q = new TermQuery(new Term("directparent", di.FullName.ToLower()));
			Hits hits = s.Search(q);
	
			for (int i = 0; i < hits.Length(); i++) 
			{
				// get the document from index
				Document doc = hits.Doc(i);

				// create a new row with the result data
				string path = doc.Get("fullname");
				if (!File.Exists(path) && !System.IO.Directory.Exists(path))
				{
					deleteDirectory(path);
					deleteDocuments(path);
				}
			}
			s.Close();
		}

		private static string next()
		{
			string task = null;

			lock (items.SyncRoot)
			{
				if (items.Count > 0)
					task = (string) items[0];
			}

			return task;
		}

		public static void Init(Directory directory)
		{
			Init(directory, new StandardAnalyzer());
		}

		public static void Init(Directory directory, Analyzer analyzer)
		{
			if (IsRunning)
				throw new ApplicationException("The IndexerQueue cannot be initialized when it is running.");

			instanceDirectory = directory;
			defaultAnalyzer = analyzer;
		}

		public static void AddParserPlugin(IParserPlugin plugin)
		{
			//			Log.Echo("Adding plugin for: " + plugin.Extensions[0]);
			if (IsRunning)
				throw new ApplicationException("The plugin cannot be added when IndexerQueue is running.");

			foreach (string extension in plugin.Extensions)
			{
				if (!extension.StartsWith("."))
				{
					Log.Debug("Parser not loaded. Extension '" + extension + "' must start with a dot.");
					continue;
				}
				string extLower = extension.ToLower();
				if (!plugins.ContainsKey(extLower))
					plugins[extLower] = plugin;
			}
		}


		private static void deleteDocuments(string fullName)
		{
			IndexReader r = IndexReader.Open(instanceDirectory);
			try
			{
				int deletedCount = r.Delete(new Term("fullname", fullName));
				if (deletedCount > 0)
				{
					dirty = true;
					Log.Echo("DELETED: " + fullName);
				}
			}
			finally
			{
				r.Close();
			}
		}

		private static void deleteDirectory(string fullName)
		{
			IndexReader r = IndexReader.Open(instanceDirectory);

			try
			{
				int deletedCount = r.Delete(new Term("parent", fullName));
				if (deletedCount > 0)
				{
					dirty = true;
					Log.Echo("DELETED (dir): " + deletedCount + " " + fullName);
				}
			}
			finally
			{
				r.Close();
			}
		}


		private static void optimizeIndex()
		{
			DateTime start = DateTime.Now;
			IndexWriter writer = new IndexWriter(instanceDirectory, defaultAnalyzer, false);
			writer.SetUseCompoundFile(true);
			try
			{
				writer.Optimize();
			}
			finally
			{
				writer.Close();
			}
			Log.Echo("OPTIMIZATION Indexing finished (" + (DateTime.Now - start) + ")");
		}
	}
}