evaluation.cs

用C#编写的一个款搜索engine的源代码！摘自<Visual c#2005 程序设计>

using System;
using System.IO;
using System.Collections;
using System.Threading;
using System.Text;
using SECompare.Kernel;
using SECompare.Structure;

namespace SECompare.Evaluation
{
	public enum Methods{Coverage,P,MAP,RP,Rank1R,CI,SP};

	/// <summary>
	/// Sequence Evaluation.
	/// Given two engines:
	/// 1) Peer Engine: The engine to be evaluated. 
    /// For each query, top Np records are stored.
	/// 2) Reference Engine: Suppose it has a perfect rank database. 
    /// For each query, top Nr records are stored. 
	/// Compute sequence score of Peer Engine, comparing to Reference Engine.
	/// 
	/// Following Directory Rule is adopted:
	/// ----[Sequence Evaluation Root]
	///   |_______[RefXXXX_PeerYYYY]  (this.m_FileRoot)
	///       |________Overall_RefXXXX_PeerYYYY.html
	///       |________Detail_RefXXXX10_PeerYYYY20.html
	///       |________Detail_RefXXXX10_PeerYYYY30.html
	///       |________Detail_Ref.......... 
	/// </summary>
	public class Evaluation:ThreadStatus
	{
		private String  m_FileRoot;
		private String  m_RefEngine;
		private String  m_PeerEngine;
		private Methods m_Method; 
		private int[]   m_CheckPoint_Ref  = {10,20,30,40,50,60,70,80,90,100};
		private int[]   m_CheckPoint_Peer = {10,20,30,40,50,60,70,80,90,100};

		/// <summary>
		/// Constructor.
		/// </summary>
		/// <param name="Directory">Directory Path to store generated evaluation reports.</param>
		/// <param name="RefEngine">Reference Engine Name</param>
		/// <param name="PeerEngine">Peer Engine Name</param>
		public Evaluation(String Directory,String RefEngine,String PeerEngine,Methods Method)
		{
			this.m_FileRoot   = Directory+"\\Ref"+RefEngine+"_Peer"+PeerEngine;
			this.m_RefEngine  = RefEngine;
			this.m_PeerEngine = PeerEngine;
			this.m_Method     = Method;
			this.threadStart = new ThreadStart(Run);
		}

		public Evaluation(String Directory,String RefEngine,String PeerEngine,Methods Method,int[] CheckPoint_Ref,int[] CheckPoint_Peer)
		{
			this.m_FileRoot   = Directory+"\\Ref"+RefEngine+"_Peer"+PeerEngine;
			this.m_RefEngine  = RefEngine;
			this.m_PeerEngine = PeerEngine;
			this.m_Method     = Method;
			this.m_CheckPoint_Peer = CheckPoint_Peer;
			this.m_CheckPoint_Ref  = CheckPoint_Ref;
			this.threadStart = new ThreadStart(Run);
		}


		/// <summary>
		/// Get full name of evaluation methods.
		/// </summary>
		/// <param name="Method">Enum of method</param>
		/// <returns>Method full name</returns>
		public static String GetMethodName(Methods Method)
		{
			switch(Method)
			{
				case Methods.Coverage:
					return "Coverage";
				case Methods.P:
					return "Precision";
				case Methods.MAP:
					return "Mean Average Precision";
				case Methods.RP:
					return "Recall Precision";
				case Methods.Rank1R:
					return "Rank 1st Relevant";
				case Methods.CI:
					return "Cumulated Index";
				case Methods.SP:
					return "Sequence Precision";
				default:
					return "";
			}
		}

		/// <summary>
		/// Sequence Evaluation.
		/// </summary>
		/// <param name="arrRank">Array of Ranks</param>
		/// <param name="RefLength">Reference List Length</param>
		/// <param name="PeerLength">Peer List Length</param>
		/// <returns>Sequence Score</returns>
		private float Evaluate(int[] arrRank,int RefLength,int PeerLength)
		{
			//Assure RefLength not exceed the range
			if(arrRank.Length<(RefLength+1))
				return Evaluate(arrRank,arrRank.Length-1,PeerLength);
			//Compute Sequence Score
			float SeqScore = 0;
			switch(this.m_Method)
			{
				case Methods.Coverage:
					SeqScore = this.SeqScore_Coverage(arrRank,RefLength,PeerLength);
					break;
				case Methods.P:
					SeqScore = this.SeqScore_P(arrRank,RefLength,PeerLength);
					break;
				case Methods.MAP:
					SeqScore = this.SeqScore_MAP(arrRank,RefLength,PeerLength);
					break;
				case Methods.RP:
					SeqScore = this.SeqScore_RP(arrRank,RefLength,PeerLength);
					break;
				case Methods.Rank1R:
					SeqScore = this.SeqScore_Rank1R(arrRank,RefLength,PeerLength);
					break;
				case Methods.CI:
					SeqScore = this.SeqScore_CI(arrRank,RefLength,PeerLength);
					break;
				case Methods.SP:
					SeqScore = this.SeqScore_SP(arrRank,RefLength,PeerLength);
					break;
			}

			return SeqScore;
		}

		/// <summary>
		/// Coverage Metric.
		/// </summary>
		/// <param name="arrRank"></param>
		/// <param name="RefLength"></param>
		/// <param name="PeerLength"></param>
		/// <returns>Score.</returns>
		private float SeqScore_Coverage(int[] arrRank,int RefLength,int PeerLength)
		{
			int count = 0;
			for(int i=1;i<=RefLength;i++)
			{
				if(arrRank[i]>0&&arrRank[i]<=PeerLength) 
					count++;
			}

			return (float)count/RefLength;
		}

		/// <summary>
		/// Precision Metric.
		/// </summary>
		/// <param name="arrRank"></param>
		/// <param name="RefLength"></param>
		/// <param name="PeerLength"></param>
		/// <returns>Score.</returns>
		private float SeqScore_P(int[] arrRank,int RefLength,int PeerLength)
		{
			int count = 0;
			for(int i=1;i<=RefLength;i++)
			{
				if(arrRank[i]>0&&arrRank[i]<=PeerLength) 
					count++;
			}

			return (float)count/PeerLength;
		}

		/// <summary>
		/// Mean Average Precision.
		/// </summary>
		/// <param name="arrRank"></param>
		/// <param name="RefLength"></param>
		/// <param name="PeerLength"></param>
		/// <returns>Score.</returns>
		private float SeqScore_MAP(int[] arrRank,int RefLength,int PeerLength)
		{
			int count = 0;   //Count of correct links
			float TP  = 0;   //Total Precision
			for(int i=1;i<=RefLength;i++)
			{
				//if it is found
				if(arrRank[i]>0&&arrRank[i]<=PeerLength) 
				{
					count++;
					TP += (float)count/i;
				}
			}

			if(count==0)
				return 0;

			return TP/count;
		}

		/// <summary>
		/// Recall-Precision.
		/// </summary>
		/// <param name="arrRank"></param>
		/// <param name="RefLength"></param>
		/// <param name="PeerLength"></param>
		/// <returns>Score.</returns>
		private float SeqScore_RP(int[] arrRank,int RefLength,int PeerLength)
		{
			int MaxRank = 0; 
			for(int i=1;i<=RefLength;i++)
			{
				//if anyone is not found, return 0
				if(arrRank[i]<=0||arrRank[i]>PeerLength)
					return 0;
				//else, if it is found
				if(arrRank[i]>MaxRank) 
				{
					MaxRank = arrRank[i];
				}
			}

			return (float)RefLength/MaxRank;
		}
		
		/// <summary>
		/// Rank of 1st Relevant.
		/// </summary>
		/// <param name="arrRank"></param>
		/// <param name="RefLength"></param>
		/// <param name="PeerLength"></param>
		/// <returns>Rank of 1st relevant record.If no peer record is found in Ref List, return NaN</returns>
		private float SeqScore_Rank1R(int[] arrRank,int RefLength,int PeerLength)
		{
			int FirstRelevant = PeerLength+1;
			for(int i=1;i<RefLength;i++)
			{
				if(arrRank[i]>0&&arrRank[i]<=PeerLength&&arrRank[i]<FirstRelevant) 
					FirstRelevant = arrRank[i];
			}

			if(FirstRelevant == (PeerLength+1))
				return float.NaN;

			return FirstRelevant;
		}

		private float SeqScore_CI(int[] arrRank, int RefLength, int PeerLength)
		{
			//Calculate Sequence Score
			//1.  Compute RefList ideal score
			int RefScore = RefLength*(RefLength-1)/2;
			//2.  Compute PeerList score
			int PeerScore = 0;
			for(int i=1;i<RefLength;i++)
			{
				if(arrRank[i]>0&&arrRank[i]<=PeerLength) 
					PeerScore += arrRank[i];
				else
					PeerScore += 1000;  //not found, give a large score (positive infinite)
			}
			//3.  Compute Sequence Score
			float SeqScore = (float)RefScore/PeerScore;

			return SeqScore;
		}

		private float SeqScore_SP(int[] arrRank, int RefLength, int PeerLength)
		{
			//Calculate Sequence Score
			//1.  Compute Correct[]
			int[] correct = new int[RefLength+1];
			correct[0] = 0;
			for(int i=1;i<correct.Length;i++)
			{
				if(arrRank[i]>0&&arrRank[i]<=i)
					correct[i] = 1;
				else
					correct[i] = 0;
			}	
			//2.  Compute Cumulated Correct[]
			for(int i=1;i<correct.Length;i++)
			{
				correct[i] += correct[i-1];
			}
			//3.  Compute Sequence Score
			float SeqScore = 0;
			for(int i=1;i<correct.Length;i++)
			{
				SeqScore += (float)correct[i]/i;
			}
			SeqScore = SeqScore/(correct.Length-1);

			return SeqScore;
		}


		/// <summary>
		/// Following Directory is generated:
		/// ----[Sequence Evaluation Root]
		///   |_______[RefXXXX_PeerYYYY]  (this.m_FileRoot)
		///       |________Overall_RefXXXX_PeerYYYY.html
		///       |________Detail_RefXXXX10_PeerYYYY20.html
		///       |________Detail_RefXXXX10_PeerYYYY30.html
		///       |________Detail_Ref.......... 
		/// </summary>
		private void Run()
		{
			//****************Create Directory*****************
			Directory.CreateDirectory(this.m_FileRoot);		
			//****************Create Overall Report************
			this.CreateOverallFiles();
			//****************Create Detail Reports************
			this.CreateDetailFiles();
			//*********Create Missing Domains Reports**********
			ArrayList MissingDomains  = new ArrayList(500);
			//*************************************************

			//Get all crawled sample files
			String[] files = Directory.GetFiles((new Config.Config()).SamplingedQueryRoot);
			//Overall average
			StringBuilder bufOverall;
			float[,] OverallTotalScore = new float[this.m_CheckPoint_Peer.Length,this.m_CheckPoint_Ref.Length];
			//for each category file
			for(int i=0;i<files.Length;i++)
			{
				//define data structure to store results
				float[,]  TotalScore = new float[this.m_CheckPoint_Peer.Length,this.m_CheckPoint_Ref.Length];
				int[,]    TotalCount = new int[this.m_CheckPoint_Peer.Length,this.m_CheckPoint_Ref.Length];
				int[,]    TotalZero  = new int[this.m_CheckPoint_Peer.Length,this.m_CheckPoint_Ref.Length];
				ArrayList[,] EntryResults = new ArrayList[this.m_CheckPoint_Peer.Length,this.m_CheckPoint_Ref.Length];
				

				FileInfo file = new FileInfo(files[i]);
				String category = file.Name;

				//************Progress Status Report*****************
				this.mProcessingFile = "Evaluating " + category;
				this.mFilePercentage = (float)i/files.Length; 
				//***************************************************

				CrawlSample sample = new CrawlSample();
				sample.Load(category,file.FullName);
				ArrayList queries = sample.GetSamples();

				//for each query
				for(int q=0;q<queries.Count;q++)
				{
					CrawlSampleUnit query = (CrawlSampleUnit)queries[q];

					//Construct Reference List