mainform.cs

利用VS C#实现的网络爬虫功能

																								this.menuItemCut,
																								this.menuItemCopy,
																								this.menuItemPaste,
																								this.menuItemDelete});
			// 
			// menuItemCut
			// 
			this.menuItemCut.Index = 0;
			this.menuItemCut.Text = "Cu&t";
			this.menuItemCut.Click += new System.EventHandler(this.menuItemCut_Click);
			// 
			// menuItemCopy
			// 
			this.menuItemCopy.Index = 1;
			this.menuItemCopy.Text = "&Copy";
			this.menuItemCopy.Click += new System.EventHandler(this.menuItemCopy_Click);
			// 
			// menuItemPaste
			// 
			this.menuItemPaste.Index = 2;
			this.menuItemPaste.Text = "&Paste";
			this.menuItemPaste.Click += new System.EventHandler(this.menuItemPaste_Click);
			// 
			// menuItemDelete
			// 
			this.menuItemDelete.Index = 3;
			this.menuItemDelete.Text = "&Delete";
			this.menuItemDelete.Click += new System.EventHandler(this.menuItemDelete_Click);
			// 
			// toolBarButton4
			// 
			this.toolBarButton4.Text = "Go";
			// 
			// buttonGo
			// 
			this.buttonGo.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
			this.buttonGo.ImageAlign = System.Drawing.ContentAlignment.MiddleLeft;
			this.buttonGo.ImageIndex = 0;
			this.buttonGo.ImageList = this.imageList1;
			this.buttonGo.Location = new System.Drawing.Point(632, 30);
			this.buttonGo.Name = "buttonGo";
			this.buttonGo.Size = new System.Drawing.Size(40, 22);
			this.buttonGo.TabIndex = 10;
			this.buttonGo.Click += new System.EventHandler(this.buttonGo_Click);
			// 
			// imageList1
			// 
			this.imageList1.ColorDepth = System.Windows.Forms.ColorDepth.Depth32Bit;
			this.imageList1.ImageSize = new System.Drawing.Size(27, 15);
			this.imageList1.ImageStream = ((System.Windows.Forms.ImageListStreamer)(resources.GetObject("imageList1.ImageStream")));
			this.imageList1.TransparentColor = System.Drawing.Color.Teal;
			// 
			// timerMem
			// 
			this.timerMem.Enabled = true;
			this.timerMem.Interval = 2000;
			this.timerMem.Tick += new System.EventHandler(this.timerMem_Tick);
			// 
			// imageListPercentage
			// 
			this.imageListPercentage.ColorDepth = System.Windows.Forms.ColorDepth.Depth32Bit;
			this.imageListPercentage.ImageSize = new System.Drawing.Size(16, 16);
			this.imageListPercentage.ImageStream = ((System.Windows.Forms.ImageListStreamer)(resources.GetObject("imageListPercentage.ImageStream")));
			this.imageListPercentage.TransparentColor = System.Drawing.Color.Transparent;
			// 
			// timerConnectionInfo
			// 
			this.timerConnectionInfo.Enabled = true;
			this.timerConnectionInfo.Interval = 15000;
			this.timerConnectionInfo.Tick += new System.EventHandler(this.timerConnectionInfo_Tick);
			// 
			// CrawlerForm
			// 
			this.AutoScaleBaseSize = new System.Drawing.Size(5, 13);
			this.ClientSize = new System.Drawing.Size(680, 313);
			this.Controls.Add(this.buttonGo);
			this.Controls.Add(this.comboBoxWeb);
			this.Controls.Add(this.tabControlRightView);
			this.Controls.Add(this.toolBarWeb);
			this.Controls.Add(this.statusBar);
			this.Controls.Add(this.toolBarMain);
			this.Icon = ((System.Drawing.Icon)(resources.GetObject("$this.Icon")));
			this.Menu = this.mainMenu;
			this.Name = "CrawlerForm";
			this.Text = "Net Crawler";
			this.Closing += new System.ComponentModel.CancelEventHandler(this.CrawlerForm_Closing);
			this.Load += new System.EventHandler(this.CrawlerForm_Load);
			((System.ComponentModel.ISupportInitialize)(this.statusBarPanelInfo)).EndInit();
			((System.ComponentModel.ISupportInitialize)(this.statusBarPanelURLs)).EndInit();
			((System.ComponentModel.ISupportInitialize)(this.statusBarPanelFiles)).EndInit();
			((System.ComponentModel.ISupportInitialize)(this.statusBarPanelByteCount)).EndInit();
			((System.ComponentModel.ISupportInitialize)(this.statusBarPanelErrors)).EndInit();
			((System.ComponentModel.ISupportInitialize)(this.statusBarPanelCPU)).EndInit();
			((System.ComponentModel.ISupportInitialize)(this.statusBarPanelMem)).EndInit();
			this.tabControlRightView.ResumeLayout(false);
			this.tabPageThreads.ResumeLayout(false);
			this.tabPageRequests.ResumeLayout(false);
			this.tabPageErrors.ResumeLayout(false);
			this.ResumeLayout(false);

		}
		#endregion

		/// <summary>
		/// The main entry point for the application.
		/// </summary>
		[STAThread]
		static void Main() 
		{
			Application.Run(new CrawlerForm());
		}

		private void CrawlerForm_Load(object sender, System.EventArgs e)
		{
			Settings.GetValue(this);

			InitValues();

			this.statusBarPanelInfo.Text = InternetGetConnectedStateString();
		}

		[DllImport("wininet")]
		public static extern int InternetGetConnectedState(ref int lpdwFlags, int dwReserved);
		[DllImport("wininet")]
		public static extern int InternetAutodial(int dwFlags, int hwndParent);
		int nFirstTimeCheckConnection = 0;
		string InternetGetConnectedStateString()
		{
			string strState = "";
			try
			{
				int nState = 0;
				// check internet connection state
				if(InternetGetConnectedState(ref nState, 0) == 0)
					return "You are currently not connected to the internet";
				if((nState & 1) == 1)
					strState = "Modem connection";
				else	if((nState & 2) == 2)
					strState = "LAN connection";
				else	if((nState & 4) == 4)
					strState = "Proxy connection";
				else	if((nState & 8) == 8)
					strState = "Modem is busy with a non-Internet connection";
				else	if((nState & 0x10) == 0x10)
					strState = "Remote Access Server is installed";
				else	if((nState & 0x20) == 0x20)
					return "Offline";
				else	if((nState & 0x40) == 0x40)
					return "Internet connection is currently configured";
					
				// get current machine IP
				IPHostEntry he = Dns.Resolve(Dns.GetHostName());
				strState += ",  Machine IP: "+he.AddressList[0].ToString();
			}
			catch
			{
			}
			return strState;
		}
		void ConnectionInfo()
		{
			try
			{
				int nState = 0;
				if(InternetGetConnectedState(ref nState, 0) == 0)
				{
					if(nFirstTimeCheckConnection++ == 0)
						// ask for dial up or DSL connection
						if(InternetAutodial(1, 0) != 0)
							// check internet connection state again
							InternetGetConnectedState(ref nState, 0);
				}
				if((nState & 2) == 2 || (nState & 4) == 4)
					// reset to reask for connection agina
					nFirstTimeCheckConnection = 0;
			}
			catch
			{
			}
			this.statusBarPanelInfo.Text = InternetGetConnectedStateString();
		}

		private void menuItemExit_Click(object sender, System.EventArgs e)
		{
			this.Close();
		}

		private void menuItemSettings_Click(object sender, System.EventArgs e)
		{
			ShowSettings(-1);
		}

		void ShowSettings(int nSelectedIndex)
		{
			SettingsForm form = new SettingsForm();
			form.SelectedIndex = nSelectedIndex;
			if(form.ShowDialog(this) == DialogResult.OK)
			{
				ThreadCount = Settings.GetValue("Threads count", 10);
				InitValues();
			}
		}

		void InitValues()
		{
			WebDepth = Settings.GetValue("Web depth", 3);
			RequestTimeout = Settings.GetValue("Request timeout", 20);
			SleepFetchTime = Settings.GetValue("Sleep fetch time", 2);
			SleepConnectTime = Settings.GetValue("Sleep connect time", 1);
			KeepSameServer = Settings.GetValue("Keep same URL server", false);
			AllMIMETypes = Settings.GetValue("Allow all MIME types", true);
			KeepAlive = Settings.GetValue("Keep connection alive", true);
			ExcludeHosts = Settings.GetValue("Exclude Hosts", ".org; .gov;").Replace("*", "").ToLower().Split(';');
			ExcludeWords = Settings.GetValue("Exclude words", "").Split(';');
			ExcludeFiles = Settings.GetValue("Exclude files", "").Replace("*", "").ToLower().Split(';');
			LastRequestCount = Settings.GetValue("View last requests count", 20);
			Downloadfolder = Settings.GetValue("Download folder", System.Environment.GetFolderPath(System.Environment.SpecialFolder.Personal));
			MIMETypes = GetMIMETypes();
			TextEncoding = GetTextEncoding();
		}

		static Encoding GetTextEncoding()
		{
			Encoding code;
			if(Settings.GetValue("Use windows default code page", true) == true)
				code = Encoding.Default;
			else
			{
				string strCodePage = Settings.GetValue("Settings code page");
				Regex reg = new Regex(@"\([0-9]*\)");
				strCodePage = reg.Match(strCodePage).Value;
				code = Encoding.GetEncoding(int.Parse(strCodePage.Trim('(', ')')));
			}
			return code;
		}
		
		// construct MIME types string from settings xml file
		static string GetMIMETypes()
		{
			string str = "";
			// check for settings xml file existence
			if(File.Exists(Application.StartupPath+"\\Settings.xml"))
			{
				XmlDocument doc = new XmlDocument();
				doc.Load(Application.StartupPath+"\\Settings.xml");
				XmlNode element = doc.DocumentElement.SelectSingleNode("SettingsForm-listViewFileMatches");
				if(element != null)
				{
					for(int n = 0; n < element.ChildNodes.Count; n++)
					{
						XmlNode xmlnode  = element.ChildNodes[n];
						XmlAttribute attribute  = xmlnode.Attributes["Checked"];
						if(attribute == null || attribute.Value.ToLower() != "true")
							continue;
						string[] items = xmlnode.InnerText.Split('\t');
						if(items.Length > 1)
						{
							str += items[0];
							if(items.Length > 2)
								str += '['+items[1]+','+items[2]+']';
							str += ';';
						}
					}
				}
			}
			return str;
		}

		private void buttonGo_Click(object sender, System.EventArgs e)
		{
			StartParsing();
		}

		void StartParsing()
		{			
			this.buttonGo.Enabled = false;
			// insert combo text in the combo list
			if(this.comboBoxWeb.FindStringExact(this.comboBoxWeb.Text) < 0)
				this.comboBoxWeb.Items.Insert(0, this.comboBoxWeb.Text);

			if(threadParse == null || threadParse.ThreadState != ThreadState.Suspended)
			{
				this.urlStorage.Clear();
				// start parsing thread
				threadParse = new Thread(new ThreadStart(RunParser));
				threadParse.Start();
			}

			// update running threads
			ThreadCount = Settings.GetValue("Threads count", 10);

			this.toolBarButtonContinue.Enabled = false;
			this.toolBarButtonPause.Enabled = true;
		}

		void ContinueParsing()
		{
			if(threadParse == null)
				return;
			if(threadParse.ThreadState == ThreadState.Suspended)
				threadParse.Resume();

			// update runnning threads
			ThreadCount = Settings.GetValue("Threads count", 10);

			this.toolBarButtonContinue.Enabled = false;
			this.toolBarButtonPause.Enabled = true;
		}

		// pause all working threads
		void PauseParsing()
		{
			if(threadParse.ThreadState == ThreadState.Running)
				threadParse.Suspend();
			for(int n = 0; n < ThreadCount; n++)
			{
				Thread thread = this.threadsRun[n];
				if(thread.ThreadState == ThreadState.Running)
					thread.Suspend();
			}
			this.toolBarButtonContinue.Enabled = true;
			this.toolBarButtonPause.Enabled = false;
		}
		
		// abort all working threads
		void StopParsing()
		{
			this.queueURLS.Clear();
			ThreadsRunning = false;
			Thread.Sleep(500);
			try
			{
				if(threadParse.ThreadState == ThreadState.Suspended)
					threadParse.Resume();
				threadParse.Abort();
			}
			catch(Exception)
			{
			}
			Monitor.Enter(this.listViewThreads);
			for(int n = 0; n < ThreadCount; n++)
			{
				try
				{
					Thread thread = this.threadsRun[n];
					ListViewItem itemLog = this.listViewThreads.Items[int.Parse(thread.Name)];
					itemLog.SubItems[2].Text = "Stop";
					itemLog.BackColor = Color.WhiteSmoke;
					itemLog.ImageIndex = 3;
					if(thread != null && thread.IsAlive)
					{