mtsearchresult1.java

利用多线程从搜索引擎下载网页并提取数据到数据库。

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.UnknownHostException;
import java.sql.SQLException;
import java.util.Date;

public class MTSearchResult1 {

	/**
	 * @param args
	 */

	static int tc;
	static int maxPage;
	static boolean dir;
	static boolean recurring = true;
	static TaskOptimizer threadOpt;
	static TaskOptimizer pageOpt;
	static Date taskStart = new Date();
	static Date recentTaskStart = new Date();
	static int maxSTPage;
	public static void main(String[] args) {
		// new Moderator(0,2);
		tc = Integer.parseInt(args[0]);
		maxPage = Integer.parseInt(args[1]);
		Moderator.minInterval = Float.parseFloat(args[2]);
		maxSTPage=Integer.parseInt(args[3]);
		// threadOpt=new TaskOptimizer(1,20,3600*2, tc);
		// pageOpt=new TaskOptimizer(10,200,3600*2, maxPage);
		// SearchResult1.threadOpt=threadOpt;
		// SearchResult1.pageOpt=pageOpt;

		deamon();
	}

	private static void deamon() {

		do {
			long timeEplased = (long) (new Date().getTime() - taskStart
					.getTime()) / 60000;
			long recentTimeEplased = (long) (new Date().getTime() - recentTaskStart
					.getTime()) / 60000;
			
			System.out.println();
			System.out
					.println("Total active thread: " + (Thread.activeCount()));

			// System.out.println("-----------TC value: "
			// + tc + "; MAXPAGE Value:" + maxPage + "-----------");
			System.out.println("Recent terms searched:"
					+ SearchResult1.recentGoodPageCount
					+ " / Recent links inserted:"
					+ SearchResult1.recentLinkInsertCount
					+ " / Recent time elapsed:" + recentTimeEplased
					+ " Minutes.");

			System.out.println("Total terms count:"
					+ SearchResult1.totalPageCount + "/Total links found:"
					+ SearchResult1.totalLinkCount
					+ " / Total links inserted: "
					+ SearchResult1.totalLinkInsertCount + ". / Time elapsed: "
					+ timeEplased + " Minutes.");
			if (Thread.activeCount() <= 2 + 0) {
				// NetHelper.restartNet();
				NetHelper.switchIP();
				reset();
				restartThreads();
			}
			try {
				// System.out
				// .println("------------Sleeping for 10 seconds.-----------");
				Thread.sleep(10000);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		} while (recurring);

	}

	private static void reset() {

		SearchResult1.recentLinkInsertCount = 0;
		SearchResult1.recentGoodPageCount = 0;
		recentTaskStart = new Date();
		// threadOpt.taskStart();
		// pageOpt.taskStart();
		// tc=threadOpt.getValue();
		// maxPage=pageOpt.getValue();
		SearchResult1.interleaving = tc;
		SearchResult1.MAXCOUNT = maxPage;
		SearchResult1.MAXSTPAGECOUNT = maxSTPage;
	}

	private static void restartThreads() {
		int i;
		String threadsName;
		// totalThreads=tc;
		StringBuilder sb = new StringBuilder();
		Thread[] threads = new Thread[Thread.activeCount()];

		Thread.enumerate(threads);

		sb.append("'");
		for (Thread t : threads) {
			sb.append(t.getName());
			sb.append("'");
		}
		threadsName = sb.toString();

		for (i = 0; i < tc; i++) {
			// int j = dir ? i : tc - i - 1;

			try {
				Thread.sleep((long) (Util.getARand() * 1000));
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
			if (!threadsName.contains("'" + String.valueOf(i) + "'")) {
				runThreads(i);
			}
		}
	}

	private static void runThreads(int t) {
		Thread thread = null;
		try {
			thread = new Thread(new SearchResult1(t), String.valueOf(t));
			thread.start();
		} catch (SQLException e) {
			e.printStackTrace();
		}
	}

}