mtsearchresult.java

利用多线程从搜索引擎下载网页并提取数据到数据库。

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.UnknownHostException;
import java.sql.SQLException;
import java.util.Date;

public class MTSearchResult {

	/**
	 * @param args
	 */

	static int tc;
	static boolean dir;
	static boolean recurring = true;

	public static void main(String[] args) {

		tc = Integer.parseInt(args[0]);
		SearchResult.MAXCOUNT = Integer.parseInt(args[1]);
		deamon();

	}

	private static void deamon() {

		do {
			System.out.println("-----------Total active thread: "
					+ (Thread.activeCount()) + "-----------");
			if (Thread.activeCount() <= 2 + 0) {
				// NetHelper.restartNet();
				NetHelper.switchIP();
				reset();
				restartThreads();
			}
			try {
				System.out
						.println("------------Sleeping for 10 seconds.-----------");
				Thread.sleep(10000);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		} while (recurring);

	}

	private static void reset() {
		SearchResult.totalLinkCount = 0;
		SearchResult.taskStart = new Date();
		SearchResult.totalLinkInsertCount = 0;
		SearchResult.totalPageCount = 0;
	}

	private static void restartThreads() {
		int i;
		String threadsName;
		// totalThreads=tc;
		StringBuilder sb = new StringBuilder();
		Thread[] threads = new Thread[Thread.activeCount()];

		Thread.enumerate(threads);

		sb.append("'");
		for (Thread t : threads) {
			sb.append(t.getName());
			sb.append("'");
		}
		threadsName = sb.toString();

		for (i = 0; i < tc; i++) {
			// int j = dir ? i : tc - i - 1;

			try {
				Thread.sleep((long) (Util.getARand() * 1000));
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
			if (!threadsName.contains("'" + String.valueOf(i) + "'")) {
				runThreads(i);
			}
		}
	}

	private static void runThreads(int t) {
		Thread thread = null;
		try {
			thread = new Thread(new SearchResult(t, tc), String.valueOf(t));
			thread.start();
		} catch (SQLException e) {
			e.printStackTrace();
		}
	}

}