crawler.java

java下的 多线程爬虫 输入线程数目

package crawler;

import java.util.ArrayList;

// 搜索Web爬行者
public class Crawler {

	// 主函数
	public static void main(String[] args) {
		
		int n = Integer.parseInt(args[0]);
		PageDownload pageDownloads[] = new PageDownload[n];
		
		//new Crawler("http://zhidao.baidu.com/browse/90?lm=2",
		//		300, "textarea");
		
		int i;
		
		GenerateUrl myGenerater = new GenerateUrl("http://zhidao.baidu.com/browse/90?lm=2&pn=", 25, 0, 1);
		UrlDatabase database = new UrlDatabase(myGenerater.generateUrls());
		ArrayList<String> limitFields = new ArrayList<String>();
		
		//limitFields.add("http://zhidao.baidu.com/browse/");
		limitFields.add("http://zhidao.baidu.com/question/");
		limitFields.add("http://passport.baidu.com/?business&aid");
		for (i = 0; i < n; i++)
		{
			pageDownloads[i] = new PageDownload(database);
			pageDownloads[i].setFields(limitFields);
			pageDownloads[i].setInterval(0);
		}
		Thread search[] = new Thread[n];
		for (i = 0; i < n; i++)
		{
			search[i]= new Thread(pageDownloads[i]);	
		}
		
		for (i = 0; i < n; i++)
			search[i].start();

		System.out.println("Start searching...");
		System.out.println("result:");
		
		
		try {
			for (i = 0; i < n; i++)
				search[i].join();
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}
}