webspider.java

基于JAVA的网络蜘蛛系统,使用JAVA实现抓取网络资源的网络蜘蛛。通过一个入口网址来扫描整个互联网的网址

package issa.webspider;

import org.apache.log4j.Logger;
import org.apache.log4j.xml.DOMConfigurator;

import java.net.MalformedURLException;
import java.net.URL;

public class WebSpider {
	/**
	 * Logger for this class
	 */
	private static final Logger logger = Logger.getLogger(WebSpider.class);

	public static void main(String[] args) {

		DOMConfigurator.configure("log4j.xml");

		if (args.length == 0) {
			System.out.println("请输入入口网址");
			System.exit(1);
		} else {
			URL entranceUrl;
			CommonVar.downLoadDir = args[1];
			try {
				entranceUrl = new URL(args[0]);
				InternalQueues.addToWaitingList(entranceUrl);
				logger.info("加入网址：" + entranceUrl);
			} catch (MalformedURLException e) {
				System.err.println("入口网址不是有效的URL地址");
				// e.printStackTrace();
				System.exit(1);
			}
		}
		System.out.println("开始下载线程………");

		for (int i = 0; i < 5; i++) {
			DownloadThread dt = new DownloadThread();
			Thread t = new Thread(dt);
			t.start();
		}
	}

}