internalqueues.java

基于JAVA的网络蜘蛛系统,使用JAVA实现抓取网络资源的网络蜘蛛。通过一个入口网址来扫描整个互联网的网址

package issa.webspider;

import java.net.URL;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;

public class InternalQueues {

	private static BlockingQueue<URL> waitingList;
	private static BlockingQueue<URL> processingList;
	private static BlockingQueue<URL> errorList;
	private static BlockingQueue<URL> completeList;

	private static ConcurrentHashMap<String, String> urlList;

	static {
		waitingList = new LinkedBlockingQueue<URL>();
		processingList = new LinkedBlockingQueue<URL>();
		errorList = new LinkedBlockingQueue<URL>();
		completeList = new LinkedBlockingQueue<URL>();
		urlList = new ConcurrentHashMap<String, String>();
	}

	public static boolean addToWaitingList(URL url) {
		if (!isURLExist(url)) {
			waitingList.add(url);
			System.out.println("Add address: " + url);
			return true;
		} else {
			return false;
		}
	}

	public static URL getFromWaitingList() throws InterruptedException {
		return waitingList.take();
	}

	public static boolean isURLExist(URL url) {
		int hashCode = url.hashCode();
		if (urlList.containsKey(((Integer) hashCode).toString())) {
			return true;
		} else {
			urlList.put(((Integer) hashCode).toString(), url.toString());
		}
		return false;
	}

	public static boolean addToProcessingList(URL url) {
		processingList.add(url);
		return true;
	}

}