storediffurl.java

是个java写的sipder,非常不错!能承受很大的压力,每天采集的数量在10000万

package cn.yicha.subject.spider.store;

import java.io.*;
import java.util.HashSet;

public class StoreDiffUrl
{	
	private static final String _DOMAIN_FILE_NAME = "DiffDomain.txt";
	private static final String _URL_FILE_NAME = "DiffUrl.txt";
	private static final String _EXCEPTION_URL_FILE_NAME = "ExceptionUrl.txt";

	// 已存储的不同二级域名的域名集合
	private static HashSet _diff_domains = new HashSet();

	// 已存储的不同二级域名的URL地址集合
	private static HashSet _diff_urls = new HashSet();

	// 已存储的异常URL地址
	private static HashSet _diff_exception_urls = new HashSet();
		
	/**
	* 保存与下载网址不同域名的二级域名信息
	*/
	public synchronized static void saveDomain(String diffDomain, String sourceDomain)
	{
		try {
			if (!_diff_domains.contains(diffDomain)) {
				String fileName = _DOMAIN_FILE_NAME;
				
				File f = new File(fileName);					
				PrintWriter pw = new PrintWriter(
					new OutputStreamWriter( new FileOutputStream(fileName, true) ) );
				pw.println(sourceDomain + "," + diffDomain);
				pw.close();

				_diff_domains.add(diffDomain);
			}
		}
		catch (java.io.FileNotFoundException e) {
			e.printStackTrace();
		}
	}

	/**
	* 保存与下载网址不同域名的URL地址信息
	*/
	public synchronized static void saveUrl(String diffUrl, String sourceDomain)
	{
		try {
			if (!_diff_urls.contains(diffUrl)) {
				String fileName = _URL_FILE_NAME;
				
				File f = new File(fileName);					
				PrintWriter pw = new PrintWriter(
					new OutputStreamWriter( new FileOutputStream(fileName, true) ) );
				pw.println(sourceDomain + "," + diffUrl);
				pw.close();

				_diff_urls.add(diffUrl);
			}
		}
		catch (java.io.FileNotFoundException e) {
			e.printStackTrace();
		}
	}

	/**
	* 保存异常URL
	*/
	public synchronized static void saveExceptionUrl(String url)
	{
		try {
			if (!_diff_exception_urls.contains(url)) {
				String fileName = _EXCEPTION_URL_FILE_NAME;
				
				File f = new File(fileName);					
				PrintWriter pw = new PrintWriter(
					new OutputStreamWriter( new FileOutputStream(fileName, true) ) );
				pw.println(url);
				pw.close();

				_diff_exception_urls.add(url);
			}
		}
		catch (java.io.FileNotFoundException e) {
			e.printStackTrace();
		}
	}
}