gamesitefetcher.java

是个java写的sipder,非常不错!能承受很大的压力,每天采集的数量在10000万

package cn.yicha.subject.spider.wapsite;

public class GameSiteFetcher {
	protected static final String _INDEX_SPLIT_CHAR = ",";
	protected static final String _INDEX_ALL_TAG = "a";
	protected static final String _DEFAULT_SITE_ID = "1";
	
	
	private int[] _index;
	private GameSite[] _gs;

	/**
	 * 将输入的站点号字符串转化为站点号数组
	 * 
	 * @param is
	 */
	protected static int[] getIndexWithIs(String is) {
		if (is == null || is.length() <= 0) {
			return null;
		}

		String[] ix = is.trim().split(_INDEX_SPLIT_CHAR);
		if (ix == null || ix.length <= 0) {
			return null;
		}

		int[] index = new int[ix.length];
		for (int i = 0; i < ix.length; i++) {
			try {
				index[i] = Integer.decode(ix[i].trim()).intValue();
			} catch (NumberFormatException nfe) {
				System.out.println("number format error");
				System.exit(0);
				return null;
			}
		}
		return index;
	}

	public GameSiteFetcher(String sfn, String is) {
		// 获得全部站点
		GameSite[] gl = GameSite.getGSList(sfn);
		
		// 获得站点id序列
		if (is.equalsIgnoreCase(_INDEX_ALL_TAG)) {
			_index = new int[gl.length];
			for (int i = 0; i < gl.length; i ++) {
				_index[i] = i + 1;
			}
		} else {
			_index = getIndexWithIs(is);
		}
		
		// 初始化站点序列
		_gs = new GameSite[_index.length];
		
		// 提取出需要的站点
		for (int i = 0; i < gl.length; i ++) {
			for (int j = 0; j < _index.length; j ++) {
				if (_index[j] == gl[i].get_id()) {
					_gs[j] = gl[i];
				}
			}
		}
		
		gl = null;
	}
	
	public GameSite[] get() {
		return get_gs();
	}

	public GameSite[] get_gs() {
		return _gs;
	}
}
//	public static void main(String args[]) {
//		GameSiteFetcher gsf = new GameSiteFetcher("34");
//		// GameSiteFetcher gsf = new GameSiteFetcher("1,29");
//		
//		GameSite[] gs = gsf.get();
//		
//		for (int i = 0; i < gs.length; i ++) {
//			System.out.println(gs[i].get_id());
//			System.out.println(gs[i].get_firstUrl());
//			System.out.println(gs[i].get_urlFilter());
//		}
//	}
//}