gamesite.java

是个java写的sipder,非常不错!能承受很大的压力,每天采集的数量在10000万

package cn.yicha.subject.spider.wapsite;

import java.util.ArrayList;

public class GameSite {
	private int _id;
	private String _firstUrl = "";
	private String _urlFilter = "";
	
	protected static final String _SITE_PROPERTY_SPLIT_CHAR = "\r\n|\n";
	
	protected void setSiteProps(String[] props) {
		try {
		_id = Integer.decode(props[0].trim()).intValue();
		_firstUrl = props[1].trim();
		_urlFilter = props[2].trim();
		} catch (Exception e) {
			System.out.println("format error: id=" + _id);
			System.exit(0);
			return;
		}
	}
	
	/**
	 * 从站点列表中得到相关站点属性
	 * @param gl
	 * @return
	 */
	public static GameSite[] getGSList(String sfn) {
		String[] gl = new SpecialSiteList(sfn).getSitesFromList();
		
		if (gl == null || gl.length <= 0) {
			System.out.println("There is no sites");
			System.exit(0);
			return null;
		}
		
		// 为站点赋上属性
		ArrayList gsList = new ArrayList();
		for (int i = 0; i < gl.length; i ++) {
			String[] props = gl[i].split(_SITE_PROPERTY_SPLIT_CHAR);
			
			GameSite gs = new GameSite();
			gs.setSiteProps(props);
			gsList.add(gs);
		}
		
		return (GameSite[]) gsList.toArray(new GameSite[0]);
	}

	public String get_firstUrl() {
		return _firstUrl;
	}

	public void set_firstUrl(String url) {
		_firstUrl = url;
	}

	public int get_id() {
		return _id;
	}

	public void set_id(int _id) {
		this._id = _id;
	}

	public String get_urlFilter() {
		return _urlFilter;
	}

	public void set_urlFilter(String filter) {
		_urlFilter = filter;
	}

}