anchorproperty.java

是个java写的sipder,非常不错!能承受很大的压力,每天采集的数量在10000万

package cn.yicha.subject.spider.store;

import java.net.URL;

public class AnchorProperty 
{
	private final String _HTTP_PREFIX = "http://";
	
	String _title;
	String _relative_url;
	String _absolute_url;

	/**
	* 根据绝对URL和相对URL，获取最终URL
	* 例如绝对URL --> http://www.sina.com.cn/news/01.htm
	*     相对URL --> catalog/02.htm
	*     返回URL --> http://www.sina.com.cn/news/catalog/02.htm
	*/
	public String getUrl()
	{
		String url = "";
		
		try {
			URL absUrl = new URL(getAbsoluteUrl());
			URL relUrl = new URL(absUrl, getRelativeUrl());			
			url = relUrl.toExternalForm();
		}
		catch (Exception e) {
			e.printStackTrace();
		}
		
		return url;
	}

	public void setTitle(String _title) {
		this._title = _title;
	}

	public String getTitle() {
		return _title;
	}

	public void setRelativeUrl(String _relative_url) {
		this._relative_url = _relative_url;
	}

	public String getRelativeUrl() {
		return _relative_url;
	}

	public void setAbsoluteUrl(String _absolute_url) {
		this._absolute_url = _absolute_url;
	}

	public String getAbsoluteUrl() {
		return _absolute_url;
	}

}