hotwebgetter.java

实现了从Google

/*
 * 创建日期 2004-10-30
 *
 * 更改所生成文件模板为
 * 窗口 > 首选项 > Java > 代码生成 > 代码和注释
 */
package com.ct.hotweb.util;

import java.net.*;
import java.util.*;

import org.apache.commons.httpclient.*;

/**
 * @author Administrator
 *
 * 更改所生成类型注释的模板为
 * 窗口 > 首选项 > Java > 代码生成 > 代码和注释
 */
public class HotWebGetter {
	private String charset = "";
	private String searchUrl = "";
	private String partKey = "";
	private String queryParmsName = "";
	private String key = "";
	private String parserClass = "";
	private int count = 10;
	private int pageCount = 10;
	private int connectionType = 0;
	private int delayTime = 0;

	private String httpContent;
	private BlockAnalyzer block;
	private ArrayList blocksVar;
	private HttpClient httpClient;

	public HotWebGetter(
		HttpClient httpClient,
		HashMap config,
		String key,
		BlockAnalyzer block) {
		this.httpClient = httpClient;
		charset = (String) config.get("charset");
		searchUrl = (String) config.get("searchUrl");
		//blockContent = (String)config.get("blockContent");
		queryParmsName = (String) config.get("queryParmsName");
		count = Integer.parseInt((String) (config.get("count")), 10);
		partKey = (String) config.get("partKey");
		parserClass = (String) config.get("parserClass");
		pageCount = Integer.parseInt((String) (config.get("pageCount")), 10);
		connectionType =
			Integer.parseInt((String) (config.get("connectionType")), 10);
		
		this.block = block;
		this.key = key;
		delayTime =
			Integer.parseInt((String)(config.get("delayTime")));
		if (delayTime >= 0)
			try {
				Thread.sleep(delayTime);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}		
	}

	/**
	 * 抓取html文件内容
	 * @throws Exception
	 * @return HotWebGetter
	 */
	public HotWebGetter getHotWeb() throws Exception {

		String url = null;
		url =
			this.searchUrl
				+ "&"
				+ this.queryParmsName
				+ "="
				+ URLEncoder.encode(key, "ISO-8859-1");

		if (connectionType == 0)
			httpContent = Utils.getContent(url, this.charset);
		else
			httpContent = Utils.getContent(httpClient, url, this.charset);

		Class c = Class.forName(parserClass);
		HttpParser hp = (HttpParser) (c.newInstance());
		blocksVar = hp.parse(httpContent, partKey, block);
		return this;
	}

	public ArrayList getBlocksVar() {
		return this.blocksVar;
	}

	/*
	public static void main(String[] args) {
	  HashMap config = new HashMap();
	  try {
	    Properties p = new Properties();
	    p.load(new FileInputStream(new File("d:/tmp/google.config")));
	    java.util.Enumeration e = p.propertyNames();
	    while (e.hasMoreElements()) {
	      String name = (String) e.nextElement();
	      String value = p.getProperty(name);
	      config.put(name, value);
	    }
	  } catch (Exception e) {
	    e.printStackTrace(); 
	  }
	  ArrayList al;
	  String blockContent = (String)config.get("blockContent");
	
	  BlockAnalyzer block = new BlockAnalyzer(new BufferedReader(new StringReader(blockContent)));
	  try {
	    block.prepareAnalyzer();
	    block.anlyzer();
	    block.afterAnlyzer();
	
	    HotWebGetter hwg = new HotWebGetter(new HttpClient(), config, "源代码", block);
	
	    al = hwg.getHotWeb().getBlocksVar();
	    System.out.println(hwg.httpContent);
	    System.out.println(al);
	  } catch (Exception e) {
	    e.printStackTrace();
	  }
	}*/
}