hotwebworkmanager.java

实现了从Google

package com.ct.hotweb.getter;

import java.io.*;
import java.util.*;

import org.apache.commons.httpclient.*;

import com.ct.hotweb.util.*;
import com.ct.hotweb.bean.*;

/**
 * 管理器, 多线程.
 * 分析->产生静态模板
 * <p>Title: </p>
 * <p>Description: </p>
 * <p>Copyright: Copyright (c) 2004</p>
 * <p>Company: </p>
 * @author not attributable
 * @version 1.0
 */

public class HotWebWorkManager {
	private boolean killed = false;
	//private SearchKeyStack keyStack = new SearchKeyStack();
	private BlockAnalyzer blockAnalyzer = null;
	private BlockAnalyzer searchBlockAnalyzer = null;
	private HashMap hotWebGetterConfig = new HashMap();
	private HashMap keyGetterConfig = new HashMap();
	private TemplateAnalyzer htmlTemplate = null,
		indexHtmlTemplate = null,
		mapIndexHtmlTemplate = null,
		mapKeysHtmlTemplate = null;
	private String htmlPath = null;
	private String charset = null;
	private String remoteSiteConfigFileName = null;
	private String staticHtmlExt = null;
	private String urlPrefix = null;
	private String mapUrlPrefix = null;
	private String searchCfgFile = null;
	private int searchKeyCount = 0;
	private int keyColumn = 3, categoryColumn = 3, similarColumn = 3;

	private MultiThreadedHttpConnectionManager httpManager = null;

	public HotWebWorkManager(HashMap config) throws Exception {
		String cfgBasePath = (String) config.get("cfgBasePath");
		htmlPath = (String) config.get("htmlPath");
		htmlPath = Utils.pathAppend(cfgBasePath, htmlPath);
		urlPrefix = (String) config.get("urlPrefix");
		//urlPrefix = Utils.pathAppend(htmlPath, urlPrefix);
		mapUrlPrefix = (String) config.get("mapUrlPrefix");
		charset = (String) config.get("charset");
		staticHtmlExt = (String) config.get("staticHtmlExt");

		keyColumn = Integer.parseInt((String) config.get("keyColumn"));
		categoryColumn =
			Integer.parseInt((String) config.get("categoryColumn"));
		similarColumn = Integer.parseInt((String) config.get("similarColumn"));
		searchKeyCount = Integer.parseInt((String) config.get("searchKeyCount"));
		//set template
		try {
			String htmlTmpFile = (String) config.get("htmlTmpFile");
			htmlTmpFile = Utils.pathAppend(cfgBasePath, htmlTmpFile);
			File f = new File(htmlTmpFile);
			this.htmlTemplate =
				new TemplateAnalyzer(new BufferedReader(new FileReader(f)));
			this.htmlTemplate.analyzer();

			String indexHtmlTmpFile = (String) config.get("indexHtmlTmpFile");
			indexHtmlTmpFile = Utils.pathAppend(cfgBasePath, indexHtmlTmpFile);
			f = new File(indexHtmlTmpFile);
			this.indexHtmlTemplate =
				new TemplateAnalyzer(new BufferedReader(new FileReader(f)));
			this.indexHtmlTemplate.analyzer();

			String mapIndexHtmlTmpFile =
				(String) config.get("mapIndexHtmlTmpFile");

			mapIndexHtmlTmpFile =
				Utils.pathAppend(cfgBasePath, mapIndexHtmlTmpFile);
			f = new File(mapIndexHtmlTmpFile);
			this.mapIndexHtmlTemplate =
				new TemplateAnalyzer(new BufferedReader(new FileReader(f)));
			this.mapIndexHtmlTemplate.analyzer();

			String mapKeysHtmlTmpFile =
				(String) config.get("mapKeysHtmlTmpFile");
			mapKeysHtmlTmpFile =
				Utils.pathAppend(cfgBasePath, mapKeysHtmlTmpFile);
			f = new File(mapKeysHtmlTmpFile);
			this.mapKeysHtmlTemplate =
				new TemplateAnalyzer(new BufferedReader(new FileReader(f)));
			this.mapKeysHtmlTemplate.analyzer();
	
		} catch (Exception e) {
			System.out.println("read html template error.");
			throw e;
		}

		//get remote config
		remoteSiteConfigFileName = (String) config.get("remoteCfgFile");
		remoteSiteConfigFileName =
			Utils.pathAppend(cfgBasePath, remoteSiteConfigFileName);
		hotWebGetterConfig = Utils.loadConfig(remoteSiteConfigFileName);
		
		searchCfgFile = (String) config.get("searchCfgFile");
		searchCfgFile = Utils.pathAppend(cfgBasePath, searchCfgFile);
		keyGetterConfig = Utils.loadConfig(searchCfgFile);

		String siteMode = (String) hotWebGetterConfig.get("blockContent");
		blockAnalyzer =
			new BlockAnalyzer(new BufferedReader(new StringReader(siteMode)));
		try {
			blockAnalyzer.prepareAnalyzer();
			blockAnalyzer.anlyzer();
			blockAnalyzer.afterAnlyzer();
		} catch (Exception e) {
			System.out.println("construct site config block error.");
			throw e;
		}
		
		String keySiteMode = (String) keyGetterConfig.get("blockContent");
		searchBlockAnalyzer =
			new BlockAnalyzer(new BufferedReader(new StringReader(keySiteMode)));
		try {
			searchBlockAnalyzer.prepareAnalyzer();
			searchBlockAnalyzer.anlyzer();
			searchBlockAnalyzer.afterAnlyzer();
		} catch (Exception e) {
			System.out.println("construct site config block error.");
			throw e;
		}
	}

	public boolean isKilled() {
		return killed;
	}

	public void kill() {
		killed = true;
	}

	public BlockAnalyzer getBlockAnalyzer() {
		return this.blockAnalyzer;
	}

	public HashMap getHotWebGetterConfig() {
		return this.hotWebGetterConfig;
	}

	public TemplateAnalyzer getHtmlTemplate() {
		return this.htmlTemplate;
	}

	public String getHtmlExtension() {
		return this.staticHtmlExt;
	}

	public String getHtmlPath() {
		return this.htmlPath;
	}

	public String getHtmlCharset() {
		return this.charset;
	}

	public void work(Category cat, List categorys, SearchKeyStack stack)
		throws Exception {
		try {
			//不采用多线程来做
			this.killed = false;
			this.httpManager = new MultiThreadedHttpConnectionManager();
			CategoryViewThread catThread =
				new CategoryViewThread(this, cat, categorys);
			catThread.stack = stack;
			//catThread.start();
			catThread.run();
			/*
			HotWebWorkThread[] threads = new HotWebWorkThread[1];
			for (int i = 0; i < threads.length; i++) {
				threads[i] =
					new HotWebWorkThread(this, this.httpManager, stack);
				threads[i].run();
			}*/
			
			HotWebWorkThread keyThread = new HotWebWorkThread(this, this.httpManager, stack);
			keyThread.run();
		} finally {
			httpManager.shutdown();
		}
	}
	
	public void workSearchKey(Category cat) {
		System.out.println("开始从www添加关键字...");
		try {
			this.httpManager = new MultiThreadedHttpConnectionManager();
			SearchKeyGetThread thread = new SearchKeyGetThread(this, cat);
			//无线程
			thread.run();
			//多线程
			//thread.start();
		} catch (RuntimeException e) {
			e.printStackTrace();
		} finally {
			this.httpManager.shutdown();
		}		
		System.out.println("从www添加关键字完毕！");
	}
	
	public void workCategory(Category cat) {
		System.out.println("开始从www添加子目录...");		
		try {
			this.httpManager = new MultiThreadedHttpConnectionManager();
			CategoryGetThread thread = new CategoryGetThread(this, cat);
			thread.run();
		} catch (RuntimeException e) {
			e.printStackTrace();
		} finally {
			this.httpManager.shutdown();
		}
		System.out.println("从www添加子目录完毕！");		
	}

	public void generateMap() {
		System.out.println("开始产生网站地图...");
		try {
			this.httpManager = new MultiThreadedHttpConnectionManager();
			MapIndexViewThread thread = new MapIndexViewThread(this);
			thread.run();
		} catch (RuntimeException e) {
			e.printStackTrace();
		} finally {
			this.httpManager.shutdown();
		}
		System.out.println("网站产生完毕！");
	}

	/*
	  public static void main(String[] args) {
	    String[] keys = {
	 "源代码", "病毒", "愤青", "歼10", "台湾", "大选", "Java", "Linus", "RedHat", "地震",
	        "软件 下载", "私服 奇迹", "仙境传说", "传奇", "木子", "腾讯", "坦克大战"};
	
	    ArrayList sks = new ArrayList();
	
	    for (int j = 0; j < 10; j ++) {
	      for (int i = 0; i < keys.length; i++) {
	        SearchKey sk = new SearchKey();
	        sk.setName(keys[i]);
	        sks.add(sk);
	      }
	    }
	
	    try {
	      HotWebWorkManager manager = new HotWebWorkManager(sks,
	          Utils.loadConfig("d:/tmp/hotweb.config"));
	      manager.work();
	    }
	    catch (Exception e) {
	      e.printStackTrace();
	    }
	  }
	 */
	/**
	 * @return
	 */
	public TemplateAnalyzer getIndexHtmlTemplate() {
		return indexHtmlTemplate;
	}

	/**
	 * @param analyzer
	 */
	public void setIndexHtmlTemplate(TemplateAnalyzer analyzer) {
		indexHtmlTemplate = analyzer;
	}

	/**
	 * @return
	 */
	public String getUrlPrefix() {
		//return urlPrefix.replace('/', '\\');
		return urlPrefix;
	}

	/**
	 * @param string
	 */
	public void setUrlPrefix(String string) {
		urlPrefix = string;
	}

	/**
	 * @return
	 */
	public MultiThreadedHttpConnectionManager getHttpManager() {
		return httpManager;
	}

	/**
	 * @param manager
	 */
	public void setHttpManager(MultiThreadedHttpConnectionManager manager) {
		httpManager = manager;
	}

	/**
	 * @return
	 */
	public String getMapUrlPrefix() {
		return mapUrlPrefix;
	}

	/**
	 * @param string
	 */
	public void setMapUrlPrefix(String string) {
		mapUrlPrefix = string;
	}

	/**
	 * @return
	 */
	public TemplateAnalyzer getMapIndexHtmlTemplate() {
		return mapIndexHtmlTemplate;
	}

	/**
	 * @return
	 */
	public TemplateAnalyzer getMapKeysHtmlTemplate() {
		return mapKeysHtmlTemplate;
	}

	/**
	 * @param analyzer
	 */
	public void setMapIndexHtmlTemplate(TemplateAnalyzer analyzer) {
		mapIndexHtmlTemplate = analyzer;
	}

	/**
	 * @param analyzer
	 */
	public void setMapKeysHtmlTemplate(TemplateAnalyzer analyzer) {
		mapKeysHtmlTemplate = analyzer;
	}

	public HttpClient getHttpClient() {
		return new HttpClient(this.httpManager);
	}

	/**
	 * @return
	 */
	public int getKeyColumn() {
		return keyColumn;
	}

	/**
	 * @return
	 */
	public int getSimilarColumn() {
		return similarColumn;
	}

	/**
	 * @return
	 */
	public String getStaticHtmlExt() {
		return staticHtmlExt;
	}

	/**
	 * @return
	 */
	public int getCategoryColumn() {
		return categoryColumn;
	}

	/**
	 * @return
	 */
	public int getSearchKeyCount() {
		return searchKeyCount;
	}

	/**
	 * @return
	 */
	public HashMap getKeyGetterConfig() {
		return keyGetterConfig;
	}

	/**
	 * @return
	 */
	public BlockAnalyzer getSearchBlockAnalyzer() {
		return searchBlockAnalyzer;
	}

}