utils.java

实现了从Google

package com.ct.hotweb.util;

import java.io.*;
import java.util.*;

import java.text.*;
import java.net.*;

import org.apache.commons.httpclient.*;
import org.apache.commons.httpclient.methods.*;

public class Utils {

	private static final int DEFAULT_INITIAL_BUFFER_SIZE = 4 * 1024;

	private static final String[] os = { "Windows 2000", "Windows 98", };

	private static final String[] broswer = { "MSIE 5.00", "MSIE 6.00", };

	private static int i = 0, j = 0;

	private static final Random rand = new Random();

	public static SimpleDateFormat mysqlDateFormat = new SimpleDateFormat(
			"yyyy-MM-dd hh:mm:ss");

	public static String myReplaceAll(String sourceStr, String s1, String s2) {
		if (sourceStr == null || sourceStr.indexOf(s1) < 0) {
			return sourceStr;
		}
		StringBuffer sb = new StringBuffer();
		int pos;
		while ((pos = sourceStr.indexOf(s1)) >= 0) {
			sb.append(sourceStr.substring(0, pos));
			sb.append(s2);
			sourceStr = sourceStr.substring(pos + s1.length(), sourceStr
					.length());
		}
		if (sourceStr.length() > 0) {
			sb.append(sourceStr);
		}
		return sb.toString();
	}

	public static String getContent(String strUrl, String charset) {
		URL url = null;
		InputStream is = null;
		int len;
		byte[] b = new byte[4 * 1024];
		ByteArrayOutputStream os = new ByteArrayOutputStream();
		String result = "";
		try {
			url = new URL(strUrl);
			url.openConnection();
			is = url.openStream();
			while ((len = is.read(b)) > 0)
				os.write(b, 0, len);

			result = new String(os.toByteArray(), charset);
		} catch (Exception e) {
		} finally {
			try {
				is.close();
			} catch (IOException e1) {
				e1.printStackTrace();
			}
		}

		return (result == null ? "" : result);
	}

	public static String getContent(HttpClient hc, String strUrl, String charset)
			throws Exception {
		//charset default = gb2312
		//google
		//String url =
		// "http://www.google.com/search?hl=zh-CN&ie=GB2312&q=fdsa&btnG=Google%CB%D1%CB%F7&lr=";
		//yahoo
		//String url =
		// "http://cn.websearch.yahoo.com/search/web_cn?stype=&p=fdsa&scch=on";
		//baidu
		//String url = "http://www.baidu.com/baidu?w=fdsafds&cl=3";

		HttpMethod hm = new GetMethod(strUrl);
		//HttpMethod hm = new UrlGetMethod(strUrl);
		try {
			String host = hm.getURI().getHost();
			hm.addRequestHeader("Host", host);
			hm.addRequestHeader("Accept", "*/*");
			hm.addRequestHeader("Referer", "http://" + host);
			hm.addRequestHeader("Accept", "*/*");
			hm.addRequestHeader("Cookie", "null");
			hm.addRequestHeader("User-Agent", "Mozilla/4.0 (compatible; "
					+ broswer[i] + "; " + os[j]);

			//hm.addRequestHeader("Pragma", "no-cache");
			//hm.addRequestHeader("Cache-Control", "no-cache");
			//hm.addRequestHeader("Connection", "Keep-Alive");
			hm.addRequestHeader("Accept-Language", "zh-cn");
			hc.executeMethod(hm);

			byte[] b = hm.getResponseBody();
			/*
			 * InputStream is = null; byte[] b = null; try { is =
			 * hm.getResponseBodyAsStream(); b = getResponseBody(is, hc); }
			 * catch (Exception ioe) { }
			 */
			if (b == null)
				return "";
			char error = '?';
			String str = new String(b, charset);
			int count = getCount(str, error);
			if (count < (b.length / 60 )) return str;
			
			str = new String(b);
			count = getCount(str, error);
			if (count < (b.length / 60 )) return str;
			
			str = new String(b, "GBK");
			count = getCount(str, error);
			if (count < (b.length / 60 )) return str;
			
			return new String(b, charset);
			
		} catch (Exception e) {
			//log.error("http getter error. " + e.getMessage());
			/*
			 * try { hm.releaseConnection(); hm = null; //Thread.sleep(1000);
			 * return getContent(hc, strUrl, charset); } catch (Exception e1) {
			 * e1.printStackTrace(); return ""; }
			 */
			System.out.println("连接被重置......");
			i = rand.nextInt() % broswer.length;
			j = rand.nextInt() % os.length;
			throw e;
		} catch (Throwable t) {
			System.out.println("high error......");
		} finally {
			if (hm != null) {
				hm.releaseConnection();
			}
		}
		return "";
	}

	public static HashMap loadConfig(String fileName) throws Exception {
		InputStream is = null;
		HashMap config = new HashMap();
		try {
			Properties p = new Properties();
			is = new FileInputStream(new File(fileName));
			p.load(is);
			java.util.Enumeration e = p.propertyNames();
			while (e.hasMoreElements()) {
				String name = (String) e.nextElement();
				String value = p.getProperty(name);
				config.put(name, value);
			}
		} catch (Exception e) {
			System.out.print("read remote site config error. file name: "
					+ fileName);
			throw e;
		} finally {
			is.close();
		}
		return config;
	}

	public static String dateFormat(java.util.Date date) {
		return mysqlDateFormat.format(date);
	}

	public static String pathAppend(String base, String path) {
		if (base == null || (base = base.trim()).equals(""))
			return path;
		if (base.endsWith("/") || base.endsWith("\\")) {
			base = base.substring(0, base.length() - 1);
		}
		if (path == null || (path = path.trim()).equals(""))
			return base;
		if (path.startsWith("/") || path.startsWith("\\")) {
			path = path.substring(1, path.length());
		}
		return base + "/" + path;
	}

	private static int getCount(String str, char split) {
		int result = 0;
		if (str == null)
			return result;

		for (int i = 0; i < str.length(); i++) {
			if (split == str.charAt(i))
				result++;
		}
		return result;
	}

	public static String charsetChange(String str, String charset) {
		try {
			String result = new String(str.getBytes("ISO-8859-1"), charset);
			return result;
		} catch (Exception e) {
			e.printStackTrace();
		}
		return str;
	}

	public static String uncharsetChange(String str, String charset) {
		try {
			//return str;
			return new String(str.getBytes(charset), "ISO-8859-1");
		} catch (Exception e) {
			e.printStackTrace();
		}
		return str;

	}

	/*
	 * public static void main(String[] args) { long start =
	 * System.currentTimeMillis(); System.out.println(dateFormat(new
	 * Date(System.currentTimeMillis()))); System.out.println("time: " +
	 * (System.currentTimeMillis() - start)); }
	 */

	private static byte[] getResponseBody(InputStream instream, HttpClient hc)
			throws Exception {
		ByteArrayOutputStream outstream = null;
		try {
			outstream = new ByteArrayOutputStream();
			byte[] buffer = new byte[DEFAULT_INITIAL_BUFFER_SIZE];
			int len;
			while ((len = instream.read(buffer)) > 0) {
				outstream.write(buffer, 0, len);
			}
			outstream.close();
			return outstream.toByteArray();
		} catch (Exception e) {
			throw e;
		} finally {
			if (outstream != null)
				outstream.close();
		}
	}

	public static List averageList(List list, int count) {
		List result = new ArrayList();
		if (count <= 0) {
			result.add(list);
			return result;
		}
		int partCount = list.size() / count;
		if (list.size() % count > 0)
			partCount++;

		int c = 0;
		for (int i = 0; i < count; i++) {
			List part = new ArrayList();
			for (int j = 0; j < partCount; j++, c++) {
				if (c >= list.size())
					break;
				part.add(list.get(c));
			}
			result.add(part);
		}
		return result;
	}

	public static void main(String[] args) {
		try {
			String test = getContent(
					"http://cn.websearch.yahoo.com/search?p=java.net.URL+%E7%BC%96%E7%A0%81+%E5%A4%B4http&ei=UTF-8&fl=1&vc=&vl=lang_zh-CN&vl=lang_zh-TW&x=wrt&meta=vl%3Dlang_zh-CN%26vl%3Dlang_zh-TW",
					"utf-8");
			File f = new File("d:/tmp.txt.txt");
			OutputStream os = new FileOutputStream(f);
			os.write(test.getBytes());
			os.flush();
			os.close();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}