urlutil.java

java写的搜索引擎网络爬虫 java写的搜索引擎网络爬虫 java写的搜索引擎网络爬虫

/* 
 * FileName UrlRegexUtil.java
 * Create Time 2006-5-17 8:48:05
 * Author shiwei
 * Descript 公用类
 * Version 
 */

package com.snoics.reptile.util;

import java.io.BufferedReader;

import com.snoics.base.net.NetTool;
import com.snoics.base.util.StringClass;
import com.snoics.base.util.file.SnoicsFile;
import com.snoics.reptile.system.common.Common;
import com.snoics.reptile.system.common.CommonObject;


public class UrlUtil {

	private CommonObject commonObject=new CommonObject();
	
	/**
	 * 获取一个页面的HTML字符串
	 * @param url
	 * @return String
	 */
	public String getHtmlString(String url) {
		String charsetName=commonObject.getConfigInfo(Common.CONFIGFILE_NODE_CHARSETNAME);
		StringBuffer htmlString=new StringBuffer();
		BufferedReader bufferedreader=null;
		try{
			charsetName=StringClass.getString(charsetName);
			if(!charsetName.equals("")){
				bufferedreader=NetTool.getUrlToBufferReader(url,charsetName);
			}else{
				bufferedreader=NetTool.getUrlToBufferReader(url);
			}
		}catch(Exception e) {
			e.printStackTrace();
		}

		if(bufferedreader==null) {
			return htmlString.toString();
		}
		
		String tempString=null;
		try {
			while ((tempString = bufferedreader.readLine()) != null) {
				htmlString.append(tempString+" \n");
			}
		}catch(Exception e) {
			e.printStackTrace();
		}
		
		return htmlString.toString();
	}
	
	/**
	 * 获取二进制文件
	 * @param url
	 * @param fileName
	 */
	public void createBinFile(String url,String fileName){
		NetTool.getUrlToFileAsDataInputStream(url,fileName);
	}
	
	public static void main(String[] args){
		UrlUtil urlUtil=new UrlUtil();
		String url="http://www.squarecn.com/ff/zhuanti/FF8/index.html";
		String filename="c:/1.html";
		String htmlString=urlUtil.getHtmlString(url);
		SnoicsFile snoicsFile=new SnoicsFile();
		snoicsFile.stringToFile(htmlString,filename);
		System.out.println(htmlString);
	}
}