getformcontent.java

本程序是专门用于从网页上自动收集cmi,cnki上的被引文献的数据

package cn.ac.cintcm.spider.cnki;

import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.List;

import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.httpclient.*;

import cn.ac.cintcm.spider.*;

public class GetFormContent {
	
	protected FormParameter formParameters;
	public static Cookie[] cookies;
			
	public GetFormContent(String configFile) throws IOException {
		formParameters = ConfigUtil.loadFormConfig(configFile);
	}
	
	public GetFormContent(FormParameter formParameters) {
		this.formParameters = formParameters;
	}
	
	protected InputStream post(String url) {
		HttpClient client = new HttpClient();
        PostMethod post = new PostMethod(url);
        
//        List<NameValuePair> list = formParameters.getNameValues();         
//        NameValuePair[] tmpArray = convertArray(formParameters.getNameValues());
       
        post.setRequestBody(convertArray(formParameters.getNameValues()));    
        post.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, 
	    		new DefaultHttpMethodRetryHandler(3, false));
        
        InputStream ret = null;
        try {
        	client.executeMethod(post);
        	InputStream responseBody = post.getResponseBodyAsStream();
        	
        	cookies = client.getState().getCookies();
        	ret = responseBody;
		} catch (HttpException e) {			
			LogEntity.logFile.log(e,"happened in PostMethod");
			e.printStackTrace();
		} catch (IOException e) {
			LogEntity.logFile.log(e,"happened in PostMethod");
			e.printStackTrace();
		}
		return ret;
	}
	
	protected static NameValuePair[] convertArray(List config) {
		NameValuePair[] pairValues= new NameValuePair[config.size()];
		for (int i=0; i < config.size(); i++) {
			pairValues[i] = (NameValuePair) config.get(i);
		}
		return pairValues;
	}
	
	public InputStream getContent() throws UnsupportedEncodingException, IOException {
		return post(formParameters.getUrl());
	}
	
	public static String slurp (InputStream in) throws IOException {
	    StringBuffer out = new StringBuffer();
	    byte[] b = new byte[4096];
	    for (int n; (n = in.read(b)) != -1;) {
	        out.append(new String(b, 0, n,"utf-8"));
	        new String();
	    }
	    return out.toString();
	    
	}

}