get2formcontent.java

本程序是专门用于从网页上自动收集cmi,cnki上的被引文献的数据

package cn.ac.cintcm.spider.cnki;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.httpclient.NameValuePair;
import org.htmlparser.Node;
import org.htmlparser.tags.InputTag;
import org.htmlparser.util.NodeList;

import cn.ac.cintcm.spider.*;

public class Get2FormContent extends GetFormContent{

	public Get2FormContent(String configFile) throws IOException {
		super(configFile);
	}
	public Get2FormContent(FormParameter formParameters) {
		super(formParameters);
	}
	
	public InputStream getContent() throws IOException {
		createHiddenPairValues();
	//	createHandPairValues();
		InputStream result = post(formParameters.getUrl2());
		if(result==null){
			 LogEntity.logFile.log("PostMehod 出错");
		}
        return result;
	}
	
//	public void createHandPairValues() throws UnsupportedEncodingException {
//		
//	}

	public void createHiddenPairValues()throws IOException {  //get the other input parameters
		List<NameValuePair> newPairsList = null;
		GetUrlContent guc=new GetUrlContent(formParameters.getUrl());
		String resource=guc.getContent();
	    if(resource==null){
	    	 LogEntity.logFile.log("网络出错!!");
	    	 return;
	    }
        NodeList list=NodeFilters.getNodeList(resource,"INPUT");      
        newPairsList = new ArrayList<NameValuePair>();
        for (Node node : list.toNodeArray()) {
        	InputTag tag = new InputTag();	       	        		        	        	
        	tag.setText(node.toHtml());
        	String name = tag.getAttribute("name");	        	
        	if (formParameters.getHiddenParameters().contains(name)) {
	        	String value = tag.getAttribute("value");		   		        			        			        			        	
	        	newPairsList.add(new NameValuePair(name, new String (value.getBytes("UTF-8"), "ISO-8859-1")));
        		//newPairsList.add(new NameValuePair(name, value));        		
        	}	        	
        }				
        if(newPairsList.size()==0||newPairsList==null){
        	LogEntity.logFile.log("没有得到http://lsg.cnki.net/grid20/Navigator.aspx?id=6对应的页面，可能网络忙");
        	return;
        }
        formParameters.getNameValues().addAll(newPairsList);       
	}

}