get2formcontent.java

本程序是专门用于从网页上自动收集cmi,cnki上的被引文献的数据

package cn.ac.cintcm.spider;

import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.httpclient.NameValuePair;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.beans.FilterBean;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.InputTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

public class Get2FormContent extends GetFormContent{

	public Get2FormContent(String configFile) throws IOException {
		super(configFile);
	}
	public Get2FormContent(FormParameter formParameters) {
		super(formParameters);
	}
	
	public InputStream getContent() throws IOException {
		createHiddenPairValues();
	//	createHandPairValues();
		InputStream result = post(formParameters.getUrl2());	
        return result;
	}
	
	public void createHandPairValues() throws UnsupportedEncodingException {
		
	}

	public void createHiddenPairValues()throws IOException {  //get the other input parameters
		GetUrlContent guc=new GetUrlContent("http://lsg.cnki.net/grid20/Navigator.aspx?id=6");
		String resource=guc.getContent();
	
//		BufferedWriter outurl = new BufferedWriter(new FileWriter("out_url.txt"));		
		
        List<NameValuePair> newPairsList = null;
        try {
			Parser parser = new Parser(resource);
	        FilterBean bean = new FilterBean ();
	        bean.setFilters(getFilter());
	        bean.setParser(parser);
	        NodeList list = bean.getNodes();
	        newPairsList = new ArrayList<NameValuePair>();
	        for (Node node : list.toNodeArray()) {
	        	InputTag tag = new InputTag();	       	        		        	        	
	        	tag.setText(node.toHtml());
	        	String name = tag.getAttribute("name");	        	
	        	if (formParameters.getHiddenParameters().contains(name)) {
		        	String value = tag.getAttribute("value");		   		        			        			        			        	
		        	newPairsList.add(new NameValuePair(name, new String (value.getBytes("UTF-8"), "ISO-8859-1")));
	        		//newPairsList.add(new NameValuePair(name, value));        		
	        	}	        	
	        }
	        formParameters.getNameValues().addAll(newPairsList);	 	        
//	        outurl.close();
		} catch (ParserException e) {
			e.printStackTrace();
		}
	}

	protected void createExtraPairValues(InputTag tag) {
		
	}

	protected static NodeFilter[] getFilter() {
        TagNameFilter filter0 = new TagNameFilter ();
        filter0.setName ("INPUT");
        NodeFilter[] array0 = new NodeFilter[1];
        array0[0] = filter0;
        return array0;
	}

}