parsedocument.java

用httpclient-4.0-alpha2 打造基于http协议的网站分析器

package com.ue.browser.util;

import java.io.StringReader;
import java.lang.reflect.Constructor;
import java.util.ArrayList;
import java.util.HashMap;

import org.cyberneko.html.parsers.DOMParser;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;

import com.ue.browser.Browser;
import com.ue.browser.core.ElementCode;
import com.ue.browser.core.HTMLElement;
import com.ue.browser.Frame;
import com.ue.browser.Iframe;

public class ParseDocument {
	private ArrayList<HTMLElement> elements = new ArrayList<HTMLElement>();
	private Browser browser;
	private HTMLElement element;
	private String text;
	private InputSource source;
	private static DOMParser parser = new DOMParser();
	
	public boolean haveIframe = false;
	public boolean haveFrame = false;
	
	public ParseDocument(Browser browser) {
		super();
		this.browser = browser;
	}

	public ArrayList<HTMLElement> getElements() {
		return elements;
	}

	public void setElements(ArrayList<HTMLElement> elements) {
		this.elements = elements;
	}
	
	public Browser getBrowser() {
		return browser;
	}

	public void setBrowser(Browser browser) {
		this.browser = browser;
	}
	
	public HTMLElement getElement() {
		return element;
	}

	public void setElement(HTMLElement element) {
		this.element = element;
	}
	
	
   
	private static HashMap<String, String> elementMap = ElementCode.getElements();
	private static String thePack = "com.ue.browser.html.";
	
	public void parse(String body){
		try{
			source = new InputSource(new StringReader(body));
			//是否允许命名空间
			parser.setFeature("http://xml.org/sax/features/namespaces", false);
			//是否允许增补缺失的标签。如果要以XML方式操作HTML文件，此值必须为真
			parser.setFeature("http://cyberneko.org/html/features/balance-tags", true);
			//是否剥掉<script>元素中的<!-- -->等注释符
			parser.setFeature("http://cyberneko.org/html/features/scanner/script/strip-comment-delims", true);
			parser.setFeature("http://cyberneko.org/html/features/augmentations", true);
			parser.setProperty("http://cyberneko.org/html/properties/names/elems", "lower");
			
			//解析HTML片段时是否作标签增补，此功能不要用在DOMParser上，而要用在DOMFragmentParser上
			//parser.setFeature("http://cyberneko.org/features/document-fragment",true);
			//添加管道过滤器
			/*XMLDocumentFilter noop = new DefaultFilter();
			XMLDocumentFilter[] filters = { noop };
			parser.setProperty("http://cyberneko.org/html/properties/filters", filters);*/

			parser.parse(source);
			
			//cyberneko解析：将httpclien读取的数据source解析成w3c的Document对象
			this.buildElements(parser.getDocument());
		}catch(Exception e){
			e.printStackTrace();
		}
	}
	
    //从w3c的Document节点对象中获取节点的属性放入对应的类中
	public void buildElements(Node node){
		int type = node.getNodeType();
		switch (type) {
			// print the document element
			case Node.DOCUMENT_NODE: {
				this.buildElements(((org.w3c.dom.Document) node).getDocumentElement());
				break;
			}
	
			// print element with attributes
			case Node.ELEMENT_NODE: {
				String name = node.getNodeName();
				String clazz = elementMap.get(name);
				try{
					if(clazz!=null){
						Class<?> tt = Class.forName(thePack+clazz);
			            Constructor<?> c = tt.getConstructor(Browser.class);
			            element = (HTMLElement)c.newInstance(this.getBrowser());
			            NamedNodeMap attrs = node.getAttributes();
						for (int i = 0; i < attrs.getLength(); i++) {
							Node attr = attrs.item(i);
							element.setAttribute(attr.getNodeName(), attr.getNodeValue());
						}
			                elements.add(element);
					}
				}catch(Exception e){
					e.printStackTrace();
				}
				
				//查询子节点表，并构建子节点属性
				NodeList children = node.getChildNodes();
				if (children != null) {
					int len = children.getLength();
					for (int i = 0; i < len; i++)
						this.buildElements(children.item(i));
				}
				break;
			}
	
			// handle entity reference nodes
			case Node.ENTITY_REFERENCE_NODE: {
				break;
			}
	
			// print cdata sections
			case Node.CDATA_SECTION_NODE: {
				break;
			}
	
			// print text
			case Node.TEXT_NODE: {
				text = node.getNodeValue();
				break;
			}

            /*//print script 注释节点<!--   -->   
			case Node.COMMENT_NODE: {
				script = node.getNodeValue();
				System.out.println(script);
				break;
			}*/
			
			// print processing instruction
			case Node.PROCESSING_INSTRUCTION_NODE: {
				break;
			}
		}

		    if (type == Node.ELEMENT_NODE) {
			   if(node.getNodeName().equals("TEXTAREA")){
				element.setAttribute("value", text);
			}
			
			if (type == Node.ELEMENT_NODE) {
				if(node.getNodeName().equals("IFRAME")){
					//如果有IFRAME节点，进行IFRAME处理
					haveIframe = true;
				}
			}
				
			if (type == Node.ELEMENT_NODE) {
				if(node.getNodeName().equals("FRAME")){
				    //如果有FRAME节点，进行FRAME处理
					haveFrame = true;	
				}
			}
			
		    if(node.getNodeName().equals("SCRIPT")){
			if(text!=null){
				element.setAttribute("daima", text);
			 }
			}
		}
	}
	
	//获得当前节点的父节点名称
	public String getParentNodeName(Node node){
		Node parent = node.getParentNode();
		String name = parent.getNodeName();
		return name;
		}

	public boolean isHaveFrame() {
		return haveFrame;
	}

	public void setHaveFrame(boolean haveFrame) {
		this.haveFrame = haveFrame;
	}

	public boolean isHaveIframe() {
		return haveIframe;
	}

	public void setHaveIframe(boolean haveIframe) {
		this.haveIframe = haveIframe;
	}

	
}