📄 extractorhtmltest.java
字号:
/* ExtractorHTMLTest * * Created on May 19, 2004 * * Copyright (C) 2004 Internet Archive. * * This file is part of the Heritrix web crawler (crawler.archive.org). * * Heritrix is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser Public License as published by * the Free Software Foundation; either version 2.1 of the License, or * any later version. * * Heritrix is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser Public License for more details. * * You should have received a copy of the GNU Lesser Public License * along with Heritrix; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */package org.archive.crawler.extractor;import java.io.File;import java.io.FileOutputStream;import java.io.IOException;import java.net.URL;import java.util.Collection;import java.util.Iterator;import javax.management.AttributeNotFoundException;import javax.management.InvalidAttributeValueException;import javax.management.MBeanException;import javax.management.ReflectionException;import org.apache.commons.collections.CollectionUtils;import org.apache.commons.collections.Predicate;import org.apache.commons.httpclient.URIException;import org.archive.crawler.datamodel.CoreAttributeConstants;import org.archive.crawler.datamodel.CrawlOrder;import org.archive.crawler.datamodel.CrawlURI;import org.archive.crawler.settings.MapType;import org.archive.crawler.settings.SettingsHandler;import org.archive.crawler.settings.XMLSettingsHandler;import org.archive.net.UURI;import org.archive.net.UURIFactory;import org.archive.util.HttpRecorder;import org.archive.util.TmpDirTestCase;/** * Test html extractor. * * @author stack * @version $Revision: 6153 $, $Date: 2009-03-03 03:13:49 +0000 (Tue, 03 Mar 2009) $ */public class ExtractorHTMLTestextends TmpDirTestCaseimplements CoreAttributeConstants { private final String ARCHIVE_DOT_ORG = "archive.org"; private final String LINK_TO_FIND = "http://www.hewlett.org/"; private HttpRecorder recorder = null; private ExtractorHTML extractor = null; protected ExtractorHTML createExtractor() throws InvalidAttributeValueException, AttributeNotFoundException, MBeanException, ReflectionException { // Hack in a settings handler. Do this by adding this extractor // to the order file (I'm adding it to a random MapType; seemingly // can only add to MapTypes post-construction). This takes care // of setting a valid SettingsHandler into the ExtractorHTML (This // shouldn't be so difficult). Of note, the order file below is // not written to disk. final String name = this.getClass().getName(); SettingsHandler handler = new XMLSettingsHandler( new File(getTmpDir(), name + ".order.xml")); handler.initialize(); return (ExtractorHTML)((MapType)handler.getOrder(). getAttribute(CrawlOrder.ATTR_RULES)).addElement(handler. getSettingsObject(null), new ExtractorHTML(name)); } protected void setUp() throws Exception { super.setUp(); this.extractor = createExtractor(); final boolean USE_NET = false; URL url = null; if (USE_NET) { url = new URL("http://" + this.ARCHIVE_DOT_ORG); } else { File f = new File(getTmpDir(), this.ARCHIVE_DOT_ORG + ".html"); url = f.toURI().toURL(); FileOutputStream fos = new FileOutputStream(f); fos.write(("<html><head><title>test</title><body>" + "<a href=" + this.LINK_TO_FIND + ">Hewlett Foundation</a>" + "</body></html>").getBytes()); fos.flush(); fos.close(); } this.recorder = HttpRecorder.wrapInputStreamWithHttpRecord(getTmpDir(), this.getClass().getName(), url.openStream(), null); } /* * @see TestCase#tearDown() */ protected void tearDown() throws Exception { super.tearDown(); } public void testInnerProcess() throws IOException { UURI uuri = UURIFactory.getInstance("http://" + this.ARCHIVE_DOT_ORG); CrawlURI curi = setupCrawlURI(this.recorder, uuri.toString()); this.extractor.innerProcess(curi); Collection links = curi.getOutLinks(); boolean foundLinkToHewlettFoundation = false; for (Iterator i = links.iterator(); i.hasNext();) { Link link = (Link)i.next(); if (link.getDestination().toString().equals(this.LINK_TO_FIND)) { foundLinkToHewlettFoundation = true; break; } } assertTrue("Did not find gif url", foundLinkToHewlettFoundation); } private CrawlURI setupCrawlURI(HttpRecorder rec, String url) throws URIException { CrawlURI curi = new CrawlURI(UURIFactory.getInstance(url)); curi.setContentSize(this.recorder.getRecordedInput().getSize()); curi.setContentType("text/html"); curi.setFetchStatus(200); curi.setHttpRecorder(rec); // Fake out the extractor that this is a HTTP transaction. curi.putObject(CoreAttributeConstants.A_HTTP_TRANSACTION, new Object()); return curi; } /** * Test single net or local filesystem page parse. * Set the uuri to be a net url or instead put in place a file * named for this class under the unit test directory. * @throws IOException * @throws ReflectionException * @throws MBeanException * @throws AttributeNotFoundException * @throws InvalidAttributeValueException */ public void testPageParse() throws InvalidAttributeValueException, AttributeNotFoundException, MBeanException, ReflectionException, IOException { UURI uuri = null; // DO// uuri = UURIFactory.getInstance("http://www.xjmu.edu.cn/");// OR// File f = new File(getTmpDir(), this.getClass().getName() +// ".html");// if (f.exists()) {// uuri = UURIFactory.getInstance("file://" +// f.getAbsolutePath());// }// OR // uuri = getUURI(URL or PATH)//// OR // Use the main method below and pass this class an argument.// if (uuri != null) { runExtractor(uuri); } } protected UURI getUURI(String url) throws URIException { url = (url.indexOf("://") > 0)? url: "file://" + url; return UURIFactory.getInstance(url); } protected void runExtractor(UURI baseUURI) throws InvalidAttributeValueException, AttributeNotFoundException, MBeanException, ReflectionException, IOException { runExtractor(baseUURI, null); } protected void runExtractor(UURI baseUURI, String encoding) throws IOException, InvalidAttributeValueException, AttributeNotFoundException, MBeanException, ReflectionException { if (baseUURI == null) { return; } this.extractor = createExtractor(); URL url = new URL(baseUURI.toString()); this.recorder = HttpRecorder. wrapInputStreamWithHttpRecord(getTmpDir(), this.getClass().getName(), url.openStream(), encoding); CrawlURI curi = setupCrawlURI(this.recorder, url.toString()); this.extractor.innerProcess(curi); System.out.println("+" + this.extractor.report()); int count = 0; Collection links = curi.getOutLinks(); System.out.println("+HTML Links (hopType="+Link.NAVLINK_HOP+"):"); if (links != null) { for (Iterator i = links.iterator(); i.hasNext();) { Link link = (Link)i.next(); if (link.getHopType()==Link.NAVLINK_HOP) { count++; System.out.println(link.getDestination()); } } } System.out.println("+HTML Embeds (hopType="+Link.EMBED_HOP+"):"); if (links != null) { for (Iterator i = links.iterator(); i.hasNext();) { Link link = (Link)i.next(); if (link.getHopType()==Link.EMBED_HOP) { count++; System.out.println(link.getDestination()); } } } System.out. println("+HTML Speculative Embeds (hopType="+Link.SPECULATIVE_HOP+"):"); if (links != null) { for (Iterator i = links.iterator(); i.hasNext();) { Link link = (Link)i.next(); if (link.getHopType()==Link.SPECULATIVE_HOP) { count++; System.out.println(link.getDestination()); } } } System.out. println("+HTML Other (all other hopTypes):"); if (links != null) { for (Iterator i = links.iterator(); i.hasNext();) {⌨️ 快捷键说明
复制代码
Ctrl + C
搜索代码
Ctrl + F
全屏模式
F11
切换主题
Ctrl + Shift + D
显示快捷键
?
增大字号
Ctrl + =
减小字号
Ctrl + -