parsertest.java

本程序用于对页面信息进行提取并分析

// HTMLParser Library $Name: v1_6_20060319 $ - A java-based parser for HTML
// http://sourceforge.org/projects/htmlparser
// Copyright (C) 2004 Somik Raha
//
// Revision Control Information
//
// $Source: /cvsroot/htmlparser/htmlparser/src/org/htmlparser/tests/ParserTest.java,v $
// $Author: derrickoswald $
// $Date: 2005/04/24 17:48:27 $
// $Revision: 1.65 $
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//

package org.htmlparser.tests;

import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Locale;

import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.PrototypicalNodeFactory;
import org.htmlparser.Remark;
import org.htmlparser.Tag;
import org.htmlparser.Text;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.lexer.InputStreamSource;
import org.htmlparser.tags.BodyTag;
import org.htmlparser.tags.ImageTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.tags.MetaTag;
import org.htmlparser.util.DefaultParserFeedback;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.htmlparser.util.SimpleNodeIterator;

public class ParserTest extends ParserTestCase
{
    static
    {
        System.setProperty ("org.htmlparser.tests.ParserTest", "ParserTest");
    }

    public ParserTest(String name) {
        super(name);
    }
    public void testElements() throws Exception {
        StringBuffer hugeData = new StringBuffer();
        for (int i=0;i<5001;i++) hugeData.append('a');
        createParser(hugeData.toString());
        int i = 0;
        for (NodeIterator e = parser.elements();e.hasMoreNodes();)
        {
            node[i++] = e.nextNode();
        }
        assertEquals("There should be 1 node identified",1,i);
        // Now try getting the elements again
//      i = 0;
//      reader.reset();
//      reader.setLineCount(1);
//      reader.setPosInLine(-1);
//      for (HTMLEnumeration e = parser.elements();e.hasMoreNodes();)
//      {
//          node[i++] = e.nextHTMLNode();
//      }
//      assertEquals("There should be 1 node identified (second call to parser.elements())",1,i);
    }

    /**
     * This testcase needs you to be online.
     */
    public void testElementsFromWeb() throws Exception {
        Parser parser;
        try {
            parser = new Parser("http://www.google.com");
        }
        catch (Exception e ){
            throw new ParserException("You must be offline! This test needs you to be connected to the internet.",e);
        }

        Node[] node = new Node[500];
        int i = 0;
        for (NodeIterator e = parser.elements();e.hasMoreNodes();)
        {
            node[i++] = e.nextNode();
        }
        int cnt = i;
        parser.reset ();
        // Now try getting the elements again
        i = 0;
        for (NodeIterator e = parser.elements();e.hasMoreNodes();)
        {
            node[i++] = e.nextNode();
        }
        assertEquals("There should be "+cnt+" nodes identified (second call to parser.elements())",cnt,i);
    }

    /**
     * Test the Parser(URLConnection) constructor.
     * This testcase needs you to be online.
     * Based on the form at Canada Post <code>http://www.canadapost.ca/tools/pcl/bin/default-e.asp</code>:
     * <pre>
     * &lt;form NAME="SearchQuick" method="POST" action="cp_search_response-e.asp"
     *  onSubmit="return runSubmit();"&gt;
     *
     * &lt;!-- begin test hidden field code --&gt;
     *   &lt;input TYPE="Hidden" NAME="app_language" value="english"&gt;
     *
     *   &lt;input TYPE="Hidden" NAME="app_response_start_row_number" value="1"&gt;
     *   &lt;input TYPE="Hidden" NAME="app_response_rows_max" value="9"&gt;
     *
     *   &lt;input TYPE="Hidden" NAME="app_source" value="quick"&gt;
     *   &lt;input TYPE="Hidden" NAME="query_source" value="q"&gt;
     *
     *   &lt;input TYPE="Hidden" NAME="name" value&gt;
     *   &lt;input TYPE="Hidden" NAME="postal_code" value&gt;
     *   &lt;input TYPE="Hidden" NAME="directory_area_name" value&gt;
     *
     *   &lt;input TYPE="Hidden" NAME="delivery_mode" value&gt;
     *   &lt;input TYPE="Hidden" NAME="Suffix" value&gt;
     *
     *   &lt;input TYPE="Hidden" NAME="street_direction" value&gt;
     *   &lt;input TYPE="Hidden" NAME="installation_type" value&gt;
     *   &lt;input TYPE="Hidden" NAME="delivery_number" value&gt;
     *   &lt;input TYPE="Hidden" NAME="installation_name" value&gt;
     *   &lt;input TYPE="Hidden" NAME="unit_number" value&gt;
     *
     *   &lt;input TYPE="Hidden" NAME="app_state" value="production"&gt;
     * &lt;!-- end test hidden field code --&gt;
     *
     * &lt;p&gt;
     *   &lt;table border="0" cellpadding="0" width="90%" cellspacing="0"&gt;
     *
     *     &lt;tr&gt;
     *       &lt;td  class="tbltitle"&gt; Street Number: &lt;/td&gt;
     *       &lt;td class="tbltitle"&gt; Street Name: &lt;/td&gt;
     *       &lt;td class="tbltitle"&gt; Street Type:&lt;/td&gt;
     *     &lt;/tr&gt;
     *     &lt;tr&gt;
     *
     *       &lt;td&gt;
     *         &lt;input type="text" name="street_number" size="10" maxlength="10"&gt;
     *       &lt;/td&gt;
     *       &lt;td&gt;
     *         &lt;input type="text" name="street_name" size="30" maxlength="40"&gt;
     *         &lt;input type="hidden" name="street_type" size="30"&gt;
     *       &lt;/td&gt;
     *       &lt;td&gt;&lt;input type="text" name="test" size="10" maxlength="30"&gt;&lt;/td&gt;
     *     &lt;/tr&gt;
     *
     *   &lt;/table&gt;
     * &lt;p&gt;
     *   &lt;table border="0" cellpadding="0" width="90%" cellspacing="0"&gt;
     *     &lt;tr&gt;
     *       &lt;td class="tbltitle"&gt;
     *         Municipality (City, Town, etc.):
     *       &lt;/td&gt;
     *       &lt;td class="tbltitle"&gt;
     *         Province:
     *       &lt;/td&gt;
     *
     *     &lt;/tr&gt;
     *     &lt;tr&gt;
     *       &lt;td&gt;
     *         &lt;input type="text" name="city" size="30" maxlength="30"&gt;
     *       &lt;/td&gt;
     *       &lt;td&gt;
     *         &lt;select size="1" name="prov"&gt;
     *           &lt;option selected value="NULL"&gt;Select&lt;/option&gt;&lt;option value="AB"&gt;AB - Alberta&lt;/option&gt;&lt;option value="BC"&gt;BC - British Columbia&lt;/option&gt;&lt;option value="MB"&gt;MB - Manitoba&lt;/option&gt;&lt;option value="NB"&gt;NB - New Brunswick&lt;/option&gt;&lt;option value="NL"&gt;NL - Newfoundland and Labrador&lt;/option&gt;&lt;option value="NS"&gt;NS - Nova Scotia&lt;/option&gt;&lt;option value="NT"&gt;NT - Northwest Territories&lt;/option&gt;&lt;option value="NU"&gt;NU - Nunavut&lt;/option&gt;&lt;option value="ON"&gt;ON - Ontario&lt;/option&gt;&lt;option value="PE"&gt;PE - Prince Edward Island&lt;/option&gt;&lt;option value="QC"&gt;QC - Quebec&lt;/option&gt;&lt;option value="SK"&gt;SK - Saskatchewan&lt;/option&gt;&lt;option value="YT"&gt;YT - Yukon&lt;/option&gt;
     *
     *         &lt;/select&gt;
     *       &lt;/td&gt;
     *     &lt;/tr&gt;
     *     &lt;tr&gt;
     *       &lt;td height="10"&gt;&amp;nbsp;&lt;/td&gt;
     *       &lt;td&gt;&amp;nbsp;&lt;/td&gt;
     *     &lt;/tr&gt;
     *     &lt;tr&gt;
     *       &lt;td colspan="2" align="right" nowrap&gt;
     *     &lt;input type="image" src="images/bb_submit-e.gif" name="Search" border="0" WIDTH="88" HEIGHT="23"&gt;
     *         &amp;nbsp; &lt;a href="#" onclick="javascript:fClearAllFields();"&gt;&lt;img src="images/bb_clear_form-e.gif" name="Clear" border="0" WIDTH="88" HEIGHT="23"&gt;&lt;/a&gt;
     *    &lt;/td&gt;
     *     &lt;/tr&gt;
     *   &lt;/table&gt;
     * &lt;p&gt;
     * &lt;/form&gt;
     * </pre>
     * Sumbits the POST and verifies the returned HTML contains an expected value.
     */
    public void testPOST() throws Exception
    {   // the form data:
        final String number = "2708";
        final String street = "Kelly";
        final String type = "Avenue";
        final String city = "Ottawa";
        final String province = "ON";
        // the correct answer
        final String postal_code = "K2B 7V4";

        Parser parser;
        URL url;
        HttpURLConnection connection;
        StringBuffer buffer;
        PrintWriter out;
        boolean pass;
        NodeIterator enumeration;
        Node node;
        Text string;

        try
        {
            url = new URL ("http://www.canadapost.ca/tools/pcl/bin/cp_search_response-e.asp");
             connection = (HttpURLConnection)url.openConnection ();
            connection.setRequestMethod ("POST");
            connection.setRequestProperty ("Referer", "http://www.canadapost.ca/tools/pcl/bin/default-e.asp");
            connection.setDoOutput (true);
            connection.setDoInput (true);
            connection.setUseCaches (false);
            buffer = new StringBuffer (1024);
            buffer.append ("app_language=");
            buffer.append ("english");
            buffer.append ("&");
            buffer.append ("app_response_start_row_number=");
            buffer.append ("1");
            buffer.append ("&");
            buffer.append ("app_response_rows_max=");
            buffer.append ("9");
            buffer.append ("&");
            buffer.append ("app_source=");
            buffer.append ("quick");
            buffer.append ("&");
            buffer.append ("query_source=");
            buffer.append ("q");
            buffer.append ("&");
            buffer.append ("name=");
            buffer.append ("&");
            buffer.append ("postal_code=");
            buffer.append ("&");
            buffer.append ("directory_area_name=");
            buffer.append ("&");
            buffer.append ("delivery_mode=");
            buffer.append ("&");
            buffer.append ("Suffix=");
            buffer.append ("&");
            buffer.append ("street_direction=");
            buffer.append ("&");
            buffer.append ("installation_type=");
            buffer.append ("&");
            buffer.append ("delivery_number=");
            buffer.append ("&");
            buffer.append ("installation_name=");
            buffer.append ("&");
            buffer.append ("unit_numbere=");
            buffer.append ("&");
            buffer.append ("app_state=");
            buffer.append ("production");
            buffer.append ("&");
            buffer.append ("street_number=");
            buffer.append (number);
            buffer.append ("&");
            buffer.append ("street_name=");
            buffer.append (street);
            buffer.append ("&");
            buffer.append ("street_type=");
            buffer.append (type);
            buffer.append ("&");
            buffer.append ("test=");
            buffer.append ("&");
            buffer.append ("city=");
            buffer.append (city);
            buffer.append ("&");
            buffer.append ("prov=");
            buffer.append (province);
            buffer.append ("&");
            buffer.append ("Search=");
            out = new PrintWriter (connection.getOutputStream ());
            out.print (buffer);
            out.close ();
            parser = new Parser (connection);
            parser.setNodeFactory (new PrototypicalNodeFactory (true));
        }
        catch (Exception e)
        {
            throw new ParserException ("You must be offline! This test needs you to be connected to the internet.", e);
        }

        pass = false;
        for (enumeration = parser.elements (); enumeration.hasMoreNodes ();)
        {
            node = enumeration.nextNode ();
            if (node instanceof Text)
            {
                string = (Text)node;
                if (-1 != string.getText ().indexOf (postal_code))
                    pass = true;
            }
        }
        assertTrue("POST operation failed.", pass);
    }

    /**
     * Tests the 'from file' Parser constructor.
     */
    public void testFile ()
    {