connectionmanager.java

html to xml convertor

// HTMLParser Library $Name: v1_6 $ - A java-based parser for HTML
// http://sourceforge.org/projects/htmlparser
// Copyright (C) 2004 Derrick Oswald
//
// Revision Control Information
//
// $Source: /cvsroot/htmlparser/htmlparser/src/org/htmlparser/http/ConnectionManager.java,v $
// $Author: derrickoswald $
// $Date: 2006/06/02 02:43:24 $
// $Revision: 1.13 $
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//

package org.htmlparser.http;

import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.net.UnknownHostException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.Vector;

import org.htmlparser.util.ParserException;

/**
 * Handles proxies, password protected URLs and request properties
 * including cookies.
 */
public class ConnectionManager
{
    /**
     * Default Request header fields.
     * So far this is just "User-Agent" and "Accept-Encoding".
     */
    protected static Hashtable mDefaultRequestProperties = new Hashtable ();
    static
    {
        mDefaultRequestProperties.put ("User-Agent", "HTMLParser/"
            + org.htmlparser.lexer.Lexer.VERSION_NUMBER);
        mDefaultRequestProperties.put ("Accept-Encoding", "gzip, deflate");
    }

    /**
     * Messages for page not there (404).
     */
    private static final String[] FOUR_OH_FOUR =
    {
        "The web site you seek cannot be located,"
            + " but countless more exist",
        "You step in the stream, but the water has moved on."
            + " This page is not here.",
        "Yesterday the page existed. Today it does not."
            + " The internet is like that.",
        "That page was so big. It might have been very useful."
            + " But now it is gone.",
        "Three things are certain: death, taxes and broken links."
            + " Guess which has occured.",
        "Chaos reigns within. Reflect, repent and enter the correct URL."
            + " Order shall return.",
        "Stay the patient course. Of little worth is your ire."
            + " The page is not found.",
        "A non-existant URL reduces your expensive computer to a simple stone.",
        "Many people have visited that page."
            + " Today, you are not one of the lucky ones.",
        "Cutting the wind with a knife. Bookmarking a URL."
            + " Both are ephemeral.",
    };

    /**
     * Base 64 character translation table.
     */
    private static final char[] BASE64_CHAR_TABLE =
         ("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
        + "abcdefghijklmnopqrstuvwxyz0123456789+/").toCharArray ();

    /**
     * Request header fields.
     */
    protected Hashtable mRequestProperties;

    /**
     * The proxy server name.
     */
    protected String mProxyHost;

    /**
     * The proxy port number.
     */
    protected int mProxyPort;

    /**
     * The proxy username name.
     */
    protected String mProxyUser;

    /**
     * The proxy user password.
     */
    protected String mProxyPassword;

    /**
     * The username name for accessing the URL.
     */
    protected String mUser;

    /**
     * The user password for accessing the URL.
     */
    protected String mPassword;

    /**
     * Cookie storage, a hashtable (by site or host) of vectors of Cookies.
     * This will be null if cookie processing is disabled (default).
     */
    protected Hashtable mCookieJar;

    /**
     * The object to be notified prior to and after each connection.
     */
    protected ConnectionMonitor mMonitor;

    /**
     * Flag determining if redirection processing is being handled manually.
     */
    protected boolean mRedirectionProcessingEnabled;

    /**
     * Cookie expiry date format for parsing.
     */
    static protected SimpleDateFormat mFormat =
        new SimpleDateFormat ("EEE, dd-MMM-yy kk:mm:ss z");

    /**
     * Create a connection manager.
     */
    public ConnectionManager ()
    {
        this (getDefaultRequestProperties ());
    }

    /**
     * Create a connection manager with the given connection properties.
     * @param properties Name/value pairs to be added to the HTTP request.
     */
    public ConnectionManager (Hashtable properties)
    {
        mRequestProperties = properties;
        mProxyHost = null;
        mProxyPort = 0;
        mProxyUser = null;
        mProxyPassword = null;
        mUser = null;
        mPassword = null;
        mCookieJar = null;
        mMonitor = null;
        mRedirectionProcessingEnabled = false;
    }

    //
    // static methods
    //

    /**
     * Get the current default request header properties.
     * A String-to-String map of header keys and values.
     * These fields are set by the parser when creating a connection.
     * @return The default set of request header properties that will
     * currently be used.
     * @see #mDefaultRequestProperties
     * @see #setRequestProperties
     */
    public static Hashtable getDefaultRequestProperties ()
    {
        return (mDefaultRequestProperties);
    }

    /**
     * Set the default request header properties.
     * A String-to-String map of header keys and values.
     * These fields are set by the parser when creating a connection.
     * Some of these can be set directly on a <code>URLConnection</code>,
     * i.e. If-Modified-Since is set with setIfModifiedSince(long),
     * but since the parser transparently opens the connection on behalf
     * of the developer, these properties are not available before the
     * connection is fetched. Setting these request header fields affects all
     * subsequent connections opened by the parser. For more direct control
     * create a <code>URLConnection</code> massage it the way you want and
     * then set it on the parser.<p>
     * From <a href="http://www.ietf.org/rfc/rfc2616.txt">
     * RFC 2616 Hypertext Transfer Protocol -- HTTP/1.1</a>: 
     * <pre>
     * 5.3 Request Header Fields
     *
     *    The request-header fields allow the client to pass additional
     *    information about the request, and about the client itself, to the
     *    server. These fields act as request modifiers, with semantics
     *    equivalent to the parameters on a programming language method
     *    invocation.
     *
     *        request-header = Accept                   ; Section 14.1
     *                       | Accept-Charset           ; Section 14.2
     *                       | Accept-Encoding          ; Section 14.3
     *                       | Accept-Language          ; Section 14.4
     *                       | Authorization            ; Section 14.8
     *                       | Expect                   ; Section 14.20
     *                       | From                     ; Section 14.22
     *                       | Host                     ; Section 14.23
     *                       | If-Match                 ; Section 14.24
     *                       | If-Modified-Since        ; Section 14.25
     *                       | If-None-Match            ; Section 14.26
     *                       | If-Range                 ; Section 14.27
     *                       | If-Unmodified-Since      ; Section 14.28
     *                       | Max-Forwards             ; Section 14.31
     *                       | Proxy-Authorization      ; Section 14.34
     *                       | Range                    ; Section 14.35
     *                       | Referer                  ; Section 14.36
     *                       | TE                       ; Section 14.39
     *                       | User-Agent               ; Section 14.43
     *
     *    Request-header field names can be extended reliably only in
     *    combination with a change in the protocol version. However, new or
     *    experimental header fields MAY be given the semantics of request-
     *    header fields if all parties in the communication recognize them to
     *    be request-header fields. Unrecognized header fields are treated as
     *    entity-header fields.
     * </pre>
     * @param properties The new set of default request header properties to
     * use. This affects all subsequently created connections.
     * @see #mDefaultRequestProperties
     * @see #setRequestProperties
     */
    public static void setDefaultRequestProperties (Hashtable properties)
    {
        mDefaultRequestProperties = properties;
    }

    /**
     * Get the current request header properties.
     * A String-to-String map of header keys and values,
     * excluding proxy items, cookies and URL authorization.
     * @return The request header properties for this connection manager.
     */
    public Hashtable getRequestProperties ()
    {
        return (mRequestProperties);
    }

    /**
     * Set the current request properties.
     * Replaces the current set of fixed request properties with the given set.
     * This does not replace the Proxy-Authorization property which is
     * constructed from the values of {@link #setProxyUser}
     * and {@link #setProxyPassword} values or the Authorization property
     * which is constructed from the {@link #setUser}
     * and {@link #setPassword} values. Nor does it replace the
     * Cookie property which is constructed from the current cookie jar.
     * @param properties The new fixed properties.
     */
    public void setRequestProperties (Hashtable properties)
    {
        mRequestProperties = properties;
    }

    /**
     * Get the proxy host name, if any.
     * @return Returns the proxy host.
     */
    public String getProxyHost ()
    {
        return (mProxyHost);
    }

    /**
     * Set the proxy host to use.
     * @param host The host to use for proxy access.
     * <em>Note: You must also set the proxy {@link #setProxyPort port}.</em>
     */
    public void setProxyHost (String host)
    {
        mProxyHost = host;
    }

    /**
     * Get the proxy port number.
     * @return Returns the proxy port.
     */
    public int getProxyPort ()
    {
        return (mProxyPort);
    }

    /**
     * Set the proxy port number.
     * @param port The proxy port.
     * <em>Note: You must also set the proxy {@link #setProxyHost host}.</em>
     */
    public void setProxyPort (int port)
    {
        mProxyPort = port;
    }

    /**
     * Get the user name for proxy authorization, if any.
     * @return Returns the proxy user,
     * or <code>null</code> if no proxy authorization is required.
     */
    public String getProxyUser ()
    {
        return (mProxyUser);
    }

    /**
     * Set the user name for proxy authorization.
     * @param user The proxy user name.
     * <em>Note: You must also set the proxy {@link #setProxyPassword password}.</em>
     */
    public void setProxyUser (String user)
    {
        mProxyUser = user;
    }

    /**
     * Set the proxy user's password.
     * @return Returns the proxy password.
     */
    public String getProxyPassword ()
    {
        return (mProxyPassword);
    }

    /**
     * Get the proxy user's password.
     * @param password The password for the proxy user.
     * <em>Note: You must also set the proxy {@link #setProxyUser user}.</em>
     */
    public void setProxyPassword (String password)
    {
        mProxyPassword = password;
    }

    /**
     * Get the user name to access the URL.
     * @return Returns the username that will be used to access the URL,
     * or <code>null</code> if no authorization is required.
     */
    public String getUser ()
    {
        return (mUser);
    }

    /**
     * Set the user name to access the URL.
     * @param user The user name for accessing the URL.
     * <em>Note: You must also set the {@link #setPassword password}.</em>
     */
    public void setUser (String user)
    {
        mUser = user;
    }

    /**
     * Get the URL users's password.
     * @return Returns the URL password.
     */
    public String getPassword ()
    {
        return (mPassword);
    }

    /**
     * Set the URL users's password.
     * @param password The password for the URL.
     */
    public void setPassword (String password)
    {
        mPassword = password;
    }

    /**
     * Predicate to determine if cookie processing is currently enabled.
     * @return <code>true</code> if cookies are being processed.
     */
    public boolean getCookieProcessingEnabled ()
    {
        return (null != mCookieJar);
    }

    /**
     * Enables and disabled cookie processing.
     * @param enable if <code>true</code> cookie processing will occur,
     * else cookie processing will be turned off.
     */