httpmethodbase.java

爬虫

/*
 * $Header: /cvsroot/archive-crawler/ArchiveOpenCrawler/src/java/org/apache/commons/httpclient/HttpMethodBase.java,v 1.10 2005/08/12 23:42:58 gojomo Exp $
 * $Revision: 1.10 $
 * $Date: 2005/08/12 23:42:58 $
 *
 * ====================================================================
 *
 *  Copyright 1999-2004 The Apache Software Foundation
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 * ====================================================================
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of the Apache Software Foundation.  For more
 * information on the Apache Software Foundation, please see
 * <http://www.apache.org/>.
 *
 */

package org.apache.commons.httpclient;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;

import org.apache.commons.httpclient.auth.AuthState;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.cookie.CookieSpec;
import org.apache.commons.httpclient.cookie.MalformedCookieException;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.httpclient.protocol.Protocol;
import org.apache.commons.httpclient.util.EncodingUtil;
import org.apache.commons.httpclient.util.ExceptionUtil;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * An abstract base implementation of HttpMethod.
 * <p>
 * At minimum, subclasses will need to override:
 * <ul>
 *   <li>{@link #getName} to return the approriate name for this method
 *   </li>
 * </ul>
 * </p>
 *
 * <p>
 * When a method requires additional request headers, subclasses will typically
 * want to override:
 * <ul>
 *   <li>{@link #addRequestHeaders addRequestHeaders(HttpState,HttpConnection)}
 *      to write those headers
 *   </li>
 * </ul>
 * </p>
 *
 * <p>
 * When a method expects specific response headers, subclasses may want to
 * override:
 * <ul>
 *   <li>{@link #processResponseHeaders processResponseHeaders(HttpState,HttpConnection)}
 *     to handle those headers
 *   </li>
 * </ul>
 * </p>
 *
 *
 * @author <a href="mailto:[e-mail]">Remy Maucherat</a>
 * @author Rodney Waldhoff
 * @author Sean C. Sullivan
 * @author <a href="mailto:[e-mail]">dIon Gillard</a>
 * @author <a href="mailto:[e-mail]">Jeff Dever</a>
 * @author <a href="mailto:[e-mail]">Davanum Srinivas</a>
 * @author Ortwin Glueck
 * @author Eric Johnson
 * @author Michael Becke
 * @author <a href="mailto:[e-mail]">Oleg Kalnichevski</a>
 * @author <a href="mailto:[e-mail]">Mike Bowler</a>
 * @author <a href="mailto:[e-mail]">Gary Gregory</a>
 * @author Christian Kohlschuetter
 *
 * @version $Revision: 1.10 $ $Date: 2005/08/12 23:42:58 $
 */
public abstract class HttpMethodBase implements HttpMethod {

    // -------------------------------------------------------------- Constants

    /** Log object for this class. */
    private static final Log LOG = LogFactory.getLog(HttpMethodBase.class);

    // ----------------------------------------------------- Instance variables 

    /** Request headers, if any. */
    private HeaderGroup requestHeaders = new HeaderGroup();

    /** The Status-Line from the response. */
    private StatusLine statusLine = null;

    /** Response headers, if any. */
    private HeaderGroup responseHeaders = new HeaderGroup();

    /** Response trailer headers, if any. */
    private HeaderGroup responseTrailerHeaders = new HeaderGroup();

    /** Path of the HTTP method. */
    private String path = null;

    /** Query string of the HTTP method, if any. */
    private String queryString = null;

    /** The response body of the HTTP method, assuming it has not be 
     * intercepted by a sub-class. */
    private InputStream responseStream = null;

    /** The connection that the response stream was read from. */
    private HttpConnection responseConnection = null;

    /** Buffer for the response */
    private byte[] responseBody = null;

    /** True if the HTTP method should automatically follow HTTP redirects.*/
    private boolean followRedirects = false;

    /** True if the HTTP method should automatically handle
    *  HTTP authentication challenges. */
    private boolean doAuthentication = true;

    /** HTTP protocol parameters. */
    private HttpMethodParams params = new HttpMethodParams();

    /** Host authentication state */
    private AuthState hostAuthState = new AuthState();

    /** Proxy authentication state */
    private AuthState proxyAuthState = new AuthState();

    /** True if this method has already been executed. */
    private boolean used = false;

    /** Count of how many times did this HTTP method transparently handle 
    * a recoverable exception. */
    private int recoverableExceptionCount = 0;

    /** the host for this HTTP method, can be null */
    private HttpHost httphost = null;

    /**
     * Handles method retries
     * 
     * @deprecated no loner used
     */
    private MethodRetryHandler methodRetryHandler;

    /** True if the connection must be closed when no longer needed */
    private boolean connectionCloseForced = false;

    /** Number of milliseconds to wait for 100-contunue response. */
    private static final int RESPONSE_WAIT_TIME_MS = 3000;

    /** HTTP protocol version used for execution of this method. */
    private HttpVersion effectiveVersion = null;

    /** Whether the execution of this method has been aborted */
    private transient boolean aborted = false;

    /** Whether the HTTP request has been transmitted to the target
     * server it its entirety */
    private boolean requestSent = false;
    
    /** Actual cookie policy */
    private CookieSpec cookiespec = null;

    /** Default initial size of the response buffer if content length is unknown. */
    private static final int DEFAULT_INITIAL_BUFFER_SIZE = 4*1024; // 4 kB
    
    // ----------------------------------------------------------- Constructors

    /**
     * No-arg constructor.
     */
    public HttpMethodBase() {
    }

    /**
     * Constructor specifying a URI.
     * It is responsibility of the caller to ensure that URI elements
     * (path & query parameters) are properly encoded (URL safe).
     *
     * @param uri either an absolute or relative URI. The URI is expected
     *            to be URL-encoded
     * 
     * @throws IllegalArgumentException when URI is invalid
     * @throws IllegalStateException when protocol of the absolute URI is not recognised
     */
    public HttpMethodBase(String uri) 
        throws IllegalArgumentException, IllegalStateException {

        try {

            // create a URI and allow for null/empty uri values
            if (uri == null || uri.equals("")) {
                uri = "/";
            }
// BEGIN IA CHANGES
//        setURI(new URI(uri, true));
          setURI(new org.archive.net.LaxURI(uri, true));
// END IA CHANGES
        } catch (URIException e) {
            throw new IllegalArgumentException("Invalid uri '" 
                + uri + "': " + e.getMessage() 
            );
        }
    }

    // ------------------------------------------- Property Setters and Getters

    /**
     * Obtains the name of the HTTP method as used in the HTTP request line,
     * for example <tt>"GET"</tt> or <tt>"POST"</tt>.
     * 
     * @return the name of this method
     */
    public abstract String getName();

    /**
     * Returns the URI of the HTTP method
     * 
     * @return The URI
     * 
     * @throws URIException If the URI cannot be created.
     * 
     * @see org.apache.commons.httpclient.HttpMethod#getURI()
     */
    public URI getURI() throws URIException {
        StringBuffer buffer = new StringBuffer();
        if (this.httphost != null) {
            buffer.append(this.httphost.getProtocol().getScheme());
            buffer.append("://");
            buffer.append(this.httphost.getHostName());
            int port = this.httphost.getPort();
            if (port != -1 && port != this.httphost.getProtocol().getDefaultPort()) {
                buffer.append(":");
                buffer.append(port);
            }
        }
        buffer.append(this.path);
        if (this.queryString != null) {
            buffer.append('?');
            buffer.append(this.queryString);
        }
        return new URI(buffer.toString(), true);
    }

    /**
     * Sets the URI for this method. 
     * 
     * @param uri URI to be set 
     * 
     * @throws URIException if a URI cannot be set
     * 
     * @since 3.0
     */
    public void setURI(URI uri) throws URIException {
        // only set the host if specified by the URI
        if (uri.isAbsoluteURI()) {
            this.httphost = new HttpHost(uri);
        }
        // set the path, defaulting to root
        setPath(
            uri.getPath() == null
            ? "/"
            : uri.getEscapedPath()
        );
        setQueryString(uri.getEscapedQuery());
    } 

    /**
     * Sets whether or not the HTTP method should automatically follow HTTP redirects 
     * (status code 302, etc.)
     * 
     * @param followRedirects <tt>true</tt> if the method will automatically follow redirects,
     * <tt>false</tt> otherwise.
     */
    public void setFollowRedirects(boolean followRedirects) {
        this.followRedirects = followRedirects;
    }

    /**
     * Returns <tt>true</tt> if the HTTP method should automatically follow HTTP redirects 
     * (status code 302, etc.), <tt>false</tt> otherwise.
     * 
     * @return <tt>true</tt> if the method will automatically follow HTTP redirects, 
     * <tt>false</tt> otherwise.
     */
    public boolean getFollowRedirects() {
        return this.followRedirects;
    }

    /** Sets whether version 1.1 of the HTTP protocol should be used per default.
     *
     * @param http11 <tt>true</tt> to use HTTP/1.1, <tt>false</tt> to use 1.0
     * 
     * @deprecated Use {@link HttpMethodParams#setVersion(HttpVersion)}
     */
    public void setHttp11(boolean http11) {
        if (http11) {
            this.params.setVersion(HttpVersion.HTTP_1_1);
        } else {
            this.params.setVersion(HttpVersion.HTTP_1_0);
        } 
    }

    /**
     * Returns <tt>true</tt> if the HTTP method should automatically handle HTTP 
     * authentication challenges (status code 401, etc.), <tt>false</tt> otherwise
     *
     * @return <tt>true</tt> if authentication challenges will be processed 
     * automatically, <tt>false</tt> otherwise.
     * 
     * @since 2.0
     */
    public boolean getDoAuthentication() {
        return doAuthentication;
    }

    /**
     * Sets whether or not the HTTP method should automatically handle HTTP 
     * authentication challenges (status code 401, etc.)
     *
     * @param doAuthentication <tt>true</tt> to process authentication challenges
     * authomatically, <tt>false</tt> otherwise.
     * 
     * @since 2.0
     */
    public void setDoAuthentication(boolean doAuthentication) {
        this.doAuthentication = doAuthentication;
    }

    // ---------------------------------------------- Protected Utility Methods

    /**
     * Returns <tt>true</tt> if version 1.1 of the HTTP protocol should be 
     * used per default, <tt>false</tt> if version 1.0 should be used.
     *