httpconnection.java

爬虫

/*
 * $Header: /cvsroot/archive-crawler/ArchiveOpenCrawler/src/java/org/apache/commons/httpclient/HttpConnection.java,v 1.10 2006/08/15 04:38:59 gojomo Exp $
 * $Revision: 1.10 $
 * $Date: 2006/08/15 04:38:59 $
 *
 * ====================================================================
 *
 *  Copyright 1999-2004 The Apache Software Foundation
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 * ====================================================================
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of the Apache Software Foundation.  For more
 * information on the Apache Software Foundation, please see
 * <http://www.apache.org/>.
 *
 */

package org.apache.commons.httpclient;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.OutputStream;
import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.Socket;
import java.net.SocketException;

import org.apache.commons.httpclient.params.HttpConnectionParams;
import org.apache.commons.httpclient.protocol.DefaultProtocolSocketFactory;
import org.apache.commons.httpclient.protocol.Protocol;
import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;
import org.apache.commons.httpclient.util.EncodingUtil;
import org.apache.commons.httpclient.util.ExceptionUtil;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

// HERITRIX import.
import org.archive.util.HttpRecorder;

/**
 * An abstraction of an HTTP {@link InputStream} and {@link OutputStream}
 * pair, together with the relevant attributes.
 * <p>
 * The following options are set on the socket before getting the input/output 
 * streams in the {@link #open()} method:
 * <table border=1><tr>
 *    <th>Socket Method
 *    <th>Sockets Option
 *    <th>Configuration
 * </tr><tr>
 *    <td>{@link java.net.Socket#setTcpNoDelay(boolean)}
 *    <td>SO_NODELAY
 *    <td>{@link HttpConnectionParams#setTcpNoDelay(boolean)}
 * </tr><tr>
 *    <td>{@link java.net.Socket#setSoTimeout(int)}
 *    <td>SO_TIMEOUT
 *    <td>{@link HttpConnectionParams#setSoTimeout(int)}
 * </tr><tr>
 *    <td>{@link java.net.Socket#setSendBufferSize(int)}
 *    <td>SO_SNDBUF
 *    <td>{@link HttpConnectionParams#setSendBufferSize(int)}
 * </tr><tr>
 *    <td>{@link java.net.Socket#setReceiveBufferSize(int)}
 *    <td>SO_RCVBUF
 *    <td>{@link HttpConnectionParams#setReceiveBufferSize(int)}
 * </tr></table>
 *
 * @author Rod Waldhoff
 * @author Sean C. Sullivan
 * @author Ortwin Glueck
 * @author <a href="mailto:[e-mail]">Jeff Dever</a>
 * @author <a href="mailto:[e-mail]">Mike Bowler</a>
 * @author <a href="mailto:[e-mail]">Oleg Kalnichevski</a>
 * @author Michael Becke
 * @author Eric E Johnson
 * @author Laura Werner
 * 
 * @version   $Revision: 1.10 $ $Date: 2006/08/15 04:38:59 $
 */
public class HttpConnection {

    // ----------------------------------------------------------- Constructors

    /**
     * Creates a new HTTP connection for the given host and port.
     *
     * @param host the host to connect to
     * @param port the port to connect to
     */
    public HttpConnection(String host, int port) {
        this(null, -1, host, null, port, Protocol.getProtocol("http"));
    }

    /**
     * Creates a new HTTP connection for the given host and port
     * using the given protocol.
     *
     * @param host the host to connect to
     * @param port the port to connect to
     * @param protocol the protocol to use
     */
    public HttpConnection(String host, int port, Protocol protocol) {
        this(null, -1, host, null, port, protocol);
    }

    /**
     * Creates a new HTTP connection for the given host with the virtual 
     * alias and port using given protocol.
     *
     * @param host the host to connect to
     * @param virtualHost the virtual host requests will be sent to
     * @param port the port to connect to
     * @param protocol the protocol to use
     */
    public HttpConnection(String host, String virtualHost, int port, Protocol protocol) {
        this(null, -1, host, virtualHost, port, protocol);
    }

    /**
     * Creates a new HTTP connection for the given host and port via the 
     * given proxy host and port using the default protocol.
     *
     * @param proxyHost the host to proxy via
     * @param proxyPort the port to proxy via
     * @param host the host to connect to
     * @param port the port to connect to
     */
    public HttpConnection(
        String proxyHost,
        int proxyPort,
        String host,
        int port) {
        this(proxyHost, proxyPort, host, null, port, Protocol.getProtocol("http"));
    }

    /**
     * Creates a new HTTP connection for the given host configuration.
     * 
     * @param hostConfiguration the host/proxy/protocol to use
     */
    public HttpConnection(HostConfiguration hostConfiguration) {
        this(hostConfiguration.getProxyHost(),
             hostConfiguration.getProxyPort(),
             hostConfiguration.getHost(),
             hostConfiguration.getPort(),
             hostConfiguration.getProtocol());
        this.localAddress = hostConfiguration.getLocalAddress();
    }

    /**
     * Creates a new HTTP connection for the given host with the virtual 
     * alias and port via the given proxy host and port using the given 
     * protocol.
     * 
     * @param proxyHost the host to proxy via
     * @param proxyPort the port to proxy via
     * @param host the host to connect to. Parameter value must be non-null.
     * @param virtualHost No longer applicable. 
     * @param port the port to connect to
     * @param protocol The protocol to use. Parameter value must be non-null.
     * 
     * @deprecated use #HttpConnection(String, int, String, int, Protocol)
     */
    public HttpConnection(
        String proxyHost,
        int proxyPort,
        String host,
        String virtualHost,
        int port,
        Protocol protocol) {
        this(proxyHost, proxyPort, host, port, protocol);
    }

    /**
     * Creates a new HTTP connection for the given host with the virtual 
     * alias and port via the given proxy host and port using the given 
     * protocol.
     * 
     * @param proxyHost the host to proxy via
     * @param proxyPort the port to proxy via
     * @param host the host to connect to. Parameter value must be non-null.
     * @param port the port to connect to
     * @param protocol The protocol to use. Parameter value must be non-null.
     */
    public HttpConnection(
        String proxyHost,
        int proxyPort,
        String host,
        int port,
        Protocol protocol) {

        if (host == null) {
            throw new IllegalArgumentException("host parameter is null");
        }
        if (protocol == null) {
            throw new IllegalArgumentException("protocol is null");
        }

        proxyHostName = proxyHost;
        proxyPortNumber = proxyPort;
        hostName = host;
        portNumber = protocol.resolvePort(port);
        protocolInUse = protocol;
    }

    // ------------------------------------------ Attribute Setters and Getters

    /**
     * Returns the connection socket.
     *
     * @return the socket.
     * 
     * @since 3.0
     */
    protected Socket getSocket() {
        return this.socket;
    }

    /**
     * Returns the host.
     *
     * @return the host.
     */
    public String getHost() {
        return hostName;
    }

    /**
     * Sets the host to connect to.
     *
     * @param host the host to connect to. Parameter value must be non-null.
     * @throws IllegalStateException if the connection is already open
     */
    public void setHost(String host) throws IllegalStateException {
        if (host == null) {
            throw new IllegalArgumentException("host parameter is null");
        }
        assertNotOpen();
        hostName = host;
    }

    /**
     * Returns the target virtual host.
     *
     * @return the virtual host.
     * 
     * @deprecated no longer applicable
     */

    public String getVirtualHost() {
        return this.hostName;
    }

    /**
     * Sets the virtual host to target.
     *
     * @param host the virtual host name that should be used instead of 
     *        physical host name when sending HTTP requests. Virtual host 
     *        name can be set to <tt> null</tt> if virtual host name is not
     *        to be used
     * 
     * @throws IllegalStateException if the connection is already open
     * 
     * @deprecated no longer applicable
     */

    public void setVirtualHost(String host) throws IllegalStateException {
        assertNotOpen();
    }

    /**
     * Returns the port of the host.
     *
     * If the port is -1 (or less than 0) the default port for
     * the current protocol is returned.
     *
     * @return the port.
     */
    public int getPort() {
        if (portNumber < 0) {
            return isSecure() ? 443 : 80;
        } else {
            return portNumber;
        }
    }

    /**
     * Sets the port to connect to.
     *
     * @param port the port to connect to
     * 
     * @throws IllegalStateException if the connection is already open
     */
    public void setPort(int port) throws IllegalStateException {
        assertNotOpen();
        portNumber = port;
    }

    /**
     * Returns the proxy host.
     *
     * @return the proxy host.
     */
    public String getProxyHost() {
        return proxyHostName;
    }

    /**
     * Sets the host to proxy through.
     *
     * @param host the host to proxy through.
     * 
     * @throws IllegalStateException if the connection is already open
     */
    public void setProxyHost(String host) throws IllegalStateException {
        assertNotOpen();
        proxyHostName = host;
    }

    /**
     * Returns the port of the proxy host.
     *
     * @return the proxy port.
     */
    public int getProxyPort() {
        return proxyPortNumber;
    }

    /**
     * Sets the port of the host to proxy through.
     *
     * @param port the port of the host to proxy through.
     * 
     * @throws IllegalStateException if the connection is already open