httpconnection.java

爬虫

     */
    public void setProxyPort(int port) throws IllegalStateException {
        assertNotOpen();
        proxyPortNumber = port;
    }

    /**
     * Returns <tt>true</tt> if the connection is established over 
     * a secure protocol.
     *
     * @return <tt>true</tt> if connected over a secure protocol.
     */
    public boolean isSecure() {
        return protocolInUse.isSecure();
    }

    /**
     * Returns the protocol used to establish the connection.
     * @return The protocol
     */
    public Protocol getProtocol() {
        return protocolInUse;
    }

    /**
     * Sets the protocol used to establish the connection
     * 
     * @param protocol The protocol to use.
     * 
     * @throws IllegalStateException if the connection is already open
     */
    public void setProtocol(Protocol protocol) {
        assertNotOpen();

        if (protocol == null) {
            throw new IllegalArgumentException("protocol is null");
        }

        protocolInUse = protocol;

    }

    /**
     * Return the local address used when creating the connection.
     * If <tt>null</tt>, the default address is used.
     * 
     * @return InetAddress the local address to be used when creating Sockets
     */
    public InetAddress getLocalAddress() {
        return this.localAddress;
    }
    
    /**
     * Set the local address used when creating the connection.
     * If unset or <tt>null</tt>, the default address is used.
     * 
     * @param localAddress the local address to use
     */
    public void setLocalAddress(InetAddress localAddress) {
        assertNotOpen();
        this.localAddress = localAddress;
    }

    /**
     * Tests if the connection is open. 
     *
     * @return <code>true</code> if the connection is open
     */
    public boolean isOpen() {
        return isOpen;
    }

    /**
     * Closes the connection if stale.
     * 
     * @return <code>true</code> if the connection was stale and therefore closed, 
     * <code>false</code> otherwise.
     * 
     * @see #isStale()
     * 
     * @since 3.0
     */
    public boolean closeIfStale() throws IOException {
        if (isOpen && isStale()) {
            LOG.debug("Connection is stale, closing...");
            close();
            return true;
        }
        return false;
    }
    
    /**
     * Tests if stale checking is enabled.
     * 
     * @return <code>true</code> if enabled
     * 
     * @see #isStale()
     * 
     * @deprecated Use {@link HttpConnectionParams#isStaleCheckingEnabled()},
     * {@link HttpConnection#getParams()}.
     */
    public boolean isStaleCheckingEnabled() {
        return this.params.isStaleCheckingEnabled();
    }

    /**
     * Sets whether or not isStale() will be called when testing if this connection is open.
     * 
     * <p>Setting this flag to <code>false</code> will increase performance when reusing
     * connections, but it will also make them less reliable.  Stale checking ensures that
     * connections are viable before they are used.  When set to <code>false</code> some
     * method executions will result in IOExceptions and they will have to be retried.</p>
     * 
     * @param staleCheckEnabled <code>true</code> to enable isStale()
     * 
     * @see #isStale()
     * @see #isOpen()
     * 
     * @deprecated Use {@link HttpConnectionParams#setStaleCheckingEnabled(boolean)},
     * {@link HttpConnection#getParams()}.
     */
    public void setStaleCheckingEnabled(boolean staleCheckEnabled) {
        this.params.setStaleCheckingEnabled(staleCheckEnabled);
    }

    /**
     * Determines whether this connection is "stale", which is to say that either
     * it is no longer open, or an attempt to read the connection would fail.
     *
     * <p>Unfortunately, due to the limitations of the JREs prior to 1.4, it is
     * not possible to test a connection to see if both the read and write channels
     * are open - except by reading and writing.  This leads to a difficulty when
     * some connections leave the "write" channel open, but close the read channel
     * and ignore the request.  This function attempts to ameliorate that
     * problem by doing a test read, assuming that the caller will be doing a
     * write followed by a read, rather than the other way around.
     * </p>
     *
     * <p>To avoid side-effects, the underlying connection is wrapped by a
     * {@link BufferedInputStream}, so although data might be read, what is visible
     * to clients of the connection will not change with this call.</p.
     *
     * @throws IOException if the stale connection test is interrupted.
     * 
     * @return <tt>true</tt> if the connection is already closed, or a read would
     * fail.
     */
    protected boolean isStale() throws IOException {
        boolean isStale = true;
        if (isOpen) {
            // the connection is open, but now we have to see if we can read it
            // assume the connection is not stale.
            isStale = false;
            try {
                if (inputStream.available() <= 0) {
                    try {
                        socket.setSoTimeout(1);
                        inputStream.mark(1);
                        int byteRead = inputStream.read();
                        if (byteRead == -1) {
                            // again - if the socket is reporting all data read,
                            // probably stale
                            isStale = true;
                        } else {
                            inputStream.reset();
                        }
                    } finally {
                        socket.setSoTimeout(this.params.getSoTimeout());
                    }
                }
            } catch (InterruptedIOException e) {
                if (!ExceptionUtil.isSocketTimeoutException(e)) {
                    throw e;
                }
                // aha - the connection is NOT stale - continue on!
            } catch (IOException e) {
                // oops - the connection is stale, the read or soTimeout failed.
                LOG.debug(
                    "An error occurred while reading from the socket, is appears to be stale",
                    e
                );
                isStale = true;
            }
        }

        return isStale;
    }

    /**
     * Returns <tt>true</tt> if the connection is established via a proxy,
     * <tt>false</tt> otherwise.
     *
     * @return <tt>true</tt> if a proxy is used to establish the connection, 
     * <tt>false</tt> otherwise.
     */
    public boolean isProxied() {
        return (!(null == proxyHostName || 0 >= proxyPortNumber));
    }

    /**
     * Set the state to keep track of the last response for the last request.
     *
     * <p>The connection managers use this to ensure that previous requests are
     * properly closed before a new request is attempted.  That way, a GET
     * request need not be read in its entirety before a new request is issued.
     * Instead, this stream can be closed as appropriate.</p>
     *
     * @param inStream  The stream associated with an HttpMethod.
     */
    public void setLastResponseInputStream(InputStream inStream) {
        lastResponseInputStream = inStream;
    }

    /**
     * Returns the stream used to read the last response's body.
     *
     * <p>Clients will generally not need to call this function unless
     * using HttpConnection directly, instead of calling {@link HttpClient#executeMethod}.
     * For those clients, call this function, and if it returns a non-null stream,
     * close the stream before attempting to execute a method.  Note that
     * calling "close" on the stream returned by this function <i>may</i> close
     * the connection if the previous response contained a "Connection: close" header. </p>
     *
     * @return An {@link InputStream} corresponding to the body of the last
     *  response.
     */
    public InputStream getLastResponseInputStream() {
        return lastResponseInputStream;
    }

    // --------------------------------------------------- Other Public Methods

    /**
     * Returns {@link HttpConnectionParams HTTP protocol parameters} associated with this method.
     *
     * @return HTTP parameters.
     *
     * @since 3.0
     */
    public HttpConnectionParams getParams() {
        return this.params;
    }

    /**
     * Assigns {@link HttpConnectionParams HTTP protocol parameters} for this method.
     * 
     * @since 3.0
     * 
     * @see HttpConnectionParams
     */
    public void setParams(final HttpConnectionParams params) {
        if (params == null) {
            throw new IllegalArgumentException("Parameters may not be null");
        }
        this.params = params;
    }

    /**
     * Set the {@link Socket}'s timeout, via {@link Socket#setSoTimeout}.  If the
     * connection is already open, the SO_TIMEOUT is changed.  If no connection
     * is open, then subsequent connections will use the timeout value.
     * <p>
     * Note: This is not a connection timeout but a timeout on network traffic!
     *
     * @param timeout the timeout value
     * @throws SocketException - if there is an error in the underlying
     * protocol, such as a TCP error.
     * 
     * @deprecated Use {@link HttpConnectionParams#setSoTimeout(int)},
     * {@link HttpConnection#getParams()}.
     */
    public void setSoTimeout(int timeout)
        throws SocketException, IllegalStateException {
        this.params.setSoTimeout(timeout);
        if (this.socket != null) {
            this.socket.setSoTimeout(timeout);
        }
    }

    /**
     * Sets <code>SO_TIMEOUT</code> value directly on the underlying {@link Socket socket}. 
     * This method does not change the default read timeout value set via 
     * {@link HttpConnectionParams}.
     *
     * @param timeout the timeout value
     * @throws SocketException - if there is an error in the underlying
     * protocol, such as a TCP error.
     * @throws IllegalStateException if not connected
     * 
     * @since 3.0
     */
    public void setSocketTimeout(int timeout)
        throws SocketException, IllegalStateException {
        assertOpen();
        if (this.socket != null) {
            this.socket.setSoTimeout(timeout);
        }
    }

    /**
     * Returns the {@link Socket}'s timeout, via {@link Socket#getSoTimeout}, if the
     * connection is already open. If no connection is open, return the value subsequent 
     * connection will use.
     * <p>
     * Note: This is not a connection timeout but a timeout on network traffic!
     *
     * @return the timeout value
     * 
     * @deprecated Use {@link HttpConnectionParams#getSoTimeout()},
     * {@link HttpConnection#getParams()}.
     */
    public int getSoTimeout() throws SocketException {
        return this.params.getSoTimeout();
    }

    /**
     * Sets the connection timeout. This is the maximum time that may be spent
     * until a connection is established. The connection will fail after this
     * amount of time.
     * @param timeout The timeout in milliseconds. 0 means timeout is not used.
     * 
     * @deprecated Use {@link HttpConnectionParams#setConnectionTimeout(int)},
     * {@link HttpConnection#getParams()}.
     */
    public void setConnectionTimeout(int timeout) {
        this.params.setConnectionTimeout(timeout);
    }

    /**
     * Establishes a connection to the specified host and port
     * (via a proxy if specified).
     * The underlying socket is created from the {@link ProtocolSocketFactory}.
     *
     * @throws IOException if an attempt to establish the connection results in an
     *   I/O error.
     */
    public void open() throws IOException {
        LOG.trace("enter HttpConnection.open()");

        final String host = (proxyHostName == null) ? hostName : proxyHostName;
        final int port = (proxyHostName == null) ? portNumber : proxyPortNumber;
        assertNotOpen();
        
        if (LOG.isDebugEnabled()) {
            LOG.debug("Open connection to " + host + ":" + port);
        }
        
        try {
            if (this.socket == null) {
                usingSecureSocket = isSecure() && !isProxied();