httpconnection.java

爬虫

                // use the protocol's socket factory unless this is a secure
                // proxied connection
                final ProtocolSocketFactory socketFactory =
                    (isSecure() && isProxied()
                            ? new DefaultProtocolSocketFactory()
                            : protocolInUse.getSocketFactory());
                this.socket = socketFactory.createSocket(
                            host, port, 
                            localAddress, 0,
                            this.params);
            }

            /*
            "Nagling has been broadly implemented across networks, 
            including the Internet, and is generally performed by default 
            - although it is sometimes considered to be undesirable in 
            highly interactive environments, such as some client/server 
            situations. In such cases, nagling may be turned off through 
            use of the TCP_NODELAY sockets option." */

            socket.setTcpNoDelay(this.params.getTcpNoDelay());
            socket.setSoTimeout(this.params.getSoTimeout());
            
            int linger = this.params.getLinger();
            if (linger >= 0) {
                socket.setSoLinger(linger > 0, linger);
            }
            
            int sndBufSize = this.params.getSendBufferSize();
            if (sndBufSize >= 0) {
                socket.setSendBufferSize(sndBufSize);
            }        
            int rcvBufSize = this.params.getReceiveBufferSize();
            if (rcvBufSize >= 0) {
                socket.setReceiveBufferSize(rcvBufSize);
            }        
            int outbuffersize = socket.getSendBufferSize();
            if ((outbuffersize > 2048) || (outbuffersize <= 0)) {
                outbuffersize = 2048;
            }
            int inbuffersize = socket.getReceiveBufferSize();
            if ((inbuffersize > 2048) || (inbuffersize <= 0)) {
                inbuffersize = 2048;
            }
            
            // START HERITRIX Change
            HttpRecorder httpRecorder = HttpRecorder.getHttpRecorder();
            if (httpRecorder == null || (isSecure() && isProxied())) {
                // no recorder, OR defer recording for pre-tunnel leg
                inputStream = new BufferedInputStream(
                    socket.getInputStream(), inbuffersize);
                outputStream = new BufferedOutputStream(
                    socket.getOutputStream(), outbuffersize);
            } else {
                inputStream = httpRecorder.inputWrap((InputStream)
                    (new BufferedInputStream(socket.getInputStream(),
                    inbuffersize)));
                outputStream = httpRecorder.outputWrap((OutputStream)
                    (new BufferedOutputStream(socket.getOutputStream(), 
                    outbuffersize)));
            }
            // END HERITRIX change.

            isOpen = true;
        } catch (IOException e) {
            // Connection wasn't opened properly
            // so close everything out
            closeSocketAndStreams();
            throw e;
        }
    }

    /**
     * Instructs the proxy to establish a secure tunnel to the host. The socket will 
     * be switched to the secure socket. Subsequent communication is done via the secure 
     * socket. The method can only be called once on a proxied secure connection.
     *
     * @throws IllegalStateException if connection is not secure and proxied or
     * if the socket is already secure.
     * @throws IOException if an attempt to establish the secure tunnel results in an
     *   I/O error.
     */
    public void tunnelCreated() throws IllegalStateException, IOException {
        LOG.trace("enter HttpConnection.tunnelCreated()");

        if (!isSecure() || !isProxied()) {
            throw new IllegalStateException(
                "Connection must be secure "
                    + "and proxied to use this feature");
        }

        if (usingSecureSocket) {
            throw new IllegalStateException("Already using a secure socket");
        }
        
        if (LOG.isDebugEnabled()) {
            LOG.debug("Secure tunnel to " + this.hostName + ":" + this.portNumber);
        }

        SecureProtocolSocketFactory socketFactory =
            (SecureProtocolSocketFactory) protocolInUse.getSocketFactory();

        socket = socketFactory.createSocket(socket, hostName, portNumber, true);
        int sndBufSize = this.params.getSendBufferSize();
        if (sndBufSize >= 0) {
            socket.setSendBufferSize(sndBufSize);
        }        
        int rcvBufSize = this.params.getReceiveBufferSize();
        if (rcvBufSize >= 0) {
            socket.setReceiveBufferSize(rcvBufSize);
        }        
        int outbuffersize = socket.getSendBufferSize();
        if (outbuffersize > 2048) {
            outbuffersize = 2048;
        }
        int inbuffersize = socket.getReceiveBufferSize();
        if (inbuffersize > 2048) {
            inbuffersize = 2048;
        }

        // START HERITRIX Change
        HttpRecorder httpRecorder = HttpRecorder.getHttpRecorder();
        if (httpRecorder == null) {
            inputStream = new BufferedInputStream(socket.getInputStream(), inbuffersize);
            outputStream = new BufferedOutputStream(socket.getOutputStream(), outbuffersize);
        } else {
            inputStream = httpRecorder.inputWrap((InputStream)
                    (new BufferedInputStream(socket.getInputStream(),
                    inbuffersize)));
            outputStream = httpRecorder.outputWrap((OutputStream)
                (new BufferedOutputStream(socket.getOutputStream(), 
                outbuffersize)));
        }
        // END HERITRIX change.

        usingSecureSocket = true;
        tunnelEstablished = true;
    }

    /**
     * Indicates if the connection is completely transparent from end to end.
     *
     * @return true if conncetion is not proxied or tunneled through a transparent
     * proxy; false otherwise.
     */
    public boolean isTransparent() {
        return !isProxied() || tunnelEstablished;
    }

    /**
     * Flushes the output request stream.  This method should be called to 
     * ensure that data written to the request OutputStream is sent to the server.
     * 
     * @throws IOException if an I/O problem occurs
     */
    public void flushRequestOutputStream() throws IOException {
        LOG.trace("enter HttpConnection.flushRequestOutputStream()");
        assertOpen();
        outputStream.flush();
    }

    /**
     * Returns an {@link OutputStream} suitable for writing the request.
     *
     * @throws IllegalStateException if the connection is not open
     * @throws IOException if an I/O problem occurs
     * @return a stream to write the request to
     */
    public OutputStream getRequestOutputStream()
        throws IOException, IllegalStateException {
        LOG.trace("enter HttpConnection.getRequestOutputStream()");
        assertOpen();
        OutputStream out = this.outputStream;
        if (Wire.CONTENT_WIRE.enabled()) {
            out = new WireLogOutputStream(out, Wire.CONTENT_WIRE);
        }
        return out;
    }

    /**
     * Return a {@link InputStream} suitable for reading the response.
     * @return InputStream The response input stream.
     * @throws IOException If an IO problem occurs
     * @throws IllegalStateException If the connection isn't open.
     */
    public InputStream getResponseInputStream()
        throws IOException, IllegalStateException {
        LOG.trace("enter HttpConnection.getResponseInputStream()");
        assertOpen();
        return inputStream;
    }

    /**
     * Tests if input data avaialble. This method returns immediately
     * and does not perform any read operations on the input socket
     * 
     * @return boolean <tt>true</tt> if input data is available, 
     *                 <tt>false</tt> otherwise.
     * 
     * @throws IOException If an IO problem occurs
     * @throws IllegalStateException If the connection isn't open.
     */
    public boolean isResponseAvailable() 
        throws IOException {
        LOG.trace("enter HttpConnection.isResponseAvailable()");
        assertOpen();
        return this.inputStream.available() > 0;
    }

    /**
     * Tests if input data becomes available within the given period time in milliseconds.
     * 
     * @param timeout The number milliseconds to wait for input data to become available 
     * @return boolean <tt>true</tt> if input data is availble, 
     *                 <tt>false</tt> otherwise.
     * 
     * @throws IOException If an IO problem occurs
     * @throws IllegalStateException If the connection isn't open.
     */
    public boolean isResponseAvailable(int timeout) 
        throws IOException {
        LOG.trace("enter HttpConnection.isResponseAvailable(int)");
        assertOpen();
        boolean result = false;
        if (this.inputStream.available() > 0) {
            result = true;
        } else {
            try {
                this.socket.setSoTimeout(timeout);
                inputStream.mark(1);
                int byteRead = inputStream.read();
                if (byteRead != -1) {
                    inputStream.reset();
                    LOG.debug("Input data available");
                    result = true;
                } else {
                    LOG.debug("Input data not available");
                }
            } catch (InterruptedIOException e) {
                if (!ExceptionUtil.isSocketTimeoutException(e)) {
                    throw e;
                }
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Input data not available after " + timeout + " ms");
                }
            } finally {
                try {
                    socket.setSoTimeout(this.params.getSoTimeout());
                } catch (IOException ioe) {
                    LOG.debug("An error ocurred while resetting soTimeout, we will assume that"
                        + " no response is available.",
                        ioe);
                    result = false;
                }
            }
        }
        return result;
    }

    /**
     * Writes the specified bytes to the output stream.
     *
     * @param data the data to be written
     * @throws IllegalStateException if not connected
     * @throws IOException if an I/O problem occurs
     * @see #write(byte[],int,int)
     */
    public void write(byte[] data)
        throws IOException, IllegalStateException {
        LOG.trace("enter HttpConnection.write(byte[])");
        this.write(data, 0, data.length);
    }

    /**
     * Writes <i>length</i> bytes in <i>data</i> starting at
     * <i>offset</i> to the output stream.
     *
     * The general contract for
     * write(b, off, len) is that some of the bytes in the array b are written
     * to the output stream in order; element b[off] is the first byte written
     * and b[off+len-1] is the last byte written by this operation.
     *
     * @param data array containing the data to be written.
     * @param offset the start offset in the data.
     * @param length the number of bytes to write.
     * @throws IllegalStateException if not connected
     * @throws IOException if an I/O problem occurs
     */
    public void write(byte[] data, int offset, int length)
        throws IOException, IllegalStateException {
        LOG.trace("enter HttpConnection.write(byte[], int, int)");

        if (offset < 0) {
            throw new IllegalArgumentException("Array offset may not be negative");
        }
        if (length < 0) {
            throw new IllegalArgumentException("Array length may not be negative");
        }
        if (offset + length > data.length) {
            throw new IllegalArgumentException("Given offset and length exceed the array length");
        }
        assertOpen();
        this.outputStream.write(data, offset, length);
    }

    /**
     * Writes the specified bytes, followed by <tt>"\r\n".getBytes()</tt> to the
     * output stream.
     *
     * @param data the bytes to be written
     * @throws IllegalStateException if the connection is not open
     * @throws IOException if an I/O problem occurs
     */
    public void writeLine(byte[] data)
        throws IOException, IllegalStateException {
        LOG.trace("enter HttpConnection.writeLine(byte[])");
        write(data);
        writeLine();
    }

    /**
     * Writes <tt>"\r\n".getBytes()</tt> to the output stream.
     *
     * @throws IllegalStateException if the connection is not open
     * @throws IOException if an I/O problem occurs
     */
    public void writeLine()
        throws IOException, IllegalStateException {
        LOG.trace("enter HttpConnection.writeLine()");
        write(CRLF);
    }

    /**
     * @deprecated Use {@link #print(String, String)}
     * 
     * Writes the specified String (as bytes) to the output stream.
     *
     * @param data the string to be written
     * @throws IllegalStateException if the connection is not open
     * @throws IOException if an I/O problem occurs
     */
    public void print(String data)
        throws IOException, IllegalStateException {
        LOG.trace("enter HttpConnection.print(String)");
        write(EncodingUtil.getBytes(data, "ISO-8859-1"));
    }

    /**
     * Writes the specified String (as bytes) to the output stream.
     *