httpmethodbase.java

Light in the box 抓取程序。 使用HttpClient

     * If the response is large this method involves lots of array copying and many object 
     * allocations, which makes it unsuitable for high-performance / low-footprint applications.
     * Those applications should use {@link #getResponseBodyAsStream()}.
     * 
     * @param maxlen the maximum content length to accept (number of bytes). 
     * @return The response body.
     * 
     * @throws IOException If an I/O (transport) problem occurs while obtaining the 
     * response body.
     */
    public byte[] getResponseBody(int maxlen) throws IOException {
        if (maxlen < 0) throw new IllegalArgumentException("maxlen must be positive");
        if (this.responseBody == null) {
            InputStream instream = getResponseBodyAsStream();
            if (instream != null) {
                // we might already know that the content is larger
                long contentLength = getResponseContentLength();
                if ((contentLength != -1) && (contentLength > maxlen)) {
                    throw new HttpContentTooLargeException(
                            "Content-Length is " + contentLength, maxlen);
                }
                
                LOG.debug("Buffering response body");
                ByteArrayOutputStream rawdata = new ByteArrayOutputStream(
                        contentLength > 0 ? (int) contentLength : DEFAULT_INITIAL_BUFFER_SIZE);
                byte[] buffer = new byte[2048];
                int pos = 0;
                int len;
                do {
                    len = instream.read(buffer, 0, Math.min(buffer.length, maxlen-pos));
                    if (len == -1) break;
                    rawdata.write(buffer, 0, len);
                    pos += len;
                } while (pos < maxlen);
                
                setResponseStream(null);
                // check if there is even more data
                if (pos == maxlen) {
                    if (instream.read() != -1)
                        throw new HttpContentTooLargeException(
                                "Content-Length not known but larger than "
                                + maxlen, maxlen);
                }
                this.responseBody = rawdata.toByteArray();
            }
        }
        return this.responseBody;
    }

    /**
     * Returns the response body of the HTTP method, if any, as an {@link InputStream}. 
     * If response body is not available, returns <tt>null</tt>. If the response has been
     * buffered this method returns a new stream object on every call. If the response
     * has not been buffered the returned stream can only be read once.
     * 
     * @return The response body or <code>null</code>.
     * 
     * @throws IOException If an I/O (transport) problem occurs while obtaining the 
     * response body.
     */
    public InputStream getResponseBodyAsStream() throws IOException {
        if (responseStream != null) {
            return responseStream;
        }
        if (responseBody != null) {
            InputStream byteResponseStream = new ByteArrayInputStream(responseBody);
            LOG.debug("re-creating response stream from byte array");
            return byteResponseStream;
        }
        return null;
    }

    /**
     * Returns the response body of the HTTP method, if any, as a {@link String}. 
     * If response body is not available or cannot be read, returns <tt>null</tt>
     * The string conversion on the data is done using the character encoding specified
     * in <tt>Content-Type</tt> header. Buffers the response and this method can be 
     * called several times yielding the same result each time.
     * 
     * Note: This will cause the entire response body to be buffered in memory. A
     * malicious server may easily exhaust all the VM memory. It is strongly
     * recommended, to use getResponseAsStream if the content length of the response
     * is unknown or resonably large.
     * 
     * @return The response body or <code>null</code>.
     * 
     * @throws IOException If an I/O (transport) problem occurs while obtaining the 
     * response body.
     */
    public String getResponseBodyAsString() throws IOException {
        byte[] rawdata = null;
        if (responseAvailable()) {
            rawdata = getResponseBody();
        }
        if (rawdata != null) {
            return EncodingUtil.getString(rawdata, getResponseCharSet());
        } else {
            return null;
        }
    }
    
    /**
     * Returns the response body of the HTTP method, if any, as a {@link String}. 
     * If response body is not available or cannot be read, returns <tt>null</tt>
     * The string conversion on the data is done using the character encoding specified
     * in <tt>Content-Type</tt> header. Buffers the response and this method can be 
     * called several times yielding the same result each time.</p>
     * 
     * Note: This will cause the entire response body to be buffered in memory. This method is
     * safe if the content length of the response is unknown, because the amount of memory used
     * is limited.<p>
     * 
     * If the response is large this method involves lots of array copying and many object 
     * allocations, which makes it unsuitable for high-performance / low-footprint applications.
     * Those applications should use {@link #getResponseBodyAsStream()}.
     * 
     * @param maxlen the maximum content length to accept (number of bytes). Note that,
     * depending on the encoding, this is not equal to the number of characters.
     * @return The response body or <code>null</code>.
     * 
     * @throws IOException If an I/O (transport) problem occurs while obtaining the 
     * response body.
     */
    public String getResponseBodyAsString(int maxlen) throws IOException {
        if (maxlen < 0) throw new IllegalArgumentException("maxlen must be positive");
        byte[] rawdata = null;
        if (responseAvailable()) {
            rawdata = getResponseBody(maxlen);
        }
        if (rawdata != null) {
            return EncodingUtil.getString(rawdata, getResponseCharSet());
        } else {
            return null;
        }
    }

    /**
     * Returns an array of the response footers that the HTTP method currently has
     * in the order in which they were read.
     *
     * @return an array of footers
     */
    public Header[] getResponseFooters() {
        return getResponseTrailerHeaderGroup().getAllHeaders();
    }

    /**
     * Gets the response footer associated with the given name.
     * Footer name matching is case insensitive.
     * <tt>null</tt> will be returned if either <i>footerName</i> is
     * <tt>null</tt> or there is no matching footer for <i>footerName</i>
     * or there are no footers available.  If there are multiple footers
     * with the same name, there values will be combined with the ',' separator
     * as specified by RFC2616.
     * 
     * @param footerName the footer name to match
     * @return the matching footer
     */
    public Header getResponseFooter(String footerName) {
        if (footerName == null) {
            return null;
        } else {
            return getResponseTrailerHeaderGroup().getCondensedHeader(footerName);
        }
    }

    /**
     * Sets the response stream.
     * @param responseStream The new response stream.
     */
    protected void setResponseStream(InputStream responseStream) {
        this.responseStream = responseStream;
    }

    /**
     * Returns a stream from which the body of the current response may be read.
     * If the method has not yet been executed, if <code>responseBodyConsumed</code>
     * has been called, or if the stream returned by a previous call has been closed,
     * <code>null</code> will be returned.
     *
     * @return the current response stream
     */
    protected InputStream getResponseStream() {
        return responseStream;
    }
    
    /**
     * Returns the status text (or "reason phrase") associated with the latest
     * response.
     * 
     * @return The status text.
     */
    public String getStatusText() {
        return statusLine.getReasonPhrase();
    }

    /**
     * Defines how strictly HttpClient follows the HTTP protocol specification  
     * (RFC 2616 and other relevant RFCs). In the strict mode HttpClient precisely
     * implements the requirements of the specification, whereas in non-strict mode 
     * it attempts to mimic the exact behaviour of commonly used HTTP agents, 
     * which many HTTP servers expect.
     * 
     * @param strictMode <tt>true</tt> for strict mode, <tt>false</tt> otherwise
     * 
     * @deprecated Use {@link org.apache.commons.httpclient.params.HttpParams#setParameter(String, Object)}
     * to exercise a more granular control over HTTP protocol strictness.
     */
    public void setStrictMode(boolean strictMode) {
        if (strictMode) {
            this.params.makeStrict();
        } else {
            this.params.makeLenient();
        }
    }

    /**
     * @deprecated Use {@link org.apache.commons.httpclient.params.HttpParams#setParameter(String, Object)}
     * to exercise a more granular control over HTTP protocol strictness.
     *
     * @return <tt>false</tt>
     */
    public boolean isStrictMode() {
        return false;
    }

    /**
     * Adds the specified request header, NOT overwriting any previous value.
     * Note that header-name matching is case insensitive.
     *
     * @param headerName the header's name
     * @param headerValue the header's value
     */
    public void addRequestHeader(String headerName, String headerValue) {
        addRequestHeader(new Header(headerName, headerValue));
    }

    /**
     * Tests if the connection should be force-closed when no longer needed.
     * 
     * @return <code>true</code> if the connection must be closed
     */
    protected boolean isConnectionCloseForced() {
        return this.connectionCloseForced;
    }

    /**
     * Sets whether or not the connection should be force-closed when no longer 
     * needed. This value should only be set to <code>true</code> in abnormal 
     * circumstances, such as HTTP protocol violations. 
     * 
     * @param b <code>true</code> if the connection must be closed, <code>false</code>
     * otherwise.
     */
    protected void setConnectionCloseForced(boolean b) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Force-close connection: " + b);
        }
        this.connectionCloseForced = b;
    }

    /**
     * Tests if the connection should be closed after the method has been executed.
     * The connection will be left open when using HTTP/1.1 or if <tt>Connection: 
     * keep-alive</tt> header was sent.
     * 
     * @param conn the connection in question
     * 
     * @return boolean true if we should close the connection.
     */
    protected boolean shouldCloseConnection(HttpConnection conn) {
        // Connection must be closed due to an abnormal circumstance 
        if (isConnectionCloseForced()) {
            LOG.debug("Should force-close connection.");
            return true;
        }

        Header connectionHeader = null;
        // In case being connected via a proxy server
        if (!conn.isTransparent()) {
            // Check for 'proxy-connection' directive
            connectionHeader = responseHeaders.getFirstHeader("proxy-connection");
        }
        // In all cases Check for 'connection' directive
        // some non-complaint proxy servers send it instread of
        // expected 'proxy-connection' directive
        if (connectionHeader == null) {
            connectionHeader = responseHeaders.getFirstHeader("connection");
        }
        // In case the response does not contain any explict connection
        // directives, check whether the request does
        if (connectionHeader == null) {
            connectionHeader = requestHeaders.getFirstHeader("connection");
        }
        if (connectionHeader != null) {
            if (connectionHeader.getValue().equalsIgnoreCase("close")) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Should close connection in response to directive: " 
                        + connectionHeader.getValue());
                }
                return true;
            } else if (connectionHeader.getValue().equalsIgnoreCase("keep-alive")) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Should NOT close connection in response to directive: " 
                        + connectionHeader.getValue());
                }
                return false;
            } else {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Unknown directive: " + connectionHeader.toExternalForm());
                }
            }
        }
        LOG.debug("Resorting to protocol version default close connection policy");
        // missing or invalid connection header, do the default
        if (this.effectiveVersion.greaterEquals(HttpVersion.HTTP_1_1)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Should NOT close connection, using " + this.effectiveVersion.toString());
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Should close connection, using " + this.effectiveVersion.toString());
            }
        }
        return this.effectiveVersion.lessEquals(HttpVersion.HTTP_1_0);
    }
    
    /**
     * Tests if the this method is ready to be executed.
     * 
     * @param state the {@link HttpState state} information associated with this method
     * @param conn the {@link HttpConnection connection} to be used
     * @throws HttpException If the method is in invalid state.
     */
    private void checkExecuteConditions(HttpState state, HttpConnection conn)
    throws HttpException {

        if (state == null) {
            throw new IllegalArgumentException("HttpState parameter may not be null");
        }
        if (conn == null) {
            throw new IllegalArgumentException("HttpConnection parameter may not be null");
        }
        if (this.aborted) {
            throw new IllegalStateException("Method has been aborted");
        }
        if (!validate()) {
            throw new ProtocolException("HttpMethodBase object not valid");
        }
    }

    /**
     * Executes this method using the specified <code>HttpConnection</code> and
     * <code>HttpState</code>. 
     *