httpmethoddirector.java

Light in the box 抓取程序。 使用HttpClient

            if (LOG.isDebugEnabled()) {
                LOG.debug("Authenticating with " + authscope);
            }
            Credentials credentials = this.state.getProxyCredentials(authscope);
            if (credentials != null) {
                String authstring = authscheme.authenticate(credentials, method);
                if (authstring != null) {
                    method.addRequestHeader(new Header(PROXY_AUTH_RESP, authstring, true));
                }
            } else {
                if (LOG.isWarnEnabled()) {
                    LOG.warn("Required proxy credentials not available for " + authscope);
                    if (method.getProxyAuthState().isPreemptive()) {
                        LOG.warn("Preemptive authentication requested but no default " +
                            "proxy credentials available"); 
                    }
                }
            }
        }
    }
    
    
    /**
     * Applies connection parameters specified for a given method
     * 
     * @param method HTTP method
     * 
     * @throws IOException if an I/O occurs setting connection parameters 
     */
    private void applyConnectionParams(final HttpMethod method) throws IOException {
        int timeout = 0;
        // see if a timeout is given for this method
        Object param = method.getParams().getParameter(HttpMethodParams.SO_TIMEOUT);
        if (param == null) {
            // if not, use the default value
            param = this.conn.getParams().getParameter(HttpConnectionParams.SO_TIMEOUT);
        }
        if (param != null) {
            timeout = ((Integer)param).intValue();
        }
        this.conn.setSocketTimeout(timeout);                    
    }
    
    /**
     * Executes a method with the current hostConfiguration.
     *
     * @throws IOException if an I/O (transport) error occurs. Some transport exceptions 
     * can be recovered from.
     * @throws HttpException  if a protocol exception occurs. Usually protocol exceptions 
     * cannot be recovered from.
     */
    private void executeWithRetry(final HttpMethod method) 
        throws IOException, HttpException {
        
        /** How many times did this transparently handle a recoverable exception? */
        int execCount = 0;
        // loop until the method is successfully processed, the retryHandler 
        // returns false or a non-recoverable exception is thrown
        try {
            while (true) {
                execCount++;
                try {

                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Attempt number " + execCount + " to process request");
                    }
                    if (this.conn.getParams().isStaleCheckingEnabled()) {
                        this.conn.closeIfStale();
                    }
                    if (!this.conn.isOpen()) {
                        // this connection must be opened before it can be used
                        // This has nothing to do with opening a secure tunnel
                        this.conn.open();
                        if (this.conn.isProxied() && this.conn.isSecure() 
                        && !(method instanceof ConnectMethod)) {
                            // we need to create a secure tunnel before we can execute the real method
                            if (!executeConnect()) {
                                // abort, the connect method failed
                                return;
                            }
                        }
                    }
                    applyConnectionParams(method);                    
                    method.execute(state, this.conn);
                    break;
                } catch (HttpException e) {
                    // filter out protocol exceptions which cannot be recovered from
                    throw e;
                } catch (IOException e) {
                    LOG.debug("Closing the connection.");
                    this.conn.close();
                    // test if this method should be retried
                    // ========================================
                    // this code is provided for backward compatibility with 2.0
                    // will be removed in the next major release
                    if (method instanceof HttpMethodBase) {
                        MethodRetryHandler handler = 
                            ((HttpMethodBase)method).getMethodRetryHandler();
                        if (handler != null) {
                            if (!handler.retryMethod(
                                    method,
                                    this.conn, 
                                    new HttpRecoverableException(e.getMessage()),
                                    execCount, 
                                    method.isRequestSent())) {
                                LOG.debug("Method retry handler returned false. "
                                        + "Automatic recovery will not be attempted");
                                throw e;
                            }
                        }
                    }
                    // ========================================
                    HttpMethodRetryHandler handler = 
                        (HttpMethodRetryHandler)method.getParams().getParameter(
                                HttpMethodParams.RETRY_HANDLER);
                    if (handler == null) {
                        handler = new DefaultHttpMethodRetryHandler();
                    }
                    if (!handler.retryMethod(method, e, execCount)) {
                        LOG.debug("Method retry handler returned false. "
                                + "Automatic recovery will not be attempted");
                        throw e;
                    }
                    if (LOG.isInfoEnabled()) {
                        LOG.info("I/O exception ("+ e.getClass().getName() +") caught when processing request: "
                                + e.getMessage());
                    }
                    if (LOG.isDebugEnabled()) {
                        LOG.debug(e.getMessage(), e);
                    }
                    LOG.info("Retrying request");
                }
            }
        } catch (IOException e) {
            if (this.conn.isOpen()) {
                LOG.debug("Closing the connection.");
                this.conn.close();
            }
            releaseConnection = true;
            throw e;
        } catch (RuntimeException e) {
            if (this.conn.isOpen()) {
                LOG.debug("Closing the connection.");
                this.conn.close();
            }
            releaseConnection = true;
            throw e;
        }
    }
    
    /**
     * Executes a ConnectMethod to establish a tunneled connection.
     * 
     * @return <code>true</code> if the connect was successful
     * 
     * @throws IOException
     * @throws HttpException
     */
    private boolean executeConnect() 
        throws IOException, HttpException {

        this.connectMethod = new ConnectMethod(this.hostConfiguration);
        this.connectMethod.getParams().setDefaults(this.hostConfiguration.getParams());
        
        int code;
        for (;;) {
            if (!this.conn.isOpen()) {
                this.conn.open();
            }
            if (this.params.isAuthenticationPreemptive()
                    || this.state.isAuthenticationPreemptive()) {
                LOG.debug("Preemptively sending default basic credentials");
                this.connectMethod.getProxyAuthState().setPreemptive();
                this.connectMethod.getProxyAuthState().setAuthAttempted(true);
            }
            try {
                authenticateProxy(this.connectMethod);
            } catch (AuthenticationException e) {
                LOG.error(e.getMessage(), e);
            }
            applyConnectionParams(this.connectMethod);                    
            this.connectMethod.execute(state, this.conn);
            code = this.connectMethod.getStatusCode();
            boolean retry = false;
            AuthState authstate = this.connectMethod.getProxyAuthState(); 
            authstate.setAuthRequested(code == HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED);
            if (authstate.isAuthRequested()) {
                if (processAuthenticationResponse(this.connectMethod)) {
                    retry = true;
                }
            }
            if (!retry) {
                break;
            }
            if (this.connectMethod.getResponseBodyAsStream() != null) {
                this.connectMethod.getResponseBodyAsStream().close();
            }
        }
        if ((code >= 200) && (code < 300)) {
            this.conn.tunnelCreated();
            // Drop the connect method, as it is no longer needed
            this.connectMethod = null;
            return true;
        } else {
            this.conn.close();
            return false;
        }
    }

    /**
     * Fake response
     * @param method
     * @return
     */
    
    private void fakeResponse(final HttpMethod method)
        throws IOException, HttpException {
        // What is to follow is an ugly hack.
        // I REALLY hate having to resort to such
        // an appalling trick
        // The only feasible solution is to split monolithic
        // HttpMethod into HttpRequest/HttpResponse pair.
        // That would allow to execute CONNECT method 
        // behind the scene and return CONNECT HttpResponse 
        // object in response to the original request that 
        // contains the correct status line, headers & 
        // response body.
        LOG.debug("CONNECT failed, fake the response for the original method");
        // Pass the status, headers and response stream to the wrapped
        // method.
        // To ensure that the connection is not released more than once
        // this method is still responsible for releasing the connection. 
        // This will happen when the response body is consumed, or when
        // the wrapped method closes the response connection in 
        // releaseConnection().
        if (method instanceof HttpMethodBase) {
            ((HttpMethodBase) method).fakeResponse(
                this.connectMethod.getStatusLine(),
                this.connectMethod.getResponseHeaderGroup(),
                this.connectMethod.getResponseBodyAsStream()
            );
            method.getProxyAuthState().setAuthScheme(
                this.connectMethod.getProxyAuthState().getAuthScheme());
            this.connectMethod = null;
        } else {
            releaseConnection = true;
            LOG.warn(
                "Unable to fake response on method as it is not derived from HttpMethodBase.");
        }
    }
    
    /**
     * Process the redirect response.
     * 
     * @return <code>true</code> if the redirect was successful
     */
    private boolean processRedirectResponse(final HttpMethod method)
     throws RedirectException {
        //get the location header to find out where to redirect to
        Header locationHeader = method.getResponseHeader("location");
        if (locationHeader == null) {
            // got a redirect response, but no location header
            LOG.error("Received redirect response " + method.getStatusCode()
                    + " but no location header");
            return false;
        }
        String location = locationHeader.getValue();
        if (LOG.isDebugEnabled()) {
            LOG.debug("Redirect requested to location '" + location + "'");
        }
        
        //rfc2616 demands the location value be a complete URI
        //Location       = "Location" ":" absoluteURI
        URI redirectUri = null;
        URI currentUri = null;

        try {
            currentUri = new URI(
                this.conn.getProtocol().getScheme(),
                null,
                this.conn.getHost(), 
                this.conn.getPort(), 
                method.getPath()
            );
            
            String charset = method.getParams().getUriCharset();
            redirectUri = new URI(location, true, charset);
            
            if (redirectUri.isRelativeURI()) {
                if (this.params.isParameterTrue(HttpClientParams.REJECT_RELATIVE_REDIRECT)) {
                    LOG.warn("Relative redirect location '" + location + "' not allowed");
                    return false;
                } else { 
                    //location is incomplete, use current values for defaults
                    LOG.debug("Redirect URI is not absolute - parsing as relative");
                    redirectUri = new URI(currentUri, redirectUri);
                }
            } else {
                // Reset the default params
                method.getParams().setDefaults(this.params);
            }
            method.setURI(redirectUri);
            hostConfiguration.setHost(redirectUri);
        } catch (URIException ex) {
            throw new InvalidRedirectLocationException(
                    "Invalid redirect location: " + location, location, ex);
        }

        if (this.params.isParameterFalse(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS)) {
            if (this.redirectLocations == null) {
                this.redirectLocations = new HashSet();
            }
            this.redirectLocations.add(currentUri);
            try {