httpmethoddirector.java

Light in the box 抓取程序。 使用HttpClient

                if(redirectUri.hasQuery()) {
                    redirectUri.setQuery(null);
                }
            } catch (URIException e) {
                // Should never happen
                return false;
            }

            if (this.redirectLocations.contains(redirectUri)) {
                throw new CircularRedirectException("Circular redirect to '" +
                    redirectUri + "'");
            }
        }

        if (LOG.isDebugEnabled()) {
            LOG.debug("Redirecting from '" + currentUri.getEscapedURI()
                + "' to '" + redirectUri.getEscapedURI());
        }
        //And finally invalidate the actual authentication scheme
        method.getHostAuthState().invalidate(); 
        return true;
    }

    /**
     * Processes a response that requires authentication
     *
     * @param method the current {@link HttpMethod HTTP method}
     *
     * @return <tt>true</tt> if the authentication challenge can be responsed to,
     *   (that is, at least one of the requested authentication scheme is supported, 
     *   and matching credentials have been found), <tt>false</tt> otherwise.
     */
    private boolean processAuthenticationResponse(final HttpMethod method) {
        LOG.trace("enter HttpMethodBase.processAuthenticationResponse("
            + "HttpState, HttpConnection)");

        try {
            switch (method.getStatusCode()) {
                case HttpStatus.SC_UNAUTHORIZED:
                    return processWWWAuthChallenge(method);
                case HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED:
                    return processProxyAuthChallenge(method);
                default:
                    return false;
            }
        } catch (Exception e) {
            if (LOG.isErrorEnabled()) {
                LOG.error(e.getMessage(), e);
            }
            return false;
        }
    }

    private boolean processWWWAuthChallenge(final HttpMethod method)
        throws MalformedChallengeException, AuthenticationException  
    {
        AuthState authstate = method.getHostAuthState();
        Map challenges = AuthChallengeParser.parseChallenges(
            method.getResponseHeaders(WWW_AUTH_CHALLENGE));
        if (challenges.isEmpty()) {
            LOG.debug("Authentication challenge(s) not found");
            return false; 
        }
        AuthScheme authscheme = null;
        try {
            authscheme = this.authProcessor.processChallenge(authstate, challenges);
        } catch (AuthChallengeException e) {
            if (LOG.isWarnEnabled()) {
                LOG.warn(e.getMessage());
            }
        }
        if (authscheme == null) {
            return false;
        }
        String host = method.getParams().getVirtualHost();
        if (host == null) {
            host = conn.getHost();
        }
        int port = conn.getPort();
        AuthScope authscope = new AuthScope(
            host, port, 
            authscheme.getRealm(), 
            authscheme.getSchemeName());
        
        if (LOG.isDebugEnabled()) {
            LOG.debug("Authentication scope: " + authscope);
        }
        if (authstate.isAuthAttempted() && authscheme.isComplete()) {
            // Already tried and failed
            Credentials credentials = promptForCredentials(
                authscheme, method.getParams(), authscope);
            if (credentials == null) {
                if (LOG.isInfoEnabled()) {
                    LOG.info("Failure authenticating with " + authscope);
                }
                return false;
            } else {
                return true;
            }
        } else {
            authstate.setAuthAttempted(true);
            Credentials credentials = this.state.getCredentials(authscope);
            if (credentials == null) {
                credentials = promptForCredentials(
                    authscheme, method.getParams(), authscope);
            }
            if (credentials == null) {
                if (LOG.isInfoEnabled()) {
                    LOG.info("No credentials available for " + authscope); 
                }
                return false;
            } else {
                return true;
            }
        }
    }

    private boolean processProxyAuthChallenge(final HttpMethod method)
        throws MalformedChallengeException, AuthenticationException
    {  
        AuthState authstate = method.getProxyAuthState();
        Map proxyChallenges = AuthChallengeParser.parseChallenges(
            method.getResponseHeaders(PROXY_AUTH_CHALLENGE));
        if (proxyChallenges.isEmpty()) {
            LOG.debug("Proxy authentication challenge(s) not found");
            return false; 
        }
        AuthScheme authscheme = null;
        try {
            authscheme = this.authProcessor.processChallenge(authstate, proxyChallenges);
        } catch (AuthChallengeException e) {
            if (LOG.isWarnEnabled()) {
                LOG.warn(e.getMessage());
            }
        }
        if (authscheme == null) {
            return false;
        }
        AuthScope authscope = new AuthScope(
            conn.getProxyHost(), conn.getProxyPort(), 
            authscheme.getRealm(), 
            authscheme.getSchemeName());  

        if (LOG.isDebugEnabled()) {
            LOG.debug("Proxy authentication scope: " + authscope);
        }
        if (authstate.isAuthAttempted() && authscheme.isComplete()) {
            // Already tried and failed
            Credentials credentials = promptForProxyCredentials(
                authscheme, method.getParams(), authscope);
            if (credentials == null) {
                if (LOG.isInfoEnabled()) {
                    LOG.info("Failure authenticating with " + authscope);
                }
                return false;
            } else {
                return true;
            }
        } else {
            authstate.setAuthAttempted(true);
            Credentials credentials = this.state.getProxyCredentials(authscope);
            if (credentials == null) {
                credentials = promptForProxyCredentials(
                    authscheme, method.getParams(), authscope);
            }
            if (credentials == null) {
                if (LOG.isInfoEnabled()) {
                    LOG.info("No credentials available for " + authscope); 
                }
                return false;
            } else {
                return true;
            }
        }
    }

    /**
     * Tests if the {@link HttpMethod method} requires a redirect to another location.
     * 
     * @param method HTTP method
     * 
     * @return boolean <tt>true</tt> if a retry is needed, <tt>false</tt> otherwise.
     */
    private boolean isRedirectNeeded(final HttpMethod method) {
        switch (method.getStatusCode()) {
            case HttpStatus.SC_MOVED_TEMPORARILY:
            case HttpStatus.SC_MOVED_PERMANENTLY:
            case HttpStatus.SC_SEE_OTHER:
            case HttpStatus.SC_TEMPORARY_REDIRECT:
                LOG.debug("Redirect required");
                if (method.getFollowRedirects()) {
                    return true;
                } else {
                    return false;
                }
            default:
                return false;
        } //end of switch
    }

    /**
     * Tests if the {@link HttpMethod method} requires authentication.
     * 
     * @param method HTTP method
     * 
     * @return boolean <tt>true</tt> if a retry is needed, <tt>false</tt> otherwise.
     */
    private boolean isAuthenticationNeeded(final HttpMethod method) {
        method.getHostAuthState().setAuthRequested(
                method.getStatusCode() == HttpStatus.SC_UNAUTHORIZED);
        method.getProxyAuthState().setAuthRequested(
                method.getStatusCode() == HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED);
        if (method.getHostAuthState().isAuthRequested() || 
            method.getProxyAuthState().isAuthRequested()) {
            LOG.debug("Authorization required");
            if (method.getDoAuthentication()) { //process authentication response
                return true;
            } else { //let the client handle the authenticaiton
                LOG.info("Authentication requested but doAuthentication is "
                        + "disabled");
                return false;
            }
        } else {
            return false;
        }
    }

    private Credentials promptForCredentials(
        final AuthScheme authScheme,
        final HttpParams params, 
        final AuthScope authscope)
    {
        LOG.debug("Credentials required");
        Credentials creds = null;
        CredentialsProvider credProvider = 
            (CredentialsProvider)params.getParameter(CredentialsProvider.PROVIDER);
        if (credProvider != null) {
            try {
                creds = credProvider.getCredentials(
                    authScheme, authscope.getHost(), authscope.getPort(), false);
            } catch (CredentialsNotAvailableException e) {
                LOG.warn(e.getMessage());
            }
            if (creds != null) {
                this.state.setCredentials(authscope, creds);
                if (LOG.isDebugEnabled()) {
                    LOG.debug(authscope + " new credentials given");
                }
            }
        } else {
            LOG.debug("Credentials provider not available");
        }
        return creds;
    }

    private Credentials promptForProxyCredentials(
        final AuthScheme authScheme,
        final HttpParams params,
        final AuthScope authscope) 
    {
        LOG.debug("Proxy credentials required");
        Credentials creds = null;
        CredentialsProvider credProvider = 
            (CredentialsProvider)params.getParameter(CredentialsProvider.PROVIDER);
        if (credProvider != null) {
            try {
                creds = credProvider.getCredentials(
                    authScheme, authscope.getHost(), authscope.getPort(), true);
            } catch (CredentialsNotAvailableException e) {
                LOG.warn(e.getMessage());
            }
            if (creds != null) {
                this.state.setProxyCredentials(authscope, creds);
                if (LOG.isDebugEnabled()) {
                    LOG.debug(authscope + " new credentials given");
                }
            }
        } else {
            LOG.debug("Proxy credentials provider not available");
        }
        return creds;
    }

    /**
     * @return
     */
    public HostConfiguration getHostConfiguration() {
        return hostConfiguration;
    }

    /**
     * @return
     */
    public HttpState getState() {
        return state;
    }

    /**
     * @return
     */
    public HttpConnectionManager getConnectionManager() {
        return connectionManager;
    }

    /**
     * @return
     */
    public HttpParams getParams() {
        return this.params;
    }
}