httpmethoddirector.java

Light in the box 抓取程序。 使用HttpClient

/*
 * $Header: /home/jerenkrantz/tmp/commons/commons-convert/cvs/home/cvs/jakarta-commons//httpclient/src/java/org/apache/commons/httpclient/HttpMethodDirector.java,v 1.34 2005/01/14 19:40:39 olegk Exp $
 * $Revision: 486658 $
 * $Date: 2006-12-13 15:05:50 +0100 (Wed, 13 Dec 2006) $
 *
 * ====================================================================
 *
 *  Licensed to the Apache Software Foundation (ASF) under one or more
 *  contributor license agreements.  See the NOTICE file distributed with
 *  this work for additional information regarding copyright ownership.
 *  The ASF licenses this file to You under the Apache License, Version 2.0
 *  (the "License"); you may not use this file except in compliance with
 *  the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 * ====================================================================
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of the Apache Software Foundation.  For more
 * information on the Apache Software Foundation, please see
 * <http://www.apache.org/>.
 *
 */

package org.apache.commons.httpclient;

import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

import org.apache.commons.httpclient.auth.AuthChallengeException;
import org.apache.commons.httpclient.auth.AuthChallengeParser;
import org.apache.commons.httpclient.auth.AuthChallengeProcessor;
import org.apache.commons.httpclient.auth.AuthScheme;
import org.apache.commons.httpclient.auth.AuthState;
import org.apache.commons.httpclient.auth.AuthenticationException;
import org.apache.commons.httpclient.auth.CredentialsProvider;
import org.apache.commons.httpclient.auth.CredentialsNotAvailableException;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.auth.MalformedChallengeException;
import org.apache.commons.httpclient.params.HostParams;
import org.apache.commons.httpclient.params.HttpClientParams;
import org.apache.commons.httpclient.params.HttpConnectionParams;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.httpclient.params.HttpParams;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * Handles the process of executing a method including authentication, redirection and retries.
 * 
 * @since 3.0
 */
class HttpMethodDirector {

    /** The www authenticate challange header. */
    public static final String WWW_AUTH_CHALLENGE = "WWW-Authenticate";

    /** The www authenticate response header. */
    public static final String WWW_AUTH_RESP = "Authorization";

    /** The proxy authenticate challange header. */
    public static final String PROXY_AUTH_CHALLENGE = "Proxy-Authenticate";

    /** The proxy authenticate response header. */
    public static final String PROXY_AUTH_RESP = "Proxy-Authorization";

    private static final Log LOG = LogFactory.getLog(HttpMethodDirector.class);

    private ConnectMethod connectMethod;
    
    private HttpState state;
    
    private HostConfiguration hostConfiguration;
    
    private HttpConnectionManager connectionManager;
    
    private HttpClientParams params;
    
    private HttpConnection conn;
    
    /** A flag to indicate if the connection should be released after the method is executed. */
    private boolean releaseConnection = false;

    /** Authentication processor */
    private AuthChallengeProcessor authProcessor = null;

    private Set redirectLocations = null; 
    
    public HttpMethodDirector(
        final HttpConnectionManager connectionManager,
        final HostConfiguration hostConfiguration,
        final HttpClientParams params,
        final HttpState state
    ) {
        super();
        this.connectionManager = connectionManager;
        this.hostConfiguration = hostConfiguration;
        this.params = params;
        this.state = state;
        this.authProcessor = new AuthChallengeProcessor(this.params);
    }
    
    
    /**
     * Executes the method associated with this method director.
     * 
     * @throws IOException
     * @throws HttpException
     */
    public void executeMethod(final HttpMethod method) throws IOException, HttpException {
        if (method == null) {
            throw new IllegalArgumentException("Method may not be null");
        }
        // Link all parameter collections to form the hierarchy:
        // Global -> HttpClient -> HostConfiguration -> HttpMethod
        this.hostConfiguration.getParams().setDefaults(this.params);
        method.getParams().setDefaults(this.hostConfiguration.getParams());
        
        // Generate default request headers
        Collection defaults = (Collection)this.hostConfiguration.getParams().
            getParameter(HostParams.DEFAULT_HEADERS);
        if (defaults != null) {
            Iterator i = defaults.iterator();
            while (i.hasNext()) {
                method.addRequestHeader((Header)i.next());
            }
        }
        
        try {
            int maxRedirects = this.params.getIntParameter(HttpClientParams.MAX_REDIRECTS, 100);

            for (int redirectCount = 0;;) {

                // make sure the connection we have is appropriate
                if (this.conn != null && !hostConfiguration.hostEquals(this.conn)) {
                    this.conn.setLocked(false);
                    this.conn.releaseConnection();
                    this.conn = null;
                }
        
                // get a connection, if we need one
                if (this.conn == null) {
                    this.conn = connectionManager.getConnectionWithTimeout(
                        hostConfiguration,
                        this.params.getConnectionManagerTimeout() 
                    );
                    this.conn.setLocked(true);
                    if (this.params.isAuthenticationPreemptive()
                     || this.state.isAuthenticationPreemptive()) 
                    {
                        LOG.debug("Preemptively sending default basic credentials");
                        method.getHostAuthState().setPreemptive();
                        method.getHostAuthState().setAuthAttempted(true);
                        if (this.conn.isProxied() && !this.conn.isSecure()) {
                            method.getProxyAuthState().setPreemptive();
                            method.getProxyAuthState().setAuthAttempted(true);
                        }
                    }
                }
                authenticate(method);
                executeWithRetry(method);
                if (this.connectMethod != null) {
                    fakeResponse(method);
                    break;
                }
                
                boolean retry = false;
                if (isRedirectNeeded(method)) {
                    if (processRedirectResponse(method)) {
                        retry = true;
                        ++redirectCount;
                        if (redirectCount >= maxRedirects) {
                            LOG.error("Narrowly avoided an infinite loop in execute");
                            throw new RedirectException("Maximum redirects ("
                                + maxRedirects + ") exceeded");
                        }
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Execute redirect " + redirectCount + " of " + maxRedirects);
                        }
                    }
                }
                if (isAuthenticationNeeded(method)) {
                    if (processAuthenticationResponse(method)) {
                        LOG.debug("Retry authentication");
                        retry = true;
                    }
                }
                if (!retry) {
                    break;
                }
                // retry - close previous stream.  Caution - this causes
                // responseBodyConsumed to be called, which may also close the
                // connection.
                if (method.getResponseBodyAsStream() != null) {
                    method.getResponseBodyAsStream().close();
                }

            } //end of retry loop
        } finally {
            if (this.conn != null) {
                this.conn.setLocked(false);
            }
            // If the response has been fully processed, return the connection
            // to the pool.  Use this flag, rather than other tests (like
            // responseStream == null), as subclasses, might reset the stream,
            // for example, reading the entire response into a file and then
            // setting the file as the stream.
            if (
                (releaseConnection || method.getResponseBodyAsStream() == null) 
                && this.conn != null
            ) {
                this.conn.releaseConnection();
            }
        }

    }

    
    private void authenticate(final HttpMethod method) {
        try {
            if (this.conn.isProxied() && !this.conn.isSecure()) {
                authenticateProxy(method);
            }
            authenticateHost(method);
        } catch (AuthenticationException e) {
            LOG.error(e.getMessage(), e);
        }
    }


    private boolean cleanAuthHeaders(final HttpMethod method, final String name) {
        Header[] authheaders = method.getRequestHeaders(name);
        boolean clean = true;
        for (int i = 0; i < authheaders.length; i++) {
            Header authheader = authheaders[i];
            if (authheader.isAutogenerated()) {
                method.removeRequestHeader(authheader);
            } else {
                clean = false;
            }
        }
        return clean;
    }
    

    private void authenticateHost(final HttpMethod method) throws AuthenticationException {
        // Clean up existing authentication headers
        if (!cleanAuthHeaders(method, WWW_AUTH_RESP)) {
            // User defined authentication header(s) present
            return;
        }
        AuthState authstate = method.getHostAuthState();
        AuthScheme authscheme = authstate.getAuthScheme();
        if (authscheme == null) {
            return;
        }
        if (authstate.isAuthRequested() || !authscheme.isConnectionBased()) {
            String host = method.getParams().getVirtualHost();
            if (host == null) {
                host = conn.getHost();
            }
            int port = conn.getPort();
            AuthScope authscope = new AuthScope(
                host, port, 
                authscheme.getRealm(), 
                authscheme.getSchemeName());  
            if (LOG.isDebugEnabled()) {
                LOG.debug("Authenticating with " + authscope);
            }
            Credentials credentials = this.state.getCredentials(authscope);
            if (credentials != null) {
                String authstring = authscheme.authenticate(credentials, method);
                if (authstring != null) {
                    method.addRequestHeader(new Header(WWW_AUTH_RESP, authstring, true));
                }
            } else {
                if (LOG.isWarnEnabled()) {
                    LOG.warn("Required credentials not available for " + authscope);
                    if (method.getHostAuthState().isPreemptive()) {
                        LOG.warn("Preemptive authentication requested but no default " +
                            "credentials available"); 
                    }
                }
            }
        }
    }


    private void authenticateProxy(final HttpMethod method) throws AuthenticationException {
        // Clean up existing authentication headers
        if (!cleanAuthHeaders(method, PROXY_AUTH_RESP)) {
            // User defined authentication header(s) present
            return;
        }
        AuthState authstate = method.getProxyAuthState();
        AuthScheme authscheme = authstate.getAuthScheme();
        if (authscheme == null) {
            return;
        }
        if (authstate.isAuthRequested() || !authscheme.isConnectionBased()) {
            AuthScope authscope = new AuthScope(
                conn.getProxyHost(), conn.getProxyPort(), 
                authscheme.getRealm(), 
                authscheme.getSchemeName());