multithreadedhttpconnectionmanager.java

Light in the box 抓取程序。 使用HttpClient

    }

    /**
     * @see HttpConnectionManager#getConnection(HostConfiguration, long)
     * 
     * @deprecated Use #getConnectionWithTimeout(HostConfiguration, long)
     */
    public HttpConnection getConnection(HostConfiguration hostConfiguration, 
        long timeout) throws HttpException {

        LOG.trace("enter HttpConnectionManager.getConnection(HostConfiguration, long)");
        try {
            return getConnectionWithTimeout(hostConfiguration, timeout);
        } catch(ConnectionPoolTimeoutException e) {
            throw new HttpException(e.getMessage());
        }
    }

    private HttpConnection doGetConnection(HostConfiguration hostConfiguration, 
        long timeout) throws ConnectionPoolTimeoutException {

        HttpConnection connection = null;

        int maxHostConnections = this.params.getMaxConnectionsPerHost(hostConfiguration);
        int maxTotalConnections = this.params.getMaxTotalConnections();
        
        synchronized (connectionPool) {

            // we clone the hostConfiguration
            // so that it cannot be changed once the connection has been retrieved
            hostConfiguration = new HostConfiguration(hostConfiguration);
            HostConnectionPool hostPool = connectionPool.getHostPool(hostConfiguration, true);
            WaitingThread waitingThread = null;

            boolean useTimeout = (timeout > 0);
            long timeToWait = timeout;
            long startWait = 0;
            long endWait = 0;

            while (connection == null) {

                if (shutdown) {
                    throw new IllegalStateException("Connection factory has been shutdown.");
                }
                
                // happen to have a free connection with the right specs
                //
                if (hostPool.freeConnections.size() > 0) {
                    connection = connectionPool.getFreeConnection(hostConfiguration);

                // have room to make more
                //
                } else if ((hostPool.numConnections < maxHostConnections) 
                    && (connectionPool.numConnections < maxTotalConnections)) {

                    connection = connectionPool.createConnection(hostConfiguration);

                // have room to add host connection, and there is at least one free
                // connection that can be liberated to make overall room
                //
                } else if ((hostPool.numConnections < maxHostConnections) 
                    && (connectionPool.freeConnections.size() > 0)) {

                    connectionPool.deleteLeastUsedConnection();
                    connection = connectionPool.createConnection(hostConfiguration);

                // otherwise, we have to wait for one of the above conditions to
                // become true
                //
                } else {
                    // TODO: keep track of which hostConfigurations have waiting
                    // threads, so they avoid being sacrificed before necessary

                    try {
                        
                        if (useTimeout && timeToWait <= 0) {
                            throw new ConnectionPoolTimeoutException("Timeout waiting for connection");
                        }
                        
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Unable to get a connection, waiting..., hostConfig=" + hostConfiguration);
                        }
                        
                        if (waitingThread == null) {
                            waitingThread = new WaitingThread();
                            waitingThread.hostConnectionPool = hostPool;
                            waitingThread.thread = Thread.currentThread();
                        } else {
                            waitingThread.interruptedByConnectionPool = false;
                        }
                                    
                        if (useTimeout) {
                            startWait = System.currentTimeMillis();
                        }
                        
                        hostPool.waitingThreads.addLast(waitingThread);
                        connectionPool.waitingThreads.addLast(waitingThread);
                        connectionPool.wait(timeToWait);
                    } catch (InterruptedException e) {
                        if (!waitingThread.interruptedByConnectionPool) {
                            LOG.debug("Interrupted while waiting for connection", e);
                            throw new IllegalThreadStateException(
                                "Interrupted while waiting in MultiThreadedHttpConnectionManager");
                        }
                        // Else, do nothing, we were interrupted by the connection pool
                        // and should now have a connection waiting for us, continue
                        // in the loop and let's get it.
                    } finally {
                        if (!waitingThread.interruptedByConnectionPool) {
                            // Either we timed out, experienced a "spurious wakeup", or were
                            // interrupted by an external thread.  Regardless we need to 
                            // cleanup for ourselves in the wait queue.
                            hostPool.waitingThreads.remove(waitingThread);
                            connectionPool.waitingThreads.remove(waitingThread);
                        }
                        
                        if (useTimeout) {
                            endWait = System.currentTimeMillis();
                            timeToWait -= (endWait - startWait);
                        }
                    }
                }
            }
        }
        return connection;
    }

    /**
     * Gets the total number of pooled connections for the given host configuration.  This 
     * is the total number of connections that have been created and are still in use 
     * by this connection manager for the host configuration.  This value will
     * not exceed the {@link #getMaxConnectionsPerHost() maximum number of connections per
     * host}.
     * 
     * @param hostConfiguration The host configuration
     * @return The total number of pooled connections
     */
    public int getConnectionsInPool(HostConfiguration hostConfiguration) {
        synchronized (connectionPool) {
            HostConnectionPool hostPool = connectionPool.getHostPool(hostConfiguration, false);
            return (hostPool != null) ? hostPool.numConnections : 0;
        }
    }

    /**
     * Gets the total number of pooled connections.  This is the total number of 
     * connections that have been created and are still in use by this connection 
     * manager.  This value will not exceed the {@link #getMaxTotalConnections() 
     * maximum number of connections}.
     * 
     * @return the total number of pooled connections
     */
    public int getConnectionsInPool() {
        synchronized (connectionPool) {
            return connectionPool.numConnections;
        }
    }
    
    /**
     * Gets the number of connections in use for this configuration.
     *
     * @param hostConfiguration the key that connections are tracked on
     * @return the number of connections in use
     * 
     * @deprecated Use {@link #getConnectionsInPool(HostConfiguration)}
     */
    public int getConnectionsInUse(HostConfiguration hostConfiguration) {
        return getConnectionsInPool(hostConfiguration);
    }

    /**
     * Gets the total number of connections in use.
     * 
     * @return the total number of connections in use
     * 
     * @deprecated Use {@link #getConnectionsInPool()}
     */
    public int getConnectionsInUse() {
        return getConnectionsInPool();
    }

    /**
     * Deletes all closed connections.  Only connections currently owned by the connection
     * manager are processed.
     * 
     * @see HttpConnection#isOpen()
     * 
     * @since 3.0
     */
    public void deleteClosedConnections() {
        connectionPool.deleteClosedConnections();
    }
    
    /**
     * @since 3.0
     */
    public void closeIdleConnections(long idleTimeout) {
        connectionPool.closeIdleConnections(idleTimeout);
        deleteClosedConnections();
    }
    
    /**
     * Make the given HttpConnection available for use by other requests.
     * If another thread is blocked in getConnection() that could use this
     * connection, it will be woken up.
     *
     * @param conn the HttpConnection to make available.
     */
    public void releaseConnection(HttpConnection conn) {
        LOG.trace("enter HttpConnectionManager.releaseConnection(HttpConnection)");

        if (conn instanceof HttpConnectionAdapter) {
            // connections given out are wrapped in an HttpConnectionAdapter
            conn = ((HttpConnectionAdapter) conn).getWrappedConnection();
        } else {
            // this is okay, when an HttpConnectionAdapter is released
            // is releases the real connection
        }

        // make sure that the response has been read.
        SimpleHttpConnectionManager.finishLastResponse(conn);

        connectionPool.freeConnection(conn);
    }

    /**
     * Gets the host configuration for a connection.
     * @param conn the connection to get the configuration of
     * @return a new HostConfiguration
     */
    private HostConfiguration configurationForConnection(HttpConnection conn) {

        HostConfiguration connectionConfiguration = new HostConfiguration();
        
        connectionConfiguration.setHost(
            conn.getHost(), 
            conn.getPort(), 
            conn.getProtocol()
        );
        if (conn.getLocalAddress() != null) {
            connectionConfiguration.setLocalAddress(conn.getLocalAddress());
        }
        if (conn.getProxyHost() != null) {
            connectionConfiguration.setProxy(conn.getProxyHost(), conn.getProxyPort());
        }

        return connectionConfiguration;
    }

    /**
     * Returns {@link HttpConnectionManagerParams parameters} associated 
     * with this connection manager.
     * 
     * @since 3.0
     * 
     * @see HttpConnectionManagerParams
     */
    public HttpConnectionManagerParams getParams() {
        return this.params;
    }

    /**
     * Assigns {@link HttpConnectionManagerParams parameters} for this 
     * connection manager.
     * 
     * @since 3.0
     * 
     * @see HttpConnectionManagerParams
     */
    public void setParams(final HttpConnectionManagerParams params) {
        if (params == null) {
            throw new IllegalArgumentException("Parameters may not be null");
        }
        this.params = params;
    }
    
    /**
     * Global Connection Pool, including per-host pools
     */
    private class ConnectionPool {
        
        /** The list of free connections */
        private LinkedList freeConnections = new LinkedList();

        /** The list of WaitingThreads waiting for a connection */
        private LinkedList waitingThreads = new LinkedList();

        /**
         * Map where keys are {@link HostConfiguration}s and values are {@link
         * HostConnectionPool}s
         */
        private final Map mapHosts = new HashMap();

        private IdleConnectionHandler idleConnectionHandler = new IdleConnectionHandler();        
        
        /** The number of created connections */
        private int numConnections = 0;

        /**
         * Cleans up all connection pool resources.
         */
        public synchronized void shutdown() {
            
            // close all free connections
            Iterator iter = freeConnections.iterator();
            while (iter.hasNext()) {
                HttpConnection conn = (HttpConnection) iter.next();
                iter.remove();
                conn.close();
            }
            
            // close all connections that have been checked out
            shutdownCheckedOutConnections(this);
            
            // interrupt all waiting threads
            iter = waitingThreads.iterator();
            while (iter.hasNext()) {
                WaitingThread waiter = (WaitingThread) iter.next();
                iter.remove();
                waiter.interruptedByConnectionPool = true;
                waiter.thread.interrupt();
            }
            
            // clear out map hosts
            mapHosts.clear();
            
            // remove all references to connections
            idleConnectionHandler.removeAll();
        }
        
        /**
         * Creates a new connection and returns it for use of the calling method.
         *
         * @param hostConfiguration the configuration for the connection
         * @return a new connection or <code>null</code> if none are available
         */
        public synchronized HttpConnection createConnection(HostConfiguration hostConfiguration) {
            HostConnectionPool hostPool = getHostPool(hostConfiguration, true);
            if (LOG.isDebugEnabled()) {
                LOG.debug("Allocating new connection, hostConfig=" + hostConfiguration);
            }
            HttpConnectionWithReference connection = new HttpConnectionWithReference(
                    hostConfiguration);
            connection.getParams().setDefaults(MultiThreadedHttpConnectionManager.this.params);
            connection.setHttpConnectionManager(MultiThreadedHttpConnectionManager.this);
            numConnections++;
            hostPool.numConnections++;
    
            // store a reference to this connection so that it can be cleaned up
            // in the event it is not correctly released
            storeReferenceToConnection(connection, hostConfiguration, this);
            return connection;
        }
    
        /**
         * Handles cleaning up for a lost connection with the given config.  Decrements any 
         * connection counts and notifies waiting threads, if appropriate.
         * 
         * @param config the host configuration of the connection that was lost
         */
        public synchronized void handleLostConnection(HostConfiguration config) {
            HostConnectionPool hostPool = getHostPool(config, true);
            hostPool.numConnections--;
            if ((hostPool.numConnections == 0) &&
                hostPool.waitingThreads.isEmpty()) {

                mapHosts.remove(config);
            }
            
            numConnections--;
            notifyWaitingThread(config);
        }

        /**
         * Get the pool (list) of connections available for the given hostConfig.
         *
         * @param hostConfiguration the configuraton for the connection pool
         * @param create <code>true</code> to create a pool if not found,
         *               <code>false</code> to return <code>null</code>
         *
         * @return a pool (list) of connections available for the given config,
         *         or <code>null</code> if neither found nor created
         */
        public synchronized HostConnectionPool getHostPool(HostConfiguration hostConfiguration, boolean create) {
            LOG.trace("enter HttpConnectionManager.ConnectionPool.getHostPool(HostConfiguration)");

            // Look for a list of connections for the given config
            HostConnectionPool listConnections = (HostConnectionPool) 
                mapHosts.get(hostConfiguration);
            if ((listConnections == null) && create) {
                // First time for this config
                listConnections = new HostConnectionPool();
                listConnections.hostConfiguration = hostConfiguration;
                mapHosts.put(hostConfiguration, listConnections);
            }
            
            return listConnections;
        }

        /**
         * If available, get a free connection for this host
         *
         * @param hostConfiguration the configuraton for the connection pool
         * @return an available connection for the given config
         */
        public synchronized HttpConnection getFreeConnection(HostConfiguration hostConfiguration) {

            HttpConnectionWithReference connection = null;
            
            HostConnectionPool hostPool = getHostPool(hostConfiguration, false);

            if ((hostPool != null) && (hostPool.freeConnections.size() > 0)) {
                connection = (HttpConnectionWithReference) hostPool.freeConnections.removeLast();
                freeConnections.remove(connection);
                // store a reference to this connection so that it can be cleaned up
                // in the event it is not correctly released
                storeReferenceToConnection(connection, hostConfiguration, this);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Getting free connection, hostConfig=" + hostConfiguration);
                }