adaptiverevisitqueuelist.java

这是个爬虫和lucece相结合最好了

    /**
     * Returns the size of the largest (deepest) queue.
     * @return the size of the largest (deepest) queue.
     */
    public long getDeepestQueueSize(){
    	long size = 0;
        for (Iterator it = sortedHostQueues.iterator(); it.hasNext();) {
            AdaptiveRevisitHostQueue hq = ((AdaptiveRevisitHostQueueWrapper)it
                    .next()).hq;
            if(hq.getSize() > size){
            	size = hq.getSize();
            }
        }
        return size;
    }
    
    /**
     * Returns the congestion ratio.
     * <p>
     * The congestion ratio is equal to the total number of queues divided
     * by the number of queues currently being processed or are snozzed (i.e. 
     * not ready). A congestion ratio of 1 indicates no congestion.
     * @return the congestion ratio
     */
    public float getCongestionRatio(){
    	int readyQueues = 0;
        for (Iterator it = sortedHostQueues.iterator(); it.hasNext();) {
            AdaptiveRevisitHostQueue hq = ((AdaptiveRevisitHostQueueWrapper)it
                    .next()).hq;
            if(hq.getState() == AdaptiveRevisitHostQueue.HQSTATE_READY){
            	readyQueues++;
            }
        }
        int totalQueues = hostQueues.size();
        
        return (float)(totalQueues) / (totalQueues-readyQueues);
    }
    
    /**
     * This method reorders the host queues. Method is only called by the
     * AdaptiveRevisitHostQueue that it 'owns' when their reported time of next
     * ready is being updated.
     * 
     * @param hq The calling HQ
     */
    protected void reorder(AdaptiveRevisitHostQueue hq){
        // Find the wrapper
        AdaptiveRevisitHostQueueWrapper wrapper = 
            (AdaptiveRevisitHostQueueWrapper)hostQueues.get(hq.getHostName());
        
        long newTime = hq.getNextReadyTime();
        
        if(newTime != wrapper.nextReadyTime){
            // Ok, the time has changed, move the queue around.
            if (logger.isLoggable(Level.FINER)) {
                logger.finer("reorder(" + hq.getHostName() + ") was "
                        + wrapper.nextReadyTime);
            }
            // Remove it from the sorted list
            sortedHostQueues.remove(wrapper);
            // Update the time on the ref.
            wrapper.nextReadyTime = newTime;
            if (logger.isLoggable(Level.FINER)) {
                logger.finer("reorder(" + hq.getHostName() + ") is "
                        + wrapper.nextReadyTime);
            }
            // Readd to the list
            sortedHostQueues.add(wrapper);
        }
    }
    
    /**
     * The total number of URIs queued in all the HQs belonging to this list.
     * 
     * @return total number of URIs queued in all the HQs belonging to this list.
     */
    public long getUriCount(){
        Iterator it = hostQueues.keySet().iterator();
        long count = 0;
        while(it.hasNext()){
            AdaptiveRevisitHostQueueWrapper wrapper =
                (AdaptiveRevisitHostQueueWrapper)it.next();
            count += wrapper.hq.getSize();
        }
        return count;
    }
        
    /**
     * This class wraps an AdaptiveRevisitHostQueue with a fixed value for next
     * ready time.
     * This is done to facilitate sorting by making sure that the value does 
     * not change while the HQ is in the sorted list. With this wrapper, it
     * is possible to remove it from the sorted list, then update the time of
     * next ready, and then readd (resort) it. 
     *
     * @author Kristinn Sigurdsson
     */
    private class AdaptiveRevisitHostQueueWrapper implements Comparable{
        long nextReadyTime;
        AdaptiveRevisitHostQueue hq;
        
        public AdaptiveRevisitHostQueueWrapper(AdaptiveRevisitHostQueue hq){
            nextReadyTime = hq.getNextReadyTime();
            this.hq = hq;
        }
        
        /**
         * Compares the this ARHQWrapper to the supplied one.
         * 
         * @param obj the HQ to compare to. If this object is not an instance
         *             of ARHostQueueWrapper then the method will throw a 
         *             ClassCastException.
         * @return a value less than 0 if this HQWrappers time of next ready
         *         value is less than the argument HQWrappers's; and a value 
         *         greater than 0 if this value is greater. If the time of
         *         next ready is equal, the hostName strings will be compared
         *         and that result returned.
         */
        public int compareTo(Object obj){
            AdaptiveRevisitHostQueueWrapper comp =
                (AdaptiveRevisitHostQueueWrapper)obj;

            long compTime = comp.nextReadyTime;
            
            if(nextReadyTime>compTime){
                return 1;
            } else if(nextReadyTime<compTime){
                return -1;
            } else {
                // Equal time. Use hostnames
                return hq.getHostName().compareTo(comp.hq.getHostName());
            }
        }
    }
    
    /**
     * Closes all HQs and the Environment. 
     */
    public void close(){
        Iterator it = sortedHostQueues.iterator();
        while(it.hasNext()){
            AdaptiveRevisitHostQueue hq =
                ((AdaptiveRevisitHostQueueWrapper)it.next()).hq;
            try {
                hq.close();
            } catch (IOException e) {
                logger.severe("IOException while closing " + hq.getHostName() +
                        "\n" + e.getMessage());
            }
        }
        try {
            hostNamesDB.close();
        } catch (DatabaseException e) {
            logger.severe("IOException while closing hostNamesDB" +
                    "\n" + e.getMessage());
        }
    }
    
    //
    // Reporter implementation
    //
    
    public String[] getReports() {
        // none but default for now
        return new String[] {};
    }
    
    /* (non-Javadoc)
     * @see org.archive.util.Reporter#singleLineReport()
     */
    public String singleLineReport() {
        return ArchiveUtils.singleLineReport(this);
    }

    /* (non-Javadoc)
     * @see org.archive.util.Reporter#reportTo(java.io.Writer)
     */
    public void reportTo(PrintWriter writer) {
        Iterator it = sortedHostQueues.iterator();
        while(it.hasNext()){
            AdaptiveRevisitHostQueueWrapper wrapper =
                (AdaptiveRevisitHostQueueWrapper)it.next();
          
            writer.print(wrapper.hq.report(10));
            writer.print("\n\n");
        }
    }
    
    public void reportTo(String name, PrintWriter writer) {
        if(name==null || hostQueues.containsKey(name)==false){
        	reportTo(writer);
        } else {
	    	AdaptiveRevisitHostQueueWrapper wrapper =
	                (AdaptiveRevisitHostQueueWrapper)hostQueues.get(name);
	          
	        writer.print(wrapper.hq.report(0));
	        writer.print("\n\n");
        }
    }
    
    public void singleLineReportTo(PrintWriter writer) {
        Iterator it = sortedHostQueues.iterator();
        int total = 0;
        int ready = 0;
        int snoozed = 0;
        int empty = 0;
        int busy = 0;
        while(it.hasNext()){
            AdaptiveRevisitHostQueueWrapper wrapper =
                (AdaptiveRevisitHostQueueWrapper)it.next();
            total++;
            switch(wrapper.hq.getState()){
                case AdaptiveRevisitHostQueue.HQSTATE_BUSY : busy++; break;
                case AdaptiveRevisitHostQueue.HQSTATE_EMPTY : empty++; break;
                case AdaptiveRevisitHostQueue.HQSTATE_READY : ready++; break;
                case AdaptiveRevisitHostQueue.HQSTATE_SNOOZED : snoozed++; break;
            }
        }
        writer.print(total + " queues: " + ready + " ready, " + snoozed + 
            " snoozed, " + busy + " busy, and " + empty + " empty");
    }
    
    /* (non-Javadoc)
     * @see org.archive.util.Reporter#singleLineLegend()
     */
    public String singleLineLegend() {
        return "total ready snoozed busy empty";
    }

}