changeevaluator.java

这是个爬虫和lucece相结合最好了

/* ChangeEvaluator
 * 
 * $Id: ChangeEvaluator.java 4654 2006-09-25 20:19:54Z paul_jack $
 * 
 * Created on 11.11.2004
 *
 * Copyright (C) 2004 Kristinn Sigurdsson.
 * 
 * This file is part of the Heritrix web crawler (crawler.archive.org).
 * 
 * Heritrix is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * any later version.
 * 
 * Heritrix is distributed in the hope that it will be useful, 
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser Public License
 * along with Heritrix; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
package org.archive.crawler.extractor;

import java.util.logging.Level;
import java.util.logging.Logger;

import org.archive.crawler.datamodel.CrawlURI;
import org.archive.crawler.framework.Processor;
import org.archive.crawler.frontier.AdaptiveRevisitAttributeConstants;
import org.archive.util.Base32;

/**
 * This processor compares the CrawlURI's current 
 * {@link org.archive.crawler.datamodel.CrawlURI#getContentDigest() content digest}
 * with the one from a previous crawl. If they are equal, then further 
 * processing is skipped (going straight to the post processor chain) and the
 * CrawlURI is marked appropriately.
 *
 * @author Kristinn Sigurdsson
 */
public class ChangeEvaluator extends Processor
implements AdaptiveRevisitAttributeConstants {

    private static final long serialVersionUID = 5547590621493534632L;
    private static final Logger logger =
        Logger.getLogger(ChangeEvaluator.class.getName());

    /**
     * Constructor
     * @param name The name of the module
     */
    public ChangeEvaluator(String name) {
        super(name, "Compares CrawlURI's current " +
                "content digest with digest from previous crawl. If " +
                "equal, further processing is skipped (going " +
                "straight to the post processor chain) and the CrawlURI is " +
                "marked appropriately. Should be located at the start of " +
                "the Extractor chain.");

        // Register persistent CrawlURI items 
        CrawlURI.addAlistPersistentMember(A_LAST_CONTENT_DIGEST);
        CrawlURI.addAlistPersistentMember(A_NUMBER_OF_VISITS);
        CrawlURI.addAlistPersistentMember(A_NUMBER_OF_VERSIONS);
    }

    protected void innerProcess(CrawlURI curi) throws InterruptedException {
        if (curi.isSuccess() == false) {
            // Early return. No point in doing comparison on failed downloads.
            if (logger.isLoggable(Level.FINEST)) {
                logger.finest("Not handling " + curi.toString()
                        + ", did not " + "succeed.");
            }
            return;
        }
        
        // If a mid fetch filter aborts the HTTP fetch because the headers
        // predict no change, we can skip the whole comparing hashes.
        if (!curi.containsKey(A_CONTENT_STATE_KEY) ||
                curi.getInt(A_CONTENT_STATE_KEY) != CONTENT_UNCHANGED) {
            String currentDigest = null;
            Object digest = curi.getContentDigest();
            if (digest != null) {
                currentDigest = Base32.encode((byte[])digest);
            }
    
            String oldDigest = null;
            if (curi.containsKey(A_LAST_CONTENT_DIGEST)) {
                oldDigest = curi.getString(A_LAST_CONTENT_DIGEST);
            }
    
            // Compare the String representation of the byte arrays.
            if (currentDigest == null && oldDigest == null) {
                // Both are null, can't do a thing
                if (logger.isLoggable(Level.FINER)) {
                    logger.finer("On " + curi.toString()
                            + " both digest are null");
                }
                // NOTE! RETURN!
                return;
            }
            
            if (currentDigest != null && oldDigest != null 
                    && currentDigest.equals(oldDigest)) { 
                // If equal, we have just downloaded a duplicate.
                if (logger.isLoggable(Level.FINER)) {
                    logger.finer("On " + curi.toString()
                            + " both digest are " + "equal. Old: " + oldDigest
                            + ", new: " + currentDigest);
                }
                curi.putInt(A_CONTENT_STATE_KEY, CONTENT_UNCHANGED);
                // TODO: In the future processors should take note of the content
                // state, removing the need for the following 'skip'
                curi.skipToProcessorChain(getController().getPostprocessorChain());
                // Make not in log
                curi.addAnnotation("unchanged");
                // Set content size to zero, we are not going to 'write it to disk'
                curi.setContentSize(0);
            } else {
                // Document has changed
                if (logger.isLoggable(Level.FINER)) {
                    logger.finer("On " + curi.toString()
                            + " digest are not " + "equal. Old: "
                            + (oldDigest == null? "null": oldDigest)
                            + ", new: "
                            + (currentDigest == null? "null": currentDigest));
                }
                // currentDigest may be null, that probably means a failed download
                curi.putInt(A_CONTENT_STATE_KEY, CONTENT_CHANGED);
                curi.putString(A_LAST_CONTENT_DIGEST, currentDigest); 
            }
        } else {
            if (logger.isLoggable(Level.FINER)) {
                logger.finer("On " + curi.toString()
                        + " content state was " + "already set as UNCHANGED.");
            }
        }
        
        // Update visit and version counters
        int visits = 1;
        if(curi.containsKey(A_NUMBER_OF_VISITS)) {
            visits = curi.getInt(A_NUMBER_OF_VISITS) + 1;
        }
        curi.putInt(A_NUMBER_OF_VISITS, visits);

        // Update versions.
        if(curi.getInt(A_CONTENT_STATE_KEY) == CONTENT_CHANGED) {
            int versions = 1;
            if(curi.containsKey(A_NUMBER_OF_VERSIONS)) {
                versions = curi.getInt(A_NUMBER_OF_VERSIONS) + 1;
            }
            curi.putInt(A_NUMBER_OF_VERSIONS,versions);
        }
    }
}