fpuriuniqfilter.java

最强的爬虫工程

/* Copyright (C) 2003 Internet Archive.
 *
 * This file is part of the Heritrix web crawler (crawler.archive.org).
 *
 * Heritrix is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * any later version.
 *
 * Heritrix is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser Public License for more details.
 *
 * You should have received a copy of the GNU Lesser Public License
 * along with Heritrix; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 * MemFPUURISet.java
 * Created on Oct 1, 2003
 *
 * $Header: /cvsroot/archive-crawler/ArchiveOpenCrawler/src/java/org/archive/crawler/util/FPUriUniqFilter.java,v 1.12 2005/10/05 20:36:51 gojomo Exp $
 */
package org.archive.crawler.util;

import java.io.Serializable;
import java.util.logging.Logger;

import org.archive.util.ArchiveUtils;
import org.archive.util.fingerprint.LongFPSet;

import st.ata.util.FPGenerator;

/**
 * UriUniqFilter storing 64-bit UURI fingerprints, using an internal LongFPSet
 * instance. 
 * 
 * The passed LongFPSet internal instance may be disk or memory based. Accesses
 * to the underlying LongFPSet are synchronized.
 *
 * @author gojomo
 */
public class FPUriUniqFilter extends SetBasedUriUniqFilter 
implements Serializable {
    // Be robust against trivial implementation changes
    private static final long serialVersionUID =
        ArchiveUtils.classnameBasedUID(FPUriUniqFilter.class, 1);
    
    private static Logger logger =
        Logger.getLogger(FPUriUniqFilter.class.getName());
    
    private LongFPSet fpset;
    private transient FPGenerator fpgen = FPGenerator.std64;
    
    /**
     * Create FPUriUniqFilter wrapping given long set
     * 
     * @param fpset
     */
    public FPUriUniqFilter(LongFPSet fpset) {
        this.fpset = fpset;
    }
    
    private long getFp(CharSequence canonical) {
        return fpgen.fp(canonical);
    }

    protected boolean setAdd(CharSequence uri) {
        return fpset.add(getFp(uri));
    }

    protected long setCount() {
        return fpset.count();
    }

    protected boolean setRemove(CharSequence uri) {
        return fpset.remove(getFp(uri));
    }
}