cachedbdbmap.java

这是个爬虫和lucece相结合最好了

            dbEnvironment.openDbCount--;
            if (dbEnvironment.openDbCount <= 0) {
                dbEnvironment.classCatalog.close();
                dbEnvironment.environment.close();
                dbEnvironmentMap.remove(dbEnvironment.dbDir.getAbsolutePath());
                dbEnvironment = null;
            }
        }
    }

    protected void finalize() throws Throwable {
        close();
        super.finalize();
    }

    /**
     * The keySet of the diskMap is all relevant keys. 
     * 
     * @see java.util.Map#keySet()
     */
    @SuppressWarnings("unchecked")
    public Set<K> keySet() {
        return diskMap.keySet();
    }
    
    public Set<Map.Entry<K,V>> entrySet() {
        // Would require complicated implementation to 
        // maintain identity guarantees, so skipping
        throw new UnsupportedOperationException();
    }

    public synchronized V get(final Object object) {
        K key = toKey(object);
        countOfGets++;
        expungeStaleEntries();
        if (countOfGets % 10000 == 0) {
            logCacheSummary();
        }
        SoftEntry<V> entry = memMap.get(key);
        if (entry != null) {
            V val = entry.get(); // get & hold, so not cleared pre-return
            if (val != null) {
                cacheHit++;
                return val;
            }
            // Explicitly clear this entry from referencequeue since its
            // value is null.
            expungeStaleEntry(entry);
        }

        // check backing diskMap
        V v = diskMapGet(key);
        if (v != null) {
            diskHit++;
            memMap.put(key, new SoftEntry<V>(key, v, refQueue));
        }
        return v;
    }

    /**
     * Info to log, if at FINE level, on every get()
     */
    private void logCacheSummary() {
        if (!logger.isLoggable((Level.FINE))) {
            return;
        }
        try {
            long cacheHitPercent = (cacheHit * 100) / (cacheHit + diskHit);
            logger.fine("DB name: " + this.db.getDatabaseName()
                + ", Cache Hit: " + cacheHitPercent
                + "%, Not in map: " + (countOfGets - (cacheHit + diskHit))
                + ", Total number of gets: " + countOfGets);
        } catch (DatabaseException e) {
            // This is just for logging so ignore DB Exceptions
        }
    }
    
    public synchronized V put(K key, V value) {
        V prevVal = get(key);
        memMap.put(key, new SoftEntry<V>(key, value, refQueue));
        diskMap.put(key,value); // dummy
        if(prevVal==null) {
            diskMapSize++;
        }
        return prevVal;
    }

    /**
     * Note that a call to this method CLOSEs the underlying bdbje.
     * This instance is no longer of any use.  It must be re-initialized.
     * We close the db here because if this BigMap is being treated as a plain
     * Map, this is only opportunity for cleanup.
     */
    public synchronized void clear() {
        this.memMap.clear();
        this.diskMap.clear();
        this.diskMapSize = 0;
        try {
            close();
        } catch (DatabaseException e) {
            e.printStackTrace();
        }
    }

    public synchronized V remove(final Object key) {
        V prevValue = get(key);
        memMap.remove(key);
        expungeStaleEntries();
        diskMap.remove(key);
        diskMapSize--;
        return prevValue;
    }

    public synchronized boolean containsKey(Object key) {
        if (quickContainsKey(key)) {
            return true;
        }
        return diskMap.containsKey(key);
    }

    public synchronized boolean quickContainsKey(Object key) {
        expungeStaleEntries();
        return memMap.containsKey(key);
    }

    public synchronized boolean containsValue(Object value) {
        if (quickContainsValue(value)) {
            return true;
        }
        return diskMap.containsValue(value);
    }

    public synchronized boolean quickContainsValue(Object value) {
        expungeStaleEntries();
        // FIXME this isn't really right, as memMap is of SoftEntries
        return memMap.containsValue(value);
    }

    public int size() {
        return diskMapSize;
    }
    
    protected String getDatabaseName() {
        String name = "DbName-Lookup-Failed";
        try {
            if (this.db != null) {
                name = this.db.getDatabaseName();
            }
        } catch (DatabaseException e) {
            // Ignore.
        }
        return name;
    }
    
    /**
     * Sync in-memory map entries to backing disk store.
     * When done, the memory map will be cleared and all entries stored
     * on disk.
     */
    public synchronized void sync() {
        String dbName = null;
        // Sync. memory and disk.
        long startTime = 0;
        if (logger.isLoggable(Level.INFO)) {
            dbName = getDatabaseName();
            startTime = System.currentTimeMillis();
            logger.info(dbName + " start sizes: disk " + this.diskMapSize +
                ", mem " + this.memMap.size());
        }
        expungeStaleEntries();
        LinkedList<SoftEntry> stale = new LinkedList<SoftEntry>(); 
        for (Iterator i = this.memMap.keySet().iterator(); i.hasNext();) {
            Object key = i.next();
            SoftEntry entry = (SoftEntry) memMap.get(key);
            if (entry != null) {
                // Get & hold so not cleared pre-return.
                Object value = entry.get();
                if (value != null) {
                    this.diskMap.put(key, value);
                } else {
                    stale.add(entry);
                }
            }
        }
        // for any entries above that had been cleared, ensure expunged
        for (SoftEntry entry : stale) {
            expungeStaleEntry(entry);
        }   
        
        // force sync of deferred-writes
        try {
            this.db.sync();
        } catch (DatabaseException e) {
            // TODO Auto-generated catch block
            throw new RuntimeException(e);
        }
        
        if (logger.isLoggable(Level.INFO)) {
            logger.info(dbName + " sync took " +
                (System.currentTimeMillis() - startTime) + "ms. " +
                "Finish sizes: disk " +
                this.diskMapSize + ", mem " + this.memMap.size());
        }
    }

    private void expungeStaleEntries() {
        int c = 0;
        for(SoftEntry entry; (entry = refQueuePoll()) != null;) {
            expungeStaleEntry(entry);
            c++;
        }
        if (c > 0 && logger.isLoggable(Level.FINER)) {
            try {
                logger.finer("DB: " + db.getDatabaseName() + ",  Expunged: "
                        + c + ", Diskmap size: " + diskMapSize
                        + ", Cache size: " + memMap.size());
            } catch (DatabaseException e) {
                // Just for logging so ignore Exceptions
            }
        }
    }
    
    private void expungeStaleEntry(SoftEntry entry) {
        // If phantom already null, its already expunged -- probably
        // because it was purged directly first from inside in
        // {@link #get(String)} and then it went on the poll queue and
        // when it came off inside in expungeStaleEntries, this method
        // was called again.
        if (entry.getPhantom() == null) {
            return;
        }
        // If the object that is in memMap is not the one passed here, then
        // memMap has been changed -- probably by a put on top of this entry.
        if (memMap.get(entry.getPhantom().getKey()) == entry) {
            memMap.remove(entry.getPhantom().getKey());
            diskMap.put(entry.getPhantom().getKey(),
                entry.getPhantom().doctoredGet());
        }
        entry.clearPhantom();
    }
    
    private class PhantomEntry<T> extends PhantomReference<T> {
        private final Object key;

        public PhantomEntry(Object key, T referent) {
            super(referent, null);
            this.key = key;
        }

        /**
         * @return Return the referent. The contract for {@link #get()}
         * always returns a null referent.  We've cheated and doctored
         * PhantomReference to return the actual referent value.  See notes
         * at {@link #referentField};
         */
        public Object doctoredGet() {
            try {
                // Here we use the referentField saved off on static
                // initialization of this class to get at this References'
                // private referent field.
                return referentField.get(this);
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        }

        /**
         * @return Returns the key.
         */
        public Object getKey() {
            return this.key;
        }
    }

    private class SoftEntry<T> extends SoftReference<T> {
        private PhantomEntry<T> phantom;

        public SoftEntry(Object key, T referent, ReferenceQueue<T> q) {
            super(referent, q);
            this.phantom = new PhantomEntry<T>(key, referent);
        }

        /**
         * @return Returns the phantom reference.
         */
        public PhantomEntry getPhantom() {
            return this.phantom;
        }
        
        public void clearPhantom() {
            this.phantom.clear();
            this.phantom = null;
            super.clear();
        }
    }
    
    private void readObject(java.io.ObjectInputStream stream)
    throws IOException, ClassNotFoundException {
        stream.defaultReadObject();
        initializeInstance();
        if (logger.isLoggable(Level.FINE)) {
            logger.fine(getDatabaseName() + " diskMapSize: " + diskMapSize);
        }
    }
    
 
    
    @SuppressWarnings("unchecked")
    private K toKey(Object o) {
        return (K)o;
    }
    
    @SuppressWarnings("unchecked")
    private V diskMapGet(K k) {
        return (V)diskMap.get(k);
    }
    
    @SuppressWarnings("unchecked")
    private SoftEntry<V> refQueuePoll() {
        return (SoftEntry)refQueue.poll();
    }
}