persistentstorage.java

pastry的java实现的2.0b版

/*************************************************************************

"FreePastry" Peer-to-Peer Application Development Substrate 

Copyright 2002, Rice University. All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

- Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.

- Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

- Neither  the name  of Rice  University (RICE) nor  the names  of its
contributors may be  used to endorse or promote  products derived from
this software without specific prior written permission.

This software is provided by RICE and the contributors on an "as is"
basis, without any representations or warranties of any kind, express
or implied including, but not limited to, representations or
warranties of non-infringement, merchantability or fitness for a
particular purpose. In no event shall RICE or contributors be liable
for any direct, indirect, incidental, special, exemplary, or
consequential damages (including, but not limited to, procurement of
substitute goods or services; loss of use, data, or profits; or
business interruption) however caused and on any theory of liability,
whether in contract, strict liability, or tort (including negligence
or otherwise) arising in any way out of the use of this software, even
if advised of the possibility of such damage.

********************************************************************************/

package rice.persistence;

/*
 *  @(#) PersistenceManager.java
 *
 *  @author Ansley Post
 *  @author Alan Mislove
 *
 *  @version $Id: PersistentStorage.java 3007 2006-02-02 13:50:02Z amislove $
 */
import java.io.*;
import java.util.*;
import java.util.zip.*;

import rice.*;
import rice.Continuation.*;
import rice.environment.Environment;
import rice.environment.logging.Logger;
import rice.environment.processing.WorkRequest;
import rice.p2p.commonapi.*;
import rice.p2p.util.*;

/**
 * This class is an implementation of Storage which provides persistent storage
 * to disk. This class also guarentees that the data will be consistent, even
 * after a crash. This class also provides these services is a non-blocking
 * fashion by launching a seperate thread which is tasked with actaully writing
 * the data to disk. This class was initially designed to support only Ids whose
 * toString() method returns a String of constant length. It has been exteneded
 * to support variable-length toString()s, but we have the caveat that not
 * toString() can be a substring of another toString() - this will cause
 * undefined behavior. <b>Additionally, the toString() method on the key Ids
 * *CANNOT* have the period ('.') or exclamation point ('!') characters in them
 * - these are used for internal purposes.<b> The serialized objects are stored
 * on-disk in a GZIPed XML format, which provides extensibility with reasonable
 * storage and processing costs. Additionally, any metadata, if provided, is
 * also stored in the on-disk file. The format of the file is [Key, Object,
 * Version, Gzipped XML] [Metadata, Gzipped XML] [persistence magic number,
 * long] [persistence version, long] [persistence revision, long] [metadata
 * length, long] The persistence package is set up to automatically upgrade
 * older versions of the on-disk format as new data is written under the key.
 * Persistence also supports the metadata interface specified in the Catalog
 * interface. All metadata is guaranteeded to be stored in memory, so fetching
 * the metadata of a given key is an efficient operation.
 *
 * @version $Id: pretty.settings 2305 2005-03-11 20:22:33Z jeffh $
 * @author jeffh
 */
public class PersistentStorage implements Storage {

  /**
   * Fields for logging based on the requests we are writing.
   */
  private Object statLock = new Object();
  private long statsLastWritten;
  private long statsWriteInterval = 60 * 1000;
  private long numWrites = 0;
  private long numReads = 0;
  private long numRenames = 0;
  private long numDeletes = 0;
  private long numMetadataWrites = 0;

  private IdFactory factory;
  // the factory used for creating ids

  private String name;
  // the name of this instance
  private File rootDirectory;
  // root directory to store stuff in
  private File backupDirectory;
  // dir for storing persistent objs
  private File appDirectory;
  // dir for storing persistent objs
  private File lostDirectory;
  // dir for lost objects

  private boolean index;
  // whether or not we are indexing the objects
  private HashMap directories;
  // the in-memory map of directories (for efficiency)
  private HashMap prefixes;
  // an in-memory cache of the directory prefixes
  private HashSet dirty;
  // the list of directories which have dirty metadata

  private ReverseTreeMap metadata;
  // the in-memory cache of object metadata

  private String rootDir;
  // rootDirectory

  private long storageSize;
  // The amount of storage allowed to be used
  private long usedSize;
  // The amount of storage currently in use

  Environment environment;
  Logger logger;

  /**
   * Static variables defining the layout of the on-disk storage
   */
  public final static long PERSISTENCE_MAGIC_NUMBER = 8038844221L;
  /**
   * DESCRIBE THE FIELD
   */
  public final static long PERSISTENCE_VERSION_2 = 2L;
  /**
   * DESCRIBE THE FIELD
   */
  public final static long PERSISTENCE_REVISION_2_0 = 0L;
  /**
   * DESCRIBE THE FIELD
   */
  public final static long PERSISTENCE_REVISION_2_1 = 1L;

  /**
   * Static variables which define the location of the storage root
   */
  public final static String BACKUP_DIRECTORY = "/FreePastry-Storage-Root/";
  /**
   * DESCRIBE THE FIELD
   */
  public final static String LOST_AND_FOUND_DIRECTORY = "lost+found";
  /**
   * DESCRIBE THE FIELD
   */
  public final static String METADATA_FILENAME = "metadata.cache";

  /**
   * The splitting factor, or the number of files in one directory
   */
  public final static int MAX_FILES = 256;

  /**
   * The maximum number of subdirectories in a directory before splitting
   */
  public final static int MAX_DIRECTORIES = 32;

  /**
   * The amount of time before re-writing the metadata file
   */
  public final static int METADATA_SYNC_TIME = 300000;

  /**
   * Special placeholder for the file whose name should be zero-length
   */
  public final static String ZERO_LENGTH_NAME = "!";

  /**
   * Builds a PersistentStorage given a root directory in which to persist the
   * data. Uses a default instance name.
   *
   * @param factory The factory to use for creating Ids.
   * @param rootDir The root directory of the persisted disk.
   * @param size the size of the storage in bytes, or -1 for unlimited
   * @param env DESCRIBE THE PARAMETER
   * @exception IOException DESCRIBE THE EXCEPTION
   */
  public PersistentStorage(IdFactory factory, String rootDir, long size, Environment env) throws IOException {
    this(factory, "default", rootDir, size, env);
  }

  /**
   * Builds a PersistentStorage given and an instance name and a root directoy
   * in which to persist the data.
   *
   * @param factory The factory to use for creating Ids.
   * @param name the name of this instance
   * @param rootDir The root directory of the persisted disk.
   * @param size the size of the storage in bytes, or -1 for unlimited
   * @param env DESCRIBE THE PARAMETER
   * @exception IOException DESCRIBE THE EXCEPTION
   */
  public PersistentStorage(IdFactory factory, String name, String rootDir, long size, Environment env) throws IOException {
    this(factory, name, rootDir, size, true, env);
  }

  /**
   * Builds a PersistentStorage given and an instance name and a root directoy
   * in which to persist the data.
   *
   * @param factory The factory to use for creating Ids.
   * @param name the name of this instance
   * @param rootDir The root directory of the persisted disk.
   * @param size the size of the storage in bytes, or -1 for unlimited
   * @param index Whether or not to index the objects
   * @param env DESCRIBE THE PARAMETER
   * @exception IOException DESCRIBE THE EXCEPTION
   */
  public PersistentStorage(IdFactory factory, String name, String rootDir, long size, boolean index, Environment env) throws IOException {
    this.environment = env;
    logger = environment.getLogManager().getLogger(PersistentStorage.class, null);
    this.factory = factory;
    this.name = name;
    this.rootDir = rootDir;
    this.storageSize = size;
    this.index = index;
    this.directories = new HashMap();
    this.prefixes = new HashMap();
    statsLastWritten = environment.getTimeSource().currentTimeMillis();
    if (index) {
      this.dirty = new HashSet();
      this.metadata = new ReverseTreeMap();
    }

    if (logger.level <= Logger.INFO) {
      logger.log("Launching persistent storage in " + rootDir + " with name " + name + " spliting factor " + MAX_FILES);
    }

    init();
  }

  /**
   * Returns the metadata associated with the provided object, or null if no
   * metadata exists. The metadata must be stored in memory, so this operation
   * is guaranteed to be fast and non-blocking.
   *
   * @param id The id for which the metadata is needed
   * @return The metadata, or null of non exists
   */
  public Serializable getMetadata(Id id) {
    if (index) {
      synchronized (metadata) {
        return (Serializable) metadata.get(id);
      }
    } else {
      throw new UnsupportedOperationException("getMetadata() not supported without indexing");
    }
  }

  /**
   * Returns the object identified by the given id.
   *
   * @param id The id of the object in question.
   * @param c The command to run once the operation is complete
   */
  public void getObject(final Id id, Continuation c) {
    printStats();

    if (index && (!exists(id))) {
      c.receiveResult(null);
    } else {
      environment.getProcessor().processBlockingIO(
        new WorkRequest(c, environment.getSelectorManager()) {
          public String toString() {
            return "getObject " + id;
          }

          public Object doWork() throws Exception {
            synchronized (statLock) {
              numReads++;
            }

            /*
             *  get the file
             */
            File objFile = getFile(id);

            try {
              /*
               *  and make sure that it exists
               */
              if ((objFile == null) || (!objFile.exists())) {
                return null;
              }

              if (logger.level <= Logger.FINER) {
                logger.log("COUNT: Fetching data under " + id.toStringFull() + " of size " + objFile.length() + " in " + name);
              }
              return readData(objFile);
            } catch (Exception e) {
              /*
               *  remove our index for this file
               */
              if (index) {
                synchronized (metadata) {
                  metadata.remove(id);
                  dirty.add(objFile.getParentFile());
                }
              }

              /*
               *  if there's a problem, move the file to the lost+found
               */
              moveToLost(objFile);

              throw e;
            }
          }
        });
    }
  }

  /**
   * Returns the total size of the stored data in bytes.The result is returned
   * via the receiveResult method on the provided Continuation with an Integer
   * representing the size.
   *
   * @return The total size, in bytes, of data stored.
   */
  public long getTotalSize() {
    return usedSize;
  }

  /**
   * Returns the number of Ids currently stored in the catalog
   *
   * @return The number of ids in the catalog
   */
  public int getSize() {
    if (index) {
      return metadata.size();
    } else {
      throw new UnsupportedOperationException("getSize() not supported without indexing");
    }
  }

  /**
   * Returns the sublist of the provided list which starts with the given
   * prefix.
   *
   * @param prefix The prefix to look for
   * @param dirNames DESCRIBE THE PARAMETER
   * @return The sublist
   */
  private String[] getMatchingDirectories(String prefix, String[] dirNames) {
    Vector result = new Vector();

    for (int i = 0; i < dirNames.length; i++) {
      if (dirNames[i].startsWith(prefix)) {
        result.add(dirNames[i]);
      }
    }

    return (String[]) result.toArray(new String[0]);
  }

  /**
   * This method takes in a list of the file names in a given directory which
   * needs to be expanded, and returns the lists of directories which should be
   * created for the expansion to happen.
   *
   * @param names The names to return the directories for
   * @return The directory names
   */
  private String[] getDirectories(String[] names) {
    int length = getPrefixLength(names);
    String prefix = names[0].substring(0, length);
    CharacterHashSet set = new CharacterHashSet();

    for (int i = 0; i < names.length; i++) {
      if (names[i].length() > length) {
        set.put(names[i].charAt(length));
      }
    }

    char[] splits = set.get();
    String[] result = new String[splits.length];

    for (int i = 0; i < result.length; i++) {
      result[i] = prefix + splits[i];
    }

    return result;
  }

  /**
   * This method takes in the list of file names in a given directory and
   * returns the longest common prefix of all of the names.
   *
   * @param names The names to find the prefix of
   * @return The longest common prefix of all of the names
   */
  private int getPrefixLength(String[] names) {
    int length = names[0].length() - 1;

    for (int i = 0; i < names.length; i++) {
      length = getPrefixLength(names[0], names[i], length);
    }

    return length;
  }

  /**
   * Method which takes in two strings and returns the length of the shared
   * prefix, or a predefined maximum.