segmentinfos.java

lucene-2.4.0 是一个全文收索的工具包

  public static int getDefaultGenFileRetryCount() {
    return defaultGenFileRetryCount;
  }

  /**
   * Advanced: set how many milliseconds to pause in between
   * attempts to load the segments.gen file.
   */
  public static void setDefaultGenFileRetryPauseMsec(int msec) {
    defaultGenFileRetryPauseMsec = msec;
  }

  /**
   * @see #setDefaultGenFileRetryPauseMsec
   */
  public static int getDefaultGenFileRetryPauseMsec() {
    return defaultGenFileRetryPauseMsec;
  }

  /**
   * Advanced: set how many times to try incrementing the
   * gen when loading the segments file.  This only runs if
   * the primary (listing directory) and secondary (opening
   * segments.gen file) methods fail to find the segments
   * file.
   */
  public static void setDefaultGenLookaheadCount(int count) {
    defaultGenLookaheadCount = count;
  }
  /**
   * @see #setDefaultGenLookaheadCount
   */
  public static int getDefaultGenLookahedCount() {
    return defaultGenLookaheadCount;
  }

  /**
   * @see #setInfoStream
   */
  public static PrintStream getInfoStream() {
    return infoStream;
  }

  private static void message(String message) {
    if (infoStream != null) {
      infoStream.println("SIS [" + Thread.currentThread().getName() + "]: " + message);
    }
  }

  /**
   * Utility class for executing code that needs to do
   * something with the current segments file.  This is
   * necessary with lock-less commits because from the time
   * you locate the current segments file name, until you
   * actually open it, read its contents, or check modified
   * time, etc., it could have been deleted due to a writer
   * commit finishing.
   */
  public abstract static class FindSegmentsFile {
    
    File fileDirectory;
    Directory directory;

    public FindSegmentsFile(File directory) {
      this.fileDirectory = directory;
    }

    public FindSegmentsFile(Directory directory) {
      this.directory = directory;
    }

    public Object run() throws CorruptIndexException, IOException {
      String segmentFileName = null;
      long lastGen = -1;
      long gen = 0;
      int genLookaheadCount = 0;
      IOException exc = null;
      boolean retry = false;

      int method = 0;

      // Loop until we succeed in calling doBody() without
      // hitting an IOException.  An IOException most likely
      // means a commit was in process and has finished, in
      // the time it took us to load the now-old infos files
      // (and segments files).  It's also possible it's a
      // true error (corrupt index).  To distinguish these,
      // on each retry we must see "forward progress" on
      // which generation we are trying to load.  If we
      // don't, then the original error is real and we throw
      // it.
      
      // We have three methods for determining the current
      // generation.  We try the first two in parallel, and
      // fall back to the third when necessary.

      while(true) {

        if (0 == method) {

          // Method 1: list the directory and use the highest
          // segments_N file.  This method works well as long
          // as there is no stale caching on the directory
          // contents (NOTE: NFS clients often have such stale
          // caching):
          String[] files = null;

          long genA = -1;

          if (directory != null)
            files = directory.list();
          else
            files = fileDirectory.list();
          
          if (files != null)
            genA = getCurrentSegmentGeneration(files);

          message("directory listing genA=" + genA);

          // Method 2: open segments.gen and read its
          // contents.  Then we take the larger of the two
          // gen's.  This way, if either approach is hitting
          // a stale cache (NFS) we have a better chance of
          // getting the right generation.
          long genB = -1;
          if (directory != null) {
            for(int i=0;i<defaultGenFileRetryCount;i++) {
              IndexInput genInput = null;
              try {
                genInput = directory.openInput(IndexFileNames.SEGMENTS_GEN);
              } catch (FileNotFoundException e) {
                message("segments.gen open: FileNotFoundException " + e);
                break;
              } catch (IOException e) {
                message("segments.gen open: IOException " + e);
              }

              if (genInput != null) {
                try {
                  int version = genInput.readInt();
                  if (version == FORMAT_LOCKLESS) {
                    long gen0 = genInput.readLong();
                    long gen1 = genInput.readLong();
                    message("fallback check: " + gen0 + "; " + gen1);
                    if (gen0 == gen1) {
                      // The file is consistent.
                      genB = gen0;
                      break;
                    }
                  }
                } catch (IOException err2) {
                  // will retry
                } finally {
                  genInput.close();
                }
              }
              try {
                Thread.sleep(defaultGenFileRetryPauseMsec);
              } catch (InterruptedException e) {
                // will retry
              }
            }
          }

          message(IndexFileNames.SEGMENTS_GEN + " check: genB=" + genB);

          // Pick the larger of the two gen's:
          if (genA > genB)
            gen = genA;
          else
            gen = genB;
          
          if (gen == -1) {
            // Neither approach found a generation
            String s;
            if (files != null) {
              s = "";
              for(int i=0;i<files.length;i++)
                s += " " + files[i];
            } else
              s = " null";
            throw new FileNotFoundException("no segments* file found in " + directory + ": files:" + s);
          }
        }

        // Third method (fallback if first & second methods
        // are not reliable): since both directory cache and
        // file contents cache seem to be stale, just
        // advance the generation.
        if (1 == method || (0 == method && lastGen == gen && retry)) {

          method = 1;

          if (genLookaheadCount < defaultGenLookaheadCount) {
            gen++;
            genLookaheadCount++;
            message("look ahead increment gen to " + gen);
          }
        }

        if (lastGen == gen) {

          // This means we're about to try the same
          // segments_N last tried.  This is allowed,
          // exactly once, because writer could have been in
          // the process of writing segments_N last time.

          if (retry) {
            // OK, we've tried the same segments_N file
            // twice in a row, so this must be a real
            // error.  We throw the original exception we
            // got.
            throw exc;
          } else {
            retry = true;
          }

        } else if (0 == method) {
          // Segment file has advanced since our last loop, so
          // reset retry:
          retry = false;
        }

        lastGen = gen;

        segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                                "",
                                                                gen);

        try {
          Object v = doBody(segmentFileName);
          if (exc != null) {
            message("success on " + segmentFileName);
          }
          return v;
        } catch (IOException err) {

          // Save the original root cause:
          if (exc == null) {
            exc = err;
          }

          message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retry=" + retry + "; gen = " + gen);

          if (!retry && gen > 1) {

            // This is our first time trying this segments
            // file (because retry is false), and, there is
            // possibly a segments_(N-1) (because gen > 1).
            // So, check if the segments_(N-1) exists and
            // try it if so:
            String prevSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                                               "",
                                                                               gen-1);

            final boolean prevExists;
            if (directory != null)
              prevExists = directory.fileExists(prevSegmentFileName);
            else
              prevExists = new File(fileDirectory, prevSegmentFileName).exists();

            if (prevExists) {
              message("fallback to prior segment file '" + prevSegmentFileName + "'");
              try {
                Object v = doBody(prevSegmentFileName);
                if (exc != null) {
                  message("success on fallback " + prevSegmentFileName);
                }
                return v;
              } catch (IOException err2) {
                message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry");
              }
            }
          }
        }
      }
    }

    /**
     * Subclass must implement this.  The assumption is an
     * IOException will be thrown if something goes wrong
     * during the processing that could have been caused by
     * a writer committing.
     */
    protected abstract Object doBody(String segmentFileName) throws CorruptIndexException, IOException;
  }

  /**
   * Returns a new SegmentInfos containg the SegmentInfo
   * instances in the specified range first (inclusive) to
   * last (exclusive), so total number of segments returned
   * is last-first.
   */
  public SegmentInfos range(int first, int last) {
    SegmentInfos infos = new SegmentInfos();
    infos.addAll(super.subList(first, last));
    return infos;
  }

  // Carry over generation numbers from another SegmentInfos
  void updateGeneration(SegmentInfos other) {
    lastGeneration = other.lastGeneration;
    generation = other.generation;
    version = other.version;
  }

  public final void rollbackCommit(Directory dir) throws IOException {
    if (pendingOutput != null) {
      try {
        pendingOutput.close();
      } catch (Throwable t) {
        // Suppress so we keep throwing the original exception
        // in our caller
      }

      // Must carefully compute fileName from "generation"
      // since lastGeneration isn't incremented:
      try {
        final String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                                             "",
                                                                             generation);
        dir.deleteFile(segmentFileName);
      } catch (Throwable t) {
        // Suppress so we keep throwing the original exception
        // in our caller
      }
      pendingOutput = null;
    }
  }

  /** Call this to start a commit.  This writes the new
   *  segments file, but writes an invalid checksum at the
   *  end, so that it is not visible to readers.  Once this
   *  is called you must call {@link #finishCommit} to complete
   *  the commit or {@link #rollbackCommit} to abort it. */
  public final void prepareCommit(Directory dir) throws IOException {
    if (pendingOutput != null)
      throw new IllegalStateException("prepareCommit was already called");
    write(dir);
  }

  public final void finishCommit(Directory dir) throws IOException {
    if (pendingOutput == null)
      throw new IllegalStateException("prepareCommit was not called");
    boolean success = false;
    try {
      pendingOutput.finishCommit();
      pendingOutput.close();
      pendingOutput = null;
      success = true;
    } finally {
      if (!success)
        rollbackCommit(dir);
    }

    // NOTE: if we crash here, we have left a segments_N
    // file in the directory in a possibly corrupt state (if
    // some bytes made it to stable storage and others
    // didn't).  But, the segments_N file includes checksum
    // at the end, which should catch this case.  So when a
    // reader tries to read it, it will throw a
    // CorruptIndexException, which should cause the retry
    // logic in SegmentInfos to kick in and load the last
    // good (previous) segments_N-1 file.

    final String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                                  "",
                                                                  generation);
    success = false;
    try {
      dir.sync(fileName);
      success = true;
    } finally {
      if (!success) {
        try {
          dir.deleteFile(fileName);
        } catch (Throwable t) {
          // Suppress so we keep throwing the original exception
        }
      }
    }

    lastGeneration = generation;

    try {
      IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN);
      try {
        genOutput.writeInt(FORMAT_LOCKLESS);
        genOutput.writeLong(generation);
        genOutput.writeLong(generation);
      } finally {
        genOutput.close();
      }
    } catch (Throwable t) {
      // It's OK if we fail to write this file since it's
      // used only as one of the retry fallbacks.
    }
  }

  /** Writes & syncs to the Directory dir, taking care to
   *  remove the segments file on exception */
  public final void commit(Directory dir) throws IOException {
    prepareCommit(dir);
    finishCommit(dir);
  }

  synchronized String segString(Directory directory) {
    StringBuffer buffer = new StringBuffer();
    final int count = size();
    for(int i = 0; i < count; i++) {
      if (i > 0) {
        buffer.append(' ');
      }
      final SegmentInfo info = info(i);
      buffer.append(info.segString(directory));
      if (info.dir != directory)
        buffer.append("**");
    }
    return buffer.toString();
  }
}