filesystem.java

hadoop:Nutch集群平台

     *    <dd> Removes (escapes) any special meaning of character <i>c</i>.
     *
     *   </dl>
     *  </dd>
     * </dl>
     *
     * @param filePattern: a regular expression specifying file pattern

     * @return an array of paths that match the file pattern
     * @throws IOException
     */
    public Path[] globPaths(Path filePattern) throws IOException {
      return globPaths(filePattern, DEFAULT_FILTER);
    }
    
    /** glob all the file names that matches filePattern
     * and is accepted by filter
     * @param 
     */
    public Path[] globPaths(Path filePattern, PathFilter filter) 
        throws IOException {
      Path [] parents = new Path[1];
      int level = 0;
      
      String filename = filePattern.toString();
      if("".equals(filename) || Path.SEPARATOR.equals(filename)) {
        parents[0] = filePattern;
        return parents;
      }
      
      String [] components = filename.split(Path.SEPARATOR);
      if(filePattern.isAbsolute()) {
        parents[0] = new Path(Path.SEPARATOR);
        level = 1;
      } else {
        parents[0] = new Path( "" );
      }
      
      Path[] results = globPathsLevel(parents, components, level, filter);
      Arrays.sort(results);
      return results;
    }
    
    private Path[] globPathsLevel(Path[] parents,
        String [] filePattern, int level, PathFilter filter) throws IOException {
      if (level == filePattern.length)
        return parents;
      GlobFilter fp = new GlobFilter(filePattern[level], filter);
      if( fp.hasPattern()) {
        parents = listPaths(parents, fp);
      } else {
        for(int i=0; i<parents.length; i++) {
          parents[i] = new Path(parents[i], filePattern[level]);
        }
      }
      return globPathsLevel(parents, filePattern, level+1, filter);      
    }
 
    private static class GlobFilter implements PathFilter {
      private PathFilter userFilter = DEFAULT_FILTER;
      private Pattern regex;
      private boolean hasPattern = false;
      
      /** Default pattern character: Escape any special meaning. */
      private static final char  PAT_ESCAPE =  '\\';
      /** Default pattern character: Any single character. */
      private static final char  PAT_ANY = '.';
      /** Default pattern character: Character set close. */
      private static final char  PAT_SET_CLOSE = ']';
      
      GlobFilter() {
      }
      
      GlobFilter(String filePattern) throws IOException {
        setRegex(filePattern);
      }
      
      GlobFilter(String filePattern, PathFilter filter) throws IOException {
        userFilter = filter;
        setRegex(filePattern);
      }
      
      void setRegex(String filePattern) throws IOException {
        int len;
        int setOpen;
        boolean setRange;
        StringBuffer fileRegex = new StringBuffer();

        // Validate the pattern
        len = filePattern.length();
        if (len == 0)
            return;

        setOpen =  0;
        setRange = false;

        for (int i = 0;  i < len;  i++)
        {
            char  pCh;

            // Examine a single pattern character
            pCh = filePattern.charAt(i);            
            if( pCh == PAT_ESCAPE ) {
              fileRegex.append( pCh );
              i++;
              if (i >= len)
                  error( "An escaped character does not present",
                      filePattern, i);
              pCh = filePattern.charAt(i);
            } else if( pCh == '.' ) {
              fileRegex.append( PAT_ESCAPE );
            } else if( pCh == '*' ) {
                fileRegex.append( PAT_ANY );
                hasPattern = true;
            } else if( pCh == '?' ) {
                pCh = PAT_ANY ;
                hasPattern = true;
            } else if( pCh == '[' && setOpen == 0 ) {
                setOpen++;
                hasPattern = true;
            } else if( pCh == '^' && setOpen > 0) {
            } else if (pCh == '-'  &&  setOpen > 0) {
                // Character set range
                setRange = true;
            } else if (pCh == PAT_SET_CLOSE  &&  setRange) {
                // Incomplete character set range
                error("Incomplete character set range", filePattern, i);
            } else if (pCh == PAT_SET_CLOSE  &&  setOpen > 0) {
                // End of a character set
                if (setOpen < 2)
                    error("Unexpected end of set", filePattern, i);
                setOpen = 0;
            } else if (setOpen > 0) {
                // Normal character, or the end of a character set range
                setOpen++;
                setRange = false;
            }
            fileRegex.append( pCh );
        }

        // Check for a well-formed pattern
        if (setOpen > 0  ||  setRange)
        {
            // Incomplete character set or character range
            error("Expecting set closure character or end of range", filePattern, len);
        }
        regex = Pattern.compile(fileRegex.toString());
      }
      
      boolean hasPattern() {
        return hasPattern;
      }
      
      public boolean accept(Path path) {
        return regex.matcher(path.getName()).matches() && userFilter.accept(path);
      }
      
      private void error(String s, String pattern, int pos) throws IOException {
        throw new IOException("Illegal file pattern: "
                                 +s+" for glob "+pattern + " at " + pos);
      }
    }
    
    /**
     * Set the current working directory for the given file system.
     * All relative paths will be resolved relative to it.
     * @param new_dir
     */
    public abstract void setWorkingDirectory(Path new_dir);
    
    /**
     * Get the current working directory for the given file system
     * @return the directory pathname
     */
    public abstract Path getWorkingDirectory();
    
    /** @deprecated Call {@link #mkdirs(Path)} instead. */
    public boolean mkdirs(File f) throws IOException {
      return mkdirs(new Path(f.toString()));
    }

    /**
     * Make the given file and all non-existent parents into
     * directories. Has the semantics of Unix 'mkdir -p'.
     * Existence of the directory hierarchy is not an error.
     */
    public abstract boolean mkdirs(Path f) throws IOException;

    /** @deprecated Call {@link #lock(Path,boolean)} instead. */
    public void lock(File f, boolean shared) throws IOException {
      lock(new Path(f.toString()), shared);
    }

    /**
     * Obtain a lock on the given Path
     */
    public abstract void lock(Path f, boolean shared) throws IOException;

    /** @deprecated Call {@link #release(Path)} instead. */
    public void release(File f) throws IOException {
      release(new Path(f.toString()));
    }

    /**
     * Release the lock
     */
    public abstract void release(Path f) throws IOException;

    /**
     * The src file is on the local disk.  Add it to FS at
     * the given dst name and the source is kept intact afterwards
     */
    public abstract void copyFromLocalFile(Path src, Path dst) throws IOException;

    /**
     * The src file is on the local disk.  Add it to FS at
     * the given dst name, removing the source afterwards.
     */
    public abstract void moveFromLocalFile(Path src, Path dst) throws IOException;

    /**
     * The src file is under FS, and the dst is on the local disk.
     * Copy it from FS control to the local dst name.
     */
    public abstract void copyToLocalFile(Path src, Path dst) throws IOException;

    /**
     * the same as copyToLocalFile(Path src, File dst), except that
     * the source is removed afterward.
     */
    // not implemented yet
    //public abstract void moveToLocalFile(Path src, File dst) throws IOException;

    /** @deprecated Call {@link #startLocalOutput(Path, Path)} instead. */
    public File startLocalOutput(File src, File dst) throws IOException {
      return new File(startLocalOutput(new Path(src.toString()),
                                       new Path(dst.toString())).toString());
    }

    /**
     * Returns a local File that the user can write output to.  The caller
     * provides both the eventual FS target name and the local working
     * file.  If the FS is local, we write directly into the target.  If
     * the FS is remote, we write into the tmp local area.
     */
    public abstract Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException;

    /** @deprecated Call {@link #completeLocalOutput(Path, Path)} instead. */
    public void completeLocalOutput(File src, File dst) throws IOException {
      completeLocalOutput(new Path(src.toString()), new Path(dst.toString()));
    }

    /**
     * Called when we're all done writing to the target.  A local FS will
     * do nothing, because we've written to exactly the right place.  A remote
     * FS will copy the contents of tmpLocalFile to the correct target at
     * fsOutputFile.
     */
    public abstract void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException;

    /**
     * No more filesystem operations are needed.  Will
     * release any held locks.
     */
    public void close() throws IOException {
        NAME_TO_FS.remove(getName());
    }

    /**
     * Report a checksum error to the file system.
     * @param f the file name containing the error
     * @param in the stream open on the file
     * @param start the position of the beginning of the bad data in the file
     * @param length the length of the bad data in the file
     * @param crc the expected CRC32 of the data
     */
    public abstract void reportChecksumFailure(Path f, FSInputStream in,
                                               long start, long length,
                                               int crc);

    /**
     * Get the size for a particular file.
     * @param f the filename
     * @return the number of bytes in a block
     */
    public abstract long getBlockSize(Path f) throws IOException;
    
    /** Return the number of bytes that large input files should be optimally
     * be split into to minimize i/o time. */
    public abstract long getDefaultBlockSize();
    
    /**
     * Get the default replication.
     */
    public abstract short getDefaultReplication();

}