filesystem.java

hadoop:Nutch集群平台

/**
 * Copyright 2005 The Apache Software Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.fs;

import java.io.*;
import java.net.*;
import java.util.*;
import java.util.regex.Pattern;

import org.apache.commons.logging.*;

import org.apache.hadoop.dfs.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.Progressable;

/****************************************************************
 * An abstract base class for a fairly generic filesystem.  It
 * may be implemented as a distributed filesystem, or as a "local"
 * one that reflects the locally-connected disk.  The local version
 * exists for small Hadopp instances and for testing.
 *
 * <p>
 *
 * All user code that may potentially use the Hadoop Distributed
 * File System should be written to use a FileSystem object.  The
 * Hadoop DFS is a multi-machine system that appears as a single
 * disk.  It's useful because of its fault tolerance and potentially
 * very large capacity.
 * 
 * <p>
 * The local implementation is {@link LocalFileSystem} and distributed
 * implementation is {@link DistributedFileSystem}.
 * @author Mike Cafarella
 *****************************************************************/
public abstract class FileSystem extends Configured {
    public static final Log LOG = LogFactory.getLog("org.apache.hadoop.dfs.DistributedFileSystem");

    private static final HashMap NAME_TO_FS = new HashMap();
    /**
     * Parse the cmd-line args, starting at i.  Remove consumed args
     * from array.  We expect param in the form:
     * '-local | -dfs <namenode:port>'
     */
    public static FileSystem parseArgs(String argv[], int i, Configuration conf) throws IOException {
        /**
        if (argv.length - i < 1) {
            throw new IOException("Must indicate filesystem type for DFS");
        }
        */
        int orig = i;
        FileSystem fs = null;
        String cmd = argv[i];
        if ("-dfs".equals(cmd)) {
            i++;
            InetSocketAddress addr = DataNode.createSocketAddr(argv[i++]);
            fs = new DistributedFileSystem(addr, conf);
        } else if ("-local".equals(cmd)) {
            i++;
            fs = new LocalFileSystem(conf);
        } else {
            fs = get(conf);                          // using default
            LOG.info("No FS indicated, using default:"+fs.getName());

        }
        System.arraycopy(argv, i, argv, orig, argv.length - i);
        for (int j = argv.length - i; j < argv.length; j++) {
            argv[j] = null;
        }
        return fs;
    }

    /** Returns the configured filesystem implementation.*/
    public static FileSystem get(Configuration conf) throws IOException {
      return getNamed(conf.get("fs.default.name", "local"), conf);
    }

    /** Returns a name for this filesystem, suitable to pass to {@link
     * FileSystem#getNamed(String,Configuration)}.*/
    public abstract String getName();
  
    /** Returns a named filesystem.  Names are either the string "local" or a
     * host:port pair, naming an DFS name server.*/
    public static FileSystem getNamed(String name, Configuration conf) throws IOException {
      FileSystem fs = (FileSystem)NAME_TO_FS.get(name);
      if (fs == null) {
        if ("local".equals(name)) {
          fs = new LocalFileSystem(conf);
        } else {
          fs = new DistributedFileSystem(DataNode.createSocketAddr(name), conf);
        }
        NAME_TO_FS.put(name, fs);
      }
      return fs;
    }

    /** Return the name of the checksum file associated with a file.*/
    public static Path getChecksumFile(Path file) {
      return new Path(file.getParent(), "."+file.getName()+".crc");
    }

    /** Return true iff file is a checksum file name.*/
    public static boolean isChecksumFile(Path file) {
      String name = file.getName();
      return name.startsWith(".") && name.endsWith(".crc");
    }

    ///////////////////////////////////////////////////////////////
    // FileSystem
    ///////////////////////////////////////////////////////////////

    protected FileSystem(Configuration conf) {
      super(conf);
    }

    /**
     * Return a 2D array of size 1x1 or greater, containing hostnames 
     * where portions of the given file can be found.  For a nonexistent 
     * file or regions, null will be returned.
     *
     * This call is most helpful with DFS, where it returns 
     * hostnames of machines that contain the given file.
     *
     * The FileSystem will simply return an elt containing 'localhost'.
     */
    public abstract String[][] getFileCacheHints(Path f, long start, long len) throws IOException;

    /** @deprecated Call {@link #open(Path)} instead. */
    public FSDataInputStream open(File f) throws IOException {
      return open(new Path(f.toString()));
    }

    /**
     * Opens an FSDataInputStream at the indicated Path.
     * @param f the file name to open
     * @param bufferSize the size of the buffer to be used.
     */
    public FSDataInputStream open(Path f, int bufferSize) throws IOException {
      return new FSDataInputStream(this, f, bufferSize, getConf());
    }
    
    /**
     * Opens an FSDataInputStream at the indicated Path.
     * @param f the file to open
     */
    public FSDataInputStream open(Path f) throws IOException {
      return new FSDataInputStream(this, f, getConf());
    }

    /**
     * Opens an InputStream for the indicated Path, whether local
     * or via DFS.
     */
    public abstract FSInputStream openRaw(Path f) throws IOException;

    /** @deprecated Call {@link #create(Path)} instead. */
    public FSDataOutputStream create(File f) throws IOException {
      return create(new Path(f.toString()));
    }

    /**
     * Opens an FSDataOutputStream at the indicated Path.
     * Files are overwritten by default.
     */
    public FSDataOutputStream create(Path f) throws IOException {
      return create(f, true, 
                    getConf().getInt("io.file.buffer.size", 4096),
                    getDefaultReplication(),
                    getDefaultBlockSize());
    }

    /**
     * Create an FSDataOutputStream at the indicated Path with write-progress
     * reporting.
     * Files are overwritten by default.
     */
    public FSDataOutputStream create(Path f, Progressable progress) throws IOException {
      return create(f, true, 
                    getConf().getInt("io.file.buffer.size", 4096),
                    getDefaultReplication(),
                    getDefaultBlockSize(), progress);
    }

    /**
     * Opens an FSDataOutputStream at the indicated Path.
     * Files are overwritten by default.
     */
    public FSDataOutputStream create(Path f, short replication)
      throws IOException {
      return create(f, true, 
                    getConf().getInt("io.file.buffer.size", 4096),
                    replication,
                    getDefaultBlockSize());
    }

    /**
     * Opens an FSDataOutputStream at the indicated Path with write-progress
     * reporting.
     * Files are overwritten by default.
     */
    public FSDataOutputStream create(Path f, short replication, Progressable progress)
      throws IOException {
      return create(f, true, 
                    getConf().getInt("io.file.buffer.size", 4096),
                    replication,
                    getDefaultBlockSize(), progress);
    }

    
    /**
     * Opens an FSDataOutputStream at the indicated Path.
     * @param f the file name to open
     * @param overwrite if a file with this name already exists, then if true,
     *   the file will be overwritten, and if false an error will be thrown.
     * @param bufferSize the size of the buffer to be used.
     */
    public FSDataOutputStream create( Path f, 
                                      boolean overwrite,
                                      int bufferSize
                                    ) throws IOException {
      return create( f, overwrite, bufferSize, 
                     getDefaultReplication(),
                     getDefaultBlockSize());
    }
    
    /**
     * Opens an FSDataOutputStream at the indicated Path with write-progress
     * reporting.
     * @param f the file name to open
     * @param overwrite if a file with this name already exists, then if true,
     *   the file will be overwritten, and if false an error will be thrown.
     * @param bufferSize the size of the buffer to be used.
     */
    public FSDataOutputStream create( Path f, 
                                      boolean overwrite,
                                      int bufferSize,
                                      Progressable progress
                                    ) throws IOException {
      return create( f, overwrite, bufferSize, 
                     getDefaultReplication(),
                     getDefaultBlockSize(), progress);
    }
    
    
    /**
     * Opens an FSDataOutputStream at the indicated Path.
     * @param f the file name to open
     * @param overwrite if a file with this name already exists, then if true,
     *   the file will be overwritten, and if false an error will be thrown.
     * @param bufferSize the size of the buffer to be used.
     * @param replication required block replication for the file. 
     */
    public FSDataOutputStream create( Path f, 
                                      boolean overwrite,
                                      int bufferSize,
                                      short replication,
                                      long blockSize
                                    ) throws IOException {
      return new FSDataOutputStream(this, f, overwrite, getConf(), 
                                    bufferSize, replication, blockSize );
    }

    /**
     * Opens an FSDataOutputStream at the indicated Path with write-progress
     * reporting.
     * @param f the file name to open
     * @param overwrite if a file with this name already exists, then if true,
     *   the file will be overwritten, and if false an error will be thrown.
     * @param bufferSize the size of the buffer to be used.
     * @param replication required block replication for the file. 
     */
    public FSDataOutputStream create( Path f, 
                                      boolean overwrite,
                                      int bufferSize,
                                      short replication,
                                      long blockSize,
                                      Progressable progress
                                    ) throws IOException {
      return new FSDataOutputStream(this, f, overwrite, getConf(), 
                                    bufferSize, replication, blockSize, progress );
    }

    /** Opens an OutputStream at the indicated Path.
     * @param f the file name to open
     * @param overwrite if a file with this name already exists, then if true,
     *   the file will be overwritten, and if false an error will be thrown.
     * @param replication required block replication for the file.