dfsshell.java

hadoop:Nutch集群平台

     */
    public void du(String src) throws IOException {
        Path items[] = fs.listPaths( fs.globPaths( new Path(src) ) );
        if (items == null) {
            System.out.println("Could not get listing for " + src);
        } else {
            System.out.println("Found " + items.length + " items");
            for (int i = 0; i < items.length; i++) {
                DfsPath cur = (DfsPath) items[i];
                System.out.println(cur + "\t" + cur.getContentsLength());
            }
        }
    }

    /**
     * Create the given dir
     */
    public void mkdir(String src) throws IOException {
        Path f = new Path(src);
        fs.mkdirs(f);
    }
    
    /**
     * Move DFS files that match the file pattern <i>srcf</i>
     * to a destination dfs file.
     * When moving mutiple files, the destination must be a directory. 
     * Otherwise, IOException is thrown.
     * @param srcf: a file pattern specifying source files
     * @param dstf: a destination local file/directory 
     * @exception: IOException  
     * @see org.apache.hadoop.fs.FileSystem.globPaths 
     */
    public void rename(String srcf, String dstf) throws IOException {
      Path [] srcs = fs.globPaths( new Path(srcf) );
      Path dst = new Path(dstf);
      if( srcs.length > 1 && !fs.isDirectory(dst)) {
        throw new IOException( "When moving multiple files, " 
            + "destination should be a directory." );
      }
      for( int i=0; i<srcs.length; i++ ) {
        if (fs.rename(srcs[i], dst)) {
            System.out.println("Renamed " + srcs[i] + " to " + dstf);
        } else {
            System.out.println("Rename failed " + srcs[i]);
        }
      }
    }

    /**
     * Copy DFS files that match the file pattern <i>srcf</i>
     * to a destination dfs file.
     * When copying mutiple files, the destination must be a directory. 
     * Otherwise, IOException is thrown.
     * @param srcf: a file pattern specifying source files
     * @param dstf: a destination local file/directory 
     * @exception: IOException  
     * @see org.apache.hadoop.fs.FileSystem.globPaths 
     */
    public void copy(String srcf, String dstf, Configuration conf) throws IOException {
      Path [] srcs = fs.globPaths( new Path(srcf) );
      Path dst = new Path(dstf);
      if( srcs.length > 1 && !fs.isDirectory(dst)) {
        throw new IOException( "When copying multiple files, " 
            + "destination should be a directory." );
      }
      for( int i=0; i<srcs.length; i++ ) {
        FileUtil.copy(fs, srcs[i], fs, dst, false, conf);
      }
    }

    /**
     * Delete all files in DFS that match the file pattern <i>srcf</i>
     * @param srcf: a file pattern specifying source files
     * @param recursive: if need to delete subdirs
     * @exception: IOException  
     * @see org.apache.hadoop.fs.FileSystem.globPaths 
     */
    public void delete(String srcf, boolean recursive) throws IOException {
      Path [] srcs = fs.globPaths( new Path(srcf) );
      for( int i=0; i<srcs.length; i++ ) {
        delete(srcs[i], recursive);
      }
    }
    
    /* delete an DFS file */
    private void delete(Path src, boolean recursive ) throws IOException {
      if (fs.isDirectory(src) && !recursive) {
        System.out.println("Cannot remove directory \"" + src +
                           "\", use -rmr instead");
        return;
      }

      if (fs.delete(src)) {
        System.out.println("Deleted " + src);
      } else {
        System.out.println("Delete failed " + src);
      }
    }

    /**
     * Return an abbreviated English-language desc of the byte length
     */
    static String byteDesc(long len) {
        double val = 0.0;
        String ending = "";
        if (len < 1024 * 1024) {
            val = (1.0 * len) / 1024;
            ending = " k";
        } else if (len < 1024 * 1024 * 1024) {
            val = (1.0 * len) / (1024 * 1024);
            ending = " Mb";
        } else {
            val = (1.0 * len) / (1024 * 1024 * 1024);
            ending = " Gb";
        }
        return limitDecimal(val, 2) + ending;
    }

    static String limitDecimal(double d, int placesAfterDecimal) {
        String strVal = Double.toString(d);
        int decpt = strVal.indexOf(".");
        if (decpt >= 0) {
            strVal = strVal.substring(0, Math.min(strVal.length(), decpt + 1 + placesAfterDecimal));
        }
        return strVal;
    }

    /**
     * Gives a report on how the FileSystem is doing
     */
    public void report() throws IOException {
      if (fs instanceof DistributedFileSystem) {
        DistributedFileSystem dfs = (DistributedFileSystem)fs;
        long raw = dfs.getRawCapacity();
        long rawUsed = dfs.getRawUsed();
        long used = dfs.getUsed();
        boolean mode = dfs.setSafeMode( FSConstants.SafeModeAction.SAFEMODE_GET );

        if( mode )
          System.out.println("Safe mode is ON" );
        System.out.println("Total raw bytes: " + raw + " (" + byteDesc(raw) + ")");
        System.out.println("Used raw bytes: " + rawUsed + " (" + byteDesc(rawUsed) + ")");
        System.out.println("% used: " + limitDecimal(((1.0 * rawUsed) / raw) * 100, 2) + "%");
        System.out.println();
        System.out.println("Total effective bytes: " + used + " (" + byteDesc(used) + ")");
        System.out.println("Effective replication multiplier: " + (1.0 * rawUsed / used));

        System.out.println("-------------------------------------------------");
        DatanodeInfo info[] = dfs.getDataNodeStats();
        System.out.println("Datanodes available: " + info.length);
        System.out.println();
        for (int i = 0; i < info.length; i++) {
          System.out.println(info[i].getDatanodeReport());
          System.out.println();
        }
      }
    }
    
    /**
     * Safe mode maintenance command.
     * 
     * Usage: java DFSShell -safemode [enter | leave | get]
     */
    public void setSafeMode( String argv[], int idx ) throws IOException {
      final String safeModeUsage = "Usage: java DFSShell -safemode [enter | leave | get]";
      if( ! (fs instanceof DistributedFileSystem) ) {
        System.out.println( "FileSystem is " + fs.getName() );
        return;
      }
      if( idx != argv.length-1 ) {
        System.out.println( safeModeUsage );
        return;
      }
      FSConstants.SafeModeAction action;
      if( "leave".equalsIgnoreCase(argv[idx]) )
        action = FSConstants.SafeModeAction.SAFEMODE_LEAVE;
      else if( "enter".equalsIgnoreCase(argv[idx]) )
        action = FSConstants.SafeModeAction.SAFEMODE_ENTER;
      else if( "get".equalsIgnoreCase(argv[idx]) )
        action = FSConstants.SafeModeAction.SAFEMODE_GET;
      else {
        System.out.println( safeModeUsage );
        return;
      }
      DistributedFileSystem dfs = (DistributedFileSystem)fs;
      boolean mode = dfs.setSafeMode( action );
      System.out.println( "Safe mode is " + ( mode ? "ON" : "OFF" ));
    }

    /**
     * run
     */
    public int run( String argv[] ) throws Exception {
        if (argv.length < 1) {
            System.out.println("Usage: java DFSShell" + 
                " [-fs <local | namenode:port>]" +
                " [-conf <configuration file>]" +
                " [-D <[property=value>]"+
                " [-ls <path>]"+
                " [-lsr <path>]"+
                " [-du <path>]"+
                " [-mv <src> <dst>]"+
                " [-cp <src> <dst>]"+
                " [-rm <path>]" +
                " [-rmr <path>]" +
                " [-put <localsrc> <dst>]"+
                " [-copyFromLocal <localsrc> <dst>]"+
                " [-moveFromLocal <localsrc> <dst>]" + 
                " [-get <src> <localdst>]"+
                " [-getmerge <src> <localdst> [addnl]]"+
                " [-cat <src>]"+
                " [-copyToLocal <src> <localdst>]" +
                " [-moveToLocal <src> <localdst>]"+
                " [-mkdir <path>]"+
                " [-report]"+
                " [-setrep [-R] <rep> <path/file>]" +
                " [-safemode enter | leave | get]");
            return -1;
        }

        // initialize DFSShell
        init();
        
        int exitCode = -1;
        int i = 0;
        String cmd = argv[i++];
        try {
            if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) {
                copyFromLocal(new Path(argv[i++]), argv[i++]);
            } else if ("-moveFromLocal".equals(cmd)) {
                moveFromLocal(new Path(argv[i++]), argv[i++]);
            } else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) {
                copyToLocal(argv[i++], argv[i++]);
            } else if ("-getmerge".equals(cmd)) {
                if(argv.length>i+2)
                    copyMergeToLocal(argv[i++], new Path(argv[i++]), Boolean.parseBoolean(argv[i++]));
                else
                    copyMergeToLocal(argv[i++], new Path(argv[i++]));
                
            } else if ("-cat".equals(cmd)) {
                cat(argv[i++]);
            } else if ("-moveToLocal".equals(cmd)) {
                moveToLocal(argv[i++], new Path(argv[i++]));
            } else if ("-setrep".equals(cmd)) {
            	setReplication(argv, i);           
            } else if ("-ls".equals(cmd)) {
                String arg = i < argv.length ? argv[i++] : "";
                ls(arg, false);
            } else if ("-lsr".equals(cmd)) {
                String arg = i < argv.length ? argv[i++] : "";
                ls(arg, true);
            } else if ("-mv".equals(cmd)) {
                rename(argv[i++], argv[i++]);
            } else if ("-cp".equals(cmd)) {
                copy(argv[i++], argv[i++], conf);
            } else if ("-rm".equals(cmd)) {
                delete(argv[i++], false);
            } else if ("-rmr".equals(cmd)) {
                delete(argv[i++], true);
            } else if ("-du".equals(cmd)) {
                String arg = i < argv.length ? argv[i++] : "";
                du(arg);
            } else if ("-mkdir".equals(cmd)) {
                mkdir(argv[i++]);
            } else if ("-report".equals(cmd)) {
                report();
            } else if ("-safemode".equals(cmd)) {
                setSafeMode(argv,i);
            }
            exitCode = 0;;
        } catch (IOException e ) {
          System.err.println( cmd.substring(1) + ": " + e.getLocalizedMessage() );  
        } finally {
            fs.close();
        }
        return exitCode;
    }

    /**
     * main() has some simple utility methods
     */
    public static void main(String argv[]) throws Exception {
        int res = new DFSShell().doMain(new Configuration(), argv);
        System.exit(res);
    }
}