dfsck.java

hadoop:Nutch集群平台

            //
            in = new DataInputStream(new BufferedInputStream(s.getInputStream()));
            long curBlockSize = in.readLong();
            long amtSkipped = in.readLong();
            if (curBlockSize != lblock.getBlock().len) {
                throw new IOException("Recorded block size is " + lblock.getBlock().len + ", but datanode reports size of " + curBlockSize);
            }
            if (amtSkipped != 0L) {
                throw new IOException("Asked for offset of " + 0L + ", but only received offset of " + amtSkipped);
            }
        } catch (IOException ex) {
            // Put chosen node into dead list, continue
            LOG.info("Failed to connect to " + targetAddr + ":" + ex);
            deadNodes.add(chosenNode);
            if (s != null) {
                try {
                    s.close();
                } catch (IOException iex) {
                }                        
            }
            s = null;
        }
    }
    if (in == null) {
      throw new Exception("Could not open data stream for " + lblock.getBlock().getBlockName());
    }
    byte[] buf = new byte[1024];
    int cnt = 0;
    boolean success = true;
    try {
      while ((cnt = in.read(buf)) > 0) {
        fos.write(buf, 0, cnt);
      }
    } catch (Exception e) {
      e.printStackTrace();
      success = false;
    } finally {
      try {in.close(); } catch (Exception e1) {}
      try {out.close(); } catch (Exception e1) {}
      try {s.close(); } catch (Exception e1) {}
    }
    if (!success)
      throw new Exception("Could not copy block data for " + lblock.getBlock().getBlockName());
  }
  
  /*
   * XXX (ab) See comment above for copyBlock().
   * 
   * Pick the best node from which to stream the data.
   * That's the local one, if available.
   */
  Random r = new Random();
  private DatanodeInfo bestNode(DatanodeInfo nodes[], TreeSet deadNodes) throws IOException {
      if ((nodes == null) || 
          (nodes.length - deadNodes.size() < 1)) {
          throw new IOException("No live nodes contain current block");
      }
      DatanodeInfo chosenNode = null;
      for (int i = 0; i < nodes.length; i++) {
          if (deadNodes.contains(nodes[i])) {
              continue;
          }
          String nodename = nodes[i].getName();
          int colon = nodename.indexOf(':');
          if (colon >= 0) {
              nodename = nodename.substring(0, colon);
          }
          if (dfs.localName.equals(nodename)) {
              chosenNode = nodes[i];
              break;
          }
      }
      if (chosenNode == null) {
          do {
              chosenNode = nodes[Math.abs(r.nextInt()) % nodes.length];
          } while (deadNodes.contains(chosenNode));
      }
      return chosenNode;
  }

  private void lostFoundInit() {
    lfInited = true;
    try {
      UTF8 lfName = new UTF8("/lost+found");
      // check that /lost+found exists
      if (!dfs.exists(lfName)) {
        lfInitedOk = dfs.mkdirs(lfName);
        lostFound = lfName;
      } else if (!dfs.isDirectory(lfName)) {
        System.err.println("Cannot use /lost+found : a regular file with this name exists.");
        lfInitedOk = false;
      } else { // exists and isDirectory
        lostFound = lfName;
        lfInitedOk = true;
      }
    } catch (Exception e) {
      e.printStackTrace();
      lfInitedOk = false;
    }
    if (lostFound == null) {
      System.err.println("Cannot initialize /lost+found .");
      lfInitedOk = false;
    }
  }
  
  /**
   * @param args
   */
  public int run(String[] args) throws Exception {
    if (args.length == 0) {
      System.err.println("Usage: DFSck <path> [-move | -delete] [-files] [-blocks [-locations]]");
      System.err.println("\t<path>\tstart checking from this path");
      System.err.println("\t-move\tmove corrupted files to /lost+found");
      System.err.println("\t-delete\tdelete corrupted files");
      System.err.println("\t-files\tprint out files being checked");
      System.err.println("\t-blocks\tprint out block report");
      System.err.println("\t-locations\tprint out locations for every block");
      return -1;
    }
    String path = args[0];
    boolean showFiles = false;
    boolean showBlocks = false;
    boolean showLocations = false;
    int fixing = FIXING_NONE;
    for (int i = 1; i < args.length; i++) {
      if (args[i].equals("-files")) showFiles = true;
      if (args[i].equals("-blocks")) showBlocks = true;
      if (args[i].equals("-locations")) showLocations = true;
      if (args[i].equals("-move")) fixing = FIXING_MOVE;
      if (args[i].equals("-delete")) fixing = FIXING_DELETE;
    }
    init(fixing, showFiles, showBlocks, showLocations);
    Result res = fsck(path);
    System.out.println();
    System.out.println(res);
    if (res.isHealthy()) {
      System.out.println("\n\nThe filesystem under path '" + args[0] + "' is HEALTHY");
    } else {
      System.out.println("\n\nThe filesystem under path '" + args[0] + "' is CORRUPT");
    }
    return 0;
  }

  public static void main(String[] args) throws Exception {
      int res = new DFSck().doMain(new Configuration(), args);
      System.exit(res);
  }

  /**
   * Result of checking, plus overall DFS statistics.
   * @author Andrzej Bialecki
   */
  public static class Result {
    private ArrayList missingIds = new ArrayList();
    private long missingSize = 0L;
    private long corruptFiles = 0L;
    private long overReplicatedBlocks = 0L;
    private long underReplicatedBlocks = 0L;
    private int replication = 0;
    private long totalBlocks = 0L;
    private long totalFiles = 0L;
    private long totalDirs = 0L;
    private long totalSize = 0L;
    
    /**
     * DFS is considered healthy if there are no missing blocks.
     * @return
     */
    public boolean isHealthy() {
      return missingIds.size() == 0;
    }
    
    /** Add a missing block name, plus its size. */
    public void addMissing(String id, long size) {
      missingIds.add(id);
      missingSize += size;
    }
    
    /** Return a list of missing block names (as list of Strings). */
    public ArrayList getMissingIds() {
      return missingIds;
    }

    /** Return total size of missing data, in bytes. */
    public long getMissingSize() {
      return missingSize;
    }

    public void setMissingSize(long missingSize) {
      this.missingSize = missingSize;
    }

    /** Return the number of over-replicsted blocks. */
    public long getOverReplicatedBlocks() {
      return overReplicatedBlocks;
    }

    public void setOverReplicatedBlocks(long overReplicatedBlocks) {
      this.overReplicatedBlocks = overReplicatedBlocks;
    }

    /** Return the actual replication factor. */
    public float getReplicationFactor() {
      if (totalBlocks != 0)
        return (float)(totalBlocks * replication + overReplicatedBlocks - underReplicatedBlocks) / (float)totalBlocks;
      else return 0.0f;
    }

    /** Return the number of under-replicated blocks. Note: missing blocks are not counted here.*/
    public long getUnderReplicatedBlocks() {
      return underReplicatedBlocks;
    }

    public void setUnderReplicatedBlocks(long underReplicatedBlocks) {
      this.underReplicatedBlocks = underReplicatedBlocks;
    }

    /** Return total number of directories encountered during this scan. */
    public long getTotalDirs() {
      return totalDirs;
    }

    public void setTotalDirs(long totalDirs) {
      this.totalDirs = totalDirs;
    }

    /** Return total number of files encountered during this scan. */
    public long getTotalFiles() {
      return totalFiles;
    }

    public void setTotalFiles(long totalFiles) {
      this.totalFiles = totalFiles;
    }

    /** Return total size of scanned data, in bytes. */
    public long getTotalSize() {
      return totalSize;
    }

    public void setTotalSize(long totalSize) {
      this.totalSize = totalSize;
    }

    /** Return the intended replication factor, against which the over/under-
     * replicated blocks are counted. Note: this values comes from the current
     * Configuration supplied for the tool, so it may be different from the
     * value in DFS Configuration.
     */
    public int getReplication() {
      return replication;
    }

    public void setReplication(int replication) {
      this.replication = replication;
    }

    /** Return the total number of blocks in the scanned area. */
    public long getTotalBlocks() {
      return totalBlocks;
    }

    public void setTotalBlocks(long totalBlocks) {
      this.totalBlocks = totalBlocks;
    }
    
    public String toString() {
      StringBuffer res = new StringBuffer();
      res.append("Status: " + (isHealthy() ? "HEALTHY" : "CORRUPT"));
      res.append("\n Total size:\t" + totalSize + " B");
      res.append("\n Total blocks:\t" + totalBlocks);
      if (totalBlocks > 0) res.append(" (avg. block size "
              + (totalSize / totalBlocks) + " B)");
      res.append("\n Total dirs:\t" + totalDirs);
      res.append("\n Total files:\t" + totalFiles);
      if (missingSize > 0) {
        res.append("\n  ********************************");
        res.append("\n  CORRUPT FILES:\t" + corruptFiles);
        res.append("\n  MISSING BLOCKS:\t" + missingIds.size());
        res.append("\n  MISSING SIZE:\t\t" + missingSize + " B");
        res.append("\n  ********************************");
      }
      res.append("\n Over-replicated blocks:\t" + overReplicatedBlocks);
      if (totalBlocks > 0) res.append(" ("
              + ((float)(overReplicatedBlocks * 100) / (float)totalBlocks)
              + " %)");
      res.append("\n Under-replicated blocks:\t" + underReplicatedBlocks);
      if (totalBlocks > 0) res.append(" ("
              + ((float)(underReplicatedBlocks * 100) / (float)totalBlocks)
              + " %)");
      res.append("\n Target replication factor:\t" + replication);
      res.append("\n Real replication factor:\t" + getReplicationFactor());
      return res.toString();
    }

    /** Return the number of currupted files. */
    public long getCorruptFiles() {
      return corruptFiles;
    }

    public void setCorruptFiles(long corruptFiles) {
      this.corruptFiles = corruptFiles;
    }
  }


}