streamutil.java

hadoop:Nutch集群平台

/**
 * Copyright 2006 The Apache Software Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.streaming;

import java.text.DecimalFormat;
import java.io.*;
import java.net.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.List;
import java.util.jar.*;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.JobConf;

/** Utilities not available elsewhere in Hadoop.
 *  
 */
public class StreamUtil {

  /** It may seem strange to silently switch behaviour when a String
   * is not a classname; the reason is simplified Usage:<pre>
   * -mapper [classname | program ]
   * instead of the explicit Usage:
   * [-mapper program | -javamapper classname], -mapper and -javamapper are mutually exclusive.
   * (repeat for -reducer, -combiner) </pre>
   */
  public static Class goodClassOrNull(String className, String defaultPackage) {
    if (className.indexOf('.') == -1 && defaultPackage != null) {
      className = defaultPackage + "." + className;
    }
    Class clazz = null;
    try {
      clazz = Class.forName(className);
    } catch (ClassNotFoundException cnf) {
    } catch (LinkageError cnf) {
    }
    return clazz;
  }

  public static String findInClasspath(String className) {
    return findInClasspath(className, StreamUtil.class.getClassLoader());
  }

  /** @return a jar file path or a base directory or null if not found.
   */
  public static String findInClasspath(String className, ClassLoader loader) {

    String relPath = className;
    relPath = relPath.replace('.', '/');
    relPath += ".class";
    java.net.URL classUrl = loader.getResource(relPath);

    String codePath;
    if (classUrl != null) {
      boolean inJar = classUrl.getProtocol().equals("jar");
      codePath = classUrl.toString();
      if (codePath.startsWith("jar:")) {
        codePath = codePath.substring("jar:".length());
      }
      if (codePath.startsWith("file:")) { // can have both
        codePath = codePath.substring("file:".length());
      }
      if (inJar) {
        // A jar spec: remove class suffix in /path/my.jar!/package/Class
        int bang = codePath.lastIndexOf('!');
        codePath = codePath.substring(0, bang);
      } else {
        // A class spec: remove the /my/package/Class.class portion
        int pos = codePath.lastIndexOf(relPath);
        if (pos == -1) {
          throw new IllegalArgumentException("invalid codePath: className=" + className
              + " codePath=" + codePath);
        }
        codePath = codePath.substring(0, pos);
      }
    } else {
      codePath = null;
    }
    return codePath;
  }

  // copied from TaskRunner  
  static void unJar(File jarFile, File toDir) throws IOException {
    JarFile jar = new JarFile(jarFile);
    try {
      Enumeration entries = jar.entries();
      while (entries.hasMoreElements()) {
        JarEntry entry = (JarEntry) entries.nextElement();
        if (!entry.isDirectory()) {
          InputStream in = jar.getInputStream(entry);
          try {
            File file = new File(toDir, entry.getName());
            file.getParentFile().mkdirs();
            OutputStream out = new FileOutputStream(file);
            try {
              byte[] buffer = new byte[8192];
              int i;
              while ((i = in.read(buffer)) != -1) {
                out.write(buffer, 0, i);
              }
            } finally {
              out.close();
            }
          } finally {
            in.close();
          }
        }
      }
    } finally {
      jar.close();
    }
  }

  final static long KB = 1024L * 1;
  final static long MB = 1024L * KB;
  final static long GB = 1024L * MB;
  final static long TB = 1024L * GB;
  final static long PB = 1024L * TB;

  static DecimalFormat dfm = new DecimalFormat("####.000");
  static DecimalFormat ifm = new DecimalFormat("###,###,###,###,###");

  public static String dfmt(double d) {
    return dfm.format(d);
  }

  public static String ifmt(double d) {
    return ifm.format(d);
  }

  public static String formatBytes(long numBytes) {
    StringBuffer buf = new StringBuffer();
    boolean bDetails = true;
    double num = numBytes;

    if (numBytes < KB) {
      buf.append(numBytes + " B");
      bDetails = false;
    } else if (numBytes < MB) {
      buf.append(dfmt(num / KB) + " KB");
    } else if (numBytes < GB) {
      buf.append(dfmt(num / MB) + " MB");
    } else if (numBytes < TB) {
      buf.append(dfmt(num / GB) + " GB");
    } else if (numBytes < PB) {
      buf.append(dfmt(num / TB) + " TB");
    } else {
      buf.append(dfmt(num / PB) + " PB");
    }
    if (bDetails) {
      buf.append(" (" + ifmt(numBytes) + " bytes)");
    }
    return buf.toString();
  }

  public static String formatBytes2(long numBytes) {
    StringBuffer buf = new StringBuffer();
    long u = 0;
    if (numBytes >= TB) {
      u = numBytes / TB;
      numBytes -= u * TB;
      buf.append(u + " TB ");
    }
    if (numBytes >= GB) {
      u = numBytes / GB;
      numBytes -= u * GB;
      buf.append(u + " GB ");
    }
    if (numBytes >= MB) {
      u = numBytes / MB;
      numBytes -= u * MB;
      buf.append(u + " MB ");
    }
    if (numBytes >= KB) {
      u = numBytes / KB;
      numBytes -= u * KB;
      buf.append(u + " KB ");
    }
    buf.append(u + " B"); //even if zero
    return buf.toString();
  }

  static Environment env;
  static String HOST;

  static {
    try {
      env = new Environment();
      HOST = env.getHost();
    } catch (IOException io) {
      io.printStackTrace();
    }
  }

  static class StreamConsumer extends Thread {

    StreamConsumer(InputStream in, OutputStream out) {
      this.bin = new LineNumberReader(new BufferedReader(new InputStreamReader(in)));
      if (out != null) {
        this.bout = new DataOutputStream(out);
      }
    }

    public void run() {
      try {
        String line;
        while ((line = bin.readLine()) != null) {
          if (bout != null) {
            bout.writeUTF(line); //writeChars
            bout.writeChar('\n');
          }
        }
        bout.flush();
      } catch (IOException io) {
      }
    }

    LineNumberReader bin;
    DataOutputStream bout;
  }

  static void exec(String arg, PrintStream log) {
    exec(new String[] { arg }, log);
  }

  static void exec(String[] args, PrintStream log) {
    try {
      log.println("Exec: start: " + Arrays.asList(args));
      Process proc = Runtime.getRuntime().exec(args);
      new StreamConsumer(proc.getErrorStream(), log).start();
      new StreamConsumer(proc.getInputStream(), log).start();
      int status = proc.waitFor();
      //if status != 0
      log.println("Exec: status=" + status + ": " + Arrays.asList(args));
    } catch (InterruptedException in) {
      in.printStackTrace();
    } catch (IOException io) {
      io.printStackTrace();
    }
  }

  static String qualifyHost(String url) {
    try {
      return qualifyHost(new URL(url)).toString();
    } catch (IOException io) {
      return url;
    }
  }

  static URL qualifyHost(URL url) {
    try {
      InetAddress a = InetAddress.getByName(url.getHost());
      String qualHost = a.getCanonicalHostName();
      URL q = new URL(url.getProtocol(), qualHost, url.getPort(), url.getFile());
      return q;
    } catch (IOException io) {
      return url;
    }
  }

  static final String regexpSpecials = "[]()?*+|.!^-\\~@";

  public static String regexpEscape(String plain) {
    StringBuffer buf = new StringBuffer();
    char[] ch = plain.toCharArray();
    int csup = ch.length;
    for (int c = 0; c < csup; c++) {
      if (regexpSpecials.indexOf(ch[c]) != -1) {
        buf.append("\\");
      }
      buf.append(ch[c]);
    }
    return buf.toString();
  }

  public static String safeGetCanonicalPath(File f) {
    try {
      String s = f.getCanonicalPath();
      return (s == null) ? f.toString() : s;
    } catch (IOException io) {
      return f.toString();
    }
  }

  static String slurp(File f) throws IOException {
    int len = (int) f.length();
    byte[] buf = new byte[len];
    FileInputStream in = new FileInputStream(f);
    String contents = null;
    try {
      in.read(buf, 0, len);
      contents = new String(buf, "UTF-8");
    } finally {
      in.close();
    }
    return contents;
  }

  static String slurpHadoop(Path p, FileSystem fs) throws IOException {
    int len = (int) fs.getLength(p);
    byte[] buf = new byte[len];
    InputStream in = fs.open(p);
    String contents = null;
    try {
      in.read(buf, 0, len);
      contents = new String(buf, "UTF-8");
    } finally {
      in.close();
    }
    return contents;
  }

  public static String rjustify(String s, int width) {
    if (s == null) s = "null";
    if (width > s.length()) {
      s = getSpace(width - s.length()) + s;
    }
    return s;
  }

  public static String ljustify(String s, int width) {
    if (s == null) s = "null";
    if (width > s.length()) {
      s = s + getSpace(width - s.length());
    }
    return s;
  }

  static char[] space;
  static {
    space = new char[300];
    Arrays.fill(space, '\u0020');
  }

  public static String getSpace(int len) {
    if (len > space.length) {
      space = new char[Math.max(len, 2 * space.length)];
      Arrays.fill(space, '\u0020');
    }
    return new String(space, 0, len);
  }

  static private Environment env_;

  static Environment env() {
    if (env_ != null) {
      return env_;
    }
    try {
      env_ = new Environment();
    } catch (IOException io) {
      io.printStackTrace();
    }
    return env_;
  }

  public static String makeJavaCommand(Class main, String[] argv) {
    ArrayList vargs = new ArrayList();
    File javaHomeBin = new File(System.getProperty("java.home"), "bin");
    File jvm = new File(javaHomeBin, "java");
    vargs.add(jvm.toString());
    // copy parent classpath
    vargs.add("-classpath");
    vargs.add("\"" + System.getProperty("java.class.path") + "\"");

    // Add main class and its arguments
    vargs.add(main.getName());
    for (int i = 0; i < argv.length; i++) {
      vargs.add(argv[i]);
    }
    return collate(vargs, " ");
  }

  public static String collate(Object[] args, String sep) {
    return collate(Arrays.asList(args), sep);
  }

  public static String collate(List args, String sep) {
    StringBuffer buf = new StringBuffer();
    Iterator it = args.iterator();
    while (it.hasNext()) {
      if (buf.length() > 0) {
        buf.append(" ");
      }
      buf.append(it.next());
    }
    return buf.toString();
  }

  // JobConf helpers

  public static FileSplit getCurrentSplit(JobConf job) {
    String path = job.get("map.input.file");
    if (path == null) {
      return null;
    }
    Path p = new Path(path);
    long start = Long.parseLong(job.get("map.input.start"));
    long length = Long.parseLong(job.get("map.input.length"));
    return new FileSplit(p, start, length);
  }

  static class TaskId {

    boolean mapTask;
    String jobid;
    int taskid;
    int execid;
  }

  public static boolean isLocalJobTracker(JobConf job) {
    return job.get("mapred.job.tracker", "local").equals("local");
  }

  public static TaskId getTaskInfo(JobConf job) {
    TaskId res = new TaskId();

    String id = job.get("mapred.task.id");
    if (isLocalJobTracker(job)) {
      // it uses difft naming 
      res.mapTask = job.getBoolean("mapred.task.is.map", true);
      res.jobid = "0";
      res.taskid = 0;
      res.execid = 0;
    } else {
      String[] e = id.split("_");
      res.mapTask = e[2].equals("m");
      res.jobid = e[1];
      res.taskid = Integer.parseInt(e[3]);
      res.execid = Integer.parseInt(e[4]);
    }
    return res;
  }

  static boolean getUseMapSideEffect(JobConf job) {
    String reduce = job.get("stream.reduce.streamprocessor");
    return StreamJob.REDUCE_NONE.equals(reduce);
  }

  public static void touch(File file) throws IOException {
    file = file.getAbsoluteFile();
    FileOutputStream out = new FileOutputStream(file);
    out.close();
    if (!file.exists()) {
      throw new IOException("touch failed: " + file);
    }
  }

  public static boolean isCygwin() {
    String OS = System.getProperty("os.name");
    return (OS.indexOf("Windows") > -1);
  }

  public static String localizeBin(String path) {
    if (isCygwin()) {
      path = "C:/cygwin/" + path;
    }
    return path;
  }
  
  /** @param name foo where &lt;junit>&lt;sysproperty key="foo" value="${foo}"/> 
   * If foo is undefined then Ant sets the unevaluated value. 
   * Take this into account when setting defaultVal. */
  public static String getBoundAntProperty(String name, String defaultVal)
  {
    String val = System.getProperty(name);
    if(val != null && val.indexOf("${") >= 0) {
      val = null;
    }
    if(val == null) {
      val = defaultVal;
    }
    return val;
  }

}