configuration.java

hadoop:Nutch集群平台

/**
 * Copyright 2005 The Apache Software Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.conf;

import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.net.URL;
import java.io.*;

import javax.xml.parsers.*;

import org.w3c.dom.*;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.Transformer;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.apache.commons.logging.*;

import org.apache.hadoop.util.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/** Provides access to configuration parameters.  Configurations are specified
 * by resources.  A resource contains a set of name/value pairs.
 *
 * <p>Each resource is named by either a String or by a Path.  If named by a
 * String, then the classpath is examined for a file with that name.  If a
 * File, then the local filesystem is examined directly, without referring to
 * the CLASSPATH.
 *
 * <p>Configuration resources are of two types: default and
 * final.  Default values are loaded first and final values are loaded last, and
 * thus override default values.
 *
 * <p>Hadoop's default resource is the String "hadoop-default.xml" and its
 * final resource is the String "hadoop-site.xml".  Other tools built on Hadoop
 * may specify additional resources.
 * 
 * <p>The values returned by most <tt>get*</tt> methods are based on String representations. 
 * This String is processed for <b>variable expansion</b>. The available variables are the 
 * <em>System properties</em> and the <em>other properties</em> defined in this Configuration.
 * <p>The only <tt>get*</tt> method that is not processed for variable expansion is
 * {@link getObject} (as it cannot assume that the returned values are String). 
 * You can use <tt>getObject</tt> to obtain the raw value of a String property without 
 * variable expansion: if <tt>(String)conf.getObject("my.jdk")</tt> is <tt>"JDK ${java.version}"</tt>
 * then conf.get("my.jdk")</tt> is <tt>"JDK 1.5.0"</tt> 
 * <p> Example XML config using variables:<br><tt>
 * &lt;name>basedir&lt;/name>&lt;value>/user/${user.name}&lt;/value><br> 
 * &lt;name>tempdir&lt;/name>&lt;value>${basedir}/tmp&lt;/value><br>
 * </tt>When conf.get("tempdir") is called:<br>
 * <tt>${basedir}</tt> is resolved to another property in this Configuration.
 * Then <tt>${user.name}</tt> is resolved to a System property.
 */
public class Configuration {
  private static final Log LOG =
    LogFactory.getLog("org.apache.hadoop.conf.Configuration");

  private ArrayList defaultResources = new ArrayList();
  private ArrayList finalResources = new ArrayList();

  private Properties properties;
  private Properties overlay;
  private ClassLoader classLoader;
  {
    classLoader = Thread.currentThread().getContextClassLoader();
    if (classLoader == null) {
      classLoader = Configuration.class.getClassLoader();
    }
  }
  
  /** A new configuration. */
  public Configuration() {
    if (LOG.isDebugEnabled()) {
      LOG.debug(StringUtils.stringifyException(new IOException("config()")));
    }
    defaultResources.add("hadoop-default.xml");
    finalResources.add("hadoop-site.xml");
  }

  /** A new configuration with the same settings cloned from another. */
  public Configuration(Configuration other) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(StringUtils.stringifyException
                (new IOException("config(config)")));
    }
    this.defaultResources = (ArrayList)other.defaultResources.clone();
    this.finalResources = (ArrayList)other.finalResources.clone();
    if (other.properties != null)
      this.properties = (Properties)other.properties.clone();
    if(other.overlay!=null)
      this.overlay = (Properties)other.overlay.clone();
  }

  /** Add a default resource. */
  public void addDefaultResource(String name) {
    addResource(defaultResources, name);
  }

  /** Add a default resource. */
  public void addDefaultResource(URL url) {
    addResource(defaultResources, url);
  }

  /** Add a default resource. */
  public void addDefaultResource(Path file) {
    addResource(defaultResources, file);
  }

  /** Add a final resource. */
  public void addFinalResource(String name) {
    addResource(finalResources, name);
  }

  /** Add a final resource. */
  public void addFinalResource(URL url) {
    addResource(finalResources, url);
  }

  /** Add a final resource. */
  public void addFinalResource(Path file) {
    addResource(finalResources, file);
  }

  private synchronized void addResource(ArrayList resources, Object resource) {
    resources.add(resource);                      // add to resources
    properties = null;                            // trigger reload
  }
  
  /**
   * Returns the value of the <code>name</code> property, or null if no such
   * property exists.
   */
  public Object getObject(String name) { return getProps().get(name);}

  /** Sets the value of the <code>name</code> property. */
  public void setObject(String name, Object value) {
    getProps().put(name, value);
  }

  /** Returns the value of the <code>name</code> property.  If no such property
   * exists, then <code>defaultValue</code> is returned.
   */
  public Object get(String name, Object defaultValue) {
    Object res = getObject(name);
    if (res != null) return res;
    else return defaultValue;
  }
  
  private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
  private static int MAX_SUBST = 20;

  private String substituteVars(String expr) {
    if(expr == null) {
      return null;
    }
    Matcher match = varPat.matcher("");
    String eval = expr;
    for(int s=0; s<MAX_SUBST; s++) {
      match.reset(eval);
      if(! match.find()) {
        return eval;
      }
      String var = match.group();
      var = var.substring(2, var.length()-1); // remove ${ .. }
      String val = System.getProperty(var);
      if(val == null) {
        val = (String)this.getObject(var);
      }
      if(val == null) {
        return eval; // return literal ${var}: var is unbound
      }
      // substitute
      eval = eval.substring(0, match.start())+val+eval.substring(match.end());
    }
    throw new IllegalStateException("Variable substitution depth too large: " 
                                    + MAX_SUBST + " " + expr);
  }
  
  /** Returns the value of the <code>name</code> property, or null if no
   * such property exists. */
  public String get(String name) {
    return substituteVars(getProps().getProperty(name));
  }

  /** Sets the value of the <code>name</code> property. */
  public void set(String name, Object value) {
    getOverlay().setProperty(name, value.toString());
    getProps().setProperty(name, value.toString());
  }
  
  private synchronized Properties getOverlay() {
    if(overlay==null){
      overlay=new Properties();
    }
    return overlay;
  }

  /** Returns the value of the <code>name</code> property.  If no such property
   * exists, then <code>defaultValue</code> is returned.
   */
  public String get(String name, String defaultValue) {
     return substituteVars(getProps().getProperty(name, defaultValue));
  }
    
  /** Returns the value of the <code>name</code> property as an integer.  If no
   * such property is specified, or if the specified value is not a valid
   * integer, then <code>defaultValue</code> is returned.
   */
  public int getInt(String name, int defaultValue) {
    String valueString = get(name);
    if (valueString == null)
      return defaultValue;
    try {
      return Integer.parseInt(valueString);
    } catch (NumberFormatException e) {
      return defaultValue;
    }
  }

  /** Sets the value of the <code>name</code> property to an integer. */
  public void setInt(String name, int value) {
    set(name, Integer.toString(value));
  }


  /** Returns the value of the <code>name</code> property as a long.  If no
   * such property is specified, or if the specified value is not a valid
   * long, then <code>defaultValue</code> is returned.
   */
  public long getLong(String name, long defaultValue) {
    String valueString = get(name);
    if (valueString == null)
      return defaultValue;
    try {
      return Long.parseLong(valueString);
    } catch (NumberFormatException e) {
      return defaultValue;
    }
  }

  /** Sets the value of the <code>name</code> property to a long. */
  public void setLong(String name, long value) {
    set(name, Long.toString(value));
  }

  /** Returns the value of the <code>name</code> property as a float.  If no
   * such property is specified, or if the specified value is not a valid
   * float, then <code>defaultValue</code> is returned.
   */
  public float getFloat(String name, float defaultValue) {
    String valueString = get(name);
    if (valueString == null)
      return defaultValue;
    try {
      return Float.parseFloat(valueString);
    } catch (NumberFormatException e) {
      return defaultValue;
    }
  }

  /** Returns the value of the <code>name</code> property as an boolean.  If no
   * such property is specified, or if the specified value is not a valid
   * boolean, then <code>defaultValue</code> is returned.  Valid boolean values
   * are "true" and "false".
   */
  public boolean getBoolean(String name, boolean defaultValue) {
    String valueString = get(name);
    if ("true".equals(valueString))
      return true;
    else if ("false".equals(valueString))
      return false;
    else return defaultValue;
  }

  /** Sets the value of the <code>name</code> property to an integer. */
  public void setBoolean(String name, boolean value) {
    set(name, Boolean.toString(value));
  }

  /** Returns the value of the <code>name</code> property as an array of
   * strings.  If no such property is specified, then <code>null</code>
   * is returned.  Values are comma delimited.
   */
  public String[] getStrings(String name) {
    String valueString = get(name);
    if (valueString == null)
      return null;
    StringTokenizer tokenizer = new StringTokenizer (valueString,",");
    List values = new ArrayList();
    while (tokenizer.hasMoreTokens()) {
      values.add(tokenizer.nextToken());
    }
    return (String[])values.toArray(new String[values.size()]);
  }

  /**
   * Load a class by name.
   * @param name the class name
   * @return the class object
   * @throws ClassNotFoundException if the class is not found
   */
  public Class getClassByName(String name) throws ClassNotFoundException {
    return Class.forName(name, true, classLoader);