hod

HADOOP 0.18.0安装源代码头文件

#!/bin/sh

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


""":"
work_dir=$(dirname $0)
base_name=$(basename $0)
original_dir=$PWD
cd $work_dir

if [ $HOD_PYTHON_HOME ]; then
    exec $HOD_PYTHON_HOME -u -OO $base_name ${1+"$@"} --hod.original-dir $original_dir
elif [ -e /usr/bin/python ]; then
    exec /usr/bin/python -u -OO $base_name ${1+"$@"} --hod.original-dir $original_dir
elif [ -e /usr/local/bin/python ]; then
    exec /usr/local/bin/python -u -OO $base_name ${1+"$@"} --hod.original-dir $original_dir
else
    exec python -u -OO $base_name ${1+"$@"} --hod.original-dir $work_dir
fi
":"""

"""The executable to be used by the user"""
import sys, os, re, pwd, threading, sys

myName          = os.path.basename(sys.argv[0])
myName          = re.sub(".*/", "", myName)
binDirectory    = os.path.realpath(sys.argv[0])
rootDirectory   = re.sub("/bin/.*", "", binDirectory)
libDirectory    = rootDirectory

sys.path.append(libDirectory)

from hodlib.Hod.hod import hodRunner
from hodlib.Common.setup import *
from hodlib.Common.descGenerator import *
from hodlib.Common.util import local_fqdn, need_to_allocate, filter_warnings,\
    get_exception_error_string, hodInterrupt, \
    HOD_INTERRUPTED_MESG, HOD_INTERRUPTED_CODE,\
    TORQUE_USER_LIMITS_COMMENT_FIELD
from hodlib.Common.tcp import tcpError, tcpSocket
from hodlib.Hod.hod import hodHelp

filter_warnings()

reVersion = re.compile(".*(\d+_\d+).*")

VERSION = None
if os.path.exists("./VERSION"):
  vFile = open("./VERSION", 'r')
  VERSION = vFile.readline()
  vFile.close()

# Always look for hodrc file here unless otherwise specified with -c:   
DEFAULT_LOC = os.path.join(rootDirectory, 'conf')
DEFAULT_HOD_DIR = os.path.join(os.environ['HOME'], ".hod")

if not os.path.isdir(DEFAULT_HOD_DIR):
  os.mkdir(DEFAULT_HOD_DIR, 0777)

DEFAULT_CONFIG = os.path.join(DEFAULT_HOD_DIR, 'hodrc')
if not os.path.exists(DEFAULT_CONFIG):
  if os.environ.has_key('HOD_CONF_DIR') and os.environ['HOD_CONF_DIR'] is not None:
    DEFAULT_CONFIG = os.path.join(os.environ['HOD_CONF_DIR'], 'hodrc')

# Definition tuple is of the form:
#  (name, type, description, help?, default value, required?, validate?, 
#   short option)
#
defList = { 'hod' : (      
             ('original-dir', 'directory', 'hod original start directory',
              False, None, True, True, 'r'),

             ('clusterdir', 'directory', 
             'Directory where cluster state information and hadoop-site.xml' +
             ' will be stored.',
              True, None, False, False, 'd'),

             ('syslog-address', 'address', 'Syslog address.',
              False, None, False, True, 'y'),
              
             ('java-home', 'directory', 'Java home directory.',
              True, None, True, True, 'j'),
            
             ('debug', 'pos_int', 'Debugging level, 0-4.',
              True, 3, True, True, 'b'),
            
             ('stream', 'bool', 'Output to stderr.',
              False, True, False, True),

             ('nodecount', 'pos_int', 
              'Number of nodes to allocate at startup. ',
              True, None, False, True, 'n'),

             ('script', 'file', 'Hadoop script to execute.',
              True, None, False, False, 's'), 

             ('userid', 'user_account', 
              'User ID the hod shell is running under.',
              False, pwd.getpwuid(os.getuid())[0], False, True, 'u'),
             
             ('allocate-wait-time', 'pos_int', 
              'Time to wait for cluster allocation.',
              False, 300, True, True, 'e'),         
              
             ('operation', 'string',
              'Initiate a hod operation. (help, allocate, deallocate ...)',
              False, None, False, True, 'o'),
             
             ('cluster-factor', 'pos_float',
              'The number of grid slots per machines', False, 1.9, False, True,
              'x'),
             
             ('cluster', 'string', 'Name of cluster being used.',
              False, None, True, True, 'w'),

             ('proxy-xrs-address', 'address', 
              'Address to Allocation Manager XML RPC proxy.',
              False, None, False, True, 'p'),
              
             ('xrs-port-range', 'range', 'XML-RPC port range n-m.',
              False, None, True, True),

             ('client-params', 'keyval', 'Hadoop client xml key/value list',
              True, None, False, True, 'C'), 

             ('hadoop-ui-log-dir', 'directory', 'Directory to store Web UI Logs of Hadoop',
              True, None, False, True),

             ('temp-dir', 'directory', 'HOD temporary directories.',
              False, None, True, False),

             ('update-worker-info', 'bool', 'Specifies whether to update Worker Info after allocation',
              False, False, False, True),

             ('job-feasibility-attr', 'string', 'Specifies whether to check job feasibility - resource manager and/or scheduler limits, also gives the attribute value',
              False, None, False, True),

             ('title', 'string', 'Title for the current HOD allocation.',
               True, "HOD", False, True, 'N'),

             ('walltime', 'pos_int', 'Walltime in seconds for the current HOD allocation',
              True, None, False, True, 'l'),

             ('script-wait-time', 'pos_int', 'Specifies the time to wait before running the script. Used with the hod.script option.',
              True, 10, False, True, 'W')),

            'resource_manager' : (
             ('id', 'string', 'Batch scheduler ID: torque|condor.',
              False, None, True, True),
             
             ('pbs-user', 'user_account', 'User ID jobs are submitted under.',
              False, None, False, True),
              
             ('pbs-account', 'string', 'User Account jobs are submitted under.',
              True, None, False, False, 'A'),
              
             ('queue', 'string', 'Queue of the batch scheduler to query.',
              True, 'batch', False, True, 'Q'),
             
             ('batch-home', 'directory', 'Scheduler installation directory.',
              False, None, True, True),
             
             ('options', 'keyval', 'Options to pass to the scheduler.',
              False, None, False, True),

             ('env-vars', 'keyval', 'Environment variables to pass to the submitted jobs.',
              False, None, False, True)),
                            
            'ringmaster' : (
             ('work-dirs', 'list', 'hod work directories',
              False, None, True, False),

             ('temp-dir', 'directory', 'Ringmaster temporary directory.',
              False, None, True, False),
              
             ('log-dir', 'directory', 'hod logging directory.', 
              False, os.path.join(rootDirectory, 'logs'), False, False),

             ('syslog-address', 'address', 'Syslog address.',
              False, None, False, True),

             ('xrs-port-range', 'range', 'XML-RPC port range n-m.',
              False, None, True, True),
              
             ('http-port-range', 'range', 'HTTP port range n-m.',
              False, None, True, True),
              
             ('debug', 'pos_int', 'Debugging level, 0-4.',
              False, 4, True,   True),
               
             ('register', 'bool', 'Register with service registry?',
              False, True, True, True),
               
             ('stream', 'bool', 'Output to stderr.',
              False, False, False, True),
              
             ('userid', 'user_account', 
              'User ID the hod shell is running under.',
              False, pwd.getpwuid(os.getuid())[0], False, True),
               
             ('svcrgy-addr', 'address', 'Download HTTP address.',
              False, None, False, False),             
             
             ('hadoop-tar-ball', 'uri', 'hadoop program tar ball.',
              True, None, False, False, 't'),

             ('max-connect','pos_int','max connections allowed for a single tarball server',
             False, 30, False, True),

             ('jt-poll-interval', 'pos_int', 'How often to poll the Job tracker for idleness',
             False, 120, False, True),

             ('idleness-limit', 'pos_int', 'Limit after which to deallocate the cluster',
             False, 3600, False, True),

             ('max-master-failures', 'pos_int', 
              'Defines how many times a master can fail before' \
              ' failing cluster allocation', False, 5, True, True)),


            'gridservice-mapred' : (
             ('external', 'bool', "Connect to an already running MapRed?",
              False, False, True, True),
              
             ('host', 'hostname', 'Mapred hostname.', 
              False, 'localhost', False, False),

             ('info_port', 'pos_int', 'Mapred info port.',
              False, None, False, False),
             
             ('tracker_port', 'pos_int', 'Mapred job tracker port.',
              False, None, False, False),
                        
             ('cmdline-params', 'keyval', 'Hadoop cmdline key/value list.',
              False, None, False, False),

             ('server-params', 'keyval', 'Hadoop xml key/value list',
              True, None, False, True, 'M'),
               
             ('envs', 'keyval', 'environment to run this package in',
              False, None, False, True),

             ('final-server-params', 'keyval', 'Hadoop final xml key/val list',
              False, None, False, True, 'F'),

             ('pkgs', 'directory', "directory where the package is installed",
              False, None, False, False)), 
               
               
            'gridservice-hdfs' : (
             ('external', 'bool', "Connect to an already running HDFS?",
              False, False, True, True),
             
             ('host', 'hostname', 'HDFS hostname.', 
              False, 'localhost', False, False),
             
             ('fs_port', 'pos_int', 'HDFS port.',
              False, None, False, False),
              
             ('info_port', 'pos_int', 'HDFS info port.',
              False, None, False, False),