setup.py

HADOOP 0.18.0安装源代码头文件

                            configValidator.add(configVarName,
                                self._configDef[section][option]['type'],
                                self._dict[section][option])
                    elif self._configDef[section][option]['req']:
                        statusMsgs.append("%s: %s.%s is not defined."
                             % (errorPrefix, section, option))
                        errorCount = errorCount + 1                         

            configValidator.validate()

            for valueInfo in configValidator.validatedInfo:
                sectionsOptions = reDot.split(valueInfo['name'])

                if valueInfo['isValid'] == 1:
                    self._dict[sectionsOptions[0]][sectionsOptions[1]] = \
                        valueInfo['normalized']
                else:
                    if valueInfo['errorData']:
                        statusMsgs.extend(self.var_error(sectionsOptions[0],
                            sectionsOptions[1], valueInfo['errorData']))
                    else:
                        statusMsgs.extend(self.var_error(sectionsOptions[0],
                            sectionsOptions[1]))
                    errorCount = errorCount + 1

            if errorCount > 1:
                statusMsgs.append( "%s: %s problems found." % (
                    errorPrefix, errorCount))
                self.var_error_suggest(statusMsgs)
                status = False
            elif errorCount > 0:
                statusMsgs.append( "%s: %s problem found." % (
                    errorPrefix, errorCount))
                self.var_error_suggest(statusMsgs)
                status = False
        
        self.__validated = True

        if self.__originalDir:
          os.chdir(oldDir)

        return status,statusMsgs

    def normalizeValue(self, section, option)  :
      return typeValidatorInstance.normalize(
                                  self._configDef[section][option]['type'],
                                  self[section][option])

    def validateValue(self, section, option):
      # Validates a section.option and exits on error
      valueInfo = typeValidatorInstance.verify(
                                  self._configDef[section][option]['type'],
                                  self[section][option])
      if valueInfo['isValid'] == 1:
        return []
      else:
        if valueInfo['errorData']:
          return self.var_error(section, option, valueInfo['errorData'])
        else:
          return self.var_error(section, option)

class config(SafeConfigParser, baseConfig):
    def __init__(self, configFile, configDef=None, originalDir=None, 
                 options=None, checkPerms=False):
        """Constructs config object.

           configFile - configuration file to read
           configDef  - definition object
           options    - options object
           checkPerms - check file permission on config file, 0660

           sample configuration file:

            [snis]
            modules_dir  = modules/       ; location of infoModules
            md5_defs_dir = etc/md5_defs   ; location of infoTree md5 defs
            info_store   = var/info       ; location of nodeInfo store
            cam_daemon   = localhost:8200 ; cam daemon address"""


        SafeConfigParser.__init__(self)
        baseConfig.__init__(self, configDef, originalDir)

        if(os.path.exists(configFile)):
          self.configFile = configFile
        else:
          raise IOError
        
        self._options = options
        
	## UNUSED CODE : checkPerms is never True
  ## zim: this code is used if one instantiates config() with checkPerms set to
  ## True.
        if checkPerms: self.__check_perms()

        self.read(configFile)

        self._configDef = configDef
        if not self._configDef:
            self._mySections = self.sections()

        self.__initialize_config_dict()

    def __initialize_config_dict(self):
        """ build a dictionary of config vars keyed by section name defined in
           configDef, if options defined override config"""

        for section in self._mySections:
            items = self.items(section)
            self._dict[section] = {}

            # First fill self._dict with whatever is given in hodrc.
            # Going by this, options given at the command line either override
            # options in hodrc, or get appended to the list, like for
            # hod.client-params. Note that after this dict has _only_ hodrc
            # params
            for keyValuePair in items:
                # stupid commenting bug in ConfigParser class, lines without an
                #  option value pair or section required that ; or # are at the
                #  beginning of the line, :(
                newValue = reCommentHack.sub("", keyValuePair[1])
                newValue = reCommentNewline.sub("", newValue)
                self._dict[section][keyValuePair[0]] = newValue
            # end of filling with options given in hodrc
            # now start filling in command line options
            if self._options:    
                for option in self._configDef[section].keys():
                    if self._options[section].has_key(option):
                        # the user has given an option
                        compoundOpt = "%s.%s" %(section,option)
                        if ( compoundOpt == \
                              'gridservice-mapred.final-server-params' \
                              or compoundOpt == \
                                    'gridservice-hdfs.final-server-params' \
                              or compoundOpt == \
                                    'gridservice-mapred.server-params' \
                              or compoundOpt == \
                                    'gridservice-hdfs.server-params' \
                              or compoundOpt == \
                                    'hod.client-params' ):
                 
                           if ( compoundOpt == \
                              'gridservice-mapred.final-server-params' \
                              or compoundOpt == \
                                    'gridservice-hdfs.final-server-params' ):
                              overwrite = False
                           else: overwrite = True

                           # Append to the current list of values in self._dict
                           if not self._dict[section].has_key(option):
                             self._dict[section][option] = ""
                           dictOpts = reKeyValList.split(self._dict[section][option])
                           dictOptsKeyVals = {}
                           for opt in dictOpts:
                              if opt != '':
                                # when dict _has_ params from hodrc
                                if reKeyVal.search(opt):
                                  (key, val) = reKeyVal.split(opt,1)
                                  # we only consider the first '=' for splitting
                                  # we do this to support passing params like
                                  # mapred.child.java.opts=-Djava.library.path=some_dir
                                  # Even in case of an invalid error like unescaped '=',
                                  # we don't want to fail here itself. We leave such errors 
                                  # to be caught during validation which happens after this
                                  dictOptsKeyVals[key] = val
                                else: 
                                  # this means an invalid option. Leaving it
                                  #for config.verify to catch
                                  dictOptsKeyVals[opt] = None
                                
                           cmdLineOpts = reKeyValList.split(self._options[section][option])

                           for opt in cmdLineOpts:
                              if reKeyVal.search(opt):
                                # Same as for hodrc options. only consider
                                # the first =
                                ( key, val ) = reKeyVal.split(opt,1)
                              else:
                                key = opt
                                val = None
                              # whatever is given at cmdline overrides
                              # what is given in hodrc only for non-final params
                              if dictOptsKeyVals.has_key(key):
                                if overwrite:
                                  dictOptsKeyVals[key] = val
                              else: dictOptsKeyVals[key] = val
                              
                           self._dict[section][option] = ""
                           for key in dictOptsKeyVals:
                              if self._dict[section][option] == "":
                                if dictOptsKeyVals[key]:
                                  self._dict[section][option] = key + "=" + \
                                    dictOptsKeyVals[key]
                                else: #invalid option. let config.verify catch
                                  self._dict[section][option] = key
                              else:
                                if dictOptsKeyVals[key]:
                                  self._dict[section][option] = \
                                    self._dict[section][option] + "," + key + \
                                      "=" + dictOptsKeyVals[key]
                                else:  #invalid option. let config.verify catch
                                  self._dict[section][option] = \
                                    self._dict[section][option] + "," + key

                        else:
                             # for rest of the options, that don't need
                            # appending business.
                            # options = cmdline opts + defaults
                            # dict    = hodrc opts only
                            # only non default opts can overwrite any opt
                            # currently in dict
                           if not self._dict[section].has_key(option):
                              # options not mentioned in hodrc
                              self._dict[section][option] = \
                                               self._options[section][option]
                           elif self._configDef[section][option]['default'] != \
                                               self._options[section][option]:
                              # option mentioned in hodrc but user has given a
                              # non-default option
                              self._dict[section][option] = \
                                               self._options[section][option]

    ## UNUSED METHOD
    ## zim: is too :)
    def __check_perms(self):
        perms = None
        if self._options:  
            try:
                perms = get_perms(self.configFile)
            except OSError, data:
                self._options.print_help()
                raise Exception("*** could not find config file: %s" % data)
                sys.exit(1)
        else:
            perms = get_perms(self.configFile)
               
        if perms != requiredPerms:
            error = "*** '%s' has invalid permission: %s should be %s\n" % \
                (self.configFile, perms, requiredPerms)
            raise Exception( error)
            sys.exit(1)

    def replace_escape_seqs(self):
      """ replace any escaped characters """
      replace_escapes(self)

class formatter(IndentedHelpFormatter):
    def format_option_strings(self, option):
        """Return a comma-separated list of option strings & metavariables."""
        if option.takes_value():
            metavar = option.metavar or option.dest.upper()
            short_opts = [sopt
                          for sopt in option._short_opts]
            long_opts = [self._long_opt_fmt % (lopt, metavar)
                         for lopt in option._long_opts]
        else:
            short_opts = option._short_opts
            long_opts = option._long_opts

        if self.short_first:
            opts = short_opts + long_opts
        else:
            opts = long_opts + short_opts

        return ", ".join(opts)    

class options(OptionParser, baseConfig):

    def __init__(self, optionDef, usage, version, originalDir=None, 
                 withConfig=False, defaultConfig=None, defaultLocation=None,
                 name=None):
        """Constructs and options object.
         
           optionDef     - definition object
           usage         - usage statement
           version       - version string
           withConfig    - used in conjunction with a configuration file
           defaultConfig - default configuration file
        
        """
        OptionParser.__init__(self, usage=usage)
        baseConfig.__init__(self, optionDef, originalDir)
        
        self.formatter = formatter(4, max_help_position=100, width=180, 
                                   short_first=1)
        
        self.__name = name
        self.__version = version
        self.__withConfig = withConfig
        self.__defaultConfig = defaultConfig
        self.__defaultLoc = defaultLocation
        self.args = []
        self.__optionList = []
        self.__compoundOpts = []
        self.__shortMap = {}
        self.__alphaString = 'abcdefghijklmnopqrstuvxyzABCDEFGHIJKLMNOPQRSTUVXYZ1234567890'
        self.__alpha = []
        self.__parsedOptions = {}
        self.__reserved = [ 'h' ]
        
        self.__orig_grps = []
        self.__orig_grp_lists = {}
        self.__orig_option_list = []
        
        self.__display_grps = []
        self.__display_grp_lists = {}
        self.__display_option_list = [] 
        
        self.config = None
        
        if self.__withConfig:
            self.__reserved.append('c')
        self.__reserved.append('v')
        
        self.__gen_alpha()            

        # build self.__optionList, so it contains all the options that are
        # possible. the list elements are of the form section.option
        for section in self._mySections:
            if self.__withConfig and section == 'config':
                raise Exception(
                    "withConfig set 'config' cannot be used as a section name")
            for option in self._configDef[section].keys():
                if '.' in option:
                    raise Exception("Options cannot contain: '.'")
                elif self.__withConfig and option == 'config':
                    raise Exception(
                        "With config set, option config is not allowed.")
                elif self.__withConfig and option == 'verbose-help':
                    raise Exception(
                        "With config set, option verbose-help is not allowed.")                 
                self.__optionList.append(self.__splice_compound(section, 
                                                                option))
        self.__build_short_map()
        self.__add_options()
        self.__init_display_options() 
        
        (self.__parsedOptions, self.args) = self.parse_args()

        # Now process the positional arguments only for the client side
        if self.__name == 'hod':

          hodhelp = hodHelp()

          _operation = getattr(self.__parsedOptions,'hod.operation')
          _script = getattr(self.__parsedOptions, 'hod.script')
          nArgs = self.args.__len__()
          if _operation:
            # -o option is given
            if nArgs != 0:
              self.error('invalid syntax : command and operation(-o) cannot coexist')
          elif nArgs == 0 and _script:
            # for a script option, without subcommand: hod -s script ...