types.py

HADOOP 0.18.0安装源代码头文件

#Licensed to the Apache Software Foundation (ASF) under one
#or more contributor license agreements.  See the NOTICE file
#distributed with this work for additional information
#regarding copyright ownership.  The ASF licenses this file
#to you under the Apache License, Version 2.0 (the
#"License"); you may not use this file except in compliance
#with the License.  You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

#Unless required by applicable law or agreed to in writing, software
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.
# $Id:types.py 6172 2007-05-22 20:26:54Z zim $
#
#------------------------------------------------------------------------------

""" Higher level data types and type related classes.

    Supported Types (Verification and Display):

      address        - validates ip:port and host:port tcp addresses
      ip_address     - validates and IP address
      net_address    - validates an IP like address, ie netmask
      hostname       - validates a hostname with DNS
      eaddress       - validates a single email address or a comma
                       seperated list of email addresses
      http_version   - validates a value is a http version (1.0/1.1)
      tcp_port       - validates a value to be a valid tcp port (2-65535)
      bool           - validates value is (0, 1, true, false) / converts
                       true -> 1 and false -> 0
      directory      - validates a values is a directory / resolves path to
                       absolute path
      file           - validates a value is a file / resolves path to absolute
                       path
      float          - validates a value is a float, converts string to float
      pos_float      - validates a value is a float and >= 0, converts string
                       to float
      pos_num        - same as pos_float
      neg_float      - validates a value is a float and < 0, converts string to
                       float
      int            - validates a value is an integer, converts string to
                       integer
      pos_int        - validates a value is an integer and >= 0, converts
                       string to integer
      neg_int        - validates a values is an integer and < 0, converts
                       striing to integer
      freq           - frequency, positive integer
      size           - validates a size in bytes, kb, mb, kb, and tb
                       (int > 0 post fixed with K, M, G, or T) also converts
                       value to integer bytes
      range          - numeric range, x-y normalized to a tuple, if a single
                       number is supplie a single element tuple is returned
      timestamp      - utc timestamp of the form YYYYMMDDHHMMSS
      user_account   - UNIX user account name
      user_group     - UNIX group name
      string         - arbitrarily long string
      list           - comma seperated list of strings of arbitrary length,
      keyval         - comma seperated list of key=value pairs, key does not 
                       need to be unique.
      uri            - a uri """

import sys, os, socket, pwd, grp, stat, re, re, string, pprint, urlparse

from tcp import tcpSocket, check_net_address, check_ip_address
from util import check_timestamp

types = { 'directory'      : { 'db'    : 'string',
                               'units' : None     },

          'address'        : { 'db'    : 'string',
                               'units' : None     },

          'ip_address'     : { 'db'    : 'string',
                               'units' : None     },

          'net_address'    : { 'db'    : 'string',
                               'units' : None     },

          'bool'           : { 'db'    : 'bool',
                               'units' : None     },

          'int'            : { 'db'    : 'integer',
                               'units' : None     },

          'float'          : { 'db'    : 'float',
                               'units' : None     },

          'pos_int'        : { 'db'    : 'integer',
                               'units' : None     },

          'neg_int'        : { 'db'    : 'integer',
                               'units' : None     },

          'pos_num'        : { 'db'    : 'float',
                               'units' : None     },

          'pos_float'      : { 'db'    : 'float',
                               'units' : None     },

          'neg_float'      : { 'db'    : 'float',
                               'units' : None     },

          'string'         : { 'db'    : 'string',
                               'units' : None     },

          'list'           : { 'db'    : 'string',
                               'units' : None     },

          'file'           : { 'db'    : 'string',
                               'units' : None     },

          'size'           : { 'db'    : 'integer',
                               'units' : 'bytes'  },

          'freq'           : { 'db'    : 'integer',
                               'units' : 'hz'     },

          'eaddress'       : { 'db'    : 'string',
                               'units' : None     },

          'tcp_port'       : { 'db'    : 'integer',
                               'units' : None     },

          'http_version'   : { 'db'    : 'float',
                               'units' : None     },

          'range'          : { 'db'    : 'string',
                               'units' : None     },

          'hostname'       : { 'db'    : 'string',
                               'units' : None     },

          'user_account'   : { 'db'    : 'string',
                               'units' : None     },

          'user_group'     : { 'db'    : 'string',
                               'units' : None     },

          'timestamp'      : { 'db'    : 'timestamp',
                               'units' : None     },

          'keyval'         : { 'db'    : 'string',
                               'units' : None     },
          
          'uri'            : { 'db'    : 'string',
                               'units' : None     },

          ''               : { 'db'    : 'string',
                               'units' : None     }}

dbTypes = { 'string'  :   { 'type'  : 'varchar',
                            'store' : 'type_strings_0',
                            'table' : True              },

            'integer' :   { 'type'  : 'bigint',
                            'store' : 'integers',
                            'table' : False             },

            'float' :     { 'type'  : 'real',
                            'store' : 'floats',
                            'table' : False             },

            'bool' :      { 'type'  : 'boolean',
                            'store' : 'bools',
                            'table' : False             },

            'timestamp' : { 'type'  : 'timestamp(0)',
                            'store' : 'timestamps',
                            'table' : False             }}

reSizeFormat = re.compile("^(\d+)(k|m|g|t|p|kb|mb|gb|tb|pb)$", flags=2)
reDash = re.compile("\s*-\s*")

sizeFactors = { 'b'     : 1,
                'bytes' : 1,
                'k'     : 1024,
                'kb'    : 1024,
                'm'     : 1048576,
                'mb'    : 1048576,
                'g'     : 1073741824,
                'gb'    : 1073741824,
                't'     : 1099511627776,
                'tb'    : 1099511627776,
                'p'     : 1125899906842624,
                'pb'    : 1125899906842624 }

freqFactors = { 'hz'  : 1,
                'khz' : 1000,
                'mhz' : 1000000,
                'ghz' : 1000000000,
                'thz' : 1000000000000,
                'phz' : 1000000000000000 }

sizeMap = [ { 'factor' : sizeFactors['b'],
              'long'   : 'byte',
              'short'  : 'byte'           },

            { 'factor' : sizeFactors['k'],
              'long'   : 'Kilobyte',
              'short'  : 'KB'             },

            { 'factor' : sizeFactors['m'],
              'long'   : 'Megabyte',
              'short'  : 'MB'             },

            { 'factor' : sizeFactors['g'],
              'long'   : 'Gigabyte',
              'short'  : 'GB'             },

            { 'factor' : sizeFactors['t'],
              'long'   : 'Terabyte',
              'short'  : 'TB'             },

            { 'factor' : sizeFactors['p'],
              'long'   : 'Petabyte',
              'short'  : 'PB'             } ]

freqMap = [ { 'factor' : freqFactors['hz'],
              'long'   : 'Hertz',
              'short'  : 'Hz'               },

            { 'factor' : freqFactors['khz'],
              'long'   : 'Kilohertz',
              'short'  : 'KHz'              },

            { 'factor' : freqFactors['mhz'],
              'long'   : 'Megahertz',
              'short'  : 'MHz'              },

            { 'factor' : freqFactors['ghz'],
              'long'   : 'Gigahertz',
              'short'  : 'GHz'              },

            { 'factor' : freqFactors['thz'],
              'long'   : 'Terahertz',
              'short'  : 'THz'              },

            { 'factor' : freqFactors['phz'],
              'long'   : 'Petahertz',
              'short'  : 'PHz'              } ]

reListString = r"(?<!\\),"
reList = re.compile(reListString)

reKeyVal = r"(?<!\\)="
reKeyVal = re.compile(reKeyVal)

class typeToString:
    """Provides method for converting normalized types to strings."""
    def __init__(self):
        self.toStringFunctions = {}
        self.__build_to_string_functions()
 
    def __call__(self, type, value):
        return self.toStringFunctions[type](value) 
 
    def __build_to_string_functions(self):
        functions = {}
        for function in dir(self):
            functions[function] = 1

        for type in types.keys():
            # kinda bad, need to find out how to know the name of the class
            #  I'm in.  But it works.
            functionName = "_typeToString__tostring_%s" % type
            if functions.has_key(functionName):
                self.toStringFunctions[type] = getattr(self, functionName)
            else:
                if type == '':
                    self.toStringFunctions[type] = self.__tostring_nothing
                else:
                    error = "To string function %s for type %s does not exist." \
                        % (functionName, type)
                    raise Exception(error)
                    sys.exit(1)        

    def __tostring(self, value):
        return str(value)

    def __tostring_directory(self, value):
        return self.__tostring(value)

    def __tostring_address(self, value):
        return "%s:%s" % (value[0], value[1])

    def __tostring_ip_address(self, value):
        return self.__tostring(value)

    def __tostring_net_address(self, value):
        return self.__tostring(value)

    def __tostring_bool(self, value):
        if value == False:
            return 'false'
        elif value == True:
            return 'true'
        else:
            return str(value)

    def __tostring_int(self, value):
        return self.__tostring(value)

    def __tostring_float(self, value):
        return self.__tostring(value)

    def __tostring_pos_int(self, value):
        return self.__tostring(value)

    def __tostring_neg_int(self, value):
        return self.__tostring(value)     

    def __tostring_freq(self, value):
        return self.__tostring(value)

    def __tostring_pos_float(self, value):
        return self.__tostring(value)

    def __tostring_pos_num(self, value):
        return self.__tostring(value)

    def __tostring_neg_float(self, value):
        return self.__tostring(value)

    def __tostring_string(self, value):
        return value

    def __tostring_keyval(self, value):
        string = '"' # to protect from shell escapes
        for key in value:
          # for item in value[key]:
          #      string = "%s%s=%s," % (string, key, item)
          # Quotes still cannot protect Double-slashes.
          # Dealing with them separately
          val = re.sub(r"\\\\",r"\\\\\\\\",value[key])

          string = "%s%s=%s," % (string, key, val)

        return string[:-1] + '"'

    def __tostring_list(self, value):
        string = ''
        for item in value:
            string = "%s%s," % (string, item)
            
        return string[:-1]

    def __tostring_file(self, value):
        return self.__tostring(value)
      
    def __tostring_size(self, value):
        return self.__tostring(value)
        
    def __tostring_eaddress(self, value):
        return self.__tostring(value)

    def __tostring_tcp_port(self, value):
        return self.__tostring(value)

    def __tostring_http_version(self, value):
        return self.__tostring(value)

    def __tostring_range(self, value):
        if len(value) < 2:
          return value[0]
        else:
          return "%s-%s" % (value[0], value[1])

    def __tostring_timestamp(self, value):
        return self.__tostring(value)

    def __tostring_hostname(self, value):
        return self.__tostring(value)

    def __tostring_user_account(self, value):
        return self.__tostring(value)

    def __tostring_user_group(self, value):
        return self.__tostring(value)

    def __tostring_uri(self, value):
        return self.__tostring(value)

    def __tostring_nothing(self, value):
        return value

class typeValidator:
    """Type validation class used to normalize values or validated 
       single/large sets of values by type."""

    def __init__(self, originalDir=None):
        self.verifyFunctions = {}
        self.__build_verify_functions()

        self.validateList = []
        self.validatedInfo = []
        self.__originalDir = originalDir

    def __getattr__(self, attrname):
        """validateList  = [ { 'func' : <bound method configValidator>,
                               'name' : 'SA_COMMON.old_xml_dir',
                               'value': 'var/data/old'                 },

                             { 'func' : <bound method configValidator>,
                               'name' : 'SA_COMMON.log_level',
                               'value': '4'                            } ]

           validatedInfo = [ { # name supplied to add()
                               'name'       : 'SA_COMMON.tmp_xml_dir',

                               # is valid or not
                               'isValid'    : 1

                               # normalized value
                               'normalized' : /var/data/tmp,

                               # error string ?
                               'errorData'  : 0                        },

                             { 'name'       : 'SA_COMMON.new_xml_dir',
                               'isValid'    : 1