logger.py

HADOOP 0.18.0安装源代码头文件

#Licensed to the Apache Software Foundation (ASF) under one
#or more contributor license agreements.  See the NOTICE file
#distributed with this work for additional information
#regarding copyright ownership.  The ASF licenses this file
#to you under the Apache License, Version 2.0 (the
#"License"); you may not use this file except in compliance
#with the License.  You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

#Unless required by applicable law or agreed to in writing, software
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.
"""hodLogger provides a customized interface to Python's core logging package.
"""

import sys, os, re, logging, logging.handlers, inspect, pprint, types
from tcp import get_address_tuple

fileFormatString    = '[%(asctime)s] %(levelname)s/%(levelno)s \
%(module)s:%(lineno)s - %(message)s'

streamFormatString  = '%(levelname)s - %(message)s'

debugStreamFormatString = '[%(asctime)s] %(levelname)s/%(levelno)s \
%(module)s:%(lineno)s - %(message)s'

syslogFormatString = '(%(process)d) %(levelname)s/%(levelno)s \
%(module)s:%(lineno)s - %(message)s'

smtpFormatString    = '[%(asctime)s] %(levelname)s/%(levelno)s \
%(module)s:%(lineno)s\n\n%(message)s'

fileFormater = logging.Formatter(fileFormatString)
streamFormater = logging.Formatter(streamFormatString)
debugStreamFormater = logging.Formatter(debugStreamFormatString)
syslogFormater = logging.Formatter(syslogFormatString)
smtpFormater = logging.Formatter(smtpFormatString)

defaultFileLevel = 3
defaultStreamLevel = 4
defaultSyslogLevel = 3
defaultSmtpLevel = 0

hodLogLevelMap = { 0 : logging.CRITICAL,
                   1 : logging.ERROR,
                   2 : logging.WARNING,
                   3 : logging.INFO,
                   4 : logging.DEBUG    }

hodStreamFormatMap = { 0 : streamFormater,
                       1 : streamFormater,
                       2 : streamFormater,
                       3 : streamFormater,
                       4 : debugStreamFormater }

rehodLogLevelMap = {}
for key in hodLogLevelMap.keys():
    rehodLogLevelMap[hodLogLevelMap[key]] = key


reModule = re.compile("^(.*)\..*$")

hodLogs = {}

class hodRotatingFileHandler(logging.handlers.RotatingFileHandler):
    """ This class needs to be used in place of RotatingFileHandler when
        the 2.4.0 Python interpreter is used."""

    def emit(self, record):
        """
        Emit a record.

        If a formatter is specified, it is used to format the record.
        The record is then written to the stream with a trailing newline
        [N.B. this may be removed depending on feedback]. If exception
        information is present, it is formatted using
        traceback.print_exception and appended to the stream.

        *****

        THIS IS A HACK, when instances of hodLogger get passed to the child of
        a child thread for some reason self.stream gets closed.  This version
        of emit re-opens self.stream if it is closed.  After testing it appears
        that self.stream is only closed once after the second thread is
        initialized so there is not performance penalty to this hack.  This
        problem only exists in python 2.4.

        *****
        """
        try:
            if self.shouldRollover(record):
                self.doRollover()
            try:
                msg = self.format(record)
                fs = "%s\n"
                if not hasattr(types, "UnicodeType"): #if no unicode support...
                    self.stream.write(fs % msg)
                else:
                    try:
                        self.stream.write(fs % msg)
                    except UnicodeError:
                        self.stream.write(fs % msg.encode("UTF-8"))
                    except ValueError:
                        self.stream = open(self.baseFilename, self.mode)
                        self.stream.write(fs % msg)

                self.flush()
            except:
                self.handleError(record)
        except:
            self.handleError(record)

    def shouldRollover(self, record):
        """
        Determine if rollover should occur.

        Basically, see if the supplied record would cause the file to exceed
        the size limit we have.

        *****

        THIS IS A HACK, when instances of hodLogger get passed to the child of
        a child thread for some reason self.stream gets closed.  This version
        of emit re-opens self.stream if it is closed.  After testing it appears
        that self.stream is only closed once after the second thread is
        initialized so there is not performance penalty to this hack. This
        problem only exists in python 2.4.

        *****
        """
        if self.maxBytes > 0:                   # are we rolling over?
            msg = "%s\n" % self.format(record)

            try:
                #due to non-posix-compliant Windows feature
                self.stream.seek(0, 2)
            except ValueError:
                self.stream = open(self.baseFilename, self.mode)
                self.stream.seek(0, 2)

            if self.stream.tell() + len(msg) >= self.maxBytes:
                return 1
        return 0

class hodCustomLoggingLogger(logging.Logger):
    """ Slight extension of the logging.Logger class used by the hodLog class.
    """
    def findCaller(self):
        """ findCaller() is supposed to return the callers file name and line
            number of the caller. This was broken when the logging package was
            wrapped by hodLog.  We should return much more relevant info now.
            """

        callerModule = ''
        callerLine = 0

        currentModule = os.path.basename(__file__)
        currentModule = reModule.sub("\g<1>", currentModule)

        frames = inspect.stack()
        for i in range(len(frames)):
            frameModule = os.path.basename(frames[i][1])
            frameModule = reModule.sub("\g<1>", frameModule)
            if frameModule == currentModule:
                previousFrameModule = os.path.basename(frames[i+1][1])
                previousFrameModule = reModule.sub("\g<1>",
                    previousFrameModule)
                callerFile = frames[i+1][1]
                callerLine = frames[i+1][2]
                continue

        returnValues = (callerFile, callerLine)
        if sys.version.startswith('2.4.4') or sys.version.startswith('2.5'):
            returnValues = (callerFile, callerLine, None)
            
        return returnValues

class hodLog:
    """ Cluster management logging class.

        logging levels: 0 - log only critical messages
                        1 - log critical and error messages
                        2 - log critical, error, and warning messages
                        3 - log critical, error, warning, and info messages
                        4 - log critical, error, warning, info, and debug
                            messages"""

    def __init__(self, appName):
        """Constructs a hodLogger object.

        appName      - name of logging application, log filenames will be
                       prepended with this name"""

        self.__appName = appName

        # initialize a dictionary to hold loggerNames
        self.__loggerNames = {}

        # initialize a dictionary to track log handlers and handler classes
        self.__logObjs = { 'file' : {}, 'smtp' : {}, 
                           'syslog' : {}, 'strm' : {} }

        # use a custom logging.Logger class
        logging.setLoggerClass(hodCustomLoggingLogger)

        # get the root app logger
        self.__logger = logging.getLogger(appName)
        self.__logger.setLevel(logging.DEBUG)
        
        hodLogs[self.__appName] = self

    def __attr__(self, attrname):
        """loggerNames  - list of defined logger names"""

        if attrname   == "loggerNames":  return self.__loggerNames.keys()
        else: raise AttributeError, attrname

    def __repr__(self):
        """Returns a string representation of a hodLog object of the form:

           LOG_NAME
                file: FILENAME (level LEVEL)
                smtp: SMTP_SERVER from FROM_ADDRESS (level LEVEL)
                strm: STRM_OBJECT (level LEVEL)
                ... """

        hodLogString = "hodLog: %s\n\n" % self.__appName
        for loggerName in self.__loggerNames.keys():
            hodLogString = "%s    logger: %s\n" % (hodLogString, loggerName)
            handlerClasses = self.__logObjs.keys()
            handlerClasses.sort()
            for handlerClass in handlerClasses:
                try:
                    loggerLevelName = logging.getLevelName(
                        self.__logObjs[handlerClass][loggerName]['level'])
                    hodLogString = "%s        %s: %s (level %s)\n" % (
                        hodLogString, handlerClass,
                        self.__logObjs[handlerClass][loggerName]['data'],
                        loggerLevelName)
                except:
                    hodLogString = "%s        %s: none\n" % (
                        hodLogString, handlerClass)
            hodLogString = "%s\n" % hodLogString

        return hodLogString

    # 'private' method which adds handlers to self.__logObjs
    def __add_to_handlers(self, handlerClass, loggerName, handler, data,
        level):
        self.__logObjs[handlerClass][loggerName] = {}
        self.__logObjs[handlerClass][loggerName]['handler'] = handler
        self.__logObjs[handlerClass][loggerName]['data'] = data
        self.__logObjs[handlerClass][loggerName]['level'] = level

    # 'private' method which determines whether a hod log level is valid and
    #   returns a valid logging.Logger level
    def __get_logging_level(self, level, defaultLevel):
        loggingLevel = ''
        try:
            loggingLevel = hodLogLevelMap[int(level)]
        except:
            loggingLevel = hodLogLevelMap[defaultLevel]

        return loggingLevel

    # make a logging.logger name rootLogger.childLogger in our case the
    #   appName.componentName
    def __get_logging_logger_name(self, loggerName):
        return "%s.%s" % (self.__appName, loggerName)

    def add_logger(self, loggerName):
        """Adds a logger of name loggerName.

           loggerName    - name of component of a given application doing the
                           logging

           Returns a hodLogger object for the just added logger."""

        try:
            self.__loggerNames[loggerName]
        except:
            loggingLoggerName = self.__get_logging_logger_name(loggerName)
            logging.getLogger(loggingLoggerName)

            self.__loggerNames[loggerName] = 1

            return hodLogger(self.__appName, loggingLoggerName)

    def add_file(self, logDirectory, maxBytes=0, backupCount=0,
        level=defaultFileLevel, addToLoggerNames=None):
        """Adds a file handler to all defined loggers or a specified set of
           loggers.  Each log file will be located in logDirectory and have a
           name of the form appName-loggerName.log.

           logDirectory     - logging directory
           maxBytes         - maximum log size to write in bytes before rotate
           backupCount      - number of rotated logs to keep
           level            - cluster management log level
           addToLoggerNames - list of logger names to which stream handling
                              will be added"""

        def add_file_handler(loggerName):
            if not self.__logObjs['file'].has_key(loggerName):
                loggingLevel = self.__get_logging_level(level,
                    defaultFileLevel)

                logFile = os.path.join(logDirectory, "%s-%s.log" % (
                    self.__appName, loggerName))

                if sys.version.startswith('2.4'):
                    fileHandler = hodRotatingFileHandler(logFile,
                        maxBytes=maxBytes, backupCount=backupCount)
                else:
                    fileHandler = logging.handlers.RotatingFileHandler(logFile,
                        maxBytes=maxBytes, backupCount=backupCount)

                fileHandler.setLevel(loggingLevel)
                fileHandler.setFormatter(fileFormater)

                loggingLoggerName = self.__get_logging_logger_name(loggerName)
                aLogger = logging.getLogger(loggingLoggerName)
                aLogger.addHandler(fileHandler)

                fileData = "%s" % logFile
                self.__add_to_handlers('file', loggerName, fileHandler,
                    fileData, loggingLevel)

        if addToLoggerNames:
            for loggerName in addToLoggerNames:
                add_file_handler(loggerName)
        else:
            for loggerName in self.__loggerNames:
                add_file_handler(loggerName)

    def add_stream(self, stream=sys.stderr, level=defaultStreamLevel,
        addToLoggerNames=None):
        """Adds a stream handler to all defined loggers or a specified set of
           loggers.

           stream           - a stream such as sys.stderr or sys.stdout
           level            - cluster management log level
           addToLoggerNames - tupple of logger names to which stream handling
                              will be added"""

        def add_stream_handler(loggerName):
            if not self.__logObjs['strm'].has_key(loggerName):
                loggingLevel = self.__get_logging_level(level,
                    defaultStreamLevel)

                streamHandler = logging.StreamHandler(stream)
                
                streamHandler.setLevel(loggingLevel)
                
                streamHandler.setFormatter(hodStreamFormatMap[int(level)])

                loggingLoggerName = self.__get_logging_logger_name(loggerName)
                aLogger = logging.getLogger(loggingLoggerName)
                aLogger.addHandler(streamHandler)

                streamData = "%s" % stream
                self.__add_to_handlers('strm', loggerName, streamHandler,
                    streamData, loggingLevel)

        if addToLoggerNames:
            for loggerName in addToLoggerNames:
                add_stream_handler(loggerName)
        else:
            for loggerName in self.__loggerNames:
                add_stream_handler(loggerName)

    def add_syslog(self, address, level=defaultSyslogLevel, 
                   addToLoggerNames=None):
        def add_syslog_handler(loggerName):
            if not self.__logObjs['syslog'].has_key(loggerName):
                loggingLevel = self.__get_logging_level(level,
                    defaultSyslogLevel)

                address[1] = int(address[1])
                syslogHandler = logging.handlers.SysLogHandler(tuple(address),
                                                               9)
                
                syslogHandler.setLevel(loggingLevel)
                
                syslogHandler.setFormatter(syslogFormater)

                loggingLoggerName = self.__get_logging_logger_name(loggerName)
                aLogger = logging.getLogger(loggingLoggerName)
                aLogger.addHandler(syslogHandler)