config.py

来自「Harvestman-最新版本」· Python 代码 · 共 1,583 行 · 第 1/5 页

                return CONFIG_INVALID_ARGUMENT


        # Error in option value
        if self._error:
            print self._error, value
            return CONFIG_INVALID_ARGUMENT

        # If need to pass config file return CONFIG_INVALID_ARGUMENT
        if cfgfile:
            return CONFIG_INVALID_ARGUMENT
        
        return CONFIG_ARGUMENTS_OK

    def check_value(self, option, value):
        """ This function checks the values for options when options
        are supplied as command line arguments. Returns 0 on any error
        and non-zero otherwise """

        # check #1: If value is a null, return 0
        if not value:
            self._error='Error in option value for option %s, value should not be empty!' % option
            return CONFIG_ARGUMENT_ERROR

        # no other checks right now
        return CONFIG_ARGUMENT_OK

    def process_value(self, value):
        """ This function maps values of command line arguments to values
        which can be used to assign to config params """

        # a 'yes' is treated as 1 and 'no' as 0
        # also an 'on' is treated as 1 and 'off' as 0
        # Other valid values: integers, strings, 'YES'/'NO'
        # 'OFF'/'ON'

        ret = OPTION_TURN_OFF
        # We expect the null check has been done before
        val = value.lower()
        if val in ('yes', 'on'):
            return OPTION_TURN_ON
        elif val in ('no', 'off'):
            return OPTION_TURN_OFF

        # convert value to int
        try:
            ret=int(val)
            return ret
        except:
            pass

        # return string value directly
        return str(value)

    def print_help(self):
        """ Prints the command-line usage information """

        print PROG_HELP % {'appname' : self.appname,
                           'version' : self.version,
                           'maturity' : self.maturity }

    def print_version_info(self):
        """ Prints version information about the program """

        print 'Version: %s %s' % (self.version, self.maturity)

    def _fix_url_protocol(self, url):
        """ Fixes errors in URL protocol string, if any """
        
        r = re.compile('www\d?\.|http://|https://|ftp://|file://',re.IGNORECASE)
        if not r.match(url):
            # Assume http url
            # prepend http:// to it
            # We prepend http:// to even FTP urls so that
            # the ftp servers can be crawled.
            url = 'http://' + url

        return url

    def add(self, url, name='', basedir='.', verbosity=logger.INFO):
        """ Adds a crawl project to the crawler. The arguments
        are the starting URL, and optional name for the project,
        a base directory for saving files and project verbosity """

        # Useful for command-line crawling
        self.projects.append({'url': url,
                              'project': name,
                              'basedir': basedir,
                              'verbosity': verbosity})
        
    def setup(self):
        """ Sets up the configuration object for full use,
        after fixing any errors in key config variables such as
        URL, project directory, project names etc """

        
        # If there is more than one url, we
        # combine all the project related
        # variables into a dictionary for easy
        # lookup.
        num=len(self.projects)
        if num==0:
            msg = 'Fatal Error: No URLs given, Aborting.\nFor command-line options run with -h option'
            sys.exit(msg)

        # Fix url error
        for x in range(len(self.projects)):
            entry = self.projects[x]
            url = entry['url']
            
            # If null url, return
            if not url: continue

            # Fix protocol strings
            url = self._fix_url_protocol(url)
            entry['url'] = url
            
            # If project is not set, set it to domain
            # name of the url.
            if entry.get('project','')=='':
                h = urlparser.HarvestManUrl(url)
                project = h.get_domain()
                entry['project'] = project

            if entry.get('basedir','')=='':
                entry['basedir'] = '.'

            if entry.get('verbosity',-1)==-1:
                if self.verbosity_override:
                    entry['verbosity'] = logger.getLogLevel(self.verbosity)
                else:
                    entry['verbosity'] = logger.getLogLevel(self.verbosity_default)
            else:
                entry['verbosity'] = logger.getLogLevel(entry['verbosity'])

        self.plugins = list(set(self.plugins))
            
        if 'swish-e' in self.plugins:
            # Disable any message output for swish-e
            self.verbosity = logger.DISABLE
            # Set verbosity to zero for all projects
            for entry in self.projects:
                entry['verbosity'] = logger.DISABLE

    def set_system_params(self):
        """ Sets config file/directory parameters for all users """

        # Need to reload sys to get setdefaultencoding attribute
        reload(sys)
        sys.setdefaultencoding('utf8')
        
        # Directory for system wide configuration files...
        if os.name == 'posix':
            #We might have to use find_packager() if somebody will use py2app py2exe
            #print os.path.split(os.path.dirname(__main__.__file__))[0]
            if os.path.isdir(sys.prefix):
                basefolder = os.path.abspath(sys.prefix)
            else:
                basefolder = os.path.dirname(sys.prefix)
            #print os.path.join(basefolder, 'etc', 'harvestman', 'config.xml')
            self.etcdir=os.path.join(basefolder, 'etc', 'harvestman')
            #self.etcdir = '/etc/harvestman'            
        elif os.name == 'nt':
            self.etcdir = os.path.join(os.environ.get("ALLUSERSPROFILE"),
                                       "Application Data", "HarvestMan", "conf")
                
    def set_user_params(self):
        """ Set config file/directory parameters specific to the current user  """
        
        if os.name == 'posix':
            homedir = os.environ.get('HOME')
            if homedir and os.path.isdir(homedir):
                harvestman_dir = os.path.join(homedir, '.harvestman')
                
        elif os.name == 'nt':
            profiledir = os.environ.get('USERPROFILE')
            if profiledir and os.path.isdir(profiledir):
                harvestman_dir = os.path.join(profiledir, 'Local Settings', 'Application Data','HarvestMan')

        if harvestman_dir:
            harvestman_conf_dir = os.path.join(harvestman_dir, 'conf')
            harvestman_sessions_dir = os.path.join(harvestman_dir, 'sessions')
            harvestman_db_dir = os.path.join(harvestman_dir, 'db')

            self.userdir = harvestman_dir
            self.userconfdir = harvestman_conf_dir
            self.usersessiondir = harvestman_sessions_dir
            self.userdbdir = harvestman_db_dir
    
    def parse_config_file(self, configfile=None):
        """ Parses the configuration file. An optional configuration file can be
        passed to this method. Otherwise it tries to parse the default configuration
        file """

        if configfile:
            cfgfile = configfile
        else:
            cfgfile = self.configfile
            
        if not os.path.isfile(cfgfile):
            logconsole('Configuration file %s not found...' % cfgfile)
        else:
            logconsole('Using configuration file %s...' % cfgfile)

        return configparser.parse_xml_config_file(self, cfgfile)
            
    def get_program_options(self):
        """ Umbrella method for reading the program configuration
        from a configuration file or the command-line or both """

        # Now load system wide configuration file...
        system_conf_file = os.path.join(self.etcdir, "config.xml")
        if os.path.isfile(system_conf_file):
            logconsole("Loading system configuration...")
            configparser.parse_xml_config_file(self, system_conf_file)

        # Then load user configuration file
        user_conf_file = os.path.join(self.userconfdir, 'config.xml')
        if os.path.isfile(user_conf_file):
            logconsole("Loading user configuration...")
            configparser.parse_xml_config_file(self, user_conf_file)

        # first check in argument list, if failed
        # check in config file
        res = self.parse_arguments()

        if res == CONFIG_INVALID_ARGUMENT:
            self.parse_config_file()

        # fix errors in config variables
        self.setup()
        
    def enable_controller(self):
        """ Return whether we need to start the controller
        thread. This is determined by whether we have
        any limits either on time, files or data """

        return (self.timelimit != -1) or \
               (self.maxfiles) or \
               (self.maxbytes)
    
    def reset_progress(self):
        """ Rests the progress bar object (used by Hget)"""
        
        self.progressobj = None
        self.progressobj = TextProgress()
        
    def __getattr__(self, name):
        """Overloaded __getattr__ method """
        
        try:
            return self[intern(name)]
        except KeyError:
            return

    def __setattr__(self, name, value):
        """ Overloaded __setattr__ method """
        
        self[intern(name)] = value

    def set_klass_plugin_func(self, klassname, funcname, func):
        """ Sets the plugin function for the given HarvestMan class
        'klassname'. The plugin target function is specified by
        'funcname' and the plugin source function is 'func' """
        
        try:
            d = self.__class__.klassmap[klassname + '_plugins']
            d[funcname] = func
        except KeyError:
            self.__class__.klassmap[klassname + '_plugins'] = { funcname: func }            

    def get_klass_plugins(self, klassname):
        """ Return the plugin function dictionary for the HarvestMan class
        with the name 'klassname' """
        
        return self.__class__.klassmap.get(klassname + '_plugins')

    def set_klass_callback_func(self, klassname, funcname, func, where):
        """ Sets the callback function for the given HarvestMan class
        'klassname'. The callback target function is specified by
        'funcname' and the callback source function is 'func'. The
        last argument specifies whether to insert the callback before
        or after the target function """
        
        try:
            d = self.__class__.klassmap[klassname + '_callbacks']
            d[funcname] = (func, where)
        except KeyError:
            self.__class__.klassmap[klassname + '_callbacks'] = { funcname: (func, where) }            

    def get_klass_callbacks(self, klassname):
        """ Return the callbacks function dictionary for the HarvestMan class
        with the name 'klassname' """
        
        return self.__class__.klassmap.get(klassname + '_callbacks')    

    def generate_projects_xml(self):
        """ Generates and returns content for the <projects> config file XML element """

        content = "<projects>\n\n"
        for x in range(len(self.projects)):
            entry = self.projects[x]
            
            project = entry.get('project')
            url = entry.get('url')
            verb = logger.getLogLevelName(entry.get('verbosity'))
            basedir = entry.get('basedir')
            
            projcontent = '<project skip="0">\n'
            projcontent += '<url>' + url + '</url>\n'
            projcontent += '<name>' + project + '</name>\n'
            projcontent += '<basedir>' + basedir + '</basedir>\n'            
            projcontent += '<verbosity level="' + str(verb) + '"/>\n'
            projcontent += '</project>\n\n'