config.py

来自「Harvestman-最新版本」· Python 代码 · 共 1,583 行 · 第 1/5 页

            if value=="": return CONFIG_VALUE_EMPTY

            # Bug fix: If someone has set the
            # value to 'True'/'False' instead of
            # 1/0, convert to bool type first.
            if type(value) in (str, unicode):
                if value.lower() == 'true':
                    value = 1
                elif value.lower() == 'false':
                    value = 0
            
            if type(value) is not tuple:
                # get the key for the option
                key = (self.xml_map[option])[0]
                # get the type of the option
                typ = (self.xml_map[option])[1]
                # do any type casting of the option
                fval = (eval(typ))(value)
                # do any negation of the option
                if type(fval) in (int,bool):
                    if negate: fval = not fval
                # set the option on the dictionary
                self[key] = fval
                
                return CONFIG_OPTION_SET
            else:
                # option is a tuple of values
                # iterate through all values of the option
                # see if the size of the value tuple and the
                # size of the values for this key match
                _values = self.xml_map[option]
                if len(_values) != len(value): return CONFIG_VALUE_MISMATCH

                for index in range(0, len(_values)):
                    _v = _values[index]
                    if len(_v) !=2: continue
                    _key, _type = _v

                    v = value[index]
                    # do any type casting on the option's value
                    fval = (eval(_type))(v)
                    # do any negation
                    if type(fval) in (int,bool):                    
                        if negate: fval = not fval
                    # set the option on the dictionary
                    self[_key] = fval

                return CONFIG_OPTION_SET

        return CONFIG_OPTION_NOT_SET

    def set_option_xml_attr(self, option, value, attrs):
        """ Sets an option from the XML config file for an XML attribute """

        # If option in things to be skipped, return
        if option in self.items_to_skip:
            return CONFIG_ITEM_SKIPPED
        
        option_val = self.xml_map.get(option, None)
        
        if option_val:
            try:
                if type(option_val) is tuple:
                    self.assign_option(option_val, value, attrs)
                elif type(option_val) is list:
                    # If the option_val is a list, there
                    # might be multiple vars to set.
                    for item in option_val:
                        # The item has to be a tuple again...
                        if type(item) is tuple:
                            # Set it
                            self.assign_option(item, value, attrs)
            except Exception, e:
                print 'Error assigning option','"'+option+'"','=>',str(e)
                if e.__class__==ValueError:
                    print '(Perhaps you invoked the wrong argument ?)'
                print 'Pass option -h for command line usage.'                    
                hexit(e)
        else:
            return CONFIG_OPTION_NOT_DEFINED

        return CONFIG_OPTION_SET

    def set_option_xml(self, option, value):
        """ Set an option from the XML config file from an XML element """
        
        # If option in things to be skipped, return
        if option in self.items_to_skip:
            return CONFIG_ITEM_SKIPPED
        
        option_val = self.xml_map.get(option, None)
        
        if option_val:
            try:
                if type(option_val) is tuple:
                    # print 'Assigning option',option_val,value
                    self.assign_option(option_val, value)
                elif type(option_val) is list:
                    # If the option_val is a list, there
                    # might be multiple vars to set.
                    for item in option_val:
                        # The item has to be a tuple again...
                        if type(item) is tuple:
                            # Set it
                            self.assign_option(item, value)
            except Exception, e:
               print 'Error assigning option','"'+option+'"','=>',str(e)
               # If this is a ValueError, mostly the wrong argument was passed
               if e.__class__==ValueError:
                   print '(Perhaps you invoked the wrong argument ?)'
               print 'Pass option -h for command line usage.'
               hexit(1)
        else:
            return CONFIG_OPTION_NOT_DEFINED

        return CONFIG_OPTION_SET        

    def set_maxbytes(self, key, val, attrdict):

        # The value could be in any of the following forms
        # <maxbytes value="5000" /> - End crawl at 5000 bytes
        # <maxbytes value="10kb" /> - End crawl at 10kb 
        # <maxbytes value="50MB" /> - End crawl at 50 MB.
        # <maxbytes value="1GB" /> - End crawl at 1 GB.
        # <maxbytes value="10k" /> - End crawl at 10kb 
        # <maxbytes value="50M" /> - End crawl at 50 MB.
        # <maxbytes value="1G" /> - End crawl at 1 GB.        
        # Any extra spaces should also be taken care of
        
        # The regexp does all the above
        items = maxbytes_re.findall(val.strip())
        if items:
            # 'item' is a pair
            item = items[0]
            # First member is the number, second the
            # specification for kb, mb, gb if any.
            limit, spec = item
            limit = int(limit)
            
            if spec != '':
                # Check for kb, mb, gb
                spec = spec.strip().lower()
                if spec.startswith('k'):
                    limit *= 1024
                elif spec.startswith('m'):
                    limit *= pow(1024, 2)
                elif spec.startswith('g'):
                    limit *= pow(1024, 3)

            # Set maxbytes
            self.maxbytes = limit

    def set_maxbandwidth(self, key, val, attrdict):

        # The value could be in any of the following forms
        # <maxbandwidth value="5" /> - Crawl at 5 bytes per sec 
        # <maxbandwidth value="5 k" /> - Crawl at 5 kbps
        # <maxbandwidth value="5 kb" /> - Crawl at 5 kbps
        # <maxbandwidth value="5 kbps" /> - Crawl at 5 kbps        
        # <maxbandwidth value="5 m" /> - Crawl at 5 mbps
        # <maxbandwidth value="5 mb" /> - Crawl at 5 mbps
        # <maxbandwidth value="5 mbps" /> - Crawl at 5 mbps        
        # Any extra spaces should also be taken care of
        
        # The regexp does all the above
        items = maxbw_re.findall(val.strip())
        if items:
            item = items[0]
            # First member is the number, second the
            # specification for kb, mb, gb if any.
            limit = int(item[0])
            spec = item[1]
            
            if spec != '':
                # Check for kb, mb, gb, kbps, mbps, gbps
                spec = spec.strip().lower()
                if spec.startswith('k'):
                    limit *= 1024
                elif spec.startswith('m'):
                    limit *= pow(1024, 2)
                elif spec.startswith('g'):
                    limit *= pow(1024, 3)

            # Set maxbandwidth
            self.bandwidthlimit = float(limit)

    def set_urlfilter(self, key, val, filterdict):

        enable = int(filterdict.get(u'enable', 1))
        if not enable:
            return
        
        casing = int(filterdict.get(u'case',0))
        flags = filterdict.get(u'flags','')

        # Append a tuple of value, casing, flags
        if key=='regexp':
            self.regexurlfilters.append((val,casing,flags))
        elif key=='path':
            self.pathurlfilters.append((val,casing,flags))
        elif key=='extension':
            self.extnurlfilters.append((val,casing,flags))

    def set_textfilter(self, key, val, filterdict):

        enable = int(filterdict.get(u'enable', 1))
        if not enable:
            return
        
        casing = int(filterdict.get(u'case',0))
        flags = filterdict.get(u'flags','')

        # Append a tuple of value, casing, flags
        if key=='content':
            self.contentfilters.append((val,casing,flags))
        elif key=='meta':
            tags = filterdict.get(u'tags','all')
            self.metafilters.append((val,casing,flags,tags))

    def set_project(self, key, val, prjdict):
        # Same function is called for url, basedir, name
        # and verbosity

        # If project is to be ignored, skip this
        if self.project_ignore:
            return

        # Project name has to match [a-zA-Z0-9_]. No other
        # characters allowed.
        if key=='project':
            if not projectname_re.match(val):
                raise HarvestManConfigError,'Project name %s is not valid' % val
            
        new_entry, recent = False, {}
        sz = len(self.projects)
        if sz==0:
            new_entry = True
        else:
            item = self.projects[-1]
            # If item is completed, new entry is True
            # else it is false
            if item.get('done',False):
                new_entry = True
            else:
                recent = item

                # Still check if this is beginning of a new entry
                # since we may not define all fields inside <project>...</project>
                if key in recent:
                    # Key already there, so treat this as a fresh entry
                    # and close current entry...
                    recent['done'] = True
                    self.projects[-1] = recent
                    recent = {}
                    new_entry = True

        if key in ('url','basedir','project'):
            recent[key] = val
        elif key=='verbosity':
            try:
                recent['verbosity'] = logger.getLogLevel(prjdict[u'level'])
                # print 'Verbosity=>',recent['verbosity']
            except KeyError:
                recent['verbosity'] = logger.getLogLevel(val)

        # If all items are present, put 'done' to True
        if len(recent)==4:
            recent['done'] = True

        # If new entry, then append, else set in position
        if new_entry:
            self.projects.append(recent)
        else:
            self.projects[-1] = recent

        # print self.projects
        
    def set_plugin(self, key, val, plugindict):
        """ Sets the state of the plugins param """

        plugin = plugindict['name']
        enable = int(plugindict['enable'])
        if enable: self.plugins.append(plugin)

    def set_datamode(self, key, val, modedict):
        """ Sets the state of the connections param """

        
        mode = modedict['type']
        if type(mode) is str:
            if mode.lower()=='flush' :
                self.datamode = CONNECTOR_DATA_MODE_FLUSH
                self.datamodename == mode.lower()
            elif mode.lower()=='mem':
                self.datamode = CONNECTOR_DATA_MODE_INMEM
                self.datamodename == mode.lower()
        elif type(mode) is int:
            self.datamode = mode
            if self.datamode>1: 
                self.datamode=1
            
    def set_parse_features(self, key, val, featuredict):
        """ Sets the state of the plugins param """

        feat = featuredict['name']
        enable = int(featuredict['enable'])
        self.htmlfeatures.append((feat, enable))
        
    def parse_arguments(self):
        """ Parses the command line arguments """

        # This function has 3 return values
        # CONFIG_INVALID_ARGUMENT => no cmd line arguments/invalid cmd line arguments
        # ,so force program to read config file.
        # PROJECT_FILE_EXISTS => existing project file supplied in cmd line
        # CONFIG_ARGUMENTS_OK => all options correctly read from cmd line