config.py

来自「Harvestman-最新版本」· Python 代码 · 共 1,583 行 · 第 1/5 页

        # if no cmd line arguments, then use config file,
        # return -1
        if len(sys.argv)==1:
            return CONFIG_INVALID_ARGUMENT

        # Otherwise parse the arguments, the command line arguments
        # are the same as the variables(dictionary keys) of this class.
        # Description
        # Options needing no arguments
        #
        # -h => prints help
        # -v => prints version info
        
        args, optdict = '',{}
        try:
            if self.appname == 'HarvestMan':
                USAGE = USAGE1
            elif self.appname == 'Hget':
                USAGE = USAGE2
                
            gopt = GenericOptionParser(options.getOptList(self.appname), usage = USAGE % self )
            optdict, args = gopt.parse_arguments()
        except GenericOptionParserError, e:
            hexit('Error: ' + str(e))

        # print optdict
        
        cfgfile = False

        if self.appname == 'HarvestMan':
            for option, value in optdict.items():
                # If an option with value of null string, skip it
                if value=='':
                   # print 'Skipping option',option
                   continue
                else:
                   # print 'Processing option',option,'value',value
                   pass

                # first parse arguments with no options
                if option=='version' and value:
                    self.print_version_info()
                    sys.exit(0)                
                elif option=='configfile':
                    if SUCCESS(self.check_value(option,value)):
                        self.set_option_xml('configfile_value', self.process_value(value))
                        cfgfile = True
                        # Continue parsing and take rest of options from cmd line
                elif option=='projectfile':
                    if SUCCESS(self.check_value(option,value)):
                        self.set_option_xml('projectfile_value', self.process_value(value))

                        projector = utils.HarvestManProjectManager()

                        if projector.read_project() == PROJECT_FILE_EXISTS:
                            # No need to parse further values
                            return PROJECT_FILE_EXISTS

                elif option=='basedir':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('basedir', self.process_value(value))
                elif option=='project':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('name', self.process_value(value))
                elif option=='retries':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('retries_value', self.process_value(value))
                elif option=='localise':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('localise_value', self.process_value(value))
                elif option=='fetchlevel':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('fetchlevel_value', self.process_value(value))
                elif option=='maxthreads':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('trackers_value', self.process_value(value))
                elif option=='maxfiles':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('maxfiles_value', self.process_value(value))
                elif option=='timelimit':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('timelimit_value', self.process_value(value))
                elif option=='workers':
                    self.set_option_xml('workers_status',1)
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('workers_size', self.process_value(value))                
                elif option=='urlfilter':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('urlfilter', self.process_value(value))
                elif option=='depth':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('depth_value', self.process_value(value))
                elif option=='robots':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('robots_value', self.process_value(value))
                elif option=='urllist':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('urllist', self.process_value(value))
                elif option=='proxy':
                    if SUCCESS(self.check_value(option,value)):
                        # Set proxyencrypted flat to False
                        self.proxyenc=False
                        self.set_option_xml('proxyserver', self.process_value(value))
                elif option=='proxyuser':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('proxyuser', self.process_value(value))                
                elif option=='proxypasswd':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('proxypasswd', self.process_value(value))
                elif option=='urlserver':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('urlserver_status', self.process_value(value))

                elif option=='cache':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('cache_status', self.process_value(value))
                elif option=='connections':
                    if SUCCESS(self.check_value(option,value)):
                        val = self.process_value(value)
                        if val>=self.connections:
                            self.connections = val + 1
                elif option=='verbosity':
                    if SUCCESS(self.check_value(option,value)):
                        self.verbosity = logger.getLogLevel(value)
                        self.verbosity_override = True
                    else:
                        print 'Check failed!'
                elif option=='subdomain':
                    if value: self.set_option_xml('subdomain_value', 0)                    
                #elif option=='savesessions':
                #    if SUCCESS(self.check_value(option,value)): self.set_option_xml('savesessions_value', self.process_value(value))
                elif option=='simulate':
                    self.set_option_xml('simulate_value', value)
                elif option=='plugins':
                    # Plugin is specified as plugin1+plugin2+...
                    plugins = value.split('+')
                    # Remove any duplicate occurence of same plugin
                    self.plugins = list(set([plugin.strip() for plugin in plugins]))
                    # Don't allow reading plugin from config file now
                    self.items_to_skip.append('plugin_name')
                elif option=='option':
                    # Value should be of the form param=value
                    if not param_re.match(value):
                        print 'Error in option value, should be of the form <param>=<value>'
                    else:
                        param,val=value.strip().split('=')
                        if param in self:
                            # Guess type of param
                            # Could be a tuple, dict or list value ?
                            if (val.startswith('(') and val.endswith(')')) or \
                               (val.startswith('[') and val.endswith(']')) or \
                               (val.startswith('{') and val.endswith('}')):

                                # Try tupling
                                try:
                                    val = eval(val)
                                except Exception:
                                    pass
                            else:
                                # Try float next
                                if float_re.match(val):
                                    val = float(val)
                                # Try int next
                                elif int_re.match(val):
                                    val = int(val)
                                else:
                                    # Plain string
                                    pass
                                
                            self[param]=val
                elif option == 'ui' and value:
                    # Start the web UI
                    from harvestman.lib import gui
                    gui.run()
                    
                    sys.exit(0)
                elif option == 'genconfig' and value:
                    #Generate configuration File
                    from harvestman.tools import genconfig
                    genconfig.main()
                    
                    sys.exit(0)
                elif option == 'selftest' and value:
                    # Run the unit-tests as self-tests
                    print 'Running self-test...'
                    sys.path.append(os.path.join(module_path, 'test'))
                    from harvestman.test import run_tests
                    
                    result = run_tests.run_all_tests()
                    print result
                    if result.wasSuccessful():
                        print 'self-test complete. All tests passed.'
                        sys.exit(0)
                    else:
                        print 'self-test failed. Please check your installation!'
                        sys.exit(1)

        elif self.appname == 'Hget':
            # Hget options
            for option, value in optdict.items():
                # If an option with value of null string, skip it
                if value=='':
                   # print 'Skipping option',option
                   continue
                else:
                   # print 'Processing option',option,'value',value
                   pass

                # first parse arguments with no options
                if option=='version' and value:
                    self.print_version_info()
                    sys.exit(0)                               
                elif option=='numparts':
                    # Setting numparts forces split downloads
                    self.numparts = abs(int(value))
                    if self.numparts == 0:
                        print 'Error: Invalid value for number of parts, value should be non-zero!'
                        sys.exit(1)
                    if self.numparts>1:
                        self.forcesplit = True
                        # If we are forcesplitting with parts>1, then disable resume automatically
                        print 'Force-split switched on, resuming will be disabled'
                        self.canresume = False
                    else:
                        print 'Warning: Setting numparts to 1 has no effect!'
                elif option=='memory':
                    if value:
                        print 'Warning: Enabling in-memory flag, data will be stored in memory!'
                        self.datamode = CONNECTOR_DATA_MODE_INMEM
                        self.datamodename = "mem"
                elif option=='currentdir':
                    if value:
                        print 'Temporary files will be saved to current directory'
                        # Do not use temporary directory for saving intermediate files
                        self.hgetnotemp = True
                elif option=='output':
                    self.hgetoutfile = value
                elif option=='outputdir':
                    self.hgetoutdir = value
                elif option=='verbose':
                    self.hgetverbose = value
                elif option=='proxy':
                    if SUCCESS(self.check_value(option,value)):
                        # Set proxyencrypted flat to False
                        self.proxyenc=False
                        self.set_option_xml('proxyserver', self.process_value(value))
                elif option=='proxyuser':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('proxyuser', self.process_value(value))
                elif option=='proxypasswd':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('proxypasswd', self.process_value(value))
                elif option=='passwd':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('passwd', self.process_value(value))
                elif option=='username':
                    if SUCCESS(self.check_value(option,value)): self.set_option_xml('username', self.process_value(value))
                elif option == 'mirrorfile':
                    filename = value.strip()
                    if os.path.isfile(filename):
                        print 'Mirrors will be loaded from %s...' % filename
                        self.mirrorfile = filename
                    else:
                        print  'Warning: Mirror file %s not found!' % filename
                elif option == 'mirrorsearch':
                    if value:
                        print  'Mirror search turned on'
                        print 'Warning: This is an experimental feature...'
                        self.mirrorsearch = True
                elif option == 'relpathidx':
                    idx = int(value.strip())
                    self.mirrorpathindex = idx
                elif option == 'norelpath':
                    if value:
                        print 'Using filename only to construct mirror URLs...'
                        self.mirroruserelpath = False
                elif option == 'resumeoff':
                    if value:
                        print 'Resume mode set to off, partial downloads will not be resumed!'
                        self.canresume = False
                elif option == 'retries':
                    self.retryfailed = int(value)

            # If both mirror search and mirror file specified, mirror file is used
            # Print some information regarding mismatch of options...
            if self.mirrorfile and self.mirrorsearch:
                print 'Both mirror search and mirror file option given, mirror search will not be done'
                self.mirrorsearch = False
            if self.mirrorpathindex and not self.mirrorfile:
                print 'Ignoring mirror path index param because no mirror file is loaded'
            if not self.mirroruserelpath and not self.mirrorfile:
                print 'Ignoring relpath flag because no mirror file is loaded'
        
        if args:
            # Any option without an argument is assumed to be a URL
            for arg in args:
                self.set_option_xml('url',self.process_value(arg))
                
            # Since we set a URL from outside, we dont want to read
            # URLs from the config file - same for plugins
            self.items_to_skip = ['url','name','basedir','verbosity_level']

        # If urlfile option set, read all URLs from a file
        # and load them.
        if self.urlfile:
            if not os.path.isfile(self.urlfile):
                print 'Error: Cannot find URL file %s!' % self.urlfile
                return CONFIG_INVALID_ARGUMENT
            
            # Open file
            try:
                lines = open(self.urlfile).readlines()
                if len(lines):
                    # Reset all...
                    self.projects = []

                    for line in lines:
                        url = line.strip()
                        # Fix URL protocol string
                        url = self._fix_url_protocol(url)
                        try:
                            # Create project name
                            h = urlparser.HarvestManUrl(url)
                            project = h.get_domain()
                            self.projects.append({'url': url,
                                                  'project': project,
                                                  'basedir': '.',
                                                  'verbosity': self.verbosity_default})
                        except urlparser.HarvestManUrlError, e:
                            continue

                    # We would now want to skip url, project,
                    # basedir etc in the config file
                    self.items_to_skip = ['url','name','basedir','verbosity_level']

            except Exception, e:
                print e