genconfig.py

网络蜘蛛

""" Genconfig.py - Script to generate config file for
    HarvestMan Program.

 Author - Anand B Pillai.
 
 Feb 10 2004  Anand      1.3.1 bug fix release.
 Jun 14 2004  Anand      1.3.9 release.
                         
 Copyright(C) 2004-2005, Anand B Pillai.
 
 This file is part of the HarvestMan module.
 
"""

import os, sys
import string
import imp


class GenConfig:

    def __init__(self):

        # Locate HarvestMan modules
        import config
        
        self.__dict__['cfg'] = config.HarvestManStateObject()
        
    def __getattr__(self, name):
        try:
            return self.cfg[name]
        except KeyError:
            return None
        
    def __setattr__(self, name, value):
        self.cfg[name] = value

    def fitString(self, s, field=35):
        return string.ljust(s, field)
    
    def GenConfigFile(self):

        if os.path.exists('config.txt'):
            try:
                os.remove('config.txt')
            except OSError, e:
                print e
                return
        try:
            cf=open('config.txt','w')
        except IOError, e:
            print e
            return
        except OSError, e:
            print e
            return
        print 'Generating config file...'
        # write values
        cf.write(';;HarvestMan Configuration File version 2.0\n\n')
        cf.write(';;project related variables\n')
        cf.write(self.fitString('project.url') + str(self.url) + '\n')
        cf.write(self.fitString('project.name') + str(self.project) + '\n')
        cf.write(self.fitString('project.basedir') + str(self.basedir) + '\n')
        cf.write(self.fitString('project.verbosity') + str(self.verbosity) + '\n\n')          
        cf.write(';;network related variables\n')
        if self.proxy:
            cf.write(self.fitString('network.proxyserver') + bin_crypt(self.proxy) + '\n')
        if self.puser:
            cf.write(self.fitString('network.proxyuser') + bin_crypt(self.puser) + '\n')
        if self.ppasswd:
            cf.write(self.fitString('network.proxypasswd') + bin_crypt(self.ppasswd) + '\n')
        #if self.proxyport != 80:
        cf.write(self.fitString('network.proxyport') + str(self.proxyport) + '\n')
        if self.intranet:
            cf.write(self.fitString('network.intranet') + str(self.intranet) + '\n')
        cf.write('\n')
        cf.write(';;url related variables\n')
        if self.siteusername:
            cf.write(self.fitString('url.username') + bin_crypt(self.siteusername) + '\n')
        if self.sitepasswd:
            cf.write(self.fitString('url.password') + bin_crypt(self.sitepasswd) + '\n')
        cf.write('\n')
        
        cf.write(';;download related variables\n')
        cf.write(self.fitString('download.linkedimages') + str(self.getimagelinks) + '\n')
        cf.write(self.fitString('download.linkedstylesheets') + str(self.getstylesheets) + '\n')
        cf.write(self.fitString('download.fetchlevel') + str(self.fetchlevel) + '\n')        
        cf.write(self.fitString('download.rename') + str(self.renamefiles) + '\n')
        cf.write(self.fitString('download.checkfiles') + str(self.checkfiles) + '\n')
        cf.write(self.fitString('download.retryfailed') + str(self.retryfailed) + '\n')
        cf.write(self.fitString('download.html') + str(self.html) + '\n')
        cf.write(self.fitString('download.images') + str(self.images) + '\n')
        cf.write(self.fitString('download.cookies') + str(self.cookies) + '\n')
        cf.write(self.fitString('download.javascript') + str(self.javascript) + '\n')
        cf.write(self.fitString('download.javaapplet') + str(self.javaapplet) + '\n\n')
        
        cf.write(';;download control variables\n')
        cf.write(self.fitString('control.extserverlinks') + str(self.eserverlinks) + '\n')
        cf.write(self.fitString('control.extpagelinks') + str(self.epagelinks) + '\n')
        cf.write(self.fitString('control.maxextservers') + str(self.maxextservers) + '\n')
        cf.write(self.fitString('control.maxextdirs') + str(self.maxextdirs) + '\n')
        cf.write(self.fitString('control.urlfilter') + str(self.urlfilter) + '\n')
        cf.write(self.fitString('control.robots') + str(self.robots) + '\n')
        cf.write(self.fitString('control.maxfiles') + str(self.maxfiles) + '\n')
        cf.write(self.fitString('control.serverfilter') + str(self.serverfilter) + '\n')
        cf.write(self.fitString('control.depth') + str(self.depth) + '\n')
        cf.write(self.fitString('control.extdepth') + str(self.extdepth) + '\n')
        cf.write(self.fitString('control.pagecache') + str(self.pagecache) + '\n')
        cf.write(self.fitString('control.connections') + str(self.connections) + '\n')                        
        cf.write(self.fitString('control.projtimeout') + str(self.projtimeout) + '\n')
        cf.write(self.fitString('control.skipqueryforms') + str(self.skipqueryforms) + '\n')
        cf.write(self.fitString('control.subdomain') + str(self.subdomain) + '\n')
        cf.write(self.fitString('control.requests') + str(self.requests) + '\n')
        cf.write(self.fitString('control.tidyhtml') + str(self.tidyhtml) + '\n')
        cf.write(self.fitString('control.urlpriority') + str(self.urlpriority) + '\n')
        cf.write(self.fitString('control.serverpriority') + str(self.serverpriority) + '\n\n')                                
        cf.write(';;system related variables\n')
        cf.write(self.fitString('system.fastmode') + str(self.fastmode) + '\n')
        cf.write(self.fitString('system.usethreads') + str(self.usethreads) + '\n')
        cf.write(self.fitString('system.maxtrackers') + str(self.maxtrackers) + '\n')
        cf.write(self.fitString('system.threadtimeout') + str(self.timeout) + '\n')
        cf.write(self.fitString('system.threadpoolsize') + str(self.threadpoolsize) + '\n\n')

        cf.write(';;indexer related variables\n')
        cf.write(self.fitString('indexer.localise') + str(self.localise) + '\n')
        cf.write(self.fitString('indexer.jitlocalise') + str(self.jitlocalise) + '\n\n')            

        cf.write(';;files related variables\n')
        cf.write(self.fitString('files.errorfile') + str(self.errorfile) + '\n')
        cf.write(self.fitString('files.urlslistfile') + str(self.urlslistfile) + '\n')
        cf.write(self.fitString('files.msgfile') + str(self.logfile) + '\n\n')

        cf.write(';;display related variables\n')
        cf.write(self.fitString('display.browsepage') + str(self.browsepage) + '\n\n')

        cf.close()
        print 'Wrote config file config.txt.'
        print 'Done.'
        # Exit system
        sys.exit(0)
        
    def MakeStringQuery(self, query, strict=1,comment=""):

        res=''
        if not strict:
            res=raw_input(query)
        
        while res=='' and strict:
            res=raw_input(query)
            if res=='':
                print 'I am sorry, you need to enter a non-empty string!'

        if res==" ":
            # Abort signalled using <space> key
            # generate the config file
            print '<Space> key pressed! '
            print 'Aborting dialog...'
            self.GenConfigFile()
            
        return res

    def MakeYesOrNoQuery(self, query, expected=1):

        res='*'
        yornquery = query + ' [y/n/(<Enter> for default)] ?'
        while res !='y' and res != 'n' and res != '' and res != " ":
            res=raw_input(yornquery)
            if res!='y' and res != 'n' and res != '' and res !=" ":