connector.py

Harvestman-最新版本

            kls.CONTENTLEN[self._index] = self._contentlen

            total = sum(kls.CONTENTLEN)
            self.__class__.NETDATALEN = total
            
            if kls.ORIGLENGTH:
               pertotal = float(100.0*total)/float(kls.ORIGLENGTH)

            if self._clength:
                per = float(100.0*self._contentlen)/float(self._clength)
                
            if curr>kls.START_TIME:
               bandwidth = float(total)/float(curr - kls.START_TIME)

            if bandwidth and kls.ORIGLENGTH:
               eta = int((kls.ORIGLENGTH - total)/float(bandwidth))
            pass
        
        if eta != -1:
            eta = str(datetime.timedelta(seconds=int(eta)))
        else:
            eta = 'NaN'
        
        return (per, pertotal, l, bandwidth, eta)

    def get_data(self):
        """ Returns the downloaded data """
        
        return self._data

    def get_datalen(self):
        """ Returns length of downloaded data """
        
        return self._contentlen

    def set_index(self, idx):
        """ Sets the index attribute (used for multipart downloads only) """

        self._index = idx

    def get_index(self):
        """ Gets the index attribute (used for multipart downloads only) """
        
        return self._index
    
    def stop(self):
        """ Stops the thread """
        
        self._flag = True

    @classmethod
    def reset_klass(cls):
        """ Resets all class attributes (classmethod) """
        
        cls.ORIGLENGTH = 0
        cls.START_TIME = 0.0
        cls.CONTENTLEN = []
        cls.MULTIPART = False
        cls.NETDATALEN = 0
        
class HarvestManNetworkConnector(object):
    """ This class keeps the Internet settings and configures the network. """

    alias = 'connmgr'                
    
    def __init__(self):
        # use proxies flag
        self._useproxy=0
        # check for ssl support in python
        self._initssl=False
        # Number of socket errors
        self._sockerrs = 0
        # Config object
        self._cfg = objects.config
        
        if hasattr(socket, 'ssl'):
            self._initssl=True
            __protocols__.append("https")

        # dictionary of protocol:proxy values
        self._proxydict = {}
        # dictionary of protocol:proxy auth values
        self._proxyauth = {}
        self.configure()
        
    def set_useproxy(self, val=True):
        """ Set the value of use-proxy flag. Also sets proxy dictionaries to default values """

        self._useproxy=val

        if val:
            proxystring = 'proxy:80'
            
            # proxy variables
            self._proxydict["http"] =  proxystring
            self._proxydict["https"] = proxystring
            self._proxydict["ftp"] = proxystring
            # set default for proxy authentication
            # tokens.
            self._proxyauth["http"] = ""
            self._proxyauth["https"] = ""
            self._proxyauth["ftp"] = ""            

    def set_ftp_proxy(self, proxyserver, proxyport, authinfo=(), encrypted=True):
        """ Sets ftp proxy information """

        if encrypted:
            self._proxydict["ftp"] = "".join((bin_decrypt(proxyserver),  ':', str(proxyport)))
        else:
            self._proxydict["ftp"] = "".join((proxyserver, ':', str(proxyport)))

        if authinfo:
            try:
                username, passwd = authinfo
            except ValueError:
                username, passwd = '', ''

            if encrypted:
                passwdstring= "".join((bin_decrypt(username), ':', bin_decrypt(passwd)))
            else:
                passwdstring = "".join((username, ':', passwd))

            self._proxyauth["ftp"] = passwdstring

    def set_https_proxy(self, proxyserver, proxyport, authinfo=(), encrypted=True):
        """ Sets https(ssl) proxy  information """

        if encrypted:
            self._proxydict["https"] = "".join((bin_decrypt(proxyserver), ':', str(proxyport)))
        else:
            self._proxydict["https"] = "".join((proxyserver, ':', str(proxyport)))

        if authinfo:
            try:
                username, passwd = authinfo
            except ValueError:
                username, passwd = '', ''

            if encrypted:
                passwdstring= "".join((bin_decrypt(username), ':', bin_decrypt(passwd)))
            else:
                passwdstring = "".join((username, ':', passwd))

            self._proxyauth["https"] = passwdstring

    def set_http_proxy(self, proxyserver, proxyport, authinfo=(), encrypted=True):
        """ Sets http proxy information """

        if encrypted:
            self._proxydict["http"] = "".join((bin_decrypt(proxyserver), ':', str(proxyport)))
        else:
            self._proxydict["http"] = "".join((proxyserver, ':', str(proxyport)))

        if authinfo:
            try:
                username, passwd = authinfo
            except ValueError:
                username, passwd = '', ''

            if encrypted:
                passwdstring= "".join((bin_decrypt(username), ':', bin_decrypt(passwd)))
            else:
                passwdstring= "".join((username, ':', passwd))

            self._proxyauth["http"] = passwdstring

    def set_proxy(self, server, port, authinfo=(), encrypted=True):
        """ Sets proxy information for all protocols """

        # For most users, only this method will be called,
        # rather than the specific method for each protocol,
        # as proxies are normally shared for all tcp/ip protocols 

        for p in __protocols__:
            # eval helps to do this dynamically
            s='self.set_' + p + '_proxy'
            func=eval(s, locals())
            
            func(server, port, authinfo, encrypted)

    def set_authinfo(self, username, passwd, encrypted=True):
        """ Set authentication information for proxy server """

        
        # Note: If this function is used all protocol specific
        # authentication will be replaced by this authentication. 

        if encrypted:
            passwdstring = "".join((bin_decrypt(username), ':', bin_decrypt(passwd)))
        else:
            passwdstring = "".join((username, ':', passwd))

        self._proxyauth = {"http" : passwdstring,
                            "https" : passwdstring,
                            "ftp" : passwdstring }

    def configure(self):
        """ Wrapper method for configuring network and protocols """

        log = objects.logger

        keepalive.DEBUG = keepalive.FakeLogger()
        keepalive.DEBUG.info = log.debug
        keepalive.DEBUG.error = log.debug
        
        self.configure_network()
        self.configure_protocols()
        
    def configure_network(self):
        """ Configure network settings for the user """

        # First: Configuration of network (proxies/intranet etc)
        
        # Check for proxies in the config object
        if self._cfg.proxy:
            self.set_useproxy(True)
            proxy = self._cfg.proxy
            
            index = proxy.rfind(':')
            if index != -1:
                port = proxy[(index+1):].strip()
                server = proxy[:index]
                # strip of any 'http://' from server
                index = server.find('http://')
                if index != -1:
                    server = server[(index+7):]

                self.set_proxy(server, int(port), (), self._cfg.proxyenc)

            else:
                port = self._cfg.proxyport
                server = self._cfg.proxy
                self.set_proxy(server, int(port), (), self._cfg.proxyenc)

            # Set proxy username and password, if specified
            puser, ppasswd = self._cfg.puser, self._cfg.ppasswd
            if puser and ppasswd: self.set_authinfo(puser, ppasswd, self._cfg.proxyenc)


    def configure_protocols(self):
        """ Configures protocol handlers """
        
        # Second: Configuration of protocol handlers.

        # TODO: Verify gopher protocol
        authhandler = urllib2.HTTPBasicAuthHandler()
        cookiehandler = None

        # No need for version checks since HarvestMan install
        # works for Python >=2.4 anyway
        socket.setdefaulttimeout( self._cfg.socktimeout )
        cj = cookielib.MozillaCookieJar()
        cookiehandler = urllib2.HTTPCookieProcessor(cj)

        # HTTP/HTTPS handlers
        if self._cfg.appname == 'Hget':
            httphandler = urllib2.HTTPHandler
            httpshandler = urllib2.HTTPSHandler
        else:
            httphandler = keepalive.HTTPHandler
            httpshandler = urllib2.HTTPSHandler #keepalive.HTTPSHandler
            
        # If we are behing proxies/firewalls
        if self._useproxy:
            if self._proxyauth['http']:
                httpproxystring = "".join(('http://',
                                           self._proxyauth['http'],
                                           '@',
                                           self._proxydict['http']))
            else:
                httpproxystring = "".join(('http://', self._proxydict['http']))

            if self._proxyauth['ftp']:
                ftpproxystring = "".join(('http://',
                                          self._proxyauth['ftp'],
                                          '@',
                                          self._proxydict['ftp']))
            else:
                ftpproxystring = "".join(('http://', self._proxydict['ftp']))

            if self._proxyauth['https']:
                httpsproxystring = "".join(('http://',
                                            self._proxyauth['https'],
                                            '@',
                                            self._proxydict['https']))
            else:
                httpsproxystring = "".join(('http://', self._proxydict['https']))

            # Set this as the new entry in the proxy dictionary
            self._proxydict['http'] = httpproxystring
            self._proxydict['ftp'] = ftpproxystring
            self._proxydict['https'] = httpsproxystring

            
            proxy_support = urllib2.ProxyHandler(self._proxydict)
            
            # build opener and install it
            if self._initssl:
                opener = urllib2.build_opener(authhandler,
                                              urllib2.HTTPRedirectHandler,
                                              proxy_support,
                                              httphandler,
                                              urllib2.HTTPDefaultErrorHandler,
                                              urllib2.FTPHandler,
                                              #urllib2.GopherHandler,
                                              httpshandler,
                                              urllib2.FileHandler,
                                              cookiehandler)
            else:
                opener = urllib2.build_opener(authhandler,
                                              urllib2.HTTPRedirectHandler,
                                              proxy_support,
                                              httphandler,
                                              urllib2.HTTPDefaultErrorHandler,
                                              urllib2.FTPHandler,
                                              #urllib2.GopherHandler,
                                              urllib2.FileHandler,
                                              cookiehandler)

        else:
            # Direct connection to internet
            if self._initssl:
                opener = urllib2.build_opener(authhandler,
                                              urllib2.HTTPRedirectHandler,
                                              httphandler,
                                              urllib2.FTPHandler,
                                              httpshandler,
                                              #urllib2.GopherHandler,
                                              urllib2.FileHandler,
                                              urllib2.HTTPDefaultErrorHandler,
                                              cookiehandler)
                opener.addheaders=[] #Need to clear default headers so we can apply our own
            else:
                opener = urllib2.build_opener( authhandler,
                                               urllib2.HTTPRedirectHandler,
                                               httphandler,