zurllib.py

来自「这是一个嵌入式linux系统下的命令工具包」· Python 代码 · 共 101 行

# Written by John Hoffman
# see LICENSE.txt for license information

from httplib import HTTPConnection, HTTPSConnection, HTTPException
from urlparse import urlparse
from bencode import bdecode
import socket
from gzip import GzipFile
from StringIO import StringIO
from urllib import quote, unquote
from __init__ import product_name, version_short

VERSION = product_name+'/'+version_short
MAX_REDIRECTS = 10


class btHTTPcon(HTTPConnection): # attempt to add automatic connection timeout
    def connect(self):
        HTTPConnection.connect(self)
        try:
            self.sock.settimeout(30)
        except:
            pass

class btHTTPScon(HTTPSConnection): # attempt to add automatic connection timeout
    def connect(self):
        HTTPSConnection.connect(self)
        try:
            self.sock.settimeout(30)
        except:
            pass 

class urlopen:
    def __init__(self, url):
        self.tries = 0
        self._open(url.strip())
        self.error_return = None

    def _open(self, url):
        self.tries += 1
        if self.tries > MAX_REDIRECTS:
            raise IOError, ('http error', 500,
                            "Internal Server Error: Redirect Recursion")
        (scheme, netloc, path, pars, query, fragment) = urlparse(url)
        if scheme != 'http' and scheme != 'https':
            raise IOError, ('url error', 'unknown url type', scheme, url)
        url = path
        if pars:
            url += ';'+pars
        if query:
            url += '?'+query
#        if fragment:
        try:
            if scheme == 'http':
                self.connection = btHTTPcon(netloc)
            else:
                self.connection = btHTTPScon(netloc)
            self.connection.request('GET', url, None,
                                { 'User-Agent': VERSION,
                                  'Accept-Encoding': 'gzip' } )
            self.response = self.connection.getresponse()
        except HTTPException, e:
            raise IOError, ('http error', str(e))
        status = self.response.status
        if status in (301,302):
            try:
                self.connection.close()
            except:
                pass
            self._open(self.response.getheader('Location'))
            return
        if status != 200:
            try:
                data = self._read()
                d = bdecode(data)
                if d.has_key('failure reason'):
                    self.error_return = data
                    return
            except:
                pass
            raise IOError, ('http error', status, self.response.reason)

    def read(self):
        if self.error_return:
            return self.error_return
        return self._read()

    def _read(self):
        data = self.response.read()
        if self.response.getheader('Content-Encoding','').find('gzip') >= 0:
            try:
                compressed = StringIO(data)
                f = GzipFile(fileobj = compressed)
                data = f.read()
            except:
                raise IOError, ('http error', 'got corrupt response')
        return data

    def close(self):
        self.connection.close()