urllib2.py

mallet是自然语言处理、机器学习领域的一个开源项目。

"""An extensible library for opening URLs using a variety of protocols

The simplest way to use this module is to call the urlopen function,
which accepts a string containing a URL or a Request object (described
below).  It opens the URL and returns the results as file-like
object; the returned object has some extra methods described below.

The OpenerDirectory manages a collection of Handler objects that do
all the actual work.  Each Handler implements a particular protocol or
option.  The OpenerDirector is a composite object that invokes the
Handlers needed to open the requested URL.  For example, the
HTTPHandler performs HTTP GET and POST requests and deals with
non-error returns.  The HTTPRedirectHandler automatically deals with
HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
deals with digest authentication.

urlopen(url, data=None) -- basic usage is that same as original
urllib.  pass the url and optionally data to post to an HTTP URL, and
get a file-like object back.  One difference is that you can also pass
a Request instance instead of URL.  Raises a URLError (subclass of
IOError); for HTTP errors, raises an HTTPError, which can also be
treated as a valid response.

build_opener -- function that creates a new OpenerDirector instance.
will install the default handlers.  accepts one or more Handlers as
arguments, either instances or Handler classes that it will
instantiate.  if one of the argument is a subclass of the default
handler, the argument will be installed instead of the default.

install_opener -- installs a new opener as the default opener.

objects of interest:
OpenerDirector --

Request -- an object that encapsulates the state of a request.  the
state can be a simple as the URL.  it can also include extra HTTP
headers, e.g. a User-Agent.

BaseHandler --

exceptions:
URLError-- a subclass of IOError, individual protocols have their own
specific subclass

HTTPError-- also a valid HTTP response, so you can treat an HTTP error
as an exceptional event or valid response

internals:
BaseHandler and parent
_call_chain conventions

Example usage:

import urllib2

# set up authentication info
authinfo = urllib2.HTTPBasicAuthHandler()
authinfo.add_password('realm', 'host', 'username', 'password')

proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})

# build a new opener that adds authentication and caching FTP handlers
opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)

# install it
urllib2.install_opener(opener)

f = urllib2.urlopen('http://www.python.org/')


"""

# XXX issues:
# If an authentication error handler that tries to perform
# authentication for some reason but fails, how should the error be
# signalled?  The client needs to know the HTTP error code.  But if
# the handler knows that the problem was, e.g., that it didn't know
# that hash algo that requested in the challenge, it would be good to
# pass that information along to the client, too.

# XXX to do:
# name!
# documentation (getting there)
# complex proxies
# abstract factory for opener
# ftp errors aren't handled cleanly
# gopher can return a socket.error
# check digest against correct (i.e. non-apache) implementation

import socket
import httplib
import inspect
import re
import base64
import types
import urlparse
import md5
import mimetypes
import mimetools
import rfc822
import ftplib
import sys
import time
import os
import stat
import gopherlib
import posixpath

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

try:
    import sha
except ImportError:
    # need 1.5.2 final
    sha = None

# not sure how many of these need to be gotten rid of
from urllib import unwrap, unquote, splittype, splithost, \
     addinfourl, splitport, splitgophertype, splitquery, \
     splitattr, ftpwrapper, noheaders

# support for proxies via environment variables
from urllib import getproxies

# support for FileHandler
from urllib import localhost, url2pathname

__version__ = "2.0a1"

_opener = None
def urlopen(url, data=None):
    global _opener
    if _opener is None:
        _opener = build_opener()
    return _opener.open(url, data)

def install_opener(opener):
    global _opener
    _opener = opener

# do these error classes make sense?
# make sure all of the IOError stuff is overridden.  we just want to be
 # subtypes.

class URLError(IOError):
    # URLError is a sub-type of IOError, but it doesn't share any of
    # the implementation.  need to override __init__ and __str__
    def __init__(self, reason):
        self.reason = reason

    def __str__(self):
        return '<urlopen error %s>' % self.reason

class HTTPError(URLError, addinfourl):
    """Raised when HTTP error occurs, but also acts like non-error return"""
    __super_init = addinfourl.__init__

    def __init__(self, url, code, msg, hdrs, fp):
        self.__super_init(fp, hdrs, url)
        self.code = code
        self.msg = msg
        self.hdrs = hdrs
        self.fp = fp
        # XXX
        self.filename = url

    def __str__(self):
        return 'HTTP Error %s: %s' % (self.code, self.msg)

    def __del__(self):
        # XXX is this safe? what if user catches exception, then
        # extracts fp and discards exception?
        if self.fp:
            self.fp.close()

class GopherError(URLError):
    pass


class Request:

    def __init__(self, url, data=None, headers={}):
        # unwrap('<URL:type://host/path>') --> 'type://host/path'
        self.__original = unwrap(url)
        self.type = None
        # self.__r_type is what's left after doing the splittype
        self.host = None
        self.port = None
        self.data = data
        self.headers = {}
        self.headers.update(headers)

    def __getattr__(self, attr):
        # XXX this is a fallback mechanism to guard against these
        # methods getting called in a non-standard order.  this may be
        # too complicated and/or unnecessary.
        # XXX should the __r_XXX attributes be public?
        if attr[:12] == '_Request__r_':
            name = attr[12:]
            if hasattr(Request, 'get_' + name):
                getattr(self, 'get_' + name)()
                return getattr(self, attr)
        raise AttributeError, attr

    def get_method(self):
        if self.has_data():
            return "POST"
        else:
            return "GET"

    def add_data(self, data):
        self.data = data

    def has_data(self):
        return self.data is not None

    def get_data(self):
        return self.data

    def get_full_url(self):
        return self.__original

    def get_type(self):
        if self.type is None:
            self.type, self.__r_type = splittype(self.__original)
            if self.type is None:
                raise ValueError, "unknown url type: %s" % self.__original
        return self.type

    def get_host(self):
        if self.host is None:
            self.host, self.__r_host = splithost(self.__r_type)
            if self.host:
                self.host = unquote(self.host)
        return self.host

    def get_selector(self):
        return self.__r_host

    def set_proxy(self, host, type):
        self.host, self.type = host, type
        self.__r_host = self.__original

    def add_header(self, key, val):
        # useful for something like authentication
        self.headers[key] = val

class OpenerDirector:
    def __init__(self):
        server_version = "Python-urllib/%s" % __version__
        self.addheaders = [('User-agent', server_version)]
        # manage the individual handlers
        self.handlers = []
        self.handle_open = {}
        self.handle_error = {}

    def add_handler(self, handler):
        added = 0
        for meth in dir(handler):
            if meth[-5:] == '_open':
                protocol = meth[:-5]
                if self.handle_open.has_key(protocol):
                    self.handle_open[protocol].append(handler)
                else:
                    self.handle_open[protocol] = [handler]
                added = 1
                continue
            i = meth.find('_')
            j = meth[i+1:].find('_') + i + 1
            if j != -1 and meth[i+1:j] == 'error':
                proto = meth[:i]
                kind = meth[j+1:]
                try:
                    kind = int(kind)
                except ValueError:
                    pass
                dict = self.handle_error.get(proto, {})
                if dict.has_key(kind):
                    dict[kind].append(handler)
                else:
                    dict[kind] = [handler]
                self.handle_error[proto] = dict
                added = 1
                continue
        if added:
            self.handlers.append(handler)
            handler.add_parent(self)

    def __del__(self):
        self.close()

    def close(self):
        for handler in self.handlers:
            handler.close()
        self.handlers = []

    def _call_chain(self, chain, kind, meth_name, *args):
        # XXX raise an exception if no one else should try to handle
        # this url.  return None if you can't but someone else could.
        handlers = chain.get(kind, ())
        for handler in handlers:
            func = getattr(handler, meth_name)

            result = func(*args)
            if result is not None:
                return result

    def open(self, fullurl, data=None):
        # accept a URL or a Request object
        if isinstance(fullurl, (types.StringType, types.UnicodeType)):
            req = Request(fullurl, data)
        else:
            req = fullurl
            if data is not None:
                req.add_data(data)
        assert isinstance(req, Request) # really only care about interface

        result = self._call_chain(self.handle_open, 'default',
                                  'default_open', req)
        if result:
            return result

        type_ = req.get_type()
        result = self._call_chain(self.handle_open, type_, type_ + \
                                  '_open', req)
        if result:
            return result

        return self._call_chain(self.handle_open, 'unknown',
                                'unknown_open', req)

    def error(self, proto, *args):
        if proto in ['http', 'https']:
            # XXX http[s] protocols are special-cased
            dict = self.handle_error['http'] # https is not different than http
            proto = args[2]  # YUCK!
            meth_name = 'http_error_%d' % proto
            http_err = 1
            orig_args = args
        else:
            dict = self.handle_error
            meth_name = proto + '_error'
            http_err = 0
        args = (dict, proto, meth_name) + args
        result = self._call_chain(*args)
        if result:
            return result

        if http_err:
            args = (dict, 'default', 'http_error_default') + orig_args
            return self._call_chain(*args)

# XXX probably also want an abstract factory that knows things like
 # the fact that a ProxyHandler needs to get inserted first.
# would also know when it makes sense to skip a superclass in favor of
 # a subclass and when it might make sense to include both

def build_opener(*handlers):
    """Create an opener object from a list of handlers.

    The opener will use several default handlers, including support
    for HTTP and FTP.  If there is a ProxyHandler, it must be at the
    front of the list of handlers.  (Yuck.)

    If any of the handlers passed as arguments are subclasses of the
    default handlers, the default handlers will not be used.
    """

    opener = OpenerDirector()
    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
                       FTPHandler, FileHandler]
    if hasattr(httplib, 'HTTPS'):
        default_classes.append(HTTPSHandler)
    skip = []
    for klass in default_classes:
        for check in handlers:
            if inspect.isclass(check):
                if issubclass(check, klass):