header.py

mallet是自然语言处理、机器学习领域的一个开源项目。

        if charset is None:
            charset = self._charset
        elif not isinstance(charset, Charset):
            charset = Charset(charset)
        # If the charset is our faux 8bit charset, leave the string unchanged
        if charset <> '8bit':
            # We need to test that the string can be converted to unicode and
            # back to a byte string, given the input and output codecs of the
            # charset.
            if isinstance(s, StringType):
                # Possibly raise UnicodeError if the byte string can't be
                # converted to a unicode with the input codec of the charset.
                incodec = charset.input_codec or 'us-ascii'
                ustr = unicode(s, incodec, errors)
                # Now make sure that the unicode could be converted back to a
                # byte string with the output codec, which may be different
                # than the iput coded.  Still, use the original byte string.
                outcodec = charset.output_codec or 'us-ascii'
                ustr.encode(outcodec, errors)
            elif isinstance(s, UnicodeType):
                # Now we have to be sure the unicode string can be converted
                # to a byte string with a reasonable output codec.  We want to
                # use the byte string in the chunk.
                for charset in USASCII, charset, UTF8:
                    try:
                        outcodec = charset.output_codec or 'us-ascii'
                        s = s.encode(outcodec, errors)
                        break
                    except UnicodeError:
                        pass
                else:
                    assert False, 'utf-8 conversion failed'
        self._chunks.append((s, charset))

    def _split(self, s, charset, maxlinelen, splitchars):
        # Split up a header safely for use with encode_chunks.
        splittable = charset.to_splittable(s)
        encoded = charset.from_splittable(splittable, True)
        elen = charset.encoded_header_len(encoded)
        # If the line's encoded length first, just return it
        if elen <= maxlinelen:
            return [(encoded, charset)]
        # If we have undetermined raw 8bit characters sitting in a byte
        # string, we really don't know what the right thing to do is.  We
        # can't really split it because it might be multibyte data which we
        # could break if we split it between pairs.  The least harm seems to
        # be to not split the header at all, but that means they could go out
        # longer than maxlinelen.
        if charset == '8bit':
            return [(s, charset)]
        # BAW: I'm not sure what the right test here is.  What we're trying to
        # do is be faithful to RFC 2822's recommendation that ($2.2.3):
        #
        # "Note: Though structured field bodies are defined in such a way that
        #  folding can take place between many of the lexical tokens (and even
        #  within some of the lexical tokens), folding SHOULD be limited to
        #  placing the CRLF at higher-level syntactic breaks."
        #
        # For now, I can only imagine doing this when the charset is us-ascii,
        # although it's possible that other charsets may also benefit from the
        # higher-level syntactic breaks.
        elif charset == 'us-ascii':
            return self._split_ascii(s, charset, maxlinelen, splitchars)
        # BAW: should we use encoded?
        elif elen == len(s):
            # We can split on _maxlinelen boundaries because we know that the
            # encoding won't change the size of the string
            splitpnt = maxlinelen
            first = charset.from_splittable(splittable[:splitpnt], False)
            last = charset.from_splittable(splittable[splitpnt:], False)
        else:
            # Binary search for split point
            first, last = _binsplit(splittable, charset, maxlinelen)
        # first is of the proper length so just wrap it in the appropriate
        # chrome.  last must be recursively split.
        fsplittable = charset.to_splittable(first)
        fencoded = charset.from_splittable(fsplittable, True)
        chunk = [(fencoded, charset)]
        return chunk + self._split(last, charset, self._maxlinelen, splitchars)

    def _split_ascii(self, s, charset, firstlen, splitchars):
        chunks = _split_ascii(s, firstlen, self._maxlinelen,
                              self._continuation_ws, splitchars)
        return zip(chunks, [charset]*len(chunks))

    def _encode_chunks(self, newchunks, maxlinelen):
        # MIME-encode a header with many different charsets and/or encodings.
        #
        # Given a list of pairs (string, charset), return a MIME-encoded
        # string suitable for use in a header field.  Each pair may have
        # different charsets and/or encodings, and the resulting header will
        # accurately reflect each setting.
        #
        # Each encoding can be email.Utils.QP (quoted-printable, for
        # ASCII-like character sets like iso-8859-1), email.Utils.BASE64
        # (Base64, for non-ASCII like character sets like KOI8-R and
        # iso-2022-jp), or None (no encoding).
        #
        # Each pair will be represented on a separate line; the resulting
        # string will be in the format:
        #
        # =?charset1?q?Mar=EDa_Gonz=E1lez_Alonso?=\n
        #  =?charset2?b?SvxyZ2VuIEL2aW5n?="
        chunks = []
        for header, charset in newchunks:
            if not header:
                continue
            if charset is None or charset.header_encoding is None:
                s = header
            else:
                s = charset.header_encode(header)
            # Don't add more folding whitespace than necessary
            if chunks and chunks[-1].endswith(' '):
                extra = ''
            else:
                extra = ' '
            _max_append(chunks, s, maxlinelen, extra)
        joiner = NL + self._continuation_ws
        return joiner.join(chunks)

    def encode(self, splitchars=';, '):
        """Encode a message header into an RFC-compliant format.

        There are many issues involved in converting a given string for use in
        an email header.  Only certain character sets are readable in most
        email clients, and as header strings can only contain a subset of
        7-bit ASCII, care must be taken to properly convert and encode (with
        Base64 or quoted-printable) header strings.  In addition, there is a
        75-character length limit on any given encoded header field, so
        line-wrapping must be performed, even with double-byte character sets.

        This method will do its best to convert the string to the correct
        character set used in email, and encode and line wrap it safely with
        the appropriate scheme for that character set.

        If the given charset is not known or an error occurs during
        conversion, this function will return the header untouched.

        Optional splitchars is a string containing characters to split long
        ASCII lines on, in rough support of RFC 2822's `highest level
        syntactic breaks'.  This doesn't affect RFC 2047 encoded lines.
        """
        newchunks = []
        maxlinelen = self._firstlinelen
        lastlen = 0
        for s, charset in self._chunks:
            # The first bit of the next chunk should be just long enough to
            # fill the next line.  Don't forget the space separating the
            # encoded words.
            targetlen = maxlinelen - lastlen - 1
            if targetlen < charset.encoded_header_len(''):
                # Stick it on the next line
                targetlen = maxlinelen
            newchunks += self._split(s, charset, targetlen, splitchars)
            lastchunk, lastcharset = newchunks[-1]
            lastlen = lastcharset.encoded_header_len(lastchunk)
        return self._encode_chunks(newchunks, maxlinelen)



def _split_ascii(s, firstlen, restlen, continuation_ws, splitchars):
    lines = []
    maxlen = firstlen
    for line in s.splitlines():
        # Ignore any leading whitespace (i.e. continuation whitespace) already
        # on the line, since we'll be adding our own.
        line = line.lstrip()
        if len(line) < maxlen:
            lines.append(line)
            maxlen = restlen
            continue
        # Attempt to split the line at the highest-level syntactic break
        # possible.  Note that we don't have a lot of smarts about field
        # syntax; we just try to break on semi-colons, then commas, then
        # whitespace.
        for ch in splitchars:
            if line.find(ch) >= 0:
                break
        else:
            # There's nothing useful to split the line on, not even spaces, so
            # just append this line unchanged
            lines.append(line)
            maxlen = restlen
            continue
        # Now split the line on the character plus trailing whitespace
        cre = re.compile(r'%s\s*' % ch)
        if ch in ';,':
            eol = ch
        else:
            eol = ''
        joiner = eol + ' '
        joinlen = len(joiner)
        wslen = len(continuation_ws.replace('\t', SPACE8))
        this = []
        linelen = 0
        for part in cre.split(line):
            curlen = linelen + max(0, len(this)-1) * joinlen
            partlen = len(part)
            onfirstline = not lines
            # We don't want to split after the field name, if we're on the
            # first line and the field name is present in the header string.
            if ch == ' ' and onfirstline and \
                   len(this) == 1 and fcre.match(this[0]):
                this.append(part)
                linelen += partlen
            elif curlen + partlen > maxlen:
                if this:
                    lines.append(joiner.join(this) + eol)
                # If this part is longer than maxlen and we aren't already
                # splitting on whitespace, try to recursively split this line
                # on whitespace.
                if partlen > maxlen and ch <> ' ':
                    subl = _split_ascii(part, maxlen, restlen,
                                        continuation_ws, ' ')
                    lines.extend(subl[:-1])
                    this = [subl[-1]]
                else:
                    this = [part]
                linelen = wslen + len(this[-1])
                maxlen = restlen
            else:
                this.append(part)
                linelen += partlen
        # Put any left over parts on a line by themselves
        if this:
            lines.append(joiner.join(this))
    return lines



def _binsplit(splittable, charset, maxlinelen):
    i = 0
    j = len(splittable)
    while i < j:
        # Invariants:
        # 1. splittable[:k] fits for all k <= i (note that we *assume*,
        #    at the start, that splittable[:0] fits).
        # 2. splittable[:k] does not fit for any k > j (at the start,
        #    this means we shouldn't look at any k > len(splittable)).
        # 3. We don't know about splittable[:k] for k in i+1..j.
        # 4. We want to set i to the largest k that fits, with i <= k <= j.
        #
        m = (i+j+1) >> 1  # ceiling((i+j)/2); i < m <= j
        chunk = charset.from_splittable(splittable[:m], True)
        chunklen = charset.encoded_header_len(chunk)
        if chunklen <= maxlinelen:
            # m is acceptable, so is a new lower bound.
            i = m
        else:
            # m is not acceptable, so final i must be < m.
            j = m - 1
    # i == j.  Invariant #1 implies that splittable[:i] fits, and
    # invariant #2 implies that splittable[:i+1] does not fit, so i
    # is what we're looking for.
    first = charset.from_splittable(splittable[:i], False)
    last  = charset.from_splittable(splittable[i:], False)
    return first, last