sb_pop3dnd.py

用python实现的邮件过滤器

#!/usr/bin/env python

"""POP3DND - provides drag'n'drop training ability for POP3 clients.

This application is a twisted cross between a POP3 proxy and an IMAP
server.  It sits between your mail client and your POP3 server (like any
other POP3 proxy).  While messages classified as ham are simply passed
through the proxy, messages that are classified as spam or unsure are
intercepted and passed to the IMAP server.  The IMAP server offers four
folders - one where messages classified as spam end up, one for messages
it is unsure about, one for training ham, and one for training spam.

In other words, to use this application, setup your mail client to connect
to localhost, rather than directly to your POP3 server.  Additionally, add
a new IMAP account, also connecting to localhost.  Setup the application
via the web interface, and you are ready to go.  Good messages will appear
as per normal, but you will also have two new incoming folders, one for
spam and one for unsure messages.

To train SpamBayes, use the 'train_as_spam' and 'train_as_ham' folders.
Any messages in these folders will be trained appropriately.

This SpamBayes application is designed to work with Outlook Express, and
provide the same sort of ease of use as the Outlook plugin.  Although the
majority of development and testing has been done with Outlook Express,
Eudora and Thunderbird, any mail client that supports both IMAP and POP3
should be able to use this application - if the client enables the user to
work with an IMAP account and POP3 account side-by-side (and move messages
between them), then it should work equally as well.
"""

from __future__ import generators

todo = """
 o The RECENT flag should be unset at some point, but when?  The
   RFC says that a message is recent if this is the first session
   to be notified about the message.  Perhaps this can be done
   simply by *not* persisting this flag - i.e. the flag is always
   loaded as not recent, and only new messages are recent.  The
   RFC says that if it is not possible to determine, then all
   messages should be recent, and this is what we currently do.
 o The Mailbox should be calling the appropriate listener
   functions (currently only newMessages is called on addMessage).
   flagsChanged should also be called on store, addMessage, or ???
 o We cannot currently get part of a message via the BODY calls
   (with the <> operands), or get a part of a MIME message (by
   prepending a number).  This should be added!
 o Suggestions?
"""

# This module is part of the spambayes project, which is Copyright 2002-4
# The Python Software Foundation and is covered by the Python Software
# Foundation license.

__author__ = "Tony Meyer <[e-mail]>"
__credits__ = "All the Spambayes folk."

try:
    True, False
except NameError:
    # Maintain compatibility with Python 2.2
    True, False = 1, 0

import os
import re
import sys
import md5
import time
import errno
import types
import email
import thread
import getopt
import socket
import imaplib
import operator
import email.Utils

try:
    import cStringIO as StringIO
except NameError:
    import StringIO

from twisted import cred
import twisted.application.app
from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet.defer import maybeDeferred
from twisted.internet.protocol import ServerFactory
from twisted.protocols.imap4 import IMessage
from twisted.protocols.imap4 import parseNestedParens, parseIdList
from twisted.protocols.imap4 import IllegalClientResponse, IAccount
from twisted.protocols.imap4 import collapseNestedLists, MessageSet
from twisted.protocols.imap4 import IMAP4Server, MemoryAccount, IMailbox
from twisted.protocols.imap4 import IMailboxListener, collapseNestedLists

from spambayes import storage
from spambayes import message
from spambayes.Stats import Stats
from spambayes.Options import options
from spambayes.tokenizer import tokenize
from spambayes import FileCorpus, Dibbler
from spambayes.Version import get_current_version
from sb_server import POP3ProxyBase, State, _addressPortStr, _recreateState

def ensureDir(dirname):
    """Ensure that the given directory exists - in other words, if it
    does not exist, attempt to create it."""
    try:
        os.mkdir(dirname)
        if options["globals", "verbose"]:
            print "Creating directory", dirname
    except OSError, e:
        if e.errno != errno.EEXIST:
            raise


class IMAPMessage(message.Message):
    '''IMAP Message base class.'''
    __implements__ = (IMessage,)

    def __init__(self, date=None, message_db=None):
        message.Message.__init__(self, message_info_db=message_db)
        # We want to persist more information than the generic
        # Message class.
        self.stored_attributes.extend(["date", "deleted", "flagged",
                                       "seen", "draft", "recent",
                                       "answered"])
        self.date = date
        self.clear_flags()

    # IMessage implementation
    def getHeaders(self, negate, *names):
        """Retrieve a group of message headers."""
        headers = {}
        for header, value in self.items():
            if (header.upper() in names and not negate) or \
               (header.upper() not in names and negate) or names == ():
                headers[header.lower()] = value
        return headers

    def getFlags(self):
        """Retrieve the flags associated with this message."""
        return self._flags_iter()

    def _flags_iter(self):
        if self.deleted:
            yield "\\DELETED"
        if self.answered:
            yield "\\ANSWERED"
        if self.flagged:
            yield "\\FLAGGED"
        if self.seen:
            yield "\\SEEN"
        if self.draft:
            yield "\\DRAFT"
        if self.recent:
            yield "\\RECENT"

    def getInternalDate(self):
        """Retrieve the date internally associated with this message."""
        assert self.date is not None, \
               "Must set date to use IMAPMessage instance."
        return self.date

    def getBodyFile(self):
        """Retrieve a file object containing the body of this message."""
        # Note: only body, not headers!
        s = StringIO.StringIO()
        s.write(self.body())
        s.seek(0)
        return s

    def getSize(self):
        """Retrieve the total size, in octets, of this message."""
        return len(self.as_string())

    def getUID(self):
        """Retrieve the unique identifier associated with this message."""
        return self.id

    def isMultipart(self):
        """Indicate whether this message has subparts."""
        return False

    def getSubPart(self, part):
        """Retrieve a MIME sub-message

        @type part: C{int}
        @param part: The number of the part to retrieve, indexed from 0.

        @rtype: Any object implementing C{IMessage}.
        @return: The specified sub-part.
        """
        raise NotImplementedError

    # IMessage implementation ends

    def clear_flags(self):
        """Set all message flags to false."""
        self.deleted = False
        self.answered = False
        self.flagged = False
        self.seen = False
        self.draft = False
        self.recent = False

    def set_flag(self, flag, value):
        # invalid flags are ignored
        flag = flag.upper()
        if flag == "\\DELETED":
            self.deleted = value
        elif flag == "\\ANSWERED":
            self.answered = value
        elif flag == "\\FLAGGED":
            self.flagged = value
        elif flag == "\\SEEN":
            self.seen = value
        elif flag == "\\DRAFT":
            self.draft = value
        else:
            print "Tried to set invalid flag", flag, "to", value

    def flags(self):
        """Return the message flags."""
        return list(self._flags_iter())

    def train(self, classifier, isSpam):
        if self.GetTrained() == (not isSpam):
            classifier.unlearn(self.tokenize(), not isSpam)
            self.RememberTrained(None)
        if self.GetTrained() is None:
            classifier.learn(self.tokenize(), isSpam)
            self.RememberTrained(isSpam)
        classifier.store()

    def structure(self, ext=False):
        """Body structure data describes the MIME-IMB
        format of a message and consists of a sequence of mime type, mime
        subtype, parameters, content id, description, encoding, and size.
        The fields following the size field are variable: if the mime
        type/subtype is message/rfc822, the contained message's envelope
        information, body structure data, and number of lines of text; if
        the mime type is text, the number of lines of text.  Extension fields
        may also be included; if present, they are: the MD5 hash of the body,
        body disposition, body language."""
        s = []
        for part in self.walk():
            if part.get_content_charset() is not None:
                charset = ("charset", part.get_content_charset())
            else:
                charset = None
            part_s = [part.get_main_type(), part.get_subtype(),
                      charset,
                      part.get('Content-Id'),
                      part.get('Content-Description'),
                      part.get('Content-Transfer-Encoding'),
                      str(len(part.as_string()))]
            #if part.get_type() == "message/rfc822":
            #    part_s.extend([envelope, body_structure_data,
            #                  part.as_string().count("\n")])
            #elif part.get_main_type() == "text":
            if part.get_main_type() == "text":
                part_s.append(str(part.as_string().count("\n")))
            if ext:
                part_s.extend([md5.new(part.as_string()).digest(),
                               part.get('Content-Disposition'),
                               part.get('Content-Language')])
            s.append(part_s)
        if len(s) == 1:
            return s[0]
        return s

    def body(self):
        rfc822 = self.as_string()
        bodyRE = re.compile(r"\r?\n(\r?\n)(.*)",
                            re.DOTALL + re.MULTILINE)
        bmatch = bodyRE.search(rfc822)
        return bmatch.group(2)

    def headers(self):
        rfc822 = self.as_string()
        bodyRE = re.compile(r"\r?\n(\r?\n)(.*)",
                            re.DOTALL + re.MULTILINE)
        bmatch = bodyRE.search(rfc822)
        return rfc822[:bmatch.start(2)]


class DynamicIMAPMessage(IMAPMessage):
    """An IMAP Message that may change each time it is loaded."""
    def __init__(self, func, mdb):
        date = imaplib.Time2Internaldate(time.time())[1:-1]
        IMAPMessage.__init__(self, date, mdb)
        self.func = func
        self.load()
    def load(self):
        # This only works for simple messages (non multi-part).
        self.set_payload(self.func(body=True))
        # This only works for simple headers (no continuations).
        for headerstr in self.func(headers=True).split('\r\n'):
            header, value = headerstr.split(':')
            self[header] = value.strip()


class IMAPFileMessage(IMAPMessage, FileCorpus.FileMessage):
    '''IMAP Message that persists as a file system artifact.'''

    def __init__(self, file_name=None, directory=None, mdb=None):
        """Constructor(message file name, corpus directory name)."""
        date = imaplib.Time2Internaldate(time.time())[1:-1]
        IMAPMessage.__init__(self, date, mdb)
        FileCorpus.FileMessage.__init__(self, file_name, directory)
        self.id = file_name


class IMAPFileMessageFactory(FileCorpus.FileMessageFactory):
    '''MessageFactory for IMAPFileMessage objects'''
    def create(self, key, directory, content=None):
        '''Create a message object from a filename in a directory'''
        if content is None:
            return IMAPFileMessage(key, directory)
        msg = email.message_from_string(content, _class=IMAPFileMessage)
        msg.id = key
        msg.file_name = key
        msg.directory = directory
        return msg


class IMAPMailbox(cred.perspective.Perspective):
    __implements__ = (IMailbox,)

    def __init__(self, name, identity_name, id):
        cred.perspective.Perspective.__init__(self, name, identity_name)
        self.UID_validity = id
        self.listeners = []

    def getUIDValidity(self):
        """Return the unique validity identifier for this mailbox."""
        return self.UID_validity

    def addListener(self, listener):
        """Add a mailbox change listener."""
        self.listeners.append(listener)

    def removeListener(self, listener):
        """Remove a mailbox change listener."""