emailtokenizer.java

spam source codejasen-0.9jASEN - java Anti Spam ENgine.zip 如标题所示

/*
 * @(#)EmailTokenizer.java 5/11/2004
 * 
 * Copyright (c) 2004, 2005  jASEN.org
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *   1. Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in
 *      the documentation and/or other materials provided with the distribution.
 *
 *   3. The names of the authors may not be used to endorse or promote products
 *      derived from this software without specific prior written permission.
 *
 *   4. Any modification or additions to the software must be contributed back
 *      to the project.
 *
 *   5. Any investigation or reverse engineering of source code or binary to
 *      enable emails to bypass the filters, and hence inflict spam and or viruses
 *      onto users who use or do not use jASEN could subject the perpetrator to
 *      criminal and or civil liability.
 *
 * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESSED OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JASEN.ORG,
 * OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
 * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */
package org.jasen.core.token;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Arrays;
import java.util.List;

import javax.mail.Header;
import javax.mail.MessagingException;
import javax.mail.internet.MimeMessage;

import org.jasen.core.linguistics.LinguisticAnalyzer;
import org.jasen.core.parsers.StandardMimeMessageParser;
import org.jasen.core.parsers.URLParser;
import org.jasen.error.JasenException;
import org.jasen.interfaces.JasenMessage;
import org.jasen.interfaces.MimeMessageTokenizer;
import org.jasen.interfaces.ParserData;
import org.jasen.interfaces.TokenErrorRecorder;
import org.jasen.util.MimeUtils;

import com.sun.mail.smtp.SMTPMessage;


/**
 *
 * <P>
 *  Converts the subject, text and html parts of a MimeMessage into discrete String "tokens".
 * </P>
 * <p>
 * 	Each token represents either a word, or a specialized representation of certain key information.
 * </p>
 * <p>
 * 	For example:
 * </p>
 * <p>
 * 	Often the subject line in a message is all that is required to identify it as spam.  This can be a very
 *  good source of information because it will almost always be free from obfuscation (not withstanding the use of non-ascii characters).
 * 	Hence, tokens found in the subject are annotated with the word "Subject" and delimited with a question mark.
 * </p>
 * <p>
 * 	For example:
 * </p>
 * The subject line "Buy viagra!" would be tokenized as:
 * <br/><br/>
 * Subject?Buy<br/>
 * Subject?viagra!
 * <br/><br/>
 * @author Jason Polites
 */
public class EmailTokenizer implements MimeMessageTokenizer
{

    private SpamTokenizer tokenizer;

    private boolean ignoreHeaders = false;

    // This can optionally limit the number of tokens extracted
    protected int tokenLimit = 20;
    protected int linguisticLimit = 3;

    /**
     * This is just a rare character user to identify mail header tokens It
     * looks like two pipes ||
     */
    public static final char HEADER_TOKEN_DELIMITER = 0x01C1;

    // If we are ignoring headers, ignore these
    /**
     * @deprecated This should be done in config
     */
    public static String[] IGNORED_HEADERS = { "thread-index", "date", "content-class", "content-type", "received", "mime-version" };

    // If we are including headers (ignoring everything else) include these
    /**
     * @deprecated This should be done in config
     */
    public static String[] INCLUDED_HEADERS = { "subject", "from", "to", "cc", "bcc", "return-path" };

    static
    {
        Arrays.sort (IGNORED_HEADERS);
        Arrays.sort (INCLUDED_HEADERS);
    }

    public EmailTokenizer() throws IOException {
        tokenizer = new SpamTokenizer ();
        tokenizer.maxTokens=tokenLimit;
        tokenizer.linguisticLimit = linguisticLimit;
    }

    protected String[] tokenize(MimeMessage mail, String realHtml, String realText, String text, String html, String tokenizee, TokenErrorRecorder recorder) throws JasenException {

        String tokenPrefix = null;

        String[] tokens = null;
        String[] tokensTemp = null;
        String[] headerTokens = null;
        Header[] headers = null;

        try
        {
            // Build a large String with the text OR HTML parts

            // ... and tokenize
            tokens = tokenizer.tokenize (tokenizee, recorder);

            // Get the headers
            headers = MimeUtils.getAllHeaders(mail);

            String headerName = null;
            int atIndex = 0;

            for (int h = 0; h < headers.length; h++)
            {

                headerName = headers[h].getName ().toLowerCase();

                headerTokens = null;

                if (ignoreHeaders)
                {
                    if (includeHeader (headerName))
                    {
                        text = headers[h].getValue ();
                        headerTokens = tokenizer.tokenize (text, recorder);
                    }
                }
                else if (!ignoreHeader (headerName))
                {

                    // If the header is the Message-ID header, we want to split it
                    // This is because the first part of the message id has no
                    // meaning
                    text = headers[h].getValue ();

                    if (headerName.equalsIgnoreCase ("Message-ID"))
                    {
                        // Split the value on the @ discarding preceding characters
                        atIndex = text.indexOf ('@');

                        if (atIndex > -1)
                        {
                            text = text.substring (atIndex, text.length ());
                        }

                        headerTokens = tokenizer.tokenize (text, true, recorder);
                    }
                    else if (headerName.equalsIgnoreCase ("Received"))
                    {
                        // Otherwise we want to tokenize the value as usual
                        headerTokens = tokenizer.tokenize (text, true, recorder);
                    }
                    else
                    {
                        // Otherwise we want to tokenize the value as usual
                        headerTokens = tokenizer.tokenize (text, recorder);
                    }
                }

                // Now we need to prepend the header identifier to each token
                if (headerTokens != null)
                {

                    if(Arrays.binarySearch(INCLUDED_HEADERS, headerName) > -1) {
                        for (int i = 0; i < headerTokens.length; i++)
                        {
                            headerTokens[i] = HEADER_TOKEN_DELIMITER + headers[h].getName () + HEADER_TOKEN_DELIMITER + headerTokens[i];
                        }
                    }

                    // Now we need to append the current temp array to the master
                    // token array
                    // Use the native array copy method for greater speed
                    if (tokens != null)
                    {
                        tokensTemp = new String[tokens.length + headerTokens.length];
                        System.arraycopy (tokens, 0, tokensTemp, 0, tokens.length);
                        System.arraycopy (headerTokens, 0, tokensTemp, tokens.length, headerTokens.length);

                        // Now point the original token array at the temp array
                        tokens = tokensTemp;
                    }
                    else
                    {
                        tokens = headerTokens;
                    }
                }
            }

            // Now add the url tokens
            if (realHtml == null && text == null)
            {
                return tokens;
            }
            else
            {

                URLParser ext = null;
                List urlList = null;

                if (realHtml != null)
                {
                    ext = new URLParser ();
                    ext.parse (realHtml);
                    urlList = ext.getUrls ();
                }

                if (realText != null)
                {
                    ext = new URLParser ();
                    ext.parse (realText);

                    if (ext.getUrls () != null)
                    {
                        if (urlList == null)
                        {
                            urlList = ext.getUrls ();
                        }
                        else
                        {
                            urlList.addAll (ext.getUrls ());
                        }
                    }
                }