imapcollection.java

很好的搜索代码,大家都很难下载!抓紧时间啊!不要错过!

/*
 * Copyright 2003-2004 Michael Franken, Zilverline.
 *
 * The contents of this file, or the files included with this file, are subject to
 * the current version of ZILVERLINE Collaborative Source License for the
 * Zilverline Search Engine (the "License"); You may not use this file except in
 * compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.zilverline.org.
 *
 * See the License for the rights, obligations and
 * limitations governing use of the contents of the file.
 *
 * The Original and Upgraded Code is the Zilverline Search Engine. The developer of
 * the Original and Upgraded Code is Michael Franken. Michael Franken owns the
 * copyrights in the portions it created. All Rights Reserved.
 *
 */

package org.zilverline.core;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Properties;

import javax.activation.DataHandler;
import javax.mail.Address;
import javax.mail.BodyPart;
import javax.mail.FetchProfile;
import javax.mail.Flags;
import javax.mail.Folder;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.NoSuchProviderException;
import javax.mail.Part;
import javax.mail.Session;
import javax.mail.Store;
import javax.mail.UIDFolder;
import javax.mail.internet.MimeMessage;
import javax.mail.internet.MimeMultipart;
import javax.mail.internet.MimePart;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import org.springframework.util.StringUtils;

import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;

import org.zilverline.extractors.HTMLExtractor;
import org.zilverline.util.StopWatch;

/**
 * A Collection is a number of documents in a directory that are indexed together.
 * 
 * @author Michael Franken
 * @version $Revision: 1.15 $
 */
public class IMAPCollection extends AbstractCollection {
    /** logger for Commons logging. */
    private static Log log = LogFactory.getLog(IMAPCollection.class);

    // field names
    private static final String F_CD = "content-description";

    private static final String F_CONTENTS = "contents";

    private static final String F_CT = "content-type";

    private static final String F_FOLDER = "folder";

    private static final String F_FROM = "from";

    private static final String F_RECEIVED = "received";

    private static final String F_REPLY_TO = "reply-to";

    private static final String F_SENT = "sent";

    private static final String F_SIZE = "size";

    private static final String F_SUBJECT = "subject";

    private static final String F_TO = "to";

    private static final String F_UID = "uid";

    private static final String F_URL = "url";

    // msg flags
    private static final Flags.Flag[] FLAGS = new Flags.Flag[] { Flags.Flag.ANSWERED, Flags.Flag.DELETED, Flags.Flag.DRAFT,
        Flags.Flag.FLAGGED, Flags.Flag.RECENT, Flags.Flag.SEEN };

    // no toString() in Flags.Flag :(
    private static final String[] SFLAGS = new String[] { "answered", "deleted", "draft", "flagged", "recent", "seen" };

    private static final FetchProfile PROFILE = new FetchProfile();

    static {
        PROFILE.add(FetchProfile.Item.ENVELOPE); // standard headers
        PROFILE.add(FetchProfile.Item.CONTENT_INFO);
        PROFILE.add(UIDFolder.FetchProfileItem.UID);
        PROFILE.add(com.sun.mail.imap.IMAPFolder.FetchProfileItem.HEADERS);
        PROFILE.add(com.sun.mail.imap.IMAPFolder.FetchProfileItem.SIZE);
    }

    private String folder;

    private String host;

    private String password;

    private String user;

    /**
     * Default Constructor setting all fields to non null defaults.
     */
    public IMAPCollection() {
        super();
        name = "";
        url = "";
        description = "";
        numberOfDocs = 0;
        version = 0;
        lastIndexed = null;
        existsOnDisk = false;
        keepCache = false;
        isKeepCacheSet = false;
    }

    /**
     * Gets the origin from where this collection's documents can be retrieved.
     * 
     * @return location such as e:/docs or InBox
     */
    public final String getRoot() {
        if (!StringUtils.hasText(folder)) {
            return "all folders";
        }
        return getFolder();
    }

    public String getFolder() {
        return folder;
    }

    /**
     * @return Returns the host.
     */
    public String getHost() {
        return host;
    }

    /**
     * @return Returns the password.
     */
    public String getPassword() {
        return password;
    }

    /**
     * @return Returns the user.
     */
    public String getUser() {
        return user;
    }

    /**
     * Index the given Collection.
     * 
     * @param fullIndex indicates whether a full or incremental index should be created
     * @throws IndexException if the Collections can not be indexed
     */
    public final void index(final boolean fullIndex) throws IndexException {
        resetCache(fullIndex);
        doIndex(fullIndex);
    }

    /**
     * Index the given Collection.
     * 
     * @param fullIndex indicates whether a full or incremental index should be created
     * @throws IndexException if the Collections can not be indexed
     * @return true if succesfull
     */
    private final boolean doIndex(boolean fullIndex) throws IndexException {
        IndexWriter writer = null;
        Store store = null;
        try {
            // record start time
            StopWatch watch = new StopWatch();

            watch.start();

            // make sure the index exists
            File indexDirectory = this.getIndexDirWithManagerDefaults();

            // reindex if the index is not there or invalid
            boolean mustReindex = fullIndex;
            if (!this.isIndexValid()) {
                mustReindex = true;
                indexDirectory.mkdirs();
            }

            // create an index(writer)
            writer = new IndexWriter(indexDirectory, this.createAnalyzer(), mustReindex);
            // see whether there are specific indexing settings in manager
            if (manager.getMergeFactor() != null) {
                writer.setMergeFactor(manager.getMergeFactor().intValue());
            }
            if (manager.getMinMergeDocs() != null) {
                writer.setMaxBufferedDocs(manager.getMinMergeDocs().intValue());
            }

            if (manager.getMaxMergeDocs() != null) {
                writer.setMaxMergeDocs(manager.getMaxMergeDocs().intValue());
            }

            resetCache(fullIndex);
            // connect to IMAP
            log.debug("Connecting to IMAP server: " + host);
            Properties props = System.getProperties();
            Session session = Session.getDefaultInstance(props, null);
            store = session.getStore("imap");
            log.debug("Connecting to " + host + " as " + user);
            store.connect(host, user, password);
            log.debug("Connected");
            // start at the proper folder
            Folder topFolder = null;
            if (StringUtils.hasText(folder)) {
                topFolder = store.getFolder(folder);
            } else {
                topFolder = store.getDefaultFolder();
            }
            indexFolder(writer, topFolder);
            // record end time and report duration of indexing
            watch.stop();
            log.info("Indexed " + writer.docCount() + " documents in " + watch.elapsedTime());
            return true;
        }
        catch (NoSuchProviderException e) {
            throw new IndexException("Can't connect to " + host, e);
        }
        catch (MessagingException e) {
            throw new IndexException("Error while accessing IMAP server " + host, e);
        }
        catch (IOException e) {
            throw new IndexException("Error indexing '" + this.getName() + "'. Possibly unable to remove old index", e);
        }
        catch (Exception e) {
            throw new IndexException("Error indexing '" + this.getName() + "'", e);
        }
        finally {
            if (writer != null) {
                try {
                    writer.optimize();
                    log.debug("Optimizing index for " + name);
                    writer.close();
                    log.debug("Closing index for " + name);
                }
                catch (IOException e1) {
                    log.error("Error closing Index for " + name, e1);
                }
            }
            if (store != null) {
                try {
                    store.close();
                }
                catch (MessagingException e1) {
                    log.error("Error closing IMAP server " + host, e1);
                }
            }
            init();
        }
    }

    private final boolean indexFolder(IndexWriter writer, Folder thisFolder) throws MessagingException {
        if (stopRequested) {
            log.info("Indexing stops, due to request");
            return false;
        }
        if ((thisFolder.getType() & Folder.HOLDS_MESSAGES) != 0) {
            thisFolder.open(Folder.READ_ONLY);
            Message[] messages = thisFolder.getMessages(); // get refs to all msgs
            if (messages == null) {
                // dummy
                messages = new Message[0];
            }

            thisFolder.fetch(messages, PROFILE); // fetch headers

            log.debug("FOLDER: " + thisFolder.getFullName() + " messages=" + messages.length);

            for (int i = 0; i < messages.length; i++) {
                try {
                    String msgID = null;
                    if (messages[i] instanceof MimeMessage) {
                        MimeMessage mm = (MimeMessage) messages[i];
                        msgID = mm.getMessageID();
                    }
                    if (!md5DocumentCache.contains(msgID)) {
                        log.debug("new message added for message: " + msgID);
                        final Document doc = new Document();
                        doc.add(Field.Keyword(F_FOLDER, thisFolder.getFullName()));
                        doc.add(Field.Keyword("collection", name));
                        // index this message