dedupe.java

spam source codejasen-0.9jASEN - java Anti Spam ENgine.zip 如标题所示

/*
 * @(#)Dedupe.java	13/11/2004
 *
 * Copyright (c) 2004, 2005  jASEN.org
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *   1. Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in
 *      the documentation and/or other materials provided with the distribution.
 *
 *   3. The names of the authors may not be used to endorse or promote products
 *      derived from this software without specific prior written permission.
 *
 *   4. Any modification or additions to the software must be contributed back
 *      to the project.
 *
 *   5. Any investigation or reverse engineering of source code or binary to
 *      enable emails to bypass the filters, and hence inflict spam and or viruses
 *      onto users who use or do not use jASEN could subject the perpetrator to
 *      criminal and or civil liability.
 *
 * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESSED OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JASEN.ORG,
 * OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
 * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */
package org.jasen.tools;

import java.io.File;

import org.jasen.util.FileUtils;

/**
 * <P>
 * 	Removes duplicate files from a given folder.
 * </P>
 * <p>
 * 	This is simply a helper tool to be used when training the engine so we can
 * 	<br/>
 * 	ensure the training corpus does not contain duplicate entries.
 * </p>
 * <p>
 * 	This does a pretty rough job of deduping... probably best not use it :)
 * </p>
 * @author Jason Polites
 */
public final class Dedupe
{
    /**
     * Dedupes the files in a folder
     * <br/>
     * See Usage statement for instructions on how to use
     * @param args
     */
    public static void main(String[] args) {
        try
        {
            if(args.length < 1) {
                System.out.println ("Usage: Dedupe <dedupe path> <duplicates path> (Optional)");
            }
            else
            {
                File dupes = null;
                String option = null;
                boolean filename = false;
                File path = null;

                if(args.length >= 1) {

                    option = args[0].trim();

                    if(option.startsWith("-")) {
                        if(option.toLowerCase().endsWith("f")) {
                            filename = true;
                        }

                        if(args.length > 1) {
                            path = new File(args[1]);

                            if(args.length > 2) {
                                dupes = new File(args[2]);
                            }
                        }
                    }
                    else
                    {
                        path = new File(args[0]);

                        if(args.length > 1) {
                            dupes = new File(args[1]);
                        }
                    }
                }


                FileUtils.dedupe(path, dupes, filename);
            }
        }
        catch (Exception e)
        {
            e.printStackTrace ();
        }
    }
}