fileutils.java

这是个爬虫和lucece相结合最好了

/* FileUtils
 *
 * $Id: FileUtils.java 4936 2007-02-23 02:19:00Z gojomo $
 *
 * Created on Feb 2, 2004
 *
 * Copyright (C) 2004 Internet Archive.
 *
 * This file is part of the Heritrix web crawler (crawler.archive.org).
 *
 * Heritrix is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * any later version.
 *
 * Heritrix is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser Public License for more details.
 *
 * You should have received a copy of the GNU Lesser Public License
 * along with Heritrix; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
package org.archive.util;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.channels.FileChannel;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;


/** Utility methods for manipulating files and directories.
 *
 * @author John Erik Halse
 */
public class FileUtils {
    private static final Logger LOGGER =
        Logger.getLogger(FileUtils.class.getName());
    
    public static final File TMPDIR =
        new File(System.getProperty("java.io.tmpdir", "/tmp"));
    
    private static final boolean DEFAULT_OVERWRITE = true;
    
    /**
     * Constructor made private because all methods of this class are static.
     */
    private FileUtils() {
        super();
    }
    
    public static int copyFiles(final File srcDir, Set srcFile,
            final File dest)
    throws IOException {
        int count = 0;
        for (Iterator i = srcFile.iterator(); i.hasNext();) {
            String name = (String)i.next();
            File src = new File(srcDir, name);
            File tgt = new File(dest, name);
            if (LOGGER.isLoggable(Level.FINE)) {
                LOGGER.fine("Before " + src.getAbsolutePath() + " " +
                    src.exists() + ", " + tgt.getAbsolutePath() + " " +
                    tgt.exists());
            }
            copyFiles(src, tgt);
            if (LOGGER.isLoggable(Level.FINE)) {
                LOGGER.fine("After " + src.getAbsolutePath() + " " +
                    src.exists() + ", " + tgt.getAbsolutePath() + " " +
                    tgt.exists());
            }
            count++;
        }
        return count;
    }

    /** Recursively copy all files from one directory to another.
     *
     * @param src file or directory to copy from.
     * @param dest file or directory to copy to.
     * @throws IOException
     */
    public static void copyFiles(File src, File dest)
    throws IOException {
        copyFiles(src, null, dest, false, true);
    }
    
    /**
     * @param src Directory of files to fetch.
     * @param filter Filter to apply to filenames.
     * @return Files in directory sorted.
     */
    public static String [] getSortedDirContent(final File src,
            final FilenameFilter filter) {
        if (!src.exists()) {
            if (LOGGER.isLoggable(Level.FINE)) {
                LOGGER.fine(src.getAbsolutePath() + " does not exist");
            }
            return null;
        }
       
        if (!src.isDirectory()) {
            if (LOGGER.isLoggable(Level.FINE)) {
                LOGGER.fine(src.getAbsolutePath() + " is not directory.");
            }
            return null;
        }
        // Go through the contents of the directory
        String [] list = (filter == null)? src.list(): src.list(filter);
        if (list != null) {
            Arrays.sort(list);
        }
        return list;
    }
        
    /**
     * Recursively copy all files from one directory to another.
     * 
     * @param src File or directory to copy from.
     * @param filter Filename filter to apply to src. May be null if no
     * filtering wanted.
     * @param dest File or directory to copy to.
     * @param inSortedOrder Copy in order of natural sort.
     * @param overwrite If target file already exits, and this parameter is
     * true, overwrite target file (We do this by first deleting the target
     * file before we begin the copy).
     * @throws IOException
     */
    public static void copyFiles(final File src, final FilenameFilter filter,
        final File dest, final boolean inSortedOrder, final boolean overwrite)
    throws IOException {
        // TODO: handle failures at any step
        if (!src.exists()) {
            if (LOGGER.isLoggable(Level.FINE)) {
                LOGGER.fine(src.getAbsolutePath() + " does not exist");
            }
            return;
        }

        if (src.isDirectory()) {
            if (LOGGER.isLoggable(Level.FINE)) {
                LOGGER.fine(src.getAbsolutePath() + " is a directory.");
            }
            // Create destination directory
            if (!dest.exists()) {
                dest.mkdirs();
            }
            // Go through the contents of the directory
            String list[] = (filter == null)? src.list(): src.list(filter);
            if (inSortedOrder) {
                Arrays.sort(list);
            }
            for (int i = 0; i < list.length; i++) {
                copyFiles(new File(src, list[i]), filter,
                    new File(dest, list[i]), inSortedOrder, overwrite);
            }
        } else {
            copyFile(src, dest, overwrite);
        }
    }

    /**
     * Copy the src file to the destination.
     * 
     * @param src
     * @param dest
     * @return True if the extent was greater than actual bytes copied.
     * @throws FileNotFoundException
     * @throws IOException
     */
    public static boolean copyFile(final File src, final File dest)
    throws FileNotFoundException, IOException {
        return copyFile(src, dest, -1, DEFAULT_OVERWRITE);
    }
    
    /**
     * Copy the src file to the destination.
     * 
     * @param src
     * @param dest
     * @param overwrite If target file already exits, and this parameter is
     * true, overwrite target file (We do this by first deleting the target
     * file before we begin the copy).
     * @return True if the extent was greater than actual bytes copied.
     * @throws FileNotFoundException
     * @throws IOException
     */
    public static boolean copyFile(final File src, final File dest,
        final boolean overwrite)
    throws FileNotFoundException, IOException {
        return copyFile(src, dest, -1, overwrite);
    }
    
    /**
     * Copy up to extent bytes of the source file to the destination
     *
     * @param src
     * @param dest
     * @param extent Maximum number of bytes to copy
     * @return True if the extent was greater than actual bytes copied.
     * @throws FileNotFoundException
     * @throws IOException
     */
    public static boolean copyFile(final File src, final File dest,
        long extent)
    throws FileNotFoundException, IOException {
        return copyFile(src, dest, extent, DEFAULT_OVERWRITE);
    }

	/**
     * Copy up to extent bytes of the source file to the destination
     *
     * @param src
     * @param dest
     * @param extent Maximum number of bytes to copy
	 * @param overwrite If target file already exits, and this parameter is
     * true, overwrite target file (We do this by first deleting the target
     * file before we begin the copy).
	 * @return True if the extent was greater than actual bytes copied.
     * @throws FileNotFoundException
     * @throws IOException
     */
    public static boolean copyFile(final File src, final File dest,
        long extent, final boolean overwrite)
    throws FileNotFoundException, IOException {
        boolean result = false;
        if (LOGGER.isLoggable(Level.FINE)) {
            LOGGER.fine("Copying file " + src + " to " + dest + " extent " +
                extent + " exists " + dest.exists());
        }
        if (dest.exists()) {
            if (overwrite) {
                dest.delete();
                LOGGER.finer(dest.getAbsolutePath() + " removed before copy.");
            } else {
                // Already in place and we're not to overwrite.  Return.
                return result;
            }
        }
        FileInputStream fis = null;
        FileOutputStream fos = null;
        FileChannel fcin = null;
        FileChannel fcout = null;
        try {
            // Get channels
            fis = new FileInputStream(src);
            fos = new FileOutputStream(dest);
            fcin = fis.getChannel();