logreader.java

Heritrix是一个开源,可扩展的web爬虫项目。Heritrix设计成严格按照robots.txt文件的排除指示和META robots标签。

/* Copyright (C) 2003 Internet Archive.
 *
 * This file is part of the Heritrix web crawler (crawler.archive.org).
 *
 * Heritrix is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * any later version.
 *
 * Heritrix is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser Public License for more details.
 *
 * You should have received a copy of the GNU Lesser Public License
 * along with Heritrix; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
package org.archive.crawler.util;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.LinkedList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.archive.crawler.framework.CrawlController;
import org.archive.io.CompositeFileReader;
import org.archive.util.ArchiveUtils;

/**
 * This class contains a variety of methods for reading log files (or other text 
 * files containing repeated lines with similar information).
 * <p>
 * All methods are static.
 *
 * @author Kristinn Sigurdsson
 */

public class LogReader
{
    /**
     * Returns the entire file. Useful for smaller files.
     *
     * @param aFileName a file name
     * @return The String representation of the entire file.
     *         Null is returned if errors occur (file not found or io exception)
     */
    public static String get(String aFileName){
        try {
            return get(new FileReader(aFileName));
        } catch (FileNotFoundException e) {
            e.printStackTrace();
            return null;
        }
    }

    /**
     * Reads entire contents of reader, returns as string.
     *
     * @param reader
     * @return String of entire contents; null for any error.
     */
    public static String get(InputStreamReader reader){
        StringBuffer ret = new StringBuffer();
        try{
            BufferedReader bf = new BufferedReader(reader, 8192);

            String line = null;
            while ((line = bf.readLine()) != null) {
                ret.append(line);
                ret.append("\n");
            }
        } catch(IOException e){
            e.printStackTrace();
            return null;
        }
        return ret.toString();
    }

    /**
     * Gets a portion of a log file. Starting at a given line number and the n-1
     * lines following that one or until the end of the log if that is reached
     * first.
     *
     * @param aFileName The filename of the log/file
     * @param lineNumber The number of the first line to get (if larger then the 
     *                   file an empty string will be returned)
     * @param n How many lines to return (total, including the one indicated by 
     *                   lineNumber). If smaller then 1 then an empty string 
     *                   will be returned.
     *
     * @return An array of two strings is returned. At index 0 a portion of the
     *         file starting at lineNumber and reaching lineNumber+n is located.
     *         At index 1 there is an informational string about how large a
     *         segment of the file is being returned.
     *         Null is returned if errors occur (file not found or io exception)
     */
    public static String[] get(String aFileName, int lineNumber, int n)
    {
        File f = new File(aFileName);
        long logsize = f.length();
        try {
            return get(new FileReader(aFileName),lineNumber,n,logsize);
        } catch (FileNotFoundException e) {
            e.printStackTrace();
            return null;
        }
    }

    /**
     * Gets a portion of a log spread across a numbered series of files.
     *
     * Starting at a given line number and the n-1 lines following that
     * one or until the end of the log if that is reached
     * first.
     *
     * @param aFileName The filename of the log/file
     * @param lineNumber The number of the first line to get (if larger then the
     *                   file an empty string will be returned)
     * @param n How many lines to return (total, including the one indicated by 
     *                   lineNumber). If smaller then 1 then an empty string 
     *                   will be returned.
     *
     * @return An array of two strings is returned. At index 0 a portion of the
     *         file starting at lineNumber and reaching lineNumber+n is located.
     *         At index 1 there is an informational string about how large a
     *         segment of the file is being returned.
     *         Null is returned if errors occur (file not found or io exception)
     */
    public static String[] getFromSeries(String aFileName, int lineNumber, int n)
    {
        File f = new File(aFileName);
        long logsize = f.length();
        try {
            return get(seriesReader(aFileName),lineNumber,n,logsize);
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }

    public static String buildDisplayingHeader(int len, long logsize)
    {
        double percent = 0.0;
        if (logsize != 0) {
            percent = ((double) len/logsize) * 100;
        }
        return "Displaying: " + ArchiveUtils.doubleToString(percent,1) +
            "% of " + ArchiveUtils.formatBytesForDisplay(logsize);
    }

    /**
     * Gets a portion of a log file. Starting at a given line number and the n-1
     * lines following that one or until the end of the log if that is reached
     * first.
     *
     * @param reader source to scan for lines
     * @param lineNumber The number of the first line to get (if larger then the
     *                   file an empty string will be returned)
     * @param n How many lines to return (total, including the one indicated by
     *                   lineNumber). If smaller then 1 then an empty string
     *                   will be returned.
     *
     * @param logsize total size of source
     * @return An array of two strings is returned. At index 0 a portion of the
     *         file starting at lineNumber and reaching lineNumber+n is located.
     *         At index 1 there is an informational string about how large a
     *         segment of the file is being returned.
     *         Null is returned if errors occur (file not found or io exception)
     */
    public static String[] get(InputStreamReader reader, 
                               int lineNumber, 
                               int n, 
                               long logsize)
    {
        StringBuffer ret = new StringBuffer();
        String info = null;
        try{
            BufferedReader bf = new BufferedReader(reader, 8192);

            String line = null;
            int i=1;
            while ((line = bf.readLine()) != null) {
                if(i >= lineNumber && i < (lineNumber+n))
                {
                    ret.append(line);
                    ret.append('\n');
                } else if( i >= (lineNumber+n)){
                    break;
                }
                i++;
            }
            info = buildDisplayingHeader(ret.length(), logsize);
        }catch(IOException e){
            e.printStackTrace();
            return null;
        }
        String[] tmp = {ret.toString(),info};
        return tmp;
    }

    /**
     * Return the line number of the first line in the
     * log/file that matches a given regular expression.
     *
     * @param aFileName The filename of the log/file
     * @param regExpr The regular expression that is to be used
     * @return The line number (counting from 1, not zero) of the first line
     *         that matches the given regular expression. -1 is returned if no
     *         line matches the regular expression. -1 also is returned if 
     *         errors occur (file not found, io exception etc.)
     */
    public static int findFirstLineContaining(String aFileName, String regExpr)
    {
        try {
            return findFirstLineContaining(new FileReader(aFileName), regExpr);
        } catch (FileNotFoundException e) {
            e.printStackTrace();
            return -1;
        }
    }

    /**
     * Return the line number of the first line in the
     * log/file that begins with the given string.
     *
     * @param aFileName The filename of the log/file
     * @param prefix The prefix string to match
     * @return The line number (counting from 1, not zero) of the first line
     *         that matches the given regular expression. -1 is returned if no
     *         line matches the regular expression. -1 also is returned if 
     *         errors occur (file not found, io exception etc.)
     */
    public static int findFirstLineBeginningFromSeries(String aFileName, 
                                                        String prefix)
    {
        try {
            return findFirstLineBeginning(seriesReader(aFileName), prefix);
        } catch (IOException e) {
            e.printStackTrace();
            return -1;
        }
    }
    
    /**
     * Return the line number of the first line in the
     * log/file that that begins with the given string.
     *
     * @param reader The reader of the log/file
     * @param prefix The prefix string to match
     * @return The line number (counting from 1, not zero) of the first line
     *         that matches the given regular expression. -1 is returned if no
     *         line matches the regular expression. -1 also is returned if 
     *         errors occur (file not found, io exception etc.)
     */
    public static int findFirstLineBeginning(InputStreamReader reader, 
                                              String prefix)
    {

        try{
            BufferedReader bf = new BufferedReader(reader, 8192);

            String line = null;
            int i = 1;
            while ((line = bf.readLine()) != null) {
                if(line.startsWith(prefix)){
                    // Found a match
                    return i;
                }
                i++;
            }
        } catch(IOException e){
            e.printStackTrace();
        }
        return -1;
    }
    
    /**
     * Return the line number of the first line in the
     * log/file that matches a given regular expression.
     *
     * @param aFileName The filename of the log/file
     * @param regExpr The regular expression that is to be used
     * @return The line number (counting from 1, not zero) of the first line
     *         that matches the given regular expression. -1 is returned if no
     *         line matches the regular expression. -1 also is returned if