logreader.java

Heritrix是一个开源,可扩展的web爬虫项目。Heritrix设计成严格按照robots.txt文件的排除指示和META robots标签。

     *         errors occur (file not found, io exception etc.)
     */
    public static int findFirstLineContainingFromSeries(String aFileName, 
                                                        String regExpr)
    {
        try {
            return findFirstLineContaining(seriesReader(aFileName), regExpr);
        } catch (IOException e) {
            e.printStackTrace();
            return -1;
        }
    }

    /**
     * Return the line number of the first line in the
     * log/file that matches a given regular expression.
     *
     * @param reader The reader of the log/file
     * @param regExpr The regular expression that is to be used
     * @return The line number (counting from 1, not zero) of the first line
     *         that matches the given regular expression. -1 is returned if no
     *         line matches the regular expression. -1 also is returned if 
     *         errors occur (file not found, io exception etc.)
     */
    public static int findFirstLineContaining(InputStreamReader reader, 
                                              String regExpr)
    {
        Pattern p = Pattern.compile(regExpr);

        try{
            BufferedReader bf = new BufferedReader(reader, 8192);

            String line = null;
            int i = 1;
            while ((line = bf.readLine()) != null) {
                if(p.matcher(line).matches()){
                    // Found a match
                    return i;
                }
                i++;
            }
        } catch(IOException e){
            e.printStackTrace();
        }
        return -1;
    }

    /**
     * Returns all lines in a log/file matching a given regular expression.  
     * Possible to get lines immediately following the matched line.  Also 
     * possible to have each line prepended by it's line number.
     *
     * @param aFileName The filename of the log/file
     * @param regExpr The regular expression that is to be used
     * @param addLines How many lines (in addition to the matched line) to add. 
     *                 A value less then 1 will mean that only the matched line 
     *                 will be included. If another matched line is hit before 
     *                 we reach this limit it will be included and this counter
     *                 effectively reset for it.
     * @param prependLineNumbers If true, then each line will be prepended by 
     *                           it's line number in the file.
     * @param skipFirstMatches The first number of matches up to this value will
     *                         be skipped over.
     * @param numberOfMatches Once past matches that are to be skipped this many
     *                        matches will be added to the return value. A
     *                        value of 0 will cause all matching lines to be
     *                        included.
     * @return An array of two strings is returned. At index 0 tall lines in a
     *         log/file matching a given regular expression is located.
     *         At index 1 there is an informational string about how large a
     *         segment of the file is being returned.
     *         Null is returned if errors occur (file not found or io exception)
     *         If a PatternSyntaxException occurs, it's error message will be
     *         returned and the informational string will be empty (not null).
     */
    public static String[] getByRegExpr(String aFileName,
                                        String regExpr,
                                        int addLines,
                                        boolean prependLineNumbers,
                                        int skipFirstMatches,
                                        int numberOfMatches) {
        try {
            File f = new File(aFileName);
            return getByRegExpr(
                    new FileReader(f), 
                    regExpr, 
                    addLines, 
                    prependLineNumbers,
                    skipFirstMatches,
                    numberOfMatches,
                    f.length());
        } catch (FileNotFoundException e) {
            e.printStackTrace();
            return null;
        }
    }

    /**
     * Returns all lines in a log/file matching a given regular expression.  
     * Possible to get lines immediately following the matched line.  Also 
     * possible to have each line prepended by it's line number.
     *
     * @param aFileName The filename of the log/file
     * @param regExpr The regular expression that is to be used
     * @param addLines How many lines (in addition to the matched line) to add. 
     *                 A value less then 1 will mean that only the matched line 
     *                 will be included. If another matched line is hit before 
     *                 we reach this limit it will be included and this counter
     *                 effectively reset for it.
     * @param prependLineNumbers If true, then each line will be prepended by 
     *                           it's line number in the file.
     * @param skipFirstMatches The first number of matches up to this value will
     *                         be skipped over.
     * @param numberOfMatches Once past matches that are to be skipped this many
     *                        matches will be added to the return value. A
     *                        value of 0 will cause all matching lines to be
     *                        included.
     * @return An array of two strings is returned. At index 0 tall lines in a
     *         log/file matching a given regular expression is located.
     *         At index 1 there is an informational string about how large a
     *         segment of the file is being returned.
     *         Null is returned if errors occur (file not found or io exception)
     *         If a PatternSyntaxException occurs, it's error message will be
     *         returned and the informational string will be empty (not null).
     */
    public static String[] getByRegExprFromSeries(String aFileName,
                                      String regExpr,
                                      int addLines,
                                      boolean prependLineNumbers,
                                      int skipFirstMatches,
                                      int numberOfMatches) {
        try {
            File f = new File(aFileName);
            return getByRegExpr(
                    seriesReader(aFileName), 
                    regExpr, 
                    addLines, 
                    prependLineNumbers,
                    skipFirstMatches,
                    numberOfMatches,
                    f.length());
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }

    /**
     * Returns all lines in a log/file matching a given regular expression.  
     * Possible to get lines immediately following the matched line.  Also 
     * possible to have each line prepended by it's line number.
     *
     * @param reader The reader of the log/file
     * @param regExpr The regular expression that is to be used
     * @param addLines How many lines (in addition to the matched line) to add. 
     *                 A value less then 1 will mean that only the matched line 
     *                 will be included. If another matched line is hit before 
     *                 we reach this limit it will be included and this counter
     *                 effectively reset for it.
     * @param prependLineNumbers If true, then each line will be prepended by 
     *                           it's line number in the file.
     * @param skipFirstMatches The first number of matches up to this value will
     *                         be skipped over.
     * @param numberOfMatches Once past matches that are to be skipped this many
     *                        matches will be added to the return value. A
     *                        value of 0 will cause all matching lines to be
     *                        included.
     * @param logsize Size of the log in bytes
     * @return An array of two strings is returned. At index 0 all lines in a
     *         log/file matching a given regular expression is located.
     *         At index 1 there is an informational string about how large a
     *         segment of the file is being returned.
     *         Null is returned if errors occur (file not found or io exception)
     *         If a PatternSyntaxException occurs, it's error message will be
     *         returned and the informational string will be empty (not null).
     */
    public static String[] getByRegExpr(InputStreamReader reader,
                                      String regExpr,
                                      int addLines,
                                      boolean prependLineNumbers,
                                      int skipFirstMatches,
                                      int numberOfMatches,
                                      long logsize) {
        StringBuffer ret = new StringBuffer();
        String info = "";

        try{
            Pattern p = Pattern.compile(regExpr);
            BufferedReader bf = new BufferedReader(reader, 8192);

            String line = null;
            int i = 1;
            boolean doAdd = false;
            int addCount = 0;
            long linesMatched = 0;
            while ((line = bf.readLine()) != null) {
                if(p.matcher(line).matches()){
                    // Found a match
                    if(numberOfMatches > 0 &&
                            linesMatched >= skipFirstMatches + numberOfMatches){
                        // Ok, we are done.
                        break;
                    }
                    linesMatched++;
                    if(linesMatched > skipFirstMatches){
                        if(prependLineNumbers){
                            ret.append(i);
                            ret.append(". ");
                        }
                        ret.append(line);
                        ret.append("\n");
                        doAdd = true;
                        addCount = 0;
                    }
                } else if(doAdd) {
                    if(addCount < addLines){
                        //Ok, still within addLines
                        linesMatched++;
                        if(prependLineNumbers){
                            ret.append(i);
                            ret.append(". ");
                        }
                        ret.append(line);
                        ret.append("\n");
                    }else{
                        doAdd = false;
                        addCount = 0;
                    }
                }
                i++;
            }
            info = buildDisplayingHeader(ret.length(), logsize);
        }catch(FileNotFoundException e){
            return null;
        }catch(IOException e){
            e.printStackTrace();
            return null;
        }catch(PatternSyntaxException e){
            ret = new StringBuffer(e.getMessage());
        }
        String[] tmp = {ret.toString(),info};
        return tmp;
    }

    /**
     * Returns all lines in a log/file matching a given regular expression.  
     * Possible to get lines immediately following the matched line.  Also 
     * possible to have each line prepended by it's line number.
     *
     * @param aFileName The filename of the log/file
     * @param regExpr The regular expression that is to be used
     * @param addLines Any lines following a match that <b>begin</b> with this 
     *                 string will also be included. We will stop including new 
     *                 lines once we hit the first that does not match.
     * @param prependLineNumbers If true, then each line will be prepended by 
     *                           it's line number in the file.
     * @param skipFirstMatches The first number of matches up to this value will
     *                         be skipped over.
     * @param numberOfMatches Once past matches that are to be skipped this many
     *                        matches will be added to the return value. A
     *                        value of 0 will cause all matching lines to be
     *                        included.
     * @return An array of two strings is returned. At index 0 tall lines in a
     *         log/file matching a given regular expression is located.
     *         At index 1 there is an informational string about how large a
     *         segment of the file is being returned.
     *         Null is returned if errors occur (file not found or io exception)
     *         If a PatternSyntaxException occurs, it's error message will be
     *         returned and the informational string will be empty (not null).
     */
    public static String[] getByRegExpr(String aFileName, 
                                        String regExpr, 
                                        String addLines, 
                                        boolean prependLineNumbers,
                                        int skipFirstMatches,
                                        int numberOfMatches){
        try {
            File f = new File(aFileName);
            return getByRegExpr(
                    new FileReader(f),
                    regExpr,
                    addLines,
                    prependLineNumbers,
                    skipFirstMatches,
                    numberOfMatches,
                    f.length());
        } catch (FileNotFoundException e) {
            e.printStackTrace();
            return null;
        }
    }

    /**
     * Returns all lines in a log/file matching a given regular expression.  
     * Possible to get lines immediately following the matched line.  Also