logreader.java

这是个爬虫和lucece相结合最好了

     * possible to have each line prepended by it's  line number.
     *
     * @param aFileName The filename of the log/file
     * @param regExpr The regular expression that is to be used
     * @param addLines Any lines following a match that <b>begin</b> with this 
     *                 string will also be included. We will stop including new 
     *                 lines once we hit the first that does not match.
     * @param prependLineNumbers If true, then each line will be prepended by 
     *                           it's line number in the file.
     * @param skipFirstMatches The first number of matches up to this value will
     *                         be skipped over.
     * @param numberOfMatches Once past matches that are to be skipped this many
     *                        matches will be added to the return value. A
     *                        value of 0 will cause all matching lines to be
     *                        included.
     * @return An array of two strings is returned. At index 0 tall lines in a
     *         log/file matching a given regular expression is located.
     *         At index 1 there is an informational string about how large a
     *         segment of the file is being returned.
     *         Null is returned if errors occur (file not found or io exception)
     *         If a PatternSyntaxException occurs, it's error message will be
     *         returned and the informational string will be empty (not null).
     */
    public static String[] getByRegExprFromSeries(String aFileName, 
                                                  String regExpr, 
                                                  String addLines, 
                                                  boolean prependLineNumbers,
                                                  int skipFirstMatches,
                                                  int numberOfMatches){
        try {
            File f = new File(aFileName);
            return getByRegExpr(
                    seriesReader(aFileName),
                    regExpr,
                    addLines,
                    prependLineNumbers,
                    skipFirstMatches,
                    numberOfMatches,
                    f.length());
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }

    /**
     * Returns all lines in a log/file matching a given regular expression.  
     * Possible to get lines immediately following the matched line.  Also 
     * possible to have each line prepended by it's line number.
     *
     * @param reader The reader of the log/file
     * @param regExpr The regular expression that is to be used
     * @param addLines Any lines following a match that <b>begin</b> with this 
     *                 string will also be included. We will stop including new 
     *                 lines once we hit the first that does not match.
     * @param prependLineNumbers If true, then each line will be prepended by 
     *                           it's line number in the file.
     * @param skipFirstMatches The first number of matches up to this value will
     *                         be skipped over.
     * @param numberOfMatches Once past matches that are to be skipped this many
     *                        matches will be added to the return value. A
     *                        value of 0 will cause all matching lines to be
     *                        included.
     * @param logsize Size of the log in bytes
     * @return An array of two strings is returned. At index 0 tall lines in a
     *         log/file matching a given regular expression is located.
     *         At index 1 there is an informational string about how large a
     *         segment of the file is being returned.
     *         Null is returned if errors occur (file not found or io exception)
     *         If a PatternSyntaxException occurs, it's error message will be
     *         returned and the informational string will be empty (not null).
     */
    public static String[] getByRegExpr(InputStreamReader reader, 
                                        String regExpr, 
                                        String addLines, 
                                        boolean prependLineNumbers,
                                        int skipFirstMatches,
                                        int numberOfMatches,
                                        long logsize) {
        StringBuffer ret = new StringBuffer();
        String info = "";
        try{
            Matcher m = Pattern.compile(regExpr).matcher("");
            BufferedReader bf = new BufferedReader(reader, 8192);

            String line = null;
            int i = 1;
            boolean doAdd = false;
            long linesMatched = 0;
            while ((line = bf.readLine()) != null) {
                m.reset(line);
                if(m.matches()){
                    // Found a match
                    if(numberOfMatches > 0 && 
                            linesMatched >= skipFirstMatches + numberOfMatches){
                        // Ok, we are done.
                        break;
                    }
                    linesMatched++;
                    if(linesMatched > skipFirstMatches){
                        if(prependLineNumbers){
                            ret.append(i);
                            ret.append(". ");
                        }
                        ret.append(line);
                        ret.append("\n");
                        doAdd = true;
                    }
                } else if(doAdd) {
                    if(line.indexOf(addLines)==0){
                        linesMatched++;
                        //Ok, line begins with 'addLines'
                        if(prependLineNumbers){
                            ret.append(i);
                            ret.append(". ");
                        }
                        ret.append(line);
                        ret.append("\n");
                    }else{
                        doAdd = false;
                    }
                }
                i++;
            }
            info = buildDisplayingHeader(ret.length(), logsize);
        }catch(FileNotFoundException e){
            return null;
        }catch(IOException e){
            e.printStackTrace();
            return null;
        }catch(PatternSyntaxException e){
            ret = new StringBuffer(e.getMessage());
        }
        String[] tmp = {ret.toString(),info};
        return tmp;
    }

    /**
     * Implementation of a unix-like 'tail' command
     *
     * @param aFileName a file name String
     * @return An array of two strings is returned. At index 0 the String
     *         representation of at most 10 last lines is located.
     *         At index 1 there is an informational string about how large a
     *         segment of the file is being returned.
     *         Null is returned if errors occur (file not found or io exception)
     */
    public static String[] tail(String aFileName) {
        return tail(aFileName, 10);
    }

    /**
     * Implementation of a unix-like 'tail -n' command
     *
     * @param aFileName a file name String
     * @param n int number of lines to be returned
     * @return An array of two strings is returned. At index 0 the String
     *         representation of at most n last lines is located.
     *         At index 1 there is an informational string about how large a
     *         segment of the file is being returned.
     *         Null is returned if errors occur (file not found or io exception)
     */
    public static String[] tail(String aFileName, int n) {
        try {
            return tail(new RandomAccessFile(new File(aFileName),"r"),n);
        } catch (FileNotFoundException e) {
            e.printStackTrace();
            return null;
        }
    }

    /**
     * Implementation of a unix-like 'tail -n' command
     *
     * @param raf a RandomAccessFile to tail
     * @param n int number of lines to be returned
     * @return An array of two strings is returned. At index 0 the String
     *         representation of at most n last lines is located.
     *         At index 1 there is an informational string about how large a
     *         segment of the file is being returned.
     *         Null is returned if errors occur (file not found or io exception)
     */
    public static String[] tail(RandomAccessFile raf, int n) {
        int BUFFERSIZE = 1024;
        long pos;
        long endPos;
        long lastPos;
        int numOfLines = 0;
        String info=null;
        byte[] buffer = new byte[BUFFERSIZE];
        StringBuffer sb = new StringBuffer();
        try {
            endPos = raf.length();
            lastPos = endPos;

            // Check for non-empty file
            // Check for newline at EOF
            if (endPos > 0) {
                byte[] oneByte = new byte[1];
                raf.seek(endPos - 1);
                raf.read(oneByte);
                if ((char) oneByte[0] != '\n') {
                    numOfLines++;
                }
            }

            do {
                // seek back BUFFERSIZE bytes
                // if length of the file if less then BUFFERSIZE start from BOF
                pos = 0;
                if ((lastPos - BUFFERSIZE) > 0) {
                    pos = lastPos - BUFFERSIZE;
                }
                raf.seek(pos);
                // If less then BUFFERSIZE avaliable read the remaining bytes
                if ((lastPos - pos) < BUFFERSIZE) {
                    int remainer = (int) (lastPos - pos);
                    buffer = new byte[remainer];
                }
                raf.readFully(buffer);
                // in the buffer seek back for newlines
                for (int i = buffer.length - 1; i >= 0; i--) {
                    if ((char) buffer[i] == '\n') {
                        numOfLines++;
                        // break if we have last n lines
                        if (numOfLines > n) {
                            pos += (i + 1);
                            break;
                        }
                    }
                }
                // reset last postion
                lastPos = pos;
            } while ((numOfLines <= n) && (pos != 0));

            // print last n line starting from last postion
            for (pos = lastPos; pos < endPos; pos += buffer.length) {
                raf.seek(pos);
                if ((endPos - pos) < BUFFERSIZE) {
                    int remainer = (int) (endPos - pos);
                    buffer = new byte[remainer];
                }
                raf.readFully(buffer);
                sb.append(new String(buffer));
            }

            info = buildDisplayingHeader(sb.length(), raf.length());
        } catch (FileNotFoundException e) {
            sb = null;
        } catch (IOException e) {
            e.printStackTrace();
            sb = null;
        } finally {
            try {
                if (raf != null) {
                    raf.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        if(sb==null){
            return null;
        }
        String[] tmp = {sb.toString(),info};
        return tmp;
    }

    /**
     * @param fileName
     * @return
     * @throws IOException
     */
    private static CompositeFileReader seriesReader(String fileName)
    throws IOException {
        LinkedList<File> filenames = new LinkedList<File>();
        int seriesNumber = 1;
        NumberFormat fmt = new DecimalFormat("00000");
        String predecessorFilename =
            fileName.substring(0,fileName.length() 
            - CrawlController.CURRENT_LOG_SUFFIX.length())
            + fmt.format(seriesNumber);
        while((new File(predecessorFilename)).exists()) {
            filenames.add(new File(predecessorFilename));
            seriesNumber++;
            predecessorFilename =
                fileName.substring(0,fileName.length()
                - CrawlController.CURRENT_LOG_SUFFIX.length())
                + fmt.format(seriesNumber);
        }
        filenames.add(new File(fileName)); // add current file
        return new CompositeFileReader(filenames);
    }
}