loganalyser.java

dspace 用j2ee架构的一个数字图书馆.开源程序

/*
 * LogAnalyser.java
 *
 * Version: $Revision: 1.5 $
 *
 * Date: $Date: 2005/12/08 09:24:44 $
 *
 * Copyright (c) 2002-2005, Hewlett-Packard Company and Massachusetts
 * Institute of Technology.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 *
 * - Neither the name of the Hewlett-Packard Company nor the name of the
 * Massachusetts Institute of Technology nor the names of their
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 * DAMAGE.
 */

package org.dspace.app.statistics;

import org.dspace.app.statistics.LogLine;

import org.dspace.core.ConfigurationManager;
import org.dspace.core.Context;
import org.dspace.storage.rdbms.DatabaseManager;
import org.dspace.storage.rdbms.TableRow;

import java.sql.SQLException;

import java.lang.Long;
import java.lang.StringBuffer;

import java.net.InetAddress;
import java.net.UnknownHostException;

import java.text.ParseException;
import java.text.SimpleDateFormat;

import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.StringTokenizer;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;

/**
 * This class performs all the actual analysis of a given set of DSpace log
 * files.  Most input can be configured; use the -help flag for a full list
 * of usage information.
 *
 * The output of this file is plain text and forms an "aggregation" file which
 * can then be used for display purposes using the related ReportGenerator
 * class.
 *
 * @author  Richard Jones
 */
public class LogAnalyser 
{
    
    // set up our class globals
    // FIXME: there are so many of these perhaps they should exist in a static
    // object of their own
    
    /////////////////
    // aggregators
    /////////////////
    
    /** aggregator for all actions performed in the system */
    private static Map actionAggregator = new HashMap();
    
    /** aggregator for all searches performed */
    private static Map searchAggregator = new HashMap();
    
    /** aggregator for user logins */
    private static Map userAggregator = new HashMap();
    
    /** aggregator for item views */
    private static Map itemAggregator = new HashMap();
    
    /** aggregator for current archive state statistics */
    private static Map archiveStats = new HashMap();
    
    /** warning counter */
    private static int warnCount = 0;
    
    /** log line counter */
    private static int lineCount = 0;
        
    //////////////////
    // config data
    //////////////////
    
    /** list of actions to be included in the general summary */
    private static List generalSummary = new ArrayList();
    
    /** list of words not to be aggregated */
    private static List excludeWords = new ArrayList();
    
    /** list of search types to be ignored, such as "author:" */
    private static List excludeTypes = new ArrayList();
    
    /** list of characters to be excluded */
    private static List excludeChars = new ArrayList();
    
    /** list of item types to be reported on in the current state */
    private static List itemTypes = new ArrayList();
    
    /** bottom limit to output for search word analysis */
    private static int searchFloor;
    
    /** bottom limit to output for item view analysis */
    private static int itemFloor;
    
    /** number of items from most popular to be looked up in the database */
    private static int itemLookup;
    
    /** mode to use for user email display */
    private static String userEmail;
    
    /** URL of the service being analysed */
    private static String url;
    
    /** Name of the service being analysed */
    private static String name;
   
    /** Name of the service being analysed */
    private static String hostName;
    
    /** the average number of views per item */
    private static int views = 0;
    
    ///////////////////////
    // regular expressions
    ///////////////////////
   
   /** Exclude characters regular expression pattern */
   private static Pattern excludeCharRX = null;
   
   /** handle indicator string regular expression pattern */
   private static Pattern handleRX = null;
   
   /** item id indicator string regular expression pattern */
   private static Pattern itemRX = null;
  
   /** query string indicator regular expression pattern */
   private static Pattern queryRX = null;
   
   /** collection indicator regular expression pattern */
   private static Pattern collectionRX = null;
   
   /** community indicator regular expression pattern */
   private static Pattern communityRX = null;
   
   /** results indicator regular expression pattern */
   private static Pattern resultsRX = null;
   
   /** single character regular expression pattern */
   private static Pattern singleRX = null;
   
   /** a pattern to match a valid log file line */
   private static Pattern valid = null;
   
   /** pattern to match valid log file names */
   private static Pattern logRegex = null;
   
   /** pattern to match commented out lines from the config file */
   private static Pattern comment = Pattern.compile("^#");
        
   /** pattern to match genuine lines from the config file */
   private static Pattern real = Pattern.compile("^(.+)=(.+)");
   
   /** pattern to match all search types */
   private static Pattern typeRX = null;
   
   /** pattern to match all search types */
   private static Pattern wordRX = null;
   
   //////////////////////////
   // Miscellaneous variables
   //////////////////////////
   
   /** process timing clock */
   private static Calendar startTime = null;
   
   /////////////////////////
   // command line options
   ////////////////////////
   
   /** the log directory to be analysed */
   private static String logDir = ConfigurationManager.getProperty("dspace.dir") + 
                        File.separator + "log";
        
   /** the regex to describe the file name format */
   private static String fileTemplate = "dspace\\.log.*";
        
   /** the config file from which to configure the analyser */
   private static String configFile = ConfigurationManager.getProperty("dspace.dir") + 
                            File.separator + "config" + File.separator +
                            "dstat.cfg";
   
   /** the output file to which to write aggregation data */
   private static String outFile = ConfigurationManager.getProperty("dspace.dir") + 
                            File.separator + "log" + File.separator + "dstat.dat";
   
   /** the starting date of the report */
   private static Date startDate = null;
        
   /** the end date of the report */
   private static Date endDate = null;
        
   /** the starting date of the report as obtained from the log files */
   private static Date logStartDate = null;
        
   /** the end date of the report as obtained from the log files */
   private static Date logEndDate = null;
   
   /** are we looking stuff up in the database */
   private static boolean lookUp = false;
        
   
    /**
     * main method to be run from command line.  See usage information for
     * details as to how to use the command line flags (-help)
     */
    public static void main(String [] argv)
        throws Exception, SQLException
    {
        // first, start the processing clock
        startTime = new GregorianCalendar();
        
        // create context as super user
        Context context = new Context();
        context.setIgnoreAuthorization(true);
        
        // set up our command line variables
        String myLogDir = null;
        String myFileTemplate = null;
        String myConfigFile = null;
        String myOutFile = null;
        Date myStartDate = null;
        Date myEndDate = null;
        boolean myLookUp = false;
        
        // read in our command line options
        for (int i = 0; i < argv.length; i++)
        {
            if (argv[i].equals("-log"))
            {
                myLogDir = argv[i+1];
            }
            
            if (argv[i].equals("-file"))
            {
                myFileTemplate = argv[i+1];
            }
            
            if (argv[i].equals("-cfg"))
            {
                myConfigFile = argv[i+1];
            }
            
            if (argv[i].equals("-out"))
            {
                myOutFile = argv[i+1];
            }
            
            if (argv[i].equals("-help"))
            {
                LogAnalyser.usage();
                System.exit(0);
            }
            
            if (argv[i].equals("-start"))
            {
                myStartDate = parseDate(argv[i+1]);
            }
            
            if (argv[i].equals("-end"))
            {
                myEndDate = parseDate(argv[i+1]);
            }
            
            if (argv[i].equals("-lookup"))
            {
                myLookUp = true;
            }
        }
        
        // now call the method which actually processes the logs
        processLogs(context, myLogDir, myFileTemplate, myConfigFile, myOutFile, myStartDate, myEndDate, myLookUp);
    }