logfileqparse.java

一个数据挖掘软件ALPHAMINERR的整个过程的JAVA版源代码

  private String qParseExecutePrefix(String toParse, char sepchar)
  {
    int pos;
    if ((pos=toParse.indexOf(sepchar))>0)
    {
      return toParse.substring(0,pos);
    }
    return null;
  }


  private boolean qParseRecurse(LogFileQParseAction node, String toParse) throws MiningException
  {
    boolean actionSucceeded;
    String newString = null;

//  execute action
    switch (node.actionType)
    {
      case PARSE_PERL_MATCH:
        actionSucceeded = qParseExecutePerlMatch(toParse, ((LogFileQParseActionPerl)node).configString)!=null?true:false;
        newString = toParse;
      break;

      case PARSE_PERL_MATCHEXTRACT:
         newString = qParseExecutePerlMatchExtract(toParse, ((LogFileQParseActionPerl)node).configString);
         actionSucceeded = newString!=null?true:false;
      break;

      case PARSE_PERL_NEGMATCH:
        actionSucceeded = qParseExecutePerlMatch(toParse, ((LogFileQParseActionPerl)node).configString)==null?true:false;
        newString = toParse;
      break;

      case PARSE_EXTRACT:
        newString = qParseExecuteExtract(toParse, ((LogFileQParseActionExtract)node).count1, ((LogFileQParseActionExtract)node).separator1, ((LogFileQParseActionExtract)node).count2, ((LogFileQParseActionExtract)node).separator2);
        actionSucceeded = newString!=null?true:false;
      break;

      case PARSE_PREFIX:
        newString = qParseExecutePrefix(toParse, ((LogFileQParseActionPrefixPostfix)node).separator);
        actionSucceeded = newString!=null?true:false;
      break;

      default:
        throw new MiningException("qParseRecurse(): Invalid action");
    }

    if (actionSucceeded)
    {
      if (node.nextAction == null)
      {
        if (node.isTransaction)  // extracted transaction successfully
        {
          haveTransaction = true;
          qParseTransactionString = newString;
          if (haveItem)
            return true;
        }
        if (node.isItem) // extracted item successfully
        {
          haveItem = true;
          qParseItemString = newString;
          if (haveTransaction)
            return true;
        }
      } else
      {
        if ((node.nextAction.isTransaction && (!haveTransaction)) || (node.nextAction.isItem && (!haveItem))) // next node is useful
        {
          qParseRecurse(node.nextAction, newString);
        }
      }
    }

    if (haveTransaction && haveItem)
      return true;

    if (node.alternativeAction == null)
      return false;

    return qParseRecurse(node.alternativeAction, toParse);
  }


private boolean qParseParse(String toParse) throws MiningException // this is a wrapper for the recursive parsing function
{
  // no transaction / item found yet
  haveTransaction = false;
  haveItem = false;

  if (parseAnchor == null)
    throw new MiningException("qParseParse(): Parsing structure not initialised!");

  return qParseRecurse(parseAnchor, toParse);
}

private boolean qParseRead() throws MiningException
{
  String logFileLine;

  while (true) // loop is exited by returning either true or false
  {
    try
    {
      // read a line from the file
      // note: bufferedReader is quite fast, so it's not urgent to optimise at that point
      logFileLine = bufferedReader.readLine();

      // if no more lines are available (eof), return false
      if (logFileLine == null)
      {
        qParseTransactionString = null;
        qParseItemString = null;
        return false;
      }
    numLinesCount++;   // successfully retrieved a line from the data source

    } catch (IOException e)
    {
      throw new MiningException("QParse: Call to readLine in qParseRead() resulted in exception with message " + e.getMessage());
    }

    // parse the string that was read from the file
    // if parsing succeeds, the result is written to qParseTransactionString and qParseItemString
    if (qParseParse(logFileLine))
      return true;

    // line was invalid
    numLinesInvalid++;
  }
}

  /**
   * next - standard method to request next mining vector.
   * This function calls qParseRead() to extract the next
   *
   * @return next position of mining vector to read
   * @throws MiningException cannot advance cursor position
   */
  public boolean next() throws MiningException
  {
    // qParseRead() returns true if a dataset was read and parsed correctly
    // extracted data is stored in "qParseTransactionString" and "qParseItemString"
    if (qParseRead())
    {
      double instance[] = new double[3];

      // assign transaction
      Category transactionCat = new Category(qParseTransactionString);
      double d0 = ( (CategoricalAttribute) attributeTransactionID).getKey(transactionCat);
      instance[0] = Category.isMissingValue(d0) ? ( (CategoricalAttribute) attributeTransactionID).addCategory(transactionCat) : d0;

      // assign item
      Category itemCat = new Category(qParseItemString);
      double d1 = ( (CategoricalAttribute) attributeItemID).getKey(itemCat);
      instance[1] = Category.isMissingValue(d1) ? ( (CategoricalAttribute) attributeItemID).addCategory(itemCat) : d1;

      // numerical attribute is not yet initialised
      instance[2] = Category.MISSING_VALUE;

      cursorVectorProcessed = new MiningVector(instance);
      cursorVectorProcessed.setMetaData(metaDataProcessed);

      return true;
    } else
      return false;
  }

  /**
 * Reads current mining vector.
 *
 * @return mining vector at current cursor position
 * @throws MiningException
 */
  public MiningVector read() throws MiningException
  {
    return cursorVectorProcessed;
  }

  /**
   * Moves to position in source relative to current cursor position
   *
   * @param position number of (valid) lines to advance
   * @throws MiningException
   * @return boolean
   */
    public boolean move( int position ) throws MiningException
    {
      // check parameter (position must be >0)
      if (position <=0)
        throw new MiningException("Cannot move backwards or advance zero lines in file stream.");

      // moving is realised using calls to next()
      boolean nextResult = false;
      for(int i=0;i<position;i++)
      {
        nextResult = next();
      };
      return nextResult;
    }


    /**
     * This function acutally adds a node to the parsing tree. See documentation for individual actions that can be added to the tree.
     *
     * @param node LogFileQParseAction
     * @param newAction LogFileQParseAction
     * @param isNextActionBranch boolean
     * @param isTransaction boolean
     * @param isItem boolean
     */
    private void addParseActionNode(LogFileQParseAction node, LogFileQParseAction newAction, boolean isNextActionBranch, boolean isTransaction, boolean isItem)
  {
    newAction.nextAction = null;
    newAction.alternativeAction = null;

    newAction.isTransaction = isTransaction;
    newAction.isItem = isItem;

    // add to appropriate position in tree
    if (node == null)
    {
      parseAnchor = newAction;
    }
    else
    {
      if (isNextActionBranch)
        node.nextAction = newAction;
      else
        node.alternativeAction = newAction;
    }
  }

  /**
   * addParseActionPerl Function that adds a perl proessor action to the parse tree.
   *
   * @param node Node to attach new node to
   * @param isNextActionBranch New node is attached to next branch if true, to alternative branch if false
   * @param isTransaction This action is required to extract the transaction id
   * @param isItem This action is required to extract the item id
   * @param type Action type, must be one of PARSE_PERL_MATCH (only checks if matches), PARSE_PERL_MATCHEXTRACT (tries match and extracts matching portion), PARSE_PERL_NEGMATCH (only matches and fails in ase of match).
   * @param config Regular expression used for matching
   *
   * @throws MiningException
   * @return LogFileQParseAction
   */
  public LogFileQParseAction addParseActionPerl(LogFileQParseAction node, boolean isNextActionBranch,  boolean isTransaction, boolean isItem, int type, String config) throws MiningException
  {
    // generate specific parsing instance
    LogFileQParseActionPerl newAction = new LogFileQParseActionPerl();
    newAction.actionType = type;

    switch (type)
    {
     case PARSE_PERL_MATCH:
     case PARSE_PERL_MATCHEXTRACT:
     case PARSE_PERL_NEGMATCH:
       newAction.configString = config;
       break;
     default:
       throw new MiningException("Invalid action type specified.");
    }

    // general function to add node to tree
    addParseActionNode(node, newAction, isNextActionBranch, isTransaction, isItem);
    return newAction;
  }

  /**
   * addParseActionExtract Function that adds an "extract" action to the parse tree.
   *
   * @param node Node to attach new node to
   * @param isNextActionBranch New node is attached to next branch if true, to alternative branch if false
   * @param isTransaction This action is required to extract the transaction id
   * @param isItem This action is required to extract the item id
   * @param type Action type, must be == PARSE_EXTRACT
   * @param count1 Number of occurences for first searator before extraction begins
   * @param sepchar1 First separator
   * @param count2 Number of occurences for second searator before extraction stops
   * @param sepchar2 Second separator
   *
   * @throws MiningException
   * @return LogFileQParseAction
   */
  public LogFileQParseAction addParseActionExtract(LogFileQParseAction node, boolean isNextActionBranch,  boolean isTransaction, boolean isItem, int type, int count1, char sepchar1, int count2, char sepchar2) throws MiningException
  {
    LogFileQParseActionExtract newAction = new LogFileQParseActionExtract();
    newAction.actionType = type;

    switch (type)
    {
     case PARSE_EXTRACT:
       newAction.count1 = count1;
       newAction.separator1 = sepchar1;
       newAction.count2 = count2;
       newAction.separator2 = sepchar2;
       break;
     default:
       throw new MiningException("Invalid action type specified.");
    }

    // general function to add node to tree
    addParseActionNode(node, newAction, isNextActionBranch, isTransaction, isItem);
    return newAction;
  }

  /**
   * addParseActionPrefix Function that adds a prefix extraction action to the parse tree.
   *
   * @param node Node to attach new node to
   * @param isNextActionBranch New node is attached to next branch if true, to alternative branch if false
   * @param isTransaction This action is required to extract the transaction id
   * @param isItem This action is required to extract the item id
   * @param type Action type, must be == PARSE_PREFIX
   * @param sepchar Delimiter of prefix
   *
   * @throws MiningException
   * @return LogFileQParseAction
   */
  public LogFileQParseAction addParseActionPrefix(LogFileQParseAction node, boolean isNextActionBranch,  boolean isTransaction, boolean isItem, int type, char sepchar) throws MiningException
  {
    LogFileQParseActionPrefixPostfix newAction = new LogFileQParseActionPrefixPostfix();
    newAction.actionType = type;

    switch (type)
    {
     case PARSE_PREFIX:
       newAction.separator = sepchar;
       break;
     default:
       throw new MiningException("Invalid action type specified.");
    }

    // general function to add node to tree
    addParseActionNode(node, newAction, isNextActionBranch, isTransaction, isItem);
    return newAction;
  }

public void addParseAction(int nodeIndex, int affectedIDs, int actionType, String config)
{
 /* if (nodeIndex==1)
  {

  } else
  {
      int tIndex = nodeIndex;
      while (tIndex!=1)
      {

      }
  }*/
}


  public static void main(String[] args)
  {
    try
    {
//      String fileName = "config\\config.properties";
//      String fileName = "data\\logs\\NCSA Combined Log File Format.log";
//      String fileName = "data\\logs\\NCSA Common Log File Format.log";
        String fileName = "data\\logs\\WebShopNCSA.log";
 //     String fileName = "data\\logs\\Extended Log File Format.log";
//      String fileName = "data\\logs\\IIS Log File Format.log";
//      String fileName = "data\\logs\\Intershop Log File Format.log";
//      String fileName = "data\\logs\\Shop Log File Format.log";
//      String fileName = "data\\logs\\UNRECOGNIZED.log";

        LogFileQParse converter = new LogFileQParse( fileName );

//        addParseActionPerl(LogFileQParseAction node, boolean isNextActionBranch,  boolean isTransaction, boolean isItem, int type, String config) throws MiningException

        LogFileQParseAction extrquote = converter.addParseActionExtract(null, true, true, true, PARSE_EXTRACT,1,'\"',1,'\"');
        LogFileQParseAction extrtransac = converter.addParseActionPerl(extrquote, true, true, false, PARSE_PERL_MATCHEXTRACT, "/[0-9]{4,10}\\-[0-9]{8}/");
        LogFileQParseAction extritem21 = converter.addParseActionPerl(extrtransac, false, false, true, PARSE_PERL_MATCHEXTRACT, "/\\.cgi\\?id=[0-9]{2,20}&p=3/");
        LogFileQParseAction extritem22 = converter.addParseActionExtract(extritem21, true, false, true, PARSE_EXTRACT,1,'=',1,'&');
        LogFileQParseAction extritem11 = converter.addParseActionPerl(extritem21, false, false, true, PARSE_PERL_MATCHEXTRACT, "/3\\-[0-9]{2,20}\\-[1-9].html/");
        LogFileQParseAction extritem12 = converter.addParseActionExtract(extritem11, true, false, true, PARSE_EXTRACT, 1,'-',1,'-');


//        LogFileQParseAction extrtransac = converter.addParseActionPrefix(null, true, true, false, PARSE_PREFIX,',');
//        LogFileQParseAction extrquote = converter.addParseActionExtract(extrtransac, false, false, true, PARSE_EXTRACT,1,'/',1,',');

        MiningDataSpecification metaData = converter.metaDataProcessed;
        System.out.println(metaData);
        converter.reset();

        // overall timer - time taken for parsing AND sorting is measured
        long start = ( new java.util.Date() ).getTime();

        converter.dump("data\\logs\\SequentialAnanlysisFile.arff", true);

        long end = ( new java.util.Date() ).getTime();
        long programTime = end - start;

        System.out.println("\nTime taken = " + programTime + " ms.");

        converter.close();
        System.out.println("Log reading was finished.");
    }
    catch (MiningException ex)
    {
        ex.printStackTrace();
    }
  }

}