nbrkstats

harvest是一个下载html网页得机器人

     print
      table({-border=>'0', -bgcolor=>"$COLOR_FRAME"}, Tr({-align=>CENTER,-valign=>TOP},[td([
        table({-border=>'0', -bgcolor=>"$COLOR_TAB",-cellpadding=>'3'}, "\n",
              Tr({-align=>CENTER,-valign=>CENTER}, [ @rows ] )),"\n" ,
      ])]));  # outer table

   print " </TD></TR>\n";
   print "</TABLE>\n<BR>\n";

   # Table (outside)
   print "\n<TABLE BORDER=\"0\" CELLPADDING=\"10\">\n";
   print " <TR VALIGN=\"TOP\" ALIGN=\"CENTER\">\n";
   print " <TD>\n";

   @rows=();

   # Print Weekday Stats
   if ( $cb_week) {
     print p(b("Weekday<BR>(Max: $WEEKDAY{$maxweekdayname}/$stat_wday{$maxweekdayname}):"))."\n";
     push(@rows,th('Weekday').th('Number').th('% of total')."\n");
     foreach $i (sort keys %stat_wday) {
        push(@rows,(td($WEEKDAY{$i}).td($stat_wday{$i}).td("$statp{\"w,$i\"} \%")."\n"));
     }
     push(@rows,(td('<B>Sum</B>').td("<B>$weekdaysum</B>").td('<B>100</B>'))."\n");
     #print table({-border=>'1',-cellpadding=>'3'}, "\n", Tr({-align=>CENTER,-valign=>CENTER}, [ @rows ] )) ;
     # outer table
     print
      table({-border=>'0', -bgcolor=>"$COLOR_FRAME"}, Tr({-align=>CENTER,-valign=>TOP},[td([
        table({-border=>'0', -bgcolor=>"$COLOR_TAB",-cellpadding=>'3'}, "\n",
              Tr({-align=>CENTER,-valign=>CENTER}, [ @rows ] )),"\n" ,
      ])]));  # outer table

     print " </TD>\n <TD>\n";
     @rows=();
   }

   # Print Hour Stats
   if ( $cb_hour) {
     print p(b("Hour<BR>(Max: $maxhour/$stat_hour{$maxhour}):"))."\n";
     push(@rows,th('Hour').th('Number').th('% of total')."\n");
     foreach $i (sort keys %stat_hour) {
        push(@rows,(td($HOURS{$i}).td($stat_hour{$i}).td("$statp{$i} \%")."\n"));
     }
     push(@rows,(td('<B>Sum</B>').td("<B>$hoursum</B>").td('<B>100</B>')."\n"));
     #print table({-border=>'1',-cellpadding=>'3'}, "\n", Tr({-align=>CENTER,-valign=>CENTER}, [ @rows ] )) ;
     # outer table
     print
      table({-border=>'0', -bgcolor=>"$COLOR_FRAME"}, Tr({-align=>CENTER,-valign=>TOP},[td([
        table({-border=>'0', -bgcolor=>"$COLOR_TAB",-cellpadding=>'3'}, "\n",
              Tr({-align=>CENTER,-valign=>CENTER}, [ @rows ] )),"\n" ,
      ])]));  # outer table

     print " </TD>\n <TD>\n";
     @rows=();
   }

   # Print Day Stats
   if ( $cb_day) {
     print p(b("Day<BR>(Max: $maxday/$stat_day{$maxday}):"))."\n";
     push(@rows,th('Day').th('Number').th('% of total')."\n");
     foreach $i ( sort { $b <=> $a } keys %stat_day ) {
        push(@rows,(td($i).td($stat_day{$i}).td("$statp{$i} \%")."\n"));
     }
     push(@rows,(td('<B>Sum</B>').td("<B>$daysum</B>").td('<B>100</B>')."\n"));
     #print table({-border=>'1',-cellpadding=>'3'}, "\n", Tr({-align=>CENTER,-valign=>CENTER}, [ @rows ] )) ;
     # outer table
     print
      table({-border=>'0', -bgcolor=>"$COLOR_FRAME"}, Tr({-align=>CENTER,-valign=>TOP},[td([
        table({-border=>'0', -bgcolor=>"$COLOR_TAB",-cellpadding=>'3'}, "\n",
              Tr({-align=>CENTER,-valign=>CENTER}, [ @rows ] )),"\n" ,
      ])]));  # outer table

     print " </TD>\n <TD>\n";
     @rows=();
   }

   # Print Query Stats
   if ( $cb_query) {
     print p(b("Queries<BR>(Max: $maxquery/$stat_query{$maxquery}):"))."\n";
     push(@rows,th('Query').th('Number').th('% of total')."\n");
     foreach $i ( sort { $stat_query{$b} <=> $stat_query{$a} } keys %stat_query ) {
        if ($stat_query{$i} < $maxdetail )  {
         push(@rows,(td('...').td('...').td('...')."\n"));
         last
        }
        push(@rows,(td($i).td($stat_query{$i}).td("$statp{$i} \%")."\n"));
     }
     push(@rows,(td('<B>Sum</B>').td("<B>$querysum</B>").td('<B>100</B>')."\n"));
     #print table({-border=>'1',-cellpadding=>'3'}, "\n", Tr({-align=>CENTER,-valign=>CENTER}, [ @rows ] )) ;
     # outer table
     print
      table({-border=>'0', -bgcolor=>"$COLOR_FRAME"}, Tr({-align=>CENTER,-valign=>TOP},[td([
        table({-border=>'0', -bgcolor=>"$COLOR_TAB",-cellpadding=>'3'}, "\n",
              Tr({-align=>CENTER,-valign=>CENTER}, [ @rows ] )),"\n" ,
      ])]));  # outer table
   }

   print " </TD></TR>\n";
   print "</TABLE>\n<BR>\n";
   print '</CENTER>',"\n";

} # end print_response

##############################################################################
# Initialize
sub init {

  @MAXDETAIL = qw(1 2 3 4 5 10);
  @HOURS     = qw(00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23);
  @DAYS      = qw(01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31);
  @MONTHS    = qw(01 02 03 04 05 06 07 08 09 10 11 12);
  @YEARS     = qw(2000 2001 2002 2003 2004 2005 2006 2007 2008 2009);
  %MONTHSLAB = ( '01'=>'January', '02'=>'February', '03'=>'March',
                 '04'=>'April',   '05'=>'May',      '06'=>'June',
                 '07'=>'July',    '08'=>'August',   '09'=>'September',
                 '10'=>'October', '11'=>'November', '12'=>'December');
  %WEEKDAY   = ( '1'=>'Mon', '2'=>'Tue', '3'=>'Wed', '4'=>'Thu', '5'=>'Fri', '6'=>'Sat', '7'=>'Sun');
  %HOURS     = ( '00'=>'0:00 - 1:00',   '01'=>'1:00 - 2:00',   '02'=>'2:00 - 3:00',   '03'=>'3:00 - 4:00',
                 '04'=>'4:00 - 5:00',   '05'=>'5:00 - 6:00',   '06'=>'6:00 - 7:00',   '07'=>'7:00 - 8:00',
                 '08'=>'8:00 - 9:00',   '09'=>'9:00 - 10:00',  '10'=>'10:00 - 11:00', '11'=>'11:00 - 12:00',
                 '12'=>'12:00 - 13:00', '13'=>'13:00 - 14:00', '14'=>'14:00 - 15:00', '15'=>'15:00 - 16:00',
                 '16'=>'16:00 - 17:00', '17'=>'17:00 - 18:00', '18'=>'18:00 - 19:00', '19'=>'19:00 - 20:00',
                 '20'=>'20:00 - 21:00', '21'=>'21:00 - 22:00', '22'=>'22:00 - 23:00', '23'=>'23:00 - 24:00');
  @CASE         = qw(NO YES);
  $CASEDEF      = $CASE[0];

  # Select default Value
  if ( ! $BROKERDEF  )   { $BROKERDEF    = $BROKER[0]; }
  if ( ! $MAXDETAILDEF ) { $MAXDETAILDEF = $MAXDETAIL[0]; }

  ############################################################################
  # do not change
  $0          =~ s#.*/##g;
  $PROGNAME   = $0;
  $AUTHOR     = 'Guido Kerkewitz';
  $MAIL       = '[e-mail]';
  $VERSION    = "Version 0.6, 06.05.2002";
  $COPY       = "written by $AUTHOR";
  ############################################################################

} # end init

##############################################################################
# Set variables
sub set_variables {

  # transfer parameter
  $broker     = param(broker);
  $case       = param(case);
  $maxdetail  = param(maxdetail);
  $ini_days   = param(ini_days);
  $ini_months = param(ini_months);
  $ini_years  = param(ini_years);
  $fin_days   = param(fin_days);
  $fin_months = param(fin_months);
  $fin_years  = param(fin_years);
  $cb_week    = param(cbox_week);
  $cb_hour    = param(cbox_hour);
  $cb_day     = param(cbox_day);
  $cb_query   = param(cbox_query);

  # Create Variables
  $logfile="${logprefix}/${broker}/broker.out";
  $brokerurl="${brokerprefix}/${broker}/${brokersuffix}";
  $date_ini="${ini_years}${ini_months}${ini_days}";
  $date_fin="${fin_years}${fin_months}${fin_days}";


  # Set defaults
  # Queries
    $querysum           = 0; $maxquery           = 0;  %stat_query         = ();
  # Weekday
    $weekdaysum         = 0; $maxweekdayname     = ''; %stat_wday          = ();
    foreach $i (sort keys %WEEKDAY) { $stat_wday{$i} = 0 };
  # Hour
    $hoursum            = 0; $maxhour            = 0;  %stat_hour          = ();
    foreach $i ( @HOURS ) { $stat_hour{$i} = 0 };
  # Day
    $daysum             = 0; $maxday             = 0;  %stat_day           = ();

  @stat=();
  # stat includes:
  #   Total Queries:                 $stat{'totalqueries'}
  #   Completed Queries:             $stat{'totalcomplet'}
  #   Total results:                 $stat{'totalres'}
  #   (gk not used) Max. returned Results:    $stat{'maxret'}
  #   Timeouts:                      $stat{'timeout'}
  #   Glimpseserver online/restart:  $stat{'glonline'}
  #   Broker online/restart:         $stat{'brkonline'}
  #   Collections started            $stat{'collect'}
  #   (gk not used) Processing Admin Command  $stat{'admin'}
  #   (gk not used) Stating cleaning          $stat{'cleaning'}
  #  Count Results:
  #   not found                      $stat{'r0'}
  #   1-5                            $stat{'r5'}
  #   5-10                           $stat{'r10'}
  #   10-20                          $stat{'r20'}
  #   20-50                          $stat{'r50'}
  #   >50                            $stat{'rn'}

  #  percent of total
  @statp=();

  foreach $i ( 'r0', 'r5', 'r10', 'r20', 'r50', 'rn')                     { $statp{$i}=0; $stat{$i}=0}
  foreach $i ( 'totalqueries', 'totalcomplet', 'totalres', 'maxret',
               'timeout', 'glonline', 'brkonline', 'collect', 'cleaning', 'admin')
             { $stat{$i}=0}

  $phrase               = 0;
  $and                  = 0;
  $or                   = 0;
  $compound             = 0;
  $field                = 0;
  $singleTerm           = 0;
  $truncation           = 0; #(gk not used)


} # end set_variables

##############################################################################
# Get Statistic from broker.out
sub get_statistic {

  open (IN,${logfile}) || die (print "<H2>Sorry, $logfile couldn't be opened.</H2>");

  while(<IN>){
    chop;
    @line=split(' ',$_);
    next if ( ($line[1] < $date_ini) || ($line[1] > $date_fin) );


    if ($_ =~ /Processing Query/){
        # Count Queries

        $q=$_;
        $q=~s/(.*?)\#END\s(.*)/$2/;
        $q=~s/\\\./\./g if ( $gkspecial );		# Site Special Transformation
        ( $q=lc($q) ) if ( $case =~ /NO/i );

        push (@queries, "$q\n");
        $stat_query{"$q"}++;

        # Count query/weekday
          $line[1]=~/(\d\d\d\d)(\d\d)(\d\d)/;
          $d=$3; $m=$2; $y=$1;
          $wday=&Date_DayOfWeek($m,$d,$y);
          $stat_wday{"$wday"}++;
        # Count query/hour
          ($hour, undef) = split(':',$line[2]);
          $stat_hour{"$hour"}++;
        # Count query/day
          $stat_day{"$line[1]"}++;
          $stat{'totalqueries'}++;

    }
    elsif ($_ =~ /Query returned/){
      # get the number of results returned

      if ($line[5] > $stat{'maxret'}) {$stat{'maxret'} = $line[5];}
      $stat{'totalres'} += $line[5];

      $stat{'totalcomplet'}++;

      if    ($line[5]==0)    { $stat{'r0'}++  }
      elsif ($line[5] <= 5)  { $stat{'r5'}++; }
      elsif ($line[5] <= 10) { $stat{'r10'}++;}
      elsif ($line[5] <= 20) { $stat{'r20'}++;}
      elsif ($line[5] <= 50) { $stat{'r50'}++;}
      else                   { $stat{'rn'}++; }

    }
    elsif ($_ =~ /Inline query timeout/)      { $stat{'timeout'}++; }
    elsif ($_ =~ /is on-line/)                { $stat{'glonline'}++; }
    elsif ($_ =~ /Broker is now on-line/)     { $stat{'brkonline'}++; }
    elsif ($_ =~ /Starting collections/)      { $stat{'collect'}++; }
    elsif ($_ =~ /Processing Admin Command/)  { $stat{'admin'}++; }
    elsif ($_ =~ /Starting Cleaning/)         { $stat{'cleaning'}++; }
    else {}#Other lines

  } #while
  close(IN);


  # Prepare the collected values for the output

  # Weekdaystats
  foreach $i (sort keys %WEEKDAY) {
                $weekdaysum+=$stat_wday{$i};
                $maxweekdayname=$i if ($stat_wday{$i}>$stat_wday{$maxweekdayname});
              }
  foreach $i (sort keys %WEEKDAY) { $statp{"w,$i"} = sprintf("%.2f", $stat_wday{$i}/$weekdaysum*100); }

  # Hourstats
  foreach $i ( @HOURS ) {
                $hoursum+=$stat_hour{$i};
                $maxhour=$i if ($stat_hour{$i}>$stat_hour{$maxhour});
              }
  foreach $i ( @HOURS ) { $statp{$i} = sprintf("%.2f", $stat_hour{$i}/$hoursum*100); }

  # Daystats
  foreach $i (sort keys %stat_day) {
                $daysum+=$stat_day{$i};
                $maxday=$i if ($stat_day{$i}>$stat_day{$maxday});
              }
  foreach $i (keys %stat_day) { $statp{$i} = sprintf("%.2f", $stat_day{$i}/$daysum*100); }


  # Queries
  foreach $i (sort { $stat_query{$b} <=> $stat_query{$a} } keys %stat_query) {
                $querysum+=$stat_query{$i};
                $maxquery=$i if ($stat_query{$i}>$stat_query{$maxquery});
  }
  foreach $i (keys %stat_query) { $statp{$i} = sprintf("%.2f", $stat_query{$i}/$querysum*100); }

  # analyse the queries in terms of their structure
  foreach (keys %stat_query) {
     if (/"/)       { $phrase++; }
     if (/'/)       { $phrase++; }
     if (/ and /i)  { $and++; }
     if (/ or /i)   { $or++; }
     if (/\*/)      { $truncation++; }
     if (/\(/)      { $compound++; }
     if (/: /)      { $field++; }
     if (!/ /)      { $singleTerm++; }
   }

  # Count 'Results found'
  foreach $i ( 'r0', 'r5', 'r10', 'r20', 'r50', 'rn') {
    $statp{$i} = sprintf("%.2f", $stat{$i}/$stat{'totalqueries'}*100);
  }

  $stat{timeout} = 0 if ( ! $stat{timeout} );

  # FORMAT:
  #broker: 20020407 13:51:50: Processing Query 55: #USER #opaque #desc #index error 0 #index maxresult 1000 #index maxfiles 50 #index maxlines 100 #index case insensitive #index matchword #attribute none #END leseliste
  #broker: 20020407 13:51:50: Query returned 22 objects.

} # end  get_statistic

################
sub print_contact {
  print p("\n", &UnixDate("today","Page generated %T at %e.%m.%Y."),br, em("Local contact: $ADMIN"));
}
################
sub print_copy {
  print p("\n",hr,em("$PROGNAME, $COPY, $VERSION"));
}