nbrkstats

harvest是一个下载html网页得机器人

#!/usr/bin/perl
##
##############################################################################
## NAME
##	nbrkstats.cgi - create HTML-output from harvest-broker statistic-file
##
## SYNOPSIS
##	./nbrkstats.cgi > /tmp/output.html or
##	http://www.mybrokerhostname.de/Harvest/cgi-bin/nbrkstats.cgi
##
## INSTALLATION:
##	- To use this script via www,
##	   * copy this script to
##	       /usr/local/harvest/cgi-bin
##	     or
##	       $PREFIX/harvest/cgi-bin
##	   * change some variables below "#### Config"
##	   * and run:
##	       http://localhost/Harvest/cgi-bin/nbrkstats.cgi
##
##	- To use this script from command line
##	   * change some variables below "#### Config"
##	   * set $WWW=0
##	   * create a configfile (nbrkstats.conf)
##	   * and run
##	       "./nbrkstats.cgi > /tmp/output.html"
##	     or put in crontab
##	       0-59/5 * * * * root /PATH/TO/nbrkstats.cgi > /tmp/output.html
##
##############################################################################
#
# CONFIG-FILE:
#   To use this script from command line you have to create a config-file with
#   the following entries:
#
#     broker=www.mybrokerhostname.de
#     case=yes        or   case=no
#     maxdetail=3
#     ini_days=01
#     ini_months=01
#     ini_years=2000
#     fin_days=31
#     fin_months=12
#     fin_years=2002
#     cbox_week=ON    or   cbox_week=OFF
#     cbox_hour=ON    or   cbox_hour=OFF
#     cbox_day=ON     or   cbox_day=OFF
#     cbox_query=ON   or   cbox_query=OFF
#
#
##############################################################################
# VERSION
#	Version 0.6, 06.05.2002, Guido Kerkewitz, [e-mail]
#	  - added run from command line
#	  - added config-file
#	  - some minor changes
#	Version 0.5a, 25.04.2002, Guido Kerkewitz, [e-mail]
#	  - fix overwriting of values in %statp
#	  - fix formatting of rounded numbers
#	  - fix double '/' in $logprefix
#	  - insert debug-option
#	Version 0.5,  12.04.2002, Guido Kerkewitz, [e-mail]
# 	  - rewrite of BrokerStats.cgi 0.7
#         - no cgi-lib.pl required, only two perl modules (CGI, Date::Manip)
#
##############################################################################
# OLD VERSIONS:
#  Program BrokerStats.cgi:  harvest broker statistics
#  Version 0.6 Hermann Straus, Madrid February 1998
#  Send comments to [e-mail]
#
#  Version 0.7 Kang-Jin Lee <[e-mail]> 30 January 2000
#  Minor changes.
##############################################################################

##############################################################################
#### Config - User configurable parameters
##############################################################################

# Use from cgi-bin  (WWW=1) or commandline (WWW=0)
  $WWW         = '1';

# Configfile for command line run
  $CONFIGFILE  = "./nbrkstats.conf";

# local hostname (for linux);
  chop($HOSTNAME=`/bin/hostname -f`);

# WWW-/Harvest-Administrator:
  $ADMIN        = "harvest\@mydomain.org";

# Optional HTML-Stylefile (may be empty: ''):
  $STYLEFILE    = '/styles/style1.css';

# Colors:
  $COLOR_FRAME  = "#000080";
  $COLOR_TAB    = "#FFFFFF";

# Your local broker names:
  @BROKER       = qw(www.mybrokerhostname1.de www.mybrokerhostname.de);
# Default brokerhost
  $BROKERDEF    = "www.mybrokerhostname1.de";

# Default shown queries
  $MAXDETAILDEF = "3";

# Path to broker.out
  $logprefix    = "/usr/local/harvest/brokers";
  $brokerprefix = "http://${HOSTNAME}/Harvest/brokers";
  $brokersuffix = "index.html";

# Default START/END date
  $DAYSINIDEF   = '1';  $DAYSFINDEF   = '31';   $MONTHSINIDEF = '01';
  $MONTHSFINDEF = '12'; $YEARSINIDEF  = '2000'; $YEARSFINDEF  = '2005';

# special transformations for my own sites
# Should be always '0'
  $gkspecial      = '0';

##############################################################################
# Do not change after this line
##############################################################################

##############################################################################
# MAIN

use CGI qw/:standard :netscape *table/;

# Use this if you have DateManip module installed:
#
#use Date::Manip;
#
# Otherwise use this:
# Add the perl library path to the @INC path
@P = split('/', $0); pop @P; $medir = join('/',@P);
unshift(@INC, $medir);
require 'Manip.pm';

# Initialize
&init;

$frame_name = path_info();
$frame_name =~ s!^/!!;

# Prepare for command line run
if ( ! $WWW ) {
  use CGI qw/-no_debug/;
  open (IN,"$CONFIGFILE") || die (print "ERROR: Configfile ($CONFIGFILE) couldn't be opened.\n");
  restore_parameters(IN);
  close IN;
  $frame_name = 'brkbot';
}

# Create the header of the html-page
print header if ( $WWW );

# If no path information is provided, then we create
# a side-by-side frame set
if (!$frame_name) {
    print_frameset();
    exit 0;
}

# If we get here, then we either create the query form
# or we create the response.
print start_html(-title=>'Broker statistic',-style=>{'src'=>$STYLEFILE}, -author=>"$MAIL",
                 -meta=>{'keywords'=>'harvest broker statistic', 'copyright'=>"copyright 2002 $AUTHOR"}
                );
print_query()    if $frame_name eq 'brktop';
print_response() if $frame_name eq 'brkbot';
print_contact()  if $frame_name eq 'brkbot';
print_copy()     ;
print end_html();


##############################################################################
# Create the frameset
sub print_frameset {
    my $script = url();

    print title('Harvest Broker Statistics'),

    frameset({-rows=>'37%,63%'},
             frame({-name=>'brktop',-src=>"$script/brktop"}),
             frame({-name=>'brkbot',-src=>"$script/brkbot"})
             );
    exit 0;
} # end print_frameset

##############################################################################
# Print the menu for selection
sub print_query {
    my $script = url();

    print
      '<CENTER>',
      h1("Harvest Broker Statistics"),

      start_form(-action=>"$script/brkbot", -target=>"brkbot"),


      # outer table
       table({-border=>'0', -bgcolor=>"$COLOR_FRAME"}, Tr({-align=>CENTER,-valign=>TOP},[td([

       table({-border=>'0', -bgcolor=>"$COLOR_TAB"},
       Tr({-align=>CENTER,-valign=>TOP},
       [
       td([
          table({-border=>'0'}, caption('<B>Query:</B>'),
            Tr({-align=>LEFT,-valign=>CENTER},
            [
             td(['Broker to obtain statistics: ',
                 table({-border=>'0'},Tr({-align=>LEFT,-valign=>CENTER}, [ td([
                 popup_menu(-name=>'broker', -default=>"$BROKERDEF",-values=>[@BROKER])
                 ]) ])),
                ]),
             td(['Count queries casesensitive: ',
                 table({-border=>'0'},Tr({-align=>LEFT,-valign=>CENTER}, [ td([
                 popup_menu(-name=>'case', -default=>"$CASEDEF",-values=>[@CASE])
                 ]) ])),
                ]),
             td(['Show querys present at least: ',
                  table({-border=>'0'},Tr({-align=>LEFT,-valign=>CENTER},
                    [ td([ popup_menu(-name=>'maxdetail', -default=>"$MAXDETAILDEF", -values=>[@MAXDETAIL]), 'times.'
                    ]) ])),
                ])
          ])), # end table
          table({-border=>'0'}, caption('<B>Ranges of dates:</B>'),
            Tr({-align=>CENTER,-valign=>CENTER},
              [ th([''               , 'Day','Month','Year']),
                td(['Initial Date: ' ,
                      popup_menu(-name=>'ini_days',    -default=>"$DAYSINIDEF",   -values=>[@DAYS]),
                      popup_menu(-name=>'ini_months',  -default=>"$MONTHSINIDEF", -values=>[@MONTHS], -labels=>\%MONTHSLAB),
                      popup_menu(-name=>'ini_years',   -default=>"$YEARSINIDEF",  -values=>[@YEARS])
                   ]),
                td(['Final Date: '   ,
                      popup_menu(-name=>'fin_days',    -default=>"$DAYSFINDEF",   -values=>[@DAYS]),
                      popup_menu(-name=>'fin_months',  -default=>"$MONTHSFINDEF", -values=>[@MONTHS], -labels=>\%MONTHSLAB),
                      popup_menu(-name=>'fin_years',   -default=>"$YEARSFINDEF",  -values=>[@YEARS]),
                   ])
                ])
          ), # end table
          table({-border=>'0'}, caption('<B>Output:</B>'),
            Tr({-align=>LEFT,-valign=>CENTER},
            [
             td(['Show Weekday: ',
                 table({-border=>'0'},Tr({-align=>LEFT,-valign=>CENTER}, [
                    td([ checkbox(-name=>'cbox_week',  -checked=>1, -value=>'ON', -label=>''), ]),
                ])), ]),
             td(['Show Hour: ',
                 table({-border=>'0'},Tr({-align=>LEFT,-valign=>CENTER}, [
                    td([ checkbox(-name=>'cbox_hour',  -checked=>1, -value=>'ON', -label=>''), ]),
                ])), ]),
             td(['Show Day: ',
                 table({-border=>'0'},Tr({-align=>LEFT,-valign=>CENTER}, [
                    td([ checkbox(-name=>'cbox_day',   -checked=>1, -value=>'ON', -label=>''), ]),
                ])), ]),
             td(['Show Query: ',
                 table({-border=>'0'},Tr({-align=>LEFT,-valign=>CENTER}, [
                    td([ checkbox(-name=>'cbox_query', -checked=>1, -value=>'ON', -label=>''), ]),
                ])), ]),
          ])), # end table
          ]),
       ])
      ), # end table

       ])])),  # outer table

      br, submit, end_form,
      '</CENTER>' ;

} # end print_query

##############################################################################
# Print the results
sub print_response {

   # No Query
   unless (param) {
      print "<CENTER>\n";
      print b("No query submitted yet.");
      print '</CENTER>';
      return;
   }

   &set_variables();

   print "<CENTER>\n";
   print h1("Statistics for the broker $broker"),"\n";
   print p("\n","Here you can access the ", a({-href=>"$brokerurl",-target=>'_new'}, "Broker main page."));

   &get_statistic();

   # Query

   # print p(b("Result statistics:")),"\n";
   print p("Statistics from ${ini_days}.${ini_months}.${ini_years} to ${fin_days}.${fin_months}.${fin_years}"), br,"\n";
   # Table (outside)
   print "\n<TABLE BORDER=\"0\" CELLPADDING=\"10\">\n";
   print " <TR VALIGN=\"TOP\" ALIGN=\"TOP\">\n";
   print " <TD>\n";

     @rows=();
     push(@rows,th('&nbsp;') .th('Number')."\n");
     push(@rows,td("<B>Total queries:</B>")      .td("$stat{'totalqueries'}")."\n");
     push(@rows,td("<B>Queries completed:</B>")  .td("$stat{'totalcomplet'}")."\n");
     push(@rows,td("<B>Total results:</B>")      .td("$stat{'totalres'}")    ."\n");
     push(@rows,td("<B>Timeouts:</B>")           .td("$stat{'timeout'}")     ."\n");
     push(@rows,td("<B>Broker restarted</B>")    .td("$stat{'brkonline'}")   ."\n");
     push(@rows,td("<B>Glimpse restarted</B>")   .td("$stat{'glonline'}")    ."\n");
     push(@rows,td("<B>Collections started</B>") .td("$stat{'collect'}")     ."\n");

     # outer table
     print
      table({-border=>'0', -bgcolor=>"$COLOR_FRAME"}, Tr({-align=>CENTER,-valign=>TOP},[td([
        table({-border=>'0', -bgcolor=>"$COLOR_TAB",-cellpadding=>'3'}, "\n",
              Tr({-align=>CENTER,-valign=>CENTER}, [ @rows ] )),"\n" ,
      ])]));  # outer table

   print " </TD>\n <TD>\n";

     @rows=();
     push(@rows,th('Results found').th('Number').th('% of total')."\n");
     push(@rows,td("not found").td("$stat{r0}").td("$statp{'r0'} \%")."\n");
     push(@rows,td("1-5")      .td("$stat{r5}").td("$statp{'r5'} \%")."\n");
     push(@rows,td("5-10")     .td("$stat{r10}").td("$statp{'r10'} \%")."\n");
     push(@rows,td("10-20")    .td("$stat{r20}").td("$statp{'r20'} \%")."\n");
     push(@rows,td("20-50")    .td("$stat{r50}").td("$statp{'r50'} \%")."\n");
     push(@rows,td(">50")      .td("$stat{rn}").td("$statp{'rn'} \%")."\n");

     # outer table
     print
      table({-border=>'0', -bgcolor=>"$COLOR_FRAME"}, Tr({-align=>CENTER,-valign=>TOP},[td([
        table({-border=>'0', -bgcolor=>"$COLOR_TAB",-cellpadding=>'3'}, "\n",
              Tr({-align=>CENTER,-valign=>CENTER}, [ @rows ] )),"\n" ,
      ])]));  # outer table

   print " </TD>\n <TD>\n";

     @rows=();
     push(@rows,th('Query type').th('Number') ."\n");
     push(@rows,td("Single terms").td("$singleTerm") ."\n");
     push(@rows,td("Phrases").td("$phrase") ."\n");
     push(@rows,td("logic AND").td("$and") ."\n");
     push(@rows,td("logic OR").td("$or") ."\n");
     push(@rows,td("Compound").td("$compound") ."\n");
     push(@rows,td("by field").td("$field") ."\n");
     #print table({-border=>'1',-cellpadding=>'3'}, "\n", Tr({-align=>CENTER,-valign=>CENTER}, [ @rows ] )),br,"\n" ;
     # outer table