brokeradmin.in

harvest是一个下载html网页得机器人

#!@PERL@
#
#############################################################################
#
#  brokerAdmin.cgi
#
#  Usage: Called as a CGI process
#  Author: Simon Wilkinson ([e-mail])
#
#############################################################################
#
#  Harvest Indexer http://harvest.sourceforge.net/
#  -----------------------------------------------
#
#  The Harvest Indexer is a continued development of code developed by
#  the Harvest Project. Development is carried out by numerous individuals
#  in the Internet community, and is not officially connected with the
#  original Harvest Project or its funding sources.
#
#  Please mail [e-mail] if you are interested in participating
#  in the development effort.
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#

$prefix = "@prefix@";
unshift (@INC, "$prefix/lib");
require 'socket.ph';

$ME = $ENV { 'SCRIPT_NAME' };

$cf = "$prefix/brokers/Brokers.cf";

open (CF, "<$cf") or error ("Can't open Broker configuration file $cf.");
$i = 0;
while (<CF>) {
    next if /^\#/;
    ($names[$i], $hosts[$i], $ports[$i]) = /(\S+)\s+(\S+)\s+(\d+)/;
    $i++;
}
close CF;

# Check that we've been called as a POST script.
error ("Reference with a METHOD of POST.")
    if (! $ENV { 'REQUEST_METHOD' } =~ /POST/);

# Check that some data actually got POSTed.
print_form () if ($ENV { 'CONTENT_LENGTH' } == 0);

# Get *all* of the input, up to the content length.
read (STDIN, $buffer, $ENV { 'CONTENT_LENGTH' });

# Split up the input into attribute,value pairs and stick it in an
# associative array. This should all really be in a library...

foreach $pair (split (/&/, $buffer))
{
    ($name, $value) = split (/=/, $pair);
    $value =~ tr/+/ /;
    $value =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex ($1))/eg;
    $value =~ s/~!/ ~!/g;
    $vars { $name } = $value;
}

error ("Broker not found.")
    if ($ports[$vars { 'broker' }] == 0 || $hosts[$vars { 'broker' }] eq "");

$commandstr = "#ADMIN";
$commandstr .= " #password " . $vars { 'password' } if ($vars { 'password' } ne "");
$commandstr .= " " . $vars { 'command' };
$commandstr .= " " . $vars { 'parameter' } if ($vars { 'parameter' } ne "");

# Get ourselves a socket to talk to the broker on.
$SOCK = &client_socket ($hosts[$vars { 'broker' }], $ports[$vars { 'broker' }]);
error ("Couldn't open socket.") unless ($SOCK);

# Send the command to the broker
print $SOCK $commandstr;

# Display the results
print_header ("Harvest Broker Command Results");
print "<PRE>\n";
while (<$SOCK>) {
    chop;
    printf ("%s\n", $_);
}
close ($SOCK);
print "</PRE>\n";
print_footer ();

exit;


sub client_socket {
        local ($host, $port) = @_;
        local ($sockaddr) = 'S n a4 x8';
        local ($connected) = 0;

        # Lookup address for remote hostname
        #
        local($w,$x,$y,$z,@thataddrs) = gethostbyname($host);
        &error("Unknown Host: $host\n") unless (@thataddrs);

        # bind local socket to INADDR_ANY
        #
        local ($thissock) = pack($sockaddr, &AF_INET, 0, "\0\0\0\0");
        &error("socket: $!\n") unless socket (SOCK, &AF_INET, &SOCK_STREAM, 0);
        &error("bind: $!\n") unless bind (SOCK, $thissock);
        # Try all addresses
        #
        foreach $thataddr (@thataddrs) {
                local ($that) = pack($sockaddr, &AF_INET, $port, $thataddr);
                @IP = unpack('C4', $thataddr);
                printf ("Trying connection to %d.%d.%d.%d<BR>\n", @IP)
                        if ($debug);
                if (connect (SOCK, $that)) {
                        $connected = 1;
                        printf ("Connected to %d.%d.%d.%d!<BR>\n", @IP)
                                if ($debug);
                        last;
                }
        }
        return () unless ($connected);
        select (SOCK); $| = 1;
        select (STDOUT);
        return (SOCK);
}


sub print_header {

    $str = shift;

    print <<EOM;
Content-type: text/html

<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html>
<head>
<title>$str</title>
</head>
<body>
<h1>$str</h1>

EOM
}


sub print_footer {

    print <<EOM;

<hr>

<a href="$ME">Return to Harvest Broker Administration Interface</a>

</body>
</html>
EOM
}


sub print_form {

    print_header ("Harvest Broker Administration Interface");

    print <<EOM;
<P>
Look here for <A HREF="/Harvest/brokers/adminhelp.html">Help on Harvest Broker
Administation Commands</A>.

<P>
<B>Note:</B> Commands may take minutes or even hours depending the size of
your Broker.

<P>
<HR>
<FORM METHOD="POST" ACTION="$ME">
<dl>
<dt>Command:<dd>
<SELECT NAME="command" SIZE=1>
<OPTION VALUE="#add-object"> 		Add objects by file
<OPTION VALUE="#close-log"> 		Close log
<OPTION VALUE="#compress"> 		Compress registry
<OPTION VALUE="#clean-broker"> 		Delete expired objects
<OPTION VALUE="#rem-object-query"> 	Delete objects by query
<OPTION VALUE="#rem-object-path"> 	Delete objects by oid
<OPTION VALUE="#uniqify">	 	Delete older objects of duplicate URLs
<OPTION VALUE="#rem-log"> 		Disable log type
<OPTION VALUE="#add-log"> 		Enable log type
<OPTION VALUE="#flush-log"> 		Flush log
<OPTION VALUE="#do-stats"> 		Generate statistics
<OPTION VALUE="#incremental-index"> 	Index changes
<OPTION VALUE="#full-index"> 		Index corpus
<OPTION VALUE="#open-log"> 		Open log
<OPTION VALUE="#restart-index-server"> 	Restart index-server
<OPTION VALUE="#restart"> 		Restart server (Replicas only)
<OPTION VALUE="#rotate-log"> 		Rotate log
<OPTION VALUE="#set"> 		        Set variable
<OPTION VALUE="#shutdown"> 		Shutdown server
<OPTION VALUE="#collection" SELECTED> 	Start collections
</SELECT>

<dt>Parameters:<dd>
<TEXTAREA ROWS=4 COLS=50 NAME="parameter"></TEXTAREA>

<dt>Password:<dd>
<INPUT TYPE="password" NAME="password" SIZE=50>

<dt>Broker:<dd>
EOM

    if ($#names == 0) {
	print "$names[0] on $hosts[0] port $ports[0]\n";
	print "<input type=\"hidden\" name=\"broker\" value=\"0\">\n";
    } else {
	print "<SELECT NAME=\"broker\">\n";
	for ($i = 0; $i <= $#names; $i++) {
	    print "<OPTION VALUE=\"$i\">$names[$i] on $hosts[$i] port $ports[$i]\n";
	}
	print "</SELECT>\n";
    }

    print <<EOM;
</dl>

<P>
<INPUT TYPE="submit" VALUE="Issue Command">
<INPUT TYPE="reset" VALUE="Clear Form">
</FORM>

<P>
<HR>
</BODY>
</HTML>
EOM

    exit;
}


sub error {

    print_header ("Error from Broker Admin CGI script");

    print <<EOM;
<PRE>
$_[0]
</PRE>
EOM

    print_footer ();
    exit;
}