createbroker.in

harvest是一个下载html网页得机器人

    $line

Please remedy this conflict by manually editing the Brokers.cf file or
abort this procedure.
=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

EOM


&get_ans("Enter the password to the broker administrative interface",
	   "Broker-Password");
&get_ans("Enter the hostname for your HTTP server.  If you want to use\na different port number than 80, append it to the hostname\nwith a colon (e.g., www.tardis.ed.ac.uk:8080)",
	   "HTTP-Host");
&get_ans("Enter your email address",
	   "Contact-Email");
$vals{"Contact-Message"} = "Please send questions and comments to $vals{'Contact-Email'}" if ($vals{'Contact-Email'} ne "");
&get_ans("Enter the name of the attribute that will be displayed to the\nuser as one-line object description in search results",
	   "Broker-Description-Tag");
$vals{"Indexer"} = "glimpse";
&get_ans("Do you want to use Glimpse, WAIS or Swish as an Indexer",
	   "Indexer");
$INDEXER = "glimpse";

if ($vals{"Indexer"} =~ /swish/io) {
	$INDEXER = "swish";
        &get_ans("Enter the pathname to the swish command", "swish-Command");
} elsif ($vals{"Indexer"} =~ /glimpse/io) {
       	$INDEXER = "glimpse";
	&get_ans("Enter the pathname to the glimpse command",
	   	"glimpse-Command");
	&get_ans("Enter the pathname to the glimpseindex command",
	   	"glimpseindex-Command");
	&get_ans("Enter the pathname to the glimpseserver command",
	   	"glimpseserver-Command");
	&get_ans("Enter the host on which to run glimpseserver",
	   	"Broker-Glimpse-Host");
} elsif ($vals{"Indexer"} =~ /wais/io) {
        $INDEXER = "Wais";
	$vals{'Broker-WAIS-Host'} = $vals{'Broker-Host'};
	$vals{'Broker-WAIS-Port'} = $vals{'Broker-Port'} + 1;
	$vals{'WAIS-Flavor'} = "WAIS";
	$vals{"YorN"} = "no";
	&get_ans("Are you using the commericial version of WAIS?", "YorN");
	if ($vals{"YorN"} =~ /^y/io) {
		&get_ans("Enter the pathname to the waisdelete command",
	   		"waisdelete-Command");
		&get_ans("Enter the pathname to the waislookup command",
	   		"waislookup-Command");
		&get_ans("Enter the pathname to the waisparse command",
	   		"waisparse-Command");
		$vals{'WAIS-Flavor'} = "Commercial-WAIS";
	} else {
		$vals{'waisdelete-Command'} = "none";
		$vals{'waislookup-Command'} = "none";
		$vals{'waisparse-Command'} = "none";
	}
	&get_ans("Enter the host on which to run the WAIS server",
	   	"Broker-WAIS-Host");
	&get_ans("Enter the port on which to run the WAIS server",
	   	"Broker-WAIS-Port");
	&get_ans("Enter the pathname to the waisindex command",
	   	"waisindex-Command");
	&get_ans("Enter the pathname to the waissearch command",
	   	"waissearch-Command");
	&get_ans("Enter the pathname to the waisserver command",
	   	"waisserver-Command");
}

&get_ans("Enter the destination Broker directory",
   	"Destination-Directory");
if (! -d $vals{"Skeleton-Directory"}) {
	print "Couldn't find the Broker skeleton directory.\n";
	&get_ans("Enter the Broker skeleton directory",
   		"Skeleton-Directory");
}
}	# end of get_user_input

sub build_broker {
	print <<EOM;

  ----------------------------------------------------------------------
  CreateBroker will now perform the following tasks:
      1. Create the directory $vals{'Destination-Directory'}
         in which the new Broker will live.
      2. Build the Broker configuration files from the skeleton Broker
         tree in $vals{'Skeleton-Directory'}
      3. Create the program $vals{'Destination-Directory'}/RunBroker
  ----------------------------------------------------------------------

EOM
	$vals{"YorN"} = "yes";
	&get_ans("Should I build this Broker now?", "YorN");
	return if ($vals{"YorN"} !~ /^y/io);

	$vals{'Broker-Indexer-Type'} = "Glimpse";
	$vals{'Broker-Indexer-Type'} = "WAIS" if ($INDEXER =~ /wais/io);
	$vals{'Broker-Indexer-Type'} = "Swish" if ($INDEXER =~ /swish/io);
	#  Make the broker directory
	mkdir("$vals{'Destination-Directory'}", 0755) || die
		"Cannot make $vals{'Destination-Directory'}: $!";
	mkdir("$vals{'Destination-Directory'}/admin", 0755) || die
		"Cannot make $vals{'Destination-Directory'}/admin: $!";

	mkdir("$vals{'Destination-Directory'}/zebra", 0755) || die
		"Cannot make $vals{'Destination-Directory'}/zebra: $!";
	mkdir("$vals{'Destination-Directory'}/zebra/index", 0755) || die
		"Cannot make $vals{'Destination-Directory'}/zebra/index: $!";
	mkdir("$vals{'Destination-Directory'}/zebra/index-shadow", 0755) || die
		"Cannot make $vals{'Destination-Directory'}/zebra/index-shadow: $!";
	mkdir("$vals{'Destination-Directory'}/zebra/log", 0755) || die
		"Cannot make $vals{'Destination-Directory'}/zebra/log: $!";
	mkdir("$vals{'Destination-Directory'}/zebra/run", 0755) || die
		"Cannot make $vals{'Destination-Directory'}/zebra/run: $!";
	mkdir("$vals{'Destination-Directory'}/zebra/tmp", 0755) || die
		"Cannot make $vals{'Destination-Directory'}/zebra/tmp: $!";

	mkdir("$ENV{'HARVEST_HOME'}/tmp", 0755);
	mkdir("$ENV{'HARVEST_HOME'}/tmp/$vals{'Broker-Short-Name'}", 0755) || die
		"Cannot make $ENV{'HARVEST_HOME'}/tmp/$vals{'Broker-Short-Name'}: $!";
	chmod(01777, "$ENV{'HARVEST_HOME'}/tmp/$vals{'Broker-Short-Name'}");

	$vals{'Admin-Process'} = "";
	@files = ("Makefile", "Query.cgi", "RunBroker",
		"query-$INDEXER.html",    "query-$INDEXER-modern.html",
		"query-$INDEXER.html.it", "query-$INDEXER-modern.html.it",
		"query-$INDEXER.html.se", "query-$INDEXER-modern.html.se",
		"query-$INDEXER.html.de", "query-$INDEXER-modern.html.de",
		"query-$INDEXER.html.nl", "query-$INDEXER-modern.html.nl",
		"query-$INDEXER-modern.html.es",
		"query-$INDEXER-modern.html.fr",
		"query-$INDEXER-modern.html.ru",
		"admin/Collection.conf", "admin/broker.conf");

	foreach $file (@files) {
		open(IN, "< $vals{'Skeleton-Directory'}/$file.in") || die
		"Cannot read $vals{'Skeleton-Directory'}/$file.in: $!\n";
		open(OUT, "> $vals{'Destination-Directory'}/$file") || die
		"Cannot create $vals{'Destination-Directory'}/$file: $!\n";
		print "Creating $vals{'Destination-Directory'}/$file\n";
		while (<IN>) {
	s/\@THIS_HARVEST_HOME\@/$ENV{'HARVEST_HOME'}/g;
	s/\@THIS_HTTP_HOST\@/$vals{'HTTP-Host'}/g;
	s/\@CONTACT_EMAIL\@/$vals{'Contact-Email'}/g;
	s/\@CONTACT_MESSAGE\@/$vals{'Contact-Message'}/g;
	s/\@THIS_BROKER_ADMIN_PROCESS\@/$vals{'Admin-Process'}/g;
	s/\@THIS_BROKER_COLLECT\@/$vals{'Broker-Collect'}/g;
	s/\@THIS_BROKER_DIR\@/$vals{'Destination-Directory'}/g;
	s/\@THIS_BROKER_NAME\@/$vals{'Broker-Name'}/g;
	s/\@THIS_BROKER_SHORT_NAME\@/$vals{'Broker-Short-Name'}/g;
	s/\@THIS_BROKER_PORT\@/$vals{'Broker-Port'}/g;
	s/\@THIS_BROKER_PASSWORD\@/$vals{'Broker-Password'}/g;
	s/\@THIS_BROKER_HOST\@/$vals{'Broker-Host'}/g;
	s/\@THIS_BROKER_SWISH\@/$vals{'swish-Command'}/g;
	s/\@THIS_BROKER_GLIMPSE\@/$vals{'glimpse-Command'}/g;
	s/\@THIS_BROKER_GLIMPSEINDEX\@/$vals{'glimpseindex-Command'}/g;
	s/\@THIS_BROKER_GLIMPSESERVER\@/$vals{'glimpseserver-Command'}/g;
	s/\@THIS_BROKER_GLIMPSE_HOST\@/$vals{'Broker-Glimpse-Host'}/g;
	s/\@THIS_BROKER_INDEXER_TYPE\@/$vals{'Broker-Indexer-Type'}/g;
	s/\@THIS_BROKER_WAIS_FLAVOR\@/$vals{'WAIS-Flavor'}/g;
	s/\@THIS_BROKER_WAISINDEX\@/$vals{'waisindex-Command'}/g;
	s/\@THIS_BROKER_WAISPARSE\@/$vals{'waisparse-Command'}/g;
	s/\@THIS_BROKER_WAISSEARCH\@/$vals{'waissearch-Command'}/g;
	s/\@THIS_BROKER_WAISSERVER\@/$vals{'waisserver-Command'}/g;
	s/\@THIS_BROKER_WAISDELETE\@/$vals{'waisdelete-Command'}/g;
	s/\@THIS_BROKER_WAISLOOKUP\@/$vals{'waislookup-Command'}/g;
	s/\@THIS_BROKER_WAIS_HOST\@/$vals{'Broker-WAIS-Host'}/g;
	s/\@THIS_BROKER_WAIS_PORT\@/$vals{'Broker-WAIS-Port'}/g;
	s/\@THIS_BROKER_DESCRIPTION\@/$vals{'Broker-Description-Tag'}/g;
	s/\@REPLICAS\@/$vals{'Broker-Replicas'}/g;
	s#\@THIS_BROKER_TOPDIR\@#$ENV{'HARVEST_HOME'}#g;
			print OUT;
		}
		close(IN);
		close(OUT);
	}
	chmod(0755, "$vals{'Destination-Directory'}/RunBroker");
	chmod(0755, "$vals{'Destination-Directory'}/Query.cgi");
	chmod(0644, "$vals{'Destination-Directory'}/query-$INDEXER.html");
	chmod(0644, "$vals{'Destination-Directory'}/query-$INDEXER-modern.html");
	system("ln -s query-$INDEXER-modern.html $vals{'Destination-Directory'}/index.html");
	system("ln -s query-$INDEXER.html $vals{'Destination-Directory'}/query.html");
	system("cp $vals{'Skeleton-Directory'}/zebra.cfg $vals{'Destination-Directory'}/");

	&add_brokers_cf ($vals{'Broker-Short-Name'},
		$vals{'Broker-Host'},
		$vals{'Broker-Port'});

	print "Finished.\n\n";
}

# Check if (name,host,port) are already in use in the Brokers.cf file
#
# If so, return the line from the file, else return an empty string.
#
sub check_brokers_cf {
	local ($name,$host,$port) = @_;
	local ($BROKERSCF) = $ENV{'HARVEST_HOME'} . '/brokers/Brokers.cf';
	return '' unless ( -s $BROKERSCF );
	open (BROKERSCF) || die "$BROKERSCF: $!\n";
	while (<BROKERSCF>) {
		chop;
		s/#.*//;
		next unless (/\s*(\S+)\s+(\S+)\s+(\d+)\s*$/);
		return $_ if ($name eq $1);
		return $_ if ($host eq $2 && $port == $3);
	}
	close (BROKERSCF);
	return '';
}

sub add_brokers_cf {
	local ($name,$host,$port) = @_;
	local ($BROKERSCF) = $ENV{'HARVEST_HOME'} . '/brokers/Brokers.cf';
	open (BROKERSCF, ">>$BROKERSCF") || die "$BROKERSCF: $!\n";
	chmod (0644, $BROKERSCF)
		unless ( -s $BROKERSCF );
	printf BROKERSCF ("%-31s %-31s %s\n", '#NAME','HOST','PORT')
		unless ( -s $BROKERSCF );
	printf BROKERSCF ("%-31s %-31s %d\n", $name,$host,$port);
	close BROKERSCF;
}

sub add_collection_point {
	local($file) = @_;

	$vals{'Collection-Host'} = $this_host_full;
	$vals{'Collection-Port'} = 8500;
	$vals{'Collection-Type'} = 3;
	$vals{'Collection-Query'} = "";

	&get_ans("Enter the host from which to collect",
		   "Collection-Host");
	&get_ans("Enter the port from which to collect",
		   "Collection-Port");
	print <<EOM;
The Broker supports various types of collections:
    0 - From Gatherer: Full collection each time,    without data compression.
    1 - From Gatherer: Incremental collections,      without data compression.
    2 - From Gatherer: Full collection each time,    with    data compression.
    3 - From Gatherer: Incremental collections,      with    data compression.
    4 - From Broker:   Full collection each time,    without data compression.
    5 - From Broker:   Incremental collections,      without data compression.
    6 - From Broker:   Collection based on a query,  without data compression.
    7 - From Broker:   Incremental based on a query, without data compression.
EOM
	&get_ans("Enter the collection type",
		   "Collection-Type");
	if ($vals{"Collection-Type"} == "6") {
		&get_ans("Enter the query filter", "Collection-Query");
	}
	if ($vals{"Collection-Type"} == "7") {
		&get_ans("Enter the query filter", "Collection-Query");
	}

	print <<EOM;

  ----------------------------------------------------------------------
EOM
	foreach $key (	"Collection-Host",
			"Collection-Type",
			"Collection-Port",
			"Collection-Query") {
		printf "%21s: ", $key;
		print $vals{$key}, "\n";
	}
	print <<EOM;
  ----------------------------------------------------------------------
EOM
	$vals{"YorN"} = "yes";
	&get_ans("Is this information correct?", "YorN");
	&add_collection_point($file), return if ($vals{"YorN"} =~ /^n/io);

	open(OUT, ">> $file") || die "Cannot append to $file: $!\n";
	print OUT $vals{'Collection-Host'}, " ";
	print OUT $vals{'Collection-Port'}, " ";
	print OUT $vals{'Collection-Type'}, " ";
	print OUT "--";
	print OUT $vals{'Collection-Query'}, "\n";
	close(OUT);
	print "Ok, the collection point was successfully added.\n\n";
}