folddb

harvest是一个下载html网页得机器人

#!/bin/sh
#
#  folddb - Folds the WORKING database into the PRODUCTION database
#           and gets the databases ready for gatherd to serve.
#           Folds in the REFRESH database as well.
#
#  Usage: folddb [--save-space] GathererIDString directory
#
#  Darren Hardy, [e-mail], July 1994
#
#  folddb,v 1.26 1995/09/05 20:18:21 hardy Exp
#
#  Copyright (c) 1994, 1995.  All rights reserved.
#
#    The Harvest software was developed by the Internet Research Task
#    Force Research Group on Resource Discovery (IRTF-RD):
#
#          Mic Bowman of Transarc Corporation.
#          Peter Danzig of the University of Southern California.
#          Darren R. Hardy of the University of Colorado at Boulder.
#          Udi Manber of the University of Arizona.
#          Michael F. Schwartz of the University of Colorado at Boulder.
#          Duane Wessels of the University of Colorado at Boulder.
#
#    This copyright notice applies to software in the Harvest
#    ``src/'' directory only.  Users should consult the individual
#    copyright notices in the ``components/'' subdirectories for
#    copyright information about other software bundled with the
#    Harvest source code distribution.
#
#  TERMS OF USE
#
#    The Harvest software may be used and re-distributed without
#    charge, provided that the software origin and research team are
#    cited in any use of the system.  Most commonly this is
#    accomplished by including a link to the Harvest Home Page
#    (http://harvest.cs.colorado.edu/) from the query page of any
#    Broker you deploy, as well as in the query result pages.  These
#    links are generated automatically by the standard Broker
#    software distribution.
#
#    The Harvest software is provided ``as is'', without express or
#    implied warranty, and with no support nor obligation to assist
#    in its use, correction, modification or enhancement.  We assume
#    no liability with respect to the infringement of copyrights,
#    trade secrets, or any patents, and are not responsible for
#    consequential damages.  Proper use of the Harvest software is
#    entirely the responsibility of the user.
#
#  DERIVATIVE WORKS
#
#    Users may make derivative works from the Harvest software, subject
#    to the following constraints:
#
#      - You must include the above copyright notice and these
#        accompanying paragraphs in all forms of derivative works,
#        and any documentation and other materials related to such
#        distribution and use acknowledge that the software was
#        developed at the above institutions.
#
#      - You must notify IRTF-RD regarding your distribution of
#        the derivative work.
#
#      - You must clearly notify users that your are distributing
#        a modified version and not the original Harvest software.
#
#      - Any derivative product is also subject to these copyright
#        and use restrictions.
#
#    Note that the Harvest software is NOT in the public domain.  We
#    retain copyright, as specified above.
#
#  HISTORY OF FREE SOFTWARE STATUS
#
#    Originally we required sites to license the software in cases
#    where they were going to build commercial products/services
#    around Harvest.  In June 1995 we changed this policy.  We now
#    allow people to use the core Harvest software (the code found in
#    the Harvest ``src/'' directory) for free.  We made this change
#    in the interest of encouraging the widest possible deployment of
#    the technology.  The Harvest software is really a reference
#    implementation of a set of protocols and formats, some of which
#    we intend to standardize.  We encourage commercial
#    re-implementations of code complying to this set of standards.
#
#
workdb="WORKING.gdbm"
refreshdb="REFRESH.gdbm"
proddb="PRODUCTION.gdbm"
indexdb="INDEX.gdbm"
mddb="MD5.gdbm"
tmpdb="folddb.$$.gdbm"
allzipped=All-Templates

if [ $# -ne 2 -a $# -ne 3 ]; then
	echo "Usage: folddb [--save-space] GathererIDString Directory"
	exit 1
fi

if [ "$1" = "--save-space" ]; then
	echo "folddb: Space efficiency enabled."
	saveflag="--save-space"
	gidstr="$2"
	dir="$3"
else
	saveflag=""
	gidstr="$1"
	dir="$2"
fi


if [ ! -d "$dir" ]; then
	echo "folddb: $dir is not a directory"
	exit 1
fi
cd "$dir"
echo "folddb: Using given directory: $dir"

if [ ! -r "$workdb" ]; then
	echo "folddb: ERROR: $workdb is not present in the current directory."
	echo "folddb: Perhaps your Gatherer did not generate a database."
	exit 1
fi

echo "folddb: Cleaning up the gathered templates..."
chmod -R +w $workdb
cleandb $workdb
if [ -r "$refreshdb" ]; then
	chmod -R +w $refreshdb
	cleandb $refreshdb
fi

echo "folddb: Consolidating databases..."
if [ -r "$refreshdb" ]; then
	consoldb $saveflag $tmpdb $workdb $refreshdb $proddb
	rm -rf $workdb $refreshdb
else
	consoldb $saveflag $tmpdb $workdb $proddb
	rm -rf $workdb
fi

echo "folddb: Building an index for the Gatherer..."
# the production database will be unavailable for a little bit
chmod -Rf +w $proddb $indexdb $mddb
rm -rf $proddb $indexdb $mddb
mkindex $tmpdb $indexdb $mddb
mv $tmpdb $proddb
chmod -R a-w $proddb $indexdb $mddb

echo "folddb: Building the gatherd cache and generating statistics of the database..."
# build a cache file to support SEND-UPDATE 0 commands
# in compressed mode very quickly.
gdbmutil print -gatherd $proddb | tee $allzipped | mkgathererstats.pl "$gidstr" > INFO.soif-new
rm -f $allzipped.gz INFO.soif
mv INFO.soif-new INFO.soif
gzip $allzipped
chmod 444 $allzipped.gz INFO.soif

echo "folddb: Databases ready for export."
exit 0