find-expired-broker-objects.pl

harvest是一个下载html网页得机器人

: # *-*-perl-*-*
    eval 'exec perl -S $0 "$@"'
    if $running_under_some_shell;  

# find-expired-broker-objects.pl 
#
# usage: 
#
#      % cd /usr/local/harvest/brokers/mybroker
#      % find objects -type f -print | find-expired-broker-objects.pl > list
#      % vi list
#      % awk '{print $NF}' list | xargs rm
#
# $Id: find-expired-broker-objects.pl,v 2.1 1997/03/21 17:18:57 sxw Exp $
#

#############################################################################
#
#  Harvest Indexer http://www.tardis.ed.ac.uk/harvest/
#  ---------------------------------------------------
#
#  The Harvest Indexer is a continued development of code developed by
#  the Harvest Project. Development is carried out by numerous individuals
#  in the Internet community, and is not officially connected with the
#  original Harvest Project or its funding sources.
# 
#  Please mail [e-mail] if you are interested in participating
#  in the development effort.
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#  
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#  
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
# 

$now = time;

while (<>) {	# stdin is a list of filenames (eg from find)
	chop ($f = $_);
	next unless open f;
		$timestamp = 0;
		$ttl = 0;
	while (<f>) {
		$timestamp = $1
			if (/^Update-Time{\d+}:\t(\d+)/io);
		$ttl = $1
			if (/^time-to-live{\d+}:\t(\d+)/io);
		last if ($ttl != 0 && $timestamp != 0);
	}
	close f;
	$exp = $timestamp + $ttl;
	#print "Timestamp: $timestamp\n";
	#print "      TTL: $ttl\n";
	#print "  Expires: $exp\n";
	if ($exp < $now) {
		printf ("Expired %4.1f days ago %s\n",
			($now - $exp) / 86400, $f);
	}
}