byurl-posix.cf

harvest是一个下载html网页得机器人

#
#  byurl.cf - URL naming conventions that yield type recognition
#
#  Format: <Type> <Regular Expression for filename match>
#
#  All regular expressions are in POSIX extended form and must have
#  a ^ first and a $ last.  The types are evaluated in the order specified.
#  All regular expressions are CASE SENSITIVE.
#
#  $Id: byurl-posix.cf,v 2.1 1997/03/21 19:21:46 sxw Exp $
#

#
#  These types must always be evaluated first
#
ApacheFancyIndex	^http:.*\?.=.$
ApacheFancyIndex	^http:.*\?C=.\&O=.$

HTTP-Query		^http:.*\?.*$
HTTP-Query		^http:.*/cgi-bin/.*$
HTTP-Query		^http:.*/htbin/.*$
HTTP-Query		^http:.*\.cgi$

HTML			^http:.*\/$

#  Any News URL with a Message ID is a news article
NewsArticle		^news:.*@.*$
NewsGroup		^news:.*$