filter.options

这是个爬虫和lucece相结合最好了

# Availible Scopes.
# Each processor class should be listed with full package info
# Lines beginning with # and empty lines are ignored
org.archive.crawler.filter.HopsFilter
org.archive.crawler.filter.OrFilter
org.archive.crawler.filter.PathDepthFilter
org.archive.crawler.filter.TransclusionFilter
org.archive.crawler.filter.URIRegExpFilter
org.archive.crawler.filter.URIListRegExpFilter
org.archive.crawler.filter.PathologicalPathFilter
org.archive.crawler.filter.SurtPrefixFilter
org.archive.crawler.filter.ContentTypeRegExpFilter
org.archive.crawler.filter.HTTPMidFetchUnchangedFilter
org.archive.crawler.deciderules.DecidingFilter