robots.pm

awstats-6.6.zip tomcat日志分析包linux

'tarantula',
'tarspider',
'techbot',
'templeton',
'titan',
'titin',
'tkwww',
'tlspider',
'ucsd',
'udmsearch',
'universalfeedparser',
'urlck',
'valkyrie',
'verticrawl',
'victoria',
'visionsearch',
'voidbot',
'vwbot',
'w3index',
'w3m2',
'wallpaper',
'wanderer',
'wapspIRLider',
'webbandit',
'webcatcher',
'webcopy',
'webfetcher',
'webfoot',
'webinator',
'weblinker',
'webmirror',
'webmoose',
'webquest',
'webreader',
'webreaper',
'websnarf',
'webspider',
'webvac',
'webwalk',
'webwalker',
'webwatch',
'whatuseek',
'whowhere',
'wired\-digital',
'wmir',
'wolp',
'wombat',
'wordpress',
'worm',
'wwwc',
'wz101',
'xget',
# Other robots reported by users
'1\-more\sscanner',
'accoona\-ai\-agent',
'activebookmark',
'adamm\sbot',
'almaden',
'aipbot',
'aleadsoftbot',
'alpha\ssearch\sagent',
'aport',
'archive\.org_bot',
'argus', 		# Must be before nutch
'arianna\.libero\.it',
'aspseek',
'asterias',
'awbot',
'baiduspider',
'becomebot',
'bender',
'biglotron',
'bittorrent\sbot',
'biz360\sspider',
'blogbridge\sservice',
'bloglines',
'blogpulse',#		added OpenWebSpider http://www.openwebspider.org/
#		added NimbleCrawler http://www.healthline.com/
#		added Mydoyouhike http://www.doyouhike.net/my
#		added PHP version tracker http://www.nexen.net/phpversion/bot.php
#		added kamano.de NewsFeedVerzeichnis http://www.kamano.de/
#		added yoogliFetchAgent http://www.yoogli.com/
#		added ETS v http://www.freetranslation.com/help/
#		added HPPrint 
#		added CFNetwork http://www.cocoadev.com/index.pl?CFNetwork
#		added STEROID  Download http://faqs.org.ru/progr/pascal/delphi_internet2.htm
#		added OSSProxy http://www.marketscore.com/FAQ.Aspx
#		added Oracle Ultra Search http://www.oracle.com/technology/products/ultrasearch/index.html
#		added Web Downloader http://www.krasu.ru/soft/chuchelo/
#		added lwp-request http://search.cpan.org/~gaas/libwww-perl-5.69/bin/lwp-request
#		added Sunrise http://www.sunrisexp.com/
#		added WordPress http://wordpress.org/
#		added Global Fetch http://www.wesonet.com/
#		added NASA Search http://www.psychedelix.com/cgi-bin/csv2html.pl?data=allagents.csv&template=detail.html&match=\bid_n_s_140506_2\b
#		added NetSprint http://www.netsprint.pl/serwis/
#		added Webdup http://www.webdup.com/en/index.html
#		added Megite http://www.megite.com/
#		added Mini-reptile 
#		added lwp-trivial http://search.cpan.org/src/GAAS/libwww-perl-5.805/lib/LWP/Simple.pm
#		added HTMLParser http://htmlparser.sourceforge.net/

'blogsearch',
'blogshares',
'blogslive',
'blogssay',
'bncf\.firenze\.sbn\.it\/raccolta\.txt',
'bobby',
'boitho\.com\-dc',
'bookmark\-manager',
'boris',
'bumblebee',
'candlelight\_favorites\_inspector',
'cbn00glebot',
'cerberian\sdrtrs',
'cfnetwork',
'cipinetbot',
'checkweb\slink\svalidator',
'commons\-httpclient',
'computer\sand\sautomation\sresearch\sinstitute\scrawler',
'converamultimediacrawler',
'converacrawler',
'cscrawler',
'cse\shtml\svalidator\slite\sonline',
'cuasarbot',
'cursor',
'custo',
'datafountains/dmoz\sdownloader',
'daviesbot',
'daypopbot',
'deepindex',
'dipsie\.bot',
'dnsgroup',
'docomo',
'domainchecker',
'domainsdb\.net',
'dulance',
'dumbot',
'dumm\.de\-bot',
'earthcom\.info',
'easydl',
'edgeio\-retriever',
'ets\sv',
'exactseek',
'extreme\_picture\_finder',
'eventax',
'everbeecrawler',
'everest\-vulcan',
'ezresult',
'enteprise',
'fast\senterprise\scrawler.*crawleradmin\.t\-info@telekom\.de',
'fast\senterprise\scrawler.*t\-info_bi_cluster\scrawleradmin\.t\-info@telekom\.de',
'matrix\ss\.p\.a\.\s\-\sfast\senterprise\scrawler', # must come before fast enterprise crawler
'fast\senterprise\scrawler',
'fast\-search\-engine',
'favorg',
'favorites\ssweeper',
'feedburner',
'feedfetcher\-google',
'feedflow',
'feedster',
'feedvalidator',
'filmkamerabot',
'findlinks',
'findexa\scrawler',
'fooky\.com\/ScorpionBot',
'g2crawler',
'gaisbot',
'geniebot',
'gigabot',
'girafabot',
'global\sfetch',
'gnodspider',
'goforit\.com',
'goforitbot',
'grub',
'gpu\sp2p\scrawler',
'henrythemiragorobot',
'heritrix',
'holmes',
'hoowwwer',
'hpprint',
'htmlparser',
'html\_link\_validator',
'httrack',
'hundesuche\.com\-bot',
'ichiro',
'iltrovatore\-setaccio',
'infobot',
'infociousbot',
'infomine',
'insurancobot',
'internet\_ninja',
'internetarchive',
'internetseer',
'internetsupervision',
'irlbot',
'isearch2006',
'iupui_research_bot',
'jrtwine\_software\_check\_favorites\_utility',
'justview',
'kalambot',
'kamano\.de\snewsfeedverzeichnis',
'kazoombot',
'kevin',
'keyoshid', # Must come before Y!J
'kinjabot',
'kinja\-imagebot',
'knowitall',
'knowledge\.com',
'kouaa\skrawler',
'krugle',
'ksibot',
'kurzor',
'lanshanbot',
'letscrawl\.com',
'libcrawl',
'linkbot',
'link\svalet\sonline',
'metager\-linkchecker',	# Must be before linkchecker
'linkchecker',
'livejournal\.com',
'lmspider',
'lwp\-request',
'lwp\-trivial',
'magpierss',
'mapoftheinternet\.com',
'mediapartners\-google',
'megite',
'metaspinner',
'microsoft\surl\scontrol',
'mini\-reptile',
'minirank',
'missigua\slocator',
'misterbot',
'miva',
'mizzu\slabs',
'mj12bot',
'mojeekbot',
'tencenttraveler', # Must be before msiecrawler
'msiecrawler',
'ms\ssearch\s4\.0\srobot',
'msrabot',
'msrbot',
'mt::telegraph::agent',
'nagios',
'nasa\ssearch',
'mydoyouhike',
'netluchs',
'netsprint',
'newsgatoronline',
'nicebot',
'nimblecrawler',
'noxtrumbot',
'npbot',
'nutchcvs',
'nutchosu\-vlib',
'nutch',  # Must come after other nutch versions
'ocelli',
'octora\sbeta\sbot',
'omniexplorer\_bot',
'onet\.pl\_sa',
'onfolio',
'opentaggerbot',
'openwebspider',
'oracle\sultra\ssearch',
'orbiter',
'outfoxbot',
'passwordmaker\.org',
'pear\shttp\srequest\sclass',
'peerbot',
'perman',
'php\_version\_tracker',
'php\sversion\stracker',
'pictureofinternet',
'ping\.blo\.gs',
'plinki',
'pluckfeedcrawler',
'pompos',
'popdexter',
'port\shuron\slabs',
'postfavorites',
'projectwf\-java\-test\-crawler',
'proodlebot',
'pyquery',
'rambler',
'redalert',
'rojo',
'rssimagesbot',
'ruffle',
'rufusbot',
'sandcrawler',
'sbider',
'schizozilla',
'scumbot',
'searchguild\_dmoz\_experiment',
'seekbot',
'sensis\sweb\scrawler',
'seznambot',