search_engines.pm

awstats-6.6.zip tomcat日志分析包linux

# AWSTATS SEARCH ENGINES DATABASE
#------------------------------------------------------------------------------
# If you want to add a Search Engine to extend AWStats database detection capabilities,
# you must add an entry in SearchEnginesSearchIDOrder, SearchEnginesHashID and in
# SearchEnginesHashLib.
# An entry if known in SearchEnginesKnownUrl is also welcome.
#------------------------------------------------------------------------------
# $Revision: 1.41 $ - $Author: eldy $ - $Date: 2006/11/15 22:30:15 $

# 2005-08-19 Sean Carlos http://www.antezeta.com/awstats.html
#            added minor italian search engines
#                  arianna http://arianna.libero.it/
#                  supereva http://search.supereva.com/
#                  kataweb http://kataweb.it/
#            corrected uk looksmart
#                  'askuk','ask=', 'bbc','q=', 'freeserve','q=', 'looksmart','key=',
#            to 
#                  'askuk','ask=', 'bbc','q=', 'freeserve','q=', 'looksmartuk','key=',
#            corrected spelling
#                     internationnal -> international
#            added 'google\.'=>'mail\.google\.', to NotSearchEnginesKeys in order to
#            avoid counting gmail referrals as search engine traffic
# 2005-08-21 Sean Carlos http://www.antezeta.com/awstats.html
#            avoid counting babelfish.altavista referrals as search engine traffic
#            avoid counting translate.google referrals as search engine traffic
# 2005-11-20 Sean Carlos
# 	     added missing 'tiscali','key=', entry.  Check order
# 2005-11-22 Sean Carlos
# 	     added Google Base & Froogle.  Froogle not tested.
# 2006-04-18 Sean Carlos http://www.antezeta.com/awstats.html
# 	     added biglotron.com (France)
# 	     added blingo http://www.blingo.com/
# 	     added Clusty & Vivisimo
# 	     added eniro.no (Norway) [https://sourceforge.net/forum/message.php?msg_id=3134783]
# 	     added GPU p2p search http://search.centraldatabase.org/
# 	     added mail.tiscali to "not search engines list" [https://sourceforge.net/forum/message.php?msg_id=3166688]
# 	     added Ask group's "mysearch"
# 	     added sify.com (India)
# 	     added sogou.com (Cina) [https://sourceforge.net/forum/message.php?msg_id=3501603]
# 	     Ask changes:
# 	     - added Ask Japan (ask.jp)	
# 	     - break out Ask new country level variants (DE, ES, FR, IT, NL)
# 	     - updated Ask name from Ask Jevees
# 	     - added Ask q= parameter - many recent searches probably not recognized; [https://sourceforge.net/forum/message.php?msg_id=3465444]
# 	     - updated Ask uk (new uk.ask.com added to older ask.co.uk)
# 	     updated voila kw|rdata parameter [https://sourceforge.net/forum/message.php?msg_id=3373912]
#	     for each new engine, added link to Search Engine.  This serves to document engine. Done for major & Italian engines as well. Requires patch
#		to AWStats to allow untranslated html.  Otherwise html will appear instead of link.
#	     reviewed mnoGoSearch (http://www.mnogosearch.org/); the search engined mentioned no longer
#		exists https://sourceforge.net/forum/message.php?msg_id=3025426
# 2006-05-13 Sean Carlos http://www.antezeta.com/awstats.html
#            added 10 Chello European broadband portals (Austria, Belgium, Czech Republic, France, Hungary, The Netherlands, Norway, Poland, Slovakia, Sweden)
#	     added Alice Internal Search (blends data with Google?) search.alice.it.master:10005
#            added detection of google cache views from IPs 66.249.93.104 72.14.203.104 72.14.207.104
#		To do: add more extensive IP list; keywords not yet detected.
#            added icerocket.com blog search http://www.icerocket.com/
#	     added live.com (msn) http://www.live.com/
# 	     added Meta motor kartoo.  Note: Kartoo does not provide search words in referrers, thus the engine will appear in the
#		search engine list but the actual search words are not available.
#	     added netluchs.de http://www.netluchs.de/
#	     added sphere.com blog search http://www.sphere.com/
#	     added wwweasel.de http://wwweasel.de
#	     added Yahoo Mindset! http://mindset.research.yahoo.com/
#            updated Mirago query parameter recognition (qry=); added breakout for each country (France, Germany, Spain, Italy, Norway, Sweden, Denmark, Netherlands, Belgium, Switzerland)
# 2006-05-13 Sean Carlos http://www.antezeta.com/awstats.html 
#	     added Google cache IPs 64.233.183.104 & 66.102.7.104
# 2006-05-20 Sean Carlos http://www.antezeta.com/awstats.html 
#		anzwers.com.au
#		schoenerbrausen.de http://www.schoenerbrausen.de/
#		added Google cache IP 216.239.59.104
#		answerbus http://www.answerbus.com/ (does not provide keywords)
# 2006-05-23 Sean Carlos http://www.antezeta.com/awstats.html
#		added Google cache IP 66.102.9.104, 64.233.161.104
# 2006-06-23 Sean Carlos http://www.antezeta.com/awstats.html 
#	     	added Alice Search search.alice.it
#		added GoodSearch http://www.goodsearch.com/ (does not provide keywords) "a Yahoo-powered search engine that donates money to your favorite charity or school each time you search the web"
#		added googlee.com, variant of Google
#		added gotuneed http://www.gotuneed.com/ Italian search engine, in beta
#		added icq.com
#		added logic to parse Google Cache search keywords. Seems to work for alpha but not numeric cache IDs, i.e. search?q=cache:lWVLmnuGJswJ: is recognized but q=cache:Yv5qxeJNuhgJ: is not recognized. The URL triggering the keywords will also appear.  The URLs are probably too varied to parse out?
#		added Nusearch http://www.nusearch.com/
#		added Polymeta www.polymeta.hu (does not provide keywords)
#		added scroogle http://www.scroogle.org/ (does not always provide keywords)
#		added Tango http://tango.hu/search.php?st=0&q=jeles+napok
#		Changed Google Cache notation 64\.233\.(161|167|179|183|187)\.104 to 64\.233\.1[0-9]{2}\.104
#		 			      72\.14\.(203|205|207|209|221)\.104 to 72\.14\.2[0-9]{2}\.104
#					      216\.239\.(51|59)\.104 to 216\.239\.5[0-9]\.104
#					      66\.102\.(7|9)\.104 to 66\.102\.[1-9]\.104
# 2006-06-27 Sean Carlos http://www.antezeta.com/awstats.html
#		added Onet.pl http://szukaj.onet.pl/ 
#		corrected name "Wirtualna Polska" from "Szukaj" (search); added link http://szukaj.wp.pl/ 
# 2006-06-30 Sean Carlos http://www.antezeta.com/awstats.html
#	Additional Polish Search Engines:
#	added Dodaj.pl http://www.dodaj.pl/
#	added Gazeta.pl http://szukaj.gazeta.pl/
#	added Gery.pl http://szukaj.gery.pl/
#	added Hoga.pl http://www.hoga.pl/
#	added Interia.pl http://www.google.interia.pl/
#	added Katalog.Onet.pl http://katalog.onet.pl/
#	added NetSprint.pl http://www.netsprint.pl/
#	added o2.pl http://szukaj2.o2.pl/
#	added Polska http://szukaj.polska.pl/
#	added Szukacz http://www.szukacz.pl/
#	added Wow.pl http://szukaj.wow.pl/
#	added Sagool http://sagool.jp/

# 2006-08-25 Social Bookmarks
#	International
#	added del.icio.us/search - for now, just search referrer. To do: consider /tag/(tagname) referrer?
# 	added stumbleupon.com - No keywords supplied.
#	added swik.net
#       added digg. Keywords sometimes supplied.
#	Italy
# 	added segnalo.alice.it - No keywords supplied.
#	added ineffabile.it - No keywords supplied.

#       added filter for google groups.  Attempt to parse group name as keyword.

# 2006-09-14 
#	added Eniro Sverige http://www.eniro.se/
#	added MyWebSearch http://search.mywebsearch.com/ 
#	added Teecno http://www.teecno.it/ Italian Open Source Search Engine

#package AWSSE;

# 2006-09-25 (Gabor Moizes)
# added 4-counter (Google alternative) http://4-counter.com/
# added Googlecom (Google alternative) http://googlecom.com/
# added Goggle (Google alternative) http://goggle.co.hu/
# added Comet toolbar http://as.starware.com
# added new IP for Yahoo: 216.109.125.130
# added Ledix http://ledix.net/
# added AT&T search (powered by Google) http://www.att.net/
# added Keresolap (Hungarian search engine) http://www.keresolap.hu/
# added Mozbot (French search engine) http://www.mozbot.fr/
# added Zoznam (Slovak search engine) http://www.zoznam.sk/
# added sapo.pt (Portuguese search engine) http://www.sapo.pt/
# added shaw.ca (powered by Google) http://start.shaw.ca/
# added Searchalot http://www.searchalot.com/
# added Copernic http://www.copernic.com/
# added 216.109.125.130 to Yahoo
# added 66.218.69.11 to Yahoo
# added Avantfind http://www.avantfind.com/
# added Steadysearch http://www.steadysearch.com/
# added Steadysearch http://www.steady-search.com/
# modified 216\.239\.5[0-9]\.104/search to 216\.239\.5[0-9]\.104


# SearchEnginesSearchIDOrder
# It contains all matching criteria to search for in log fields. This list is
# used to know in which order to search Search Engines IDs.
# Most frequent one are in list1, used when LevelForSearchEnginesDetection is 1 or more
# Minor robots are in list2, used when LevelForSearchEnginesDetection is 2 or more
# Note: Regex IDs are in lower case and ' ' and '+' are changed into '_'
#------------------------------------------------------------------------------
@SearchEnginesSearchIDOrder_list1=(
# Major international search engines
'base\.google\.',
'froogle\.google\.',
'groups\.google\.',
'images\.google\.',
'google\.',
'googlee\.',
'googlecom\.com',
'goggle\.co\.hu',
'216\.239\.(35|37|39|51)\.100',
'216\.239\.(35|37|39|51)\.101', 
'216\.239\.5[0-9]\.104', 
'64\.233\.1[0-9]{2}\.104',
'66\.102\.[1-9]\.104',
'66\.249\.93\.104',
'72\.14\.2[0-9]{2}\.104',
'msn\.',
'live\.com',
'voila\.',
'mindset\.research\.yahoo',
'yahoo\.','(66\.218\.71\.225|216\.109\.117\.135|216\.109\.125\.130|66\.218\.69\.11)',
'search\.aol\.co',
'tiscali\.',
'lycos\.',
'alexa\.com',
'alltheweb\.com',
'altavista\.',
'a9\.com',
'dmoz\.org',
'netscape\.',
'search\.terra\.',
'www\.search\.com',
'search\.sli\.sympatico\.ca', 
'excite\.'
);

@SearchEnginesSearchIDOrder_list2=(
# Minor international search engines
'4\-counter\.com',
'att\.net',
'northernlight\.',
'hotbot\.',
'kvasir\.',
'webcrawler\.',
'metacrawler\.',
'go2net\.com',
'(^|\.)go\.com',
'euroseek\.',
'looksmart\.',
'spray\.',
'nbci\.com\/search',
'de\.ask.\com', # break out Ask country specific engines.  (.jp is in Japan section)
'es\.ask.\com',
'fr\.ask.\com',
'it\.ask.\com',
'nl\.ask.\com',
'uk\.ask.\com',
'(^|\.)ask\.com',
'atomz\.',
'overture\.com',		# Replace 'goto\.com','Goto.com',
'teoma\.',
'findarticles\.com',
'infospace\.com',
'mamma\.',
'dejanews\.',
'dogpile\.com',
'wisenut\.com',
'ixquick\.com',
'search\.earthlink\.net', 
'i-une\.com',
'blingo\.com',
'centraldatabase\.org',
'clusty\.com',
'mysearch\.',
'vivisimo\.com',
'kartoo\.com',
'icerocket\.com',
'sphere\.com',
'ledix\.net',
'start\.shaw\.ca',
'searchalot\.com',
'copernic\.com',
'avantfind\.com',
'steadysearch\.com',
'steady-search\.com',
# Chello Portals
'chello\.at',
'chello\.be',
'chello\.cz',
'chello\.fr',
'chello\.hu',
'chello\.nl',
'chello\.no',
'chello\.pl',
'chello\.se',
'chello\.sk',
'chello', # required as catchall for new countries not yet known
# Mirago 
'mirago\.be',
'mirago\.ch',
'mirago\.de',
'mirago\.dk',
'es\.mirago\.com',
'mirago\.fr',
'mirago\.it',
'mirago\.nl',
'no\.mirago\.com',
'mirago\.se',
'mirago\.co\.uk',
'mirago', # required as catchall for new countries not yet known
'answerbus\.com',
'icq\.com\/search',
'nusearch\.com',
'goodsearch\.com',
'scroogle\.org',
'questionanswering\.com',
'mywebsearch\.com',
'as\.starware\.com',
# Social Bookmarking Services
'del\.icio\.us',
'digg\.com',
'stumbleupon\.com',
'swik\.net',
'segnalo\.alice\.it',
'ineffabile\.it',
# Minor Australian search engines
'anzwers\.com\.au',
# Minor brazilian search engines
'engine\.exe', 'miner\.bol\.com\.br',
# Minor chinese search engines
'baidu\.com','search\.sina\.com','search\.sohu\.com', 'sogou\.com',
# Minor czech search engines
'atlas\.cz','seznam\.cz','quick\.cz','centrum\.cz','jyxo\.(cz|com)','najdi\.to','redbox\.cz',
# Minor danish search-engines 
'opasia\.dk', 'danielsen\.com', 'sol\.dk', 'jubii\.dk', 'find\.dk', 'edderkoppen\.dk', 'netstjernen\.dk', 'orbis\.dk', 'tyfon\.dk', '1klik\.dk', 'ofir\.dk',
# Minor dutch search engines
'ilse\.','vindex\.',
# Minor english search engines
'(^|\.)ask\.co\.uk','bbc\.co\.uk/cgi-bin/search','ifind\.freeserve','looksmart\.co\.uk','splut\.','spotjockey\.','ukdirectory\.','ukindex\.co\.uk','ukplus\.','searchy\.co\.uk',
# Minor finnish search engines
'haku\.www\.fi',
# Minor french search engines
'recherche\.aol\.fr','ctrouve\.','francite\.','\.lbb\.org','rechercher\.libertysurf\.fr', 'search[\w\-]+\.free\.fr', 'recherche\.club-internet\.fr',
'toile\.com', 'biglotron\.com', 
'mozbot\.fr', 
# Minor german search engines
'sucheaol\.aol\.de',
'fireball\.de','infoseek\.de','suche\d?\.web\.de','[a-z]serv\.rrzn\.uni-hannover\.de',
'suchen\.abacho\.de','brisbane\.t-online\.de','allesklar\.de','meinestadt\.de',
'212\.227\.33\.241',
'(161\.58\.227\.204|161\.58\.247\.101|212\.40\.165\.90|213\.133\.108\.202|217\.160\.108\.151|217\.160\.111\.99|217\.160\.131\.108|217\.160\.142\.227|217\.160\.176\.42)',
'wwweasel\.de',
'netluchs\.de',
'schoenerbrausen\.de',
# Minor Hungarian search engines
'heureka\.hu','vizsla\.origo\.hu','lapkereso\.hu','goliat\.hu','index\.hu','wahoo\.hu','webmania\.hu','search\.internetto\.hu',
'tango\.hu',
'keresolap\.hu',
'polymeta\.hu',
# Minor Indian search engines
'sify\.com',
# Minor Italian search engines
'virgilio\.it','arianna\.libero\.it','supereva\.com','kataweb\.it','search\.alice\.it\.master','search\.alice\.it','gotuneed\.com',
'godado','jumpy\.it','shinyseek\.it','teecno\.it',
# Minor Japanese search engines
'ask\.jp','sagool\.jp',
# Minor Norwegian search engines
'sok\.start\.no', 'eniro\.no',
# Minor Polish search engines
'szukaj\.wp\.pl','szukaj\.onet\.pl','dodaj\.pl','gazeta\.pl','gery\.pl','hoga\.pl','netsprint\.pl','interia\.pl','katalog\.onet\.pl','o2\.pl','polska\.pl','szukacz\.pl','wow\.pl',
# Minor russian search engines
'ya(ndex)?\.ru', 'aport\.ru', 'rambler\.ru', 'turtle\.ru', 'metabot\.ru',