robots.txt

来自「cetus网络开发资源指南,包括近2万个开发网址」· 文本 代码 · 共 54 行

# robots.txt for http://www.cetus-links.org/
# See http://info.webcrawler.com/mak/projects/robots/norobots.html for details

# Copyright (c) 1996-2000 by Manfred Schneider

User-agent: *
Disallow: /.remote/
Disallow: /.glimpse*
Disallow: /.nh*
Disallow: /.wg*
Disallow: /allpages*
Disallow: /archive.cfg
Disallow: /bin/
Disallow: /cgi-bin/
Disallow: /checkout.html
Disallow: /cronjob*
Disallow: /email.html
Disallow: /empty.html
Disallow: /frames*
Disallow: /ftp
Disallow: /ftp-private
Disallow: /guest_book.html
Disallow: /indexing*
Disallow: /misc/
Disallow: /logs/
Disallow: /noframes.html
Disallow: /oo_agents.html
Disallow: /oo_diagram_layout.html
Disallow: /oo_entry.html
Disallow: /oo_index.html
Disallow: /oo_java_language.html
Disallow: /oo_java_old.html
Disallow: /oo_java_xxx.html
Disallow: /oo_libraries.html
Disallow: /oo_links.html
Disallow: /oo_numerics.html
Disallow: /oo_opendoc.html
Disallow: /oo_repository.html
Disallow: /oo_template.html
Disallow: /oo_xxx.html
Disallow: /private/
Disallow: /stats/
Disallow: /tar
Disallow: /test*
Disallow: /top_advanced.html
Disallow: /top_databases.html
Disallow: /top_distributed_objects.html
Disallow: /top_general.html
Disallow: /top_java.html
Disallow: /top_ooa_ood_methods_tools.html
Disallow: /top_services_companies.html
Disallow: /update
Disallow: /wg*