larbin.conf

larbin是一种开源的网络爬虫/网络蜘蛛

###############################################
# Who are you ?
# mail of the one who launched larbin (YOUR mail)
From [e-mail]
# name of the bot (sent with http headers)
UserAgent larbin_2.6.3

############################################
# What are the inputs and ouputs of larbin
# port on which is launched the http statistic webserver
# if unset or set to 0, no webserver is launched
httpPort 8081
# port on which you can submit urls to fetch
# no input is possible if you comment this line or use port 0
#inputPort 1976

############################################
# parameters to adapt depending on your network
# Number of connexions in parallel (to adapt depending of your network speed)
pagesConnexions 100
# Number of dns calls in parallel
dnsConnexions 5
# How deep do you want to go in a site
depthInSite 5
# do you want to follow external links
#noExternalLinks
# time between 2 calls on the same server (in sec) : NEVER less than 30
waitDuration 60
# Make requests through a proxy (use with care)
#proxy www 8080

##############################################
# now, let's customize the search

# first page to fetch (you can specify several urls)
startUrl http://slashdot.org/

# Do you want to limit your search to a specific domain ?
# if yes, uncomment the following line
#limitToDomain .fr .dk .uk end

# What are the extensions you surely don't want
# never forbid .html, .htm and so on : larbin needs them
forbiddenExtensions
.tar .gz .tgz .zip .Z .rpm .deb
.ps .dvi .pdf
.png .jpg .jpeg .bmp .smi .tiff .gif
.mov .avi .mpeg .mpg .mp3 .qt .wav .ram .rm
.jar .java .class .diff
.doc .xls .ppt .mdb .rtf .exe .pps .so .psd
end