pavuk.1.in

网络爬虫程序

.TH pavuk 1 "@REVISION@" "@VERSION@" "Internet utils"
.SH NAME
pavuk \- HTTP, HTTP over SSL, FTP, FTP over SSL and Gopher recursive document retrieval program
.SH SYNOPSIS
.PP
.B pavuk [-mode {normal | resumeregets | singlepage | singlereget | sync | dontstore | ftpdir | mirror}]
.I [-X] [-runX] [-bg/-nobg] [prefs/-noprefs] [-h] [-v] [-progress/-noprogress] [-stime/-nostime] [-xmaxlog $nr]
.I [-logfile $file] [-slogfile $file] [-auth_file $file] [-msgcat $dir] [-language $str] [-gui_font $font] [-quiet/-verbose
.I [-read_css/-noread_css]
.I [-cdir $dir]
.I [-scndir $dir]
.I [-scenario $str] [-dumpscn $filename]
.I [-lmax $nr] [-dmax $nr]
.I [-leave_level $nr]
.I [-maxsize $nr] [-minsize $nr]
.I [-asite $list] [-dsite $list]
.I [-adomain $list] [-ddomain $list]
.I [-asfx $list] [-dsfx $list]
.I [-aprefix $list] [-dprefix $list]
.I [-amimt $list] [-dmimet $list]
.I [-pattern $pattern] [-url_pattern $pattern]
.I [-rpattern $regexp] [-url_rpattern $regexp]
.I [-skip_pattern $pattern] [-skip_url_pattern $pattern]
.I [-skip_rpattern $regexp] [-skip_url_rpattern $regexp]
.I [-newer_than $time] [-older_than $time]
.I [-schedule $time] [-reschedule $nr]
.I [-dont_leave_site/-leave_site] [-dont_leave_dir/-leave_dir]
.I [-http_proxy $site[:$port]] [-ftp_proxy $site[:$port]]
.I [-ssl_proxy $site[:$port]] [-gopher_proxy $site[:$port]]
.I [-ftp_httpgw/-noftp_httpgw] [-ftp_dirtyproxy/-noftp_dirtyproxy]
.I [-gopher_httpgw/-nogopher_httpgw]
.I [-noFTP/-FTP] [-noHTTP/-HTTP] [-noSSL/-SSL] [-noGopher/-Gopher]
.I [-FTPdir/-noFTPdir]
.I [-noCGI/-CGI]
.I [-FTPlist/-noFTPlist] [-FTPhtml/-noFTPhtml]
.I [-noRelocate/-Relocate] [-force_reget/-noforce_reget]
.I [-nocache/-cache] [-check_size/-nocheck_size]
.I [-noRobots/-Robots] [-noEnc/-Enc]
.I [-auth_name $user] [-auth_passwd $pass] [-auth_scheme 1/2/3/4/user/Basic/Digest/NTLM]
.I [-auth_reuse_nonce/-no_auth_reuse_nonce]
.I [-http_proxy_user $user] [-http_proxy_pass $pass] [-http_proxy_auth 1/2/3/4/user/Basic/Digest/NTLM]
.I [-auth_reuse_proxy_nonce/-no_auth_reuse_proxy_nonce]
.I [-ssl_key_file $file] [-ssl_cert_file $file] [-ssl_cert_passwd $pass]
.I [-from $email] [-send_from/-nosend_from] [-identity $str] [-auto_referer/-noauto_referer]
.I [-referer/-noreferer]
.I [-alang $list] [-acharset $list]
.I [-retry $nr] [-nregets $nr] [-nredirs $nr] [-rollback $nr] [-sleep $nr] [-timeout $nr]
.I [-preserve_time/-nopreserve_time] [-preserve_perm/-nopreserve_perm] [-preserve_slinks/-nopreserve_slinks]
.I [-bufsize $nr] [-maxrate $nr] [-minrate $nr]
.I [-user_condition $str]
.I [-cookie_file $file] [-cookie_send/-nocookie_send] [-cookie_recv/-nocookie_recv] [-cookie_update/-nocookie_update] [-cookies_max $nr] [-disabled_cookie_domains $list]
.I [-disable_html_tag $TAG,[$ATTRIB][;...]] [-enable_html_tag $TAG,[$ATTRIB][;...]]
.I [-tr_del_chr $str] [-tr_str_str $str1 $str2] [-tr_chr_chr $chrset1 $chrset2]
.I [-index_name $str] [-store_index/-nostore_index]
.I [-store_name $str] [-debug/-nodebug] [-debug_level $level]
.I [-browser $str]
.I [-urls_file $file]
.I [-file_quota $nr] [-trans_quota $nr] [-fs_quota $nr] [-enable_js/-disable_js]
.I [-fnrules $t $m $r] [-store_info/-nostore_info]
.I [-all_to_local/-noall_to_local] [-sel_to_local/-nosel_to_local]
.I [-all_to_remote/-noall_to_remote]
.I [-url_strategie $strategie]
.I [-remove_adv/-noremove_adv] [-adv_re $RE]
.I [-check_bg/-nocheck_bg]
.I [-send_if_range/-nosend_if_range]
.I [-sched_cmd $str]
.I [-unique_log/-nounique_log]
.I [-post_cmd $str] [-ssl_version $v] [-unique_sslid/-nounique_sslid]
.I [-aip_pattern $re] [-dip_pattern $re]
.I [-use_http11/-nouse_http11]
.I [-local_ip $addr]
.I [-request $req]
.I [-formdata $req] [-httpad $str]
.I [-nthreads $nr] [-immesg/-noimmesg]
.I [-dumpfd $nr] [-dump_urlfd $nr]
.I [-unique_name/-nounique_name]
.I [-leave_site_enter_dir/-dont_leave_site_enter_dir]
.I [-max_time $nr]
.I [-del_after/-nodel_after] [-singlepage/-nosinglepage]
.I [-dump_after/-nodump_after] [-dump_response/-nodump_response]
.I [-auth_ntlm_domain $str] [-auth_proxy_ntlm_domain $str]
.I [-js_pattern $re] [-follow_cmd $str]
.I [-retrieve_symlink/-noretrieve_symlink]
.I [-js_transform $p $t $h $a] [-js_transform2 $p $t $h $a]
.I [-ftp_proxy_user $str] [-ftp_proxy_pass $str]
.I [-limit_inlines/-dont_limit_inlines]
.I [-ftp_list_options $str] [-fix_wuftpd_list/-nofix_wuftpd_list]
.I [-post_update/-nopost_update] [-info_dir $dir]
.I [-mozcache_dir $dir] [-aport $list] [-dport $list]
.I [-hack_add_index/-nohack_add_index]
.I [-default_prefix $str] [-rsleep/-norsleep]
.I [-ftp_login_handshake $host $handshake]
.I [-js_script_file $file]
.I [-dont_touch_url_pattern $pat]
.I [-dont_touch_url_rpattern $pat]
.I [-dont_touch_tag_rpattern $pat]
.I [-tag_pattern $tag $attrib $url]
.I [-tag_rpattern $tag $attrib $url]
.I [-nss_cert_dir $dir]
.I [-nss_accept_unknown_cert/-nonss_accept_unknown_cert]
.I [-nss_domestic_policy/-nss_export_policy]
.I [-[no]verify]
.I [-tlogfile $file]
.I [-trelative {object | program}]
.I [-transparent_proxy FQDN[:port]]
.I [-transparent_ssl_proxy FQDN[:port]]
.I [-sdemo]
.I [-noencode]
.I [URLs]
.br
.PP
.B pavuk -mode {normal | singlepage | singlereget}
.I [-base_level $nr]
.br
.PP
.B pavuk -mode sync
.I [-ddays $nr] [-subdir $dir] [-remove_old/-noremove_old]
.br
.PP
.B pavuk -mode resumeregets
.I [-subdir $dir]
.br
.PP
.B pavuk -mode linkupdate
.I [-X] [-h] [-v] [-cdir $dir] [-subdir $dir] [-scndir $dir] [-scenario $str]
.PP
.B pavuk -mode reminder
.I [-remind_cmd $str]
.br
.PP
.B pavuk -mode mirror
.I [-subdir $dir] [-remove_old/-noremove_old] [-remove_before_store/-noremove_before_store] [-always_mdtm/-noalways_mdtm]
.br
.PP
.SH DESCRIPTION
.sp
This manual page describes how to use pavuk. Pavuk can be used to
mirror contents of internet/intranet servers and to maintain copies in
a local tree of documents.  Pavuk stores retrieved documents in
locally mapped disk space. The structure of the local tree is the same
as the one on the remote server. Each supported service (protocol) has
its own subdirectory in the local tree.  Each referenced server has
its own subdirectory in these protocols subdirectories; followed by
the port number on which the service resides, delimited by character
'_'. Then the directory structure follows.  As of version 0.9pl19 this
can be be changed. With the option
.B -fnrules
you can change the default layout of the local document tree,
without losing link consistency.
.br
With
.B pavuk
it is possible to have up-to-date copies of remote documents in the
local disk space.
.br
As of version 0.3pl2, pavuk can automatically restart broken connections, and reget
partial content from an FTP server (which must support the
.B REST
command), from a properly configured HTTP/1.1 server, or from a HTTP/1.0
server which supports
.B Ranges.
.br
As of version 0.6 it is possible to handle configurations via so
called scenarios.  The best way to create such a configuration file is
to use the X Window interface and simply save the created
configuration. The other way is to use the -dumpscn switch.
.br
As of version 0.7pl1 it is possible to store authentification information
into an authinfo file, which pavuk can then parse and use.
.br
As of version 0.8pl4 pavuk can fetch documents for use in a local
proxy/cache server without storing them to local documents tree.
.br
As of version 0.9pl4 pavuk supports
.B SOCKS (4/5)
proxies if you have the required libraries.
.br
As of version 0.9pl12 pavuk can preserve permissions of remote files and
symbolic links, so it can be used for powerful FTP mirroring.
.br
Pavuk supports SSL connections to FTP servers, if you specify ftps://
URL instead of ftp://.
.br
Pavuk can automatically handle file names with unsafe characters for filesystem.
This is yet implemented only for Win32 platform and it is hard coded.
.br
Pavuk can now use \fBHTTP/1.1\fR protocol for communication with HTTP servers.
It can use persistent connections, so one TCP connection should be used to
transfer several documents without closing it. This feature saves network
bandwidth and also speedup network communication.
.br
Pavuk can do configurable \fBPOST\fR requests to HTTP servers and support
also file uploading via HTTP POST request.
.br
Pavuk can automatically fill found HTML forms, if user will supply data
for its fields before with option \fB-formdata\fR.
.br
Pavuk can run configurable number of concurrently running downloading threads when
compiled with multithreading support.

.SH Format of supported URLs
.sp
.sp
\fIHTTP\fR
.br
http://[[user][:password]@]host[:port][/document]
.br
[[user][:password]@]host[:port][/document]
.sp
\fIHTTPS\fR
.br
https://[[user][:password]@]host[:port][/document]
.br
ssl[.domain][:port][/document]
.sp
\fIFTP\fR
.br
ftp://[[user][:password]@]host[:port][/relative_path][;type=x]
.br
ftp://[[user][:password]@]host[:port][//absolute_path][;type=x]
.br
ftp[.domain][:port][/document][;type=x]
.sp
\fIFTPS\fR
.br
ftps://[[user][:password]@]host[:port][/relative_path][;type=x]
.br
ftps://[[user][:password]@]host[:port][//absolute_path][;type=x]
.br
ftps[.domain][:port][/document][;type=x]
.sp
\fIGopher\fR
.br
gopher://host[:port][/type[document]]
.br
gopher[.domain][:port][/type[document]]

.SH Default mapping of URLs to local filenames
.sp
.sp
\fIHTTP\fR
.br
http://[[user][:password]@]host[:port][/document][?query]
.br
to
.br
http/host_port/[document][?query]
.sp
\fIHTTPS\fR
.br
https://[[user][:password]@]host[:port][/document][?query]
.br
to
.br
https/host_port/[document][?query]
.sp
\fIFTP\fR
.br
ftp://[[user][:password]@]host[:port][/path]
.br
to
.br
ftp/host_port/[path]
.sp
\fIFTPS\fR
.br
ftps://[[user][:password]@]host[:port][/path]
.br
to
.br
ftps/host_port/[path]
.sp
\fIGopher\fR
.br
gopher://host[:port][/type[document]]
.br
to
.br
gopher/host_port/[type[document]]

\fINOTE:\fR Pavuk will use the string with which it queries the target
server as the name of the results file. This file name may, in some cases,
contain punctuations such as \fB$,?,=,&\fR etc. Such punctuation can cause
problems when you are trying to browse downloaded files with your browser or
you are trying to process downloaded files with shell scripts or view files
with file management utilities which reference the name of the results file.
If you believe that this maybe causing problems for you, then you can remove
all punctuation from the result file name with the option:
\fB-tr_del_chr [:punct:]\fR or with other options for adjusting filenames.


.SH OPTIONS
.sp
 All options are case insensitive.

.SH List of options chapters
.sp
.I Mode
.br
.I Help
.br
.I Indicate/Logging/Interface options
.br
.I Netli options
.br
.I Special start
.br
.I Scenario/Task options
.br
.I Directory options
.br
.I Preserve options
.br
.I Proxy options
.br
.I Proxy Authentification
.br
.I Protocol/Download Option
.br
.I Authentification
.br
.I Site/Domain/Port Limitation Options
.br
.I Limitation Document properties
.br
.I Limitation Document name
.br
.I Limitation Protocol Option
.br
.I Other Limitation Options
.br