📄 webautomation.mht
字号:
color=3D#00cd00>'me</FONT><FONT color=3D#cdad00>@wherever</FONT><FONT =
color=3D#00cd00>.com'</FONT><FONT color=3D#00ffff>);</FONT>
<FONT color=3D#bebebe>#-----------------------------
</FONT><FONT color=3D#cdcd00>403</FONT> <FONT =
color=3D#00ffff>(</FONT>Forbidden<FONT color=3D#00ffff>)</FONT> =
Forbidden by robots.txt
<FONT color=3D#bebebe>#-----------------------------
#% GET http://www.webtechniques.com/robots.txt</FONT><FONT =
color=3D#43cd80> </FONT><FONT color=3D#bebebe>
#User-agent: *
#
# Disallow: /stats
#
# Disallow: /db
#
# Disallow: /logs
#
# Disallow: /store
#
# Disallow: /forms
#
# Disallow: /gifs
#
# Disallow: /wais-src
#
# Disallow: /scripts
#
# Disallow: /config
#-----------------------------
#% GET http://www.cnn.com/robots.txt | head
## robots, scram
#
## $I d : robots.txt,v 1.2 1998/03/10 18:27:01 mreed Exp $
#
#User-agent: *
#
#Disallow: /
#
#User-agent: Mozilla/3.01 (hotwired-test/0.1)
#
#Disallow: /cgi-bin
#
#Disallow: /TRANSCRIPTS
#
#Disallow: /development
#-----------------------------
</FONT></PRE></FONT></TD></TR></TBODY></TABLE></DIV>
<DIV class=3DSECT2>
<H2 class=3DSECT2><A name=3DAEN1090>Parsing a Web Server Log =
File</A></H2>
<TABLE width=3D"100%" bgColor=3D#2f4f4f border=3D0>
<TBODY>
<TR>
<TD><PRE class=3DSCREEN><FONT color=3D#f5deb3 size=3D+1><FONT =
color=3D#bebebe>#-----------------------------
</FONT><FONT color=3D#ffa500>while</FONT> <FONT =
color=3D#00ffff>(</FONT><LOGFILE><FONT color=3D#00ffff>)</FONT> =
<FONT color=3D#00ffff>{</FONT>
<FONT color=3D#ffa500>my</FONT> <FONT color=3D#00ffff>(</FONT><FONT =
color=3D#9ac0cd>$client</FONT>, <FONT color=3D#9ac0cd>$identuser</FONT>, =
<FONT color=3D#9ac0cd>$authuser</FONT>, <FONT =
color=3D#9ac0cd>$date</FONT>, <FONT color=3D#9ac0cd>$time</FONT>, <FONT =
color=3D#9ac0cd>$tz</FONT>, <FONT color=3D#9ac0cd>$method</FONT>,
$url, $protocol, $status, $bytes<FONT color=3D#00ffff>)</FONT> =3D
/^<FONT color=3D#00ffff>(</FONT>\S+<FONT color=3D#00ffff>)</FONT> =
<FONT color=3D#00ffff>(</FONT>\S+<FONT color=3D#00ffff>)</FONT> <FONT =
color=3D#00ffff>(</FONT>\S+<FONT color=3D#00ffff>)</FONT> \<FONT =
color=3D#00ffff>[([</FONT>^:<FONT color=3D#00ffff>]</FONT>+<FONT =
color=3D#00ffff>)</FONT>:<FONT color=3D#00ffff>(</FONT>\d+:\d+:\d+<FONT =
color=3D#00ffff>)</FONT> <FONT color=3D#00ffff>([</FONT>^\<FONT =
color=3D#00ffff>]]</FONT>+<FONT color=3D#00ffff>)</FONT> <FONT =
color=3D#00cd00>"(\S+) (.*?) (\S+)"</FONT> <FONT =
color=3D#00ffff>(</FONT>\S+<FONT color=3D#00ffff>)</FONT> <FONT =
color=3D#00ffff>(</FONT>\S+<FONT color=3D#00ffff>)</FONT>$/ <FONT =
color=3D#98fb98>or</FONT> <FONT color=3D#ffa500>next</FONT><FONT =
color=3D#00ffff>;</FONT>
<FONT color=3D#bebebe># ...
</FONT><FONT color=3D#00ffff>}</FONT>
<FONT color=3D#bebebe>#-----------------------------
</FONT></PRE></FONT></TD></TR></TBODY></TABLE></DIV>
<DIV class=3DSECT2>
<H2 class=3DSECT2><A name=3DAEN1093>Processing Server Logs</A></H2>
<TABLE width=3D"100%" bgColor=3D#2f4f4f border=3D0>
<TBODY>
<TR>
<TD><PRE class=3DSCREEN><FONT color=3D#f5deb3 size=3D+1><FONT =
color=3D#bebebe>#-----------------------------
# <FONT size=3D-1><A =
href=3D"http://pleac.sourceforge.net/include/perl/ch20/sumwww">download =
the following standalone program</A></FONT>
#!/usr/bin/perl -w
# sumwww - summarize web server log activity
</FONT>
$lastdate =3D <FONT color=3D#00cd00>""</FONT><FONT =
color=3D#00ffff>;</FONT>
daily_logs<FONT color=3D#00ffff>();</FONT>
summary<FONT color=3D#00ffff>();</FONT>
<FONT color=3D#ffa500>exit</FONT><FONT color=3D#00ffff>;</FONT>
<FONT color=3D#bebebe># read CLF files and tally hits from the host and =
to the URL
</FONT><FONT color=3D#ffa500>sub</FONT> <FONT =
color=3D#b2dfee>daily_logs</FONT> <FONT color=3D#00ffff>{</FONT>
<FONT color=3D#ffa500>while</FONT> <FONT =
color=3D#00ffff>(</FONT><><FONT color=3D#00ffff>)</FONT> <FONT =
color=3D#00ffff>{</FONT>
<FONT color=3D#00ffff>(</FONT>$type, $what<FONT =
color=3D#00ffff>)</FONT> =3D /<FONT color=3D#00cd00>"(GET|POST)\s+(\S+?) =
\S+"</FONT>/ <FONT color=3D#98fb98>or</FONT> <FONT =
color=3D#ffa500>next</FONT><FONT color=3D#00ffff>;</FONT>
<FONT color=3D#00ffff>(</FONT>$host, <FONT =
color=3D#ff7f50>undef</FONT>, <FONT color=3D#ff7f50>undef</FONT>, =
$datetime<FONT color=3D#00ffff>)</FONT> =3D <FONT =
color=3D#ff7f50>split</FONT><FONT color=3D#00ffff>;</FONT>
<FONT color=3D#00ffff>(</FONT>$bytes<FONT =
color=3D#00ffff>)</FONT> =3D /\s<FONT color=3D#00ffff>(</FONT>\d+<FONT =
color=3D#00ffff>)</FONT>\s*$/ <FONT color=3D#98fb98>or</FONT> <FONT =
color=3D#ffa500>next</FONT><FONT color=3D#00ffff>;</FONT>
<FONT color=3D#00ffff>(</FONT>$date<FONT =
color=3D#00ffff>)</FONT> =3D <FONT color=3D#00ffff>(</FONT>$datetime =
=3D~ /\<FONT color=3D#00ffff>[([</FONT>^:<FONT =
color=3D#00ffff>]</FONT>*<FONT color=3D#00ffff>)</FONT>/<FONT =
color=3D#00ffff>);</FONT>
$posts +=3D <FONT color=3D#00ffff>(</FONT>$type <FONT =
color=3D#98fb98>eq</FONT> POST<FONT color=3D#00ffff>);</FONT>
$home++ <FONT color=3D#ffa500>if</FONT> <FONT =
color=3D#ff7f50>m</FONT>, / ,<FONT color=3D#00ffff>;</FONT>
<FONT color=3D#ffa500>if</FONT> <FONT =
color=3D#00ffff>(</FONT>$date <FONT color=3D#98fb98>ne</FONT> =
$lastdate<FONT color=3D#00ffff>)</FONT> <FONT color=3D#00ffff>{</FONT>
<FONT color=3D#ffa500>if</FONT> <FONT =
color=3D#00ffff>(</FONT>$lastdate<FONT color=3D#00ffff>)</FONT> <FONT =
color=3D#00ffff>{</FONT> <FONT color=3D#98fb98>write</FONT>_report<FONT =
color=3D#00ffff>()</FONT> <FONT color=3D#00ffff>}</FONT>
<FONT color=3D#ffa500>else</FONT> <FONT =
color=3D#00ffff>{</FONT> $lastdate =3D $date <FONT =
color=3D#00ffff>}</FONT>
<FONT color=3D#00ffff>}</FONT>
$count++<FONT color=3D#00ffff>;</FONT>
<FONT color=3D#cdcd00>$hosts</FONT><FONT =
color=3D#00ffff>{</FONT>$host<FONT color=3D#00ffff>}</FONT>++<FONT =
color=3D#00ffff>;</FONT>
<FONT color=3D#cdcd00>$what</FONT><FONT =
color=3D#00ffff>{</FONT>$what<FONT color=3D#00ffff>}</FONT>++<FONT =
color=3D#00ffff>;</FONT>
$bytesum +=3D $bytes<FONT color=3D#00ffff>;</FONT>
<FONT color=3D#00ffff>}</FONT>
<FONT color=3D#98fb98>write</FONT>_report<FONT =
color=3D#00ffff>()</FONT> <FONT color=3D#ffa500>if</FONT> $count<FONT =
color=3D#00ffff>;</FONT>
<FONT color=3D#00ffff>}</FONT>
<FONT color=3D#bebebe># use *typeglob aliasing of global variables for =
cheap copy
</FONT><FONT color=3D#ffa500>sub</FONT> <FONT =
color=3D#b2dfee>summary</FONT> <FONT color=3D#00ffff>{</FONT>
$lastdate =3D <FONT color=3D#00cd00>"Grand Total"</FONT><FONT =
color=3D#00ffff>;</FONT>
*count =3D *sumcount<FONT color=3D#00ffff>;</FONT>
*bytesum =3D *bytesumsum<FONT color=3D#00ffff>;</FONT>
*hosts =3D *allhosts<FONT color=3D#00ffff>;</FONT>
*posts =3D *allposts<FONT color=3D#00ffff>;</FONT>
*what =3D *allwhat<FONT color=3D#00ffff>;</FONT>
*home =3D *allhome<FONT color=3D#00ffff>;</FONT>
<FONT color=3D#98fb98>write</FONT><FONT color=3D#00ffff>;</FONT>
<FONT color=3D#00ffff>}</FONT>
<FONT color=3D#bebebe># display the tallies of hosts and URLs, using =
formats
</FONT><FONT color=3D#ffa500>sub</FONT> <FONT =
color=3D#98fb98>write</FONT>_report <FONT color=3D#00ffff>{</FONT>
<FONT color=3D#98fb98>write</FONT><FONT color=3D#00ffff>;</FONT>
<FONT color=3D#bebebe># add to summary data
</FONT> $lastdate =3D $date<FONT color=3D#00ffff>;</FONT>
$sumcount +=3D $count<FONT color=3D#00ffff>;</FONT>
$bytesumsum +=3D $bytesum<FONT color=3D#00ffff>;</FONT>
$allposts +=3D $posts<FONT color=3D#00ffff>;</FONT>
$allhome +=3D $home<FONT color=3D#00ffff>;</FONT>
<FONT color=3D#bebebe># reset daily data
</FONT> $posts =3D $count =3D $bytesum =3D $home =3D <FONT =
color=3D#cdcd00>0</FONT><FONT color=3D#00ffff>;</FONT>
<FONT color=3D#cdcd00>@allwhat</FONT><FONT =
color=3D#00ffff>{</FONT><FONT color=3D#ff7f50>keys</FONT> <FONT =
color=3D#cdcd00>%what</FONT><FONT color=3D#00ffff>}</FONT> =3D <FONT =
color=3D#ff7f50>keys</FONT> <FONT color=3D#cdcd00>%what</FONT><FONT =
color=3D#00ffff>;</FONT>
<FONT color=3D#cdcd00>@allhosts</FONT><FONT =
color=3D#00ffff>{</FONT><FONT color=3D#ff7f50>keys</FONT> <FONT =
color=3D#cdcd00>%hosts</FONT><FONT color=3D#00ffff>}</FONT> =3D <FONT =
color=3D#ff7f50>keys</FONT> <FONT color=3D#cdcd00>%hosts</FONT><FONT =
color=3D#00ffff>;</FONT>
<FONT color=3D#cdcd00>%hosts</FONT> =3D <FONT =
color=3D#cdcd00>%what</FONT> =3D <FONT color=3D#00ffff>();</FONT>
<FONT color=3D#00ffff>}</FONT>
<FONT color=3D#ff7f50>format</FONT> <FONT =
color=3D#b2dfee>STDOUT_TOP</FONT> =3D
@|||||||||| @|||||| @||||||| @||||||| @|||||| @|||||| @|||||||||||||
<FONT color=3D#00cd00>"Date"</FONT>, <FONT =
color=3D#00cd00>"Hosts"</FONT>, <FONT color=3D#00cd00>"Accesses"</FONT>, =
<FONT color=3D#00cd00>"Unidocs"</FONT>, <FONT =
color=3D#00cd00>"POST"</FONT>, <FONT color=3D#00cd00>"Home"</FONT>, =
<FONT color=3D#00cd00>"Bytes"</FONT>
----------- ------- -------- -------- ------- ------- --------------
.
<FONT color=3D#ff7f50>format</FONT> <FONT color=3D#b2dfee>STDOUT</FONT> =
=3D
@>>>>>>>>>> @>>>>>> =
@>>>>>>> @>>>>>>> =
@>>>>>> @>>>>>> =
@>>>>>>>>>>>>>
$lastdate, <FONT color=3D#ff7f50>scalar</FONT><FONT =
color=3D#00ffff>(</FONT><FONT color=3D#ff7f50>keys</FONT> <FONT =
color=3D#cdcd00>%hosts</FONT><FONT color=3D#00ffff>)</FONT>,<FONT =
color=3D#43cd80> </FONT>
$count, <FONT color=3D#ff7f50>scalar</FONT><FONT =
color=3D#00ffff>(</FONT><FONT color=3D#ff7f50>keys</FONT> <FONT =
color=3D#cdcd00>%what</FONT><FONT color=3D#00ffff>)</FONT>,<FONT =
color=3D#43cd80> </FONT>
$posts, $home, $bytesum
.
<FONT color=3D#bebebe>#-----------------------------
# Date Hosts Accesses Unidocs POST Home Bytes
#</FONT><FONT color=3D#43cd80> </FONT><FONT color=3D#bebebe>
# ----------- ------- -------- -------- ------- ------- --------------
#</FONT><FONT color=3D#43cd80> </FONT><FONT color=3D#bebebe>
# 19/May/1998 353 6447 3074 352 51 16058246
#</FONT><FONT color=3D#43cd80> </FONT><FONT color=3D#bebebe>
# 20/May/1998 1938 23868 4288 972 350 61879643
#</FONT><FONT color=3D#43cd80> </FONT><FONT color=3D#bebebe>
# 21/May/1998 1775 27872 6596 1064 376 64613798
#</FONT><FONT color=3D#43cd80> </FONT><FONT color=3D#bebebe>
# 22/May/1998 1680 21402 4467 735 285 52437374
#</FONT><FONT color=3D#43cd80> </FONT><FONT color=3D#bebebe>
# 23/May/1998 1128 21260 4944 592 186 55623059
#</FONT><FONT color=3D#43cd80> </FONT><FONT color=3D#bebebe>
# Grand Total 6050 100849 10090 3715 1248 250612120
#-----------------------------
# <FONT size=3D-1><A =
href=3D"http://pleac.sourceforge.net/include/perl/ch20/aprept">download =
the following standalone program</A></FONT>
#!/usr/bin/perl -w
# aprept - report on Apache logs
</FONT>
<FONT color=3D#ffa500>use</FONT> <FONT =
color=3D#b2dfee>Logfile::Apache</FONT><FONT color=3D#00ffff>;</FONT>
$l =3D Logfile::Apache->new<FONT color=3D#00ffff>(</FONT>
<FONT color=3D#00cd00>File</FONT> =3D> <FONT =
color=3D#00cd00>"-"</FONT>, <FONT color=3D#bebebe># =
STDIN
</FONT> <FONT color=3D#00cd00>Group</FONT> =3D> <FONT =
color=3D#00ffff>[</FONT> Domain, File <FONT color=3D#00ffff>]);</FONT>
$l->report<FONT color=3D#00ffff>(</FONT><FONT =
color=3D#00cd00>Group</FONT> =3D> Domain, <FONT =
color=3D#00cd00>Sort</FONT> =3D> Records<FONT =
color=3D#00ffff>);</FONT>
$l->report<FONT color=3D#00ffff>(</FONT><FONT =
color=3D#00cd00>Group</FONT> =3D> File, <FONT =
color=3D#00cd00>List</FONT> =3D> <FONT =
color=3D#00ffff>[</FONT>Bytes,Records<FONT color=3D#00ffff>]);</FONT>
<FONT color=3D#bebebe>#-----------------------------
# Domain Records</FONT><FONT color=3D#43cd80> =
</FONT><FONT color=3D#be⌨️ 快捷键说明
复制代码
Ctrl + C
搜索代码
Ctrl + F
全屏模式
F11
切换主题
Ctrl + Shift + D
显示快捷键
?
增大字号
Ctrl + =
减小字号
Ctrl + -