statisticssummary.html

一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫

<A NAME="totalDnsMimeSize"><!-- --></A><H3>
totalDnsMimeSize</H3>
<PRE>
protected long <B>totalDnsMimeSize</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="totalHostSize"><!-- --></A><H3>
totalHostSize</H3>
<PRE>
protected long <B>totalHostSize</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="totalDnsHostSize"><!-- --></A><H3>
totalDnsHostSize</H3>
<PRE>
protected long <B>totalDnsHostSize</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="totalTldDocuments"><!-- --></A><H3>
totalTldDocuments</H3>
<PRE>
protected long <B>totalTldDocuments</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="totalTldSize"><!-- --></A><H3>
totalTldSize</H3>
<PRE>
protected long <B>totalTldSize</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="totalHosts"><!-- --></A><H3>
totalHosts</H3>
<PRE>
protected long <B>totalHosts</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="durationTime"><!-- --></A><H3>
durationTime</H3>
<PRE>
protected java.lang.String <B>durationTime</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="processedDocsPerSec"><!-- --></A><H3>
processedDocsPerSec</H3>
<PRE>
protected java.lang.String <B>processedDocsPerSec</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="bandwidthKbytesPerSec"><!-- --></A><H3>
bandwidthKbytesPerSec</H3>
<PRE>
protected java.lang.String <B>bandwidthKbytesPerSec</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="totalDataWritten"><!-- --></A><H3>
totalDataWritten</H3>
<PRE>
protected java.lang.String <B>totalDataWritten</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="mimeTypeDistribution"><!-- --></A><H3>
mimeTypeDistribution</H3>
<PRE>
protected java.util.Hashtable <B>mimeTypeDistribution</B></PRE>
<DL>
<DD>Keep track of the file types we see (mime type -> count)
<P>
<DL>
</DL>
</DL>
<HR>

<A NAME="mimeTypeBytes"><!-- --></A><H3>
mimeTypeBytes</H3>
<PRE>
protected java.util.Hashtable <B>mimeTypeBytes</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="mimeTypeDnsDistribution"><!-- --></A><H3>
mimeTypeDnsDistribution</H3>
<PRE>
protected java.util.Hashtable <B>mimeTypeDnsDistribution</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="mimeTypeDnsBytes"><!-- --></A><H3>
mimeTypeDnsBytes</H3>
<PRE>
protected java.util.Hashtable <B>mimeTypeDnsBytes</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="statusCodeDistribution"><!-- --></A><H3>
statusCodeDistribution</H3>
<PRE>
protected java.util.Hashtable <B>statusCodeDistribution</B></PRE>
<DL>
<DD>Keep track of status codes
<P>
<DL>
</DL>
</DL>
<HR>

<A NAME="dnsStatusCodeDistribution"><!-- --></A><H3>
dnsStatusCodeDistribution</H3>
<PRE>
protected java.util.Hashtable <B>dnsStatusCodeDistribution</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="hostsDistribution"><!-- --></A><H3>
hostsDistribution</H3>
<PRE>
protected java.util.Hashtable <B>hostsDistribution</B></PRE>
<DL>
<DD>Keep track of hosts
<P>
<DL>
</DL>
</DL>
<HR>

<A NAME="hostsBytes"><!-- --></A><H3>
hostsBytes</H3>
<PRE>
protected java.util.Hashtable <B>hostsBytes</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="hostsDnsDistribution"><!-- --></A><H3>
hostsDnsDistribution</H3>
<PRE>
protected java.util.Hashtable <B>hostsDnsDistribution</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="hostsDnsBytes"><!-- --></A><H3>
hostsDnsBytes</H3>
<PRE>
protected java.util.Hashtable <B>hostsDnsBytes</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="tldDistribution"><!-- --></A><H3>
tldDistribution</H3>
<PRE>
protected java.util.Hashtable <B>tldDistribution</B></PRE>
<DL>
<DD>Keep track of TLDs
<P>
<DL>
</DL>
</DL>
<HR>

<A NAME="tldBytes"><!-- --></A><H3>
tldBytes</H3>
<PRE>
protected java.util.Hashtable <B>tldBytes</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="tldHostDistribution"><!-- --></A><H3>
tldHostDistribution</H3>
<PRE>
protected java.util.Hashtable <B>tldHostDistribution</B></PRE>
<DL>
<DL>
</DL>
</DL>
<HR>

<A NAME="processedSeedsRecords"><!-- --></A><H3>
processedSeedsRecords</H3>
<PRE>
protected transient java.util.Map <B>processedSeedsRecords</B></PRE>
<DL>
<DD>Keep track of processed seeds
<P>
<DL>
</DL>
</DL>

<!-- ========= CONSTRUCTOR DETAIL ======== -->

<A NAME="constructor_detail"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="1"><FONT SIZE="+2">
<B>Constructor Detail</B></FONT></TH>
</TR>
</TABLE>

<A NAME="StatisticsSummary(org.archive.crawler.admin.CrawlJob)"><!-- --></A><H3>
StatisticsSummary</H3>
<PRE>
public <B>StatisticsSummary</B>(<A HREF="../../../../org/archive/crawler/admin/CrawlJob.html" title="class in org.archive.crawler.admin">CrawlJob</A>&nbsp;cjob)</PRE>
<DL>
<DD>Constructor
<P>
<DL>
<DT><B>Parameters:</B><DD><CODE>cjob</CODE> - Completed crawl job</DL>
</DL>

<!-- ============ METHOD DETAIL ========== -->

<A NAME="method_detail"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="1"><FONT SIZE="+2">
<B>Method Detail</B></FONT></TH>
</TR>
</TABLE>

<A NAME="incrementMapCount(java.util.Map, java.lang.String)"><!-- --></A><H3>
incrementMapCount</H3>
<PRE>
protected static void <B>incrementMapCount</B>(java.util.Map&nbsp;map,
                                        java.lang.String&nbsp;key)</PRE>
<DL>
<DD>Increment a counter for a key in a given HashMap. Used for various
 aggregate data.
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>map</CODE> - The HashMap<DD><CODE>key</CODE> - The key for the counter to be incremented, if it does not
               exist it will be added (set to 1).  If null it will
            increment the counter "unknown".</DL>
</DD>
</DL>
<HR>

<A NAME="incrementMapCount(java.util.Map, java.lang.String, long)"><!-- --></A><H3>
incrementMapCount</H3>
<PRE>
protected static void <B>incrementMapCount</B>(java.util.Map&nbsp;map,
                                        java.lang.String&nbsp;key,
                                        long&nbsp;increment)</PRE>
<DL>
<DD>Increment a counter for a key in a given HashMap by an arbitrary amount.
 Used for various aggregate data. The increment amount can be negative.
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>map</CODE> - The HashMap<DD><CODE>key</CODE> - The key for the counter to be incremented, if it does not
            exist it will be added (set to equal to
            <code>increment</code>).
            If null it will increment the counter "unknown".<DD><CODE>increment</CODE> - The amount to increment counter related to the
            <code>key</code>.</DL>
</DD>
</DL>
<HR>

<A NAME="getMimeDistribution()"><!-- --></A><H3>
getMimeDistribution</H3>
<PRE>
public java.util.Hashtable <B>getMimeDistribution</B>()</PRE>
<DL>
<DD>Returns a HashMap that contains information about distributions of
  encountered mime types.  Key/value pairs represent
  mime type -> count.
 <p>
 <b>Note:</b> All the values are wrapped with a
 <A HREF="../../../../org/archive/util/LongWrapper.html" title="class in org.archive.util"><CODE>LongWrapper</CODE></A>
<P>
<DD><DL>

<DT><B>Returns:</B><DD>mimeTypeDistribution</DL>
</DD>
</DL>
<HR>

<A NAME="getTotalMimeTypeDocuments()"><!-- --></A><H3>
getTotalMimeTypeDocuments</H3>
<PRE>
public long <B>getTotalMimeTypeDocuments</B>()</PRE>
<DL>
<DD><DL>
</DL>
</DD>
</DL>
<HR>

<A NAME="getTotalDnsMimeTypeDocuments()"><!-- --></A><H3>
getTotalDnsMimeTypeDocuments</H3>
<PRE>
public long <B>getTotalDnsMimeTypeDocuments</B>()</PRE>
<DL>
<DD><DL>
</DL>
</DD>
</DL>
<HR>

<A NAME="getTotalMimeSize()"><!-- --></A><H3>
getTotalMimeSize</H3>
<PRE>
public long <B>getTotalMimeSize</B>()</PRE>
<DL>
<DD><DL>
</DL>
</DD>
</DL>
<HR>

<A NAME="getTotalDnsMimeSize()"><!-- --></A><H3>
getTotalDnsMimeSize</H3>
<PRE>
public long <B>getTotalDnsMimeSize</B>()</PRE>
<DL>
<DD><DL>
</DL>
</DD>
</DL>
<HR>

<A NAME="getStatusCodeDistribution()"><!-- --></A><H3>
getStatusCodeDistribution</H3>
<PRE>
public java.util.Hashtable <B>getStatusCodeDistribution</B>()</PRE>
<DL>
<DD>Return a HashMap representing the distribution of HTTP status codes for
 successfully fetched curis, as represented by a hashmap where key -&gt;
 val represents (string)code -&gt; (integer)count.
 
 <b>Note: </b> All the values are wrapped with a
 <A HREF="../../../../org/archive/util/LongWrapper.html" title="class in org.archive.util"><CODE>LongWrapper</CODE></A>
<P>
<DD><DL>

<DT><B>Returns:</B><DD>statusCodeDistribution</DL>
</DD>
</DL>
<HR>

<A NAME="getDnsStatusCodeDistribution()"><!-- --></A><H3>
getDnsStatusCodeDistribution</H3>
<PRE>
public java.util.Hashtable <B>getDnsStatusCodeDistribution</B>()</PRE>
<DL>
<DD>Return a HashMap representing the distribution of DNS status codes for
 successfully fetched curis, as represented by a hashmap where key -&gt;
 val represents (string)code -&gt; (integer)count.
 
 <b>Note: </b> All the values are wrapped with a
 <A HREF="../../../../org/archive/util/LongWrapper.html" title="class in org.archive.util"><CODE>LongWrapper</CODE></A>
<P>
<DD><DL>