bdbfrontier.html

一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫

<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#BdbFrontier(java.lang.String, java.lang.String)">BdbFrontier</A></B>(java.lang.String&nbsp;name,
            java.lang.String&nbsp;description)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Create the BdbFrontier</TD>
</TR>
</TABLE>
&nbsp;
<!-- ========== METHOD SUMMARY =========== -->

<A NAME="method_summary"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="2"><FONT SIZE="+2">
<B>Method Summary</B></FONT></TH>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#closeQueue()">closeQueue</A></B>()</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#crawlCheckpoint(java.io.File)">crawlCheckpoint</A></B>(java.io.File&nbsp;checkpointDir)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Called by <A HREF="../../../../org/archive/crawler/framework/CrawlController.html" title="class in org.archive.crawler.framework"><CODE>CrawlController</CODE></A> when checkpointing.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;<A HREF="../../../../org/archive/crawler/datamodel/UriUniqFilter.html" title="interface in org.archive.crawler.datamodel">UriUniqFilter</A></CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#createAlreadyIncluded()">createAlreadyIncluded</A></B>()</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Create a UriUniqFilter that will serve as record 
 of already seen URIs.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;<A HREF="../../../../org/archive/crawler/datamodel/UriUniqFilter.html" title="interface in org.archive.crawler.datamodel">UriUniqFilter</A></CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#deserializeAlreadySeen(java.lang.Class, java.io.File)">deserializeAlreadySeen</A></B>(java.lang.Class&nbsp;cls,
                       java.io.File&nbsp;dir)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;<A HREF="../../../../org/archive/crawler/framework/FrontierMarker.html" title="interface in org.archive.crawler.framework">FrontierMarker</A></CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#getInitialMarker(java.lang.String, boolean)">getInitialMarker</A></B>(java.lang.String&nbsp;regexpr,
                 boolean&nbsp;inCacheOnly)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get a <code>URIFrontierMarker</code> initialized with the given
 regular expression at the 'start' of the Frontier.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;<A HREF="../../../../org/archive/crawler/frontier/WorkQueue.html" title="class in org.archive.crawler.frontier">WorkQueue</A></CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#getQueueFor(org.archive.crawler.datamodel.CrawlURI)">getQueueFor</A></B>(<A HREF="../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Return the work queue for the given CrawlURI's classKey.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;<A HREF="../../../../org/archive/crawler/frontier/WorkQueue.html" title="class in org.archive.crawler.frontier">WorkQueue</A></CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#getQueueFor(java.lang.String)">getQueueFor</A></B>(java.lang.String&nbsp;classKey)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Return the work queue for the given classKey, or null
 if no such queue exists.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;java.util.ArrayList</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#getURIsList(org.archive.crawler.framework.FrontierMarker, int, boolean)">getURIsList</A></B>(<A HREF="../../../../org/archive/crawler/framework/FrontierMarker.html" title="interface in org.archive.crawler.framework">FrontierMarker</A>&nbsp;marker,
            int&nbsp;numberOfMatches,
            boolean&nbsp;verbose)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Return list of urls.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;<A HREF="../../../../org/archive/crawler/frontier/BdbMultipleWorkQueues.html" title="class in org.archive.crawler.frontier">BdbMultipleWorkQueues</A></CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#getWorkQueues()">getWorkQueues</A></B>()</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#initialize(org.archive.crawler.framework.CrawlController)">initialize</A></B>(<A HREF="../../../../org/archive/crawler/framework/CrawlController.html" title="class in org.archive.crawler.framework">CrawlController</A>&nbsp;c)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Initializes the Frontier, given the supplied CrawlController.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#initQueue()">initQueue</A></B>()</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;boolean</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html#workQueueDataOnDisk()">workQueueDataOnDisk</A></B>()</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns <code>true</code> if the WorkQueue implementation of this
 Frontier stores its workload on disk instead of relying
 on serialization mechanisms.</TD>
</TR>
</TABLE>
&nbsp;<A NAME="methods_inherited_from_class_org.archive.crawler.frontier.WorkQueueFrontier"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
<TH ALIGN="left"><B>Methods inherited from class org.archive.crawler.frontier.<A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html" title="class in org.archive.crawler.frontier">WorkQueueFrontier</A></B></TH>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD><CODE><A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#appendQueueReports(java.io.PrintWriter, java.util.Iterator, int, int)">appendQueueReports</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#asCrawlUri(org.archive.crawler.datamodel.CandidateURI)">asCrawlUri</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#averageDepth()">averageDepth</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#congestionRatio()">congestionRatio</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#considerIncluded(org.archive.net.UURI)">considerIncluded</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#crawlEnded(java.lang.String)">crawlEnded</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#deepestUri()">deepestUri</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#deleted(org.archive.crawler.datamodel.CrawlURI)">deleted</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#deleteURIs(java.lang.String)">deleteURIs</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#discoveredUriCount()">discoveredUriCount</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#finished(org.archive.crawler.datamodel.CrawlURI)">finished</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#forget(org.archive.crawler.datamodel.CrawlURI)">forget</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#getGroup(org.archive.crawler.datamodel.CrawlURI)">getGroup</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#getReports()">getReports</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#isEmpty()">isEmpty</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#kickUpdate()">kickUpdate</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#next()">next</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#receive(org.archive.crawler.datamodel.CandidateURI)">receive</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#reportTo(java.lang.String, java.io.PrintWriter)">reportTo</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#schedule(org.archive.crawler.datamodel.CandidateURI)">schedule</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#sendToQueue(org.archive.crawler.datamodel.CrawlURI)">sendToQueue</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#singleLineLegend()">singleLineLegend</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#singleLineReportTo(java.io.PrintWriter)">singleLineReportTo</A>, <A HREF="../../../../org/archive/crawler/frontier/WorkQueueFrontier.html#wakeQueues()">wakeQueues</A></CODE></TD>