bdbmultipleworkqueues.html

一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫

<DT><B>Throws:</B>
<DD><CODE>com.sleepycat.je.DatabaseException</CODE></DL>
</DL>

<!-- ============ METHOD DETAIL ========== -->

<A NAME="method_detail"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="1"><FONT SIZE="+2">
<B>Method Detail</B></FONT></TH>
</TR>
</TABLE>

<A NAME="deleteMatchingFromQueue(java.lang.String, java.lang.String, com.sleepycat.je.DatabaseEntry)"><!-- --></A><H3>
deleteMatchingFromQueue</H3>
<PRE>
public long <B>deleteMatchingFromQueue</B>(java.lang.String&nbsp;match,
                                    java.lang.String&nbsp;queue,
                                    com.sleepycat.je.DatabaseEntry&nbsp;headKey)
                             throws com.sleepycat.je.DatabaseException</PRE>
<DL>
<DD>Delete all CrawlURIs matching the given expression.
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>match</CODE> - <DD><CODE>queue</CODE> - <DD><CODE>headKey</CODE> - 
<DT><B>Returns:</B><DD>count of deleted items
<DT><B>Throws:</B>
<DD><CODE>com.sleepycat.je.DatabaseException</CODE>
<DD><CODE>com.sleepycat.je.DatabaseException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="getFrom(org.archive.crawler.framework.FrontierMarker, int)"><!-- --></A><H3>
getFrom</H3>
<PRE>
public java.util.List <B>getFrom</B>(<A HREF="../../../../org/archive/crawler/framework/FrontierMarker.html" title="interface in org.archive.crawler.framework">FrontierMarker</A>&nbsp;m,
                              int&nbsp;maxMatches)
                       throws com.sleepycat.je.DatabaseException</PRE>
<DL>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>m</CODE> - marker<DD><CODE>maxMatches</CODE> - 
<DT><B>Returns:</B><DD>list of matches starting from marker position
<DT><B>Throws:</B>
<DD><CODE>com.sleepycat.je.DatabaseException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="getInitialMarker(java.lang.String)"><!-- --></A><H3>
getInitialMarker</H3>
<PRE>
public <A HREF="../../../../org/archive/crawler/framework/FrontierMarker.html" title="interface in org.archive.crawler.framework">FrontierMarker</A> <B>getInitialMarker</B>(java.lang.String&nbsp;regexpr)</PRE>
<DL>
<DD>Get a marker for beginning a scan over all contents
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>regexpr</CODE> - 
<DT><B>Returns:</B><DD>a marker pointing to the first item</DL>
</DD>
</DL>
<HR>

<A NAME="getFirstKey()"><!-- --></A><H3>
getFirstKey</H3>
<PRE>
protected com.sleepycat.je.DatabaseEntry <B>getFirstKey</B>()
                                              throws com.sleepycat.je.DatabaseException</PRE>
<DL>
<DD><DL>

<DT><B>Returns:</B><DD>the key to the first item in the database
<DT><B>Throws:</B>
<DD><CODE>com.sleepycat.je.DatabaseException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="get(com.sleepycat.je.DatabaseEntry)"><!-- --></A><H3>
get</H3>
<PRE>
public <A HREF="../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A> <B>get</B>(com.sleepycat.je.DatabaseEntry&nbsp;headKey)
             throws com.sleepycat.je.DatabaseException</PRE>
<DL>
<DD>Get the next nearest item after the given key. Relies on 
 external discipline -- we'll look at the queues count of how many
 items it has -- to avoid asking for something from a
 range where there are no associated items --
 otherwise could get first item of next 'queue' by mistake. 
 
 <p>TODO: hold within a queue's range
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>headKey</CODE> - Key prefix that demarks the beginning of the range
 in <code>pendingUrisDB</code> we're interested in.
<DT><B>Returns:</B><DD>CrawlURI.
<DT><B>Throws:</B>
<DD><CODE>com.sleepycat.je.DatabaseException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="getNextNearestItem(com.sleepycat.je.DatabaseEntry, com.sleepycat.je.DatabaseEntry)"><!-- --></A><H3>
getNextNearestItem</H3>
<PRE>
protected com.sleepycat.je.OperationStatus <B>getNextNearestItem</B>(com.sleepycat.je.DatabaseEntry&nbsp;headKey,
                                                              com.sleepycat.je.DatabaseEntry&nbsp;result)
                                                       throws com.sleepycat.je.DatabaseException</PRE>
<DL>
<DD><DL>

<DT><B>Throws:</B>
<DD><CODE>com.sleepycat.je.DatabaseException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="put(org.archive.crawler.datamodel.CrawlURI)"><!-- --></A><H3>
put</H3>
<PRE>
public void <B>put</B>(<A HREF="../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)
         throws com.sleepycat.je.DatabaseException</PRE>
<DL>
<DD>Put the given CrawlURI in at the appropriate place.
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>curi</CODE> - 
<DT><B>Throws:</B>
<DD><CODE>com.sleepycat.je.DatabaseException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="calculateOriginKey(java.lang.String)"><!-- --></A><H3>
calculateOriginKey</H3>
<PRE>
static byte[] <B>calculateOriginKey</B>(java.lang.String&nbsp;classKey)</PRE>
<DL>
<DD>Calculate the 'origin' key for a virtual queue of items
 with the given classKey. This origin key will be a 
 prefix of the keys for all items in the queue.
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>classKey</CODE> - String key to derive origin byte key from
<DT><B>Returns:</B><DD>a byte array key</DL>
</DD>
</DL>
<HR>

<A NAME="calculateInsertKey(org.archive.crawler.datamodel.CrawlURI)"><!-- --></A><H3>
calculateInsertKey</H3>
<PRE>
static com.sleepycat.je.DatabaseEntry <B>calculateInsertKey</B>(<A HREF="../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)</PRE>
<DL>
<DD>Calculate the insertKey that places a CrawlURI in the
 desired spot. First bytes are always classKey (usu. host)
 based -- ensuring grouping by host -- terminated by a zero
 byte. Then 8 bytes of data ensuring desired ordering 
 within that 'queue' are used. The first byte of these 8 is
 priority -- allowing 'immediate' and 'soon' items to 
 sort above regular. Next 1 byte is 'cost'. Last 6 bytes 
 are ordinal serial number, ensuring earlier-discovered 
 URIs sort before later. 
 
 NOTE: Dangers here are:
 (1) priorities or costs over 2^7 (signed byte comparison)
 (2) ordinals over 2^48
 
 Package access & static for testing purposes.
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>curi</CODE> - 
<DT><B>Returns:</B><DD>a DatabaseEntry key for the CrawlURI</DL>
</DD>
</DL>
<HR>

<A NAME="delete(org.archive.crawler.datamodel.CrawlURI)"><!-- --></A><H3>
delete</H3>
<PRE>
public void <B>delete</B>(<A HREF="../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;item)
            throws com.sleepycat.je.DatabaseException</PRE>
<DL>
<DD>Delete the given CrawlURI from persistent store. Requires
 the key under which it was stored be available.
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>item</CODE> - 
<DT><B>Throws:</B>
<DD><CODE>com.sleepycat.je.DatabaseException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="sync()"><!-- --></A><H3>
sync</H3>
<PRE>
void <B>sync</B>()</PRE>
<DL>
<DD>Method used by BdbFrontier during checkpointing.
 <p>The backing bdbje database has been marked deferred write so we save
 on writes to disk.  Means no guarantees disk will have whats in memory
 unless a sync is called (Calling sync on the bdbje Environment is not
 sufficent).
 <p>Package access only because only Frontiers of this package would ever
 need access.
<P>
<DD><DL>
<DT><B>See Also:</B><DD><a href="http://www.sleepycat.com/jedocs/GettingStartedGuide/DB.html">Deferred Write Databases</a></DL>
</DD>
</DL>
<HR>

<A NAME="close()"><!-- --></A><H3>
close</H3>
<PRE>
public void <B>close</B>()</PRE>
<DL>
<DD>clean up
<P>
<DD><DL>
</DL>
</DD>
</DL>
<HR>

<A NAME="addCap(byte[])"><!-- --></A><H3>
addCap</H3>
<PRE>
public void <B>addCap</B>(byte[]&nbsp;origin)</PRE>
<DL>
<DD>Add a dummy 'cap' entry at the given insertion key. Prevents
 'seeks' to queue heads from holding lock on last item of 
 'preceding' queue. See:
 http://sourceforge.net/tracker/index.php?func=detail&aid=1262665&group_id=73833&atid=539102
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>origin</CODE> - key at which to insert the cap</DL>
</DD>
</DL>
<!-- ========= END OF CLASS DATA ========= -->
<HR>


<!-- ======= START OF BOTTOM NAVBAR ====== -->
<A NAME="navbar_bottom"><!-- --></A>
<A HREF="#skip-navbar_bottom" title="Skip navigation links"></A>
<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0" SUMMARY="">
<TR>
<TD COLSPAN=2 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
<A NAME="navbar_bottom_firstrow"><!-- --></A>
<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3" SUMMARY="">
  <TR ALIGN="center" VALIGN="top">
  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="../../../../overview-summary.html"><FONT CLASS="NavBarFont1"><B>Overview</B></FONT></A>&nbsp;</TD>
  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="package-summary.html"><FONT CLASS="NavBarFont1"><B>Package</B></FONT></A>&nbsp;</TD>
  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="class-use/BdbMultipleWorkQueues.html"><FONT CLASS="NavBarFont1"><B>Use</B></FONT></A>&nbsp;</TD>
  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="package-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="../../../../deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="../../../../index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="../../../../help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
  </TR>
</TABLE>
</TD>
<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
</EM>
</TD>
</TR>

<TR>
<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
&nbsp;<A HREF="../../../../org/archive/crawler/frontier/BdbFrontier.html" title="class in org.archive.crawler.frontier"><B>PREV CLASS</B></A>&nbsp;
&nbsp;<A HREF="../../../../org/archive/crawler/frontier/BdbMultipleWorkQueues.BdbFrontierMarker.html" title="class in org.archive.crawler.frontier"><B>NEXT CLASS</B></A></FONT></TD>
<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
  <A HREF="../../../../index.html?org/archive/crawler/frontier/BdbMultipleWorkQueues.html" target="_top"><B>FRAMES</B></A>  &nbsp;
&nbsp;<A HREF="BdbMultipleWorkQueues.html" target="_top"><B>NO FRAMES</B></A>  &nbsp;
&nbsp;<SCRIPT type="text/javascript">
  <!--
  if(window==top) {
    document.writeln('<A HREF="../../../../allclasses-noframe.html"><B>All Classes</B></A>');
  }
  //-->
</SCRIPT>
<NOSCRIPT>
  <A HREF="../../../../allclasses-noframe.html"><B>All Classes</B></A>
</NOSCRIPT>


</FONT></TD>
</TR>
<TR>
<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
  SUMMARY:&nbsp;<A HREF="#nested_class_summary">NESTED</A>&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
</TR>
</TABLE>
<A NAME="skip-navbar_bottom"></A>
<!-- ======== END OF BOTTOM NAVBAR ======= -->

<HR>
Copyright &copy; 2003-2006 Internet Archive. All Rights Reserved.
</BODY>
</HTML>