crawluri.html

用JAVA编写的,在做实验的时候留下来的,本来想删的,但是传上来,大家分享吧

<TD><CODE><B>StatisticsTracker.</B><B><A HREF="../../../../../org/archive/crawler/admin/StatisticsTracker.html#crawledURISuccessful(org.archive.crawler.datamodel.CrawlURI)">crawledURISuccessful</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
</TABLE>
&nbsp;
<P>

<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
<TH ALIGN="left" COLSPAN="2">Constructors in <A HREF="../../../../../org/archive/crawler/admin/package-summary.html">org.archive.crawler.admin</A> with parameters of type <A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A></FONT></TH>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD><CODE><B><A HREF="../../../../../org/archive/crawler/admin/SeedRecord.html#SeedRecord(org.archive.crawler.datamodel.CrawlURI, java.lang.String)">SeedRecord</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi,
           java.lang.String&nbsp;disposition)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Create a record from the given CrawlURI and disposition string</TD>
</TR>
</TABLE>
&nbsp;
<P>
<A NAME="org.archive.crawler.datamodel"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="2"><FONT SIZE="+2">
Uses of <A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A> in <A HREF="../../../../../org/archive/crawler/datamodel/package-summary.html">org.archive.crawler.datamodel</A></FONT></TH>
</TR>
</TABLE>
&nbsp;
<P>

<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
<TH ALIGN="left" COLSPAN="2">Methods in <A HREF="../../../../../org/archive/crawler/datamodel/package-summary.html">org.archive.crawler.datamodel</A> that return <A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A></FONT></TH>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>static&nbsp;<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A></CODE></FONT></TD>
<TD><CODE><B>CrawlURI.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html#from(org.archive.crawler.datamodel.CandidateURI, long)">from</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CandidateURI.html" title="class in org.archive.crawler.datamodel">CandidateURI</A>&nbsp;caUri,
     long&nbsp;ordinal)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Make a <code>CrawlURI</code> from the passed <code>CandidateURI</code>.</TD>
</TR>
</TABLE>
&nbsp;
<P>

<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
<TH ALIGN="left" COLSPAN="2">Methods in <A HREF="../../../../../org/archive/crawler/datamodel/package-summary.html">org.archive.crawler.datamodel</A> with parameters of type <A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A></FONT></TH>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;boolean</CODE></FONT></TD>
<TD><CODE><B>RobotsExclusionPolicy.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/RobotsExclusionPolicy.html#disallows(org.archive.crawler.datamodel.CrawlURI, java.lang.String)">disallows</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi,
          java.lang.String&nbsp;userAgent)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
<TD><CODE><B>CrawlOrder.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/CrawlOrder.html#getFrom(org.archive.crawler.datamodel.CrawlURI)">getFrom</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
<TD><CODE><B>CrawlOrder.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/CrawlOrder.html#getUserAgent(org.archive.crawler.datamodel.CrawlURI)">getUserAgent</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;boolean</CODE></FONT></TD>
<TD><CODE><B>RobotsHonoringPolicy.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/RobotsHonoringPolicy.html#shouldMasquerade(org.archive.crawler.datamodel.CrawlURI)">shouldMasquerade</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;This method returns true if the crawler should masquerade as the user agent
 which restrictions it opted to use.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;java.util.Set</CODE></FONT></TD>
<TD><CODE><B>CredentialStore.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/CredentialStore.html#subset(org.archive.crawler.datamodel.CrawlURI, java.lang.Class)">subset</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;context,
       java.lang.Class&nbsp;type)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Return set made up of all credentials of the passed
 <code>type</code>.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;java.util.Set</CODE></FONT></TD>
<TD><CODE><B>CredentialStore.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/CredentialStore.html#subset(org.archive.crawler.datamodel.CrawlURI, java.lang.Class, java.lang.String)">subset</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;context,
       java.lang.Class&nbsp;type,
       java.lang.String&nbsp;rootUri)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Return set made up of all credentials of the passed
 <code>type</code>.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;void</CODE></FONT></TD>
<TD><CODE><B>CrawlSubstats.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/CrawlSubstats.html#tally(org.archive.crawler.datamodel.CrawlURI)">tally</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;void</CODE></FONT></TD>
<TD><CODE><B>CrawlServer.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/CrawlServer.html#updateRobots(org.archive.crawler.datamodel.CrawlURI)">updateRobots</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Update the robots exclusion policy.</TD>
</TR>
</TABLE>
&nbsp;
<P>
<A NAME="org.archive.crawler.datamodel.credential"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="2"><FONT SIZE="+2">
Uses of <A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A> in <A HREF="../../../../../org/archive/crawler/datamodel/credential/package-summary.html">org.archive.crawler.datamodel.credential</A></FONT></TH>
</TR>
</TABLE>
&nbsp;
<P>

<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
<TH ALIGN="left" COLSPAN="2">Methods in <A HREF="../../../../../org/archive/crawler/datamodel/credential/package-summary.html">org.archive.crawler.datamodel.credential</A> with parameters of type <A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A></FONT></TH>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;void</CODE></FONT></TD>
<TD><CODE><B>Credential.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/credential/Credential.html#attach(org.archive.crawler.datamodel.CrawlURI)">attach</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Attach this credentials avatar to the passed <code>curi</code> .</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;void</CODE></FONT></TD>
<TD><CODE><B>Credential.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/credential/Credential.html#attach(org.archive.crawler.datamodel.CrawlURI, java.lang.String)">attach</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi,
       java.lang.String&nbsp;payload)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Attach this credentials avatar to the passed <code>curi</code> .</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;boolean</CODE></FONT></TD>
<TD><CODE><B>Credential.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/credential/Credential.html#detach(org.archive.crawler.datamodel.CrawlURI)">detach</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Detach this credential from passed curi.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;boolean</CODE></FONT></TD>
<TD><CODE><B>Credential.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/credential/Credential.html#detachAll(org.archive.crawler.datamodel.CrawlURI)">detachAll</A></B>(<A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Detach all credentials of this type from passed curi.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>static&nbsp;<A HREF="../../../../../org/archive/crawler/datamodel/credential/Rfc2617Credential.html" title="class in org.archive.crawler.datamodel.credential">Rfc2617Credential</A></CODE></FONT></TD>
<TD><CODE><B>Rfc2617Credential.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/credential/Rfc2617Credential.html#getByRealm(java.util.Set, java.lang.String, org.archive.crawler.datamodel.CrawlURI)">getByRealm</A></B>(java.util.Set&nbsp;rfc2617Credentials,
           java.lang.String&nbsp;realm,
           <A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;context)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Convenience method that does look up on passed set using realm for key.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;<A HREF="../../../../../org/archive/crawler/datamodel/credential/Credential.html" title="class in org.archive.crawler.datamodel.credential">Credential</A></CODE></FONT></TD>
<TD><CODE><B>CredentialAvatar.</B><B><A HREF="../../../../../org/archive/crawler/datamodel/credential/CredentialAvatar.html#getCredential(org.archive.crawler.settings.SettingsHandler, org.archive.crawler.datamodel.CrawlURI)">getCredential</A></B>(<A HREF="../../../../../org/archive/crawler/settings/SettingsHandler.html" title="class in org.archive.crawler.settings">SettingsHandler</A>&nbsp;handler,
              <A HREF="../../../../../org/archive/crawler/datamodel/CrawlURI.html" title="class in org.archive.crawler.datamodel">CrawlURI</A>&nbsp;curi)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">