adaptiverevisitfrontier.html

来自「网络爬虫开源代码」· HTML 代码 · 共 674 行 · 第 1/5 页

<a name="532" href="#532">532</a>             <strong>long</strong> overdue = (currT-temp);
<a name="533" href="#533">533</a>             <strong>if</strong>(logger.isLoggable(Level.FINER)){
<a name="534" href="#534">534</a>                 String waitI = <span class="string">"not set"</span>;
<a name="535" href="#535">535</a>                 <strong>if</strong>(curi.containsKey(A_WAIT_INTERVAL)){
<a name="536" href="#536">536</a>                     waitI = ArchiveUtils.formatMillisecondsToConventional(
<a name="537" href="#537">537</a>                             curi.getLong(A_WAIT_INTERVAL));
<a name="538" href="#538">538</a>                 }
<a name="539" href="#539">539</a>                 logger.finer(<span class="string">"Wait interval: "</span> + waitI + 
<a name="540" href="#540">540</a>                         <span class="string">", Time of next proc: "</span> + temp +
<a name="541" href="#541">541</a>                         <span class="string">", Current time: "</span> + currT +
<a name="542" href="#542">542</a>                         <span class="string">", Overdue by: "</span> + overdue + <span class="string">"ms"</span>);
<a name="543" href="#543">543</a>             }
<a name="544" href="#544">544</a>             <strong>if</strong>(overdue &lt; 0){
<a name="545" href="#545">545</a>                 <em class="comment">// This should never happen.</em>
<a name="546" href="#546">546</a>                 logger.severe(<span class="string">"Time overdue for "</span> + curi.toString() + 
<a name="547" href="#547">547</a>                         <span class="string">"is negative ("</span> + overdue + <span class="string">")!"</span>);
<a name="548" href="#548">548</a>             }
<a name="549" href="#549">549</a>             curi.putLong(A_FETCH_OVERDUE,overdue);
<a name="550" href="#550">550</a>             <strong>return</strong> curi;
<a name="551" href="#551">551</a>         } <strong>catch</strong> (IOException e) {
<a name="552" href="#552">552</a>             <em class="comment">// TODO: Need to handle this in an intelligent manner. </em>
<a name="553" href="#553">553</a>             <em class="comment">//       Is probably fatal?</em>
<a name="554" href="#554">554</a>             e.printStackTrace();
<a name="555" href="#555">555</a>         }
<a name="556" href="#556">556</a> 
<a name="557" href="#557">557</a>         <strong>return</strong> <strong>null</strong>;
<a name="558" href="#558">558</a>     }
<a name="559" href="#559">559</a> 
<a name="560" href="#560">560</a>     <em class="comment">/*<em class="comment"> (non-Javadoc)</em></em>
<a name="561" href="#561">561</a> <em class="comment">     * @see org.archive.crawler.framework.Frontier#isEmpty()</em>
<a name="562" href="#562">562</a> <em class="comment">     */</em>
<a name="563" href="#563">563</a>     <strong>public</strong> <strong>boolean</strong> isEmpty() {
<a name="564" href="#564">564</a>         <em class="comment">// Technically, the Frontier should never become empty since URIs are</em>
<a name="565" href="#565">565</a>         <em class="comment">// only discarded under exceptional circumstances.</em>
<a name="566" href="#566">566</a>         <strong>return</strong> hostQueues.getSize() == 0;
<a name="567" href="#567">567</a>     }
<a name="568" href="#568">568</a> 
<a name="569" href="#569">569</a>     <em class="comment">/*<em class="comment"> (non-Javadoc)</em></em>
<a name="570" href="#570">570</a> <em class="comment">     * @see org.archive.crawler.framework.Frontier#schedule(org.archive.crawler.datamodel.CandidateURI)</em>
<a name="571" href="#571">571</a> <em class="comment">     */</em>
<a name="572" href="#572">572</a>     <strong>public</strong> <strong>void</strong> schedule(<a href="../../../../org/archive/crawler/datamodel/CandidateURI.html">CandidateURI</a> caURI) {
<a name="573" href="#573">573</a>         batchSchedule(caURI);        
<a name="574" href="#574">574</a>     }
<a name="575" href="#575">575</a> 
<a name="576" href="#576">576</a>     <em class="comment">/*<em class="comment"> (non-Javadoc)</em></em>
<a name="577" href="#577">577</a> <em class="comment">     * @see org.archive.crawler.framework.Frontier#finished(org.archive.crawler.datamodel.CrawlURI)</em>
<a name="578" href="#578">578</a> <em class="comment">     */</em>
<a name="579" href="#579">579</a>     <strong>public</strong> <strong>synchronized</strong> <strong>void</strong> finished(<a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi) {
<a name="580" href="#580">580</a>         logger.fine(curi.toString()+ <span class="string">" "</span> + 
<a name="581" href="#581">581</a>                 CrawlURI.fetchStatusCodesToString(curi.getFetchStatus()));
<a name="582" href="#582">582</a>         curi.incrementFetchAttempts();
<a name="583" href="#583">583</a>         logLocalizedErrors(curi);
<a name="584" href="#584">584</a> 
<a name="585" href="#585">585</a>         innerFinished(curi);
<a name="586" href="#586">586</a>     }
<a name="587" href="#587">587</a>     
<a name="588" href="#588">588</a>     <strong>protected</strong> <strong>synchronized</strong> <strong>void</strong> innerFinished(<a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi) {
<a name="589" href="#589">589</a>         <strong>try</strong> {
<a name="590" href="#590">590</a>             innerBatchFlush();
<a name="591" href="#591">591</a>             
<a name="592" href="#592">592</a>             <strong>if</strong> (curi.isSuccess()) {
<a name="593" href="#593">593</a>                 successDisposition(curi);
<a name="594" href="#594">594</a>             } <strong>else</strong> <strong>if</strong> (needsPromptRetry(curi)) {
<a name="595" href="#595">595</a>                 <em class="comment">// Consider statuses which allow nearly-immediate retry</em>
<a name="596" href="#596">596</a>                 <em class="comment">// (like deferred to allow precondition to be fetched)</em>
<a name="597" href="#597">597</a>                 reschedule(curi,false);
<a name="598" href="#598">598</a>             } <strong>else</strong> <strong>if</strong> (needsRetrying(curi)) {
<a name="599" href="#599">599</a>                 <em class="comment">// Consider errors which can be retried</em>
<a name="600" href="#600">600</a>                 reschedule(curi,<strong>true</strong>);
<a name="601" href="#601">601</a>                 controller.fireCrawledURINeedRetryEvent(curi);
<a name="602" href="#602">602</a>             } <strong>else</strong> <strong>if</strong>(isDisregarded(curi)) {
<a name="603" href="#603">603</a>                 <em class="comment">// Check for codes that mean that while the crawler did</em>
<a name="604" href="#604">604</a>                 <em class="comment">// manage to get it it must be disregarded for any reason.</em>
<a name="605" href="#605">605</a>                 disregardDisposition(curi);
<a name="606" href="#606">606</a>             } <strong>else</strong> {
<a name="607" href="#607">607</a>                 <em class="comment">// In that case FAILURE, note &amp; log</em>
<a name="608" href="#608">608</a>                 failureDisposition(curi);
<a name="609" href="#609">609</a>             }
<a name="610" href="#610">610</a> 
<a name="611" href="#611">611</a>             <em class="comment">// New items might be available, let waiting threads know</em>
<a name="612" href="#612">612</a>             <em class="comment">// More then one queue might have become available due to </em>
<a name="613" href="#613">613</a>             <em class="comment">// scheduling of items outside the parent URIs host, so we</em>
<a name="614" href="#614">614</a>             <em class="comment">// wake all waiting threads.</em>
<a name="615" href="#615">615</a>             notifyAll();
<a name="616" href="#616">616</a>         } <strong>catch</strong> (RuntimeException e) {
<a name="617" href="#617">617</a>             curi.setFetchStatus(S_RUNTIME_EXCEPTION);
<a name="618" href="#618">618</a>             <em class="comment">// store exception temporarily for logging</em>
<a name="619" href="#619">619</a>             logger.warning(<span class="string">"RTE in innerFinished() "</span> +
<a name="620" href="#620">620</a>                 e.getMessage());
<a name="621" href="#621">621</a>             e.printStackTrace();
<a name="622" href="#622">622</a>             curi.putObject(A_RUNTIME_EXCEPTION, e);
<a name="623" href="#623">623</a>             failureDisposition(curi);
<a name="624" href="#624">624</a>         } <strong>catch</strong> (AttributeNotFoundException e) {
<a name="625" href="#625">625</a>             logger.severe(e.getMessage());
<a name="626" href="#626">626</a>         }
<a name="627" href="#627">627</a>     }
<a name="628" href="#628">628</a> 
<a name="629" href="#629">629</a>     <em>/**<em>*</em></em>
<a name="630" href="#630">630</a> <em>     * Take note of any processor-local errors that have</em>
<a name="631" href="#631">631</a> <em>     * been entered into the CrawlURI.</em>
<a name="632" href="#632">632</a> <em>     * @param curi CrawlURI with errors.</em>
<a name="633" href="#633">633</a> <em>     */</em>
<a name="634" href="#634">634</a>     <strong>private</strong> <strong>void</strong> logLocalizedErrors(<a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi) {
<a name="635" href="#635">635</a>         <strong>if</strong>(curi.containsKey(A_LOCALIZED_ERRORS)) {
<a name="636" href="#636">636</a>             List localErrors = (List)curi.getObject(A_LOCALIZED_ERRORS);
<a name="637" href="#637">637</a>             Iterator iter = localErrors.iterator();
<a name="638" href="#638">638</a>             <strong>while</strong>(iter.hasNext()) {
<a name="639" href="#639">639</a>                 Object array[] = {curi, iter.next()};
<a name="640" href="#640">640</a>                 controller.localErrors.log(Level.WARNING,
<a name="641" href="#641">641</a>                     curi.getUURI().toString(), array);
<a name="642" href="#642">642</a>             }
<a name="643" href="#643">643</a>             <em class="comment">// once logged, discard</em>
<a name="644" href="#644">644</a>             curi.remove(A_LOCALIZED_ERRORS);
<a name="645" href="#645">645</a>         }
<a name="646" href="#646">646</a>     }
<a name="647" href="#647">647</a>     
<a name="648" href="#648">648</a>     <em>/**<em>*</em></em>
<a name="649" href="#649">649</a> <em>     * The CrawlURI has been successfully crawled. </em>
<a name="650" href="#650">650</a> <em>     *</em>
<a name="651" href="#651">651</a> <em>     * @param curi The CrawlURI</em>
<a name="652" href="#652">652</a> <em>     */</em>
<a name="653" href="#653">653</a>     <strong>protected</strong> <strong>void</strong> successDisposition(<a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi) {
<a name="654" href="#654">654</a>         curi.aboutToLog();
<a name="655" href="#655">655</a> 
<a name="656" href="#656">656</a>         <strong>long</strong> waitInterval = 0;
<a name="657" href="#657">657</a>         
<a name="658" href="#658">658</a>         <strong>if</strong>(curi.containsKey(A_WAIT_INTERVAL)){
<a name="659" href="#659">659</a>             waitInterval = curi.getLong(A_WAIT_INTERVAL);
<a name="660" href="#660">660</a>             curi.addAnnotation(<span class="string">"wt:"</span> + 
<a name="661" href="#661">661</a>                     ArchiveUtils.formatMillisecondsToConventional(
<a name="662" href="#662">662</a>                             waitInterval));
<a name="663" href="#663">663</a>         } <strong>else</strong> {
<a name="664" href="#664">664</a>             logger.severe(<span class="string">"Missing wait interval for "</span> + curi.toString() +
<a name="665" href="#665">665</a>                     <span class="string">" WaitEvalua