fetchhttp.html

用JAVA编写的,在做实验的时候留下来的,本来想删的,但是传上来,大家分享吧

<a name="254" href="#254">254</a>     <strong>static</strong> <strong>final</strong> String SSL_FACTORY_KEY = <span class="string">"heritrix.ssl.factory"</span>;
<a name="255" href="#255">255</a>     
<a name="256" href="#256">256</a>     <em>/**<em>**</em></em>
<a name="257" href="#257">257</a> <em>     * Socket factory that has the configurable trust manager installed.</em>
<a name="258" href="#258">258</a> <em>     */</em>
<a name="259" href="#259">259</a>     <strong>private</strong> SSLSocketFactory sslfactory = <strong>null</strong>;
<a name="260" href="#260">260</a>     
<a name="261" href="#261">261</a> 
<a name="262" href="#262">262</a>     <em>/**<em>*</em></em>
<a name="263" href="#263">263</a> <em>     * Constructor.</em>
<a name="264" href="#264">264</a> <em>     *</em>
<a name="265" href="#265">265</a> <em>     * @param name Name of this processor.</em>
<a name="266" href="#266">266</a> <em>     */</em>
<a name="267" href="#267">267</a>     <strong>public</strong> <a href="../../../../org/archive/crawler/fetcher/FetchHTTP.html">FetchHTTP</a>(String name) {
<a name="268" href="#268">268</a>         <strong>super</strong>(name, <span class="string">"HTTP Fetcher"</span>);
<a name="269" href="#269">269</a>         <strong>this</strong>.midfetchfilters = (MapType) addElementToDefinition(
<a name="270" href="#270">270</a>             <strong>new</strong> <a href="../../../../org/archive/crawler/settings/MapType.html">MapType</a>(MIDFETCH_ATTR_FILTERS, <span class="string">"Filters applied after"</span> +
<a name="271" href="#271">271</a>                 <span class="string">" receipt of HTTP response headers but before we start to"</span> +
<a name="272" href="#272">272</a>                 <span class="string">" download the body. If any filter returns"</span> +
<a name="273" href="#273">273</a>                 <span class="string">" FALSE, the fetch is aborted. Prerequisites such as"</span> +
<a name="274" href="#274">274</a>                 <span class="string">" robots.txt by-pass filtering (i.e. they cannot be"</span> +
<a name="275" href="#275">275</a>                 <span class="string">" midfetch aborted."</span>, Filter.<strong>class</strong>));
<a name="276" href="#276">276</a> <em class="comment">// see [ 1379040 ] regex for midfetch filter not being stored in crawl order</em>
<a name="277" href="#277">277</a> <em class="comment">// http://sourceforge.net/support/tracker.php?aid=1379040</em>
<a name="278" href="#278">278</a> <em class="comment">//        this.midfetchfilters.setExpertSetting(true);</em>
<a name="279" href="#279">279</a>         
<a name="280" href="#280">280</a>         addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_TIMEOUT_SECONDS,
<a name="281" href="#281">281</a>             <span class="string">"If the fetch is not completed in this number of seconds,"</span>
<a name="282" href="#282">282</a>             + <span class="string">" give up (and retry later). For optimal configuration, "</span> +
<a name="283" href="#283">283</a>             <span class="string">" ensure this value is > "</span> + ATTR_TIMEOUT_SECONDS + <span class="string">"."</span>,
<a name="284" href="#284">284</a>             DEFAULT_TIMEOUT_SECONDS));
<a name="285" href="#285">285</a>         <a href="../../../../org/archive/crawler/settings/Type.html">Type</a> e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_SOTIMEOUT_MS,
<a name="286" href="#286">286</a>             <span class="string">"If the socket is unresponsive for this number of milliseconds, "</span> +
<a name="287" href="#287">287</a>             <span class="string">" give up.  Set to zero for no timeout (Not."</span> +
<a name="288" href="#288">288</a>             <span class="string">" recommended. Could hang a thread on an unresponsive server)."</span> +
<a name="289" href="#289">289</a>             <span class="string">" This timeout is used timing out socket opens "</span> +
<a name="290" href="#290">290</a>             <span class="string">" and for timing out each socket read.  Make sure this "</span> +
<a name="291" href="#291">291</a>             <span class="string">" value is &lt; "</span> + ATTR_TIMEOUT_SECONDS + <span class="string">" for optimal "</span> +
<a name="292" href="#292">292</a>             <span class="string">" configuration: ensures at least one retry read."</span>,
<a name="293" href="#293">293</a>                 DEFAULT_SOTIMEOUT_MS));
<a name="294" href="#294">294</a>         e.setExpertSetting(<strong>true</strong>);
<a name="295" href="#295">295</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_FETCH_BANDWIDTH_MAX,
<a name="296" href="#296">296</a>             <span class="string">"The maximum KB/sec to use when fetching data from a server. "</span> +
<a name="297" href="#297">297</a>             <span class="string">"0 means no maximum.  Default: "</span>+ DEFAULT_FETCH_BANDWIDTH_MAX
<a name="298" href="#298">298</a>              + <span class="string">"."</span>, DEFAULT_FETCH_BANDWIDTH_MAX));
<a name="299" href="#299">299</a>         e.setExpertSetting(<strong>true</strong>);
<a name="300" href="#300">300</a>         e.setOverrideable(<strong>true</strong>);
<a name="301" href="#301">301</a>         addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_MAX_LENGTH_BYTES,
<a name="302" href="#302">302</a>             <span class="string">"Maximum length in bytes to fetch.\n"</span> +
<a name="303" href="#303">303</a>             <span class="string">"Fetch is truncated at this length. A value of 0 means no limit."</span>,
<a name="304" href="#304">304</a>             DEFAULT_MAX_LENGTH_BYTES));
<a name="305" href="#305">305</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_IGNORE_COOKIES,
<a name="306" href="#306">306</a>             <span class="string">"Disable cookie-handling."</span>, DEFAULT_IGNORE_COOKIES));
<a name="307" href="#307">307</a>         e.setOverrideable(<strong>true</strong>);
<a name="308" href="#308">308</a>         e.setExpertSetting(<strong>true</strong>);
<a name="309" href="#309">309</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_BDB_COOKIES,
<a name="310" href="#310">310</a>                 <span class="string">"Store cookies in BDB-backed map."</span>, DEFAULT_BDB_COOKIES));
<a name="311" href="#311">311</a>         e.setExpertSetting(<strong>true</strong>);
<a name="312" href="#312">312</a> 
<a name="313" href="#313">313</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_LOAD_COOKIES,
<a name="314" href="#314">314</a>             <span class="string">"File to preload cookies from"</span>, <span class="string">""</span>));
<a name="315" href="#315">315</a>         e.setExpertSetting(<strong>true</strong>);
<a name="316" href="#316">316</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_SAVE_COOKIES,
<a name="317" href="#317">317</a>             <span class="string">"When crawl finishes save cookies to this file"</span>, <span class="string">""</span>));
<a name="318" href="#318">318</a>         e.setExpertSetting(<strong>true</strong>);
<a name="319" href="#319">319</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_TRUST,
<a name="320" href="#320">320</a>             <span class="string">"SSL certificate trust level.  Range is from the default 'open'"</span>
<a name="321" href="#321">321</a>             + <span class="string">" (trust all certs including expired, selfsigned, and those for"</span>
<a name="322" href="#322">322</a>             + <span class="string">" which we do not have a CA) through 'loose' (trust all valid"</span>
<a name="323" href="#323">323</a>             + <span class="string">" certificates including selfsigned), 'normal' (all valid"</span>
<a name="324" href="#324">324</a>             + <span class="string">" certificates not including selfsigned) to 'strict' (Cert is"</span>
<a name="325" href="#325">325</a>             + <span class="string">" valid and DN must match servername)"</span>,
<a name="326" href="#326">326</a>             ConfigurableX509TrustManager.DEFAULT,
<a name="327" href="#327">327</a>             ConfigurableX509TrustManager.LEVELS_AS_ARRAY));
<a name="328" href="#328">328</a>         e.setOverrideable(false);
<a name="329" href="#329">329</a>         e.setExpertSetting(<strong>true</strong>);
<a name="330" href="#330">330</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/StringList.html">StringList</a>(ATTR_ACCEPT_HEADERS,
<a name="331" href="#331">331</a>             <span class="string">"Accept Headers to include in each request. Each must be the"</span>
<a name="332" href="#332">332</a>             + <span class="string">" complete header, e.g., 'Accept-Language: en'"</span>));
<a name="333" href="#333">333</a>         e.setExpertSetting(<strong>true</strong>);
<a name="334" href="#334">334</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_HTTP_PROXY_HOST,
<a name="335" href="#335">335</a>             <span class="string">"Proxy host IP (set only if needed)."</span>, <span class="string">""</span>));
<a name="336" href="#336">336</a>         e.setExpertSetting(<strong>true</strong>);
<a name="337" href="#337">337</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_HTTP_PROXY_PORT,
<a name="338" href="#338">338</a>             <span class="string">"Proxy port (set only if needed)"</span>, <span class="string">""</span>));
<a name="339" href="#339">339</a>         e.setExpertSetting(<strong>true</strong>);
<a name="340" href="#340">340</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_DEFAULT_ENCODING,
<a name="341" href="#341">341</a>             <span class="string">"The character encoding to use for files that do not have one"</span> +
<a name="342" href="#342">342</a>             <span class="string">" specified in the HTTP response headers.  Default: "</span> +
<a name="343" href="#343">343</a>             DEFAULT_CONTENT_CHARSET + <span class="string">"."</span>,
<a name="344" href="#344">344</a>             DEFAULT_CONTENT_CHARSET));
<a name="345" href="#345">345</a>         e.setExpertSetting(<strong>true</strong>);
<a name="346" href="#346">346</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_SHA1_CONTENT,
<a name="347" href="#347">347</a>                 <span class="string">"Whether or not to perform an on-the-fly SHA1 hash of"</span> +
<a name="348" href="#348">348</a>                 <span class="string">"retrieved content-bodies."</span>,
<a name="349" href="#349">349</a>                 DEFAULT_SHA1_CONTENT));
<a name="350" href="#350">350</a>         e.setExpertSetting(<strong>true</strong>);
<a name="351" href="#351">351</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_SEND_CONNECTION_CLOSE,
<a name="352" href="#352">352</a>             <span class="string">"Send 'Connection: close' header with every request."</span>,
<a name="353" href="#353">353</a>              <strong>new</strong> Boolean(<strong>true</strong>)));
<a name="354" href="#354">354</a>         e.setOverrideable(<strong>true</strong>);
<a name="355" href="#355">355</a>         e.setExpertSetting(<strong>true</strong>);
<a name="356" href="#356">356</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_SEND_REFERER,
<a name="357" href="#357">357</a>              <span class="string">"Send 'Referer' header with every request.\n"</span> +
<a name="358" href="#358">358</a>              <span class="string">"The 'Referer' header contans the location the crawler came "</span> +
<a name="359" href="#359">359</a>              <span class="string">" from, "</span> +
<a name="360" href="#360">360</a>              <span class="string">"the page the current URI was discovered in. The 'Referer' "</span> +
<a name="361" href="#361">361</a>              <span class="string">"usually is "</span> +
<a name="362" href="#362">362</a>              <span class="string">"logged on the remote server and can be of assistance to "</span> +
<a name="363" href="#363">363</a>              <span class="string">"webmasters trying to figure how a crawler got to a "</span> +
<a name="364" href="#364">364</a>              <span class="string">"particular area on a site."</span>,
<a name="365" href="#365">365</a>              <strong>new</strong> Boolean(<strong>true</strong>)));
<a name="366" href="#366">366</a>         e.setOverrideable(<strong>true</strong>);
<a name="367" href="#367">367</a>         e.setExpertSetting(<strong>true</strong>);
<a name="368" href="#368">368</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_SEND_RANGE,
<a name="369" href="#369">369</a>               <span class="string">"Send 'Range' header when a limit ("</span> + ATTR_MAX_LENGTH_BYTES +
<a name="370" href="#370">370</a>               <span class="string">") on document size.\n"</span> +
<a name="371" href="#371">371</a>               <span class="string">"Be polite to the HTTP servers and send the 'Range' header,"</span> +
<a name="372" href="#372">372</a>               <span class="string">"stating that you are only interested in the first n bytes. "</span> +
<a name="373" href="#373">373</a>               <span class="string">"Only pertinent if "</span> + ATTR_MAX_LENGTH_BYTES + <span class="string">" > 0. "</span> +
<a name="374" href="#374">374</a>               <span class="string">"Sending the 'Range' header results in a "</span> +
<a name="375" href="#375">375</a>               <span class="string">"'206 Partial Content' status response, which is better than "</span> +
<a name="376" href="#376">376</a>               <span class="string">"just cutting the response mid-download. On rare occasion, "</span> +
<a name="377" href="#377">377</a>               <span class="string">" sending 'Range' will "</span> +
<a name="378" href="#378">378</a>               <span class="string">"generate '416 Request Range Not Satisfiable' response."</span>,
<a name="379" href="#379">379</a>               <strong>new</strong> Boolean(false)));
<a name="380" href="#380">380</a>            e.setOverrideable(<strong>true</strong>);
<a name="381" href="#381">381</a>            e.setExpertSetting(<strong>true</strong>);
<a name="382" href="#382">382</a>            e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_LOCAL_ADDRESS,
<a name="383" href="#383">383</a>                <span class="string">"Local IP address or hostname to use when making connections "</span> +
<a name="384" href="#384">384</a>                <span class="string">"(binding sockets). When not specified, uses default local"</span> +