fetchhttp.html

来自「网络爬虫开源代码」· HTML 代码 · 共 643 行 · 第 1/5 页

<a name="250" href="#250">250</a>     <em>/**<em>*</em></em>
<a name="251" href="#251">251</a> <em>     * Name of cookie BDB Database</em>
<a name="252" href="#252">252</a> <em>     */</em>
<a name="253" href="#253">253</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String COOKIEDB_NAME = <span class="string">"http_cookies"</span>;
<a name="254" href="#254">254</a>     
<a name="255" href="#255">255</a>     <strong>static</strong> {
<a name="256" href="#256">256</a>     	Protocol.registerProtocol(<span class="string">"http"</span>, <strong>new</strong> Protocol(<span class="string">"http"</span>,
<a name="257" href="#257">257</a>             <strong>new</strong> <a href="../../../../org/archive/crawler/fetcher/HeritrixProtocolSocketFactory.html">HeritrixProtocolSocketFactory</a>(), 80));
<a name="258" href="#258">258</a>     	<strong>try</strong> {
<a name="259" href="#259">259</a> 			Protocol.registerProtocol(<span class="string">"https"</span>,
<a name="260" href="#260">260</a> 			    <strong>new</strong> Protocol(<span class="string">"https"</span>, ((ProtocolSocketFactory)
<a name="261" href="#261">261</a> 			        <strong>new</strong> HeritrixSSLProtocolSocketFactory()), 443));
<a name="262" href="#262">262</a> 		} <strong>catch</strong> (KeyManagementException e) {
<a name="263" href="#263">263</a> 			e.printStackTrace();
<a name="264" href="#264">264</a> 		} <strong>catch</strong> (KeyStoreException e) {
<a name="265" href="#265">265</a> 			e.printStackTrace();
<a name="266" href="#266">266</a> 		} <strong>catch</strong> (NoSuchAlgorithmException e) {
<a name="267" href="#267">267</a> 			e.printStackTrace();
<a name="268" href="#268">268</a> 		}
<a name="269" href="#269">269</a>     }
<a name="270" href="#270">270</a>     <strong>static</strong> <strong>final</strong> String SERVER_CACHE_KEY = <span class="string">"heritrix.server.cache"</span>;
<a name="271" href="#271">271</a>     <strong>static</strong> <strong>final</strong> String SSL_FACTORY_KEY = <span class="string">"heritrix.ssl.factory"</span>;
<a name="272" href="#272">272</a>     
<a name="273" href="#273">273</a>     <em>/**<em>**</em></em>
<a name="274" href="#274">274</a> <em>     * Socket factory that has the configurable trust manager installed.</em>
<a name="275" href="#275">275</a> <em>     */</em>
<a name="276" href="#276">276</a>     <strong>private</strong> SSLSocketFactory sslfactory = <strong>null</strong>;
<a name="277" href="#277">277</a>     
<a name="278" href="#278">278</a> 
<a name="279" href="#279">279</a>     <em>/**<em>*</em></em>
<a name="280" href="#280">280</a> <em>     * Constructor.</em>
<a name="281" href="#281">281</a> <em>     *</em>
<a name="282" href="#282">282</a> <em>     * @param name Name of this processor.</em>
<a name="283" href="#283">283</a> <em>     */</em>
<a name="284" href="#284">284</a>     <strong>public</strong> <a href="../../../../org/archive/crawler/fetcher/FetchHTTP.html">FetchHTTP</a>(String name) {
<a name="285" href="#285">285</a>         <strong>super</strong>(name, <span class="string">"HTTP Fetcher"</span>);
<a name="286" href="#286">286</a> 
<a name="287" href="#287">287</a>         addElementToDefinition(
<a name="288" href="#288">288</a>             <strong>new</strong> <a href="../../../../org/archive/crawler/deciderules/DecideRuleSequence.html">DecideRuleSequence</a>(ATTR_MIDFETCH_DECIDE_RULES, 
<a name="289" href="#289">289</a>                 <span class="string">"DecideRules which, if final decision is REJECT, "</span> +
<a name="290" href="#290">290</a>                 <span class="string">"abort fetch after headers before all content is"</span> +
<a name="291" href="#291">291</a>                 <span class="string">"read."</span>));
<a name="292" href="#292">292</a>         
<a name="293" href="#293">293</a>         addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_TIMEOUT_SECONDS,
<a name="294" href="#294">294</a>             <span class="string">"If the fetch is not completed in this number of seconds, "</span>
<a name="295" href="#295">295</a>             + <span class="string">"even if it is making progress, give up. The URI will be "</span>
<a name="296" href="#296">296</a>             + <span class="string">"annotated as timeTrunc. Set to zero for no timeout. "</span>
<a name="297" href="#297">297</a>             + <span class="string">"(This is not recommended: threads could wait indefinitely "</span>
<a name="298" href="#298">298</a>             + <span class="string">"for the fetch to end.)"</span>,
<a name="299" href="#299">299</a>             DEFAULT_TIMEOUT_SECONDS));
<a name="300" href="#300">300</a>         <a href="../../../../org/archive/crawler/settings/Type.html">Type</a> e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_SOTIMEOUT_MS,
<a name="301" href="#301">301</a>             <span class="string">"If a socket is unresponsive for this number of milliseconds, "</span> +
<a name="302" href="#302">302</a>             <span class="string">"give up on that connects/read. (This does not necessarily give "</span> +
<a name="303" href="#303">303</a>             <span class="string">"up on the fetch immediately; connects are subject to retries "</span> +
<a name="304" href="#304">304</a>             <span class="string">"and reads will be retried until "</span> + ATTR_TIMEOUT_SECONDS +
<a name="305" href="#305">305</a>             <span class="string">" have elapsed. Set to zero for no socket timeout. (This is "</span> +
<a name="306" href="#306">306</a>             <span class="string">"note recommended: a socket operation could hand indefinitely."</span>,
<a name="307" href="#307">307</a>                 DEFAULT_SOTIMEOUT_MS));
<a name="308" href="#308">308</a>         e.setExpertSetting(<strong>true</strong>);
<a name="309" href="#309">309</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_FETCH_BANDWIDTH_MAX,
<a name="310" href="#310">310</a>             <span class="string">"The maximum KB/sec to use when fetching data from a server. "</span> +
<a name="311" href="#311">311</a>             <span class="string">"0 means no maximum.  Default: "</span>+ DEFAULT_FETCH_BANDWIDTH_MAX
<a name="312" href="#312">312</a>              + <span class="string">"."</span>, DEFAULT_FETCH_BANDWIDTH_MAX));
<a name="313" href="#313">313</a>         e.setExpertSetting(<strong>true</strong>);
<a name="314" href="#314">314</a>         e.setOverrideable(<strong>true</strong>);
<a name="315" href="#315">315</a>         addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_MAX_LENGTH_BYTES,
<a name="316" href="#316">316</a>             <span class="string">"Maximum length in bytes to fetch.\n"</span> +
<a name="317" href="#317">317</a>             <span class="string">"Fetch is truncated at this length. A value of 0 means no limit."</span>,
<a name="318" href="#318">318</a>             DEFAULT_MAX_LENGTH_BYTES));
<a name="319" href="#319">319</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_IGNORE_COOKIES,
<a name="320" href="#320">320</a>             <span class="string">"Disable cookie-handling."</span>, DEFAULT_IGNORE_COOKIES));
<a name="321" href="#321">321</a>         e.setOverrideable(<strong>true</strong>);
<a name="322" href="#322">322</a>         e.setExpertSetting(<strong>true</strong>);
<a name="323" href="#323">323</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_BDB_COOKIES,
<a name="324" href="#324">324</a>                 <span class="string">"Store cookies in BDB-backed map."</span>, DEFAULT_BDB_COOKIES));
<a name="325" href="#325">325</a>         e.setExpertSetting(<strong>true</strong>);
<a name="326" href="#326">326</a> 
<a name="327" href="#327">327</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_LOAD_COOKIES,
<a name="328" href="#328">328</a>             <span class="string">"File to preload cookies from"</span>, <span class="string">""</span>));
<a name="329" href="#329">329</a>         e.setExpertSetting(<strong>true</strong>);
<a name="330" href="#330">330</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_SAVE_COOKIES,
<a name="331" href="#331">331</a>             <span class="string">"When crawl finishes save cookies to this file"</span>, <span class="string">""</span>));
<a name="332" href="#332">332</a>         e.setExpertSetting(<strong>true</strong>);
<a name="333" href="#333">333</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_TRUST,
<a name="334" href="#334">334</a>             <span class="string">"SSL certificate trust level.  Range is from the default 'open'"</span>
<a name="335" href="#335">335</a>             + <span class="string">" (trust all certs including expired, selfsigned, and those for"</span>
<a name="336" href="#336">336</a>             + <span class="string">" which we do not have a CA) through 'loose' (trust all valid"</span>
<a name="337" href="#337">337</a>             + <span class="string">" certificates including selfsigned), 'normal' (all valid"</span>
<a name="338" href="#338">338</a>             + <span class="string">" certificates not including selfsigned) to 'strict' (Cert is"</span>
<a name="339" href="#339">339</a>             + <span class="string">" valid and DN must match servername)"</span>,
<a name="340" href="#340">340</a>             ConfigurableX509TrustManager.DEFAULT,
<a name="341" href="#341">341</a>             ConfigurableX509TrustManager.LEVELS_AS_ARRAY));
<a name="342" href="#342">342</a>         e.setOverrideable(false);
<a name="343" href="#343">343</a>         e.setExpertSetting(<strong>true</strong>);
<a name="344" href="#344">344</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/StringList.html">StringList</a>(ATTR_ACCEPT_HEADERS,
<a name="345" href="#345">345</a>             <span class="string">"Accept Headers to include in each request. Each must be the"</span>
<a name="346" href="#346">346</a>             + <span class="string">" complete header, e.g., 'Accept-Language: en'"</span>));
<a name="347" href="#347">347</a>         e.setExpertSetting(<strong>true</strong>);
<a name="348" href="#348">348</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_HTTP_PROXY_HOST,
<a name="349" href="#349">349</a>             <span class="string">"Proxy host IP (set only if needed)."</span>, <span class="string">""</span>));
<a name="350" href="#350">350</a>         e.setExpertSetting(<strong>true</strong>);
<a name="351" href="#351">351</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_HTTP_PROXY_PORT,
<a name="352" href="#352">352</a>             <span class="string">"Proxy port (set only if needed)"</span>, <span class="string">""</span>));
<a name="353" href="#353">353</a>         e.setExpertSetting(<strong>true</strong>);
<a name="354" href="#354">354</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_DEFAULT_ENCODING,
<a name="355" href="#355">355</a>             <span class="string">"The character encoding to use for files that do not have one"</span> +
<a name="356" href="#356">356</a>             <span class="string">" specified in the HTTP response headers.  Default: "</span> +
<a name="357" href="#357">357</a>             DEFAULT_CONTENT_CHARSET + <span class="string">"."</span>,
<a name="358" href="#358">358</a>             DEFAULT_CONTENT_CHARSET));
<a name="359" href="#359">359</a>         e.setExpertSetting(<strong>true</strong>);
<a name="360" href="#360">360</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_DIGEST_CONTENT,
<a name="361" href="#361">361</a>                 <span class="string">"Whether or not to perform an on-the-fly digest hash of"</span> +
<a name="362" href="#362">362</a>                 <span class="string">" retrieved content-bodies."</span>,
<a name="363" href="#363">363</a>                 DEFAULT_DIGEST_CONTENT));
<a name="364" href="#364">364</a>         e.setExpertSetting(<strong>true</strong>);
<a name="365" href="#365">365</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_DIGEST_ALGORITHM,
<a name="366" href="#366">366</a>                 <span class="string">"Which algorithm (for example MD5 or SHA-1) to use to perform an on-the-fly digest"</span> +
<a name="367" href="#367">367</a>                 <span class="string">" hash of retrieved content-bodies."</span>,
<a name="368" href="#368">368</a>                 DEFAULT_DIGEST_ALGORITHM, DIGEST_ALGORITHMS));
<a name="369" href="#369">369</a>         e.setExpertSetting(<strong>true</strong>);
<a name="370" href="#370">370</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_SEND_IF_MODIFIED_SINCE,
<a name="371" href="#371">371</a>                 <span class="string">"Send 'If-Modified-Since' header, if previous 'Last-Modified' "</span> +
<a name="372" href="#372">372</a>                 <span class="string">"fetch history information is available in URI history."</span>,
<a name="373" href="#373">373</a>                  <strong>new</strong> Boolean(<strong>true</strong>)));
<a name="374" href="#374">374</a>         e.setOverrideable(<strong>true</strong>);
<a name="375" href="#375">375</a>         e.setExpertSetting(<strong>true</strong>);
<a name="376" href="#376">376</a>         e = addElementToDefinition(<strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_SEND_IF_NONE_MATCH,
<a name="377" href="#377">377</a>                 <span class="string">"Send 'If-None-Match' header, if previous 'Etag' fetch "</span> +
<a name="378" href="#378">378</a>                 <span class="string">"history information is available in URI history."</span>,