mirrorwriterprocessor.html

来自「网络爬虫开源代码」· HTML 代码 · 共 696 行 · 第 1/5 页

<a name="412" href="#412">412</a>             segs[i].addToPath(r);
<a name="413" href="#413">413</a>             <strong>if</strong> (r.longerThan(maxLen)) {
<a name="414" href="#414">414</a>                 <strong>return</strong> <strong>null</strong>;
<a name="415" href="#415">415</a>             }
<a name="416" href="#416">416</a>         }
<a name="417" href="#417">417</a>         <strong>return</strong> r;
<a name="418" href="#418">418</a>     }
<a name="419" href="#419">419</a> 
<a name="420" href="#420">420</a>     <em>/**<em>*</em></em>
<a name="421" href="#421">421</a> <em>       Ensures that a list contains an even number of elements.</em>
<a name="422" href="#422">422</a> <em>       If not, the last element is removed.</em>
<a name="423" href="#423">423</a> <em>       @param list the list</em>
<a name="424" href="#424">424</a> <em>    */</em>
<a name="425" href="#425">425</a>     <strong>private</strong> <strong>void</strong> ensurePairs(<a href="../../../../org/archive/crawler/settings/ListType.html">ListType</a> list) {
<a name="426" href="#426">426</a>         <strong>if</strong> (1 == (list.size() % 2)) {
<a name="427" href="#427">427</a>             list.remove(list.size() - 1);
<a name="428" href="#428">428</a>         }
<a name="429" href="#429">429</a>     }
<a name="430" href="#430">430</a> 
<a name="431" href="#431">431</a>     <em>/**<em>*</em></em>
<a name="432" href="#432">432</a> <em>       Makes a path in which a resource can be stored.</em>
<a name="433" href="#433">433</a> <em>       @param baseDir the path to the starting directory</em>
<a name="434" href="#434">434</a> <em>       @param curi the URI</em>
<a name="435" href="#435">435</a> <em>       @return a path to the file in which to store the resource</em>
<a name="436" href="#436">436</a> <em>       @throws AttributeNotFoundException</em>
<a name="437" href="#437">437</a> <em>       if a needed setting is missing</em>
<a name="438" href="#438">438</a> <em>       @throws IOException</em>
<a name="439" href="#439">439</a> <em>       if a needed directory could not be created</em>
<a name="440" href="#440">440</a> <em>       @throws IOException</em>
<a name="441" href="#441">441</a> <em>       if a needed directory is not writeable</em>
<a name="442" href="#442">442</a> <em>       @throws IOException</em>
<a name="443" href="#443">443</a> <em>       if a non-directory file exists with the same path as a needed directory</em>
<a name="444" href="#444">444</a> <em>    */</em>
<a name="445" href="#445">445</a>     <strong>private</strong> URIToFileReturn uriToFile(String baseDir, <a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi)
<a name="446" href="#446">446</a>         throws AttributeNotFoundException, IOException {
<a name="447" href="#447">447</a>         <a href="../../../../org/archive/net/UURI.html">UURI</a> uuri = curi.getUURI(); <em class="comment">// Current URI.</em>
<a name="448" href="#448">448</a>         String host = <strong>null</strong>;
<a name="449" href="#449">449</a>         Boolean hd = (Boolean) getAttribute(ATTR_HOST_DIRECTORY, curi);
<a name="450" href="#450">450</a>         <strong>if</strong> (hd.booleanValue()) {
<a name="451" href="#451">451</a>             host = uuri.getHost();
<a name="452" href="#452">452</a>             <a href="../../../../org/archive/crawler/settings/StringList.html">StringList</a> hostMap = (StringList) getAttribute(ATTR_HOST_MAP, curi);
<a name="453" href="#453">453</a>             <strong>if</strong> ((<strong>null</strong> != hostMap) &amp;&amp; (hostMap.size() > 1)) {
<a name="454" href="#454">454</a>                 ensurePairs(hostMap);
<a name="455" href="#455">455</a>                 Iterator&lt;String> i = hostMap.typesafe().iterator();
<a name="456" href="#456">456</a>                 <strong>for</strong> (<strong>boolean</strong> more = <strong>true</strong>; more &amp;&amp; i.hasNext();) {
<a name="457" href="#457">457</a>                     String h1 = i.next();
<a name="458" href="#458">458</a>                     String h2 = i.next();
<a name="459" href="#459">459</a>                     <strong>if</strong> (host.equalsIgnoreCase(h1)) {
<a name="460" href="#460">460</a>                         more = false;
<a name="461" href="#461">461</a>                         <strong>if</strong> ((<strong>null</strong> != h2) &amp;&amp; (0 != h2.length())) {
<a name="462" href="#462">462</a>                             host = h2;
<a name="463" href="#463">463</a>                         }
<a name="464" href="#464">464</a>                     }
<a name="465" href="#465">465</a>                 }
<a name="466" href="#466">466</a>             }
<a name="467" href="#467">467</a>         }
<a name="468" href="#468">468</a> 
<a name="469" href="#469">469</a>         <strong>int</strong> port =
<a name="470" href="#470">470</a>             ((Boolean) getAttribute(ATTR_PORT_DIRECTORY, curi)).booleanValue()
<a name="471" href="#471">471</a>             ? uuri.getPort()
<a name="472" href="#472">472</a>             : -1;
<a name="473" href="#473">473</a> 
<a name="474" href="#474">474</a>         String suffix = <strong>null</strong>; <em class="comment">// Replacement suffix.</em>
<a name="475" href="#475">475</a>         <a href="../../../../org/archive/crawler/settings/StringList.html">StringList</a> ctm = (StringList) getAttribute(ATTR_CONTENT_TYPE_MAP, curi);
<a name="476" href="#476">476</a>         <strong>if</strong> ((<strong>null</strong> != ctm) &amp;&amp; (ctm.size() > 1)) {
<a name="477" href="#477">477</a>             ensurePairs(ctm);
<a name="478" href="#478">478</a>             String contentType = curi.getContentType().toLowerCase();
<a name="479" href="#479">479</a>             Iterator i = ctm.iterator();
<a name="480" href="#480">480</a>             <strong>for</strong> (<strong>boolean</strong> more = <strong>true</strong>; more &amp;&amp; i.hasNext();) {
<a name="481" href="#481">481</a>                 String ct = (String) i.next();
<a name="482" href="#482">482</a>                 String suf = (String) i.next();
<a name="483" href="#483">483</a>                 <strong>if</strong> ((<strong>null</strong> != ct) &amp;&amp; contentType.startsWith(ct.toLowerCase())) {
<a name="484" href="#484">484</a>                     more = false;
<a name="485" href="#485">485</a>                     <strong>if</strong> ((<strong>null</strong> != suf) &amp;&amp; (0 != suf.length())) {
<a name="486" href="#486">486</a>                         suffix = suf;
<a name="487" href="#487">487</a>                     }
<a name="488" href="#488">488</a>                 }
<a name="489" href="#489">489</a>             }
<a name="490" href="#490">490</a>         }
<a name="491" href="#491">491</a> 
<a name="492" href="#492">492</a>         <strong>int</strong> maxSegLen =
<a name="493" href="#493">493</a>             ((Integer) getAttribute(ATTR_MAX_SEG_LEN, curi)).intValue();
<a name="494" href="#494">494</a>         <strong>if</strong> (maxSegLen &lt; 2) {
<a name="495" href="#495">495</a>             maxSegLen = DEFAULT_MAX_SEG_LEN;
<a name="496" href="#496">496</a>         }
<a name="497" href="#497">497</a> 
<a name="498" href="#498">498</a>         <strong>int</strong> maxPathLen =
<a name="499" href="#499">499</a>             ((Integer) getAttribute(ATTR_MAX_PATH_LEN, curi)).intValue();
<a name="500" href="#500">500</a>         <strong>if</strong> (maxPathLen &lt; 2) {
<a name="501" href="#501">501</a>             maxPathLen = DEFAULT_MAX_PATH_LEN;
<a name="502" href="#502">502</a>         }
<a name="503" href="#503">503</a> 
<a name="504" href="#504">504</a>         Map&lt;String,String> characterMap = EMPTY_MAP;
<a name="505" href="#505">505</a>         <a href="../../../../org/archive/crawler/settings/StringList.html">StringList</a> cm = (StringList) getAttribute(ATTR_CHAR_MAP, curi);
<a name="506" href="#506">506</a>         <strong>if</strong> ((<strong>null</strong> != cm) &amp;&amp; (cm.size() > 1)) {
<a name="507" href="#507">507</a>             ensurePairs(cm);
<a name="508" href="#508">508</a>             characterMap = <strong>new</strong> HashMap&lt;String,String>(cm.size()); 
<a name="509" href="#509">509</a>             <em class="comment">// Above will be half full.</em>
<a name="510" href="#510">510</a>             <strong>for</strong> (Iterator i = cm.iterator(); i.hasNext();) {
<a name="511" href="#511">511</a>                 String s1 = (String) i.next();
<a name="512" href="#512">512</a>                 String s2 = (String) i.next();
<a name="513" href="#513">513</a>                 <strong>if</strong> ((<strong>null</strong> != s1) &amp;&amp; (1 == s1.length()) &amp;&amp; (<strong>null</strong> != s2)
<a name="514" href="#514">514</a>                         &amp;&amp; (0 != s2.length())) {
<a name="515" href="#515">515</a>                     characterMap.put(s1, s2);
<a name="516" href="#516">516</a>                 }
<a name="517" href="#517">517</a>             }
<a name="518" href="#518">518</a>         }
<a name="519" href="#519">519</a> 
<a name="520" href="#520">520</a>         String dotBegin = (String) getAttribute(ATTR_DOT_BEGIN, curi);
<a name="521" href="#521">521</a>         <strong>if</strong> (<span class="string">"."</span>.equals(dotBegin)) {
<a name="522" href="#522">522</a>             dotBegin = <strong>null</strong>;
<a name="523" href="#523">523</a>         }
<a name="524" href="#524">524</a> 
<a name="525" href="#525">525</a>         String dotEnd = (String) getAttribute(ATTR_DOT_END, curi);
<a name="526" href="#526">526</a>         <strong>if</strong> (<span class="string">"."</span>.equals(dotEnd)) {
<a name="527" href="#527">527</a>             dotEnd = <strong>null</strong>;
<a name="528" href="#528">528</a>         }
<a name="529" href="#529">529</a> 
<a name="530" href="#530">530</a>         String tld = (String) getAttribute(ATTR_TOO_LONG_DIRECTORY, curi);
<a name="531" href="#531">531</a>         <strong>if</strong> ((<strong>null</strong> == tld) || (0 == tld.length())
<a name="532" href="#532">532</a>                 || (-1 != tld.indexOf(File.separatorChar))) {
<a name="533" href="#533">533</a>             tld = DEFAULT_TOO_LONG_DIRECTORY;
<a name="534" href="#534">534</a>         }
<a name="535" href="#535">535</a> 
<a name="536" href="#536">536</a>         Set&lt;String> underscoreSet = <strong>null</strong>;
<a name="537" href="#537">537</a>         <a href="../../../../org/archive/crawler/settings/StringList.html">StringList</a> us = (StringList) getAttribute(ATTR_UNDERSCORE_SET, curi);
<a name="538" href="#538">538</a>         <strong>if</strong> ((<strong>null</strong> != us) &amp;&amp; (0 != us.size())) {
<a name="539" href="#539">539</a>             underscoreSet = <strong>new</strong> HashSet&lt;String>(us.size(), 0.5F);
<a name="540" href="#540">540</a>             <strong>for</strong> (String s: us.typesafe()) {
<a name="541" href="#541">541</a>                 <strong>if</strong> ((<strong>null</strong> != s) &amp;&amp; (0 != s.length())) {
<a name="542" href="#542">542</a>                     underscoreSet.add(s.toLowerCase());
<a name="543" href="#543">543</a>                 }
<a name="544" href="#544">544</a>             }
<a name="545" href="#545">545</a>         }
<a name="546" href="#546">546</a> 
<a name="547" href="#547">547</a>         <strong>return</strong> uriToFile(curi, host, port, uuri.getPath(), uuri.getQuery(),
<a name="548" href="#548">548</a>             suffix, baseDir, maxSegLen, maxPathLen,
<a name="549" href="#549">549</a>             ((Boolean) getAttribute(ATTR_CASE_SENSITIVE, curi)).booleanValue(),
<a name="550" href="#550">550</a>             (String) getAttribute(ATTR_DIRECTORY_FILE, curi),
<a name="551" href="#551">551</a>             characterMap, dotBegin, dotEnd, tld,