mirrorwriterprocessor.html

用JAVA编写的,在做实验的时候留下来的,本来想删的,但是传上来,大家分享吧

<a name="548" href="#548">548</a>             underscoreSet);
<a name="549" href="#549">549</a>     }
<a name="550" href="#550">550</a> 
<a name="551" href="#551">551</a>     <em>/**<em>*</em></em>
<a name="552" href="#552">552</a> <em>       Makes a path in which a resource can be stored.</em>
<a name="553" href="#553">553</a> <em>       @param curi the URI</em>
<a name="554" href="#554">554</a> <em>       @param host the host part of the URI, or null if the host name</em>
<a name="555" href="#555">555</a> <em>       should not be part of the returned path</em>
<a name="556" href="#556">556</a> <em>       @param port the port part of the URI, or -1 if the port</em>
<a name="557" href="#557">557</a> <em>       should not be part of the returned path</em>
<a name="558" href="#558">558</a> <em>       @param uriPath the path part of the URI (must be absolute)</em>
<a name="559" href="#559">559</a> <em>       @param query the query part of the URI, or null if none</em>
<a name="560" href="#560">560</a> <em>       @param suffix if non-null, use this as the suffix in preference to</em>
<a name="561" href="#561">561</a> <em>       any suffix that uriPath might have</em>
<a name="562" href="#562">562</a> <em>       @param baseDir the path to the starting directory</em>
<a name="563" href="#563">563</a> <em>       @param maxSegLen the maximum number of characters allowed in one</em>
<a name="564" href="#564">564</a> <em>       file system path segment (component)</em>
<a name="565" href="#565">565</a> <em>       @param maxPathLen the maximum number of characters allowed in a</em>
<a name="566" href="#566">566</a> <em>       file system path</em>
<a name="567" href="#567">567</a> <em>       @param caseSensitive if true, the file system is assumed to be</em>
<a name="568" href="#568">568</a> <em>       case-sensitive; otherwise the file system is assumed to be</em>
<a name="569" href="#569">569</a> <em>       case-insensitive but case-preserving</em>
<a name="570" href="#570">570</a> <em>       @param dirFile the simple file name to append to a URI path</em>
<a name="571" href="#571">571</a> <em>       ending in '/'</em>
<a name="572" href="#572">572</a> <em>       @param characterMap a map from characters (as length-1 String values) in</em>
<a name="573" href="#573">573</a> <em>       the URI path and query to replacement String values</em>
<a name="574" href="#574">574</a> <em>       @param dotBegin if non-null, this replaces a '.' at</em>
<a name="575" href="#575">575</a> <em>       the beginning of a segment</em>
<a name="576" href="#576">576</a> <em>       @param dotEnd if non-null, this replaces a '.' that appears at the end</em>
<a name="577" href="#577">577</a> <em>       of a directory name</em>
<a name="578" href="#578">578</a> <em>       @param tooLongDir if the path length would exceed or be close to</em>
<a name="579" href="#579">579</a> <em>       exceeding maxPathLen then this simple name is used as a directory</em>
<a name="580" href="#580">580</a> <em>       under baseDir instead</em>
<a name="581" href="#581">581</a> <em>       @param suffixAtEnd if true, the suffix is placed at the end of the</em>
<a name="582" href="#582">582</a> <em>       path, after the query (if any); otherwise, the suffix is placed</em>
<a name="583" href="#583">583</a> <em>       before the query</em>
<a name="584" href="#584">584</a> <em>       @param underscoreSet if non-null and a segment, after conversion</em>
<a name="585" href="#585">585</a> <em>       to lower case, is in this set, then prepend an underscore</em>
<a name="586" href="#586">586</a> <em>       to the segment</em>
<a name="587" href="#587">587</a> <em>       @return a path to the file in which to store the resource</em>
<a name="588" href="#588">588</a> <em>       @throws IOException</em>
<a name="589" href="#589">589</a> <em>       if a needed directory could not be created</em>
<a name="590" href="#590">590</a> <em>       @throws IOException</em>
<a name="591" href="#591">591</a> <em>       if a needed directory is not writeable</em>
<a name="592" href="#592">592</a> <em>       @throws IOException</em>
<a name="593" href="#593">593</a> <em>       if a non-directory file exists with the same path as a needed directory</em>
<a name="594" href="#594">594</a> <em>    */</em>
<a name="595" href="#595">595</a>     <strong>private</strong> URIToFileReturn uriToFile(<a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi, String host, <strong>int</strong> port,
<a name="596" href="#596">596</a>             String uriPath, String query, String suffix, String baseDir,
<a name="597" href="#597">597</a>             <strong>int</strong> maxSegLen, <strong>int</strong> maxPathLen, <strong>boolean</strong> caseSensitive,
<a name="598" href="#598">598</a>             String dirFile, Map characterMap, String dotBegin, String dotEnd,
<a name="599" href="#599">599</a>             String tooLongDir, <strong>boolean</strong> suffixAtEnd, Set underscoreSet)
<a name="600" href="#600">600</a>             throws IOException {
<a name="601" href="#601">601</a>         assert (<strong>null</strong> == host) || (0 != host.length());
<a name="602" href="#602">602</a>         assert 0 != uriPath.length();
<a name="603" href="#603">603</a>         assert '/' == uriPath.charAt(0) : <span class="string">"uriPath: "</span> + uriPath;
<a name="604" href="#604">604</a>         assert -1 == uriPath.indexOf(<span class="string">"//"</span>) : <span class="string">"uriPath: "</span> + uriPath;
<a name="605" href="#605">605</a>         assert -1 == uriPath.indexOf(<span class="string">"/./"</span>) : <span class="string">"uriPath: "</span> + uriPath;
<a name="606" href="#606">606</a>         assert !uriPath.endsWith(<span class="string">"/."</span>) : <span class="string">"uriPath: "</span> + uriPath;
<a name="607" href="#607">607</a>         assert (<strong>null</strong> == query) || (-1 == query.indexOf('/'))
<a name="608" href="#608">608</a>             : <span class="string">"query: "</span> + query;
<a name="609" href="#609">609</a>         assert (<strong>null</strong> == suffix)
<a name="610" href="#610">610</a>             || ((0 != suffix.length()) &amp;&amp; (-1 == suffix.indexOf('/')))
<a name="611" href="#611">611</a>             : <span class="string">"suffix: "</span> + suffix;
<a name="612" href="#612">612</a>         assert 0 != baseDir.length();
<a name="613" href="#613">613</a>         assert maxSegLen > 2 : <span class="string">"maxSegLen: "</span> + maxSegLen;
<a name="614" href="#614">614</a>         assert maxPathLen > 1;
<a name="615" href="#615">615</a>         assert maxPathLen >= maxSegLen
<a name="616" href="#616">616</a>             : <span class="string">"maxSegLen: "</span> + maxSegLen + <span class="string">" maxPathLen: "</span> + maxPathLen;
<a name="617" href="#617">617</a>         assert 0 != dirFile.length();
<a name="618" href="#618">618</a>         assert -1 == dirFile.indexOf(<span class="string">"/"</span>) : <span class="string">"dirFile: "</span> + dirFile;
<a name="619" href="#619">619</a>         assert <strong>null</strong> != characterMap;
<a name="620" href="#620">620</a>         assert (<strong>null</strong> == dotBegin) || (0 != dotBegin.length());
<a name="621" href="#621">621</a>         assert (<strong>null</strong> == dotEnd) || !dotEnd.endsWith(<span class="string">"."</span>) : <span class="string">"dotEnd: "</span> + dotEnd;
<a name="622" href="#622">622</a>         assert 0 != tooLongDir.length();
<a name="623" href="#623">623</a>         assert '/' != tooLongDir.charAt(0) : <span class="string">"tooLongDir: "</span> + tooLongDir;
<a name="624" href="#624">624</a> 
<a name="625" href="#625">625</a>         <strong>int</strong> nSegs = 0; <em class="comment">// Number of segments in the URI path.</em>
<a name="626" href="#626">626</a>         <strong>for</strong> (<strong>int</strong> i = 0; uriPath.length() != i; ++i) {
<a name="627" href="#627">627</a>             <strong>if</strong> ('/' == uriPath.charAt(i)) {
<a name="628" href="#628">628</a>                 ++nSegs; <em class="comment">// Just count slashes.</em>
<a name="629" href="#629">629</a>             }
<a name="630" href="#630">630</a>         }
<a name="631" href="#631">631</a>         assert nSegs > 0 : <span class="string">"uriPath: "</span> + uriPath;
<a name="632" href="#632">632</a>         PathSegment[] segs = <strong>new</strong> PathSegment[nSegs]; <em class="comment">// The segments.</em>
<a name="633" href="#633">633</a>         <strong>int</strong> slashIndex = 0; <em class="comment">// Index in uriPath of current /.</em>
<a name="634" href="#634">634</a>         <strong>for</strong> (<strong>int</strong> i = 0; (segs.length - 1) != i; ++i) {
<a name="635" href="#635">635</a>             <strong>int</strong> nsi = uriPath.indexOf('/', slashIndex + 1); <em class="comment">// Next index.</em>
<a name="636" href="#636">636</a>             assert nsi > slashIndex : <span class="string">"uriPath: "</span> + uriPath;
<a name="637" href="#637">637</a>             segs[i] = <strong>new</strong> DirSegment(uriPath, slashIndex + 1, nsi,
<a name="638" href="#638">638</a>                                      maxSegLen, caseSensitive, curi,
<a name="639" href="#639">639</a>                                      characterMap, dotBegin, dotEnd,
<a name="640" href="#640">640</a>                                      underscoreSet);
<a name="641" href="#641">641</a>             slashIndex = nsi;
<a name="642" href="#642">642</a>         }
<a name="643" href="#643">643</a>         <strong>if</strong> (slashIndex &lt; (uriPath.length() - 1)) {
<a name="644" href="#644">644</a> 
<a name="645" href="#645">645</a>             <em class="comment">// There's something after the last /.</em>
<a name="646" href="#646">646</a>             segs[segs.length - 1] = <strong>new</strong> EndSegment(uriPath, slashIndex + 1,
<a name="647" href="#647">647</a>                     uriPath.length(), maxSegLen, caseSensitive, curi,
<a name="648" href="#648">648</a>                     characterMap, dotBegin, query, suffix, maxPathLen,
<a name="649" href="#649">649</a>                     suffixAtEnd);
<a name="650" href="#650">650</a>         } <strong>else</strong> {
<a name="651" href="#651">651</a> 
<a name="652" href="#652">652</a>             <em class="comment">// The URI ends with a /.</em>
<a name="653" href="#653">653</a>             segs[segs.length - 1] = <strong>new</strong> EndSegment(dirFile, 0, dirFile.length(),
<a name="654" href="#654">654</a>                     maxSegLen, caseSensitive, curi, characterMap, <strong>null</strong>,
<a name="655" href="#655">655</a>                     query, suffix, maxPathLen, suffixAtEnd);
<a name="656" href="#656">656</a>         }
<a name="657" href="#657">657</a>         URIToFileReturn r = dirPath(baseDir, host, port, segs,
<a name="658" href="#658">658</a>                                     maxPathLen - maxSegLen);
<a name="659" href="#659">659</a>         <strong>if</strong> (<strong>null</strong> == r) {
<a name="660" href="#660">660</a> 
<a name="661" href="#661">661</a>             <em class="comment">// The path is too long.</em>
<a name="662" href="#662">662</a>             <em class="comment">// Replace all the segment directories by tooLongDir.</em>
<a name="663" href="#663">663</a>             PathSegment endSegment = segs[segs.length - 1];
<a name="664" href="#664">664</a>             segs = <strong>new</strong> PathSegment[2];
<a name="665" href="#665">665</a>             segs[0] = <strong>new</strong> DirSegment(tooLongDir, 0, tooLongDir.length(),
<a name="666" href="#666">666</a>                                      maxSegLen, caseSensitive, curi, EMPTY_MAP,
<a name="667" href="#667">667</a>                                      <strong>null</strong>, <strong>null</strong>, <strong>null</strong>);
<a name="668" href="#668">668</a>             segs[1] = endSegment;
<a name="669" href="#669">669</a>             r = dirPath(baseDir, host, port, segs, maxPathLen - maxSegLen);
<a name="670" href="#670">670</a>         }
<a name="671" href="#671">671</a>         segs[segs.length - 1].addToPath(r);
<a name="672" href="#672">672</a>         <strong>return</strong> r;
<a name="673" href="#673">673</a>     }
<a name="674" href="#674">674</a> 
<a name="675" href="#675">675</a>     <em>/**<em>*</em></em>
<a name="676" href="#676">676</a> <em>       Copies a resource into a file.</em>
<a name="677" href="#677">677</a> <em>       A temporary file is created and then atomically renamed to</em>
<a name="678" href="#678">678</a> <em>       the destination file.</em>
<a name="679" href="#679">679</a> <em>       This prevents leaving a partial file in case of a crash.</em>
<a name="680" href="#680">680</a> <em>       @param recis the RecordingInputStream that recorded the contents</em>
<a name="681" href="#681">681</a> <em>       of the resource</em>
<a name="682" href="#682">682</a> <em>       @param dest the destination file</em>
<a name="683" href="#683">683</a> <em>       @throws IOException on I/O error</em>
<a name="684" href="#684">684</a> <em>       @throws IOException if</em>
<a name="685" href="#685">685</a> <em>       the file rename fails</em>
<a name="686" h