jerichoextractorhtmltest.html

来自「网络爬虫开源代码」· HTML 代码 · 共 176 行 · 第 1/2 页

<a name="80" href="#80">80</a>                  getSettingsObject(<strong>null</strong>), <strong>new</strong> <a href="../../../../org/archive/crawler/extractor/JerichoExtractorHTML.html">JerichoExtractorHTML</a>(name));
<a name="81" href="#81">81</a>      }
<a name="82" href="#82">82</a>      
<a name="83" href="#83">83</a>      <strong>protected</strong> <strong>void</strong> setUp() throws Exception {
<a name="84" href="#84">84</a>          <strong>super</strong>.setUp();
<a name="85" href="#85">85</a>          <strong>this</strong>.extractor = createExtractor();
<a name="86" href="#86">86</a>          <strong>final</strong> <strong>boolean</strong> USE_NET = false;
<a name="87" href="#87">87</a>          URL url = <strong>null</strong>;
<a name="88" href="#88">88</a>          <strong>if</strong> (USE_NET) {
<a name="89" href="#89">89</a>              url = <strong>new</strong> URL(<span class="string">"http://"</span> + <strong>this</strong>.ARCHIVE_DOT_ORG);
<a name="90" href="#90">90</a>          } <strong>else</strong> {
<a name="91" href="#91">91</a>              File f = <strong>new</strong> File(getTmpDir(), <strong>this</strong>.ARCHIVE_DOT_ORG + <span class="string">".html"</span>);
<a name="92" href="#92">92</a>              url = <strong>new</strong> URL(<span class="string">"file://"</span> + f.getAbsolutePath());
<a name="93" href="#93">93</a>              FileOutputStream fos = <strong>new</strong> FileOutputStream(f);
<a name="94" href="#94">94</a>              fos.write((<span class="string">"&lt;html>&lt;head>&lt;title>test&lt;/title>&lt;body>"</span> +
<a name="95" href="#95">95</a>                  <span class="string">"&lt;a href="</span> + <strong>this</strong>.LINK_TO_FIND + <span class="string">">Hewlett Foundation&lt;/a>"</span> +
<a name="96" href="#96">96</a>                  <span class="string">"&lt;/body>&lt;/html>"</span>).getBytes());
<a name="97" href="#97">97</a>              fos.flush();
<a name="98" href="#98">98</a>              fos.close();
<a name="99" href="#99">99</a>          }
<a name="100" href="#100">100</a>         <strong>this</strong>.recorder = HttpRecorder.wrapInputStreamWithHttpRecord(getTmpDir(),
<a name="101" href="#101">101</a>             <strong>this</strong>.getClass().getName(), url.openStream(), <strong>null</strong>);
<a name="102" href="#102">102</a>     }
<a name="103" href="#103">103</a> 
<a name="104" href="#104">104</a> 
<a name="105" href="#105">105</a>     <strong>public</strong> <strong>void</strong> testInnerProcess() throws IOException {
<a name="106" href="#106">106</a>         <a href="../../../../org/archive/net/UURI.html">UURI</a> uuri = UURIFactory.getInstance(<span class="string">"http://"</span> + <strong>this</strong>.ARCHIVE_DOT_ORG);
<a name="107" href="#107">107</a>         <a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi = setupCrawlURI(<strong>this</strong>.recorder, uuri.toString());
<a name="108" href="#108">108</a>         <strong>this</strong>.extractor.innerProcess(curi);
<a name="109" href="#109">109</a>         Collection links = curi.getOutLinks();
<a name="110" href="#110">110</a>         <strong>boolean</strong> foundLinkToHewlettFoundation = false;
<a name="111" href="#111">111</a>         <strong>for</strong> (Iterator i = links.iterator(); i.hasNext();) {
<a name="112" href="#112">112</a>             <a href="../../../../org/archive/crawler/extractor/Link.html">Link</a> link = (Link)i.next();
<a name="113" href="#113">113</a>             <strong>if</strong> (link.getDestination().toString().equals(<strong>this</strong>.LINK_TO_FIND)) {
<a name="114" href="#114">114</a>                 foundLinkToHewlettFoundation = <strong>true</strong>;
<a name="115" href="#115">115</a>                 <strong>break</strong>;
<a name="116" href="#116">116</a>             }
<a name="117" href="#117">117</a>         }
<a name="118" href="#118">118</a>         assertTrue(<span class="string">"Did not find gif url"</span>, foundLinkToHewlettFoundation);
<a name="119" href="#119">119</a>     }
<a name="120" href="#120">120</a>     
<a name="121" href="#121">121</a>     <strong>private</strong> <a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> setupCrawlURI(<a href="../../../../org/archive/util/HttpRecorder.html">HttpRecorder</a> rec, String url)
<a name="122" href="#122">122</a>     		throws URIException {
<a name="123" href="#123">123</a>         <a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi = <strong>new</strong> <a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a>(UURIFactory.getInstance(url));
<a name="124" href="#124">124</a>         curi.setContentSize(<strong>this</strong>.recorder.getRecordedInput().getSize());
<a name="125" href="#125">125</a>         curi.setContentType(<span class="string">"text/html"</span>);
<a name="126" href="#126">126</a>         curi.setFetchStatus(200);
<a name="127" href="#127">127</a>         curi.setHttpRecorder(rec);
<a name="128" href="#128">128</a>         <em class="comment">// Fake out the extractor that this is a HTTP transaction.</em>
<a name="129" href="#129">129</a>         curi.putObject(CoreAttributeConstants.A_HTTP_TRANSACTION,
<a name="130" href="#130">130</a>             <strong>new</strong> Object());
<a name="131" href="#131">131</a>         <strong>return</strong> curi;
<a name="132" href="#132">132</a>     }
<a name="133" href="#133">133</a> 
<a name="134" href="#134">134</a>     
<a name="135" href="#135">135</a>     <em>/**<em>*</em></em>
<a name="136" href="#136">136</a> <em>     * Test a forms link extraction</em>
<a name="137" href="#137">137</a> <em>     * </em>
<a name="138" href="#138">138</a> <em>     * @throws URIException</em>
<a name="139" href="#139">139</a> <em>     */</em>
<a name="140" href="#140">140</a>     <strong>public</strong> <strong>void</strong> testFormsLink() throws URIException {
<a name="141" href="#141">141</a>         <a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi =
<a name="142" href="#142">142</a>             <strong>new</strong> <a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a>(UURIFactory.getInstance(<span class="string">"http://www.example.org"</span>));
<a name="143" href="#143">143</a>         CharSequence cs = 
<a name="144" href="#144">144</a>         	<span class="string">"&lt;form name=\&quot;testform\&quot; method=\&quot;POST\&quot; action=\&quot;redirect_me?form=true\&quot;> "</span> +
<a name="145" href="#145">145</a>         	<span class="string">"  &lt;INPUT TYPE=CHECKBOX NAME=\&quot;checked[]\&quot; VALUE=\&quot;1\&quot; CHECKED> "</span>+
<a name="146" href="#146">146</a>         	<span class="string">"  &lt;INPUT TYPE=CHECKBOX NAME=\&quot;unchecked[]\&quot; VALUE=\&quot;1\&quot;> "</span> +
<a name="147" href="#147">147</a>         	<span class="string">"  &lt;select name=\&quot;selectBox\&quot;>"</span> +
<a name="148" href="#148">148</a>         	<span class="string">"    &lt;option value=\&quot;selectedOption\&quot; selected>option1&lt;/option>"</span> +
<a name="149" href="#149">149</a>         	<span class="string">"    &lt;option value=\&quot;nonselectedOption\&quot;>option2&lt;/option>"</span> +
<a name="150" href="#150">150</a>         	<span class="string">"  &lt;/select>"</span> +
<a name="151" href="#151">151</a>         	<span class="string">"  &lt;input type=\&quot;submit\&quot; name=\&quot;test\&quot; value=\&quot;Go\&quot;>"</span> +
<a name="152" href="#152">152</a>         	<span class="string">"&lt;/form>"</span>;   
<a name="153" href="#153">153</a>         <strong>this</strong>.extractor.extract(curi,cs);
<a name="154" href="#154">154</a>         curi.getOutLinks();
<a name="155" href="#155">155</a>         assertTrue(CollectionUtils.exists(curi.getOutLinks(), <strong>new</strong> Predicate() {
<a name="156" href="#156">156</a>             <strong>public</strong> <strong>boolean</strong> evaluate(Object object) {
<a name="157" href="#157">157</a>                 <strong>return</strong> ((Link) object).getDestination().toString().indexOf(
<a name="158" href="#158">158</a>                         <span class="string">"/redirect_me?form=true&amp;checked[]=1&amp;unchecked[]=&amp;selectBox=selectedOption&amp;test=Go"</span>)>=0;
<a name="159" href="#159">159</a>             }
<a name="160" href="#160">160</a>         }));
<a name="161" href="#161">161</a>     }
<a name="162" href="#162">162</a> }
</pre>
<hr/><div id="footer">This page was automatically generated by <a href="http://maven.apache.org/">Maven</a></div></body>
</html>