lowdiskpauseprocessor.html

用JAVA编写的,在做实验的时候留下来的,本来想删的,但是传上来,大家分享吧

<a name="82" href="#82">82</a>  <em>     */</em>
<a name="83" href="#83">83</a>      <strong>public</strong> <a href="../../../../org/archive/crawler/postprocessor/LowDiskPauseProcessor.html">LowDiskPauseProcessor</a>(String name) {
<a name="84" href="#84">84</a>          <strong>super</strong>(name, <span class="string">"LowDiskPause processor"</span>);
<a name="85" href="#85">85</a>          <a href="../../../../org/archive/crawler/settings/Type.html">Type</a> e = addElementToDefinition(
<a name="86" href="#86">86</a>              <strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_MONITOR_MOUNTS, 
<a name="87" href="#87">87</a>                      <span class="string">"Space-delimited list of filessystem mounts whose "</span> +
<a name="88" href="#88">88</a>                      <span class="string">"'available' space should be monitored via 'df' "</span> +
<a name="89" href="#89">89</a>                      <span class="string">"(if available)."</span>,
<a name="90" href="#90">90</a>                  DEFAULT_MONITOR_MOUNTS));
<a name="91" href="#91">91</a>          e.setOverrideable(false);
<a name="92" href="#92">92</a>          e = addElementToDefinition(
<a name="93" href="#93">93</a>              <strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_PAUSE_THRESHOLD, 
<a name="94" href="#94">94</a>                      <span class="string">"When available space on any monitored mounts falls "</span> +
<a name="95" href="#95">95</a>                      <span class="string">"below this threshold, the crawl will be paused. "</span>,
<a name="96" href="#96">96</a>                      <strong>new</strong> Integer(DEFAULT_PAUSE_THRESHOLD)));
<a name="97" href="#97">97</a>          e = addElementToDefinition(
<a name="98" href="#98">98</a>              <strong>new</strong> <a href="../../../../org/archive/crawler/settings/SimpleType.html">SimpleType</a>(ATTR_RECHECK_THRESHOLD, 
<a name="99" href="#99">99</a>                      <span class="string">"Available space via 'df' is rechecked after every "</span> +
<a name="100" href="#100">100</a>                     <span class="string">"increment of this much content (uncompressed) is "</span> +
<a name="101" href="#101">101</a>                     <span class="string">"observed. "</span>,
<a name="102" href="#102">102</a>                     <strong>new</strong> Integer(DEFAULT_RECHECK_THRESHOLD)));
<a name="103" href="#103">103</a>         e.setOverrideable(false);
<a name="104" href="#104">104</a>     } 
<a name="105" href="#105">105</a>     
<a name="106" href="#106">106</a>     <em>/**<em>*</em></em>
<a name="107" href="#107">107</a> <em>     * Notes a CrawlURI's content size in its running tally. If the </em>
<a name="108" href="#108">108</a> <em>     * recheck increment of content has passed through since the last</em>
<a name="109" href="#109">109</a> <em>     * available-space check, checks available space and pauses the </em>
<a name="110" href="#110">110</a> <em>     * crawl if any monitored mounts are below the configured threshold. </em>
<a name="111" href="#111">111</a> <em>     * </em>
<a name="112" href="#112">112</a> <em>     * @param curi CrawlURI to process.</em>
<a name="113" href="#113">113</a> <em>     */</em>
<a name="114" href="#114">114</a>     <strong>protected</strong> <strong>void</strong> innerProcess(<a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi) {
<a name="115" href="#115">115</a>         contentSinceCheck += curi.getContentSize();
<a name="116" href="#116">116</a>         <strong>synchronized</strong> (<strong>this</strong>) {
<a name="117" href="#117">117</a>             <strong>if</strong> (contentSinceCheck/1024 > ((Integer) getUncheckedAttribute(<strong>null</strong>,
<a name="118" href="#118">118</a>                     ATTR_RECHECK_THRESHOLD)).intValue()) {
<a name="119" href="#119">119</a>                 checkAvailableSpace(curi);
<a name="120" href="#120">120</a>                 contentSinceCheck = 0;
<a name="121" href="#121">121</a>             }
<a name="122" href="#122">122</a>         }
<a name="123" href="#123">123</a>     }
<a name="124" href="#124">124</a> 
<a name="125" href="#125">125</a> 
<a name="126" href="#126">126</a>     <em>/**<em>*</em></em>
<a name="127" href="#127">127</a> <em>     * Probe via 'df' to see if monitored mounts have fallen</em>
<a name="128" href="#128">128</a> <em>     * below the pause available threshold. If so, request a </em>
<a name="129" href="#129">129</a> <em>     * crawl pause. </em>
<a name="130" href="#130">130</a> <em>     * @param curi Current context.</em>
<a name="131" href="#131">131</a> <em>     */</em>
<a name="132" href="#132">132</a>     <strong>private</strong> <strong>void</strong> checkAvailableSpace(<a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi) {
<a name="133" href="#133">133</a>         <strong>try</strong> {
<a name="134" href="#134">134</a>             String df = IoUtils.readFullyAsString(Runtime.getRuntime().exec(
<a name="135" href="#135">135</a>                     <span class="string">"df -k"</span>).getInputStream());
<a name="136" href="#136">136</a>             Matcher matcher = VALID_DF_OUTPUT.matcher(df);
<a name="137" href="#137">137</a>             <strong>if</strong>(!matcher.matches()) {
<a name="138" href="#138">138</a>                 logger.severe(<span class="string">"'df -k' output unacceptable for low-disk checking"</span>);
<a name="139" href="#139">139</a>                 <strong>return</strong>;
<a name="140" href="#140">140</a>             }
<a name="141" href="#141">141</a>             List monitoredMounts = Arrays.asList(((String) getUncheckedAttribute(<strong>null</strong>,
<a name="142" href="#142">142</a>                     ATTR_MONITOR_MOUNTS)).split(<span class="string">"&#47;&#47;s*"</span>));
<a name="143" href="#143">143</a>             matcher = AVAILABLE_EXTRACTOR.matcher(df);
<a name="144" href="#144">144</a>             <strong>while</strong> (matcher.find()) {
<a name="145" href="#145">145</a>                 String mount = matcher.group(2);
<a name="146" href="#146">146</a>                 <strong>if</strong> (monitoredMounts.contains(mount)) {
<a name="147" href="#147">147</a>                     <strong>long</strong> availKilobytes = Long.parseLong(matcher.group(1));
<a name="148" href="#148">148</a>                     <strong>int</strong> thresholdKilobytes = ((Integer) getUncheckedAttribute(
<a name="149" href="#149">149</a>                             <strong>null</strong>, ATTR_PAUSE_THRESHOLD)).intValue();
<a name="150" href="#150">150</a>                     <strong>if</strong> (availKilobytes &lt; thresholdKilobytes ) {
<a name="151" href="#151">151</a>                         getController().requestCrawlPause();
<a name="152" href="#152">152</a>                         logger.log(Level.SEVERE, <span class="string">"Low Disk Pause"</span>,
<a name="153" href="#153">153</a>                                 availKilobytes + <span class="string">"K available on "</span> + mount
<a name="154" href="#154">154</a>                                         + <span class="string">" (below threshold "</span>
<a name="155" href="#155">155</a>                                         + thresholdKilobytes + <span class="string">"K)"</span>);
<a name="156" href="#156">156</a>                         <strong>break</strong>;
<a name="157" href="#157">157</a>                     }
<a name="158" href="#158">158</a>                 }
<a name="159" href="#159">159</a>             }
<a name="160" href="#160">160</a>         } <strong>catch</strong> (IOException e) {
<a name="161" href="#161">161</a>             curi.addLocalizedError(<strong>this</strong>.getName(), e,
<a name="162" href="#162">162</a>                     <span class="string">"problem checking available space via 'df'"</span>);
<a name="163" href="#163">163</a>         }
<a name="164" href="#164">164</a>     }
<a name="165" href="#165">165</a> }
</pre>
<hr/><div id="footer">This page was automatically generated by <a href="http://maven.apache.org/">Maven</a></div></body>
</html>