preprocessutility.cs

只是中科院分词系统的SharpICTCLAS分词系统

      }
      return sb.ToString();
   }

   //===================================================================
   // 预处理，去掉不必要的HTML信息，并完成繁体、简体转换
   //===================================================================
   public static string HTML2Text(string content)
   {
      //去掉多余空白符号
      content = Regex.Replace(content, @"\s+", " ");

      //将换行符更换成回车换行标记
      content = Regex.Replace(content, @"<br>", "\n\r", RegexOptions.IgnoreCase);
      content = Regex.Replace(content, @"</br>", "\n\r", RegexOptions.IgnoreCase);
      content = Regex.Replace(content, @"<p>", "\n\r", RegexOptions.IgnoreCase);
      content = Regex.Replace(content, @"</p>", "\n\r", RegexOptions.IgnoreCase);
      content = Regex.Replace(content, @"<p( )*/>", "\n\r", RegexOptions.IgnoreCase);
      content = Regex.Replace(content, @"<br( )*/>", "\n\r", RegexOptions.IgnoreCase);

      //去掉所有HTML标记
      content = Regex.Replace(content, @"<[^>]+?>", " ");

      //将所有代换符还原
      content = Regex.Replace(content, @"&nbsp;", " ");
      content = Regex.Replace(content, @"&gt;", ">");
      content = Regex.Replace(content, @"&lt;", "<");
      content = Regex.Replace(content, @"&amp;", "&");
      content = Regex.Replace(content, @"&quot;", "\"");

      //去掉空行
      content = Regex.Replace(content, @"\n[\s| ]*\r", "\n\r");

      //将多个连续空格替换成一个空格
      content = Regex.Replace(content, @"( ){2,}", " ");

      //去掉首尾空格
      content = Regex.Replace(content, @"(^\s*)|(\s*$)", "");

      //去掉行首空格
      content = Regex.Replace(content, @"\n\r( )*", "\n\r");

      //去掉行尾空格
      content = Regex.Replace(content, @"( )*\n\r", "\n\r");

      return content;
   }
}