websearch.java

网页搜索小程序,包含网页爬虫

package web;
import java.io.BufferedReader;
import java.io.InputStreamReader;//
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
//Access 
public class Access implements Runnable {
//错误信息列表保存在Err_list中,搜索到的结果保存在Res_list中
 ArrayList<String> Err_list = new ArrayList<String>(),Res_list = new ArrayList<String>();
 String fiestUrl;// 开始搜索的起点
 int pageNum;// 最大处理的页面数
 String searchString;// 要搜索的字符串(限于英文)
 boolean caseSensitive = false,limitHost =false;// 是否区分大小写,是否在限制的主机内搜索
//outofcatch 用于识别网站根目录下规定不能读取的文件
 private HashMap<String, ArrayList<String>> outofcatch = new HashMap<String, ArrayList<String>>();
//类构造器  初始化工作
 public Access(String firstUrl, int pageNum, String searchString) {
  this.fiestUrl = firstUrl;
  this.pageNum = pageNum;
  this.searchString = searchString;
 }

 public ArrayList<String> getRes_list() {
  return Res_list;
 }

 public void run() {// 启动搜索线程
  crawl(fiestUrl, pageNum, searchString, limitHost, caseSensitive);
 }

 // 检测URL格式
 private URL verifyUrl(String url) {
  //此处 只处理HTTP格式的URLs.
  if (!url.toLowerCase().startsWith("http://"))
   return null;
  URL verifiedUrl = null;
  try {
   verifiedUrl = new URL(url);
  } catch (Exception e) {
   return null;
  }
  return verifiedUrl;
 }

// 检测是否有不允许访问给出的URL.
 private boolean isRobotAllowed(URL urlToCheck) {
  String host = urlToCheck.getHost().toLowerCase();// 获取给出RUL的主机
// 获取主机不允许搜索的URL缓存
  ArrayList<String> outofca_list = outofcatch.get(host);
  if (outofca_list == null) {
   outofca_list = new ArrayList<String>();
   try {
    URL robotsFileUrl = new URL("http://" + host + "/robots.txt");
    BufferedReader reader = new BufferedReader(
      new InputStreamReader(robotsFileUrl.openStream()));
    String line;
    while ((line = reader.readLine()) != null) {
     if (line.indexOf("Disallow:") == 0) {// 是否包含"Disallow:"
      String disallowPath = line.substring("Disallow:".length());// 获取不允许访问路径
      //注释检查。
      int commentok = disallowPath.indexOf("#");
      if (commentok != -1) {
       disallowPath = disallowPath.substring(0,
         commentok); 
      }
      disallowPath = disallowPath.trim();
      outofca_list.add(disallowPath);
     }
    }
    // 保存此主机不允许访问的路径。
    outofcatch.put(host, outofca_list);
   } catch (Exception e) {
    return true; 
   }
  }
  String file = urlToCheck.getFile();
  for (int i = 0; i < outofca_list.size(); i++) {
   String disallow = outofca_list.get(i);
   if (file.startsWith(disallow)) {
    return false;
   }
  }
  return true;
 }

 private String downloadPage(URL pageUrl) {
  try {
   // 创建打开读入流对象
   BufferedReader reader = new BufferedReader(new InputStreamReader(pageUrl.openStream()));
   String line;
   StringBuffer urlBuffer = new StringBuffer();
   //文件读入缓存
   while ((line = reader.readLine()) != null) {
    urlBuffer.append(line);
   }
   return urlBuffer.toString();
  } catch (Exception e) {
  }

  return null;
 }
 
 private String durceWwww(String url) {
  int index = url.indexOf("://www.");
  if (index != -1) {
   return url.substring(0, index + 3) + url.substring(index + 7);
  }
  return (url);
 }

 // 开始解析页面 
 private ArrayList<String> retrieveLinks(URL pageUrl, String pageContents,
   HashSet crawledList, boolean limitHost) {
  Pattern model = Pattern.compile("<a\\s+href\\s*=\\s*\"?(.*?)[\"|>]",
    Pattern.CASE_INSENSITIVE);
  Matcher m = model.matcher(pageContents);
  ArrayList<String> linkList = new ArrayList<String>();
  while (m.find()) {
   String link = m.group(1).trim();
   //忽略到此页的连接和JavaScript;
   if (link.length() < 1)  continue;
   if (link.charAt(0) == '#') continue;
   if (link.indexOf("mailto:") != -1)continue;
   if (link.toLowerCase().indexOf("javascript") != -1) continue;
   if (link.indexOf("://") == -1) {
   if (link.charAt(0) == '/') //绝对地址
     link = "http://" + pageUrl.getHost() + ":"+ pageUrl.getPort() + link;
   else {
     String file = pageUrl.getFile();
     if (file.indexOf('/') == -1) {//相对地址
      link = "http://" + pageUrl.getHost() + ":"
        + pageUrl.getPort() + "/" + link;
     } else {
      String path = file.substring(0,
        file.lastIndexOf('/') + 1);
      link = "http://" + pageUrl.getHost() + ":"
        + pageUrl.getPort() + path + link;
     }
    }
   }
   int index = link.indexOf('#');
   if (index != -1) 
       link = link.substring(0, index);
   link = durceWwww(link);
   URL verifiedLink = verifyUrl(link);
   if (verifiedLink == null) {
    continue;
   }

   /*  限定主机? */
   if (limitHost && !pageUrl.getHost().toLowerCase().equals(verifiedLink.getHost().toLowerCase())) 
    continue;
   if (crawledList.contains(link)) {
    continue;
   }
   linkList.add(link);
  }
  return (linkList);
 }
 // 搜索下载Web页面的内容，判断 有无匹配 的字符串
 private boolean searchStringMatches(String pageContents,
   String searchString, boolean caseSensitive) {
  String searchContents = pageContents;
  if (!caseSensitive) {//不区分大小写
   searchContents = pageContents.toLowerCase();
  }
  Pattern p = Pattern.compile("[\\s]+");
  String[] terms = p.split(searchString);
  for (int i = 0; i < terms.length; i++) {
   if (caseSensitive) {
    if (searchContents.indexOf(terms[i]) == -1) {
     return false;
    }
   } else {
    if (searchContents.indexOf(terms[i].toLowerCase()) == -1) {
     return false;
    }
   }
  }

  return true;
 }

 //开始真正搜索操作
 public ArrayList<String> crawl(String startUrl, int pageNums,
   String searchString, boolean limithost, boolean caseSensitive) {

  HashSet<String> crawledList = new HashSet<String>();
  LinkedHashSet<String> toCrawlList = new LinkedHashSet<String>();

  if (pageNums < 1) {
   Err_list.add("要搜索的url数至少为 1.");
   System.out.println("要搜索的url数至少为 1.");
  }
  if (searchString.length() < 1) {
   Err_list.add("要收索的关键字长度至少为  1.");
   System.out.println("关键字长度出错。");
  }

  if (Err_list.size() > 0) {
   System.out.println("出现错误。");
   return Err_list;
  }
  //去除url中www标志
  startUrl = durceWwww(startUrl);
  toCrawlList.add(startUrl);
  while (toCrawlList.size() > 0) {
   if (pageNums != -1) {
    if (crawledList.size() == pageNums) {
     break;
    }
   }
   // 获取url.
   String url = toCrawlList.iterator().next();
   // 去除url
   toCrawlList.remove(url);
   URL verifiedUrl = verifyUrl(url);
   if (!isRobotAllowed(verifiedUrl)) {
    continue;
   }
   // 增加已处理的URL到crawledList
   crawledList.add(url);
   String pageContents = downloadPage(verifiedUrl);
   if (pageContents != null && pageContents.length() > 0) {
    ArrayList<String> links = retrieveLinks(verifiedUrl,
      pageContents, crawledList, limitHost);
    toCrawlList.addAll(links);
    if (searchStringMatches(pageContents, searchString,
      caseSensitive)) {
     Res_list.add(url);
     System.out.println(url);
    }
   }
  }
  return Res_list;
 }
 // 这里开始 
 public static void main(String[] args) {
  Access crawler = new Access("http://www.sina.com.cn/",10,"news");
  
  Thread search = new Thread(crawler);
  //System.out.println("请务必保持连接到Internet网\n");
  System.out.println("在 \"http://www.sina.com.cn/\" 网站上搜索含关键字：\"news\"的网页\n" +
  		"返回url数目：10\n" + "(url，返回数目，关键字，都可自行修改)\n"+"开始搜索...\n");
  System.out.println("结果url列表:");
  search.start();
  try {
   search.join();
  } catch (InterruptedException e) {
   e.printStackTrace();
  }
 }
}