mannergather.java

一个主题相关的网络爬虫,实现与某一主题相关的网页的爬取

package com.manner;

import java.net.*;
import java.util.*;
import java.io.*;

//礼貌采集类
public class MannerGather {
	// 判断urlToCheck对应的网页是否允许采集
	public boolean isRobotAllowed(URL urlToCheck) {
		HashMap<String, ArrayList<String>> disallowListCache = new HashMap<String, ArrayList<String>>();

		String host = urlToCheck.getHost().toLowerCase();// 获取给出RUL的主机

		// 获取主机不允许搜索的URL缓存
		ArrayList<String> disallowList = disallowListCache.get(host);

		// 如果还没有缓存,下载并缓存。
		if (disallowList == null) {
			disallowList = new ArrayList<String>();
			try {
				URL robotsFileUrl = new URL("http://" + host + "/robots.txt");
				BufferedReader reader = new BufferedReader(
						new InputStreamReader(robotsFileUrl.openStream()));

				// 读robot文件，创建不允许访问的路径列表。
				String line;
				while ((line = reader.readLine()) != null) {
					if (line.indexOf("Disallow:") == 0) {// 是否包含"Disallow:"
						String disallowPath = line.substring("Disallow:"
								.length());// 获取不允许访问路径

						// 检查是否有注释。
						int commentIndex = disallowPath.indexOf("#");
						if (commentIndex != -1) {
							disallowPath = disallowPath.substring(0,
									commentIndex);// 去掉注释
						}

						disallowPath = disallowPath.trim();
						disallowList.add(disallowPath);
					}
				}

				// 缓存此主机不允许访问的路径。
				disallowListCache.put(host, disallowList);
			} catch (Exception e) {
				return true; // web站点根目录下没有robots.txt文件,返回真
			}
		}

		String file = urlToCheck.getFile();
		for (int i = 0; i < disallowList.size(); i++) {
			String disallow = disallowList.get(i);
			if (file.startsWith(disallow)) {
				return false;
			}
		}

		return true;
	}

}