frontierschedulerfor163mobile.java

来自「一个搜索引擎,希望对大家有用」· Java 代码 · 共 41 行

package my.postprocessor;

import java.util.logging.Logger;

import org.archive.crawler.datamodel.CandidateURI;
import org.archive.crawler.postprocessor.FrontierScheduler;

public class FrontierSchedulerFor163Mobile extends FrontierScheduler {

	private static Logger LOGGER = Logger
			.getLogger(FrontierSchedulerFor163Mobile.class.getName());

	public FrontierSchedulerFor163Mobile(String name) {
		super(name);
	}

	protected void schedule(CandidateURI caUri) {
		String url = caUri.toString();

		try {
			if (url.indexOf("mobile.163.com/0011/product/0011000B/product") != -1
					|| url.indexOf("mobile.163.com/0011/product/0011000B/mark") != -1
					|| url.endsWith(".gif")
					|| url.endsWith(".jpg")
					|| url.endsWith(".jpeg")
					|| url.indexOf("robots.txt") != -1
					|| url.indexOf("dns:") != -1) {

				if (url.indexOf("#") == -1) {
					getController().getFrontier().schedule(caUri);
				}
			} else {
				return;
			}
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
		}
	}
}