stopdownload.java

真正的网络爬虫的源代码啊,希望大家好好阅读,写出心得体会啊

package net.matuschek.examples;

/*********************************************
    Copyright (c) 2001 by Daniel Matuschek
*********************************************/

import java.net.URL;

import net.matuschek.spider.WebRobot;
import net.matuschek.spider.WebRobotCallback;

import org.apache.log4j.BasicConfigurator;

/**
 * This example program shows how it is possible
 * to use the WebRobotCallback to stop after a given number
 * of documents has been retrieved.
 *
 * @author Daniel Matuschek 
 * @version $Revision: 1.3 $
 */
public class StopDownload {

  class DownloadStopper implements WebRobotCallback {
    
    /** maximal number of documents to retrieve */
    int max = 0;

    /** current number of retrieved documents */
    int count = 0;
    
    /** WebRobot to control */
    WebRobot robot = null;


    public DownloadStopper(int max, WebRobot robot) {
      this.max = max;
      this.robot = robot;
    }
    
    /**
     * Increases the number of retrieved documents and stops
     * the robot, if the number has reached the maximum
     */
    public void webRobotRetrievedDoc(String url, int size) {
      count ++;
      if (count >= max) {
	robot.stopRobot();
      }
    }

    // ignore these methods
    public void webRobotDone() {};
    public void webRobotSleeping(boolean sleeping) {}
    public void webRobotUpdateQueueStatus(int length) {}
  }



  public StopDownload() {
  }



  public void run() throws Exception {
    WebRobot robby = new WebRobot();
    robby.setStartURL(new URL("http://www.matuschek.net"));
    robby.setMaxDepth(1);
    robby.setSleepTime(0);

    // download only the first 5 documents
    DownloadStopper stopit = new DownloadStopper(5,robby);
    robby.setWebRobotCallback(stopit);

    robby.run();
  }


  public static void main(String[] args) 
    throws Exception
  {
    BasicConfigurator.configure();
    StopDownload stopper = new StopDownload();
    stopper.run();
  }
}