logurl.java

真正的网络爬虫的源代码啊,希望大家好好阅读,写出心得体会啊

package net.matuschek.examples;

import java.io.FileWriter;
import java.net.URL;
import net.matuschek.http.URLLogger;
import net.matuschek.spider.WebRobot;
/*********************************************
    Copyright (c) 2001 by Daniel Matuschek
*********************************************/



/**
 * This example program downloads a web page. It does not
 * store the documents but only logs the visited URLs.
 *
 * @author Daniel Matuschek 
 * @version $Revision: 1.2 $
 */
public class LogURL {

  public static void main(String[] args) 
    throws Exception
  {
    System.out.println("URLs will be logged to urls.txt\n\n");

    WebRobot robby = new WebRobot();
    robby.setStartURL(new URL("http://www.matuschek.net"));
    robby.setMaxDepth(1);
    robby.setSleepTime(0);

    FileWriter logfile = new FileWriter("urls.txt");
    URLLogger log = new URLLogger(logfile);
    robby.setDocManager(log);

    robby.run();
    logfile.close();
  }
}