eventlog.java

一个用java语言编写的网络爬虫程序

/*
 * WebSPHINX web crawling toolkit
 * Copyright (C) 1998,1999 Carnegie Mellon University 
 * 
 * This library is free software; you can redistribute it
 * and/or modify it under the terms of the GNU Library
 * General Public License as published by the Free Software 
 * Foundation, version 2.
 *
 * WebSPHINX homepage: http://www.cs.cmu.edu/~rcm/websphinx/
 */
package websphinx;

import java.io.File;
import java.io.OutputStream;
import java.io.IOException;
import java.util.Date;
//#ifdef JDK1.1
import java.io.PrintWriter;
//#endif JDK1.1
/*#ifdef JDK1.0
import java.io.PrintStream;
#endif JDK1.0*/

/**
 * Crawling monitor that writes messages to standard output or a file.
 * Acts as both a CrawlListener (monitoring start and end of the crawl)
 * and as a LinkListener (monitoring page retrieval).
 */
public class EventLog implements CrawlListener, LinkListener {

//#ifdef JDK1.1
    PrintWriter stream;
//#endif JDK1.1
/*#ifdef JDK1.0
    PrintStream stream;
#endif JDK1.0*/
    boolean onlyNetworkEvents = true;

    /**
     * Make a EventLog that writes to standard output.
     */
    public EventLog () {
        this (System.out);
    }

    /**
     * Make a EventLog that writes to a stream.
     */
    public EventLog (OutputStream out) {
/*#ifdef JDK1.0
        stream = new PrintStream (out, true);
#endif JDK1.0*/
//#ifdef JDK1.1
        stream = new PrintWriter (out, true);
//#endif JDK1.1
    }

    /**
     * Make a EventLog that writes to a file.  The file is overwritten.
     * @param filename File to which crawling event messages are written
     */
    public EventLog (String filename) throws IOException {
/*#ifdef JDK1.0
        stream = new PrintStream (SecurityPolicy.getPolicy ().writeFile (new File(filename), false));
#endif JDK1.0*/
//#ifdef JDK1.1
        stream = new PrintWriter (SecurityPolicy.getPolicy ().writeFile (new File(filename), false));
//#endif JDK1.1
    }

    /**
     * Set whether logger prints only network-related LinkEvents.
     * If true, then the logger only prints LinkEvents where
     * LinkEvent.isNetworkEvent() returns true.  If false,
     * then the logger prints all LinkEvents.  Default is true.
     * @param flag true iff only network LinkEvents should be logged
     */
    public void setOnlyNetworkEvents (boolean flag) {
        onlyNetworkEvents = flag;
    }
    /**
     * Test whether logger prints only network-related LinkEvents.
     * If true, then the logger only prints LinkEvents where
     * LinkEvent.isNetworkEvent() returns true.  If false,
     * then the logger prints all LinkEvents.  Default is true.
     * @return true iff only network LinkEvents are logged
     */
    public boolean getOnlyNetworkEvents () {
        return onlyNetworkEvents;
    }

    /**
     * Notify that the crawler started.
     */
    public void started (CrawlEvent event) {
        stream.println (new Date() + ": *** started " + event.getCrawler());
    }

    /**
     * Notify that the crawler has stopped.
     */
    public void stopped (CrawlEvent event) {
        stream.println (new Date() + ": *** finished " + event.getCrawler());
    }

    /**
     * Notify that the crawler's state was cleared.
     */
    public void cleared (CrawlEvent event) {
        stream.println (new Date() + ": *** cleared " + event.getCrawler());
    }

    /**
     * Notify that the crawler timed out.
     */
    public void timedOut (CrawlEvent event) {
        stream.println (new Date() + ": *** timed out " + event.getCrawler());
    }

    /**
     * Notify that the crawler paused.
     */
    public void paused (CrawlEvent event) {
        stream.println (new Date() + ": *** paused " + event.getCrawler());
    }

    /**
     * Notify that a link event occured.
     */
    public void crawled (LinkEvent event) {
        switch (event.getID()) {
          case LinkEvent.RETRIEVING:
          case LinkEvent.DOWNLOADED:
          case LinkEvent.VISITED:
          case LinkEvent.ERROR:
            break;
          default:
            if (onlyNetworkEvents)
                return;
            break;
        }
        stream.println (new Date () + ": "
                        + event);

        Throwable exc = event.getException();
        if (exc != null && ! (exc instanceof IOException))
            exc.printStackTrace (stream);
    }

    /**
     * Create a EventLog that prints to standard error and attach it to a crawler.
     * This is a convenience method.
     * @param crawler Crawler to be monitored
     */
    public static void monitor (Crawler crawler) {
        EventLog logger = new EventLog (System.err);
        crawler.addCrawlListener (logger);
        crawler.addLinkListener (logger);
    }
}