workbench.java

一个Web爬虫（机器人

/*
 * WebSPHINX web crawling toolkit
 * Copyright (C) 1998,1999 Carnegie Mellon University 
 * 
 * This library is free software; you can redistribute it
 * and/or modify it under the terms of the GNU Library
 * General Public License as published by the Free Software 
 * Foundation, version 2.
 *
 * WebSPHINX homepage: http://www.cs.cmu.edu/~rcm/websphinx/
 */
package websphinx.workbench;

import java.awt.*;
import java.io.*;
import java.net.URL;
import websphinx.*;
import websphinx.util.Constrain;
import websphinx.util.PopupDialog;
import websphinx.util.TabPanel;
import websphinx.util.BorderPanel;
import websphinx.util.ClosableFrame;

public class Workbench extends Panel implements CrawlListener {

    Crawler crawler;

    String currentFilename = "";

    // panel wrappers
    Panel workbenchPanel; // contains menuPanel, configPanel, buttonPanel
    GridBagConstraints workbenchConstraints;
    WorkbenchVizPanel vizPanel; // contains graph, outline, and statistics
    GridBagConstraints vizConstraints;

    // GUI event listeners
    WebGraph graph;
    WebOutline outline;
    Statistics statistics;
    EventLog logger;

    // menu bar (for frame mode)
    MenuBar menubar;
    Menu fileMenu;
    MenuItem newCrawlerItem;
    MenuItem openCrawlerItem;
    MenuItem saveCrawlerItem;
    MenuItem createCrawlerItem;
    MenuItem exitItem;

    // menu panel (for container mode)
    Panel menuPanel;
    Button newCrawlerButton;
    Button openCrawlerButton;
    Button saveCrawlerButton;
    Button createCrawlerButton;

    WorkbenchTabPanel configPanel;
    Panel simplePanel;
    Panel crawlPanel;
    Panel limitsPanel;
    Panel classifiersPanel;
    Panel linksPanel;
    Panel actionPanel;

    CrawlerEditor crawlerEditor;
    ClassifierListEditor classifierListEditor;
    DownloadParametersEditor downloadParametersEditor;
    LinkPredicateEditor linkPredicateEditor;
    PagePredicateEditor pagePredicateEditor;
    ActionEditor actionEditor;
    SimpleCrawlerEditor simpleCrawlerEditor;

    boolean advancedMode = false;
    boolean tornOff = false;
    
    Button startButton, pauseButton, stopButton, clearButton;
    
    boolean allowExit;

    // Frames
    Frame workbenchFrame;
    Frame vizFrame;
    
    static final int MARGIN = 8;  // pixel border around configPanel
    
    public Workbench () {
        this (makeDefaultCrawler ());
        return;
    }
    
    private static Crawler makeDefaultCrawler () {
        Crawler c = new Crawler ();
        c.setDomain (Crawler.SUBTREE);
        return c;
    }
    
    public Workbench (String filename) throws Exception {
        //#ifdef JDK1.1
        this (loadCrawler (new FileInputStream (filename)));
        //#endif JDK1.1
        /*#ifdef JDK1.0
        throw new RuntimeException ("Crawler load/save not supported under Java 1.0");
        #endif JDK1.0*/
    }

    public Workbench (URL url) throws Exception {
        //#ifdef JDK1.1
        this (loadCrawler (url.openStream ())); // FIX: Netscape 4 refuses to load off local disk
        //#endif JDK1.1
        /*#ifdef JDK1.0
        throw new RuntimeException ("Crawler load/save not supported under Java 1.0");
        #endif JDK1.0*/
    }
   
    public Workbench (Crawler _crawler) {
        Browser browser = Context.getBrowser ();

        setLayout (new BorderLayout ());
        setBackground (Color.lightGray);
        
        setLayout (new GridLayout (2, 1));
        
        add (workbenchPanel = new Panel ());
        workbenchPanel.setLayout (new GridBagLayout ());
        
        // menu buttons panel
        makeMenus ();
        Constrain.add (workbenchPanel, menuPanel, Constrain.labelLike (0, 0));
        
        // configuration panel                
        configPanel = new WorkbenchTabPanel ();
        Constrain.add (workbenchPanel, configPanel, Constrain.areaLike (0, 1));
        simplePanel = makeSimplePanel();
        crawlPanel = makeCrawlPanel();
        linksPanel = makeLinksPanel();
        actionPanel = makeActionPanel();
        classifiersPanel = makeClassifiersPanel ();
        limitsPanel = makeLimitsPanel ();

        // start/pause/stop button panel
        Constrain.add (workbenchPanel, makeButtonPanel(), Constrain.fieldLike (0, 2));

        // visualization panel
        add (vizPanel = new WorkbenchVizPanel (this));

        // graph visualization
        graph = new WebGraph ();
        graph.setBackground (Color.white);
        if (browser != null)
            graph.addLinkViewListener (browser);
        vizPanel.addTabPanel ("Graph", true, graph);

        // outline visualization
        outline = new WebOutline ();
        outline.setBackground (Color.white);
        if (browser != null)
            outline.addLinkViewListener (browser);
        vizPanel.addTabPanel ("Outline", true, outline);

        // statistics visualization
        statistics = new Statistics ();
        Panel p = new Panel ();
        p.setLayout (new FlowLayout());
        p.add (statistics);
        vizPanel.addTabPanel ("Statistics", true, p);

        // event logger (sends to System.err -- no GUI presence)
        logger = new EventLog ();

        // now that the GUI is set up, we can initialize it with the
        // crawler
        setCrawler (_crawler);        
    }

    public Frame makeFrame () {
        if (workbenchFrame == null) {
            Dimension screen = Toolkit.getDefaultToolkit().getScreenSize();
            workbenchFrame = new WorkbenchFrame (this);
            workbenchFrame.setForeground (getForeground());
            workbenchFrame.setBackground (getBackground());
            workbenchFrame.setFont (getFont());
            workbenchFrame.setTitle ("Crawler Workbench: " + 
                                     (crawler != null ? crawler.getName() : ""));
            workbenchFrame.setLayout (new GridLayout (1,1));
            workbenchFrame.add (this);
            workbenchPanel.remove (menuPanel);
            workbenchFrame.setMenuBar (menubar);
            workbenchFrame.reshape (0, 0, 
                                    Math.min (550, screen.width),
                                    screen.height - 50);
        }
        return workbenchFrame;
    }

    public void setAllowExit (boolean yes) {
        allowExit = yes;
    }
    
    public boolean getAllowExit () {
        return allowExit;
    }
    
    public synchronized void setAdvancedMode (boolean adv) {
        if (advancedMode == adv)
            return;

        configureCrawler ();  // write current mode's settings back to crawler
        advancedMode = adv;
        setCrawler (crawler); // read new mode's settings from crawler

        configPanel.advancedButton.setLabel (advancedMode 
                                             ? "<< Simple" : ">> Advanced");
        validate ();
    }
    
    public boolean getAdvancedMode () {
        return advancedMode;
    }

    /* * * * * * * * * * * * * * * * * * * * * * *
     *   GUI Construction
     * * * * * * * * * * * * * * * * * * * * * * */

    static void setVisible (Component comp, boolean visible) {
        if (visible)
            comp.show();
        else
            comp.hide ();
    }

    static void setEnabled (Component comp, boolean enabled) {
        if (enabled)
            comp.enable();
        else
            comp.disable();
    }

    static void setEnabled (MenuItem item, boolean enabled) {
        if (enabled)
            item.enable();
        else
            item.disable();
    }

    Panel makeMenus () {
        // menubar
        menubar = new MenuBar ();
        menuPanel = new Panel ();
        menuPanel.setLayout (new FlowLayout()); // for horizontal layout

        // FILE menu
        menubar.add (fileMenu = new Menu ("File"));
        
        // File/New Crawler
        fileMenu.add (newCrawlerItem = new MenuItem ("New Crawler"));
        menuPanel.add (newCrawlerButton = new Button ("New"));
               
        // File/Open Crawler
        fileMenu.add (openCrawlerItem = new MenuItem ("Open Crawler..."));
        /*#ifdef JDK1.0
        setEnabled (openCrawlerItem, false);
        #endif JDK1.0*/
        menuPanel.add (openCrawlerButton = new Button ("Open..."));
        /*#ifdef JDK1.0
        setEnabled (openCrawlerButton, false);
        #endif JDK1.0*/
        

        // File/Save Crawler
        fileMenu.add (saveCrawlerItem = new MenuItem ("Save Crawler..."));
        /*#ifdef JDK1.0
        setEnabled (saveCrawlerItem, false);
        #endif JDK1.0*/
        menuPanel.add (saveCrawlerButton = new Button ("Save..."));
        /*#ifdef JDK1.0
        setEnabled (saveCrawlerButton, false);
        #endif JDK1.0*/

        // File/Create Crawler
        fileMenu.add (createCrawlerItem = new MenuItem ("Create Crawler From Class..."));
        menuPanel.add (createCrawlerButton = new Button ("Create..."));
               

        // File/Exit
        fileMenu.add (exitItem = new MenuItem ("Exit"));
        
        return menuPanel;
    }
        
    private Panel makeSimplePanel () {
        return simpleCrawlerEditor = new SimpleCrawlerEditor ();
    }

    // FIX: add onlyHyperLinks, synchronous, ignoreVisitedLinks
    private Panel makeCrawlPanel () {
        return crawlerEditor = new CrawlerEditor ();
    }

    private Panel makeLinksPanel () {
        Panel panel = new Panel ();
        panel.setLayout (new GridBagLayout ());

        Constrain.add (panel, new Label("Follow:"), Constrain.labelLike (0, 0));
        Constrain.add (panel, linkPredicateEditor = new LinkPredicateEditor (),
             Constrain.areaLike (1, 0));

        return panel;
    }

    private Panel makeActionPanel () {
        Panel panel = new Panel ();
        panel.setLayout (new GridBagLayout ());

        Constrain.add (panel, new Label ("Action:"), Constrain.labelLike (0, 0));
        Constrain.add (panel, actionEditor = new ActionEditor (), Constrain.areaLike (1, 0));

        Constrain.add (panel, new Label("on pages:"), Constrain.labelLike (0, 1));
        Constrain.add (panel, pagePredicateEditor = new PagePredicateEditor (),
             Constrain.areaLike (1, 1));
        return panel;
    }
    
    private Panel makeClassifiersPanel () {
        classifierListEditor = new ClassifierListEditor ();
        return classifierListEditor;
    }

    private Panel makeLimitsPanel () {
        downloadParametersEditor = new DownloadParametersEditor ();
        return downloadParametersEditor;
    }

    private Panel makeButtonPanel () {
        Panel panel = new Panel ();
        panel.setLayout (new FlowLayout ());
        
        panel.add (startButton = new Button ("Start"));
        panel.add (pauseButton = new Button ("Pause"));
        panel.add (stopButton = new Button ("Stop"));
        panel.add (clearButton = new Button ("Clear"));
        enableButtons (true, false, false, false);
        return panel;
    }

    String getCrawlerClassName (String label) {
        String className = label;
        if (className != null) {
            if (className.equals ("Crawler"))
                className = "websphinx.Crawler";
            else if (className.equals ("Load Class..."))
                className = null;
        }
        return className;
    }    
       
    public boolean handleEvent (Event event) {
        if (doEvent (event))
            return true;
        else
            return super.handleEvent (event);
    }

    boolean doEvent (Event event) {
        if (event.id == Event.ACTION_EVENT) {
            if (event.target instanceof MenuItem) {
                MenuItem item = (MenuItem)event.target;
                
                if (item == newCrawlerItem)
                    newCrawler ();
                //#ifdef JDK1.1
                else if (item == openCrawlerItem)
                    openCrawler ();
                else if (item == saveCrawlerItem)
                    saveCrawler ();
                //#endif JDK1.1
                else if (item == createCrawlerItem)
                    createCrawler (null);
                else if (item == exitItem)
                    close ();
                else
                    return false;
            }
            else if (event.target == newCrawlerButton)
                newCrawler ();
            //#ifdef JDK1.1
            else if (event.target == openCrawlerButton)
                openCrawler ();
            else if (event.target == saveCrawlerButton)
                saveCrawler ();
            //#endif JDK1.1
            else if (event.target == createCrawlerButton)
                createCrawler (null);
            else if (event.target == configPanel.advancedButton)
                setAdvancedMode (!advancedMode);
            else if (event.target == vizPanel.optionsButton)
                new WorkbenchControlPanel (graph, outline).show ();
            else if (event.target == vizPanel.tearoffButton)
                if (tornOff)
                    dockVisualizations ();
                else
                    tearoffVisualizations ();
            else if (event.target == startButton)
                start ();
            else if (event.target == pauseButton)
                pause ();
            else if (event.target == stopButton)
                stop ();
            else if (event.target == clearButton)
                clear ();
            else
                return false;
        }
        else
            return false;
            
        return true;
    }

    
    /* * * * * * * * * * * * * * * * * * * * * * *
     *   Command handling
     * * * * * * * * * * * * * * * * * * * * * * */
    
    protected void finalize () {
        // FIX: dispose of frames
    }
    
    void close () {
        if (!allowExit)
            return;