joboswing.java

真正的网络爬虫的源代码啊,希望大家好好阅读,写出心得体会啊

package net.matuschek.jobo;

import java.awt.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DecimalFormat;
import java.util.Date;
import javax.swing.*;

import net.matuschek.http.HttpTool;
import net.matuschek.http.HttpToolCallback;
import net.matuschek.http.cookie.Cookie;
import net.matuschek.http.cookie.CookieException;
import net.matuschek.spider.WebRobotCallback;
import net.matuschek.swing.OptionPanel;
import net.matuschek.swing.VerticalAlignPanel;

import org.apache.log4j.BasicConfigurator;

/**
 * The JoBo interface with Swing GUI.
 * 
 * @author  Daniel Matuschek
 * @version $Revision: 1.52 $
 */
public class JoBoSwing 
  extends JFrame 
  implements HttpToolCallback,
	     WebRobotCallback
{
  /** JoBo version */
  final static String VERSION="1.4";
  
//private static Category log = Category.getInstance("");
  private static LogFrame logFrame;
  private static LogFrameAppender lfAppend=null;
  private JoBoBase jobobase = null;
  private Thread robotThread=null;
  private Date docDownloadStarted=new Date();

  private long httpToolDocSize = 0;
  private int robotCount = 0;
  private long robotSize = 0;
  private int robotQueueSize = 0;
  
  // Swing variables declaration
  private JTextField urlField;
  private JTextField directoryField;
  private JProgressBar progressBar;
  private JTextField currentUrlField;
  private JTextField queuedField;
  private JTextField retrievedField;

  private JButton runStopButton;
  private JButton sleepButton;
  // End of Swing variables


  // Other frames 

  private RobotConfigFrame robotConfigFrame = null;
  private URLCheckConfigFrame urlCheckConfigFrame = null;
  private FilterConfigFrame filterConfigFrame = null;
  private AllowedListFrame allowedURLsFrame = null;
  // End of other frames
  

  /** Creates new form JoBoSwing */
  public JoBoSwing() {

    try {
      jobobase = JoBoBase.createFromXML();
    } catch (ClassNotFoundException e) {
      System.out.println("Could not initialize WebRobot: "+e);
      System.exit(1);
    }

    jobobase.registerHttpToolCallback(this);
    jobobase.registerWebRobotCallback(this);

    initComponents ();
    pack ();

    updateDialogFromRobot();
  }


  private void initComponents() {
    getContentPane().setLayout(new BorderLayout());

    setTitle("JoBo/Swing: http://www.matuschek.net/jobo/");
    addWindowListener(new java.awt.event.WindowAdapter() {
	public void windowClosing(java.awt.event.WindowEvent evt) {
	  exitApp();
	}
      }
		      );
    
    /** Heading **/
    VerticalAlignPanel headPanel = new VerticalAlignPanel();

    JLabel heading1 = new JLabel();
    heading1.setText("JoBo "+VERSION);
    heading1.setHorizontalAlignment(JTextField.CENTER);
    heading1.setFont(new java.awt.Font ("Dialog", 0, 24));
    
    JLabel heading2 = new JLabel();
    heading2.setText("look at http://www.matuschek.net/jobo/");
    heading2.setHorizontalAlignment(JTextField.CENTER);
    heading2.setFont(new java.awt.Font ("Dialog", 0, 12));
    
    JLabel heading3 = new JLabel();
    heading3.setText("for more details");
    heading3.setHorizontalAlignment(JTextField.CENTER);
    heading3.setFont(new java.awt.Font ("Dialog", 0, 12));
    
    headPanel.add(heading1);
    headPanel.add(heading2);
    headPanel.add(heading3);

    getContentPane().add(headPanel,BorderLayout.NORTH);

    
    /* fields */
    OptionPanel optPanel = new OptionPanel(2);

    urlField = new JTextField();
    urlField.setColumns(40);
    urlField.setText("http://");
    optPanel.add("URL:",urlField);

    directoryField = new JTextField();
    directoryField.setText(jobobase.getStorageDirectory());
    directoryField.setColumns(40);
    optPanel.add("Storage directory:",directoryField);
      
    currentUrlField = new JTextField();
    currentUrlField.setEditable(false);
    currentUrlField.setColumns(40);
    currentUrlField.setText("-");
    optPanel.add("Current URL:",currentUrlField);
          
    retrievedField = new JTextField();
    retrievedField.setEditable(false);
    retrievedField.setColumns(40);
    optPanel.add("Retrieved:",retrievedField);
      
    queuedField = new JTextField();
    queuedField.setEditable(false);
    queuedField.setColumns(40);
    optPanel.add("Queued:",queuedField);
      
    progressBar = new JProgressBar();
    progressBar.setStringPainted(true);
    optPanel.add("Progress:",progressBar,GridBagConstraints.HORIZONTAL);

    getContentPane().add(optPanel);    

    /** Buttons **/
    JPanel buttonPanel = new JPanel();
    JButton butt = null;
            
    runStopButton = new JButton();
    runStopButton.setText("Run");
    runStopButton.addActionListener(new java.awt.event.ActionListener() {
	public void actionPerformed(java.awt.event.ActionEvent evt) {
	  runStopButtonActionPerformed();
	}
      }
				    );
    buttonPanel.add(runStopButton);
             
    sleepButton = new JButton();
    sleepButton.setText("Sleep");
    sleepButton.addActionListener(new java.awt.event.ActionListener() {
	public void actionPerformed(java.awt.event.ActionEvent evt) {
	  sleepButtonActionPerformed();
	}
      }
				  );
    buttonPanel.add(sleepButton);
      
    butt = new JButton();
    butt.setText("Robot settings");
    butt.addActionListener(new java.awt.event.ActionListener() {
	public void actionPerformed(java.awt.event.ActionEvent evt) {
	  doRobotSettingsDialog();
	}
      }
				  );
    buttonPanel.add(butt);
      
    butt = new JButton();
    butt.setText("Save");
    butt.addActionListener(new java.awt.event.ActionListener() {
	public void actionPerformed(java.awt.event.ActionEvent evt) {
	  saveSettings();
	}
      }
				  );
    //    buttonPanel.add(butt);
      
    butt = new JButton();
    butt.setText("URLCheck");
    butt.addActionListener(new java.awt.event.ActionListener() {
	public void actionPerformed(java.awt.event.ActionEvent evt) {
	  configureURLCheck();
	}
      }
				  );
    //    buttonPanel.add(butt);

    butt = new JButton();
    butt.setText("Filter configuration");
    butt.addActionListener(new java.awt.event.ActionListener() {
	public void actionPerformed(java.awt.event.ActionEvent evt) {
	  configureFilters();
	}
      }
			   );
    
    // buttonPanel.add(butt);
    
      
    butt = new JButton();
    butt.setText("Allowed URLs");
    butt.addActionListener(new java.awt.event.ActionListener() {
	public void actionPerformed(java.awt.event.ActionEvent evt) {
	  configureAllowedURLs();
	}
      });
    buttonPanel.add(butt);
    

    butt = new JButton();
    butt.setText("Add cookie");
    butt.addActionListener(new java.awt.event.ActionListener() {
	public void actionPerformed(java.awt.event.ActionEvent evt) {
	  addCookie();
	}
      });
    buttonPanel.add(butt);
    

    

    butt = new JButton();
    butt.setText("Log");
    butt.addActionListener(new java.awt.event.ActionListener() {
	public void actionPerformed(java.awt.event.ActionEvent evt) {
	  logButtonActionPerformed();
	}
      });
    buttonPanel.add(butt);
      
        
    butt = new JButton();
    butt.setText("Exit");
    butt.addActionListener(new java.awt.event.ActionListener() {
	public void actionPerformed(java.awt.event.ActionEvent evt) {
	  exitApp();
	}
      });
    buttonPanel.add(butt);        
        
    getContentPane().add(buttonPanel,BorderLayout.SOUTH);
  }

  private void sleepButtonActionPerformed() {

    // is there a robotThread running ?
    if ((robotThread != null) && 
	(robotThread.isAlive())) {
      
      if (jobobase.getRobot().isSleeping()) {
	jobobase.getRobot().setSleep(false);
	sleepButton.setText("Sleep");
      } else {
	jobobase.getRobot().setSleep(true);
	sleepButton.setText("Wake up");
      }
    }
  } 


  private void logButtonActionPerformed() {
      logFrame.setVisible(true);
  }

  private void runStopButtonActionPerformed() {

    // is there another robotThread running ?
    if ((robotThread != null) && 
	(robotThread.isAlive())) {
      jobobase.getRobot().stopRobot();
    } else {
      if (updateRobotFromDialog()) {
	runStopButton.setText("Stop");
	// clear cookies from last run
	//jobobase.getRobot().clearCookies();
	robotThread = new Thread(jobobase.getRobot());
	robotThread.start();
      }
    }
  }

  /**
   * open the robot settings dialog
   */
  private void doRobotSettingsDialog() {
    if (robotConfigFrame == null) {
      robotConfigFrame = new RobotConfigFrame(jobobase);
    }
    robotConfigFrame.setVisible(true);
  }

  /**
   * configure URLCheck
   */
  private void configureURLCheck() {
    if (urlCheckConfigFrame == null) {