robotconfigframe.java

真正的网络爬虫的源代码啊,希望大家好好阅读,写出心得体会啊

package net.matuschek.jobo;

import javax.swing.BoxLayout;
import javax.swing.JButton;
import javax.swing.JCheckBox;
import javax.swing.JOptionPane;
import javax.swing.JPanel;
import javax.swing.JTextField;

import net.matuschek.http.HttpException;
import net.matuschek.spider.WebRobot;
import net.matuschek.swing.JHideFrame;
import net.matuschek.swing.OptionPanel;
import net.matuschek.swing.SwingHelper;
/*********************************************
    Copyright (c) 2001 by Daniel Matuschek
 *********************************************/



/**
 * Configuration dialog for the robot
 * 
 * @author Daniel Matuschek
 * @version $Id: RobotConfigFrame.java,v 1.20 2004/08/09 13:06:56 matuschd Exp $
 */
public class RobotConfigFrame extends JHideFrame {

	private static final long serialVersionUID = 8977779246579784214L;

	public RobotConfigFrame(JoBoBase jobobase) {    
		super("Robot configuration");
		this.robot = jobobase.getRobot();
		this.jobobase = jobobase;
		initComponents();
	}


	protected void initComponents() {
		JPanel mainPanel = new JPanel();
		mainPanel.setLayout(new BoxLayout(mainPanel,BoxLayout.Y_AXIS));
		setContentPane(mainPanel);

		OptionPanel dialogPanel = new OptionPanel(2);

		JPanel buttonPanel = new JPanel();
		buttonPanel.setLayout(new BoxLayout(buttonPanel,BoxLayout.X_AXIS));

		mainPanel.add(dialogPanel);
		mainPanel.add(buttonPanel);

		/**
       Input fields
		 **/
		agentNameField = SwingHelper.createInputField(25);
		dialogPanel.add("Agent name:",agentNameField);

		startRefererField = SwingHelper.createInputField(25);
		dialogPanel.add("Start referer:",startRefererField);

		proxyField = SwingHelper.createInputField(25);
		dialogPanel.add("Proxy (host:port or empty):",proxyField);

		maxDepthField = SwingHelper.createInputField(5);
		dialogPanel.add("Maximal search depth:",maxDepthField);

		sleepField = SwingHelper.createInputField(5);
		dialogPanel.add("Sleep time (in seconds)",sleepField);

		bandwidthField = SwingHelper.createInputField(5);
		dialogPanel.add("Bandwidth limit in Bytes/s (0 to disable)",
				bandwidthField);

		maxAgeField = SwingHelper.createInputField(5);
		dialogPanel.add("Maximum age in days (leave empty to disable)",
				maxAgeField);

		cookiesEnabled = new JCheckBox();
		dialogPanel.add("Enable cookies",cookiesEnabled);

		allowWholeHost = new JCheckBox();
		dialogPanel.add("Allow all URLs on start host",allowWholeHost);

		allowWholeDomain = new JCheckBox();
		dialogPanel.add("Allow all URLs in the same domain",allowWholeDomain);

		flexibleHostCheck = new JCheckBox();
		dialogPanel.add("Do flexible host checking",flexibleHostCheck);

		ignoreRobotsTxt = new JCheckBox();
		dialogPanel.add("Ignore robots.txt",ignoreRobotsTxt);

		localizeLinks = new JCheckBox();
		dialogPanel.add("Localize links",localizeLinks);

		storeCGI = new JCheckBox();
		dialogPanel.add("Save dynamic pages",storeCGI);

		allowCaching = new JCheckBox();
		dialogPanel.add("Don't retrieve pages that are already on disk "
				+"(resume download)",
				allowCaching);

		/** End of input fields */

		/** Button panel */
		JButton okButton = new JButton();
		okButton.setText("OK");
		okButton.addActionListener(new java.awt.event.ActionListener() {
			public void actionPerformed(java.awt.event.ActionEvent evt) {
				updateAndHide();
			}
		});
		buttonPanel.add(okButton);

		JButton closeButton = new JButton();
		closeButton.setText("Cancel");
		closeButton.addActionListener(new java.awt.event.ActionListener() {
			public void actionPerformed(java.awt.event.ActionEvent evt) {
				exitForm();
			}
		});
		buttonPanel.add(closeButton);

		/** End of button panel */

		pack ();
		updateFormFromRobot();
	}


	/**
	 * update the robot settings from the form contents and hide window
	 */
	protected void updateAndHide() {
		if (updateRobotFromForm()) {
			this.setVisible(false);
		}
	}


	/** 
	 * update the content of the form field from the current robot
	 * settings
	 */
	protected void updateFormFromRobot() {
		agentNameField.setText(robot.getAgentName());
		startRefererField.setText(robot.getStartReferer());
		proxyField.setText(robot.getProxy());
		maxDepthField.setText(String.valueOf(robot.getMaxDepth()));
		sleepField.setText(String.valueOf(robot.getSleepTime()));
		bandwidthField.setText(String.valueOf(robot.getBandwidth()));

		// a day has 86400 seconds
		long maxAge=robot.getMaxDocumentAge();
		if (maxAge>0) {
			maxAgeField.setText(Long.toString(maxAge/86400));
		} else {
			maxAgeField.setText("");
		}

		cookiesEnabled.setSelected(robot.getEnableCookies());
		allowWholeHost.setSelected(robot.getAllowWholeHost());
		allowWholeDomain.setSelected(robot.getAllowWholeDomain());
		flexibleHostCheck.setSelected(robot.getFlexibleHostCheck());
		ignoreRobotsTxt.setSelected(robot.getIgnoreRobotsTxt());
		localizeLinks.setSelected(jobobase.getLocalizeLinks());
		storeCGI.setSelected(jobobase.getStoreCGI());
		allowCaching.setSelected(robot.getAllowCaching());
	}


	/**
	 * update the robot settings from the current fields
	 * @return true if everything is okay, false otherwise
	 */
	protected boolean updateRobotFromForm() {
		robot.setAgentName(agentNameField.getText());
		robot.setStartReferer(startRefererField.getText());
		try {
			robot.setProxy(proxyField.getText());
		} catch (HttpException e) {}

		try {
			robot.setMaxDepth(Integer.parseInt(maxDepthField.getText()));
		} catch (NumberFormatException e) {
			JOptionPane.showMessageDialog(this, 
					"Maximal depth must be a number", 
					"Error", 
					JOptionPane.ERROR_MESSAGE); 
			return false;
		}

		try {
			robot.setSleepTime(Integer.parseInt(sleepField.getText()));
		} catch (NumberFormatException e) {
			JOptionPane.showMessageDialog(this, 
					"Sleep time must be a number", 
					"Error", 
					JOptionPane.ERROR_MESSAGE); 
			return false;
		}

		// bandwidth limitation
		try {
			robot.setBandwidth(Integer.parseInt(bandwidthField.getText()));
		} catch (NumberFormatException e) {
			JOptionPane.showMessageDialog(this, 
					"Bandwidth must be an integer", 
					"Error", 
					JOptionPane.ERROR_MESSAGE); 
			return false;
		}

		// maximum age of files to download
		if (! maxAgeField.getText().equals("")) {
			try {
				int age = Integer.parseInt(maxAgeField.getText());
				// a day has 86400 seconds
				robot.setMaxDocumentAge(age*86400);
			} catch (NumberFormatException e) {
				JOptionPane.showMessageDialog(this, 
						"Age must be an integer", 
						"Error", 
						JOptionPane.ERROR_MESSAGE); 
				return false;
			}
		} else {
			robot.setMaxDocumentAge(-1);
		}


		robot.setEnableCookies(cookiesEnabled.isSelected());
		robot.setAllowWholeHost(allowWholeHost.isSelected());
		robot.setAllowWholeDomain(allowWholeDomain.isSelected());
		robot.setFlexibleHostCheck(flexibleHostCheck.isSelected());
		robot.setIgnoreRobotsTxt(ignoreRobotsTxt.isSelected());
		robot.setAllowCaching(allowCaching.isSelected());

		jobobase.setLocalizeLinks(localizeLinks.isSelected());
		jobobase.setStoreCGI(storeCGI.isSelected());

		return true;
	}

	/**********************************************************************/
	/**********************************************************************/
	/**********************************************************************/

	/** the web robot to configure */
	WebRobot robot = null;

	/** jobobase to configure */
	JoBoBase jobobase = null;


	// input fields
	JTextField agentNameField = null;
	JTextField startRefererField = null;
	JTextField proxyField = null;
	JTextField maxDepthField = null;
	JTextField sleepField = null;
	JTextField bandwidthField = null;
	JTextField maxAgeField = null;
	JCheckBox cookiesEnabled = null;
	JCheckBox allowWholeHost = null;
	JCheckBox allowWholeDomain = null;
	JCheckBox flexibleHostCheck = null;
	JCheckBox ignoreRobotsTxt = null;
	JCheckBox localizeLinks = null;
	JCheckBox storeCGI = null;
	JCheckBox allowCaching = null;
	//

} // RobotConfigFrame