joboswing.java

真正的网络爬虫的源代码啊,希望大家好好阅读,写出心得体会啊

      urlCheckConfigFrame = new URLCheckConfigFrame(jobobase.getURLCheck());
    }
    urlCheckConfigFrame.setVisible(true);
  }

  /**
   * configure URLCheck
   */
  private void addCookie() {
  	String cookieStr = JOptionPane.showInputDialog(this, "Cookie string:");
  	String domain = JOptionPane.showInputDialog(this, "Domain:");
  	URL url;
	try {
		url = new URL("http://"+domain);
	} catch (MalformedURLException e1) {
		JOptionPane.showMessageDialog(this, "Domain invalid: "+e1.getMessage());
		return;
	}
	try {
  		if (cookieStr.startsWith("Set-Cookie")) {
  			Cookie cookie;
			cookie = new Cookie(cookieStr,url);
			jobobase.getRobot().getCookieManager().add(cookie);
  			JOptionPane.showMessageDialog(this, "Cookie added");
  		} else {
  			Cookie[] cookies = Cookie.cookieStringToCookies(cookieStr, domain);
  			for (int i=0; i<cookies.length; i++) {
  				System.out.println(cookies[i]);
  				jobobase.getRobot().getCookieManager().add(cookies[i]);
  			}
  			JOptionPane.showMessageDialog(this, cookies.length+" cookies added");
  		}
	} catch (CookieException e) {
		JOptionPane.showMessageDialog(this, "Cookie string invalid: "+e.getMessage());
	}

  }

  /**
   * configure document filters
   */
  private void configureFilters() {
    if (filterConfigFrame == null) {
      filterConfigFrame = new FilterConfigFrame();
    }
    filterConfigFrame.setVisible(true);
  }

  /**
   * configure allowed URls
   */
  private void configureAllowedURLs() {
    if (allowedURLsFrame == null) {
      allowedURLsFrame = 
	new AllowedListFrame(jobobase.getRobot().getAllowedURLs());
    }
    allowedURLsFrame.setVisible(true);
  }

  
  /**
   * store current settings
   */
  private void saveSettings() {
    jobobase.saveConfig("test.xml");
  }

  
  /** 
   * Exit the Application 
   */
  private void exitApp() {
    System.exit(0);
  }


  
  /**
   * @param args the command line arguments
   */
  public static void main (String[] args) {

    // create a new frame for logging
    logFrame = new LogFrame();
    logFrame.addMsg("Starting ...");

    // configure Log4J subsystem
    lfAppend = new LogFrameAppender(logFrame);
    BasicConfigurator.configure(lfAppend);

    // other command line arguments
    for (int i=0; i<args.length; i++) {
      if (args[i].equals("-debug")) {
	// send all log output to standard out (including level debug)
	BasicConfigurator.configure();
      }
    }

    new JoBoSwing().setVisible(true);
  }

  /**
   * update the WebRobot setting using the values of the input fields
   * @return true if everything is ok, false otherwise
   */
  private boolean updateRobotFromDialog() {

    // start URL
    String startUrl = urlField.getText();
    try {
      URL u = new URL(startUrl);
      jobobase.getRobot().setStartURL(u);
    } catch (MalformedURLException e) {
      JOptionPane.showMessageDialog(this,"URL "+startUrl+" is invalid");
      return false;
    }

    // storage directory
    String storageDir = directoryField.getText();
    jobobase.setStorageDirectory(storageDir);
      
    return true;
  }


  /**
   * update the dialog field from the current setting in the webrobot
   * will only be called on startup !
   */
  private void updateDialogFromRobot() {

    // start URL
    URL u = jobobase.getRobot().getStartURL();
    if (u != null) {
      urlField.setText(u.toString());
    } else {
      urlField.setText("");
    }

  }


  /***********************************************************************
   ***********************************************************************
   
   Callback interface for httpTool

   ***********************************************************************
   ***********************************************************************/

  public void setHttpToolDocUrl(String url) {
    try {
      currentUrlField.setText(urlToString(new URL(url)));
    } catch (MalformedURLException e) {}
  }

  public void setHttpToolDocSize(int size) {
    this.httpToolDocSize = size;
    progressBar.setMaximum(size);
  }

  public void setHttpToolDocCurrentSize(int size) {
    StringBuffer progressStr = new StringBuffer();
    progressBar.setValue(size);
    int kB = size / 1024;
    long longsize = size;

    Date current = new Date();

    long millisecs = (current.getTime()-docDownloadStarted.getTime());
    long ratio = 0;
    if (millisecs > 0) {
      ratio = ((longsize*1000)/millisecs);
    } else {
      ratio = 0;
    }

    progressStr.append(kB);
    progressStr.append(" kB");

    // document size known ?
    if ( this.httpToolDocSize > 0) {
      progressStr.append(" of ");
      progressStr.append(httpToolDocSize / 1024);
      progressStr.append(" kB");
    }

    // ratio
    DecimalFormat myFormat = new DecimalFormat("####0.000");
    progressStr.append(", ");
    progressStr.append(myFormat.format((float)(ratio)/1000));
    progressStr.append(" kB/s");

    
    // time left
    if (this.httpToolDocSize > 0) {
    	long secleft;
    	if (ratio > 0)
    		secleft = (this.httpToolDocSize - longsize)/ratio;
    	else 
    		secleft=100000;
    	
      long min = secleft/60;
      int sec = (int)(secleft%60);
      String secStr = Integer.toString(sec);
      if (secStr.length()<2) {
	secStr = "0"+secStr;
      }
      progressStr.append(", ");
      progressStr.append(min);
      progressStr.append(":");
      progressStr.append(secStr);
      progressStr.append(" min left");
    }

    progressBar.setString(progressStr.toString());
  }

  public void setHttpToolStatus(int status) {
    if (status == HttpTool.STATUS_CONNECTING) {
      progressBar.setString("Connecting");
      this.setHttpToolDocSize(0);
    } else if (status == HttpTool.STATUS_CONNECTED) {
      progressBar.setString("Connected");
      docDownloadStarted=new Date();
    } else if (status == HttpTool.STATUS_RETRIEVING) {
      progressBar.setString("Retrieving");
    } else if (status == HttpTool.STATUS_DONE) {
      progressBar.setString("Finished");
    } else if (status == HttpTool.STATUS_DENIEDBYRULE) {
      progressBar.setString("Denied by rule");  
    } else {
      progressBar.setString("Unknown status ("+status+")");
    }
  }

  /**
   * converts a URL to a textual representation where the authentication
   * info is hidden
   * @param u
   * @return a textual representation
   */
  protected String urlToString(URL u) {
    if (u==null) {
      return null;
    }

    String userInfo=u.getUserInfo();
    if ((userInfo != null) &&
	(! userInfo.equals(""))) {
      userInfo="authenticated@";
    } else {
      userInfo="";
    }

    return u.getProtocol()+"://"+userInfo+u.getHost()+u.getPath();
  }


  /***********************************************************************
   ***********************************************************************
   
   End of callback interface for httpTool

   ***********************************************************************
   ***********************************************************************/

  public void webRobotRetrievedDoc(String url, int size) {
    robotCount++;
    robotSize += size;
    updateControls();
  }

  public void webRobotUpdateQueueStatus(int length) {
    robotQueueSize=length;
    updateControls();
  }

  public void webRobotDone() {
    progressBar.setString("Download completed");
    robotCount=0;
    robotSize=0;
    runStopButton.setText("Run");
  }

  public void webRobotSleeping(boolean sleeping) {
    if (sleeping) {
      progressBar.setString("sleeping");
    } else {
      progressBar.setString("");
    }
  }

  /***********************************************************************
   ***********************************************************************
   
   End of callback interface for webRobot

   ***********************************************************************
   ***********************************************************************/


  /**
   * update status of dynamic elements
   */
  protected void updateControls() {
    DecimalFormat myFormat = new DecimalFormat("###,###,###.0");
    String retrievedContent=
      robotCount+" files with "
      +myFormat.format((float)robotSize/1024)
      +" kB";
    String queuedContent=robotQueueSize+"";
    retrievedField.setText(retrievedContent);
    queuedField.setText(queuedContent);
  }



}