httpshelper.java

真正的网络爬虫的源代码啊,希望大家好好阅读,写出心得体会啊

package net.matuschek.http.connection;

/*********************************************
    Copyright (c) 2002 by Daniel Matuschek
*********************************************/

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.Socket;

import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;

/*
 * How to use SSL with a proxy:
 * - http://java.sun.com/j2se/1.4/docs/guide/security/jsse/
 *    samples/sockets/client/SSLSocketClientWithTunneling.java
 */


/**
 * An helper class to provide SSL connections with and without proxy
 *
 * @author Daniel Matuschek <[e-mail]>
 * @version $Id: HttpsHelper.java,v 1.5 2002/09/06 13:03:52 matuschd Exp $
 */
public class HttpsHelper {

  /** should it use a proxy server ? */
  boolean useProxy = false;

  /** Proxy host */
  InetAddress proxyHost = null;

  /** Proxy port */
  int proxyPort = 0;

  

  /**
   * Simple costructor that initialized an HttpsHelper
   */
  public HttpsHelper() {
  }


  /**
   * Constructor that initializes the HttpsHelper an
   * also sets the proxy settings.
   */
  public HttpsHelper(InetAddress proxyHost, int proxyPort, boolean useProxy) {
    this.proxyHost = proxyHost;
    this.proxyPort = proxyPort;
    this.useProxy = useProxy;
  }



  /**
   * Creates a new HTTPS connection to the defined host/port
   *
   * @param host full qualified hostname or ip address of the host 
   * to contact
   * @param port destination prot on the server to connect
   *
   * @exception IOException, if the connection cannot be established
   *
   * @return an HttpConnection object with the established conection
   */
  public HttpConnection createHttpsConnection(String host, int port) 
    throws IOException
  {
    HttpConnection connection = null;
    SSLSocketFactory factory = (SSLSocketFactory)SSLSocketFactory.getDefault();
    SSLSocket socket = null;

    if (! useProxy) {
      
      socket = (SSLSocket)factory.createSocket(host,port);

    } else {
      
      Socket tunnel = new Socket(proxyHost, proxyPort);
      doTunnelHandshake(tunnel, host, port);
      
      /*
       * Ok, let's overlay the tunnel socket with SSL.
       */
      socket = (SSLSocket)factory.createSocket(tunnel, host, port, true);
      
      /*
       * send http request
       *
       * See SSLSocketClient.java for more information about why
       * there is a forced handshake here when using PrintWriters.
       */
      socket.startHandshake();
    }

    connection = new HttpConnection(socket);

    return connection;

  }


  /**
   * Tell our tunnel where we want to CONNECT, and look for the
   * right reply.  Throw IOException if anything goes wrong.
   */
  private void doTunnelHandshake(Socket tunnel, String host, int port)
    throws IOException
  {
    OutputStream out = tunnel.getOutputStream();
    String msg = "CONNECT " + host + ":" + port + " HTTP/1.0\n"
      + "User-Agent: JoBo/1.4beta"
      + "\r\n\r\n";
    byte[] b;
    try {
      /*
       * We really do want ASCII7 -- the http protocol doesn't change
       * with locale.
       */
      b = msg.getBytes("ASCII7");
    } catch (UnsupportedEncodingException ignored) {
      /*
       * If ASCII7 isn't there, something serious is wrong, but
       * Paranoia Is Good (tm)
       */
      b = msg.getBytes();
    }
    out.write(b);
    out.flush();
    
    /*
     * We need to store the reply so we can create a detailed
     * error message to the user.
     */
    byte[] reply = new byte[200];
    int	replyLen = 0;
    int newlinesSeen = 0;
    boolean headerDone = false;	/* Done on first newline */
    
    InputStream	in = tunnel.getInputStream();
    while (newlinesSeen < 2) {
      int i = in.read();
      if (i < 0) {
	throw new IOException("Unexpected EOF from proxy");
      }
      if (i == '\n') {
	headerDone = true;
	++newlinesSeen;
      } else if (i != '\r') {
	newlinesSeen = 0;
	if (!headerDone && replyLen < reply.length) {
	  reply[replyLen++] = (byte) i;
	}
      }
    }
    
    /*
     * Converting the byte array to a string is slightly wasteful
     * in the case where the connection was successful, but it's
     * insignificant compared to the network overhead.
     */
    String replyStr;
    try {
      replyStr = new String(reply, 0, replyLen, "ASCII7");
    } catch (UnsupportedEncodingException ignored) {
      replyStr = new String(reply, 0, replyLen);
    }
    
    /* We asked for HTTP/1.0, so we should get that back */
    if (!replyStr.startsWith("HTTP/1.0 200")) {
      throw new IOException("Unable to tunnel through proxy"
			    + ".  Proxy returns \"" + replyStr + "\"");
    }
    
    /* tunneling Handshake was successful! */
  }
  


}