webcrawler.java

lucene和nutch书中配套代码

package chapter2;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.URL;
import java.net.URLConnection;
import java.net.UnknownHostException;

public class WebCrawler { 

private static String Text_File_Path = "c:\\workshop\\ch2\\htmlsrc.html";

 public static void main(String[] args) throws IOException {
        try { 
        	
			File file = new File(Text_File_Path);
			FileWriter fpWriter = new FileWriter(file);
        	
           // 生成下载对象
			URL url = new URL("http://www.bnu.edu.cn");
            // 创建代理服务器
            InetSocketAddress addr = new InetSocketAddress("172.17.18.84",
                    8080);
            Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); // http 代理
            // 如果我们知道代理server的名字, 可以直接使用
            // 结束
            URLConnection conn = url.openConnection(proxy);
            InputStream in = conn.getInputStream();
            
           //Socket webclient = new Socket("www.bnu.edu.cn", 80); 
           //PrintWriter result = new PrintWriter(webclient.getOutputStream(), true); 
           //BufferedReader receiver = new BufferedReader(new InputStreamReader(webclient.getInputStream()));
            BufferedReader receiver = new BufferedReader(new InputStreamReader(in));

           //发送HTTP request请求
           /*result.println("GET / HTTP/1.1"); 
           result.println("Host: bnu.edu.cn"); 
           result.println("Connection: Close"); 
           result.println();*/
           
           //接收HTTP Response 返回的结果信息        
           boolean bRet = true;
           StringBuffer sBuffer = new StringBuffer(8096); 
           while (bRet) { 
              if (receiver.ready()) {
                int idx = 0;
                while (idx != -1) {
                	idx = receiver.read();
                	if(idx == '<')
                		break;
                }
                while (idx != -1) {
                	sBuffer.append((char) idx);
                	idx = receiver.read();
                }
                bRet = false; 
            } 
          }
           
          // 显示获得的网页正文，打印到控制台  
          System.out.println(sBuffer.toString());
          fpWriter.write(sBuffer.toString());
          //webclient.close();
          fpWriter.close();
       } catch (UnknownHostException e) {
            System.err.println("无法访问指定主机."); 
            System.exit(1);
       } catch (IOException e) { 
           System.err.println("下载失败，请检查输入地址是否正确。");  
           System.exit(1);
       } 
  }
 
}