webcrawler.java

Lucene+nuctch一书的全部源码 测试源码 和几个简单的项目

package chapter2;

import java.io.*;
import java.net.*;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;

public class WebCrawler { 

private static String Text_File_Path = "D:\\workshop\\ch2\\htmlsrc.html";

 public static void main(String[] args) throws IOException {
        try { 
        	
			File file = new File(Text_File_Path);
			FileWriter fpWriter = new FileWriter(file);
        	
           // 生成下载对象
           Socket webclient = new Socket("www.bnu.edu.cn", 80); 
           PrintWriter result = new PrintWriter(webclient.getOutputStream(), true); 
           BufferedReader receiver = new BufferedReader(new InputStreamReader(webclient.getInputStream()));

           //发送HTTP request请求
           result.println("GET / HTTP/1.1"); 
           result.println("Host: bnu.edu.cn"); 
           result.println("Connection: Close"); 
           result.println();
           
           //接收HTTP Response 返回的结果信息        
           boolean bRet = true;
           StringBuffer sBuffer = new StringBuffer(8096); 
           while (bRet) { 
              if (receiver.ready()) {
                int idx = 0;
                while (idx != -1) {
                	idx = receiver.read();
                	if(idx == '<')
                		break;
                }
                while (idx != -1) {
                	sBuffer.append((char) idx);
                	idx = receiver.read();
                }
                bRet = false; 
            } 
          }
           
          // 显示获得的网页正文，打印到控制台  
          System.out.println(sBuffer.toString());
          fpWriter.write(sBuffer.toString());
          webclient.close();
          fpWriter.close();
       } catch (UnknownHostException e) {
            System.err.println("无法访问指定主机."); 
            System.exit(1);
       } catch (IOException e) { 
           System.err.println("下载失败，请检查输入地址是否正确。");  
           System.exit(1);
       } 
  }
 
}