webcrawler.java

java实现的web搜索引擎

/*
 * 下载html源码
 */

package com.catking.webSearcher;

import java.net.*;
import java.io.*;


public class WebCrawler {

	/**
	 * @param args
	 */
	public static String htmlFileName = "index.html";
	
	public static void main(String[] args) throws IOException{
		// TODO Auto-generated method stub
		
		try{
			
			Socket webClient = new Socket("www.jiangnan.edu.cn", 80);
			PrintWriter result = new PrintWriter(webClient.getOutputStream(), true);
			
			BufferedReader receiver = new BufferedReader(
										new InputStreamReader(webClient.getInputStream()));
			
			File htmlFile = new File(htmlFileName);
			FileWriter writer = new FileWriter(htmlFile);
			
			//发送Http请求
			result.println("GET / HTTP/1.1");
			result.println("Host: ");
			result.println("Connection: Close");
			result.println();
			
			//接收Http Response 返回的数据
			boolean mark = true;
			StringBuffer buffer = new StringBuffer();
			while(mark){
				if(receiver.ready()){
					
					int i = 0;
					while(i != -1){
						i = receiver.read();
						buffer.append((char)i);
					}
					mark = false;
				}
			}
			System.out.println(buffer);
			writer.append(buffer);
			
			webClient.close();
			writer.close();
			
				
		}catch(UnknownHostException e){
			System.err.println("找不到主机");
			System.exit(1);
		}catch(IOException e){
			System.err.println("下载失败");
			System.exit(1);
		}
	}
	

}