webcontentgetter.java

基于JAVA的网络蜘蛛系统,使用JAVA实现抓取网络资源的网络蜘蛛。通过一个入口网址来扫描整个互联网的网址

package issa.webspider.demo;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;

public class WebContentGetter {

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		if (args.length > 0) {
			try {
				URL u = new URL(args[0]);
				URLConnection uc = u.openConnection();

				Class[] types = {String.class, Reader.class, InputStream.class};
				
				Object o = uc.getContent(types);
				System.out.println("I got a " + o.getClass().getName());
				
				if (o instanceof String) {
					System.out.println("String");
					System.out.println(o);
				} else if (o instanceof Reader) {
					System.out.println("Reader");
					int c;
					Reader r = (Reader) o;
					while ((c=r.read()) != -1) {
						System.out.print((char) c);
					}
				} else if (o instanceof InputStream) {
					System.out.println("InputStream");
					int c;
					InputStream in = (InputStream) o;
					Reader r= new InputStreamReader(in);
					while ((c=r.read()) != -1) {
						System.out.print((char) c);
					}
					while ((c=r.read()) != -1) {
						System.out.print((char) c);
					}
				} else if (o == null) {
					System.out.println("None of the requested types were available.");
				} else {
					System.out.println("Error: unexpected type " + o.getClass());
				}
				
				System.out.println("Content Type " + uc.getContentType());
				
			} catch (MalformedURLException ex) {
				System.err.println(args[0] + "is not a parseable URL");
			} catch (IOException e) {
				System.err.println(e);
			}

		}

	}

}