wsdlcrawler.java

Java Crawler with domain knowledge path

package ie.vnit.webcrawler;
import ie.vnit.threads.*;
import java.net.*;
import java.io.File;
import java.io.FileWriter;
public class WSDLCrawler implements MessageReceiver {
    static String keyword;
   static StringBuffer sb =new StringBuffer();
	public WSDLCrawler(Queue q, int maxLevel, int maxThreads)
		throws InstantiationException, IllegalAccessException {
		ThreadController tc = new ThreadController(WSDLCrawlerThread.class,
												   maxThreads,
												   maxLevel,
												   q,
												   0,
												   this);
	}

	public void finishedAll() {
        try {
             File outFile = new File("C://Log.txt");
              FileWriter out = new FileWriter(outFile);
		      out.write(sb.toString());
              out.close();
       // System.out.println(sb.toString());
		System.out.println("Finished All");
        } catch (Exception e) {
			System.err.println("An error occured: ");
			e.printStackTrace();
			// System.err.println(e.toString());
		}
	}

	public void receiveMessage(Object o, int threadId) {
		// In our case, the object is already string, but that doesn't matter

       sb.append(o.toString()+"\n");
      // System.out.println("sb"+sb.toString());
		System.out.println("[" + threadId + "] " + o.toString());
	}

	public void finished(int threadId) {
       // sb.append("\n[" + threadId + "] finished"+"\n");
		System.out.println("[" + threadId + "] finished");
	}

	public static void main(String[] args) {
		try {
			int maxLevel = 5;
			int maxThreads = 10;
            keyword="teaching";
			/*if (args.length == 4) {
				maxThreads = Integer.parseInt(args[3]);
			}
			if (args.length >= 3) {
				maxLevel = Integer.parseInt(args[2]);
			}*/
			//if (args.length >= 2) {
				URLQueue q = new URLQueue();
				q.setFilenamePrefix("viiiils.txt");
				q.push(new URL("http://www.rediff.com"), 0);
				new WSDLCrawler(q, maxLevel, maxThreads);

          // File f =new File("C://Log.txt");
              //  System.out.println("sb"+sb.toString());
          



				return;
			//}
		} catch (Exception e) {
			System.err.println("An error occured: ");
			e.printStackTrace();
			// System.err.println(e.toString());
		}
		System.err.println("Usage: java WSDLCrawler <url> <filenamePrefix> [<maxLevel> [<maxThreads>]]");
		System.err.println("Crawls the web for WSDL descriptions.");
	}
}