main.java

一个网络爬虫

package org.flaviotordini.arale;

import java.util.*;
import java.text.*;
import java.net.*;
import java.io.*;

/**
 *  Arale main class
 *
 *@author     Flavio Tordini
 *@created    16 dicembre 2001
 */
public class Main {

    /**
     *  The main program for the Arale class
     *
     *@param  args           The command line arguments
     *@exception  Exception  Description of Exception
     *@since                 26 novembre 2001
     */
    public static void main(String[] args) throws Exception {

        File propfile = null;
        File localpath = null;
        URL homepage = null;

        for (int j = 0; j < args.length; j++) {
            String s4 = args[j];
            if (s4 != null) {
                if (s4.length() == 0) {
                    args[j] = null;
                } else if (s4.startsWith("-")) {
                    if (s4.equals("-help") || s4.equals("-usage")) {
                        printVersion();
                        printUsage();
                        return;
                    } else if (s4.equals("-version")) {
                        printVersion();
                        return;
                    } else if (s4.equals("-settings")) {
                        try {
                            propfile = new File(args[j + 1]);
                            j++;
                        } catch (ArrayIndexOutOfBoundsException aioob) {
                            String errmsg = "You must specify a property file when using the -settings argument";
                            System.out.println(errmsg);
                            return;
                        }
                    } else if (s4.equals("-output")) {
                        try {
                            localpath = new File(args[j + 1] + File.separator);
                            j++;
                        } catch (ArrayIndexOutOfBoundsException aioob) {
                            String errmsg = "You must specify a directory when using the -output argument";
                            System.out.println(errmsg);
                            return;
                        }
                    }
                } else if (j == 0) {
                    try {
                        homepage = new URL(s4);
                    } catch (MalformedURLException e) {
                        System.out.println("invalid URL: " + e);
                        return;
                    }
                }
            }
        }

        if (propfile == null) {
            propfile = new File("arale.properties");
        }
        System.out.println("Using properties: " + propfile);

        AraleSettings settings = new AraleSettings();
        settings.localpath = localpath;
        settings.homepage = homepage;
        settings.setParameters(propfile);

        Arale arale = new Arale();
        arale.settings = settings;

        System.out.println("Output directory: " + arale.settings.localpath.getCanonicalPath());

        ContextualURL starturl =
                new ContextualURL(arale.settings.homepage, arale.settings.homepage, 1);
        starturl.scannable = true;
        //AraleUtilities.StringContainsToken(urlstring, scanTokens);
        starturl.downloadable = AraleUtilities.StringContainsToken(arale.settings.homepage.toString(), arale.settings.downloadTokens);

        arale.startTime = System.currentTimeMillis();
        arale.followLink(starturl);

    }


    /**
     *  Description of the Method
     */
    private static void printUsage() {
        System.out.println("Usage: arale [<URL>] [<options>]");
        System.out.println("\t-settings <file>: Use specified property file");
        System.out.println("\t-output <dir>: Use specified output directory");
        System.out.println("\t-version: Print Arale version and exit");
        System.out.println("\t-help: Print this message and exit");
        System.out.println();
        System.out.println("If URL is omitted it will be read from the property file.");
        System.out.println();
        System.out.println("Report bugs to Flavio Tordini <[e-mail]>.");
    }


    /**
     *  Description of the Method
     */
    private static void printVersion() {
        String version = Arale.ARALE_MAJOR_VERSION + "."
                 + Arale.ARALE_MINOR_VERSION + " " + Arale.ARALE_RELEASE_TYPE;
        System.out.println(Arale.ARALE_APPNAME + " " + version);
    }

}