heritrix.java

高性能分词算法

/* Heritrix
 *
 * $Id: Heritrix.java 6081 2008-12-09 00:58:14Z gojomo $
 *
 * Created on May 15, 2003
 *
 * Copyright (C) 2003 Internet Archive.
 *
 * This file is part of the Heritrix web crawler (crawler.archive.org).
 *
 * Heritrix is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * any later version.
 *
 * Heritrix is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser Public License for more details.
 *
 * You should have received a copy of the GNU Lesser Public License
 * along with Heritrix; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
package org.archive.crawler;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.InetAddress;
import java.net.URL;
import java.net.URLConnection;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.TimeZone;
import java.util.Vector;
import java.util.logging.Level;
import java.util.logging.LogManager;
import java.util.logging.Logger;

import javax.management.Attribute;
import javax.management.AttributeList;
import javax.management.AttributeNotFoundException;
import javax.management.DynamicMBean;
import javax.management.InstanceAlreadyExistsException;
import javax.management.InstanceNotFoundException;
import javax.management.InvalidAttributeValueException;
import javax.management.MBeanInfo;
import javax.management.MBeanNotificationInfo;
import javax.management.MBeanOperationInfo;
import javax.management.MBeanRegistration;
import javax.management.MBeanRegistrationException;
import javax.management.MBeanServer;
import javax.management.MBeanServerFactory;
import javax.management.MalformedObjectNameException;
import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
import javax.management.ReflectionException;
import javax.management.RuntimeOperationsException;
import javax.management.openmbean.CompositeData;
import javax.management.openmbean.CompositeDataSupport;
import javax.management.openmbean.CompositeType;
import javax.management.openmbean.OpenDataException;
import javax.management.openmbean.OpenMBeanAttributeInfoSupport;
import javax.management.openmbean.OpenMBeanConstructorInfoSupport;
import javax.management.openmbean.OpenMBeanInfoSupport;
import javax.management.openmbean.OpenMBeanOperationInfoSupport;
import javax.management.openmbean.OpenMBeanParameterInfo;
import javax.management.openmbean.OpenMBeanParameterInfoSupport;
import javax.management.openmbean.OpenType;
import javax.management.openmbean.SimpleType;
import javax.management.openmbean.TabularData;
import javax.management.openmbean.TabularDataSupport;
import javax.management.openmbean.TabularType;
import javax.naming.CompoundName;
import javax.naming.Context;
import javax.naming.NameNotFoundException;
import javax.naming.NamingException;
import javax.naming.NoInitialContextException;

import org.apache.commons.cli.Option;
import org.archive.crawler.admin.CrawlJob;
import org.archive.crawler.admin.CrawlJobErrorHandler;
import org.archive.crawler.admin.CrawlJobHandler;
import org.archive.crawler.datamodel.CredentialStore;
import org.archive.crawler.datamodel.credential.Credential;
import org.archive.crawler.event.CrawlStatusListener;
import org.archive.crawler.framework.AlertManager;
import org.archive.crawler.framework.CrawlController;
import org.archive.crawler.framework.exceptions.FatalConfigurationException;
import org.archive.crawler.framework.exceptions.InitializationException;
import org.archive.crawler.selftest.SelfTestCrawlJobHandler;
import org.archive.crawler.settings.XMLSettingsHandler;
import org.archive.io.SinkHandler;
import org.archive.io.SinkHandlerLogRecord;
import org.archive.net.UURI;
import org.archive.util.FileUtils;
import org.archive.util.IoUtils;
import org.archive.util.JmxUtils;
import org.archive.util.JndiUtils;
import org.archive.util.PropertyUtils;
import org.archive.util.TextUtils;

import sun.net.www.protocol.file.FileURLConnection;


/**
 * Main class for Heritrix crawler.
 *
 * Heritrix is usually launched by a shell script that backgrounds heritrix
 * that redirects all stdout and stderr emitted by heritrix to a log file.  So
 * that startup messages emitted subsequent to the redirection of stdout and
 * stderr show on the console, this class prints usage or startup output
 * such as where the web UI can be found, etc., to a STARTLOG that the shell
 * script is waiting on.  As soon as the shell script sees output in this file,
 * it prints its content and breaks out of its wait.
 * See ${HERITRIX_HOME}/bin/heritrix.
 * 
 * <p>Heritrix can also be embedded or launched by webapp initialization or
 * by JMX bootstrapping.  So far I count 4 methods of instantiation:
 * <ol>
 * <li>From this classes main -- the method usually used;</li>
 * <li>From the Heritrix UI (The local-instances.jsp) page;</li>
 * <li>A creation by a JMX agent at the behest of a remote JMX client; and</li>
 * <li>A container such as tomcat or jboss.</li>
 * </ol>
 *
 * @author gojomo
 * @author Kristinn Sigurdsson
 * @author Stack
 */
public class Heritrix implements DynamicMBean, MBeanRegistration {
    /**
     * Heritrix logging instance.
     */
    private static final Logger logger =
        Logger.getLogger(Heritrix.class.getName());
    
    private static final File TMPDIR =
        new File(System.getProperty("java.io.tmpdir", "/tmp"));

    /**
     * Name of the heritrix properties file.
     */
    private static final String PROPERTIES = "heritrix.properties";

    /**
     * Name of the key to use specifying alternate heritrix properties on
     * command line.
     */
    private static final String PROPERTIES_KEY = PROPERTIES;
    
    /**
     * Prefix used on our properties we'll add to the System.properties list.
     */
    private static final String HERITRIX_PROPERTIES_PREFIX = "heritrix.";

    /**
     * Prefix used on other properties we'll add to the System.properties 
     * list (after stripping this prefix). 
     */
    private static final String SYSTEM_PREFIX = "system.";

    /**
     * Instance of web server if one was started.
     */
    private static SimpleHttpServer httpServer = null;

    /**
     * CrawlJob handler. Manages multiple crawl jobs at runtime.
     */
    private CrawlJobHandler jobHandler = null;

    /**
     * Heritrix start log file.
     *
     * This file contains standard out produced by this main class for startup
     * only.  Used by heritrix shell script.  Name here MUST match that in the
     * <code>bin/heritrix</code> shell script.  This is a DEPENDENCY the shell
     * wrapper has on this here java heritrix.
     */
    private static final String STARTLOG = "heritrix_dmesg.log";

    /**
     * Default encoding.
     * 
     * Used for content when fetching if none specified.
     */
	public static final String DEFAULT_ENCODING = "ISO-8859-1";

    /**
     * Heritrix stderr/stdout log file.
     *
     * This file should have nothing in it except messages over which we have
     * no control (JVM stacktrace, 3rd-party lib emissions).  The wrapper
     * startup script directs stderr/stdout here. This is an INTERDEPENDENCY
     * this program has with the wrapper shell script.  Shell can actually
     * pass us an alternate to use for this file.
     */
    private static String DEFAULT_HERITRIX_OUT = "heritrix_out.log";

    /**
     * Where to write this classes startup output.
     * 
     * This out should only be used if Heritrix is being run from the
     * command-line.
     */
    private static PrintWriter out = null;

    /**
     * The org.archive package
     */
    private static final String ARCHIVE_PACKAGE = "org.archive.";

    /**
     * The crawler package.
     */
	private static final String CRAWLER_PACKAGE = Heritrix.class.getName().
        substring(0, Heritrix.class.getName().lastIndexOf('.'));
    
    /**
     * The root context for a webapp.
     */
    private static final String ROOT_CONTEXT = "/";

    /**
     * Set to true if application is started from command line.
     */
    private static boolean commandLine = false;
    
    /**
     * True if container initialization has been run.
     */
    private static boolean containerInitialized = false;
    
    /**
     * True if properties have been loaded.
     */
    private static boolean propertiesLoaded = false;
    
    private static final String JAR_SUFFIX = ".jar";
    
    private AlertManager alertManager;

    /**
     * The context of the GUI webapp.  Default is root.
     */
    private static String adminContext = ROOT_CONTEXT;
    
    /**
     * True if we're to put up a GUI.
     * Cmdline processing can override.
     */
    private static boolean gui =
        !PropertyUtils.getBooleanProperty("heritrix.cmdline.nowui");
    
    /**
     * Port to put the GUI up on.
     * Cmdline processing can override.
     */
    private static int guiPort = SimpleHttpServer.DEFAULT_PORT;

    
    /**
     * A collection containing only localhost.  Used as default value
     * for guiHosts, and passed to SimpleHttpServer when doing selftest.
     */
    final private static Collection<String> LOCALHOST_ONLY =
     Collections.unmodifiableList(Arrays.asList(new String[] { "127.0.0.1" }));

    
    /**
     * Hosts to bind the GUI webserver to.
     * By default, only contans localhost.
     * Set to an empty collection to indicate that all available network
     * interfaces should be used for the webserver.
     */
    private static Collection<String> guiHosts = LOCALHOST_ONLY;
    
    
    /**
     * Web UI server, realm, context name.
     */
    private static String ADMIN = "admin";
    
    // OpenMBean support.
    /**
     * The MBean server we're registered with (May be null).
     */
    private MBeanServer mbeanServer = null;
    
    /**
     * MBean name we were registered as.
     */
    private ObjectName mbeanName = null;
    
    /**
     * Keep reference to all instances of Heritrix.
     * Used by the UI to figure which of the local Heritrice it should
     * be going against and to figure what to shutdown on the way out (If
     * there was always a JMX Agent, we wouldn't need to keep this list.  We
     * could always ask the JMX Agent for all instances. UPDATE: True we could
     * always ask the JMX Agent but we might keep around this local reference
     * because it will allow faster, less awkward -- think of marshalling the args
     * for JMX invoke operation -- access to local Heritrix instances.  A new
     * usage for this instances Map is in CrawlJob#preRegister to find the hosting
     * Heritrix instance).
     */
    private static Map<String,Heritrix> instances
     = new Hashtable<String,Heritrix>();
    
    private OpenMBeanInfoSupport openMBeanInfo;
    private final static String STATUS_ATTR = "Status";
    private final static String VERSION_ATTR = "Version";
    private final static String ISRUNNING_ATTR = "IsRunning";
    private final static String ISCRAWLING_ATTR = "IsCrawling";
    private final static String ALERTCOUNT_ATTR = "AlertCount";
    private final static String NEWALERTCOUNT_ATTR = "NewAlertCount";
    private final static String CURRENTJOB_ATTR = "CurrentJob";
    private final static List ATTRIBUTE_LIST;
    static {
        ATTRIBUTE_LIST = Arrays.asList(new String [] {STATUS_ATTR,
            VERSION_ATTR, ISRUNNING_ATTR, ISCRAWLING_ATTR,
            ALERTCOUNT_ATTR, NEWALERTCOUNT_ATTR, CURRENTJOB_ATTR});