heritrix.java

爬虫

	}

	/**
	 * Shutdown all running heritrix instances and the JVM. Assumes stop has
	 * already been called.
	 * 
	 * @param exitCode
	 *            Exit code to pass system exit.
	 */
	public static void shutdown(final int exitCode)
	{
		getShutdownThread(true, exitCode, "Heritrix shutdown").start();
	}

	protected static Thread getShutdownThread(final boolean sysexit,
			final int exitCode, final String name)
	{
		Thread t = new Thread(name)
		{
			public void run()
			{
				Heritrix.prepareHeritrixShutDown();
				if (sysexit)
				{
					Heritrix.performHeritrixShutDown(exitCode);
				}
			}
		};
		t.setDaemon(true);
		return t;
	}

	public static void shutdown()
	{
		shutdown(0);
	}

	/**
	 * Register Heritrix with JNDI, JMX, and with the static hashtable of all
	 * Heritrix instances known to this JVM.
	 * 
	 * If launched from cmdline, register Heritrix MBean if an agent to register
	 * ourselves with. Usually this method will only have effect if we're
	 * running in a 1.5.0 JDK and command line options such as
	 * '-Dcom.sun.management.jmxremote.port=8082
	 * -Dcom.sun.management.jmxremote.authenticate=false
	 * -Dcom.sun.management.jmxremote.ssl=false' are supplied. See <a
	 * href="http://java.sun.com/j2se/1.5.0/docs/guide/management/agent.html">Monitoring
	 * and Management Using JMX</a> for more on the command line options and
	 * how to connect to the Heritrix bean using the JDK 1.5.0 jconsole tool. We
	 * register currently with first server we find (TODO: Make configurable).
	 * 
	 * <p>
	 * If we register successfully with a JMX agent, then part of the
	 * registration will include our registering ourselves with JNDI.
	 * 
	 * <p>
	 * Finally, add the heritrix instance to the hashtable of all the Heritrix
	 * instances floating in the current VM. This latter registeration happens
	 * whether or no there is a JMX agent to register with. This is a list we
	 * keep out of convenience so its easy iterating over all all instances
	 * calling stop when main application is going down.
	 * 
	 * @param h
	 *            Instance of heritrix to register.
	 * @param name
	 *            Name to use for this Heritrix instance.
	 * @param jmxregister
	 *            True if we are to register this instance with JMX.
	 * @throws NullPointerException
	 * @throws MalformedObjectNameException
	 * @throws NotCompliantMBeanException
	 * @throws MBeanRegistrationException
	 * @throws InstanceAlreadyExistsException
	 */
	protected static void registerHeritrix(final Heritrix h, final String name,
			final boolean jmxregister) throws MalformedObjectNameException,
			InstanceAlreadyExistsException, MBeanRegistrationException,
			NotCompliantMBeanException
	{
		MBeanServer server = getMBeanServer();
		if (server != null)
		{
			// Are we to manage the jmx registration? Or is it being done for
			// us by an external process: e.g. This instance was created by
			// MBeanAgent.
			if (jmxregister)
			{
				ObjectName objName = (name == null || name.length() <= 0)
						? getJmxObjectName() : getJmxObjectName(name);
				registerMBean(server, h, objName);
			}
		}
		else
		{
			// JMX ain't available. Put this instance into the list of Heritrix
			// instances so findable by the UI (Normally this is done in the
			// JMX postRegister routine below). When no JMX, can only have
			// one instance of Heritrix so no need to do the deregisteration.
			Heritrix.instances.put(h.getNoJmxName(), h);
		}
	}

	protected static void unregisterHeritrix(final Heritrix h)
			throws InstanceNotFoundException, MBeanRegistrationException,
			NullPointerException
	{
		MBeanServer server = getMBeanServer();
		if (server != null)
		{
			server.unregisterMBean(h.mbeanName);
		}
		else
		{
			// JMX ain't available. Remove from list of Heritrix instances.
			// Usually this is done by the JMX postDeregister below.
			Heritrix.instances.remove(h.getNoJmxName());
		}
	}

	/**
	 * Get MBeanServer. Currently uses first MBeanServer found. This will
	 * definetly not be whats always wanted. TODO: Make which server settable.
	 * Also, if none, put up our own MBeanServer.
	 * 
	 * @return An MBeanServer to register with or null.
	 */
	public static MBeanServer getMBeanServer()
	{
		MBeanServer result = null;
		List servers = MBeanServerFactory.findMBeanServer(null);
		if (servers == null)
		{
			return result;
		}
		for (Iterator i = servers.iterator(); i.hasNext();)
		{
			MBeanServer server = (MBeanServer) i.next();
			if (server == null)
			{
				continue;
			}
			result = server;
			break;
		}
		return result;
	}

	public static MBeanServer registerMBean(final Object objToRegister,
			final String name, final String type)
			throws InstanceAlreadyExistsException, MBeanRegistrationException,
			NotCompliantMBeanException
	{
		MBeanServer server = getMBeanServer();
		if (server != null)
		{
			server = registerMBean(server, objToRegister, name, type);
		}
		return server;
	}

	public static MBeanServer registerMBean(final MBeanServer server,
			final Object objToRegister, final String name, final String type)
			throws InstanceAlreadyExistsException, MBeanRegistrationException,
			NotCompliantMBeanException
	{
		try
		{
			Hashtable ht = new Hashtable();
			ht.put(JmxUtils.NAME, name);
			ht.put(JmxUtils.TYPE, type);
			registerMBean(server, objToRegister, new ObjectName(
				CRAWLER_PACKAGE, ht));
		}
		catch (MalformedObjectNameException e)
		{
			e.printStackTrace();
		}
		return server;
	}

	public static MBeanServer registerMBean(final MBeanServer server,
			final Object objToRegister, final ObjectName objName)
			throws InstanceAlreadyExistsException, MBeanRegistrationException,
			NotCompliantMBeanException
	{
		server.registerMBean(objToRegister, objName);
		return server;
	}

	public static void unregisterMBean(final MBeanServer server,
			final String name, final String type)
	{
		if (server == null)
		{
			return;
		}
		try
		{
			unregisterMBean(server, getJmxObjectName(name, type));
		}
		catch (MalformedObjectNameException e)
		{
			e.printStackTrace();
		}
	}

	public static void unregisterMBean(final MBeanServer server,
			final ObjectName name)
	{
		try
		{
			server.unregisterMBean(name);
			logger.info("Unregistered bean " + name.getCanonicalName());
		}
		catch (InstanceNotFoundException e)
		{
			e.printStackTrace();
		}
		catch (MBeanRegistrationException e)
		{
			e.printStackTrace();
		}
		catch (NullPointerException e)
		{
			e.printStackTrace();
		}
	}

	/**
	 * @return Name to use when no JMX agent available.
	 */
	protected String getNoJmxName()
	{
		return this.getClass().getName();
	}

	public static ObjectName getJmxObjectName()
			throws MalformedObjectNameException, NullPointerException
	{
		return getJmxObjectName("Heritrix", JmxUtils.SERVICE);
	}

	public static ObjectName getJmxObjectName(final String name)
			throws MalformedObjectNameException, NullPointerException
	{
		return getJmxObjectName(name, JmxUtils.SERVICE);
	}

	public static ObjectName getJmxObjectName(final String name,
			final String type) throws MalformedObjectNameException,
			NullPointerException
	{
		Hashtable ht = new Hashtable();
		ht.put(JmxUtils.NAME, name);
		ht.put(JmxUtils.TYPE, type);
		return new ObjectName(CRAWLER_PACKAGE, ht);
	}

	/**
	 * @return Returns true if Heritrix was launched from the command line.
	 *         (When launched from command line, we do stuff like put up a web
	 *         server to manage our web interface and we register ourselves with
	 *         the first available jmx agent).
	 */
	public static boolean isCommandLine()
	{
		return Heritrix.commandLine;
	}

	/**
	 * @return True if heritrix has been started.
	 */
	public boolean isStarted()
	{
		return this.jobHandler != null;
	}

	public String getStatus()
	{
		StringBuffer buffer = new StringBuffer();
		if (this.getJobHandler() != null)
		{
			buffer.append("isRunning=");
			buffer.append(this.getJobHandler().isRunning());
			buffer.append(" isCrawling=");
			buffer.append(this.getJobHandler().isCrawling());
			buffer.append(" alertCount=");
			buffer.append(getAlertsCount());
			buffer.append(" newAlertCount=");
			buffer.append(getNewAlertsCount());
			if (this.getJobHandler().isCrawling())
			{
				buffer.append(" currentJob=");
				buffer.append(this.getJobHandler().getCurrentJob()
					.getJmxJobName());
			}
		}
		return buffer.toString();
	}

	// Alert methods.
	public int getAlertsCount()
	{
		return this.alertManager.getCount();
	}

	public int getNewAlertsCount()
	{
		return this.alertManager.getNewCount();
	}

	public Vector getAlerts()
	{
		return this.alertManager.getAll();
	}

	public Vector getNewAlerts()
	{
		return this.alertManager.getNewAll();
	}

	public SinkHandlerLogRecord getAlert(final String id)
	{
		return this.alertManager.get(id);
	}

	public void readAlert(final String id)
	{
		this.alertManager.read(id);
	}

	public void removeAlert(final String id)
	{
		this.alertManager.remove(id);
	}

	/**
	 * Start Heritrix.
	 * 
	 * Used by JMX and webapp initialization for starting Heritrix. Not by the
	 * cmdline launched Heritrix. Idempotent. If start is called by JMX, then
	 * new instance of Heritrix is automatically registered w/ JMX Agent. If
	 * started by webapp, need to register the new Heritrix instance.
	 */
	public void start()
	{
		// Don't start if we've been launched from the command line.
		// Don't start if already started.
		if (!Heritrix.isCommandLine() && !isStarted())
		{
			try
			{
				logger.info(launch());
			}
			catch (Exception e)
			{
				e.printStackTrace();
			}
		}
	}

	/**
	 * Stop Heritrix.
	 * 
	 * Used by JMX and webapp initialization for stopping Heritrix.
	 */
	public void stop()
	{
		if (this.jobHandler != null)
		{
			this.jobHandler.stop();
		}
	}

	public String interrupt(String threadName)
	{
		String result = "Thread " + threadName + " not found";
		ThreadGroup group = Thread.currentThread().getThreadGroup();
		if (group == null)
		{
			return result;
		}
		// Back up to the root threadgroup before starting
		// to iterate over threads.
		ThreadGroup parent = null;
		while ((parent = group.getParent()) != null)
		{
			group = parent;
		}
		// Do an array that is twice the size of active
		// thread count. That should be big enough.
		final int max = group.activeCount() * 2;
		Thread[] threads = new Thread[max];
		int threadCount = group.enumerate(threads, true);
		if (threadCount >= max)
		{
			logger.info("Some threads not found...array too small: " + max);
		}
		for (int j = 0; j < threadCount; j++)
		{
			if (threads[j].getName().equals(threadName))
			{
				threads[j].interrupt();
				result = "Interrupt sent to " + threadName;
				break;
			}
		}
		return result;
	}

	// OpenMBean implementation.

	/**
	 * Build up the MBean info for Heritrix main.
	 * 
	 * @return Return created mbean info instance.
	 */
	protected OpenMBeanInfoSupport buildMBeanInfo()
	{
		OpenMBeanAttributeInfoSupport[] attributes = new OpenMBeanAttributeInfoSupport[Heritrix.ATTRIBUTE_LIST
			.size()];
		OpenMBeanConstructorInfoSupport[] constructors = new OpenMBeanConstructorInfoSupport[1];
		OpenMBeanOperationInfoSupport[] operations = new OpenMBeanOperationInfoSupport[Heritrix.OPERATION_LIST
			.size()];
		MBeanNotificationInfo[] notifications = new MBeanNotificationInfo[0];

		// Attributes.
		attributes[0] = new OpenMBeanAttributeInfoSupport(Heritrix.STATUS_ATTR,
			"Short basic status message", SimpleType.STRING, true, false, false);
		// Attributes.
		attributes[1] = new OpenMBeanAttributeInfoSupport(
			Heritrix.VERSION_ATTR, "Heritrix version", SimpleType.STRING, true,
			false, false);

		// Constructors.
		constructors[0] = new OpenMBeanConstructorInfoSupport(
			"HeritrixOpenMBean", "Constructs Heritrix OpenMBean instance ",
			new OpenMBeanParameterInfoSupport[0]);

		// Operations.
		operations[0] = new OpenMBeanOperationInfoSupport(Heritrix.START_OPER,
			"Start Heritrix instance", null, SimpleType.VOID,
			MBeanOperationInfo.ACTION);

		operations[1] = new OpenMBeanOperationInfoSupport(Heritrix.STOP_OPER,
			"Stop Heritrix instance", null, SimpleType.VOID,
			MBeanOperationInfo.ACTION);

		OpenMBeanParameterInfo[] args = new OpenMBeanParameterInfoSupport[1];
		args[0] = new OpenMBeanParameterInfoSupport("threadName",
			"Name of thread to send interrupt", SimpleType.STR