complextype.java

这是个爬虫和lucece相结合最好了

/* ComplexType
 *
 * $Id: ComplexType.java 5028 2007-03-29 23:21:48Z gojomo $
 *
 * Created on Dec 17, 2003
 *
 * Copyright (C) 2004 Internet Archive.
 *
 * This file is part of the Heritrix web crawler (crawler.archive.org).
 *
 * Heritrix is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * any later version.
 *
 * Heritrix is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser Public License for more details.
 *
 * You should have received a copy of the GNU Lesser Public License
 * along with Heritrix; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
package org.archive.crawler.settings;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Stack;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.management.Attribute;
import javax.management.AttributeList;
import javax.management.AttributeNotFoundException;
import javax.management.DynamicMBean;
import javax.management.InvalidAttributeValueException;
import javax.management.MBeanAttributeInfo;
import javax.management.MBeanException;
import javax.management.MBeanInfo;
import javax.management.ReflectionException;

import org.apache.commons.httpclient.URIException;
import org.archive.crawler.datamodel.CandidateURI;
import org.archive.crawler.datamodel.CrawlOrder;
import org.archive.crawler.datamodel.CrawlURI;
import org.archive.crawler.settings.Constraint.FailedCheck;
import org.archive.net.UURI;

/** Superclass of all configurable modules.
 *
 * This class is in many ways the heart of the settings framework. All modules
 * that should be configurable extends this class or one of its subclasses.
 *
 * All subclasses of this class will automatically conform to the
 * JMX DynamicMBean. You could then use the {@link #getMBeanInfo()} method to
 * investigate which attributes this module supports and then use the
 * {@link #getAttribute(String)} and {@link #setAttribute(Attribute)} methods to
 * alter the attributes values.
 *
 * Because the settings framework supports per domain/host settings there is
 * also available context sensitive versions of the DynamicMBean methods.
 * If you use the non context sensitive methods, it is the global settings
 * that will be altered.
 *
 * @author John Erik Halse
 */
public abstract class ComplexType extends Type implements DynamicMBean {
    private static Logger logger =
        Logger.getLogger("org.archive.crawler.settings.ComplexType");

    private transient SettingsHandler settingsHandler;
    private transient ComplexType parent;
    private String description;
    private String absoluteName;
    protected final List<Type> definition = new ArrayList<Type>();
    protected final Map<String,Type> definitionMap = new HashMap<String,Type>();
    private boolean initialized = false;
    private String[] preservedFields = new String[0];

    /**
     * Private constructor to make sure that no one
     * instantiates this class with the empty constructor.
     */
    private ComplexType() {
        super(null, null);
    }

    /** Creates a new instance of ComplexType.
     *
     * @param name the name of the element.
     * @param description the description of the element.
     */
    public ComplexType(String name, String description) {
        super(name, null);
        this.description = description.intern();
    }

    protected void setAsOrder(SettingsHandler settingsHandler)
    throws InvalidAttributeValueException {
        this.settingsHandler = settingsHandler;
        this.absoluteName = "";
        globalSettings().addTopLevelModule((CrawlOrder) this);
        addComplexType(settingsHandler.getSettingsObject(null), this);
        this.parent = null;
    }

    /** Get the global settings object (aka order).
     *
     * @return the global settings object.
     */
    public CrawlerSettings globalSettings() {
        if (settingsHandler == null) {
            return null;
        }
        return settingsHandler.getSettingsObject(null);
    }

    public Type addElement(CrawlerSettings settings, Type type)
        throws InvalidAttributeValueException {
        getOrCreateDataContainer(settings).addElementType(type);
        if (type instanceof ComplexType) {
            addComplexType(settings, (ComplexType) type);
        }
        return type;
    }

    private ComplexType addComplexType(CrawlerSettings settings,
            ComplexType object) throws InvalidAttributeValueException {

        if (this.settingsHandler == null) {
            throw new IllegalStateException("Can't add ComplexType to 'free' ComplexType");
        }
        setupVariables(object);
        settings.addComplexType(object);
        if (!object.initialized) {
            Iterator it = object.definition.iterator();
            while (it.hasNext()) {
                Type t = (Type) it.next();
                object.addElement(settings, t);
            }
            object.earlyInitialize(settings);
        }
        object.initialized = true;

        return object;
    }

    private ComplexType replaceComplexType(CrawlerSettings settings,
            ComplexType object) throws InvalidAttributeValueException,
            AttributeNotFoundException {
        if (this.settingsHandler == null) {
            throw new IllegalStateException(
                    "Can't add ComplexType to 'free' ComplexType");
        }
        String[] preservedFields = object.getPreservedFields();

        setupVariables(object);

        DataContainer oldData = settings.getData(object);
        settings.addComplexType(object);
        DataContainer newData = settings.getData(object);

        if (!object.initialized) {
            Iterator it = object.definition.iterator();
            while (it.hasNext()) {
                Type t = (Type) it.next();

                // Check if attribute should be copied from old object.
                boolean found = false;
                if (preservedFields.length > 0) {
                    for (int i = 0; i < preservedFields.length; i++) {
                        if (preservedFields[i].equals(t.getName())) {
                            found = true;
                            break;
                        }
                    }
                }
                if (found && oldData.copyAttribute(t.getName(), newData)) {
                    if (t instanceof ComplexType) {
                        object.setupVariables((ComplexType) t);
                    }
                } else {
                    object.addElement(settings, t);
                }
            }
            object.earlyInitialize(settings);
        }
        object.initialized = true;

        return object;
    }

    /** Set a list of attribute names that the complex type should attempt to
     * preserve if the module is exchanged with an other one.
     *
     * @param preservedFields array of attributenames to preserve.
     */
    protected void setPreservedFields(String[] preservedFields) {
        this.preservedFields = preservedFields;
    }

    /** Get a list of attribute names that the complex type should attempt to
     * preserve if the module is exchanged with an other one.
     *
     * @return an array of attributenames to preserve.
     */
    protected String[] getPreservedFields() {
        return this.preservedFields;
    }

    /** Get the active data container for this ComplexType for a specific
     * settings object.
     *
     * If no value has been overridden on the settings object for this
     * ComplexType, then it traverses up until it find a DataContainer with
     * values for this ComplexType.
     *
     * This method should probably not be called from user code. It is a helper
     * method for the settings framework.
     *
     * @param context Context from which we get settings.
     * @return the active DataContainer.
     */
    protected DataContainer getDataContainerRecursive(Context context) {
        if (context.settings == null) {
            return null;
        }
        DataContainer data = context.settings.getData(this);
        if (data == null && context.settings.getParent(context.uri) != null) {
            context.settings = context.settings.getParent(context.uri);
            data = getDataContainerRecursive(context);
        }
        return data;
    }

    /** Get the active data container for this ComplexType for a specific
     * settings object.
     *
     * If the key has not been overridden on the settings object for this
     * ComplexType, then it traverses up until it find a DataContainer with
     * the key for this ComplexType.
     *
     * This method should probably not be called from user code. It is a helper
     * method for the settings framework.
     *
     * @param context the settings object for which the {@link DataContainer}
     *                 is active.
     * @param key the key to look for.
     * @return the active DataContainer.
     * @throws AttributeNotFoundException
     */
    protected DataContainer getDataContainerRecursive(Context context,
            String key) throws AttributeNotFoundException {
        Context c = new Context(context.settings, context.uri);
        DataContainer data = getDataContainerRecursive(c);
        while (data != null) {
            if (data.containsKey(key)) {
                return data;
            }
            c.settings = data.getSettings().getParent(c.uri);
            data = getDataContainerRecursive(c);
        }
        throw new AttributeNotFoundException(key);
    }

    /** Sets up some variables for a new complex type.
     *
     * The complex type is set up to be an attribute of
     * this complex type.
     *
     * @param object to be set up.
     */
    private void setupVariables(ComplexType object) {
        object.parent = this;
        object.settingsHandler = getSettingsHandler();
        object.absoluteName =
            (getAbsoluteName() + '/' + object.getName()).intern();
    }

    public SettingsHandler getSettingsHandler() {
        return settingsHandler;
    }

    /** Get the absolute name of this ComplexType.
     *
     * The absolute name is like a file path with the name of the element
     * prepended by all the parents names separated by slashes.
     * @return Absolute name.
     */
    public String getAbsoluteName() {
        return absoluteName;
    }

    /**
     * Get settings object valid for a URI.
     * <p/>
     * This method takes an object,
     * try to convert it into a {@link CrawlURI} and then tries to get the
     * settings object from it. If this fails, then the global settings object
     * is returned.
     * <p/>
     * If the requested attribute is not set on this settings
     * object it tries its parent until it gets a settings object where this
     * attribute is set is found. If nothing is found, global settings is
     * returned.
     *
     * @param o possible {@link CrawlURI}.
     * @param attributeName the attribute that should have a value set on the
     *            returned settings object.
     * @return the settings object valid for the URI.
     */
    Context getSettingsFromObject(Object o, String attributeName) {
        Context context;
        if (o == null) {
            context = null;
        } else if (o instanceof Context) {
            context = (Context) o;
        } else if (o instanceof CrawlerSettings) {
            context = new Context((CrawlerSettings) o, null);
        } else if (o instanceof UURI || o instanceof CandidateURI) {
            // Try to get settings for URI that has no references to a
            // CrawlServer [SIC - CrawlURI may have CrawlServer -gjm]
            context = new Context();
            context.uri = (o instanceof CandidateURI)?
                ((CandidateURI) o).getUURI(): (UURI)o;
            try {
               context.settings = getSettingsHandler().
                   getSettings(context.uri.getReferencedHost(),
                       context.uri);
            }
            catch (URIException e1) {
                logger.severe("Failed to get host");
            }

            if (attributeName != null) {
                try {
                    context.settings =
                        getDataContainerRecursive(context, attributeName).
                            getSettings();
                } catch (AttributeNotFoundException e) {
                    // Nothing found, globals will be used
                }
            }
        } else {
            logger.warning("Unknown object type: " +
                o.getClass().getName());
            context = null;
        }

        // if settings could not be resolved use globals.
        if (context == null) {
            context = new Context(globalSettings(), null);
        }
        return context;
    }

    /** Get settings object valid for a URI.
    *
    * This method takes an object, try to convert it into a {@link CrawlURI}
    * and then tries to get the settings object from it. If this fails, then
    * the global settings object is returned.
    *
    * @param o possible {@link CrawlURI}.
    * @return the settings object valid for the URI.
    */
    Context getSettingsFromObject(Object o) {
        return getSettingsFromObject(o, null);
    }

    /** Returns true if an element is overridden for this settings object.
     *
     * @param settings the settings object to investigate.