complextype.java

这是个爬虫和lucece相结合最好了

     * @param name the name of the element to check.
     * @return true if element is overridden for this settings object, false
     *              if not set here or is first defined here.
     * @throws AttributeNotFoundException if element doesn't exist.
     */
    public boolean isOverridden(CrawlerSettings settings, String name)
            throws AttributeNotFoundException {
        settings = settings == null ? globalSettings() : settings;
        DataContainer data = settings.getData(this);
        if (data == null || !data.containsKey(name)) {
            return false;
        }

        // Try to find attribute, will throw an exception if not found.
        Context context = new Context(settings.getParent(), null);
        getDataContainerRecursive(context, name);
        return true;
    }

    /** Obtain the value of a specific attribute from the crawl order.
     *
     * If the attribute doesn't exist in the crawl order, the default
     * value will be returned.
     *
     * @param name the name of the attribute to be retrieved.
     * @return The value of the attribute retrieved.
     * @throws AttributeNotFoundException
     * @throws MBeanException
     * @throws ReflectionException
     */
    public Object getAttribute(String name)
        throws AttributeNotFoundException, MBeanException, ReflectionException {
        return getAttribute(null, name);
    }

    /** Obtain the value of a specific attribute that is valid for a
     * specific CrawlURI.
     *
     * This method will try to get the attribute from the host settings
     * valid for the CrawlURI. If it is not found it will traverse the
     * settings up to the order and as a last resort deliver the default
     * value. This is also the case if the CrawlURI is null or if the CrawlURI
     * hasn't been assigned a CrawlServer.
     *
     * @param name the name of the attribute to be retrieved.
     * @param uri the CrawlURI that this attribute should be valid for.
     * @return The value of the attribute retrieved.
     * @see #getAttribute(Object settings, String name)
     * @throws AttributeNotFoundException
     */
    public Object getAttribute(String name, CrawlURI uri)
        throws AttributeNotFoundException {
        return getAttribute(uri, name);
    }
    
    /**
     * Obtain the value of a specific attribute that is valid for a specific
     * CrawlerSettings object.<p>
     * 
     * This method will first try to get a settings object from the supplied
     * context, then try to look up the attribute from this settings object. If
     * it is not found it will traverse the settings up to the order and as a
     * last resort deliver the default value.
     * 
     * @param context the object to get the settings from.
     * @param name the name of the attribute to be retrieved.
     * @return The value of the attribute retrieved.
     * @see CrawlerSettings
     * @throws AttributeNotFoundException
     */
    public Object getAttribute(Object context, String name)
        throws AttributeNotFoundException {
        Context ctxt = getSettingsFromObject(context);

        // If settings is not set, return the default value
        if (ctxt.settings == null) {
            try {
                return ((Type) definitionMap.get(name)).getDefaultValue();
            } catch (NullPointerException e) {
                throw new AttributeNotFoundException(
                        "Could not find attribute: " + name);
            }
        }

        return getDataContainerRecursive(ctxt, name).get(name);
    }

    /**
     * Obtain the value of a specific attribute that is valid for a specific
     * CrawlerSettings object.
     * <p>
     * 
     * This method will first try to get a settings object from the supplied
     * context, then try to look up the attribute from this settings object. If
     * it is not found it will traverse the settings up to the order and as a
     * last resort deliver the default value.
     * <p>
     * 
     * The only difference from the {@link #getAttribute(Object, String)}is
     * that this method doesn't throw any checked exceptions. If an undefined
     * attribute is requested from a ComplexType, it is concidered a bug and a
     * runtime exception is thrown instead.
     * 
     * @param context the object to get the settings from.
     * @param name the name of the attribute to be retrieved.
     * @return The value of the attribute retrieved.
     * @see #getAttribute(Object, String)
     * @see CrawlerSettings
     * @throws IllegalArgumentException
     */
    public Object getUncheckedAttribute(Object context, String name) {
        try {
            return getAttribute(context, name);
        } catch (AttributeNotFoundException e) {
            throw new IllegalArgumentException("Was passed '" + name +
                "' and got this exception: " + e);
        }
    }

    /** Obtain the value of a specific attribute that is valid for a
     * specific CrawlerSettings object.
     *
     * This method will try to get the attribute from the supplied host
     * settings object. If it is not found it will return <code>null</code>
     * and not try to investigate the hierarchy of settings.
     *
     * @param settings the CrawlerSettings object to search for this attribute.
     * @param name the name of the attribute to be retrieved.
     * @return The value of the attribute retrieved or null if its not set.
     * @see CrawlerSettings
     * @throws AttributeNotFoundException is thrown if the attribute doesn't
     *         exist.
     */
    public Object getLocalAttribute(CrawlerSettings settings, String name)
            throws AttributeNotFoundException {

        settings = settings == null ? globalSettings() : settings;

        DataContainer data = settings.getData(this);
        if (data != null && data.containsKey(name)) {
            // Attribute was found return it.
            return data.get(name);
        }
        // Try to find the attribute, will throw an exception if not found.
        Context context = new Context(settings, null);
        getDataContainerRecursive(context, name);
        return null;
    }

    /** Set the value of a specific attribute of the ComplexType.
     *
     * This method sets the specific attribute for the order file.
     *
     * @param attribute The identification of the attribute to be set and the
     *                  value it is to be set to.
     * @throws AttributeNotFoundException is thrown if there is no attribute
     *         with this name.
     * @throws InvalidAttributeValueException is thrown if the attribute is of
     *         wrong type and cannot be converted to the right type.
     * @throws MBeanException this is to conform to the MBean specification, but
     *         this exception is never thrown, though this might change in the
     *         future.
     * @throws ReflectionException this is to conform to the MBean specification, but
     *         this exception is never thrown, though this might change in the
     *         future.
     * @see javax.management.DynamicMBean#setAttribute(javax.management.Attribute)
     */
    public synchronized final void setAttribute(Attribute attribute)
        throws
            AttributeNotFoundException,
            InvalidAttributeValueException,
            MBeanException,
            ReflectionException {
        setAttribute(settingsHandler.getSettingsObject(null), attribute);
    }

    /** Set the value of a specific attribute of the ComplexType.
     *
     * This method is an extension to the Dynamic MBean specification so that
     * it is possible to set the value for a CrawlerSettings object other than
     * the settings object representing the order.
     *
     * @param settings the settings object for which this attributes value is valid
     * @param attribute The identification of the attribute to be set and the
     *                  value it is to be set to.
     * @throws AttributeNotFoundException is thrown if there is no attribute
     *         with this name.
     * @throws InvalidAttributeValueException is thrown if the attribute is of
     *         wrong type and cannot be converted to the right type.
     * @see javax.management.DynamicMBean#setAttribute(javax.management.Attribute)
     */
    public synchronized final void setAttribute(CrawlerSettings settings,
            Attribute attribute) throws InvalidAttributeValueException,
            AttributeNotFoundException {

        if(settings==null){
            settings = globalSettings();
        }

        DataContainer data = getOrCreateDataContainer(settings);
        Object value = attribute.getValue();

        ModuleAttributeInfo attrInfo = (ModuleAttributeInfo) getAttributeInfo(
                settings.getParent(), attribute.getName());

        ModuleAttributeInfo localAttrInfo = (ModuleAttributeInfo) data
                .getAttributeInfo(attribute.getName());

        // Check if attribute exists
        if (attrInfo == null && localAttrInfo == null) {
            throw new AttributeNotFoundException(attribute.getName());
        }

        // Check if we are overriding and if that is allowed for this attribute
        if (localAttrInfo == null) {
            if (!attrInfo.isOverrideable()) {
                throw new InvalidAttributeValueException(
                        "Attribute not overrideable: " + attribute.getName());
            }
            localAttrInfo = new ModuleAttributeInfo(attrInfo);
        }

        // Check if value is of correct type. If not, see if it is
        // a string and try to turn it into right type
        Class typeClass = getDefinition(attribute.getName()).getLegalValueType();
        if (!(typeClass.isInstance(value)) && value instanceof String) {
            try {
                value = SettingsHandler.StringToType((String) value,
                        SettingsHandler.getTypeName(typeClass.getName()));
            } catch (ClassCastException e) {
                throw new InvalidAttributeValueException(
                        "Unable to decode string '" + value + "' into type '"
                                + typeClass.getName() + "'");
            }
        }

        // Check if the attribute value is legal
        FailedCheck error = checkValue(settings, attribute.getName(), value);
        if (error != null) {
            if (error.getLevel() == Level.SEVERE) {
                throw new InvalidAttributeValueException(error.getMessage());
            } else if (error.getLevel() == Level.WARNING) {
                if (!getSettingsHandler().fireValueErrorHandlers(error)) {
                    throw new InvalidAttributeValueException(error.getMessage());
                }
            } else {
                getSettingsHandler().fireValueErrorHandlers(error);
            }
        }

        // Everything ok, set it
        localAttrInfo.setType(value);
        Object oldValue = data.put(attribute.getName(), localAttrInfo, value);

        // If the attribute is a complex type other than the old value,
        // make sure that all sub attributes are correctly set
        if (value instanceof ComplexType && value != oldValue) {
            ComplexType complex = (ComplexType) value;
            replaceComplexType(settings, complex);
        }
    }

    /**
     * Get the content type definition for an attribute.
     *
     * @param attributeName the name of the attribute to get definition for.
     * @return the content type definition for the attribute.
     */
    Type getDefinition(String attributeName) {
        return (Type) definitionMap.get(attributeName);
    }

    /**
     * Check an attribute to see if it fulfills all the constraints set on the
     * definition of this attribute.
     *
     * @param settings the CrawlerSettings object for which this check was
     *            executed.
     * @param attributeName the name of the attribute to check.
     * @param value the value to check.
     * @return null if everything is ok, otherwise it returns a FailedCheck
     *         object with detailed information of what went wrong.
     */
    public FailedCheck checkValue(CrawlerSettings settings,
            String attributeName, Object value) {
        return checkValue(settings, attributeName,
                getDefinition(attributeName), value);
    }

    FailedCheck checkValue(CrawlerSettings settings, String attributeName,
            Type definition, Object value) {
        FailedCheck res = null;

        // Check if value fulfills any constraints
        List constraints = definition.getConstraints();
        if (constraints != null) {
            for (Iterator it = constraints.iterator(); it.hasNext()
                    && res == null;) {
                res = ((Constraint) it.next()).check(settings, this,
                        definition, value);
            }
        }

        return res;
    }

    /** Unset an attribute on a per host level.
     *
     * This methods removes an override on a per host or per domain level.
     *
     * @param settings the settings object for which the attribute should be
     *        unset.
     * @param name the name of the attribute.
     * @return The removed attribute or null if nothing was removed.
     * @throws AttributeNotFoundException is thrown if the attribute name
     *         doesn't exist.
     */
    public Object unsetAttribute(CrawlerSettings settings, String name)
            throws AttributeNotFoundException {

        if (settings == globalSettings()) {
            throw new IllegalArgumentException(
                "Not allowed to unset attributes in Crawl Order.");
        }

        DataContainer data = settings.getData(this);
        if (data != null && data.containsKey(name)) {
            // Remove value
            return data.removeElement(name);
        }

        // Value not found. Check if we should return null or throw an exception
        // This method throws an exception if not found.
        Context context = new Context(settings, null);
        getDataContainerRecursive(context, name);
        return null;
    }

    private DataContainer getOrCreateDataContainer(CrawlerSettings settings)
        throws InvalidAttributeValueException {

        // Get this ComplexType's data container for the submitted settings
        DataContainer data = settings.getData(this);

        // If there isn't a container, create one
        if (data == null) {
            ComplexType parent = getParent();
            if (parent == null) {
                settings.addTopLevelModule((ModuleType) this);
            } else {
                DataContainer parentData =
                    settings.getData(parent);
                if (parentData == null) {
                    if (this instanceof ModuleType) {
                        settings.addTopLevelModule((ModuleType) this);
                    } else {
                        settings.addTopLevelModule((ModuleType) parent);
                        try {
                            parent.setAttribute(settings, this);
                        } catch (AttributeNotFoundException e) {
                            logger.severe(e.getMessage());
                        }
                    }
                } else {
                    globalSettings().getData(parent).copyAttributeInfo(
                        getName(),
                        parentData);
                }
            }

            // Create fresh DataContainer
            data = settings.addComplexType(this);
        }

        // Make sure that the DataContainer references right type
        if (data.getComplexType() != this) {