clinearfafeaturecalculator.h

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#ifndef CLINEARFAFEATURE_H
#define  CLINEARFAFEATURE_H



#include "cstate.h"
#include "cstatemodifier.h"
#include "ril_debug.h"

/// Combines 2 or more feature states with an "Or" operatation.
/** 
The "or" operator gives you the possibility to use different feature states simultanously that are independent. You can add several feature calculators, all calculated features from the different modifiers can now be active simultanously in the same feature state. It is understood that you have to distuingish between the different features from the different feature states, so, to get a unique feature index again, the sum off all feature state sizes from the previously added feature states is added to the feature index of a particular feature state. So our feature state size is the sum off all feature state sizes, and the number of active features is naturally the sum of all number of active features.\par
The feature operator or is used to combine 2 or more (more or less) indepedent states, for example tilings or RBF networks with different offsets/resolutions.
\par
You always have to init the feature operator with the function initFeatureOperator after ALL feature states have been added and before you add the feature operator to the agent's modifier list.*/
class CFeatureOperatorOr : public CStateMultiModifier, public CFeatureCalculator
{
protected:
	std::map<CStateModifier *, rlt_real> *featureFactors;
public:
	///Creates empty operator
	CFeatureOperatorOr();
	virtual ~CFeatureOperatorOr();

/// Calculates the feature state
	/** 
	The "or" operator gives you the possibility to use different feature states simultanously. You can add several feature calculators, all calculated features from the different modifiers can now be active simultanously in the same feature state. It is understood that you have to distuingish between the different features from the different feature states, so, to get a unique feature index again, the sum off all feature state sizes from the previously added feature states is added to the feature index of a particular feature state. So our feature state size is the sum off all feature state sizes, and the number of active features is naturally the sum of all number of active features.
	*/
	virtual void getModifiedState(CStateCollection *state, CState *modifiedState);
	/// Adds a feature state to the modifier
	/** 
	The state modifier HAS to be a feature calculator or a discretizer. After the operator has been initialized, no feature calculators can be added any more.
	*/
	virtual void addStateModifier(CStateModifier *featCalc, rlt_real factor = 1.0);

	/// Inits the feature operator
	/** 
You always have to init the feature operator after ALL feature states have been added and before you use the feature operator (e.g. add the feature operator to the agent's modifier list).	
	*/
	virtual void initFeatureOperator();

	/// returns the feature calculator for a given feature index
	CStateModifier *getStateModifier(int feature);

	/// Doesn't work, not used, use numeric derivations instead.
	void getFeatureDerivationX(int feature, CStateCollection *state, CMyVector *targetVector);
};

/// Combines 2 or more feature states with an "And" operatation.
/** 
The "and" operator gives you the possibility to use different feature states simulatanously that are dependent. This class works primary like the discrete operator and class, it combines several dependent feature state to one. The feature state size of the operator is the product off al feature state sizes, the number of active states is the product of all numbers of active states. Each active feature in the operator is assigned to different active states from the feature states (there is one active feature in the operator for each possibility of choosing one active feature from each feature calculator). A new feature index is calculated according to the feature indices of the assigned active features, the feature factor is the product of all feature factors.
\par
The feature operator and is used to combine 2 or more depedent states, for example single state RBF networks (CSingleStateRBFFeatureCalculator).
\par
You always have to init the feature operator with the function initFeatureOperator after ALL feature states have been added and before you add the feature operator to the agent's modifier list.*/
class CFeatureOperatorAnd : public CStateMultiModifier, public CFeatureCalculator
{
protected:
public:
	CFeatureOperatorAnd();

/// Calculates the feature state
/**The "and" operator gives you the possibility to use different feature states simulatanously that are dependent. This class works primary like the discrete operator and class, it combines several dependent feature state to one. The feature state size of the operator is the product off al feature state sizes, the number of active states is the product of all numbers of active states. Each active feature in the operator is assigned to different active states from the feature states (there is one active feature in the operator for each possibility of choosing one active feature from each feature calculator). A new feature index is calculated according to the feature indices of the assigned active features, the feature factor is the product of all feature factors.*/
	virtual void getModifiedState(CStateCollection *state, CState *modifiedState);
	
	/// Adds a feature state to the modifier
	/** 
	The state modifier HAS to be a feature calculator or a discretizer. After the operator has been initialized, no feature calculators can be added any more.
	*/
	virtual void addStateModifier(CStateModifier *featCalc);

	/// Inits the feature operator
	/** 
	You always have to init the feature operator after ALL feature states have been added and before you use the feature operator (e.g. add the feature operator to the agent's modifier list).	
	*/
	virtual void initFeatureOperator();

	/// Doesn't work, not used, use numeric derivations instead.
	void getFeatureDerivationX(int feature, CStateCollection *state, CMyVector *targetVector);
};


/// Abstract Superclass for all feature calculator that partition the state space with a grid.
/** 
This class lays a grid over a specified sub-space of the model space. How this grid is used (e.g. Tiling or RBF network) is specified by the sub-classes. You can specify which dimensions you want to use (dimensions array), how much partitions you need for each dimension and also an offset for each dimension which is added to the grid centers. The size of the feature space is the product of all numbers of partitions.
\par
The class provides methods for retrieving the position of a feature (getFeaturePosition), retrieving the active feature (feature nearest to the current state) (getActiveFeature), and retrieving the current partitions of all dimensions (getSingleActiveFeature).
\par
This class always works with the normalized continuous state variable values (scaled to the intervall [0, 1], so consider that when choosing your offsets. 
*/
class CGridFeatureCalculator : public CFeatureCalculator
{
protected:
	unsigned int *partitions;
	unsigned int *dimensions;
	rlt_real *offsets;
	
	unsigned int numDim;

	unsigned int *dimensionSize;

	rlt_real *gridScale;

public:
	
	/** 
	This class lays a grid over a specified sub-space of the model space. How this grid is used (e.g. Tiling or RBF network) is specified by the sub-classes. You can specify which dimensions you want to use, how much partitions you need for each dimension and also an offset for each dimension which is added to the grid centers. The first parameter is the number of dimensions you want to use so its the size of your arrays. The size of the feature space is the product of all numbers of partitions.
	*/
	CGridFeatureCalculator(unsigned int numDim, unsigned int dimensions[], unsigned int partitions[], rlt_real offsets[], unsigned int numActiveFeatures);
	~CGridFeatureCalculator();

	void setGridScale(int dimension, rlt_real scale);

	/// Returns the number of dimensions you use
	int unsigned getNumDimensions();

	/// returns the feature index of the given position array (active partitions of the single state variables)
	int unsigned getFeatureIndex(int position[]);

	/// Stores the position of the specified feature in the position vector
	virtual void getFeaturePosition(unsigned int feature, CMyVector *position);
	
	/// Returns the number of the nearest feature to the current state
	virtual unsigned int getActiveFeature(CState *state);
	
	/// retrievs the current partitions of all dimensions
	virtual void getSingleActiveFeature(CState *state, unsigned int *activeFeature);
	
	/// Interface method for subclasses
	virtual void getModifiedState(CStateCollection *state, CState *featState) = 0;
};


/// Tilings represent a grid layed over the state space
/** 
The grid is specified the same way as it is in the superclass. A tiling always has just one active state (factor = 1.0), which is the feature containing the current model state. So a single tiling is exactly seen a discrete state. Use CFeatureOperatorOr to combine more tiling objects.
*/
class CTilingFeatureCalculator : public CGridFeatureCalculator
{

public:
	CTilingFeatureCalculator(unsigned int numDim, unsigned int dimensions[], unsigned int partitions[], rlt_real offsets[]);
	~CTilingFeatureCalculator();
	
	/// Returns the active feature
	virtual void getModifiedState(CStateCollection *state, CState *featState);
};

/// Super class for all feature calculators that use a grid and where more than one feature can be active.
/** 
If we use CLinearMultiFeatureCalculator more than one feature can be active in the grid. The class mantains an additional array which mantains the number of active features in each direction, so a value of 1 for dimension 0 means that there are 3 active features for dimension 0 (the current feature itself and it's left and right neighbors) . This array has to be set by the subclasses with the function initAreaSize (which has also to be called by the subclasses in the constructor). For each feature that is in the "active" area, the feature factor is calculated by the function getFeatureFactor, which also has to be implemented by the subclasses.

*/