clinearfafeaturecalculator.h

强化学习算法（R-Learning）难得的珍贵资料

class CLinearMultiFeatureCalculator : public CGridFeatureCalculator
{
protected:
	unsigned int *areaSize;
	unsigned int areaNumPart;

	CMyVector *activePosition;
	CMyVector *featurePosition;
	unsigned int *actualPartition;
	unsigned int *singleStateFeatures;

/// Interface function for returning the feature activation factor
	/** 
	As input the method gets the current model state and the position of the feature as vector.
	*/
	virtual rlt_real getFeatureFactor(CState *state, CMyVector *featPos) = 0;
	/// Interface function for setting the areaSize array
	/** 
	Function has to set the number of active features in each direction. o a value of 1 for dimension 0 means that there are 3 active features for dimension 0 (the current feature itself and it's left and right neighbors).
	Has to be called by the subclass's constructor and has to call the calcNumActiveFeatures method.
	*/
	virtual void initAreaSize();

	/// calculates the number of active feature and stores it in areaNumPart
	/** 
	Has to be called by initAreaSize. Uses the areaSize array to calculate the number of active features. The number of active features is 2 * areaSize[i] + 1 for each dimension i.
	*/
	virtual void calcNumActiveFeatures();
public:
/// For details about creating the grid see the super class.
	CLinearMultiFeatureCalculator(unsigned int numDim, unsigned int dimensions[], unsigned int partitions[], rlt_real offsets[], unsigned int numFeatures);
	~CLinearMultiFeatureCalculator();
	
	/// Calculates the feature state with multiple active features.
	/**
	The class mantains an additional array which mantains the number of active features in each direction, so a value of 1 for dimension 0 means that there are 3 active features for dimension 0 (the current feature itself and it's left and right neighbors) . This array has to be set by the subclasses with the function initAreaSize (which has also to be called by the subclasses in the constructor). For each feature that is in the "active" area, the feature factor is calculated by the function getFeatureFactor, which also has to be implemented by the subclasses. For periodic states the active area is mirrored into the state intervall again if it leaves the intervall, for non-periodic states the active area is cut off. At the end all feature factors get normalized (sum = 1.0)*/
	virtual void getModifiedState(CStateCollection *state, CState *featState);
};

/// Represents a normalized RBF network
/** 
This class lays a grid of RBF-Functions over a specified sub-space of the model space.  You can specify which dimensions you want to use (dimensions array), how much partitions you need for each dimension, an offset for each dimension which is added to the RBF centers and the sigmas for each dimension. When talking about grids, the RBF center is always in the center of the partition. The size of the feature space is the product of all numbers of partitions. 
\par
This class always works with the normalized continuous state variable values (scaled to the intervall [0, 1], so consider that when choosing your offsets and sigmas.
\par
The number of active states for each dimension is choosen according the sigma value. All features are active that are within a range of 2 * sigma. So if you want only the neighbors to be active too, use the thumbrule sigma[i] = 1 / (2 * numPartitions[i] ), so you get 3 active features per dimension.
*/
class CRBFFeatureCalculator : public CLinearMultiFeatureCalculator
{
protected:
	rlt_real *sigma;
	rlt_real sigmaMaxSize;
	
	/// Calculates the feature factor with the RBF function
	virtual rlt_real getFeatureFactor(CState *state, CMyVector *featPos);
	/// set the areaSize array
	/** 
	The number of active states for each dimension is choosen according the sigma value. All features are active that are within a range of 2 * sigma. So if you want only the neighbors to be active too, use the thumbrule sigma[i] = 1 / (2 * numPartitions[i] ), so you get 3 active features per dimension.
	*/
	virtual void initAreaSize();

public:
	CRBFFeatureCalculator(unsigned int numDim, unsigned int dimensions[], unsigned int partitions[], rlt_real offsets[], rlt_real sigma[]);
	CRBFFeatureCalculator(unsigned int numDim, unsigned int dimensions[], unsigned int partitions[], rlt_real offsets[], rlt_real sigma[], unsigned int areaSize[]);

	~CRBFFeatureCalculator();

	/// Doesn't work, use numeric derivations instead.
	virtual void getFeatureDerivationX(int feature, CStateCollection *state, CMyVector *targetVector);
};

class CLinearInterpolationFeatureCalculator : public CLinearMultiFeatureCalculator
{
	virtual rlt_real getFeatureFactor(CState *state, CMyVector *featPos);
	
	virtual void initAreaSize();

public:
	CLinearInterpolationFeatureCalculator(unsigned int numDim, unsigned int dimensions[], unsigned int partitions[], rlt_real offsets[]);
	~CLinearInterpolationFeatureCalculator();

};

/// Superclass for calculating features from a single continuous state variable
/** 
This class allows you, similiar to CSingleStateDiscretizer to calculate your features from a single continuous state variable. Therefore a 1-dimensional grid is layed over the specified continous state variable. How the grid is used is specified by the subclasses (e.g. RBF-Centers). For the current partition of the grid and the neighbors (as much as numActiveFeatures set in the constructor) the feature factor is calculated by the interface function getFeatureFactor. For the current feature and the neighbors calculation periodic state variables get also considered.

At the end the features get normalized (factor sum = 1).
\par
Use CFeatureOperatorOr to combine the feature states comming from different continuous state variables.
*/
class CSingleStateFeatureCalculator : public CFeatureCalculator
{
protected:
	int dimension;
	int numPartitions;
	rlt_real *partitions;

	/// Interface function for calculating the feature activation factor
	/** 
	Gets the number of the partition (= featureindex), the value of the continuous state variable and the distance to the feature center to the current state varaible value as input. In this distance periodic states are already considered.
	*/
	virtual rlt_real getFeatureFactor(int partition, rlt_real contState, rlt_real difference) = 0;
public:
	/// Creates the feature calculator
	/** 
	You have to set the dimension of the modelstate you want to calculate your features from, the size of your grid, the grid itself as a rlt_real array and how much features should be active. If you set numActiveFeatures for example to 3, the current feature (feature nearest to the current state) and its left and right neighbor are active.
	*/
	CSingleStateFeatureCalculator(int dimension, int numPartitions, rlt_real *partitions, int numActiveFeatures);
	virtual ~CSingleStateFeatureCalculator();

	/// Calculates the feature state from one continupus state variable
	/**For the current partition of the grid and the neighbors (as much as numActiveFeatures set in the constructor) the feature factor is calculated by the interface function getFeatureFactor. At the end the features get normalized (factor sum = 1).*/
	virtual void getModifiedState(CStateCollection *state, CState *featState);
};

/// Represents a 1-D RBF-network layed over a specified continous state variable.
/** 
You can specifiy the centers of the RBF-Functions (partitions array) and the sigma of each RBF-Function. The RBF-centers have to be in an ascending order. If you don't specify any sigma, the sigma gets automatically calculated so that the distance between two neighbored features is equal 2 * sigma. 
The rest of the parameters are the same as for the superclass.
*/
class CSingleStateRBFFeatureCalculator : public CSingleStateFeatureCalculator
{
protected:
	rlt_real *sigma;
	
	/// Calculates the feature factor according the RBF formular
	virtual rlt_real getFeatureFactor(int partition, rlt_real contState, rlt_real difference);
public:
	CSingleStateRBFFeatureCalculator(int dimension, int numPartitions, rlt_real *partitions, int numActiveFeatures, rlt_real *sigma);
	CSingleStateRBFFeatureCalculator(int dimension, int numPartitions, rlt_real *partitions, int numActiveFeatures);
	
	void setSigma(int index, rlt_real sigma);
	rlt_real getSigma(int index);

	/// Not working, not used
	virtual void getFeatureDerivationX(int feature, CStateCollection *state, CMyVector *targetVector);

};


/// Class for linear interpolation features of a single continuous state variable
/** 
This class represent a linear interpolator for a single continuous state variable. You can specify the centers of the features and which dimension of the modelstate you want to use. There are always 2 features active, the nearest feature to the left and to the right. The feature factors are calculated per linear interpolation, so if dist is the distance of 2 neighbored features and dist_left is the distance to the left feature from the current state x, the feature factor of the left feature is 1 - dist_left / dist and the factor of the right feature is naturally dist_left/dist. 
*/

class CSingleStateLinearInterpolationFeatureCalculator : public CFeatureCalculator
{
protected:
	int dimension;
	int numPartitions;
	rlt_real *partitions;

public:
	CSingleStateLinearInterpolationFeatureCalculator(int dimension, int numPartitions, rlt_real *partitions);
	~CSingleStateLinearInterpolationFeatureCalculator();
	
	/// Calculates the feature factors of the 2 nearest features to the current state with linear interpolation
	/**There are always 2 features active, the nearest feature to the left and to the right. The feature factors are calculated per linear interpolation, so if dist is the distance of 2 neighbored features and dist_left is the distance to the left feature from the current state x, the feature factor of the left feature is 1 - dist_left / dist and the factor of the right feature is naturally dist_left/dist. */
	virtual void getModifiedState(CStateCollection *state, CState *featState);

	/// Not working, not used
	virtual void getFeatureDerivationX(int feature, CStateCollection *state, CMyVector *targetVector);

};

class CFeatureStateNNInput : public CStateModifier
{
protected:
	CFeatureCalculator *featureStateCalc;
	CState *featureState;
public:
	CFeatureStateNNInput(CFeatureCalculator *featureStateCalc);
	~CFeatureStateNNInput();


	virtual void getModifiedState(CStateCollection *state, CState *featState);
};

#endif