ccontinuoustime.h

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#ifndef C_CONTINUOUSTIME
#define C_CONTINUOUSTIME

#include "cstate.h"
#include "cpolicies.h"
#include "ctransitionfunction.h"
#include "cvfunction.h"
#include "ccontinuousactiongradientpolicy.h"
#include "ril_debug.h"

class CContinuousTimeParameters
{
public:
	static rlt_real getGammaFromSgamma(rlt_real sgamma, rlt_real dt);
	static rlt_real getLambdaFromKappa(rlt_real kappa, rlt_real sgamma, rlt_real dt);
};

class CContinuousTimeVMPolicy : public CQStochasticPolicy
{
protected:
	CVFunctionInputDerivationCalculator *vfunction;
	CDynamicContinuousTimeModel *model;
public:

	CContinuousTimeVMPolicy(CActionSet *actions, CActionDistribution *distribution, CVFunctionInputDerivationCalculator *vFunction, CDynamicContinuousTimeModel *model, CRewardFunction *rewardFunction);
	~CContinuousTimeVMPolicy();

	CContinuousTimeQFunctionFromTransitionFunction *getQFunctionFromTransitionFunction();

};

class CContinuousTimeAndActionVMPolicy : public CContinuousActionController
{
protected:
	CVFunctionInputDerivationCalculator *dVFunction;
	CTransitionFunction *model;

	CMyVector *actionValues;
	CMyVector *derivationX;
	CMyMatrix *derivationU;

	virtual void getActionValues(CMyVector *actionValues, CMyVector *noise) = 0;
public:
	CContinuousTimeAndActionVMPolicy(CContinuousAction *action, CVFunctionInputDerivationCalculator *dVFunction, CTransitionFunction *model);
	~CContinuousTimeAndActionVMPolicy();

	virtual void getNextContinuousAction(CStateCollection *state, CContinuousActionData *contAction);

	
};

class CContinuousTimeAndActionSigmoidVMPolicy : public CContinuousTimeAndActionVMPolicy
{
protected:

	CMyVector *c;

	void getActionValues(CMyVector *actionValues, CMyVector *noise);

public:
	CContinuousTimeAndActionSigmoidVMPolicy(CContinuousAction *action, CVFunctionInputDerivationCalculator *vfunction, CTransitionFunction *model);
	~CContinuousTimeAndActionSigmoidVMPolicy();

	void setC(int index, rlt_real value);
	rlt_real getC(int index);

	CMyVector *getC() {return c;};

	virtual void getNoise(CStateCollection *state, CContinuousActionData *action, CContinuousActionData *noise);
};

class CContinuousTimeAndActionSigmoidVMGradientPolicy : public CContinuousActionGradientPolicy
{
protected:
	CGradientVFunction *vFunction;
	CStateCollectionImpl *derivationState;

	CFeatureList *gradient1;
	CFeatureList *gradient2;

	virtual void updateWeights(CFeatureList *dParams);

	
	CVFunctionInputDerivationCalculator *dVFunction;
	CTransitionFunction *model;
	CMyVector *actionValues;
	CMyVector *derivationX;
	CMyMatrix *derivationU;

	CMyVector *c;

	void getActionValues(CMyVector *actionValues, CMyVector *noise);
	virtual void getGradientActionValues(CMyVector *actionValues, CMyVector *noise) {};




public:
	CContinuousTimeAndActionSigmoidVMGradientPolicy(CContinuousAction *action, CGradientVFunction *gradVFunction, CVFunctionInputDerivationCalculator *vfunction, CTransitionFunction *model, std::list<CStateModifier *> *modifiers);
	virtual ~CContinuousTimeAndActionSigmoidVMGradientPolicy();

	virtual int getNumWeights();

	virtual void getWeights(rlt_real *parameters);
	virtual void setWeights(rlt_real *parameters);

	virtual void getGradient(CStateCollection *inputState, int outputDimension, CFeatureList *gradientFeatures);

	virtual void resetData();

	virtual void getNextContinuousAction(CStateCollection *state, CContinuousActionData *contAction);

	void setC(int index, rlt_real value);
	rlt_real getC(int index);

	CMyVector *getC() {return c;};

	virtual void getNoise(CStateCollection *state, CContinuousActionData *action, CContinuousActionData *noise);
};

class CContinuousTimeAndActionBangBangVMPolicy : public CContinuousTimeAndActionVMPolicy
{
protected:
	virtual void getActionValues(CMyVector *actionValues, CMyVector *noise);
 
public:
	CContinuousTimeAndActionBangBangVMPolicy(CContinuousAction *action, CVFunctionInputDerivationCalculator *vfunction, CTransitionFunction *model);

	virtual void getNoise(CStateCollection *state, CContinuousActionData *action, CContinuousActionData *noise);
};

class CContinuousActionSmoother : public CContinuousActionController
{
protected:
	CContinuousActionController *policy;
	rlt_real *actionValues;

	rlt_real alpha;
public:
	CContinuousActionSmoother(CContinuousAction *action, CContinuousActionController *policy, rlt_real alpha = 0.3);
	~CContinuousActionSmoother();

	virtual void getNextContinuousAction(CStateCollection *state, CContinuousActionData *contAction);

	void setAlpha(rlt_real alpha);
	virtual rlt_real getAlpha();

};

#endif