ccontinuoustime.cpp

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "ril_debug.h"
#include "ccontinuoustime.h"
#include <assert.h>
#include <math.h>


rlt_real CContinuousTimeParameters::getGammaFromSgamma(rlt_real sgamma, rlt_real dt)
{
	return 1 - dt * sgamma;
}

rlt_real CContinuousTimeParameters::getLambdaFromKappa(rlt_real kappa, rlt_real sgamma, rlt_real dt)
{
	return (kappa - dt) / (1 / sgamma - dt);
}

CContinuousTimeVMPolicy::CContinuousTimeVMPolicy(CActionSet *actions, CActionDistribution *distribution, CVFunctionInputDerivationCalculator *vFunction, CDynamicContinuousTimeModel *model, CRewardFunction *rewardFunction) : CQStochasticPolicy(actions, distribution, new CContinuousTimeQFunctionFromTransitionFunction(actions, vFunction, model, rewardFunction))
{
	this->vfunction = vFunction;
	this->model = model;

	addParameters(vFunction);

//	assert(vfunction->getNumContinuousStates() == model->getNumContinuousStates());
}

CContinuousTimeVMPolicy::~CContinuousTimeVMPolicy()
{
	delete this->qfunction;
}

CContinuousTimeQFunctionFromTransitionFunction *CContinuousTimeVMPolicy::getQFunctionFromTransitionFunction()
{
	return dynamic_cast<CContinuousTimeQFunctionFromTransitionFunction *>(qfunction);
}

CContinuousTimeAndActionVMPolicy::CContinuousTimeAndActionVMPolicy(CContinuousAction *action, CVFunctionInputDerivationCalculator *dvFunction, CTransitionFunction *model) : CContinuousActionController(action, false)
{
	this->dVFunction = dvFunction;
	this->model =  model;

	assert(this->dVFunction->getNumInputs() == model->getNumContinuousStates());
	assert(model->isType(DM_DERIVATIONUMODEL));

	actionValues = new CMyVector(getContinuousActionProperties()->getNumActionValues());
	derivationU = new CMyMatrix(model->getNumContinuousStates(), getContinuousActionProperties()->getNumActionValues());
	derivationX = new CMyVector(model->getNumContinuousStates());
	//noise = new CContinuousActionData(getContinuousActionProperties());

	randomControllerMode = INTERN_RANDOM_CONTROLLER;

	addParameters(dvFunction);
}

CContinuousTimeAndActionVMPolicy::~CContinuousTimeAndActionVMPolicy()
{
	delete actionValues;
	delete derivationU;
	delete derivationX;
	//delete noise;
}




void CContinuousTimeAndActionVMPolicy::getNextContinuousAction(CStateCollection *state, CContinuousActionData *action)
{
	model->getDerivationU(state->getState(model->getStateProperties()), derivationU);
	dVFunction->getInputDerivation(state, derivationX);

	derivationX->multMatrix(derivationU, actionValues);

	noise->initVector(0.0);

	if (randomController && randomControllerMode == INTERN_RANDOM_CONTROLLER)
	{
		randomController->getNextContinuousAction(state, noise);
	}

	getActionValues(actionValues, noise);

	action->setVector(actionValues);
}



CContinuousTimeAndActionSigmoidVMPolicy::CContinuousTimeAndActionSigmoidVMPolicy(CContinuousAction *action, CVFunctionInputDerivationCalculator *vfunction, CTransitionFunction *model) : CContinuousTimeAndActionVMPolicy(action, vfunction, model)
{
	c = new CMyVector(action->getContinuousActionProperties()->getNumActionValues());
	c->initVector(1.0);

	addParameter("SigmoidPolicyCFactor", 100.0);
}

CContinuousTimeAndActionSigmoidVMPolicy::~CContinuousTimeAndActionSigmoidVMPolicy()
{
	delete c;
}



void CContinuousTimeAndActionSigmoidVMPolicy::getActionValues(CMyVector *actionValues, CMyVector *noise)
{
	actionValues->dotVector(c);
	actionValues->multScalar(getParameter("SigmoidPolicyCFactor"));
	actionValues->addVector(noise);

	for (unsigned int i = 0; i < actionValues->getNumDimensions(); i++)
	{
		rlt_real umax = getContinuousActionProperties()->getMaxActionValue(i);
		rlt_real umin = getContinuousActionProperties()->getMinActionValue(i);
		if (actionValues->getElement(i) < - 400)
		{
			actionValues->setElement(i, -400);
		}
		rlt_real s = 1/(1 + exp(-(actionValues->getElement(i))));
		actionValues->setElement(i, umin + s * (umax - umin));
	}
}

void CContinuousTimeAndActionSigmoidVMPolicy::getNoise(CStateCollection *state, CContinuousActionData *action, CContinuousActionData *l_noise)
{
	rlt_real generalC = getParameter("SigmoidPolicyCFactor");
	if (randomControllerMode == INTERN_RANDOM_CONTROLLER)
	{
		CMyVector tempVector(contAction->getNumDimensions());
		model->getDerivationU(state->getState(model->getStateProperties()), derivationU);
		dVFunction->getInputDerivation(state, derivationX);

		derivationX->multMatrix(derivationU, l_noise);
		tempVector.setVector(action);

		for (int i = 0; i < tempVector.getNumDimensions(); i ++)
		{
			rlt_real umax = getContinuousActionProperties()->getMaxActionValue(i);
			rlt_real umin = getContinuousActionProperties()->getMinActionValue(i);

			rlt_real actionValue = tempVector.getElement(i);
			actionValue = (actionValue - umin) / (umax - umin);

			actionValue = - log(1 / actionValue - 1) / generalC / c->getElement(i);

			tempVector.setElement(i, actionValue);
		}

		l_noise->multScalar(-1.0);
		l_noise->addVector(&tempVector);
		
	}
	else
	{
		CContinuousActionController::getNoise(state, action, l_noise);
	}
}


void CContinuousTimeAndActionSigmoidVMPolicy::setC(int index, rlt_real value)
{
	c->setElement(index, value);
}

rlt_real CContinuousTimeAndActionSigmoidVMPolicy::getC(int index)
{
	return c->getElement(index);
}



CContinuousTimeAndActionSigmoidVMGradientPolicy::CContinuousTimeAndActionSigmoidVMGradientPolicy(CContinuousAction *action, CGradientVFunction *gradVFunction, CVFunctionInputDerivationCalculator *dvFunction, CTransitionFunction *model, std::list<CStateModifier *> *modifiers) : CContinuousActionGradientPolicy(action, model->getStateProperties())
{
	vFunction = gradVFunction;

	derivationState = new CStateCollectionImpl(model->getStateProperties(), modifiers);

	gradient1 = new CFeatureList();
	gradient2 = new CFeatureList();

	c = new CMyVector(action->getContinuousActionProperties()->getNumActionValues());
	c->initVector(1.0);

	addParameter("SigmoidPolicyCFactor", 1.0);

	this->dVFunction = dvFunction;
	this->model =  model;

	assert(this->dVFunction->getNumInputs() == model->getNumContinuousStates());
	assert(model->isType(DM_DERIVATIONUMODEL));

	actionValues = new CMyVector(getContinuousActionProperties()->getNumActionValues());
	derivationU = new CMyMatrix(model->getNumContinuousStates(), getContinuousActionProperties()->getNumActionValues());
	derivationX = new CMyVector(model->getNumContinuousStates());
	//noise = new CContinuousActionData(getContinuousActionProperties());

	randomControllerMode = INTERN_RANDOM_CONTROLLER;

	addParameters(dvFunction);
}

CContinuousTimeAndActionSigmoidVMGradientPolicy::~CContinuousTimeAndActionSigmoidVMGradientPolicy()