cpegasus.cpp

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "cpegasus.h"
#include <math.h>

CTransitionFunctionInputDerivationCalculator::CTransitionFunctionInputDerivationCalculator(CDynamicContinuousTimeAndActionModel *dynModel)
{
	this->dynModel = dynModel;
}

CTransitionFunctionInputDerivationCalculator::~CTransitionFunctionInputDerivationCalculator()
{
}

CTransitionFunctionNumericalInputDerivationCalculator::CTransitionFunctionNumericalInputDerivationCalculator(CDynamicContinuousTimeAndActionModel *dynModel, rlt_real stepSize = 0.001) : CTransitionFunctionInputDerivationCalculator(dynModel)
{
	addParameter("InputDerivationCalculatorStepSize", stepSize);

	nextState1 = new CState(dynModel->getStateProperties());
	nextState2 = new CState(dynModel->getStateProperties());

	buffState = new CState(dynModel->getStateProperties());

	buffData = dynamic_cast<CContinuousActionData *>(dynModel->getContinuousAction()->getNewActionData());

}

CTransitionFunctionNumericalInputDerivationCalculator::~CTransitionFunctionNumericalInputDerivationCalculator()
{
	delete buffState;
	delete nextState1;
	delete nextState2;

	delete buffData;

}

void CTransitionFunctionNumericalInputDerivationCalculator::getInputDerivation(CState *currentState, CContinuousActionData *data, CMyMatrix *dModelInput)
{
	int column = 0;
	CStateProperties *stateProp = dynModel->getStateProperties();
	rlt_real stepSize = getParameter("InputDerivationCalculatorStepSize");
	for (unsigned int i = 0; i < currentState->getNumContinuousStates(); i++, column ++)
	{
		rlt_real stepSize_i = stepSize * (stateProp->getMaxValue(i) - stateProp->getMinValue(i));
		buffState->setState(currentState);

		buffState->setContinuousState(i, currentState->getContinuousState(i) - stepSize_i);
		dynModel->transitionFunction(buffState, dynModel->getContinuousAction(), nextState1, data);

		buffState->setContinuousState(i, currentState->getContinuousState(i) + stepSize_i);
		dynModel->transitionFunction(buffState, dynModel->getContinuousAction(), nextState2, data);

		for (unsigned int row = 0; row < stateProp->getNumContinuousStates(); row ++)
		{
			dModelInput->setElement(row, column, (nextState2->getSingleStateDifference(row, nextState1->getContinuousState(row))) / ( 2 * stepSize_i));
		}
	}

	CContinuousActionProperties *actionProp = dynModel->getContinuousAction()->getContinuousActionProperties();

	for (unsigned int i = 0; i < data->getNumDimensions(); i++, column ++)
	{
		rlt_real stepSize_i = stepSize * (actionProp->getMaxActionValue(i) - actionProp->getMinActionValue(i));
		
		buffData->setData(data);
		buffData->setElement(i, data->getElement(i) - stepSize_i);
		dynModel->transitionFunction(buffState, dynModel->getContinuousAction(), nextState1, buffData);

		buffData->setElement(i, data->getElement(i) + stepSize_i);
		dynModel->transitionFunction(buffState, dynModel->getContinuousAction(), nextState2, buffData);

		for (unsigned int row = 0; row < stateProp->getNumContinuousStates(); row ++)
		{
			dModelInput->setElement(row, column, (nextState2->getSingleStateDifference(row, nextState1->getContinuousState(row))) / ( 2 * stepSize_i));
		}
	}
}

CPEGASUSPolicyGradientCalculator::CPEGASUSPolicyGradientCalculator(CContinuousActionGradientPolicy *policy, CTransitionFunctionEnvironment *dynModel, int numStartStates, int horizon, rlt_real gamma) 
: CPolicyGradientCalculator(policy)
{
	addParameter("DiscountFactor", gamma);
	addParameter("PEGASUSHorizon", horizon);
	addParameter("PEGASUSNumStartStates", numStartStates);
	addParameter("PEGASUSUseNewStartStates", 0.0);

	this->dynModel = dynModel;

	startStates = new CStateList(dynModel->getStateProperties());

	this->policy = policy;

}

CPEGASUSPolicyGradientCalculator::~CPEGASUSPolicyGradientCalculator()
{
	delete startStates;
}

void CPEGASUSPolicyGradientCalculator::getGradient(CFeatureList *gradient)
{
	bool bUseNewStates = getParameter("PEGASUSUseNewStartStates") > 0.5;
	if (bUseNewStates || startStates->getNumStates() == 0)
	{
		setRandomStartStates();
	}
	getPEGASUSGradient(gradient, startStates);
}

CStateList* CPEGASUSPolicyGradientCalculator::getStartStates()
{
	return startStates;
}

void CPEGASUSPolicyGradientCalculator::setStartStates(CStateList *startStates)
{
	this->startStates->clear();
	CState *state = new CState(dynModel->getStateProperties());
	for (unsigned int i = 0; i < startStates->getNumStates(); i++)
	{
		startStates->getState(i, state);
		this->startStates->addState(state);
	}
	delete state;
}

void CPEGASUSPolicyGradientCalculator::setRandomStartStates()
{
	this->startStates->clear();
	CStateProperties *properties = dynModel->getStateProperties();
	CState *state = new CState(properties);
	for (int i = 0; i < my_round(getParameter("PEGASUSNumStartStates")); i++)
	{
		dynModel->resetModel();
		dynModel->getState(state);
		this->startStates->addState(state);
	}
	delete state;
}


CPEGASUSAnalyticalPolicyGradientCalculator::CPEGASUSAnalyticalPolicyGradientCalculator(CAgent *agent, CContinuousActionGradientPolicy *policy, CCAGradientPolicyInputDerivationCalculator *policydInput, CTransitionFunctionEnvironment *dynModel, CTransitionFunctionInputDerivationCalculator *dynModeldInput, CStateReward *rewardFunction, int numStartStates,int horizon, rlt_real gamma ) : CPEGASUSPolicyGradientCalculator(policy,dynModel, numStartStates, horizon, gamma)
{
	addParameters(policydInput, "DPolicy");
	addParameters(dynModeldInput, "DModel");

	int numActionValues = policy->getContinuousActionProperties()->getNumActionValues();

	this->rewardFunction = rewardFunction;
	this->agent = agent;
	this->dynModeldInput = dynModeldInput;
	this->policydInput = policydInput;

	this->dReward = new CMyVector(dynModel->getNumContinuousStates());
	stateGradient1 = new std::list<CFeatureList *>();

	for (unsigned int i = 0; i < dynModel->getNumContinuousStates(); i ++)
	{
		stateGradient1->push_back(new CFeatureList());
	}

	stateGradient2 = new std::list<CFeatureList *>();

	for (unsigned int i = 0; i < dynModel->getNumContinuousStates(); i ++)
	{
		stateGradient2->push_back(new CFeatureList());
	}

	dModelGradient = new std::list<CFeatureList *>();

	for (int i = 0; i < numActionValues; i ++)
	{
		dModelGradient->push_back(new CFeatureList());
	}

	episodeGradient = new CFeatureList();

	dPolicy = new CMyMatrix(numActionValues, dynModel->getNumContinuousStates());
	dModelInput = new CMyMatrix(dynModel->getNumContinuousStates(), numActionValues + dynModel->getNumContinuousStates());

	steps = 0;
}

CPEGASUSAnalyticalPolicyGradientCalculator::~CPEGASUSAnalyticalPolicyGradientCalculator()
{
	delete dReward;

	std::list<CFeatureList *>::iterator it = stateGradient1->begin();

	for (; it != stateGradient1->end(); it ++)
	{
		delete *it;
	}

	delete stateGradient1;

	it = stateGradient2->begin();

	for (; it != stateGradient2->end(); it ++)
	{
		delete *it;
	}

	it = dModelGradient->begin();

	for (; it != dModelGradient->end(); it ++)
	{
		delete *it;