ccontinuousactiongradientpolicy.cpp

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "ccontinuousactiongradientpolicy.h"
#include <math.h>

CContinuousActionGradientPolicy::CContinuousActionGradientPolicy(CContinuousAction *contAction, CStateProperties *modelState) : CContinuousActionController(contAction), CStateObject(modelState)
{
	this->modelState = modelState;
}

CContinuousActionGradientPolicy::~CContinuousActionGradientPolicy()
{
}

int CContinuousActionGradientPolicy::getNumInputs()
{
	return modelState->getNumContinuousStates();
}

int CContinuousActionGradientPolicy::getNumOutputs()
{
	return contAction->getNumDimensions();
}

void CContinuousActionGradientPolicy::getFunctionValue(CMyVector *input, CMyVector *output)
{
	CState *state = new CState(modelState);
	CContinuousActionData *data = dynamic_cast<CContinuousActionData *>(contAction->getNewActionData());
	state->setVector(input);
	getNextContinuousAction(state, data);
	output->setVector(data);
	delete state;
	delete data;
}

void CContinuousActionGradientPolicy::getGradient(CMyVector *input, CMyVector *outputErrors, CFeatureList *gradientFeatures)
{
	CFeatureList *featureList = new CFeatureList();
	CState *state = new CState(modelState);
	state->setVector(input);

	for (int i = 0; i < getNumOutputs(); i++)
	{
		getGradient(state, i, featureList);
		gradientFeatures->add(featureList, outputErrors->getElement(i));
	}
	delete featureList;
	delete state;
}


CContinuousActionPolicyFromGradientFunction::CContinuousActionPolicyFromGradientFunction(CContinuousAction *contAction, CGradientFunction *gradientFunction, CStateProperties *modelState) : CContinuousActionGradientPolicy(contAction, modelState)
{
	assert(contAction->getNumDimensions() == gradientFunction->getNumOutputs());
	this->gradientFunction = gradientFunction;

	outputError = new CMyVector(gradientFunction->getNumOutputs());
}

CContinuousActionPolicyFromGradientFunction::~CContinuousActionPolicyFromGradientFunction()
{
	delete outputError;
}

void CContinuousActionPolicyFromGradientFunction::updateWeights(CFeatureList *dParams)
{
	gradientFunction->updateGradient(dParams, 1.0);
}


void CContinuousActionPolicyFromGradientFunction::getNextContinuousAction(CStateCollection *state, CContinuousActionData *action)
{
	gradientFunction->getFunctionValue(state->getState(modelState), action);
}

int CContinuousActionPolicyFromGradientFunction::getNumWeights()
{
	return gradientFunction->getNumWeights();
}

void CContinuousActionPolicyFromGradientFunction::getWeights(rlt_real *parameters)
{
	gradientFunction->getWeights(parameters);
}

void CContinuousActionPolicyFromGradientFunction::setWeights(rlt_real *parameters)
{
	gradientFunction->setWeights(parameters);
}

void CContinuousActionPolicyFromGradientFunction::getGradient(CStateCollection *inputState, int outputDimension, CFeatureList *gradientFeatures)
{
	outputError->initVector(0.0);
	outputError->setElement(outputDimension, 1.0);
	gradientFunction->getGradient(inputState->getState(modelState), outputError, gradientFeatures);
}

void CContinuousActionPolicyFromGradientFunction::getInputDerivation(CStateCollection *inputState, CMyMatrix *targetVector)
{
	gradientFunction->getInputDerivation(inputState->getState(modelState), targetVector);
}

void CContinuousActionPolicyFromGradientFunction::resetData()
{
	gradientFunction->resetData();
}

CContinuousActionFeaturePolicy::CContinuousActionFeaturePolicy(CContinuousAction *contAction, CStateProperties *modelState, std::list<CFeatureCalculator *> *featureCalculators) : CContinuousActionGradientPolicy(contAction, modelState)
{
	this->featureCalculators = featureCalculators;
	localGradient = new CFeatureList();

	numWeights = 0;

	std::list<CFeatureCalculator *>::iterator it = featureCalculators->begin();

	std::list<CStateModifier *> *modifiers = new std::list<CStateModifier *>();
	
	for (it = featureCalculators->begin(); it != featureCalculators->end(); it ++)
	{
		modifiers->push_back(*it);
	}

	featureFunctions = new std::list<CFeatureVFunction *>();

	inputDerivationFunctions = new std::map<CFeatureVFunction *, CVFunctionInputDerivationCalculator *>();

	for (it = featureCalculators->begin(); it != featureCalculators->end(); it ++)
	{
		CFeatureVFunction *featureFunction = new CFeatureVFunction(*it);
		featureFunctions->push_back(featureFunction);
		(*inputDerivationFunctions)[featureFunction] = new CVFunctionNumericInputDerivationCalculator(modelState,featureFunction, 0.005, modifiers);
		numWeights += (*it)->getNumFeatures();
	}
	inputDerivation = new CMyVector(modelState->getNumContinuousStates());


}

CContinuousActionFeaturePolicy::~CContinuousActionFeaturePolicy()
{
	delete localGradient;

	std::list<CFeatureVFunction *>::iterator it = featureFunctions->begin();

	for (; it != featureFunctions->end(); it ++)
	{
		delete (*it);
	}

	delete inputDerivation;
	delete inputDerivationFunctions;
}

void CContinuousActionFeaturePolicy::updateWeights(CFeatureList *dParams)
{
	unsigned int weightIndexStart = 0;
	unsigned int weightIndexStop = 0;

	std::list<CFeatureVFunction *>::iterator it = featureFunctions->begin();

	for (; it != featureFunctions->end(); it ++)
	{
		weightIndexStop += (*it)->getNumFeatures();
		CFeatureList::iterator itFeat = dParams->begin();
		localGradient->clear();
		for (; itFeat != dParams->end(); itFeat ++)
		{
			if ((*itFeat)->featureIndex >= weightIndexStart && (*itFeat)->featureIndex < weightIndexStop)
			{
				localGradient->update((*itFeat)->featureIndex, (*itFeat)->factor);
			}
		}
		(*it)->updateFeatureList(localGradient, 1.0);
	}
}

void CContinuousActionFeaturePolicy::getNextContinuousAction(CStateCollection *stateCol, CContinuousActionData *action)
{
	std::list<CFeatureVFunction *>::iterator itFunc = featureFunctions->begin();
	std::list<CFeatureCalculator *>::iterator itCalc = featureCalculators->begin();

	for (int i = 0; itFunc != featureFunctions->end(); itFunc ++, itCalc ++, i++)
	{
		action->setElement(i, (*itFunc)->getValue(stateCol->getState((*itFunc)->getStateProperties())));
	}
}

int CContinuousActionFeaturePolicy::getNumWeights()
{
	return numWeights;
}

void CContinuousActionFeaturePolicy::getWeights(rlt_real *parameters)
{
	std::list<CFeatureVFunction *>::iterator it = featureFunctions->begin();

	int weightIndex = 0;
	for (; it != featureFunctions->end(); it ++)
	{
		(*it)->getWeights(parameters + weightIndex);
		weightIndex += (*it)->getNumWeights();
	}
}

void CContinuousActionFeaturePolicy::setWeights(rlt_real *parameters)
{
	std::list<CFeatureVFunction *>::iterator it = featureFunctions->begin();

	int weightIndex = 0;
	for (; it != featureFunctions->end(); it ++)
	{
		(*it)->setWeights(parameters + weightIndex);
		weightIndex += (*it)->getNumWeights();
	}
}

void CContinuousActionFeaturePolicy::getGradient(CStateCollection *inputState, int outputDimension, CFeatureList *gradientFeatures)
{
	std::list<CFeatureVFunction *>::iterator it = featureFunctions->begin();

	int weightIndex = 0;
	for (; it != featureFunctions->end(); it ++)
	{
		localGradient->clear();
		(*it)->getGradient(inputState, localGradient);
		localGradient->addIndexOffset(weightIndex);
		gradientFeatures->add(localGradient, 1.0);
		weightIndex += (*it)->getNumWeights();
	}
}

void CContinuousActionFeaturePolicy::getInputDerivation(CStateCollection *inputState, CMyMatrix *targetVector)
{
	std::list<CFeatureVFunction *>::iterator it = featureFunctions->begin();

	for (unsigned int row = 0; it != featureFunctions->end(); it ++, row ++)
	{
		(*inputDerivationFunctions)[(*it)]->getInputDerivation(inputState, inputDerivation);
		
		for (unsigned int col = 0; col < inputDerivation->getNumDimensions(); col ++)
		{