cvfunction.cpp

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "ril_debug.h"
#include "cvfunction.h"
#include "cvfunctionfromqfunction.h"
#include <assert.h>
#include <math.h>
#include <sstream>


CAbstractVFunction::CAbstractVFunction(CStateProperties *prop) : CStateObject(prop)
{
//	gamma = 0.95;
	type = 0;
	mayDiverge = false;
}

CAbstractVFunction::~CAbstractVFunction()
{
}


void CAbstractVFunction::updateValue(CStateCollection *state, rlt_real td)
{
	updateValue(state->getState(properties), td);
}

void CAbstractVFunction::setValue(CStateCollection *state, rlt_real qValue)
{
	setValue(state->getState(properties), qValue);
}

rlt_real CAbstractVFunction::getValue(CStateCollection *state)
{
	return getValue(state->getState(properties));
}

void CAbstractVFunction::updateValue(CState *state, rlt_real td)
{
	setValue(state, getValue(state) + td);
}

int CAbstractVFunction::getType()
{
	return type;
}

bool CAbstractVFunction::isType(int isT)
{
	int temp = type & isT;
	return (temp == isT);
}

void CAbstractVFunction::addType(int newType)
{
	type = type | newType;
}

/*void CAbstractVFunction::setGamma(rlt_real gamma)
{
	this->gamma = gamma;
}*/

void CAbstractVFunction::saveData(FILE *file)
{
    fprintf(file, "V-Function:\n");
    fprintf(file, "\n");
}

void CAbstractVFunction::loadData(FILE *file)
{
    assert(fscanf(file, "V-Function:\n") == 0);
    assert(fscanf(file, "\n") == 0);
}

CAbstractVETraces *CAbstractVFunction::getStandardETraces()
{
	return new CStateVETraces(this, properties);
}

CZeroVFunction::CZeroVFunction() : CAbstractVFunction(NULL)
{

}

rlt_real CZeroVFunction::getValue(CState *state)
{
	return 0;
}

CVFunctionSum::CVFunctionSum() : CAbstractVFunction(NULL)
{
	vFunctions = new std::map<CAbstractVFunction *, rlt_real>;
}

CVFunctionSum::~CVFunctionSum()
{
	delete vFunctions;
}


rlt_real CVFunctionSum::getValue(CStateCollection *state)
{
	std::map<CAbstractVFunction *, rlt_real>::iterator it = vFunctions->begin();

	rlt_real sum = 0.0;
	for (;it != vFunctions->end(); it ++)
	{
		CAbstractVFunction *vFunc = (*it).first;
		
		sum += (*it).second * vFunc->getValue(state);
	}
	return sum;
}

rlt_real CVFunctionSum::getVFunctionFactor(CAbstractVFunction *vFunction)
{
	return (*vFunctions)[vFunction];
}

void CVFunctionSum::setVFunctionFactor(CAbstractVFunction *vFunction, rlt_real factor)
{
	(*vFunctions)[vFunction] = factor;
}

void CVFunctionSum::addVFunction(CAbstractVFunction *vFunction, rlt_real factor)
{
	(*vFunctions)[vFunction] = factor;
}

void CVFunctionSum::removeVFunction(CAbstractVFunction *vFunction)
{
	std::map<CAbstractVFunction *, rlt_real>::iterator it = vFunctions->find(vFunction);

	vFunctions->erase(it);
}

void CVFunctionSum::normFactors(rlt_real factor)
{
	std::map<CAbstractVFunction *, rlt_real>::iterator it = vFunctions->begin();

	rlt_real sum = 0.0;
	for (;it != vFunctions->end(); it ++)
	{
		sum += (*it).second;
	}
	for (;it != vFunctions->end(); it ++)
	{
		(*it).second *= factor / sum;
	}
}

CDivergentVFunctionException::CDivergentVFunctionException(string vFunctionName, CAbstractVFunction *vFunction, CState *state, rlt_real value) : CMyException(101, "DivergentVFunction")
{
	this->vFunction = vFunction;
	this->vFunctionName = vFunctionName;
	this->state = state;
	this->value = value;
}

string CDivergentVFunctionException::getInnerErrorMsg()
{
	char errorMsg[1000];

	sprintf(errorMsg, "%s diverges (value = %f, |value| > 100000)\n", vFunctionName.c_str(), value);
	
	return string(errorMsg);
}

CGradientVFunction::CGradientVFunction(CStateProperties *properties) : CAbstractVFunction(properties)
{
	this->addType(GRADIENTVFUNCTION);

	//this->gradientFeatures = new CFeatureList();
}



CGradientVFunction::~CGradientVFunction()
{
	//delete gradientFeatures;
}

void CGradientVFunction::updateValue(CStateCollection *state, rlt_real td)
{
	localGradientFeatureBuffer->clear();
	getGradient(state, localGradientFeatureBuffer);

	updateGradient(localGradientFeatureBuffer, td);
}

void CGradientVFunction::updateValue(CState *state, rlt_real td)
{
	localGradientFeatureBuffer->clear();
	getGradient(state, localGradientFeatureBuffer);

	updateGradient(localGradientFeatureBuffer, td);
}

CAbstractVETraces *CGradientVFunction::getStandardETraces()
{
	return new CGradientVETraces(this);
}


CGradientDelayedUpdateVFunction::CGradientDelayedUpdateVFunction(CGradientVFunction *vFunction) : CGradientVFunction(vFunction->getStateProperties()), CGradientDelayedUpdateFunction(vFunction)
{
	this->vFunction = vFunction;
}

rlt_real CGradientDelayedUpdateVFunction::getValue(CState *state)
{
	return vFunction->getValue(state);
}
	
void CGradientDelayedUpdateVFunction::getGradient(CStateCollection *state, CFeatureList *gradientFeatures)
{
	vFunction->getGradient(state, gradientFeatures);
}


CVFunctionInputDerivationCalculator::CVFunctionInputDerivationCalculator(CStateProperties *modelState)
{
	this->modelState = modelState;
}

int CVFunctionInputDerivationCalculator::getNumInputs()
{
	return modelState->getNumContinuousStates();
}


CVFunctionNumericInputDerivationCalculator::CVFunctionNumericInputDerivationCalculator(CStateProperties *modelState, CAbstractVFunction *vFunction, rlt_real stepSize, std::list<CStateModifier *> *modifiers) : CVFunctionInputDerivationCalculator(modelState)
{
	this->vFunction = vFunction;
	this->stateBuffer = new CStateCollectionImpl(modelState, modifiers);

	addParameter("NumericInputDerivationStepSize", stepSize);
}

CVFunctionNumericInputDerivationCalculator::~CVFunctionNumericInputDerivationCalculator()
{
	delete stateBuffer;
}

void CVFunctionNumericInputDerivationCalculator::getInputDerivation( CStateCollection *state, CMyVector *targetVector)
{
	CState *inputState = stateBuffer->getState(modelState);
	inputState->setState(state->getState(modelState));

	rlt_real stepSize = getParameter("NumericInputDerivationStepSize");

	for (unsigned int i = 0; i < modelState->getNumContinuousStates(); i++)
	{
		rlt_real stepSize_i = (modelState->getMaxValue(i) - modelState->getMinValue(i)) * stepSize;
		inputState->setContinuousState(i, inputState->getContinuousState(i) + stepSize_i);
		stateBuffer->newModelState();
		rlt_real vPlus = vFunction->getValue(stateBuffer);
		inputState->setContinuousState(i, inputState->getContinuousState(i) - 2 * stepSize_i);
		stateBuffer->newModelState();
		rlt_real vMinus = vFunction->getValue(stateBuffer);

		inputState->setContinuousState(i, inputState->getContinuousState(i) + stepSize_i);
		targetVector->setElement(i, (vPlus - vMinus) / (2 * stepSize_i));
	}
}

CFeatureVFunction::CFeatureVFunction(CStateProperties *prop) : CGradientVFunction(prop), CFeatureFunction(prop->getDiscreteStateSize(0))
{
}



CFeatureVFunction::CFeatureVFunction(CFeatureQFunction *qfunction, CStochasticPolicy *policy) : CGradientVFunction(qfunction->getFeatureCalculator()) ,CFeatureFunction(qfunction->getFeatureCalculator()->getDiscreteStateSize())
{
	setVFunctionFromQFunction(qfunction, policy);
}

CFeatureVFunction::~CFeatureVFunction()
{
}
	
void CFeatureVFunction::setVFunctionFromQFunction(CFeatureQFunction *qfunction, CStochasticPolicy *policy)
{
	CStateProperties properties(0, 1, DISCRETESTATE);
	properties.setDiscreteStateSize(0, numFeatures);
	CState discState(&properties);
	
	CAbstractVFunction *tempFunction;
	
	if (policy)
	{
		tempFunction = new CVFunctionFromQFunction(qfunction, policy, qfunction->getFeatureCalculator());
	}
	else