cresiduals.cpp

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <math.h>

#include "ril_debug.h"
#include "cresiduals.h"



CDiscreteResidual::CDiscreteResidual(rlt_real gamma)
{
	addParameter("DiscountFactor", gamma);
}

rlt_real CDiscreteResidual::getResidual(rlt_real oldV, rlt_real reward, rlt_real duration, rlt_real newV)
{
	return reward + pow(getParameter("DiscountFactor"), duration) * newV - oldV;
}


void CDiscreteResidual::getResidualGradient(CFeatureList *oldGradient, CFeatureList *newGradient, rlt_real duration, CFeatureList *gradientResidualFeatures)
{
	gradientResidualFeatures->add(oldGradient, -1.0);
	gradientResidualFeatures->add(newGradient, pow(getParameter("DiscountFactor"), duration));
}

CContinuousEulerResidual::CContinuousEulerResidual(rlt_real dt, rlt_real sgamma)
{
	addParameter("TimeIntervall", dt);
	addParameter("ContinuousDiscountFactor", sgamma);
}


rlt_real CContinuousEulerResidual::getResidual(rlt_real oldV, rlt_real reward, rlt_real duration, rlt_real newV)
{
	rlt_real dt = getParameter("TimeIntervall");
	return (reward + (1 / (dt * duration) - getParameter("ContinuousDiscountFactor")) * newV - oldV / (dt * duration)) * dt;
}

void CContinuousEulerResidual::getResidualGradient(CFeatureList *oldGradient, CFeatureList *newGradient, rlt_real duration, CFeatureList *gradientResidualFeatures)
{
	rlt_real dt = getParameter("TimeIntervall");

	gradientResidualFeatures->add(oldGradient, - 1 );
	gradientResidualFeatures->add(newGradient, (1  -  getParameter("ContinuousDiscountFactor") * (dt * duration)));
}

CContinuousCoulomResidual::CContinuousCoulomResidual(rlt_real dt, rlt_real sgamma) 
{
	addParameter("TimeIntervall", dt);
	addParameter("ContinuousDiscountFactor", sgamma);
}

rlt_real CContinuousCoulomResidual::getResidual(rlt_real oldV, rlt_real reward, rlt_real duration, rlt_real newV)
{
	rlt_real dt = getParameter("TimeIntervall");

	return (reward + (1 / (dt * duration) - getParameter("ContinuousDiscountFactor") / 2) * newV - oldV *(1 / (dt * duration) + getParameter("ContinuousDiscountFactor") / 2)) * dt;
}

void CContinuousCoulomResidual::getResidualGradient(CFeatureList *oldGradient, CFeatureList *newGradient, rlt_real duration, CFeatureList *gradientResidualFeatures)
{
	rlt_real dt = getParameter("TimeIntervall");

	gradientResidualFeatures->add(oldGradient, (- 1  - getParameter("ContinuousDiscountFactor") * (dt * duration) / 2));
	gradientResidualFeatures->add(newGradient, (1 - getParameter("ContinuousDiscountFactor") *( dt * duration) / 2));
}

CConstantBetaCalculator::CConstantBetaCalculator(rlt_real beta)
{
	addParameter("ResidualBeta", beta);
}

rlt_real CConstantBetaCalculator::getBeta(CFeatureList *oldGradient, CFeatureList *newGradient)
{
	return getParameter("ResidualBeta");
}


CVariableBetaCalculator::CVariableBetaCalculator(rlt_real mu, rlt_real maxBeta)
{
	addParameter("ResidualBetaMu", mu);
	addParameter("ResidualMaxBeta", maxBeta);
}

rlt_real CVariableBetaCalculator::getBeta(CFeatureList *directGradient, CFeatureList *residualGradient)
{
	

	rlt_real numerator = 0.0;
	rlt_real denominator = 0.0;

	rlt_real beta = 0.0;

	if (DebugIsEnabled('v'))
	{
		DebugPrint('v', "ResidualGradient: ");
		residualGradient->saveASCII(DebugGetFileHandle('v'));
		DebugPrint('v', "DirectGradient: ");
		directGradient->saveASCII(DebugGetFileHandle('v'));
	}

	numerator = residualGradient->multFeatureList(directGradient);
	denominator = residualGradient->multFeatureList(residualGradient) - numerator;


	if (fabs(denominator) < 0.0000001)
	{
		beta = 0.0;
	}
	else
	{
		beta = - numerator / denominator + getParameter("ResidualBetaMu");
	}

	DebugPrint('v', "Residual %f, %f: Beta = %f", numerator, denominator, beta);

	if (beta < 0.0 || beta > 1.0)
	{
		beta = 0.0;
	}
	else
	{
		if (beta > getParameter("ResidualMaxBeta"))
		{
			beta = getParameter("ResidualMaxBeta");
		}
	}
	DebugPrint('v', "beschr鋘kt: %f\n", beta);


	return beta;
}

CResidualBetaFunction::CResidualBetaFunction(CAbstractBetaCalculator *betaCalculator, CResidualGradientFunction *residualGradient)
{
	this->betaCalculator = betaCalculator;
	this->residualGradient = residualGradient;

	addParameters(betaCalculator);
	addParameters(residualGradient);
}

void CResidualBetaFunction::getResidualGradient(CFeatureList *oldGradient, CFeatureList *newGradient, rlt_real duration, CFeatureList *residualGradientFeatures)
{
	residualGradient->getResidualGradient(oldGradient, newGradient, duration, residualGradientFeatures);

	rlt_real beta = betaCalculator->getBeta(oldGradient, residualGradientFeatures);
	residualGradientFeatures->multFactor(beta);
	residualGradientFeatures->add(oldGradient, -(1 - beta));
}

void CDirectGradient::getResidualGradient(CFeatureList *oldGradient, CFeatureList *newGradient, rlt_real duration, CFeatureList *residualGradient)
{
	residualGradient->add(oldGradient, -1);
}