crewardfunction.cpp

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "ril_debug.h"
#include "crewardfunction.h"
#include "crewardmodel.h"

CSemiMDPLastNRewardFunction::CSemiMDPLastNRewardFunction(CRewardFunction *rewardFunction, rlt_real gamma) : CRewardEpisode(rewardFunction)
{
	this->gamma = gamma;
}

CSemiMDPLastNRewardFunction::~CSemiMDPLastNRewardFunction()
{
}


CRewardEpisode::CRewardEpisode(CRewardFunction *rewardFunction) : CSemiMDPRewardListener(rewardFunction)
{
	rewards = new std::vector<rlt_real>();
}

CRewardEpisode::~CRewardEpisode()
{
	delete rewards;
}

void CRewardEpisode::nextStep(CStateCollection *oldState, CAction *action, rlt_real reward, CStateCollection *newState)
{
	rewards->push_back(reward);
}

void CRewardEpisode::newEpisode()
{
	rewards->clear();
}

int CRewardEpisode::getNumRewards()
{
	return rewards->size();
}

rlt_real CRewardEpisode::getReward(int index)
{
	return (*rewards)[index];
}

rlt_real CRewardEpisode::getMeanReward()
{
	assert(getNumRewards() > 0);
	rlt_real mean = 0;
	for (int i = 0; i < getNumRewards(); i ++)
	{
		mean += getReward(i);
	}
	return mean / getNumRewards();
}

rlt_real CRewardEpisode::getLastStepsMeanReward(int Steps)
{
	assert(getNumRewards() > 0);
	rlt_real mean = 0;
	for (int i = getNumRewards() - Steps; i < getNumRewards(); i ++)
	{
		mean += getReward(i);
	}
	return mean / (getNumRewards() - Steps);
}

rlt_real CSemiMDPLastNRewardFunction::getReward(CStateCollection *oldState, CAction *action, CStateCollection *newState)
{
	rlt_real reward = 0;
	unsigned int duration = action->getDuration();
	
	for (unsigned int index = 0; index < duration; index ++)
	{
		reward = reward * gamma;
		
		assert(rewards->size() >= duration);
		reward = reward + (*rewards)[rewards->size() - duration + index];
	}
	return reward;
}

CFeatureRewardFunction::CFeatureRewardFunction(CStateModifier * discretizer) : CStateObject(discretizer)
{
	this->discretizer = discretizer;
}

CFeatureRewardFunction::~CFeatureRewardFunction()
{
	
}

rlt_real CFeatureRewardFunction::getReward(CStateCollection *oldState, CAction *action, CStateCollection *newState)
{
	
	return getReward(oldState->getState(properties), action, newState->getState(properties));
}

rlt_real CFeatureRewardFunction::getReward(CFeatureList *oldState, CAction *action, CFeatureList *newState)
{
	rlt_real reward = 0;

	CFeatureList::iterator oldIt;
	CFeatureList::iterator newIt;

	for (oldIt = oldState->begin(); oldIt != oldState->end(); oldIt++)
	{
		for (newIt = newState->begin(); newIt != newState->end(); newIt++)
		{
			reward += getReward((*oldIt)->featureIndex, action, (*newIt)->featureIndex) * (*oldIt)->factor * (*newIt)->factor;
		}
	}
	return reward;
}

rlt_real CFeatureRewardFunction::getReward(CState *oldState, CAction *action, CState *newState)
{
	rlt_real reward = 0;
	
	int type = oldState->getStateProperties()->getType() & (FEATURESTATE | DISCRETESTATE);
	switch (type)
	{
		case FEATURESTATE:
			{
				for (unsigned int oldS = 0; oldS < oldState->getNumDiscreteStates(); oldS++)
				{
					for (unsigned int newS = 0; newS < newState->getNumDiscreteStates(); newS++)
					{
						reward += getReward(oldState->getDiscreteState(oldS), action, newState->getDiscreteState(newS)) * oldState->getContinuousState(oldS) * newState->getContinuousState(newS);
					}
				}
				break;
			}
		case DISCRETESTATE:
			{
				reward = getReward(oldState->getDiscreteState(0), action, newState->getDiscreteState(0));
				break;
			}
		default:
			{
			}
	}
	return reward;
}

CStateReward::CStateReward(CStateProperties *l_properties) : CStateObject(l_properties)
{
	this->properties = l_properties;
}

rlt_real CStateReward::getReward(CStateCollection *oldState, CAction *action, CStateCollection *newState)
{
	return this->getStateReward(newState->getState(properties));
}


CRewardAsVFunction::CRewardAsVFunction(CStateReward *reward) : CAbstractVFunction(reward->getStateProperties())
{
	this->reward = reward;
}

rlt_real CRewardAsVFunction::getValue(CState *state)
{
	return reward->getStateReward(state);
}