ctheoreticalmodel.cpp

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "ril_debug.h"
#include "ctheoreticalmodel.h"
#include <math.h>


CTransition::CTransition(int startState, int endState, rlt_real prop)
{
	this->startState = startState;
	this->endState = endState;
	this->propability = prop;

	type = 1;
}

bool CTransition::isType(int Type)
{
	return (this->type & Type) > 0;
}


int CTransition::getStartState()
{
	return startState;
}

int CTransition::getEndState()
{
	return endState;
}

rlt_real CTransition::getPropability()
{
	return propability;
}

void CTransition::setPropability(rlt_real prop)
{
	propability = prop;
}

void CTransition::saveASCII(FILE *stream, bool forward)
{
	int state;
	if (forward)
	{
		state = getEndState();
	}
	else
	{
		state = getStartState();
	}
	fprintf(stream, "(%d %lf)", state, getPropability());
}

void CTransition::loadASCII(FILE *stream, int fixedState, bool forward)
{
	int state;
	fscanf(stream, "(%d %lf)", &state, &propability);
	if (forward)
	{
		endState = state;
		startState = fixedState;
	}
	else
	{
		startState = state;
		endState = fixedState;
	}
}

CSemiMDPTransition::CSemiMDPTransition(int startState, int endState, rlt_real prop) : CTransition(startState, endState, prop)
{
	durations = new std::map<int, rlt_real>();
	type = type | SEMIMDPTRANSITION;
}

CSemiMDPTransition::~CSemiMDPTransition()
{
	delete durations;
}

std::map<int, rlt_real> *CSemiMDPTransition::getDurations()
{
	return durations;
}

void CSemiMDPTransition::addDuration(int duration, rlt_real factor)
{
	rlt_real normalize = (1 - factor);
	std::map<int, rlt_real>::iterator it = durations->begin();
	for (; it != durations->end(); it++)
	{
		(*it).second *= normalize;
	}
		
	it = durations->find(duration);
	if (it != durations->end())
	{
		(*it).second += factor;
	}
	else
	{
		(*durations)[duration] = factor;
	}
}

rlt_real CSemiMDPTransition::getDurationFaktor(int duration)
{
	return (*durations)[duration];
}


rlt_real CSemiMDPTransition::getDurationPropability(int duration)
{
	return getPropability() * getDurationFaktor(duration);
}

rlt_real CSemiMDPTransition::getSemiMDPFaktor(rlt_real gamma)
{
	rlt_real factor = 0.0;
	std::map<int,rlt_real>::iterator itDurations = getDurations()->begin();
	for (; itDurations != getDurations()->end(); itDurations++)
	{
		factor += pow(gamma, (*itDurations).first - 1) * (*itDurations).second;
	}
	return factor;
}

void CSemiMDPTransition::loadASCII(FILE *stream, int fixedState, bool forward)
{
	int bufDuration;
	rlt_real bufFaktor;
	char buf;

	CTransition::loadASCII(stream, fixedState, forward);
	
	fscanf(stream, "[");
	fread(&buf, sizeof(char), 1, stream);
	while (buf != ']')
	{
		fscanf(stream, "%d %lf)", &bufDuration, &bufFaktor);
		(*durations)[bufDuration] = bufFaktor;
		fread(&buf, sizeof(char), 1, stream);
	}
}

void CSemiMDPTransition::saveASCII(FILE *stream, bool forward)
{
	CTransition::saveASCII(stream, forward);
	
	std::map<int, rlt_real>::iterator it =  this->getDurations()->begin();

	fprintf(stream, "[");
	for (; it != getDurations()->end(); it ++)
	{
		fprintf(stream, "(%d %f)", (*it).first, (*it).second);
	}
	fprintf(stream, "]");
}


CTransitionList::CTransitionList(bool forwardList)
{
	this->forwardList = forwardList;
}

CTransitionList::iterator CTransitionList::getTransitionIterator(int featureIndex)
{
	CTransitionList::iterator it = begin();
	
	if (forwardList)
	{
		while(it != end() && (*it)->getEndState() < featureIndex)
		{
			it ++;
		}
	}
	else
	{
		while(it != end() && (*it)->getStartState() < featureIndex)
		{
			it ++;
		}
	}
	return it;
}

bool CTransitionList::isMember(int featureIndex)
{
	CTransitionList::iterator it = getTransitionIterator(featureIndex);
	
	if (forwardList)
	{
		return it != end() && (*it)->getEndState() == featureIndex;
	}
	else
	{
		return it != end() && (*it)->getStartState() == featureIndex;
	}
}

bool CTransitionList::isForwardList()
{
	return forwardList;
}

void CTransitionList::addTransition(CTransition *transition)
{	
	CTransitionList::iterator it;
	
	if (isForwardList())
	{
		it = getTransitionIterator(transition->getEndState());
	}
	else
	{
		it = getTransitionIterator(transition->getStartState());
	}
	insert(it, transition);
}

CTransition *CTransitionList::getTransition(int featureIndex)
{
	CTransitionList::iterator it = getTransitionIterator(featureIndex);
	
	if (it != end())
	{
		return (*it);
	}
	else return NULL;
}

void CTransitionList::clearAndDelete()
{
	CTransitionList::iterator it = begin();
	
	for (;it != end(); it ++)
	{
		delete (*it);
	}
	clear();
}

CStateActionTransitions::CStateActionTransitions()
{
	forwardList = new CTransitionList(true);
	backwardList = new CTransitionList(false);
}

CStateActionTransitions::~CStateActionTransitions()
{
	forwardList->clearAndDelete();
	backwardList->clear();
	
	delete forwardList;
	delete backwardList;
}


CTransitionList* CStateActionTransitions::getForwardTransitions()
{
	return forwardList;
}

CTransitionList* CStateActionTransitions::getBackwardTransitions()
{
	return backwardList;
}

unsigned int CAbstractFeatureStochasticModel::getNumFeatures()
{
	return numFeatures;
}

CAbstractFeatureStochasticModel::CAbstractFeatureStochasticModel(CActionSet *actions, int numFeatures) : CActionObject(actions)
{
	this->numFeatures = numFeatures;
}

rlt_real CAbstractFeatureStochasticModel::getPropability(int oldState, CAction *action, int newState)
{
	int index = getActions()->getIndex(action);

	return getPropability(oldState, index, newState);
}

rlt_real CAbstractFeatureStochasticModel::getPropability(CFeatureList *oldList, CAction *action, CFeatureList *newList)
{
	rlt_real propability = 0.0;
	CFeatureList::iterator itOld ;
	CFeatureList::iterator itNew = newList->begin();

	int actoinIndex = getActions()->getIndex(action);

	for (itOld = oldList->begin(); itOld != oldList->end(); itOld ++)
	{
		for (itNew = newList->begin(); itNew != newList->end(); itNew ++)
		{
			propability += getPropability((*itOld)->featureIndex, actoinIndex, (*itNew)->featureIndex) * (*itNew)->factor * (*itOld)->factor;
		}
	}
	return propability;
}

CFeatureStochasticModel::CFeatureStochasticModel(CActionSet *actions, int numFeatures, FILE *stream) : CAbstractFeatureStochasticModel(actions, numFeatures)
{
	stateTransitions = new CMyArray2D<CStateActionTransitions *>(getNumActions(), numFeatures);
	
	for (int i = 0; i < stateTransitions->getSize(); i++)
	{
		stateTransitions->set1D(i, new CStateActionTransitions());
	}