cepisodehistory.cpp

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "ril_debug.h"
#include "cepisodehistory.h"
#include <assert.h>

CEpisodeHistory::CEpisodeHistory(CStateProperties *prop, CActionSet *actions) : CStateModifiersObject(prop), CStepHistory(properties, actions)
{
}

int CEpisodeHistory::getNumSteps()
{
	int sum = 0;

	int i = 0;

	for (i = 0; i < getNumEpisodes(); i++)
	{
		sum += getEpisode(i)->getNumSteps();
	}

	return sum;
}


void CEpisodeHistory::getStep(int index, CStep *step)
{
	assert(index < getNumSteps());

	int lnum = index;
	
	int i = 0;
	
	CEpisode *episode;

	while (lnum >= (episode = getEpisode(i))->getNumSteps())
	{	
		lnum -= episode->getNumSteps();
		i ++;
	}
	
	episode->getStep(lnum, step);
}
/*
void CEpisodeHistory::simulateEpisode(int index, CSemiMDPListener *listener)
{
	assert(index < this->getNumEpisodes());
	
	CEpisode *episode = getEpisode(index);
	
	CStep *step = new CStep(getStateProperties(), getStateModifiers());

	listener->newEpisode();

	int i = 0;

	

	for (i = 0; i < episode->getNumSteps(); i++)
	{
		episode->getStep(i, step);

		listener->nextStep(step->oldState, step->action, step->newState);
	}

	listener->newEpisode();

	delete step;
}

void CEpisodeHistory::simulateAllEpisodes(CSemiMDPListener *listener)
{
	for (int i = 0; i < getNumEpisodes(); i++)
	{
		simulateEpisode(i, listener);
	}
}

void CEpisodeHistory::simulateNRandomEpisodes(int numEpisodes, CSemiMDPListener *listener)
{

	std::list<int> *episodeIndex = new std::list<int>();
	for (int i = 0; i < getNumEpisodes(); i++)
	{
		episodeIndex->push_back(i);
	}
	for (int i = 0; i < numEpisodes && episodeIndex->size() > 0; i++)
	{
		int nRand = rand() % episodeIndex->size();
		std::list<int>::iterator it = episodeIndex->begin();
		for (int j = 0; j < nRand && it != episodeIndex->end(); j ++, it ++);
		nRand = *it;
		episodeIndex->erase(it);
		simulateEpisode(nRand, listener);
	}
}
*/

CStoredEpisodeModel::CStoredEpisodeModel(CEpisodeHistory *history) : CEnvironmentModel(history->getStateProperties()), CAgentController(history->getActions())
{
	this->history = history;

	this->numEpisode = -1;
	this->numStep = 0;

	doResetModel();
}

CStoredEpisodeModel::~CStoredEpisodeModel()
{
}

void CStoredEpisodeModel::doNextState(CPrimitiveAction *action)
{
	numStep ++;

	if (numStep >= currentEpisode->getNumSteps())
	{
		reset = true;
	}

}

void CStoredEpisodeModel::doResetModel() 
{
	if (reset == true)
	{
		numEpisode ++;
		if (numEpisode >= history->getNumEpisodes())
		{
			numEpisode = 0;
		}
	}
	currentEpisode = history->getEpisode(numEpisode);
	numStep = 0;
	if (currentEpisode->getNumSteps() == 0)
	{
		doResetModel();
	}
}

CEpisodeHistory* CStoredEpisodeModel::getEpisodeHistory()
{
	return history;
}

void CStoredEpisodeModel::setEpisodeHistory(CEpisodeHistory *hist)
{
	history = hist;
}


void CStoredEpisodeModel::getState(CState *state)
{
	currentEpisode->getState(numStep, state);
}

void CStoredEpisodeModel::getState(CStateCollectionImpl *stateCollection)
{
	stateCollection->newModelState();
	currentEpisode->getStateCollection(numStep, stateCollection);
}


CAction* CStoredEpisodeModel::getNextAction(CStateCollection *state)
{
	if (numStep >= currentEpisode->getNumSteps())
	{
		return NULL;
	}
	else
	{
		return currentEpisode->getAction(numStep);	
	}
}

CBatchEpisodeUpdate::CBatchEpisodeUpdate(CSemiMDPListener *listener, CEpisodeHistory *logger, int numEpisodes, std::list<CStateModifier *> *modifiers)
{
	this->listener = listener;
	this->logger = logger;
	this->numEpisodes = numEpisodes;

	dataSet= new CActionDataSet(logger->getActions());

	step = new CStep(logger->getStateProperties(), modifiers, logger->getActions());

}

CBatchEpisodeUpdate::~CBatchEpisodeUpdate()
{
	delete dataSet;
	delete step;
}

void CBatchEpisodeUpdate::newEpisode()
{
	simulateNRandomEpisodes(numEpisodes, listener);
}

void CBatchEpisodeUpdate::simulateEpisode(int index, CSemiMDPListener *listener)
{
	assert(index < logger->getNumEpisodes());

	CEpisode *episode = logger->getEpisode(index);


	listener->newEpisode();

	int i = 0;

	CActionSet::iterator it = logger->getActions()->begin();
	for (; it != logger->getActions()->end(); it ++)
	{
		dataSet->setActionData(*it, (*it)->getActionData());
	}

	for (i = 0; i < episode->getNumSteps(); i++)
	{
		episode->getStep(i, step);
		step->action->loadActionData(step->actionData->getActionData(step->action));

		listener->nextStep(step->oldState, step->action, step->newState);
	}
	it = logger->getActions()->begin();
	for (; it != logger->getActions()->end(); it ++)
	{
		(*it)->loadActionData(dataSet->getActionData(*it));
	}

	listener->newEpisode();
}

void CBatchEpisodeUpdate::simulateAllEpisodes(CSemiMDPListener *listener)
{
	for (int i = 0; i < logger->getNumEpisodes(); i++)
	{
		simulateEpisode(i, listener);
	}
}

void CBatchEpisodeUpdate::simulateNRandomEpisodes(int numEpisodes, CSemiMDPListener *listener)
{

	std::list<int> *episodeIndex = new std::list<int>();
	for (int i = 0; i < logger->getNumEpisodes(); i++)
	{
		episodeIndex->push_back(i);
	}
	for (int i = 0; i < numEpisodes && episodeIndex->size() > 0; i++)
	{
		int nRand = rand() % episodeIndex->size();
		std::list<int>::iterator it = episodeIndex->begin();
		for (int j = 0; j < nRand && it != episodeIndex->end(); j ++, it ++);
		nRand = *it;
		episodeIndex->erase(it);
		simulateEpisode(nRand, listener);
	}
	delete episodeIndex;
}