canalyzer.cpp

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "ril_debug.h"
#include "canalyzer.h"
#include <math.h>
#include <assert.h>

CVFunctionAnalyzer::CVFunctionAnalyzer(CAbstractVFunction *vFunction, CStateProperties *modelState, std::list<CStateModifier *> *modifiers)
{
	this->vFunction = vFunction;
	this->modelStateProperties = modelState;

	stateCollection = new CStateCollectionImpl(modelState, modifiers);
}

CVFunctionAnalyzer::~CVFunctionAnalyzer()
{
	delete stateCollection;
}

void CVFunctionAnalyzer::save1DValues(FILE *stream, CState *initState, int dim1, int part1)
{
	rlt_real width = modelStateProperties->getMaxValue(dim1) - modelStateProperties->getMinValue(dim1);
	CState *modelState = stateCollection->getState(modelStateProperties);

	modelState->setState(initState);
	fprintf(stream, "VFunction Analyzer 1D-Table for Dimension %d\n", dim1);
	fprintf(stream, "InitialState: ");
	initState->saveASCII(stream);
	fprintf(stream, "\n");

	for (int i = 0; i <= part1; i ++)
	{
		rlt_real x = modelStateProperties->getMinValue(dim1) + i * width / part1;
		modelState->setContinuousState(dim1, x);
		stateCollection->newModelState();
		rlt_real value = vFunction->getValue(stateCollection);
		fprintf(stream, "(%f, %f)\n", x, value);
	}
}

void CVFunctionAnalyzer::save2DValues(FILE *stream, CState *initstate, int dim1, int part1, int dim2, int part2)
{
	rlt_real width1 = modelStateProperties->getMaxValue(dim1) - modelStateProperties->getMinValue(dim1);
	rlt_real width2 = modelStateProperties->getMaxValue(dim2) - modelStateProperties->getMinValue(dim2);

	CState *modelState = stateCollection->getState(modelStateProperties);

	fprintf(stream, "VFunction Analyzer 2D-Table for Dimensions %d and %d\n", dim1, dim2);
	fprintf(stream, "InitialState: ");
	initstate->saveASCII(stream);
	fprintf(stream, "\n");

	for (int i = 0; i <= part1; i ++)
	{
		rlt_real x = modelStateProperties->getMinValue(dim1) + i * width1 / part1;
		initstate->setContinuousState(dim1, x);

		for (int j = 0; j <= part2; j ++)
		{
			rlt_real y = modelStateProperties->getMinValue(dim2) + j * width2 / part2;
			modelState->setContinuousState(dim2, y);
			stateCollection->newModelState();

			rlt_real value = vFunction->getValue(stateCollection);
			fprintf(stream, "(%f, %f, %f)\n", x, y, value);
		}
	}
}

void CVFunctionAnalyzer::saveStateValues(FILE *stream, CStateList *states)
{
	CState *modelState = stateCollection->getState(modelStateProperties);

	fprintf(stream, "VFunction Analyzer State Value-Table\n");

	for (unsigned int i = 0; i < states->getNumStates(); i ++)
	{
		states->getState(i, modelState);
		stateCollection->newModelState();

		rlt_real value = vFunction->getValue(stateCollection);
		fprintf(stream, "(");
		modelState->saveASCII(stream);
		fprintf(stream, ", %f)\n", value);
	}
}

CQFunctionAnalyzer::CQFunctionAnalyzer(CAbstractQFunction *qFunction, CStateProperties *modelState, std::list<CStateModifier *> *modifiers)
{
	this->qFunction = qFunction;
	this->modelStateProperties = modelState;

	stateCollection = new CStateCollectionImpl(modelState, modifiers);
}

CQFunctionAnalyzer::~CQFunctionAnalyzer()
{
	delete stateCollection;
}

void CQFunctionAnalyzer::save1DValues(FILE *stream, CActionSet *actions, CState *initState, int dim1, int part1)
{
	rlt_real width = modelStateProperties->getMaxValue(dim1) - modelStateProperties->getMinValue(dim1);
	CState *modelState = stateCollection->getState(modelStateProperties);

	modelState->setState(initState);
	fprintf(stream, "QFunction Analyzer 1D-Table for Dimension %d\n", dim1);
	fprintf(stream, "InitialState: ");
	initState->saveASCII(stream);
	fprintf(stream, "\n");

	fprintf(stream, "Actions: %d\n", actions->size());

	for (int i = 0; i <= part1; i ++)
	{
		rlt_real x = modelStateProperties->getMinValue(dim1) + i * width / part1;
		modelState->setContinuousState(dim1, x);
		stateCollection->newModelState();
		fprintf(stream, "(%f: ", x);
		CActionSet::iterator it = actions->begin();

		for (; it != actions->end(); it ++)
		{
			rlt_real value = qFunction->getValue(stateCollection, *it);
			fprintf(stream, "%f ", value);
		}
		fprintf(stream, ")\n");
	}
}

void CQFunctionAnalyzer::save2DValues(FILE *stream, CActionSet *actions, CState *initstate, int dim1, int part1, int dim2, int part2)
{
	rlt_real width1 = modelStateProperties->getMaxValue(dim1) - modelStateProperties->getMinValue(dim1);
	rlt_real width2 = modelStateProperties->getMaxValue(dim2) - modelStateProperties->getMinValue(dim2);

	CState *modelState = stateCollection->getState(modelStateProperties);

	fprintf(stream, "QFunction Analyzer 2D-Table for Dimensions %d and %d\n", dim1, dim2);
	fprintf(stream, "InitialState: ");
	initstate->saveASCII(stream);
	fprintf(stream, "\n");

	fprintf(stream, "Actions: %d\n", actions->size());

	for (int i = 0; i <= part1; i ++)
	{
		rlt_real x = modelStateProperties->getMinValue(dim1) + i * width1 / part1;
		initstate->setContinuousState(dim1, x);

		for (int j = 0; j <= part2; j ++)
		{
			rlt_real y = modelStateProperties->getMinValue(dim2) + j * width2 / part2;
			modelState->setContinuousState(dim2, y);
			stateCollection->newModelState();
			fprintf(stream, "(%f, %f: ", x, y);

			CActionSet::iterator it = actions->begin();

			for (; it != actions->end(); it ++)
			{
				rlt_real value = qFunction->getValue(stateCollection, *it);
				fprintf(stream, "%f ", value);
			}

			fprintf(stream, ")\n");
		}
	}
}

void CQFunctionAnalyzer::saveStateValues(FILE *stream, CActionSet *actions, CStateList *states)
{
	CState *modelState = stateCollection->getState(modelStateProperties);

	fprintf(stream, "QFunction Analyzer State Value-Table\n");

	for (unsigned int i = 0; i < states->getNumStates(); i ++)
	{
		states->getState(i, modelState);
		stateCollection->newModelState();

		fprintf(stream, "(");
		modelState->saveASCII(stream);
		fprintf(stream, ": ");

		CActionSet::iterator it = actions->begin();

		for (; it != actions->end(); it ++)
		{
			rlt_real value = qFunction->getValue(stateCollection, *it);
			fprintf(stream, "%f ", value);
		}
		fprintf(stream, ")\n");
	}
}

CFunctionComperator::CFunctionComperator(CStateProperties *modelState, std::list<CStateModifier *> *modifiers)
{
	this->modelStateProperties = modelState;
	stateCollection = new CStateCollectionImpl(modelStateProperties, modifiers);
}

CFunctionComperator::~CFunctionComperator()
{
	delete stateCollection;
}

rlt_real CFunctionComperator::getDifference(CStateCollection *state, int errorFunction)
{
	rlt_real value1 = getValue(1, state);
	rlt_real value2 = getValue(2, state);

	rlt_real error = 0;

	switch (errorFunction)
	{
	case MSE:
		{
			error = pow(value1 - value2, 2);
			break;
		}
	case MAE:
		{
			error = fabs((rlt_real)(value1 - value2));
		}
	}
	return error;
}

void CFunctionComperator::getRandomState(CState *state)
{
	for (unsigned int i = 0; i < state->getNumContinuousStates(); i ++)
	{
		rlt_real width = state->getStateProperties()->getMaxValue(i) - state->getStateProperties()->getMinValue(i);
		rlt_real randVal = ((rlt_real) rand() / (rlt_real) RAND_MAX) * width + state->getStateProperties()->getMaxValue(i);

		state->setContinuousState(i, randVal);
	}
}

rlt_real CFunctionComperator::compareFunctionsRandom(int nSamples, int errorFunction )
{
	CState *modelState = stateCollection->getState(modelStateProperties);
	rlt_real error = 0.0;
	for (int i = 0; i < nSamples; i++)
	{
		getRandomState(modelState);
		stateCollection->newModelState();

		switch(errorFunction)
		{
		case MSE:
		case MAE:
			{
				rlt_real value = getDifference(stateCollection, errorFunction);
				error += value;
				break;
			}
		case MAXERROR:
			{
				rlt_real value = getDifference(stateCollection, MAE);
				if (value > error)
				{
					error = value;
				}
				break;
			}

		}
	}
	if (errorFunction == MSE || errorFunction == MAE)
	{
		error /= nSamples;
	}
	return error;
}

rlt_real CFunctionComperator::compareFunctionsStates(CStateList *states, int errorFunction)
{
	CState *modelState = stateCollection->getState(modelStateProperties);
	rlt_real error = 0.0;
	for (unsigned int i = 0; i < states->getNumStates(); i++)
	{
		states->getState(i, modelState);
		stateCollection->newModelState();
		switch(errorFunction)
		{
		case MSE:
		case MAE:
			{
				rlt_real value = getDifference(stateCollection, errorFunction);
				error += value;
				break;
			}
		case MAXERROR:
			{
				rlt_real value = getDifference(stateCollection, MAE);
				if (value > error)
				{
					error = value;
				}
				break;
			}

		}
	}
	if (errorFunction == MSE || errorFunction == MAE)
	{
		error /= states->getNumStates();
	}
	return error;
}

rlt_real CVFunctionComperator::getValue(int numFunc, CStateCollection *state)
{
	if (numFunc == 1)
	{
		return vFunction1->getValue(state);
	}
	else
	{
		return vFunction2->getValue(state);
	}
}

CVFunctionComperator::CVFunctionComperator(CStateProperties *modelState, std::list<CStateModifier *> *modifiers, CAbstractVFunction *vFunction1, CAbstractVFunction *vFunction2) : CFunctionComperator(modelState, modifiers)
{
	this->vFunction1 = vFunction1;
	this->vFunction2 = vFunction2;
}

rlt_real CQFunctionComperator::getValue(int numFunc, CStateCollection *state)
{
	if (numFunc == 1)
	{
		return qFunction1->getValue(state, action);
	}
	else
	{
		return qFunction2->getValue(state, action);
	}
}

CQFunctionComperator::CQFunctionComperator(CStateProperties *modelState, std::list<CStateModifier *> *modifiers, CAbstractQFunction *qFunction1, CAbstractQFunction *qFunction2, CAction *action) : CFunctionComperator(modelState, modifiers)
{
	this->qFunction1 = qFunction1;
	this->qFunction2 = qFunction2;

	this->action = action;
}


CControllerAnalyzer::CControllerAnalyzer(CStateList *states, CAgentController *controller, CActionSet *actions) : CActionObject(actions)
{
	this->states = states;
	this->controller = controller;
}

CControllerAnalyzer::~CControllerAnalyzer()
{
}

CStateList *CControllerAnalyzer::getStateList()
{
	return this->states;
}

void CControllerAnalyzer::setStateList(CStateList *states)
{
	this->states = states;
}

CAgentController *CControllerAnalyzer::getController()
{
	return this->controller;
}

void CControllerAnalyzer::setController(CAgentController *controller)
{
	this->controller = controller;
}

void CControllerAnalyzer::saveActions(FILE *stream, std::list<CStateModifier *> *modifiers)
{
	fprintf(stream, "State - selected Action Table\n");
	CState *modelState = new CState(states->getStateProperties());
	CAction *action;
    unsigned int i = 1;
	CStateCollectionImpl *stateCollection = new CStateCollectionImpl(states->getStateProperties(), modifiers);

	for (unsigned int stateNum = 0; stateNum < states->getNumStates(); stateNum ++)
	{
		states->getState(stateNum, modelState);
		stateCollection->setState(modelState);
		stateCollection->calculateModifiedStates();

		action = controller->getNextAction(stateCollection);
		assert(action != NULL);
		fprintf(stream, "State# %d: Action %d\n", i++, actions->getIndex(action));
	}
	delete modelState;
	delete stateCollection;
}


CActionStatisticAnalyzer::CActionStatisticAnalyzer(CAgentStatisticController *master) : CAgentStatisticController(NULL)
{
	this->master = master;
	controllers = new std::map<void *, std::pair<char *, long>*>();
}

CActionStatisticAnalyzer::~CActionStatisticAnalyzer()
{
	if (controllers->size() > 0)
	{
		std::map<void *, std::pair<char *, long>*>::iterator it = controllers->begin();
        for(;it != controllers->end(); it++)
		{
			delete (*it).second;
		}
		controllers->clear();
	}
    delete controllers;
}

void CActionStatisticAnalyzer::addController(CAgentStatisticController *newcontroller, char *name)
{
	std::pair<char *, long>* temp = new std::pair<char *, long>(name, 0);
	(*controllers)[(void *)newcontroller] = temp;
}

void CActionStatisticAnalyzer::printStatistics()
{
	if (controllers->size() > 0)
	{
		printf("ActionOwners: ");
		std::map<void *, std::pair<char *, long>*>::iterator it = controllers->begin();
        for(;it != controllers->end(); it++)
		{
			printf("%s: %ld; ", (*it).second->first, (*it).second->second);
		}
		printf("\n");
	}
}

void CActionStatisticAnalyzer::init()
{
	if (controllers->size() > 0)
	{
		std::map<void *, std::pair<char *, long>*>::iterator it = controllers->begin();
        for(;it != controllers->end(); it++)
		{
			(*it).second->second = 0;
		}
	}
}

CAction* CActionStatisticAnalyzer::getNextAction(CStateCollection *state, CActionStatistics *stat)
{
	CAction *action = master->getNextAction(state, NULL, stat);
	assert (stat != NULL);
    if (controllers->find(stat->owner) != controllers->end())
	{
		(*controllers)[(void *)stat->owner]->second ++;
	}
	return action;
}