cmulticontroller.cpp

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "ril_debug.h"
#include "cmulticontroller.h"
#include <assert.h>
#include <list>

CMultiControllerPolicy::CMultiControllerPolicy()
{
}

CMultiControllerPolicy::~CMultiControllerPolicy()
{
}


CMultiControllerRandomPolicy::CMultiControllerRandomPolicy()
{
}

CMultiControllerRandomPolicy::~CMultiControllerRandomPolicy()
{
}

CActionStatistics *CMultiControllerRandomPolicy::getNextAction(std::list<CActionStatistics *> *actionstats)
{
    assert (actionstats->size() > 0);
    int index = rand() % actionstats->size();
    std::list<CActionStatistics *>::iterator it = actionstats->begin();
    for (int i = 0; i < index; i++, it++);
    return (*it);
}


CMultiControllerGreedyPolicy::CMultiControllerGreedyPolicy(CActionStatisticsComparator *comp)
{
	this->comp = comp;
}

CMultiControllerGreedyPolicy::~CMultiControllerGreedyPolicy()
{
	if (this->comp != NULL) delete this->comp;
}

CActionStatistics *CMultiControllerGreedyPolicy::getNextAction(std::list<CActionStatistics *> *actionstats)
{
	CActionStatistics * maxstatistics= NULL;
	assert(actionstats->size() > 0);
		
	std::list<CActionStatistics *>::iterator it = actionstats->begin();
    maxstatistics = (*it++);
	for (; it != actionstats->end(); it++)
	{
        if (comp->compare(maxstatistics, (*it)) < 0)
		{
			maxstatistics = (*it);
		}
	}
    return maxstatistics;
}


CMultiController::CMultiController(CActionSet *actions, CMultiControllerPolicy* policy) : CAgentStatisticController(actions)
{
    this->policy = policy;
    controllers = new std::list<std::pair<CAgentStatisticController *, CActionStatistics *> *>();
}

CMultiController::~CMultiController()
{
    delete controllers;
}

void CMultiController::addController(CAgentStatisticController *controller) {
    controllers->push_back(new std::pair<CAgentStatisticController *, CActionStatistics *>(controller, new CActionStatistics()));
}

void CMultiController::removeController(CAgentStatisticController *controller) {
    std::list<std::pair<CAgentStatisticController *, CActionStatistics *> *>::iterator it;
    for (it = controllers->begin(); ((*it)->first != controller) && it != controllers->end(); it++);
    if (it != controllers->end())
    {
        delete (*it)->second;
        delete *it;
        controllers->remove(*it);
    }
}

CAction* CMultiController::getNextAction(CStateCollection *state, CActionDataSet *dataset, CActionStatistics *statistics) {
    std::list<std::pair<CAgentStatisticController *, CActionStatistics *> *>::iterator it;
    std::list<CActionStatistics *> *actionstats = new std::list<CActionStatistics *>();
    CActionStatistics *selectedstats;
    CAction* action = NULL;

	for (it = controllers->begin(); it != controllers->end(); it++)
	{
	     action = (*it)->first->getNextAction(state, dataset,(*it)->second);
	     if (action != NULL)
	     {
	         actionstats->push_back((*it)->second);
	     }
	}
	selectedstats = policy->getNextAction(actionstats);
	action = selectedstats->action;
   	if (statistics != NULL) statistics->copy(selectedstats);
    delete actionstats;
    //printf("CMultiController::getNextAction: (Action %d)\n", actions->getIndex());
    return action;
}