cqtcontrol.cpp

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//#define RL_TOOLBOX_USE_QT




#include "cqtcontrol.h"

#include <sstream>
#include <string>

#ifdef RL_TOOLBOX_USE_QT

#include <qgrid.h>
#include <qapplication.h>
#include <qpushbutton.h>
#include <qcheckbox.h>
#include <qfont.h>
#include <qlayout.h>
#include <qlabel.h>
#include <math.h>
#include <qgroupbox.h>
#include <qlistbox.h>
#include <qlineedit.h>
#include <qspinbox.h>
#include <qtextedit.h>
#include <qmessagebox.h>
#include <qfiledialog.h>

CQTControl::CQTControl(CMyTestSuiteCollection *collection, QWidget *parent, const char *name) : qtControl(parent, name)
{
	this->testSuiteChooser = new CQTTestSuiteChooser(collection);
	this->testCollection = collection;
	
	initQTControl(collection->agent);
}

CQTControl::CQTControl(CAgent *agent, QWidget *parent, const char *name) : qtControl(parent, name)
{
	this->testSuiteChooser = NULL;
	this->testCollection = NULL;

	initQTControl(agent);
}

CQTControl::~CQTControl()
{
	exit = true;
	startSimulationCondition->wakeAll();
	wait();

	delete startSimulationCondition;
	delete startSimulationMutex;

	delete agentControllerList;
	delete agentListenerList;

	delete selectedListeners;

	if (testSuiteChooser)
	{
		delete testSuiteChooser;
	}
	

	delete policyEvaluator;
}

void CQTControl::initQTControl(CAgent *agent)
{
	this->agent = agent;
	this->model = agent->getEnvironmentModel();

	agent->addSemiMDPListener(this);

	connect( quit, SIGNAL(clicked()), qApp, SLOT(quit()) );

	delay = realTime->value();
	episodesToGo = 1;
	stepsToGo = 0;
	stepsPerEpisode = 0;
	numSteps = 0;
	threadMode = 0;
	exit = false;

	startSimulationCondition = new QWaitCondition();
	startSimulationMutex = new QMutex();

	agentControllers->insertItem("TestSuite Controller");
	start();

	connect(this->startSimButton, SIGNAL(clicked()), this, SLOT(startSimulationClicked()));
	connect(this->resetModelButton, SIGNAL(clicked()), this, SLOT(resetModelClicked()));
	connect(this->realTime, SIGNAL(valueChanged (int)), this, SLOT(realTimeChanged(int)));
	connect(this->agentControllers, SIGNAL(activated(int)), this, SLOT(controllerChanged(int)));


	connect(this->chooseTestSuiteButton, SIGNAL(clicked()), this, SLOT(chooseNewTestSuiteClicked()));

	if (testSuiteChooser == NULL)
	{
		chooseTestSuiteButton->setDisabled(true);
	}

	connect(this->changeParameters, SIGNAL(nextLine()), this, SLOT(nextParameters()));
	connect(this->changeParameters, SIGNAL(prevLine()), this, SLOT(previousParameters()));

	connect(this->resetDataButton, SIGNAL(clicked()), this, SLOT(resetLearnDataClicked()));
	connect(this->loadDataButton, SIGNAL(clicked()), this, SLOT(loadLearnDataClicked()));
	connect(this->saveDataButton, SIGNAL(clicked()), this, SLOT(saveLearnDataClicked()));

	connect(this->browseButton, SIGNAL(clicked()), this, SLOT(browseButtonClicked()));
	connect(this->newParameterButton, SIGNAL(clicked()),  this, SLOT(newParametersButtonClicked()));

	connect(this->bestParameterButton, SIGNAL(clicked()),  this, SLOT(getBestParametersButtonClicked()));


	connect(this->learnTestSuiteButton, SIGNAL(clicked()), this, SLOT(startLearningClicked()));


	//connect(this->agentListeners, SIGNAL(selectionChanged()), this, SLOT(listenersChanged()));

	agentControllerList = new std::list<CAgentController *>();
	agentListenerList = new std::list<CSemiMDPListener *>();

	selectedListeners = new std::list<bool>();

	testSuite = NULL;
	this->parameterCalculator = NULL;

	policyEvaluator = NULL;
}

void CQTControl::resetModelClicked(void)
{
	startSimulationMutex->lock();
	agent->startNewEpisode();
	nepisodes->setText(QString::number(agent->getCurrentEpisodeNumber()));
	threadMode = 0;
	startSimButton->setText("Start Simulation");
	startSimulationMutex->unlock();
}

void CQTControl::startSimulationClicked(void)
{
	startSimulationMutex->lock();
	if (threadMode == SIMULATE)
	{
		startSimButton->setText("Start Simulation");
		threadMode = 0;
	}
	else
	{
		startSimButton->setText("Pause");
		episodesToGo = simulateEpisodes->value();
		stepsToGo = simSteps->value();

		stepsPerEpisode = steps_Episode->value();

		threadMode = SIMULATE;

		startSimulationCondition->wakeAll();
	}
	startSimulationMutex->unlock();
}

void CQTControl::realTimeChanged(int value)
{
	startSimulationMutex->lock();
	this->delay = value;
	startSimulationMutex->unlock();
}

void CQTControl::run()
{
	while (!exit)
	{
		startSimulationMutex->lock();
		switch (threadMode)
		{
		case SIMULATE:
			{
				if (episodesToGo > 0 || stepsToGo > 0)
				{
					if (agent->getController() != NULL)
					{
						agent->doControllerStep();
					}
					else
					{
						threadMode = 0;
						QMessageBox::warning(this, "Error", "Please choose a valid Controller or Testsuite", QMessageBox::Ok, QMessageBox::NoButton);

					}
					
					numSteps ++;
					if (stepsToGo > 0)
					{
						stepsToGo--;
						simSteps->setValue(stepsToGo);
					}
					if (model->isReset() || (stepsPerEpisode > 0 && agent->getCurrentStep() >= stepsPerEpisode))
					{
						agent->startNewEpisode();
						episodesToGo--;
						simulateEpisodes->setValue(episodesToGo);
						nepisodes->setText(QString::number(agent->getCurrentEpisodeNumber()));

					}
					steps->setText(QString::number(numSteps));

				}
				else
				{
					threadMode = 0;
					startSimButton->setText("Start Simulation");
				}
				break;
			}
		case LEARN:
			{
				if (parameterCalculator)
				{
					testSuite->resetLearnedData();
					parameterCalculator->evaluateParameters(testSuite, true);
					setParameters(testSuite);
					threadMode = 0;
				}
				break;
			}
		default:
			{
			
				startSimulationCondition->wait(startSimulationMutex);
			}
		}

		startSimulationMutex->unlock();
	}
}

void CQTControl::nextStep(CStateCollection *oldState, CAction *action, CStateCollection *newState)
{
	msleep(delay);
}

void CQTControl::addAgentController(CAgentController *controller, char *name, bool selected)
{
	agentControllers->insertItem(name);
	agentControllerList->push_back(controller);

	if (selected)
	{
		agentControllers->setCurrentItem(agentControllerList->size() - 1);
	}
}

void CQTControl::removeAgentController(CAgentController *controller)
{
	std::list<CAgentController *>::iterator it;
	int i = 0;

	for (it = agentControllerList->begin(); it != agentControllerList->end(); it++, i++)
	{
		if (*it == controller)
		{
			agentControllers->removeItem(i);
			agentControllerList->erase(it);
		}
	}
}

/*void CQTControl::addAgentListener(CSemiMDPListener *listener, char *name, bool selected )
{
	agentListeners->insertItem(name);
	agentListenerList->push_back(listener);
	selectedListeners->push_back(selected);

	if (selected)
	{
		agentListeners->setSelected(agentListenerList->size() - 1, selected); 
	}
}