ctestsuit.h

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


#ifndef C_TESTSuite__H
#define C_TESTSuite__H

#include <time.h>
#include <stdio.h>

#include "cpolicies.h"
#include "cagent.h"
#include "ril_debug.h"
#include "ctdlearner.h"
#include "clinearfafeaturecalculator.h"
#include "cvfunctionlearner.h"
#include "crewardmodel.h"
#include "cadvantagelearning.h"
#include "cactorcritic.h"
#include "cparameters.h"
#include "cmontecarlo.h"
#include "cpolicygradient.h"
#include "cvaluepolicygradientlearner.h"

#include <map>
#include <string>
#include <iostream>

using namespace std;

#define ARCF_IDENTITY 1
#define ARCF_LINEAR 2
#define ARCF_AVERAGE 3

#define EVALUATION_DIRECTORY "../../../RL_Toolbox_Linux/executables"

class CTestSuite :  virtual public CParameterObject
{
protected:

	CAgentController *controller;
	CAgentController *evaluationController;
	std::list<CLearnDataObject *> *learnDataObjects;

	CAgent *agent;

	string testSuiteName;
	string learnDataFileName;

public:
	CTestSuite(CAgent *agent, CAgentController *controller, CLearnDataObject *vFunction, char *testSuiteName);
	CTestSuite(CAgent *agent, CAgentController *controller, CAgentController *evaluationController, CLearnDataObject *vFunction, char *testSuiteName);
	virtual ~CTestSuite();
	
	virtual void saveLearnedData(FILE *stream);
	virtual void loadLearnedData(FILE *stream);

	virtual void resetLearnedData();
	
	void addLearnDataObject(CLearnDataObject *learnDataObject);

	virtual void learn(int nEpisodes, int nStepsPerEpisode) = 0;

	virtual CAgentController *getController();
	virtual void setController(CAgentController *controller);
	virtual CAgentController *getEvaluationController();
	virtual void setEvaluationController(CAgentController *evaluationController);


	string getTestSuiteName();
	void setTestSuiteName(string name);
};



class CListenerTestSuite : public CTestSuite
{
protected:
	std::list<CSemiMDPListener *> *learnerObjects;
	std::map<CSemiMDPListener *, bool> *addToAgent;
public:
	CListenerTestSuite(CAgent *agent, CSemiMDPListener *learner, CAgentController *controller, CLearnDataObject *vFunction, char *testSuiteName);
	CListenerTestSuite(CAgent *agent, CSemiMDPListener *learner, CAgentController *controller, CAgentController *evaluationController, CLearnDataObject *vFunction, char *testSuiteName);

	virtual ~CListenerTestSuite();

	virtual void addLearnersToAgent();
	virtual void removeLearnersFromAgent();

	void addLearnerObject(CSemiMDPListener *listener, bool addParams = true, bool addBack = true, bool addToAgent = true);

	virtual void learn(int nEpisodes, int nStepsPerEpisode);

	std::list<CSemiMDPListener *> *getLearnerList() {return learnerObjects;};
};

class CPolicyGradientTestSuite : public CTestSuite
{
protected:
	CPolicyGradientLearner *learner;

public:
	CPolicyGradientTestSuite(CAgent *agent, CPolicyGradientLearner *learner, CAgentController *controller, CLearnDataObject *vFunction, char *testSuiteName, int nMaxGradientUpdates = 1);
	CPolicyGradientTestSuite(CAgent *agent, CPolicyGradientLearner *learner, CAgentController *controller, CAgentController *evaluationController, CLearnDataObject *vFunction, char *testSuiteName, int nMaxGradientUpdates = 1);

	virtual ~CPolicyGradientTestSuite();

	virtual void learn(int nEpisodes, int nStepsPerEpisode);
};

class CTestSuiteCollection 
{
protected:
	std::map<string, CTestSuite *> *testSuiteMap;
	
public:
	CTestSuiteCollection();
	virtual ~CTestSuiteCollection();

	void addTestSuite(CTestSuite *testSuite);
	void removeTestSuite(CTestSuite *testSuite);

	void removeAllTestSuites();

	int getNumTestSuites();
	CTestSuite *getTestSuite(string testSuiteName);
	CTestSuite *getTestSuite(int index);
};



class CTestSuiteEvaluator : virtual public CParameterObject
{
protected:
	CAgent *agent;

	string evaluatorDirectory;
	string testSuiteCollectionName;

	std::list<rlt_real *> *values;
	unsigned int nTrials;
	int numValuesPerTrial;

	bool exception;

	virtual void getXLabel(char *xLabel, int i);
public:
	CTestSuiteEvaluator(CAgent *agent, string testSuiteCollectionName, int nTrials, int nValuesPerTrial);
	virtual ~CTestSuiteEvaluator();

	virtual rlt_real evaluateTestSuite(CTestSuite *testSuite, bool loadEvaluationTrial = true);
	
	string getEvaluatorDirectory();
	string getEvaluationFileName(CTestSuite *testSuite);
	string getLearnDataFileName(CTestSuite *testSuite);

	void checkDirectories();

	virtual rlt_real getEvaluationValue(std::list<rlt_real *> *values) = 0;

	virtual void loadEvaluationData(CParameters *testSuite, const char *filename) = 0;

	virtual void doEvaluationTrial(CTestSuite *testSuiteName, FILE *evaluationFile, const char *learnDataFileName) = 0;

	virtual void clearValues();

	virtual void saveMatlabData(CParameters *testSuite,  char *outFileName, char *inFileName = NULL);
};

class CTestSuiteNeededStepsEvaluator : public CTestSuiteEvaluator
{
protected:
	std::list<rlt_real *> *succeded;

	unsigned int totalLearnEpisodes;
	unsigned int stepsLearnEpisode;
	unsigned int nTrials;
	unsigned int episodesBeforeEvaluate;

	unsigned int nValues;

	bool maxStepsSucceded;
public: 
	CTestSuiteNeededStepsEvaluator(CAgent *agent, string testSuiteCollectionName, int totalLearnEpisodes, int stepsLearnEpisode, int episodesBeforeEvaluate, int nTrials, bool maxStepsSucceded = false);
	virtual ~CTestSuiteNeededStepsEvaluator();

	virtual void loadEvaluationData(CParameters *testSuite,const char  *fileName);

	virtual void doEvaluationTrial(CTestSuite *testSuiteName, FILE *evaluationFile, const char *learnDataFileName);

	virtual rlt_real getEvaluationValue(std::list<rlt_real *> *values);

	rlt_real getPercentageSucceded();

	virtual void clearValues();

};

class CAverageRewardTestSuiteEvaluator : public CTestSuiteEvaluator
{
protected:
	std::list<CPolicyEvaluator *> *evaluators;

	void getXLabel(char *xLabel, int i);

public:
	unsigned int episodesBeforeEvaluate;
	unsigned int totalLearnEpisodes;
	unsigned int stepsLearnEpisode;

	unsigned int nAverageRewards;
	int evaluationFunction;


	CAverageRewardTestSuiteEvaluator(CAgent *agent, string testSuiteCollectionName, CPolicyEvaluator *evaluator, int totalLearnEpisodes, int episodesBeforeEvaluate, int stepsLearnEpisode, int nTrials);
	virtual ~CAverageRewardTestSuiteEvaluator();

	virtual rlt_real evaluateTestSuite(CTestSuite *testSuite, int evaluationFunction = 0, bool loadEvaluationTrial = true);

	virtual void loadEvaluationData(CParameters *testSuite, const char *fileName);

	virtual void doEvaluationTrial(CTestSuite *testSuiteName, FILE *evaluationFile, const char *learnDataFileName);

	virtual std::list<rlt_real *> *getTrialAverageRewards();

	virtual rlt_real getEvaluationValue(std::list<rlt_real *> *values);

	virtual void addPolicyEvaluator(CPolicyEvaluator *evaluator);
};

class CTestSuiteParameterCalculator 
{
protected:

	CTestSuiteEvaluator *evaluator;
	CTestSuite *testSuite;
	
public:
	FILE *parameterFile;

	CTestSuiteParameterCalculator(CTestSuiteEvaluator *evaluator, CTestSuite *testSuite);
	virtual ~CTestSuiteParameterCalculator();
	
	virtual CParameters* calculateBestParameters(std::list<rlt_real *> *parameters, int *paramSize, std::list<string> *paramNames);
	virtual CParameters* calculateBestParameters(std::list<CParameters *> *parameterList);

	virtual rlt_real calculateSingleBestParameter(std::list<rlt_real> *values, string paramNames);

	rlt_real searchBestParameterValue(string paramName, rlt_real startValue, int maxIterations = 5, rlt_real minValue = 0.0, rlt_real maxValue = 10000);


	static std::list<CParameters *> *getParameterList(std::list<rlt_real *> *parameters, int *paramSize, std::list<string> *paramNames);
    
	virtual rlt_real evaluateParameters(CParameters *parameters, bool newEvaluation = false);

	CTestSuite *getTestSuite();
};

class CResidualChooser 
{
public:
	static CResidualFunction *getResidual(int resNum, rlt_real dt);
	static CResidualGradientFunction *getResidualGradient(int resNum, CResidualGradientFunction *residual);

	static int getResidualFromInput();
	static int getResidualGradientFromInput();

	static CAbstractBetaCalculator *getBetaCalculator(int numBeta);
	static int getBetaCalculatorFromInput();

};

class CPolicyChooser
{
public:
	static CActionDistribution *getDistribution(int number);
	static int getDistributionFromInput();
};

class CTDLearnerChooser
{
public:
	static CTDLearner *getQLearner(int estimationPolicyNum, int learnModeNum, int residualNum, int residualGradientNum, CGradientQFunction *qFunction, CRewardFunction *rewardFunction, rlt_real dt = 1.0);

	static CTDLearner *getQLearnerFromInput(CGradientQFunction *qFunction, CRewardFunction *rewardFunction, rlt_real dt = 1.0);
};

class CVFunctionLearnerChooser
{
public:
	static CVFunctionLearner *getVFunctionLearner(int learnModeNum, int residualNum, int residualGradientNum, CGradientVFunction *qFunction, CRewardFunction *rewardFunction, rlt_real dt = 1.0);

	static CVFunctionLearner *getVFunctionLearnerFromInput(CGradientVFunction *qFunction, CRewardFunction *rewardFunction, rlt_real dt = 1.0);
};


#endif