ctestscripts.h

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


#ifndef C_PENDULUMTESTSCRIPT__H
#define C_PENDULUMTESTSCRIPT__H

#include "cpendulummodel.h"
#include "cpolicies.h"
#include "cagent.h"
#include "ril_debug.h"
#include "ctdlearner.h"
#include "clinearfafeaturecalculator.h"
#include "cvfunctionlearner.h"
#include "crewardmodel.h"
#include "ctorchvfunction.h"
#include "canalyzer.h"
#include "ccontinuoustime.h"
#include "cmyexception.h"
#include "cadaptivesoftmaxnetwork.h"


#include <list>
#include <string>
#include <iostream>

class CMyTestSuiteCollection : public CTestSuiteCollection
{
protected:

	//virtual void initModelVariables(rlt_real dt, CFeatureCalculator *rbfCalc);
public:
	CAgent *agent;
	char *testSuiteDirectory;

	CDynamicContinuousTimeAndActionModel *dynModel;
	CTransitionFunctionEnvironment *modelEnvironment;

	CFeatureCalculator *rbfCalculator;
	CFeatureVFunction *rbfVFunction;
	CVFunctionFromGradientFunction *nnVFunction;

	CFeatureQFunction *rbfQFunction;
	CQFunction *nnQFunctionSeperate;
	CQFunctionFromGradientFunction *nnQFunction;

	CActionSet *staticContActions;
	CStateReward *rewardFunction;

	CStateModifier *nnState;

	CStateModifier *gaussSigmoidState;
	CVFunctionFromGradientFunction *gaussSigmoidVFunction;

//	CAdaptiveParameterFromAverageRewardCalculator *discreteExplorationCalculator;
//	CAdaptiveParameterFromAverageRewardCalculator *continuousExplorationCalculator;

//	CActionDistribution *discExploration;
	CContinuousActionRandomPolicy *contExploration;

	CContinuousActionGradientPolicy *gradientPolicyNeuralNet;
	CContinuousActionGradientPolicy *gradientPolicyRBF;

	CAdaptiveSoftMaxVFunction *adaptiveRBFFunction;
	CAdaptiveSoftMaxNetwork *softMaxNet;

	CMyTestSuiteCollection(char *testSuiteDirectory);
	virtual ~CMyTestSuiteCollection();

	void addVRBFTestSuites();
	void addVAdaptiveRBFTestSuites();
	void addVNNTestSuites();
	void addVNNDelayedUpdateTestSuites();

	void addVNNTestSuitesStateETraces();

	void addVNNTestSuiteAdaptiveLearningRate();


	void addVLearnModelTestSuites();

	void addQRBFTestSuites();
	void addQNNTestSuites();
	void addQNNSeperateTestSuites();


	void addSarsaRBFTestSuites();
	void addSarsaNNTestSuites();

	void addQNNTestSuitesAdaptiveLearningRate();

	void addAdvantageTestSuites();
	void addActorCriticTestSuites(CResidualFunction *residual = NULL, CResidualGradientFunction *gradient  = NULL, string suffix = "");

	void addReinforceTestSuites();

	void addCONJMDPTestSuites();
	void addPEGASUSTestSuites();

	void addNumericPEGASUSTestSuites();

	void addPrioritizedSweepingTestSuites();

	void addVPolicyTestSuites(CResidualFunction *residual = NULL, CResidualGradientFunction *gradient  = NULL, string suffix = "");

	void addVGaussianSigmoidNetworksTestSuites(); 
};

class CPendulumTestSuiteCollection : public CMyTestSuiteCollection
{
protected:

	virtual void initModelVariables(rlt_real dt, CFeatureCalculator *rbfCalc);
public:
	CPendulumTestSuiteCollection(rlt_real dt = 0.05);
	CPendulumTestSuiteCollection(rlt_real dt, CFeatureCalculator *rbfCalc);

	virtual ~CPendulumTestSuiteCollection();
};

class CCartPoleTestSuiteCollection : public CMyTestSuiteCollection
{
protected:

	virtual void initModelVariables(rlt_real dt, CFeatureCalculator *rbfCalc);
public:
	CCartPoleTestSuiteCollection(rlt_real dt = 0.05);
	CCartPoleTestSuiteCollection(rlt_real dt, CFeatureCalculator *rbfCalc);

	virtual ~CCartPoleTestSuiteCollection();
};

class CAcroBotTestSuiteCollection : public CMyTestSuiteCollection
{
protected:

	virtual void initModelVariables(rlt_real dt, CFeatureCalculator *rbfCalc, rlt_real uMax = 2.0);
	int rewardFunctionType;
public:
	CAcroBotTestSuiteCollection(rlt_real dt = 0.05, int rewardFunctionType = 0, rlt_real uMax = 2);
	CAcroBotTestSuiteCollection(rlt_real dt, CFeatureCalculator *rbfCalc, int rewardFunctionType = 0, rlt_real uMax = 2);

	virtual ~CAcroBotTestSuiteCollection();
};

/*
class CTestScriptCommand
{
protected:
	std::list<CTestScriptCommand *> *childCommands;
public:
	CTestScriptCommand();
	virtual ~CTestScriptCommand();

	virtual void doCommand(CTestSuiteParameterCalculator *paramCalculator, CTestSuiteEvaluator *evaluator, CTestSuite *testSuite) = 0;

	void addChildCommand(CTestScriptCommand *command);

	virtual void executeChildCommands(CTestSuiteParameterCalculator *paramCalculator, CTestSuiteEvaluator *evaluator, CTestSuite *testSuite);
};

class CConditionCommand : public CTestScriptCommand
{
protected:
	std::list<string> *testConditionParameters;
	std::list<bool> *testCondition;
public:
	CConditionCommand();
	virtual ~CConditionCommand();

	virtual void doCommand(CTestSuiteParameterCalculator *paramCalculator, CTestSuiteEvaluator *evaluator, CTestSuite *testSuite);

	void addTestCondition(string Parameter, bool member);
};

class CForCommand : public CTestScriptCommand
{
protected:
	string paramName;
	std::list<rlt_real> *values;
public:
	CForCommand(string paramName);
	virtual ~CForCommand();

	virtual void doCommand(CTestSuiteParameterCalculator *paramCalculator, CTestSuiteEvaluator *evaluator, CTestSuite *testSuite);
	void addForValue(rlt_real value);
};

class CSetCommand : public CTestScriptCommand, public CParameters
{
protected:
	
public:
	CSetCommand();
	virtual ~CSetCommand();

	virtual void doCommand(CTestSuiteParameterCalculator *paramCalculator, CTestSuiteEvaluator *evaluator, CTestSuite *testSuite);
};

class CSearchCommand : public CTestScriptCommand
{
protected:
	rlt_real startValue;
	int maxIterations;
	string paramName;
public:
	CSearchCommand(string parameterName, rlt_real startValue, int maxIterations);
	virtual ~CSearchCommand() {};

	virtual void doCommand(CTestSuiteParameterCalculator *paramCalculator, CTestSuiteEvaluator *evaluator, CTestSuite *testSuite);
};

class CEvaluateCommand : public CTestScriptCommand
{
protected:
public:
	CEvaluateCommand();
	virtual ~CEvaluateCommand() {};
	virtual void doCommand(CTestSuiteParameterCalculator *paramCalculator, CTestSuiteEvaluator *evaluator, CTestSuite *testSuite);
};

class CTestCommand : public CTestScriptCommand
{
protected:
	string paramName;
	std::list<rlt_real> *values;
public:
	CTestCommand(string paramName);
	virtual ~CTestCommand();

	virtual void doCommand(CTestSuiteParameterCalculator *paramCalculator, CTestSuiteEvaluator *evaluator, CTestSuite *testSuite);
	void addTestValue(rlt_real value);
};

class CChooseBestParametersCommand : public CTestScriptCommand
{
protected:
	CParameters *parameters;
	std::map<string, char> *compareMethod;
public:
	CChooseBestParametersCommand();
	virtual ~CChooseBestParametersCommand();

	virtual void doCommand(CTestSuiteParameterCalculator *paramCalculator, CTestSuiteEvaluator *evaluator, CTestSuite *testSuite);
	bool testParameters(CParameters *paramObject);
	void addParameterCondition(string name, char method, rlt_real value);
};


class CTestScript : public CTestScriptCommand
{
protected:


public:
	CMyTestSuiteCollection *testSuiteCollection;

	CTestSuiteParameterCalculator *paramCalculator;
	CTestSuiteEvaluator *evaluator;
	CPolicyEvaluator *policyEvaluator;

	CTestScript();
	virtual ~CTestScript();

	virtual void doCommand(CTestSuiteParameterCalculator *paramCalculator, CTestSuiteEvaluator *evaluator, CTestSuite *testSuite);
	virtual void loadTestScript(FILE *stream);

	virtual void executeTestScript();
	virtual void executeTestScriptOnTestSuite(CTestSuite *testSuite);

	void loadChildCommands(CTestScriptCommand *parentCommand, string endCommand,  FILE *stream);
};

class CWrongTestScriptFormatException : public CMyException
{
protected:
	virtual string getInnerErrorMsg();
	string errorName;
public:
	CWrongTestScriptFormatException(string errorName);
	virtual ~CWrongTestScriptFormatException() {};
};
*/
#endif