ctestscripts.cpp

强化学习算法（R-Learning）难得的珍贵资料

	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscDirect, contSigPolicy, rbfVFunction, "VRBFDiscDirectContSigPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerDirect, contSigPolicy, rbfVFunction, "VRBFEulerDirectContSigPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomDirect, contSigPolicy, rbfVFunction, "VRBFCoulomDirectContSigPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscConstBeta, contSigPolicy, rbfVFunction, "VRBFDiscConstBetaContSigPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomConstBeta, contSigPolicy, rbfVFunction, "VRBFCoulomConstBetaContSigPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerConstBeta, contSigPolicy, rbfVFunction, "VRBFEulerConstBetaContSigPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscVarBeta, contSigPolicy, rbfVFunction, "VRBFDiscVarBetaContSigPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerVarBeta, contSigPolicy, rbfVFunction, "VRBFEulerVarBetaContSigPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomVarBeta, contSigPolicy, rbfVFunction, "VRBFCoulomVarBetaContSigPolicy"));

	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscDirect, contBangBangPolicy, rbfVFunction, "VRBFDiscDirectContBangBangPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerDirect, contBangBangPolicy, rbfVFunction, "VRBFEulerDirectContBangBangPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomDirect, contBangBangPolicy, rbfVFunction, "VRBFCoulomDirectContBangBangPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscConstBeta, contBangBangPolicy, rbfVFunction, "VRBFDiscConstBetaContBangBangPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomConstBeta, contBangBangPolicy, rbfVFunction, "VRBFCoulomConstBetaContBangBangPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerConstBeta, contBangBangPolicy, rbfVFunction, "VRBFEulerConstBetaContBangBangPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscVarBeta, contBangBangPolicy, rbfVFunction, "VRBFDiscVarBetaContBangBangPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerVarBeta, contBangBangPolicy, rbfVFunction, "VRBFEulerVarBetaContBangBangPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomVarBeta, contBangBangPolicy, rbfVFunction, "VRBFCoulomVarBetaContBangBangPolicy"));

	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscDirect, contAddPolicy, rbfVFunction, "VRBFDiscDirectContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerDirect, contAddPolicy, rbfVFunction, "VRBFEulerDirectContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomDirect, contAddPolicy, rbfVFunction, "VRBFCoulomDirectContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscConstBeta, contAddPolicy, rbfVFunction, "VRBFDiscConstBetaContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomConstBeta, contAddPolicy, rbfVFunction, "VRBFCoulomConstBetaContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerConstBeta, contAddPolicy, rbfVFunction, "VRBFEulerConstBetaContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscVarBeta, contAddPolicy, rbfVFunction, "VRBFDiscVarBetaContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerVarBeta, contAddPolicy, rbfVFunction, "VRBFEulerVarBetaContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomVarBeta, contAddPolicy, rbfVFunction, "VRBFCoulomVarBetaContAddPolicy"));


	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscDirect, discAddPolicy, rbfVFunction, "VRBFDiscDirectDiscAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerDirect, discAddPolicy, rbfVFunction, "VRBFEulerDirectDiscAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomDirect, discAddPolicy, rbfVFunction, "VRBFCoulomDirectDiscAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscConstBeta, discAddPolicy, rbfVFunction, "VRBFDiscConstBetaDiscAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomConstBeta, discAddPolicy, rbfVFunction, "VRBFCoulomConstBetaDiscAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerConstBeta, discAddPolicy, rbfVFunction, "VRBFEulerConstBetaDiscAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscVarBeta, discAddPolicy, rbfVFunction, "VRBFDiscVarBetaDiscAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerVarBeta, discAddPolicy, rbfVFunction, "VRBFEulerVarBetaDiscAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomVarBeta, discAddPolicy, rbfVFunction, "VRBFCoulomVarBetaDiscAddPolicy"));


	/*char *discActionTestSuites[] = {"VRBFDiscDirectDiscVMPolicy", "VRBFEulerDirectDiscVMPolicy", "VRBFCoulomDirectDiscVMPolicy", "VRBFDiscConstBetaDiscVMPolicy", "VRBFCoulomConstBetaDiscVMPolicy", "VRBFEulerConstBetaDiscVMPolicy", "VRBFDiscVarBetaDiscVMPolicy", "VRBFEulerVarBetaDiscVMPolicy", "VRBFCoulomVarBetaDiscVMPolicy", "VRBFDiscDirectContVMPolicy", "VRBFEulerDirectContVMPolicy", "VRBFCoulomDirectContVMPolicy", "VRBFDiscConstBetaContVMPolicy", "VRBFCoulomConstBetaContVMPolicy", "VRBFEulerConstBetaContVMPolicy", "VRBFDiscVarBetaContVMPolicy", "VRBFEulerVarBetaContVMPolicy", "VRBFCoulomVarBetaContVMPolicy"}
	
	char *contActionTestSuites[] = {"VRBFDiscDirectContSigPolicy", "VRBFEulerDirectContSigPolicy", "VRBFCoulomDirectContSigPolicy", "VRBFDiscConstBetaContSigPolicy", "VRBFCoulomConstBetaContSigPolicy", "VRBFEulerConstBetaContSigPolicy", "VRBFDiscVarBetaContSigPolicy", "VRBFEulerVarBetaContSigPolicy", "VRBFCoulomVarBetaContSigPolicy", "VRBFDiscDirectContBangBangPolicy", "VRBFEulerDirectContBangBangPolicy", "VRBFCoulomDirectContBangBangPolicy", "VRBFDiscConstBetaContBangBangPolicy", "VRBFCoulomConstBetaContBangBangPolicy", "VRBFEulerConstBetaContBangBangPolicy", "VRBFDiscVarBetaContBangBangPolicy", "VRBFEulerVarBetaContBangBangPolicy", "VRBFCoulomVarBetaContBangBangPolicy", "VRBFDiscDirectContAddPolicy", "VRBFEulerDirectContAddPolicy", "VRBFCoulomDirectContAddPolicy", "VRBFDiscConstBetaContAddPolicy", "VRBFCoulomConstBetaContAddPolicy", "VRBFEulerConstBetaContAddPolicy", "VRBFDiscVarBetaContAddPolicy", "VRBFEulerVarBetaContAddPolicy", "VRBFCoulomVarBetaContAddPolicy","VRBFDiscDirectDiscAddPolicy", "VRBFEulerDirectDiscAddPolicy", "VRBFCoulomDirectDiscAddPolicy", "VRBFDiscConstBetaDiscAddPolicy", "VRBFCoulomConstBetaDiscAddPolicy", "VRBFEulerConstBetaDiscAddPolicy", "VRBFDiscVarBetaDiscAddPolicy", "VRBFEulerVarBetaDiscAddPolicy", "VRBFCoulomVarBetaDiscAddPolicy"};

	for (int i = 0; i < 18; i ++)
	{
		CListenerTestSuite *testSuite = (CListenerTestSuite *) getTestSuite(discActionTestSuites[i]);

		testSuite->add
	}
*/
}


void CMyTestSuiteCollection::addVAdaptiveRBFTestSuites()
{
	agent->addStateModifier(softMaxNet);

	// VFunctionLearner
	CVFunctionNumericInputDerivationCalculator  *vFunctionInputDerivation = new CVFunctionNumericInputDerivationCalculator(dynModel->getStateProperties(), adaptiveRBFFunction, 0.025, agent->getStateModifiers());

	CAbstractQFunction *qFunctionFromTransitionFunction = new CQFunctionFromTransitionFunction(staticContActions, adaptiveRBFFunction, dynModel, rewardFunction, agent->getStateModifiers());

	CAbstractQFunction *contqFunctionFromTransitionFunction = new CContinuousTimeQFunctionFromTransitionFunction(staticContActions, vFunctionInputDerivation, dynModel, rewardFunction, agent->getStateModifiers());

	CVFunctionLearner *vLearnerDiscDirect = new CVFunctionGradientLearner(rewardFunction, adaptiveRBFFunction, new CDiscreteResidual(0.95), new CDirectGradient());
	vLearnerDiscDirect->addErrorListener(adaptiveRBFFunction);

	CVFunctionLearner *vLearnerEulerDirect = new CVFunctionGradientLearner(rewardFunction, adaptiveRBFFunction, new CContinuousEulerResidual(dynModel->getTimeIntervall(), 1.0), new CDirectGradient());
	vLearnerEulerDirect->addErrorListener(adaptiveRBFFunction);

	CVFunctionLearner *vLearnerCoulomDirect = new CVFunctionGradientLearner(rewardFunction, adaptiveRBFFunction, new CContinuousCoulomResidual(dynModel->getTimeIntervall(), 1.0), new CDirectGradient());
	vLearnerCoulomDirect->addErrorListener(adaptiveRBFFunction);


	CVFunctionLearner *vLearnerDiscConstBeta = new CVFunctionGradientLearner(rewardFunction, adaptiveRBFFunction, new CDiscreteResidual(0.95), new CResidualBetaFunction(new CConstantBetaCalculator(0.4), new CDiscreteResidual(0.95)));

	CVFunctionLearner *vLearnerEulerConstBeta = new CVFunctionGradientLearner(rewardFunction, adaptiveRBFFunction, new CContinuousEulerResidual(dynModel->getTimeIntervall(), 1.0),new CResidualBetaFunction(new CConstantBetaCalculator(0.4), new CContinuousEulerResidual(dynModel->getTimeIntervall(), 1.0)));

	CVFunctionLearner *vLearnerCoulomConstBeta = new CVFunctionGradientLearner(rewardFunction, adaptiveRBFFunction, new CContinuousCoulomResidual(dynModel->getTimeIntervall(), 1.0), new CResidualBetaFunction(new CConstantBetaCalculator(0.4), new CContinuousCoulomResidual(dynModel->getTimeIntervall(), 1.0)));

	vLearnerDiscConstBeta->addErrorListener(adaptiveRBFFunction);
	vLearnerEulerConstBeta->addErrorListener(adaptiveRBFFunction);
	vLearnerCoulomConstBeta->addErrorListener(adaptiveRBFFunction);

	CVFunctionLearner *vLearnerDiscVarBeta = new CVFunctionResidualLearner(rewardFunction, adaptiveRBFFunction, new CDiscreteResidual(0.95), new CDiscreteResidual(0.95),  new CVariableBetaCalculator(0.01, 0.9));
	vLearnerDiscVarBeta->addErrorListener(adaptiveRBFFunction);

	CVFunctionLearner *vLearnerEulerVarBeta = new CVFunctionResidualLearner(rewardFunction, adaptiveRBFFunction, new CContinuousEulerResidual(dynModel->getTimeIntervall(), 1.0), new CContinuousEulerResidual(dynModel->getTimeIntervall(), 1.0), new CVariableBetaCalculator(0.01, 0.9));
	vLearnerEulerVarBeta->addErrorListener(adaptiveRBFFunction);

	CVFunctionLearner *vLearnerCoulomVarBeta = new CVFunctionResidualLearner(rewardFunction, adaptiveRBFFunction, new CContinuousCoulomResidual(dynModel->getTimeIntervall(), 1.0), new CContinuousCoulomResidual(dynModel->getTimeIntervall(), 1.0), new CVariableBetaCalculator(0.01, 0.9));
	vLearnerCoulomVarBeta->addErrorListener(adaptiveRBFFunction);

	/*CContinuousActionPolicy *discVMPolicy = new CContinuousActionPolicy(dynModel->getContinuousAction(), new CGreedyDistribution(), qFunctionFromTransitionFunction, staticContActions, -1.0);

	CContinuousActionPolicy *contAddPolicy = new CContinuousActionPolicy(dynModel->getContinuousAction(), new CSoftMaxDistribution(), contqFunctionFromTransitionFunction, staticContActions, -1.0);*/

	CAgentController *discVMPolicy = new CVMStochasticPolicy(staticContActions, new CSoftMaxDistribution(10), adaptiveRBFFunction, dynModel, rewardFunction, agent->getStateModifiers());

	CAgentController *contVMPolicy = new CContinuousTimeVMPolicy(staticContActions, new CSoftMaxDistribution(10), vFunctionInputDerivation, dynModel, rewardFunction);

	CContinuousTimeAndActionSigmoidVMPolicy *contSigPolicy = new CContinuousTimeAndActionSigmoidVMPolicy(dynModel->getContinuousAction(), vFunctionInputDerivation, dynModel);

	contSigPolicy->setParameter("SigmoidPolicyCFactor", 100);

	CContinuousTimeAndActionBangBangVMPolicy *contBangBangPolicy = new CContinuousTimeAndActionBangBangVMPolicy(dynModel->getContinuousAction(), vFunctionInputDerivation, dynModel);

	contSigPolicy->setRandomController(contExploration);
	contBangBangPolicy->setRandomController(contExploration);

	CContinuousActionPolicy *discAddPolicy = new CContinuousActionPolicy(dynModel->getContinuousAction(), new CSoftMaxDistribution(1000.0), qFunctionFromTransitionFunction, staticContActions, 5.5);

	CContinuousActionPolicy *contAddPolicy = new CContinuousActionPolicy(dynModel->getContinuousAction(), new CSoftMaxDistribution(1000.0), contqFunctionFromTransitionFunction, staticContActions, 5.5);

	contSigPolicy->setRandomController(contExploration);
	contBangBangPolicy->setRandomController(contExploration);


	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscDirect, discVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFDiscDirectDiscVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerDirect, discVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFEulerDirectDiscVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomDirect, discVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFCoulomDirectDiscVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscConstBeta, discVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFDiscConstBetaDiscVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomConstBeta, discVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFCoulomConstBetaDiscVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerConstBeta, discVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFEulerConstBetaDiscVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscVarBeta, discVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFDiscVarBetaDiscVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerVarBeta, discVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFEulerVarBetaDiscVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomVarBeta, discVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFCoulomVarBetaDiscVMPolicy"));

	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscDirect, contVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFDiscDirectContVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerDirect, contVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFEulerDirectContVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomDirect, contVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFCoulomDirectContVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscConstBeta, contVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFDiscConstBetaContVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerCoulomConstBeta, contVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFCoulomConstBetaContVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerConstBeta, contVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFEulerConstBetaContVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerDiscVarBeta, contVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFDiscVarBetaContVMPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vLearnerEulerVarBeta, contVMPolicy, adaptiveRBFFunction, "VAdaptiveRBFEulerVarBetaContVMPolicy"));