ctestscripts.cpp

强化学习算法（R-Learning）难得的珍贵资料

	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscVarBeta, NNcontBangBangPolicy, nnVFunction, "VNNDiscVarBetaContBangBangPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerVarBeta, NNcontBangBangPolicy, nnVFunction, "VNNEulerVarBetaContBangBangPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomVarBeta, NNcontBangBangPolicy, nnVFunction, "VNNCoulomVarBetaContBangBangPolicy"));

	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscDirect, contAddPolicy, nnVFunction, "VNNDiscDirectContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerDirect, contAddPolicy, nnVFunction, "VNNEulerDirectContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomDirect, contAddPolicy, nnVFunction, "VNNCoulomDirectContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscConstBeta, contAddPolicy, nnVFunction, "VNNDiscConstBetaContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomConstBeta, contAddPolicy, nnVFunction, "VNNCoulomConstBetaContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerConstBeta, contAddPolicy, nnVFunction, "VNNEulerConstBetaContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscVarBeta, contAddPolicy, nnVFunction, "VNNDiscVarBetaContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerVarBeta, contAddPolicy, nnVFunction, "VNNEulerVarBetaContAddPolicy"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomVarBeta, contAddPolicy, nnVFunction, "VNNCoulomVarBetaContAddPolicy"));
}

void CMyTestSuiteCollection::addVNNDelayedUpdateTestSuites()
{
	agent->addStateModifier(nnState);

	// NN V Functions
	CVFunctionNumericInputDerivationCalculator  *vFunctionInputDerivation = new CVFunctionNumericInputDerivationCalculator(dynModel->getStateProperties(), nnVFunction, 0.025, agent->getStateModifiers());

	CGradientDelayedUpdateVFunction *delayedVFunction = new CGradientDelayedUpdateVFunction(nnVFunction);

	CSemiMDPListener *updater = new CDelayedFunctionUpdater(delayedVFunction, 100, 1);

	agent->addSemiMDPListener(updater);


	CAbstractQFunction *qFunctionFromTransitionFunction = new CQFunctionFromTransitionFunction(staticContActions, rbfVFunction, dynModel, rewardFunction, agent->getStateModifiers());

	CVFunctionLearner *vNNLearnerDiscDirect = new CVFunctionGradientLearner(rewardFunction, delayedVFunction, new CDiscreteResidual(0.95), new CDirectGradient());

	CVFunctionLearner *vNNLearnerEulerDirect = new CVFunctionGradientLearner(rewardFunction, delayedVFunction, new CContinuousEulerResidual(dynModel->getTimeIntervall(), 1.0), new CDirectGradient());

	CVFunctionLearner *vNNLearnerCoulomDirect = new CVFunctionGradientLearner(rewardFunction, delayedVFunction, new CContinuousCoulomResidual(dynModel->getTimeIntervall(), 1.0), new CDirectGradient());

	CVFunctionLearner *vNNLearnerDiscConstBeta = new CVFunctionResidualLearner(rewardFunction, delayedVFunction, new CDiscreteResidual(0.95), new CDiscreteResidual(0.95), new CConstantBetaCalculator(0.4));

	CVFunctionLearner *vNNLearnerEulerConstBeta = new CVFunctionResidualLearner(rewardFunction, delayedVFunction, new CContinuousEulerResidual(dynModel->getTimeIntervall(), 1.0), new CContinuousEulerResidual(dynModel->getTimeIntervall(), 1.0), new CConstantBetaCalculator(0.4));

	CVFunctionLearner *vNNLearnerCoulomConstBeta = new CVFunctionResidualLearner(rewardFunction, delayedVFunction, new CContinuousCoulomResidual(dynModel->getTimeIntervall(), 1.0), new CContinuousCoulomResidual(dynModel->getTimeIntervall(), 1.0), new CConstantBetaCalculator(0.4));

	CVFunctionLearner *vNNLearnerDiscVarBeta = new CVFunctionResidualLearner(rewardFunction, delayedVFunction, new CDiscreteResidual(0.95), new CDiscreteResidual(0.95),  new CVariableBetaCalculator(0.01, 0.9));

	CVFunctionLearner *vNNLearnerEulerVarBeta = new CVFunctionResidualLearner(rewardFunction, delayedVFunction, new CContinuousEulerResidual(dynModel->getTimeIntervall(), 1.0), new CContinuousEulerResidual(dynModel->getTimeIntervall(), 1.0), new CVariableBetaCalculator(0.01, 0.9));

	CVFunctionLearner *vNNLearnerCoulomVarBeta = new CVFunctionResidualLearner(rewardFunction, delayedVFunction, new CContinuousCoulomResidual(dynModel->getTimeIntervall(), 1.0), new CContinuousCoulomResidual(dynModel->getTimeIntervall(), 1.0), new CVariableBetaCalculator(0.01, 0.9));



	CAgentController *NNdiscVMPolicy = new CVMStochasticPolicy(staticContActions, new CSoftMaxDistribution(10.0), nnVFunction, dynModel, rewardFunction, agent->getStateModifiers());

	CAgentController *NNdiscEpsVMPolicy = new CVMStochasticPolicy(staticContActions, new CEpsilonGreedyDistribution(0.1), nnVFunction, dynModel, rewardFunction, agent->getStateModifiers());

	CAgentController *NNcontVMPolicy = new CContinuousTimeVMPolicy(staticContActions, new CSoftMaxDistribution(10.0), vFunctionInputDerivation, dynModel, rewardFunction);

	CContinuousTimeAndActionSigmoidVMPolicy *NNcontSigPolicy = new CContinuousTimeAndActionSigmoidVMPolicy(dynModel->getContinuousAction(), vFunctionInputDerivation, dynModel);

	NNcontSigPolicy->setParameter("SigmoidPolicyCFactor", 100.0);

	CContinuousTimeAndActionBangBangVMPolicy *NNcontBangBangPolicy = new CContinuousTimeAndActionBangBangVMPolicy(dynModel->getContinuousAction(), vFunctionInputDerivation, dynModel);

	CContinuousActionPolicy *contAddPolicy = new CContinuousActionPolicy(dynModel->getContinuousAction(), new CSoftMaxDistribution(10.0), qFunctionFromTransitionFunction, staticContActions, 5.5);


	NNcontSigPolicy->setRandomController(contExploration);
	NNcontBangBangPolicy->setRandomController(contExploration);

	contAddPolicy->setRandomController(contExploration);

	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscDirect, NNdiscVMPolicy, nnVFunction, "VNNDiscDirectDiscVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerDirect, NNdiscVMPolicy, nnVFunction, "VNNEulerDirectDiscVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomDirect, NNdiscVMPolicy, nnVFunction, "VNNCoulomDirectDiscVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscConstBeta, NNdiscVMPolicy, nnVFunction, "VNNDiscConstBetaDiscVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomConstBeta, NNdiscVMPolicy, nnVFunction, "VNNCoulomConstBetaDiscVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerConstBeta, NNdiscVMPolicy, nnVFunction, "VNNEulerConstBetaDiscVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscVarBeta, NNdiscVMPolicy, nnVFunction, "VNNDiscVarBetaDiscVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerVarBeta, NNdiscVMPolicy, nnVFunction, "VNNEulerVarBetaDiscVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomVarBeta, NNdiscVMPolicy, nnVFunction, "VNNCoulomVarBetaDiscVMPolicyDelay"));

	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscDirect, NNdiscEpsVMPolicy, nnVFunction, "VNNDiscDirectDiscVMPolicy1Delay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerDirect, NNdiscEpsVMPolicy, nnVFunction, "VNNEulerDirectDiscVMPolicy1Delay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomDirect, NNdiscEpsVMPolicy, nnVFunction, "VNNCoulomDirectDiscVMPolicy1Delay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscConstBeta, NNdiscEpsVMPolicy, nnVFunction, "VNNDiscConstBetaDiscVMPolicy1Delay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomConstBeta, NNdiscEpsVMPolicy, nnVFunction, "VNNCoulomConstBetaDiscVMPolicy1Delay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerConstBeta, NNdiscEpsVMPolicy, nnVFunction, "VNNEulerConstBetaDiscVMPolicy1Delay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscVarBeta, NNdiscEpsVMPolicy, nnVFunction, "VNNDiscVarBetaDiscVMPolicy1Delay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerVarBeta, NNdiscEpsVMPolicy, nnVFunction, "VNNEulerVarBetaDiscVMPolicy1Delay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomVarBeta, NNdiscEpsVMPolicy, nnVFunction, "VNNCoulomVarBetaDiscVMPolicy1Delay"));


	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscDirect, NNcontVMPolicy, nnVFunction, "VNNDiscDirectContVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerDirect, NNcontVMPolicy, nnVFunction, "VNNEulerDirectContVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomDirect, NNcontVMPolicy, nnVFunction, "VNNCoulomDirectContVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscConstBeta, NNcontVMPolicy, nnVFunction, "VNNDiscConstBetaContVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomConstBeta, NNcontVMPolicy, nnVFunction, "VNNCoulomConstBetaContVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerConstBeta, NNcontVMPolicy, nnVFunction, "VNNEulerConstBetaContVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscVarBeta, NNcontVMPolicy, nnVFunction, "VNNDiscVarBetaContVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerVarBeta, NNcontVMPolicy, nnVFunction, "VNNEulerVarBetaContVMPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomVarBeta, NNcontVMPolicy, nnVFunction, "VNNCoulomVarBetaContVMPolicyDelay"));

	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscDirect, NNcontSigPolicy, nnVFunction, "VNNDiscDirectContSigPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerDirect, NNcontSigPolicy, nnVFunction, "VNNEulerDirectContSigPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomDirect, NNcontSigPolicy, nnVFunction, "VNNCoulomDirectContSigPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscConstBeta, NNcontSigPolicy, nnVFunction, "VNNDiscConstBetaContSigPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomConstBeta, NNcontSigPolicy, nnVFunction, "VNNCoulomConstBetaContSigPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerConstBeta, NNcontSigPolicy, nnVFunction, "VNNEulerConstBetaContSigPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscVarBeta, NNcontSigPolicy, nnVFunction, "VNNDiscVarBetaContSigPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerVarBeta, NNcontSigPolicy, nnVFunction, "VNNEulerVarBetaContSigPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomVarBeta, NNcontSigPolicy, nnVFunction, "VNNCoulomVarBetaContSigPolicyDelay"));

	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscDirect, NNcontBangBangPolicy, nnVFunction, "VNNDiscDirectContBangBangPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerDirect, NNcontBangBangPolicy, nnVFunction, "VNNEulerDirectContBangBangPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomDirect, NNcontBangBangPolicy, nnVFunction, "VNNCoulomDirectContBangBangPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscConstBeta, NNcontBangBangPolicy, nnVFunction, "VNNDiscConstBetaContBangBangPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomConstBeta, NNcontBangBangPolicy, nnVFunction, "VNNCoulomConstBetaContBangBangPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerConstBeta, NNcontBangBangPolicy, nnVFunction, "VNNEulerConstBetaContBangBangPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscVarBeta, NNcontBangBangPolicy, nnVFunction, "VNNDiscVarBetaContBangBangPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerVarBeta, NNcontBangBangPolicy, nnVFunction, "VNNEulerVarBetaContBangBangPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomVarBeta, NNcontBangBangPolicy, nnVFunction, "VNNCoulomVarBetaContBangBangPolicyDelay"));

	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscDirect, contAddPolicy, nnVFunction, "VNNDiscDirectContAddPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerDirect, contAddPolicy, nnVFunction, "VNNEulerDirectContAddPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomDirect, contAddPolicy, nnVFunction, "VNNCoulomDirectContAddPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscConstBeta, contAddPolicy, nnVFunction, "VNNDiscConstBetaContAddPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomConstBeta, contAddPolicy, nnVFunction, "VNNCoulomConstBetaContAddPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerConstBeta, contAddPolicy, nnVFunction, "VNNEulerConstBetaContAddPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerDiscVarBeta, contAddPolicy, nnVFunction, "VNNDiscVarBetaContAddPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerEulerVarBeta, contAddPolicy, nnVFunction, "VNNEulerVarBetaContAddPolicyDelay"));
	addTestSuite(new CListenerTestSuite(agent, vNNLearnerCoulomVarBeta, contAddPolicy, nnVFunction, "VNNCoulomVarBetaContAddPolicyDelay"));
}

void CMyTestSuiteCollection::addVNNTestSuitesStateETraces()
{
	agent->addStateModifier(nnState);

	std::list<CStateModifier *> *list = new std::list<CStateModifier *>();
	list->push_back(this->nnState);
	CAbstractVETraces *vETraces = new CStateVETraces(nnVFunction, dynModel->getStateProperties(), list);