clearneddynamicmodel.h

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#ifndef C__LEARNEDMODEL__H
#define C__LEARNEDMODEL__H

#include "cstatecollection.h"
#include "caction.h"
#include "cagentlistener.h"
#include "csupervisedlearner.h"
#include "ctransitionfunction.h"
#include "cagent.h"
#include "ril_debug.h"
#include <map>

class CLearnedTransitionFunction : public CTransitionFunction, public CSemiMDPListener, public CLearnDataObject
{
protected:
	CMyVector *input;
	CMyVector *target;

	CSupervisedLearner *learner;
	CAction *actionToLearn;

	CState *modelState;

	bool learnDuration;

	CActionData *lActionData;
	CMyVector *outputError;

	virtual void getInputVector(CState *oldState, CActionData *data, CMyVector *input);
	virtual void getTargetVector(CState *newState, CActionData *data, CMyVector *target);
public:
	CLearnedTransitionFunction(CStateProperties *properties, CAction *action, CSupervisedLearner *learner, bool learnDuration = false);
	~CLearnedTransitionFunction();

	virtual void nextStep(CStateCollection *oldState, CAction *action, CStateCollection *newState); 
	virtual void doLearnStep(CState *oldState, CAction *, CActionData *data, CState *newState);

	virtual void transitionFunction(CState *oldState, CAction *action, CState *newState, CActionData *data);

	virtual void saveData(FILE *stream);
	virtual void loadData(FILE *stream);

	virtual void resetData();

};

class CComposedLearnedTransitionFunction : public CComposedTransitionFunction, public CSemiMDPListener, CLearnDataObject
{
protected:
	std::map<CTransitionFunction *, CLearnedTransitionFunction *> *learnedModels;
public:
	CComposedLearnedTransitionFunction(CStateProperties *properties);
	~CComposedLearnedTransitionFunction();

	virtual void addTransitionFunction(CLearnedTransitionFunction *model);

	virtual void nextStep(CStateCollection *oldState, CAction *action, CStateCollection *newState); 

	virtual void saveData(FILE *stream);
	virtual void loadData(FILE *stream);

	virtual void resetData();
};

class CStochasticModelTrainer : public CActionObject, public CSemiMDPSender
{
protected:
	CSemiMDPListener *learnModel;
	CTransitionFunction *trainModel;

	CState *currentState;
	CState *nextState;

	CActionDataSet *actionDataSet;

public:
	CStochasticModelTrainer(CTransitionFunction *trainModel, CActionSet *actions, CSemiMDPListener *learnModel);
	~CStochasticModelTrainer();

	void doRandomLearnStep();
	void doNLearnSteps(int N);
};

class CTransitionFunctionComparer : public CSemiMDPListener
{
protected:
	CTransitionFunction *compareModel;

	rlt_real contMse;
	int discreteFailures;

	rlt_real maxMse;
	rlt_real maxDiscreteFailure;

	int nExamples;

	CState *compareState1;
	CStateProperties *compareState2Properties;
public:
	CTransitionFunctionComparer(CTransitionFunction *dynModel, CStateProperties *compareState);

	virtual void nextStep(CStateCollection *oldState, CAction *action, CStateCollection *nextState);

	virtual void resetStatistics();

	rlt_real getContinuousMSE();
	rlt_real getContinuousMaxSquaredError();

	rlt_real getDiscreteFailures();
	rlt_real getMaxDiscreteFailure();
};

#endif