caction.h

强化学习算法（R-Learning）难得的珍贵资料

// Copyright (C) 2003
// Gerhard Neumann ([e-mail])

//                
// This file is part of RL Toolbox.
// http://www.igi.tugraz.at/ril_toolbox
//
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#ifndef CACTION_H
#define CACTION_H

#include <stdio.h> 
#include <vector>
#include <list>
#include <map>

#include "ril_debug.h"

class CStateCollection;
class CStateProperties;

#define EXTENDEDACTION 1
#define MULTISTEPACTION 2
#define PRIMITIVEACTION 4
#define CONTINUOUSACTION 16
#define CONTINUOUSSTATICACTION 32

/// interface for saving changable data of an action
/** Since the "Id" of an action is the pointer itself, and this pointer is used in every action set, we can't set the changeable data of an action that easily, since it would change the action's data everywhere in the program. If this isn't wanted (as usual), we have to use action data objects. New action data objects can be retrieved from any action class (if there is no changeable data for an action the action will return a NULL pointer). This action data object is now local, and not global any more and can therefore be changed easily. If an action has changeable data, the action pointer and a coresponding action data object determine the action. This is used in many function-arguments. Normally, if the action-data object is missing (= NULL) a new action data object is retrieved from the action.
\par
At the momemt its  used for saving the data of MultiStepActions (duration, finished), and ContinousActions (saving the continuous action value)
all other classes return NULL when asking them for a CActionData Object. 
@see CAction
@see CMultiStepActionData
@see CActionList
*/
class CActionData
{
protected:
	bool bIsChangeAble;
public:
	CActionData();
	
	virtual void saveASCII(FILE *stream) = 0;
	virtual void loadASCII(FILE *stream) = 0;

	virtual void saveBIN(FILE *stream) = 0;
	virtual void loadBIN(FILE *stream) = 0;

	/// Set the changeable data according to the actiondata object
	virtual void setData(CActionData *actionData) = 0;

	bool isChangeAble();
	void setIsChangeAble(bool changeAble);
};


/// class for saving the duration and the finished flag from a MultiStepAction
/**
@see CActionData
*/
class CMultiStepActionData : public CActionData
{
public:
	CMultiStepActionData();

	int duration;
	bool finished;

	virtual void saveASCII(FILE *stream);
	virtual void loadASCII(FILE *stream);

	virtual void saveBIN(FILE *stream);
	virtual void loadBIN(FILE *stream);

	/// Set the changeable data according to the actiondata object
	virtual void setData(CActionData *actionData);

};



class CContinuousActionProperties;
class CActionDataSet;


///Class Representing an action the agent or any other SMDP can choose from. 
/** 
The class represents an interface for all kind of actions. The interface consists of an action type, methods for handling the action data object of the action and the isAvailable Method.
The action itself is identified with the pointer of the action abject together with an specific actionSet. So if you want to know the index of an action, you first must have an action set and then call its getIndex(CAction *) method. 
The class is just an interface for all other actions. It maintains type field to determine which type the action is. There are
following types:
<ul>
<li> EXTENDEDACTION </li>
<li> MULTISTEPACTION </li>
<li> PRIMITIVEACTION </li>
<li> CONTINUOUSACTION </li>
<li> CONTINUOUSSTATICACTION </li>
</ul>

Each type belongs to a specific class. Don't add a type if the class isn't subclass of the class belonging to the type! This will lead to type cast errors. If the type is added the class has to be cast-able to the class representing the type.
\par 
All the direct subclasses of CAction have CAction as a virtual base class, so you can combine the attributes of the different actions.
For example you can create an PrimitiveAction which can last for several steps (deviated from CPrimitiveAction and CMultiStepAction). Be carefull, due to the virtual base class you ALWAYS have to do a dynamic_cast instead of a normal type cast if you want to cast any CAction class object.
\par
It also has the member isAvailable(CStateCollection *), which is per default always true. With this function you can exclude
specific actions in some states, when they are not avialable for the agent. Befor choosing an action, each Controller first determines
 the list of available actions from its action set to choose a certain action. 
 Since the Method isAvailable gets an state collection you don't have to use the model state, you also have acces to other modified states (for example a discrete state). 
The index of action can't be identified by the action itself, you need always an actionset to get the index of the action.
*/
class CAction
{
protected:
/**
The type field is an integer, but it serves as a bitmask. So if you want to add an specific type to the Action 
you have to add it with an or mask. If you want to add a new general typ of Actions you have to create a new Typenumber. 
Since the bitmask only 2^n numbers are allowed.
*/
	int type;
	CActionData *actionData;

	CAction(CActionData *actionData);

public:
    CAction();
	virtual ~CAction();
	
    int getType();
	bool isType(int type);

/// adds a specific type to the type field bitmap
/**
So The parameter should be a power of 2, because al bits in the "Type" parameter gets set with an OR mask
to the internal type.
*/
	void addType(int Type);

	
	/// Returns an new CActionData object of the specific sub-class.
	/**
	The CActionData Object is created with new and must be deleted by the programmer!
	*/
	virtual CActionData *getNewActionData();

	
	/// Set the changeable data according to the actiondata object
	virtual void loadActionData(CActionData *actionData);
	/// Returns an actiondata object initialised with the actions values
	/**
	The CActionData Object is created with new and must be deleted by the programmer!
	*/
	virtual CActionData * getActionData();

	

	virtual bool isAvailable(CStateCollection *) {return true ;};//return isAvailable(state->getState());};
	//virtual bool isAvailable(CState *state) {return true;};

	///returns the duration of the action, per default 1
	virtual int getDuration() {return 1;};

	///Determines wether the 2 actions represent the same action
	virtual bool equals(CAction *action);

	/// Compares the action and the actionData obejcts.
	virtual bool isSameAction(CAction *action, CActionData *data);


};

/// A list of Actions representing the actual hierarchical Stack
/** The actual hierarchical stack contains alls ExtendedActions which are aktiv at the moment and the 
PrimitiveAction which is returned by the last extended action. 
*/
class CHierarchicalStack: public std::list<CAction *> 
{
protected:
	
public:
	CHierarchicalStack();
	~CHierarchicalStack();
	
	void clearAndDelete();
};