cvfunction.cpp

强化学习算法（R-Learning）难得的珍贵资料

	{
		tempFunction = new COptimalVFunctionFromQFunction(qfunction, qfunction->getFeatureCalculator());
	}

	for (unsigned int i = 0; i < numFeatures; i++)
	{
		discState.setDiscreteState(0, i);
		setFeature(i, tempFunction->getValue(&discState));
	}
	delete tempFunction;
}


void CFeatureVFunction::getGradient(CStateCollection *stateCol, CFeatureList *gradient)
{
	CState *state = stateCol->getState(properties);

	int type = state->getStateProperties()->getType() & (DISCRETESTATE | FEATURESTATE);
	switch (type)
	{
	case FEATURESTATE:
		{
			for (unsigned int i = 0; i < state->getNumActiveDiscreteStates(); i ++)
			{
				gradient->set(state->getDiscreteState(i), state->getContinuousState(i));
			}
			break;
		}
	case DISCRETESTATE:
		{
			gradient->set(state->getDiscreteState(0), 1.0);
			break;
		}
	default:
		{
		}
	}
	if (DebugIsEnabled('v'))
	{
		DebugPrint('v', "Calculating feature Gradient List : ");
		gradient->saveASCII(DebugGetFileHandle('v'));
		DebugPrint('v', "\n");
	}
}

void CFeatureVFunction::updateValue(CState *state, rlt_real td)
{
	if (DebugIsEnabled('v'))
	{
		DebugPrint('v', "Update V-Value: %f", td);
		state->saveASCII(DebugGetFileHandle('v'));
		DebugPrint('v', "\n");
	}

	int type = state->getStateProperties()->getType() & (DISCRETESTATE | FEATURESTATE);
	switch (type)
	{
		case FEATURESTATE:
			{
				for (unsigned int i = 0; i < state->getNumActiveDiscreteStates(); i++)
				{
					updateFeature(state->getDiscreteState(i), td * state->getContinuousState(i));
				}
				break;
			}
		case DISCRETESTATE:
			{
				updateFeature(state->getDiscreteState(0), td);
				break;
			}
		default:
			{
			}
	}
}

void CFeatureVFunction::setValue(CState *state, rlt_real qValue)
{
	if (DebugIsEnabled('v'))
	{
		DebugPrint('v', "Set V-Value: %f", qValue);
		state->saveASCII(DebugGetFileHandle('v'));
		DebugPrint('v', "\n");
	}

	int type = state->getStateProperties()->getType() & (DISCRETESTATE | FEATURESTATE);
	switch (type)
	{
		case FEATURESTATE:
			{
				for (unsigned int i = 0; i < state->getNumActiveDiscreteStates(); i++)
				{
					setFeature(state->getDiscreteState(i), qValue * state->getContinuousState(i));
				}
				break;
			}
		case DISCRETESTATE:
			{
				setFeature(state->getDiscreteState(0), qValue);
				break;
			}
		default:
			{
			}
	}
}

rlt_real CFeatureVFunction::getValue(CState *state)
{
	rlt_real value = 0;

	int type = state->getStateProperties()->getType() & (DISCRETESTATE | FEATURESTATE);
	switch (type)
	{
		case FEATURESTATE:
			{
				for (unsigned int i = 0; i < state->getNumActiveDiscreteStates(); i++)
				{
					value += getFeature(state->getDiscreteState(i)) * state->getContinuousState(i);
				}
				break;
			}
		case DISCRETESTATE:
			{
				value = getFeature(state->getDiscreteState(0));
				break;
			}
		default:
			{
			}
	}
	
	if (DebugIsEnabled('v'))
	{
		DebugPrint('v', "Get V-Value: %f", value);
		state->saveASCII(DebugGetFileHandle('v'));
		DebugPrint('v', "\n");
	}
	if (! mayDiverge && (value < - DIVERGENTVFUNCTIONVALUE || value > DIVERGENTVFUNCTIONVALUE))
	{
		throw new CDivergentVFunctionException("Feature Function", this, state, value);
	}
	return value;
} 

void CFeatureVFunction::saveData(FILE *file)
{
	CGradientVFunction::saveData(file);
}

void CFeatureVFunction::loadData(FILE *file)
{
	CAbstractVFunction::loadData(file);

	//CGradientVFunction::loadData(file);

	CGradientVFunction::loadData(file);
}

void CFeatureVFunction::printValues()
{
	CAbstractVFunction::printValues();
	printFeatures();
}

CAbstractVETraces *CFeatureVFunction::getStandardETraces()
{
	return new CFeatureVETraces(this);
}

/*CStateProperties *CFeatureVFunction::getGradientCalculator()
{
	return properties;
}*/

void CFeatureVFunction::updateWeights(CFeatureList *gradientFeatures)
{
	this->updateFeatureList(gradientFeatures, 1.0);
}

int CFeatureVFunction::getNumWeights()
{
	return this->numFeatures;
}

void CFeatureVFunction::resetData()
{
	CFeatureFunction::init(0.0);
}

void CFeatureVFunction::getWeights(rlt_real *parameters)
{
	memcpy(parameters, this->features, sizeof(rlt_real) * getNumFeatures());
}

void CFeatureVFunction::setWeights(rlt_real *parameters)
{
	memcpy(this->features, parameters, sizeof(rlt_real) * getNumFeatures());
}


CFeatureVFunctionInputDerivationCalculator::CFeatureVFunctionInputDerivationCalculator(CStateProperties *inputState, CFeatureVFunction *vFunction) : CVFunctionInputDerivationCalculator(inputState)
{
	this->vFunction = vFunction;
	featureInputDerivation = new CMyVector(getNumInputs());
	fiDerivationSum = new CMyVector(getNumInputs());

	normalizedFeatures = true;
}

CFeatureVFunctionInputDerivationCalculator::~CFeatureVFunctionInputDerivationCalculator()
{
	delete featureInputDerivation;
	delete fiDerivationSum;
}

void CFeatureVFunctionInputDerivationCalculator::getInputDerivation( CStateCollection *state, CMyVector *targetVector)
{
	CState *featureState = state->getState(vFunction->getStateProperties());

	targetVector->initVector(0.0);

	CFeatureCalculator *featCalc = dynamic_cast<CFeatureCalculator *>(vFunction->getStateProperties());
	
	if (!normalizedFeatures)
	{
		for (unsigned int i = 0; i < featureState->getNumContinuousStates(); i ++)
		{
			featureInputDerivation->initVector(0.0);
			featCalc->getFeatureDerivationX(featureState->getDiscreteState(i), state, featureInputDerivation);

			/*if (DebugIsEnabled('v'))
			{
				DebugPrint('v',"Feature VFunction Input Gradient for Feature %d (%f): ", featureState->getDiscreteState(i), featureState->getContinuousState(i));
				for (int i = 0; i < featureInputDerivation->getNumDimensions(); i ++)
				{
					DebugPrint('v', "%f, ", featureInputDerivation->getElement(i));
				}
				DebugPrint('v', "\n");
			}*/
			featureInputDerivation->multScalar(vFunction->getFeature(featureState->getDiscreteState(i)));

			targetVector->addVector(featureInputDerivation);
		}
	}
	else
	{
		rlt_real fiSum = 0;
		fiDerivationSum->initVector(0.0);
		
		for (unsigned int i = 0; i < featureState->getNumContinuousStates(); i ++)
		{
			featureInputDerivation->initVector(0.0);
			featCalc->getFeatureDerivationX(featureState->getDiscreteState(i), state, featureInputDerivation);

			fiSum += featureState->getContinuousState(i);
			fiDerivationSum->addVector(featureInputDerivation);

			/*if (DebugIsEnabled('v'))
			{
				DebugPrint('v',"Feature VFunction Input Gradient for Feature %d (%f): ", featureState->getDiscreteState(i), featureState->getContinuousState(i));
				for (int i = 0; i < featureInputDerivation->getNumDimensions(); i ++)
				{
					DebugPrint('v', "%f, ", featureInputDerivation->getElement(i));
				}
				DebugPrint('v', "\n");
			}*/
			featureInputDerivation->multScalar(vFunction->getFeature(featureState->getDiscreteState(i)));

			targetVector->addVector(featureInputDerivation);
		}

		targetVector->multScalar(1 / fiSum);
		fiDerivationSum->multScalar(- vFunction->getValue(featureState));
		targetVector->addVector(fiDerivationSum);
	}
	
	rlt_real length = targetVector->getLength();

	if (length > 0.01)
	{
		targetVector->multScalar(1 / length);
	}
	

	if (DebugIsEnabled('v'))
	{
		DebugPrint('v',"Feature VFunction Input Gradient: ");
		for (unsigned int i = 0; i < targetVector->getNumDimensions(); i ++)
		{
			DebugPrint('v', "%f, ", targetVector->getElement(i));
		}
		DebugPrint('v', "\n");
	}
}


CVTable::CVTable(CAbstractStateDiscretizer *discretizer) : CFeatureVFunction(discretizer)
{
}

CVTable::~CVTable() 
{
}

void CVTable::setDiscretizer(CAbstractStateDiscretizer *discretizer)
{
	assert(discretizer == NULL || discretizer->getDiscreteStateSize() == numFeatures);

	this->properties = discretizer;
}

CAbstractStateDiscretizer *CVTable::getDiscretizer()
{
	return (CAbstractStateDiscretizer*) properties;
}


int CVTable::getNumStates()
{
	return getNumFeatures();
}