ctestsuit.cpp

强化学习算法（R-Learning）难得的珍贵资料

	if (parameterFile == NULL)
	{
		return;
	}
 
	while (!feof(parameterFile))
	{
		char buffer[256];

		int results = fscanf(parameterFile, "%s\n", buffer);
		while (results == 1 && strcmp(buffer, "<testsuiteevaluation>") != 0 && !feof(parameterFile)) 
		{
			results =  fscanf(parameterFile, "%s\n", buffer);
		}

		if (feof(parameterFile))
		{
			break;
		}

		CParameters *parameters = new CParameters();
		parameters->loadParametersXML(parameterFile);
		
		if ((*testSuite) == (*parameters))
		{
			fscanf(parameterFile, "<evaluationdata>\n");
			fscanf(parameterFile, "%s\n", buffer);

			rlt_real time = 0.0;

			int bufEpisode;
			int bufSteps;

			rlt_real *trialValues = new rlt_real[nAverageRewards];
			memset(trialValues, 0, sizeof(rlt_real) * nAverageRewards);
			values->push_back(trialValues);

			results = 1;
			bool bOk = true;
			int i = 0;
			while (strcmp(buffer, "</evaluationdata>") != 0 )
			{
				if (buffer[0] == '<' || results <= 0 || feof(parameterFile))
				{
					bOk = false;
					break;
				}
				
				sscanf(buffer, "%d,", &bufEpisode);

				int n_result = fscanf(parameterFile, "%d: %lf\n", &bufSteps, &trialValues[i]);

				if (n_result != 2 || trialValues[i] < getParameter("DivergentEvaluationValue") + 1)
				{
					exception = true;
				}

				results = fscanf(parameterFile, "%s\n", buffer);
				i ++;
			}
			if (bOk)
			{
				while (results == 1 && strcmp(buffer, "</testsuiteevaluation>") != 0 && !feof(parameterFile)) 
				{
					results =  fscanf(parameterFile, "%s\n", buffer);
				}
			}
		}
		delete parameters;	
	}
	fclose(parameterFile);
}

rlt_real CAverageRewardTestSuiteEvaluator::evaluateTestSuite(CTestSuite *testSuite, int evaluationFunction, bool loadEvaluationTrial)
{
	int tmpEvalF = this->evaluationFunction;
	this->evaluationFunction = evaluationFunction;
	rlt_real value = evaluateTestSuite(testSuite,  loadEvaluationTrial);

	this->evaluationFunction = tmpEvalF;
	return value;
}

/*
rlt_real CAverageRewardTestSuiteEvaluator::evaluateTestSuite(CTestSuite *testSuite,int evaluationFunction, bool loadEvaluationTrial)
{
	checkDirectories();
	
	exception = false;
	
	std::list<rlt_real *>::iterator it = values->begin();
	for (; it != values->end(); it ++)
	{
		delete *it;
	}
	values->clear();

	printf("Evaluating TestSuite %s with Parameters:\n", testSuite->getTestSuiteName().c_str());
	testSuite->saveParameters(stdout);

	if (loadEvaluationTrial)
	{
		loadEvaluationData(testSuite);
	}
	printf("Loaded %d Trials for Evaluation!\n", values->size());
	
	FILE *evaluationFile = fopen(getEvaluationFileName(testSuite).c_str(), "a");
	while (values->size() < nTrials && ! exception)
	{
		doEvaluationTrial(testSuite, evaluationFile, getLearnDataFileName(testSuite).c_str());
	}
	fclose(evaluationFile);

	rlt_real evaluationValue = 0.0;
	
	if(exception)
	{
		evaluationValue = getParameter("DivergentEvaluationValue");
	}
	else
	{
		evaluationValue = getEvaluationValue(values, evaluationFunction);
	}

	return evaluationValue;
}*/

std::list<rlt_real *> *CAverageRewardTestSuiteEvaluator::getTrialAverageRewards()
{
	return values;
}

rlt_real CAverageRewardTestSuiteEvaluator::getEvaluationValue(std::list<rlt_real *> *values)
{
	std::list<rlt_real *>::iterator it = values->begin();

	rlt_real average = 0.0;
	switch (evaluationFunction)
	{
		case ARCF_IDENTITY:
		{
			for (; it != values->end();it++)
			{
				for (int i = 0; i < nAverageRewards; i++)
				{
					average += (*it)[i];
				}

			}
			break;
		}
		case ARCF_LINEAR:
		{
			for (; it != values->end();it++)
			{
				for (int i = 0; i < nAverageRewards; i++)
				{
					average += (*it)[i] * (i + 1) / nAverageRewards;
				}

			}
			break;
		}
		case ARCF_AVERAGE:
		{
			for (; it != values->end();it++)
			{
				rlt_real l_average = 0;
				for (int i = 0; i < nAverageRewards; i++)
				{
					l_average += (*it)[i];
				}
				average += l_average / nAverageRewards;

			}
			break;
		}
		default:
		{
			for (; it != values->end();it++)
			{
				rlt_real l_average = 0;
				for (int i = 0; i < nAverageRewards; i++)
				{
					l_average += (*it)[i];
				}
				average += l_average / nAverageRewards;

			}

		}
	}
	average = average / values->size();
	printf("FINISHED with evaluation !!!\n");
	printf("Evaluated Average Value : %f\n\n\n", average );
	return average;
}



CTestSuiteParameterCalculator::CTestSuiteParameterCalculator(CTestSuiteEvaluator *evaluator, CTestSuite *testSuite)
{
	this->evaluator = evaluator;
	this->testSuite = testSuite;
}


CTestSuiteParameterCalculator::~CTestSuiteParameterCalculator()
{
}
/*
void  CTestSuiteParameterCalculator::setFilenamesAndDirectories(char *testSuiteCollectionDirectory)
{
	char evaluationFileDirectory[255];
	char evaluationFileName[255];
	char checkDirSystemCall[255];

	sprintf(evaluationFileDirectory, "%s/%s/", testSuiteCollectionDirectory, evaluator->getEvaluatorDirectory().c_str());

	sprintf(evaluationFileName, "%s/%s_params.txt", evaluationFileDirectory, testSuite->getTestSuiteName().c_str());

	sprintf(this->learnDataFileDirectory, "%s/LearnData/", evaluationFileDirectory);

	sprintf(checkDirSystemCall, "checkdir.bat %s", evaluationFileDirectory);

	system(checkDirSystemCall);


	evaluatedTestsuiteParameters = new std::map<CParameters *, ParameterData>();

	FILE *input = fopen(evaluationFileName, "r");
	if (input)
	{
		loadEvaluatedParameters(input);
		fclose(input);
	}

	this->parameterFile = fopen(evaluationFileName, "a");
	assert(parameterFile != NULL);
}


void CTestSuiteParameterCalculator::loadEvaluatedParameters(FILE *parameterFile)
{
	CParameters *testSuiteParameters = new CParameters(*testSuite);
	while (!feof(parameterFile))
	{
		char buffer[256];
		ParameterData data;
		//fscanf(input, "TestSuiteEvaluation: ");
	

		int results = fscanf(parameterFile, "%s\n", buffer);
		while (strcmp(buffer, "<testsuiteevaluation>") != 0 && !feof(parameterFile)) 
		{
			results =  fscanf(parameterFile, "%s\n", buffer);
		}
		
		if (feof(parameterFile))
		{
			break;
		}

		testSuite->loadParametersXML(parameterFile);
		CParameters *parameters = new CParameters(*testSuite);

		fscanf(parameterFile, "<evaluationdata>\n");
		fscanf(parameterFile, "%s\n", buffer);

		rlt_real time = 0.0;

		if (parameters->getNumParameters() > 0)
		{
			results = 1;
			bool bOk = true;
			while (strcmp(buffer, "</evaluationdata>") != 0 )
			{
				if (buffer[0] == '<' || results <= 0 || feof(parameterFile))
				{
					bOk = false;
					break;
				}
				results = fscanf(parameterFile, "%s\n", buffer);
			}
			if (bOk)
			{
				data.learnDataFile = "";

 				fscanf(parameterFile, "<evaluatedvalue> %lf </evaluatedvalue>\n", &data.value);

				fscanf(parameterFile, "%s", buffer);

				if (strcmp(buffer, "<learntime>") == 0)
				{
					fscanf(parameterFile, " %lf </learntime>\n", &time);
				}
				if (strcmp(buffer, "<learndatafile>") == 0)
				{
					results = fscanf(parameterFile, "%s </learndatafile>\n", buffer);
					if (results == 1)
					{
						data.learnDataFile = string(buffer);
					}
				}

				fscanf(parameterFile, "</testsuiteevaluation>\n");

				(*evaluatedTestsuiteParameters)[parameters] = data;

			}
			else
			{
				delete parameters;
			}
		}
		else
		{
			delete parameters;
		}
		testSuite->setParameters(testSuiteParameters);
	}
	delete testSuiteParameters;
	printf("%d Parameter-Sets Loaded\n", this->getNumEvaluatedParameters());
}*/


CParameters *CTestSuiteParameterCalculator::calculateBestParameters(std::list<rlt_real *> *parameters, int *paramSize, std::list<string> *paramNames )
{
	std::list<CParameters *> *paramList = getParameterList(parameters, paramSize, paramNames);

	CParameters *params = new CParameters(*calculateBestParameters(paramList));

	std::list<CParameters *>::iterator it = paramList->begin();

	for (; it != paramList->end(); it ++)
	{
		delete (*it);
	}

	delete paramList;

	return params;
}

CParameters *CTestSuiteParameterCalculator::calculateBestParameters(std::list<CParameters *> *parameterList)
{
	std::list<CParameters *>::iterator it;
	CParameters *parameters = NULL;

	rlt_real max = 0.0;
	for (it = parameterList->begin(); it != parameterList->end(); it ++)
	{
		rlt_real value = evaluateParameters(*it);
		if (value > max || it == parameterList->begin())
		{
			max = value;
			parameters = *it;
		}
	}
	return parameters;
}

rlt_real CTestSuiteParameterCalculator::searchBestParameterValue(string paramName, rlt_real startValue, int maxIterations, rlt_real minParamValue, rlt_real maxParamValue)
{	
	rlt_real parameters[3];
	rlt_real values[3];

	parameters[0] = startValue;
	parameters[1] = startValue / 2;
	parameters[2] = startValue * 2;

	if (testSuite->getParameterIndex(paramName) < 0)
	{
		printf("Parameter %s not Member of the Testsuite!!!\n", paramName.c_str());
		return 0;
	}

	printf("Searching Value for Parameter %s, StartValue: %f\n", paramName.c_str(), parameters[0]);
	
	std::list<rlt_real> *beginValues = new std::list<rlt_real>();
	std::list<rlt_real> *beginParameters = new std::list<rlt_real>();

	printf("Testing StartParameter %f\n", parameters[0]);
	testSuite->setParameter(paramName, parameters[0]);
	values[0] = evaluateParameters(testSuite);

	beginValues->push_back(values[0]);
	beginParameters->push_back(parameters[0]);
	

	rlt_real maxValue = values[0];
	rlt_real minValue = values[0];


	printf("StartValue: (%f: %f)\n",parameters[0],values[0]);

	int iterations = 0;
	
	int divIterations = 0;
	while (maxValue - minValue < 0.25 * fabs(minValue) && iterations < maxIterations + divIterations && divIterations < 20)
	{
		iterations ++;
		printf("Iteration: %d, No distinctive Maximum found, searching in booth directions\n", iterations);

		rlt_real value = 0.0;
		rlt_real minParam = *beginParameters->begin();
		rlt_real maxParam = *beginParameters->rbegin();

		
		bool left = ((*beginValues->begin()) < (this->evaluator->getParameter("DivergentEvaluationValue") - 1)) && (minParam > minParamValue);
		bool right = ((*beginValues->rbegin()) < (this->evaluator->getParameter("DivergentEvaluationValue") - 1)) && (maxValue < maxParamValue);


		if (left == right)
		{
			if ((minParam < minParamValue) && (maxParam > maxParamValue))
			{
				break;
			}
			rlt_real meanVal = ((*beginValues->begin()) + (*beginValues->rbegin())) / 2;

			if (left && fabs((*beginValues->begin() - *beginValues->rbegin()) / meanVal) > 0.15)
			{
				left = (*beginValues->begin() > *beginValues->rbegin());
				right = !left;

			}
			else
			{
				left = iterations % 2 == 1;
				right = !left;
			}
		}
		if (left)
		{
			rlt_real param = *beginParameters->begin() / 2;
			printf("Searching Left : %f ...\n", param);

			testSuite->setParameter(paramName, param);
			value = evaluateParameters(testSuite);

			beginParameters->push_front(param);
			beginValues->push_front(value);
		}
		else
		{
			rlt_real param = *beginParameters->rbegin() * 2;
			printf("Searching Right : %f ...\n", param);

			testSuite->setParameter(paramName, param);
			value = evaluateParameters(testSuite);

			beginParameters->push_back(param);
			beginValues->push_back(value);

		}
		

		if (value > maxValue) 
		{
			maxValue = value;
		}	
		if ((value < minValue && value >  evaluator->getParameter("DivergentEvaluationValue")) || (minValue <= evaluator->getParameter("DivergentEvaluationValue") - 1))
		{
			minValue = value;
		}
		if (value < (this->evaluator->getParameter("DivergentEvaluationValue") - 1))
		{
			divIterations ++;
		}
	}
	
	if (maxValue > this->evaluator->getParameter("DivergentEvaluationValue"))
	{
		iterations -= divIterations;
	}

	while (maxValue <= *beginValues->begin() && iterations < maxIterations &&  *beginParameters->begin() > minParamValue)
	{
		iterations ++;

		rlt_real param = *beginParameters->begin() / 2;
		printf("Iteration %d, Maximum on the left side ... searching left: %f\n", iterations, param);

		testSuite->setParameter(paramName, param);
		rlt_real value = evaluateParameters(testSuite);

		beginParameters->push_front(param);
		beginValues->push_front(value);

		if (value > maxValue) 
		{
			maxValue = value;
		}
	}

	

	while (maxValue <= *beginValues->rbegin() && iterations < maxIterations && *beginParameters->rbegin() < maxParamValue)
	{
		rlt_real param = *beginParameters->rbegin() * 2;
		iterations ++;

		printf("Iteration %d: Maximum on the right side ... searching right: %f\n", iterations, param);

		testSuite->setParameter(paramName, param);
		rlt_real value = evaluateParameters(testSuite);

		beginParameters->push_back(param);
		beginValues->push_back(value);

		if (value > maxValue) 
		{
			maxValue = value;
		}	
	}

	std::list<rlt_real>::iterator it = beginValues->begin();
	std::list<rlt_real>::iterator itParam = beginParameters->begin();

	values[0] = *it;
	parameters[0] = *itParam;
	while ((*it) < maxValue - 0.0001 && it != beginValues->end())
	{
		parameters[0] = *itParam;
		values[0] = *it;

		it ++;
		itParam ++;
	}

	if (it != beginValues->end())
	{
		values[1] = *it;
		parameters[1] = *itParam;
	}

	it ++;