xcsr.cpp

- XCS for Dynamic Environments + Continuous versions of XCS + Test problem: real multiplexer +

/*	XCSR_DE1.0
*	--------------------------------------------------------
*	Learning classifier system based on accuracy in dynamic environments
*
*	by  Huong Hai (Helen) Dam
*	[e-mail]
*	UNSW @ ADFA, Canberra Australia
*	Artificial Life and Adaptive Robotics Laboratory
*	http://www.itee.adfa.edu.au/~alar
*
*	Last modified: 24-11-2005
*
*/
#include <iostream.h>
#include <fstream.h>
#include "population.h"
#include "relearningscheme.h"
#include "ga.h"
#include "declare.h"
#include "random.h"
#include "string"
#include "environment.h"

int problemExplore(int time);
int problemExploit(int time);
void trackConceptDrift(double,int);
void learningFunction(double current_error);
Population *pop;
GA *ga;
ReLearningScheme *my_learning; 
Environment *env;
double current_error;
double real_threshold;
double previous_error;
int covering_type;
int fast_learning_rate;

int main(int argc, char* argv[])
{
	char  stsname[50];
	double performance[CNEPOCHS/WINDOW_SIZE];
	double system_error[CNEPOCHS/WINDOW_SIZE];
	double macro_classifier_fraction[CNEPOCHS/WINDOW_SIZE];
	int population_size[CNEPOCHS/WINDOW_SIZE];
	double num_of_classifier_in_regions[5][CNEPOCHS/WINDOW_SIZE];
	double num_of_rule[17][CNEPOCHS/WINDOW_SIZE];
	int num_new_rules[CNEPOCHS/WINDOW_SIZE];
	int num_delete_rules[CNEPOCHS/WINDOW_SIZE];
	fast_learning_rate = FALSE;
	previous_error = REWARD;
	if(argc != 10)
	{
		cout<<"Error! please input:					"<<endl;
		cout<<"seed: random seed					"<<endl; 
		cout<<"noise: noise level					"<<endl;
		cout<<"population: maximum population size	"<<endl;
		cout<<"crate: crossover rate				"<<endl;
		cout<<"mrate: mutation rate					"<<endl;
		cout<<"representation type: continuous-valued"<<endl;
		cout<<"recovering concept drift strategy	"<<endl;
		cout<<"a real threshold before changing		"<<endl;
		cout<<"a real threshold after changing		"<<endl;
		return 1;
	}
	
	int seed = atoi(argv[1]);
	double noise = atof(argv[2]);
	int population = atoi(argv[3]);
	double crate = atof(argv[4]);
	double mrate = atof(argv[5]);
	int rep_type = atoi(argv[6]);
	covering_type = atoi(argv[7]);
	double vol1 = atof(argv[8]);
	double vol2 = atof(argv[9]);
	env = new Environment(seed, noise);
	pop = new Population(seed); // Read the initial population;
	my_learning = new ReLearningScheme(seed, rep_type, population);
	ga = new GA(seed, population, crate,  mrate, rep_type);
	int correct = 0;
	double error = 0.0;
	int k;
	real_threshold = env->setThreshold(vol1);
	for(k = 0; k < CNEPOCHS/WINDOW_SIZE; k++)
	{
		performance[k] =0.;
		system_error[k] = 0.;
		macro_classifier_fraction[k] =0;
		population_size[k] =0;
		num_of_classifier_in_regions[1][k] =0;
		num_of_classifier_in_regions[2][k] =0;
		num_of_classifier_in_regions[3][k] =0;
		num_of_classifier_in_regions[4][k] =0;
		
	}
	for(int iteration = 0; iteration < CNEPOCHS; iteration++)
	{
		problemExplore( iteration);
		if(problemExploit(iteration) == TRUE)
		{
			correct++;
			//			cout<<time<<" Correct"<<endl;
		}
		error += current_error;
		//testfile<<"Population at "<<i<<endl;	
		//pop->snapshotReport(testfile);
		if(iteration%WINDOW_SIZE == 0)
		{		
			performance[iteration/WINDOW_SIZE] = (double)correct/(WINDOW_SIZE);
			system_error[iteration/WINDOW_SIZE] = error/(WINDOW_SIZE);
			macro_classifier_fraction[iteration/WINDOW_SIZE] = pop->getNumMacro();
			population_size[iteration/WINDOW_SIZE] = pop->getPopSize();
			num_new_rules[iteration/WINDOW_SIZE] =pop->getNumNewRules();
			num_delete_rules[iteration/WINDOW_SIZE] = pop->getNumDeleteRules();
			double current_error = error/(WINDOW_SIZE);
			if(covering_type != ADAPTIVE_LEARNING_MODEL)
			{	
				trackConceptDrift(current_error,i);
			}
			else
			{
				my_learning->adaptLearningRate(current_error, previous_error);
			}
			previous_error = current_error;
			
			
		if(iteration % DYN_CYCLE==0)
		{
			if(i != 0)
			{
				real_threshold =	env->setThreshold(vol2);
			}
			cout<<iteration<<" Change in environment "<<real_threshold<<endl;
		}
	}
	//Record system information including performance, population size, system error, regions' proportion.
	sprintf(stsname, "result%dP%dC%.1fM%.2fS%dV1%.1fV2%.1fN%.2f.dat", covering_type, population, crate, mrate, seed, vol1, vol2, noise);
	ofstream stsfile (stsname);
	for(k = 0; k < CNEPOCHS/WINDOW_SIZE; k++)
	{
		stsfile<<performance[k]<<"\t";
		stsfile<<system_error[k]<<"\t";
		stsfile<<macro_classifier_fraction[k] <<"\t";
		stsfile<<population_size[k]<<"\t";
		stsfile<<num_new_rules[k]<<"\t";
		stsfile<<num_delete_rules[k]<<endl;
	}
	stsfile.close();
	
	sprintf(stsname, "Population%dP%dC%.1fM%.2fS%dV1%.1fV2%.1fN%.2f.dat", covering_type, population, crate, mrate, seed, vol1, vol2, noise);
	ofstream pop_file (stsname);
	pop->snapshotReport(pop_file);
	pop_file.close();
	
	return 0;
}

void trackConceptDrift(double error, int id)
{
	//Dynamic environment
	double delta = (error - previous_error)/REWARD;
	if(delta >DELTA_ERROR ) 
	{
		int number_cover = my_learning->getCoveringCounter();
		if(number_cover == 0)//concept drift occurs
		{
			cout<<id<<": Concept Drift, current error: "<<error<<", previous: "<<previous_error<<", delta: "<<delta<<endl;	
			if(covering_type == REINIT_POP_MODEL)
				pop->emptyPopulation();
			if(covering_type == REINIT_PARAM_MODEL)
				pop->reinitParam(); 
			if(covering_type == PARTIAL_DELETE_RULE_MODEL)
			{
				pop->deletePartialPopulation(0.5);
				pop->reinitParam();
				my_learning->setFastLearningRate(TRUE);
				fast_learning_rate = TRUE;
			}
			
		}
	}
	else if(fast_learning_rate == TRUE && error/REWARD < ACCEPTED_ERROR)
	{
		fast_learning_rate = FALSE;
		my_learning->setFastLearningRate(FALSE);
	}
	my_learning->resetCoveringCounter();
}
int problemExplore( int time)
{
	int outputN;
	double reward = 0;
	int naset;
	int result = FALSE;
	input_t *theinput = env->getInput(TRAINING);
	int *actset= new int[MAX_POP_SIZE + 1];
	//Output from XCS
	
	outputN= my_learning->performanceComponent( ga, pop,theinput );
	//Reinforcement learning
	if(outputN == theinput->action) //if expected output
	{
		reward = REWARD;
		result = TRUE;
	}
	//Pay Reward 
	my_learning->adjustActionSet(pop, reward);
	naset = my_learning->returnActionSet(actset); //get an action set
	ga->discoveryComponent(pop, actset, naset,  time); // run GAa
	delete actset;
	delete theinput;
	return result;
}
int problemExploit( int time)
{
	int output;
	double reward=0;
	int result = FALSE;
	input_t *theinput = env->getInput(TESTING);
	output=my_learning->getXCSOutput(pop, theinput);
	if(output == theinput->action)
	{
		result = TRUE;
		reward = REWARD;
	}
	current_error = my_learning->getAvgSystemError(output, reward); // Get System Error
	delete theinput;
	return result;
}