fann_cascade.c

python 神经网络 数据挖掘 python实现的神经网络算法

/*
  Fast Artificial Neural Network Library (fann)
  Copyright (C) 2003 Steffen Nissen ([e-mail])
  
  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.
  
  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.
  
  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

#include "config.h"
#include "fann.h"
#include "string.h"

#ifndef FIXEDFANN

/* #define CASCADE_DEBUG */
/* #define CASCADE_DEBUG_FULL */

void fann_print_connections_raw(struct fann *ann)
{
	unsigned int i;

	for(i = 0; i < ann->total_connections_allocated; i++)
	{
		if(i == ann->total_connections)
		{
			printf("* ");
		}
		printf("%f ", ann->weights[i]);
	}
	printf("\n\n");
}

/* Cascade training directly on the training data.
   The connected_neurons pointers are not valid during training,
   but they will be again after training.
 */
FANN_EXTERNAL void FANN_API fann_cascadetrain_on_data(struct fann *ann, struct fann_train_data *data,
										unsigned int max_neurons,
										unsigned int neurons_between_reports,
										float desired_error)
{
	float error;
	unsigned int i;
	unsigned int total_epochs = 0;
	int desired_error_reached;

	if(neurons_between_reports && ann->callback == NULL)
	{
		printf("Max neurons %3d. Desired error: %.6f\n", max_neurons, desired_error);
	}

	for(i = 1; i <= max_neurons; i++)
	{
		/* train output neurons */
		total_epochs += fann_train_outputs(ann, data, desired_error);
		error = fann_get_MSE(ann);
		desired_error_reached = fann_desired_error_reached(ann, desired_error);

		/* print current error */
		if(neurons_between_reports &&
		   (i % neurons_between_reports == 0
			|| i == max_neurons || i == 1 || desired_error_reached == 0))
		{
			if(ann->callback == NULL)
			{
				printf
					("Neurons     %3d. Current error: %.6f. Total error:%8.4f. Epochs %5d. Bit fail %3d",
					 i, error, ann->MSE_value, total_epochs, ann->num_bit_fail);
				if((ann->last_layer-2) != ann->first_layer)
				{
					printf(". candidate steepness %.2f. function %s", 
					   (ann->last_layer-2)->first_neuron->activation_steepness,
					   FANN_ACTIVATIONFUNC_NAMES[(ann->last_layer-2)->first_neuron->activation_function]);
				}
				printf("\n");
			}
			else if((*ann->callback) (ann, data, max_neurons, 
				neurons_between_reports, desired_error, total_epochs) == -1) 
			{
				/* you can break the training by returning -1 */
				break;
			}					 
		}

		if(desired_error_reached == 0)
			break;

		if(fann_initialize_candidates(ann) == -1)
		{
			/* Unable to initialize room for candidates */
			break;
		}

		/* train new candidates */
		total_epochs += fann_train_candidates(ann, data);

		/* this installs the best candidate */
		fann_install_candidate(ann);
	}

	/* Train outputs one last time but without any desired error */
	total_epochs += fann_train_outputs(ann, data, 0.0);

	if(neurons_between_reports && ann->callback == NULL)
	{
		printf("Train outputs    Current error: %.6f. Epochs %6d\n", fann_get_MSE(ann),
			   total_epochs);
	}

	/* Set pointers in connected_neurons
	 * This is ONLY done in the end of cascade training,
	 * since there is no need for them during training.
	 */
	fann_set_shortcut_connections(ann);
}

FANN_EXTERNAL void FANN_API fann_cascadetrain_on_file(struct fann *ann, const char *filename,
													  unsigned int max_neurons,
													  unsigned int neurons_between_reports,
													  float desired_error)
{
	struct fann_train_data *data = fann_read_train_from_file(filename);

	if(data == NULL)
	{
		return;
	}
	fann_cascadetrain_on_data(ann, data, max_neurons, neurons_between_reports, desired_error);
	fann_destroy_train(data);
}

int fann_train_outputs(struct fann *ann, struct fann_train_data *data, float desired_error)
{
	float error, initial_error, error_improvement;
	float target_improvement = 0.0;
	float backslide_improvement = -1.0e20f;
	unsigned int i;
	unsigned int max_epochs = ann->cascade_max_out_epochs;
	unsigned int stagnation = max_epochs;

	/* TODO should perhaps not clear all arrays */
	fann_clear_train_arrays(ann);

	/* run an initial epoch to set the initital error */
	initial_error = fann_train_outputs_epoch(ann, data);

	if(fann_desired_error_reached(ann, desired_error) == 0)
		return 1;

	for(i = 1; i < max_epochs; i++)
	{
		error = fann_train_outputs_epoch(ann, data);

		/*printf("Epoch %6d. Current error: %.6f. Bit fail %d.\n", i, error, ann->num_bit_fail); */

		if(fann_desired_error_reached(ann, desired_error) == 0)
		{
#ifdef CASCADE_DEBUG
			printf("Error %f < %f\n", error, desired_error);
#endif
			return i + 1;
		}

		/* Improvement since start of train */
		error_improvement = initial_error - error;

		/* After any significant change, set a new goal and
		 * allow a new quota of epochs to reach it */
		if((error_improvement > target_improvement) || (error_improvement < backslide_improvement))
		{
			/*printf("error_improvement=%f, target_improvement=%f, backslide_improvement=%f, stagnation=%d\n", error_improvement, target_improvement, backslide_improvement, stagnation); */

			target_improvement = error_improvement * (1.0f + ann->cascade_output_change_fraction);
			backslide_improvement = error_improvement * (1.0f - ann->cascade_output_change_fraction);
			stagnation = i + ann->cascade_output_stagnation_epochs;
		}

		/* No improvement in allotted period, so quit */
		if(i >= stagnation)
		{
			return i + 1;
		}
	}

	return max_epochs;
}

float fann_train_outputs_epoch(struct fann *ann, struct fann_train_data *data)
{
	unsigned int i;

	fann_reset_MSE(ann);

	for(i = 0; i < data->num_data; i++)
	{
		fann_run(ann, data->input[i]);
		fann_compute_MSE(ann, data->output[i]);
		fann_update_slopes_batch(ann, ann->last_layer - 1, ann->last_layer - 1);
	}

	switch (ann->training_algorithm)
	{
		case FANN_TRAIN_RPROP:
			fann_update_weights_irpropm(ann, (ann->last_layer - 1)->first_neuron->first_con,
										ann->total_connections);
			break;
		case FANN_TRAIN_QUICKPROP:
			fann_update_weights_quickprop(ann, data->num_data,
										  (ann->last_layer - 1)->first_neuron->first_con,
										  ann->total_connections);
			break;
		case FANN_TRAIN_BATCH:
		case FANN_TRAIN_INCREMENTAL:
			fann_error((struct fann_error *) ann, FANN_E_CANT_USE_TRAIN_ALG);
	}

	return fann_get_MSE(ann);
}

int fann_reallocate_connections(struct fann *ann, unsigned int total_connections)
{
	/* The connections are allocated, but the pointers inside are
	 * first moved in the end of the cascade training session.
	 */

#ifdef CASCADE_DEBUG
	printf("realloc from %d to %d\n", ann->total_connections_allocated, total_connections);
#endif
	ann->connections =
		(struct fann_neuron **) realloc(ann->connections,
										total_connections * sizeof(struct fann_neuron *));
	if(ann->connections == NULL)
	{
		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
		return -1;
	}

	ann->weights = (fann_type *) realloc(ann->weights, total_connections * sizeof(fann_type));
	if(ann->weights == NULL)
	{
		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
		return -1;
	}

	ann->train_slopes =
		(fann_type *) realloc(ann->train_slopes, total_connections * sizeof(fann_type));
	if(ann->train_slopes == NULL)
	{
		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
		return -1;
	}

	ann->prev_steps = (fann_type *) realloc(ann->prev_steps, total_connections * sizeof(fann_type));
	if(ann->prev_steps == NULL)
	{
		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
		return -1;
	}

	ann->prev_train_slopes =
		(fann_type *) realloc(ann->prev_train_slopes, total_connections * sizeof(fann_type));
	if(ann->prev_train_slopes == NULL)
	{
		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
		return -1;
	}

	ann->total_connections_allocated = total_connections;

	return 0;
}

int fann_reallocate_neurons(struct fann *ann, unsigned int total_neurons)
{
	struct fann_layer *layer_it;
	struct fann_neuron *neurons;
	unsigned int num_neurons = 0;
	unsigned int num_neurons_so_far = 0;

	neurons =
		(struct fann_neuron *) realloc(ann->first_layer->first_neuron,
									   total_neurons * sizeof(struct fann_neuron));
	ann->total_neurons_allocated = total_neurons;

	if(neurons == NULL)
	{
		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
		return -1;
	}

	/* Also allocate room for more train_errors */
	ann->train_errors = (fann_type *) realloc(ann->train_errors, total_neurons * sizeof(fann_type));
	if(ann->train_errors == NULL)
	{
		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
		return -1;
	}

	if(neurons != ann->first_layer->first_neuron)
	{
		/* Then the memory has moved, also move the pointers */

#ifdef CASCADE_DEBUG_FULL
		printf("Moving neuron pointers\n");
#endif

		/* Move pointers from layers to neurons */
		for(layer_it = ann->first_layer; layer_it != ann->last_layer; layer_it++)
		{
			num_neurons = layer_it->last_neuron - layer_it->first_neuron;
			layer_it->first_neuron = neurons + num_neurons_so_far;
			layer_it->last_neuron = layer_it->first_neuron + num_neurons;
			num_neurons_so_far += num_neurons;
		}
	}

	return 0;
}

int fann_initialize_candidates(struct fann *ann)
{
	/* The candidates are allocated after the normal neurons and connections,