fann_cascade.c

python 神经网络 数据挖掘 python实现的神经网络算法

	 * but there is an empty place between the real neurons and the candidate neurons,
	 * so that it will be possible to make room when the chosen candidate are copied in
	 * on the desired place.
	 */
	unsigned int neurons_to_allocate, connections_to_allocate;
	unsigned int num_candidates = fann_get_cascade_num_candidates(ann);
	unsigned int num_neurons = ann->total_neurons + num_candidates + 1;
	unsigned int candidate_connections_in = ann->total_neurons - ann->num_output;
	unsigned int candidate_connections_out = ann->num_output;

	/* the number of connections going into a and out of a candidate is
	 * ann->total_neurons */
	unsigned int num_connections =
		ann->total_connections + (ann->total_neurons * (num_candidates + 1));
	unsigned int first_candidate_connection = ann->total_connections + ann->total_neurons;
	unsigned int first_candidate_neuron = ann->total_neurons + 1;
	unsigned int connection_it, i, j, k, candidate_index;
	struct fann_neuron *neurons;
	fann_type initial_slope;
	
	/* First make sure that there is enough room, and if not then allocate a
	 * bit more so that we do not need to allocate more room each time.
	 */
	if(num_neurons > ann->total_neurons_allocated)
	{
		/* Then we need to allocate more neurons
		 * Allocate half as many neurons as already exist (at least ten)
		 */
		neurons_to_allocate = num_neurons + num_neurons / 2;
		if(neurons_to_allocate < num_neurons + 10)
		{
			neurons_to_allocate = num_neurons + 10;
		}

		if(fann_reallocate_neurons(ann, neurons_to_allocate) == -1)
		{
			return -1;
		}
	}

	if(num_connections > ann->total_connections_allocated)
	{
		/* Then we need to allocate more connections
		 * Allocate half as many connections as already exist
		 * (at least enough for ten neurons)
		 */
		connections_to_allocate = num_connections + num_connections / 2;
		if(connections_to_allocate < num_connections + ann->total_neurons * 10)
		{
			connections_to_allocate = num_connections + ann->total_neurons * 10;
		}

		if(fann_reallocate_connections(ann, connections_to_allocate) == -1)
		{
			return -1;
		}
	}

	/* Set the neurons.
	 */
	connection_it = first_candidate_connection;
	neurons = ann->first_layer->first_neuron;
	candidate_index = first_candidate_neuron;

	for(i = 0; i < ann->cascade_activation_functions_count; i++)
	{
		for(j = 0; j < ann->cascade_activation_steepnesses_count; j++)
		{
			for(k = 0; k < ann->cascade_num_candidate_groups; k++)
			{
				/* TODO candidates should actually be created both in
				 * the last layer before the output layer, and in a new layer.
				 */
				neurons[candidate_index].value = 0;
				neurons[candidate_index].sum = 0;
				
				neurons[candidate_index].activation_function =
					ann->cascade_activation_functions[i];
				neurons[candidate_index].activation_steepness =
					ann->cascade_activation_steepnesses[j];
				
				neurons[candidate_index].first_con = connection_it;
				connection_it += candidate_connections_in;
				neurons[candidate_index].last_con = connection_it;
				/* We have no specific pointers to the output weights, but they are
				 * available after last_con */
				connection_it += candidate_connections_out;
				ann->train_errors[candidate_index] = 0;
				candidate_index++;
			}
		}
	}

	/* Now randomize the weights and zero out the arrays that needs zeroing out.
	 */
#ifdef CASCADE_DEBUG_FULL
	printf("random cand weight [%d ... %d]\n", first_candidate_connection, num_connections - 1);
#endif
	if(ann->training_algorithm == FANN_TRAIN_RPROP)
	{
		initial_slope = ann->rprop_delta_zero;
	}
	else
	{
		initial_slope = 0.0;
	}
	for(i = first_candidate_connection; i < num_connections; i++)
	{
		ann->weights[i] = fann_random_weight();
		/*ann->weights[i] = fann_rand(-0.25,0.25);*/
		ann->train_slopes[i] = 0;
		ann->prev_steps[i] = 0;
		ann->prev_train_slopes[i] = initial_slope;
	}

	return 0;
}

int fann_train_candidates(struct fann *ann, struct fann_train_data *data)
{
	fann_type best_cand_score = 0.0;
	fann_type target_cand_score = 0.0;
	fann_type backslide_cand_score = -1.0e20f;
	unsigned int i;
	unsigned int max_epochs = ann->cascade_max_cand_epochs;
	unsigned int stagnation = max_epochs;

	if(ann->cascade_candidate_scores == NULL)
	{
		ann->cascade_candidate_scores =
			(fann_type *) malloc(fann_get_cascade_num_candidates(ann) * sizeof(fann_type));
		if(ann->cascade_candidate_scores == NULL)
		{
			fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
			return 0;
		}
	}

	for(i = 0; i < max_epochs; i++)
	{
		best_cand_score = fann_train_candidates_epoch(ann, data);

		if(best_cand_score / ann->MSE_value > ann->cascade_candidate_limit)
		{
#ifdef CASCADE_DEBUG
			printf("above candidate limit %f/%f > %f", best_cand_score, ann->MSE_value,
				   ann->cascade_candidate_limit);
#endif
			return i + 1;
		}

		if((best_cand_score > target_cand_score) || (best_cand_score < backslide_cand_score))
		{
#ifdef CASCADE_DEBUG_FULL
			printf("Best candidate score %f, real score: %f\n", ann->MSE_value - best_cand_score,
				   best_cand_score);
			/* printf("best_cand_score=%f, target_cand_score=%f, backslide_cand_score=%f, stagnation=%d\n", best_cand_score, target_cand_score, backslide_cand_score, stagnation); */
#endif

			target_cand_score = best_cand_score * (1.0f + ann->cascade_candidate_change_fraction);
			backslide_cand_score = best_cand_score * (1.0f - ann->cascade_candidate_change_fraction);
			stagnation = i + ann->cascade_candidate_stagnation_epochs;
		}

		/* No improvement in allotted period, so quit */
		if(i >= stagnation)
		{
#ifdef CASCADE_DEBUG
			printf("Stagnation with %d epochs, best candidate score %f, real score: %f\n", i + 1,
				   ann->MSE_value - best_cand_score, best_cand_score);
#endif
			return i + 1;
		}
	}

#ifdef CASCADE_DEBUG
	printf("Max epochs %d reached, best candidate score %f, real score: %f\n", max_epochs,
		   ann->MSE_value - best_cand_score, best_cand_score);
#endif
	return max_epochs;
}

void fann_update_candidate_slopes(struct fann *ann)
{
	struct fann_neuron *neurons = ann->first_layer->first_neuron;
	struct fann_neuron *first_cand = neurons + ann->total_neurons + 1;
	struct fann_neuron *last_cand = first_cand + fann_get_cascade_num_candidates(ann);
	struct fann_neuron *cand_it;
	unsigned int i, j, num_connections;
	unsigned int num_output = ann->num_output;
	fann_type max_sum, cand_sum, activation, derived, error_value, diff, cand_score;
	fann_type *weights, *cand_out_weights, *cand_slopes, *cand_out_slopes;
	fann_type *output_train_errors = ann->train_errors + (ann->total_neurons - ann->num_output);

	for(cand_it = first_cand; cand_it < last_cand; cand_it++)
	{
		cand_score = ann->cascade_candidate_scores[cand_it - first_cand];
		error_value = 0.0;

		/* code more or less stolen from fann_run to fast forward pass
		 */
		cand_sum = 0.0;
		num_connections = cand_it->last_con - cand_it->first_con;
		weights = ann->weights + cand_it->first_con;

		/* unrolled loop start */
		i = num_connections & 3;	/* same as modulo 4 */
		switch (i)
		{
			case 3:
				cand_sum += weights[2] * neurons[2].value;
			case 2:
				cand_sum += weights[1] * neurons[1].value;
			case 1:
				cand_sum += weights[0] * neurons[0].value;
			case 0:
				break;
		}

		for(; i != num_connections; i += 4)
		{
			cand_sum +=
				weights[i] * neurons[i].value +
				weights[i + 1] * neurons[i + 1].value +
				weights[i + 2] * neurons[i + 2].value + weights[i + 3] * neurons[i + 3].value;
		}
		/*
		 * for(i = 0; i < num_connections; i++){
		 * cand_sum += weights[i] * neurons[i].value;
		 * }
		 */
		/* unrolled loop end */

		max_sum = 150/cand_it->activation_steepness;
		if(cand_sum > max_sum)
			cand_sum = max_sum;
		else if(cand_sum < -max_sum)
			cand_sum = -max_sum;
		
		activation =
			fann_activation(ann, cand_it->activation_function, cand_it->activation_steepness,
							cand_sum);
		/* printf("%f = sigmoid(%f);\n", activation, cand_sum); */

		cand_it->sum = cand_sum;
		cand_it->value = activation;

		derived = fann_activation_derived(cand_it->activation_function,
										  cand_it->activation_steepness, activation, cand_sum);

		/* The output weights is located right after the input weights in
		 * the weight array.
		 */
		cand_out_weights = weights + num_connections;

		cand_out_slopes = ann->train_slopes + cand_it->first_con + num_connections;
		for(j = 0; j < num_output; j++)
		{
			diff = (activation * cand_out_weights[j]) - output_train_errors[j];
#ifdef CASCADE_DEBUG_FULL
			/* printf("diff = %f = (%f * %f) - %f;\n", diff, activation, cand_out_weights[j], output_train_errors[j]); */
#endif
			cand_out_slopes[j] -= 2.0f * diff * activation;
#ifdef CASCADE_DEBUG_FULL
			/* printf("cand_out_slopes[%d] <= %f += %f * %f;\n", j, cand_out_slopes[j], diff, activation); */
#endif
			error_value += diff * cand_out_weights[j];
			cand_score -= (diff * diff);
#ifdef CASCADE_DEBUG_FULL
			/* printf("cand_score[%d][%d] = %f -= (%f * %f)\n", cand_it - first_cand, j, cand_score, diff, diff); */

			printf("cand[%d]: error=%f, activation=%f, diff=%f, slope=%f\n", cand_it - first_cand,
				   output_train_errors[j], (activation * cand_out_weights[j]), diff,
				   -2.0 * diff * activation);
#endif
		}

		ann->cascade_candidate_scores[cand_it - first_cand] = cand_score;
		error_value *= derived;

		cand_slopes = ann->train_slopes + cand_it->first_con;
		for(i = 0; i < num_connections; i++)
		{
			cand_slopes[i] -= error_value * neurons[i].value;
		}
	}
}

void fann_update_candidate_weights(struct fann *ann, unsigned int num_data)
{
	struct fann_neuron *first_cand = (ann->last_layer - 1)->last_neuron + 1;	/* there is an empty neuron between the actual neurons and the candidate neuron */
	struct fann_neuron *last_cand = first_cand + fann_get_cascade_num_candidates(ann) - 1;

	switch (ann->training_algorithm)
	{
		case FANN_TRAIN_RPROP:
			fann_update_weights_irpropm(ann, first_cand->first_con,
										last_cand->last_con + ann->num_output);
			break;
		case FANN_TRAIN_QUICKPROP:
			fann_update_weights_quickprop(ann, num_data, first_cand->first_con,
										  last_cand->last_con + ann->num_output);
			break;
		case FANN_TRAIN_BATCH:
		case FANN_TRAIN_INCREMENTAL:
			fann_error((struct fann_error *) ann, FANN_E_CANT_USE_TRAIN_ALG);
			break;
	}
}

fann_type fann_train_candidates_epoch(struct fann *ann, struct fann_train_data *data)
{
	unsigned int i, j;
	unsigned int best_candidate;
	fann_type best_score;
	unsigned int num_cand = fann_get_cascade_num_candidates(ann);
	fann_type *output_train_errors = ann->train_errors + (ann->total_neurons - ann->num_output);
	struct fann_neuron *output_neurons = (ann->last_layer - 1)->first_neuron;

	for(i = 0; i < num_cand; i++)
	{
		/* The ann->MSE_value is actually the sum squared error */
		ann->cascade_candidate_scores[i] = ann->MSE_value;
	}
	/*printf("start score: %f\n", ann->MSE_value); */

	for(i = 0; i < data->num_data; i++)
	{
		fann_run(ann, data->input[i]);

		for(j = 0; j < ann->num_output; j++)
		{
			/* TODO only debug, but the error is in opposite direction, this might be usefull info */