nakayama.java

一个纯java写的神经网络源代码

/*
 * Nakayama.java
 *
 * Created on October 22, 2004, 1:30 PM
 */

package org.joone.structure;

import java.util.*;
import org.joone.engine.*;
import org.joone.engine.listeners.*;
import org.joone.log.*;
import org.joone.net.*;

/**
 * This class performs the method of optimizing activation functions as described
 * in: <p>
 * K.Nakayama and Y.Kimura, <i>"Optimization of activation functions in multilayer 
 * neural network applied to pattern classification"</i>, Proc. IEEE ICNN'94 Florida, 
 * pp.431-436, June 1994.
 * <p>
 * <p>
 * <i>This techniques probably fails whenever the <code>NeuralNet.join()</code> method 
 * is called because this optimization technique stops the network to perform the 
 * optimization, use a <code>NeuralNetListener</code> instead.</i>
 * 
 *
 * @author  Boris Jansen
 */
public class Nakayama implements NeuralNetListener, NeuralValidationListener, ConvergenceListener, java.io.Serializable {
    
    /** Logger for this class. */
    private static final ILogger log = LoggerFactory.getLogger(Nakayama.class);
    
    /** Constant indicating the neuron will not be removed (should be zero). */
    private static final int NO_REMOVE = 0;
    
    /** Constant indicating a neuron will be removed based on its information value Ij. */
    private static final int INFO_REMOVE = 1;
    
    /** Constant indicating a neuron will be removed based on its variance value Vj. */
    private static final int VARIANCE_REMOVE = 2;
    
    /** Constant indicating a neuron MIGHT be removed based on its correlation value Rjmin. */
    private static final int CORRELATION_POSSIBLE_REMOVE = 3;
    
    /** Constant indicating a neuron WILL be removed based on its correlation value Rjmin. */
    private static final int CORRELATION_REMOVE = 4;
    
    /** Constant indicating that a neuron is removed. */
    private static final int REMOVE_DONE = -1;
    
    /** List with layers to be optimized (layers should be on the same (layer) level). */
    private List layers = new ArrayList();
    
    /** Flag to remember if the network was running when the request come to optimize
     * the network. If so we will re-start it after optimization. */
    private boolean isRunning;
    
    /** The net to optimize. */
    private NeuralNet net;
    
    /** The clone of a network to collect information (outputs of neurons after a pattern has been
     * forwarded through the network). */
    private NeuralNet clone;
    
    /** The threshold to decide if the neuron should be deleted or not, i.e. if
     * Cj = Ij / max-j{Ij} * Vj / max-j{Vj} * Rjmin <= epsilon. */
    private double epsilon = 0.05; // default value
    
    /** The original listeners of the neural network that will be temporarely removed.
     * We will add them again after optimization when we restart the network. */
    private List listeners = new ArrayList();
    
    /** Holds the outputs of the neurons after each pattern [pattern->layer->neuron]. */
    private List outputsAfterPattern;
    
    /** Holds the information for every neuron (excluding input and output neurons)
     * to its output layers [layer->neuron]. */
    private List information;
    
    /** The maximum value of the information from a neuron to its output layer. */
    private double infoMax;
    
    /** Holds the average output over all patterns for every neuron (excluding input and
     * output neurons) [layer->neuron]. */
    private List averageOutputs;  
    
    /** Holds the variance for every neuron. */
    private List variance;
    
    /** The maximum variance value [layer->neuron]. */
    private double varianceMax;
    
    /** Holds the gamma value for every neuron [layer->neuron->layer->neuron]. */
    private List gamma;
    
    /** Flag indicating if the network is optimized, i.e. if neurons are removed. */
    private boolean optimized = false;
    
    /** 
     * Creates a new instance of Nakayama.
     *
     * @param aNet the network to be optimized.
     */
    public Nakayama(NeuralNet aNet) {
        net = aNet;
    }
    
    /**
     * Adds layers to this optimizer. The layers will be optimized. Layers should
     * be on the same (layer) level, otherwise the optimization does not make sense.
     * 
     * @param aLayer the layer to be added.
     */
    public void addLayer(Layer aLayer) {
        layers.add(aLayer);
    }
    
    /**
     * Adds all the hidden layers to this optimizer. The layers will be optimized.
     * The neuron network should consist of only one hidden layer, i.e. the hidden
     * layers should be on the same level else this method doesn't make any sense.
     * If the hidden layers are not all on the same level then the layers should be
     * added individually my using {@ling addLayer(Layer)}, adding only the layers
     * that are on the same hidden level.
     *
     * @param aNeuralNet the network holding the hidden layers.
     */
    public void addLayers(NeuralNet aNeuralNet) {
        for(int i = 0; i < aNeuralNet.getLayers().size(); i++) {
            Object myLayer = aNeuralNet.getLayers().get(i);
            if(myLayer != aNeuralNet.getInputLayer() && myLayer != aNeuralNet.getOutputLayer()) {
                layers.add(myLayer);
            }
        }
    }
    
    /**
     * Optimizes the activation functions of the neural network.
     *
     * @return whether the network was optimized or not, i.e. if neurons where deleted
     * or not.
     */
    public boolean optimize() {
        // init (throw away any old values from previous optimization round)
        outputsAfterPattern = new ArrayList();
        information = new ArrayList();
        infoMax = 0;
        averageOutputs = new ArrayList();
        variance = new ArrayList();
        varianceMax = 0;
        gamma = new ArrayList();
        optimized = false;
        
        log.debug("Optimization request [cycle : " + net.getMonitor().getCurrentCicle() + "]");
        
        isRunning = net.isRunning();
        if(isRunning) {
            log.debug("Stopping network...");
            removeAllListeners();
            net.addNeuralNetListener(this);
            net.getMonitor().Stop();
            // runValidation() will be called from cicleTerminated() after the network has been stopped
        } else {
            runValidation();
        }
        return optimized;
    }
    
    /**
     * Runs the network with a validator, this way we are able to collect information
     * related to the different patterns. Everytime a pattern is forwarded to the 
     * network we collect certain info {@link patternFinished()}.
     */
    protected void runValidation() {
        net.getMonitor().setExporting(true);
        clone = net.cloneNet();
        net.getMonitor().setExporting(false);
        
        clone.removeAllListeners();
        // add the following synapse so everytime a pattern has been forwarded through
        // the network patternFinished() is called so we can collect certain info
        clone.getOutputLayer().addOutputSynapse(new PatternForwardedSynapse(this));

        // run the network to collect information
        log.debug("Validating network...");
        NeuralNetValidator myValidator = new NeuralNetValidator(clone);
        myValidator.addValidationListener(this);
        myValidator.useTrainingData(true); // just use the normal training data
        myValidator.start();
    }
    
    /**
     * Optimizes the activation functions of the network.
     */
    protected void doOptimize() {
        log.debug("Optimizing...");
        
        evaluateNeurons();
        selectNeurons();
        
        log.debug("Optimization done.");
        cleanUp();
        
        // debug info
        Layer myLayer;
        for(int i = 0; i < net.getLayers().size(); i++) {
            myLayer = (Layer)net.getLayers().get(i);
            log.debug("Layer [" + myLayer.getClass().getName() + "] - neurons : " + myLayer.getRows());
        }
        // end debug
    }
    
    /**
     * This method is called after optimization, e.g. to restore the listeners.
     */
    protected void cleanUp() {
        log.debug("Cleaning up...");
        outputsAfterPattern = null;
        information = null;
        averageOutputs = null;
        variance = null;
        gamma = null;
        clone = null;

        // remove layers that have no input and output synapses
        Layer myLayer;
        for(int i = 0; i < layers.size(); i++) {
            myLayer = (Layer)layers.get(i);
            if(myLayer.getRows() == 0) {
                log.debug("Remove layer [" + myLayer.getClass().getName() + "]");
                net.removeLayer(myLayer);
                layers.remove(i);
                i--; // layer is removed so index changes
            }
        }
        
        net.removeNeuralNetListener(this);
        restoreAllListeners();
        log.debug("Clean ;)");
        if(isRunning) {
            log.debug("Restarting net...");
            net.start();
            net.getMonitor().runAgain();
        }
        
    }
    
    /**
     * Selects neurons to be deleted based on the information calculated by
     * <code>evaluateNeurons()</code>. It selects neurons to be deleted and 
     * performs the deletion.
     */
    protected void selectNeurons() {
        log.debug("Selecting neurons...");
        
        Layer myLayer;
        List myStatuses = new ArrayList(); // will hold the status for every neuron
        int [] myStatus; // holds the status of neurons for a single layer
        double myScaledInfo, myScaledVariance; // scaled info Ij^ = Ij / max-j{Ij}, scaled variance Vj^ = Vj / max-j{Vj}
        double[] myMinCorrelation; // array holding the minimum correlation, together with the index of the neuron j and j' 
                                   // which have the minimum correlation