nakayama.java

一个纯java写的神经网络源代码

        Matrix myBiases, myWeights;
        double myAverageOutput;
        Layer myOutputLayer, myLayer = (Layer)layers.get(aLayer);
        for(int i = 0; i < myLayer.getAllOutputs().size(); i++) {
            myElement = (NeuralElement)myLayer.getAllOutputs().get(i);
            if(!(myElement instanceof Synapse)) {
                // TODO how to deal with outputs that are not synpases?
                throw new org.joone.exception.JooneRuntimeException("Unable to optimize. Output of layer is not a synapse.");
            }
            mySynapse = (Synapse)myElement;
            myOutputLayer = findOutputLayer(mySynapse);
            myBiases = myOutputLayer.getBias();
            myWeights = mySynapse.getWeights();
            myAverageOutput = ((double[])averageOutputs.get(aLayer))[aNeuronOriginal];
            for(int r = 0; r < myOutputLayer.getRows(); r++) {
                myBiases.value[r][0] += myWeights.value[aNeuron][r] * myAverageOutput;
                myBiases.delta[r][0] = 0;
            } 
        }
    }
    
    /**
     * Removes a neuron.
     *
     * @param aLayer the index of the layer in which we should remove the neuron.
     * @param aNeuron the index of the neuron to be removed (taking into account previous
     * deletions).
     */
    protected void removeNeuron(int aLayer, int aNeuron) {
        Layer myLayer = (Layer)layers.get(aLayer);
        
        NeuralElement myElement;
        Synapse mySynapse;
        Matrix myWeights;
        if(myLayer.getRows() > 1) {
            for(int i = 0; i < myLayer.getAllInputs().size(); i++) {
                myElement = (NeuralElement)myLayer.getAllInputs().get(i);
                if(!(myElement instanceof Synapse)) {
                    // TODO how to deal with inputs that are not synpases?
                    throw new org.joone.exception.JooneRuntimeException("Unable to optimize. Input of layer is not a synapse.");
                }
                mySynapse = (Synapse)myElement;
                myWeights = mySynapse.getWeights();
                myWeights.removeColumn(aNeuron);
                mySynapse.setOutputDimension(mySynapse.getOutputDimension() - 1);
                mySynapse.setWeights(myWeights);
            }
            for(int i = 0; i < myLayer.getAllOutputs().size(); i++) {
                myElement = (NeuralElement)myLayer.getAllOutputs().get(i);
                if(!(myElement instanceof Synapse)) {
                    // TODO how to deal with outputs that are not synpases?
                    throw new org.joone.exception.JooneRuntimeException("Unable to optimize. Output of layer is not a synapse.");
                }
                mySynapse = (Synapse)myElement;
                myWeights = mySynapse.getWeights();
                myWeights.removeRow(aNeuron);
                mySynapse.setInputDimension(mySynapse.getInputDimension() - 1);
                mySynapse.setWeights(myWeights);
            }
            myWeights = myLayer.getBias();
            myWeights.removeRow(aNeuron);
            myLayer.setRows(myLayer.getRows() - 1);
            myLayer.setBias(myWeights);
        } else {
            // we are going to remove the last neuron so remove the layer and its input an output synapses
            for(int i = 0; i < myLayer.getAllInputs().size(); i++) {
                myElement = (NeuralElement)myLayer.getAllInputs().get(i);
                if(!(myElement instanceof Synapse)) {
                    // TODO how to deal with inputs that are not synpases?
                    throw new org.joone.exception.JooneRuntimeException("Unable to optimize. Input of layer is not a synapse.");
                }
                mySynapse = (Synapse)myElement;
                Layer myInputLayer = findInputLayer(mySynapse);
                myInputLayer.removeOutputSynapse(mySynapse);
            }
            for(int i = 0; i < myLayer.getAllOutputs().size(); i++) {
                myElement = (NeuralElement)myLayer.getAllOutputs().get(i);
                if(!(myElement instanceof Synapse)) {
                    // TODO how to deal with outputs that are not synpases?
                    throw new org.joone.exception.JooneRuntimeException("Unable to optimize. Output of layer is not a synapse.");
                }
                mySynapse = (Synapse)myElement;
                Layer myOutputLayer = findOutputLayer(mySynapse);
                myOutputLayer.removeInputSynapse(mySynapse);
            }
            // if we remove the layer here from the network the index of layers goes out of order,
            // after optimization is done we will remove the layer
            myWeights = myLayer.getBias();
            myWeights.removeRow(aNeuron);
            myLayer.setRows(myLayer.getRows() - 1);
            myLayer.setBias(myWeights);
        }
    }
    
    /**
     * Finds the input layer of a synapse.
     *
     * @param aSynapse the synapse to find the input layer for.
     */
    protected Layer findInputLayer(Synapse aSynapse) {
        Layer myLayer;
        for(int i = 0; i < net.getLayers().size(); i++) {
            myLayer = (Layer)net.getLayers().get(i);
            if(myLayer.getAllOutputs().contains(aSynapse)) {
                return myLayer;
            }
        }
        return null;
    }
    
    /**
     * Finds the output layer of a synapse.
     *
     * @param aSynapse the synapse to find the output layer for.
     */
    protected Layer findOutputLayer(Synapse aSynapse) {
        Layer myLayer;
        for(int i = 0; i < net.getLayers().size(); i++) {
            myLayer = (Layer)net.getLayers().get(i);
            if(myLayer.getAllInputs().contains(aSynapse)) {
                return myLayer;
            }
        }
        return null;
    }
    
    /**
     * Evaluates neurons, that is, this function calculates information related to
     * the contribution of the activation functions, based on the following three
     * criteria: <br>
     * <ul>
     *      <li>Information from neurons to its output layers. </li>
     *      <li>Variance of the output of the neurons.</li>
     *      <li>Correlation between outputs of neurons.</li>
     * </ul>
     * This information will be used in a next stage to select neurons to delete.
     */
    protected void evaluateNeurons() {
        log.debug("Evaluation of neurons...");
        
        Layer myLayer; // help variable
        int myNrOfPatterns = net.getMonitor().getTrainingPatterns(); // number of patterns
        double [] myInfo; // Ij (for a single layer)
        double [] myAvgOutputs; // the average output over all patterns (for a single layer)
        
        // first we will calculate the information from the jth hidden unit to its output layer (Ij)
        // Ij = 1/M * sum{p=1:M}(sum{k=1:No}(|wjk*vpj|)), this is equal to
        //    = 1/M * sum{p=1:M}(|vpj|)) * sum{k=1:No}(|wjk|)
        // M is number of patterns, No is number of output units, 
        // vpj = the output of a neuron j for pattern p, wjk is the weight from neuron j to k
        // during this the calculation of Ij we also calculate the average output for each neuron j
        // _vj = 1/M sum{p=1:M}(vpj)
        
        for(int i = 0; i < layers.size(); i++) {
            myLayer = (Layer)layers.get(i);
            myInfo = new double[myLayer.getRows()];
            myAvgOutputs = new double[myLayer.getRows()]; 
            for(int n = 0; n < myLayer.getRows(); n++) {
                // for all neurons in a layer
                double myTempSumWeights = getSumAbsoluteWeights(myLayer, n); // get sum{k=1:No}(|wjk|)
                double[] myTempSumOutputs = getSumOutputs(i, n); // get sum{p=1:M}(vpj) and sum{p=1:M}(|vpj|)

                myInfo[n] = (myTempSumWeights * myTempSumOutputs[1]) / myNrOfPatterns;
                if(myInfo[n] > infoMax) {
                    // also find max value of the information max-j{Ij}
                    infoMax = myInfo[n];
                }
                myAvgOutputs[n] = myTempSumOutputs[0] / myNrOfPatterns;
            }
            information.add(myInfo);
            averageOutputs.add(myAvgOutputs);
        }
        
        // at this moment we have calculated (A) information from each neuron j to its output 
        // layer (Ij and maxj{Ij}) and we have calculated the average of outputs for each neuron
        // ((B) _vj)
        
        // In the next step we will calculate the variance (B) Vj
        // Vj = sum{p=1:M}(vpj - _vj)^2, _vj the average is calculate above
        // we will also store (vpj - _vj) for all neurons, so in the following step it will be easier 
        // to calculate the correlation
        
        double [] myVariance; // variances for a layer
        double [] myTempDifferences; // differences (output-pattern - avg-output <=> vjp - _vj) for a single layer and single pattern
        List myDifferences = new ArrayList(); // all differences (all layers and over all patterns) 
                                              // vpj - _vj [layer->pattern->neuron], NOTE outputsAfterPattern is [pattern->layer->neuron]
        List myDifferencesForLayer; // differences of all patterns for a single layer [pattern->neuron]
        for(int i = 0; i < layers.size(); i++) {
            myLayer = (Layer)layers.get(i);
            myVariance = new double[myLayer.getRows()]; // Vj
            myDifferencesForLayer = new ArrayList();
            for(int p = 0; p < outputsAfterPattern.size(); p++) {
                myTempDifferences = new double[myLayer.getRows()]; // differences for a single pattern for a single layer
                for(int n = 0; n < myLayer.getRows(); n++) {
                    List myOutputs = (List)outputsAfterPattern.get(p);
                    myTempDifferences[n] = ((double[])myOutputs.get(i))[n] 
                        - ((double[])averageOutputs.get(i))[n]; // vpj - _vj
                    myVariance[n] += myTempDifferences[n] * myTempDifferences[n];
                }
                myDifferencesForLayer.add(myTempDifferences);
            }
            for(int n = 0; n < myLayer.getRows(); n++) {
                // also find max variance
                if(myVariance[n] > varianceMax) {
                    varianceMax = myVariance[n];
                }
            }
            myDifferences.add(myDifferencesForLayer);
            variance.add(myVariance);
        }
        
        // Now we have calculated the variance for each neuron (B) Vj and myDifferences holds the differences
        // between the output of a neuron and the average output over all patterns and all layers. Now we will
        // calculate gamma (C), which is closely related to the correlation, which will be needed later
        // Ajj' = sum{p=1:M}((vpj - _vj) * (vpj' - _vj'))
        // Bjj' = sum{p=1:M}(vpj - _vj)^2 * sum{p=1:M}(vpj' - _vj')^2
        //      = Vj * Vj'
        // Gammajj' = Ajj' / Bjj'^1/2
        // The correlation between units is defined by:
        // Rjj' = 1 - |Gammajj'|, however we will not calculate Rjj' here
        
        Layer myLayer1, myLayer2;
        List myTempDifferencesForLayer1, myTempDifferencesForLayer2; // differences between vpj - _vj and vpj' - _vj' (j = n1, j' = n2)
        // Pointers to construct the tree to save the gammas [gamma->layer1->neuron1->layer2->neuron2]
        List [] myNeurons1Pointer;
        double [] myNeurons2Pointer;
        
        for(int l1 = 0; l1 < layers.size(); l1++) { // l1 = layer 1
            myLayer1 = (Layer)layers.get(l1);
            myNeurons1Pointer = new List[myLayer1.getRows()];
            gamma.add(myNeurons1Pointer); // so gamma.get(an index) gets a neuron pointer of layer index
            for(int n1 = 0; n1 < myLayer1.getRows(); n1++) {
                myNeurons1Pointer[n1] = new ArrayList();
                for(int l2 = 0; l2 < layers.size(); l2++) { // l2 = layer 1
                    myLayer2 = (Layer)layers.get(l2);
                    if(l2 < l1) {
                        myNeurons1Pointer[n1].add(new double[0]); // a little waste of memory, but this way it allows us to 
                                                                  // index layers easily later on, because the index of layers
                                                                  // will be still be matching
                    } else {
                        myNeurons2Pointer = new double[myLayer2.getRows()];
                        myNeurons1Pointer[n1].add(myNeurons2Pointer);
                        for(int n2 = (l1 == l2 ? n1+1 : 0); n2 < myLayer2.getRows(); n2++) {
                            double myA = 0, myB = 0;
                            for(int p = 0; p < myNrOfPatterns; p++) {
                                myTempDifferencesForLayer1 = (List)myDifferences.get(l1);
                                myTempDifferencesForLayer2 = (List)myDifferences.get(l2);

                                myA += ((double[])myTempDifferencesForLayer1.get(p))[n1] *
                                    ((double[])myTempDifferencesForLayer2.get(p))[n2];
                            }
                            myB = ((double[])variance.get(l1))[n1] * ((double[])variance.get(l2))[n2];
                            myNeurons2Pointer[n2] = myA / Math.sqrt(myB);
                        }
                    }
                }
            }
        }
        log.debug("Evaluation done.");
    }
    
    /**
     * Gets the minimum correlation for a certain neuron j. A correlation between neuron