nakayama.java

一个纯java写的神经网络源代码

        List myMinCorrelationPointers = new ArrayList(); // if the min correlation Rjj' is the lowest of Ij^, Vj^ and Rjj'
                                                         // then we save the min correlation info, because the neuron that
                                                         // is part of this min correlation needs more investigation later
                                                         // to decide if the neuron should be deleted or not
        for(int i = 0; i < layers.size(); i++) {
            myLayer = (Layer)layers.get(i);
            myStatus = new int[myLayer.getRows()]; // initially all elements equal 0, so status is NO_REMOVE
            for(int n = 0; n < myLayer.getRows(); n++) {
                myScaledInfo = ((double[])information.get(i))[n] / infoMax;
                myScaledVariance = ((double[])variance.get(i))[n] / varianceMax;
                myMinCorrelation = getMinCorrelation(i, n);
                
                // log.debug("Info : " + myScaledInfo + ", variance : " + myScaledVariance + ", correlation : " + myMinCorrelation[0]);
                
                if(myScaledInfo * myScaledVariance * myMinCorrelation[0] <= epsilon) {
                    if(myScaledInfo <= myScaledVariance && myScaledInfo <= myMinCorrelation[0]) {
                        myStatus[n] = INFO_REMOVE; // scaled info is the smallest, so neuron should be removed based on its info
                    } else if(myScaledVariance < myScaledInfo && myScaledVariance <= myMinCorrelation[0]) {
                        myStatus[n] = VARIANCE_REMOVE; // scaled variance is the smallest, so neuron should be removed based on its variance
                    } else {
                        myStatus[n] = CORRELATION_POSSIBLE_REMOVE; // set it to possible remove, because it needs more 
                                                                   // examination to decide if it really should be removed. 
                        myMinCorrelationPointers.add(myMinCorrelation); // save the pointer for later investigation
                    }
                }
            }
            myStatuses.add(myStatus);
        }
        
        // now we will investigate which CORRELATION_POSSIBLE_REMOVE neurons should be really removed
        List myCorrelations = new ArrayList(); // this list will hold the arrays indicating the index of the neuron that will be deleted
                                               // together with the index of the neuron it was most closely correlated to (this neuron will
                                               // take over the weights of the other neuron that will be deleted).
        int mySingleStatus; // status of a single neuron
        for(int i = 0; i < myMinCorrelationPointers.size(); i++) {
            myMinCorrelation = (double[])myMinCorrelationPointers.get(i);
            if(myMinCorrelation[5] < 0 && // also check if status is still CORRELATION_POSSIBLE_REMOVE, it might have been changed
                ((int[])myStatuses.get((int)myMinCorrelation[1]))[(int)myMinCorrelation[2]] == CORRELATION_POSSIBLE_REMOVE) 
            {
                // position 1, 2 contain the argument (CORRELATION_POSSIBLE_REMOVE) which should
                // be checked with status neuron argument 3, 4
                mySingleStatus = ((int[])myStatuses.get((int)myMinCorrelation[3]))[(int)myMinCorrelation[4]];
                if(mySingleStatus == INFO_REMOVE || mySingleStatus == VARIANCE_REMOVE || mySingleStatus == CORRELATION_REMOVE) {
                    // neuron that caused the minimum correlation will be removed, so we don't need to remove the other neuron
                    ((int[])myStatuses.get((int)myMinCorrelation[1]))[(int)myMinCorrelation[2]] = NO_REMOVE;
                } else if(((double[])variance.get((int)myMinCorrelation[1]))[(int)myMinCorrelation[2]] <= 
                    ((double[])variance.get((int)myMinCorrelation[3]))[(int)myMinCorrelation[4]]) 
                {
                    ((int[])myStatuses.get((int)myMinCorrelation[1]))[(int)myMinCorrelation[2]] = CORRELATION_REMOVE;
                    myCorrelations.add(new int[] {(int)myMinCorrelation[1], (int)myMinCorrelation[2], (int)myMinCorrelation[3], (int)myMinCorrelation[4]});
                } else {
                    ((int[])myStatuses.get((int)myMinCorrelation[1]))[(int)myMinCorrelation[2]] = NO_REMOVE;
                    ((int[])myStatuses.get((int)myMinCorrelation[3]))[(int)myMinCorrelation[4]] = CORRELATION_REMOVE;
                    myCorrelations.add(new int[] {(int)myMinCorrelation[3], (int)myMinCorrelation[4], (int)myMinCorrelation[1], (int)myMinCorrelation[2]});
                }
            } else if(myMinCorrelation[5] > 0 && // also check if status is still CORRELATION_POSSIBLE_REMOVE, it might have been changed
                ((int[])myStatuses.get((int)myMinCorrelation[3]))[(int)myMinCorrelation[4]] == CORRELATION_POSSIBLE_REMOVE) 
            {
                mySingleStatus = ((int[])myStatuses.get((int)myMinCorrelation[1]))[(int)myMinCorrelation[2]];
                if(mySingleStatus == INFO_REMOVE || mySingleStatus == VARIANCE_REMOVE || mySingleStatus == CORRELATION_REMOVE) {
                    ((int[])myStatuses.get((int)myMinCorrelation[3]))[(int)myMinCorrelation[4]] = NO_REMOVE;
                } else if(((double[])variance.get((int)myMinCorrelation[1]))[(int)myMinCorrelation[2]] >= 
                    ((double[])variance.get((int)myMinCorrelation[3]))[(int)myMinCorrelation[4]]) 
                {
                    ((int[])myStatuses.get((int)myMinCorrelation[3]))[(int)myMinCorrelation[4]] = CORRELATION_REMOVE;
                    myCorrelations.add(new int[] {(int)myMinCorrelation[3], (int)myMinCorrelation[4], (int)myMinCorrelation[1], (int)myMinCorrelation[2]});
                } else {
                    ((int[])myStatuses.get((int)myMinCorrelation[3]))[(int)myMinCorrelation[4]] = NO_REMOVE;
                    ((int[])myStatuses.get((int)myMinCorrelation[1]))[(int)myMinCorrelation[2]] = CORRELATION_REMOVE;
                    myCorrelations.add(new int[] {(int)myMinCorrelation[1], (int)myMinCorrelation[2], (int)myMinCorrelation[3], (int)myMinCorrelation[4]});
                }
            }
        }
        
        int myNeuron; // used to index neurons taking into account the effect of neurons deleted before
        for(int l = 0; l < myStatuses.size(); l++) {
            myStatus = (int[])myStatuses.get(l);
            myNeuron = 0;
            for(int n = 0; n < myStatus.length; n++) {
                if(myStatus[n] == INFO_REMOVE) {
                    log.debug("Remove[info]: " + l + " " + n);
                    removeNeuron(l, myNeuron);
                    optimized = true; // neurons are moved, so the network is really optimized (changed)
                    myStatus[n] = REMOVE_DONE;
                } else if(myStatus[n] == VARIANCE_REMOVE) {
                    log.debug("Remove[variance]: " + l + " " + n);
                    weightsUpdateVariance(l, n, myNeuron);
                    removeNeuron(l, myNeuron);
                    optimized = true;
                    myStatus[n] = REMOVE_DONE;
                } else if(myStatus[n] == CORRELATION_REMOVE) {
                    log.debug("Remove[correlation]: " + l + " " + n);
                    weightsUpdateCorrelation(myStatuses, myCorrelations, l, n);
                    removeNeuron(l, myNeuron);
                    optimized = true;
                    myStatus[n] = REMOVE_DONE;
                } else if(myStatus[n] == NO_REMOVE) {
                    // neuron is not removed so move to next neuron in layer
                    myNeuron++;
                }
            }
        }
        log.debug("Selection done.");
    }
    
    /**
     * Updates weights before a neuron is removed (because of its similar correlation).
     *
     * @param aStatuses the status of the neurons (used to find the correct neuron taking into account
     * any deletions of neurons).
     * @param aCorrelations a list holding all the correlations (neurons to be removed and the
     * correlated neuron (which will take over the weights)).
     * @param aLayer the layer of the neuron to be removed.
     * @param aNeuron the neuron to be removed.
     */
    protected void weightsUpdateCorrelation(List aStatuses, List aCorrelations, int aLayer, int aNeuron) {
        int [] myCorrelatedNeuron = null, myTemp = findCorrelation(aCorrelations, aLayer, aNeuron);

        // the correlated neuron of the neuron to be removed might be part of another correlation and will be removed
        // so we'll search in the chain of correlated neurons for the neuron that will not be removed (myCorrelatedNeuron)
        while(myTemp != null) {
            myCorrelatedNeuron = myTemp;
            myTemp = findCorrelation(aCorrelations, myCorrelatedNeuron[0], myCorrelatedNeuron[1]);
        }
        
        // take into account any deletions of previous neurons
        int myAdjustedNeuron = findIndex(aStatuses, aLayer, aNeuron);
        int myAdjustedCorrelatedNeuron = findIndex(aStatuses, myCorrelatedNeuron[0], myCorrelatedNeuron[1]);
        
        // the weights of the correlated neuron j will be updated in the following way taking into account
        // the effect of the neuron j that will be removed:
        // wjk = wjk + a * wj'k
        // bk = bk + wj'k * (_vj' - a * _vj)
        // a = sign(gammajj') * {Vj' / Vj}^1/2
        
        double myAlpha = (getGamma(aLayer, aNeuron, myCorrelatedNeuron[0], myCorrelatedNeuron[1]) >= 0 ? 1 : -1) * 
            Math.sqrt(((double[])variance.get(aLayer))[aNeuron] / ((double[])variance.get(aLayer))[myCorrelatedNeuron[1]]);
        
        NeuralElement myElement, myElementCorrelation;
        Synapse mySynapse, mySynapseCorrelation = null;
        Matrix myBiases, myWeights, myWeightsCorrelation;
        Layer myOutputLayer, myLayer = (Layer)layers.get(aLayer), myLayerCorrelation = (Layer)layers.get(myCorrelatedNeuron[0]);
        if(myLayer.getAllOutputs().size() != myLayerCorrelation.getAllInputs().size()) {
            throw new org.joone.exception.JooneRuntimeException("Unable to optimize. #output layers for neuron and correlated neuron are not equal.");
        }

        for(int i = 0; i < myLayer.getAllOutputs().size(); i++) {
            myElement = (NeuralElement)myLayer.getAllOutputs().get(i);
            if(!(myElement instanceof Synapse)) {
                // TODO how to deal with outputs that are not synpases?
                throw new org.joone.exception.JooneRuntimeException("Unable to optimize. Output of layer is not a synapse.");
            }
            mySynapse = (Synapse)myElement;
            myOutputLayer = findOutputLayer(mySynapse);
            
            // find synapse from correlation layer to the same output layer
            for(int j = 0; j < myLayerCorrelation.getAllOutputs().size() && mySynapseCorrelation == null; j++) {
                myElementCorrelation = (NeuralElement)myLayerCorrelation.getAllOutputs().get(j);
                if(myElementCorrelation instanceof Synapse) {
                    mySynapseCorrelation = (Synapse)myElementCorrelation;
                    if(findOutputLayer(mySynapseCorrelation) != myOutputLayer) {
                        mySynapseCorrelation = null;
                    }
                }
            }
            if(mySynapseCorrelation == null) {
                throw new org.joone.exception.JooneRuntimeException("Unable to optimize. Unable to find same output layer for correlated layer.");
            }
            
            myBiases = myOutputLayer.getBias();
            myWeights = mySynapse.getWeights();
            myWeightsCorrelation = mySynapseCorrelation.getWeights();
            for(int r = 0; r < myOutputLayer.getRows(); r++) {
                myBiases.value[r][0] += myWeights.value[myAdjustedNeuron][r] * 
                    (((double[])averageOutputs.get(aLayer))[aNeuron] - myAlpha * ((double[])averageOutputs.get(myCorrelatedNeuron[0]))[myCorrelatedNeuron[1]]);
                myBiases.delta[r][0] = 0;
                myWeightsCorrelation.value[myAdjustedCorrelatedNeuron][r] += myWeights.value[myAdjustedNeuron][r];
                myWeightsCorrelation.delta[myAdjustedCorrelatedNeuron][r] = 0;
            } 
        }
    }
    
    /**
     * Gets gammajj' (is equal to gammaj'j).
     *
     * @param aLayer1 the layer of neuron j.
     * @param aNeuron1 the neuron j.
     * @param aLayer2 the layer of neuron j'.
     * @param aNeuron2 the neuron j'.
     * return gammajj'.
     */
    protected double getGamma(int aLayer1, int aNeuron1, int aLayer2, int aNeuron2) {
        int mySwapLayer, mySwapNeuron;
        
        if(aLayer1 > aLayer2 || (aLayer1 == aLayer2 && aNeuron1 > aNeuron2)) {
            mySwapLayer = aLayer1;
            mySwapNeuron = aNeuron1;
            aLayer1 = aLayer2;
            aNeuron1 = aNeuron2;
            aLayer2 = mySwapLayer;
            aNeuron2 = mySwapNeuron;
        }
        return ((double[])((List[])gamma.get(aLayer1))[aNeuron1].get(aLayer2))[aNeuron2];
    }
    
    /**
     * Finds the index of a neuron taking into account the deletion of previous neurons.
     *
     * @param aStatuses the status of neurons.
     * @param aLayer the layer of the neuron.
     * @param aNeuron the index of the neuron, not considering any deletions.
     * @return the index of the neuron taking into account any previous deletions.
     */
    protected int findIndex(List aStatuses, int aLayer, int aNeuron) {
        int[] myStatusLayer = (int[])aStatuses.get(aLayer);
        int myNewIndex = aNeuron;
        
        for(int i = 0; i < aNeuron; i++) {
            if(myStatusLayer[i] == REMOVE_DONE) {
                myNewIndex--;
            }
        }
        return myNewIndex;
    }
    
    /**
     * Finds a correlation from a given list of correlations.
     *
     * @param aCorrelations a list holding correlations.
     * @param aLayer the layer of the neuron to find the correlation for.
     * @param aNeuron the neuron to find the correlation for.
     * @return the index of the neuron of the correlation with <code>(aLayer, aNeuron)</code>,
     * return <code>null</code> in case the correlation is not found.
     */
    protected int[] findCorrelation(List aCorrelations, int aLayer, int aNeuron) {
        int[] myCorrelation;

        for(int i = 0; i < aCorrelations.size(); i++) {
            myCorrelation = (int [])aCorrelations.get(i);
            if(myCorrelation[0] == aLayer && myCorrelation[1] == aNeuron) {
                return new int [] {myCorrelation[2], myCorrelation[3]};
            }
        }
        return null;
    }
    
    /**
     * Updates weights before a neuron is removed (because of its low variance).
     *
     * @param aLayer the index of the layer of the neuron.
     * @param aNeuronOriginal the index of the neuron to be removed (NOT taking into account
     * previous deletions).
     * @param aNeuron the index of the neuron to be removed (taking into account previous
     * deletions).
     */
    protected void weightsUpdateVariance(int aLayer, int aNeuronOriginal, int aNeuron) {
        // the biases of the neurons in the output layer will be updated by taking
        // the effects of the neuron that will be removed into account:
        // bk = bk + wjk * _vj, b = bias, k = index output neuron, j = aNeuron, 
        // _vj is average output neuron aNeuron
        
        NeuralElement myElement;
        Synapse mySynapse;