gneuralnet.cpp

一个非常有用的开源代码

/*
	Copyright (C) 2006, Mike Gashler

	This library is free software; you can redistribute it and/or
	modify it under the terms of the GNU Lesser General Public
	License as published by the Free Software Foundation; either
	version 2.1 of the License, or (at your option) any later version.

	see http://www.gnu.org/copyleft/lesser.html
*/

#include "GNeuralNet.h"
#include "GArff.h"
#include "GMath.h"
#include "GMacros.h"
#include "GArray.h"
#include "GBits.h"

class GNeuron;


#define INIT_THRESH .15
#define OUTPUT_MIN .1
#define OUTPUT_MIDDLE .5
#define OUTPUT_RANGE .8
#define INPUT_MIN -.7
#define INPUT_RANGE 1.4


struct GSynapse
{
	double m_dWeight; // Synapse
	double m_dWeightDelta;
	GNeuron* m_pInput;
	GNeuron* m_pOutput;

	GSynapse()
	{
		m_dWeight = (GBits::GetRandomDouble() * INIT_THRESH) - (INIT_THRESH / 2);
		m_dWeightDelta = 0;
		m_pInput = NULL;
		m_pOutput = NULL;
	}
};



class GNeuron
{
public:
	GNeuron() {}
	virtual ~GNeuron() {}

	virtual double PullEvalDownStream() = 0;
	virtual double PullErrorBackUpStream() = 0;
	virtual double GetOutput() = 0;
	virtual void SetOutput(double d) = 0;
	virtual void SetError(double d) = 0;
	virtual void AjustWeights(double dLearningRate, double dMomentum) = 0;
	virtual int SerializeWeights(double* pBuffer) = 0;
	virtual int DeserializeWeights(double* pBuffer) = 0;
	virtual void AddInput(GNeuron* pNeuron) = 0;
	virtual void Print() = 0;
	virtual void BatchUpdateDeltas(double dLearningRate) = 0;
	virtual void BatchUpdateWeights() = 0;

	// For internal use only
	virtual void ConnectOutputToSynapse(GSynapse* pSynapse) = 0;
	virtual void RemapSynapse(GSynapse* pOld, GSynapse* pNew) = 0;
};



class GStandardNeuron : public GNeuron
{
protected:
	double m_dOutput; // Axon
	double m_dError;
	int m_nInputs;
	int m_nInputSpace;
	struct GSynapse* m_pInputs;
	int m_nOutputs;
	int m_nOutputSpace;
	struct GSynapse** m_pOutputs;

public:
	GStandardNeuron() : GNeuron()
	{
		m_dOutput = 1e50;
		m_dError = 1e50;
		m_nInputs = 0;
		m_nInputSpace = 0;
		m_pInputs = NULL;
		AddInput(NULL); // Add the constant 1 (for bias) input
		m_nOutputs = 0;
		m_nOutputSpace = 0;
		m_pOutputs = NULL;
	}

	virtual ~GStandardNeuron()
	{
		delete [] m_pInputs;
		delete(m_pOutputs);
	}

	virtual void Print()
	{
		int n;
		for(n = 0; n < m_nInputs; n++)
			printf("\t%lf\n", m_pInputs[n].m_dWeight);
	}

	virtual double PullEvalDownStream()
	{
		if(m_dOutput == 1e50)
		{
			// Sum up the weighted inputs
			double dSum = m_pInputs[0].m_dWeight;
			int n;
			for(n = 1; n < m_nInputs; n++)
				dSum += (m_pInputs[n].m_dWeight * m_pInputs[n].m_pInput->PullEvalDownStream());

			// Squash the sum
			m_dOutput = GMath::sigmoid(dSum, 1);
		}
		return m_dOutput;
	}

	virtual double PullErrorBackUpStream()
	{
		GAssert(m_dOutput != 1e50, "output was not calculated");
		if(m_dError == 1e50)
		{
			// Sum up the errors from each output
			double dSum = 0;
			int n;
			for(n = 0; n < m_nOutputs; n++)
				dSum += (m_pOutputs[n]->m_dWeight * m_pOutputs[n]->m_pOutput->PullErrorBackUpStream());

			// Multiply by derivative of squashing function
			m_dError = dSum * m_dOutput * ((double)1 - m_dOutput);
		}
		return m_dError;
	}

	virtual double GetOutput()
	{
		GAssert(m_dOutput != 1e50, "output was not calculated");
		return m_dOutput;
	}

	virtual void SetOutput(double d)
	{
		m_dOutput = d;
	}

	virtual void SetError(double d)
	{
		m_dError = d;
	}

	virtual void AjustWeights(double dLearningRate, double dMomentum)
	{
		GAssert(m_dError != 1e50, "output was not calculated");
		GSynapse* pNeuronRef;
		pNeuronRef = &m_pInputs[0];
		pNeuronRef->m_dWeightDelta *= dMomentum;
		pNeuronRef->m_dWeightDelta += (dLearningRate * m_dError);
		pNeuronRef->m_dWeight += pNeuronRef->m_dWeightDelta;
		int n;
		for(n = 1; n < m_nInputs; n++)
		{
			pNeuronRef = &m_pInputs[n];
			pNeuronRef->m_dWeightDelta *= dMomentum;
			pNeuronRef->m_dWeightDelta += (dLearningRate * m_dError * pNeuronRef->m_pInput->GetOutput());
			pNeuronRef->m_dWeight += pNeuronRef->m_dWeightDelta;
		}
	}

	virtual void BatchUpdateDeltas(double dLearningRate)
	{
		GAssert(m_dError != 1e50, "output was not calculated");
		GSynapse* pNeuronRef;
		pNeuronRef = &m_pInputs[0];
		pNeuronRef->m_dWeightDelta += (dLearningRate * m_dError);
		int n;
		for(n = 1; n < m_nInputs; n++)
		{
			pNeuronRef = &m_pInputs[n];
			pNeuronRef->m_dWeightDelta += (dLearningRate * m_dError * pNeuronRef->m_pInput->GetOutput());
		}
	}

	virtual void BatchUpdateWeights()
	{
		GSynapse* pNeuronRef;
		pNeuronRef = &m_pInputs[0];
		pNeuronRef->m_dWeight += pNeuronRef->m_dWeightDelta;
		pNeuronRef->m_dWeightDelta = 0;
		int n;
		for(n = 1; n < m_nInputs; n++)
		{
			pNeuronRef = &m_pInputs[n];
			pNeuronRef->m_dWeight += pNeuronRef->m_dWeightDelta;
			pNeuronRef->m_dWeightDelta = 0;
		}
	}

	virtual int SerializeWeights(double* pBuffer)
	{
		if(pBuffer)
		{
			int n;
			for(n = 0; n < m_nInputs; n++)
				pBuffer[n] = m_pInputs[n].m_dWeight;
		}
		return m_nInputs;
	}

	virtual int DeserializeWeights(double* pBuffer)
	{
		int n;
		for(n = 0; n < m_nInputs; n++)
			m_pInputs[n].m_dWeight = pBuffer[n];
		return m_nInputs;
	}

	virtual void AddInput(GNeuron* pNeuron)
	{
		if(m_nInputs >= m_nInputSpace)
		{
			// Reallocate input space
			int nInputSpace = MAX(4, m_nInputSpace * 2);
			GSynapse* pInputs = new GSynapse[nInputSpace];
			memcpy(pInputs, m_pInputs, sizeof(GSynapse) * m_nInputSpace);

			// Remap everything
			int n;
			for(n = 0; n < m_nInputs; n++)
			{
				if(m_pInputs[n].m_pInput)
					m_pInputs[n].m_pInput->RemapSynapse(&m_pInputs[n], &pInputs[n]);
			}
			delete(m_pInputs);
			m_pInputs = pInputs;
			m_nInputSpace = nInputSpace;
		}
		m_pInputs[m_nInputs].m_pInput = pNeuron;
		m_pInputs[m_nInputs].m_pOutput = this;
		if(pNeuron)
			pNeuron->ConnectOutputToSynapse(&m_pInputs[m_nInputs]);
		else
		{
			GAssert(m_nInputs == 0, "only the first input should be NULL");
		}
		m_nInputs++;
	}

	virtual void RemapSynapse(GSynapse* pOld, GSynapse* pNew)
	{
		int n;
		for(n = 0; n < m_nOutputs; n++)
		{
			if(m_pOutputs[n] == pOld)
			{
				m_pOutputs[n] = pNew;
				break;
			}
		}
	}

protected:
	virtual void ConnectOutputToSynapse(GSynapse* pSynapse)
	{
		if(m_nOutputs >= m_nOutputSpace)
		{
			// Reallocate output space
			int nOutputSpace = MAX(4, m_nOutputSpace * 2);
			GSynapse** pOutputs = new GSynapse*[nOutputSpace];
			memcpy(pOutputs, m_pOutputs, sizeof(GSynapse*) * m_nOutputSpace);
			delete(m_pOutputs);
			m_pOutputs = pOutputs;
			m_nOutputSpace = nOutputSpace;
		}
		m_pOutputs[m_nOutputs++] = pSynapse;
	}
};




// ----------------------------------------------------------------------

GNeuralNet::GNeuralNet(GArffRelation* pRelation)
: GSupervisedLearner(pRelation)
{
	m_pInternalRelation = NULL;
	m_pNeurons = new GPointerArray(64);
	m_pBestSet = NULL;
	m_nWeightCount = 0;
	m_nInputStart = 0;
	m_nLayerStart = 0;
	m_nLayerSize = 0;
	m_pMinAndRanges = NULL;
	MakeInternalRelationAndOutputLayer();

	// Default settings
	m_dLearningRate = .215;
	m_dLearningDecay = 1;
	m_dMomentum = .9;
	m_nRunEpochs = 4000;
	m_nMaximumEpochs = 50000;
	m_nEpochsPerValidationCheck = 5;
	m_dAcceptableMeanSquareError = 0.000001;
	m_dTrainingPortion = .65;

	// Step training
	m_pTrainingDataInternal = NULL;
	m_pValidationDataInternal = NULL;
}

GNeuralNet::~GNeuralNet()
{
	int nCount = m_pNeurons->GetSize();
	int n;
	for(n = 0; n < nCount; n++)
		delete((GNeuron*)m_pNeurons->GetPointer(n));
	delete(m_pNeurons);
	delete(m_pBestSet);
	delete(m_pMinAndRanges);
	delete(m_pInternalRelation);
	ReleaseInternalData();
}

void GNeuralNet::ReleaseInternalData()
{
	if(m_pValidationDataInternal != m_pTrainingDataInternal)
		delete(m_pValidationDataInternal);
	delete(m_pTrainingDataInternal);
	m_pTrainingDataInternal = NULL;
	m_pValidationDataInternal = NULL;
}

void GNeuralNet::MakeInternalRelationAndOutputLayer()
{
	// Make the internal relation
	GAssert(m_pInternalRelation == NULL, "already created the internal relation");
	m_pInternalRelation = new GArffRelation();

	// Add the internal input nodes
	GArffAttribute* pAttr;
	int nValueCount;
	int nInputCount = m_pRelation->GetInputCount();
	int n, i;
	for(n = 0; n < nInputCount; n++)
	{
		pAttr = m_pRelation->GetAttribute(m_pRelation->GetInputIndex(n));
		if(pAttr->IsContinuous())
			m_pInternalRelation->AddAttribute(new GArffAttribute(true, 0, NULL));
		else
		{
			nValueCount = pAttr->GetValueCount();
			if(nValueCount <= 2)
				m_pInternalRelation->AddAttribute(new GArffAttribute(true, 0, NULL));
			else
			{
				for(i = 0; i < nValueCount; i++)
					m_pInternalRelation->AddAttribute(new GArffAttribute(true, 0, NULL));
			}
		}
	}

	// Add the internal output nodes
	int nOutputCount = m_pRelation->GetOutputCount();
	for(n = 0; n < nOutputCount; n++)
	{
		pAttr = m_pRelation->GetAttribute(m_pRelation->GetOutputIndex(n));
		if(pAttr->IsContinuous())
			m_pInternalRelation->AddAttribute(new GArffAttribute(false, 0, NULL));
		else
		{
			nValueCount = pAttr->GetValueCount();
			if(nValueCount <= 2)
				m_pInternalRelation->AddAttribute(new GArffAttribute(false, 0, NULL));
			else
			{
				for(i = 0; i < nValueCount; i++)
					m_pInternalRelation->AddAttribute(new GArffAttribute(false, 0, NULL));
			}
		}
	}

	// Make the output layer
	AddLayer(m_pInternalRelation->GetOutputCount());
}

void GNeuralNet::MakeInputLayer()
{
	GAssert(m_nInputStart == 0, "already made the input layer");
	m_nInputStart = m_pNeurons->GetSize();
	AddLayer(m_pInternalRelation->GetInputCount());
}

void GNeuralNet::InputsToInternal(double* pExternal, double* pInternal)
{
	GAssert(m_pMinAndRanges, "min and ranges not calculated yet");
	GArffAttribute* pAttr;
	int nValueCount;
	int nInputCount = m_pRelation->GetInputCount();
	int nInternalIndex = 0;
	int n, i, nExternalIndex;
	for(n = 0; n < nInputCount; n++)
	{
		nExternalIndex = m_pRelation->GetInputIndex(n);
		pAttr = m_pRelation->GetAttribute(nExternalIndex);
		if(pAttr->IsContinuous())
			pInternal[nInternalIndex++] = GArffData::Normalize(pExternal[nExternalIndex], m_pMinAndRanges[nExternalIndex + nExternalIndex], m_pMinAndRanges[nExternalIndex + nExternalIndex + 1], INPUT_MIN, INPUT_RANGE);
		else
		{
			nValueCount = pAttr->GetValueCount();
			if(nValueCount <= 2)
				pInternal[nInternalIndex++] = (pExternal[nExternalIndex] < .5 ? INPUT_MIN : INPUT_MIN + INPUT_RANGE);
			else
			{
				for(i = 0; i < nValueCount; i++)
					pInternal[nInternalIndex + i] = INPUT_MIN;
				GAssert((int)pExternal[nExternalIndex] >= 0 && (int)pExternal[nExternalIndex] < nValueCount, "out of range");
				pInternal[nInternalIndex + (int)pExternal[nExternalIndex]] = INPUT_MIN + INPUT_RANGE;
				nInternalIndex += nValueCount;
			}
		}
	}
	GAssert(nInternalIndex == m_pInternalRelation->GetInputCount(), "error");
}

void GNeuralNet::OutputsToInternal(double* pExternal, double* pInternal)
{
	GAssert(m_pMinAndRanges, "min and ranges not calculated yet");
	GArffAttribute* pAttr;
	int nValueCount;
	int nOutputCount = m_pRelation->GetOutputCount();
	int nInternalIndex = m_pInternalRelation->GetOutputIndex(0);
	int n, i, nExternalIndex;
	for(n = 0; n < nOutputCount; n++)
	{
		nExternalIndex = m_pRelation->GetOutputIndex(n);
		pAttr = m_pRelation->GetAttribute(nExternalIndex);
		if(pAttr->IsContinuous())
			pInternal[nInternalIndex++] = GArffData::Normalize(pExternal[nExternalIndex], m_pMinAndRanges[nExternalIndex + nExternalIndex], m_pMinAndRanges[nExternalIndex + nExternalIndex + 1], OUTPUT_MIN, OUTPUT_RANGE);
		else
		{
			nValueCount = pAttr->GetValueCount();
			if(nValueCount <= 2)
				pInternal[nInternalIndex++] = (pExternal[nExternalIndex] < .5 ? OUTPUT_MIN : OUTPUT_MIN + OUTPUT_RANGE);
			else
			{
				for(i = 0; i < nValueCount; i++)
					pInternal[nInternalIndex + i] = OUTPUT_MIN;
				GAssert((int)pExternal[nExternalIndex] >= 0 && (int)pExternal[nExternalIndex] < nValueCount, "out of range");
				pInternal[nInternalIndex + (int)pExternal[nExternalIndex]] = OUTPUT_MIN + OUTPUT_RANGE;
				nInternalIndex += nValueCount;
			}
		}
	}