mnistdoc.cpp

基于神经网络的手写体识别程序

				
				while (sRowp1 >= m_cRows ) sRowp1 -= m_cRows;
				while (sRowp1 < 0 ) sRowp1 += m_cRows;
				
				while (sColp1 >= m_cCols ) sColp1 -= m_cCols;
				while (sColp1 < 0 ) sColp1 += m_cCols;
				
				// perform bi-linear interpolation
				
				sourceValue =	w1 * At( inputVector, sRow  , sCol   ) +
					w2 * At( inputVector, sRow  , sColp1 ) +
					w3 * At( inputVector, sRowp1, sCol   ) +
					w4 * At( inputVector, sRowp1, sColp1 );
			}
			else
			{
				// At least one supporting pixel for the "phantom" pixel is outside the
				// bounds of the character grid. Set its value to "background"

				sourceValue = 1.0;  // "background" color in the -1 -> +1 range of inputVector
			}
			
			mappedVector[ row ][ col ] = 0.5 * ( 1.0 - sourceValue );  // conversion to 0->1 range we are using for mappedVector
			
		}
	}
	
	// now, invert again while copying back into original vector
	
	for ( row=0; row<m_cRows; ++row )
	{
		for ( col=0; col<m_cCols; ++col )
		{
			At( inputVector, row, col ) = 1.0 - 2.0 * mappedVector[ row ][ col ];
		}
	}			
	
}



void CMNistDoc::CalculateNeuralNet(double *inputVector, int count, 
								   double* outputVector /* =NULL */, int oCount /* =0 */,
								   std::vector< std::vector< double > >* pNeuronOutputs /* =NULL */,
								   BOOL bDistort /* =FALSE */ )
{
	// wrapper function for neural net's Calculate() function, needed because the NN is a protected member
	// waits on the neural net mutex (using the CAutoMutex object, which automatically releases the
	// mutex when it goes out of scope) so as to restrict access to one thread at a time
	
	CAutoMutex tlo( m_utxNeuralNet );
	
	if ( bDistort != FALSE )
	{
		GenerateDistortionMap();
		ApplyDistortionMap( inputVector );
	}
	
	
	m_NN.Calculate( inputVector, count, outputVector, oCount, pNeuronOutputs );
	
}


void CMNistDoc::BackpropagateNeuralNet(double *inputVector, int iCount, double* targetOutputVector, 
									   double* actualOutputVector, int oCount, 
									   std::vector< std::vector< double > >* pMemorizedNeuronOutputs, 
									   BOOL bDistort )
{
	// function to backpropagate through the neural net. 
	
	ASSERT( (inputVector != NULL) && (targetOutputVector != NULL) && (actualOutputVector != NULL) );

	///////////////////////////////////////////////////////////////////////
	//
	// CODE REVIEW NEEDED:
	//
	// It does not seem worthwhile to backpropagate an error that's very small.  "Small" needs to be defined
	// and for now, "small" is set to a fixed size of pattern error ErrP <= 0.10 * MSE, then there will
	// not be a backpropagation of the error.  The current MSE is updated from the neural net dialog CDlgNeuralNet

	BOOL bWorthwhileToBackpropagate;  /////// part of code review
	
	{	
		// local scope for capture of the neural net, only during the forward calculation step,
		// i.e., we release neural net for other threads after the forward calculation, and after we
		// have stored the outputs of each neuron, which are needed for the backpropagation step
		
		CAutoMutex tlo( m_utxNeuralNet );
		
		// determine if it's time to adjust the learning rate
		
		if ( (( m_cBackprops % m_nAfterEveryNBackprops ) == 0) && (m_cBackprops != 0) )
		{
			double eta = m_NN.m_etaLearningRate;
			eta *= m_dEtaDecay;
			if ( eta < m_dMinimumEta )
				eta = m_dMinimumEta;
			m_NN.m_etaLearningRatePrevious = m_NN.m_etaLearningRate;
			m_NN.m_etaLearningRate = eta;
		}
		
		
		// determine if it's time to adjust the Hessian (currently once per epoch)
		
		if ( (m_bNeedHessian != FALSE) || (( m_cBackprops % ::GetPreferences().m_nItemsTrainingImages ) == 0) )
		{
			// adjust the Hessian.  This is a lengthy operation, since it must process approx 500 labels
			
			CalculateHessian();
			
			m_bNeedHessian = FALSE;
		}
		
		
		// determine if it's time to randomize the sequence of training patterns (currently once per epoch)
		
		if ( ( m_cBackprops % ::GetPreferences().m_nItemsTrainingImages ) == 0 )
		{
			RandomizeTrainingPatternSequence();
		}
		
		
		// increment counter for tracking number of backprops
		
		m_cBackprops++;
		
		
		// forward calculate through the neural net
		
		CalculateNeuralNet( inputVector, iCount, actualOutputVector, oCount, pMemorizedNeuronOutputs, bDistort );


		// calculate error in the output of the neural net
		// note that this code duplicates that found in many other places, and it's probably sensible to 
		// define a (global/static ??) function for it

		double dMSE = 0.0;
		for ( int ii=0; ii<10; ++ii )
		{
			dMSE += ( actualOutputVector[ii]-targetOutputVector[ii] ) * ( actualOutputVector[ii]-targetOutputVector[ii] );
		}
		dMSE /= 2.0;

		if ( dMSE <= ( 0.10 * m_dEstimatedCurrentMSE ) )
		{
			bWorthwhileToBackpropagate = FALSE;
		}
		else
		{
			bWorthwhileToBackpropagate = TRUE;
		}

		
		if ( (bWorthwhileToBackpropagate != FALSE) && (pMemorizedNeuronOutputs == NULL) )
		{
			// the caller has not provided a place to store neuron outputs, so we need to
			// backpropagate now, while the neural net is still captured.  Otherwise, another thread
			// might come along and call CalculateNeuralNet(), which would entirely change the neuron
			// outputs and thereby inject errors into backpropagation 
			
			m_NN.Backpropagate( actualOutputVector, targetOutputVector, oCount, NULL );
			
			SetModifiedFlag( TRUE );
			
			// we're done, so return
			
			return;
		}
		
	}
	
	// if we have reached here, then the mutex for the neural net has been released for other 
	// threads.  The caller must have provided a place to store neuron outputs, which we can 
	// use to backpropagate, even if other threads call CalculateNeuralNet() and change the outputs
	// of the neurons

	if ( (bWorthwhileToBackpropagate != FALSE) )
	{
		m_NN.Backpropagate( actualOutputVector, targetOutputVector, oCount, pMemorizedNeuronOutputs );
		
		// set modified flag to prevent closure of doc without a warning
		
		SetModifiedFlag( TRUE );
	}
	
}



void CMNistDoc::CalculateHessian()
{
	// controls the Neural network's calculation if the diagonal Hessian for the Neural net
	// This will be called from a thread, so although the calculation is lengthy, it should not interfere
	// with the UI
	
	// we need the neural net exclusively during this calculation, so grab it now
	
	CAutoMutex tlo( m_utxNeuralNet );
	
	double inputVector[841] = {0.0};  // note: 29x29, not 28x28
	double targetOutputVector[10] = {0.0};
	double actualOutputVector[10] = {0.0};
	
	unsigned char grayLevels[g_cImageSize * g_cImageSize] = { 0 };
	int label = 0;
	int ii, jj;
	UINT kk;
	
	// calculate the diagonal Hessian using 500 random patterns, per Yann LeCun 1998 "Gradient-Based Learning
	// Applied To Document Recognition"
	
	// message to dialog that we are commencing calculation of the Hessian
	
	if ( m_hWndForBackpropPosting != NULL )
	{
		// wParam == 4L -> related to Hessian, lParam == 1L -> commenced calculation
		::PostMessage( m_hWndForBackpropPosting, UWM_BACKPROPAGATION_NOTIFICATION, 4L, 1L );
	}
	
	
	// some of this code is similar to the BackpropagationThread() code
	
	m_NN.EraseHessianInformation();
	
	UINT numPatternsSampled = ::GetPreferences().m_nNumHessianPatterns ;
	
	for ( kk=0; kk<numPatternsSampled; ++kk )
	{
		GetRandomTrainingPattern( grayLevels, &label, TRUE );
		
		if ( label < 0 ) label = 0;
		if ( label > 9 ) label = 9;
		
		
		// pad to 29x29, convert to double precision
		
		for ( ii=0; ii<841; ++ii )
		{
			inputVector[ ii ] = 1.0;  // one is white, -one is black
		}
		
		// top row of inputVector is left as zero, left-most column is left as zero 
		
		for ( ii=0; ii<g_cImageSize; ++ii )
		{
			for ( jj=0; jj<g_cImageSize; ++jj )
			{
				inputVector[ 1 + jj + 29*(ii+1) ] = (double)((int)(unsigned char)grayLevels[ jj + g_cImageSize*ii ])/128.0 - 1.0;  // one is white, -one is black
			}
		}
		
		// desired output vector
		
		for ( ii=0; ii<10; ++ii )
		{
			targetOutputVector[ ii ] = -1.0;
		}
		targetOutputVector[ label ] = 1.0;
		
		
		// apply distortion map to inputVector.  It's not certain that this is needed or helpful.
		// The second derivatives do NOT rely on the output of the neural net (i.e., because the 
		// second derivative of the MSE function is exactly 1 (one), regardless of the actual output
		// of the net).  However, since the backpropagated second derivatives rely on the outputs of
		// each neuron, distortion of the pattern might reveal previously-unseen information about the
		// nature of the Hessian.  But I am reluctant to give the full distortion, so I set the
		// severityFactor to only 2/3 approx
		
		GenerateDistortionMap( 0.65 );
		ApplyDistortionMap( inputVector );
		
		
		// forward calculate the neural network
		
		m_NN.Calculate( inputVector, 841, actualOutputVector, 10, NULL );
		
		
		// backpropagate the second derivatives
		
		m_NN.BackpropagateSecondDervatives( actualOutputVector, targetOutputVector, 10 );
		
		
		// progress message to dialog that we are calculating the Hessian
		
		if ( kk%50 == 0 )
		{
			// every 50 iterations ...
			if ( m_hWndForBackpropPosting != NULL )
			{
				// wParam == 4L -> related to Hessian, lParam == 2L -> progress indicator
				::PostMessage( m_hWndForBackpropPosting, UWM_BACKPROPAGATION_NOTIFICATION, 4L, 2L );
			}
		}
		
		if ( m_bBackpropThreadAbortFlag != FALSE )
			break;
		
	}
	
	m_NN.DivideHessianInformationBy( (double)numPatternsSampled );
	
	// message to dialog that we are finished calculating the Hessian
	
	if ( m_hWndForBackpropPosting != NULL )
	{
		// wParam == 4L -> related to Hessian, lParam == 4L -> finished calculation
		::PostMessage( m_hWndForBackpropPosting, UWM_BACKPROPAGATION_NOTIFICATION, 4L, 4L );
	}
	
}



BOOL CMNistDoc::CanCloseFrame(CFrameWnd* pFrame) 
{
	// check if any threads are running before we allow the main frame to close down
	
	BOOL bRet = TRUE;
	
	if ( (m_bBackpropThreadsAreRunning != FALSE) || (m_bTestingThreadsAreRunning != FALSE) )
	{
		CString str;
		int iRet;
		
		str.Format( _T( "This will stop backpropagation and/or testing threads \n" )
			_T( "Are you certain that you wish to close the application? \n\n" )
			_T( "Click \"Yes\" to stop all threads and close the application \n" )
			_T( "Click \"No\" or \"Cancel\" to continue running the threads and the application " ) );
		
		iRet = ::MessageBox( NULL, str, _T( "Threads Are Running" ), MB_ICONEXCLAMATION | MB_YESNOCANCEL );
		
		if ( (iRet == IDYES) || (iRet == IDOK) )
		{
			bRet = TRUE;
			
			if ( m_bBackpropThreadsAreRunning != FALSE )
			{
				StopBackpropagation();
			}
			
			if ( m_bTestingThreadsAreRunning != FALSE )
			{
				StopTesting();
			}
		}
		else
		{
			bRet = FALSE;
		}
	}
	
	if ( bRet != FALSE )
	{
		// only call the base class if, so far, it's still safe to close.  If we
		// always called the base class, then we would get needless reminders to save the
		// current document.  These reminders are not needed, since we're not closing 
		// anyway
		
		bRet &= COleDocument::CanCloseFrame(pFrame);
	}
	
	return bRet;
}



BOOL CMNistDoc::StartBackpropagation(UINT iStartPattern /* =0 */, UINT iNumThreads /* =2 */, HWND hWnd /* =NULL */,
									 double initialEta /* =0.005 */, double minimumEta /* =0.000001 */, double etaDecay /* =0.990 */, 
									 UINT nAfterEvery  /* =1000 */, BOOL bDistortPatterns /* =TRUE */, double estimatedCurrentMSE /* =1.0 */)
{
	if ( m_bBackpropThreadsAreRunning != FALSE )
		return FALSE;
	
	m_bBackpropThreadAbortFlag = FALSE;
	m_bBackpropThreadsAreRunning = TRUE;
	m_iNumBackpropThreadsRunning = 0;
	m_iBackpropThreadIdentifier = 0;
	m_cBackprops = iStartPattern;
	m_bNeedHessian = TRUE;
	
	m_iNextTrainingPattern = iStartPattern;
	m_hWndForBackpropPosting = hWnd;
	
	if ( m_iNextTrainingPattern < 0 ) 
		m_iNextTrainingPattern = 0;
	if ( m_iNextTrainingPattern >= ::GetPreferences().m_nItemsTrainingImages )
		m_iNextTrainingPattern = ::GetPreferences().m_nItemsTrainingImages - 1;
	
	if ( iNumThreads < 1 ) 
		iNumThreads = 1;
	if ( iNumThreads > 10 )  // 10 is arbitrary upper limit
		iNumThreads = 10;
	
	m_NN.m_etaLearningRate = initialEta;
	m_NN.m_etaLearningRatePrevious = initialEta;
	m_dMinimumEta = minimumEta;
	m_dEtaDecay = etaDecay;
	m_nAfterEveryNBackprops = nAfterEvery;
	m_bDistortTrainingPatterns = bDistortPatterns;

	m_dEstimatedCurrentMSE = estimatedCurrentMSE;  // estimated number that will define whether a forward calculation's error is significant enough to warrant backpropagation

	RandomizeTrainingPatternSequence();
	
	for ( UINT ii=0; ii<iNumThreads; ++ii )
	{
		CWinThread* pThread = ::AfxBeginThread( BackpropagationThread, (LPVOID)this, 
			THREAD_PRIORITY_BELOW_NORMAL, 0, CREATE_SUSPENDED, NULL );
		
		if ( pThread == NULL )
		{
			// creation failed; un-do everything
			
			StopBackpropagation();
			return FALSE;
		}
		
		pThread->m_bAutoDelete = FALSE;		 
		m_pBackpropThreads[ ii ] = pThread;
		pThread->ResumeThread();
		m_iNumBackpropThreadsRunning++;
	}
	
	return TRUE;
	
}




void CMNistDoc::StopBackpropagation()
{
	// stops all the backpropagation threads
	
	if ( m_bBackpropThreadsAreRunning == FALSE )
	{
		// it's curious to select "stop" if no threads are running, but perform some
		// shutdown safeguards, just to be certain
		
		m_bBackpropThreadAbortFlag = TRUE;
		m_bBackpropThreadsAreRunning = FALSE;
		m_iNumBackpropThreadsRunning = 0;
		m_iBackpropThreadIdentifier = 0;
		m_cBackprops = 0;