tokenmaker.cpp

用VC实现的词法分析器

#include "StdAfx.h"

#include "tokenmaker.h"


inline bool IsDigit(char c)
{
	return c >= '0' && c <= '9';
}

int TokenMaker::MakeTokens(MallocList<char*> * pInput, CppList<CodeObject*> * pTokens)
{
	int  nBufferIndex;
	char buffer[256];
	int nLine = 1;

	char * pChar;
	char * pStart;

	CodeObject * pNew;

	for(MallocList<char*>::iterator _Iter = pInput->begin(); _Iter != pInput->end(); _Iter++)
	{
		pChar = *_Iter;
		while(*pChar)
		{
			if (*pChar == ' ' || *pChar == '\t')
			{
				pChar++;
				continue;
			}
			else if (IsDigit(*pChar))
			{
				nBufferIndex = 0;
				pStart = pChar;
				while(*pChar)
				{
					if (IsDigit(*pChar))
						buffer[nBufferIndex++] = *pChar++;
					else
						break;
				}
				buffer[nBufferIndex] = '\0';
				pNew = new CodeObject;
				pNew->Line = nLine;
				pNew->Token = strdup(buffer);
				pNew->Type = Integer;
				pTokens->push_back(pNew);
				continue;
			}
			else if (*pChar == '(')
			{				
				pNew = new CodeObject;
				pNew->Line = nLine;
				pNew->Token = NULL;
				pNew->Type = OpenParenthesis;
				pTokens->push_back(pNew);
			}
			else if (*pChar == ')')
			{				
				pNew = new CodeObject;
				pNew->Line = nLine;
				pNew->Token = NULL;
				pNew->Type = CloseParenthesis;
				pTokens->push_back(pNew);
			}
			else if (*pChar == '+')
			{				
				pNew = new CodeObject;
				pNew->Line = nLine;
				pNew->Token = NULL;
				pNew->Type = PlusSign;
				pTokens->push_back(pNew);
			}
			else
			{
				LastError = "Failure - Character not recognized.";
				return -1;
			}
			pChar++;
		}
		nLine++;
	}

	pNew = new CodeObject;
	pNew->Line = nLine-1;
	pNew->Token = NULL;
	pNew->Type = EndOfFile;
	pTokens->push_back(pNew);
	return 0;

}