cpplex.cpp

这是一个很不错的词法语法分析器! 很适合计算机专业的大学生学习参考

#include "StdAfx.h"
#include "CppLex.h"
#include <functional>
#include <algorithm>

namespace CppParser
{
using namespace std;

static char* KEYWORDS[]={"else","main","struct","enum","switch","auto","template","explicit","this",
	"bool","extern","mutable","break","false","throw","case","namespace","true",
	"catch","new","try","float","noreturn","char","for","operator","typedef",
	"class","friend","private","typeid","const","goto","protected","typename","const_cast","if","public",
	"union","continue","inline","register","unsigned","reinterpret_cast","using",
	"default","int","return","uuid","delete","short","signed","virtual","sizeof","void","do","static",
	"volatile","double","static_cast","dynamic_cast","long","while"};

static char* OPERATORS[]={"<<=",">>=","->*","+=","-=","^=","&=","|=","<<",">>",
	"<=",">=","&&","||","++","--","*=","/=","%=","==","!=","->",".*","::",
	"+","-",",","*","/","%","^","!","=","<",">","(",")","[","]","&","|","~",",",".","?",":"};

void StreamContextHelper::LocalInfo::setName(const string& name)
{
	mName=name;
}

string StreamContextHelper::LocalInfo::getName(void) const
{
	return mName;
}

StreamContextHelper::InputStreamHelper::InputStreamHelper(istream& istr):mInputStream(istr)
{
}

bool StreamContextHelper::InputStreamHelper::hasChar()
{
	if(mCharsRead.size())
		return true;
	const int numOfByteReadOnce=10240;
	char s[numOfByteReadOnce+1];
	mInputStream.read(s,numOfByteReadOnce);
	int count=mInputStream.gcount();
	if(count==0)
		return false;
	s[count]=0;
	_strrev(s);
	mCharsRead=s;
	return true;
}

char StreamContextHelper::InputStreamHelper::getChar()
{
	if(!hasChar())
		throw 1;
	char c=*(mCharsRead.end()-1);
	mCharsRead.erase(mCharsRead.end()-1);
	return c;
}

void StreamContextHelper::InputStreamHelper::ungetChar(char c)
{
	mCharsRead+=c;
}

void StreamContextHelper::InputStreamHelper::ungetString(const string& s)
{
	for(int i=s.size()-1;i>=0;--i)
		mCharsRead+=s[i];
}

char StreamContextHelper::getEscapedChar(string& s)
{
	char ch=mInputStreamHelper.getChar();
	if(ch=='\\'&&mInputStreamHelper.hasChar())
	{
		ch=mInputStreamHelper.getChar();
		if(ch==0x0d&&mInputStreamHelper.hasChar())
		{
			ch=mInputStreamHelper.getChar();
			if(ch==0x0a)
			{
				s="\\\x0d\x0a";
				if(mInputStreamHelper.hasChar())
					ch=mInputStreamHelper.getChar();
				else
					throw 1;
				s+=ch;
				return ch;
			}
			mInputStreamHelper.ungetChar(ch);
		}
		s="\\";
		s+=ch;
		switch(ch)
		{
		case '\'':
			ch='\'';
			break;
		case '\"':
			ch='\"';
			break;
		case '?':
			ch='\?';
			break;
		case '\\':
			ch='\\';
			break;
		case 'a':
			ch='\a';
			break;
		case 'b':
			ch='\b';
			break;
		case 'f':
			ch='\f';
			break;
		case 'n':
			ch='\n';
			break;
		case 'r':
			ch='\r';
			break;
		case 't':
			ch='\t';
			break;
		case 'v':
			ch='\v';
			break;
		case '\x0a':
			if(mInputStreamHelper.hasChar())
				ch=mInputStreamHelper.getChar();
			else
				throw 1;
			s+=ch;
			break;
		}
		return ch;
	}
	else if(ch=='?'&&mInputStreamHelper.hasChar())
	{
		ch=mInputStreamHelper.getChar();
		if(ch=='?'&&mInputStreamHelper.hasChar())
		{
			ch=mInputStreamHelper.getChar();
			switch(ch)
			{
			case '=':
				s="?\?=";
				ch='#';
				return ch;
				break;
			case '(':
				s="?\?(";
				ch='[';
				return ch;
				break;
			case '/':
				s="?\?/";
				ch='\\';
				return ch;
				break;
			case ')':
				s="?\?)";
				ch=']';
				return ch;
				break;
			case '\'':
				s="?\?\'";
				ch='^';
				return ch;
				break;
			case '<':
				s="?\?<";
				ch='{';
				return ch;
				break;
			case '!':
				s="?\?!";
				ch='|';
				return ch;
				break;
			case '>':
				s="?\?>";
				ch='}';
				return ch;
				break;
			case '-':
				s="?\?-";
				ch='~';
				return ch;
				break;
			}
			mInputStreamHelper.ungetChar(ch);
			mInputStreamHelper.ungetChar('?');
		}
		else
		{
			mInputStreamHelper.ungetChar(ch);
			ch='?';
		}
	}
	s=ch;
	return ch;
}

StreamContextHelper::StreamContextHelper(InputStreamHelper& istr,Parser& parser)
	:mParser(parser),mInputStreamHelper(istr),mLastType(INVALID)
{
	addWord(mPPKeyword);
	addWord(mKeyword);
	addWord(mIdentifier);
	addWord(mWhiteSpace);
	addWord(mLineFeed);
	addWord(mSemicolonDelimiter);
	addWord(mLeftBracketDelimiter);
	addWord(mRightBracketDelimiter);
	addWord(mOperator);
	addWord(mSingleLineComment);
	addWord(mStringLiteral);
	addWord(mCharLiteral);
	addWord(mDecimalLiteral);
	addWord(mOctLiteral);
	addWord(mHexLiteral);
	addWord(mIntegralFloatLiteral);
	addWord(mNonIntegralFloatLiteral);
	addWord(mExpIntegralFloatLiteral);
	addWord(mExpNonIntegralFloatLiteral);
	addWord(mBlockComment);
}

StreamContextHelper::~StreamContextHelper()
{
}

void StreamContextHelper::run()
{
	while(runOnce())
		mParser.parse(mLastType,mLastWord,*this);
}

const Info& StreamContextHelper::getInfo() const
{
	return mInfo;
}

void StreamContextHelper::addWord(Word& w)
{
	mWordList.push_back(&w);
}

bool StreamContextHelper::runOnce(void)
{
	mLastWord.erase();
	mLastType=INVALID;
	vector<Word*>::iterator iter;
	vector<Word*>::iterator iterLast=mWordList.end();
	for(;;)
	{
		for(iter=mWordList.begin();iter!=iterLast;++iter)
			if((*iter)->end(*this))
			{
				mLastType=(*iter)->getType();
				break;
			}
		if(!mInputStreamHelper.hasChar())
			break;
		string s;
		char ch=getEscapedChar(s);
		if(s.size()==2)
			ch='-';// a hack for escape of " and '
		for(iter=mWordList.begin();iter!=iterLast;++iter)
			if(!(*iter)->addChar(ch,*this))
			{
				(*iter)->clean();
				swap(*iter,*(iterLast-1));
				--iterLast;
				--iter;
			}
		if(mWordList.begin()==iterLast)
		{
			mInputStreamHelper.ungetString(s);
			break;
		}
		mLastWord+=s;
	}
	for(iter=mWordList.begin();iter!=iterLast;++iter)
		(*iter)->clean();
	return mLastType!=INVALID;
}

const string& StreamContextHelper::getLastWord(void) const
{
	return mLastWord;
}

Type StreamContextHelper::getLastType(void) const
{
	return mLastType;
}

void StreamContextHelper::setInfoName(const string& name)
{
	mInfo.setName(name);
}

string StreamContextHelper::getInfoName(void) const
{
	return mInfo.getName();
}

WordBase::WordBase(Type t)
{
	mType=t;
}

Type WordBase::getType() const
{
	return mType;
}

bool WordBase::isWhiteSpace(char ch)
{
	return ch==' '||ch=='\t';
}

bool WordBase::isIdentifierStart(char ch)
{
	return ch>='a'&&ch<='z'||ch>='A'&&ch<='Z'||ch=='_';
}

bool WordBase::isIdentifier(char ch)
{
	return ch>='a'&&ch<='z'||ch>='A'&&ch<='Z'||ch=='_'||ch>='0'&&ch<='9';
}

bool WordBase::isDigit(char ch)
{
	return ch>='0'&&ch<='9';
}

RegularExpressionWordHelper::RegularExpressionWordHelper(Type t,const string& re)
	:WordBase(t),mRegularExpression(re)
{
	clean();
}

void RegularExpressionWordHelper::clean()
{
	mCurrentOffset=0;
	mCurrentCode=mRegularExpression[mCurrentOffset];
	mCurrentRepitition=mRegularExpression[mCurrentOffset+1];
}

const char RegularExpressionWordHelper::REP_ONE_OR_MORE='+';
const char RegularExpressionWordHelper::REP_ONE='$';
const char RegularExpressionWordHelper::REP_ZERO_OR_MORE='*';
const char RegularExpressionWordHelper::REP_ZERO_OR_ONE='-';

// adghiopqstwxz
const char RegularExpressionWordHelper::CODE_WHITE_SPACE='w';
const char RegularExpressionWordHelper::CODE_IDENTIFIED_START='s';
const char RegularExpressionWordHelper::CODE_IDENTIFIER='i';
const char RegularExpressionWordHelper::CODE_DIGIT='d';