tokenlist.cpp

一个小语言的词法分析器

#include"wordDefine.h"

const MAXRESERVED=21;

typedef enum
{ 
	START,INASSIGN,INRANGE,INCOMMENT,INNUM,INID,INCHAR,DONE
}StateType;

struct
{ 
	char * str;
	char * tok;
} reservedWords[MAXRESERVED]= 
{ 
	{"program","PROGRAM"},{"procedure","PROCEDURE"},{"type","TYPE"},{"var","VAR"}
	,{"if","IF"},{"then","THEN"},{"else","ELSE"},{"fi","FI"},{"while","WHILE"}
	  ,{"do","DO"},{"endwh","ENDWH"}
	   ,{"begin","BEGIN"},{"end","END"}, {"read","READ"},{"write","WRITE"},{"array","ARRAY"},{"of","OF"},{"record","RECORD"}
      ,{"return","RETURN"},{"integer","INTEGER"},{"char","CHAR"}  
};

extern void parse();
extern void parseLL1();
list<word> Token;
list<word>::iterator pos;
static int CurLine = 1;
char getch()
{
	char nextChar=cin.get();
	while(nextChar=='\n')
	{
		CurLine++;
		nextChar=cin.get();
	}
	return nextChar;
}
void scanner()
{
	StateType CurState = START;
	word CurWord;
	int save=1;//是否注释
	int ERR=0; //错误
	string TempStr;
	while(1)
	{
		char CurChar;
		switch(CurState)
		{
		case START:
			CurChar=getch();
			if(isalpha(CurChar))
			{
				TempStr=CurChar;
				CurState=INID;//标识符状态；
				break;
			}
			if(isdigit(CurChar))
			{
				TempStr=CurChar;
				CurState=INNUM;//数字状态；
				break;
			}
			switch(CurChar)
			{
			case '+':
				CurWord.Lex = "PLUS";
				CurWord.Kind=27;
				break;
			case '-':
                CurWord.Lex = "MINUS";
				CurWord.Kind=28;
                break;
			case '*':
                CurWord.Lex = "TIMES";
				CurWord.Kind=29;
                break;
			case '/':
                CurWord.Lex = "OVER";
				CurWord.Kind=30;
                break;
			case '(':
                CurWord.Lex = "LPAREN";
				CurWord.Kind=31;
                break;
			case ')':
                CurWord.Lex = "RPAREN";
				CurWord.Kind=32;
                break;
			case ';':
                CurWord.Lex = "SEMI";
				CurWord.Kind=35;
                break;
			case '[':
			    CurWord.Lex = "LMIDPAREN";
				CurWord.Kind=37;
			    break;
			case ']':
			    CurWord.Lex = "RMIDPAREN";
				CurWord.Kind=38;
			    break;
			case '=':
                CurWord.Lex = "EQ";
				CurWord.Kind=25;
                break;
			case '<':
                CurWord.Lex = "LT";
				CurWord.Kind=26;
                break;
			case ',':
				CurWord.Lex="COMMA";
				CurWord.Kind=36;
				break;
			case EOF://词法分析结束，输出
				{
			     
			     for(pos=Token.begin();pos!=Token.end();++pos)
				 {
				    cout<<pos->Line<<" "<<pos->Lex<<"     "<<pos->Sem<<"     "<<pos->Kind<<endl;
				 }
			     return ;
				}
				
			case ' ':
			case '\t':
				break;
			case ':':
				CurState = INASSIGN;//赋值状态；
				break;
			case '{':
				save=1;//表明是注释；
				while(!(cin.peek()=='}'||cin.peek()==EOF))
					CurChar=getch();
				break;//处理注释；
			case '.':
				CurState=INRANGE;//数组下标状态；
				break;
			case '\'':
				CurState=INCHAR;//字符标志状态；
				break;
			default:
				ERR=1;
				
			}
			if(CurChar!='\t'&&CurChar!=' '&&CurState==START&&save)
			{
				CurWord.Line=CurLine;
				CurWord.Sem="";
				Token.push_back(CurWord);
			}
			if(cin.peek()=='}')
			{
				CurChar=getch();
				CurState=START;
				save=1;
			}
			break;
		case INID:
			if(isalnum(cin.peek()))
			{
				CurChar=getch();
				TempStr+=CurChar;
			}
			else
			{
				CurState=START;	
				int i;
				for(i=0;i<MAXRESERVED;i++)
				{
					if(TempStr==reservedWords[i].str)
					{
						CurWord.Lex=reservedWords[i].tok;
						CurWord.Line=CurLine;
						CurWord.Sem=reservedWords[i].str;
						CurWord.Kind=i;
						break;
					}
				}
				if(i==MAXRESERVED)
				{
					CurWord.Lex="ID";
					CurWord.Line=CurLine;
					CurWord.Kind=21;
					CurWord.Sem=TempStr;
				}

				TempStr="";
				Token.push_back(CurWord);

			}
			break;
		case INNUM:
			if(isalnum(cin.peek()))
			{
				CurChar=getch();
				TempStr+=CurChar;
			}
			else
			{
				CurState=START;
				CurWord.Lex="INTC";
				CurWord.Kind=22;
				CurWord.Line=CurLine;
				CurWord.Sem=TempStr;
				Token.push_back(CurWord);
				TempStr="";
			}
			break;
		case INASSIGN:
			CurChar=getch();
			if(CurChar=='=')
			{				
				CurState=START;
				CurWord.Lex="ASSIGN";
				CurWord.Kind=24;
				CurWord.Line=CurLine;
				CurWord.Sem="";
				Token.push_back(CurWord);
				TempStr="";
			}
			
			break;
		case INRANGE:		
			if(cin.peek()=='.')
			{
				CurChar=getch();
				CurState=START;
				CurWord.Lex="UNDERANGE";
				CurWord.Line=CurLine;
				CurWord.Kind=39;
				CurWord.Sem="";
				Token.push_back(CurWord);
				TempStr="";
			}
			else
			{
				CurState=START;
				CurWord.Lex="DOT";
				CurWord.Line=CurLine;
				CurWord.Kind=33;
				CurWord.Sem="";
				Token.push_back(CurWord);
				TempStr="";
			}
			break;
		case INCHAR:
			{CurChar=getch();
			char CurNext=getch();
			if(isalnum(CurChar)&&CurNext=='\'')
			{
				CurState=START;
				CurWord.Lex="CHARC";
				CurWord.Kind=23;
				CurWord.Sem=CurChar;
				CurWord.Line=CurLine;
				Token.push_back(CurWord);
				TempStr="";
			}}
			break;
		default:ERR=1;
		}
		if(ERR)
		{
			cerr<<"ERROR IN"<<CurLine<<endl;
			return;
		}
	}
}
void main()
{
	freopen("in.txt","r",stdin);
	freopen("out.txt","w",stdout);
	cout<<"/************TokenList************/"<<endl;
	scanner();
	parse();
	parseLL1();
	cout<<endl<<endl;
}