gettoken.cpp

简单词法分析程序,文件输入文件输出,输出token序列.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
FILE * source;
int EOF_Flag = 0; 
int linepos = 0; 
int Tracescan = 1;
char tokenstring[41];
int bufsize = 0;
int echosource = 1;
int lineno=0;

typedef enum 
{ENDFILE,ERROR,
IF,ELSE,INT,RETURN,VOID,WHILE,
ID,NUM,
LBPAREN,RBPAREN,
LMPAREN,RMPAREN, 
STOP,UNDERLINE,
ASSIGN,NQ,EQ,LT,GT,LPAREN,RPAREN,SEMI,PLUS,MINUS,TIMES,OVER
}TokenType;
typedef enum {START,INNUM,DONE,INSSIGN,INID,NONEQ} StateType;
struct
{
char *str;
TokenType tok;	
}reservedWords[6]
={{"if",IF},{"else",ELSE},{"int",INT},{"return",RETURN},{"void",VOID},{"while",WHILE}};
char getNextChar()
{	 
	char linebuf[256];
    if(!(linepos<bufsize))
    {
        lineno++;
        if(fgets(linebuf,255,source))
        {
			bufsize = strlen(linebuf);
			linepos = 0;
			return linebuf[linepos++];                            
        }
        else
        {
			EOF_Flag =1;
			return EOF;	
		}
    }
    else return linebuf[linepos++];
}
void ungetNextChar()
{
	if(!EOF_Flag)
		linepos--;	
}
void printToken(TokenType token,const char *tokenstring)
{
	switch(token)
	{
		case IF:
		case ELSE:
		case INT:
		case RETURN:
		case VOID:
		case WHILE:
			fprintf(stdout,"reserved word:%s\n",tokenstring);
			break;
		case ASSIGN: fprintf(stdout,"=\n");break;
		case STOP: fprintf(stdout,",\n");break;
		case LT: fprintf(stdout,"<\n");break;
		case GT: fprintf(stdout,">\n");break;
		case EQ: fprintf(stdout,"==\n");break;
		case NQ: fprintf(stdout,"!=\n");break;
		case UNDERLINE: fprintf(stdout,"_\n");break;
		case LPAREN: fprintf(stdout,"(\n");break;
		case RPAREN: fprintf(stdout,")\n");break;
		case LBPAREN: fprintf(stdout,"{\n");break;
		case RBPAREN: fprintf(stdout,"}\n");break;
		case LMPAREN: fprintf(stdout,"[\n");break;
		case RMPAREN: fprintf(stdout,"]\n");break;  
		case SEMI: fprintf(stdout,";\n");break;
		case PLUS: fprintf(stdout,"+\n");break;	
		case MINUS: fprintf(stdout,"-\n");break;
		case TIMES: fprintf(stdout,"*\n");break;
		case OVER: fprintf(stdout,"/\n");break;
		case ENDFILE: fprintf(stdout,"EOF");break;
		case NUM:
			fprintf(stdout,"NUM,val = %s\n",tokenstring);
			break;
		case ID:
			fprintf(stdout,"ID,name = %s\n",tokenstring);
			break;
		case ERROR:
			fprintf(stdout,"ERROR: %s\n",tokenstring);
			break;
		default:
			fprintf(stdout,"unknown token:%d\n",token);
	}	
}
TokenType reservedLookup(char *s)
{
	int i;
	for(i =0;i<6;i++)
	{
		if(!strcmp(s,reservedWords[i].str))
			return reservedWords[i].tok;
	}	
	return ID;
}
TokenType getToken()
{
	int tokenstringindex = 0;
	TokenType currentToken;
	StateType state = START;
	int save;
	while(state!=DONE)
	{
		int c = getNextChar();
		save = 1;
		switch(state)
		{
			case START:
				if(c=='=')
					state = INSSIGN;
				else if(c=='!')
					state = NONEQ;
				else if(isalpha(c))
					state = INID;
				else if(isdigit(c))
					state = INNUM;
				else if((c==' ')||(c=='\t')||(c=='\n'))
					save = 0;
				else
				{
					state = DONE;
					switch(c)
					{
						case EOF:
							save = 0;
							currentToken = ENDFILE;
							break;
						case '+':
							currentToken = PLUS;
							break;
						case '-':
							currentToken = MINUS;
							break;
						case ',':
							currentToken = STOP;
							break;
						case '*':
							currentToken = TIMES;
							break;
						case '/':
							currentToken = OVER;
							break;
						case '<':
							currentToken = LT;
						case '>':
							currentToken = GT;
							break;	
						case '_':
							currentToken =UNDERLINE;
							break;
						case ';':
							currentToken = SEMI;
							break;
						case '(':
							currentToken = LPAREN;
							break;
						case ')':
							currentToken = RPAREN;
							break;
						case '{':
							currentToken = LBPAREN;
							break;
						case '}':
							currentToken = RBPAREN;
							break;
						case '[':
							currentToken = LMPAREN;
							break;
						case ']':
							currentToken = RMPAREN;
							break;
						default:
							currentToken = ERROR;
							break;
					}	
				}
				break;
				case NONEQ:
					state = DONE;
					if(c=='=')
						currentToken = NQ;
					else 
					{
						ungetNextChar();
						save = 0;
						currentToken = ERROR;	
					}
					break;
				case INSSIGN:
					state =  DONE;
					if(c=='=')
						currentToken = EQ;
					else
					{
						ungetNextChar();
						save = 0;
						currentToken = ASSIGN;	
					}
					break;
				case INNUM:
					if(!isdigit(c))
					{
						ungetNextChar();
						save = 0;
						state = DONE;
						currentToken = NUM;
					}
					break;
				case INID:
					if(!isalpha(c))
					{
						ungetNextChar();
						save = 0;
						state = DONE;
						currentToken = ID;	
					}
				case DONE:
					break;
				default:
					fprintf(stdout,"state= %d\n",state);
					state = DONE;	
					currentToken = ERROR;
					break;
			}
			if(save&&(tokenstringindex<=40))
				tokenstring[tokenstringindex++] = (char) c;
			if(state == DONE)
			{
				tokenstring[tokenstringindex++] = '\0';	
				if(currentToken == ID)
					currentToken = reservedLookup(tokenstring);
			}
		}
		if(Tracescan)
		{
			fprintf(stdout,"\t%d:",lineno);
			printToken(currentToken,tokenstring);
		}
	return currentToken;	
}
int main(int ac,char *av[])
{	
	char pgm[120];
	if(ac!=2)
	{
		printf("error!\n");
		exit(1);
	}
	strcpy(pgm,av[1]);
	if(strchr(pgm,'.')==NULL)
		strcat(pgm,".c");
	source = fopen(pgm,"r");
	if(source==NULL)
	{
		printf("File %s not found\n",pgm);
		exit(1);
	}
	while(getToken()!=ENDFILE);
	fclose(source);
	return 0;
}