token.h

用、Visual++6.0 开发的控制台程序

#include<iostream>
#include<string>

using namespace std;
const int TOKEN_LEN = 120;

typedef enum 
{
	ENDOFFILE=1,   //文件结尾
	IF,ELSE,THEN,END,REPEAT,UNTIL,READ,WRITE, //保留字
	ADD,SUB,MUL,DIV,//运算符
	L_BRACKET,R_BRACKET,COMMA,ASSIGN,LESS,SEMICOLON,EQUAL,//特殊字符
	NUM,//常量
	ID,//变量
	ERROR//错误
}TokenType;
struct Token
{
	TokenType type;
	char *lex;
};
static Token TokenTab[]=
{
	{IF,"If",},
	{ELSE,"Else"},
	{THEN,"Then"},
	{END,"End"},
	{REPEAT,"Repeat"},
	{UNTIL,"Until"},
	{READ,"Read"},
	{WRITE,"Write"}
};

int lineNo;
FILE *infile;
char TokenBuf[TOKEN_LEN];
int InitScanner(char *);
Token GetToken();
void CloseScanner();
void PrintToken(Token token)
{
	cout<<"Line "<<lineNo<<": ( "<<token.type<<" , "<<token.lex<<" )"<<endl;
}


int InitScanner(char *file)
{
	lineNo = 1;
	if((infile = fopen(file,"r")) == NULL)
	{
		cout<<"Can't open file!"<<endl;
		return 0;
	}
	return 1;
}
void CloseScanner()
{
	if(infile != NULL) fclose(infile);
}
 char GetChar()
 {
	 char ch =getc(infile);
	 return ch;
 }
 void BackChar(char ch)
 {
	 if(ch != EOF)
		 ungetc(ch,infile);
 }
void AddToken( char ch)
{
	int len = strlen(TokenBuf);
	if(len + 1 >= sizeof(TokenBuf))return ;
	TokenBuf[len]=ch;
	TokenBuf[len+1]='\0';
}
void Empty()
{
	memset(TokenBuf,0,TOKEN_LEN);
}
Token IsKey(char *str)
{
	for(int i=0;i<sizeof(TokenTab)/sizeof(TokenTab[0]);i++)
	{
		if(strcmp(TokenTab[i].lex,str) == 0) return TokenTab[i];
	}
	Token token;
	token.type=ID;
	return token;
}
Token GetToken()
{
	Token token;
	char ch;
	memset(&token,0,sizeof(Token));
	Empty();
	token.lex = TokenBuf;
	while(true)
	{
		ch = GetChar();
		if(ch == EOF)
		{
			token.type = ENDOFFILE;
	//		token.lex =" EndOfFile";
			return token;
		}
		if(ch == '\n') lineNo++;
		if(ch=='{')
		{
			do
			{
				ch=GetChar();
				if(ch=='\n')lineNo++;
			}while(ch!='}');
			ch=GetChar();
		}
		while(ch=='\t')ch=GetChar();
		if(!isspace(ch))break;
	}
	AddToken(ch);
	if(isalpha(ch))
	{
		while(true)
		{
			ch = GetChar();
			if(isalnum(ch))AddToken(ch);
			else break;
		}
		BackChar(ch);
		token = IsKey(TokenBuf);
		token.lex=TokenBuf;
		return token;
	}
	if(isdigit(ch))
	{
		while(true)
		{
			ch = GetChar();
			if(isdigit(ch))AddToken(ch);
			else break;
		}
		BackChar(ch);
		token.type = NUM;
		return token;

	}
	else
	{
		
		switch(ch)
		{
		case '+':
			token.type = ADD;
			break;
		case '-':
			token.type= SUB;
			break;
		case '*':
			token.type =MUL;
			break;
		case '/':
			token.type= DIV;
			break;
		case '=':
			token.type= ASSIGN;
			break;
		case '<':
			token.type = LESS;
			break;
		case ';':
			token.type= SEMICOLON;
			break;
		case ',':
			token.type = COMMA;
			break;
		case ')':
			token.type = R_BRACKET;
			break;
		case ':':
			ch = GetChar();
			if(ch == '=')
			{
				AddToken(ch);
				token.type=EQUAL;
			}
			else
			{
				token.type=ERROR;
				return token;
			}
			break;
		case '(':
			token.type = L_BRACKET;
			break;
		default :
			token.type =ERROR;
			return token;
		}
		return token;
	}
}