ctoken.cpp

一个简单实现了词法分析

/*
#include "CToken.h"
#include "StdAfx.h"
#include "stdio.h"
*/
//根本没有想到的事：就是这样的三个头文件顺序问题，会造成好几个错误
#include "stdafx.h"
#include "stdio.h"
#include "CToken.h"
#include "string.h"
#if! defined _CToken_h_
#define _CToken_h_

CToken::CToken()
{
	strcpy(CTokenName,"");
}
CToken::~CToken()
{}
int IsNumChar(char ch)
{
	if(ch<0x30 || ch>0x39)
		return 0;
	else 
		return 1;
}
int IsFLtNum(char* str)
{
	int cout=0;
	int len=0;
	int i=0;
	int t=0;
	len=strlen(str);
	if((int)str[0]==0)
	{
		if(str[1]!='.')
			return 0;
	}
	if(str[len-1]=='.' ) 
	{return 0;}	 
	else
	{
		while(i<len)
		{
		 
			if(IsNumChar(str[i])!=1 && str[i]!='.')
				return 0;
			if(str[i]=='.')
			{
				cout++;
			}
			i++;
		}
		if(cout>1)
			return 0;
	}
	return 1;
}
int IsIntNum(char* str)
{
	int len=0;
	len=strlen(str);
	int i=0;
	if(str[0]==0x30||IsNumChar(str[0])!=1)
		return 0;
	else
	{
		for(i=0;i<len;i++)
		{
			if(IsNumChar(str[i])!=1)
				return 0;
		}
		return 1;
	}
}
CTokens::CTokens()
{}
CTokens::~CTokens()
{}
int CTokens::IsKeyWord(char* str)
{
	if(strcmp(str,"if")==0)
		return 1;	
	if(strcmp(str,"else")==0)
		return 1;
	if(strcmp(str,"program")==0)
		return 1;
	if(strcmp(str,"begin")==0)
		return 1;
	if(strcmp(str,"end")==0)
		return 1;
	if(strcmp(str,"var")==0)
		return 1;
	else return 0;
}
int CTokens::IsOperator(char ch)
{
	if((int)ch<=47 && (int)ch>=33 &&(int)ch!=46)
		return 1;
	if((int)ch<=64 &&(int)ch>=58)
		return 1;
	if((int)ch<=96 && (int)ch>=91)
		return 1;
	if((int)ch<=127 && (int)ch>=123)
		return 1;
	else return 0;
}
void CTokens::ZeroToken(CToken* pToken)
{
	pToken->CTokenChar=' ';
	pToken->CTokenkind=0;
	strcpy(pToken->CTokenName,"");
	pToken->CTokenValue=0;
}
int CTokens::ReadWord(int* line,char* buffer,char* cho,int* re)
{
	int len=0;
	char ch;
	ch=fgetc(FHandle);	
	while(ch==' '||ch==0x09)//排除掉Tab键
	{ch=fgetc(FHandle);}
	while((int)ch!=-1)
	{
		if(ch=='\n')
		{
			*line=*line+1;
			return 3;
		}
		if(IsOperator(ch)==1)
		{
			*cho=ch;
			if(len==0)
			{
				buffer[len]='\0';
				return 1;//operator
			}
			else//assume x1+x2;
			{
				buffer[len]='\0';//文件需要从新定位，回一个字符
				fseek(FHandle,-1,1);
				/*
				 fseek(fp,long/int,0/1/2)
				 0:at file beginning  
				 1:at present point
				 2:at file end
				 第二个参数是代表移动的字节数，正数是向前走的数目，负数是后退的数目。
				*/
				return 3;//var
			}
		}
 		if(ch==' ')//判断不严密，给后面留下了隐患，可能是空串
		{
			buffer[len]='\0';
			*re++;
			return 3;//Just consider as var
		} 
		else
		{		
			buffer[len]=ch;			 
			len++;
			ch=fgetc(FHandle);
		}
	}
	return 3;
	
}
int CTokens::Open(char* FileName)		
{
	FHandle=fopen(FileName,"r");
	if(FHandle==0)
		return 0;
	else return 1;
} 
int CTokens::ReadToken(int* ErrorLine,int* ErrorToken)
{//int CTokens::ReadWord(int* line,char* buffer,char* cho,int* re)
//	char buffer[10]="";刚开始的时候放在这，结果造成了文本文挡结尾是的空格处理不了
	char cho='a';
	int rev=0;
	CToken TmpToken;
	ZeroToken(&TmpToken);
	while(feof(FHandle)==0)
	{   char buffer[10]="";//作用域问题/
		rev=ReadWord(ErrorLine,buffer,&cho,ErrorToken);
		if(rev==0)
			return 0;
		if(rev==1)
		{
			ZeroToken(&TmpToken);
			TmpToken.CTokenChar=cho;
			TmpToken.CTokenkind=1;//1:operator
			TokenV.push_back(TmpToken);
			goto space;
		}
		if(rev==3)
		{
			if(IsKeyWord(buffer)==1)
			{
				ZeroToken(&TmpToken);
				TmpToken.CTokenkind=2;//2:keyword
				strcpy(TmpToken.CTokenName,buffer);
				TokenV.push_back(TmpToken);
				goto space; 
			}
			if(IsFLtNum(buffer)==1)
			{
				ZeroToken(&TmpToken);
				TmpToken.CTokenkind=4;//4:sonstant
				TmpToken.CTokenValue=atof(buffer);
				TokenV.push_back(TmpToken);
				goto space;
			}
			if(IsIntNum(buffer)==1)
			{
				ZeroToken(&TmpToken);
				TmpToken.CTokenkind=4;//4:sonstant
				TmpToken.CTokenValue=atoi(buffer);
				TokenV.push_back(TmpToken);
				goto space;
			}
			else
			{//主要是因为在上面的ReadWord(..)函数中，判断不严，有可能是个空的串也被当成了Word
				int len=0;
				len=strlen(buffer);
				if(len!=0)
				{
					ZeroToken(&TmpToken);
					TmpToken.CTokenkind=3;//3:varification
					strcpy(TmpToken.CTokenName,buffer);
					TokenV.push_back(TmpToken);
					goto space;
				}
				else goto space;
				
			}
		}
space: int i=0;
	   while(i<10){i++;}
	}
	return 1;
} 
int CTokens::GetTokenNum()
{
	int re=0;
	re=TokenV.size();
	return re;
}
void CTokens::GetToken(CToken* pToken,int index)
{
	*pToken=TokenV[index];
}
void printToken(CToken Token)
{
	switch(Token.CTokenkind)
	{
	case 1:
		{
			cout<<"Operator token  "<<Token.CTokenChar<<endl;
			break;
		}
	case 2:
		{
			cout<<"KeyWord token  "<<Token.CTokenName<<endl;
			break;
		}
	case 3:
		{
			cout<<"var token  "<<Token.CTokenName<<endl;
			break;
		}
	case 4:
		{
			cout<<"const token  "<<Token.CTokenValue<<endl;
			break;
		}
	}
}
#endif