token.c

在VC6环境下开发

#include "eDbInit.h"
typedef struct Keyword Keyword;
struct Keyword {
	char *zName;						/* The keyword name */
	unsigned char tokenType;			/* Token value for this keyword */
	unsigned char len;					/* Length of this keyword */
	unsigned char iNext;				/* Index in aKeywordTable[] of next with same hash */
};

static Keyword aKeywordTable[] = {
	{ "CREATE",            TK_CREATE,       },
	{ "TABLE",             TK_TABLE,        },
	{ "DROP",              TK_DROP,         },
	{ "INDEX",             TK_INDEX,        },
	{ "ON",                TK_ON,           },
	{ "INSERT",            TK_INSERT,       },
	{ "INTO",              TK_INTO,         },
	{ "VALUES",            TK_VALUES,       },
	{ "DELETE",            TK_DELETE,       },
	{ "FROM",              TK_FROM,         },
	{ "WHERE",             TK_WHERE,        },
	{ "AND",               TK_AND,          },
	{ "OR",                TK_OR,           },
	{ "UPDATE",            TK_UPDATE,       },
	{ "SET",               TK_SET,          },
	{ "SELECT",            TK_SELECT,       },
	{ "GROUP",             TK_GROUP,        },
	{ "BY",				   TK_BY,			},
	{ "HAVING",            TK_HAVING,       },
	{ "ORDER",             TK_ORDER,        },
	{ "PRIMARY",           TK_PRIMARY,      },
	{ "KEY",			   TK_KEY,			},
	{ "UNIQUE",			   TK_UNIQUE,		}

};

#define KEY_HASH_SIZE 101
static unsigned char aiHashTable[KEY_HASH_SIZE];
int eDbKeywordCode(const char *z, int n){
	int h, i;
	Keyword *p;
	static char needInit = 1;
	if( needInit ){
		/* Initialize the keyword hash table */
		//eDbOsEnterMutex();
		if( needInit ){
			int nk;
			nk = sizeof(aKeywordTable)/sizeof(aKeywordTable[0]);
			for(i=0; i<nk; i++){
				aKeywordTable[i].len = strlen(aKeywordTable[i].zName);
				h = eDbHashNoCase(aKeywordTable[i].zName, aKeywordTable[i].len);
				h %= KEY_HASH_SIZE;
				aKeywordTable[i].iNext = aiHashTable[h];
				aiHashTable[h] = i+1;
			}
			needInit = 0;
		}
		//eDbOsLeaveMutex();
	}
	h = eDbHashNoCase(z, n) % KEY_HASH_SIZE;
	for(i=aiHashTable[h]; i; i=p->iNext){
		p = &aKeywordTable[i-1];
		if( p->len==n && eDbStrNICmp(p->zName, z, n)==0 ){
		  return p->tokenType;
		}
	}
	return TK_ID;
}

static const char isIdChar[] = {
	/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 2x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
};

int ISDIGIT(int c){
	switch(c){
		case '0': case '1': case '2': case '3': case '4':
    	case '5': case '6': case '7': case '8': case '9':
    		return 1;    	
	}
	return 0;
}
static int GetToken(const unsigned char *z, int *tokenType){
	int i;
	switch( *z ){
	case ' ': case '\t': case '\n': case '\f': case '\r': {
		for(i=1; isspace(z[i]); i++){}
		*tokenType = TK_SPACE;
		return i;
	}    
	case '\'': case '"': {
		int delim = z[0];
		for(i=1; z[i]; i++){
			if( z[i]==delim ){
				if( z[i+1]==delim ){
					i++;
				}else{
					break;
				}
			}
		}
		if( z[i] ) i++;
		*tokenType = TK_STRING;
		return i;
    }
		case '=': {
		*tokenType = TK_EQ;
		return 1 + (z[1]=='=');
    }
    case '(': {
		*tokenType = TK_LP;
		return 1;
    }
    case ')': {
		*tokenType = TK_RP;
		return 1;
    }
    case ';': {
		*tokenType = TK_SEMI;
		return 1;
    }
	case ',': {
		*tokenType = TK_COMMA;
		return 1;
    }
	case '0': case '1': case '2': case '3': case '4':
    case '5': case '6': case '7': case '8': case '9': {
		*tokenType = TK_INTEGER;
		for(i=1; ISDIGIT(z[i]); i++){}
		/*
		if( z[i]=='.' && ISDIGIT(z[i+1]) ){
			i += 2;
			while( ISDIGIT(z[i]) ){ i++; }
			*tokenType = TK_FLOAT;
		}
		if( (z[i]=='e' || z[i]=='E') && ( ISDIGIT(z[i+1]) 
		|| ((z[i+1]=='+' || z[i+1]=='-') && ISDIGIT(z[i+2]))
		)){
			i += 2;
			while( ISDIGIT(z[i]) ){ i++; }
			*tokenType = TK_FLOAT;
		}
		*/
		return i;
	}
	
    case '<': {
		if( z[1]=='=' ){
			*tokenType = TK_LE;
			return 2;
		}else if( z[1]=='>' ){
			*tokenType = TK_NE;
			return 2;
		}else if( z[1]=='<' ){
			*tokenType = TK_LSHIFT;
			return 2;
		}else{
			*tokenType = TK_LT;
			return 1;
		}
    }
    case '>': {
		if( z[1]=='=' ){
			*tokenType = TK_GE;
			return 2;
		}else if( z[1]=='>' ){
			*tokenType = TK_RSHIFT;
			return 2;
		}else{
			*tokenType = TK_GT;
			return 1;
		}
    }
    case '!': {
		if( z[1]!='=' ){
			*tokenType = TK_ILLEGAL;
			return 2;
		}else{
			*tokenType = TK_NE;
			return 2;
		}
    }
	case '*': {
		*tokenType = TK_STAR;
		return 1;
    }
	case '.': {
		*tokenType = TK_DOT;
		return 1;
    }    
/*
	case '-': {
		if( z[1]=='-' ){
		for(i=2; z[i] && z[i]!='\n'; i++){}
			*tokenType = TK_COMMENT;
			return i;
		}
		*tokenType = TK_MINUS;
		return 1;
    }
    case '+': {
		*tokenType = TK_PLUS;
		return 1;
    }
    
    case '/': {
		if( z[1]!='*' || z[2]==0 ){
			*tokenType = TK_SLASH;
			return 1;
		}
		for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){}
		if( z[i] ) i++;
		*tokenType = TK_COMMENT;
		return i;
    }
    case '%': {
		*tokenType = TK_REM;
		return 1;
    }    
    case '|': {
		if( z[1]!='|' ){
			*tokenType = TK_BITOR;
			return 1;
		}else{
			*tokenType = TK_CONCAT;
			return 2;
		}
    }    
    case '&': {
		*tokenType = TK_BITAND;
		return 1;
    }
    case '~': {
		*tokenType = TK_BITNOT;
		return 1;
    }    
    
	case '[': {
		for(i=1; z[i] && z[i-1]!=']'; i++){}
		*tokenType = TK_ID;
		return i;
	}
	case '?': {
		*tokenType = TK_VARIABLE;
		return 1;
	}
*/
    default: {
		if( (*z&0x80)==0 && !isIdChar[*z] ){
			break;
		}
		for(i=1; (z[i]&0x80)!=0 || isIdChar[z[i]]; i++){}
			*tokenType = eDbKeywordCode((char*)z, i);
			return i;
		}
	}
	*tokenType = TK_ILLEGAL;
	return 1;
}


int eDbRunParser(Parser *pParse, const char *zSql, char **pzErrMsg){
	int nErr = 0;
	int i;
	void *pEngine;
	int tokenType;
	int lastTokenParsed = -1;
	int isSEMI = 0;
	extern void *eDbParserAlloc(void*(*)(size_t));
	extern void eDbParserFree(void*, void(*)(void*));
	extern void eDbParser(void*, u8, Token, Parser*);

	pParse->rc = 0;
	i = 0;
	pEngine = eDbParserAlloc((void*(*)(size_t))_MALLOC_);
	if( pEngine==0 ){
		eDbSetString(pzErrMsg, "out of memory", (char*)0);
		return 1;
	}
	pParse->sLastToken.dyn = 0;
	pParse->zTail = zSql;
	while(eDb_malloc_failed==0 && zSql[i]!=0 ){
		pParse->sLastToken.z = &zSql[i];
		pParse->sLastToken.n = GetToken((unsigned char*)&zSql[i], &tokenType);
		i += pParse->sLastToken.n;
		switch( tokenType ){
			case TK_SPACE:
			case TK_COMMENT: {        
				break;
			}
			case TK_ILLEGAL: {        
				goto abort_parse;
			}
			case TK_SEMI: {
				pParse->zTail = &zSql[i];
				isSEMI = 1;
				/* Fall thru into the default case */
			}
			default: {
				eDbParser(pEngine, (u8)tokenType, pParse->sLastToken, pParse);
				lastTokenParsed = tokenType;
				if( pParse->rc!=eDb_OK ){
					goto abort_parse;
				}
				break;
			}
		}
		if(isSEMI) break;
	}
abort_parse:
	if( zSql[i]==0 && nErr==0 && pParse->rc==eDb_OK ){
		if( lastTokenParsed!=TK_SEMI ){
			eDbParser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
			pParse->zTail = &zSql[i];
		}
		eDbParser(pEngine, 0, pParse->sLastToken, pParse);
	}
	eDbParserFree(pEngine, _FREE_); 	
	if( pParse->zErrMsg ){
		if( pzErrMsg && *pzErrMsg==0 ){
			*pzErrMsg = pParse->zErrMsg;
		}else{
			eDbFree(pParse->zErrMsg);
		}
		pParse->zErrMsg = 0;
		if( !nErr ) nErr++;
		if( pParse->pNewTable ){
			eDbDeleteTable(pParse->db, pParse->pNewTable);
			pParse->pNewTable = 0;
		}
	}			
	if( nErr>0 && pParse->rc==eDb_OK ){
		pParse->rc = eDb_ERROR;
	}
	return nErr;
}