scanner.l

Shorthand是一个强大的脚本语言

%option never-interactive
%option nostdinit
%{
///////////////////////////////////////////////////////////////////////////////
// $Header: $
//-----------------------------------------------------------------------------
// Project: ShortHand interpreter
// Author: Andrei Remenchuk <[e-mail]>
//-----------------------------------------------------------------------------
// scanner.l: definition of lexical scanner (.l file is processed by Flex).
///////////////////////////////////////////////////////////////////////////////
#include <string.h>
#include <stdlib.h>
#include <math.h>
#include <limits.h>

#ifdef WIN32
#include "io.h"
#define isatty _isatty
#endif

#include "shorthand.h"
#include "utils.h"
#include "nodes.h"
#include "mman.h"


#include "yydef.h"
#include "FlexLexer.h"
#include "parser.h"


#define PP(n) LEXER->yypos+=n;
#define PP1   LEXER->yypos++
#define PP2   LEXER->yypos+=2
#define PPT   LEXER->yypos+=yyleng

#define LEXER ((ShortHandLexer*)this)

#define TOKEN ((YYSTYPE*)LEXER->yylval)
#define RETURN_TOKEN(f) return (TOKEN->num=f)



#define YY_USER_ACTION \
  { LEXER->yylloc.first_line   = LEXER->yylloc.last_line = LEXER->yyline; \
    LEXER->yylloc.first_column = LEXER->yypos; \
    LEXER->yylloc.last_column  = LEXER->yypos+yyleng; \
  }

typedef struct _keyword {
  const char *token;
  int value;
} keyword;

#define KEYWORD_COUNT sizeof(keywords)/sizeof(keyword)
static int compare_keywords(const void *left, const void *right);

keyword keywords[] = {
  {"ALL",       ALL},
  {"AND",       AND},
  {"AS",        AS},
  {"BREAK",     BREAK},
  {"CONTINUE",  CONTINUE},
  {"ELSE",      ELSE},
  {"ELSEIF",    ELSEIF},
  {"ELSIF",     ELSEIF},
  {"END",       END},
  {"FOR",       FOR},
  {"FOREACH",   FOREACH},
  {"FUNCTION",  FUNCTION},
  {"GRID",      GRID},
  {"IF",        IF},
  {"INCLUDE",   INCLUDE},
  {"JUMP",      JUMP},
  {"LOCAL",     LOCAL},
  {"NEW",       K_NEW},
  {"NOT",       NOT},
  {"NULL",      K_NULL},
  {"OR",        OR},
  {"PRINT",     PRINT},
  {"PRINTLN",   PRINTLN},
  {"RETURN",    K_RETURN},
  {"THEN",      THEN},
  {"TO",        TO},
  {"WHILE",     WHILE}
};
static bool keywords_sorted = false;

#define FREE_TAG      1
#define QUESTION_TAG  2
#define PERCENT_TAG   3

#define INLINE_TAG    10


%}



DIGIT       [0-9]
LETTER      [a-zA-Z_]
INTEGER     {DIGIT}+
FLOAT       {INTEGER}\.{DIGIT}+
IDENT       {LETTER}({DIGIT}|{LETTER})*

QUOTED2     \"([^\"\n\r]|(\\\"))*[^\\\n\r]\"
QUOTED1     \'([^\'\n\r]|(\\\'))*[^\\\n\r]\'

%x TAG
%x VTAG
%x QCOMM
%x PAST_END

%%

"<~@"             |
"<?="             |
"<%="             { PPT; BEGIN TAG; 
                    LEXER->tag = INLINE_TAG;
                    LEXER->escape = yytext[1];
                    TOKEN->str = LEXER->flush_html(); 
                    if (TOKEN->str != NULL) return HTML_AND_PRINT; }

"<~"              |
"<?"              |
"<%"              { PPT; BEGIN TAG; 
                    LEXER->tag = FREE_TAG;
                    LEXER->escape = yytext[1];
                    // printf("<~: escape = %c\n", LEXER->escape);
                    TOKEN->str = LEXER->flush_html();
                    if (TOKEN->str != NULL) return BARE_HTML; }


<TAG>"?>"	  |
<TAG>"%>"	  |
<TAG>"~>"         { PPT; 
                    if (LEXER->escape == yytext[0]) { BEGIN 0; LEXER->tag = 0; }
		    //else printf("~>: escape = %c; yytext[1] = %c\n", LEXER->escape, yytext[1]);
		  }



<TAG>{INTEGER}     { PPT; 
                     TOKEN->num = atoi(yytext);
                     return NUMBER; }

<TAG>{FLOAT}       { PPT; 
                     TOKEN->fnum = atof(yytext);
                     return _FLOAT; }


<TAG>"!="          |
<TAG>"<>"          { PPT; return NE; }
<TAG>"<="          { PPT; return LE; }
<TAG>">="          { PPT; return GE; }
<TAG>"=>"          { PPT; return EQL; }

<TAG>{IDENT}  { PPT;

                static keyword key;
                key.token = yytext; //uppercase_string(yytext);
                keyword* kw = (keyword*) 
                bsearch(&key, keywords, KEYWORD_COUNT, sizeof(keyword), compare_keywords);
                if (kw) return kw->value;

                //TRACE((7, "unrecognized word: \"%s\"\n", yytext));
                if (yy_flex_debug) printf(" \"%s\" ", yytext);
                TOKEN->str = LEXER->m_scratch.strdup(yytext);

				int ident_type = LEXER->override_ident_type;
				if (ident_type == 0) ident_type = IDENT;
				LEXER->override_ident_type = IDENT;
                return ident_type; 
                } 


<TAG>\'\'          |
<TAG>\"\"          |
<TAG>{QUOTED1}     |
<TAG>{QUOTED2}     { PPT;
                     TOKEN->str = (char*) LEXER->m_scratch.malloc(yyleng-1);
                     char q = yytext[0];
                     register char *s = yytext+1;
                     register char *d = (char*) TOKEN->str;
                     while(*s != '\0' && *s != q)
                    {
		            if (*s == '\\')
		                 switch(*++s) 
                         {
		                 case 'n': *d++ = '\n';s++; break;
		                 case 't': *d++ = '\t';s++; break;
                                 case 'r': *d++ = '\r';s++; break;
		                 default: *d++ = *s++; break;
                         }
		            else *d++ = *s++;
                }
	        *d = '\0';
                //printf("quoted string: >>>>\n%s>>>\n", yytext);
	        return STRING; }

<TAG>\'\r?n    |
<TAG>\'[^\'\n]*\r?\n { 
                LEXER->yyline++; LEXER->yypos = 1; 
                //printf("single line comment: >>>>\n%s>>>\n", yytext);
                return '\n'; }



<TAG>[ \t]+    { PPT; }

<TAG>\r         /* */

<TAG>\n         { LEXER->yyline++; LEXER->yypos=1; 
                //TRACE((12, "returned newline\n"));
                return '\n'; }

<TAG>[\x80-\xff]  { PPT;
                TOKEN->num = 0; 
                TRACE((10, "invalid character: 0x%02x\n", yytext[0] & 0xff));
	        //compiler->add_scan_error(10302, "礤 觐痧尻蝽