lexscan.flex

java语法解释器生成器

                                  functionName = "next_token";
                                if (tokenType == null)
                                  tokenType = "java_cup.runtime.Symbol";
                                if (eofVal == null)
                                  eofVal = "return new java_cup.runtime.Symbol("+cupSymbol+".EOF);";
                                if (!Options.jlex) eofclose = true;
                              }
  "%cupsym"{WSP}+{QualIdent} {WSP}*  { cupSymbol = yytext().substring(8).trim(); 
                                if (cupCompatible) Out.warning(ErrorMessages.CUPSYM_AFTER_CUP, yyline); }
  "%cupsym"{WSP}+{NNL}*       { throw new ScannerException(file,ErrorMessages.QUIL_CUPSYM, yyline); }
  "%cupdebug"                 { cupDebug = true; }
  "%eofclose"({WSP}+"true")?  { eofclose = true; }
  "%eofclose"({WSP}+"false")  { eofclose = false; }
  "%class"{WSP}+{ClassT} {WSP}*     { className = yytext().substring(7).trim();  }
  "%ctorarg"{WSP}+{ArrType}{WSP}+   { yybegin(CTOR_ARG); ctorTypes.add(yytext().substring(8).trim()); }
  "%function"{WSP}+{Ident} {WSP}*   { functionName = yytext().substring(10).trim(); }
  "%type"{WSP}+{ArrType} {WSP}*     { tokenType = yytext().substring(6).trim(); }
  "%integer"|"%int"           { isInteger = true;  }
  "%intwrap"                  { isIntWrap = true;  }
  "%yyeof"                    { isYYEOF = true;  }
  "%notunix"                  { notUnix = true;  }
  "%7bit"                     { return symbol(ASCII); }
  "%full"|"%8bit"             { return symbol(FULL); }
  "%unicode"|"%16bit"         { return symbol(UNICODE);  }
  "%caseless"|"%ignorecase"   { caseless = true; }
  "%implements"{WSP}+.*       { isImplementing = concExc(isImplementing, yytext().substring(12).trim());  }
  "%extends"{WSP}+{QClassT}{WSP}* { isExtending = yytext().substring(9).trim(); }
  "%public"                   { isPublic = true; }
  "%apiprivate"               { visibility = "private"; Skeleton.makePrivate(); }
  "%final"                    { isFinal = true; }
  "%abstract"                 { isAbstract = true; }
  "%debug"                    { debugOption = true; }
  "%standalone"               { standalone = true; isInteger = true; }
  "%switch"                   { packed = false; useRowMap = false; }
  "%table"                    { packed = false; useRowMap = true; }
  "%pack"                     { packed = true; useRowMap = true; }
  "%include" {WSP}+ .*        { File f = new File(yytext().substring(9).trim());
                                if ( !f.canRead() )
                                  throw new ScannerException(file,ErrorMessages.NOT_READABLE, yyline); 
                                // check for cycle
                                if (files.search(f) > 0)
                                  throw new ScannerException(file,ErrorMessages.FILE_CYCLE, yyline);
                                try {
                                  yypushStream( new FileReader(f) );
                                  files.push(file);
                                  file = f;
                                  Out.println("Including \""+file+"\"");
                                }
                                catch (FileNotFoundException e) {
                                  throw new ScannerException(file,ErrorMessages.NOT_READABLE, yyline); 
                                } 
                              }
  "%buffer" {WSP}+ {Number} {WSP}*   { bufferSize = Integer.parseInt(yytext().substring(8).trim()); }
  "%buffer" {WSP}+ {NNL}*     { throw new ScannerException(file,ErrorMessages.NO_BUFFER_SIZE, yyline); }
  "%initthrow" {WSP}+ {QUIL} {WSP}* { initThrow = concExc(initThrow,yytext().substring(11).trim()); }
  "%initthrow" {WSP}+ {NNL}*  { throw new ScannerException(file,ErrorMessages.QUIL_INITTHROW, yyline); }
  "%eofthrow"  {WSP}+ {QUIL} {WSP}*  { eofThrow = concExc(eofThrow,yytext().substring(10).trim()); }
  "%eofthrow"  {WSP}+ {NNL}*  { throw new ScannerException(file,ErrorMessages.QUIL_EOFTHROW, yyline); }
  "%yylexthrow"{WSP}+ {QUIL} {WSP}*  { lexThrow = concExc(lexThrow,yytext().substring(12).trim()); }
  "%throws"    {WSP}+ {QUIL} {WSP}*  { lexThrow = concExc(lexThrow,yytext().substring(8).trim()); }
  "%yylexthrow"{WSP}+ {NNL}*  { throw new ScannerException(file,ErrorMessages.QUIL_YYLEXTHROW, yyline); }
  "%throws"    {WSP}+ {NNL}*  { throw new ScannerException(file,ErrorMessages.QUIL_THROW, yyline); }
  "%scanerror" {WSP}+ {QualIdent} {WSP}* { scanErrorException = yytext().substring(11).trim(); }
  "%scanerror" {WSP}+ {NNL}*  { throw new ScannerException(file,ErrorMessages.QUIL_SCANERROR, yyline); }

  {Ident}                     { return symbol(IDENT, yytext()); }
  "="{WSP}*                   { yybegin(REGEXP); return symbol(EQUALS); }

  "/*"                        { nextState = MACROS; yybegin(COMMENT); }
  
  {EndOfLineComment}          { }

  /* no {NL} at the end of this expression, because <REGEXPSTART> 
     needs at least one {WSPNL} to start a regular expression! */   
  ^"%%" {NNL}*                { macroDefinition = false; yybegin(REGEXPSTART); return symbol(DELIMITER); }
  "%"{Ident}                  { throw new ScannerException(file,ErrorMessages.UNKNOWN_OPTION, yyline, yycolumn); }
  "%"                         { throw new ScannerException(file,ErrorMessages.UNKNOWN_OPTION, yyline, yycolumn); }
  ^{WSP}+"%"                  { Out.warning(ErrorMessages.NOT_AT_BOL, yyline); yypushback(1); }

  {WSP}+                      { }
  {NL}+                       { }                        
  <<EOF>>                     { if ( yymoreStreams() ) {
                                  file = (File) files.pop();
                                  yypopStream();
                                }
                                else
                                  throw new ScannerException(file,ErrorMessages.EOF_IN_MACROS); 
                              }
}

<CTOR_ARG> {
  {Ident} {WSP}*   { yybegin(MACROS); ctorArgs.add(yytext().trim()); }
  [^]              { throw new ScannerException(file,ErrorMessages.CTOR_ARG,yyline,yycolumn); }
}

<REGEXPSTART> {
  {WSPNL}* "/*"               { nextState = REGEXPSTART; yybegin(COMMENT); }
  {WSPNL}+                    { yybegin(REGEXP); }
  {WSPNL}* "<"                { yybegin(STATES); return symbol_countUpdate(LESSTHAN, null); }
  {WSPNL}* "}"                { return symbol_countUpdate(RBRACE, null); }
  {WSPNL}* "//" {NNL}*        { }  
  {WSPNL}* "<<EOF>>" {WSPNL}* "{" 
                              { actionText.setLength(0); yybegin(JAVA_CODE); 
                                Symbol s = symbol_countUpdate(EOFRULE, null);
                                action_line = s.left+1; 
                                return s;
                              }
}

<STATES> {
  {Ident}                     { return symbol(IDENT, yytext()); }
  ","                         { return symbol(COMMA); }
  {WSPNL}+                    { }

  // "{" will be caught in REGEXP  
  ">"{WSPNL}*                 { yybegin(REGEXP); return symbol(MORETHAN); }

  <<EOF>>                     { throw new ScannerException(file,ErrorMessages.EOF_IN_STATES); }
}


<REGEXP> {
  "<<EOF>>" {WSPNL}+ "{"  { actionText.setLength(0); yybegin(JAVA_CODE); action_line = yyline+1; return symbol(EOFRULE); }
  "<<EOF>>"               { throw new ScannerException(file,ErrorMessages.EOF_WO_ACTION); }

  {WSPNL}*"|"{WSP}*$      { if (macroDefinition) {
                              yybegin(EATWSPNL);
                              return symbol(BAR); 
                            }
                            else { 
                              yybegin(REGEXPSTART); 
                              return symbol(NOACTION); 
                            }
                          }

  // stategroup
  "{"          { yybegin(REGEXPSTART); return symbol(LBRACE); }

  {WSPNL}*"|"  { return symbol(BAR); }

  {WSPNL}*\"   { string.setLength(0); nextState = REGEXP; yybegin(STRING_CONTENT); }
  {WSPNL}*"!"  { return symbol(BANG); }
  {WSPNL}*"~"  { return symbol(TILDE); }
  {WSPNL}*"("  { return symbol(OPENBRACKET); }
  {WSPNL}*")"  { return symbol(CLOSEBRACKET); }
  {WSPNL}*"*"  { return symbol(STAR); }
  {WSPNL}*"+"  { return symbol(PLUS); }
  {WSPNL}*"?"  { return symbol(QUESTION); }
  {WSPNL}*"$"  { return symbol(DOLLAR); }
  {WSPNL}*"^"  { bolUsed = true; return symbol(HAT); }
  {WSPNL}*"."  { return symbol(POINT); }
  {WSPNL}*"["  { yybegin(CHARCLASS); return symbol(OPENCLASS); }
  {WSPNL}*"/"  { return symbol(LOOKAHEAD); }
  
  {WSPNL}* "{" {WSP}* {Ident} {WSP}* "}" { return symbol_countUpdate(MACROUSE, makeMacroIdent()); }
  {WSPNL}* "{" {WSP}* {Number}   { yybegin(REPEATEXP); return symbol(REPEAT, new Integer(yytext().trim().substring(1).trim())); }

  {WSPNL}+ "{"    { actionText.setLength(0); yybegin(JAVA_CODE); action_line = yyline+1; return symbol(REGEXPEND); }
  {NL}            { if (macroDefinition) { yybegin(MACROS); } return symbol(REGEXPEND); }

  {WSPNL}*"/*"    { nextState = REGEXP; yybegin(COMMENT); }

  {WSPNL}*"//"{NNL}*  { }

  {WSP}+          { }

  <CHARCLASS> {
    {WSPNL}*"[:jletter:]"  { return symbol(JLETTERCLASS); }
    {WSPNL}*"[:jletterdigit:]" { return symbol(JLETTERDIGITCLASS); }
    {WSPNL}*"[:letter:]"     { return symbol(LETTERCLASS); }
    {WSPNL}*"[:digit:]"      { return symbol(DIGITCLASS); }
    {WSPNL}*"[:uppercase:]"  { return symbol(UPPERCLASS); }
    {WSPNL}*"[:lowercase:]"  { return symbol(LOWERCLASS); }
  }

  . { return symbol(CHAR, new Character(yytext().charAt(0))); }
}

<EATWSPNL> {WSPNL}+  { yybegin(REGEXP); }


<REPEATEXP> {
  "}"          { yybegin(REGEXP); return symbol(RBRACE); }
  "," {WSP}* {Number}  { return symbol(REPEAT, new Integer(yytext().substring(1).trim())); }
  {WSP}+       { }

  <<EOF>>                 { throw new ScannerException(file,ErrorMessages.EOF_IN_REGEXP); }
}

<CHARCLASS> {
  "{"{Ident}"}" { return symbol(MACROUSE, yytext().substring(1,yytext().length()-1)); }
  "["  { balance++; return symbol(OPENCLASS); }
  "]"  { if (balance > 0) balance--; else yybegin(REGEXP); return symbol(CLOSECLASS); }
  "^"  { return symbol(HAT); }
  "-"  { return symbol(DASH); }

  // this is a hack to keep JLex compatibilty with char class 
  // expressions like [+-]
  "-]" { yypushback(1); yycolumn--; return symbol(CHAR, new Character(yytext().charAt(0))); }  

  \"   { string.setLength(0); nextState = CHARCLASS; yybegin(STRING_CONTENT); }

  .    { return symbol(CHAR, new Character(yytext().charAt(0))); }

  \n   { throw new ScannerException(file,ErrorMessages.EOL_IN_CHARCLASS,yyline,yycolumn); }

  <<EOF>>     { throw new ScannerException(file,ErrorMessages.EOF_IN_REGEXP); }
}

<STRING_CONTENT> {
  \"       { yybegin(nextState); return symbol(STRING, string.toString()); }
  \\\"     { string.append('\"'); }
  [^\"\\\u2028\u2029\u000A\u000B\u000C\u000D\u0085]+ { string.append(yytext()); }

  {NL}     { throw new ScannerException(file,ErrorMessages.UNTERMINATED_STR, yyline, yycolumn); }

  {HexNumber} { string.append( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16)); }
  {Unicode}   { string.append( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16)); }
  {OctNumber} { string.append( (char) Integer.parseInt(yytext().substring(1,yytext().length()), 8)); }

  \\b { string.append('\b'); }
  \\n { string.append('\n'); }
  \\t { string.append('\t'); }
  \\f { string.append('\f'); }
  \\r { string.append('\r'); }

  \\. { string.append(yytext().charAt(1)); }

  <<EOF>>     { throw new ScannerException(file,ErrorMessages.EOF_IN_STRING); }
}


<REGEXP, CHARCLASS> {
  {HexNumber} { return symbol(CHAR, new Character( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16))); }
  {Unicode} { return symbol(CHAR, new Character( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16))); }
  {OctNumber} { return symbol(CHAR, new Character( (char) Integer.parseInt(yytext().substring(1,yytext().length()), 8))); }

  \\b { return symbol(CHAR,new Character('\b')); }
  \\n { return symbol(CHAR,new Character('\n')); }
  \\t { return symbol(CHAR,new Character('\t')); }
  \\f { return symbol(CHAR,new Character('\f')); }
  \\r { return symbol(CHAR,new Character('\r')); }

  \\. { return symbol(CHAR, new Character(yytext().charAt(1))); }
}


<JAVA_CODE> {
  "{"        { balance++; actionText.append('{'); }
  "}"        { if (balance > 0) {
                 balance--;     
                 actionText.append('}'); 
               }
               else {
                 yybegin(REGEXPSTART); 
                 Action a = new Action(actionText.toString(), action_line);
                 actions.addElement(a);
                 return symbol(ACTION, a);
               }
             } 
           
  {JavaCode}     { actionText.append(yytext()); } 

  <<EOF>>     { throw new ScannerException(file,ErrorMessages.EOF_IN_ACTION, action_line-1); }
}

<COMMENT> {
   
  "/"+ "*"  { commentbalance++; }
  "*"+ "/"  { if (commentbalance > 0) 
                commentbalance--; 
              else
                yybegin(nextState); 
            }
  
  {JFlexComment} { /* ignore */ }

  <<EOF>>     { throw new ScannerException(file,ErrorMessages.EOF_IN_COMMENT); }
}


.  { throw new ScannerException(file,ErrorMessages.UNEXPECTED_CHAR, yyline, yycolumn); }
\n { throw new ScannerException(file,ErrorMessages.UNEXPECTED_NL, yyline, yycolumn); }

<<EOF>>  { if ( yymoreStreams() ) {
             file = (File) files.pop();
             yypopStream();
           }
           else 
             return symbol(EOF); }