python.jjt

Python Development Environment (Python IDE plugin for Eclipse). Features editor, code completion, re

    matchedToken.image = image.toString(); } : DEFAULT}
<IN_USTRING13> TOKEN : { <TRIPLE_USTRING: "'''"> {
    matchedToken.image = image.toString(); } : DEFAULT}
<IN_USTRING23> TOKEN : { <TRIPLE_USTRING2: "\"\"\""> {
    matchedToken.image = image.toString(); } : DEFAULT}

<IN_STRING11> MORE:
{
    <"\\\r\n">           { image.setLength(image.length()-3); } : IN_STRING1NLC
|   <("\\" ("\n"|"\r"))> { image.setLength(image.length()-2); } : IN_STRING1NLC
}

<IN_STRING21> MORE:
{
    <"\\\r\n">           { image.setLength(image.length()-3); } : IN_STRING2NLC
|   <("\\" ("\n"|"\r"))> { image.setLength(image.length()-2); } : IN_STRING2NLC
}

<IN_USTRING11> MORE:
{
    <"\\\r\n">           { image.setLength(image.length()-3); } : IN_USTRING1NLC
|   <("\\" ("\n"|"\r"))> { image.setLength(image.length()-2); } : IN_USTRING1NLC
}

<IN_USTRING21> MORE:
{
    <"\\\r\n">           { image.setLength(image.length()-3); } : IN_USTRING2NLC
|   <("\\" ("\n"|"\r"))> { image.setLength(image.length()-2); } : IN_USTRING2NLC
}

<IN_STRING1NLC> MORE:
{
 <""> : IN_STRING11
}

<IN_STRING2NLC> MORE:
{
 <""> : IN_STRING21
}

<IN_USTRING1NLC> MORE:
{
 <""> : IN_USTRING11
}

<IN_USTRING2NLC> MORE:
{
 <""> : IN_USTRING21
}

<IN_STRING11, IN_USTRING11> MORE: { <("\\" ("\\"|"'")) | ~["\n","\r"]> }
<IN_STRING21, IN_USTRING21> MORE: { <("\\" ("\\"|"\"")) | ~["\n","\r"]> }

/* This is a test to see if we can make the loading of strings more efficient (and maybe replace the MORE that is declared below.
I stopped this because I've seen that making the CharStream was apparently the number 1 thing to do, but it might be worth
comming back to this approach later).
<IN_STRING23> MORE:
{
    <~[]> 
    {
         try {
                 while(true){
                         char c = input_stream.readChar();
                         image.append(c);
                         int len = image.length();
                         if(len > 3 && image.charAt(len-3) == '"' && image.charAt(len-2) == '"'  && image.charAt(len-1) == '"' ){
                                 input_stream.backup(3);
                                 image.delete(image.length()-3, image.length());
                                 break;
                         }
                 }
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
    }
}
*/

<IN_STRING13, IN_STRING23, IN_USTRING13, IN_USTRING23> MORE:
{
    <"\r\n"> {
        int l = image.length();
        image.setLength(l-1);
        image.setCharAt(l-2, '\n');
    }
|   <"\n">
|   <"\r"> { image.setCharAt(image.length()-1, '\n'); }
|   <~["\n","\r"]>
|   <"\\" ~["\n","\r"]>
}

//single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
// apparently CPython coalesces newlines, we don't
modType single_input(): {
    token_source.single_input=true;
}
{
    (LOOKAHEAD(2) <NEWLINE>)* [(simple_stmt() | compound_stmt() <NEWLINE>)] (<NEWLINE>)* <EOF>
    { return (modType) jjtree.popNode(); }
}

//file_input: (NEWLINE | stmt)* ENDMARKER
modType file_input(): {token_source.single_input=false;}
{
    (<NEWLINE> | stmt())* <EOF>
    { return (modType) jjtree.popNode(); }
}

//eval_input: NEWLINE* testlist NEWLINE* ENDMARKER
modType eval_input(): {token_source.single_input=false;}
{
    (<NEWLINE>)* SmartTestList() (<NEWLINE>)* <EOF>
    { return (modType) jjtree.popNode(); }
}

//funcdef: 'def' NAME parameters ':' suite
void funcdef(): {}
{ 
    decorators()
    <DEF> AnyName() parameters() <COLON> {this.addSpecialToken(":");} suite() 
}

//@ call, name, attr ... <nl>
void decorators(): {}
{
    (begin_decorator() (<LPAREN>  {this.addSpecialToken("(", STRATEGY_BEFORE_NEXT);} insidetuporcall() <RPAREN>  {this.findTokenAndAdd(")");} )* <NEWLINE> )* 
}
void begin_decorator(): {}
{ <AT> dotted_name()
}

//parameters: '(' [varargslist] ')'
void parameters() #void: {}
{ <LPAREN>{this.findTokenAndAdd("(");}
  [varargslist()] 
  <RPAREN>{this.findTokenAndAdd(")");} 
  }

//varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME] | ('**'|'*' '*') NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [',']
void varargslist() #void: {}
{
    defaultarg() (LOOKAHEAD(2) {this.addSpecialToken(",");} <COMMA> defaultarg())*
        [LOOKAHEAD(3) {this.addSpecialToken(",");} <COMMA> ExtraArgList()]
        [LOOKAHEAD(2) {this.addSpecialToken(",");} <COMMA> ExtraKeywordList()]
        [{this.addSpecialToken(",");} <COMMA>]
    |   ( LOOKAHEAD(2) ExtraArgList() [{this.addSpecialToken(",");} <COMMA> ExtraKeywordList()]
    | ExtraKeywordList()
    )
}

void ExtraArgList(): {}
{ <MULTIPLY> {this.addSpecialToken("*", STRATEGY_BEFORE_NEXT);} Name() }

void ExtraKeywordList(): {}
{ (<POWER>{this.addSpecialToken("**", STRATEGY_BEFORE_NEXT);}|<MULTIPLY> {this.addSpecialToken("*", STRATEGY_BEFORE_NEXT);}{this.addSpecialToken("*", STRATEGY_BEFORE_NEXT);} <MULTIPLY>) Name() }

void defaultarg(): {}
{ fpdef() [<EQUAL> test()] }

//fpdef: NAME | '(' fplist ')'
void fpdef() #void: {}
{ Name() | <LPAREN>  {this.addSpecialToken("(",STRATEGY_BEFORE_NEXT);}   fplist() <RPAREN> {this.findTokenAndAdd(")");}  }

//fplist: fpdef (',' fpdef)* [',']
void fplist() #tuple: {}
{ fpdef() (LOOKAHEAD(2) {this.addSpecialToken(",");} <COMMA> fpdef())* [{this.addSpecialToken(",");} <COMMA>] }


//stmt: simple_stmt | compound_stmt
void stmt() #void: {}
{ simple_stmt() | compound_stmt() }

//simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
void simple_stmt() #void: {}
{ small_stmt() (LOOKAHEAD(2) <SEMICOLON> small_stmt())* [<SEMICOLON>] <NEWLINE>
}

//small_stmt: expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt
void small_stmt() #void: {SimpleNode simpleNode;}
{
    expr_stmt()
|   print_stmt()
|   del_stmt()
|   pass_stmt() {addToPeek("pass", false); }
|   flow_stmt()
|   import_stmt()
|   global_stmt()
|   exec_stmt()
|   assert_stmt() {addToPeek("assert ", false); }
}

//expr_stmt: testlist (augassign testlist | ('=' testlist)*)
void expr_stmt() #void: {}
{
    SmartTestList() (
    <PLUSEQ> SmartTestList() #aug_plus(2)
|   <MINUSEQ> SmartTestList() #aug_minus(2)
|   <MULTIPLYEQ> SmartTestList() #aug_multiply(2)
|   <DIVIDEEQ> SmartTestList() #aug_divide(2)
|   <FLOORDIVIDEEQ> SmartTestList() #aug_floordivide(2)
|   <MODULOEQ> SmartTestList() #aug_modulo(2)
|   <ANDEQ> SmartTestList() #aug_and(2)
|   <OREQ> SmartTestList() #aug_or(2)
|   <XOREQ> SmartTestList() #aug_xor(2)
|   <LSHIFTEQ> SmartTestList() #aug_lshift(2)
|   <RSHIFTEQ> SmartTestList() #aug_rshift(2)
|   <POWEREQ> SmartTestList() #aug_power(2)
|   (<EQUAL> SmartTestList())* #expr_stmt(jjtree.nodeArity()+1)
    )
}

//print_stmt: 'print' (test ',')* [test] | 'print' '>>' test (, test)+ [,]
void print_stmt() #void: {}
{
    LOOKAHEAD(2) <PRINT> <RSHIFT>  
    (test() [ (LOOKAHEAD(2) {this.addSpecialToken(",");} <COMMA> test())+ [Comma()] ] )#printext_stmt
    
|   LOOKAHEAD(2) <PRINT> 
    (test() (LOOKAHEAD(2) {this.addSpecialToken(",");} <COMMA> test())* [Comma()])#print_stmt

|   <PRINT>  #print_stmt

}


//del_stmt: 'del' exprlist
void del_stmt(): {}
{ begin_del_stmt() exprlist() }

void begin_del_stmt(): {}
{ <DEL> {this.addToPeek("del ",false);}
}

//pass_stmt: 'pass'
void pass_stmt(): {}
{ <PASS> }

//flow_stmt: break_stmt | continue_stmt | return_stmt | yield_stmt | raise_stmt
void flow_stmt() #void: {}
{
    <BREAK> {addToPeek("break",true);} #break_stmt(0)
|   <CONTINUE>  {addToPeek("continue",true);} #continue_stmt(0)
|   return_stmt()
|   yield_stmt()
|   raise_stmt()
}

//return_stmt: 'return' [testlist]
void return_stmt(): {}
{ begin_return_stmt() [SmartTestList()] }

void begin_return_stmt(): {}
{ <RETURN> {this.addToPeek("return ",false);}
}


//yield_stmt: 'yield' [testlist]
void yield_stmt(): {}
{ <YIELD> SmartTestList() {this.addToPeek("yield ",false, Yield.class);}}

//raise_stmt: 'raise' [test [',' test [',' test]]]
void raise_stmt(): {}
{ <RAISE> {this.addSpecialToken("raise ", STRATEGY_BEFORE_NEXT);} [test() [{this.addSpecialToken(",");} <COMMA> test() [{this.addSpecialToken(",");} <COMMA> test()]]] }

//import_stmt: 'import' dotted_name (',' dotted_name)* | 'from' dotted_name 'import' ('*' | NAME (',' NAME)*)
void import_stmt() #void: {Import imp;}
{  <IMPORT> imp = Import() {imp.addSpecial("import ",false);} 
  |<FROM> {this.addSpecialToken("from ",STRATEGY_BEFORE_NEXT);} ImportFrom() 
}


Import Import(): {}
{ dotted_as_name() ({this.addSpecialToken(",");} <COMMA> dotted_as_name())* 
  {return (Import)jjtree.peekNode();}
}

//TODO: allways allowed (treat different versions in other places, to see if this is correct or not)
void ImportFrom(): { String mod; String name;int state=0; }
{
    mod=dotted_name() <IMPORT> {this.addSpecialToken(" import ");}
    (
        //from xxx import *
        <MULTIPLY> {this.addSpecialToken("*",STRATEGY_ADD_AFTER_PREV);}//from xx import *
        
        //from xxx import a,b,c
        | (name=import_as_name() ({this.addSpecialToken(",");} <COMMA> import_as_name())*) 
        
        //from xxx import (a,b,c)
        | <LPAREN>  {this.addSpecialToken("(",STRATEGY_BEFORE_NEXT);}   
          name=import_as_name() 
          (
           ({
             if(state!=0){
                 throw new ParseException("Invalid syntax: 2 commas cannot be grouped.", getToken(1));
             }
             state=1; 
             this.addSpecialToken(",");
             } 
             <COMMA> ( {state=0;} import_as_name())? )* 
           <RPAREN>  {this.findTokenAndAdd(")");} 
          )
    )
}

//dotted_as_name: dotted_name [NAME NAME]
void dotted_as_name(): {}
{ dotted_name() [<AS> {this.addSpecialToken(" as ");} Name()] }

//dotted_name: NAME ('.' NAME)*
String dotted_name(): { Token t; StringBuffer sb = new StringBuffer(); }
{ t=AnyName() { sb.append(t.image); }
    (<DOT> t=AnyName() { sb.append("." + t.image); } )*
        { return sb.toString(); }
}

//import_as_name: NAME [NAME NAME]
String import_as_name(): { Token t; }
{ t=AnyName() [<AS> {this.addSpecialToken(" as ");} Name()] { return t.image; } }

//global_stmt: 'global' NAME (',' NAME)*
void global_stmt(): {}
{ <GLOBAL> {this.addSpecialToken("global ", STRATEGY_BEFORE_NEXT);} Name() ({this.addSpecialToken(",");} <COMMA> Name())* }

//exec_stmt: 'exec' expr ['in' test [',' test]]
void exec_stmt(): {}
{ <EXEC>{this.addSpecialToken("exec ", STRATEGY_BEFORE_NEXT);} expr() [<IN>{this.addSpecialToken(" in ");} test() [{this.addSpecialToken(",");} <COMMA> test()]] }

//assert_stmt: 'assert' test [',' test]
void assert_stmt(): {}
{ <ASSERT> test() [{this.addSpecialToken(",");} <COMMA> test()] }

//compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
void compound_stmt() #void : { token_source.compound = true; }
{if_stmt() | while_stmt() | for_stmt() | try_stmt()  | funcdef() | classdef()}

//if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
void if_stmt(): {}
{
    begin_if_stmt() test() <COLON> {this.addSpecialToken(":");} suite()
         (begin_elif_stmt() test() <COLON> {this.addSpecialToken(":");} suite())* 
             [ <ELSE> <COLON> {this.findTokenAndAdd("else","else:",true);} suite()]
}

void begin_if_stmt(): {Object spStr;}
{ {spStr = createSpecialStr("if","if ", false);} <IF> {this.addToPeek(spStr,false);} 
}

void begin_elif_stmt(): {}
{ <ELIF> {this.addToPeek("elif",false);}
}


//while_stmt: 'while' test ':' suite ['else' ':' suite]
void while_stmt(): {}
{ begin_while_stmt() test() <COLON> {this.addSpecialToken(":");} suite() 
  [ begin_else_stmt()  suite()] }

void begin_while_stmt(): {}
{ {this.addSpecialToken("while ",STRATEGY_BEFORE_NEXT);} <WHILE> 
}
void begin_else_stmt(): {}
{ <ELSE> {this.addSpecialToken("else",STRATEGY_BEFORE_NEXT);} {this.addSpecialToken(":",STRATEGY_BEFORE_NEXT);} <COLON>  
}

//for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
void for_stmt(): {}
{   begin_for_stmt() exprlist() <IN> {this.addSpecialToken(" in ");} SmartTestList() <COLON> {this.addSpecialToken(":");} suite()
    [begin_for_else_stmt() suite()]
    
} 

void begin_for_stmt(): {}
{ <FOR> {this.addToPeek("for ",false);} 
}