python.jjt

Python Development Environment (Python IDE plugin for Eclipse). Features editor, code completion, re

            }

            return false;
        } catch (TokenMgrError e1) {
            return false;
        }
    }

   // constructors taking a IParserHost impl

   public PythonGrammar24(CharStream stream,IParserHost host) {
       this(stream);
       hostLiteralMkr = host;
   }

   public PythonGrammar24(PythonGrammar24TokenManager tm,
                        IParserHost host)
   {
        this(tm);
        hostLiteralMkr = host;
   }

}


PARSER_END(PythonGrammar24)

TOKEN_MGR_DECLS:
{
    int indentation[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
    int level = 0;
    int dedents = 0;
    int parens = 0;
    int indent;

    boolean expect_indent = false;

    boolean compound = false;

    public boolean single_input = false;

    public List specialTokens = new ArrayList();

    // parsing of partial sentence (interactive) mode
    public boolean partial = false;
    // control whether empty new lines on EOF force sentence closing NEWLINE even if indent
    // is expected,i.e. classic behavior at jython prompt (different from codeop behavior)
    public boolean stdprompt = false;

    static Token addDedent(Token previous) {
        Token t = new Token();
        t.kind = DEDENT;
        t.beginLine = previous.beginLine;
        t.endLine = previous.endLine;
        t.beginColumn = previous.beginColumn;
        t.endColumn = previous.endColumn;
        t.image = "<DEDENT>";
        t.specialToken = null;
        t.next = null;
        previous.next = t;
        return t;
    }
    
    void CommonTokenAction(Token t) {
        /*
           if not partial: EOF is expanded to token sequences comprising
               if single_input: [NEWLINE] necessary DEDENT NEWLINE (afterward EOF)
               otherwise      : [NEWLINE] necessary DEDENT EOF
           if partial: EOF expansion happens only if EOF preceded by empty line (etc),
           i.e. lexer is in MAYBE_FORCE_NEWLINE_IF_EOF state
           System.out.println("Token:'"+t+"'");
           System.out.println("Special:'"+t.specialToken+"'");
        */

        int i = specialTokens.size();
        while(t.specialToken != null){
                this.specialTokens.add(i, t.specialToken);
                t = t.specialToken;
        }
        
        if (t.kind == EOF) {
            // System.out.println("EOF: "+single_input+", "+curLexState+", "+level);
            if (!partial || curLexState == MAYBE_FORCE_NEWLINE_IF_EOF) {
                if (curLexState == DEFAULT) {
                    t.kind = NEWLINE;
                }
                else {
                    t.kind = DEDENT;
                    if (level >= 0) level -= 1;
                }
                while (level >= 0) {
                    level--;
                    t = addDedent(t);
                }
                if (!single_input) {
                    t.kind = EOF;
                    t.image = "<EOF>";
                } else {
                    t.kind = NEWLINE;
                    t.image = "<FORCENL>";
                    single_input = false;
                }
            }
        } 
    }

    void indenting(int ind) {
        indent = ind;
        if (indent == indentation[level])
            SwitchTo(INDENTATION_UNCHANGED);
        else
            SwitchTo(INDENTING);
    }
}


SKIP :
{
    <SPACE: " ">
|   "\t"
|   "\014"
|   <CONTINUATION: ("\\") ("\r\n"|"\n"|"\r")>
|   <NEWLINE1: ("\r\n"|"\n"|"\r")>
        {
            if (parens == 0) {
                indent = 0;
                input_stream.backup(1);
                if (level == 0)
                    SwitchTo(FORCE_NEWLINE1);
                else
                    SwitchTo(FORCE_NEWLINE2);
            }
        }
}


<FORCE_NEWLINE1> TOKEN :
{ <NEWLINE: ("\n" | "\r")> : INDENTATION_UNCHANGED }


<FORCE_NEWLINE2> TOKEN :
{ <NEWLINE2: ("\n" | "\r")> { matchedToken.kind = NEWLINE; }: INDENTING }


// causes expected warning
<MAYBE_FORCE_NEWLINE_IF_EOF> SKIP :
{
    <""> { indenting(0); }
}

<INDENTING, INDENTATION_UNCHANGED> SKIP :
{
    "\t"
        { indenting((indent/8+1)*8); }
|   " "
        { indenting(indent+1); }
|   "\014"
        { indenting(0); }
|   <CRLF1: ("\r\n" | "\n" | "\r")>
        {
            //System.out.println("empty line");
            // if partial single_input (interactive) mode,
            // empty line (indent==0), and no parens open
            // or indentetion expected (if stdprompt == true, ovveride last cond)
            // consider forcing sentence closing NEWLINE if EOF
            if (partial && single_input && indent == 0 &&
                           parens == 0  && (stdprompt || !expect_indent)) {
                //System.out.println("force newline");
                //backup a character!
                // - input_stream.backup(1); -
                SwitchTo(MAYBE_FORCE_NEWLINE_IF_EOF);
            }
            else
                indenting(0);
        }
}


<INDENTATION_UNCHANGED> SKIP :
{
    <""> : DEFAULT
}


<INDENTING> TOKEN :
{
    <DEDENT: "">
        {
            if (indent > indentation[level]) {
                level++;
                indentation[level] = indent;
                matchedToken.kind=INDENT;
                matchedToken.image = "<INDENT>";
            }
            else if (level > 0) {
                Token t = matchedToken;
                level -= 1;
                while (level > 0 && indent < indentation[level]) {
                    level--;
                    t = addDedent(t);
                }
                if (indent != indentation[level]) {
                    throw new TokenMgrError("inconsistent dedent",
                                            t.endLine, t.endColumn);
                }
                t.next = null;
            }
        } : DEFAULT
}


<UNREACHABLE> TOKEN :
{
    < INDENT:"<INDENT>">
//|     < DEDENT:"<DEDENT>">
}


<DEFAULT> SPECIAL_TOKEN: /* COMMENTS 1*/
{
    <TRAILING_COMMENT: "#" (~["\n","\r"])* >{
//    System.out.println("TRAILING_COMMENT "+image);
//    matchedToken.image = image.toString();
    }
}

<INDENTING, INDENTATION_UNCHANGED> SPECIAL_TOKEN: /* COMMENTS 2*/
{
    <SINGLE_LINE_COMMENT: "#" (~["\n","\r"])* ("\r\n" | "\n" | "\r")> {
//    System.out.println("SINGLE_LINE_COMMENT "+image);
//    matchedToken.image = image.toString();
        indenting(0);
    }
}

TOKEN : /* SEPARATORS */
{
    < LPAREN: "(" > {parens++;}
|   < RPAREN: ")" > {parens--;}
|   < LBRACE: "{" > {parens++;}
|   < RBRACE: "}" > {parens--;}
|   < LBRACKET: "[" > {parens++;}
|   < RBRACKET: "]" > {parens--;}
|   < SEMICOLON: ";" >
|   < COMMA: "," >
|   < DOT: "." >
|   < COLON: ":" >
}


TOKEN : /* OPERATORS */
{
    < PLUS: "+" >
|   < MINUS: "-" >
|   < MULTIPLY: "*" >
|   < DIVIDE: "/" >
|   < FLOORDIVIDE: "//" >
|   < POWER: "**" >
|   < LSHIFT: "<<" >
|   < RSHIFT: ">>" >
|   < MODULO: "%" >
|   < NOT: "~" >
|   < XOR: "^" >
|   < OR: "|" >
|   < AND: "&" >
|   < EQUAL: "=" >
|   < GREATER: ">" >
|   < LESS: "<" >
|   < EQEQUAL: "==" >
|   < EQLESS: "<=" >
|   < EQGREATER: ">=" >
|   < LESSGREATER: "<>" >
|   < NOTEQUAL: "!=" >
|   < PLUSEQ: "+=" >
|   < MINUSEQ: "-=" >
|   < MULTIPLYEQ: "*=" >
|   < DIVIDEEQ: "/=" >
|   < FLOORDIVIDEEQ: "//=" >
|   < MODULOEQ: "%=" >
|   < ANDEQ: "&=" >
|   < OREQ: "|=" >
|   < XOREQ: "^=" >
|   < LSHIFTEQ: "<<=" >
|   < RSHIFTEQ: ">>=" >
|   < POWEREQ: "**=" >
}

TOKEN : /* KEYWORDS */
{
    < OR_BOOL: "or" >
|   < AND_BOOL: "and" >
|   < NOT_BOOL: "not" >
|   < IS: "is" >
|   < IN: "in" >
|   < LAMBDA: "lambda" >
|   < IF: "if" >
|   < ELSE: "else" >
|   < ELIF: "elif" >
|   < WHILE: "while" >
|   < FOR: "for" >
|   < TRY: "try" >
|   < EXCEPT: "except" >
|   < DEF: "def" >
|   < CLASS: "class" >
|   < FINALLY: "finally" >
|   < PRINT: "print" >
|   < PASS: "pass" >
|   < BREAK: "break" >
|   < CONTINUE: "continue" >
|   < RETURN: "return" >
|   < YIELD: "yield" >
|   < IMPORT: "import" >
|   < FROM: "from" >
|   < DEL: "del" >
|   < RAISE: "raise" >
|   < GLOBAL: "global" >
|   < EXEC: "exec" >
|   < ASSERT: "assert" >
|   < AS: "as" >
|   < AT: "@" >
}


TOKEN : /* Python identifiers */
{
    < NAME: <LETTER> ( <LETTER> | <DIGIT>)* >
|   < #LETTER: ["_","a"-"z","A"-"Z"] >
}


TOKEN : /* Numeric literals */
{
    < DECNUMBER:
        ["1"-"9"] (["0"-"9"])* (["l", "L"])?
      | "0"
      >
|   < HEXNUMBER: "0" ["x","X"] (["0"-"9","a"-"f","A"-"F"])+ (["l","L"])? >
|   < OCTNUMBER: "0" (["0"-"7"])* (["l","L"])? >
|
    < FLOAT:
        (["0"-"9"])+ "." (["0"-"9"])* (<EXPONENT>)?
      | "." (["0"-"9"])+ (<EXPONENT>)?
      | (["0"-"9"])+ <EXPONENT>
      >
|   < COMPLEX: (<DECNUMBER> | <FLOAT> | "0" <DECNUMBER> ) ["j", "J"]>
|   < #EXPONENT: ["e","E"] (["+","-"])? (["0"-"9"])+ >
|   < #DIGIT: ["0" - "9"] >
}


MORE : /* Strings */
{
    < (["u", "U"]) (["r", "R"])? "'" > :  IN_USTRING11
|   < (["u", "U"]) (["r", "R"])? "\"" > :  IN_USTRING21
|   < (["u", "U"]) (["r", "R"])? "'''" > :  IN_USTRING13
|   < (["u", "U"]) (["r", "R"])? "\"\"\"" > :  IN_USTRING23
|   < (["r", "R"])? "'" > :  IN_STRING11
|   < (["r", "R"])? "\"" > :  IN_STRING21
|   < (["r", "R"])? "'''" > :  IN_STRING13
|   < (["r", "R"])? "\"\"\"" > :  IN_STRING23
}

<IN_STRING11> TOKEN : { <SINGLE_STRING: "'"> {
    matchedToken.image = image.toString(); } : DEFAULT}
<IN_STRING21> TOKEN : { <SINGLE_STRING2: "\""> {
    matchedToken.image = image.toString(); } : DEFAULT}
<IN_STRING13> TOKEN : { <TRIPLE_STRING: "'''"> {
    matchedToken.image = image.toString(); } : DEFAULT}
<IN_STRING23> TOKEN : { <TRIPLE_STRING2: "\"\"\""> {
    matchedToken.image = image.toString(); } : DEFAULT}

<IN_USTRING11> TOKEN : { <SINGLE_USTRING: "'"> {
    matchedToken.image = image.toString(); } : DEFAULT}
<IN_USTRING21> TOKEN : { <SINGLE_USTRING2: "\""> {