pythongrammar25tokenmanager.java

Python Development Environment (Python IDE plugin for Eclipse). Features editor, code completion, re

/* Generated By:JJTree&JavaCC: Do not edit this line. PythonGrammar25TokenManager.java */
package org.python.pydev.parser.grammar25;
import org.python.pydev.parser.jython.ast.modType;
import java.util.List;
import java.util.ArrayList;
import java.util.Iterator;
import org.python.pydev.parser.jython.ast.Call;
import org.python.pydev.parser.jython.ast.Str;
import org.python.pydev.parser.jython.ast.Import;
import org.python.pydev.parser.jython.ast.Num;
import org.python.pydev.parser.jython.ast.Yield;
import org.python.pydev.parser.jython.ast.Tuple;
import org.python.pydev.parser.jython.ast.ImportFrom;
import org.python.pydev.parser.jython.ast.NameTok;
import org.python.pydev.core.IPythonNature;
import org.python.pydev.parser.jython.IParserHost;
import org.python.pydev.parser.jython.SimpleNode;
import org.python.pydev.parser.jython.Node;
import org.python.pydev.parser.jython.Token;
import org.python.pydev.parser.jython.ParseException;
import org.python.pydev.parser.jython.SpecialStr;
import org.python.pydev.parser.jython.TokenMgrError;
import org.python.pydev.parser.jython.CharStream;
import org.python.pydev.parser.IGrammar;

public class PythonGrammar25TokenManager implements PythonGrammar25Constants
{
    int indentation[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
    int level = 0;
    int dedents = 0;
    int parens = 0;
    int indent;

    boolean expect_indent = false;

    boolean compound = false;

    public boolean single_input = false;

    public List specialTokens = new ArrayList();

    // parsing of partial sentence (interactive) mode
    public boolean partial = false;
    // control whether empty new lines on EOF force sentence closing NEWLINE even if indent
    // is expected,i.e. classic behavior at jython prompt (different from codeop behavior)
    public boolean stdprompt = false;

    static Token addDedent(Token previous) {
        Token t = new Token();
        t.kind = DEDENT;
        t.beginLine = previous.beginLine;
        t.endLine = previous.endLine;
        t.beginColumn = previous.beginColumn;
        t.endColumn = previous.endColumn;
        t.image = "<DEDENT>";
        t.specialToken = null;
        t.next = null;
        previous.next = t;
        return t;
    }

    void CommonTokenAction(Token t) {
        /*
           if not partial: EOF is expanded to token sequences comprising
               if single_input: [NEWLINE] necessary DEDENT NEWLINE (afterward EOF)
               otherwise      : [NEWLINE] necessary DEDENT EOF
           if partial: EOF expansion happens only if EOF preceded by empty line (etc),
           i.e. lexer is in MAYBE_FORCE_NEWLINE_IF_EOF state
           System.out.println("Token:'"+t+"'");
           System.out.println("Special:'"+t.specialToken+"'");
        */

        int i = specialTokens.size();
        while(t.specialToken != null){
                this.specialTokens.add(i, t.specialToken);
                t = t.specialToken;
        }

        if (t.kind == EOF) {
            // System.out.println("EOF: "+single_input+", "+curLexState+", "+level);
            if (!partial || curLexState == MAYBE_FORCE_NEWLINE_IF_EOF) {
                if (curLexState == DEFAULT) {
                    t.kind = NEWLINE;
                }
                else {
                    t.kind = DEDENT;
                    if (level >= 0) level -= 1;
                }
                while (level >= 0) {
                    level--;
                    t = addDedent(t);
                }
                if (!single_input) {
                    t.kind = EOF;
                    t.image = "<EOF>";
                } else {
                    t.kind = NEWLINE;
                    t.image = "<FORCENL>";
                    single_input = false;
                }
            }
        }
    }

    void indenting(int ind) {
        indent = ind;
        if (indent == indentation[level])
            SwitchTo(INDENTATION_UNCHANGED);
        else
            SwitchTo(INDENTING);
    }
  public  java.io.PrintStream debugStream = System.out;
  public  void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
private final int jjStopStringLiteralDfa_4(int pos, long active0)
{
   switch (pos)
   {
      default :
         return -1;
   }
}
private final int jjStartNfa_4(int pos, long active0)
{
   return jjMoveNfa_4(jjStopStringLiteralDfa_4(pos, active0), pos + 1);
}
private final int jjStopAtPos(int pos, int kind)
{
   jjmatchedKind = kind;
   jjmatchedPos = pos;
   return pos + 1;
}
private final int jjStartNfaWithStates_4(int pos, int kind, int state)
{
   jjmatchedKind = kind;
   jjmatchedPos = pos;
   try { curChar = input_stream.readChar(); }
   catch(java.io.IOException e) { return pos + 1; }
   return jjMoveNfa_4(state, pos + 1);
}
private final int jjMoveStringLiteralDfa0_4()
{
   switch(curChar)
   {
      case 9:
         return jjStopAtPos(0, 9);
      case 12:
         return jjStopAtPos(0, 11);
      case 32:
         return jjStopAtPos(0, 10);
      default :
         return jjMoveNfa_4(1, 0);
   }
}
private final void jjCheckNAdd(int state)
{
   if (jjrounds[state] != jjround)
   {
      jjstateSet[jjnewStateCnt++] = state;
      jjrounds[state] = jjround;
   }
}
private final void jjAddStates(int start, int end)
{
   do {
      jjstateSet[jjnewStateCnt++] = jjnextStates[start];
   } while (start++ != end);
}
private final void jjCheckNAddTwoStates(int state1, int state2)
{
   jjCheckNAdd(state1);
   jjCheckNAdd(state2);
}
private final void jjCheckNAddStates(int start, int end)
{
   do {
      jjCheckNAdd(jjnextStates[start]);
   } while (start++ != end);
}
private final void jjCheckNAddStates(int start)
{
   jjCheckNAdd(jjnextStates[start]);
   jjCheckNAdd(jjnextStates[start + 1]);
}
static final long[] jjbitVec0 = {
   0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
};
static final long[] jjbitVec2 = {
   0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
};
private final int jjMoveNfa_4(int startState, int curPos)
{
   int[] nextStates;
   int startsAt = 0;
   jjnewStateCnt = 8;
   int i = 1;
   jjstateSet[0] = startState;
   int j, kind = 0x7fffffff;
   for (;;)
   {
      if (++jjround == 0x7fffffff)
         ReInitRounds();
      if (curChar < 64)
      {
         long l = 1L << curChar;
         MatchLoop: do
         {
            switch(jjstateSet[--i])
            {
               case 1:
                  if ((0x2400L & l) != 0L)
                  {
                     if (kind > 12)
                        kind = 12;
                  }
                  else if (curChar == 35)
                     jjCheckNAddStates(0, 2);
                  if (curChar == 13)
                     jjstateSet[jjnewStateCnt++] = 0;
                  break;
               case 0:
                  if (curChar == 10 && kind > 12)
                     kind = 12;
                  break;
               case 2:
                  if ((0x2400L & l) != 0L && kind > 12)
                     kind = 12;
                  break;
               case 3:
                  if (curChar == 35)
                     jjCheckNAddStates(0, 2);
                  break;
               case 4:
                  if ((0xffffffffffffdbffL & l) != 0L)
                     jjCheckNAddStates(0, 2);
                  break;
               case 5:
                  if (curChar == 10 && kind > 17)
                     kind = 17;
                  break;
               case 6:
                  if (curChar == 13)
                     jjstateSet[jjnewStateCnt++] = 5;
                  break;
               case 7:
                  if ((0x2400L & l) != 0L && kind > 17)
                     kind = 17;
                  break;
               default : break;
            }
         } while(i != startsAt);
      }
      else if (curChar < 128)
      {
         long l = 1L << (curChar & 077);
         MatchLoop: do
         {
            switch(jjstateSet[--i])
            {
               case 4:
                  jjAddStates(0, 2);
                  break;
               default : break;
            }
         } while(i != startsAt);
      }
      else
      {
         int hiByte = (int)(curChar >> 8);
         int i1 = hiByte >> 6;
         long l1 = 1L << (hiByte & 077);
         int i2 = (curChar & 0xff) >> 6;
         long l2 = 1L << (curChar & 077);
         MatchLoop: do
         {
            switch(jjstateSet[--i])
            {
               case 4:
                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                     jjAddStates(0, 2);
                  break;
               default : break;
            }
         } while(i != startsAt);
      }
      if (kind != 0x7fffffff)
      {
         jjmatchedKind = kind;
         jjmatchedPos = curPos;
         kind = 0x7fffffff;
      }
      ++curPos;
      if ((i = jjnewStateCnt) == (startsAt = 8 - (jjnewStateCnt = startsAt)))
         return curPos;
      try { curChar = input_stream.readChar(); }
      catch(java.io.IOException e) { return curPos; }
   }
}
private final int jjMoveStringLiteralDfa0_15()
{
   return 1;
}
private final int jjStopStringLiteralDfa_13(int pos, long active0, long active1, long active2)
{
   switch (pos)
   {
      case 0:
         if ((active1 & 0x10000000000000L) != 0L)
         {
            jjmatchedKind = 135;
            return -1;
         }
         return -1;
      case 1:
         if ((active1 & 0x10000000000000L) != 0L)
         {
            if (jjmatchedPos == 0)
            {
               jjmatchedKind = 135;
               jjmatchedPos = 0;
            }
            return -1;
         }
         return -1;
      default :
         return -1;
   }
}
private final int jjStartNfa_13(int pos, long active0, long active1, long active2)
{
   return jjMoveNfa_13(jjStopStringLiteralDfa_13(pos, active0, active1, active2), pos + 1);
}
private final int jjStartNfaWithStates_13(int pos, int kind, int state)
{
   jjmatchedKind = kind;
   jjmatchedPos = pos;
   try { curChar = input_stream.readChar(); }
   catch(java.io.IOException e) { return pos + 1; }
   return jjMoveNfa_13(state, pos + 1);
}
private final int jjMoveStringLiteralDfa0_13()
{
   switch(curChar)
   {
      case 10:
         return jjStopAtPos(0, 133);
      case 13:
         jjmatchedKind = 134;
         return jjMoveStringLiteralDfa1_13(0x0L, 0x10L);
      case 39:
         return jjMoveStringLiteralDfa1_13(0x10000000000000L, 0x0L);
      default :
         return jjMoveNfa_13(0, 0);
   }
}
private final int jjMoveStringLiteralDfa1_13(long active1, long active2)
{
   try { curChar = input_stream.readChar(); }
   catch(java.io.IOException e) {
      jjStopStringLiteralDfa_13(0, 0L, active1, active2);
      return 1;
   }
   switch(curChar)
   {
      case 10:
         if ((active2 & 0x10L) != 0L)
            return jjStopAtPos(1, 132);
         break;
      case 39:
         return jjMoveStringLiteralDfa2_13(active1, 0x10000000000000L, active2, 0L);
      default :
         break;
   }
   return jjStartNfa_13(0, 0L, active1, active2);
}
private final int jjMoveStringLiteralDfa2_13(long old1, long active1, long old2, long active2)
{
   if (((active1 &= old1) | (active2 &= old2)) == 0L)
      return jjStartNfa_13(0, 0L, old1, old2); 
   try { curChar = input_stream.readChar(); }
   catch(java.io.IOException e) {
      jjStopStringLiteralDfa_13(1, 0L, active1, 0L);
      return 2;
   }
   switch(curChar)
   {
      case 39:
         if ((active1 & 0x10000000000000L) != 0L)
            return jjStopAtPos(2, 116);
         break;
      default :
         break;
   }
   return jjStartNfa_13(1, 0L, active1, 0L);
}
private final int jjMoveNfa_13(int startState, int curPos)
{
   int[] nextStates;
   int startsAt = 0;
   jjnewStateCnt = 3;
   int i = 1;
   jjstateSet[0] = startState;
   int j, kind = 0x7fffffff;
   for (;;)
   {
      if (++jjround == 0x7fffffff)
         ReInitRounds();
      if (curChar < 64)
      {
         long l = 1L << curChar;
         MatchLoop: do
         {
            switch(jjstateSet[--i])
            {
               case 0:
                  if ((0xffffffffffffdbffL & l) != 0L && kind > 135)
                     kind = 135;
                  break;
               case 2:
                  if ((0xffffffffffffdbffL & l) != 0L && kind > 136)
                     kind = 136;
                  break;
               default : break;
            }
         } while(i != startsAt);
      }
      else if (curChar < 128)
      {
         long l = 1L << (curChar & 077);
         MatchLoop: do
         {
            switch(jjstateSet[--i])
            {
               case 0:
                  if (kind > 135)
                     kind = 135;
                  if (curChar == 92)
                     jjstateSet[jjnewStateCnt++] = 2;
                  break;
               case 1:
                  if (curChar == 92)
                     jjstateSet[jjnewStateCnt++] = 2;