lexer.java

来自「有关编译器的编译器.」· Java 代码 · 共 1,040 行 · 第 1/2 页

              Token token = new29(
                              getText(accept_length),
                              start_line + 1,
                              start_pos + 1);
              pushBack(accept_length);
              pos = accept_pos;
              line = accept_line;
              return token;
            }
          case 30:
            {
              Token token = new30(
                              getText(accept_length),
                              start_line + 1,
                              start_pos + 1);
              pushBack(accept_length);
              pos = accept_pos;
              line = accept_line;
              return token;
            }
          case 31:
            {
              Token token = new31(
                              getText(accept_length),
                              start_line + 1,
                              start_pos + 1);
              pushBack(accept_length);
              pos = accept_pos;
              line = accept_line;
              return token;
            }
          case 32:
            {
              Token token = new32(
                              getText(accept_length),
                              start_line + 1,
                              start_pos + 1);
              pushBack(accept_length);
              pos = accept_pos;
              line = accept_line;
              return token;
            }
          case 33:
            {
              Token token = new33(
                              getText(accept_length),
                              start_line + 1,
                              start_pos + 1);
              pushBack(accept_length);
              pos = accept_pos;
              line = accept_line;
              return token;
            }
          case 34:
            {
              Token token = new34(
                              getText(accept_length),
                              start_line + 1,
                              start_pos + 1);
              pushBack(accept_length);
              pos = accept_pos;
              line = accept_line;
              return token;
            }
          }
        }
        else
        {
          if(text.length() > 0)
          {
            throw new LexerException(
              "[" + (start_line + 1) + "," + (start_pos + 1) + "]" +
              " Unknown token: " + text);
          }
          else
          {
            EOF token = new EOF(
                          start_line + 1,
                          start_pos + 1);
            return token;
          }
        }
      }
    }
  }

  Token new0(String text, int line, int pos)
  {
    return new TPkgId(text, line, pos);
  }
  Token new1(int line, int pos)
  {
    return new TPackage(line, pos);
  }
  Token new2(int line, int pos)
  {
    return new TStates(line, pos);
  }
  Token new3(int line, int pos)
  {
    return new THelpers(line, pos);
  }
  Token new4(int line, int pos)
  {
    return new TTokens(line, pos);
  }
  Token new5(int line, int pos)
  {
    return new TIgnored(line, pos);
  }
  Token new6(int line, int pos)
  {
    return new TProductions(line, pos);
  }
  Token new7(int line, int pos)
  {
    return new TTokenSpecifier(line, pos);
  }
  Token new8(int line, int pos)
  {
    return new TProductionSpecifier(line, pos);
  }
  Token new9(int line, int pos)
  {
    return new TDot(line, pos);
  }
  Token new10(int line, int pos)
  {
    return new TDDot(line, pos);
  }
  Token new11(int line, int pos)
  {
    return new TSemicolon(line, pos);
  }
  Token new12(int line, int pos)
  {
    return new TEqual(line, pos);
  }
  Token new13(int line, int pos)
  {
    return new TLBkt(line, pos);
  }
  Token new14(int line, int pos)
  {
    return new TRBkt(line, pos);
  }
  Token new15(int line, int pos)
  {
    return new TLPar(line, pos);
  }
  Token new16(int line, int pos)
  {
    return new TRPar(line, pos);
  }
  Token new17(int line, int pos)
  {
    return new TLBrace(line, pos);
  }
  Token new18(int line, int pos)
  {
    return new TRBrace(line, pos);
  }
  Token new19(int line, int pos)
  {
    return new TPlus(line, pos);
  }
  Token new20(int line, int pos)
  {
    return new TMinus(line, pos);
  }
  Token new21(int line, int pos)
  {
    return new TQMark(line, pos);
  }
  Token new22(int line, int pos)
  {
    return new TStar(line, pos);
  }
  Token new23(int line, int pos)
  {
    return new TBar(line, pos);
  }
  Token new24(int line, int pos)
  {
    return new TComma(line, pos);
  }
  Token new25(int line, int pos)
  {
    return new TSlash(line, pos);
  }
  Token new26(int line, int pos)
  {
    return new TArrow(line, pos);
  }
  Token new27(int line, int pos)
  {
    return new TColon(line, pos);
  }
  Token new28(String text, int line, int pos)
  {
    return new TId(text, line, pos);
  }
  Token new29(String text, int line, int pos)
  {
    return new TChar(text, line, pos);
  }
  Token new30(String text, int line, int pos)
  {
    return new TDecChar(text, line, pos);
  }
  Token new31(String text, int line, int pos)
  {
    return new THexChar(text, line, pos);
  }
  Token new32(String text, int line, int pos)
  {
    return new TString(text, line, pos);
  }
  Token new33(String text, int line, int pos)
  {
    return new TBlank(text, line, pos);
  }
  Token new34(String text, int line, int pos)
  {
    return new TComment(text, line, pos);
  }

  private int getChar() throws IOException
  {
    if(eof)
    {
      return -1;
    }

    int result = in.read();

    if(result == -1)
    {
      eof = true;
    }

    return result;
  }

  private void pushBack(int acceptLength) throws IOException
  {
    int length = text.length();
    for(int i = length - 1; i >= acceptLength; i--)
    {
      eof = false;

      in.unread(text.charAt(i));
    }
  }

  protected void unread(Token token) throws IOException
  {
    String text = token.getText();
    int length = text.length();

    for(int i = length - 1; i >= 0; i--)
    {
      eof = false;

      in.unread(text.charAt(i));
    }

    pos = token.getPos() - 1;
    line = token.getLine() - 1;
  }

  private String getText(int acceptLength)
  {
    StringBuffer s = new StringBuffer(acceptLength);
    for(int i = 0; i < acceptLength; i++)
    {
      s.append(text.charAt(i));
    }

    return s.toString();
  }

  private static int[][][][] gotoTable;
  /*  {
          { // NORMAL
              {{4, 4, 1}, {9, 9, 2}, {10, 10, 3}, {13, 13, 4}, {32, 32, 5}, {39, 39, 6}, {40, 40, 7}, {41, 41, 8}, {42, 42, 9}, {43, 43, 10}, {44, 44, 11}, {45, 45, 12}, {46, 46, 13}, {47, 47, 14}, {48, 48, 15}, {49, 57, 16}, {58, 58, 17}, {59, 59, 18}, {61, 61, 19}, {63, 63, 20}, {72, 72, 21}, {73, 73, 22}, {80, 80, 23}, {83, 83, 24}, {84, 84, 25}, {91, 91, 26}, {93, 93, 27}, {97, 122, 28}, {123, 123, 29}, {124, 124, 30}, {125, 125, 31}, },
              {{4, 32, -2}, },
              {{4, 32, -2}, },
              {{4, 32, -2}, },
              {{4, 9, -2}, {10, 10, 32}, {13, 32, -2}, },
              {{4, 32, -2}, },
              {{0, 9, 33}, {11, 12, 33}, {14, 38, 33}, {39, 39, 34}, {40, 65535, 33}, },
              {},
              {},
              {},
              {},
              {},
              {{62, 62, 35}, },
              {{46, 46, 36}, },
              {{42, 42, 37}, {47, 47, 38}, },
              {{48, 57, 16}, {88, 88, 39}, {120, 120, 40}, },
              {{48, 57, 16}, },
              {},
              {},
              {},
              {},
              {{101, 101, 41}, },
              {{103, 103, 42}, },
              {{97, 97, 43}, {114, 114, 44}, },
              {{116, 116, 45}, },
              {{111, 111, 46}, },
              {},
              {},
              {{48, 57, 47}, {95, 95, 48}, {97, 122, 49}, },
              {},
              {},
              {},
              {{4, 32, -2}, },
              {{0, 9, 50}, {11, 12, 50}, {14, 38, 50}, {39, 39, 51}, {40, 65535, 50}, },
              {{39, 39, 52}, },
              {},
              {},
              {{0, 41, 53}, {42, 42, 54}, {43, 65535, 53}, },
              {{0, 9, 55}, {10, 10, 56}, {11, 12, 55}, {13, 13, 57}, {14, 65535, 55}, },
              {{48, 57, 58}, {65, 70, 58}, {97, 102, 58}, },
              {{48, 102, -41}, },
              {{108, 108, 59}, },
              {{110, 110, 60}, },
              {{99, 99, 61}, },
              {{111, 111, 62}, },
              {{97, 97, 63}, },
              {{107, 107, 64}, },
              {{48, 122, -30}, },
              {{97, 122, 65}, },
              {{48, 122, -30}, },
              {{0, 38, -35}, {39, 39, 66}, {40, 65535, 50}, },
              {},
              {},
              {{0, 65535, -39}, },
              {{0, 41, 67}, {42, 42, 54}, {43, 46, 67}, {47, 47, 68}, {48, 65535, 67}, },
              {{0, 65535, -40}, },
              {},
              {{10, 10, 69}, },
              {{48, 102, -41}, },
              {{112, 112, 70}, },
              {{111, 111, 71}, },
              {{107, 107, 72}, },
              {{100, 100, 73}, },
              {{116, 116, 74}, },
              {{101, 101, 75}, },
              {{48, 57, 76}, {95, 95, 48}, {97, 122, 77}, },
              {},
              {{0, 41, 78}, {42, 42, 79}, {43, 65535, 78}, },
              {},
              {},
              {{101, 101, 80}, },
              {{114, 114, 81}, },
              {{97, 97, 82}, },
              {{117, 117, 83}, },
              {{101, 101, 84}, },
              {{110, 110, 85}, },
              {{48, 122, -67}, },
              {{48, 122, -67}, },
              {{0, 65535, -69}, },
              {{0, 41, 67}, {42, 42, 79}, {43, 65535, -56}, },
              {{114, 114, 86}, },
              {{101, 101, 87}, },
              {{103, 103, 88}, },
              {{99, 99, 89}, },
              {{115, 115, 90}, },
              {{115, 115, 91}, },
              {{115, 115, 92}, },
              {{100, 100, 93}, },
              {{101, 101, 94}, },
              {{116, 116, 95}, },
              {},
              {},
              {},
              {},
              {},
              {{105, 105, 96}, },
              {{111, 111, 97}, },
              {{110, 110, 98}, },
              {{115, 115, 99}, },
              {},
          }
          { // PACKAGE
              {{4, 4, 1}, {9, 9, 2}, {10, 10, 3}, {13, 13, 4}, {32, 32, 5}, {36, 36, 6}, {39, 39, 7}, {40, 40, 8}, {41, 41, 9}, {42, 42, 10}, {43, 43, 11}, {44, 44, 12}, {45, 45, 13}, {46, 46, 14}, {47, 47, 15}, {48, 48, 16}, {49, 57, 17}, {58, 58, 18}, {59, 59, 19}, {61, 61, 20}, {63, 63, 21}, {65, 71, 22}, {72, 72, 23}, {73, 73, 24}, {74, 79, 22}, {80, 80, 25}, {81, 82, 22}, {83, 83, 26}, {84, 84, 27}, {85, 90, 22}, {91, 91, 28}, {93, 93, 29}, {95, 95, 30}, {97, 122, 31}, {123, 123, 32}, {124, 124, 33}, {125, 125, 34}, },
              {{4, 32, -2}, },
              {{4, 32, -2}, },
              {{4, 32, -2}, },
              {{4, 9, -2}, {10, 10, 35}, {13, 32, -2}, },
              {{4, 32, -2}, },
              {{36, 36, 36}, {48, 57, 37}, {65, 90, 38}, {95, 95, 39}, {97, 122, 40}, },
              {{0, 9, 41}, {11, 12, 41}, {14, 38, 41}, {39, 39, 42}, {40, 65535, 41}, },
              {},
              {},
              {},
              {},
              {},
              {{62, 62, 43}, },
              {{46, 46, 44}, },
              {{42, 42, 45}, {47, 47, 46}, },
              {{48, 57, 17}, {88, 88, 47}, {120, 120, 48}, },
              {{48, 57, 17}, },
              {},
              {},
              {},
              {},
              {{36, 122, -8}, },
              {{36, 95, -8}, {97, 100, 40}, {101, 101, 49}, {102, 122, 40}, },
              {{36, 95, -8}, {97, 102, 40}, {103, 103, 50}, {104, 122, 40}, },
              {{36, 95, -8}, {97, 113, 40}, {114, 114, 51}, {115, 122, 40}, },
              {{36, 95, -8}, {97, 115, 40}, {116, 116, 52}, {117, 122, 40}, },
              {{36, 95, -8}, {97, 110, 40}, {111, 111, 53}, {112, 122, 40}, },
              {},
              {},
              {{36, 122, -8}, },
              {{36, 36, 36}, {48, 57, 54}, {65, 90, 38}, {95, 95, 55}, {97, 122, 56}, },
              {},
              {},
              {},
              {{4, 32, -2}, },
              {{36, 122, -8}, },
              {{36, 122, -8}, },
              {{36, 122, -8}, },
              {{36, 122, -8}, },
              {{36, 122, -8}, },
              {{0, 9, 57}, {11, 12, 57}, {14, 38, 57}, {39, 39, 58}, {40, 65535, 57}, },
              {{39, 39, 59}, },
              {},
              {},
              {{0, 41, 60}, {42, 42, 61}, {43, 65535, 60}, },
              {{0, 9, 62}, {10, 10, 63}, {11, 12, 62}, {13, 13, 64}, {14, 65535, 62}, },
              {{48, 57, 65}, {65, 70, 65}, {97, 102, 65}, },
              {{48, 102, -49}, },
              {{36, 95, -8}, {97, 107, 40}, {108, 108, 66}, {109, 122, 40}, },
              {{36, 95, -8}, {97, 109, 40}, {110, 110, 67}, {111, 122, 40}, },
              {{36, 110, -29}, {111, 111, 68}, {112, 122, 40}, },
              {{36, 95, -8}, {97, 97, 69}, {98, 122, 40}, },
              {{36, 95, -8}, {97, 106, 40}, {107, 107, 70}, {108, 122, 40}, },
              {{36, 122, -33}, },
              {{36, 95, -8}, {97, 122, 71}, },
              {{36, 122, -33}, },
              {{0, 38, -43}, {39, 39, 72}, {40, 65535, 57}, },
              {},
              {},
              {{0, 65535, -47}, },
              {{0, 41, 73}, {42, 42, 61}, {43, 46, 73}, {47, 47, 74}, {48, 65535, 73}, },
              {{0, 65535, -48}, },
              {},
              {{10, 10, 75}, },
              {{48, 102, -49}, },
              {{36, 95, -8}, {97, 111, 40}, {112, 112, 76}, {113, 122, 40}, },
              {{36, 110, -29}, {111, 111, 77}, {112, 122, 40}, },
              {{36, 95, -8}, {97, 99, 40}, {100, 100, 78}, {101, 122, 40}, },
              {{36, 115, -28}, {116, 116, 79}, {117, 122, 40}, },
              {{36, 100, -25}, {101, 101, 80}, {102, 122, 40}, },
              {{36, 36, 36}, {48, 57, 81}, {65, 95, -33}, {97, 122, 82}, },
              {},
              {{0, 41, 83}, {42, 42, 84}, {43, 65535, 83}, },
              {},
              {},
              {{36, 100, -25}, {101, 101, 85}, {102, 122, 40}, },
              {{36, 113, -27}, {114, 114, 86}, {115, 122, 40}, },
              {{36, 95, -8}, {97, 116, 40}, {117, 117, 87}, {118, 122, 40}, },
              {{36, 100, -25}, {101, 101, 88}, {102, 122, 40}, },
              {{36, 109, -52}, {110, 110, 89}, {111, 122, 40}, },
              {{36, 122, -73}, },
              {{36, 122, -73}, },
              {{0, 65535, -75}, },
              {{0, 41, 73}, {42, 42, 84}, {43, 65535, -63}, },
              {{36, 113, -27}, {114, 114, 90}, {115, 122, 40}, },
              {{36, 100, -25}, {101, 101, 91}, {102, 122, 40}, },
              {{36, 95, -8}, {97, 98, 40}, {99, 99, 92}, {100, 122, 40}, },
              {{36, 95, -8}, {97, 114, 40}, {115, 115, 93}, {116, 122, 40}, },
              {{36, 114, -90}, {115, 115, 94}, {116, 122, 40}, },
              {{36, 114, -90}, {115, 115, 95}, {116, 122, 40}, },
              {{36, 99, -70}, {100, 100, 96}, {101, 122, 40}, },
              {{36, 115, -28}, {116, 116, 97}, {117, 122, 40}, },
              {{36, 122, -8}, },
              {{36, 122, -8}, },
              {{36, 122, -8}, },
              {{36, 122, -8}, },
              {{36, 95, -8}, {97, 104, 40}, {105, 105, 98}, {106, 122, 40}, },
              {{36, 110, -29}, {111, 111, 99}, {112, 122, 40}, },
              {{36, 109, -52}, {110, 110, 100}, {111, 122, 40}, },
              {{36, 114, -90}, {115, 115, 101}, {116, 122, 40}, },
              {{36, 122, -8}, },
          }
      };*/

  private static int[][] accept;
  /*  {
          // NORMAL
          {-1, 33, 33, 33, 33, 33, -1, 15, 16, 22, 19, 24, 20, 9, 25, 30, 30, 27, 11, 12, 21, -1, -1, 8, -1, 7, 13, 14, 28, 17, 23, 18, 33, -1, -1, 26, 10, -1, 34, -1, -1, -1, -1, -1, -1, -1, -1, 28, -1, 28, -1, 29, 29, -1, -1, 34, 34, 34, 31, -1, -1, -1, -1, -1, -1, 28, 32, -1, 34, 34, -1, -1, -1, -1, -1, -1, 28, 28, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 4, 3, 5, 1, -1, -1, -1, -1, 6, },
          // PACKAGE
          {-1, 33, 33, 33, 33, 33, 0, -1, 15, 16, 22, 19, 24, 20, 9, 25, 30, 30, 27, 11, 12, 21, 0, 0, 0, 0, 0, 0, 13, 14, 0, 0, 17, 23, 18, 33, 0, 0, 0, 0, 0, -1, -1, 26, 10, -1, 34, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, -1, 29, 29, -1, -1, 34, 34, 34, 31, 0, 0, 0, 0, 0, 0, 32, -1, 34, 34, 0, 0, 0, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, },
   
      };*/

  public static class State
  {
    public final static State NORMAL = new State(0);
    public final static State PACKAGE = new State(1);

    private int id;

    private State(int id)
    {
      this.id = id;
    }

    public int id()
    {
      return id;
    }
  }
}