javascanner.java

MSP IM Source code 2008 update

  boolean readDigits(int radix) {
    if (start >= end)
      return false;
    char c = buffer[start];
    if (c == '\\')
      c = next();
    if (Character.digit(c, radix) == -1)
      return false;
    while (Character.digit(c, radix) != -1) {
      start = start + charlength;
      charlength = 1;
      if (start >= end)
        return true;
      c = buffer[start];
      if (c == '\\')
        c = next();
    }
    return true;
  }

  void readSuffix() {
    if (start >= end)
      return;
    char c = buffer[start];
    if (c == '\\')
      c = next();
    switch (c) {
    case 'f':
    case 'F':
    case 'd':
    case 'D':
    case 'l':
    case 'L':
      start = start + charlength;
      charlength = 1;
    }
  }

  private int readDot() {
    if (start >= end)
      return SEPARATOR;
    char c2 = buffer[start];
    if (c2 == '\\')
      c2 = next();
    if (Character.isDigit(c2)) {
      return readNumber('.');
    }
    if (start + 1 >= end || version < 15)
      return SEPARATOR;
    if (c2 != '.' || buffer[start + 1] != '.')
      return SEPARATOR;
    start = start + 2;
    return SEPARATOR;
  }

  private boolean readEscapeSequence() {
    if (start >= end)
      return false;
    char c2 = buffer[start];
    if (c2 == '\\')
      c2 = next();

    switch (c2) {
    case 'b':
    case 't':
    case 'n':
    case 'f':
    case 'r':
    case '\"':
    case '\'':
    case '\\':
      start = start + charlength;
      charlength = 1;
      return true;
    case '0':
    case '1':
    case '2':
    case '3':
      return readOctal(3);
    case '4':
    case '5':
    case '6':
    case '7':
      return readOctal(2);
    default:
      return false;
    }
  }

  boolean readOctal(int maxlength) {
    if (start >= end)
      return false;
    char c = buffer[start];
    if (c == '\\')
      c = next();

    int i, val = 0;
    for (i = 0; i < maxlength; i++) {
      if (Character.digit(c, 8) != -1) {
        val = 8 * val + Character.digit(c, 8);
        start = start + charlength;
        charlength = 1;
        if (start >= end)
          break;
        c = buffer[start];
        if (c == '\\')
          c = next();
      } else
        break;
    }
    if ((i == 0) || (val > 0xFF))
      return false;
    return true;
  }

  // A malformed or incomplete token has a negative type
  private int bad(int type) {
    return -type;
  }

  // Look ahead at the next character or unicode escape.
  // For efficiency, replace c = next(); with
  // c = buffer[start]; if (c == '\\') c = next();
  // To accept the character after looking at it, use:
  // start = start + charlength; charlength = 1;

  // Record the number of source code characters used up. To deal with an odd
  // or even number of backslashes preceding a unicode escape, whenever a
  // second backslash is coming up, mark its position as a pair.

  private int charlength = 1;

  private int pair = 0;

  private char next() {
    if (start >= end)
      return 26; // EOF
    char c = buffer[start];
    if (c != '\\')
      return c;
    if (start == pair) {
      pair = 0;
      return '\\';
    }
    if (start + 1 >= end)
      return '\\';

    c = buffer[start + 1];
    if (c == '\\')
      pair = start + 1;
    if (c != 'u')
      return '\\';

    int pos = start + 2;
    while (pos < end && buffer[pos] == 'u')
      pos++;
    if (pos + 4 > end) {
      charlength = end - start;
      return '\0';
    }

    c = 0;
    for (int j = 0; j < 4; j++) {
      int d = Character.digit(buffer[pos + j], 16);
      if (d < 0) {
        charlength = pos + j - start;
        return '\0';
      }
      c = (char) (c * 16 + d);
    }
    charlength = pos + 4 - start;
    return c;
  }

  // Override initSymbolTable

  protected void initSymbolTable() {
    lookup(KEYWORD, "abstract");
    if (version >= 14)
      lookup(KEYWORD, "assert");
    lookup(KEYWORD, "boolean");
    lookup(KEYWORD, "break");
    lookup(KEYWORD, "byte");
    lookup(KEYWORD, "case");
    lookup(KEYWORD, "catch");
    lookup(KEYWORD, "char");
    lookup(KEYWORD, "class");
    lookup(KEYWORD, "const");
    lookup(KEYWORD, "continue");
    lookup(KEYWORD, "default");
    lookup(KEYWORD, "do");
    lookup(KEYWORD, "double");
    lookup(KEYWORD, "else");
    if (version >= 15)
      lookup(KEYWORD, "enum");
    lookup(KEYWORD, "extends");
    lookup(KEYWORD, "final");
    lookup(KEYWORD, "finally");
    lookup(KEYWORD, "float");
    lookup(KEYWORD, "for");
    lookup(KEYWORD, "goto");
    lookup(KEYWORD, "if");
    lookup(KEYWORD, "implements");
    lookup(KEYWORD, "import");
    lookup(KEYWORD, "instanceof");
    lookup(KEYWORD, "int");
    lookup(KEYWORD, "interface");
    lookup(KEYWORD, "long");
    lookup(KEYWORD, "native");
    lookup(KEYWORD, "new");
    lookup(KEYWORD, "package");
    lookup(KEYWORD, "private");
    lookup(KEYWORD, "protected");
    lookup(KEYWORD, "public");
    lookup(KEYWORD, "return");
    lookup(KEYWORD, "short");
    lookup(KEYWORD, "static");
    if (version >= 12)
      lookup(KEYWORD, "strictfp");
    lookup(KEYWORD, "super");
    lookup(KEYWORD, "switch");
    lookup(KEYWORD, "synchronized");
    lookup(KEYWORD, "this");
    lookup(KEYWORD, "throw");
    lookup(KEYWORD, "throws");
    lookup(KEYWORD, "transient");
    lookup(KEYWORD, "try");
    lookup(KEYWORD, "void");
    lookup(KEYWORD, "volatile");
    lookup(KEYWORD, "while");

    lookup(LITERAL, "true");
    lookup(LITERAL, "false");
    lookup(LITERAL, "null");
  }

  // *** Override lookup, but what about unicode escape translation?

  private Symbol temp = new Symbol(0, null);

  protected Symbol lookup(int type, String name) {
    if (type != IDENTIFIER)
      return super.lookup(type, name);
    temp.type = KEYWORD;
    temp.name = name;
    Symbol sym = symbolTable.get(temp);
    if (sym != null)
      return sym;
    temp.type = LITERAL;
    sym = symbolTable.get(temp);
    if (sym != null)
      return sym;
    return super.lookup(type, name);
  }

  // Classify the ascii characters using an array of kinds, and classify all
  // other unicode characters using an array indexed by unicode category.
  // See the source file java/lang/Character.java for the categories.
  // To find the classification of a character, use:
  // if (c < 128) k = kind[c]; else k = unikind[Character.getType(c)];

  private static final byte[] kind = new byte[128];

  private static final byte[] unikind = new byte[31];

  // Initialise the two classification arrays using static initializer code.
  // Token types from the TokenTypes class are used to classify characters.

  private void initKind() {
    for (char c = 0; c < 128; c++)
      kind[c] = -1;
    for (char c = 0; c < 128; c++)
      switch (c) {
      case 0:
      case 1:
      case 2:
      case 3:
      case 4:
      case 5:
      case 6:
      case 7:
      case 8:
      case 11:
      case 13:
      case 14:
      case 15:
      case 16:
      case 17:
      case 18:
      case 19:
      case 20:
      case 21:
      case 22:
      case 23:
      case 24:
      case 25:
      case 27:
      case 28:
      case 29:
      case 30:
      case 31:
      case 127:
      case '#':
      case '@':
      case '`':
      case '\\':
        kind[c] = UNRECOGNIZED;
        break;
      case '\t':
      case '\n':
      case ' ':
      case '\f':
      case 26:
        kind[c] = WHITESPACE;
        break;
      case '!':
      case '%':
      case '&':
      case '*':
      case '+':
      case '-':
      case ':':
      case '<':
      case '=':
      case '>':
      case '?':
      case '^':
      case '|':
      case '~':
        kind[c] = OPERATOR;
        break;
      case '"':
        kind[c] = STRING;
        break;
      case '\'':
        kind[c] = CHARACTER;
        break;
      case '.':
        kind[c] = PUNCTUATION;
        break;
      case '/':
        kind[c] = COMMENT;
        break;
      case '$':
      case 'A':
      case 'B':
      case 'C':
      case 'D':
      case 'E':
      case 'F':
      case 'G':
      case 'H':
      case 'I':
      case 'J':
      case 'K':
      case 'L':
      case 'M':
      case 'N':
      case 'O':
      case 'P':
      case 'Q':
      case 'R':
      case 'S':
      case 'T':
      case 'U':
      case 'V':
      case 'W':
      case 'X':
      case 'Y':
      case 'Z':
      case '_':
      case 'a':
      case 'b':
      case 'c':
      case 'd':
      case 'e':
      case 'f':
      case 'g':
      case 'h':
      case 'i':
      case 'j':
      case 'k':
      case 'l':
      case 'm':
      case 'n':
      case 'o':
      case 'p':
      case 'q':
      case 'r':
      case 's':
      case 't':
      case 'u':
      case 'v':
      case 'w':
      case 'x':
      case 'y':
      case 'z':
        kind[c] = IDENTIFIER;
        break;
      case '0':
      case '1':
      case '2':
      case '3':
      case '4':
      case '5':
      case '6':
      case '7':
      case '8':
      case '9':
        kind[c] = NUMBER;
        break;
      case '(':
      case ')':
      case '[':
      case ']':
      case '{':
      case '}':
        kind[c] = BRACKET;
        break;
      case ',':
      case ';':
        kind[c] = SEPARATOR;
        break;
      }
    for (char c = 0; c < 128; c++)
      if (kind[c] == -1)
        System.out.println("Char " + ((int) c) + " hasn't been classified");
  }

  private void initUniKind() {
    for (byte b = 0; b < 31; b++)
      unikind[b] = -1;
    for (byte b = 0; b < 31; b++)
      switch (b) {
      case Character.UNASSIGNED:
      case Character.ENCLOSING_MARK:
      case Character.OTHER_NUMBER:
      case Character.SPACE_SEPARATOR:
      case Character.LINE_SEPARATOR:
      case Character.PARAGRAPH_SEPARATOR:
      case Character.CONTROL:
      case 17: // category 17 is unused
      case Character.PRIVATE_USE:
      case Character.SURROGATE:
      case Character.DASH_PUNCTUATION:
      case Character.START_PUNCTUATION:
      case Character.END_PUNCTUATION:
      case Character.OTHER_PUNCTUATION:
      case Character.MATH_SYMBOL:
      case Character.MODIFIER_SYMBOL:
      case Character.OTHER_SYMBOL:
      case Character.INITIAL_QUOTE_PUNCTUATION:
      case Character.FINAL_QUOTE_PUNCTUATION:
        unikind[b] = UNRECOGNIZED;
        break;
      case Character.UPPERCASE_LETTER:
      case Character.LOWERCASE_LETTER:
      case Character.TITLECASE_LETTER:
      case Character.MODIFIER_LETTER:
      case Character.OTHER_LETTER:
      case Character.LETTER_NUMBER:
      case Character.CONNECTOR_PUNCTUATION: // maybe NUMBER
      case Character.CURRENCY_SYMBOL:
        // Characters where Other_ID_Start is true
        unikind[b] = IDENTIFIER;
        break;
      case Character.NON_SPACING_MARK:
      case Character.COMBINING_SPACING_MARK:
      case Character.DECIMAL_DIGIT_NUMBER:
      case Character.FORMAT:
        unikind[b] = NUMBER;
        break;
      }
    for (byte b = 0; b < 31; b++)
      if (unikind[b] == -1)
        System.out.println("Unicode cat " + b + " hasn't been classified");
  }

}