scanners.scala

JAVA 语言的函数式编程扩展

          this copyFrom prev
        }
      } else if (token == IDENTIFIER && name == nme.MIXINkw) { //todo: remove eventually
        prev.copyFrom(this)
        fetchToken()
        if (token == CLASS)
          warning(prev.pos, "`mixin' is no longer a reserved word; you should use `trait' instead of `mixin class'");
        next.copyFrom(this)
        this.copyFrom(prev)
      }

      if (afterLineEnd() && inLastOfStat(lastToken) && inFirstOfStat(token) &&
          (sepRegions.isEmpty || sepRegions.head == RBRACE)) {
        next copyFrom this
        pos = in.lineStartPos
        token = if (in.lastBlankLinePos > lastPos) NEWLINES else NEWLINE
      }
    }

    private def afterLineEnd() = (
      lastPos < in.lineStartPos && 
      (in.lineStartPos <= pos ||
       lastPos < in.lastLineStartPos && in.lastLineStartPos <= pos)
    )

    /** read next token
     */
    private def fetchToken() {
      if (token == EOF) return
      lastPos = in.cpos - 1 // Position.encode(in.cline, in.ccol)
      //var index = bp
      while (true) {
        in.ch match {
          case ' ' | '\t' | CR | LF | FF =>
            in.next
          case _ =>
            pos = in.cpos // Position.encode(in.cline, in.ccol)
            in.ch match {
              case '\u21D2' => 
                in.next; token = ARROW
                return
              case 'A' | 'B' | 'C' | 'D' | 'E' |
                   'F' | 'G' | 'H' | 'I' | 'J' |
                   'K' | 'L' | 'M' | 'N' | 'O' |
                   'P' | 'Q' | 'R' | 'S' | 'T' |
                   'U' | 'V' | 'W' | 'X' | 'Y' |
                   'Z' | '$' | '_' |
                   'a' | 'b' | 'c' | 'd' | 'e' |
                   'f' | 'g' | 'h' | 'i' | 'j' |
                   'k' | 'l' | 'm' | 'n' | 'o' |
                   'p' | 'q' | 'r' | 's' | 't' |
                   'u' | 'v' | 'w' | 'x' | 'y' |  // scala-mode: need to understand multi-line case patterns
                   'z' =>
                putChar(in.ch)
                in.next
                getIdentRest  // scala-mode: wrong indent for multi-line case blocks
                return

            case '<' => // is XMLSTART?
              val last = in.last
              in.next
              last match {
              case ' '|'\t'|'\n'|'{'|'('|'>' if xml.Parsing.isNameStart(in.ch) || in.ch == '!' || in.ch == '?' =>
                token = XMLSTART
              case _ =>
                // Console.println("found '<', but last is '"+in.last+"'"); // DEBUG
                putChar('<')
                getOperatorRest
              }
              return

            case '~' | '!' | '@' | '#' | '%' |
                 '^' | '*' | '+' | '-' | /*'<' | */
                 '>' | '?' | ':' | '=' | '&' | 
                 '|' | '\\' =>
               putChar(in.ch)
               in.next
               getOperatorRest; // XXX
               return
              case '/' =>
                in.next
                if (!skipComment()) {
                  putChar('/')
                  getOperatorRest
                  return
                }
              case '0' =>
                putChar(in.ch)
                in.next
                if (in.ch == 'x' || in.ch == 'X') {
                  in.next
                  base = 16
                } else {
                  base = 8
                }
                getNumber
                return   
              case '1' | '2' | '3' | '4' |
                   '5' | '6' | '7' | '8' | '9' =>
                base = 10
                getNumber
                return
              case '`' => 
                in.next
                getStringLit('`', BACKQUOTED_IDENT)
                return
              case '\"' => 
                in.next
                if (in.ch == '\"') {
                  in.next
                  if (in.ch == '\"') {
                    in.next
                    val saved = in.lineStartPos
                    getMultiLineStringLit
                    if (in.lineStartPos != saved) // ignore linestarts within a mulit-line string 
                      in.lastLineStartPos = saved
                  } else {
                    token = STRINGLIT
                    name = nme.EMPTY
                  }
                } else {
                  getStringLit('\"', STRINGLIT)
                }
                return
              case '\'' =>
                in.next
                in.ch match {
                  case 'A' | 'B' | 'C' | 'D' | 'E' |
                       'F' | 'G' | 'H' | 'I' | 'J' |
                       'K' | 'L' | 'M' | 'N' | 'O' |
                       'P' | 'Q' | 'R' | 'S' | 'T' |
                       'U' | 'V' | 'W' | 'X' | 'Y' |
                       'Z' | '$' | '_' |
                       'a' | 'b' | 'c' | 'd' | 'e' |
                       'f' | 'g' | 'h' | 'i' | 'j' |
                       'k' | 'l' | 'm' | 'n' | 'o' |
                       'p' | 'q' | 'r' | 's' | 't' |
                       'u' | 'v' | 'w' | 'x' | 'y' |
                       'z' | 
                       '0' | '1' | '2' | '3' | '4' |
                       '5' | '6' | '7' | '8' | '9' =>
                    putChar(in.ch)
                    in.next
                    if (in.ch != '\'') {
                      getIdentRest
                      token = SYMBOLLIT
                      return
                    }
                  case _ =>
                    if (Character.isUnicodeIdentifierStart(in.ch)) {
                      putChar(in.ch)
                      in.next
                      if (in.ch != '\'') {
                        getIdentRest
                        token = SYMBOLLIT
                        return
                      }
                    } else if (isSpecial(in.ch)) {
                      putChar(in.ch)
                      in.next
                      if (in.ch != '\'') {
                        getOperatorRest
                        token = SYMBOLLIT
                        return
                      }
                    } else {
                      getlitch()
                    }
                }
                if (in.ch == '\'') {
                  in.next
                  token = CHARLIT
                  setName
                } else {
                  syntaxError("unclosed character literal")
                }
                return
              case '.' =>
                in.next
                if ('0' <= in.ch && in.ch <= '9') {
                  putChar('.'); getFraction
                } else {
                  token = DOT
                }
                return
              case ';' =>
                in.next; token = SEMI
                return
              case ',' =>
                in.next; token = COMMA
                return
              case '(' =>   //scala-mode: need to understand character quotes
                in.next; token = LPAREN
                return
              case '{' =>
                in.next; token = LBRACE
                return
              case ')' =>
                in.next; token = RPAREN
                return
              case '}' =>
                in.next; 
                token = RBRACE
                return
              case '[' =>
                in.next; token = LBRACKET
                return
              case ']' =>
                in.next; token = RBRACKET
                return
              case SU =>
                if (!in.hasNext) token = EOF
                else {
                  syntaxError("illegal character")
                  in.next
                }
                return
              case _ =>
                if (Character.isUnicodeIdentifierStart(in.ch)) {
                  putChar(in.ch)
                  in.next
                  getIdentRest
                } else if (isSpecial(in.ch)) {
                  putChar(in.ch)
                  getOperatorRest
                } else {
                  syntaxError("illegal character")
                  in.next
                }
                return
            }
        }
      }
    }

    private def skipComment(): Boolean = {
      if (in.ch == '/') {
        do {
          in.next
        } while ((in.ch != CR) && (in.ch != LF) && (in.ch != SU))
        true
      } else if (in.ch == '*') {
        docBuffer = null
        var openComments = 1
        in.next
        val scalaDoc = ("/**", "*/")
        if (in.ch == '*' && onlyPresentation)
          docBuffer = new StringBuilder(scalaDoc._1)
        while (openComments > 0) {
          do {
            do {
              if (in.ch == '/') {
                in.next; putDocChar(in.ch)
                if (in.ch == '*') {
                  in.next; putDocChar(in.ch)
                  openComments = openComments + 1
                }
              }
              if (in.ch != '*' && in.ch != SU) {
                in.next; putDocChar(in.ch)
              }
            } while (in.ch != '*' && in.ch != SU)
            while (in.ch == '*') {
              in.next; putDocChar(in.ch)
            }
          } while (in.ch != '/' && in.ch != SU)
          if (in.ch == '/') in.next
          else incompleteInputError("unclosed comment")
          openComments -= 1
        }
        true
      } else {
        false
      }
    }

    def inFirstOfStat(token: Int) = token match {
      case EOF | CASE | CATCH | ELSE | EXTENDS | FINALLY | FORSOME | MATCH |
           REQUIRES | WITH | YIELD | COMMA | SEMI | NEWLINE | NEWLINES | DOT |
           USCORE | COLON | EQUALS | ARROW | LARROW | SUBTYPE | VIEWBOUND |
           SUPERTYPE | HASH | RPAREN | RBRACKET | RBRACE => // todo: add LBRACKET
        false
      case _ =>
        true
    }

    def inLastOfStat(token: Int) = token match {
      case CHARLIT | INTLIT | LONGLIT | FLOATLIT | DOUBLELIT | STRINGLIT | SYMBOLLIT |
           IDENTIFIER | BACKQUOTED_IDENT | THIS | NULL | TRUE | FALSE | RETURN | USCORE | 
           TYPE | XMLSTART | RPAREN | RBRACKET | RBRACE =>
        true
      case _ =>
        false
    }

// Identifiers ---------------------------------------------------------------

    def isIdentStart(c: Char): Boolean = (
      ('A' <= c && c <= 'Z') ||
      ('a' <= c && c <= 'a') ||
      (c == '_') || (c == '$') ||
      Character.isUnicodeIdentifierStart(c)
    )

    def isIdentPart(c: Char) = (
      isIdentStart(c) || 
      ('0' <= c && c <= '9') ||
      Character.isUnicodeIdentifierPart(c)
    )

    def isSpecial(c: Char) = {
      val chtp = Character.getType(c)
      chtp == Character.MATH_SYMBOL || chtp == Character.OTHER_SYMBOL
    }

    private def getIdentRest {
      while (true) {
        in.ch match {
          case 'A' | 'B' | 'C' | 'D' | 'E' |
               'F' | 'G' | 'H' | 'I' | 'J' |
               'K' | 'L' | 'M' | 'N' | 'O' |
               'P' | 'Q' | 'R' | 'S' | 'T' |
               'U' | 'V' | 'W' | 'X' | 'Y' |
               'Z' | '$' |
               'a' | 'b' | 'c' | 'd' | 'e' |
               'f' | 'g' | 'h' | 'i' | 'j' |
               'k' | 'l' | 'm' | 'n' | 'o' |
               'p' | 'q' | 'r' | 's' | 't' |
               'u' | 'v' | 'w' | 'x' | 'y' |
               'z' |
               '0' | '1' | '2' | '3' | '4' |
               '5' | '6' | '7' | '8' | '9' =>
            putChar(in.ch)
            in.next
            
          case '_' =>
            putChar(in.ch)
            in.next
            getIdentOrOperatorRest
            return
          case SU =>
            setName
            token = ScannerConfiguration.name2token(name)
            return
          case _ =>
            if (Character.isUnicodeIdentifierPart(in.ch)) {