newscanners.scala

来自「JAVA 语言的函数式编程扩展」· SCALA 代码 · 共 896 行 · 第 1/2 页

            in.scratch append escapeCode(in.offset - 1); true
          case c => in.scratch.append(c); true
          }) {}
          val ret = value(STRINGLIT, in.scratch.toString)
          in.scratch setLength 0
          ret
        }
      case '`' => 
        in.scratch setLength 0
        while (in.head match {
        case '`' => in.next; false
        case CR | LF | FF | SU | EOF => 
        in.error(offset, "unterminated quoted identifier")
        false
        case _ => true
        }) in.scratch append in.next
        val name = global.newTermName(in.scratch.toString)
        value(BACKQUOTED_IDENT, (name))
      case c if (c == CR | c == LF | c == FF) => 
        var multiple = false
        in.readWhile{
        case d if isNewLine(d) =>
          multiple = multiple || d == c; true
        case ' ' | '\t' => true // eat the whitespace after newlines.
        case _ => false
        }
        xmlOk = true
        (if (multiple) NEWLINES else NEWLINE)
      case c => 
        if (Character.isUnicodeIdentifierStart(c)) {
          in.scratch.setLength(0)
          in.scratch append c
          getIdentRest
          val name = global.newTermName(in.scratch.toString)
          in.scratch.setLength(0)
          val code = name2token(name)
          value(code, (name))
        } else if (isSpecial(c)) {
          in.scratch.setLength(0)
          in.scratch append c
          getOperatorRest
          val name = global.newTermName(in.scratch.toString)
          in.scratch.setLength(0)
          val code = name2token(name)
          value(code, (name))
        } else {
          in.error(offset, "illegal character: \'" + c + "\'")
          (ERROR)
        }
      }
      current.length = length
    }
    def intVal(offset : Int, token : Int, name0 : RandomAccessSeq[Char], negated: Boolean, base : Int): Long = {
      if (name0.length == 1 && name0(0) == '0') return 0

      var name = name0
      if (name.length > 2 && name(0) == '0' && (name(1) match {
        case 'x'|'X' => true
        case _ => false
      })) name = name.drop(2)

      while (name.last match {
        case 'l'|'L' => true
        case _ => false
      }) name = name.take(name.length - 1)
      
      if (token == CHARLIT && !negated) {
        if (name.length > 0) name(0) else 0
      } else {
        var value: Long = 0
        val divider = if (base == 10) 1 else 2
        val limit: Long =
          if (token == LONGLIT) Math.MAX_LONG else Math.MAX_INT
        var i = 0
        val len = name.length
        while (i < len) {
          val d = digit2int(name(i), base)
          if (d < 0) {
            in.error(offset, "malformed integer number")
            return 0
          }
          if (value < 0 ||
              limit / (base / divider) < value ||
              limit - (d / divider) < value * (base / divider) &&
              !(negated && limit == value * base - 1 + d)) {
                in.error(offset, "integer number too large")
                return 0
              }
          value = value * base + d
          i += 1
        }
        if (negated) -value else value
      }
    }


    /** convert name, base to double value
    */
    def floatVal(offset : Int, token : Int, name0 : RandomAccessSeq[Char], negated: Boolean): Double = {
      var name = name0
      while (name.last match {
          case 'f'|'F'|'d'|'D' => true
          case _ => false
      }) name = name.take(name.length - 1)

      val limit: Double = 
        if (token == DOUBLELIT) Math.MAX_DOUBLE else Math.MAX_FLOAT
      try {
        val value: Double = java.lang.Double.valueOf(name.mkString).doubleValue()
        if (value > limit)
          in.error(offset, "floating point number too large")
        if (negated) -value else value
      } catch {
        case _: NumberFormatException => 
          in.error(offset, "malformed floating point number")
          0.0
      }
    }
  }
  
  
  // utility functions 
  def isSpecial(c : Char) : Boolean = {
    val chtp = Character.getType(c)
    chtp == Character.MATH_SYMBOL || chtp == Character.OTHER_SYMBOL
  }
  def isDigit(c : Char) : Boolean = digit2int(c, 10) >= 0

  def isIdentifierStart(c: Char): Boolean = c match {
  case 'A' | 'B' | 'C' | 'D' | 'E' |
       'F' | 'G' | 'H' | 'I' | 'J' |
       'K' | 'L' | 'M' | 'N' | 'O' |
       'P' | 'Q' | 'R' | 'S' | 'T' |
       'U' | 'V' | 'W' | 'X' | 'Y' |
       'Z' | '$' | '_' |
       'a' | 'b' | 'c' | 'd' | 'e' |
       'f' | 'g' | 'h' | 'i' | 'j' |
       'k' | 'l' | 'm' | 'n' | 'o' |
       'p' | 'q' | 'r' | 's' | 't' |
       'u' | 'v' | 'w' | 'x' | 'y' |  // scala-mode: need to understand multi-line case patterns
       'z' => true
  case _ => false
  }
  def isIdentifierPart(c: Char) : Boolean = c match {
  case ('A' | 'B' | 'C' | 'D' | 'E' |
        'F' | 'G' | 'H' | 'I' | 'J' |
        'K' | 'L' | 'M' | 'N' | 'O' |
     'P' | 'Q' | 'R' | 'S' | 'T' |
     'U' | 'V' | 'W' | 'X' | 'Y' |
     'Z' | '$' | '_' |
     'a' | 'b' | 'c' | 'd' | 'e' |
     'f' | 'g' | 'h' | 'i' | 'j' |
     'k' | 'l' | 'm' | 'n' | 'o' |
     'p' | 'q' | 'r' | 's' | 't' |
     'u' | 'v' | 'w' | 'x' | 'y' |
     'z') => true
  case '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9' => true
  case c if Character.isUnicodeIdentifierPart(c) => true
  case _ => false
  }
  //isIdentifierStart(c) || isDigit(c) || isUnicodeIdentifierPart(c)
  def isOperatorPart(c : Char) : Boolean = c match {
  case '~' | '!' | '@' | '#' | '%' | 
       '^' | '*' | '+' | '-' | '<' |
       '>' | '?' | ':' | '=' | '&' | 
       '|' | '/' | '\\' => true
  case c if isSpecial(c) => true
  case _ => false
  }

  private def getOperatorRest(implicit in : CoreScannerInput) : Unit = {
    in.readWhile{
    case ('/') if in.startsWith('/','*') || in.startsWith('/','/') => false
    case ('*') if in.startsWith('*','/') => false
    case (c) if isOperatorPart(c) => in.scratch append c; true
    case _ => false
    }
  }
  private def isFraction(c0 : Char, c1 : Char) = 
    isDigit(c0) || (c0 match {
    case 'd'|'D'|'f'|'F' if !isIdentifierPart(c1) => true
    case 'e'|'E' if isDigit(c1) => true
    case _ => false  
    })
  private def getFraction(hasWhole : Boolean)(implicit in : CoreScannerInput) : Option[Int] = {
    val hasDigits = in.readWhile(isDigit)
    if (!hasDigits && !hasWhole) return None

    def end(code : Int) : Option[Int] = {
      if (!hasDigits && isIdentifierPart(in.peek(1))) None
      else in.next; Some(code) 
    }
    in.head match {
    case 'f'|'F' => return end(FLOATLIT)
    case 'd'|'D' => return end(DOUBLELIT)
    case 'e'|'E' if {
        val peek = in.peek(1)
        peek == '-' || peek == '+' || isDigit(peek)
    } =>
      in.next // eat the e.
      var hasDigit = isDigit(in.next) // eat +/-/digit 
      hasDigit = in.readWhile(isDigit) || hasDigit
      in.readIf{ // eat an optional f or d
      case 'f'|'F' => FLOATLIT
      case 'd'|'D' => DOUBLELIT
      } orElse Some(DOUBLELIT)
    case _ if hasDigits => Some(DOUBLELIT)
    case _ => None // we didn't read anything
    }
  }
  private def getNumber(offset : Int, base : Int, prefix : String)(implicit in : CoreScannerInput) : Int = {
    val hasBody = in.readWhile{
    case at if at >= '0' && at <= '9' => true
    case at if base == 16 && ((at >= 'a' && at <= 'f') || (at >= 'A' && at <= 'F')) => true
    case _ => false
    }
    if (!hasBody) base match {
      // because Java does this
    case 16 => 
      in.error(offset, "Invalid hex literal number")
      return INTLIT
    case _ => 
    }
    val code = if (in.head == '.') {
      in.peek(1) match {
      case c if isDigit(c) => in.next; getFraction(true).get
      case 'f'|'F'|'d'|'D' if !isIdentifierPart(in.peek(2)) => in.next; getFraction(true).get
      case 'e'|'E' if {
        val peek = in.peek(2)
        isDigit(peek) || peek == '-' || peek == '+'
      } => 
        in.next // consume the dot
        in.next // consume the e
        in.next // consume the +/-/digit
        in.readWhile(isDigit) // consume remaining digits
        in.readIf{
        case 'f'|'F' => FLOATLIT
        case 'd'|'D' => DOUBLELIT
        } getOrElse DOUBLELIT
      case c if isIdentifierStart(c) => INTLIT
      case _ => in.next; DOUBLELIT
      }
    } else (in.readIf{
    case 'l'|'L' => LONGLIT
    case 'f'|'F' => FLOATLIT
    case 'd'|'D' => DOUBLELIT
    } getOrElse {
      if (in.head == 'e' || in.head == 'E') {
        in.next
        if (in.head == '-' || in.head == '+') in.next
        in.readWhile(isDigit)
        in.readIf{
        case 'f'|'F' => FLOATLIT
        case 'd'|'D' => DOUBLELIT
        } getOrElse DOUBLELIT
      } else INTLIT
    })
    if (in.readWhile(isIdentifierPart))
      in.error(offset, "Invalid literal number")
    code
  }
  def inFirstOfStat(token: Int) = token match {
  case EOF | /*CASE |*/ CATCH | ELSE | EXTENDS | FINALLY | MATCH | REQUIRES | WITH | YIELD |
       COMMA | SEMI | NEWLINE | NEWLINES | DOT | USCORE | COLON | EQUALS | ARROW |
       LARROW | SUBTYPE | VIEWBOUND | SUPERTYPE | HASH | // todo: add LBRACKET
       RPAREN | RBRACKET | RBRACE => false
  case _ => true
  }
  def inLastOfStat(token: Int) = token match {
  case CHARLIT | INTLIT | LONGLIT | FLOATLIT | DOUBLELIT | STRINGLIT | SYMBOLLIT |
       IDENTIFIER | BACKQUOTED_IDENT | THIS | NULL | TRUE | FALSE | RETURN | USCORE | 
       TYPE | XMLSTART | RPAREN | RBRACKET | RBRACE => true
  case _ => false
  }
  
  def digit(c : Char, radix : Int) = c match {
  case c if c >= '0' && c <= '7' => c - '0'
  case c if c >= '8' && c <= '9' && radix >= 10 => c - '0'
  case c if c >= '8' && c <= '9' && radix == 8 => throw new NumberFormatException("Malformed octal number")
  case c if c >= 'a' && c <= 'f' && radix == 16 => c - 'a' + 9
  case c if c >= 'A' && c <= 'F' && radix == 16 => c - 'A' + 9
  }
  private val simpleEscape : PartialFunction[Char,Char] = {
  case 'b' => '\b'
  case 't' => '\t'
  case 'n' => '\n'
  case 'f' => '\f'
  case 'r' => '\r'
  case '\"' => '\"'
  case '\'' => '\''
  case '\\' => '\\'
  }

  def digit2int(ch: Char, base: Int): Int = {
    if ('0' <= ch && ch <= '9' && ch < '0' + base)
      ch - '0'
    else if ('A' <= ch && ch < 'A' + base - 10)
      ch - 'A' + 10
    else if ('a' <= ch && ch < 'a' + base - 10)
      ch - 'a' + 10
    else
      -1
  }
  
  object ScannerConfiguration {
    private var key: Array[Byte] = _
    private var maxKey = 0
    private var tokenName = new Array[global.Name](128);
    {
      var tokenCount = 0
      // Enter keywords
      def enterKeyword(n: global.Name, tokenId: Int) {
        while (tokenId >= tokenName.length) {
          val newTokName = new Array[global.Name](tokenName.length * 2)
          Array.copy(tokenName, 0, newTokName, 0, newTokName.length)
          tokenName = newTokName
        }
        tokenName(tokenId) = n
        if (n.start > maxKey) maxKey = n.start
        if (tokenId >= tokenCount) tokenCount = tokenId + 1
      }
      import global.nme

      enterKeyword(nme.ABSTRACTkw, ABSTRACT)
      enterKeyword(nme.CASEkw, CASE)
      enterKeyword(nme.CATCHkw, CATCH)
      enterKeyword(nme.CLASSkw, CLASS)
      enterKeyword(nme.DEFkw, DEF)
      enterKeyword(nme.DOkw, DO)
      enterKeyword(nme.ELSEkw, ELSE)
      enterKeyword(nme.EXTENDSkw, EXTENDS)
      enterKeyword(nme.FALSEkw, FALSE)
      enterKeyword(nme.FINALkw, FINAL)
      enterKeyword(nme.FINALLYkw, FINALLY)
      enterKeyword(nme.FORkw, FOR)
      enterKeyword(nme.FORSOMEkw, FORSOME)
      enterKeyword(nme.IFkw, IF)
      enterKeyword(nme.IMPLICITkw, IMPLICIT)
      enterKeyword(nme.IMPORTkw, IMPORT)
      enterKeyword(nme.LAZYkw, LAZY)
      enterKeyword(nme.MATCHkw, MATCH)
      enterKeyword(nme.NEWkw, NEW)
      enterKeyword(nme.NULLkw, NULL)
      enterKeyword(nme.OBJECTkw, OBJECT)
      enterKeyword(nme.OVERRIDEkw, OVERRIDE)
      enterKeyword(nme.PACKAGEkw, PACKAGE)
      enterKeyword(nme.PRIVATEkw, PRIVATE)
      enterKeyword(nme.PROTECTEDkw, PROTECTED)
      enterKeyword(nme.REQUIRESkw, REQUIRES)
      enterKeyword(nme.RETURNkw, RETURN)
      enterKeyword(nme.SEALEDkw, SEALED)
      enterKeyword(nme.SUPERkw, SUPER)
      enterKeyword(nme.THISkw, THIS)
      enterKeyword(nme.THROWkw, THROW)
      enterKeyword(nme.TRAITkw, TRAIT)
      enterKeyword(nme.TRUEkw, TRUE)
      enterKeyword(nme.TRYkw, TRY)
      enterKeyword(nme.TYPEkw, TYPE)
      enterKeyword(nme.VALkw, VAL)
      enterKeyword(nme.VARkw, VAR)
      enterKeyword(nme.WHILEkw, WHILE)
      enterKeyword(nme.WITHkw, WITH)
      enterKeyword(nme.YIELDkw, YIELD)
      enterKeyword(nme.DOTkw, DOT)
      enterKeyword(nme.USCOREkw, USCORE)
      enterKeyword(nme.COLONkw, COLON)
      enterKeyword(nme.EQUALSkw, EQUALS)
      enterKeyword(nme.ARROWkw, ARROW)
      enterKeyword(nme.LARROWkw, LARROW)
      enterKeyword(nme.SUBTYPEkw, SUBTYPE)
      enterKeyword(nme.VIEWBOUNDkw, VIEWBOUND)
      enterKeyword(nme.SUPERTYPEkw, SUPERTYPE)
      enterKeyword(nme.HASHkw, HASH)
      enterKeyword(nme.ATkw, AT)

      // Build keyword array
      key = new Array[Byte](maxKey + 1)
      for (i <- 0 to maxKey)
        key(i) = IDENTIFIER
      for (j <- 0 until tokenCount)
        if (tokenName(j) ne null)
          key(tokenName(j).start) = j.asInstanceOf[Byte]

    }
//Token representation -----------------------------------------------------

  /** Convert name to token */
  def name2token(name: global.Name): Int =
    if (name.start <= maxKey) key(name.start) else IDENTIFIER
  
  def isKeyword(code : Int) = code match {
  case code if code >= IF && code <= REQUIRES => true
  case _ => false
  }
  /** Returns the string representation of given token. */
  def token2string(token: Int): String = token match {
    case IDENTIFIER | BACKQUOTED_IDENT => "identifier"
    case CHARLIT => "character literal"
    case INTLIT => "integer literal"
    case LONGLIT => "long literal"
    case FLOATLIT => "float literal"
    case DOUBLELIT => "double literal"
    case STRINGLIT => "string literal"
    case SYMBOLLIT => "symbol literal"
    case LPAREN => "'('"
    case RPAREN => "')'"
    case LBRACE => "'{'"
    case RBRACE => "'}'"
    case LBRACKET => "'['"
    case RBRACKET => "']'"
    case EOF => "eof"
    case ERROR => "something"
    case SEMI => "';'"
    case NEWLINE => "';'"
    case NEWLINES => "';'"
    case COMMA => "','"
    case CASECLASS =>
      "case class"
    case CASEOBJECT =>
      "case object"
    case XMLSTART => 
      "$XMLSTART$<"
    case COMMENT => "cmnt"
    case WHITESPACE => "ws"
    case IGNORE => "ig"
    case _ =>
      try {
        "'" + tokenName(token) + "'"
      } catch {
        case _: ArrayIndexOutOfBoundsException =>
          "'<" + token + ">'"
        case _: NullPointerException =>
          "'<(" + token + ")>'"
      }
    }
  } 
  class UnitScanner(unit: CompilationUnit) extends ParserScanner {
    implicit val in = 
      new DefaultInput(new NewCharArrayReader(unit.source.content, !settings.nouescape.value, error)) {
        override def error(offset : Int, msg : String) : Unit = UnitScanner.this.error(offset, msg)
        override def incompleteError(offset : Int, msg : String) = 
          unit.incompleteInputError(new OffsetPosition(unit.source, offset), msg)
      }
    init
    private def error(offset : Int, msg : String) : Unit = unit.error(new OffsetPosition(unit.source,offset), msg)
  }
}