scanners.scala

JAVA 语言的函数式编程扩展

/* NSC -- new Scala compiler
 * Copyright 2005-2008 LAMP/EPFL
 * @author  Martin Odersky
 */
// $Id: Scanners.scala 13738 2008-01-18 19:53:56Z michelou $

package scala.tools.nsc.ast.parser

import scala.tools.nsc.util._
import SourceFile.{LF, FF, CR, SU}
import Tokens._

trait Scanners {
  val global : Global
  import global._
  abstract class AbstractTokenData {
    def token: Int
    type ScanPosition
    val NoPos: ScanPosition
    def pos: ScanPosition
    def currentPos: ScanPosition 
    def name: Name
  }

  /** A class for representing a token's data. */
  trait TokenData extends AbstractTokenData {
    type ScanPosition = Int

    val NoPos: Int = -1
    /** the next token */
    var token: Int = EMPTY
    /** the token's position */
    var pos: Int = 0
    override def currentPos: Int = pos - 1

    /** the first character position after the previous token */
    var lastPos: Int = 0

    /** the name of an identifier or token */
    var name: Name = null

    /** the base of a number */
    var base: Int = 0

    def copyFrom(td: TokenData) = {
      this.token = td.token
      this.pos = td.pos
      this.lastPos = td.lastPos
      this.name = td.name
      this.base = td.base
    }
  }

  /** ...
   */
  abstract class AbstractScanner extends AbstractTokenData {
    implicit def p2g(pos: Position): ScanPosition
    implicit def g2p(pos: ScanPosition): Position
    def warning(pos: ScanPosition, msg: String): Unit
    def error  (pos: ScanPosition, msg: String): Unit
    def incompleteInputError(pos: ScanPosition, msg: String): Unit
    def deprecationWarning(pos: ScanPosition, msg: String): Unit
    /** the last error position
     */
    var errpos: ScanPosition 
    var lastPos: ScanPosition
    def skipToken: ScanPosition
    def nextToken: Unit
    def next: AbstractTokenData
    def intVal(negated: Boolean): Long
    def floatVal(negated: Boolean): Double
    def intVal: Long = intVal(false)
    def floatVal: Double = floatVal(false)
    //def token2string(token : Int) : String = configuration.token2string(token)
    /** return recent scala doc, if any */
    def flushDoc: String
  }

  object ScannerConfiguration {
//  Keywords -----------------------------------------------------------------
    /** Keyword array; maps from name indices to tokens */
    private var key: Array[Byte] = _
    private var maxKey = 0
    private var tokenName = new Array[Name](128)

    {
      var tokenCount = 0

      // Enter keywords

      def enterKeyword(n: Name, tokenId: Int) {
        while (tokenId >= tokenName.length) {
          val newTokName = new Array[Name](tokenName.length * 2)
          Array.copy(tokenName, 0, newTokName, 0, newTokName.length)
          tokenName = newTokName
        }
        tokenName(tokenId) = n
        if (n.start > maxKey) maxKey = n.start
        if (tokenId >= tokenCount) tokenCount = tokenId + 1
      }

      enterKeyword(nme.ABSTRACTkw, ABSTRACT)
      enterKeyword(nme.CASEkw, CASE)
      enterKeyword(nme.CATCHkw, CATCH)
      enterKeyword(nme.CLASSkw, CLASS)
      enterKeyword(nme.DEFkw, DEF)
      enterKeyword(nme.DOkw, DO)
      enterKeyword(nme.ELSEkw, ELSE)
      enterKeyword(nme.EXTENDSkw, EXTENDS)
      enterKeyword(nme.FALSEkw, FALSE)
      enterKeyword(nme.FINALkw, FINAL)
      enterKeyword(nme.FINALLYkw, FINALLY)
      enterKeyword(nme.FORkw, FOR)
      enterKeyword(nme.FORSOMEkw, FORSOME)
      enterKeyword(nme.IFkw, IF)
      enterKeyword(nme.IMPLICITkw, IMPLICIT)
      enterKeyword(nme.IMPORTkw, IMPORT)
      enterKeyword(nme.LAZYkw, LAZY)
      enterKeyword(nme.MATCHkw, MATCH)
      enterKeyword(nme.NEWkw, NEW)
      enterKeyword(nme.NULLkw, NULL)
      enterKeyword(nme.OBJECTkw, OBJECT)
      enterKeyword(nme.OVERRIDEkw, OVERRIDE)
      enterKeyword(nme.PACKAGEkw, PACKAGE)
      enterKeyword(nme.PRIVATEkw, PRIVATE)
      enterKeyword(nme.PROTECTEDkw, PROTECTED)
      enterKeyword(nme.REQUIRESkw, REQUIRES)
      enterKeyword(nme.RETURNkw, RETURN)
      enterKeyword(nme.SEALEDkw, SEALED)
      enterKeyword(nme.SUPERkw, SUPER)
      enterKeyword(nme.THISkw, THIS)
      enterKeyword(nme.THROWkw, THROW)
      enterKeyword(nme.TRAITkw, TRAIT)
      enterKeyword(nme.TRUEkw, TRUE)
      enterKeyword(nme.TRYkw, TRY)
      enterKeyword(nme.TYPEkw, TYPE)
      enterKeyword(nme.VALkw, VAL)
      enterKeyword(nme.VARkw, VAR)
      enterKeyword(nme.WHILEkw, WHILE)
      enterKeyword(nme.WITHkw, WITH)
      enterKeyword(nme.YIELDkw, YIELD)
      enterKeyword(nme.DOTkw, DOT)
      enterKeyword(nme.USCOREkw, USCORE)
      enterKeyword(nme.COLONkw, COLON)
      enterKeyword(nme.EQUALSkw, EQUALS)
      enterKeyword(nme.ARROWkw, ARROW)
      enterKeyword(nme.LARROWkw, LARROW)
      enterKeyword(nme.SUBTYPEkw, SUBTYPE)
      enterKeyword(nme.VIEWBOUNDkw, VIEWBOUND)
      enterKeyword(nme.SUPERTYPEkw, SUPERTYPE)
      enterKeyword(nme.HASHkw, HASH)
      enterKeyword(nme.ATkw, AT)

      // Build keyword array
      key = new Array[Byte](maxKey + 1)
      for (i <- 0 to maxKey)
        key(i) = IDENTIFIER
      for (j <- 0 until tokenCount)
        if (tokenName(j) ne null)
          key(tokenName(j).start) = j.asInstanceOf[Byte]

    }
//Token representation -----------------------------------------------------

  /** Convert name to token */
  def name2token(name: Name): Int =
    if (name.start <= maxKey) key(name.start) else IDENTIFIER

  /** Returns the string representation of given token. */
  def token2string(token: Int): String = token match {
    case IDENTIFIER | BACKQUOTED_IDENT =>
      "identifier"/* + \""+name+"\""*/
    case CHARLIT =>
      "character literal"
    case INTLIT =>
      "integer literal"
    case LONGLIT =>
      "long literal"
    case FLOATLIT =>
      "float literal"
    case DOUBLELIT =>
      "double literal"
    case STRINGLIT =>
      "string literal"
    case SYMBOLLIT =>
      "symbol literal"
    case LPAREN =>
      "'('"
    case RPAREN =>
      "')'"
    case LBRACE =>
      "'{'"
    case RBRACE =>
      "'}'"
    case LBRACKET =>
      "'['"
    case RBRACKET =>
      "']'"
    case EOF =>
      "eof"
    case ERROR =>
      "something"
    case SEMI =>
      "';'"
    case NEWLINE =>
      "';'"
    case NEWLINES =>
      "';'"
    case COMMA =>
      "','"
    case CASECLASS =>
      "case class"
    case CASEOBJECT =>
      "case object"
    case XMLSTART =>
      "$XMLSTART$<"
    case _ =>
      try {
        "'" + tokenName(token) + "'"
      } catch {
        case _: ArrayIndexOutOfBoundsException =>
          "'<" + token + ">'"
        case _: NullPointerException =>
          "'<(" + token + ")>'"
      }
    }
  }
  
  
  /** A scanner for the programming language Scala.
   *
   *  @author     Matthias Zenger, Martin Odersky, Burak Emir
   *  @version    1.1
   */
  abstract class Scanner extends AbstractScanner with TokenData {
    override def intVal = super.intVal
    override def floatVal = super.floatVal
    override var errpos: Int = NoPos

    val in: CharArrayReader

    /** character buffer for literals
     */  
    val cbuf = new StringBuilder()

    /** append Unicode character to "lit" buffer
    */
    protected def putChar(c: Char) { cbuf.append(c) }

    /** Clear buffer and set name */
    private def setName {
      name = newTermName(cbuf.toString())
      cbuf.setLength(0)
    }

    /** buffer for the documentation comment
     */
    var docBuffer: StringBuilder = null

    def flushDoc = {
      val ret = if (docBuffer != null) docBuffer.toString else null
      docBuffer = null
      ret
    }
  
    /** add the given character to the documentation buffer 
     */
    protected def putDocChar(c: Char) {
      if (docBuffer ne null) docBuffer.append(c)
    }

    private class TokenData0 extends TokenData

    /** we need one token lookahead
     */
    val next : TokenData = new TokenData0
    val prev : TokenData = new TokenData0

    /** a stack which indicates whether line-ends can be statement separators
     */
    var sepRegions: List[Int] = List()

    /** A new line was inserted where in version 1.0 it would not be.
     *  Only significant if settings.migrate.value is set
     */
    var newNewLine = false

    /** Parser is currently skipping ahead because of an error. 
     *  Only significant if settings.migrate.value is set
     */
    var skipping = false

// Get next token ------------------------------------------------------------

    /** read next token and return last position
     */
    def skipToken: Int = {
      val p = pos; nextToken
      // XXX: account for off by one error //???
      (p - 1)
    }
    
    def nextToken {
      if (token == LPAREN) {
        sepRegions = RPAREN :: sepRegions
      } else if (token == LBRACKET) {
        sepRegions = RBRACKET :: sepRegions
      } else if  (token == LBRACE) {
        sepRegions = RBRACE :: sepRegions
      } else if (token == CASE) {
        sepRegions = ARROW :: sepRegions
      } else if (token == RBRACE) {
        while (!sepRegions.isEmpty && sepRegions.head != RBRACE)
          sepRegions = sepRegions.tail
        if (!sepRegions.isEmpty)
          sepRegions = sepRegions.tail
      } else if (token == RBRACKET || token == RPAREN || token == ARROW) {
        if (!sepRegions.isEmpty && sepRegions.head == token)
          sepRegions = sepRegions.tail
      }

      val lastToken = token
      if (next.token == EMPTY) {
        fetchToken()
      } else {
        this copyFrom next
        next.token = EMPTY
      }
      if (token == CASE) {
        prev copyFrom this
        fetchToken()
        if (token == CLASS) {
          token = CASECLASS
          lastPos = prev.lastPos
        } else if (token == OBJECT) {
          token = CASEOBJECT
          lastPos = prev.lastPos
        } else {
          next copyFrom this
          this copyFrom prev
        }
      } else if (token == SEMI) {
        prev copyFrom this
        fetchToken()
        if (token != ELSE) {
          next copyFrom this