lexer.java

RESIN 3.2 最新源码

/*
 * Copyright (c) 1998-2006 Caucho Technology -- all rights reserved
 *
 * This file is part of Resin(R) Open Source
 *
 * Each copy or derived work must preserve the copyright notice and this
 * notice unmodified.
 *
 * Resin Open Source is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * Resin Open Source is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, or any warranty
 * of NON-INFRINGEMENT.  See the GNU General Public License for more
 * details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Resin Open Source; if not, write to the
 *   Free SoftwareFoundation, Inc.
 *   59 Temple Place, Suite 330
 *   Boston, MA 02111-1307  USA
 *
 * @author Scott Ferguson
 */

package com.caucho.es.parser;

import com.caucho.es.*;
import com.caucho.java.LineMap;
import com.caucho.util.CharBuffer;
import com.caucho.util.L10N;
import com.caucho.vfs.ReadStream;

import java.io.CharConversionException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.regex.Pattern;

/**
 * JavaScript lexer.
 */
class Lexer {
  private static final L10N L = new L10N(Lexer.class);
  
  final static int ERROR = -3;
  final static int START = -2;
  final static int EOF = -1;

  final static int RESERVED = 256;
  final static int LITERAL = RESERVED + 1;
  final static int REGEXP = LITERAL + 1;
  final static int IDENTIFIER = REGEXP + 1;
  final static int THIS = IDENTIFIER + 1;

  final static int HASH_DEF = THIS + 1;
  final static int HASH_REF = HASH_DEF + 1;

  final static int BIN_OP = HASH_REF + 1;
  final static int UNARY_OP = BIN_OP + 1;
  final static int BANDU_OP = UNARY_OP + 1;

  final static int RSHIFT = BANDU_OP + 1;
  final static int URSHIFT = RSHIFT + 1;
  final static int LSHIFT = URSHIFT + 1;
  final static int BITAND = LSHIFT + 1;
  final static int BITOR = BITAND + 1;

  final static int GEQ = BITOR + 1;
  final static int LEQ = GEQ + 1;
  final static int EQ = LEQ + 1;
  final static int NEQ = EQ + 1;

  final static int STRICT_EQ = NEQ + 1;
  final static int STRICT_NEQ = STRICT_EQ + 1;

  final static int AND = STRICT_NEQ + 1;
  final static int OR = AND + 1;

  final static int ASSIGN_OP = OR + 1;

  final static int PREFIX = ASSIGN_OP + 1;
  final static int POSTFIX = PREFIX + 1;
  final static int DELETE = POSTFIX + 1;
  final static int VOID = DELETE + 1;
  final static int TYPEOF = VOID + 1;

  final static int IF = TYPEOF + 1;
  final static int ELSE = IF + 1;

  final static int SWITCH = ELSE + 1;
  final static int CASE = SWITCH + 1;
  final static int DEFAULT = CASE + 1;

  final static int WHILE = DEFAULT + 1;
  final static int DO = WHILE + 1;
  final static int FOR = DO + 1;
  final static int IN = FOR + 1;
  final static int BREAK = IN + 1;
  final static int CONTINUE = BREAK + 1;

  final static int FUNCTION = CONTINUE + 1;
  final static int CONSTRUCTOR = FUNCTION;
  final static int RETURN = CONSTRUCTOR + 1;

  final static int NEW = RETURN + 1;
  final static int VAR = NEW + 1;
  final static int WITH = VAR + 1;

  final static int NULL = WITH + 1;
  final static int UNDEFINED = NULL + 1;
  final static int TRUE = UNDEFINED + 1;
  final static int FALSE = TRUE + 1;
  final static int EVAL = FALSE + 1;

  final static int CLASS = EVAL + 1;
  final static int EXTENDS = CLASS + 1;

  final static int SYNCHRONIZED = EXTENDS + 1;

  final static int TRY = SYNCHRONIZED + 1;
  final static int CATCH = TRY + 1;
  final static int FINALLY = CATCH + 1;
  final static int THROW = FINALLY + 1;

  final static int IMPORT = THROW + 1;
  final static int STATIC = IMPORT + 1;
  
  final static int LAST_LEXEME = STATIC;

  static HashMap ops;
  static HashMap reserved;
  
  Global resin;
  ReadStream is;
  int peek = -1;
  int peek2 = -1;

  ArrayList macros = new ArrayList();

  CharBuffer macroText;
  int macroIndex;
  int macroOldLine;

  int _flags;

  int state;
  int lbrace;
  int stringClose;
  boolean isRegexp;

  LineMap lineMap;
  String filename;
  String lastFilename;
  String beginFilename;

  int lastLine;
  int beginLine;
  int beginLineCh;
  int line;
  int lineCh;

  Op op;
  int lexeme;
  int lastLexeme;
  CharBuffer text;
  CharBuffer lineText = new CharBuffer();
  boolean isEof = false;
  ESId id;
  ESBase literal;
  int intValue;
  boolean hasLf;
  boolean regexpOk;
  String writeln;

  CharBuffer temp = new CharBuffer();

  Lexer(ReadStream is, String filename, int line, LineMap lineMap)
  {
    this.filename = filename;
    this.line = line;
    this.lastFilename = filename;
    this.lastLine = line;
    this.lineMap = lineMap;
    this.is = is;
    peek = -1;
    peek2 = -1;
    text = new CharBuffer();
    lexeme = START;
    lastLexeme = START;
    regexpOk = true;
    macroText = null;
    macroIndex = 0;

    // Initialize the operator table
    if (ops == null) {
      ops = new HashMap();
      opsPut(".", '.', '.', Parser.PREC_DOT, false);
      opsPut("++", '+', POSTFIX, Parser.PREC_DOT, false);
      opsPut("--", '-', POSTFIX, Parser.PREC_DOT, false);

      opsPut("@", '@', '@', Parser.PREC_DOT, false);
      
      opsPut("~", '~', UNARY_OP, Parser.PREC_UMINUS, false);
      opsPut("!", '!', UNARY_OP, Parser.PREC_UMINUS, false);

      opsPut("*", '*', BIN_OP, Parser.PREC_TIMES, false);
      opsPut("/", '/', BIN_OP, Parser.PREC_TIMES, false);
      opsPut("%", '%', BIN_OP, Parser.PREC_TIMES, false);

      opsPut("+", '+', BANDU_OP, Parser.PREC_PLUS, false);
      opsPut("-", '-', BANDU_OP, Parser.PREC_PLUS, false);

      opsPut(">>", RSHIFT, BIN_OP, Parser.PREC_SHIFT, false);
      opsPut(">>>", URSHIFT, BIN_OP, Parser.PREC_SHIFT, false);
      opsPut("<<", LSHIFT, BIN_OP, Parser.PREC_SHIFT, false);

      opsPut(">", '>', BIN_OP, Parser.PREC_CMP, false);
      opsPut(">=", GEQ, BIN_OP, Parser.PREC_CMP, false);
      opsPut("<", '<', BIN_OP, Parser.PREC_CMP, false);
      opsPut("<=", LEQ, BIN_OP, Parser.PREC_CMP, false);
      opsPut("==", EQ, BIN_OP, Parser.PREC_CMP, false);
      opsPut("!=", NEQ, BIN_OP, Parser.PREC_CMP, false);
      opsPut("===", STRICT_EQ, BIN_OP, Parser.PREC_CMP, false);
      opsPut("!==", STRICT_NEQ, BIN_OP, Parser.PREC_CMP, false);

      opsPut("&", '&', BIN_OP, Parser.PREC_BITAND, false);
      opsPut("^", '^', BIN_OP, Parser.PREC_BITXOR, false);
      opsPut("|", '|', BIN_OP, Parser.PREC_BITOR, false);

      opsPut("&&", AND, BIN_OP, Parser.PREC_AND, false);
      opsPut("||", OR, BIN_OP, Parser.PREC_OR, false);

      opsPut("?", '?', '?', Parser.PREC_COND, false);

      opsPut("=", '=', '=', Parser.PREC_ASSIGN, true);
      opsPut("*=", '*', '=', Parser.PREC_ASSIGN, true);
      opsPut("/=", '/', '=', Parser.PREC_ASSIGN, true);
      opsPut("%=", '%', '=', Parser.PREC_ASSIGN, true);
      opsPut("+=", '+', '=', Parser.PREC_ASSIGN, true);
      opsPut("-=", '-', '=', Parser.PREC_ASSIGN, true);
      opsPut(">>=", RSHIFT, '=', Parser.PREC_ASSIGN, true);
      opsPut(">>>=", URSHIFT, '=', Parser.PREC_ASSIGN, true);
      opsPut("<<=", LSHIFT, '=', Parser.PREC_ASSIGN, true);
      opsPut("&=", '&', '=', Parser.PREC_ASSIGN, true);
      opsPut("^=", '^', '=', Parser.PREC_ASSIGN, true);
      opsPut("|=", '|', '=', Parser.PREC_ASSIGN, true);

      opsPut(",", ',', ',', Parser.PREC_COMMA, false);

      reserved = new HashMap();
      resPut("new", NEW);
      resPut("var", VAR);
      resPut("delete", DELETE);
      resPut("void", VOID);
      resPut("typeof", TYPEOF);

      resPut("if", IF);
      resPut("else", ELSE);
      resPut("switch", SWITCH);
      resPut("case", CASE);
      resPut("default", DEFAULT);

      resPut("while", WHILE);
      resPut("do", DO);
      resPut("for", FOR);
      resPut("in", IN);
      resPut("break", BREAK);
      resPut("continue", CONTINUE);

      resPut("null", NULL);
      resPut("undefined", UNDEFINED);
      resPut("true", TRUE);
      resPut("false", FALSE);
      resPut("this", THIS);
      resPut("eval", EVAL);

      resPut("function", FUNCTION);
      //resPut("constructor", CONSTRUCTOR);
      resPut("return", RETURN);

      resPut("with", WITH);

      resPut("class", CLASS);
      resPut("extends", EXTENDS);

      resPut("synchronized", SYNCHRONIZED);

      resPut("try", TRY);
      resPut("catch", CATCH);
      resPut("finally", FINALLY);
      resPut("throw", THROW);

      resPut("import", IMPORT);
      resPut("static", STATIC);
      
      resPut("const", RESERVED);
      resPut("debugger", RESERVED);
      resPut("enum", RESERVED);
      resPut("export", RESERVED);
      resPut("super", RESERVED);
/*
      resPut("boolean", RESERVED);
      resPut("byte", RESERVED);
      resPut("char", RESERVED);
      resPut("double", RESERVED);
      resPut("float", RESERVED);
      resPut("int", RESERVED);
      resPut("long", RESERVED);
      resPut("short", RESERVED);
*/    
      resPut("public", RESERVED);
      resPut("private", RESERVED);
      resPut("protected", RESERVED);
      resPut("throws", RESERVED);
    }
  }
  
  Lexer(ReadStream is, String filename, int line)
  {
    this(is, filename, line, null);
  }
  
  Lexer(ReadStream is, LineMap lineMap)
  {
    this(is, null, 1, lineMap);
  }

  void setLineMap(LineMap lineMap)
  {
    this.lineMap = lineMap;
  }

  private void opsPut(String name, int code, int lex, int prec, boolean flag)
  {
    ops.put(new CharBuffer(name), new Op(code, lex, prec, flag));
  }

  private void resPut(String name, int code)
  {
    reserved.put(new CharBuffer(name), new Integer(code));
  }

  int peek() throws ESParseException
  {
    try {
      if (lexeme == START) {
	lexeme = lex();
      }
      
      lastLexeme = lexeme;

      return lexeme;
    } catch (ESParseException e) {
      throw e;
    } catch (Exception e) {
      e.printStackTrace();
      throw error(e.toString());
    }
  }

  int next() throws ESParseException
  {
    try {
      int value = lexeme;

      if (value == START) {
	value = lex();
      }

      lastLexeme = value;
      lexeme = START;

      lastFilename = beginFilename;
      lastLine = beginLine;

      return value;
    } catch (ESParseException e) {
      throw e;
    } catch (Exception e) {
      e.printStackTrace();
      throw error(e == null ? "" : e.toString());
    }
  }

  int prev()
  {
    if (lastLexeme == START)
      throw new RuntimeException();

    lexeme = lastLexeme;

    lastLexeme = START;

    return lexeme;
  }

  int last()
  {
    if (lastLexeme == START)
      throw new RuntimeException();

    return lastLexeme;
  }

  private int peekCh() throws ESParseException
  {
    try {
      int ch = read();
      ungetc(ch);
      return (ch);
    } catch (Exception e) {
      return -1;
    }
  }

  /**
   * Returns the next lexeme
   */
  private int lex() throws ESParseException
  {
    lastFilename = beginFilename;
    lastLine = beginLine;
    
    hasLf = false;

    while (true) {
      beginFilename = filename;
      beginLine = line;
      beginLineCh = lineCh;

      int ch = read();

      switch (ch) {
      case -1:
	isEof = true;
	return EOF;

      case ' ': case '\t': case '\f': case 0x0b: /* vertical tab */
	break;
	
      case '\n': 
	newline();
	hasLf = true;
	break;

      case '+': case '-': case '*': case '!': case ',': case '^':
      case '<': case '>': case '&': case '|': case '=': case '~':
      case '?':
	regexpOk = true; // exception ++/--
	return lexOp(ch);

      case ')': case ']': 
	regexpOk = false;
	return ch;

      case ':': case ';': case '(': 
      case '[': case '{': case '}':
	regexpOk = true;
	return ch;

      case '.':
	{
	  int ch2 = read();

	  if (ch2 >= '0' && ch2 <= '9') {
	    regexpOk = false;
	    return lexFloat(0, ch2);
	  }
	  else {
	    regexpOk = true;
	    ungetc(ch2);
	    return lexOp(ch);
	  }
	}

      case '/':
	{
	  int ch2 = read();

	  if (ch2 == '/') {
	    for (ch2 = read(); 
		 ch2 > 0 && ch2 != '\n';
		 ch2 = read()) {
	    }

	    ungetc(ch2);
	    break;
	  }
	  else if (ch2 == '*') {
	    boolean seenStar = false;
	    for (ch2 = read(); 
		 ch2 > 0 && (! seenStar || ch2 != '/');
		 ch2 = read()) {
	      if (ch2 == '/') {
		ch2 = read();
		if (ch2 == '*')
		  throw error(L.l("comments can't nest"));
	      }

	      seenStar = ch2 == '*';

	      if (ch2 == '\n') {
		newline();
		hasLf = true;
              }
	    }
	    break;
	  }
	  else if (regexpOk) {
	    regexpOk = false;

	    ungetc(ch2);
	    lexString('/', null, true, false);

	    readRegexpFlags();
	    try {
	      Pattern regexp = Pattern.compile(literal.toString(), _flags);
	      // checking for errors
	    } catch (Exception e) {
	      // e.printStackTrace();
	      throw error(String.valueOf(e));
	    }

	    return REGEXP;
	  } else {
	    ungetc(ch2);
	    return lexOp(ch);
	  }
	}

      case '0': case '1': case '2': case '3': case '4': 
      case '5': case '6': case '7': case '8': case '9':
	regexpOk = false;
	return lexNumber(ch);

      case '"': case '\'':
	regexpOk = false;
	return lexString((char) ch, null, false, false);

      case '@':
	{
	  int ch2 = read();

	  switch (ch2) {
	  case '"':
	    CharBuffer macro = new CharBuffer();