groovylexer.java

来自「大名鼎鼎的java动态脚本语言。已经通过了sun的认证」· Java 代码 · 共 2,085 行 · 第 1/5 页

// $ANTLR 2.7.2: "groovy.g" -> "GroovyLexer.java"$

package org.codehaus.groovy.antlr.parser;
import org.codehaus.groovy.antlr.*;
import java.util.*;
import java.io.InputStream;
import java.io.Reader;
import antlr.InputBuffer;
import antlr.LexerSharedInputState;

import java.io.InputStream;
import antlr.TokenStreamException;
import antlr.TokenStreamIOException;
import antlr.TokenStreamRecognitionException;
import antlr.CharStreamException;
import antlr.CharStreamIOException;
import antlr.ANTLRException;
import java.io.Reader;
import java.util.Hashtable;
import antlr.CharScanner;
import antlr.InputBuffer;
import antlr.ByteBuffer;
import antlr.CharBuffer;
import antlr.Token;
import antlr.CommonToken;
import antlr.RecognitionException;
import antlr.NoViableAltForCharException;
import antlr.MismatchedCharException;
import antlr.TokenStream;
import antlr.ANTLRHashString;
import antlr.LexerSharedInputState;
import antlr.collections.impl.BitSet;
import antlr.SemanticException;

public class GroovyLexer extends antlr.CharScanner implements GroovyTokenTypes, TokenStream
 {

    /** flag for enabling the "assert" keyword */
    private boolean assertEnabled = true;
    /** flag for enabling the "enum" keyword */
    private boolean enumEnabled = true;
    /** flag for including whitespace tokens (for IDE preparsing) */
    private boolean whitespaceIncluded = false;

    /** Enable the "assert" keyword */
    public void enableAssert(boolean shouldEnable) { assertEnabled = shouldEnable; }
    /** Query the "assert" keyword state */
    public boolean isAssertEnabled() { return assertEnabled; }
    /** Enable the "enum" keyword */
    public void enableEnum(boolean shouldEnable) { enumEnabled = shouldEnable; }
    /** Query the "enum" keyword state */
    public boolean isEnumEnabled() { return enumEnabled; }

    /** Include whitespace tokens.  Note that this breaks the parser.   */
    public void setWhitespaceIncluded(boolean z) { whitespaceIncluded = z; }
    /** Are whitespace tokens included? */
    public boolean isWhitespaceIncluded() { return whitespaceIncluded; }

    {
        // Initialization actions performed on construction.
        setTabSize(1);  // get rid of special tab interpretation, for IDEs and general clarity
    }

    /** Bumped when inside '[x]' or '(x)', reset inside '{x}'.  See ONE_NL.  */
    protected int parenLevel = 0;
    protected int suppressNewline = 0;  // be really mean to newlines inside strings
    protected static final int SCS_TYPE = 3, SCS_VAL = 4, SCS_LIT = 8, SCS_LIMIT = 16;
    protected static final int SCS_SQ_TYPE = 0, SCS_TQ_TYPE = 1, SCS_RE_TYPE = 2;
    protected int stringCtorState = 0;  // hack string and regexp constructor boundaries
    /** Push parenLevel here and reset whenever inside '{x}'. */
    protected ArrayList parenLevelStack = new ArrayList();
    protected int lastSigTokenType = EOF;  // last returned non-whitespace token

    protected void pushParenLevel() {
        parenLevelStack.add(new Integer(parenLevel*SCS_LIMIT + stringCtorState));
        parenLevel = 0;
        stringCtorState = 0;
    }
    protected void popParenLevel() {
        int npl = parenLevelStack.size();
        if (npl == 0)  return;
        int i = ((Integer) parenLevelStack.remove(--npl)).intValue();
        parenLevel      = i / SCS_LIMIT;
        stringCtorState = i % SCS_LIMIT;
    }

    protected void restartStringCtor(boolean expectLiteral) {
        if (stringCtorState != 0) {
            stringCtorState = (expectLiteral? SCS_LIT: SCS_VAL) + (stringCtorState & SCS_TYPE);
        }
    }
    
    protected boolean allowRegexpLiteral() {
        return !isExpressionEndingToken(lastSigTokenType);
    }

    /** Return true for an operator or punctuation which can end an expression.
     *  Return true for keywords, identifiers, and literals.
     *  Return true for tokens which can end expressions (right brackets, ++, --).
     *  Return false for EOF and all other operator and punctuation tokens.
     *  Used to suppress the recognition of /foo/ as opposed to the simple division operator '/'.
     */
    // Cf. 'constant' and 'balancedBrackets' rules in the grammar.)
    protected static boolean isExpressionEndingToken(int ttype) {
        switch (ttype) {
        case INC:               // x++ / y
        case DEC:               // x-- / y
        case RPAREN:            // (x) / y
        case RBRACK:            // f[x] / y
        case RCURLY:            // f{x} / y
        case STRING_LITERAL:    // "x" / y
        case STRING_CTOR_END:   // "$x" / y
        case NUM_INT:           // 0 / y
        case NUM_FLOAT:         // 0f / y
        case NUM_LONG:          // 0l / y
        case NUM_DOUBLE:        // 0.0 / y
        case NUM_BIG_INT:       // 0g / y
        case NUM_BIG_DECIMAL:   // 0.0g / y
        case IDENT:             // x / y
        // and a bunch of keywords (all of them; no sense picking and choosing):
        case LITERAL_any:
        case LITERAL_as:
        case LITERAL_assert:
        case LITERAL_boolean:
        case LITERAL_break:
        case LITERAL_byte:
        case LITERAL_case:
        case LITERAL_catch:
        case LITERAL_char:
        case LITERAL_class:
        case LITERAL_continue:
        case LITERAL_def:
        case LITERAL_default:
        case LITERAL_double:
        case LITERAL_else:
        case LITERAL_enum:
        case LITERAL_extends:
        case LITERAL_false:
        case LITERAL_finally:
        case LITERAL_float:
        case LITERAL_for:
        case LITERAL_if:
        case LITERAL_implements:
        case LITERAL_import:
        case LITERAL_in:
        case LITERAL_instanceof:
        case LITERAL_int:
        case LITERAL_interface:
        case LITERAL_long:
        case LITERAL_native:
        case LITERAL_new:
        case LITERAL_null:
        case LITERAL_package:
        case LITERAL_private:
        case LITERAL_protected:
        case LITERAL_public:
        case LITERAL_return:
        case LITERAL_short:
        case LITERAL_static:
        case LITERAL_super:
        case LITERAL_switch:
        case LITERAL_synchronized:
        case LITERAL_this:
        case LITERAL_threadsafe:
        case LITERAL_throw:
        case LITERAL_throws:
        case LITERAL_transient:
        case LITERAL_true:
        case LITERAL_try:
        case LITERAL_void:
        case LITERAL_volatile:
        case LITERAL_while:
        case LITERAL_with:
            return true;
        default:
            return false;
        }
    }

    protected void newlineCheck(boolean check) throws RecognitionException {
        if (check && suppressNewline > 0) {
            require(suppressNewline == 0,
                "end of line reached within a simple string 'x' or \"x\" or /x/",
                "for multi-line literals, use triple quotes '''x''' or \"\"\"x\"\"\"");
            suppressNewline = 0;  // shut down any flood of errors
        }
        newline();
    }
    
    protected boolean atValidDollarEscape() throws CharStreamException {
        // '$' (('*')? ('{' | LETTER)) =>
        int k = 1;
        char lc = LA(k++);
        if (lc != '$')  return false;
        lc = LA(k++);
        if (lc == '*')  lc = LA(k++);
        return (lc == '{' || (lc != '$' && Character.isJavaIdentifierStart(lc)));
    }

    /** This is a bit of plumbing which resumes collection of string constructor bodies,
     *  after an embedded expression has been parsed.
     *  Usage:  new GroovyRecognizer(new GroovyLexer(in).plumb()).
     */
    public TokenStream plumb() {
        return new TokenStream() {
            public Token nextToken() throws TokenStreamException {
                if (stringCtorState >= SCS_LIT) {
                    // This goo is modeled upon the ANTLR code for nextToken:
                    int quoteType = (stringCtorState & SCS_TYPE);
                    stringCtorState = 0;  // get out of this mode, now
                    resetText();
                    try {
                        switch (quoteType) {
                        case SCS_SQ_TYPE:
                            mSTRING_CTOR_END(true, /*fromStart:*/false, false); break;
                        case SCS_TQ_TYPE:
                            mSTRING_CTOR_END(true, /*fromStart:*/false, true); break;
                        case SCS_RE_TYPE:
                            mREGEXP_CTOR_END(true, /*fromStart:*/false); break;
                        default:  throw new AssertionError(false);
                        }
                        lastSigTokenType = _returnToken.getType();
                        return _returnToken;
                    } catch (RecognitionException e) {
                        throw new TokenStreamRecognitionException(e);
                    } catch (CharStreamException cse) {
                        if ( cse instanceof CharStreamIOException ) {
                            throw new TokenStreamIOException(((CharStreamIOException)cse).io);
                        }
                        else {
                            throw new TokenStreamException(cse.getMessage());
                        }
                    }
                }
                Token token = GroovyLexer.this.nextToken();
                int lasttype = token.getType();
                if (whitespaceIncluded) {
                    switch (lasttype) {  // filter out insignificant types
                    case WS:
                    case ONE_NL:
                    case SL_COMMENT:
                    case ML_COMMENT:
                        lasttype = lastSigTokenType;  // back up!
                    }
                }
                lastSigTokenType = lasttype;
                return token;
            }
        };
    }

        // stuff to adjust ANTLR's tracing machinery
    public static boolean tracing = false;  // only effective if antlr.Tool is run with -traceLexer
    public void traceIn(String rname) throws CharStreamException {
        if (!GroovyLexer.tracing)  return;
        super.traceIn(rname);
    }
    public void traceOut(String rname) throws CharStreamException {
        if (!GroovyLexer.tracing)  return;
        if (_returnToken != null)  rname += tokenStringOf(_returnToken);
        super.traceOut(rname);
    }
    private static java.util.HashMap ttypes;
    private static String tokenStringOf(Token t) {
        if (ttypes == null) {
            java.util.HashMap map = new java.util.HashMap();
            java.lang.reflect.Field[] fields = GroovyTokenTypes.class.getDeclaredFields();
            for (int i = 0; i < fields.length; i++) {
                if (fields[i].getType() != int.class)  continue;
                try {
                    map.put(fields[i].get(null), fields[i].getName());
                } catch (IllegalAccessException ee) {
                }
            }
            ttypes = map;
        }
        Integer tt = new Integer(t.getType());
        Object ttn = ttypes.get(tt);
        if (ttn == null)  ttn = "<"+tt+">";
        return "["+ttn+",\""+t.getText()+"\"]";
    }

    protected GroovyRecognizer parser;  // little-used link; TODO: get rid of
    private void require(boolean z, String problem, String solution) throws SemanticException {
        // TODO: Direct to a common error handler, rather than through the parser.
        if (!z)  parser.requireFailed(problem, solution);
    }    
public GroovyLexer(InputStream in) {
	this(new ByteBuffer(in));
}
public GroovyLexer(Reader in) {
	this(new CharBuffer(in));
}
public GroovyLexer(InputBuffer ib) {
	this(new LexerSharedInputState(ib));
}
public GroovyLexer(LexerSharedInputState state) {
	super(state);
	caseSensitiveLiterals = true;
	setCaseSensitive(true);
	literals = new Hashtable();
	literals.put(new ANTLRHashString("byte", this), new Integer(101));
	literals.put(new ANTLRHashString("public", this), new Integer(112));
	literals.put(new ANTLRHashString("case", this), new Integer(150));
	literals.put(new ANTLRHashString("short", this), new Integer(103));
	literals.put(new ANTLRHashString("break", this), new Integer(144));
	literals.put(new ANTLRHashString("while", this), new Integer(138));
	literals.put(new ANTLRHashString("new", this), new Integer(192));
	literals.put(new ANTLRHashString("instanceof", this), new Integer(178));
	literals.put(new ANTLRHashString("implements", this), new Integer(128));
	literals.put(new ANTLRHashString("synchronized", this), new Integer(117));
	literals.put(new ANTLRHashString("const", this), new Integer(40));
	literals.put(new ANTLRHashString("float", this), new Integer(105));
	literals.put(new ANTLRHashString("package", this), new Integer(78));
	literals.put(new ANTLRHashString("return", this), new Integer(143));
	literals.put(new ANTLRHashString("throw", this), new Integer(146));
	literals.put(new ANTLRHashString("null", this), new Integer(195));
	literals.put(new ANTLRHashString("def", this), new Integer(81));
	literals.put(new ANTLRHashString("threadsafe", this), new Integer(116));
	literals.put(new ANTLRHashString("protected", this), new Integer(113));
	literals.put(new ANTLRHashString("class", this), new Integer(88));
	literals.put(new ANTLRHashString("throws", this), new Integer(127));
	literals.put(new ANTLRHashString("do", this), new Integer(41));
	literals.put(new ANTLRHashString("strictfp", this), new Integer(42));
	literals.put(new ANTLRHashString("super", this), new Integer(93));
	literals.put(new ANTLRHashString("with", this), new Integer(139));
	literals.put(new ANTLRHashString("transient", this), new Integer(114));
	literals.put(new ANTLRHashString("native", this), new Integer(115));
	literals.put(new ANTLRHashString("interface", this), new Integer(89));
	literals.put(new ANTLRHashString("final", this), new Integer(37));
	literals.put(new ANTLRHashString("any", this), new Integer(108));
	literals.put(new ANTLRHashString("if", this), new Integer(136));
	literals.put(new ANTLRHashString("double", this), new Integer(107));
	literals.put(new ANTLRHashString("volatile", this), new Integer(118));
	literals.put(new ANTLRHashString("as", this), new Integer(110));
	literals.put(new ANTLRHashString("assert", this), new Integer(147));
	literals.put(new ANTLRHashString("catch", this), new Integer(153));
	literals.put(new ANTLRHashString("try", this), new Integer(151));
	literals.put(new ANTLRHashString("goto", this), new Integer(39));
	literals.put(new ANTLRHashString("enum", this), new Integer(90));
	literals.put(new ANTLRHashString("int", this), new Integer(104));
	literals.put(new ANTLRHashString("for", this), new Integer(141));
	literals.put(new ANTLRHashString("extends", this), new Integer(92));
	literals.put(new ANTLRHashString("boolean", this), new Integer(100));
	literals.put(new ANTLRHashString("char", this), new Integer(102));
	literals.put(new ANTLRHashString("private", this), new Integer(111));
	literals.put(new ANTLRHashString("default", this), new Integer(126));
	literals.put(new ANTLRHashString("false", this), new Integer(194));
	literals.put(new ANTLRHashString("this", this), new Integer(129));
	literals.put(new ANTLRHashString("static", this), new Integer(80));
	literals.put(new ANTLRHashString("abstract", this), new Integer(38));
	literals.put(new ANTLRHashString("continue", this), new Integer(145));
	literals.put(new ANTLRHashString("finally", this), new Integer(152));
	literals.put(new ANTLRHashString("else", this), new Integer(137));
	literals.put(new ANTLRHashString("import", this), new Integer(79));
	literals.put(new ANTLRHashString("in", this), new Integer(142));
	literals.put(new ANTLRHashString("void", this), new Integer(99));
	literals.put(new ANTLRHashString("switch", this), new Integer(140));
	literals.put(new ANTLRHashString("true", this), new Integer(193));
	literals.put(new ANTLRHashString("long", this), new Integer(106));
}

public Token nextToken() throws TokenStreamException {
	Token theRetToken=null;
tryAgain:
	for (;;) {
		Token _token = null;
		int _ttype = Token.INVALID_TYPE;
		resetText();
		try {   // for char stream error handling
			try {   // for lexical error handling
				switch ( LA(1)) {
				case '(':
				{
					mLPAREN(true);
					theRetToken=_returnToken;
					break;
				}
				case ')':
				{
					mRPAREN(true);
					theRetToken=_returnToken;
					break;
				}
				case '[':
				{
					mLBRACK(true);
					theRetToken=_returnToken;
					break;
				}
				case ']':
				{
					mRBRACK(true);
					theRetToken=_returnToken;
					break;
				}
				case '{':
				{
					mLCURLY(true);
					theRetToken=_returnToken;
					break;
				}
				case '}':
				{
					mRCURLY(true);
					theRetToken=_returnToken;
					break;
				}
				case ':':
				{
					mCOLON(true);
					theRetToken=_returnToken;
					break;
				}
				case ',':
				{
					mCOMMA(true);