testsymboldefinitions.java

antlr最新版本V3源代码

/*
 [The "BSD licence"]
 Copyright (c) 2005-2006 Terence Parr
 All rights reserved.

 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions
 are met:
 1. Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.
 2. Redistributions in binary form must reproduce the above copyright
    notice, this list of conditions and the following disclaimer in the
    documentation and/or other materials provided with the distribution.
 3. The name of the author may not be used to endorse or promote products
    derived from this software without specific prior written permission.

 THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.test;

import org.antlr.Tool;
import org.antlr.analysis.Label;
import org.antlr.codegen.CodeGenerator;
import org.antlr.stringtemplate.StringTemplate;
import org.antlr.tool.*;

import java.io.StringReader;
import java.util.*;

public class TestSymbolDefinitions extends BaseTest {

    /** Public default constructor used by TestRig */
    public TestSymbolDefinitions() {
    }

	public void testParserSimpleTokens() throws Exception {
		Grammar g = new Grammar(
				"parser grammar t;\n"+
				"a : A | B;\n" +
				"b : C ;");
		String rules = "a, b";
		String tokenNames = "A, B, C";
		checkSymbols(g, rules, tokenNames);
	}

	public void testParserTokensSection() throws Exception {
		Grammar g = new Grammar(
				"parser grammar t;\n" +
				"tokens {\n" +
				"  C;\n" +
				"  D;" +
				"}\n"+
				"a : A | B;\n" +
				"b : C ;");
		String rules = "a, b";
		String tokenNames = "A, B, C, D";
		checkSymbols(g, rules, tokenNames);
	}

	public void testLexerTokensSection() throws Exception {
		Grammar g = new Grammar(
				"lexer grammar t;\n" +
				"tokens {\n" +
				"  C;\n" +
				"  D;" +
				"}\n"+
				"A : 'a';\n" +
				"C : 'c' ;");
		String rules = "A, C, Tokens";
		String tokenNames = "A, C, D";
		checkSymbols(g, rules, tokenNames);
	}

	public void testTokensSectionWithAssignmentSection() throws Exception {
		Grammar g = new Grammar(
				"grammar t;\n" +
				"tokens {\n" +
				"  C='c';\n" +
				"  D;" +
				"}\n"+
				"a : A | B;\n" +
				"b : C ;");
		String rules = "a, b";
		String tokenNames = "A, B, C, D, 'c'";
		checkSymbols(g, rules, tokenNames);
	}

	public void testCombinedGrammarLiterals() throws Exception {
		Grammar g = new Grammar(
				"grammar t;\n"+
				"a : 'begin' b 'end';\n" +
				"b : C ';' ;\n" +
				"ID : 'a' ;\n" +
				"FOO : 'foo' ;\n" +  // "foo" is not a token name
				"C : 'c' ;\n");        // nor is 'c'
		String rules = "a, b";
		String tokenNames = "C, FOO, ID, 'begin', 'end', ';'";
		checkSymbols(g, rules, tokenNames);
	}

	public void testLiteralInParserAndLexer() throws Exception {
		// 'x' is token and char in lexer rule
		Grammar g = new Grammar(
				"grammar t;\n" +
				"a : 'x' E ; \n" +
				"E: 'x' '0' ;\n");        // nor is 'c'
		String literals = "['x']";
		String foundLiterals = g.getStringLiterals().toString();
		assertEquals(literals, foundLiterals);

		String implicitLexer =
			"lexer grammar t;\n" +
			"\n" +
			"T5 : 'x' ;\n" +
			"\n" +
			"// $ANTLR src \"<string>\" 3\n" +
			"E: 'x' '0' ;\n";
		assertEquals(implicitLexer, g.getLexerGrammar());
	}

	public void testCombinedGrammarWithRefToLiteralButNoTokenIDRef() throws Exception {
		Grammar g = new Grammar(
				"grammar t;\n"+
				"a : 'a' ;\n" +
				"A : 'a' ;\n");
		String rules = "a";
		String tokenNames = "A, 'a'";
		checkSymbols(g, rules, tokenNames);
	}

	public void testSetDoesNotMissTokenAliases() throws Exception {
		Grammar g = new Grammar(
				"grammar t;\n"+
				"a : 'a'|'b' ;\n" +
				"A : 'a' ;\n" +
				"B : 'b' ;\n");
		String rules = "a";
		String tokenNames = "A, 'a', B, 'b'";
		checkSymbols(g, rules, tokenNames);
	}

	public void testSimplePlusEqualLabel() throws Exception {
		Grammar g = new Grammar(
				"parser grammar t;\n"+
				"a : ids+=ID ( COMMA ids+=ID )* ;\n");
		String rule = "a";
		String tokenLabels = "ids";
		String ruleLabels = null;
		checkPlusEqualsLabels(g, rule, tokenLabels, ruleLabels);
	}

	public void testMixedPlusEqualLabel() throws Exception {
		Grammar g = new Grammar(
				"grammar t;\n"+
				"options {output=AST;}\n" +
				"a : id+=ID ( ',' e+=expr )* ;\n" +
				"expr : 'e';\n" +
				"ID : 'a';\n");
		String rule = "a";
		String tokenLabels = "id";
		String ruleLabels = "e";
		checkPlusEqualsLabels(g, rule, tokenLabels, ruleLabels);
	}

	// T E S T  L I T E R A L  E S C A P E S

	public void testParserCharLiteralWithEscape() throws Exception {
		Grammar g = new Grammar(
				"grammar t;\n"+
				"a : '\\n';\n");
		Set literals = g.getStringLiterals();
		// must store literals how they appear in the antlr grammar
		assertEquals("'\\n'", literals.toArray()[0]);
	}

	public void testTokenInTokensSectionAndTokenRuleDef() throws Exception {
		// this must return A not I to the parser; calling a nonfragment rule
		// from a nonfragment rule does not set the overall token.
		String grammar =
			"grammar P;\n" +
			"tokens { B='}'; }\n"+
			"a : A B {System.out.println(input);} ;\n"+
			"A : 'a' ;\n" +
			"B : '}' ;\n"+
			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
		String found = execParser("P.g", grammar, "PParser", "PLexer",
								  "a", "a}", false);
		assertEquals("a}\n", found);
	}

	public void testTokenInTokensSectionAndTokenRuleDef2() throws Exception {
		// this must return A not I to the parser; calling a nonfragment rule
		// from a nonfragment rule does not set the overall token.
		String grammar =
			"grammar P;\n" +
			"tokens { B='}'; }\n"+
			"a : A '}' {System.out.println(input);} ;\n"+
			"A : 'a' ;\n" +
			"B : '}' {/* */} ;\n"+
			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
		String found = execParser("P.g", grammar, "PParser", "PLexer",
								  "a", "a}", false);
		assertEquals("a}\n", found);
	}


	public void testRefToRuleWithNoReturnValue() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue);

		String grammarStr =
			"grammar P;\n" +
			"a : x=b ;\n" +
			"b : B ;\n" +
			"B : 'b' ;\n";
		Grammar g = new Grammar(grammarStr);

		Tool antlr = newTool();
		CodeGenerator generator = new CodeGenerator(antlr, g, "Java");
		g.setCodeGenerator(generator);
		StringTemplate recogST = generator.genRecognizer();
		String code = recogST.toString();
		assertTrue("not expecting label", code.indexOf("x=b();")<0);

		assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size());
	}

	// T E S T  E R R O R S

	public void testParserStringLiterals() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue);
		Grammar g = new Grammar(
				"parser grammar t;\n"+
				"a : 'begin' b ;\n" +
				"b : C ;");
		Object expectedArg = "'begin'";
		int expectedMsgID = ErrorManager.MSG_LITERAL_NOT_ASSOCIATED_WITH_LEXER_RULE;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkError(equeue, expectedMessage);
	}

	public void testParserCharLiterals() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue);
		Grammar g = new Grammar(
				"parser grammar t;\n"+
				"a : '(' b ;\n" +
				"b : C ;");
		Object expectedArg = "'('";
		int expectedMsgID = ErrorManager.MSG_LITERAL_NOT_ASSOCIATED_WITH_LEXER_RULE;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkError(equeue, expectedMessage);
	}

	public void testEmptyNotChar() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue);
		Grammar g = new Grammar(
				"grammar foo;\n" +
				"a : (~'x')+ ;\n");
		g.createNFAs();
		Object expectedArg = "'x'";
		int expectedMsgID = ErrorManager.MSG_EMPTY_COMPLEMENT;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkError(equeue, expectedMessage);
	}

	public void testEmptyNotToken() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue);
		Grammar g = new Grammar(
				"grammar foo;\n" +
				"a : (~A)+ ;\n");
		g.createNFAs();
		Object expectedArg = "A";
		int expectedMsgID = ErrorManager.MSG_EMPTY_COMPLEMENT;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkError(equeue, expectedMessage);
	}

	public void testEmptyNotSet() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue);
		Grammar g = new Grammar(
				"grammar foo;\n" +
				"a : (~(A|B))+ ;\n");
		g.createNFAs();
		Object expectedArg = null;
		int expectedMsgID = ErrorManager.MSG_EMPTY_COMPLEMENT;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkError(equeue, expectedMessage);
	}

	public void testStringLiteralInParserTokensSection() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue); // unique listener per thread
		Grammar g = new Grammar(
				"parser grammar t;\n" +
				"tokens {\n" +
				"  B='begin';\n" +
				"}\n"+
				"a : A B;\n" +
				"b : C ;");
		Object expectedArg = "'begin'";
		int expectedMsgID = ErrorManager.MSG_LITERAL_NOT_ASSOCIATED_WITH_LEXER_RULE;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkError(equeue, expectedMessage);
	}

	public void testCharLiteralInParserTokensSection() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue); // unique listener per thread
		Grammar g = new Grammar(
				"parser grammar t;\n" +
				"tokens {\n" +
				"  B='(';\n" +
				"}\n"+
				"a : A B;\n" +
				"b : C ;");
		Object expectedArg = "'('";
		int expectedMsgID = ErrorManager.MSG_LITERAL_NOT_ASSOCIATED_WITH_LEXER_RULE;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkError(equeue, expectedMessage);
	}

	public void testCharLiteralInLexerTokensSection() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue); // unique listener per thread
		Grammar g = new Grammar(
				"lexer grammar t;\n" +
				"tokens {\n" +
				"  B='(';\n" +
				"}\n"+
				"ID : 'a';\n");
		Object expectedArg = "'('";
		int expectedMsgID = ErrorManager.MSG_CANNOT_ALIAS_TOKENS_IN_LEXER;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkError(equeue, expectedMessage);
	}

	public void testRuleRedefinition() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue); // unique listener per thread
		Grammar g = new Grammar(
				"parser grammar t;\n"+
				"a : A | B;\n" +
				"a : C ;");

		Object expectedArg = "a";
		int expectedMsgID = ErrorManager.MSG_RULE_REDEFINITION;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkError(equeue, expectedMessage);
	}

	public void testLexerRuleRedefinition() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue); // unique listener per thread
		Grammar g = new Grammar(
				"lexer grammar t;\n"+
				"ID : 'a' ;\n" +
				"ID : 'd' ;");

		Object expectedArg = "ID";
		int expectedMsgID = ErrorManager.MSG_RULE_REDEFINITION;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkError(equeue, expectedMessage);
	}

	public void testCombinedRuleRedefinition() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue); // unique listener per thread
		Grammar g = new Grammar(
				"grammar t;\n"+
				"x : ID ;\n" +
				"ID : 'a' ;\n" +
				"x : ID ID ;");

		Object expectedArg = "x";
		int expectedMsgID = ErrorManager.MSG_RULE_REDEFINITION;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkError(equeue, expectedMessage);
	}

	public void testUndefinedToken() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue); // unique listener per thread
		Grammar g = new Grammar(
				"grammar t;\n"+
				"x : ID ;");

		Object expectedArg = "ID";
		int expectedMsgID = ErrorManager.MSG_NO_TOKEN_DEFINITION;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkWarning(equeue, expectedMessage);
	}

	public void testUndefinedTokenOkInParser() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue); // unique listener per thread
		Grammar g = new Grammar(
				"parser grammar t;\n"+
				"x : ID ;");
        assertEquals("should not be an error", 0, equeue.errors.size());
	}

	public void testUndefinedRule() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue); // unique listener per thread
		Grammar g = new Grammar(
				"grammar t;\n"+
				"x : r ;");

		Object expectedArg = "r";
		int expectedMsgID = ErrorManager.MSG_UNDEFINED_RULE_REF;
		GrammarSemanticsMessage expectedMessage =
			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg);
		checkError(equeue, expectedMessage);
	}

	public void testLexerRuleInParser() throws Exception {
		ErrorQueue equeue = new ErrorQueue();
		ErrorManager.setErrorListener(equeue); // unique listener per thread
		Grammar g = new Grammar(
				"parser grammar t;\n"+
				"X : ;");