codegenerator.java

antlr最新版本V3源代码

/*
[The "BSD licence"]
Copyright (c) 2005-2006 Terence Parr
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.codegen;

import antlr.RecognitionException;
import antlr.TokenStreamRewriteEngine;
import antlr.collections.AST;
import org.antlr.Tool;
import org.antlr.analysis.*;
import org.antlr.misc.BitSet;
import org.antlr.misc.*;
import org.antlr.stringtemplate.*;
import org.antlr.stringtemplate.language.AngleBracketTemplateLexer;
import org.antlr.tool.*;

import java.io.IOException;
import java.io.StringReader;
import java.io.Writer;
import java.util.*;

/** ANTLR's code generator.
 *
 *  Generate recognizers derived from grammars.  Language independence
 *  achieved through the use of StringTemplateGroup objects.  All output
 *  strings are completely encapsulated in the group files such as Java.stg.
 *  Some computations are done that are unused by a particular language.
 *  This generator just computes and sets the values into the templates;
 *  the templates are free to use or not use the information.
 *
 *  To make a new code generation target, define X.stg for language X
 *  by copying from existing Y.stg most closely releated to your language;
 *  e.g., to do CSharp.stg copy Java.stg.  The template group file has a
 *  bunch of templates that are needed by the code generator.  You can add
 *  a new target w/o even recompiling ANTLR itself.  The language=X option
 *  in a grammar file dictates which templates get loaded/used.
 *
 *  Some language like C need both parser files and header files.  Java needs
 *  to have a separate file for the cyclic DFA as ANTLR generates bytecodes
 *  directly (which cannot be in the generated parser Java file).  To facilitate
 *  this,
 *
 * cyclic can be in same file, but header, output must be searpate.  recognizer
 *  is in outptufile.
 */
public class CodeGenerator {
	/** When generating SWITCH statements, some targets might need to limit
	 *  the size (based upon the number of case labels).  Generally, this
	 *  limit will be hit only for lexers where wildcard in a UNICODE
	 *  vocabulary environment would generate a SWITCH with 65000 labels.
	 */
	public int MAX_SWITCH_CASE_LABELS = 300;
	public int MIN_SWITCH_ALTS = 3;
	public boolean GENERATE_SWITCHES_WHEN_POSSIBLE = true;
	public static boolean GEN_ACYCLIC_DFA_INLINE = true;
	public static boolean EMIT_TEMPLATE_DELIMITERS = false;

	public String classpathTemplateRootDirectoryName =
		"org/antlr/codegen/templates";

	/** Which grammar are we generating code for?  Each generator
	 *  is attached to a specific grammar.
	 */
	public Grammar grammar;

	/** What language are we generating? */
	protected String language;

	/** The target specifies how to write out files and do other language
	 *  specific actions.
	 */
	public Target target = null;

	/** Where are the templates this generator should use to generate code? */
	protected StringTemplateGroup templates;

	/** The basic output templates without AST or templates stuff; this will be
	 *  the templates loaded for the language such as Java.stg *and* the Dbg
	 *  stuff if turned on.  This is used for generating syntactic predicates.
	 */
	protected StringTemplateGroup baseTemplates;

	protected StringTemplate recognizerST;
	protected StringTemplate outputFileST;
	protected StringTemplate headerFileST;

	/** Used to create unique labels */
	protected int uniqueLabelNumber = 1;

	/** A reference to the ANTLR tool so we can learn about output directories
	 *  and such.
	 */
	protected Tool tool;

	/** Generate debugging event method calls */
	protected boolean debug;

	/** Create a Tracer object and make the recognizer invoke this. */
	protected boolean trace;

	/** Track runtime parsing information about decisions etc...
	 *  This requires the debugging event mechanism to work.
	 */
	protected boolean profile;

	protected int lineWidth = 72;

	/** I have factored out the generation of acyclic DFAs to separate class */
	public ACyclicDFACodeGenerator acyclicDFAGenerator =
		new ACyclicDFACodeGenerator(this);

	/** I have factored out the generation of cyclic DFAs to separate class */
	/*
	public CyclicDFACodeGenerator cyclicDFAGenerator =
		new CyclicDFACodeGenerator(this);
		*/

	public static final String VOCAB_FILE_EXTENSION = ".tokens";
	protected final static String vocabFilePattern =
		"<tokens:{<attr.name>=<attr.type>\n}>" +
		"<literals:{<attr.name>=<attr.type>\n}>";

	public CodeGenerator(Tool tool, Grammar grammar, String language) {
		this.tool = tool;
		this.grammar = grammar;
		this.language = language;
		loadLanguageTarget(language);
	}

	protected void loadLanguageTarget(String language) {
		String targetName = "org.antlr.codegen."+language+"Target";
		try {
			Class c = Class.forName(targetName);
			target = (Target)c.newInstance();
		}
		catch (ClassNotFoundException cnfe) {
			target = new Target(); // use default
		}
		catch (InstantiationException ie) {
			ErrorManager.error(ErrorManager.MSG_CANNOT_CREATE_TARGET_GENERATOR,
							   targetName,
							   ie);
		}
		catch (IllegalAccessException cnfe) {
			ErrorManager.error(ErrorManager.MSG_CANNOT_CREATE_TARGET_GENERATOR,
							   targetName,
							   cnfe);
		}
	}

	/** load the main language.stg template group file */
	public void loadTemplates(String language) {
		// get a group loader containing main templates dir and target subdir
		String templateDirs =
			classpathTemplateRootDirectoryName+":"+
			classpathTemplateRootDirectoryName+"/"+language;
		//System.out.println("targets="+templateDirs.toString());
		StringTemplateGroupLoader loader =
			new CommonGroupLoader(templateDirs,
								  ErrorManager.getStringTemplateErrorListener());
		StringTemplateGroup.registerGroupLoader(loader);
		StringTemplateGroup.registerDefaultLexer(AngleBracketTemplateLexer.class);

		// first load main language template
		StringTemplateGroup coreTemplates =
			StringTemplateGroup.loadGroup(language);
		baseTemplates = coreTemplates;
		if ( coreTemplates ==null ) {
			ErrorManager.error(ErrorManager.MSG_MISSING_CODE_GEN_TEMPLATES,
							   language);
			return;
		}

		// dynamically add subgroups that act like filters to apply to
		// their supergroup.  E.g., Java:Dbg:AST:ASTDbg.
		String outputOption = (String)grammar.getOption("output");
		if ( outputOption!=null && outputOption.equals("AST") ) {
			if ( debug && grammar.type!=Grammar.LEXER ) {
				StringTemplateGroup dbgTemplates =
					StringTemplateGroup.loadGroup("Dbg", coreTemplates);
				baseTemplates = dbgTemplates;
				StringTemplateGroup astTemplates =
					StringTemplateGroup.loadGroup("AST",dbgTemplates);
				StringTemplateGroup astDbgTemplates =
					StringTemplateGroup.loadGroup("ASTDbg", astTemplates);
				templates = astDbgTemplates;
			}
			else {
				templates = StringTemplateGroup.loadGroup("AST", coreTemplates);
			}
		}
		else if ( outputOption!=null && outputOption.equals("template") ) {
			if ( debug && grammar.type!=Grammar.LEXER ) {
				StringTemplateGroup dbgTemplates =
					StringTemplateGroup.loadGroup("Dbg", coreTemplates);
				baseTemplates = dbgTemplates;
				StringTemplateGroup stTemplates =
					StringTemplateGroup.loadGroup("ST",dbgTemplates);
				/*
				StringTemplateGroup astDbgTemplates =
					StringTemplateGroup.loadGroup("STDbg", astTemplates);
				*/
				templates = stTemplates;
			}
			else {
				templates = StringTemplateGroup.loadGroup("ST", coreTemplates);
			}
		}
		else if ( debug && grammar.type!=Grammar.LEXER ) {
			templates = StringTemplateGroup.loadGroup("Dbg", coreTemplates);
			baseTemplates = templates;
		}
		else {
			templates = coreTemplates;
		}

		if ( EMIT_TEMPLATE_DELIMITERS ) {
			templates.emitDebugStartStopStrings(true);
			templates.doNotEmitDebugStringsForTemplate("codeFileExtension");
			templates.doNotEmitDebugStringsForTemplate("headerFileExtension");
		}
	}

	/** Given the grammar to which we are attached, walk the AST associated
	 *  with that grammar to create NFAs.  Then create the DFAs for all
	 *  decision points in the grammar by converting the NFAs to DFAs.
	 *  Finally, walk the AST again to generate code.
	 *
	 *  Either 1 or 2 files are written:
	 *
	 * 		recognizer: the main parser/lexer/treewalker item
	 * 		header file: language like C/C++ need extern definitions
	 *
	 *  The target, such as JavaTarget, dictates which files get written.
	 */
	public StringTemplate genRecognizer() {
		// LOAD OUTPUT TEMPLATES
		loadTemplates(language);
		if ( templates==null ) {
			return null;
		}

		// CHECK FOR LEFT RECURSION; Make sure we can actually do analysis
		grammar.checkAllRulesForLeftRecursion();

		// was there a severe problem while reading in grammar?
		if ( ErrorManager.doNotAttemptAnalysis() ) {
			return null;
		}

		// CREATE NFA FROM GRAMMAR, CREATE DFA FROM NFA
		target.performGrammarAnalysis(this, grammar);

		// some grammar analysis errors will not yield reliable DFA
		if ( ErrorManager.doNotAttemptCodeGen() ) {
			return null;
		}

		// OPTIMIZE DFA
		DFAOptimizer optimizer = new DFAOptimizer(grammar);
		optimizer.optimize();

		// OUTPUT FILE (contains recognizerST)
		outputFileST = templates.getInstanceOf("outputFile");

		// HEADER FILE
		if ( templates.isDefined("headerFile") ) {
			headerFileST = templates.getInstanceOf("headerFile");
		}
		else {
			// create a dummy to avoid null-checks all over code generator
			headerFileST = new StringTemplate(templates,"");
			headerFileST.setName("dummy-header-file");
		}

		boolean filterMode = grammar.getOption("filter")!=null &&
							  grammar.getOption("filter").equals("true");
		boolean canBacktrack = grammar.getSyntacticPredicates()!=null ||
							   filterMode;

		// TODO: move this down further because generating the recognizer
		// alters the model with info on who uses predefined properties etc...
		// The actions here might refer to something.

		// The only two possible output files are available at this point.
		// Verify action scopes are ok for target and dump actions into output
		// Templates can say <actions.parser.header> for example.
		Map actions = grammar.getActions();
		verifyActionScopesOkForTarget(actions);
		// translate $x::y references
		translateActionAttributeReferences(actions);
		Map actionsForGrammarScope =
			(Map)actions.get(grammar.getDefaultActionScope(grammar.type));
		if ( filterMode &&
			 (actionsForGrammarScope==null ||
			 !actionsForGrammarScope.containsKey(Grammar.SYNPREDGATE_ACTION_NAME)) )
		{
			// if filtering, we need to set actions to execute at backtracking
			// level 1 not 0.  Don't set this action if a user has though
			StringTemplate gateST = templates.getInstanceOf("filteringActionGate");
			if ( actionsForGrammarScope==null ) {
				actionsForGrammarScope=new HashMap();
				actions.put(grammar.getDefaultActionScope(grammar.type),
							actionsForGrammarScope);
			}
			actionsForGrammarScope.put(Grammar.SYNPREDGATE_ACTION_NAME,
									   gateST);
		}
		headerFileST.setAttribute("actions", actions);
		outputFileST.setAttribute("actions", actions);

		headerFileST.setAttribute("buildTemplate", new Boolean(grammar.buildTemplate()));
		outputFileST.setAttribute("buildTemplate", new Boolean(grammar.buildTemplate()));
		headerFileST.setAttribute("buildAST", new Boolean(grammar.buildAST()));
		outputFileST.setAttribute("buildAST", new Boolean(grammar.buildAST()));

		String rewrite = (String)grammar.getOption("rewrite");
		outputFileST.setAttribute("rewrite",
								  Boolean.valueOf(rewrite!=null&&rewrite.equals("true")));
		headerFileST.setAttribute("rewrite",
								  Boolean.valueOf(rewrite!=null&&rewrite.equals("true")));

		outputFileST.setAttribute("backtracking", Boolean.valueOf(canBacktrack));
		headerFileST.setAttribute("backtracking", Boolean.valueOf(canBacktrack));
		String memoize = (String)grammar.getOption("memoize");
		outputFileST.setAttribute("memoize",
								  Boolean.valueOf(memoize!=null&&memoize.equals("true")&&
									          canBacktrack));
		headerFileST.setAttribute("memoize",
								  Boolean.valueOf(memoize!=null&&memoize.equals("true")&&
									          canBacktrack));


		outputFileST.setAttribute("trace", Boolean.valueOf(trace));
		headerFileST.setAttribute("trace", Boolean.valueOf(trace));

		outputFileST.setAttribute("profile", Boolean.valueOf(profile));
		headerFileST.setAttribute("profile", Boolean.valueOf(profile));

		// RECOGNIZER
		if ( grammar.type==Grammar.LEXER ) {
			recognizerST = templates.getInstanceOf("lexer");
			outputFileST.setAttribute("LEXER", Boolean.valueOf(true));
			headerFileST.setAttribute("LEXER", Boolean.valueOf(true));
			recognizerST.setAttribute("filterMode",
									  Boolean.valueOf(filterMode));
		}
		else if ( grammar.type==Grammar.PARSER ||
			grammar.type==Grammar.COMBINED )
		{
			recognizerST = templates.getInstanceOf("parser");
			outputFileST.setAttribute("PARSER", Boolean.valueOf(true));
			headerFileST.setAttribute("PARSER", Boolean.valueOf(true));
		}
		else {
			recognizerST = templates.getInstanceOf("treeParser");
			outputFileST.setAttribute("TREE_PARSER", Boolean.valueOf(true));
			headerFileST.setAttribute("TREE_PARSER", Boolean.valueOf(true));
		}
		outputFileST.setAttribute("recognizer", recognizerST);
		headerFileST.setAttribute("recognizer", recognizerST);
		outputFileST.setAttribute("actionScope",
								  grammar.getDefaultActionScope(grammar.type));
		headerFileST.setAttribute("actionScope",
								  grammar.getDefaultActionScope(grammar.type));

		String targetAppropriateFileNameString =
			target.getTargetStringLiteralFromString(grammar.getFileName());
		outputFileST.setAttribute("fileName", targetAppropriateFileNameString);
		headerFileST.setAttribute("fileName", targetAppropriateFileNameString);
		outputFileST.setAttribute("ANTLRVersion", Tool.VERSION);
		headerFileST.setAttribute("ANTLRVersion", Tool.VERSION);