examplelexer.java

java 词法分析器,用于一般的C,C++,VB,PS/SQL 语句的翻译

package fri.patterns.interpreter.parsergenerator.lexer.examples;

import java.io.*;
import fri.patterns.interpreter.parsergenerator.Token;
import fri.patterns.interpreter.parsergenerator.Lexer;
import fri.patterns.interpreter.parsergenerator.lexer.*;
import fri.patterns.interpreter.parsergenerator.syntax.Syntax;
import fri.patterns.interpreter.parsergenerator.syntax.builder.SyntaxSeparation;

/**
	Sample showing how to build a Lexer from StandardLexerRules building blocks.
	This Lexer scans C/Java stringdefs, chardefs, integer, float, identifiers, comments,
	and everything else as "verbatim".
	
	@author (c) 2002, Fritz Ritzberger
*/

public abstract class ExampleLexer
{
	/** Test main. */
	public static void main(String [] args)	{
		String [][] rules = {
			{ "token", "stringdef" },
			{ "token", "chardef" },
			{ "token", "identifier" },
			{ "token", "float" },
			{ "token", "integer" },
			{ "token", "verbatim" },
			{ "verbatim", "char",
						Token.BUTNOT, "space",
						Token.BUTNOT, "newline",
						Token.BUTNOT, "letter",
						Token.BUTNOT, "digit",
						Token.BUTNOT, "stringdef",
						Token.BUTNOT, "chardef",
						Token.BUTNOT, "comment" },
			{ "ignored", "comment" },
			{ "ignored", "spaces" },
			{ "ignored", "newlines" },
		};
		Syntax syntax = new Syntax(rules);

		try	{
			fri.util.TimeStopper ts = new fri.util.TimeStopper();

			SyntaxSeparation separation = new SyntaxSeparation(syntax);
			LexerBuilder builder = new LexerBuilder(separation.getLexerSyntax(), separation.getIgnoredSymbols());
			Lexer lexer = builder.getLexer();
			lexer.setDebug(true);
			lexer.setTerminals(separation.getTokenSymbols());

			System.err.println("time to build lexer was: "+ts.getInterval());

			InputStream in = ExampleLexer.class.getResourceAsStream("ExampleLexer.java");	//Reader in = new FileReader(args[0]);
			lexer.setInput(in);

			System.err.println("time to build input was: "+ts.getInterval());
			
			Token t;
			do	{
				t = lexer.getNextToken(null);
				//System.err.println(ts.getInterval()+" was time to read token "+t.symbol+" -> "+t.text);
				
				if (t.symbol == null)	{
					lexer.dump(System.err);
					throw new LexerException("Uninterpretable input!");
				}
				System.out.println(t.symbol+" "+">"+t.text+"<");
			}
			while (Token.isEpsilon(t) == false);

			System.err.println("time to scan input was: "+ts.getTimeMillis());
		}
		catch (Exception e)	{
			e.printStackTrace();
		}


		// some complicated Java expressions, for test lexing this source file
		char testChar1 = 'c';
		char testChar2 = '\r';
		char testChar3 = '\000';
		char testChar4 = '\377';
		String testString1 = "\\\"\\/* comment in string */";
		String testString2 = "///*|<>@,;.:-_#'+*~碻咪鳇\"闹躛哀%&/[][]{}";
		float testFloat = 123.456E-12f;
		double testDouble = .456d;
		int testInt1 = 0X12aB;
		int testInt2 = 07654;
		System.err.println("Zahl 07658 ist: "+testInt2);
		long testLong1 = 123456L;
	}

}