smclexer.sm

SMC takes a state machine stored in a .sm file and generates a State pattern in twelve programming l

// -*- tab-width: 4; -*-

%{
/*
 * The contents of this file are subject to the Mozilla Public
 * License Version 1.1 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy
 * of the License at http://www.mozilla.org/MPL/
 * 
 * Software distributed under the License is distributed on an
 * "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * rights and limitations under the License.
 * 
 * The Original Code is State Machine Compiler (SMC).
 * 
 * The Initial Developer of the Original Code is Charles W. Rapp.
 * Portions created by Charles W. Rapp are
 * Copyright (C) 2000 - 2006. Charles W. Rapp.
 * All Rights Reserved.
 * 
 * Contributor(s):
 *   Eitan Suez contributed examples/Ant.
 *   (Name withheld) contributed the C# code generation and
 *   examples/C#.
 *   Francois Perrad contributed the Python code generation and
 *   examples/Python.
 *   Chris Liscio contributed the Objective-C code generation
 *   and examples/ObjC.
 *
 * smcLexer --
 *
 *  This statemap finds tokens in a statemap source file.
 *
 * RCS ID
 * $Id: SmcLexer.sm,v 1.9 2007/12/28 12:34:41 cwrapp Exp $
 *
 * CHANGE LOG
 * $Log: SmcLexer.sm,v $
 * Revision 1.9  2007/12/28 12:34:41  cwrapp
 * Version 5.0.1 check-in.
 *
 * Revision 1.8  2007/01/15 00:23:51  cwrapp
 * Release 4.4.0 initial commit.
 *
 * Revision 1.7  2006/09/16 15:04:29  cwrapp
 * Initial v. 4.3.3 check-in.
 *
 * Revision 1.6  2005/07/07 12:10:41  fperrad
 * Add a new token '$' for Perl language.
 *
 * Revision 1.5  2005/05/28 19:28:42  cwrapp
 * Moved to visitor pattern.
 *
 * Revision 1.5  2005/02/21 18:14:01  charlesr
 * Removed unknown FSM transition, using unicode transition
 * instead.
 *
 * Revision 1.4  2005/02/21 15:36:08  charlesr
 * Added Francois Perrad to Contributors section for Python work.
 *
 * Revision 1.3  2004/10/30 16:05:34  charlesr
 * Added unicode support.
 *
 * Revision 1.2  2004/09/06 16:40:22  charlesr
 * Added C# support.
 *
 * Revision 1.1  2004/01/29 02:14:49  charlesr
 * Add Default state containing single transition which
 * handles an unexpected asterisk.
 *
 * Revision 1.0  2003/12/14 21:04:00  charlesr
 * Initial revision
 *
 */
%}

%start TokenMap::Start
%class SmcLexer
%package net.sf.smc

%map TokenMap
%%
Start
{
	// Is this the start of a comment?
	slash           CommentStart		{}

	// Multi-character tokens.
	percent 		PercentStart		{startToken();
                                         addToToken();}
	alpha           Word				{startToken();
                                         addToToken();}
	underscore      Word				{startToken();
                                         addToToken();}
    colon           Colon               {startToken();
                                         addToToken();}

	// Single character tokens.
	left_brace      nil					{startToken();
                                         addToToken();
										 endToken(SmcLexer.LEFT_BRACE);}
	right_brace     nil					{startToken();
                                         addToToken();
										 endToken(SmcLexer.RIGHT_BRACE);}
	left_bracket	nil					{startToken();
                                         addToToken();
										 endToken(SmcLexer.LEFT_BRACKET);}
	left_paren      nil					{startToken();
                                         addToToken();
										 endToken(SmcLexer.LEFT_PAREN);}
	right_paren     nil					{startToken();
                                         addToToken();
										 endToken(SmcLexer.RIGHT_PAREN);}
	comma           nil					{startToken();
                                         addToToken();
										 endToken(SmcLexer.COMMA);}
	semicolon       nil					{startToken();
                                         addToToken();
										 endToken(SmcLexer.SEMICOLON);}
    equal           nil                 {startToken();
                                         addToToken();
                                         endToken(SmcLexer.EQUAL);}

	dollar          nil					{startToken();
                                         addToToken();
										 endToken(SmcLexer.DOLLAR);}

	// Ignore whitespace and end-of-line.
	whitespace      nil					{}
	EOL             nil					{}

	unicode         nil					{startToken();
                                                 addToToken();
                                                 outputChar();
									      badToken("Unknown character");}
}

CommentStart
{
	asterisk	    push(OldCommentMap::Start)	{}
	slash           push(NewCommentMap::Start)
                                        {}
	commentDone     Start				{}

	// Any other character following a slash means this is
    // a plain-old slash.
	Default 	   Start			    {ungetChar();
										 endToken(SmcLexer.SLASH);}
}

PercentStart
{
	// This the start of a source code.
	left_brace      push(SourceMap::Start)
                                        {startToken();}
	sourceDone      Start				{}

	// This is the map delimiter.
	percent 		Start				{addToToken();
										 endToken(SmcLexer.EOD);}

	// Look for %start, %map, %class, %header or %package.
	alpha            PercentKeyword		{addToToken();}

	// Any other character following a percent is a malformed token.
	right_brace     Start				{addToToken();
										 badToken("End-of-source appears without matching start-of-source");}

	Default         Start			    {addToToken();
										 badToken("Unknown % directive");}
}

PercentKeyword
{
	// Keep collecting characters until whitespace or EOL is reached.
	whitespace      Start				{checkPercentKeyword();}
	EOL             Start				{checkPercentKeyword();}

	// Only letters are allowed after the percent.
	alpha           nil					{addToToken();}

	// Anything else is a bad token.
	Default         Start				{addToToken();
										 badToken("Unknown % directive");}
}

Word
{
	alpha           nil					{addToToken();}
	digit           nil					{addToToken();}
	underscore      nil					{addToToken();}
	period          nil					{addToToken();}
    colon           Scope               {}

	whitespace      Start				{checkKeyword();}
	EOL             Start				{checkKeyword();}

	// All other characters are a part of other tokens. "Unread"
	// the tokens so they can be read the next call to nextToken()
	// and then pass this token back.
	left_brace      Start				{ungetChar();
										 checkKeyword();}
	right_brace  	Start				{ungetChar();
										 checkKeyword();}
	left_bracket	Start				{ungetChar();
										 checkKeyword();}
	left_paren      Start				{ungetChar();
										 checkKeyword();}
	right_paren     Start				{ungetChar();
										 checkKeyword();}
	comma           Start				{ungetChar();
										 checkKeyword();}
	semicolon       Start				{ungetChar();
										 checkKeyword();}
	slash           Start				{ungetChar();
										 checkKeyword();}
	asterisk        Start               {ungetChar();
                                         checkKeyword();}
	equal           Start               {ungetChar();
                                         checkKeyword();}
	dollar          Start               {ungetChar();
                                         checkKeyword();}

	// The remaining characters form an invalid token.
	Default         Start				{badToken("Unknown token");}
}

Scope
{
    colon           Word                {addToToken("::");}
    Default         Start               {ungetChar();
                                         ungetChar();
                                         checkKeyword();}
}

Colon
{
    colon           Word                {addToToken("::");}

    // This is a token by itself.
    Default         Start               {ungetChar();
                                         addToToken(":");
                                         endToken(SmcLexer.COLON);}
}

Default
{
    asterisk        Start               {badToken("Unknown token");}
}
%%

%map OldCommentMap
%%
Start
{
    // This grammar correctly handles nested comments!
	slash           CommentStart		{}

    // A comment's end is */.
	asterisk	    CommentEnd			{}
}

CommentStart
{
	asterisk		push(OldCommentMap::Start)	{}
	slash           push(NewCommentMap::Start)	{}
	commentDone		Start				{}
}

CommentEnd
{
	asterisk	    nil					{}
	slash           pop(commentDone)	{}
}

Default
{
	alpha           Start				{}
	digit           Start				{}
	percent         Start				{}
	slash           Start				{}
	asterisk		Start				{}
	left_brace      Start				{}
	right_brace     Start				{}
	left_bracket	Start				{}
	left_paren      Start				{}
	right_paren     Start				{}
	period          Start				{}
	colon           Start				{}
	comma           Start				{}
	semicolon		Start				{}
	underscore      Start				{}
    equal           Start               {}
	dollar          Start				{}
	whitespace   	Start				{}
	EOL             Start				{}
    unicode         Start               {}
}
%%

%map NewCommentMap
%%
// Wait here for the end of the // comment.
Start
{
	// The end of line marks the end of the comment.
	EOL             pop(commentDone)	{}
}

Default
{
	alpha           Start				{}
	digit           Start				{}
	percent  		Start				{}
	slash           Start				{}
	asterisk	    Start				{}
	left_brace      Start				{}
	right_brace     Start				{}
	left_bracket	Start				{}
	left_paren      Start				{}
	right_paren     Start				{}
	period          Start				{}
	colon           Start				{}
	comma           Start				{}
	semicolon		Start				{}
	underscore      Start				{}
    equal           Start               {}
	dollar          Start				{}
	whitespace      Start				{}
    unicode         Start               {}
}
%%

%map SourceMap
%%
// Look for the end of the source block.
Start
{
	// Is this the end of the raw source code?
	percent 		SourceEnd			{}

	// No, the end has not come.
	Default		    Start               {addToToken();}
}

SourceEnd
{
	// Yes, we are add the end of the raw source code.
	right_brace     pop(sourceDone)		{endToken(SmcLexer.SOURCE);}

	// Perhaps this is the end of the raw source.
	percent 		nil					{addToToken("%");}

	// Nope, this is not the end of the raw source.
	Default		    Start				{addToToken("%");
										 addToToken();}
}

// This state is never used. It is here to force smc to generate
// all the necessary default transitions.
NeverUsed
{
	alpha           nil					{}
	digit           nil					{}
	slash           nil					{}
	asterisk	    nil					{}
	left_brace      nil					{}
	right_brace     nil					{}
	left_bracket	nil					{}
	left_paren      nil					{}
	right_paren     nil					{}
	period          nil					{}
	colon           nil					{}
	comma           nil					{}
	semicolon		nil					{}
	underscore      nil					{}
    equal           nil                 {}
	dollar          nil					{}
	whitespace      nil					{}
    unicode         nil                 {}
	EOL             nil					{}
}
%%