lexer.cs

charp compiler

//-----------------------------------------------------------------------------
// Hand coded lexer
//-----------------------------------------------------------------------------

using System;
using System.IO;
using System.Diagnostics;
using System.Collections;

//using ErrorLog = Blue.Utilities.ErrorLog;
using ILexer = Blue.Public.ILexer;

namespace ManualParser
{
//-----------------------------------------------------------------------------
// Represent a position in the file
//-----------------------------------------------------------------------------
public struct CursorPos
{
    public int row;
    public int col;
}

//-----------------------------------------------------------------------------
// Token information
// Single base class that operates as a variant.
// @todo - this could probably be more efficient.
//-----------------------------------------------------------------------------

/// <summary>
/// A single lexeme from a source file. These are produced by a <see cref="Blue.Public.ILexer"/>
/// and consumed by a parser.
/// </summary>
public class Token
{
#region Enum of all the possible Token types
    public enum Type
    {
        cId,
    // Literals        
        cInt,
        cChar,
        cString,
        cBool,
        
        cClass,
        cInterface,
        cStruct,
        cEnum,
        cDelegate,
        cEvent,
        cOperator,

        cReturn,
        cNew,
        cIf,
        cSwitch,
        cElse,        
        cDo,
        cWhile,
        cFor,
        cForEach,
        cIn,
        cGoto,
        cBreak,
        cContinue,
        cDefault,
        cCase,
        
        cUsing,
        cNamespace,
        
        cNull,

        cTry,
        cCatch,
        cFinally,
        cThrow,

        cRef,
        cOut,
        cParams,
                
        cGet,
        cSet,
        //cValue,
        
        cAttrPublic,
        cAttrPrivate,
        cAttrProtected,
        cAttrStatic,
        cAttrVirtual,
        cAttrAbstract,
        cAttrOverride,
        cAttrInternal,
        cAttrSealed,
        cAttrReadOnly,
        cAttrConst,
        
        cLParen,
        cRParen,
        cLCurly,
        cRCurly,
        cLSquare,
        cRSquare,
        cLRSquare,  // includes int:Dimension
        cSemi,
        cComma,
        cDot,
        cAssign,
        cColon,
        cQuestion,
        
        // Boolean
        cAnd,
        cOr,
        cNot,
        
        // Bitwise
        cBitwiseAnd,
        cBitwiseAndEqual,
        cBitwiseOr,
        cBitwiseOrEqual,
        cBitwiseXor,
        cBitwiseXorEqual,
        
        cBitwiseNot,
        
        cShiftLeft,
        cShiftLeftEqual,
        cShiftRight,
        cShiftRightEqual,
        
        cPlus,
        cPlusPlus,
        cPlusEqual,
        cMinus,
        cMinusMinus,
        cMinusEqual,
        cMul,
        cMulEqual,
        cDiv,
        cDivEqual,
        cMod,
        cModEqual,
        
        cTypeOf,
        
        cEqu,
        cNeq,
        cGTE,
        cLTE,
        cGT,
        cLT,

        cIs,
        cAs,
        
        cError,
        cEOF,

    // The order here is important.
        MARKER_LastNormalToken,
        
        MARKER_FirstPreProcToken,
        
    // Preprocessor tokens.
        cPP_EOL,
        cPP_If,
        cPP_ElseIf,
        cPP_Else,
        cPP_Endif,
        cPP_Define,
        cPP_Undef,
        cPP_Region,
        cPP_EndRegion,
    }
#endregion

#region Checks            
    // Pretty print this token's type & content
    public override string ToString()
    {
        string stValue = null;
        
        switch(m_type)
        {
        case Type.cId:
            stValue = Text; break;
            
        case Type.cString:
            stValue = Text; break;
            
        case Type.cInt:
            stValue = m_nValue.ToString(); break;        
        
        case Type.cChar:
            stValue = "\'"  + (char) m_nValue + "\'"; break; // @todo - for escapes
            
        case Type.cBool:
            stValue = m_fValue.ToString(); break;            
            
        case Type.cError:
            stValue = "starting with:"+m_text; break;
        }
        
        if (stValue == null) {
            return m_type.ToString();
        } else {
            return m_type.ToString() + ":" + stValue;
        }
    }            
#endregion    
    
#region Constructors    
    // String data (for string literals & identifiers)
    public Token(Type type, bool fValue, FileRange pos)
    {   
        Debug.Assert(type == Type.cBool);
        m_filepos = pos;
        
        m_type = type;
        m_fValue = fValue;
        
    }
    
    // String data (for string literals & identifiers)
    public Token(Type type, string stText, FileRange pos)
    {   
        Debug.Assert(type == Type.cId || type == Type.cString);
        m_filepos = pos;
        
        m_type = type;
        m_text = stText;
    }
    
    // Int data (for integers)
    public Token(Type type, int nValue, FileRange pos)
    {   
        Debug.Assert(type == Type.cInt || type == Type.cLRSquare);
        m_filepos = pos;        
        
        m_type = type;
        m_nValue = nValue;
    }
    
    // For chars
    public Token(Type type, char chValue, FileRange pos)
    {
        Debug.Assert(type == Type.cChar);
        m_filepos = pos;
        
        m_type = type;
        m_nValue = (int) chValue;
    }
    
    // For errors
    public Token(Type type, char ch1, int chPeek, FileRange pos)
    {   
        Debug.Assert(type == Type.cError);
        m_filepos = pos;
        
        m_type = type;
        if (chPeek == -1 || Lexer.IsWhitespace(chPeek))
        {
            m_text = "" + ch1;
        } else {
            m_text = "" + ch1 + (char) chPeek;
        }
        
        
    }
    
    // No data (for practically everything else)
    public Token(Type type, FileRange pos)
    {   
        m_filepos = pos;
        
        m_type = type;
        m_text = "";
    }
#endregion

#region Helpers
    public bool IsPreprocToken
    {
        get {
            return (this.m_type >= Token.Type.MARKER_FirstPreProcToken);
        }
    }
    

#endregion
    
#region Properties & Data        
    // Note where this lexeme exists (to propogate to AST for error info)
    protected FileRange m_filepos;
    public FileRange Location
    {
        get { return m_filepos; }
    }
    
    // What type of token are we?
    internal Type m_type;            
    public Type TokenType
    {
        get { return m_type; }
    }
        
    // Raw text of the token, 
    internal string m_text;
    public string Text
    {
        get { Debug.Assert(m_type == Type.cId || m_type == Type.cString); return m_text; }
    }
    
    // Get the identifier (text & location)
    public Identifier Id
    {
        get { 
            Debug.Assert(m_type == Type.cId);
            return new Identifier(m_text, m_filepos); 
        }
    }
    
    // Integer   
    internal int m_nValue;
    public int IntValue
    {
        get { Debug.Assert(m_type == Type.cInt); return m_nValue; }
    }
    
    // Chars
    public char CharValue
    {
        get { Debug.Assert(m_type == Type.cChar); return (char) m_nValue; }    
    }
    
    // Rank Specifier
    public int Dimension
    {
        get { Debug.Assert(m_type == Type.cLRSquare); return m_nValue; }        
    }
    
    // Boolean
    internal bool m_fValue;
    public bool BoolValue
    {
        get { Debug.Assert(m_type == Type.cBool); return m_fValue; }
    }
#endregion    
    
}

//-----------------------------------------------------------------------------
// Exceptions to help route errors.
// Used internally to manage control flow when we hit an error. 
// The lexer handles errors by passing a special error token to the outside
// world
//-----------------------------------------------------------------------------
internal class LexerException : ErrorException
{
    internal LexerException(Lexer.ErrorCode c, FileRange location, string stMessage) : 
        base(c, location, stMessage)
    {
    }
}

//-----------------------------------------------------------------------------
// Scanner to produce tokens
//-----------------------------------------------------------------------------

/// <summary>
/// A lexer to convert a <see cref="TextReader"/> into a stream of <see cref="Token"/> objects.
/// </summary>
/// <remarks>
/// <see cref="ILexer"/> implementation used by the <see cref="ManualParser.Parser"/> to parse a source file.
/// Also handles the preprocessor.
/// </remarks>
public class Lexer : ILexer
{   
#region Construction
    // Create a lexer on the given reader
    // The filename is only used to fill out the FileRange structures. (it
    // can be any string)
    public Lexer(string stFilenameHint, TextReader reader)
        : this(stFilenameHint, reader, null)
    {
    
    }
    
    public Lexer(string stFilenameHint, TextReader reader, string[] stDefines)