lexer.cs

charp compiler

                // it may throw an exception, and we just want to check for
                // preproc directives. So do it manually.
                if (Lexer.IsFirstIdChar(Peek()))
                {                    
                    st = ReadPreProcSymbol();
                    
                    switch (st)
                    {
                        // Handle nested #if...#endif
                        case "if":
                            cIfDepth++;
                            break;
                        
                        case "endif":
                            if (cIfDepth == 0)
                                return Token.Type.cPP_Endif;
                            cIfDepth--;
                                                   
                            break;                        
                            
                        case "elif": return Token.Type.cPP_ElseIf;
                        case "else": return Token.Type.cPP_Else;
                            
                    }
                }
                // Ignore it
#endif                            
            }
            
            // Discard the rest of the line and try again    
            st = ReadLine(); // null on EOF
        } while(st != null);
        
        return Token.Type.cEOF;
        
    }
    

//-----------------------------------------------------------------------------
// Skip to the next #endif, being nest aware (ie, skip over nested #if..#endif)
//-----------------------------------------------------------------------------
    protected void SkipToEndif()
    {    
        int iRowBefore = m_row;
        Token.Type tt;
        do
        {               
            // Just eat everything in our path...
            tt = GetNextDeadToken();
            if (tt == Token.Type.cEOF)                 
                ThrowError(E_MissingEndifBeforeEOF());
                
        } while (tt != Token.Type.cPP_Endif);
    }
    
//-----------------------------------------------------------------------------
// Skip for the #if on false. We skip to: #elif, #else, #endif
// and then act accordingly. Return which token we land on.
// Must be nest aware.
//-----------------------------------------------------------------------------    
    protected Token.Type SkipWhenIfIsFalse()    
    {   
        int iRowBefore = m_row;  
                
        do
        {
            Token.Type tt = GetNextDeadToken();
            if (tt == Token.Type.cEOF)                 
                ThrowError(E_MissingEndifBeforeEOF());
            
            // If we hit a #elif/#else/#endif on the top level, return it
            
            if ((tt == Token.Type.cPP_Else) || 
                (tt == Token.Type.cPP_ElseIf) ||
                (tt == Token.Type.cPP_Endif))
                    return tt;                  
        } while(true);
    }
    
//-----------------------------------------------------------------------------
// Preprocessor filter. Intercepts the token stream to do pre-processing.
//-----------------------------------------------------------------------------    
    int m_cRegionDepth = 0;
    
    protected Token GetNextToken_PreprocessorFilter()
    {
        do
        {
            Token t = GetNextTokenWorker();
            
            // If this isn't a preprocesor token, we can just pass it straight through
            if (!t.IsPreprocToken)
                return t;
                
            switch(t.TokenType)
            {
            // Control flow
            // '#if' exp
            case Token.Type.cPP_If:
            {         
// @dogfood - need to implement a 'goto case X', and then remove this label.
PREPROC_IF:            
                bool f = this.ReadBooleanExp();
                EnsureAtEOL();
                if (f)
                {
                    // If the (exp) is true, then we just ignore the #if, and
                    // continue returning tokens
                    break;
                }
                else 
                {                    
                    // If the (exp) is false, then we have to stategically skip.
                    Token.Type tt = SkipWhenIfIsFalse();
                    // For #else, #endif, resume normally
                    
                    // but for #elif we have to evaluate the expression.
                    // This means our lexer had better have left us off before the start of the expression
                    // So at this point, #elsif behaves just like #if, so we go back to the start
                    // of the case.
                    if (tt == Token.Type.cPP_ElseIf)
                    {
                        //goto case Token.Type.cPP_If;
                        goto PREPROC_IF;
                    }
                }                    
            }
                break;
                
            // If we're hitting a #else/#elif live, then we must have just executed
            // an #if, so we can safely skip to the #endif
            // '#elif' exp
            // '#else'            
            case Token.Type.cPP_ElseIf:
            case Token.Type.cPP_Else:
                EnsureAtEOL();
                
                SkipToEndif();
                break;

            // If we hit the #endif live, we can ignore it. (We just stay live).
            // #endif is really only useful to terminate skipping.
            // '#endif'            
            case Token.Type.cPP_Endif:
                EnsureAtEOL();
                break;
                
            // Modify the lexer's set of defined symbols            
            // '#define' symbol
            case Token.Type.cPP_Define:
            {
                string st = this.ReadPreProcSymbol();
                AddSymbol(st);
                EnsureAtEOL();
            }
                break;
                
            // '#undef' symbol
            case Token.Type.cPP_Undef:
            {
                string st = this.ReadPreProcSymbol();
                EnsureAtEOL();
                RemoveSymbol(st);            
            }
                break;
            
            // Regions - really just cosmetic
            // '#region' implicit comment to eol
            case Token.Type.cPP_Region:
                ConsumeRestOfLine();
                m_cRegionDepth++;
                break;
                
            // '#endregion' implicit comment to eol            
            case Token.Type.cPP_EndRegion:
                ConsumeRestOfLine();
                m_cRegionDepth--;
                if (m_cRegionDepth < 0)
                {                    
                    ThrowError(E_MissingEndRegion());
                }
                    
                break;            
            
            }
            
        } while(true);
            
    }
#endregion

#region Preprocessor Symbols
//-----------------------------------------------------------------------------
// Manage Symbols for preprocessor
//-----------------------------------------------------------------------------    
    protected Hashtable m_tblPreprocSymbols;
    void AddSymbol(string st)
    {
        m_tblPreprocSymbols.Add(st, null); // don't care what the value is.
    }
    
    void RemoveSymbol(string st)
    {
        m_tblPreprocSymbols.Remove(st);
    }
    
    bool IsSymbolDefined(string st)
    {
        return m_tblPreprocSymbols.ContainsKey(st);
    }
#endregion    
    
#region Preprocessor Parsing helpers    
//-----------------------------------------------------------------------------
// The preprocessor needs a super-mini parser in it.
//-----------------------------------------------------------------------------    
    
    // Skip past whitespace on this line (not including newling
    void SkipWhiteSpace()
    {
        int iCh = Peek();
        while (IsWhitespace(iCh) && (iCh != '\n'))
        {
            Read();
            iCh = Peek();
        }
    }
    
    // For #if, #elif
    bool ReadBooleanExp()
    {
        string st = ReadPreProcSymbol();
        if (st == "true")
            return true;
        if (st == "false")  
            return false;
        return IsSymbolDefined(st);
    }
    
    // Read an expected preproc symbol, return as a string.
    string ReadPreProcSymbol()
    {
        SkipWhiteSpace();
        
        int iCh = Peek();
        Debug.Assert(IsFirstIdChar(iCh));
        
        System.Text.StringBuilder sb = new System.Text.StringBuilder();
        sb.Append((char) iCh);
        Read();
                
        iCh = Peek();
        while(IsIdChar(iCh))
        {            
            sb.Append((char) iCh);
            Read(); // consume                
            iCh = Peek();
        }
        
        return sb.ToString();
    }
    
    // Make sure that we're at the end of the line (and that we don't have any extra tokens
    // Throw exception if not.
    void EnsureAtEOL()
    {
        // @todo - skip comments?
        SkipWhiteSpace();
        Debug.Assert(Peek() == '\n');
    }
    
    // Used when we don't care what's after us.
    void ConsumeRestOfLine()
    {
        ReadLine();
    }
#endregion
    
#endregion    
    
//-----------------------------------------------------------------------------
// Do the real work to get the next token
// Returns preprocessor tokens. 
//-----------------------------------------------------------------------------
    protected Token GetNextTokenWorker()
    {   
        int iCh;
               
        do
        {      
            // Record position of start of the lexeme after we've skipped whitespace
            m_StartPos.col = m_col;
            m_StartPos.row = m_row;
                                  
            iCh = Read();
        
            if (IsWhitespace(iCh))
                continue;
                
#region Comments & Division operators        
            // Check for comments
            if (iCh == '/') 
            {
                // Found eol comment, read until eol                    
                if (Peek() == '/') 
                {                
                    do
                    {
                        iCh = Read();                        
                        if (iCh == -1)
                        {                
                            return new Token(Token.Type.cEOF, CalcCurFileRange());
                        }
                    } while (iCh != '\n');
                    
                    continue;
                }  
                                
                // Multiline comment
                // Goes between /* .... */
                else if (Peek() == '*') {
                    Read(); // consume the '*'
                    
                    do 
                    {
                        iCh = Read();
                        if (iCh == -1)
                        {
                            //Debug.Assert(false, "Multiline comment terminated be EOF");
                            //return new Token(Token.Type.cEOF, CalcCurFileRange());
                            ThrowError(E_UnterminatedComment());
                        }
                    } while ((iCh != '*') || (Peek() != '/'));
                    
                    Read(); // consume the '/'
                    continue;
                }   
                
                else if (Peek() == '=')
                {
                    Read(); // consume '='
                    // DivisionEqual symbol
                    m_fStartOfLine = false;
                    return new Token(Token.Type.cDivEqual, CalcCurFileRange());
                }
                         
                else 
                {
                    // Division symbol
                    m_fStartOfLine = false;
                    return new Token(Token.Type.cDiv, CalcCurFileRange());
                }
            }
#endregion

            // Check EOF
            if (iCh == -1)
            {                
                return new Token(Token.Type.cEOF, CalcCurFileRange());
            }

#region Preprocessor tokens
            if (iCh == '#')
            {
                if (!m_fStartOfLine)
                {
                    // @todo - fix this error
                    //Debug.Assert(false, "Preprocessor directives only allowed at start of new line");                
                    ThrowError(E_PreProcDirMustBeAtStartOfLine());                    
                }
                
                // Read the whole preprocessor directive in
                System.Text.StringBuilder sb = new System.Text.StringBuilder();
                sb.Append("#");
                iCh = Peek();
                while(IsIdChar(iCh))
                {
                    Read(); // consume
                    sb.Append((char) iCh);
                    iCh = Peek();
                }
                
                string st = sb.ToString();
                
                object o = m_keywords[st];
                if (o == null)
                {
                    //Debug.Assert(false, "Preproc directive '" + st + "' not valid");
                    ThrowError(E_InvalidPreProcDir(st));
                }
                
                Token.Type e = (Token.Type) o;