xlex_lexer.hpp

Boost provides free peer-reviewed portable C++ source libraries. We emphasize libraries that work

    TOKEN_DATA(T_STATICCAST, "static_cast"),
    TOKEN_DATA(T_STATIC, "static"),
    TOKEN_DATA(T_STRUCT, "struct"),
    TOKEN_DATA(T_SWITCH, "switch"),
    TOKEN_DATA(T_TEMPLATE, "template"),
    TOKEN_DATA(T_THIS, "this"),
    TOKEN_DATA(T_THROW, "throw"),
    TOKEN_DATA(T_TRY, "try"),
    TOKEN_DATA(T_TYPEDEF, "typedef"),
    TOKEN_DATA(T_TYPEID, "typeid"),
    TOKEN_DATA(T_TYPENAME, "typename"),
    TOKEN_DATA(T_UNION, "union"),
    TOKEN_DATA(T_UNSIGNED, "unsigned"),
    TOKEN_DATA(T_USING, "using"),
    TOKEN_DATA(T_VIRTUAL, "virtual"),
    TOKEN_DATA(T_VOID, "void"),
    TOKEN_DATA(T_VOLATILE, "volatile"),
    TOKEN_DATA(T_WCHART, "wchar_t"),
    TOKEN_DATA(T_WHILE, "while"),
#if BOOST_WAVE_SUPPORT_MS_EXTENSIONS != 0
    TOKEN_DATA(T_MSEXT_INT8, "__int8"),
    TOKEN_DATA(T_MSEXT_INT16, "__int16"),
    TOKEN_DATA(T_MSEXT_INT32, "__int32"),
    TOKEN_DATA(T_MSEXT_INT64, "__int64"),
    TOKEN_DATA(T_MSEXT_BASED, "_?" "_based"),
    TOKEN_DATA(T_MSEXT_DECLSPEC, "_?" "_declspec"),
    TOKEN_DATA(T_MSEXT_CDECL, "_?" "_cdecl"),
    TOKEN_DATA(T_MSEXT_FASTCALL, "_?" "_fastcall"),
    TOKEN_DATA(T_MSEXT_STDCALL, "_?" "_stdcall"),
    TOKEN_DATA(T_MSEXT_TRY , "__try"),
    TOKEN_DATA(T_MSEXT_EXCEPT, "__except"),
    TOKEN_DATA(T_MSEXT_FINALLY, "__finally"),
    TOKEN_DATA(T_MSEXT_LEAVE, "__leave"),
    TOKEN_DATA(T_MSEXT_INLINE, "_?" "_inline"),
    TOKEN_DATA(T_MSEXT_ASM, "_?" "_asm"),
    TOKEN_DATA(T_MSEXT_PP_REGION, POUNDDEF PPSPACE "region"),
    TOKEN_DATA(T_MSEXT_PP_ENDREGION, POUNDDEF PPSPACE "endregion"),
#endif // BOOST_WAVE_SUPPORT_MS_EXTENSIONS != 0
    TOKEN_DATA(T_PP_DEFINE, POUNDDEF PPSPACE "define"),
    TOKEN_DATA(T_PP_IFDEF, POUNDDEF PPSPACE "ifdef"),
    TOKEN_DATA(T_PP_IFNDEF, POUNDDEF PPSPACE "ifndef"),
    TOKEN_DATA(T_PP_IF, POUNDDEF PPSPACE "if"),
    TOKEN_DATA(T_PP_ELSE, POUNDDEF PPSPACE "else"),
    TOKEN_DATA(T_PP_ELIF, POUNDDEF PPSPACE "elif"),
    TOKEN_DATA(T_PP_ENDIF, POUNDDEF PPSPACE "endif"),
    TOKEN_DATA(T_PP_ERROR, POUNDDEF PPSPACE "error"),
    TOKEN_DATA(T_PP_QHEADER, POUNDDEF PPSPACE \
        INCLUDEDEF PPSPACE Q("\"") "[^\n\r\"]+" Q("\"")),
    TOKEN_DATA(T_PP_HHEADER, POUNDDEF PPSPACE \
        INCLUDEDEF PPSPACE "<" "[^\n\r>]+" ">"),
    TOKEN_DATA(T_PP_INCLUDE, POUNDDEF PPSPACE \
        INCLUDEDEF PPSPACE),
    TOKEN_DATA(T_PP_LINE, POUNDDEF PPSPACE "line"),
    TOKEN_DATA(T_PP_PRAGMA, POUNDDEF PPSPACE "pragma"),
    TOKEN_DATA(T_PP_UNDEF, POUNDDEF PPSPACE "undef"),
    TOKEN_DATA(T_PP_WARNING, POUNDDEF PPSPACE "warning"),
    TOKEN_DATA(T_FLOATLIT, 
        "(" DIGIT "*" Q(".") DIGIT "+" OR DIGIT "+" Q(".") ")" 
        EXPONENT "?" FLOAT_SUFFIX "?" OR
        DIGIT "+" EXPONENT FLOAT_SUFFIX "?"),
    TOKEN_DATA(T_LONGINTLIT, INTEGER LONGINTEGER_SUFFIX),
    TOKEN_DATA(T_INTLIT, INTEGER INTEGER_SUFFIX "?"),
#if BOOST_WAVE_USE_STRICT_LEXER != 0
    TOKEN_DATA(T_IDENTIFIER, "([a-zA-Z_]" OR UNIVERSALCHAR ")([a-zA-Z0-9_]" OR UNIVERSALCHAR ")*"),
#else
    TOKEN_DATA(T_IDENTIFIER, "([a-zA-Z_$]" OR UNIVERSALCHAR ")([a-zA-Z0-9_$]" OR UNIVERSALCHAR ")*"),
#endif
    TOKEN_DATA(T_SPACE, BLANK "+"),
    TOKEN_DATA(T_SPACE2, "[\v\f]+"),
    TOKEN_DATA(T_CONTLINE, Q("\\") "\n"), 
    TOKEN_DATA(T_NEWLINE, NEWLINEDEF),
    TOKEN_DATA(T_POUND_POUND, "##"),
    TOKEN_DATA(T_POUND_POUND_TRIGRAPH, TRI("=") TRI("=")),
    TOKEN_DATA(T_POUND, "#"),
    TOKEN_DATA(T_POUND_TRIGRAPH, TRI("=")),
    TOKEN_DATA(T_ANY_TRIGRAPH, TRI(Q("/"))),
    TOKEN_DATA(T_QUESTION_MARK, Q("?")),
    TOKEN_DATA(T_DOT, Q(".")),
    TOKEN_DATA(T_ANY, "."),
    { token_id(0) }       // this should be the last entry
};

///////////////////////////////////////////////////////////////////////////////
// C++ only token definitions
template <typename Iterator, typename Position>
typename lexer<Iterator, Position>::lexer_data const 
lexer<Iterator, Position>::init_data_cpp[] = 
{
    TOKEN_DATA(T_AND_ALT, "bitand"),
    TOKEN_DATA(T_ANDASSIGN_ALT, "and_eq"),
    TOKEN_DATA(T_ANDAND_ALT, "and"),
    TOKEN_DATA(T_OR_ALT, "bitor"),
    TOKEN_DATA(T_ORASSIGN_ALT, "or_eq"),
    TOKEN_DATA(T_OROR_ALT, "or"),
    TOKEN_DATA(T_XORASSIGN_ALT, "xor_eq"),
    TOKEN_DATA(T_XOR_ALT, "xor"),
    TOKEN_DATA(T_NOTEQUAL_ALT, "not_eq"),
    TOKEN_DATA(T_NOT_ALT, "not"),
    TOKEN_DATA(T_COMPL_ALT, "compl"),
    TOKEN_DATA(T_ARROWSTAR, Q("->") Q("*")),
    TOKEN_DATA(T_DOTSTAR, Q(".") Q("*")),
    TOKEN_DATA(T_COLON_COLON, "::"),
    { token_id(0) }       // this should be the last entry
};

///////////////////////////////////////////////////////////////////////////////
//  undefine macros, required for regular expression definitions
#undef INCLUDEDEF
#undef POUNDDEF
#undef CCOMMENT
#undef PPSPACE
#undef DIGIT
#undef OCTALDIGIT
#undef HEXDIGIT
#undef SIGN
#undef EXPONENT
#undef LONGINTEGER_SUFFIX
#undef INTEGER_SUFFIX
#undef INTEGER
#undef FLOAT_SUFFIX
#undef CHAR_SPEC
#undef BACKSLASH    
#undef ESCAPESEQ    
#undef HEXQUAD      
#undef UNIVERSALCHAR

#undef Q
#undef TRI
#undef OR

#undef TOKEN_DATA
#undef TOKEN_DATA_EX

///////////////////////////////////////////////////////////////////////////////
// initialize cpp lexer 
template <typename Iterator, typename Position>
inline
lexer<Iterator, Position>::lexer(Iterator const &first, 
        Iterator const &last, Position const &pos, 
        boost::wave::language_support language) 
:   first(first), last(last), 
    filename(pos.get_file()), line(0), at_eof(false), language(language)
{
// if in C99 mode, some of the keywords/operators are not valid    
    if (!boost::wave::need_c99(language)) {
        for (int j = 0; 0 != init_data_cpp[j].tokenid; ++j) {
            xlexer.register_regex(init_data_cpp[j].tokenregex, 
                init_data_cpp[j].tokenid, init_data_cpp[j].tokencb);
        }
    }

// tokens valid for C++ and C99    
    for (int i = 0; 0 != init_data[i].tokenid; ++i) {
        xlexer.register_regex(init_data[i].tokenregex, init_data[i].tokenid, 
            init_data[i].tokencb);
    }
}

///////////////////////////////////////////////////////////////////////////////
//  get the next token from the input stream
template <typename Iterator, typename Position>
inline boost::wave::cpplexer::lex_token<Position> 
lexer<Iterator, Position>::get()
{
    using namespace boost::wave;    // to import token ids to this scope

    if (at_eof) 
        return cpplexer::lex_token<Position>();  // return T_EOI

    std::string tokval;
    token_id id = xlexer.next_token(first, last, tokval);
    string_type value = tokval.c_str();
    
    if ((token_id)(-1) == id)
        id = T_EOF;     // end of input reached

    if (T_IDENTIFIER == id) {
    // test identifier characters for validity (throws if invalid chars found)
        if (!boost::wave::need_no_character_validation(language)) {
            cpplexer::impl::validate_identifier_name(value, line, -1, filename); 
        }
    }
    else if (T_STRINGLIT == id || T_CHARLIT == id) {
    // test literal characters for validity (throws if invalid chars found)
        if (!boost::wave::need_no_character_validation(language)) {
            cpplexer::impl::validate_literal(value, line, -1, filename); 
        }
    }
    else if (T_EOF == id) {
    // T_EOF is returned as a valid token, the next call will return T_EOI,
    // i.e. the actual end of input
        at_eof = true;
        value.clear();
    }
    
#if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0
    return guards.detect_guard(cpplexer::lex_token<Position>(id, value, 
        Position(filename, line, -1)));
#else
    return cpplexer::lex_token<Position>(id, value, Position(filename, line, -1));
#endif
}

///////////////////////////////////////////////////////////////////////////////
//   
//  lex_functor
//   
///////////////////////////////////////////////////////////////////////////////
template <
    typename Iterator, 
    typename Position = boost::wave::util::file_position_type
>
class xlex_functor 
:   public xlex_input_interface<typename lexer<Iterator, Position>::token_type>
{    
public:

    typedef typename lexer<Iterator, Position>::token_type   token_type;
    
    xlex_functor(Iterator const &first, Iterator const &last, 
            Position const &pos, boost::wave::language_support language)
    :   lexer(first, last, pos, language)
    {}
    virtual ~xlex_functor() {}
    
// get the next token from the input stream
    token_type get() { return lexer.get(); }
    void set_position(Position const &pos) { lexer.set_position(pos); }

#if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0
    bool has_include_guards(std::string& guard_name) const 
        { return lexer.has_include_guards(guard_name); }
#endif    

private:
    lexer<Iterator, Position> lexer;
};

}   // namespace lexer

///////////////////////////////////////////////////////////////////////////////
//  
//  The new_lexer_gen<>::new_lexer function (declared in cpp_slex_token.hpp)
//  should be defined inline, if the lex_functor shouldn't be instantiated 
//  separately from the lex_iterator.
//
//  Separate (explicit) instantiation helps to reduce compilation time.
//
///////////////////////////////////////////////////////////////////////////////

#if BOOST_WAVE_SEPARATE_LEXER_INSTANTIATION != 0
#define BOOST_WAVE_XLEX_NEW_LEXER_INLINE
#else
#define BOOST_WAVE_XLEX_NEW_LEXER_INLINE inline
#endif 

///////////////////////////////////////////////////////////////////////////////
//
//  The 'new_lexer' function allows the opaque generation of a new lexer object.
//  It is coupled to the iterator type to allow to decouple the lexer/iterator 
//  configurations at compile time.
//
//  This function is declared inside the xlex_interface.hpp file, which is 
//  referenced by the source file calling the lexer and the source file, which
//  instantiates the lex_functor. But is is defined here, so it will be 
//  instantiated only while compiling the source file, which instantiates the 
//  lex_functor. While the xlex_interface.hpp file may be included everywhere,
//  this file (xlex_lexer.hpp) should be included only once. This allows
//  to decouple the lexer interface from the lexer implementation and reduces 
//  compilation time.
//
///////////////////////////////////////////////////////////////////////////////

template <typename Iterator, typename Position>
BOOST_WAVE_XLEX_NEW_LEXER_INLINE
lex_input_interface<boost::wave::cpplexer::lex_token<Position> > *
new_lexer_gen<Iterator, Position>::new_lexer(Iterator const &first,
    Iterator const &last, Position const &pos, 
    wave::language_support language)
{
    return new lexer::xlex_functor<Iterator, Position>(
        first, last, pos, language);
}

#undef BOOST_WAVE_XLEX_NEW_LEXER_INLINE

///////////////////////////////////////////////////////////////////////////////
}   // namespace xlex
}   // namespace cpplexer
}   // namespace wave
}   // namespace boost
     
#endif // !defined(XLEX_LEXER_HPP)