lexertl_lexer.hpp

Boost provides free peer-reviewed portable C++ source libraries. We emphasize libraries that work

        first = end_token;
        return wave::token_id(id);
    }
    return T_EOF;
}

#if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0
///////////////////////////////////////////////////////////////////////////////
//  load the DFA tables to/from a stream
template <typename Iterator, typename Position>
inline bool
lexertl<Iterator, Position>::load (std::istream& instrm)
{
#if !defined(BOOST_WAVE_LEXERTL_GENERATE_CPP_CODE)
    std::size_t version = 0;
    ::lexertl::serialise::load_as_binary(instrm, state_machine_, version);
    if (version != (std::size_t)get_compilation_time())
        return false;       // too new for us
    return instrm.good();
#else
    return false;   // always create the dfa when generating the C++ code
#endif
}

///////////////////////////////////////////////////////////////////////////////
//  save the DFA tables to/from a stream
template <typename Iterator, typename Position>
inline bool
lexertl<Iterator, Position>::save (std::ostream& outstrm)
{
#if defined(BOOST_WAVE_LEXERTL_GENERATE_CPP_CODE)
    cpp_code::generate(state_machine_, outstrm);
#else
    ::lexertl::serialise::save_as_binary(state_machine_, outstrm, 
        (std::size_t)get_compilation_time());
#endif
    return outstrm.good();
}
#endif // #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0

///////////////////////////////////////////////////////////////////////////////
}   // namespace lexer

///////////////////////////////////////////////////////////////////////////////
template <typename Iterator, typename Position = wave::util::file_position_type>
class lexertl_functor 
:   public lexertl_input_interface<wave::cpplexer::lex_token<Position> >
{
public:
    typedef wave::util::position_iterator<Iterator, Position> iterator_type;
    typedef typename boost::detail::iterator_traits<Iterator>::value_type 
        char_type;
    typedef BOOST_WAVE_STRINGTYPE string_type;
    typedef wave::cpplexer::lex_token<Position> token_type;

    lexertl_functor(Iterator const &first_, Iterator const &last_, 
            Position const &pos_, wave::language_support language)
    :   first(first_, last_, pos_), language(language), at_eof(false)
    {
        lexer_.init_dfa(language, pos_);
    }
    ~lexertl_functor() {}

// get the next token from the input stream
    token_type& get(token_type& result)
    {
        if (lexer_.is_initialized() && !at_eof) {
            do {
            // generate and return the next token
            string_type token_val;
            Position pos = first.get_position();   // begin of token position
            wave::token_id id = lexer_.next_token(first, last, token_val);
            
                if (T_CONTLINE != id) {
                //  The cast should avoid spurious warnings about missing case labels 
                //  for the other token ids's.
                    switch (static_cast<unsigned int>(id)) {   
                    case T_IDENTIFIER:
                    // test identifier characters for validity (throws if 
                    // invalid chars found)
                        if (!wave::need_no_character_validation(language)) {
                            using wave::cpplexer::impl::validate_identifier_name;
                            validate_identifier_name(token_val, 
                                pos.get_line(), pos.get_column(), pos.get_file()); 
                        }
                        break;

                    case T_STRINGLIT:
                    case T_CHARLIT:
                    // test literal characters for validity (throws if invalid 
                    // chars found)
                        if (wave::need_convert_trigraphs(language)) {
                            using wave::cpplexer::impl::convert_trigraphs;
                            token_val = convert_trigraphs(token_val); 
                        }
                        if (!wave::need_no_character_validation(language)) {
                            using wave::cpplexer::impl::validate_literal;
                            validate_literal(token_val, 
                                pos.get_line(), pos.get_column(), pos.get_file()); 
                        }
                        break;
                        
                    case T_LONGINTLIT:  // supported in C99 and long_long mode
                        if (!wave::need_long_long(language)) {
                        // syntax error: not allowed in C++ mode
                            BOOST_WAVE_LEXER_THROW(
                                wave::cpplexer::lexing_exception, 
                                invalid_long_long_literal, token_val.c_str(), 
                                pos.get_line(), pos.get_column(), 
                                pos.get_file().c_str());
                        }
                        break;

#if BOOST_WAVE_SUPPORT_INCLUDE_NEXT != 0
                    case T_PP_HHEADER:
                    case T_PP_QHEADER:
                    case T_PP_INCLUDE:
                    // convert to the corresponding ..._next token, if appropriate
                        {
                        // Skip '#' and whitespace and see whether we find an 
                        // 'include_next' here.
                            typename string_type::size_type start = token_val.find("include");
                            if (0 == token_val.compare(start, 12, "include_next", 12))
                                id = token_id(id | AltTokenType);
                        }
                        break;
#endif // BOOST_WAVE_SUPPORT_INCLUDE_NEXT != 0

                    case T_EOF:
                    // T_EOF is returned as a valid token, the next call will 
                    // return T_EOI, i.e. the actual end of input
                        at_eof = true;
                        token_val.clear();
                        break;
                        
                    case T_OR_TRIGRAPH:
                    case T_XOR_TRIGRAPH:
                    case T_LEFTBRACE_TRIGRAPH:
                    case T_RIGHTBRACE_TRIGRAPH:
                    case T_LEFTBRACKET_TRIGRAPH:
                    case T_RIGHTBRACKET_TRIGRAPH:
                    case T_COMPL_TRIGRAPH:
                    case T_POUND_TRIGRAPH:
                    case T_ANY_TRIGRAPH:
                        if (wave::need_convert_trigraphs(language))
                        {
                            using wave::cpplexer::impl::convert_trigraph;
                            token_val = convert_trigraph(token_val);
                        }
                        break;
                        
                    case T_ANYCTRL:
                        // matched some unexpected character
                        {
                            // 21 is the max required size for a 64 bit integer 
                            // represented as a string
                            char buffer[22];
                            string_type msg("invalid character in input stream: '0x");

                            // for some systems sprintf is in namespace std
                            using namespace std;    
                            sprintf(buffer, "%02x'", token_val[0]);
                            msg += buffer;
                            BOOST_WAVE_LEXER_THROW(
                                wave::cpplexer::lexing_exception, 
                                generic_lexing_error, 
                                msg.c_str(), pos.get_line(), pos.get_column(), 
                                pos.get_file().c_str());
                        }
                        break;
                    }

                    result = token_type(id, token_val, pos);
#if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0
                    return guards.detect_guard(result);
#else
                    return result;
#endif
                }
            } while (true);     // skip the T_CONTLINE token
        }
        return result = token_type();           // return T_EOI
    }

    void set_position(Position const &pos) 
    { 
        // set position has to change the file name and line number only
        first.get_position().set_file(pos.get_file()); 
        first.get_position().set_line(pos.get_line()); 
    }

#if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0
    bool has_include_guards(std::string& guard_name) const 
        { return guards.detected(guard_name); }
#endif    

private:
    iterator_type first;
    iterator_type last;

    wave::language_support language;
    bool at_eof;
#if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0
    include_guards<token_type> guards;
#endif
    
    static lexer::lexertl<iterator_type, Position> lexer_;
};

template <typename Iterator, typename Position>
lexer::lexertl<
    typename lexertl_functor<Iterator, Position>::iterator_type, Position> 
        lexertl_functor<Iterator, Position>::lexer_;

#undef INIT_DATA_SIZE
#undef INIT_DATA_CPP_SIZE
#undef INIT_DATA_PP_NUMBER_SIZE
#undef INIT_MACRO_DATA_SIZE
#undef T_ANYCTRL

///////////////////////////////////////////////////////////////////////////////
//  
//  The new_lexer_gen<>::new_lexer function (declared in lexertl_interface.hpp)
//  should be defined inline, if the lex_functor shouldn't be instantiated 
//  separately from the lex_iterator.
//
//  Separate (explicit) instantiation helps to reduce compilation time.
//
///////////////////////////////////////////////////////////////////////////////

#if BOOST_WAVE_SEPARATE_LEXER_INSTANTIATION != 0
#define BOOST_WAVE_FLEX_NEW_LEXER_INLINE
#else
#define BOOST_WAVE_FLEX_NEW_LEXER_INLINE inline
#endif 

///////////////////////////////////////////////////////////////////////////////
//
//  The 'new_lexer' function allows the opaque generation of a new lexer object.
//  It is coupled to the iterator type to allow to decouple the lexer/iterator 
//  configurations at compile time.
//
//  This function is declared inside the xlex_interface.hpp file, which is 
//  referenced by the source file calling the lexer and the source file, which
//  instantiates the lex_functor. But is is defined here, so it will be 
//  instantiated only while compiling the source file, which instantiates the 
//  lex_functor. While the xlex_interface.hpp file may be included everywhere,
//  this file (xlex_lexer.hpp) should be included only once. This allows
//  to decouple the lexer interface from the lexer implementation and reduces 
//  compilation time.
//
///////////////////////////////////////////////////////////////////////////////

template <typename Iterator, typename Position>
BOOST_WAVE_FLEX_NEW_LEXER_INLINE
wave::cpplexer::lex_input_interface<wave::cpplexer::lex_token<Position> > *
new_lexer_gen<Iterator, Position>::new_lexer(Iterator const &first,
    Iterator const &last, Position const &pos, wave::language_support language)
{
    return new lexertl_functor<Iterator, Position>(first, last, pos, language);
}

#undef BOOST_WAVE_FLEX_NEW_LEXER_INLINE

///////////////////////////////////////////////////////////////////////////////
}}}}   // namespace boost::wave::cpplexer::lexertl

#endif // !defined(BOOST_WAVE_LEXERTL_LEXER_HPP_INCLUDED)