lexertl_token.hpp

Boost provides free peer-reviewed portable C++ source libraries. We emphasize libraries that work

        //  by the template parameter 'AttributeTypes' and may be:
        //  
        //     omitted:           no data item is stored with the token 
        //                        instance (this is handled by the 
        //                        specializations of the lexertl_token class
        //                        below)
        //     mpl::vector0<>:    each token instance stores a pair of 
        //                        iterators pointing to the matched input 
        //                        sequence
        //     mpl::vector<...>:  each token instance stores a variant being 
        //                        able to store the pair of iterators pointing 
        //                        to the matched input sequence, or any of the 
        //                        types a specified in the mpl::vector<>
        //
        //  All this is done to ensure the token type is as small (in terms 
        //  of its byte-size) as possible.
        ///////////////////////////////////////////////////////////////////////
        template <typename IteratorPair, typename AttributeTypes>
        struct token_value_type
        {
            typedef 
                typename mpl::eval_if<
                    is_same<AttributeTypes, mpl::vector0<> >,
                    mpl::identity<IteratorPair>,
                    token_value_typesequence<IteratorPair, AttributeTypes>
                >::type 
            type;
        };
    }

    template <typename Iterator, typename AttributeTypes, typename HasState>
    struct lexertl_token : lexertl_token<Iterator, omitted, HasState>
    {
    private: // precondition assertions
#if !BOOST_WORKAROUND(BOOST_MSVC, <= 1300)
        BOOST_STATIC_ASSERT((mpl::is_sequence<AttributeTypes>::value || 
                            is_same<AttributeTypes, omitted>::value));
#endif
        typedef lexertl_token<Iterator, omitted, HasState> base_type;
        
    protected: 
        //  If no additional token value types are given, the the token will 
        //  hold the plain pair of iterators pointing to the matched range
        //  in the underlying input sequence. Otherwise the token value is 
        //  stored as a variant and will again hold the pair of iterators but
        //  is able to hold any of the given data types as well. The conversion 
        //  from the iterator pair to the required data type is done when it is
        //  accessed for the first time.
        typedef iterator_range<Iterator> iterpair_type;
        typedef 
            typename detail::token_value_type<iterpair_type, AttributeTypes>::type 
        token_value_type;

    public:
        typedef Iterator iterator_type;

        //  default constructed tokens correspond to EOI tokens
        lexertl_token() 
          : value_(iterpair_type(iterator_type(), iterator_type())) 
        {}
        
        //  construct an invalid token
        lexertl_token(int)
          : base_type(0), value_(iterpair_type(iterator_type(), iterator_type())) 
        {}
        
        lexertl_token(std::size_t id, std::size_t state, 
                Iterator first, Iterator last)
          : base_type(id, state), value_(iterpair_type(first, last))
        {}

        token_value_type& value() { return value_; }
        token_value_type const& value() const { return value_; }
        
    protected:
        token_value_type value_; // token value, by default a pair of iterators
    };
    
    ///////////////////////////////////////////////////////////////////////////
    //  tokens are considered equal, if their id's match (these are unique)
    template <typename Iterator, typename AttributeTypes, typename HasState>
    inline bool 
    operator== (lexertl_token<Iterator, AttributeTypes, HasState> const& lhs, 
                lexertl_token<Iterator, AttributeTypes, HasState> const& rhs)
    {
        return lhs.id() == rhs.id();
    }
    
    ///////////////////////////////////////////////////////////////////////////
    //  This overload is needed by the multi_pass/functor_input_policy to 
    //  validate a token instance. It has to be defined in the same namespace 
    //  as the token class itself to allow ADL to find it.
    ///////////////////////////////////////////////////////////////////////////
    template <typename Iterator, typename AttributeTypes, typename HasState>
    inline bool 
    token_is_valid(lexertl_token<Iterator, AttributeTypes, HasState> const& t)
    {
        return 0 != t.id() && std::size_t(boost::lexer::npos) != t.id();
    }
    
    ///////////////////////////////////////////////////////////////////////////
    //  We have to provide overloads for the construct() function allowing
    //  to extract the needed value from the token. These overloads have to be
    //  defined in the same namespace as the token class itself to allow ADL to 
    //  find them.
    ///////////////////////////////////////////////////////////////////////////
    
    //  This is called from the parse function of token_def if the token_def
    //  has been defined to carry a special attribute type
    template <typename Attribute, typename Iterator, typename AttributeTypes,
        typename HasState>
    inline void construct(Attribute& attr, 
        lexertl_token<Iterator, AttributeTypes, HasState>& t)
    {
    //  The goal of this function is to avoid the conversion of the pair of
    //  iterators (to the matched character sequence) into the token value 
    //  of the required type being done more than once. For this purpose it 
    //  checks whether the stored value type is still the default one (pair 
    //  of iterators) and if yes, replaces the pair of iterators with the 
    //  converted value to be returned from subsequent calls.
        
        if (0 == t.value().which()) {
        //  first access to the token value
            typedef iterator_range<Iterator> iterpair_type;
            iterpair_type const& ip = get<iterpair_type>(t.value());
            
        // Interestingly enough we use the assign_to() framework defined in 
        // Spirit.Qi allowing to convert the pair of iterators to almost any 
        // required type (assign_to(), if available, uses the standard Spirit 
        // parsers to do the conversion, and falls back to boost::lexical_cast
        // otherwise).
            qi::detail::assign_to(ip.begin(), ip.end(), attr);

        //  If you get an error during the compilation of the following 
        //  assignment expression, you probably forgot to list one or more 
        //  types used as token value types (in your token_def<...> 
        //  definitions) in your definition of the token class. I.e. any token 
        //  value type used for a token_def<...> definition has to be listed 
        //  during the declaration of the token type to use. For instance let's 
        //  assume we have two token_def's:
        //
        //      token_def<int> number; number = "...";
        //      token_def<std::string> identifier; identifier = "...";
        //
        //  Then you'll have to use the following token type definition 
        //  (assuming you are using the lexertl_token class):
        //
        //      typedef mpl::vector<int, std::string> token_values;
        //      typedef lexertl_token<base_iter_type, token_values> token_type;
        //
        //  where: base_iter_type is the iterator type used to expose the 
        //         underlying input stream.
        //
        //  This token_type has to be used as the second template parameter 
        //  to the lexer class:
        //
        //      typedef lexertl_lexer<base_iter_type, token_type> lexer_type;
        //
        //  again, assuming you're using the lexertl_lexer<> template for your 
        //  tokenization.
        
            t.value() = attr;   // re-assign value
        }
        else {
        // reuse the already assigned value
            qi::detail::assign_to(get<Attribute>(t.value()), attr);
        }
    }
    
    //  This is called from the parse function of token_def if the token type
    //  has no special attribute type assigned 
    template <typename Attribute, typename Iterator, typename HasState>
    inline void construct(Attribute& attr, 
        lexertl_token<Iterator, mpl::vector0<>, HasState>& t)
    {
    //  The default type returned by the token_def parser component (if it
    //  has no token value type assigned) is the pair of iterators to the 
    //  matched character sequence.
        
        qi::detail::assign_to(t.value().begin(), t.value().end(), attr);
    }
    
    //  This is called from the parse function of token_def if the token type
    //  has been explicitly omitted (i.e. no attribute value is used), which
    //  essentially means that every attribute gets initialized using default 
    //  constructed values.
    template <typename Attribute, typename Iterator, typename HasState>
    inline void
    construct(Attribute& attr, lexertl_token<Iterator, omitted, HasState>& t)
    {
    }
    
    //  This is called from the parse function of token_set or lexer_def_
    template <typename Iterator, typename AttributeTypes, typename HasState>
    inline void
    construct(fusion::vector<std::size_t, iterator_range<Iterator> >& attr,
        lexertl_token<Iterator, AttributeTypes, HasState> const& t)
    {
    //  The type returned by the token_set and lexer_def_ parser components
    //  is a fusion::vector containing the token id of the matched token 
    //  and the pair of iterators to the matched character sequence.
        
        typedef iterator_range<Iterator> iterpair_type;
        typedef 
            fusion::vector<std::size_t, iterator_range<Iterator> > 
        attribute_type;
        
        iterpair_type const& ip = get<iterpair_type>(t.value());
        attr = attribute_type(t.id(), get<iterpair_type>(t.value()));
    }

}}}

#endif