lexertl_token.hpp

Boost provides free peer-reviewed portable C++ source libraries. We emphasize libraries that work

//  Copyright (c) 2001-2008 Hartmut Kaiser
// 
//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

#if !defined(BOOST_SPIRIT_LEX_LEXERTL_TOKEN_FEB_10_2008_0751PM)
#define BOOST_SPIRIT_LEX_LEXERTL_TOKEN_FEB_10_2008_0751PM

#if defined(_MSC_VER) && (_MSC_VER >= 1020)
#pragma once      // MS compatible compilers support #pragma once
#endif

#include <boost/spirit/home/qi/detail/assign_to.hpp>
#include <boost/spirit/home/support/placeholders.hpp>
#include <boost/spirit/home/support/detail/lexer/generator.hpp>
#include <boost/spirit/home/support/detail/lexer/rules.hpp>
#include <boost/spirit/home/support/detail/lexer/consts.hpp>
#include <boost/fusion/include/vector.hpp>
#include <boost/fusion/include/at.hpp>
#include <boost/fusion/include/value_at.hpp>
#include <boost/detail/iterator.hpp>
#include <boost/variant.hpp>
#include <boost/mpl/vector.hpp>
#include <boost/mpl/insert.hpp>
#include <boost/mpl/begin.hpp>
#include <boost/mpl/bool.hpp>
#include <boost/mpl/identity.hpp>
#include <boost/mpl/if.hpp>
#include <boost/range/iterator_range.hpp>

namespace boost { namespace spirit { namespace lex 
{ 
    ///////////////////////////////////////////////////////////////////////////
    //
    //  The lexertl_token is the type of the objects returned by default by the 
    //  lexertl_iterator.
    //
    //    template parameters:
    //        Iterator        The type of the iterator used to access the
    //                        underlying character stream.
    //        AttributeTypes  A mpl sequence containing the types of all 
    //                        required different token values to be supported 
    //                        by this token type.
    //        HasState        A mpl::bool_ indicating, whether this token type
    //                        should support lexer states.
    //
    //  It is possible to use other token types with the spirit::lex 
    //  framework as well. If you plan to use a different type as your token 
    //  type, you'll need to expose the following things from your token type 
    //  to make it compatible with spirit::lex:
    //
    //    typedefs
    //        iterator_type   The type of the iterator used to access the
    //                        underlying character stream.
    //
    //        id_type         The type of the token id used.
    //
    //    methods
    //        default constructor
    //                        This should initialize the token as an end of 
    //                        input token.
    //        constructors    The prototype of the other required 
    //                        constructors should be:
    //
    //              token(int)
    //                        This constructor should initialize the token as 
    //                        an invalid token (not carrying any specific 
    //                        values)
    //
    //              where:  the int is used as a tag only and its value is 
    //                      ignored
    //
    //                        and:
    //
    //              token(std::size_t id, std::size_t state, 
    //                    iterator_type first, iterator_type last);
    //
    //              where:  id:           token id
    //                      state:        lexer state this token was matched in
    //                      first, last:  pair of iterators marking the matched 
    //                                    range in the underlying input stream 
    //
    //        accessors
    //              id()      return the token id of the matched input sequence
    //
    //              state()   return the lexer state this token was matched in
    //
    //              value()   return the token value
    //
    //  Additionally, you will have to implement a couple of helper functions
    //  in the same namespace as the token type: a comparison operator==() to 
    //  compare your token instances, a token_is_valid() function and different 
    //  construct() function overloads as described below.
    //
    ///////////////////////////////////////////////////////////////////////////
    template <
        typename Iterator = char const*, 
        typename AttributeTypes = mpl::vector0<>, 
        typename HasState = mpl::true_
    >
    struct lexertl_token;

    ///////////////////////////////////////////////////////////////////////////
    //  This specialization of the token type doesn't contain any item data and
    //  doesn't support working with lexer states.
    ///////////////////////////////////////////////////////////////////////////
    template <typename Iterator>
    struct lexertl_token<Iterator, omitted, mpl::false_>
    {
        typedef Iterator iterator_type;
        typedef mpl::false_ has_state;
        typedef std::size_t id_type;
        
        //  default constructed tokens correspond to EOI tokens
        lexertl_token() 
          : id_(boost::lexer::npos) 
        {}
        
        //  construct an invalid token
        lexertl_token(int)
          : id_(0) 
        {}
        
        lexertl_token(std::size_t id, std::size_t)
          : id_(id)
        {}
        
        lexertl_token(std::size_t id, std::size_t, 
                Iterator const&, Iterator const&)
          : id_(id)
        {}
        
        //  this default conversion operator is needed to allow the direct 
        //  usage of tokens in conjunction with the primitive parsers defined 
        //  in Qi
        operator std::size_t() const { return id_; }
        
        std::size_t id() const { return id_; }
        std::size_t state() const { return 0; }   // always '0' (INITIAL state)

    protected:
        std::size_t id_;         // token id, 0 if nothing has been matched
    };
    
    ///////////////////////////////////////////////////////////////////////////
    //  This specialization of the token type doesn't contain any item data but
    //  supports working with lexer states.
    ///////////////////////////////////////////////////////////////////////////
    template <typename Iterator>
    struct lexertl_token<Iterator, omitted, mpl::true_>
      : lexertl_token<Iterator, omitted, mpl::false_>
    {
    private:
        typedef lexertl_token<Iterator, omitted, mpl::false_> base_type;

    public:
        typedef Iterator iterator_type;
        typedef mpl::true_ has_state;

        //  default constructed tokens correspond to EOI tokens
        lexertl_token() 
          : state_(boost::lexer::npos) 
        {}
        
        //  construct an invalid token
        lexertl_token(int)
          : base_type(0), state_(boost::lexer::npos) 
        {}
        
        lexertl_token(std::size_t id, std::size_t state)
          : base_type(id, boost::lexer::npos), state_(state)
        {}
        
        lexertl_token(std::size_t id, std::size_t state, 
                Iterator const&, Iterator const&)
          : base_type(id, boost::lexer::npos), state_(state)
        {}
        
        std::size_t state() const { return state_; }

    protected:
        std::size_t state_;      // lexer state this token was matched in
    };
    
    ///////////////////////////////////////////////////////////////////////////
    //  The generic version of the lexertl_token type derives from the 
    //  specialization above and adds a single data member holding the item 
    //  data carried by the token instance.
    ///////////////////////////////////////////////////////////////////////////
    namespace detail
    {
        ///////////////////////////////////////////////////////////////////////
        //  Metafunction to calculate the type of the variant data item to be 
        //  stored with each token instance.
        //
        //  Note: The iterator pair needs to be the first type in the list of 
        //        types supported by the generated variant type (this is being 
        //        used to identify whether the stored data item in a particular 
        //        token instance needs to be converted from the pair of 
        //        iterators (see the first of the construct() functions below).
        ///////////////////////////////////////////////////////////////////////
        template <typename IteratorPair, typename AttributeTypes>
        struct token_value_typesequence
        {
            typedef typename 
                mpl::insert<
                    AttributeTypes, 
                    typename mpl::begin<AttributeTypes>::type, 
                    IteratorPair
                >::type
            sequence_type;
            typedef typename make_variant_over<sequence_type>::type type;
        };
        
        ///////////////////////////////////////////////////////////////////////
        //  The type of the data item stored with a token instance is defined