lexer.hpp

Boost provides free peer-reviewed portable C++ source libraries. We emphasize libraries that work

    : node(x)
    , m_left(x.m_left->clone())
    , m_right(x.m_right->clone())
{
}

inline node *
cat_node::clone() const
{
    return new cat_node(m_left->clone(), m_right->clone());
}

inline bool
cat_node::nullable() const
{
    return m_left->nullable() && m_right->nullable();
}

inline node_set
cat_node::firstpos() const
{
    if (m_left->nullable())
    {
        node_set rval;
        node_set l = m_left->firstpos();
        node_set r = m_right->firstpos();
        std::set_union(l.begin(), l.end(), r.begin(), r.end(),
                std::inserter(rval, rval.begin()));
        return rval;
    }
    else
    {
        return m_left->firstpos();
    }
}

inline node_set
cat_node::lastpos() const
{
    if (m_right->nullable())
    {
        node_set rval;
        node_set l = m_left->lastpos();
        node_set r = m_right->lastpos();
        std::set_union(l.begin(), l.end(), r.begin(), r.end(),
                std::inserter(rval, rval.begin()));
        return rval;
    }
    else
    {
        return m_right->lastpos();
    }
}

inline void
cat_node::compute_followpos(followpos_t& followpos) const
{
    node_set l = m_left->lastpos();
    for (node_set::iterator i = l.begin();
            i != l.end();
            ++i)
    {
        node_set rf = m_right->firstpos();
        followpos[*i].insert(rf.begin(), rf.end());
    }

    m_left->compute_followpos(followpos);
    m_right->compute_followpos(followpos);
}

inline void
cat_node::compute_state_match(state_match_t& state_match) const
{
    m_left->compute_state_match(state_match);
    m_right->compute_state_match(state_match);
}

inline void
cat_node::get_eof_ids(node_set& eof_nodes) const
{
    m_left->get_eof_ids(eof_nodes);
    m_right->get_eof_ids(eof_nodes);
}

inline void
cat_node::assign_node_ids(node_id_t& node_count)
{
    m_left->assign_node_ids(node_count);
    m_right->assign_node_ids(node_count);
}

#if defined(BOOST_SPIRIT_DEBUG) && (BOOST_SPIRIT_DEBUG_FLAGS & BOOST_SPIRIT_DEBUG_FLAGS_SLEX)
inline void
cat_node::dump(std::ostream& out) const
{
    m_left->dump(out);

    out << "\ncat_node";
    out << " nullable() = " << (nullable() ? "true" : "false");
    out << " firstpos() = ";
    node_set fp = firstpos();
    std::copy(fp.begin(), fp.end(),
            std::ostream_iterator<node_id_t>(out, ","));

    out << " lastpos() = ";
    node_set lp = lastpos();
    std::copy(lp.begin(), lp.end(),
            std::ostream_iterator<node_id_t>(out, ","));

    m_right->dump(out);
}
#endif


class star_node : public node
{

public:

    star_node(node* left);
    star_node(const star_node& x);
    virtual ~star_node(){}

    virtual node* clone() const;
    virtual bool nullable() const;
    virtual node_set firstpos() const;
    virtual node_set lastpos() const;
    virtual void compute_followpos(followpos_t& followpos) const;
    virtual void compute_state_match(state_match_t& state_match ) const;
    virtual void get_eof_ids(node_set& eof_set) const;
    virtual void assign_node_ids(node_id_t& node_count);
#if defined(BOOST_SPIRIT_DEBUG) && (BOOST_SPIRIT_DEBUG_FLAGS & BOOST_SPIRIT_DEBUG_FLAGS_SLEX)
    virtual void dump(std::ostream& out) const;
#endif

private:

    std::auto_ptr<node> m_left;
};

inline
star_node::star_node(node* left)
    : node()
    , m_left(left)
{
}

inline
star_node::star_node(const star_node& x)
    : node(x)
    , m_left(x.m_left->clone())
{
}

inline node *
star_node::clone() const
{
    return new star_node(m_left->clone());
}

inline bool
star_node::nullable() const
{
    return true;
}

inline node_set
star_node::firstpos() const
{
    return m_left->firstpos();
}

inline node_set
star_node::lastpos() const
{
    return m_left->lastpos();
}

inline void
star_node::compute_followpos(followpos_t& followpos) const
{
    node_set lp = this->lastpos();
    for (node_set::iterator i = lp.begin();
            i != lp.end();
            ++i)
    {
        node_set fp = this->firstpos();
        followpos[*i].insert(fp.begin(), fp.end());
    }

    m_left->compute_followpos(followpos);
}

inline void
star_node::compute_state_match(state_match_t& state_match) const
{
    m_left->compute_state_match(state_match);
}

inline void
star_node::get_eof_ids(node_set& eof_nodes) const
{
    m_left->get_eof_ids(eof_nodes);
}

inline void
star_node::assign_node_ids(node_id_t& node_count)
{
    m_left->assign_node_ids(node_count);
}

#if defined(BOOST_SPIRIT_DEBUG) && (BOOST_SPIRIT_DEBUG_FLAGS & BOOST_SPIRIT_DEBUG_FLAGS_SLEX)
inline void
star_node::dump(std::ostream& out) const
{
    m_left->dump(out);

    out << "\nstar_node";
    out << " nullable() = " << (nullable() ? "true" : "false");
    out << " firstpos() = ";
    node_set fp = firstpos();
    std::copy(fp.begin(), fp.end(),
            std::ostream_iterator<node_id_t>(out, ","));

    out << " lastpos() = ";
    node_set lp = lastpos();
    std::copy(lp.begin(), lp.end(),
            std::ostream_iterator<node_id_t>(out, ","));

}
#endif


class eof_node : public node
{

public:

    eof_node();
    eof_node(const eof_node& x);
    virtual ~eof_node(){}

    virtual node* clone() const;
    virtual bool nullable() const;
    virtual node_set firstpos() const;
    virtual node_set lastpos() const;
    virtual void compute_followpos(followpos_t& followpos) const;
    virtual void compute_state_match(state_match_t& state_match ) const;
    virtual void get_eof_ids(node_set& eof_set) const;
    virtual void assign_node_ids(node_id_t& node_count);
#if defined(BOOST_SPIRIT_DEBUG) && (BOOST_SPIRIT_DEBUG_FLAGS & BOOST_SPIRIT_DEBUG_FLAGS_SLEX)
    virtual void dump(std::ostream& out) const;
#endif

private:

    node_id_t m_node_num;
};

inline
eof_node::eof_node()
    : node()
    , m_node_num(0)
{
}

inline
eof_node::eof_node(const eof_node& x)
    : node(x)
    , m_node_num(x.m_node_num)
{
}

inline node *
eof_node::clone() const
{
    return new eof_node(*this);
}

inline bool
eof_node::nullable() const
{
    return false;
}

inline node_set
eof_node::firstpos() const
{
    node_set rval;
    rval.insert(m_node_num);
    return rval;
}

inline node_set
eof_node::lastpos() const
{
    node_set rval;
    rval.insert(m_node_num);
    return rval;
}

inline void
eof_node::compute_followpos(followpos_t&) const
{
    return;
}

inline void
eof_node::compute_state_match(state_match_t& state_match) const
{
    if (state_match.size() < m_node_num + 1)
        state_match.resize(m_node_num + 1);
    state_match[m_node_num].resize(256, 0);
}

inline void
eof_node::get_eof_ids(node_set& eof_nodes) const
{
    eof_nodes.insert(m_node_num);
}

inline void
eof_node::assign_node_ids(node_id_t& node_count)
{
    m_node_num = node_count++;
}

#if defined(BOOST_SPIRIT_DEBUG) && (BOOST_SPIRIT_DEBUG_FLAGS & BOOST_SPIRIT_DEBUG_FLAGS_SLEX)
inline void
eof_node::dump(std::ostream& out) const
{
    out << "\neof_node";
    out << " m_node_num = " << m_node_num;
    out << " nullable() = " << (nullable() ? "true" : "false");
    out << " firstpos() = ";
    node_set fp = firstpos();
    std::copy(fp.begin(), fp.end(),
            std::ostream_iterator<node_id_t>(out, ","));

    out << " lastpos() = ";
    node_set lp = lastpos();
    std::copy(lp.begin(), lp.end(),
            std::ostream_iterator<node_id_t>(out, ","));
}
#endif

class ccl_node : public node
{

public:

    ccl_node(const std::vector<uchar>& v);
    ccl_node(const uchar c1, const uchar c2);
    ccl_node(const ccl_node& x);
    virtual ~ccl_node(){}

    virtual node* clone() const;
    virtual bool nullable() const;
    virtual node_set firstpos() const;
    virtual node_set lastpos() const;
    virtual void compute_followpos(followpos_t& followpos) const;
    virtual void compute_state_match(state_match_t& state_match ) const;
    virtual void get_eof_ids(node_set& eof_set) const;
    virtual void assign_node_ids(node_id_t& node_count);
#if defined(BOOST_SPIRIT_DEBUG) && (BOOST_SPIRIT_DEBUG_FLAGS & BOOST_SPIRIT_DEBUG_FLAGS_SLEX)
    virtual void dump(std::ostream& out) const;
#endif

private:

    std::vector<uchar> m_match;
    node_id_t m_node_num;
};

inline
ccl_node::ccl_node(const std::vector<uchar>& v)
    : node()
    , m_match(v)
    , m_node_num(0)
{
    m_match.resize(256); // make sure it's the right size
}

inline
ccl_node::ccl_node(const uchar c1, const uchar c2)
    : node()
    , m_match(256, uchar(0))
    , m_node_num(0)
{
    BOOST_ASSERT(c1 < c2);
    for (std::size_t i = c1; i <= std::size_t(c2); ++i)
    {
        m_match[i] = 1;
    }
}

inline
ccl_node::ccl_node(const ccl_node& x)
    : node(x)
    , m_match(x.m_match)
    , m_node_num(x.m_node_num)
{
}

inline node *
ccl_node::clone() const
{
    return new ccl_node(*this);
}

inline bool
ccl_node::nullable() const
{
    return false;
}

inline node_set
ccl_node::firstpos() const
{
    node_set rval;
    rval.insert(m_node_num);
    return rval;
}

inline node_set
ccl_node::lastpos() const
{
    return firstpos();
}

inline void
ccl_node::compute_followpos(followpos_t&) const
{
    return;
}

inline void
ccl_node::compute_state_match(state_match_t& state_match) const
{
    if (state_match.size() < m_node_num + 1)
        state_match.resize(m_node_num + 1);
    state_match[m_node_num] = m_match;
}

inline void
ccl_node::get_eof_ids(node_set&) const
{
    return;
}

inline void
ccl_node::assign_node_ids(node_id_t& node_count)
{
    m_node_num = node_count++;
}

#if defined(BOOST_SPIRIT_DEBUG) && (BOOST_SPIRIT_DEBUG_FLAGS & BOOST_SPIRIT_DEBUG_FLAGS_SLEX)
inline void
ccl_node::dump(std::ostream& out) const
{
    out << "\nccl_node m_match = ";
    for (std::size_t i = 0; i < m_match.size(); ++i)
    {
        if (m_match[i])
            out << i << ", ";
    }
    out << " m_node_num = " << m_node_num;
    out << " nullable() = " << (nullable() ? "true" : "false");
    out << " firstpos() = ";
    node_set fp = firstpos();
    std::copy(fp.begin(), fp.end(),
            std::ostream_iterator<node_id_t>(out, ","));

    out << " lastpos() = ";
    node_set lp = lastpos();
    std::copy(lp.begin(), lp.end(),
            std::ostream_iterator<node_id_t>(out, ","));
}
#endif

template <typename ScannerT>
class make_concat
{
    typedef typename ScannerT::iterator_t iterator_type;

public:

    make_concat(std::stack<node*>& the_stack)
        : m_stack(the_stack)
        {}

    void operator()(iterator_type const &, iterator_type const &) const
    {
        node* right = m_stack.top();
        m_stack.pop();
        node* left = m_stack.top();
        m_stack.pop();
        node* newnode = new cat_node(left, right);
        m_stack.push(newnode);