cpp_iterator.hpp

来自「Boost provides free peer-reviewed portab」· HPP 代码 · 共 1,775 行 · 第 1/5 页

        token_sequence_type &result);

private:
    ContextT &ctx;              // context, this iterator is associated with
    boost::shared_ptr<base_iteration_context_type> iter_ctx;
    
    bool seen_newline;              // needed for recognizing begin of line
    bool skipped_newline;           // a newline has been skipped since last one
    bool must_emit_line_directive;  // must emit a line directive
    result_type act_token;          // current token
    typename result_type::position_type &act_pos;   // current fileposition (references the macromap)
        
    token_sequence_type unput_queue;     // tokens to be preprocessed again
    token_sequence_type pending_queue;   // tokens already preprocessed
    
    // detect whether to insert additional whitespace in between two adjacent 
    // tokens, which otherwise would form a different token type, if 
    // re-tokenized
    boost::wave::util::insert_whitespace_detection whitespace; 
};

///////////////////////////////////////////////////////////////////////////////
//  eof token
template <typename ContextT>
typename pp_iterator_functor<ContextT>::result_type const
    pp_iterator_functor<ContextT>::eof;

///////////////////////////////////////////////////////////////////////////////
//
//  returned_from_include()
// 
//      Tests if it is necessary to pop the include file context (eof inside
//      a file was reached). If yes, it pops this context. Preprocessing will
//      continue with the next outer file scope.
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline bool 
pp_iterator_functor<ContextT>::returned_from_include()
{
    if (iter_ctx->first == iter_ctx->last && ctx.get_iteration_depth() > 0) {
    // call the include policy trace function
#if BOOST_WAVE_USE_DEPRECIATED_PREPROCESSING_HOOKS != 0
        ctx.get_hooks().returning_from_include_file();
#else
        ctx.get_hooks().returning_from_include_file(ctx.derived());
#endif

    // restore the previous iteration context after finishing the preprocessing 
    // of the included file
        BOOST_WAVE_STRINGTYPE oldfile = iter_ctx->real_filename;
        position_type old_pos (act_pos);
        
    // if this file has include guards handle it as if it had a #pragma once
#if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0
        if (need_include_guard_detection(ctx.get_language())) {
            std::string guard_name;
            if (iter_ctx->first.has_include_guards(guard_name))
                ctx.add_pragma_once_header(ctx.get_current_filename(), guard_name);
        }
#endif 
        iter_ctx = ctx.pop_iteration_context();

        must_emit_line_directive = true;
        seen_newline = true;

    // restore current file position
        act_pos.set_file(iter_ctx->filename);
        act_pos.set_line(iter_ctx->line);
        act_pos.set_column(0);
        
    // restore the actual current file and directory 
#if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0
        ctx.set_current_filename(iter_ctx->real_filename.c_str());
#endif 
        ctx.set_current_directory(iter_ctx->real_filename.c_str());

    // ensure the integrity of the #if/#endif stack
    // report unbalanced #if/#endif now to make it possible to recover properly
        if (iter_ctx->if_block_depth != ctx.get_if_block_depth()) {
            using boost::wave::util::impl::escape_lit;
            BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, unbalanced_if_endif, 
                escape_lit(oldfile).c_str(), old_pos);
        }
        return true;
    }
    return false;
}

///////////////////////////////////////////////////////////////////////////////
//
//  operator()(): get the next preprocessed token
//
//      throws a preprocess_exception, if appropriate
//
///////////////////////////////////////////////////////////////////////////////
namespace impl {

    //  It may be necessary to emit a #line directive either 
    //  - when comments need to be preserved: if the current token is not a 
    //    whitespace, except comments
    //  - when comments are to be skipped: if the current token is not a 
    //    whitespace token.
    template <typename ContextT> 
    bool consider_emitting_line_directive(ContextT const& ctx, token_id id)
    {
        if (need_preserve_comments(ctx.get_language())) {
            if (!IS_CATEGORY(id, EOLTokenType) && !IS_CATEGORY(id, EOFTokenType))
            {
                return true;
            }
        }
        if (!IS_CATEGORY(id, WhiteSpaceTokenType) && 
            !IS_CATEGORY(id, EOLTokenType) && !IS_CATEGORY(id, EOFTokenType))
        {
          return true;
        }
        return false;
    }
}

template <typename ContextT> 
inline typename pp_iterator_functor<ContextT>::result_type const &
pp_iterator_functor<ContextT>::operator()()
{
    using namespace boost::wave;

    // make sure the cwd has been initialized
    ctx.init_context();
    
    // loop over skip able whitespace until something significant is found
    bool was_seen_newline = seen_newline;
    token_id id = T_UNKNOWN;
        
    try {   // catch lexer exceptions
        do {
        // get_next_token assigns result to act_token member
            if (skipped_newline)
                seen_newline = true;
            get_next_token();

        // if comments shouldn't be preserved replace them with newlines
            id = token_id(act_token);
            if (!need_preserve_comments(ctx.get_language()) &&
                (T_CPPCOMMENT == id || context_policies::util::ccomment_has_newline(act_token)))
            {
                act_token.set_token_id(id = T_NEWLINE);
                act_token.set_value("\n");
            }
            
        } while (ctx.get_hooks().may_skip_whitespace(ctx.derived(), act_token, skipped_newline));
    }
    catch (boost::wave::cpplexer::lexing_exception const& e) {
    // dispatch any lexer exceptions to the context hook function
        ctx.get_hooks().throw_exception(ctx.derived(), e);
        return act_token;
    }
        
// if there were skipped any newlines, we must emit a #line directive
    if ((must_emit_line_directive || (was_seen_newline && skipped_newline)) && 
        impl::consider_emitting_line_directive(ctx, id)) 
    {
    // must emit a #line directive
        if (need_emit_line_directives(ctx.get_language()) && emit_line_directive()) 
        {
            skipped_newline = false;
            ctx.get_hooks().may_skip_whitespace(ctx.derived(), act_token, skipped_newline);     // feed ws eater FSM
            id = token_id(act_token);
        }
    }
    
// cleanup of certain tokens required
    seen_newline = false;
    switch (static_cast<unsigned int>(id)) {
    case T_NONREPLACABLE_IDENTIFIER:
        act_token.set_token_id(id = T_IDENTIFIER);
        break;

    case T_GENERATEDNEWLINE:  // was generated by emit_line_directive()
        act_token.set_token_id(id = T_NEWLINE);
        ++iter_ctx->emitted_lines;
        seen_newline = true;
        break;
        
    case T_NEWLINE:
    case T_CPPCOMMENT:
        seen_newline = true;
        ++iter_ctx->emitted_lines;
        break;

    case T_CCOMMENT:          // will come here only if whitespace is preserved
        iter_ctx->emitted_lines += 
            context_policies::util::ccomment_count_newlines(act_token);
        break;
        
    case T_PP_NUMBER:        // re-tokenize the pp-number
        {
            token_sequence_type rescanned;
            
            std::string pp_number(
                util::to_string<std::string>(act_token.get_value()));

            lexer_type it = lexer_type(pp_number.begin(), 
                pp_number.end(), act_token.get_position(), 
                ctx.get_language());
            lexer_type end = lexer_type();
            
            for (/**/; it != end && T_EOF != token_id(*it); ++it) 
                rescanned.push_back(*it);
                
            pending_queue.splice(pending_queue.begin(), rescanned);
            act_token = pending_queue.front();
            id = token_id(act_token);
            pending_queue.pop_front();
        }
        break;
        
    case T_EOF:
        seen_newline = true;
        break;

    default:    // make sure whitespace at line begin keeps seen_newline status
        if (IS_CATEGORY(id, WhiteSpaceTokenType))
            seen_newline = was_seen_newline;
        break;
    }

    if (whitespace.must_insert(id, act_token.get_value())) {
    // must insert some whitespace into the output stream to avoid adjacent
    // tokens, which would form different (and wrong) tokens
        whitespace.shift_tokens(T_SPACE);
        pending_queue.push_front(act_token);        // push this token back
        return act_token = result_type(T_SPACE, 
            typename result_type::string_type(" "), 
            act_token.get_position());
    }
    whitespace.shift_tokens(id);
    return ctx.get_hooks().generated_token(ctx.derived(), act_token);
}

///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline typename pp_iterator_functor<ContextT>::result_type const &
pp_iterator_functor<ContextT>::get_next_token()
{
    using namespace boost::wave;
    
// if there is something in the unput_queue, then return the next token from
// there (all tokens in the queue are preprocessed already)
    if (!pending_queue.empty() || !unput_queue.empty()) 
        return pp_token();      // return next token
    
// test for EOF, if there is a pending input context, pop it back and continue
// parsing with it
bool returned_from_include_file = returned_from_include();
    
// try to generate the next token 
    if (iter_ctx->first != iter_ctx->last) {
        do {
        // If there are pending tokens in the queue, we'll have to return 
        // these. This may happen from a #pragma directive, which got replaced
        // by some token sequence.
            if (!pending_queue.empty()) {
            util::on_exit::pop_front<token_sequence_type> 
                pop_front_token(pending_queue);

                return act_token = pending_queue.front();
            }
            
        // adjust the current position (line and column)
        bool was_seen_newline = seen_newline || returned_from_include_file;

        // fetch the current token        
            act_token = *iter_ctx->first;
            act_pos = act_token.get_position();
            
        // act accordingly on the current token
        token_id id = token_id(act_token);
        
            if (T_EOF == id) {
            // returned from an include file, continue with the next token
                whitespace.shift_tokens(T_EOF);
                ++iter_ctx->first;

            // now make sure this line has a newline
                if ((!seen_newline || act_pos.get_column() > 1) && 
                    !need_single_line(ctx.get_language())) 
                {
                // warn, if this file does not end with a newline
                    BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, 
                        last_line_not_terminated, "", act_pos);
                }
                continue;   // if this is the main file, the while loop breaks
            }
            else if (T_NEWLINE == id || T_CPPCOMMENT == id) {   
            // a newline is to be returned ASAP, a C++ comment too
            // (the C++ comment token includes the trailing newline)
                seen_newline = true;
                ++iter_ctx->first;
                
                if (!ctx.get_if_block_status()) {
                // skip this token because of the disabled #if block
                    whitespace.shift_tokens(id);  // whitespace controller
#if BOOST_WAVE_USE_DEPRECIATED_PREPROCESSING_HOOKS != 0
                    ctx.get_hooks().skipped_token(act_token);
#else
                    ctx.get_hooks().skipped_token(ctx.derived(), act_token);
#endif
                    continue;
                }
                return act_token; 
            }
            seen_newline = false;

            if (was_seen_newline && pp_directive()) {
            // a pp directive was found
//                 seen_newline = true;
//                 must_emit_line_directive = true;

            // loop to the next token to analyze
            // simply fall through, since the iterator was already adjusted 
            // correctly
            }
            else if (ctx.get_if_block_status()) {
            // preprocess this token, eat up more, if appropriate, return 
            // the next preprocessed token
                return pp_token();
            }
            else {
            // compilation condition is false: if the current token is a 
            // newline, account for it, otherwise discard the actual token and 
            // try the next one
                if (T_NEWLINE == token_id(act_token)) {
                    seen_newline = true;
                    must_emit_line_directive = true;
                }

            // next token
#if BOOST_WAVE_USE_DEPRECIATED_PREPROCESSING_HOOKS != 0
                ctx.get_hooks().skipped_token(act_token);
#else
                ctx.get_hooks().skipped_token(ctx.derived(), act_token);
#endif
                ++iter_ctx->first;
            }
            
        } while ((iter_ctx->first != iter_ctx->last) || 
                 (returned_from_include_file = returned_from_include()));

    // overall eof reached
        if (ctx.get_if_block_depth() > 0 && !need_single_line(ctx.get_language())) 
        {
        // missing endif directive(s)
            BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, 
                missing_matching_endif, "", act_pos);