token_ids.hpp

来自「support vector clustering for vc++」· HPP 代码 · 共 346 行 · 第 1/2 页

/*=============================================================================
    Boost.Wave: A Standard compliant C++ preprocessor library
    The definition of a default set of token identifiers and related 
    functions.
    
    http://www.boost.org/

    Copyright (c) 2001-2007 Hartmut Kaiser. Distributed under the Boost
    Software License, Version 1.0. (See accompanying file
    LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
=============================================================================*/

#if !defined(TOKEN_IDS_HPP_414E9A58_F079_4789_8AFF_513815CE475B_INCLUDED)
#define TOKEN_IDS_HPP_414E9A58_F079_4789_8AFF_513815CE475B_INCLUDED

#include <string>

#include <boost/wave/wave_config.hpp>

// this must occur after all of the includes and before any code appears
#ifdef BOOST_HAS_ABI_HEADERS
#include BOOST_ABI_PREFIX
#endif

///////////////////////////////////////////////////////////////////////////////
//  Allow external redefinition of the token identifiers to use
#if !defined(BOOST_WAVE_TOKEN_IDS_DEFINED)
#define BOOST_WAVE_TOKEN_IDS_DEFINED

#if defined (__FreeBSD__) && defined (T_DIVIDE)
#undef T_DIVIDE
#endif

///////////////////////////////////////////////////////////////////////////////
namespace boost {
namespace wave {

///////////////////////////////////////////////////////////////////////////////
//  assemble tokenid's
#define TOKEN_FROM_ID(id, cat)   ((id) | (cat))
#define ID_FROM_TOKEN(tok)       ((tok) & ~TokenTypeMask)
#define BASEID_FROM_TOKEN(tok)   ((tok) & ~ExtTokenTypeMask)

///////////////////////////////////////////////////////////////////////////////
//  the token_category helps to classify the different token types 
enum token_category {
    IdentifierTokenType         = 0x10000000,
    ParameterTokenType          = 0x11000000,
    ExtParameterTokenType       = 0x11100000,
    KeywordTokenType            = 0x20000000,
    OperatorTokenType           = 0x30000000,
    LiteralTokenType            = 0x40000000,
    IntegerLiteralTokenType     = 0x41000000,
    FloatingLiteralTokenType    = 0x42000000,
    StringLiteralTokenType      = 0x43000000,
    CharacterLiteralTokenType   = 0x44000000,
    BoolLiteralTokenType        = 0x45000000,
    PPTokenType                 = 0x50000000,
    PPConditionalTokenType      = 0x50800000,

    UnknownTokenType            = 0xA0000000,
    EOLTokenType                = 0xB0000000,
    EOFTokenType                = 0xC0000000,
    WhiteSpaceTokenType         = 0xD0000000,
    InternalTokenType           = 0xE0000000,
    
    TokenTypeMask               = 0xFF000000,
    AltTokenType                = 0x00100000,
    TriGraphTokenType           = 0x00200000,
    AltExtTokenType             = 0x00500000,   // and, bit_and etc.
    ExtTokenTypeMask            = 0xFFF00000,
    ExtTokenOnlyMask            = 0x00F00000,
    TokenValueMask              = 0x000FFFFF,
    MainTokenMask               = 0xFF0FFFFF    // TokenTypeMask|TokenValueMask
};

///////////////////////////////////////////////////////////////////////////////
//  the token_id assigns unique numbers to the different C++ lexemes
enum token_id {
    T_UNKNOWN      = 0,
    T_FIRST_TOKEN  = 256,
    T_AND          = TOKEN_FROM_ID(T_FIRST_TOKEN, OperatorTokenType),
    T_AND_ALT      = TOKEN_FROM_ID(T_FIRST_TOKEN, OperatorTokenType|AltExtTokenType),
    T_ANDAND       = TOKEN_FROM_ID(257, OperatorTokenType),
    T_ANDAND_ALT   = TOKEN_FROM_ID(257, OperatorTokenType|AltExtTokenType),
    T_ASSIGN       = TOKEN_FROM_ID(258, OperatorTokenType),
    T_ANDASSIGN    = TOKEN_FROM_ID(259, OperatorTokenType),
    T_ANDASSIGN_ALT     = TOKEN_FROM_ID(259, OperatorTokenType|AltExtTokenType),
    T_OR           = TOKEN_FROM_ID(260, OperatorTokenType),
    T_OR_ALT       = TOKEN_FROM_ID(260, OperatorTokenType|AltExtTokenType),
    T_OR_TRIGRAPH  = TOKEN_FROM_ID(260, OperatorTokenType|TriGraphTokenType),
    T_ORASSIGN     = TOKEN_FROM_ID(261, OperatorTokenType),
    T_ORASSIGN_ALT          = TOKEN_FROM_ID(261, OperatorTokenType|AltExtTokenType),
    T_ORASSIGN_TRIGRAPH     = TOKEN_FROM_ID(261, OperatorTokenType|TriGraphTokenType),
    T_XOR          = TOKEN_FROM_ID(262, OperatorTokenType),
    T_XOR_ALT      = TOKEN_FROM_ID(262, OperatorTokenType|AltExtTokenType),
    T_XOR_TRIGRAPH = TOKEN_FROM_ID(262, OperatorTokenType|TriGraphTokenType),
    T_XORASSIGN    = TOKEN_FROM_ID(263, OperatorTokenType),
    T_XORASSIGN_ALT         = TOKEN_FROM_ID(263, OperatorTokenType|AltExtTokenType),
    T_XORASSIGN_TRIGRAPH    = TOKEN_FROM_ID(263, OperatorTokenType|TriGraphTokenType),
    T_COMMA        = TOKEN_FROM_ID(264, OperatorTokenType),
    T_COLON        = TOKEN_FROM_ID(265, OperatorTokenType),
    T_DIVIDE       = TOKEN_FROM_ID(266, OperatorTokenType),
    T_DIVIDEASSIGN = TOKEN_FROM_ID(267, OperatorTokenType),
    T_DOT          = TOKEN_FROM_ID(268, OperatorTokenType),
    T_DOTSTAR      = TOKEN_FROM_ID(269, OperatorTokenType),
    T_ELLIPSIS     = TOKEN_FROM_ID(270, OperatorTokenType),
    T_EQUAL        = TOKEN_FROM_ID(271, OperatorTokenType),
    T_GREATER      = TOKEN_FROM_ID(272, OperatorTokenType),
    T_GREATEREQUAL = TOKEN_FROM_ID(273, OperatorTokenType),
    T_LEFTBRACE    = TOKEN_FROM_ID(274, OperatorTokenType),
    T_LEFTBRACE_ALT         = TOKEN_FROM_ID(274, OperatorTokenType|AltTokenType),
    T_LEFTBRACE_TRIGRAPH    = TOKEN_FROM_ID(274, OperatorTokenType|TriGraphTokenType),
    T_LESS         = TOKEN_FROM_ID(275, OperatorTokenType),
    T_LESSEQUAL    = TOKEN_FROM_ID(276, OperatorTokenType),
    T_LEFTPAREN    = TOKEN_FROM_ID(277, OperatorTokenType),
    T_LEFTBRACKET  = TOKEN_FROM_ID(278, OperatorTokenType),
    T_LEFTBRACKET_ALT       = TOKEN_FROM_ID(278, OperatorTokenType|AltTokenType),
    T_LEFTBRACKET_TRIGRAPH  = TOKEN_FROM_ID(278, OperatorTokenType|TriGraphTokenType),
    T_MINUS        = TOKEN_FROM_ID(279, OperatorTokenType),
    T_MINUSASSIGN  = TOKEN_FROM_ID(280, OperatorTokenType),
    T_MINUSMINUS   = TOKEN_FROM_ID(281, OperatorTokenType),
    T_PERCENT      = TOKEN_FROM_ID(282, OperatorTokenType),
    T_PERCENTASSIGN = TOKEN_FROM_ID(283, OperatorTokenType),
    T_NOT          = TOKEN_FROM_ID(284, OperatorTokenType),
    T_NOT_ALT      = TOKEN_FROM_ID(284, OperatorTokenType|AltExtTokenType),
    T_NOTEQUAL     = TOKEN_FROM_ID(285, OperatorTokenType),
    T_NOTEQUAL_ALT      = TOKEN_FROM_ID(285, OperatorTokenType|AltExtTokenType),
    T_OROR         = TOKEN_FROM_ID(286, OperatorTokenType),
    T_OROR_ALT     = TOKEN_FROM_ID(286, OperatorTokenType|AltExtTokenType),
    T_OROR_TRIGRAPH     = TOKEN_FROM_ID(286, OperatorTokenType|TriGraphTokenType),
    T_PLUS         = TOKEN_FROM_ID(287, OperatorTokenType),
    T_PLUSASSIGN   = TOKEN_FROM_ID(288, OperatorTokenType),
    T_PLUSPLUS     = TOKEN_FROM_ID(289, OperatorTokenType),
    T_ARROW        = TOKEN_FROM_ID(290, OperatorTokenType),
    T_ARROWSTAR    = TOKEN_FROM_ID(291, OperatorTokenType),
    T_QUESTION_MARK = TOKEN_FROM_ID(292, OperatorTokenType),
    T_RIGHTBRACE   = TOKEN_FROM_ID(293, OperatorTokenType),
    T_RIGHTBRACE_ALT        = TOKEN_FROM_ID(293, OperatorTokenType|AltTokenType),
    T_RIGHTBRACE_TRIGRAPH   = TOKEN_FROM_ID(293, OperatorTokenType|TriGraphTokenType),
    T_RIGHTPAREN   = TOKEN_FROM_ID(294, OperatorTokenType),
    T_RIGHTBRACKET = TOKEN_FROM_ID(295, OperatorTokenType),
    T_RIGHTBRACKET_ALT      = TOKEN_FROM_ID(295, OperatorTokenType|AltTokenType),
    T_RIGHTBRACKET_TRIGRAPH = TOKEN_FROM_ID(295, OperatorTokenType|TriGraphTokenType),
    T_COLON_COLON  = TOKEN_FROM_ID(296, OperatorTokenType),
    T_SEMICOLON    = TOKEN_FROM_ID(297, OperatorTokenType),
    T_SHIFTLEFT    = TOKEN_FROM_ID(298, OperatorTokenType),
    T_SHIFTLEFTASSIGN = TOKEN_FROM_ID(299, OperatorTokenType),
    T_SHIFTRIGHT   = TOKEN_FROM_ID(300, OperatorTokenType),
    T_SHIFTRIGHTASSIGN = TOKEN_FROM_ID(301, OperatorTokenType),
    T_STAR         = TOKEN_FROM_ID(302, OperatorTokenType),
    T_COMPL        = TOKEN_FROM_ID(303, OperatorTokenType),
    T_COMPL_ALT         = TOKEN_FROM_ID(303, OperatorTokenType|AltExtTokenType),
    T_COMPL_TRIGRAPH    = TOKEN_FROM_ID(303, OperatorTokenType|TriGraphTokenType),
    T_STARASSIGN   = TOKEN_FROM_ID(304, OperatorTokenType),
    T_ASM          = TOKEN_FROM_ID(305, KeywordTokenType),
    T_AUTO         = TOKEN_FROM_ID(306, KeywordTokenType),
    T_BOOL         = TOKEN_FROM_ID(307, KeywordTokenType),
    T_FALSE        = TOKEN_FROM_ID(308, BoolLiteralTokenType),
    T_TRUE         = TOKEN_FROM_ID(309, BoolLiteralTokenType),
    T_BREAK        = TOKEN_FROM_ID(310, KeywordTokenType),
    T_CASE         = TOKEN_FROM_ID(311, KeywordTokenType),
    T_CATCH        = TOKEN_FROM_ID(312, KeywordTokenType),
    T_CHAR         = TOKEN_FROM_ID(313, KeywordTokenType),
    T_CLASS        = TOKEN_FROM_ID(314, KeywordTokenType),
    T_CONST        = TOKEN_FROM_ID(315, KeywordTokenType),
    T_CONSTCAST    = TOKEN_FROM_ID(316, KeywordTokenType),
    T_CONTINUE     = TOKEN_FROM_ID(317, KeywordTokenType),
    T_DEFAULT      = TOKEN_FROM_ID(318, KeywordTokenType),
    T_DEFINED      = TOKEN_FROM_ID(319, KeywordTokenType),
    T_DELETE       = TOKEN_FROM_ID(320, KeywordTokenType),
    T_DO           = TOKEN_FROM_ID(321, KeywordTokenType),
    T_DOUBLE       = TOKEN_FROM_ID(322, KeywordTokenType),