cntokens.pas

CnCodeFormater Readme Author: passos Email: [e-mail]@21cn.com Last Update:20

{******************************************************************************}
{                       CnPack For Delphi/C++Builder                           }
{                     中国人自己的开放源码第三方开发包                         }
{                   (C)Copyright 2001-2009 CnPack 开发组                       }
{                   ------------------------------------                       }
{                                                                              }
{            本开发包是开源的自由软件，您可以遵照 CnPack 的发布协议来修        }
{        改和重新发布这一程序。                                                }
{                                                                              }
{            发布这一开发包的目的是希望它有用，但没有任何担保。甚至没有        }
{        适合特定目的而隐含的担保。更详细的情况请参阅 CnPack 发布协议。        }
{                                                                              }
{            您应该已经和开发包一起收到一份 CnPack 发布协议的副本。如果        }
{        还没有，可访问我们的网站：                                            }
{                                                                              }
{            网站地址：http://www.cnpack.org                                   }
{            电子邮件：[e-mail]                                       }
{                                                                              }
{******************************************************************************}

unit CnTokens;
{* |<PRE>
================================================================================
* 软件名称：CnPack 代码格式化专家
* 单元名称：Object Pascal 词法分析用标记
* 单元作者：CnPack开发组
* 备    注：该单元Object Pascal 词法分析用标记
* 开发平台：Win2003 + Delphi 5.0
* 兼容测试：not test yet
* 本 地 化：本单元内容无需支持本地化字符。
* 单元标识：$Id: CnTokens.pas,v 1.19 2009/01/02 08:36:28 liuxiao Exp $
* 修改记录：2003-12-16 V0.5
                建立。需要补充注释相关的标记。
================================================================================
|</PRE>}

interface

uses
  Classes, SysUtils;

type
  TPascalToken = (
    tokNoToken,
    tokUnknown,

    tokEOF,
    tokBlank,
    tokCRLF,
    tokSymbol,

    // Data Type
    tokChar,
    tokString,
    tokInteger,
    tokFloat,
    tokWString,
    tokComment,

    // Const value
    tokTrue,
    tokFalse,

    // Operator
    tokPlus,
    tokMinus,
    tokStar,
    tokDiv,

    tokGreat,
    tokLess,
    tokGreatOrEqu,
    tokLessOrEqu,
    tokNotEqual,
    tokEQUAL,
    
    // Sign
    tokDot,
    tokHat,
    tokAtSign,

    tokLB,
    tokRB,
    tokSLB,
    tokSRB,
    tokAssign,
    tokColon,
    tokSemicolon, 
    tokComma,
    tokRange, 

    // Keyword
    tokKeyword_BEGIN,
    tokKeywordAbsolute,
    tokKeywordAnd,
    tokKeywordArray,
    tokKeywordAs,
    tokKeywordAsm,
    tokKeywordAutomated,
    tokKeywordBegin,
    tokKeywordCase,
    tokKeywordClass,
    tokKeywordConst,
    tokKeywordConstructor,
    tokKeywordDestructor,
    tokKeywordDispinterface,
    tokKeywordDiv,
    tokKeywordDo,
    tokKeywordDownto,
    tokKeywordElse,
    tokKeywordEnd,
    tokKeywordExcept,
    tokKeywordExports,
    tokKeywordFile,
    tokKeywordFinalization,
    tokKeywordFinally,
    tokKeywordFor,
    tokKeywordFunction,
    tokKeywordGoto,
    tokKeywordIf,
    tokKeywordImplementation,
    tokKeywordIn,
    tokKeywordInherited,
    tokKeywordInitialization,
    tokKeywordInline,
    tokKeywordInterface,
    tokKeywordIs,
    tokKeywordLabel,
    tokKeywordLibrary,
    tokKeywordMod,
    tokKeywordNil,
    tokKeywordNot,
    tokKeywordObject,
    tokKeywordOf,
    tokKeywordOr,
    tokKeywordOn,
    tokKeywordOut,
    tokKeywordPacked,
    tokKeywordPrivate,
    tokKeywordProcedure,
    tokKeywordProgram,
    tokKeywordProperty,
    tokKeywordProtected,
    tokKeywordPublic,
    tokKeywordPublished,
    tokKeywordRaise,
    tokKeywordRecord,
    tokKeywordRepeat,
    tokKeywordResourcestring,
    tokKeywordSealed,
    tokKeywordSet,
    tokKeywordShl,
    tokKeywordShr,
    tokKeywordStatic,
    tokKeywordStrict,
    tokKeywordString,
    tokKeywordThen,
    tokKeywordThreadvar,
    tokKeywordTo,
    tokKeywordTry,
    tokKeywordType,
    tokKeywordUnit,
    tokKeywordUnsafe,
    tokKeywordUntil,
    tokKeywordUses,
    tokKeywordVar,
    tokKeywordWhile,
    tokKeywordWith,
    tokKeywordXor,
    tokKeyword_END,

    tokDirective_BEGIN,
    tokDirectiveABSTRACT,
    tokDirectiveASSEMBLER,
    tokDirectiveAUTOMATED,
    tokDirectiveCDECL,
    tokDirectiveDEPRECATED,
    tokDirectiveDISPID,
    tokDirectiveDYNAMIC,
    tokDirectiveEXPORT,
    tokDirectiveEXTERNAL,
    tokDirectiveFAR,
    tokDirectiveFORWARD,
    tokDirectiveMESSAGE,
    tokDirectiveNEAR,
    tokDirectiveOVERRIDE,
    tokDirectiveOVERLOAD,
    tokDirectivePASCAL,
    tokDirectiveREGISTER,
    tokDirectiveREINTRODUCE,
    tokDirectiveRESIDENT,
    tokDirectivePLATFORM,
    tokDirectiveSAFECALL,
    tokDirectiveSTDCALL,
    tokDirectiveVARARGS,
    tokDirectiveVIRTUAL,
    tokDirective_END,

    tokComplex_BEGIN,
    tokComplexContains,
    tokComplexDefault,
    tokComplexExperimental,
    tokComplexImplements,
    tokComplexIndex,
    tokComplexLocal,
    tokComplexName,
    tokComplexNodefault,
    tokComplexPackage,
    tokComplexRead,
    tokComplexReadonly,
    tokComplexRequires,
    tokComplexResident,
    tokComplexStored,
    tokComplexWrite,
    tokComplexWriteonly,
    tokComplex_END
  );

  TPascalTokenSet = set of TPascalToken;

const
  KeywordTokens = [tokKeyword_BEGIN .. tokKeyword_END];

  // The reserved word inline and are maintained for backward compatibility
  // only. They have no effect on the compiler
  DirectiveTokens = [tokDirective_BEGIN .. tokDirective_END, tokKeywordInline,
    tokKeywordLibrary, tokComplexDefault, tokComplexIndex, tokComplexName,
    tokComplexNoDefault, tokComplexRead, tokComplexReadOnly, tokComplexStored,
    tokComplexWrite, tokComplexWriteOnly];
    
  ComplexTokens = [tokComplex_BEGIN .. tokComplex_END, tokDirectiveMessage,
    tokDirectiveRegister, tokDirectiveForward];
  
  RelOpTokens = [tokGreat, tokLess, tokGreatOrEqu, tokLessOrEqu, tokNotEqual,
    tokEqual, tokKeywordIn, tokKeywordAs, tokKeywordIs];
  AddOPTokens = [tokPlus, tokMinus, tokKeywordOR, tokKeywordXOR];
  MulOpTokens = [tokStar, tokKeywordDIV, tokDiv, tokKeywordMod, tokKeywordAnd];
  ShiftOpTokens = [tokKeywordShl, tokKeywordShr];

  ConstTokens = [tokInteger, tokFloat, tokChar, tokString, tokWString, tokTrue,
    tokFalse, tokKeywordNIL, tokSymbol];
  FactorTokens = [tokSymbol, tokInteger, tokString, tokWString, tokFloat, tokTrue,
    tokFalse, tokKeywordNOT, tokSLB]; //, tokTypeId
  
  ExprTokens = [tokPlus, tokMinus] + FactorTokens;
  SimpStmtTokens = [tokSymbol, tokKeywordGoto, tokKeywordInherited, tokAtSign, tokLB];

  StructStmtTokens = [tokKeywordAsm, tokKeywordBegin, tokKeywordIf,
    tokKeywordCase, tokKeywordFor, tokKeywordWhile, tokKeywordRepeat,
    tokKeywordWith, tokKeywordTry, tokKeywordRaise];

  StmtTokens = [tokKeywordLabel] + SimpStmtTokens + StructStmtTokens;

  RestrictedTypeTokens = [tokKeywordObject, tokKeywordClass, tokKeywordInterface,
    tokKeywordDispinterface];

  StructTypeTokens = [tokKeywordPacked, tokKeywordArray, tokKeywordSet,
    tokKeywordFile, tokKeywordRecord];

  ClassMethodTokens = [tokKeywordClass, tokKeywordProcedure, tokKeywordFunction,
    tokKeywordConstructor, tokKeywordDestructor];

  ClassVisibilityTokens = [tokKeywordPublic, tokKeywordPublished, tokKeywordStrict,
    tokKeywordProtected, tokKeywordPrivate];

  ClassMemberTokens = [tokSymbol, tokKeywordProperty, tokKeywordClass] + ClassMethodTokens;