utokenparser.pas

DelphiDoc is a program for automatic generation of documentation on a Delphi-Project. At the momen

{  JADD - Just Another DelphiDoc: Documentation from Delphi Source Code

Copyright (C) 2003-2008   Gerold Veith

This file is part of JADD - Just Another DelphiDoc.

DelphiDoc is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License version 3 as
published by the Free Software Foundation.

DelphiDoc is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
}


unit UTokenParser;

{Contains the base classes for parsing pascal files. ~[link EParseException]
 will be raised, whenever a parsing error is encountered.
 ~[link TCommentFileParser] saves a file into internal structures and parses
 if a line starts within a comment for faster access. ~[link TTokenParser]
 defines functions to get the content of the file token by token. It also
 defines a position-stack to save and reset the position, from which the next
 token should be searched.
 ~[UserDoc ..\UserDocumentation.txt ] }


interface

uses Windows, SysUtils, Classes,
     UPascalConsts,
     UBaseIdents;




  { * * *  ***  * * *  ***   EParseException   ***  * * *  ***  * * *  }


type
  //general reason of the error
  TExceptType = (
    //not a pascal file; invalid characters, unbalanced ' or braces of comments
    etNoPascal,
    //the precompiler (conditional compiling) can't parse the file correctly
    etPreCompiler,
    //syntax can't be parsed correctly
    etSyntax,
    //error in parser; f.i. a feature not yet implemented
    etParser,
    //parameters to the parser invalid or library incorrect used
    etParameters,
    //no special kind
    etNone);

      //name/short description of kind of error
const ExceptTypeName: array[TExceptType] of String =
               ('No Pascal Code',  'Pre-Compiler',          'Syntax',
                'Error in Parser', 'Incorrect Utilization', '');





type
  TCommentFileParser = class;

  //the kinds of messages to be generated by the parsers
  TParseMessageKind = (
                       //an error message generated by a parser
                       pmkError,
                       //a warning message generated by a parser
                       pmkWarning,
                       //a hint generated by a parser
                       pmkHint);


  //the kinds of messages to be generated by the parsers
  TParseMessageKinds = set of TParseMessageKind;


  {Contains all information of messages generated while parsing pascal code. }
  TParseMessageInformation = record
    //kind of the message
    MessageKind: TParseMessageKind;

    //general reason of the error/warning
    MessageType: TExceptType;

    //the file in which the message was generated
    TheFile: TPascalFile;
    //the position in that file
    ErrorPosition: TPosition;

    //the effective file in which the message was generated
    EffectiveFile: TPascalFile;
    //the position in the effective file
    EffectiveErrorPosition: TPosition;

    //the error message
    Message: String;
  end;



  {The class for exceptions raised by a parser. It saves the position (file
   name and position in that file) and the general reason of the error. It is
   also used to handle warnings. }
  EParseException = class(Exception)
  private
    //general reason of the error
    FExceptType: TExceptType;

    //the file in which the exception was raised
    FTheFile: TPascalFile;
    //the position in the file where the exception was raised
    FErrorPosition: TPosition;

    //the file in which the exception was raised
    FEffectiveFile: TPascalFile;
    //the position in the effective file
    FEffectiveErrorPosition: TPosition;
  public
    //Creates a new exception object.
    constructor Create(ExceptType: TExceptType; const Msg: String);
    //Creates a new exception object.
    constructor CreatePos(ExceptType: TExceptType; ErrorPosition: TPosition;
                          const Msg: String);

    //Extracts all information about the exception/warning.
    procedure CopyInformationTo(var Information: TParseMessageInformation);




    property ExceptType: TExceptType read FExceptType write FExceptType;

    property TheFile: TPascalFile read FTheFile write FTheFile;
    property ErrorPosition: TPosition read FErrorPosition write FErrorPosition;

    property EffectiveFile: TPascalFile read FEffectiveFile
                                        write FEffectiveFile;
    property EffectiveErrorPosition: TPosition read FEffectiveErrorPosition
                                               write FEffectiveErrorPosition;
  end;









  { * * *  ***  * * *  ***   TCommentFileParser   ***  * * *  ***  * * *  }




  {The base class of all pascal parsers. It has dimished in its usefulness,
   now it only holds the text to be parsed and defines methods to raise errors
   or warnings. I guess it should be deleted and integrated in
   ~[link TTokenParser]. }
  TCommentFileParser = class
  private
    //if the content in FLines has to be freed
    FOwnedContent: Boolean;


    //Gets the number of lines.
    function GetLineCount: Integer;

    //Raises a Parser-Exception (EParseException).
    procedure ExceptionPos(const ErrPos: TPosition;
                           ExceptType: TExceptType; const Msg: String);
  protected
    //the content of the file
    //only used in ~[link UConditionalParser.TConditionalParser] besides here
    FLines: TStrings;

    //the value every position is moved by
    FStartPos: TPosition;




    //Initializes an Parser-Exception-Object before it gets raised.
    function InitExceptionObject(ExcpObj: Exception): Exception; virtual;
    //Handles a warning; here raises the exception.
    procedure HandleWarningMessage(ExcpObj: Exception); virtual;



    //The number of lines of the pascal data.
    property LineCount: Integer read GetLineCount;
  public
    //Creates a new TCommentFileParser-object.
    constructor Create;
    //Destroys the TCommentFileParser-object.
    destructor Destroy; override;


    //Raises a Parser-Exception (EParseException) at an identifier.
    procedure ExceptionIdent(Identifier: Tidentifier;
                             ExceptType: TExceptType; const Msg: String);
    //Raises a Parser-Exception (EParseException) at an identifier.
    procedure ExceptionIdentFmt(Identifier: Tidentifier;
                                ExceptType: TExceptType;
                                const Fmt: String;
                                const Params: array of const);
    //Raises a Parser-Exception (EParseException) at the beginning of the file.
    procedure ExceptionAtStart(ExceptType: TExceptType; const Msg: String);




    //Sets the content of the file by assigning the list.
    procedure SetContentToParse(Content: TStrings);
    //Parses the string.
    procedure ParseString(const Text: String; RelPosRow: Integer = 0;
                                              RelPosColumn: Integer = 1);
  end;









  { * * *  ***  * * *  ***   TTokenParser   ***  * * *  ***  * * *  }


  {Information about positions of tokens. }
  TTokenPosition = record
    //position in this file, when the token has been read
    FPosition: TPosition;
    //the file from which the token has been read
    FEffectiveFile: TPascalFile;
    //the position in the effective file where the token was
    FEffectivePosition: TPosition;
  end;



  {A token and its position for the stack of tokens for look-ahead. }
  TTokenInfo = record
    //the token
    Token: String;
    //position where the token has been read
    Position: TTokenPosition;
  end;



{$UNDEF ListTemplateItemIsObject}
{$UNDEF ListTemplateItemMayBeFreed}
{$UNDEF ListTemplateItemFreedByDefault}

  //the items for the template list to use
  TListTemplateListItem = TTokenInfo;

  //forward declaration to be able to define an alias so the template can
  //reference itself
  TTokenList = class;

  //alias for the list to be used by the implementations of the methods
  TListTemplate = TTokenList;

{$INCLUDE ..\General\Templates\ListTemplate.inc}

  //a list of tokens
  TTokenList

{$INCLUDE ..\General\Templates\ListTemplate.inc}





  {Extends its base class by the ability to extract the content of the file
   without the comments token by token. ~[link GetToken] gets a single token,
   ~[link GetIdentWithPointsToken] gets a chain of identifiers separated by a
   dot '.' and ~[link GetBalancedToken] gets all tokens up to a closing
   brace/bracket if the first token is an opening brace/bracket. The position
   where the search for then next token should be started can be pushed on a
   stack with ~[link PushPosition] and pulled back with ~[link PopPosition] or
   the position can be dropped from the stack with ~[link LosePosition]. }
  TTokenParser = class(TCommentFileParser)
  private
    //the pascal dialect of the file to be parsed
    FPascalDialect: TPascalDialect;


    //The absolute position of the token that has been returned by the last
    //call to one of the Get*Token-functions.
    FAbsoluteLastTokenStartPos: TPosition;
    //The file that contains the token that has been returned by the last call
    //to one of the Get*Token-functions.
    FEffectiveLastTokenFile: TPascalFile;
    //The absolute effective position of the token that has been returned by
    //the last call to one of the Get*Token-functions.
    FEffectiveLastTokenStartPos: TPosition;



    //the position of the last token before the first token on the stack
    FBeforeStackTokenPosition: TTokenPosition;

    //The memory for the position stack.
    FTokenPosStack: PInteger;
    //The size of the memory for the position stack.
    FTokenPosStackSize: Integer;
    //The index of the first unused item in the position stack.
    FTokenPosStackFreeIndex: Integer;

    //The stack for tokens to return by ~[link PopPosition].
    FTokenStack: TTokenList;
    //The index of the next item on the stack to return; -1 = don't use stack.
    FTokenOnStackIndex: Integer;


    //The position where the search for the next token should be
    //started/resumed.
    FTokenSearchPos: TPosition;


    //Saves the position of the last read token.
    procedure SetPositionOfLastToken;

  protected


{$IFOPT C+}
    //Returns if the stack of positions is empty.
    function PositionStackEmpty: Boolean;
{$ENDIF}


    //Sets the effective file and position of the last token.
    procedure SetEffectivePositionOfLastToken(TheFile: TPascalFile;
                                              Pos: TPosition);





    //Returns the next token (after the current position).
    function DoDoGetToken(var ResToken: String): Boolean;

    //Returns the next token (after the current position).
    function DoGetToken(var Token: String): Boolean; virtual;

    //Returns the next token, and if it is a opening brace or bracket the whole
    //expression to the closing brace/bracked is returned.
    function GetBalancedToken(var Token: String): Boolean;

    //Returns the positions of the last returned token.
    function GetLastTokenPositions: TTokenPosition; virtual;



    //The absolute position (moved by FStartPos) of the token that has been
    //returned by the last call to one of the Get*Token-functions.
    property AbsoluteLastTokenStartPos: TPosition
                                               read FAbsoluteLastTokenStartPos;
  public
    //Creates a new TTokenParser-object.
    constructor Create;
    //Frees the position stack.
    destructor Destroy; override;


    //Raises a Parser-Exception at the current absolute position.
    procedure Exception(ExceptType: TExceptType; const Msg: String);
    //Raises a Parser-Exception at the current absolute position.
    procedure ExceptionFmt(ExceptType: TExceptType;
                           const Fmt: String; const Params: array of const);
    //Raises a Parser-Exception (EParseException) at the positions.
    procedure ExceptionPos(const Positions: TTokenPosition;
                           ExceptType: TExceptType; const Msg: String);
    //Raises a Parser-Exception (EParseException) at the positions.
    procedure ExceptionPosFmt(const Positions: TTokenPosition;
                              ExceptType: TExceptType;
                              const Fmt: String; const Params: array of const);
    //Generates a warning message at the current absolute position.
    procedure WarningMessage(WarnType: TExceptType; const Msg: String);
    //Generates a formatted warning message at the current absolute position.
    procedure WarningMessageFmt(WarnType: TExceptType;
                                const Fmt: String;
                                const Params: array of const);



    //Returns the next token (after the current position).
    function GetToken(var Token: String): Boolean;
    //Returns the next token, and if it is a chain of identifiers separated by
    //dots, the whole chain is returned.
    function GetIdentWithPointsToken(var Token: String): Boolean;



    //Puts the position to start the search for the next token on the stack.
    procedure PushPosition;
    //Pulls the position to start the search for the next token from the stack.
    procedure PopPosition;
    //Deletes the topmost position from the stack.
    procedure LosePosition;


    property PascalDialect: TPascalDialect read FPascalDialect
                                           write FPascalDialect;
  end;