cnscaners.pas

CnCodeFormater Readme Author: passos Email: [e-mail]@21cn.com Last Update:20

    end;
  end
  else
    Error(SInvalidBookmark);

  if Clear then
    ClearBookmark(Bookmark);
end;

procedure TAbstractScaner.SaveBookmark(var Bookmark: TScannerBookmark);
begin
  Bookmark := TScannerBookmark.Create;
  with Bookmark do
  begin
    OriginBookmark := FOrigin;
    SourcePtrBookmark := FSourcePtr;
    TokenBookmark := FToken;
    TokenPtrBookmark := FTokenPtr;
  end;
  FBookmarks.Add(Bookmark);
end;

procedure TAbstractScaner.ClearBookmark(var Bookmark: TScannerBookmark);
begin
  Bookmark := TScannerBookmark(FBookmarks.Extract(Bookmark));
  if Assigned(Bookmark) then
    FreeAndNil(Bookmark);
end;

function TAbstractScaner.ForwardToken(Count: Integer): TPascalToken;
var
  Bookmark: TScannerBookmark;
  I: Integer;
begin
  Result := Token;

  SaveBookmark(Bookmark);

  for I := 0 to Count - 1 do
  begin
    Result := NextToken;
    if Result = tokEOF then
      Exit;
  end;

  LoadBookmark(Bookmark);
end;

function TAbstractScaner.BlankString: string;
var
  L: Integer;
begin
  L := FBlankStringEnd - FBlankStringBegin;
  SetString(Result, FBlankStringBegin, L);
end;

{ TScaner }

constructor TScaner.Create(AStream: TStream; ACodeGen: TCnCodeGenerator);
begin
  AStream.Seek(0, soFromBeginning);
  FStream := AStream;
  FCodeGen := ACodeGen;

  inherited Create(AStream);
end;

constructor TScaner.Create(AStream: TStream);
begin
  Create(AStream, nil); //TCnCodeGenerator.Create);
end;

destructor TScaner.Destroy;
begin
  inherited Destroy;
end;

function TScaner.ForwardToken(Count: Integer): TPascalToken;
begin
  FCodeGen.LockOutput;
  try
    Result := inherited ForwardToken(Count);
  finally
    FCodeGen.UnLockOutput;
  end;
end;

function TScaner.NextToken: TPascalToken;

  procedure SkipTo(var P: PChar; TargetChar: Char);
  begin
    while (P^ <> TargetChar) do
    begin
      Inc(P);

      if (P^ = #0) then
      begin
        ReadBuffer;
        if FSourcePtr^ = #0 then
          Break;
      end;
    end;
  end;

var
  IsWideStr: Boolean;
  P: PChar;
begin
  SkipBlanks;
  P := FSourcePtr;
  FTokenPtr := P;
  case P^ of
    'A'..'Z', 'a'..'z', '_':
      begin
        Inc(P);
        while P^ in ['A'..'Z', 'a'..'z', '0'..'9', '_'] do Inc(P);
        Result := tokSymbol;
      end;
    '#', '''':
      begin
        IsWideStr := False;
        // parser string like this: 'abc'#10#13'def'#10#13
        while True do
          case P^ of
            '#':
              begin
                IsWideStr := True;
                Inc(P);
                while P^ in ['$', '0'..'9', 'a'..'f', 'A'..'F'] do Inc(P);
              end;
            '''':
              begin
                Inc(P);
                while True do
                  case P^ of
                    #0, #10, #13:
                      Error(SInvalidString);
                    '''':
                      begin
                        Inc(P);
                        Break;
                      end;
                  else
                    Inc(P);
                  end;
              end;
          else
            Break;
          end; // case P^ of

        FStringPtr := P;
        
        if IsWideStr then
          Result := tokWString
        else
          Result := tokString;
      end; // '#', '''': while True do

    '"':
      begin
        Inc(P);
        while not (P^ in ['"', #0, #10, #13]) do Inc(P);
        Result := tokString;
        if P^ = '"' then  // 和单引号字符串一样跳过最后一个双引号
          Inc(P);
        FStringPtr := P;
      end;

    '$':
      begin
        Inc(P);
        while P^ in ['0'..'9', 'A'..'F', 'a'..'f'] do
          Inc(P);
        Result := tokInteger;
      end;

    '*':
      begin
        Inc(P);
        Result := tokStar;
      end;

    '{':
      begin
        Inc(P);
        { TODO: Check Directive sign $}

        Result := tokComment;
        while ((P^ <> #0) and (P^ <> '}')) do
          Inc(P);

        if P^ = '}' then
        begin
          FBlankLinesAfterComment := 0;
          Inc(P);
          while P^ in [' ', #9] do
            Inc(P);
          if P^ = #13 then
          begin
            if not FASMMode then // ASM 模式下，换行作为语句结束符，不在注释内处理
            begin
              Inc(P);
              if P^ = #10 then
              begin
                Inc(FBlankLinesAfterComment);
                Inc(P);
              end;
            end;
          end;
        end
        else
          Error(SEndOfCommentExpected);
      end;

    '/':
      begin
        Inc(P);

        if P^ = '/' then
        begin
          Result := tokComment;
          while (P^ <> #0) and (P^ <> #13)
            do Inc(P); // 找行尾

          FBlankLinesAfterComment := 0;
          if P^ = #13 then
          begin
            if not FASMMode then // ASM 模式下，换行作为语句结束符，不在注释内处理
            begin
              Inc(P);
              if P^ = #10 then
              begin
                Inc(FBlankLinesAfterComment);
                Inc(P);
              end;
            end;
          end
          else
            Error(SEndOfCommentExpected);
        end
        else
          Result := tokDiv;
      end;

    '(':
      begin
        Inc(P);
        Result := tokLB;

        if P^ = '*' then
        begin
          Result := tokComment;

          Inc(P);
          FBlankLinesAfterComment := 0;
          while P^ <> #0 do
          begin
            if P^ = '*' then
            begin
              Inc(P);
              if P^ = ')' then
              begin
                Inc(P);
                while P^ in [' ', #9] do
                  Inc(P);

                if P^ = #13 then
                begin
                  if not FASMMode then // ASM 模式下，换行作为语句结束符，不在注释内处理
                  begin
                    Inc(P);
                    if P^ = #10 then
                    begin
                      Inc(FBlankLinesAfterComment);
                      Inc(P);
                    end;
                  end;
                end;

                Break;
              end;
            end
            else
              Inc(P);
          end;
        end;
      end;

    ')':
      begin
        Inc(P);
        Result := tokRB;
      end;

    '[':
      begin
        Inc(P);
        Result := tokSLB;
      end;

    ']':
      begin
        Inc(P);
        Result := tokSRB;
      end;

    '^':
      begin
        Inc(P);
        Result := tokHat;
      end;

    '=':
      begin
        Inc(P);
        Result := tokEQUAL;
      end;

    ':':
      begin
        Inc(P);
        if (P^ = '=') then
        begin
          Inc(P);
          Result := tokAssign;
        end else
          Result := tokColon;
      end;
      
    ';':
      begin
        Inc(P);
        Result := tokSemicolon;
      end;

    '.':
      begin
        Inc(P);

        if P^ = '.' then
        begin
          Result := tokRange;
          Inc(P);
        end else
          Result := tokDot;
      end;

    ',':
      begin
        Inc(P);
        Result := tokComma;
      end;

    '>':
      begin
        Inc(P);
        Result := tokGreat;
          
        if P^ = '<' then
        begin
          Result := tokNotEqual;
          Inc(P);
        end else
        if P^ = '=' then
        begin
          Result := tokGreatOrEqu;
          Inc(P);
        end;
      end;

    '<':
      begin
        Inc(P);
        Result := tokLess;
        
        if P^ = '=' then
        begin
          Result := tokLessOrEqu;
          Inc(P);
        end;

        if P^ = '>' then
        begin
          Result := tokNotEqual;
          Inc(P);
        end;
      end;

    '@':
      begin
        Inc(P);
        Result := tokAtSign;
      end;

    '+', '-':
      begin
        if P^ = '+' then
          Result := tokPlus
        else
          Result := tokMinus;

        Inc(P);
      end;

    '0'..'9':
      begin
        Inc(P);
        while P^ in ['0'..'9'] do Inc(P);
        Result := tokInteger;

        if (P^ = '.') and ((P+1)^ <> '.') then
        begin
          Inc(P);
          while P^ in ['0'..'9'] do Inc(P);
          Result := tokFloat;
        end;

        if P^ in ['e', 'E'] then
        begin
          Inc(P);
          if P^ in ['-', '+'] then
            Inc(P);
          while P^ in ['0'..'9'] do
            Inc(P);
          Result := tokFloat;
        end;

        if (P^ in ['c', 'C', 'd', 'D', 's', 'S']) then
        begin
          Result := tokFloat;
          FFloatType := P^;
          Inc(P);
        end
        else
          FFloatType := #0;
      end;
    #10:  // 如果有回车则处理，以 #10 为准
      begin
        Result := tokCRLF;
        Inc(P);
      end;
  else
    if P^ = #0 then
      Result := tokEOF
    else
      Result := tokUnknown;

    if Result <> tokEOF then
      Inc(P);
  end;

  FSourcePtr := P;
  FToken := Result;

  if Result = tokComment then // 当前是 Comment
  begin
    if Assigned(FCodeGen) then
    begin
      FCodeGen.Write(BlankString);
      FCodeGen.Write(TokenString);
    end;

    if not FFirstCommentInBlock then // 第一次碰到 Comment 时设置这个
    begin
      FFirstCommentInBlock := True;
      FBlankLinesBefore := FBlankLines;
    end;

    FPreviousIsComment := True;
    Result := NextToken;
    // 进入递归寻找下一个 Token，
    // 进入后 FFirstCommentInBlock 为 True，因此不会重新记录 FBlankLinesBefore
    FPreviousIsComment := False;
  end
  else
  begin
    // 只要当前不是 Comment 就设置非第一个 Comment 的标记
    FFirstCommentInBlock := False;

    if FPreviousIsComment then // 上一个是 Comment，记录这个到 上一个Comment的空行数
    begin
      // 最后一块注释的在递归最外层赋值，因此FBlankLinesAfter会被层层覆盖，
      // 代表最后一块注释后的空行数
      FBlankLinesAfter := FBlankLines + FBlankLinesAfterComment;
    end
    else // 上一个不是 Comment，当前也不是 Comment。全清0
    begin
      FBlankLinesAfter := 0;
      FBlankLinesBefore := 0;
      FBlankLines := 0;
    end;

    if FBackwardToken = tokComment then // 当前不是 Comment，但前一个是 Comment
      FCodeGen.Write(BlankString);

    if (Result = tokString) and (Length(TokenString) = 1) then
      Result := tokChar
    else if Result = tokSymbol then
      Result := StringToToken(TokenString);

    FToken := Result;
    FBackwardToken := FToken;
  end;
end;

end.