vmtokenizer.cpp

TOOL (Tiny Object Oriented Language) is an easily-embedded, object-oriented, C++-like-language inter

/*****************************************************************************/
/*                              SOURCE FILE                                  */
/*****************************************************************************/
/*
       $Archive:   $

      $Revision:   $
          $Date:   $
        $Author:   $

    Description:   Tokenizer Object Implementation

                      TOOL And XML FORMS License
                      ==========================

                      Except where otherwise noted, all of the documentation 
                      and software included in the TOOL package is 
                      copyrighted by Michael Swartzendruber.

                      Copyright (C) 2005 Michael John Swartzendruber. 
                      All rights reserved.

                      Access to this code, whether intentional or accidental,
                      does NOT IMPLY any transfer of rights.

                      This software is provided "as-is," without any express 
                      or implied warranty. In no event shall the author be held
                      liable for any damages arising from the use of this software.

                      Permission is granted to anyone to use this software for 
                      any purpose, including commercial applications, and to 
                      alter and redistribute it, provided that the following 
                      conditions are met:

                      1. All redistributions of source code files must retain 
                         all copyright notices that are currently in place, 
                         and this list of conditions without modification.

                      2. The origin of this software must not be misrepresented;
                         you must not claim that you wrote the original software.

                      3. If you use this software in another product, an acknowledgment
                         in the product documentation would be appreciated but is
                         not required.

                      4. Modified versions in source or binary form must be plainly 
                         marked as such, and must not be misrepresented as being 
                         the original software.
*/
static char OBJECT_ID[] = "$Revision: 1 $ : $Date: 3/08/00 11:37a $";
/*****************************************************************************/


#include "../../../stdafx.h"
#include "VMTokenizer.h"




/*****************************************************************************/
/*
     FUNCTION NAME:  VMTokenizer::VMTokenizer

       DESCRIPTION:  ctor. inits this

             INPUT:  oString - the string to tokenize
                     oDelims - the delimiters for tokens
            OUTPUT:  

           RETURNS:  none
*/
VMTokenizer::VMTokenizer( VMString oString, VMString oDelims, VMString oFieldWrappers )
{
  m_oToTokenize     = oString;
  m_oLeftToTokenize = oString;
  m_oTokenDelimiter = oDelims;
  m_oFieldWrapper   = oFieldWrappers;
  m_bFoundAny       = false;
  m_bFinalToken     = false;
}
/* End of function "VMTokenizer::VMTokenizer"
/*****************************************************************************/


/*****************************************************************************/
/*
     FUNCTION NAME:  VMTokenizer::~VMTokenizer

       DESCRIPTION:  dtor. does nothing

             INPUT:  void
            OUTPUT:  none

           RETURNS:  none
*/
VMTokenizer::~VMTokenizer( void )
{
}
/* End of function "VMTokenizer::~VMTokenizer"
/*****************************************************************************/


/*****************************************************************************/
/*
     FUNCTION NAME:  VMTokenizer::HasMoreTokens

       DESCRIPTION:  Checks to see if there are more tokens that can be retrieved

             INPUT:  void
            OUTPUT:  none

           RETURNS:  true if there is, false if not
*/
bool VMTokenizer::HasMoreTokens( void )
{
  if ( ( m_oLeftToTokenize.Find( m_oTokenDelimiter,0 ) == -1 ) && ( m_bFinalToken ) )
  {
    return( true );   // The string that is left over is a token
  }
  else 
  if ( ( m_oLeftToTokenize.Find( m_oTokenDelimiter, 0 ) == -1 ) && ( !m_bFinalToken ) )
  {
    return( false ); // The string contains no tokens
  }
  else
    return( true );
}
/* End of function "VMTokenizer::HasMoreTokens"
/*****************************************************************************/


/*****************************************************************************/
/*
     FUNCTION NAME:  VMTokenizer::GetNextToken

       DESCRIPTION:  returns the next token to the caller

             INPUT:  void
            OUTPUT:  none

           RETURNS:  the next token 
*/
VMString VMTokenizer::GetNextToken( void )
{
  if ( m_bFinalToken )
  {
    m_bFinalToken = false;
    VMString oReturn = m_oLeftToTokenize;
    m_oLeftToTokenize = "";
    return( oReturn );
  }
	
  int iPosition;
  int iLength;

  VMString oNewToken = m_oLeftToTokenize; // Our own working copy of the string left

  iLength   = oNewToken.GetLength();

  if ( m_oFieldWrapper.GetLength() == 0 )
  {
    iPosition = oNewToken.Find( m_oTokenDelimiter, 0 );

    if ( -1 == iPosition )
    {
      // This should never happen
      //
      return( VMString( "" ) );
    }

    m_bFoundAny = true;
    oNewToken = oNewToken.Left( iPosition );
    iPosition++;
    m_oLeftToTokenize = m_oLeftToTokenize.Right( iLength - iPosition );

    //Check to see if there anymore tokens
    //
    if ( m_oLeftToTokenize.Find( m_oTokenDelimiter, 0 ) == -1 )
    {
      // No more tokens but the string left is the remaining token
      //
      m_bFinalToken = true;
    }
  }
  else
  {
    int iFieldStart = oNewToken.Find( m_oFieldWrapper, 0 );
    int iFieldEnd   = oNewToken.FindNthOf( 2, m_oFieldWrapper );

    iPosition = oNewToken.Find( m_oTokenDelimiter, 0 );

    if ( -1 == iPosition )
    {
      // This should never happen
      //
      return( VMString( "" ) );
    }

    if ( iFieldStart < iPosition && iPosition < iFieldEnd )
    {
      int iStep = 2;
      while( iFieldStart < iPosition && iPosition < iFieldEnd )
      {
        iPosition = oNewToken.FindNthOf( iStep, m_oTokenDelimiter );
        iStep++;
      }
    }

    m_bFoundAny = true;
    oNewToken = oNewToken.Left( iPosition );
    iPosition++;
    m_oLeftToTokenize = m_oLeftToTokenize.Right( iLength - iPosition );

    //Check to see if there anymore tokens
    //
    if ( m_oLeftToTokenize.Find( m_oTokenDelimiter, 0 ) == -1 )
    {
      // No more tokens but the string left is the remaining token
      //
      m_bFinalToken = true;
    }
  }

  return( oNewToken );
}
/* End of function "VMTokenizer::GetNextToken"
/*****************************************************************************/




/*****************************************************************************/
/* Check-in history */
/*
 *$Log:   $
*/
/*****************************************************************************/