standardtokenizer.cpp

clucene是c++版的全文检索引擎,完全移植于lucene,采用 stl 编写.

/*------------------------------------------------------------------------------
* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
* 
* Distributable under the terms of either the Apache License (Version 2.0) or 
* the GNU Lesser General Public License, as specified in the COPYING file.
------------------------------------------------------------------------------*/
#include "CLucene/StdHeader.h"
#include "StandardTokenizer.h"

CL_NS_USE(analysis)
CL_NS_USE(util)
CL_NS_DEF2(analysis,standard)

  /* A bunch of shortcut macros, many of which make assumptions about variable
  ** names.  These macros enhance readability, not just convenience! */
  #define EOS           (ch==-1 || rd->Eos())
  #define SPACE         (_istspace((TCHAR)ch) != 0)
  #define ALPHA         (_istalpha((TCHAR)ch) != 0)
  #define ALNUM         (_istalnum(ch) != 0)
  #define DIGIT         (_istdigit(ch) != 0)
  #define UNDERSCORE    (ch == '_')
  
  #define _CJK			(  (ch>=0x3040 && ch<=0x318f) || \
  						   (ch>=0x3300 && ch<=0x337f) || \
  						   (ch>=0x3400 && ch<=0x3d2d) || \
  						   (ch>=0x4e00 && ch<=0x9fff) || \
  						   (ch>=0xf900 && ch<=0xfaff) || \
  						   (ch>=0xac00 && ch<=0xd7af) ) //korean

  
  #define DASH          (ch == '-')
  #define NEGATIVE_SIGN_ DASH
  //#define POSITIVE_SIGN_ (ch == '+')
  //#define SIGN          (NEGATIVE_SIGN_ || POSITIVE_SIGN_)

  #define DOT             (ch == '.')
  #define DECIMAL         DOT


  //freebsd seems to have a problem with defines over multiple lines, so this has to be one long line
  #define _CONSUME_AS_LONG_AS(conditionFails) while (true) { ch = readChar(); if (ch==-1 || (!(conditionFails) || str.len >= LUCENE_MAX_WORD_LEN)) { break; } str.appendChar(ch);}

  #define CONSUME_ALPHAS _CONSUME_AS_LONG_AS(ALPHA)

  #define CONSUME_DIGITS _CONSUME_AS_LONG_AS(DIGIT)

  /* otherMatches is a condition (possibly compound) under which a character
  ** that's not an ALNUM or UNDERSCORE can be considered not to break the
  ** span.  Callers should pass false if only ALNUM/UNDERSCORE are acceptable. */
  #define CONSUME_WORD                  _CONSUME_AS_LONG_AS(ALNUM || UNDERSCORE)
  
  /*
  ** Consume CJK characters
  */
  #define CONSUME_CJK                   _CONSUME_AS_LONG_AS(_CJK)


  /* It is considered that "nothing of value" has been read if:
  ** a) The "read head" hasn't moved since specialCharPos was established.
  ** or
  ** b) The "read head" has moved by one character, but that character was
  **    either whitespace or not among the characters found in the body of
  **    a token (deliberately doesn't include the likes of '@'/'&'). */
  #define CONSUMED_NOTHING_OF_VALUE (rdPos == specialCharPos || (rdPos == specialCharPos+1 && ( SPACE || !(ALNUM || DOT || DASH || UNDERSCORE) )))

  #define RIGHTMOST(sb) (sb.getBuffer()[sb.len-1])
  #define RIGHTMOST_IS(sb, c) (RIGHTMOST(sb) == c)
  /* To discard the last character in a StringBuffer, we decrement the buffer's
  ** length indicator and move the terminator back by one character. */
  #define SHAVE_RIGHTMOST(sb) (sb.getBuffer()[--sb.len] = '\0')

  //#define REMOVE_TRAILING_CHARS(sb, charMatchesCondition) { TCHAR* sbBuf = sb.getBuffer(); for (int32_t i = sb.len-1; i >= 0; i--) { TCHAR c = sbBuf[i]; if (charMatchesCondition) { sbBuf[--sb.len] = '\0'; } else {break;}}}

  /* Does StringBuffer sb contain any of the characters in string ofThese? */
  #define CONTAINS_ANY(sb, ofThese) (_tcscspn(sb.getBuffer(), _T(ofThese)) != static_cast<size_t>(sb.len))


  StandardTokenizer::StandardTokenizer(Reader* reader):
    rd(_CLNEW FastCharStream(reader)),
    /* rdPos is zero-based.  It starts at -1, and will advance to the first
    ** position when readChar() is first called. */
    rdPos(-1),
    tokenStart(-1)
  {
  }

  StandardTokenizer::~StandardTokenizer() {
    _CLDELETE(rd);
  }

  int StandardTokenizer::readChar() {
    /* Increment by 1 because we're speaking in terms of characters, not
    ** necessarily bytes: */
    rdPos++;
    return rd->GetNext();
  }

  void StandardTokenizer::unReadChar() {
    rd->UnGet();
    rdPos--;
  }

  inline bool StandardTokenizer::setToken(Token* t, StringBuffer* sb, TokenTypes tokenCode) {
    t->setStartOffset(tokenStart);
	t->setEndOffset(tokenStart+sb->length());
	t->setType(tokenImage[tokenCode]);
	sb->getBuffer(); //null terminates the buffer
	t->resetTermTextLen();
	return true;
  }

  /*inline Token* StandardTokenizer::createToken(const TCHAR* text, int32_t tokenLength, TokenTypes tokenCode) {
    CND_PRECONDITION (tokenLength > 0,"Token length is less than or equal to zero");
    return _CLNEW Token(text, tokenStart, tokenStart+tokenLength, tokenImage[tokenCode]);
  }*/

  bool StandardTokenizer::next(Token* t) {
    int ch=0;
	while (!EOS) {
      ch = readChar();

	  if ( ch == 0 || ch == -1 ){
		continue;
	  } else if (SPACE) {
        continue;
      } else if (ALPHA || UNDERSCORE) {
        tokenStart = rdPos;
        return ReadAlphaNum(ch,t);
      } else if (DIGIT || NEGATIVE_SIGN_ || DECIMAL) {
        tokenStart = rdPos;
        /* ReadNumber returns NULL if it fails to extract a valid number; in
        ** that case, we just continue. */
        if (ReadNumber(NULL, ch,t))
          return true;
      } else if ( _CJK ){
      	if ( ReadCJK(ch,t) )
      		return true;
      }
    }
    return false;
  }

  bool StandardTokenizer::ReadNumber(const TCHAR* previousNumber, const TCHAR prev,Token* t) {
    /* previousNumber is only non-NULL if this function already read a complete
    ** number in a previous recursion, yet has been asked to read additional
    ** numeric segments.  For example, in the HOST "192.168.1.3", "192.168" is
    ** a complete number, but this function will recurse to read the "1.3",
    ** generating a single HOST token "192.168.1.3". */
    t->growBuffer(LUCENE_MAX_WORD_LEN+1);//make sure token can hold the next word
    StringBuffer str(t->_termText,t->bufferLength(),true); //use stringbuffer to read data onto the termText
    TokenTypes tokenType;
    bool decExhausted;
    if (previousNumber != NULL) {
      str.prepend(previousNumber);
      tokenType = CL_NS2(analysis,standard)::HOST;
      decExhausted = false;
    } else {
      tokenType = CL_NS2(analysis,standard)::NUM;
      decExhausted = (prev == '.');
    }
	if (  str.len >= LUCENE_MAX_WORD_LEN ){
		//if a number is too long, i would say there is no point
		//storing it, because its going to be the wrong number anyway?
		//what do people think?
		return false; 
	}
    str.appendChar(prev);

    const bool signExhausted = (prev == '-');
    int ch = prev;

    CONSUME_DIGITS;

    if (str.len < 2 /* CONSUME_DIGITS didn't find any digits. */
        && (
                (signExhausted && !DECIMAL)
             || (decExhausted /* && !DIGIT is implied, since CONSUME_DIGITS stopped on a non-digit. */)
           )
       )
    {
      /* We have either:
      **   a) a negative sign that's not followed by either digit(s) or a decimal
      **   b) a decimal that's not followed by digit(s)
      ** so this is not a valid number. */
      if (!EOS) {
        /* Unread the character that stopped CONSUME_DIGITS: */
        unReadChar();
      }
      return false;
    }

    /* We just read a group of digits.  Is it followed by a decimal symbol,
    ** implying that there might be another group of digits available? */
    if (!EOS) {
      if (DECIMAL) {
		if (  str.len >= LUCENE_MAX_WORD_LEN )
			return false; //read above for rationale
        str.appendChar(ch);
      } else {
        unReadChar();
        goto SUCCESSFULLY_EXTRACTED_NUMBER;
      }

      CONSUME_DIGITS;
      if (!DIGIT && !DECIMAL) {
        unReadChar();
      } else if (!EOS && DECIMAL && _istdigit(rd->Peek())) {
        /* We just read the fractional digit group, but it's also followed by
        ** a decimal symbol and at least one more digit, so this must be a
        ** HOST rather than a real number. */
        return ReadNumber(str.getBuffer(), '.',t);
      }
    }

    SUCCESSFULLY_EXTRACTED_NUMBER:
    TCHAR rightmost = RIGHTMOST(str);
    /* Don't including a trailing decimal point. */
    if (rightmost == '.') {
      SHAVE_RIGHTMOST(str);