standardtokenizerimpl.java

Lucene a java open-source SearchEngine Framework

/* The following code was generated by JFlex 1.4.1 on 4/15/08 8:53 AM */

package org.apache.lucene.analysis.standard;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import org.apache.lucene.analysis.Token;


/**
 * This class is a scanner generated by 
 * <a href="http://www.jflex.de/">JFlex</a> 1.4.1
 * on 4/15/08 8:53 AM from the specification file
 * <tt>/tango/mike/src/lucene.23.clean/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex</tt>
 */
class StandardTokenizerImpl {

  /** This character denotes the end of file */
  public static final int YYEOF = -1;

  /** initial size of the lookahead buffer */
  private static final int ZZ_BUFFERSIZE = 16384;

  /** lexical states */
  public static final int YYINITIAL = 0;

  /** 
   * Translates characters to character classes
   */
  private static final String ZZ_CMAP_PACKED = 
    "\11\0\1\0\1\16\1\0\1\0\1\15\22\0\1\0\5\0\1\3"+
    "\1\1\4\0\1\7\1\5\1\2\1\7\12\11\6\0\1\4\32\10"+
    "\4\0\1\6\1\0\32\10\105\0\27\10\1\0\37\10\1\0\u0568\10"+
    "\12\12\206\10\12\12\u026c\10\12\12\166\10\12\12\166\10\12\12\166\10"+
    "\12\12\166\10\12\12\167\10\11\12\166\10\12\12\166\10\12\12\166\10"+
    "\12\12\340\10\12\12\166\10\12\12\u0166\10\12\12\266\10\u0100\10\u0e00\10"+
    "\u1040\0\u0150\14\140\0\20\14\u0100\0\200\14\200\0\u19c0\14\100\0\u5200\14"+
    "\u0c00\0\u2bb0\13\u2150\0\u0200\14\u0465\0\73\14\75\10\43\0";

  /** 
   * Translates characters to character classes
   */
  private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED);

  /** 
   * Translates DFA states to action switch labels.
   */
  private static final int [] ZZ_ACTION = zzUnpackAction();

  private static final String ZZ_ACTION_PACKED_0 =
    "\1\0\1\1\4\2\1\3\1\1\6\0\2\2\6\0"+
    "\1\4\4\5\2\6\2\0\1\7\1\0\1\7\3\5"+
    "\6\7\3\5\1\10\1\0\1\11\2\0\1\10\1\11"+
    "\1\0\2\11\2\10\2\5\1\12";

  private static int [] zzUnpackAction() {
    int [] result = new int[61];
    int offset = 0;
    offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
    return result;
  }

  private static int zzUnpackAction(String packed, int offset, int [] result) {
    int i = 0;       /* index in packed string  */
    int j = offset;  /* index in unpacked array */
    int l = packed.length();
    while (i < l) {
      int count = packed.charAt(i++);
      int value = packed.charAt(i++);
      do result[j++] = value; while (--count > 0);
    }
    return j;
  }


  /** 
   * Translates a state to a row index in the transition table
   */
  private static final int [] ZZ_ROWMAP = zzUnpackRowMap();

  private static final String ZZ_ROWMAP_PACKED_0 =
    "\0\0\0\17\0\36\0\55\0\74\0\113\0\17\0\132"+
    "\0\151\0\170\0\207\0\226\0\245\0\264\0\303\0\322"+
    "\0\341\0\360\0\377\0\u010e\0\u011d\0\u012c\0\u013b\0\u014a"+
    "\0\u0159\0\u0168\0\u0177\0\207\0\u0186\0\u0195\0\u01a4\0\u01b3"+
    "\0\u01c2\0\u01d1\0\u01e0\0\u01ef\0\u01fe\0\u020d\0\u021c\0\u022b"+
    "\0\u023a\0\u0249\0\u0258\0\u0267\0\u0276\0\u0285\0\u0294\0\u02a3"+
    "\0\u02b2\0\u02c1\0\u02d0\0\u02df\0\u02ee\0\u02fd\0\u012c\0\341"+
    "\0\170\0\u011d\0\u030c\0\u031b\0\u032a";

  private static int [] zzUnpackRowMap() {
    int [] result = new int[61];
    int offset = 0;
    offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
    return result;
  }

  private static int zzUnpackRowMap(String packed, int offset, int [] result) {
    int i = 0;  /* index in packed string  */
    int j = offset;  /* index in unpacked array */
    int l = packed.length();
    while (i < l) {
      int high = packed.charAt(i++) << 16;
      result[j++] = high | packed.charAt(i++);
    }
    return j;
  }

  /** 
   * The transition table of the DFA
   */
  private static final int [] ZZ_TRANS = zzUnpackTrans();

  private static final String ZZ_TRANS_PACKED_0 =
    "\10\2\1\3\1\4\1\5\1\6\1\7\1\10\1\2"+
    "\20\0\1\11\1\12\1\13\1\14\2\15\1\16\1\17"+
    "\1\4\1\20\1\6\5\0\1\21\1\0\1\22\2\23"+
    "\1\24\3\4\1\6\4\0\1\11\1\25\1\13\1\14"+
    "\2\23\1\24\1\20\1\4\1\20\1\6\5\0\1\26"+
    "\1\0\1\22\2\15\1\16\4\6\21\0\1\2\10\0"+
    "\1\27\1\0\1\27\14\0\1\30\1\31\1\32\1\33"+
    "\13\0\1\34\1\0\1\34\14\0\1\35\1\36\1\35"+
    "\1\36\13\0\1\37\2\40\1\41\13\0\1\16\2\42"+
    "\5\0\1\11\1\26\1\13\1\14\2\15\1\16\1\17"+
    "\1\4\1\20\1\6\4\0\1\11\1\21\1\13\1\14"+
    "\2\23\1\24\1\20\1\4\1\20\1\6\13\0\1\43"+
    "\2\44\1\45\13\0\4\36\13\0\1\46\2\47\1\50"+
    "\13\0\1\51\2\52\1\53\13\0\1\54\1\44\1\55"+
    "\1\45\13\0\1\56\2\31\1\33\4\0\1\11\6\0"+
    "\1\27\1\0\1\27\6\0\1\57\1\0\1\22\2\60"+
    "\1\0\1\56\2\31\1\33\5\0\1\61\1\0\1\22"+
    "\2\62\1\63\3\31\1\33\5\0\1\64\1\0\1\22"+
    "\2\62\1\63\3\31\1\33\5\0\1\65\1\0\1\22"+
    "\2\60\1\0\4\33\5\0\1\66\2\0\1\66\2\0"+
    "\1\35\1\36\1\35\1\36\5\0\1\66\2\0\1\66"+
    "\2\0\4\36\5\0\1\60\1\0\1\22\2\60\1\0"+
    "\1\37\2\40\1\41\5\0\1\62\1\0\1\22\2\62"+
    "\1\63\3\40\1\41\5\0\1\60\1\0\1\22\2\60"+
    "\1\0\4\41\5\0\1\63\2\0\3\63\3\42\6\0"+
    "\1\67\1\0\1\22\2\15\1\16\1\43\2\44\1\45"+
    "\5\0\1\70\1\0\1\22\2\23\1\24\3\44\1\45"+
    "\5\0\1\67\1\0\1\22\2\15\1\16\4\45\5\0"+
    "\1\15\1\0\1\22\2\15\1\16\1\46\2\47\1\50"+
    "\5\0\1\23\1\0\1\22\2\23\1\24\3\47\1\50"+
    "\5\0\1\15\1\0\1\22\2\15\1\16\4\50\5\0"+
    "\1\16\2\0\3\16\1\51\2\52\1\53\5\0\1\24"+
    "\2\0\3\24\3\52\1\53\5\0\1\16\2\0\3\16"+
    "\4\53\5\0\1\71\1\0\1\22\2\15\1\16\1\43"+
    "\2\44\1\45\5\0\1\72\1\0\1\22\2\23\1\24"+
    "\3\44\1\45\5\0\1\65\1\0\1\22\2\60\1\0"+
    "\1\56\2\31\1\33\13\0\1\73\1\33\1\73\1\33"+
    "\13\0\4\41\13\0\4\45\13\0\4\50\13\0\4\53"+
    "\13\0\1\74\1\45\1\74\1\45\13\0\4\33\13\0"+
    "\4\75\5\0\1\57\1\0\1\22\2\60\1\0\4\33"+
    "\5\0\1\71\1\0\1\22\2\15\1\16\4\45\5\0"+
    "\1\66\2\0\1\66\2\0\4\75\3\0";

  private static int [] zzUnpackTrans() {
    int [] result = new int[825];
    int offset = 0;
    offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
    return result;
  }

  private static int zzUnpackTrans(String packed, int offset, int [] result) {
    int i = 0;       /* index in packed string  */
    int j = offset;  /* index in unpacked array */
    int l = packed.length();
    while (i < l) {
      int count = packed.charAt(i++);
      int value = packed.charAt(i++);
      value--;
      do result[j++] = value; while (--count > 0);
    }
    return j;
  }


  /* error codes */
  private static final int ZZ_UNKNOWN_ERROR = 0;
  private static final int ZZ_NO_MATCH = 1;
  private static final int ZZ_PUSHBACK_2BIG = 2;

  /* error messages for the codes above */
  private static final String ZZ_ERROR_MSG[] = {
    "Unkown internal scanner error",
    "Error: could not match input",
    "Error: pushback value was too large"
  };

  /**
   * ZZ_ATTRIBUTE[aState] contains the attributes of state <code>aState</code>
   */
  private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();

  private static final String ZZ_ATTRIBUTE_PACKED_0 =
    "\1\0\1\11\4\1\1\11\1\1\6\0\2\1\6\0"+
    "\7\1\2\0\1\1\1\0\16\1\1\0\1\1\2\0"+
    "\2\1\1\0\7\1";

  private static int [] zzUnpackAttribute() {
    int [] result = new int[61];
    int offset = 0;
    offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
    return result;
  }

  private static int zzUnpackAttribute(String packed, int offset, int [] result) {
    int i = 0;       /* index in packed string  */
    int j = offset;  /* index in unpacked array */
    int l = packed.length();
    while (i < l) {
      int count = packed.charAt(i++);
      int value = packed.charAt(i++);
      do result[j++] = value; while (--count > 0);
    }
    return j;
  }

  /** the input device */
  private java.io.Reader zzReader;

  /** the current state of the DFA */
  private int zzState;

  /** the current lexical state */
  private int zzLexicalState = YYINITIAL;

  /** this buffer contains the current text to be matched and is
      the source of the yytext() string */
  private char zzBuffer[] = new char[ZZ_BUFFERSIZE];

  /** the textposition at the last accepting state */
  private int zzMarkedPos;

  /** the textposition at the last state to be included in yytext */
  private int zzPushbackPos;

  /** the current text position in the buffer */
  private int zzCurrentPos;

  /** startRead marks the beginning of the yytext() string in the buffer */
  private int zzStartRead;

  /** endRead marks the last character in the buffer, that has been read
      from input */
  private int zzEndRead;

  /** number of newlines encountered up to the start of the matched text */
  private int yyline;

  /** the number of characters up to the start of the matched text */
  private int yychar;

  /**
   * the number of characters from the last newline up to the start of the 
   * matched text
   */
  private int yycolumn;

  /** 
   * zzAtBOL == true <=> the scanner is currently at the beginning of a line
   */
  private boolean zzAtBOL = true;

  /** zzAtEOF == true <=> the scanner is at the EOF */
  private boolean zzAtEOF;

  /* user code: */

public static final int ALPHANUM          = StandardTokenizer.ALPHANUM;
public static final int APOSTROPHE        = StandardTokenizer.APOSTROPHE;
public static final int ACRONYM           = StandardTokenizer.ACRONYM;
public static final int COMPANY           = StandardTokenizer.COMPANY;
public static final int EMAIL             = StandardTokenizer.EMAIL;
public static final int HOST              = StandardTokenizer.HOST;
public static final int NUM               = StandardTokenizer.NUM;
public static final int CJ                = StandardTokenizer.CJ;
/**
 * @deprecated this solves a bug where HOSTs that end with '.' are identified
 *             as ACRONYMs. It is deprecated and will be removed in the next
 *             release.
 */
public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;

public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;

public final int yychar()
{
    return yychar;
}

/**
 * Fills Lucene token with the current token text.
 */
final void getText(Token t) {
  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
}


  /**
   * Creates a new scanner
   * There is also a java.io.InputStream version of this constructor.
   *
   * @param   in  the java.io.Reader to read input from.
   */
  StandardTokenizerImpl(java.io.Reader in) {
    this.zzReader = in;
  }

  /**
   * Creates a new scanner.
   * There is also java.io.Reader version of this constructor.
   *
   * @param   in  the java.io.Inputstream to read input from.