standardtokenizertokenmanager.cs

介绍有关全文检索的类库,以lucene为例,在.net环境下实现多种类型文档的全文检索.

							case 27: 
								if (!JjCanMove_4(hiByte, i1, i2, l1, l2))
									break;
								if (kind > 7)
									kind = 7;
								JjCheckNAddTwoStates(22, 27);
								break;
							
							case 28: 
								if (JjCanMove_3(hiByte, i1, i2, l1, l2))
									JjCheckNAddStates(12, 17);
								break;
							
							case 29: 
								if (!JjCanMove_4(hiByte, i1, i2, l1, l2))
									break;
								if (kind > 1)
									kind = 1;
								JjCheckNAddStates(0, 11);
								break;
							
							case 30: 
								if (!JjCanMove_4(hiByte, i1, i2, l1, l2))
									break;
								if (kind > 1)
									kind = 1;
								JjCheckNAdd(30);
								break;
							
							case 31: 
								if (JjCanMove_4(hiByte, i1, i2, l1, l2))
									JjCheckNAddStates(24, 26);
								break;
							
							case 33: 
								if (JjCanMove_4(hiByte, i1, i2, l1, l2))
									JjCheckNAddStates(27, 29);
								break;
							
							case 35: 
								if (JjCanMove_4(hiByte, i1, i2, l1, l2))
									JjCheckNAddTwoStates(35, 36);
								break;
							
							case 37: 
								if (!JjCanMove_4(hiByte, i1, i2, l1, l2))
									break;
								if (kind > 5)
									kind = 5;
								JjCheckNAddTwoStates(36, 37);
								break;
							
							case 38: 
								if (JjCanMove_4(hiByte, i1, i2, l1, l2))
									JjCheckNAddTwoStates(38, 39);
								break;
							
							case 40: 
								if (!JjCanMove_4(hiByte, i1, i2, l1, l2))
									break;
								if (kind > 6)
									kind = 6;
								JjCheckNAddTwoStates(39, 40);
								break;
							
							case 41: 
								if (JjCanMove_4(hiByte, i1, i2, l1, l2))
									JjCheckNAddTwoStates(41, 42);
								break;
							
							case 43: 
								if (JjCanMove_2(hiByte, i1, i2, l1, l2))
									JjAddStates(52, 53);
								break;
							
							case 44: 
								if (!JjCanMove_3(hiByte, i1, i2, l1, l2))
									break;
								if (kind > 7)
									kind = 7;
								JjCheckNAdd(45);
								break;
							
							case 45: 
								if (!JjCanMove_2(hiByte, i1, i2, l1, l2))
									break;
								if (kind > 7)
									kind = 7;
								JjCheckNAdd(45);
								break;
							
							case 46: 
								if (JjCanMove_4(hiByte, i1, i2, l1, l2))
									JjCheckNAddTwoStates(46, 47);
								break;
							
							case 48: 
								if (JjCanMove_2(hiByte, i1, i2, l1, l2))
									JjAddStates(54, 55);
								break;
							
							case 49: 
								if (JjCanMove_3(hiByte, i1, i2, l1, l2))
									JjCheckNAddTwoStates(50, 51);
								break;
							
							case 50: 
								if (JjCanMove_2(hiByte, i1, i2, l1, l2))
									JjCheckNAddTwoStates(50, 51);
								break;
							
							case 52: 
								if (!JjCanMove_4(hiByte, i1, i2, l1, l2))
									break;
								if (kind > 7)
									kind = 7;
								JjCheckNAddTwoStates(47, 52);
								break;
							
							case 53: 
								if (JjCanMove_4(hiByte, i1, i2, l1, l2))
									JjCheckNAddTwoStates(53, 54);
								break;
							
							case 55: 
								if (JjCanMove_2(hiByte, i1, i2, l1, l2))
									JjAddStates(56, 57);
								break;
							
							case 56: 
								if (JjCanMove_3(hiByte, i1, i2, l1, l2))
									JjCheckNAddTwoStates(57, 58);
								break;
							
							case 57: 
								if (JjCanMove_2(hiByte, i1, i2, l1, l2))
									JjCheckNAddTwoStates(57, 58);
								break;
							
							case 59: 
								if (JjCanMove_4(hiByte, i1, i2, l1, l2))
									JjAddStates(58, 59);
								break;
							
							case 61: 
								if (JjCanMove_2(hiByte, i1, i2, l1, l2))
									JjAddStates(60, 61);
								break;
							
							case 62: 
								if (!JjCanMove_3(hiByte, i1, i2, l1, l2))
									break;
								if (kind > 7)
									kind = 7;
								JjCheckNAddTwoStates(58, 63);
								break;
							
							case 63: 
								if (!JjCanMove_2(hiByte, i1, i2, l1, l2))
									break;
								if (kind > 7)
									kind = 7;
								JjCheckNAddTwoStates(58, 63);
								break;
							
							case 64: 
								if (JjCanMove_2(hiByte, i1, i2, l1, l2))
									JjCheckNAddStates(30, 35);
								break;
							
							case 65: 
								if (JjCanMove_2(hiByte, i1, i2, l1, l2))
									JjCheckNAddTwoStates(65, 66);
								break;
							
							case 67: 
								if (!JjCanMove_2(hiByte, i1, i2, l1, l2))
									break;
								if (kind > 2)
									kind = 2;
								JjCheckNAddTwoStates(66, 67);
								break;
							
							case 68: 
								if (JjCanMove_2(hiByte, i1, i2, l1, l2))
									JjCheckNAddTwoStates(68, 69);
								break;
							
							case 70: 
								if (JjCanMove_2(hiByte, i1, i2, l1, l2))
									JjAddStates(44, 45);
								break;
							
							case 72: 
								if (JjCanMove_2(hiByte, i1, i2, l1, l2))
									JjCheckNAddTwoStates(72, 73);
								break;
							
							case 74: 
								if (!JjCanMove_2(hiByte, i1, i2, l1, l2))
									break;
								if (kind > 4)
									kind = 4;
								jjstateSet[jjnewStateCnt++] = 74;
								break;
							
							default:  break;
							
						}
					}
					while (i != startsAt);
				}
				if (kind != 0x7fffffff)
				{
					jjmatchedKind = kind;
					jjmatchedPos = curPos;
					kind = 0x7fffffff;
				}
				++curPos;
				if ((i = jjnewStateCnt) == (startsAt = 75 - (jjnewStateCnt = startsAt)))
					return curPos;
				try
				{
					curChar = input_stream.ReadChar();
				}
				catch (System.IO.IOException)
				{
					return curPos;
				}
			}
		}
		internal static readonly int[] jjnextStates = new int[]{30, 31, 32, 34, 38, 39, 41, 42, 46, 47, 53, 54, 5, 6, 10, 11, 19, 20, 3, 4, 8, 9, 17, 18, 31, 32, 34, 32, 33, 34, 65, 66, 68, 69, 72, 73, 5, 6, 19, 20, 25, 26, 50, 51, 70, 71, 12, 13, 14, 15, 23, 24, 43, 44, 48, 49, 55, 56, 59, 60, 61, 62};
		private static bool JjCanMove_0(int hiByte, int i1, int i2, ulong l1, ulong l2)
		{
			switch (hiByte)
			{
				
				case 48: 
					return ((jjbitVec2[i2] & l2) != (ulong) 0L);
				
				case 49: 
					return ((jjbitVec3[i2] & l2) != (ulong) 0L);
				
				case 51: 
					return ((jjbitVec4[i2] & l2) != (ulong) 0L);
				
				case 77: 
					return ((jjbitVec5[i2] & l2) != (ulong) 0L);
				
				case 255: 
					return ((jjbitVec6[i2] & l2) != (ulong) 0L);
				
				default: 
					if ((jjbitVec0[i1] & l1) != (ulong) 0L)
						return true;
					return false;
				
			}
		}
		private static bool JjCanMove_1(int hiByte, int i1, int i2, ulong l1, ulong l2)
		{
			switch (hiByte)
			{
				
				case 215: 
					return ((jjbitVec8[i2] & l2) != (ulong) 0L);
				
				default: 
					if ((jjbitVec7[i1] & l1) != (ulong) 0L)
						return true;
					return false;
				
			}
		}
		private static bool JjCanMove_2(int hiByte, int i1, int i2, ulong l1, ulong l2)
		{
			switch (hiByte)
			{
				
				case 0: 
					return ((jjbitVec10[i2] & l2) != (ulong) 0L);
				
				case 255: 
					return ((jjbitVec11[i2] & l2) != (ulong) 0L);
				
				default: 
					if ((jjbitVec9[i1] & l1) != (ulong) 0L)
						return true;
					return false;
				
			}
		}
		private static bool JjCanMove_3(int hiByte, int i1, int i2, ulong l1, ulong l2)
		{
			switch (hiByte)
			{
				
				case 6: 
					return ((jjbitVec14[i2] & l2) != (ulong) 0L);
				
				case 11: 
					return ((jjbitVec15[i2] & l2) != (ulong) 0L);
				
				case 13: 
					return ((jjbitVec16[i2] & l2) != (ulong) 0L);
				
				case 14: 
					return ((jjbitVec17[i2] & l2) != (ulong) 0L);
				
				case 16: 
					return ((jjbitVec18[i2] & l2) != (ulong) 0L);
				
				default: 
					if ((jjbitVec12[i1] & l1) != (ulong) 0L)
						if ((jjbitVec13[i2] & l2) == (ulong) 0L)
							return false;
						else
							return true;
					return false;
				
			}
		}
		private static bool JjCanMove_4(int hiByte, int i1, int i2, ulong l1, ulong l2)
		{
			switch (hiByte)
			{
				
				case 0: 
					return ((jjbitVec10[i2] & l2) != (ulong) 0L);
				
				case 215: 
					return ((jjbitVec8[i2] & l2) != (ulong) 0L);
				
				case 255: 
					return ((jjbitVec11[i2] & l2) != (ulong) 0L);
				
				default: 
					if ((jjbitVec19[i1] & l1) != (ulong) 0L)
						return true;
					return false;
				
			}
		}
		public static readonly System.String[] jjstrLiteralImages = new System.String[]{"", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null};
		public static readonly System.String[] lexStateNames = new System.String[]{"DEFAULT"};
		internal static readonly long[] jjtoToken = new long[]{0x30ffL};
		internal static readonly long[] jjtoSkip = new long[]{0x8000L};
		protected internal CharStream input_stream;
		private uint[] jjrounds = new uint[75];
		private int[] jjstateSet = new int[150];
		protected internal char curChar;
		public StandardTokenizerTokenManager(CharStream stream)
		{
			InitBlock();
			input_stream = stream;
		}
		public StandardTokenizerTokenManager(CharStream stream, int lexState) : this(stream)
		{
			SwitchTo(lexState);
		}
		public virtual void  ReInit(CharStream stream)
		{
			jjmatchedPos = jjnewStateCnt = 0;
			curLexState = defaultLexState;
			input_stream = stream;
			ReInitRounds();
		}
		private void  ReInitRounds()
		{
			int i;
			jjround = 0x80000001;
			for (i = 75; i-- > 0; )
				jjrounds[i] = 0x80000000;
		}
		public virtual void  ReInit(CharStream stream, int lexState)
		{
			ReInit(stream);
			SwitchTo(lexState);
		}
		public virtual void  SwitchTo(int lexState)
		{
			if (lexState >= 1 || lexState < 0)
				throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE);
			else
				curLexState = lexState;
		}
		
		protected internal virtual Token JjFillToken()
		{
			Token t = Token.NewToken(jjmatchedKind);
			t.kind = jjmatchedKind;
			System.String im = jjstrLiteralImages[jjmatchedKind];
			t.image = (im == null) ? input_stream.GetImage() : im;
			t.beginLine = input_stream.GetBeginLine();
			t.beginColumn = input_stream.GetBeginColumn();
			t.endLine = input_stream.GetEndLine();
			t.endColumn = input_stream.GetEndColumn();
			return t;
		}
		
		internal int curLexState = 0;
		internal int defaultLexState = 0;
		internal int jjnewStateCnt;
		internal uint jjround;
		internal int jjmatchedPos;
		internal int jjmatchedKind;
		
		public virtual Token GetNextToken()
		{
			int kind;
			Token specialToken = null;
			Token matchedToken;
			int curPos = 0;
			
			for (; ; )
			{
				try
				{
					curChar = input_stream.BeginToken();
				}
				catch (System.IO.IOException)
				{
					jjmatchedKind = 0;
					matchedToken = JjFillToken();
					return matchedToken;
				}
				
				jjmatchedKind = 0x7fffffff;
				jjmatchedPos = 0;
				curPos = JjMoveStringLiteralDfa0_0();
				if (jjmatchedPos == 0 && jjmatchedKind > 15)
				{
					jjmatchedKind = 15;
				}
				if (jjmatchedKind != 0x7fffffff)
				{
					if (jjmatchedPos + 1 < curPos)
						input_stream.Backup(curPos - jjmatchedPos - 1);
					if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 63))) != 0L)
					{
						matchedToken = JjFillToken();
						return matchedToken;
					}
					else
					{
						goto EOFLoop;
					}
				}
				int error_line = input_stream.GetEndLine();
				int error_column = input_stream.GetEndColumn();
				System.String error_after = null;
				bool EOFSeen = false;
				try
				{
					input_stream.ReadChar(); input_stream.Backup(1);
				}
				catch (System.IO.IOException)
				{
					EOFSeen = true;
					error_after = curPos <= 1?"":input_stream.GetImage();
					if (curChar == '\n' || curChar == '\r')
					{
						error_line++;
						error_column = 0;
					}
					else
						error_column++;
				}
				if (!EOFSeen)
				{
					input_stream.Backup(1);
					error_after = curPos <= 1?"":input_stream.GetImage();
				}
				throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);

EOFLoop: ;
			}
		}
	}
}