lexer.cs

根据cs源码解析为codedom

						if (unsigned)
							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), UInt32.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
						else
							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int32.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
					} else if (ch == '&' || ch == 'L') {
						ReaderRead();
						if (unsigned)
							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), UInt64.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
						else
							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int64.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
					} else if (ishex) {
						ulong number = UInt64.Parse(digit, NumberStyles.HexNumber);
						if (number > uint.MaxValue) {
							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((long)number));
						} else {
							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((int)number));
						}
					}
				} catch (OverflowException ex) {
					errors.Error(Line, Col, ex.Message);
					return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
				}
			}
			Token nextToken = null; // if we accedently read a 'dot'
			if (!isdouble && ReaderPeek() == '.') { // read floating point number
				ReaderRead();
				if (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) {
					isdouble = true; // double is default
					if (ishex || isokt) {
						errors.Error(Line, Col, String.Format("No hexadecimal or oktadecimal floating point values allowed"));
					}
					digit += '.';
					while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())){ // read decimal digits beyond the dot
						digit += (char)ReaderRead();
					}
				} else {
					nextToken = new Token(Tokens.Dot, Col - 1, Line);
				}
			}
			
			if (ReaderPeek() != -1 && PeekUpperChar() == 'E') { // read exponent
				isdouble = true;
				digit +=  (char)ReaderRead();
				if (ReaderPeek() != -1 && (ReaderPeek() == '-' || ReaderPeek() == '+')) {
					digit += (char)ReaderRead();
				}
				while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) { // read exponent value
					digit += (char)ReaderRead();
				}
			}
			
			if (ReaderPeek() != -1) {
				switch (PeekUpperChar()) {
					case 'R':
					case '#':
						ReaderRead();
						isdouble = true;
						break;
					case 'D':
					case '@':
						ReaderRead();
						isdecimal = true;
						break;
					case 'F':
					case '!':
						ReaderRead();
						issingle = true;
						break;
				}
			}
			
			try {
				if (issingle) {
					return new Token(Tokens.LiteralSingle, x, y, sb.ToString(), Single.Parse(digit, CultureInfo.InvariantCulture));
				}
				if (isdecimal) {
					return new Token(Tokens.LiteralDecimal, x, y, sb.ToString(), Decimal.Parse(digit, NumberStyles.Currency | NumberStyles.AllowExponent, CultureInfo.InvariantCulture));
				}
				if (isdouble) {
					return new Token(Tokens.LiteralDouble, x, y, sb.ToString(), Double.Parse(digit, CultureInfo.InvariantCulture));
				}
			} catch (FormatException) {
				errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit));
				if (issingle)
					return new Token(Tokens.LiteralSingle, x, y, sb.ToString(), 0f);
				if (isdecimal)
					return new Token(Tokens.LiteralDecimal, x, y, sb.ToString(), 0m);
				if (isdouble)
					return new Token(Tokens.LiteralDouble, x, y, sb.ToString(), 0.0);
			}
			Token token;
			try {
				token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int32.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
			} catch (Exception) {
				try {
					token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int64.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
				} catch (FormatException) {
					errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit));
					// fallback, when nothing helps :)
					token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
				} catch (OverflowException) {
					errors.Error(Line, Col, String.Format("{0} is too long for a integer literal", digit));
					// fallback, when nothing helps :)
					token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
				}
			}
			token.next = nextToken;
			return token;
		}
		
		void ReadPreprocessorDirective()
		{
			Location start = new Location(Col - 1, Line);
			string directive = ReadIdent('#');
			string argument  = ReadToEndOfLine();
			this.specialTracker.AddPreprocessingDirective(directive, argument.Trim(), start, new Location(start.X + directive.Length + argument.Length, start.Y));
		}
		
		string ReadDate()
		{
			char ch = '\0';
			sb.Length = 0;
			int nextChar;
			while ((nextChar = ReaderRead()) != -1) {
				ch = (char)nextChar;
				if (ch == '#') {
					break;
				} else if (ch == '\n') {
					errors.Error(Line, Col, String.Format("No return allowed inside Date literal"));
				} else {
					sb.Append(ch);
				}
			}
			if (ch != '#') {
				errors.Error(Line, Col, String.Format("End of File reached before Date literal terminated"));
			}
			return sb.ToString();
		}
		
		string ReadString()
		{
			char ch = '\0';
			sb.Length = 0;
			int nextChar;
			while ((nextChar = ReaderRead()) != -1) {
				ch = (char)nextChar;
				if (ch == '"') {
					if (ReaderPeek() != -1 && ReaderPeek() == '"') {
						sb.Append('"');
						ReaderRead();
					} else {
						break;
					}
				} else if (ch == '\n') {
					errors.Error(Line, Col, String.Format("No return allowed inside String literal"));
				} else {
					sb.Append(ch);
				}
			}
			if (ch != '"') {
				errors.Error(Line, Col, String.Format("End of File reached before String terminated "));
			}
			return sb.ToString();
		}
		
		void ReadComment()
		{
			Location startPos = new Location(Col, Line);
			sb.Length = 0;
			StringBuilder curWord = specialCommentHash != null ? new StringBuilder() : null;
			int missingApostrophes = 2; // no. of ' missing until it is a documentation comment
			int nextChar;
			while ((nextChar = ReaderRead()) != -1) {
				char ch = (char)nextChar;
				
				if (HandleLineEnd(ch)) {
					break;
				}
				
				sb.Append(ch);
				
				if (missingApostrophes > 0) {
					if (ch == '\'' || ch == '\u2018' || ch == '\u2019') {
						if (--missingApostrophes == 0) {
							specialTracker.StartComment(CommentType.Documentation, startPos);
							sb.Length = 0;
						}
					} else {
						specialTracker.StartComment(CommentType.SingleLine, startPos);
						missingApostrophes = 0;
					}
				}
				
				if (specialCommentHash != null) {
					if (Char.IsLetter(ch)) {
						curWord.Append(ch);
					} else {
						string tag = curWord.ToString();
						curWord.Length = 0;
						if (specialCommentHash.ContainsKey(tag)) {
							Location p = new Location(Col, Line);
							string comment = ch + ReadToEndOfLine();
							this.TagComments.Add(new TagComment(tag, comment, p, new Location(Col, Line)));
							sb.Append(comment);
							break;
						}
					}
				}
			}
			if (missingApostrophes > 0) {
				specialTracker.StartComment(CommentType.SingleLine, startPos);
			}
			specialTracker.AddString(sb.ToString());
			specialTracker.FinishComment(new Location(Col, Line));
		}
		
		Token ReadOperator(char ch)
		{
			int x = Col - 1;
			int y = Line;
			switch(ch) {
				case '+':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.PlusAssign, x, y);
						default:
							break;
					}
					return new Token(Tokens.Plus, x, y);
				case '-':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.MinusAssign, x, y);
						default:
							break;
					}
					return new Token(Tokens.Minus, x, y);
				case '*':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.TimesAssign, x, y);
						default:
							break;
					}
					return new Token(Tokens.Times, x, y, "*");
				case '/':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.DivAssign, x, y);
						default:
							break;
					}
					return new Token(Tokens.Div, x, y);
				case '\\':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.DivIntegerAssign, x, y);
						default:
							break;
					}
					return new Token(Tokens.DivInteger, x, y);
				case '&':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.ConcatStringAssign, x, y);
						default:
							break;
					}
					return new Token(Tokens.ConcatString, x, y);
				case '^':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.PowerAssign, x, y);
						default:
							break;
					}
					return new Token(Tokens.Power, x, y);
				case ':':
					return new Token(Tokens.Colon, x, y);
				case '=':
					return new Token(Tokens.Assign, x, y);
				case '<':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.LessEqual, x, y);
						case '>':
							ReaderRead();
							return new Token(Tokens.NotEqual, x, y);
						case '<':
							ReaderRead();
							switch (ReaderPeek()) {
								case '=':
									ReaderRead();
									return new Token(Tokens.ShiftLeftAssign, x, y);
								default:
									break;
							}
							return new Token(Tokens.ShiftLeft, x, y);
					}
					return new Token(Tokens.LessThan, x, y);
				case '>':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.GreaterEqual, x, y);
						case '>':
							ReaderRead();
							if (ReaderPeek() != -1) {
								switch (ReaderPeek()) {
									case '=':
										ReaderRead();
										return new Token(Tokens.ShiftRightAssign, x, y);
									default:
										break;
								}
							}
							return new Token(Tokens.ShiftRight, x, y);
					}
					return new Token(Tokens.GreaterThan, x, y);
				case ',':
					return new Token(Tokens.Comma, x, y);
				case '.':
					// Prevent OverflowException when Peek returns -1
					int tmp = ReaderPeek();
					if (tmp > 0 && Char.IsDigit((char)tmp)) {
						return ReadDigit('.', Col);
					}
					return new Token(Tokens.Dot, x, y);
				case '(':
					return new Token(Tokens.OpenParenthesis, x, y);
				case ')':
					return new Token(Tokens.CloseParenthesis, x, y);
				case '{':
					return new Token(Tokens.OpenCurlyBrace, x, y);
				case '}':
					return new Token(Tokens.CloseCurlyBrace, x, y);
				case '?':
					return new Token(Tokens.QuestionMark, x, y);
			}
			return null;
		}
		
		public override void SkipCurrentBlock(int targetToken)
		{
			int lastKind = -1;
			int kind = base.lastToken.kind;
			while (kind != Tokens.EOF &&
			       !(lastKind == Tokens.End && kind == targetToken))
			{
				lastKind = kind;
				NextToken();
				kind = lastToken.kind;
			}
		}
	}
}