lexer.cs

c#源代码

				ishex = true;
			} else if (!reader.Eos() && ch == '&' && Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture) == 'O') {
				const string okt = "01234567";
				sb.Append(reader.GetNext()); // skip 'O'
				++col;
				while (!reader.Eos() && okt.IndexOf(Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture)) != -1) {
					ch = reader.GetNext();
					sb.Append(ch); 
					digit += Char.ToUpper(ch, System.Globalization.CultureInfo.InvariantCulture);
					++col;
				}
				isokt = true;
			} else {
				while (!reader.Eos() && Char.IsDigit(reader.Peek())) {
					ch = reader.GetNext();;
					digit += ch;
					sb.Append(ch);
					++col;
				}
			}
			if (!reader.Eos() && "%&SIL".IndexOf(Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture)) != -1 || ishex || isokt) {
				ch = reader.GetNext();
				sb.Append(ch); 
				ch = Char.ToUpper(ch, System.Globalization.CultureInfo.InvariantCulture);
				++col;
				if (isokt) {
					long number = 0L;
					for (int i = 0; i < digit.Length; ++i) {
						number = number * 8 + digit[i] - '0';
					}
					if (ch == 'S') {
						return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (short)number);
					} else if (ch == '%' || ch == 'I') {
						return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (int)number);
					} else if (ch == '&' || ch == 'L') {
						return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (long)number);
					} else {
						if (number > int.MaxValue || number < int.MinValue) {
							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (long)number);
						} else {
							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (int)number);
						}
					}
				}
				if (ch == 'S') {
					return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int16.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
				} else if (ch == '%' || ch == 'I') {
					return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int32.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
				} else if (ch == '&' || ch == 'L') {
					return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int64.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
				} else if (ishex) {
					reader.UnGet();
					--col;
					long number = Int64.Parse(digit, NumberStyles.HexNumber);
					if (number > int.MaxValue || number < int.MinValue) {
						return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), number);
					} else {
						return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (int)number);
					}
				}
			}
			if (!reader.Eos() && reader.Peek() == '.') { // read floating point number
				reader.GetNext();
				if (!Char.IsDigit(reader.Peek())) {
					reader.UnGet();
				} else {
					isdouble = true; // double is default
					if (ishex || isokt) {
						errors.Error(line, col, String.Format("No hexadecimal or oktadecimal floating point values allowed"));
					}
					digit += '.';
					++col;
					while (!reader.Eos() && Char.IsDigit(reader.Peek())){ // read decimal digits beyond the dot
						digit += reader.GetNext();
						++col;
					}
				}
			}
			
			if (!reader.Eos() && Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture) == 'E') { // read exponent
				isdouble = true;
				digit +=  reader.GetNext();
				++col;
				if (!reader.Eos() && (reader.Peek() == '-' || reader.Peek() == '+')) {
					digit += reader.GetNext();
					++col;
				}
				while (!reader.Eos() && Char.IsDigit(reader.Peek())) { // read exponent value
					digit += reader.GetNext();
					++col;
				}
			}
			
			if (!reader.Eos()) {
				if (Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture) == 'R' || Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture) == '#') { // double type suffix (obsolete, double is default)
					reader.GetNext();
					++col;
					isdouble = true;
				} else if (Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture) == 'D' || Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture) == '@') { // decimal value
					reader.GetNext();
					++col;
					isdecimal = true;
				} else if (Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture) == 'F' || Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture) == '!') { // decimal value
					reader.GetNext();
					++col;
					issingle = true;
				}
			}
			
			if (issingle) {
				NumberFormatInfo mi = new NumberFormatInfo();
				mi.CurrencyDecimalSeparator = ".";
				return new Token(Tokens.LiteralSingle, x, y, sb.ToString(), Single.Parse(digit, mi));
			}
			if (isdecimal) {
				NumberFormatInfo mi = new NumberFormatInfo();
				mi.CurrencyDecimalSeparator = ".";
				return new Token(Tokens.LiteralDecimal, x, y, sb.ToString(), Decimal.Parse(digit, mi));
			}
			if (isdouble) {
				NumberFormatInfo mi = new NumberFormatInfo();
				mi.CurrencyDecimalSeparator = ".";
				return new Token(Tokens.LiteralDouble, x, y, sb.ToString(), Double.Parse(digit, mi));
			}
			try {
				return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int32.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
			} catch (Exception) {
				try {
					return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int64.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
				} catch (Exception) {
					errors.Error(line, col, String.Format("{0} is not a parseable number (too long?)", sb.ToString()));
					// fallback, when nothing helps :)
					return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
				}
			}
		}
		
		void ReadPreprocessorDirective()
		{
			Point start = new Point(col - 1, line);
			string directive = ReadIdent('#');
			string argument  = ReadToEOL();
			this.specialTracker.AddPreProcessingDirective(directive, argument.Trim(), start, new Point(start.X + directive.Length + argument.Length, start.Y));
		}
		
		string ReadToEOL()
		{
			StringBuilder sb = new StringBuilder();
			if (!reader.Eos()) {
				char ch = reader.GetNext();
				while (!reader.Eos()) {
					if (ch == '\r') {
						if (reader.Peek() == '\n') {
							ch = reader.GetNext();
						}
					}
					if (ch == '\n') {
						++line;
						col = 1;
						return sb.ToString();;
					} else {
						sb.Append(ch);
					}
					ch = reader.GetNext();
					++col;
				}
			}
			return sb.ToString();
		}
		
		string ReadDate()
		{
			char ch = '\0';
			StringBuilder sb = new StringBuilder();
			while (!reader.Eos()) {
				ch = reader.GetNext();
				++col;
				if (ch == '#') {
					break;
				} else if (ch == '\n') {
					errors.Error(line, col, String.Format("No return allowed inside Date literal"));
				} else {
					sb.Append(ch);
				}
			}
			if (ch != '#') {
				errors.Error(line, col, String.Format("End of File reached before Date literal terminated"));
			}
			return sb.ToString();
		}
		
		string ReadString()
		{
			char ch = '\0';
			StringBuilder s = new StringBuilder();
			while (!reader.Eos()) {
				ch = reader.GetNext();
				++col;
				if (ch == '"') {
					if (!reader.Eos() && reader.Peek() == '"') {
						s.Append('"');
						reader.GetNext();
						++col;
					} else {
						break;
					}
				} else if (ch == '\n') {
					errors.Error(line, col, String.Format("No return allowed inside String literal"));
				} else {
					s.Append(ch);
				}
			}
			if (ch != '"') {
				errors.Error(line, col, String.Format("End of File reached before String terminated "));
			}
			return s.ToString();
		}
		
		protected bool HandleLineEnd(char ch)
		{
			if (WasLineEnd(ch)) {
				++line;
				col = 1;
				return true;
			}
			return false;
		}
		
		protected bool WasLineEnd(char ch)
		{
			// Handle MS-DOS or MacOS line ends.
			if (ch == '\r') {
				if (reader.Peek() == '\n') { // MS-DOS line end '\r\n'
					ch = (char)reader.GetNext();
				} else { // assume MacOS line end which is '\r'
					ch = '\n';
				}
			}
			return ch == '\n';
		}
		
		void ReadComment()
		{
			StringBuilder curWord = new StringBuilder();
			StringBuilder comment = new StringBuilder();
			
			int x = col;
			int y = line;
			int nextChar;
			while ((nextChar = reader.GetNext()) != -1) {
				char ch = (char)nextChar;
				comment.Append(ch);
				++col;
				if (HandleLineEnd(ch) || nextChar == 0) {
					specialTracker.StartComment(CommentType.SingleLine, new Point(col, line));
					specialTracker.AddString(comment.ToString());
					specialTracker.FinishComment();
					return;
				}
				
				if (Char.IsLetter(ch)) {
					curWord.Append(ch);
				} else {
					string tag = curWord.ToString();
					curWord = new StringBuilder();
					if (specialCommentHash != null && specialCommentHash[tag] != null) {
						Point p = new Point(col, line);
						string commentStr = ReadToEOL();
						tagComments.Add(new TagComment(tag, commentStr, p));
						return;
					}
				}
			}
		}
		
		Token ReadOperator(char ch)
		{
			int x = col;
			int y = line;
			switch(ch) {
				case '+':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.PlusAssign, x, y, "+=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.Plus, x, y, "+");
				case '-':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.MinusAssign, x, y, "-=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.Minus, x, y, "-");
				case '*':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.TimesAssign, x, y, "*=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.Times, x, y, "*");
				case '/':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.DivAssign, x, y, "/=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.Div, x, y, "/");
				case '\\':
					switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.DivIntegerAssign, x, y, "\\=");
							default:
								reader.UnGet();
								break;
						}
					return new Token(Tokens.DivInteger, x, y, "\\");
				case '&':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.ConcatStringAssign, x, y, "&=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.ConcatString, x, y, "&");
				case '^':
					switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.PowerAssign, x, y, "^=");
							default:
								reader.UnGet();
								break;
						}
					return new Token(Tokens.Power, x, y, "^");
				case ':':
					return new Token(Tokens.Colon, x, y, ":");
				case '=':
					return new Token(Tokens.Assign, x, y, "=");
				case '<':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.LessEqual, x, y, "<=");
							case '>':
								++col;
								return new Token(Tokens.NotEqual, x, y, "<>");
							case '<':
								if (!reader.Eos()) {
									switch (reader.GetNext()) {
										case '=':
											col += 2;
											return new Token(Tokens.ShiftLeftAssign, x, y, "<<=");
										default:
											++col;
											reader.UnGet();
											break;
									}
								}
								return new Token(Tokens.ShiftLeft, x, y, "<<");
							default:
								reader.UnGet();
								return new Token(Tokens.LessThan, x, y, "<");
						}
					}
					return new Token(Tokens.LessThan, x, y, "<");
				case '>':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.GreaterEqual, x, y, ">=");
							case '>':
								if (!reader.Eos()) {
									switch (reader.GetNext()) {
										case '=':
											col += 2;
											return new Token(Tokens.ShiftRightAssign, x, y, ">>=");
										default:
											++col;
											reader.UnGet();
											break;
									}
								}
								return new Token(Tokens.ShiftRight, x, y, ">>");
							default:
								reader.UnGet();
								return new Token(Tokens.GreaterThan, x, y, ">");
						}
					}
					return new Token(Tokens.GreaterThan, x, y, "<=");
				case ',':
					return new Token(Tokens.Comma, x, y, ",");
				case '.':
					if (Char.IsDigit(reader.Peek())) {
						 reader.UnGet();
						 --col;
						 return ReadDigit('0', col);
					}
					return new Token(Tokens.Dot, x, y, ".");
				case '(':
					return new Token(Tokens.OpenParenthesis, x, y, "(");
				case ')':
					return new Token(Tokens.CloseParenthesis, x, y, ")");
				case '{':
					return new Token(Tokens.OpenCurlyBrace, x, y, "{");
				case '}':
					return new Token(Tokens.CloseCurlyBrace, x, y, "}");
				case '[':
					return new Token(Tokens.OpenSquareBracket, x, y, "[");
				case ']':
					return new Token(Tokens.CloseSquareBracket, x, y, "]");
			}
			return null;
		}
	}
}