lexer.cs

根据cs源码解析为codedom

// <file>
//     <copyright see="prj:///doc/copyright.txt"/>
//     <license see="prj:///doc/license.txt"/>
//     <owner name="Andrea Paatz" email="[e-mail]"/>
//     <version>$Revision: 2559 $</version>
// </file>

using System;
using System.Globalization;
using System.IO;
using System.Text;

namespace ICSharpCode.NRefactory.Parser.VB
{
	internal sealed class Lexer : AbstractLexer
	{
		bool lineEnd = true;
		
		public Lexer(TextReader reader) : base(reader)
		{
		}
		
		public override Token NextToken()
		{
			if (curToken == null) { // first call of NextToken()
				curToken = Next();
				specialTracker.InformToken(curToken.kind);
				//Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val);
				return curToken;
			}
			
			lastToken = curToken;
			
			if (curToken.next == null) {
				curToken.next = Next();
				specialTracker.InformToken(curToken.next.kind);
			}
			
			curToken = curToken.next;
			
			if (curToken.kind == Tokens.EOF && !(lastToken.kind == Tokens.EOL)) { // be sure that before EOF there is an EOL token
				curToken = new Token(Tokens.EOL, curToken.col, curToken.line, "\n");
				specialTracker.InformToken(curToken.kind);
				curToken.next = new Token(Tokens.EOF, curToken.col, curToken.line, "\n");
				specialTracker.InformToken(curToken.next.kind);
			}
			//Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val);
			return curToken;
		}
		
		protected override Token Next()
		{
			unchecked {
				int nextChar;
				while ((nextChar = ReaderRead()) != -1) {
					char ch = (char)nextChar;
					if (Char.IsWhiteSpace(ch)) {
						int x = Col - 1;
						int y = Line;
						if (HandleLineEnd(ch)) {
							if (lineEnd) {
								// second line end before getting to a token
								// -> here was a blank line
								specialTracker.AddEndOfLine(new Location(x, y));
							} else {
								lineEnd = true;
								return new Token(Tokens.EOL, x, y);
							}
						}
						continue;
					}
					if (ch == '_') {
						if (ReaderPeek() == -1) {
							errors.Error(Line, Col, String.Format("No EOF expected after _"));
							return new Token(Tokens.EOF);
						}
						if (!Char.IsWhiteSpace((char)ReaderPeek())) {
							int x = Col - 1;
							int y = Line;
							string s = ReadIdent('_');
							lineEnd = false;
							return new Token(Tokens.Identifier, x, y, s);
						}
						ch = (char)ReaderRead();
						
						lineEnd = false;
						while (Char.IsWhiteSpace(ch)) {
							if (HandleLineEnd(ch)) {
								lineEnd = true;
								break;
							}
							if (ReaderPeek() != -1) {
								ch = (char)ReaderRead();
							} else {
								errors.Error(Line, Col, String.Format("No EOF expected after _"));
								return new Token(Tokens.EOF);
							}
						}
						if (!lineEnd) {
							errors.Error(Line, Col, String.Format("Return expected"));
						}
						lineEnd = false;
						continue;
					}
					
					if (ch == '#') {
						while (Char.IsWhiteSpace((char)ReaderPeek())) {
							ReaderRead();
						}
						if (Char.IsDigit((char)ReaderPeek())) {
							int x = Col - 1;
							int y = Line;
							string s = ReadDate();
							DateTime time = new DateTime(1, 1, 1, 0, 0, 0);
							try {
								time = DateTime.Parse(s, System.Globalization.CultureInfo.InvariantCulture, DateTimeStyles.NoCurrentDateDefault);
							} catch (Exception e) {
								errors.Error(Line, Col, String.Format("Invalid date time {0}", e));
							}
							return new Token(Tokens.LiteralDate, x, y, s, time);
						} else {
							ReadPreprocessorDirective();
							continue;
						}
					}
					
					if (ch == '[') { // Identifier
						lineEnd = false;
						if (ReaderPeek() == -1) {
							errors.Error(Line, Col, String.Format("Identifier expected"));
						}
						ch = (char)ReaderRead();
						if (ch == ']' || Char.IsWhiteSpace(ch)) {
							errors.Error(Line, Col, String.Format("Identifier expected"));
						}
						int x = Col - 1;
						int y = Line;
						string s = ReadIdent(ch);
						if (ReaderPeek() == -1) {
							errors.Error(Line, Col, String.Format("']' expected"));
						}
						ch = (char)ReaderRead();
						if (!(ch == ']')) {
							errors.Error(Line, Col, String.Format("']' expected"));
						}
						return new Token(Tokens.Identifier, x, y, s);
					}
					if (Char.IsLetter(ch)) {
						int x = Col - 1;
						int y = Line;
						string s = ReadIdent(ch);
						int keyWordToken = Keywords.GetToken(s);
						if (keyWordToken >= 0) {
							lineEnd = false;
							return new Token(keyWordToken, x, y, s);
						}
						
						// handle 'REM' comments
						if (s.Equals("REM", StringComparison.InvariantCultureIgnoreCase)) {
							ReadComment();
							if (!lineEnd) {
								lineEnd = true;
								return new Token(Tokens.EOL, Col, Line, "\n");
							}
							continue;
						}
						
						lineEnd = false;
						return new Token(Tokens.Identifier, x, y, s);
						
					}
					if (Char.IsDigit(ch)) {
						lineEnd = false;
						return ReadDigit(ch, Col - 1);
					}
					if (ch == '&') {
						lineEnd = false;
						if (ReaderPeek() == -1) {
							return ReadOperator('&');
						}
						ch = (char)ReaderPeek();
						if (Char.ToUpper(ch, CultureInfo.InvariantCulture) == 'H' || Char.ToUpper(ch, CultureInfo.InvariantCulture) == 'O') {
							return ReadDigit('&', Col - 1);
						}
						return ReadOperator('&');
					}
					if (ch == '\'' || ch == '\u2018' || ch == '\u2019') {
						int x = Col - 1;
						int y = Line;
						ReadComment();
						if (!lineEnd) {
							lineEnd = true;
							return new Token(Tokens.EOL, x, y, "\n");
						}
						continue;
					}
					if (ch == '"') {
						lineEnd = false;
						int x = Col - 1;
						int y = Line;
						string s = ReadString();
						if (ReaderPeek() != -1 && (ReaderPeek() == 'C' || ReaderPeek() == 'c')) {
							ReaderRead();
							if (s.Length != 1) {
								errors.Error(Line, Col, String.Format("Chars can only have Length 1 "));
							}
							if (s.Length == 0) {
								s = "\0";
							}
							return new Token(Tokens.LiteralCharacter, x, y, '"' + s  + "\"C", s[0]);
						}
						return new Token(Tokens.LiteralString, x, y, '"' + s + '"', s);
					}
					Token token = ReadOperator(ch);
					if (token != null) {
						lineEnd = false;
						return token;
					}
					errors.Error(Line, Col, String.Format("Unknown char({0}) which can't be read", ch));
				}
				
				return new Token(Tokens.EOF);
			}
		}
		
		string ReadIdent(char ch)
		{
			sb.Length = 0;
			sb.Append(ch);
			int peek;
			while ((peek = ReaderPeek()) != -1 && (Char.IsLetterOrDigit(ch = (char)peek) || ch == '_')) {
				ReaderRead();
				sb.Append(ch.ToString());
			}
			if (peek == -1) {
				return sb.ToString();
			}
			
			if ("%&@!#$".IndexOf((char)peek) != -1) {
				ReaderRead();
			}
			return sb.ToString();
		}
		
		char PeekUpperChar()
		{
			return Char.ToUpper((char)ReaderPeek(), CultureInfo.InvariantCulture);
		}
		
		[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1818:DoNotConcatenateStringsInsideLoops")]
		Token ReadDigit(char ch, int x)
		{
			sb.Length = 0;
			sb.Append(ch);
			
			int y = Line;
			string digit = "";
			if (ch != '&') {
				digit += ch;
			}
			
			bool ishex      = false;
			bool isokt      = false;
			bool issingle   = false;
			bool isdouble   = false;
			bool isdecimal  = false;
			
			if (ReaderPeek() == -1) {
				if (ch == '&') {
					errors.Error(Line, Col, String.Format("digit expected"));
				}
				return new Token(Tokens.LiteralInteger, x, y, sb.ToString() ,ch - '0');
			}
			if (ch == '.') {
				if (Char.IsDigit((char)ReaderPeek())) {
					isdouble = true; // double is default
					if (ishex || isokt) {
						errors.Error(Line, Col, String.Format("No hexadecimal or oktadecimal floating point values allowed"));
					}
					while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())){ // read decimal digits beyond the dot
						digit += (char)ReaderRead();
					}
				}
			} else if (ch == '&' && PeekUpperChar() == 'H') {
				const string hex = "0123456789ABCDEF";
				sb.Append((char)ReaderRead()); // skip 'H'
				while (ReaderPeek() != -1 && hex.IndexOf(PeekUpperChar()) != -1) {
					ch = (char)ReaderRead();
					sb.Append(ch);
					digit += Char.ToUpper(ch, CultureInfo.InvariantCulture);
				}
				ishex = true;
			} else if (ReaderPeek() != -1 && ch == '&' && PeekUpperChar() == 'O') {
				const string okt = "01234567";
				sb.Append((char)ReaderRead()); // skip 'O'
				while (ReaderPeek() != -1 && okt.IndexOf(PeekUpperChar()) != -1) {
					ch = (char)ReaderRead();
					sb.Append(ch);
					digit += Char.ToUpper(ch, CultureInfo.InvariantCulture);
				}
				isokt = true;
			} else {
				while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) {
					ch = (char)ReaderRead();;
					digit += ch;
					sb.Append(ch);
				}
			}
			
			if (digit.Length == 0) {
				errors.Error(Line, Col, String.Format("digit expected"));
				return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
			}
			
			if (ReaderPeek() != -1 && ("%&SILU".IndexOf(PeekUpperChar()) != -1 || ishex || isokt)) {
				ch = (char)ReaderPeek();
				sb.Append(ch);
				ch = Char.ToUpper(ch, CultureInfo.InvariantCulture);
				bool unsigned = ch == 'U';
				if (unsigned) {
					ReaderRead(); // read the U
					ch = (char)ReaderPeek();
					sb.Append(ch);
					ch = Char.ToUpper(ch, CultureInfo.InvariantCulture);
					if (ch != 'I' && ch != 'L' && ch != 'S') {
						errors.Error(Line, Col, "Invalid type character: U" + ch);
					}
				}
				try {
					if (isokt) {
						ReaderRead();
						ulong number = 0L;
						for (int i = 0; i < digit.Length; ++i) {
							number = number * 8 + digit[i] - '0';
						}
						if (ch == 'S') {
							if (unsigned)
								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (ushort)number);
							else
								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (short)number);
						} else if (ch == '%' || ch == 'I') {
							if (unsigned)
								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (uint)number);
							else
								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (int)number);
						} else if (ch == '&' || ch == 'L') {
							if (unsigned)
								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (ulong)number);
							else
								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (long)number);
						} else {
							if (number > uint.MaxValue) {
								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((long)number));
							} else {
								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((int)number));
							}
						}
					}
					if (ch == 'S') {
						ReaderRead();
						if (unsigned)
							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), UInt16.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
						else
							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int16.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
					} else if (ch == '%' || ch == 'I') {
						ReaderRead();