lexer.cs

来自「全功能c#编译器」· CS 代码 · 共 681 行 · 第 1/2 页

// <file>
//     <copyright see="prj:///doc/copyright.txt"/>
//     <license see="prj:///doc/license.txt"/>
//     <owner name="Andrea Paatz" email="[e-mail]"/>
//     <version value="$version"/>
// </file>

using System;
using System.IO;
using System.Collections;
using System.Drawing;
using System.Diagnostics;
using System.Globalization;
using System.Text;
using ICSharpCode.CsVbRefactory.Parser;

namespace ICSharpCode.CsVbRefactory.Parser.VB
{
	public class Lexer : AbstractLexer
	{
		bool lineEnd = false;
		
		public ArrayList PreProcessingDirectives = new ArrayList();
		
		public Lexer(TextReader reader) : base(reader)
		{
		}
		
		public override Token NextToken()
		{
			if (curToken == null) { // first call of NextToken()
				curToken = Next();
				return curToken;
			}
			
			lastToken = curToken;
			
			if (curToken.next == null) {
				curToken.next = Next();
			}
			
			curToken = curToken.next;
			
			if (curToken.kind == Tokens.EOF && !(lastToken.kind == Tokens.EOL)) { // be sure that before EOF there is an EOL token
				curToken = new Token(Tokens.EOL, curToken.col, curToken.line, "\n");
				curToken.next = new Token(Tokens.EOF, curToken.col, curToken.line, "\n");
			}
			return curToken;
		}
		
		protected override Token Next()
		{
			int nextChar;
			while ((nextChar = reader.Read()) != -1) {
				char ch = (char)nextChar;
				
				++col;
				if (Char.IsWhiteSpace(ch)) {
					if (ch == '\n') {
						int x = col - 1;
						int y = line;
						++line;
						col = 1;
						if (reader.Peek() == '\r') {
							reader.Read();
							if (!lineEnd) {
								lineEnd = true;
								return new Token(Tokens.EOL, x -1 , y, "\n\r");
							}
						}
						if (!lineEnd) {
							lineEnd = true;
							return new Token(Tokens.EOL, x, y, "\n");
						}
					}
					continue;

				}
				if (ch == '_') {
					if (reader.Peek() == -1) {
						errors.Error(line, col, String.Format("No EOF expected after _"));
					}
					++col;
					if (!Char.IsWhiteSpace((char)reader.Peek())) {
						--col;
						int x = col;
						int y = line;
						string s = ReadIdent('_');
						lineEnd = false;

						return new Token(Tokens.Identifier, x, y, s);
					}
					ch = (char)reader.Read();
					
					while (Char.IsWhiteSpace(ch)) {
						if (ch == '\n') {
							++line;
							col = 0;
							break;
						}
						if (reader.Peek() != -1) {
							ch = (char)reader.Read();
							++col;
						}
					}
					if (ch != '\n') {
						errors.Error(line, col, String.Format("Return expected"));
					}
					continue;
				}
				
				if (ch == '#') {
					while (Char.IsWhiteSpace((char)reader.Peek())) {
						++col;
						reader.Read();
					}
					if (Char.IsDigit((char)reader.Peek())) {
						int x = col;
						int y = line;
						string s = ReadDate();
						DateTime time = DateTime.Now;
						try {
							time = System.DateTime.Parse(s, System.Globalization.CultureInfo.InvariantCulture);
						} catch (Exception e) {
							errors.Error(line, col, String.Format("Invalid date time {0}", e));
						}
						return new Token(Tokens.LiteralDate, x, y, s, time);
					} else {
						ReadPreprocessorDirective();
						continue;
					}
				}
				
				if (ch == '[') { // Identifier
					lineEnd = false;
					if (reader.Peek() == -1) {
						errors.Error(line, col, String.Format("Identifier expected"));
					}
					ch = (char)reader.Read();
					++col;
					if (ch == ']' || Char.IsWhiteSpace(ch)) {
						errors.Error(line, col, String.Format("Identifier expected"));
					}
					int x = col - 1;
					int y = line;
					string s = ReadIdent(ch);
					if (reader.Peek() == -1) {
						errors.Error(line, col, String.Format("']' expected"));
					}
					ch = (char)reader.Read();
					++col;
					if (!(ch == ']')) {
						errors.Error(line, col, String.Format("']' expected"));
					}
//					Console.WriteLine(">" + s + "<");
					return new Token(Tokens.Identifier, x, y, s);
				}
				if (Char.IsLetter(ch)) {
					int x = col - 1;
					int y = line;
					string s = ReadIdent(ch);
					if (Keywords.IsKeyword(s)) {
						lineEnd = false;
						return new Token(Keywords.GetToken(s), x, y, s);
					}
					
					// handle 'REM' comments 
					if (s.ToUpper() == "REM") {
						ReadComment();
						if (!lineEnd) {
							lineEnd = true;
							return new Token(Tokens.EOL, x, y, "\n");
						}
						continue;
					}
						
					lineEnd = false;
					return new Token(Tokens.Identifier, x, y, s);
				
				}
				if (Char.IsDigit(ch)) {
					lineEnd = false;
					return ReadDigit(ch, col);
				}
				if (ch == '&') {
					lineEnd = false;
					if (reader.Peek() == -1) {
						return ReadOperator('&');
					}
					ch = (char)reader.Peek();
					++col;
					if (Char.ToUpper(ch) == 'H' || Char.ToUpper(ch) == 'O') {
						--col;
						return ReadDigit('&', col);
					} 
					return ReadOperator('&');
				}
				if (ch == '\'') {
					int x = col - 1;
					int y = line;
					ReadComment();
					if (!lineEnd) {
						lineEnd = true;
						return new Token(Tokens.EOL, x, y, "\n");
					}
					continue;
				}
				if (ch == '"') {
					lineEnd = false;
					int x = col - 1;
					int y = line;
					string s = ReadString();
					if (reader.Peek() != -1 && (reader.Peek() == 'C' || reader.Peek() == 'c')) {
						reader.Read();
						++col;
						if (s.Length != 1) {
							errors.Error(line, col, String.Format("Chars can only have Length 1 "));
						}
						return new Token(Tokens.LiteralCharacter, x, y, String.Concat('"', s , "\"C") , s[0]);
					}
					return new Token(Tokens.LiteralString, x, y,  String.Concat('"', s , '"'), s);
				}
				Token token = ReadOperator(ch);
				if (token != null) {
					lineEnd = false;
					return token;
				}
				errors.Error(line, col, String.Format("Unknown char({0}) which can't be read", ch));
			}
			
			return new Token(Tokens.EOF);
		}
		
		string ReadIdent(char ch) 
		{
			sb.Length = 0;
			sb.Append(ch);
			int peek;
			while ((peek = reader.Peek()) != -1 && (Char.IsLetterOrDigit(ch = (char)peek) || ch == '_')) {
				reader.Read();
				++col;
				sb.Append(ch.ToString());
			}
			++col;
			if (peek == -1) {
				--col;
				return sb.ToString();
			}
			
			--col;
			if (peek != -1 && "%&@!#$".IndexOf((char)peek) != -1) {
				reader.Read();
				++col;
			}
			return sb.ToString();
		}
		
		Token ReadDigit(char ch, int x)
		{
			sb.Length = 0;
			sb.Append(ch);
			
			int y = line;
			string digit = "";
			if (ch != '&') {
				digit += ch;
			}
			
			bool ishex      = false;
			bool isokt      = false;
			bool issingle   = false;
			bool isdouble   = false;
			bool isdecimal  = false;
			
			if (reader.Peek() == -1) {
				if (ch == '&') {
					errors.Error(line, col, String.Format("digit expected"));
				}
				return new Token(Tokens.LiteralInteger, x, y, sb.ToString() ,ch - '0');
			}
			if (ch == '.') {
				if (Char.IsDigit((char)reader.Peek())) {
					isdouble = true; // double is default
					if (ishex || isokt) {
						errors.Error(line, col, String.Format("No hexadecimal or oktadecimal floating point values allowed"));
					}
					digit += '.';
					++col;
					while (reader.Peek() != -1 && Char.IsDigit((char)reader.Peek())){ // read decimal digits beyond the dot
						digit += (char)reader.Read();
						++col;
					}
				}
			} else if (ch == '&' && Char.ToUpper((char)reader.Peek()) == 'H') {
				const string hex = "0123456789ABCDEF";
				sb.Append((char)reader.Read()); // skip 'H'
				++col;
				while (reader.Peek() != -1 && hex.IndexOf(Char.ToUpper((char)reader.Peek())) != -1) {
					ch = (char)reader.Read();
					sb.Append(ch); 
					digit += Char.ToUpper(ch);
					++col;
				}
				ishex = true;
			} else if (reader.Peek() != -1 && ch == '&' && Char.ToUpper((char)reader.Peek()) == 'O') {
				const string okt = "01234567";
				sb.Append((char)reader.Read()); // skip 'O'
				++col;
				while (reader.Peek() != -1 && okt.IndexOf(Char.ToUpper((char)reader.Peek())) != -1) {
					ch = (char)reader.Read();
					sb.Append(ch); 
					digit += Char.ToUpper(ch);
					++col;
				}
				isokt = true;
			} else {
				while (reader.Peek() != -1 && Char.IsDigit((char)reader.Peek())) {
					ch = (char)reader.Read();;
					digit += ch;
					sb.Append(ch);
					++col;
				}
			}
			
			if (reader.Peek() != -1 && "%&SIL".IndexOf(Char.ToUpper((char)reader.Peek())) != -1 || ishex || isokt) {
				ch = (char)reader.Peek();
				sb.Append(ch); 
				ch = Char.ToUpper(ch);
				++col;
				if (isokt) {
					reader.Read();
					long number = 0L;
					for (int i = 0; i < digit.Length; ++i) {
						number = number * 8 + digit[i] - '0';
					}
					if (ch == 'S') {
						return new Token(Tokens.LiteralSingle, x, y, sb.ToString(), (short)number);
					} else if (ch == '%' || ch == 'I') {
						return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (int)number);
					} else if (ch == '&' || ch == 'L') {
						return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (long)number);