lexer.cs

c#源代码

// <file>
//     <copyright see="prj:///doc/copyright.txt"/>
//     <license see="prj:///doc/license.txt"/>
//     <owner name="Andrea Paatz" email="[e-mail]"/>
//     <version value="$version"/>
// </file>

using System;
using System.Collections;
using System.Drawing;
using System.Diagnostics;
using System.Globalization;
using System.Text;
using ICSharpCode.SharpRefactory.Parser;

namespace ICSharpCode.SharpRefactory.Parser.VB
{
	public class Token
	{
		public int kind;
		
		public int col;
		public int line;
		
		public object    literalValue = null;
		public string    val;
		public Token     next;
		public ArrayList specials;
		
		public Point EndLocation {
			get {
				return new Point(col + val.Length - 1, line);
			}
		}
		
		public Point Location {
			get {
				return new Point(col, line);
			}
		}
		
		public Token()
		{
		}
		
		public Token(int kind)
		{
			this.kind = kind;
		}
		
		public Token(int kind, int col, int line, string val)
		{
			this.kind = kind;
			this.col  = col;
			this.line = line;
			this.val  = val;
		}
		
		public Token(int kind, int col, int line, string val, object literalValue)
		{
			this.kind         = kind;
			this.col          = col;
			this.line         = line;
			this.val          = val;
			this.literalValue = literalValue;
		}
	}
	
	public class Lexer
	{
		IReader reader;
		static  Hashtable keywords = new Hashtable();
		
		int col  = 1;
		int line = 1;
		
		bool lineEnd = false;
		
		Errors errors   = new Errors();
		
		SpecialTracker specialTracker = new SpecialTracker();
		
		Token lastToken = null;
		Token curToken  = null;
		Token peekToken = null;
		
		string[]       specialCommentTags = null;
		Hashtable      specialCommentHash = null;
		ArrayList      tagComments = new ArrayList();
		
		public ArrayList TagComments
		{
			get {
				return tagComments;
			}
		}
		
		public string[] SpecialCommentTags
		{
			get {
				return specialCommentTags;
			}
			set {
				specialCommentTags = value;
				specialCommentHash = new Hashtable();
				if (specialCommentTags != null) {
					foreach (string str in specialCommentTags) {
						specialCommentHash[str] = 0;
					}
				}
			}
		}
		
		public SpecialTracker SpecialTracker
		{
			get {
				return specialTracker;
			}
		}
		
		public Errors Errors
		{
			get {
				return errors;
			}
		}
		
		public Token Token
		{
			get {
				return lastToken;
			}
		}
		
		public Token LookAhead
		{
			get {
				return curToken;
			}
		}
		
		public void StartPeek()
		{
			peekToken = curToken;
		}
		
		public Token Peek()
		{
			if (peekToken.next == null) {
				peekToken.next = Next();
				specialTracker.InformToken(peekToken.next.kind);
			}
			peekToken = peekToken.next;
			return peekToken;
		}
		
		public Token NextToken()
		{
			if (curToken == null) { // first call of NextToken()
				curToken = Next();
				specialTracker.InformToken(curToken.kind);
				return curToken;
			}
			
			lastToken = curToken;
			
			if (curToken.next == null) {
				curToken.next = Next();
				specialTracker.InformToken(curToken.next.kind);
			}
			
			curToken = curToken.next;
			
			if (curToken.kind == Tokens.EOF && !(lastToken.kind == Tokens.EOL)) { // be sure that before EOF there is an EOL token
				curToken = new Token(Tokens.EOL, curToken.col, curToken.line, "\n");
				curToken.next = new Token(Tokens.EOF, curToken.col, curToken.line, "\n");
				specialTracker.InformToken(curToken.next.kind);
			}
			
			return curToken;
		}
		
		public ArrayList RetriveComments()
		{
			return specialTracker.RetrieveSpecials();
		}
		
//		public ArrayList RetrieveSpecials()
//		{
//			if (lastToken == null) {
//				return this.specialTracker.RetrieveSpecials();
//			}
//			
//			Debug.Assert(lastToken.specials != null);
//			
//			ArrayList tmp = lastToken.specials;
//			lastToken.specials = null;
//			return tmp;
//		}
//		
		public Lexer(IReader reader)
		{
			this.reader = reader;
		}
		
		public Token Next()
		{
			while (!reader.Eos()) {

				char ch = reader.GetNext();
		
				++col;
				if (Char.IsWhiteSpace(ch)) {
				
					if (ch == '\n') {
						int x = col - 1;
						int y = line;
						++line;
						col = 1;
						if (reader.Peek() == '\r') {
							reader.GetNext();
							if (!lineEnd) {
								lineEnd = true;
								return new Token(Tokens.EOL, x -1 , y, "\n\r");
							}
						}
						if (!lineEnd) {
							lineEnd = true;
							return new Token(Tokens.EOL, x, y, "\n");
						}
					}
					continue;

				}
				if (ch == '_') {
					if (reader.Eos()) {
						errors.Error(line, col, String.Format("No EOF expected after _"));
					}
					ch = reader.GetNext();
					++col;
					if (!Char.IsWhiteSpace(ch)) {
						reader.UnGet();
						--col;
						int x = col;
						int y = line;
						string s = ReadIdent('_');
						lineEnd = false;

						return new Token(Tokens.Identifier, x, y, s);
					}
					while (Char.IsWhiteSpace(ch)) {
						if (ch == '\n') {
							++line;
							col = 0;
							break;
						}
						if (!reader.Eos()) {
							ch = reader.GetNext();
							++col;
						}
					}
					if (ch != '\n') {
						errors.Error(line, col, String.Format("Return expected"));
					}
					continue;
				}
				
				if (ch == '#') {
					while (Char.IsWhiteSpace(reader.Peek())) {
						++col;
						reader.GetNext();
					}
					if (Char.IsDigit(reader.Peek())) {
						int x = col;
						int y = line;
						string s = ReadDate();
						DateTime time = DateTime.Now;
						try {
							time = System.DateTime.Parse(s, System.Globalization.CultureInfo.InvariantCulture);
						} catch (Exception e) {
							errors.Error(line, col, String.Format("Invalid date time {0}", e));
						}
						return new Token(Tokens.LiteralDate, x, y, s, time);
					} else {
						ReadPreprocessorDirective();
						continue;
					}
				}
				
				if (ch == '[') { // Identifier
					lineEnd = false;
					if (reader.Eos()) {
						errors.Error(line, col, String.Format("Identifier expected"));
					}
					ch = reader.GetNext();
					++col;
					if (ch == ']' || Char.IsWhiteSpace(ch)) {
						errors.Error(line, col, String.Format("Identifier expected"));
					}
					int x = col - 1;
					int y = line;
					string s = ReadIdent(ch);
					if (reader.Eos()) {
						errors.Error(line, col, String.Format("']' expected"));
					}
					ch = reader.GetNext();
					++col;
					if (!(ch == ']')) {
						errors.Error(line, col, String.Format("']' expected"));
					}
//					Console.WriteLine(">" + s + "<");
					return new Token(Tokens.Identifier, x, y, s);
				}
				if (Char.IsLetter(ch)) {
					int x = col - 1;
					int y = line;
					string s = ReadIdent(ch);
					if (Keywords.IsKeyword(s)) {
						lineEnd = false;
						return new Token(Keywords.GetToken(s), x, y, s);
					}
					
					// handle 'REM' comments 
					if (s.ToUpper(System.Globalization.CultureInfo.InvariantCulture) == "REM") {
						ReadComment();
						if (!lineEnd) {
							lineEnd = true;
							return new Token(Tokens.EOL, x, y, "\n");
						}
						continue;
					}
						
					lineEnd = false;
					return new Token(Tokens.Identifier, x, y, s);
				
				}
				if (Char.IsDigit(ch)) {
					lineEnd = false;
					return ReadDigit(ch, col);
				}
				if (ch == '&') {
					lineEnd = false;
					if (reader.Eos()) {
						return ReadOperator('&');
					}
					ch = reader.GetNext();
					++col;
					if (Char.ToUpper(ch, System.Globalization.CultureInfo.InvariantCulture) == 'H' || Char.ToUpper(ch, System.Globalization.CultureInfo.InvariantCulture) == 'O') {
						reader.UnGet();
						--col;
						return ReadDigit('&', col);
					} else {
						reader.UnGet();
						return ReadOperator('&');
					}
				}
				if (ch == '\'') {
					int x = col - 1;
					int y = line;
					ReadComment();
					if (!lineEnd) {
						lineEnd = true;
						return new Token(Tokens.EOL, x, y, "\n");
					}
					continue;
				}
				if (ch == '"') {
					lineEnd = false;
					int x = col - 1;
					int y = line;
					string s = ReadString();
					if (!reader.Eos() && (reader.Peek() == 'C' || reader.Peek() == 'c')) {
						reader.GetNext();
						++col;
						if (s.Length != 1) {
							errors.Error(line, col, String.Format("Chars can only have Length 1 "));
						}
						return new Token(Tokens.LiteralCharacter, x, y, String.Concat('"', s , "\"C") , s[0]);
					}
					return new Token(Tokens.LiteralString, x, y,  String.Concat('"', s , '"'), s);
				}
				Token token = ReadOperator(ch);
				if (token != null) {
					lineEnd = false;
					return token;
				}
				errors.Error(line, col, String.Format("Unknown char({0}) which can't be read", ch));
			}
			
			return new Token(Tokens.EOF);
		}
		
		string ReadIdent(char ch) 
		{
			StringBuilder s = new StringBuilder(ch.ToString());
			while (!reader.Eos() && (Char.IsLetterOrDigit(ch = reader.GetNext()) || ch == '_')) {
				++col;
				s.Append(ch.ToString());
			}
			++col;
			if (reader.Eos()) {
				--col;
				return s.ToString();
			}
			reader.UnGet();
			--col;
			if (!reader.Eos() && "%&@!#$".IndexOf(Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture)) != -1) {
				reader.GetNext();
				++col;
			}
			return s.ToString();
		}
		
		Token ReadDigit(char ch, int x)
		{
			StringBuilder sb = new StringBuilder(ch.ToString());
			int y = line;
			string digit = "";
			if (ch != '&') {
				digit += ch;
			}
			
			bool ishex      = false;
			bool isokt      = false;
			bool issingle   = false;
			bool isdouble   = false;
			bool isdecimal  = false;
			
			if (reader.Eos()) {
				if (ch == '&') {
					errors.Error(line, col, String.Format("digit expected"));
				}
				return new Token(Tokens.LiteralInteger, x, y, sb.ToString() ,ch - '0');
			}
			if (ch == '&' && Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture) == 'H') {
				const string hex = "0123456789ABCDEF";
				sb.Append(reader.GetNext()); // skip 'H'
				++col;
				while (!reader.Eos() && hex.IndexOf(Char.ToUpper(reader.Peek(), System.Globalization.CultureInfo.InvariantCulture)) != -1) {
					ch = reader.GetNext();
					sb.Append(ch); 
					digit += Char.ToUpper(ch, System.Globalization.CultureInfo.InvariantCulture);
					++col;
				}