lexer.cs

c#源代码

// <file>
//     <copyright see="prj:///doc/copyright.txt"/>
//     <license see="prj:///doc/license.txt"/>
//     <owner name="Andrea Paatz" email="[e-mail]"/>
//     <version value="$version"/>
// </file>

using System;
using System.Collections;
using System.Drawing;
using System.Diagnostics;
using System.Globalization;
using System.Text;
using ICSharpCode.SharpRefactory.Parser;

namespace ICSharpCode.SharpRefactory.Parser
{
	public class Token
	{
		public int kind;
		
		public int col;
		public int line;
		
		public object literalValue = null;
		public string val;
		public Token  next;
		
		public Point EndLocation {
			get {
				return new Point(col + val.Length, line);
			}
		}
		public Point Location {
			get {
				return new Point(col, line);
			}
		}
		
		public Token()
		{
		}
		
		public Token(int kind)
		{
			this.kind = kind;
		}
		
		public Token(int kind, int col, int line, string val)
		{
			this.kind = kind;
			this.col  = col;
			this.line = line;
			this.val  = val;
		}
		
		public Token(int kind, int col, int line, string val, object literalValue)
		{
			this.kind         = kind;
			this.col          = col;
			this.line         = line;
			this.val          = val;
			this.literalValue = literalValue;
		}
	}
	
	public class Lexer
	{
		IReader reader;
		
		int col  = 1;
		int line = 1;
		
		Errors         errors   = new Errors();
		SpecialTracker specialTracker = new SpecialTracker();
		Token          lastToken = null;
		Token          curToken  = null;
		Token          peekToken = null;
		string[]       specialCommentTags = null;
		Hashtable      specialCommentHash = null;
		ArrayList      tagComments = new ArrayList();
		
		public ArrayList TagComments {
			get {
				return tagComments;
			}
		}
		
		public string[] SpecialCommentTags {
			get {
				return specialCommentTags;
			}
			set {
				specialCommentTags = value;
				specialCommentHash = new Hashtable();
				if (specialCommentTags != null) {
					foreach (string str in specialCommentTags) {
						specialCommentHash[str] = 0;
					}
				}
			}
		}
		
		public SpecialTracker SpecialTracker {
			get {
				return specialTracker;
			}
		}
		
		public Errors Errors {
			get {
				return errors;
			}
		}
		
		public Token Token {
			get {
				return lastToken;
			}
		}
		
		public Token LookAhead {
			get {
				return curToken;
			}
		}
		
		public void StartPeek()
		{
			peekToken = curToken;
		}
		
		public Token Peek()
		{
			if (peekToken.next == null) {
				peekToken.next = Next();
				specialTracker.InformToken(peekToken.next.kind);
			}
			peekToken = peekToken.next;
			return peekToken;
		}
		
		public Token NextToken()
		{
			if (curToken == null) {
				curToken = Next();
				specialTracker.InformToken(curToken.kind);
				return curToken;
			}
			
			lastToken = curToken;
			
			if (curToken.next == null) {
				curToken.next = Next();
				specialTracker.InformToken(curToken.next.kind);
			}
			
			curToken  = curToken.next;
			return curToken;
		}
		
		public Lexer(IReader reader)
		{
			this.reader = reader;
		}
		
		Token Next()
		{
			while (!reader.Eos()) {
				char ch = reader.GetNext();
				
				if (Char.IsWhiteSpace(ch)) {
					++col;
					
					if (ch == '\n') {
						specialTracker.AddEndOfLine();
						++line;
						col = 1;
					}
					continue;
				}
				
				if (Char.IsLetter(ch) || ch == '_') {
					int x = col;
					int y = line;
					string s = ReadIdent(ch);
					if (Keywords.IsKeyword(s)) {
						return new Token(Keywords.GetToken(s), x, y, s);
					}
					return new Token(Tokens.Identifier, x, y, s);
				}
				
				if (Char.IsDigit(ch)) {
					return ReadDigit(ch, col);
				}
				
				if (ch == '/') {
					if (reader.Peek() == '/' || reader.Peek() == '*') {
						++col;
						ReadComment();
						continue;
					}
				} else if (ch == '#') {
					Point start = new Point(col, line);
					++col;
					string directive = ReadIdent('#');
					string argument  = ReadToEOL();
					this.specialTracker.AddPreProcessingDirective(directive, argument, start, new Point(start.X + directive.Length + argument.Length, start.Y));
					continue;
				}
				
				if (ch == '"') {
					++col;
					return ReadString();
				}
				
				if (ch == '\'') {
					++col;
					return ReadChar();
				}
				
				if (ch == '@') {
					int x = col;
					int y = line;
					ch = reader.GetNext();
					++col;
					if (ch == '"') {
						return ReadVerbatimString();
					}
					if (Char.IsLetterOrDigit(ch)) {
						return new Token(Tokens.Identifier, x, y, ReadIdent(ch));
					}
					errors.Error(y, x, String.Format("Unexpected char in Lexer.Next() : {0}", ch));
				}
				
				Token token = ReadOperator(ch);
				
				// try error recovery :)
				if (token == null) {
					return Next();
				}
				return token;
			}
			
			return new Token(Tokens.EOF, col, line, String.Empty);
		}
		
		string ReadIdent(char ch)
		{
			StringBuilder s = new StringBuilder(ch.ToString());
			++col;
			while (!reader.Eos() && (Char.IsLetterOrDigit(ch = reader.GetNext()) || ch == '_')) {
				s.Append(ch.ToString());
				++col;
			}
			if (!reader.Eos()) {
				reader.UnGet();
			}
			return s.ToString();
		}
		
		Token ReadDigit(char ch, int x)
		{
			int y = line;
			++col;
			StringBuilder sb = new StringBuilder(ch.ToString());
			StringBuilder prefix = new StringBuilder();
			StringBuilder suffix = new StringBuilder();
			
			bool ishex      = false;
			bool isunsigned = false;
			bool islong     = false;
			bool isfloat    = false;
			bool isdouble   = false;
			bool isdecimal  = false;
			
			if (ch == '0' && Char.ToUpper(reader.Peek()) == 'X') {
				const string hex = "0123456789ABCDEF";
				reader.GetNext(); // skip 'x'
				++col;
				while (hex.IndexOf(Char.ToUpper(reader.Peek())) != -1) {
					sb.Append(Char.ToUpper(reader.GetNext()));
					++col;
				}
				ishex = true;
				prefix.Append("0x");
			} else {
				while (Char.IsDigit(reader.Peek())) {
					sb.Append(reader.GetNext());
					++col;
				}
			}
			
			if (reader.Peek() == '.') { // read floating point number
				reader.GetNext();
				if (!Char.IsDigit(reader.Peek())) {
					reader.UnGet();
				} else {
					isdouble = true; // double is default
					if (ishex) {
						errors.Error(y, x, String.Format("No hexadecimal floating point values allowed"));
					}
					sb.Append('.');
					
					
					++col;
					
					while (Char.IsDigit(reader.Peek())) { // read decimal digits beyond the dot
						sb.Append(reader.GetNext());
						++col;
					}
				}
			}
			
			if (Char.ToUpper(reader.Peek()) == 'E') { // read exponent
				isdouble = true;
				sb.Append(reader.GetNext());
				++col;
				if (reader.Peek() == '-' || reader.Peek() == '+') {
					sb.Append(reader.GetNext());
					++col;
				}
				while (Char.IsDigit(reader.Peek())) { // read exponent value
					sb.Append(reader.GetNext());
					++col;
				}
				isunsigned = true;
			}
			
			if (Char.ToUpper(reader.Peek()) == 'F') { // float value
				suffix.Append(reader.Peek());
				reader.GetNext();
				++col;
				isfloat = true;
			} else if (Char.ToUpper(reader.Peek()) == 'D') { // double type suffix (obsolete, double is default)
				suffix.Append(reader.Peek());
				reader.GetNext();
				++col;
				isdouble = true;
			} else if (Char.ToUpper(reader.Peek()) == 'M') { // decimal value
				suffix.Append(reader.Peek());
				reader.GetNext();
				++col;
				isdecimal = true;
			} else if (!isdouble) {
				if (Char.ToUpper(reader.Peek()) == 'U') {
					suffix.Append(reader.Peek());
					reader.GetNext();
					++col;
					isunsigned = true;
				}
				
				if (Char.ToUpper(reader.Peek()) == 'L') {
					suffix.Append(reader.Peek());
					reader.GetNext();
					++col;
					islong = true;
					if (!isunsigned && Char.ToUpper(reader.Peek()) == 'U') {
						suffix.Append(reader.Peek());
						reader.GetNext();
						++col;
						isunsigned = true;
					}
				}
			}
			
			string digit = sb.ToString();
			StringBuilder stringValue = new StringBuilder(prefix.ToString());
			stringValue.Append(digit);
			stringValue.Append(suffix.ToString());
			if (isfloat) {
				try {
					NumberFormatInfo numberFormatInfo = new NumberFormatInfo();
					numberFormatInfo.CurrencyDecimalSeparator = ".";
					return new Token(Tokens.Literal, x, y, stringValue.ToString(), Single.Parse(digit, numberFormatInfo));
				} catch (Exception) {
					errors.Error(y, x, String.Format("Can't parse float {0}", digit));
					return new Token(Tokens.Literal, x, y, stringValue.ToString(), 0f);
				}
			}
			if (isdecimal) {
				try {
					NumberFormatInfo numberFormatInfo = new NumberFormatInfo();
					numberFormatInfo.CurrencyDecimalSeparator = ".";
					return new Token(Tokens.Literal, x, y, stringValue.ToString(), Decimal.Parse(digit, numberFormatInfo));
				} catch (Exception) {
					errors.Error(y, x, String.Format("Can't parse decimal {0}", digit));
					return new Token(Tokens.Literal, x, y, stringValue.ToString(), 0m);
				}
			}
			if (isdouble) {
				try {
					NumberFormatInfo numberFormatInfo = new NumberFormatInfo();
					numberFormatInfo.CurrencyDecimalSeparator = ".";
					return new Token(Tokens.Literal, x, y, stringValue.ToString(), Double.Parse(digit, numberFormatInfo));
				} catch (Exception) {
					errors.Error(y, x, String.Format("Can't parse double {0}", digit));
					return new Token(Tokens.Literal, x, y, stringValue.ToString(), 0d);
				}
			}
			
			double d = 0;
			if (!Double.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Integer, null, out d)) {
				errors.Error(y, x, String.Format("Can't parse integral constant {0}", digit));
				return new Token(Tokens.Literal, x, y, stringValue.ToString(), 0);
			}
			if (d < long.MinValue || d > long.MaxValue) {
				islong = true;
				isunsigned = true;	
			}
			else if (d < uint.MinValue || d > uint.MaxValue) {
				islong = true;	
			}
			else if (d < int.MinValue || d > int.MaxValue) {
				isunsigned = true;	
			}
			if (islong) {
				if (isunsigned) {
					try {
						return new Token(Tokens.Literal, x, y, stringValue.ToString(), UInt64.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
					} catch (Exception) {
						errors.Error(y, x, String.Format("Can't parse unsigned long {0}", digit));
						return new Token(Tokens.Literal, x, y, stringValue.ToString(), 0UL);
					}
				} else {
					try {
						return new Token(Tokens.Literal, x, y, stringValue.ToString(), Int64.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
					} catch (Exception) {
						errors.Error(y, x, String.Format("Can't parse long {0}", digit));
						return new Token(Tokens.Literal, x, y, stringValue.ToString(), 0L);
					}
				}
			} else {
				if (isunsigned) {
					try {
						return new Token(Tokens.Literal, x, y, stringValue.ToString(), UInt32.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
					} catch (Exception) {
						errors.Error(y, x, String.Format("Can't parse unsigned int {0}", digit));
						return new Token(Tokens.Literal, x, y, stringValue.ToString(), 0U);
					}
				} else {
					try {
						return new Token(Tokens.Literal, x, y, stringValue.ToString(), Int32.Parse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number));
					} catch (Exception) {
						errors.Error(y, x, String.Format("Can't parse int {0}", digit));
						return new Token(Tokens.Literal, x, y, stringValue.ToString(), 0);
					}
				}
			}
		}
		
		Token ReadString()
		{
			int x = col;
			int y = line;
			
			char ch = '\0';
			StringBuilder s             = new StringBuilder();
			StringBuilder originalValue = new StringBuilder();
			originalValue.Append('"');
			while (!reader.Eos() && ((ch = reader.GetNext()) != '"')) {
				++col;
				if (ch == '\\') {
					originalValue.Append('\\');
					originalValue.Append(ReadEscapeSequence(out ch));
					s.Append(ch);
				} else if (ch == '\n') {
					errors.Error(y, x, String.Format("No new line is allowed inside a string literal"));
					break;
				} else {
					originalValue.Append(ch);
					s.Append(ch);
				}
			}
			if (ch != '"') {
				errors.Error(y, x, String.Format("End of file reached inside string literal"));
			}
			originalValue.Append('"');
			return new Token(Tokens.Literal, x, y, originalValue.ToString(), s.ToString());
		}
		
		Token ReadVerbatimString()
		{
			int x = col;
			int y = line;
			char ch = '\0';