lexer.cs

根据cs源码解析为codedom

			if (nextChar == -1) {
				errors.Error(Line, Col, String.Format("End of file reached inside escape sequence"));
				ch = '\0';
				return String.Empty;
			}
			int number;
			char c = (char)nextChar;
			int curPos              = 1;
			escapeSequenceBuffer[0] = c;
			switch (c)  {
				case '\'':
					ch = '\'';
					break;
				case '\"':
					ch = '\"';
					break;
				case '\\':
					ch = '\\';
					break;
				case '0':
					ch = '\0';
					break;
				case 'a':
					ch = '\a';
					break;
				case 'b':
					ch = '\b';
					break;
				case 'f':
					ch = '\f';
					break;
				case 'n':
					ch = '\n';
					break;
				case 'r':
					ch = '\r';
					break;
				case 't':
					ch = '\t';
					break;
				case 'v':
					ch = '\v';
					break;
				case 'u':
				case 'x':
					// 16 bit unicode character
					c = (char)ReaderRead();
					number = GetHexNumber(c);
					escapeSequenceBuffer[curPos++] = c;
					
					if (number < 0) {
						errors.Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", c));
					}
					for (int i = 0; i < 3; ++i) {
						if (IsHex((char)ReaderPeek())) {
							c = (char)ReaderRead();
							int idx = GetHexNumber(c);
							escapeSequenceBuffer[curPos++] = c;
							number = 16 * number + idx;
						} else {
							break;
						}
					}
					ch = (char)number;
					break;
				case 'U':
					// 32 bit unicode character
					number = 0;
					for (int i = 0; i < 8; ++i) {
						if (IsHex((char)ReaderPeek())) {
							c = (char)ReaderRead();
							int idx = GetHexNumber(c);
							escapeSequenceBuffer[curPos++] = c;
							number = 16 * number + idx;
						} else {
							errors.Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", (char)ReaderPeek()));
							break;
						}
					}
					if (number > 0xffff) {
						ch = '\0';
						surrogatePair = char.ConvertFromUtf32(number);
					} else {
						ch = (char)number;
					}
					break;
				default:
					errors.Error(Line, Col, String.Format("Unexpected escape sequence : {0}", c));
					ch = '\0';
					break;
			}
			return new String(escapeSequenceBuffer, 0, curPos);
		}
		
		Token ReadChar()
		{
			int x = Col - 1;
			int y = Line;
			int nextChar = ReaderRead();
			if (nextChar == -1) {
				errors.Error(y, x, String.Format("End of file reached inside character literal"));
				return null;
			}
			char ch = (char)nextChar;
			char chValue = ch;
			string escapeSequence = String.Empty;
			if (ch == '\\') {
				string surrogatePair;
				escapeSequence = ReadEscapeSequence(out chValue, out surrogatePair);
				if (surrogatePair != null) {
					errors.Error(y, x, String.Format("The unicode character must be represented by a surrogate pair and does not fit into a System.Char"));
				}
			}
			
			unchecked {
				if ((char)ReaderRead() != '\'') {
					errors.Error(y, x, String.Format("Char not terminated"));
				}
			}
			return new Token(Tokens.Literal, x, y, "'" + ch + escapeSequence + "'", chValue);
		}
		
		Token ReadOperator(char ch)
		{
			int x = Col - 1;
			int y = Line;
			switch (ch) {
				case '+':
					switch (ReaderPeek()) {
						case '+':
							ReaderRead();
							return new Token(Tokens.Increment, x, y);
						case '=':
							ReaderRead();
							return new Token(Tokens.PlusAssign, x, y);
					}
					return new Token(Tokens.Plus, x, y);
				case '-':
					switch (ReaderPeek()) {
						case '-':
							ReaderRead();
							return new Token(Tokens.Decrement, x, y);
						case '=':
							ReaderRead();
							return new Token(Tokens.MinusAssign, x, y);
						case '>':
							ReaderRead();
							return new Token(Tokens.Pointer, x, y);
					}
					return new Token(Tokens.Minus, x, y);
				case '*':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.TimesAssign, x, y);
						default:
							break;
					}
					return new Token(Tokens.Times, x, y);
				case '/':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.DivAssign, x, y);
					}
					return new Token(Tokens.Div, x, y);
				case '%':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.ModAssign, x, y);
					}
					return new Token(Tokens.Mod, x, y);
				case '&':
					switch (ReaderPeek()) {
						case '&':
							ReaderRead();
							return new Token(Tokens.LogicalAnd, x, y);
						case '=':
							ReaderRead();
							return new Token(Tokens.BitwiseAndAssign, x, y);
					}
					return new Token(Tokens.BitwiseAnd, x, y);
				case '|':
					switch (ReaderPeek()) {
						case '|':
							ReaderRead();
							return new Token(Tokens.LogicalOr, x, y);
						case '=':
							ReaderRead();
							return new Token(Tokens.BitwiseOrAssign, x, y);
					}
					return new Token(Tokens.BitwiseOr, x, y);
				case '^':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.XorAssign, x, y);
						default:
							break;
					}
					return new Token(Tokens.Xor, x, y);
				case '!':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.NotEqual, x, y);
					}
					return new Token(Tokens.Not, x, y);
				case '~':
					return new Token(Tokens.BitwiseComplement, x, y);
				case '=':
					switch (ReaderPeek()) {
						case '=':
							ReaderRead();
							return new Token(Tokens.Equal, x, y);
					}
					return new Token(Tokens.Assign, x, y);
				case '<':
					switch (ReaderPeek()) {
						case '<':
							ReaderRead();
							switch (ReaderPeek()) {
								case '=':
									ReaderRead();
									return new Token(Tokens.ShiftLeftAssign, x, y);
								default:
									break;
							}
							return new Token(Tokens.ShiftLeft, x, y);
						case '=':
							ReaderRead();
							return new Token(Tokens.LessEqual, x, y);
					}
					return new Token(Tokens.LessThan, x, y);
				case '>':
					switch (ReaderPeek()) {
							// Removed because of generics:
//						case '>':
//							ReaderRead();
//							if (ReaderPeek() != -1) {
//								switch ((char)ReaderPeek()) {
//									case '=':
//										ReaderRead();
//										return new Token(Tokens.ShiftRightAssign, x, y);
//									default:
//										break;
//								}
//							}
//							return new Token(Tokens.ShiftRight, x, y);
						case '=':
							ReaderRead();
							return new Token(Tokens.GreaterEqual, x, y);
					}
					return new Token(Tokens.GreaterThan, x, y);
				case '?':
					if (ReaderPeek() == '?') {
						ReaderRead();
						return new Token(Tokens.DoubleQuestion, x, y);
					}
					return new Token(Tokens.Question, x, y);
				case ';':
					return new Token(Tokens.Semicolon, x, y);
				case ':':
					if (ReaderPeek() == ':') {
						ReaderRead();
						return new Token(Tokens.DoubleColon, x, y);
					}
					return new Token(Tokens.Colon, x, y);
				case ',':
					return new Token(Tokens.Comma, x, y);
				case '.':
					// Prevent OverflowException when ReaderPeek returns -1
					int tmp = ReaderPeek();
					if (tmp > 0 && Char.IsDigit((char)tmp)) {
						return ReadDigit('.', Col - 1);
					}
					return new Token(Tokens.Dot, x, y);
				case ')':
					return new Token(Tokens.CloseParenthesis, x, y);
				case '(':
					return new Token(Tokens.OpenParenthesis, x, y);
				case ']':
					return new Token(Tokens.CloseSquareBracket, x, y);
				case '[':
					return new Token(Tokens.OpenSquareBracket, x, y);
				case '}':
					return new Token(Tokens.CloseCurlyBrace, x, y);
				case '{':
					return new Token(Tokens.OpenCurlyBrace, x, y);
				default:
					return null;
			}
		}
		
		void ReadComment()
		{
			switch (ReaderRead()) {
				case '*':
					ReadMultiLineComment();
					break;
				case '/':
					if (ReaderPeek() == '/') {
						ReaderRead();
						ReadSingleLineComment(CommentType.Documentation);
					} else {
						ReadSingleLineComment(CommentType.SingleLine);
					}
					break;
				default:
					errors.Error(Line, Col, String.Format("Error while reading comment"));
					break;
			}
		}
		
		string ReadCommentToEOL()
		{
			if (specialCommentHash == null) {
				return ReadToEndOfLine();
			}
			sb.Length = 0;
			StringBuilder curWord = new StringBuilder();
			
			int nextChar;
			while ((nextChar = ReaderRead()) != -1) {
				char ch = (char)nextChar;
				
				if (HandleLineEnd(ch)) {
					break;
				}
				
				sb.Append(ch);
				if (IsIdentifierPart(nextChar)) {
					curWord.Append(ch);
				} else {
					string tag = curWord.ToString();
					curWord.Length = 0;
					if (specialCommentHash.ContainsKey(tag)) {
						Location p = new Location(Col, Line);
						string comment = ch + ReadToEndOfLine();
						this.TagComments.Add(new TagComment(tag, comment, p, new Location(Col, Line)));
						sb.Append(comment);
						break;
					}
				}
			}
			return sb.ToString();
		}
		
		void ReadSingleLineComment(CommentType commentType)
		{
			if (this.SkipAllComments) {
				SkipToEndOfLine();
			} else {
				specialTracker.StartComment(commentType, new Location(Col, Line));
				specialTracker.AddString(ReadCommentToEOL());
				specialTracker.FinishComment(new Location(Col, Line));
			}
		}
		
		void ReadMultiLineComment()
		{
			int nextChar;
			if (this.SkipAllComments) {
				while ((nextChar = ReaderRead()) != -1) {
					char ch = (char)nextChar;
					if (ch == '*' && ReaderPeek() == '/') {
						ReaderRead();
						return;
					} else {
						HandleLineEnd(ch);
					}
				}
			} else {
				specialTracker.StartComment(CommentType.Block, new Location(Col, Line));
				
				// sc* = special comment handling (TO DO markers)
				string scTag = null; // is set to non-null value when we are inside a comment marker
				StringBuilder scCurWord = new StringBuilder(); // current word, (scTag == null) or comment (when scTag != null)
				Location scStartLocation = Location.Empty;
				
				while ((nextChar = ReaderRead()) != -1) {
					char ch = (char)nextChar;
					
					if (HandleLineEnd(ch)) {
						if (scTag != null) {
							this.TagComments.Add(new TagComment(scTag, scCurWord.ToString(), scStartLocation, new Location(Col, Line)));
							scTag = null;
						}
						scCurWord.Length = 0;
						specialTracker.AddString(Environment.NewLine);
						continue;
					}
					
					// End of multiline comment reached ?
					if (ch == '*' && ReaderPeek() == '/') {
						if (scTag != null) {
							this.TagComments.Add(new TagComment(scTag, scCurWord.ToString(), scStartLocation, new Location(Col, Line)));
						}
						ReaderRead();
						specialTracker.FinishComment(new Location(Col, Line));
						return;
					}
					specialTracker.AddChar(ch);
					if (scTag != null || IsIdentifierPart(ch)) {
						scCurWord.Append(ch);
					} else {
						if (specialCommentHash != null && specialCommentHash.ContainsKey(scCurWord.ToString())) {
							scTag = scCurWord.ToString();
							scStartLocation = new Location(Col, Line);
						}
						scCurWord.Length = 0;
					}
				}
				specialTracker.FinishComment(new Location(Col, Line));
			}
			// Reached EOF before end of multiline comment.
			errors.Error(Line, Col, String.Format("Reached EOF before the end of a multiline comment"));
		}
		
		/// <summary>
		/// Skips to the end of the current code block.
		/// For this, the lexer must have read the next token AFTER the token opening the
		/// block (so that Lexer.Token is the block-opening token, not Lexer.LookAhead).
		/// After the call, Lexer.LookAhead will be the block-closing token.
		/// </summary>
		public override void SkipCurrentBlock(int targetToken)
		{
			int braceCount = 0;
			while (curToken != null) {
				if (curToken.kind == Tokens.OpenCurlyBrace) {
					++braceCount;
				} else if (curToken.kind == Tokens.CloseCurlyBrace) {
					if (--braceCount < 0)
						return;
				}
				lastToken = curToken;
				curToken = curToken.next;
			}
			int nextChar;
			while ((nextChar = ReaderRead()) != -1) {
				switch (nextChar) {
					case '{':
						braceCount++;
						break;
					case '}':
						if (--braceCount < 0) {
							curToken = new Token(Tokens.CloseCurlyBrace, Col - 1, Line);
							return;
						}
						break;
					case '/':
						int peek = ReaderPeek();
						if (peek == '/' || peek == '*') {
							ReadComment();
						}
						break;
					case '#':
						ReadPreProcessingDirective();
						break;
					case '"':
						ReadString();
						break;
					case '\'':
						ReadChar();
						break;
					case '\r':
					case '\n':
						HandleLineEnd((char)nextChar);
						break;
					case '@':
						int next = ReaderRead();
						if (next == -1) {
							errors.Error(Line, Col, String.Format("EOF after @"));
						} else if (next == '"') {
							ReadVerbatimString();
						}
						break;
				}
			}
			curToken = new Token(Tokens.EOF, Col, Line);
		}
	}
}