lexer.cs

c#源代码

			StringBuilder s = new StringBuilder();
			while (!reader.Eos()) {
				ch = reader.GetNext();
				if (ch == '"') {
					if (reader.Peek() != '"') {
						break;
					}
					reader.GetNext();
				}
				++col;
				if (ch == '\n') {
					++line;
					col = 1;
				}
				s.Append(ch);
			}
			if (ch != '"') {
				errors.Error(y, x, String.Format("End of file reached inside verbatim string literal"));
			}
			
			StringBuilder res = new StringBuilder("@\"");
			res.Append(s.ToString());
			res.Append('"');
			
			return new Token(Tokens.Literal, x, y, res.ToString(), s.ToString());
		}
		
		string hexdigits = "0123456789ABCDEF";
		
		string ReadEscapeSequence(out char ch)
		{
			StringBuilder s = new StringBuilder();
			if (reader.Eos()) {
				errors.Error(line, col, String.Format("End of file reached inside escape sequence"));
			}
			char c = reader.GetNext();
			s.Append(c);
			++col;
			switch (c)  {
				case '\'':
					ch = '\'';
					break;
				case '\"':
					ch = '\"';
					break;
				case '\\':
					ch = '\\';
					break;
				case '0':
					ch = '\0';
					break;
				case 'a':
					ch = '\a';
					break;
				case 'b':
					ch = '\b';
					break;
				case 'f':
					ch = '\f';
					break;
				case 'n':
					ch = '\n';
					break;
				case 'r':
					ch = '\r';
					break;
				case 't':
					ch = '\t';
					break;
				case 'v':
					ch = '\v';
					break;
				case 'u':
				case 'x':
					c = reader.GetNext();
					int number = hexdigits.IndexOf(Char.ToUpper(c));
					if (number < 0) {
						errors.Error(line, col, String.Format("Invalid char in literal : {0}", c));
					}
					s.Append(c);
					for (int i = 0; i < 3; ++i) {
						c = reader.GetNext();
						int idx = hexdigits.IndexOf(Char.ToUpper(c));
						if (idx >= 0) {
							s.Append(c);
							number = idx * (16 * (i + 1)) + number;
						} else {
							reader.UnGet();
							break;
						}
					}
					ch = (char)number;
					break;
				default:
					errors.Error(line, col, String.Format("Unexpected escape sequence : {0}", c));
					ch = '\0';
					break;
			}
			return s.ToString();
		}
		
		Token ReadChar()
		{
			int x = col;
			int y = line;
			
			if (reader.Eos()) {
				errors.Error(y, x, String.Format("End of file reached inside character literal"));
			}
			StringBuilder originalValue = new StringBuilder();
			char  ch = reader.GetNext();
			originalValue.Append("'");
			originalValue.Append(ch);
			++col;
			
			if (ch == '\\') {
				originalValue.Append(ReadEscapeSequence(out ch));
			}
			
			if (reader.Eos()) {
				errors.Error(y, x, String.Format("End of file reached inside character literal"));
			}
			if (reader.GetNext() != '\'') {
				errors.Error(y, x, String.Format("Char not terminated"));
			}
			originalValue.Append("'");
			return new Token(Tokens.Literal, x, y, originalValue.ToString(), ch);
		}
		
		Token ReadOperator(char ch)
		{
			int x = col;
			int y = line;
			++col;
			switch (ch) {
				case '+':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '+':
								++col;
								return new Token(Tokens.Increment, x, y, "++");
							case '=':
								++col;
								return new Token(Tokens.PlusAssign, x, y, "+=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.Plus, x, y, "+");
				case '-':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '-':
								++col;
								return new Token(Tokens.Decrement, x, y, "--");
							case '=':
								++col;
								return new Token(Tokens.MinusAssign, x, y, "-=");
							case '>':
								++col;
								return new Token(Tokens.Pointer, x, y, "->");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.Minus, x, y, "-");
				case '*':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.TimesAssign, x, y, "*=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.Times, x, y, "*");
				case '/':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.DivAssign, x, y, "/=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.Div, x, y, "/");
				case '%':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.ModAssign, x, y, "%=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.Mod, x, y, "%");
				case '&':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '&':
								++col;
								return new Token(Tokens.LogicalAnd, x, y, "&&");
							case '=':
								++col;
								return new Token(Tokens.BitwiseAndAssign, x, y, "&=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.BitwiseAnd, x, y, "&");
				case '|':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '|':
								++col;
								return new Token(Tokens.LogicalOr, x, y, "||");
							case '=':
								++col;
								return new Token(Tokens.BitwiseOrAssign, x, y, "|=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.BitwiseOr, x, y, "|");
				case '^':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.XorAssign, x, y, "^=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.Xor, x, y, "^");
				case '!':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.NotEqual, x, y, "!=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.Not, x, y, "!");
				case '~':
					return new Token(Tokens.BitwiseComplement, x, y, "~");
				case '=':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '=':
								++col;
								return new Token(Tokens.Equal, x, y, "==");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.Assign, x, y, "=");
				case '<':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '<':
								if (!reader.Eos()) {
									switch (reader.GetNext()) {
										case '=':
											col += 2;
											return new Token(Tokens.ShiftLeftAssign, x, y, "<<=");
										default:
											++col;
											reader.UnGet();
											break;
									}
								}
								return new Token(Tokens.ShiftLeft, x, y, "<<");
							case '=':
								++col;
								return new Token(Tokens.LessEqual, x, y, "<=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.LessThan, x, y, "<");
				case '>':
					if (!reader.Eos()) {
						switch (reader.GetNext()) {
							case '>':
								if (!reader.Eos()) {
									switch (reader.GetNext()) {
										case '=':
											col += 2;
											return new Token(Tokens.ShiftRightAssign, x, y, ">>=");
										default:
											++col;
											reader.UnGet();
											break;
									}
								}
								return new Token(Tokens.ShiftRight, x, y, ">>");
							case '=':
								++col;
								return new Token(Tokens.GreaterEqual, x, y, ">=");
							default:
								reader.UnGet();
								break;
						}
					}
					return new Token(Tokens.GreaterThan, x, y, ">");
				case '?':
					return new Token(Tokens.Question, x, y, "?");
				case ';':
					return new Token(Tokens.Semicolon, x, y, ";");
				case ':':
					return new Token(Tokens.Colon, x, y, ":");
				case ',':
					return new Token(Tokens.Comma, x, y, ",");
				case '.':
					if (Char.IsDigit(reader.Peek())) {
						 reader.UnGet();
						 col -= 2;
						 return ReadDigit('0', col + 1);
					}
					return new Token(Tokens.Dot, x, y, ".");
				case ')':
					return new Token(Tokens.CloseParenthesis, x, y, ")");
				case '(':
					return new Token(Tokens.OpenParenthesis, x, y, "(");
				case ']':
					return new Token(Tokens.CloseSquareBracket, x, y, "]");
				case '[':
					return new Token(Tokens.OpenSquareBracket, x, y, "[");
				case '}':
					return new Token(Tokens.CloseCurlyBrace, x, y, "}");
				case '{':
					return new Token(Tokens.OpenCurlyBrace, x, y, "{");
				default:
					--col;
					return null;
			}
		}
		
		void ReadComment()
		{
			char ch = reader.GetNext();
			++col;
			switch (ch) {
				case '*':
					ReadMultiLineComment();
					break;
				case '/':
					if (reader.GetNext() == '/') {
						ReadSingleLineComment(CommentType.Documentation);
					} else {
						reader.UnGet();
						ReadSingleLineComment(CommentType.SingleLine);
					}
					break;
				default:
					errors.Error(line, col, String.Format("Error while reading comment"));
					break;
			}
		}
		
		string ReadToEOL()
		{
			StringBuilder sb = new StringBuilder();
			if (!reader.Eos()) {
				char ch = reader.GetNext();
				while (!reader.Eos()) {
					if (ch == '\r') {
						if (reader.Peek() == '\n') {
							ch = reader.GetNext();
						}
					}
					if (ch == '\n') {
						++line;
						col = 1;
						return sb.ToString();;
					} else {
						sb.Append(ch);
					}
					ch = reader.GetNext();
					++col;
				}
			}
			return sb.ToString();
		}
		
		string ReadCommentToEOL()
		{
			StringBuilder sb = new StringBuilder();
			StringBuilder curWord = new StringBuilder();
			if (!reader.Eos()) {
				char ch = reader.GetNext();
				while (!reader.Eos()) {
					if (Char.IsLetter(ch)) {
						curWord.Append(ch);
					} else {
						string tag = curWord.ToString();
						curWord = new StringBuilder();
						if (specialCommentHash != null && specialCommentHash[tag] != null) {
							Point p = new Point(col ,line);
							string comment = ReadToEOL();
							tagComments.Add(new TagComment(tag, comment, p));
							sb.Append(tag);
							sb.Append(comment);
							return sb.ToString();
						}
					}
					if (ch == '\r') {
						if (reader.Peek() == '\n') {
							ch = reader.GetNext();
						}
					}
					if (ch == '\n') {
						++line;
						col = 1;
						return sb.ToString();;
					} else {
						sb.Append(ch);
					}
					ch = reader.GetNext();
					++col;
				}
			}
			return sb.ToString();
		}
		
		void ReadSingleLineComment(CommentType commentType)
		{
			specialTracker.StartComment(commentType, new Point(line, col));
			specialTracker.AddString(ReadCommentToEOL());
			specialTracker.FinishComment();
		}
		
		void ReadMultiLineComment()
		{
			specialTracker.StartComment(CommentType.Block, new Point(line, col));
			int x = col;
			int y = line;
			while (!reader.Eos()) {
				char ch;
				switch (ch = reader.GetNext()) {
					case '\n':
						specialTracker.AddChar('\n');
						++line;
						col = 1;
						break;
					case '*':
						++col;
						switch (reader.Peek()) {
							case '/':
								reader.GetNext();
								++col;
								specialTracker.FinishComment();
								return;
							default:
								specialTracker.AddChar('*');
								continue;
						}
					default:
						specialTracker.AddChar(ch);
						++col;
						break;
				}
			}
			specialTracker.FinishComment();
		}
	}
}