cs-tokenizer.cs

C#编译器源代码。Micorsoft开放源代码

					else
						state &= ~TAKING;
					
					ifstack.Push (state | ELSE_SEEN);
					
					return ret;
				}
			}

			//
			// These are only processed if we are in a `taking' block
			//
			if (!caller_is_taking)
				return false;
					
			switch (cmd){
			case "define":
				if (any_token_seen){
					Error_TokensSeen ();
					return true;
				}
				PreProcessDefinition (true, arg);
				return true;

			case "undef":
				if (any_token_seen){
					Error_TokensSeen ();
					return true;
				}
				PreProcessDefinition (false, arg);
				return true;

			case "error":
				Report.Error (1029, Location, "#error: '" + arg + "'");
				return true;

			case "warning":
				Report.Warning (1030, Location, "#warning: `{0}'", arg);
				return true;
			}

			Report.Error (1024, Location, "Wrong preprocessor directive");
			return true;

		}

		private int consume_string (bool quoted)
		{
			int c;
			string_builder.Length = 0;
								
			while ((c = getChar ()) != -1){
				if (c == '"'){
					if (quoted && peekChar () == '"'){
						string_builder.Append ((char) c);
						getChar ();
						continue;
					} else {
						val = string_builder.ToString ();
						return Token.LITERAL_STRING;
					}
				}

				if (c == '\n'){
					if (!quoted)
						Report.Error (1010, Location, "Newline in constant");
				}

				if (!quoted){
					c = escape (c);
					if (c == -1)
						return Token.ERROR;
				}
				string_builder.Append ((char) c);
			}

			Report.Error (1039, Location, "Unterminated string literal");
			return Token.EOF;
		}

		private int consume_identifier (int s)
		{
			int res = consume_identifier (s, false);

			if (doc_state == XmlCommentState.Allowed)
				doc_state = XmlCommentState.NotAllowed;
			switch (res) {
			case Token.USING:
			case Token.NAMESPACE:
				check_incorrect_doc_comment ();
				break;
			}

			if (res == Token.PARTIAL) {
				// Save current position and parse next token.
				int old = reader.Position;
				int old_putback = putback_char;
				int old_ref_line = ref_line;
				int old_col = col;

				putback_char = -1;

				int next_token = token ();
				bool ok = (next_token == Token.CLASS) ||
					(next_token == Token.STRUCT) ||
					(next_token == Token.INTERFACE) ||
					(next_token == Token.ENUM); // "partial" is a keyword in 'partial enum', even though it's not valid

				reader.Position = old;
				ref_line = old_ref_line;
				col = old_col;
				putback_char = old_putback;

				if (ok)
					return res;
				else {
					val = new LocatedToken (Location, "partial");
					return Token.IDENTIFIER;
				}
			}

			return res;
		}

		private int consume_identifier (int s, bool quoted) 
		{
			int pos = 1;
			int c = -1;
			
			id_builder [0] = (char) s;

			current_location = new Location (ref_line, Col);

			while ((c = getChar ()) != -1) {
				if (is_identifier_part_character ((char) c)){
					if (pos == max_id_size){
						Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
						return Token.ERROR;
					}
					
					id_builder [pos++] = (char) c;
//					putback_char = -1;
				} else {
//					putback_char = c;
					putback (c);
					break;
				}
			}

			//
			// Optimization: avoids doing the keyword lookup
			// on uppercase letters and _
			//
			if (!quoted && (s >= 'a' || s == '_')){
				int keyword = GetKeyword (id_builder, pos);
				if (keyword != -1) {
					val = Location;
					return keyword;
				}
			}

			//
			// Keep identifiers in an array of hashtables to avoid needless
			// allocations
			//

			if (identifiers [pos] != null) {
				val = identifiers [pos][id_builder];
				if (val != null) {
					val = new LocatedToken (Location, (string) val);
					if (quoted)
						escapedIdentifiers.Add (val);
					return Token.IDENTIFIER;
				}
			}
			else
				identifiers [pos] = new CharArrayHashtable (pos);

			val = new String (id_builder, 0, pos);
			if (RootContext.Version == LanguageVersion.ISO_1) {
				for (int i = 1; i < id_builder.Length; i += 3) {
					if (id_builder [i] == '_' && (id_builder [i - 1] == '_' || id_builder [i + 1] == '_')) {
						Report.Error (1638, Location, 
							"`{0}': Any identifier with double underscores cannot be used when ISO language version mode is specified", val);
						break;
					}
				}
			}

			char [] chars = new char [pos];
			Array.Copy (id_builder, chars, pos);

			identifiers [pos] [chars] = val;

			val = new LocatedToken (Location, (string) val);
			if (quoted)
				escapedIdentifiers.Add (val);
			return Token.IDENTIFIER;
		}
		
		public int xtoken ()
		{
			int t;
			bool doread = false;
			int c;

			// Whether we have seen comments on the current line
			bool comments_seen = false;
			
			val = null;
			// optimization: eliminate col and implement #directive semantic correctly.
			for (;(c = getChar ()) != -1;) {
				if (c == ' ')
					continue;
				
				if (c == '\t') {
					continue;
				}
				
				if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)
					continue;

				if (c == '\r') {
					if (peekChar () == '\n')
						getChar ();

					any_token_seen |= tokens_seen;
					tokens_seen = false;
					comments_seen = false;
					continue;
				}

				// Handle double-slash comments.
				if (c == '/'){
					int d = peekChar ();
				
					if (d == '/'){
						getChar ();
						if (RootContext.Documentation != null && peekChar () == '/') {
							getChar ();
							// Don't allow ////.
							if ((d = peekChar ()) != '/') {
								update_comment_location ();
								if (doc_state == XmlCommentState.Allowed)
									handle_one_line_xml_comment ();
								else if (doc_state == XmlCommentState.NotAllowed)
									warn_incorrect_doc_comment ();
							}
						}
						while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
						if (d == '\n'){
						}
						any_token_seen |= tokens_seen;
						tokens_seen = false;
						comments_seen = false;
						continue;
					} else if (d == '*'){
						getChar ();
						bool docAppend = false;
						if (RootContext.Documentation != null && peekChar () == '*') {
							getChar ();
							update_comment_location ();
							// But when it is /**/, just do nothing.
							if (peekChar () == '/') {
								getChar ();
								continue;
							}
							if (doc_state == XmlCommentState.Allowed)
								docAppend = true;
							else if (doc_state == XmlCommentState.NotAllowed)
								warn_incorrect_doc_comment ();
						}

						int current_comment_start = 0;
						if (docAppend) {
							current_comment_start = xml_comment_buffer.Length;
							xml_comment_buffer.Append (Environment.NewLine);
						}

						Location start_location = Location;

						while ((d = getChar ()) != -1){
							if (d == '*' && peekChar () == '/'){
								getChar ();
								comments_seen = true;
								break;
							}
							if (docAppend)
								xml_comment_buffer.Append ((char) d);
							
							if (d == '\n'){
								any_token_seen |= tokens_seen;
								tokens_seen = false;
								// 
								// Reset 'comments_seen' just to be consistent.
								// It doesn't matter either way, here.
								//
								comments_seen = false;
							}
						}
						if (!comments_seen)
							Report.Error (1035, start_location, "End-of-file found, '*/' expected");

						if (docAppend)
							update_formatted_doc_comment (current_comment_start);
						continue;
					}
					goto is_punct_label;
				}

				
				if (is_identifier_start_character ((char)c)){
					tokens_seen = true;
					return consume_identifier (c);
				}

			is_punct_label:
				current_location = new Location (ref_line, Col);
				if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
					tokens_seen = true;
					if (doread){
						getChar ();
					}
					return t;
				}

				// white space
				if (c == '\n'){
					any_token_seen |= tokens_seen;
					tokens_seen = false;
					comments_seen = false;
					continue;
				}

				if (c >= '0' && c <= '9'){
					tokens_seen = true;
					return is_number (c);
				}

				if (c == '.'){
					tokens_seen = true;
					int peek = peekChar ();
					if (peek >= '0' && peek <= '9')
						return is_number (c);
					return Token.DOT;
				}
				
				/* For now, ignore pre-processor commands */
				// FIXME: In C# the '#' is not limited to appear
				// on the first column.
				if (c == '#') {
					// return NONE if we're not processing directives (during token peeks)
					if (!process_directives)
						return Token.NONE;

					bool cont = true;
					if (tokens_seen || comments_seen) {
                                               error_details = "Preprocessor directives must appear as the first" +
					       " non-whitespace character on a line.";

                                               Report.Error (1040, Location, error_details);

                                               return Token.ERROR;
                                       }
					
				start_again:
					
					cont = handle_preprocessing_directive (cont);

					if (cont){
						continue;
					}

					bool skipping = false;
					for (;(c = getChar ()) != -1;){
						if (c == '\n'){
							skipping = false;
						} else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
							continue;
						else if (c != '#')
							skipping = true;
						if (c == '#' && !skipping)
							goto start_again;
					}
					any_token_seen |= tokens_seen;
					tokens_seen = false;
					if (c == -1)
						Report.Error (1027, Location, "Expected `#endif' directive");
					continue;
				}
				
				if (c == '"') 
					return consume_string (false);

				if (c == '\''){
					c = getChar ();
					tokens_seen = true;
					if (c == '\''){
						error_details = "Empty character literal";
						Report.Error (1011, Location, error_details);
						return Token.ERROR;
					}
					if (c == '\r' || c == '\n') {
						Report.Error (1010, Location, "Newline in constant");
						return Token.ERROR;
					}
					c = escape (c);
					if (c == -1)
						return Token.ERROR;
					val = new System.Char ();
					val = (char) c;
					c = getChar ();

					if (c != '\''){
						error_details = "Too many characters in character literal";
						Report.Error (1012, Location, error_details);

						// Try to recover, read until newline or next "'"
						while ((c = getChar ()) != -1){
							if (c == '\n'){
								break;
							}
							else if (c == '\'')
								break;
						}
						return Token.ERROR;
					}
					return Token.LITERAL_CHARACTER;
				}
				
				if (c == '@') {
					c = getChar ();
					if (c == '"') {
						tokens_seen = true;
						return consume_string (true);
					} else if (is_identifier_start_character ((char) c)){
						return consume_identifier (c, true);
					} else {
						Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
					}
				}

				error_details = ((char)c).ToString ();
				
				return Token.ERROR;
			}

			return Token.EOF;
		}

		//
		// Handles one line xml comment
		//
		private void handle_one_line_xml_comment ()
		{
			int c;
			while ((c = peekChar ()) == ' ')
				getChar (); // skip heading whitespaces.
			while ((c = peekChar ()) != -1 && c != '\n' && c != '\r') {
				xml_comment_buffer.Append ((char) getChar ());
			}
			if (c == '\r' || c == '\n')
				xml_comment_buffer.Append (Environment.NewLine);
		}

		//
		// Remove heading "*" in Javadoc-like xml documentation.
		//
		private void update_formatted_doc_comment (int current_comment_start)
		{
			int length = xml_comment_buffer.Length - current_comment_start;
			string [] lines = xml_comment_buffer.ToString (
				current_comment_start,
				length).Replace ("\r", "").Split ('\n');
			
			// The first line starts with /**, thus it is not target
			// for the format check.
			for (int i = 1; i < lines.Length; i++) {
				string s = lines [i];
				int idx = s.IndexOf ('*');
				string head = null;
				if (idx < 0) {
					if (i < lines.Length - 1)
						return;
					head = s;
				} else
					head = s.Substring (0, idx);
				foreach (char c in head)
					if (c != ' ')
						return;
				lines [i] = s.Substring (idx + 1);
			}
			xml_comment_buffer.Remove (current_comment_start, length);
			xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
		}

		//
		// Updates current comment location.
		//
		private void update_comment_location ()
		{
			if (current_comment_location.IsNull) {
				// "-2" is for heading "//" or "/*"
				current_comment_location =
					new Location (ref_line, col - 2);
			}
		}

		//
		// Checks if there was incorrect doc comments and raise
		// warnings.
		//
		public void check_incorrect_doc_comment ()
		{
			if (xml_comment_buffer.Length > 0)
				warn_incorrect_doc_comment ();
		}

		//
		// Raises a warning when tokenizer found incorrect doccomment
		// markup.
		//
		private void warn_incorrect_doc_comment ()
		{
			if (doc_state != XmlCommentState.Error) {
				doc_state = XmlCommentState.Error;
				// in csc, it is 'XML comment is not placed on 
				// a valid language element'. But that does not
				// make sense.
				Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
			}
		}

		//
		// Consumes the saved xml comment lines (if any)
		// as for current target member or type.
		//
		public string consume_doc_comment ()
		{
			if (xml_comment_buffer.Length > 0) {
				string ret = xml_comment_buffer.ToString ();
				reset_doc_comment ();
				return ret;
			}
			return null;
		}

		void reset_doc_comment ()
		{
			xml_comment_buffer.Length = 0;
			current_comment_location = Location.Null;
		}

		public void cleanup ()
		{
			if (ifstack != null && ifstack.Count >= 1) {
				int state = (int) ifstack.Pop ();
				if ((state & REGION) != 0)
					Report.Error (1038, Location, "#endregion directive expected");
				else 
					Report.Error (1027, "Expected `#endif' directive");
			}
				
		}
	}

	//
	// Indicates whether it accepts XML documentation or not.
	//
	public enum XmlCommentState {
		// comment is allowed in this state.
		Allowed,
		// comment is not allowed in this state.
		NotAllowed,
		// once comments appeared when it is NotAllowed, then the
		// state is changed to it, until the state is changed to
		// .Allowed.
		Error
	}
}