cs-tokenizer.cs

C#编译器源代码。Micorsoft开放源代码

			case Token.LITERAL_STRING:
			case Token.BASE:
			case Token.CHECKED:
			case Token.FALSE:
			case Token.FIXED:
			case Token.NEW:
			case Token.NULL:
			case Token.SIZEOF:
			case Token.THIS:
			case Token.THROW:
			case Token.TRUE:
			case Token.TYPEOF:
			case Token.UNCHECKED:
			case Token.UNSAFE:

				//
				// These can be part of a member access
				//
			case Token.INT:
			case Token.UINT:
			case Token.SHORT:
			case Token.USHORT:
			case Token.LONG:
			case Token.ULONG:
			case Token.DOUBLE:
			case Token.FLOAT:
			case Token.CHAR:
				return true;

			default:
				return false;
			}
		}

		public int token ()
                {
			current_token = xtoken ();
                        return current_token;
                }

		static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
		
		void get_cmd_arg (out string cmd, out string arg)
		{
			int c;
			
			tokens_seen = false;
			arg = "";
			static_cmd_arg.Length = 0;

			// skip over white space
			while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
				;


			while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
				if (is_identifier_part_character ((char) c)){
					static_cmd_arg.Append ((char) c);
					c = getChar ();
				} else {
					putback (c);
					break;
				}
			}

			cmd = static_cmd_arg.ToString ();

			if (c == '\n'){
				return;
			}

			// skip over white space
			while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
				;

			if (c == '\n'){
				return;
			} else if (c == '\r'){
				return;
			} else if (c == -1){
				arg = "";
				return;
			}
			
			static_cmd_arg.Length = 0;
			static_cmd_arg.Append ((char) c);
			
			while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
				static_cmd_arg.Append ((char) c);
			}

			arg = static_cmd_arg.ToString ().Trim ();
		}

		//
		// Handles the #line directive
		//
		bool PreProcessLine (string arg)
		{
			if (arg == "")
				return false;

			if (arg == "default"){
				ref_line = line;
				ref_name = file_name;
				Location.Push (ref_name, line);
				return true;
			} else if (arg == "hidden"){
				//
				// We ignore #line hidden
				//
				return true;
			}

			try {
				int pos;

				if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
					ref_line = System.Int32.Parse (arg.Substring (0, pos));
					pos++;
					
					char [] quotes = { '\"' };
					
					string name = arg.Substring (pos). Trim (quotes);
					ref_name = Location.LookupFile (name);
					file_name.HasLineDirective = true;
					ref_name.HasLineDirective = true;
					Location.Push (ref_name, ref_line);
				} else {
					ref_line = System.Int32.Parse (arg);
				}
			} catch {
				return false;
			}
			
			return true;
		}

		//
		// Handles #define and #undef
		//
		void PreProcessDefinition (bool is_define, string arg)
		{
			if (arg == "" || arg == "true" || arg == "false"){
				Report.Error (1001, Location, "Missing identifer to pre-processor directive");
				return;
			}

			char[] whitespace = { ' ', '\t' };
			if (arg.IndexOfAny (whitespace) != -1){
				Report.Error (1025, Location, "Single-line comment or end-of-line expected");
				return;
			}

			if (!is_identifier_start_character (arg [0]))
				Report.Error (1001, Location, "Identifier expected: " + arg);
			
			foreach (char c in arg.Substring (1)){
				if (!is_identifier_part_character (c)){
					Report.Error (1001, Location, "Identifier expected: " + arg);
					return;
				}
			}

			if (is_define){
				if (defines == null)
					defines = new Hashtable ();
				define (arg);
			} else {
				if (defines == null)
					return;
				if (defines.Contains (arg))
					defines.Remove (arg);
			}
		}

		/// <summary>
		/// Handles #pragma directive
		/// </summary>
		void PreProcessPragma (string arg)
		{
			const string warning = "warning";
			const string w_disable = "warning disable";
			const string w_restore = "warning restore";

			if (arg == w_disable) {
				Report.RegisterWarningRegion (Location).WarningDisable (line);
				return;
			}

			if (arg == w_restore) {
				Report.RegisterWarningRegion (Location).WarningEnable (line);
				return;
			}

			if (arg.StartsWith (w_disable)) {
				int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
				foreach (int code in codes) {
					if (code != 0)
						Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
				}
				return;
			}

			if (arg.StartsWith (w_restore)) {
				int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
				Hashtable w_table = Report.warning_ignore_table;
				foreach (int code in codes) {
					if (w_table != null && w_table.Contains (code))
						Report.Warning (1635, 1, Location, "Cannot restore warning `CS{0:0000}' because it was disabled globally", code);
					Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
				}
				return;
			}

			if (arg.StartsWith (warning)) {
				Report.Warning (1634, 1, Location, "Expected disable or restore");
				return;
			}

			Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
		}

		int[] ParseNumbers (string text)
		{
			string[] string_array = text.Split (',');
			int[] values = new int [string_array.Length];
			int index = 0;
			foreach (string string_code in string_array) {
				try {
					values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
				}
				catch (FormatException) {
					Report.Warning (1692, Location, "Invalid number");
				}
			}
			return values;
		}

		bool eval_val (string s)
		{
			if (s == "true")
				return true;
			if (s == "false")
				return false;
			
			if (defines == null)
				return false;
			if (defines.Contains (s))
				return true;

			return false;
		}

		bool pp_primary (ref string s)
		{
			s = s.Trim ();
			int len = s.Length;

			if (len > 0){
				char c = s [0];
				
				if (c == '('){
					s = s.Substring (1);
					bool val = pp_expr (ref s);
					if (s.Length > 0 && s [0] == ')'){
						s = s.Substring (1);
						return val;
					}
					Error_InvalidDirective ();
					return false;
				}
				
				if (is_identifier_start_character (c)){
					int j = 1;

					while (j < len){
						c = s [j];
						
						if (is_identifier_part_character (c)){
							j++;
							continue;
						}
						bool v = eval_val (s.Substring (0, j));
						s = s.Substring (j);
						return v;
					}
					bool vv = eval_val (s);
					s = "";
					return vv;
				}
			}
			Error_InvalidDirective ();
			return false;
		}
		
		bool pp_unary (ref string s)
		{
			s = s.Trim ();
			int len = s.Length;

			if (len > 0){
				if (s [0] == '!'){
					if (len > 1 && s [1] == '='){
						Error_InvalidDirective ();
						return false;
					}
					s = s.Substring (1);
					return ! pp_primary (ref s);
				} else
					return pp_primary (ref s);
			} else {
				Error_InvalidDirective ();
				return false;
			}
		}
		
		bool pp_eq (ref string s)
		{
			bool va = pp_unary (ref s);

			s = s.Trim ();
			int len = s.Length;
			if (len > 0){
				if (s [0] == '='){
					if (len > 2 && s [1] == '='){
						s = s.Substring (2);
						return va == pp_unary (ref s);
					} else {
						Error_InvalidDirective ();
						return false;
					}
				} else if (s [0] == '!' && len > 1 && s [1] == '='){
					s = s.Substring (2);

					return va != pp_unary (ref s);

				} 
			}

			return va;
				
		}
		
		bool pp_and (ref string s)
		{
			bool va = pp_eq (ref s);

			s = s.Trim ();
			int len = s.Length;
			if (len > 0){
				if (s [0] == '&'){
					if (len > 2 && s [1] == '&'){
						s = s.Substring (2);
						return (va & pp_and (ref s));
					} else {
						Error_InvalidDirective ();
						return false;
					}
				} 
			}
			return va;
		}
		
		//
		// Evaluates an expression for `#if' or `#elif'
		//
		bool pp_expr (ref string s)
		{
			bool va = pp_and (ref s);
			s = s.Trim ();
			int len = s.Length;
			if (len > 0){
				char c = s [0];
				
				if (c == '|'){
					if (len > 2 && s [1] == '|'){
						s = s.Substring (2);
						return va | pp_expr (ref s);
					} else {
						Error_InvalidDirective ();
						return false;
					}
				} 
			}
			
			return va;
		}

		bool eval (string s)
		{
			bool v = pp_expr (ref s);
			s = s.Trim ();
			if (s.Length != 0){
				Error_InvalidDirective ();
				return false;
			}

			return v;
		}
		
		void Error_InvalidDirective ()
		{
			Report.Error (1517, Location, "Invalid preprocessor directive");
		}

		void Error_UnexpectedDirective (string extra)
		{
			Report.Error (
				1028, Location,
				"Unexpected processor directive (" + extra + ")");
		}

		void Error_TokensSeen ()
		{
			Report.Error (1032, Location,
				"Cannot define or undefine preprocessor symbols after first token in file");
		}
		
		//
		// Set to false to stop handling preprocesser directives
		// 
		bool process_directives = true;

		//
		// if true, then the code continues processing the code
		// if false, the code stays in a loop until another directive is
		// reached.
		//
		bool handle_preprocessing_directive (bool caller_is_taking)
		{
			string cmd, arg;
			bool region_directive = false;

			get_cmd_arg (out cmd, out arg);

			// Eat any trailing whitespaces and single-line comments
			if (arg.IndexOf ("//") != -1)
				arg = arg.Substring (0, arg.IndexOf ("//"));
			arg = arg.TrimEnd (' ', '\t');

			//
			// The first group of pre-processing instructions is always processed
			//
			switch (cmd){
			case "pragma":
				if (RootContext.Version == LanguageVersion.ISO_1) {
					Report.FeatureIsNotStandardized (Location, "#pragma");
					return caller_is_taking;
				}

				PreProcessPragma (arg);
				return caller_is_taking;

			case "line":
				if (!PreProcessLine (arg))
					Report.Error (
						1576, Location,
						"The line number specified for #line directive is missing or invalid");
				return caller_is_taking;

			case "region":
				region_directive = true;
				arg = "true";
				goto case "if";

			case "endregion":
				region_directive = true;
				goto case "endif";
				
			case "if":
				if (arg == ""){
					Error_InvalidDirective ();
					return true;
				}
				bool taking = false;
				if (ifstack == null)
					ifstack = new Stack ();

				if (ifstack.Count == 0){
					taking = true;
				} else {
					int state = (int) ifstack.Peek ();
					if ((state & TAKING) != 0)
						taking = true;
				}

				if (eval (arg) && taking){
					int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
					if (region_directive)
						push |= REGION;
					ifstack.Push (push);
					return true;
				} else {
					int push = (taking ? PARENT_TAKING : 0);
					if (region_directive)
						push |= REGION;
					ifstack.Push (push);
					return false;
				}
				
			case "endif":
				if (ifstack == null || ifstack.Count == 0){
					Error_UnexpectedDirective ("no #if for this #endif");
					return true;
				} else {
					int pop = (int) ifstack.Pop ();
					
					if (region_directive && ((pop & REGION) == 0))
						Report.Error (1027, Location, "Expected `#endif' directive");
					else if (!region_directive && ((pop & REGION) != 0))
						Report.Error (1038, Location, "#endregion directive expected");
					
					if (!region_directive && arg.Length != 0) {
						Report.Error (1025, Location, "Single-line comment or end-of-line expected");
					}
					
					if (ifstack.Count == 0)
						return true;
					else {
						int state = (int) ifstack.Peek ();

						if ((state & TAKING) != 0)
							return true;
						else
							return false;
					}
				}

			case "elif":
				if (ifstack == null || ifstack.Count == 0){
					Error_UnexpectedDirective ("no #if for this #elif");
					return true;
				} else {
					int state = (int) ifstack.Peek ();

					if ((state & REGION) != 0) {
						Report.Error (1038, Location, "#endregion directive expected");
						return true;
					}

					if ((state & ELSE_SEEN) != 0){
						Error_UnexpectedDirective ("#elif not valid after #else");
						return true;
					}

					if ((state & (TAKEN_BEFORE | TAKING)) != 0)
						return false;

					if (eval (arg) && ((state & PARENT_TAKING) != 0)){
						state = (int) ifstack.Pop ();
						ifstack.Push (state | TAKING | TAKEN_BEFORE);
						return true;
					} else 
						return false;
				}

			case "else":
				if (ifstack == null || ifstack.Count == 0){
					Error_UnexpectedDirective ("no #if for this #else");
					return true;
				} else {
					int state = (int) ifstack.Peek ();

					if ((state & REGION) != 0) {
						Report.Error (1038, Location, "#endregion directive expected");
						return true;
					}

					if ((state & ELSE_SEEN) != 0){
						Error_UnexpectedDirective ("#else within #else");
						return true;
					}

					ifstack.Pop ();

					bool ret;
					if ((state & TAKEN_BEFORE) == 0){
						ret = ((state & PARENT_TAKING) != 0);
					} else
						ret = false;
					
					if (ret)
						state |= TAKING;