standardlexerrules.java

java 词法分析器,用于一般的C,C++,VB,PS/SQL 语句的翻译

		/*190*/	{ "letter",  "0x1154", Token.UPTO, "0x1155" },
		/*191*/	{ "letter",  "0x1159" },
		/*192*/	{ "letter",  "0x115F", Token.UPTO, "0x1161" },
		/*193*/	{ "letter",  "0x1163" },
		/*194*/	{ "letter",  "0x1165" },
		/*195*/	{ "letter",  "0x1167" },
		/*196*/	{ "letter",  "0x1169" },
		/*197*/	{ "letter",  "0x116D", Token.UPTO, "0x116E" },
		/*198*/	{ "letter",  "0x1172", Token.UPTO, "0x1173" },
		/*199*/	{ "letter",  "0x1175" },
		/*200*/	{ "letter",  "0x119E" },
		/*201*/	{ "letter",  "0x11A8" },
		/*202*/	{ "letter",  "0x11AB" },
		/*203*/	{ "letter",  "0x11AE", Token.UPTO, "0x11AF" },
		/*204*/	{ "letter",  "0x11B7", Token.UPTO, "0x11B8" },
		/*205*/	{ "letter",  "0x11BA" },
		/*206*/	{ "letter",  "0x11BC", Token.UPTO, "0x11C2" },
		/*207*/	{ "letter",  "0x11EB" },
		/*208*/	{ "letter",  "0x11F0" },
		/*209*/	{ "letter",  "0x11F9" },
		/*210*/	{ "letter",  "0x1E00", Token.UPTO, "0x1E9B" },
		/*211*/	{ "letter",  "0x1EA0", Token.UPTO, "0x1EF9" },
		/*212*/	{ "letter",  "0x1F00", Token.UPTO, "0x1F15" },
		/*213*/	{ "letter",  "0x1F18", Token.UPTO, "0x1F1D" },
		/*214*/	{ "letter",  "0x1F20", Token.UPTO, "0x1F45" },
		/*215*/	{ "letter",  "0x1F48", Token.UPTO, "0x1F4D" },
		/*216*/	{ "letter",  "0x1F50", Token.UPTO, "0x1F57" },
		/*217*/	{ "letter",  "0x1F59" },
		/*218*/	{ "letter",  "0x1F5B" },
		/*219*/	{ "letter",  "0x1F5D" },
		/*220*/	{ "letter",  "0x1F5F", Token.UPTO, "0x1F7D" },
		/*221*/	{ "letter",  "0x1F80", Token.UPTO, "0x1FB4" },
		/*222*/	{ "letter",  "0x1FB6", Token.UPTO, "0x1FBC" },
		/*223*/	{ "letter",  "0x1FBE" },
		/*224*/	{ "letter",  "0x1FC2", Token.UPTO, "0x1FC4" },
		/*225*/	{ "letter",  "0x1FC6", Token.UPTO, "0x1FCC" },
		/*226*/	{ "letter",  "0x1FD0", Token.UPTO, "0x1FD3" },
		/*227*/	{ "letter",  "0x1FD6", Token.UPTO, "0x1FDB" },
		/*228*/	{ "letter",  "0x1FE0", Token.UPTO, "0x1FEC" },
		/*229*/	{ "letter",  "0x1FF2", Token.UPTO, "0x1FF4" },
		/*230*/	{ "letter",  "0x1FF6", Token.UPTO, "0x1FFC" },
		/*231*/	{ "letter",  "0x2126" },
		/*232*/	{ "letter",  "0x212A", Token.UPTO, "0x212B" },
		/*233*/	{ "letter",  "0x212E" },
		/*234*/	{ "letter",  "0x2180", Token.UPTO, "0x2182" },
		/*235*/	{ "letter",  "0x3041", Token.UPTO, "0x3094" },
		/*236*/	{ "letter",  "0x30A1", Token.UPTO, "0x30FA" },
		/*237*/	{ "letter",  "0x3105", Token.UPTO, "0x312C" },
		/*238*/	{ "letter",  "0xAC00", Token.UPTO, "0xD7A3" },
		// Ideographic
		/*239*/	{ "letter",  "0x4E00", Token.UPTO, "0x9FA5" },
		/*240*/	{ "letter",  "0x3007" },
		/*241*/	{ "letter",  "0x3021", Token.UPTO, "0x3029" },

		// helper rules

		/*242*/	{ "spaces", "spaces", "space" },
		/*243*/	{ "spaces", "space" },
			
		/*244*/	{ "digits",  "digits", "digit" },
		/*245*/	{ "digits",  "digit" },

		/*246*/	{ "hexdigits", "hexdigits", "hexdigit" },
		/*247*/	{ "hexdigits", "hexdigit" },

		/*248*/	{ "bnf_chardef", "\"'\"", "char", "\"'\"" },
		/*249*/	{ "bnf_chardef", "\"0x\"", "hexdigits" },
		/*250*/	{ "bnf_chardef", "\"0X\"", "hexdigits" },
		/*251*/	{ "bnf_chardef", "\"'\\''\"" },	// single quote
		/*252*/	{ "bnf_chardef", "\"'\\n'\"" },	// newline
		/*253*/	{ "bnf_chardef", "\"'\\r'\"" },	// carriage return
		/*254*/	{ "bnf_chardef", "\"'\\t'\"" },	// tabulator
		/*255*/	{ "bnf_chardef", "\"'\\f'\"" },	// formfeed
		/*256*/	{ "bnf_chardef", "\"'\\b'\"" },	// backspace
		/*257*/	{ "bnf_chardef", "\"'\\\\'\"" },	// backslash
		// bell \a was removed in favor of backslash and too much work re-numbering ...
		/*258*/	{ "bnf_chardef", "digits" },

		/*259*/	{ "identifier", "letter_or_uscore", "letter_or_digit_list_opt" },
		/*260*/	{ "letter_or_uscore", "letter" },
		/*261*/	{ "letter_or_uscore", "'_'" },
		/*262*/	{ "letter_or_digit", "letter_or_uscore" },
		/*263*/	{ "letter_or_digit", "digit" },
		/*264*/	{ "letter_or_digit_list", "letter_or_digit_list", "letter_or_digit" },
		/*265*/	{ "letter_or_digit_list", "letter_or_digit" },
		/*266*/	{ "letter_or_digit_list_opt", "letter_or_digit_list" },
		/*267*/	{ "letter_or_digit_list_opt" /*nothing*/ },

		/*268*/	{ "stringdef", "'\"'", "stringpart_list_opt", "'\"'" },
		/*269*/	{ "char_minus_doublequote_list", "char_minus_doublequote_list", "char_minus_doublequote" },
		/*270*/	{ "char_minus_doublequote_list", "char_minus_doublequote" },
		/*271*/	{ "char_minus_doublequote", "char", Token.BUTNOT, "'\"'", Token.BUTNOT, "'\\'" },	// does not contain " or \
		/*272*/	{ "char_minus_doublequote_list_opt", "char_minus_doublequote_list" },
		/*273*/	{ "char_minus_doublequote_list_opt" /*nothing*/ },
		/*274*/	{ "backslash_char_list", "backslash_char_list", "backslash_char" },
		/*275*/	{ "backslash_char_list", "backslash_char" },
		/*276*/	{ "backslash_char", "'\\'", "char" },	// escaped character like "\""
		/*277*/	{ "backslash_char_list_opt", "backslash_char_list" },
		/*278*/	{ "backslash_char_list_opt" /*nothing*/ },
		/*279*/	{ "stringpart", "char_minus_doublequote_list_opt", "backslash_char_list_opt" },  
		/*280*/	{ "stringpart_list", "stringpart_list", "stringpart" },
		/*281*/	{ "stringpart_list", "stringpart" },
		/*282*/	{ "stringpart_list_opt", "stringpart_list" },
		/*283*/	{ "stringpart_list_opt" /*nothing*/ },

		/*284*/	{ "cstylecomment",  "\"/*\"", "char_minus_star_slash_list_opt", "\"*/\"" },
		/*285*/	{ "char_minus_star_slash", "char", Token.BUTNOT, "\"*/\"" },
		/*286*/	{ "char_minus_star_slash_list", "char_minus_star_slash_list", "char_minus_star_slash" },
		/*287*/	{ "char_minus_star_slash_list", "char_minus_star_slash" },
		/*288*/	{ "char_minus_star_slash_list_opt", "char_minus_star_slash_list" },
		/*289*/	{ "char_minus_star_slash_list_opt" /*nothing*/ },

		/*290*/	{ "cstylecomment",  "\"//\"", "char_minus_newline_list_opt" },
		/*291*/	{ "char_minus_newline", "char", Token.BUTNOT, "newline" },
		/*292*/	{ "char_minus_newline_list", "char_minus_newline_list", "char_minus_newline" },
		/*293*/	{ "char_minus_newline_list", "char_minus_newline" },
		/*294*/	{ "char_minus_newline_list_opt", "char_minus_newline_list" },
		/*295*/	{ "char_minus_newline_list_opt" /*nothing*/ },
		/*296*/	{ "shellstylecomment",  "'#'", "char_minus_newline_list_opt" },

		/*297*/	{ "ruleref", "'"+Token.COMMAND_QUOTE+"'", "identifier", "'"+Token.COMMAND_QUOTE+"'" },
		/*298*/	{ Token.TOKEN, "ruleref" },

		/*299*/	{ "comment", "cstylecomment" },
		/*300*/	{ "comment", "shellstylecomment" },
	};

	/** XML Char definitions of W3C. */
	public static final String [][] xmlCharRules = {
		/*0*/	{	"xmlchar", "0x9" },
		/*1*/	{	"xmlchar", "0xA" },
		/*2*/	{	"xmlchar", "0xD" },
		/*3*/	{	"xmlchar", "0x20", Token.UPTO, "0xD7FF" },
		/*4*/	{	"xmlchar", "0xE000", Token.UPTO, "0xFFFD" },
		/*5*/	{	"xmlchar", "0x10000", Token.UPTO, "0x10FFFF" },
	};
	
	/** XML CombiningChar and XML Extender definitions of W3C. */
	public static final String [][] xmlCombinigAndExtenderRules = {
		/*0*/	{	"combiningchar", "0x0300", Token.UPTO, "0x0345" },
		/*1*/	{	"combiningchar", "0x0360", Token.UPTO, "0x0361" },
		/*2*/	{	"combiningchar", "0x0483", Token.UPTO, "0x0486" },
		/*3*/	{	"combiningchar", "0x0591", Token.UPTO, "0x05A1" },
		/*4*/	{	"combiningchar", "0x05A3", Token.UPTO, "0x05B9" },
		/*5*/	{	"combiningchar", "0x05BB", Token.UPTO, "0x05BD" },
		/*6*/	{	"combiningchar", "0x05BF" },
		/*7*/	{	"combiningchar", "0x05C1", Token.UPTO, "0x05C2" },
		/*8*/	{	"combiningchar", "0x05C4" },
		/*9*/	{	"combiningchar", "0x064B", Token.UPTO, "0x0652" },
		/*10*/	{	"combiningchar", "0x0670" },
		/*11*/	{	"combiningchar", "0x06D6", Token.UPTO, "0x06DC" },
		/*12*/	{	"combiningchar", "0x06DD", Token.UPTO, "0x06DF" },
		/*13*/	{	"combiningchar", "0x06E0", Token.UPTO, "0x06E4" },
		/*14*/	{	"combiningchar", "0x06E7", Token.UPTO, "0x06E8" },
		/*15*/	{	"combiningchar", "0x06EA", Token.UPTO, "0x06ED" },
		/*16*/	{	"combiningchar", "0x0901", Token.UPTO, "0x0903" },
		/*17*/	{	"combiningchar", "0x093C" },
		/*18*/	{	"combiningchar", "0x093E", Token.UPTO, "0x094C" },
		/*19*/	{	"combiningchar", "0x094D" },
		/*20*/	{	"combiningchar", "0x0951", Token.UPTO, "0x0954" },
		/*21*/	{	"combiningchar", "0x0962", Token.UPTO, "0x0963" },
		/*22*/	{	"combiningchar", "0x0981", Token.UPTO, "0x0983" },
		/*23*/	{	"combiningchar", "0x09BC" },
		/*24*/	{	"combiningchar", "0x09BE" },
		/*25*/	{	"combiningchar", "0x09BF" },
		/*26*/	{	"combiningchar", "0x09C0", Token.UPTO, "0x09C4" },
		/*27*/	{	"combiningchar", "0x09C7", Token.UPTO, "0x09C8" },
		/*28*/	{	"combiningchar", "0x09CB", Token.UPTO, "0x09CD" },
		/*29*/	{	"combiningchar", "0x09D7" },
		/*30*/	{	"combiningchar", "0x09E2", Token.UPTO, "0x09E3" },
		/*31*/	{	"combiningchar", "0x0A02" },
		/*32*/	{	"combiningchar", "0x0A3C" },
		/*33*/	{	"combiningchar", "0x0A3E" },
		/*34*/	{	"combiningchar", "0x0A3F" },
		/*35*/	{	"combiningchar", "0x0A40", Token.UPTO, "0x0A42" },
		/*36*/	{	"combiningchar", "0x0A47", Token.UPTO, "0x0A48" },
		/*37*/	{	"combiningchar", "0x0A4B", Token.UPTO, "0x0A4D" },
		/*38*/	{	"combiningchar", "0x0A70", Token.UPTO, "0x0A71" },
		/*39*/	{	"combiningchar", "0x0A81", Token.UPTO, "0x0A83" },
		/*40*/	{	"combiningchar", "0x0ABC" },
		/*41*/	{	"combiningchar", "0x0ABE", Token.UPTO, "0x0AC5" },
		/*42*/	{	"combiningchar", "0x0AC7", Token.UPTO, "0x0AC9" },
		/*43*/	{	"combiningchar", "0x0ACB", Token.UPTO, "0x0ACD" },
		/*44*/	{	"combiningchar", "0x0B01", Token.UPTO, "0x0B03" },
		/*45*/	{	"combiningchar", "0x0B3C" },
		/*46*/	{	"combiningchar", "0x0B3E", Token.UPTO, "0x0B43" },
		/*47*/	{	"combiningchar", "0x0B47", Token.UPTO, "0x0B48" },
		/*48*/	{	"combiningchar", "0x0B4B", Token.UPTO, "0x0B4D" },
		/*49*/	{	"combiningchar", "0x0B56", Token.UPTO, "0x0B57" },
		/*50*/	{	"combiningchar", "0x0B82", Token.UPTO, "0x0B83" },
		/*51*/	{	"combiningchar", "0x0BBE", Token.UPTO, "0x0BC2" },
		/*52*/	{	"combiningchar", "0x0BC6", Token.UPTO, "0x0BC8" },
		/*53*/	{	"combiningchar", "0x0BCA", Token.UPTO, "0x0BCD" },
		/*54*/	{	"combiningchar", "0x0BD7" },
		/*55*/	{	"combiningchar", "0x0C01", Token.UPTO, "0x0C03" },
		/*56*/	{	"combiningchar", "0x0C3E", Token.UPTO, "0x0C44" },
		/*57*/	{	"combiningchar", "0x0C46", Token.UPTO, "0x0C48" },
		/*58*/	{	"combiningchar", "0x0C4A", Token.UPTO, "0x0C4D" },
		/*59*/	{	"combiningchar", "0x0C55", Token.UPTO, "0x0C56" },
		/*60*/	{	"combiningchar", "0x0C82", Token.UPTO, "0x0C83" },
		/*61*/	{	"combiningchar", "0x0CBE", Token.UPTO, "0x0CC4" },
		/*62*/	{	"combiningchar", "0x0CC6", Token.UPTO, "0x0CC8" },
		/*63*/	{	"combiningchar", "0x0CCA", Token.UPTO, "0x0CCD" },
		/*64*/	{	"combiningchar", "0x0CD5", Token.UPTO, "0x0CD6" },
		/*65*/	{	"combiningchar", "0x0D02", Token.UPTO, "0x0D03" },
		/*66*/	{	"combiningchar", "0x0D3E", Token.UPTO, "0x0D43" },
		/*67*/	{	"combiningchar", "0x0D46", Token.UPTO, "0x0D48" },
		/*68*/	{	"combiningchar", "0x0D4A", Token.UPTO, "0x0D4D" },
		/*69*/	{	"combiningchar", "0x0D57" },
		/*70*/	{	"combiningchar", "0x0E31" },
		/*71*/	{	"combiningchar", "0x0E34", Token.UPTO, "0x0E3A" },
		/*72*/	{	"combiningchar", "0x0E47", Token.UPTO, "0x0E4E" },
		/*73*/	{	"combiningchar", "0x0EB1" },
		/*74*/	{	"combiningchar", "0x0EB4", Token.UPTO, "0x0EB9" },
		/*75*/	{	"combiningchar", "0x0EBB", Token.UPTO, "0x0EBC" },
		/*76*/	{	"combiningchar", "0x0EC8", Token.UPTO, "0x0ECD" },
		/*77*/	{	"combiningchar", "0x0F18", Token.UPTO, "0x0F19" },
		/*78*/	{	"combiningchar", "0x0F35" },
		/*79*/	{	"combiningchar", "0x0F37" },
		/*80*/	{	"combiningchar", "0x0F39" },
		/*81*/	{	"combiningchar", "0x0F3E" },
		/*82*/	{	"combiningchar", "0x0F3F" },
		/*83*/	{	"combiningchar", "0x0F71", Token.UPTO, "0x0F84" },
		/*84*/	{	"combiningchar", "0x0F86", Token.UPTO, "0x0F8B" },
		/*85*/	{	"combiningchar", "0x0F90", Token.UPTO, "0x0F95" },
		/*86*/	{	"combiningchar", "0x0F97" },
		/*87*/	{	"combiningchar", "0x0F99", Token.UPTO, "0x0FAD" },
		/*88*/	{	"combiningchar", "0x0FB1", Token.UPTO, "0x0FB7" },
		/*89*/	{	"combiningchar", "0x0FB9" },
		/*90*/	{	"combiningchar", "0x20D0", Token.UPTO, "0x20DC" },
		/*91*/	{	"combiningchar", "0x20E1" },
		/*92*/	{	"combiningchar", "0x302A", Token.UPTO, "0x302F" },
		/*93*/	{	"combiningchar", "0x3099" },
		/*94*/	{	"combiningchar", "0x309A" },

		/*95*/	{	"extenderchar", "0x00B7" },
		/*96*/	{	"extenderchar", "0x02D0" },
		/*97*/	{	"extenderchar", "0x02D1" },
		/*98*/	{	"extenderchar", "0x0387" },
		/*99*/	{	"extenderchar", "0x0640" },
		/*100*/	{	"extenderchar", "0x0E46" },
		/*101*/	{	"extenderchar", "0x0EC6" },
		/*102*/	{	"extenderchar", "0x3005" },
		/*103*/	{	"extenderchar", "0x3031", Token.UPTO, "0x3035" },
		/*104*/	{	"extenderchar", "0x309D", Token.UPTO, "0x309E" },
		/*105*/	{	"extenderchar", "0x30FC", Token.UPTO, "0x30FE" },
	};


	/** Numerical rules for binary and octal <b>digits</b>. */
	public static final String [][] digitRules = {
		/*0*/	{ "octdigit", "'0'", Token.UPTO, "'7'" },
		/*1*/	{ "octdigits", "octdigits", "octdigit" },
		/*2*/	{ "octdigits", "octdigit" },

		/*3*/	{ "bindigit", "'0'", Token.UPTO, "'1'" },
		/*4*/	{ "bindigits", "bindigits", "bindigit" },
		/*5*/	{ "bindigits", "bindigit" },
	};

	/** Numerical rules for <b>numbers</b> within sourcecode: number ::= integer | float. */
	public static final String [][] numberRules = {
		// number = float | integer (incl. hexnumber)
		/*0*/	{ "number", "float" },
		/*1*/	{ "number", "integer" },
		/*2*/	{ "float", "wholenumber", "'.'", "mantissa", "float_opt" },
		/*3*/	{ "wholenumber", "digits" },
		/*4*/	{ "wholenumber" /*nothing*/ },
		/*5*/	{ "mantissa", "digits", "mantissa_opt" },
		/*6*/	{ "mantissa_opt", "exponent", "digits" },
		/*7*/	{ "mantissa_opt" /*nothing*/ },
		/*8*/	{ "exponent", "exponentletter", "exponentsign" },
		/*9*/	{ "exponentletter", "'e'" },
		/*10*/	{ "exponentletter", "'E'" },
		/*11*/	{ "exponentsign", "'-'" },
		/*12*/	{ "exponentsign", "'+'" },
		/*13*/	{ "exponentsign" /*nothing*/ },
		/*14*/	{ "float_opt", "'f'" },
		/*15*/	{ "float_opt", "'F'" },
		/*16*/	{ "float_opt", "'d'" },
		/*17*/	{ "float_opt", "'D'" },
		/*18*/	{ "float_opt" /*nothing*/ },
		/*19*/	{ "integer", "\"0X\"", "hexdigits" },
		/*20*/	{ "integer", "\"0x\"", "hexdigits" },
		/*21*/	{ "integer", "digits", "integer_opt" },
		/*22*/	{ "integer_opt", "'l'" },	// "long" marker
		/*23*/	{ "integer_opt", "'L'" },	// "long" marker
		/*24*/	{ "integer_opt" /*nothing*/ },
	};
	
	/** Rules describing one or more newlines. */
	public static final String [][] newlinesRules = {
		{ "newlines", "newlines", "newline" },
		{ "newlines", "newline" },
	};
	
	/** Rules describing C/Java-like character definitions: 'c', '\r', '\007'. */
	public static final String [][] chardefRules = {
		/*0*/	{ "chardef", "\"'\\\"", "'0'", Token.UPTO, "'3'", "octdigit", "octdigit", "\"'\"" },
		/*1*/	{ "chardef", "bnf_chardef" },	// but only 248 and 251 - 258 !!!
	};
	
	/** Rules describing whitespace: newlines and spaces, minimum one. */
	public static final String [][] whitespaceRules = {
		/*0*/	{ "whitespace", "newline" },
		/*1*/	{ "whitespace", "space" },
		/*2*/	{ "whitespaces", "whitespaces", "whitespace" },
		/*3*/	{ "whitespaces", "whitespace" },
	};
	

	private StandardLexerRules()	{}

}