standardlexerrules.java

java 词法分析器,用于一般的C,C++,VB,PS/SQL 语句的翻译

	
	/** Rules to scan # shell-style comments. */
	public static final String [][] getShellStyleCommentRules()	{
		String [][] sarr0 = getUnicodeCharRules();
		String [][] sarr1 = getNewlineRules();
		String [][] sarr2 = getSomeRules(291, 297);
		return catenizeRules(new String [][][] { sarr0, sarr1, sarr2 });
	}
	
	/** Rules for XML combining chars. */
	public static final String [][] getUnicodeXmlCharRules()	{
		return xmlCharRules;
	}
	
	/** Rules for XML combining chars. */
	public static final String [][] getUnicodeCombiningCharRules()	{
		String [][] sarr = getSomeRules(0, 95, xmlCombinigAndExtenderRules);
		return sarr;
	}
	
	/** Rules for XML extender chars. */
	public static final String [][] getUnicodeExtenderCharRules()	{
		String [][] sarr = getSomeRules(95, 106, xmlCombinigAndExtenderRules);
		return sarr;
	}
	
	/** Rules for octal number chars. */
	public static final String [][] getOctDigitsRules()	{
		String [][] sarr = getSomeRules(0, 3, digitRules);
		return sarr;
	}
	
	/** Rules for binary number chars. */
	public static final String [][] getBinDigitsRules()	{
		String [][] sarr = getSomeRules(3, 6, digitRules);
		return sarr;
	}
	
	/** Rules for general number chars (integer, float). */
	public static final String [][] getNumberRules()	{
		String [][] sarr2 = getIntegerRules();
		String [][] sarr1 = getFloatRules();
		String [][] sarr0 = getSomeRules(0, 2, numberRules);
		return catenizeRules(new String [][][] { sarr0, sarr1, sarr2 });
	}
	
	/** Rules for integer number chars. */
	public static final String [][] getIntegerRules()	{
		String [][] sarr2 = getHexDigitsRules();
		String [][] sarr1 = getUnicodeDigitsRules();
		String [][] sarr0 = getSomeRules(19, 25, numberRules);
		return catenizeRules(new String [][][] { sarr0, sarr1, sarr2 });
	}
	
	/** Rules for float number chars. */
	public static final String [][] getFloatRules()	{
		String [][] sarr1 = getUnicodeDigitsRules();
		String [][] sarr0 = getSomeRules(2, 19, numberRules);
		return catenizeRules(new String [][][] { sarr0, sarr1 });
	}
	

	private static final String [][] getSomeRules(int startIncl, int endExcl)	{
		return getSomeRules(startIncl, endExcl, lexerSyntax);
	}

	private static final String [][] getSomeRules(int startIncl, int endExcl, String [][] rules)	{
		String [][] sarr = new String [endExcl - startIncl] [];
		int j = 0;
		for (int i = startIncl; i < endExcl; i++, j++)
			sarr[j] = rules[i];
		return sarr;
	}


	/** Print a grammar to System.out. */
	public static void printRules(String [][] syntax)	{
		for (int i = 0; i < syntax.length; i++)	{
			for (int j = 0; j < syntax[i].length; j++)
				System.out.print(j == 1 ? " ::= "+syntax[i][j]+" " : syntax[i][j]+" ");
			System.out.println();
		}
	}


	/** Catenizes some rule sets to one rule set. Does not check for uniqueness. */
	public static final String [][] catenizeRules(String [][][] arrays)	{
		int len = 0;
		for (int i = 0; i < arrays.length; i++)
			len += arrays[i].length;
			
		String [][] sarr = new String [len][];
		
		int k = 0;
		for (int i = 0; i < arrays.length; i++)	{
			for (int j = 0; j < arrays[i].length; j++)	{
				sarr[k] = arrays[i][j];
				k++;
			}
		}

		return sarr;
	}




	/** Premade lexer syntax used to scan textual EBNF-like syntax specifications. */
	public static final String [][] lexerSyntax = {
	
		// CAUTION: Do NOT edit without changing indexes in source above!!!
		
		/*0*/	{ Token.TOKEN, "identifier" },
		/*1*/	{ Token.TOKEN, "bnf_chardef" },
		/*2*/	{ Token.TOKEN, "stringdef" },
		/*3*/	{ Token.TOKEN, "quantifier" },	// see 297: ruleref

		/*4*/	{ Token.IGNORED,  "spaces" },
		/*5*/	{ Token.IGNORED,  "newline" },
		/*6*/	{ Token.IGNORED,  "comment" },

		/*7*/	{ "quantifier", "'*'" },
		/*8*/	{ "quantifier", "'+'" },
		/*9*/	{ "quantifier", "'?'" },

		// programmer digits
		
		/*10*/	{ "hexdigit", "'0'", Token.UPTO, "'9'" },
		/*11*/	{ "hexdigit", "'A'", Token.UPTO, "'F'" },
		/*12*/	{ "hexdigit", "'a'", Token.UPTO, "'f'" },
		
		// formatting characters
		
		/*13*/	{ "space",   "0x20" },
		/*14*/	{ "space",   "0x9" },
		/*15*/	{ "space",   "0xC" },	// formfeed
		
		/*16*/	{ "cr", "'\\r'" },	// 0xD
		/*17*/	{ "nl", "'\\n'" },	// 0xA
		/*18*/	{ "newline",   "cr", "nl" },
		/*19*/	{ "newline",   "cr" },
		/*20*/	{ "newline",   "nl" },

		// UNICODE character set
		
		/*21*/	{ "char",    "0x0", Token.UPTO, "0xFFFF" },

		/*22*/	{ "digit",   "0x0030", Token.UPTO, "0x0039" },
		/*23*/	{ "digit",   "0x0660", Token.UPTO, "0x0669" },
		/*24*/	{ "digit",   "0x06F0", Token.UPTO, "0x06F9" },
		/*25*/	{ "digit",   "0x0966", Token.UPTO, "0x096F" },
		/*26*/	{ "digit",   "0x09E6", Token.UPTO, "0x09EF" },
		/*27*/	{ "digit",   "0x0A66", Token.UPTO, "0x0A6F" },
		/*28*/	{ "digit",   "0x0AE6", Token.UPTO, "0x0AEF" },
		/*29*/	{ "digit",   "0x0B66", Token.UPTO, "0x0B6F" },
		/*30*/	{ "digit",   "0x0BE7", Token.UPTO, "0x0BEF" },
		/*31*/	{ "digit",   "0x0C66", Token.UPTO, "0x0C6F" },
		/*32*/	{ "digit",   "0x0CE6", Token.UPTO, "0x0CEF" },
		/*33*/	{ "digit",   "0x0D66", Token.UPTO, "0x0D6F" },
		/*34*/	{ "digit",   "0x0E50", Token.UPTO, "0x0E59" },
		/*35*/	{ "digit",   "0x0ED0", Token.UPTO, "0x0ED9" },
		/*36*/	{ "digit",   "0x0F20", Token.UPTO, "0x0F29" },

		/*37*/	{ "letter",  "0x0041", Token.UPTO, "0x005A" },	// BaseChar
		/*38*/	{ "letter",  "0x0061", Token.UPTO, "0x007A" },
		/*39*/	{ "letter",  "0x00C0", Token.UPTO, "0x00D6" },
		/*40*/	{ "letter",  "0x00D8", Token.UPTO, "0x00F6" },
		/*41*/	{ "letter",  "0x00F8", Token.UPTO, "0x00FF" },
		/*42*/	{ "letter",  "0x0100", Token.UPTO, "0x0131" },
		/*43*/	{ "letter",  "0x0134", Token.UPTO, "0x013E" },
		/*44*/	{ "letter",  "0x0141", Token.UPTO, "0x0148" },
		/*45*/	{ "letter",  "0x014A", Token.UPTO, "0x017E" },
		/*46*/	{ "letter",  "0x0180", Token.UPTO, "0x01C3" },
		/*47*/	{ "letter",  "0x01CD", Token.UPTO, "0x01F0" },
		/*48*/	{ "letter",  "0x01F4", Token.UPTO, "0x01F5" },
		/*49*/	{ "letter",  "0x01FA", Token.UPTO, "0x0217" },
		/*50*/	{ "letter",  "0x0250", Token.UPTO, "0x02A8" },
		/*51*/	{ "letter",  "0x02BB", Token.UPTO, "0x02C1" },
		/*52*/	{ "letter",  "0x0386" },
		/*53*/	{ "letter",  "0x0388", Token.UPTO, "0x038A" },
		/*54*/	{ "letter",  "0x038C" },
		/*55*/	{ "letter",  "0x038E", Token.UPTO, "0x03A1" },
		/*56*/	{ "letter",  "0x03A3", Token.UPTO, "0x03CE" },
		/*57*/	{ "letter",  "0x03D0", Token.UPTO, "0x03D6" },
		/*58*/	{ "letter",  "0x03DA" },
		/*59*/	{ "letter",  "0x03DC" },
		/*60*/	{ "letter",  "0x03DE" },
		/*61*/	{ "letter",  "0x03E0" },
		/*62*/	{ "letter",  "0x03E2", Token.UPTO, "0x03F3" },
		/*63*/	{ "letter",  "0x0401", Token.UPTO, "0x040C" },
		/*64*/	{ "letter",  "0x040E", Token.UPTO, "0x044F" },
		/*65*/	{ "letter",  "0x0451", Token.UPTO, "0x045C" },
		/*66*/	{ "letter",  "0x045E", Token.UPTO, "0x0481" },
		/*67*/	{ "letter",  "0x0490", Token.UPTO, "0x04C4" },
		/*68*/	{ "letter",  "0x04C7", Token.UPTO, "0x04C8" },
		/*69*/	{ "letter",  "0x04CB", Token.UPTO, "0x04CC" },
		/*70*/	{ "letter",  "0x04D0", Token.UPTO, "0x04EB" },
		/*71*/	{ "letter",  "0x04EE", Token.UPTO, "0x04F5" },
		/*72*/	{ "letter",  "0x04F8", Token.UPTO, "0x04F9" },
		/*73*/	{ "letter",  "0x0531", Token.UPTO, "0x0556" },
		/*74*/	{ "letter",  "0x0559" },
		/*75*/	{ "letter",  "0x0561", Token.UPTO, "0x0586" },
		/*76*/	{ "letter",  "0x05D0", Token.UPTO, "0x05EA" },
		/*77*/	{ "letter",  "0x05F0", Token.UPTO, "0x05F2" },
		/*78*/	{ "letter",  "0x0621", Token.UPTO, "0x063A" },
		/*79*/	{ "letter",  "0x0641", Token.UPTO, "0x064A" },
		/*80*/	{ "letter",  "0x0671", Token.UPTO, "0x06B7" },
		/*81*/	{ "letter",  "0x06BA", Token.UPTO, "0x06BE" },
		/*82*/	{ "letter",  "0x06C0", Token.UPTO, "0x06CE" },
		/*83*/	{ "letter",  "0x06D0", Token.UPTO, "0x06D3" },
		/*84*/	{ "letter",  "0x06D5" },
		/*85*/	{ "letter",  "0x06E5", Token.UPTO, "0x06E6" },
		/*86*/	{ "letter",  "0x0905", Token.UPTO, "0x0939" },
		/*87*/	{ "letter",  "0x093D" },
		/*88*/	{ "letter",  "0x0958", Token.UPTO, "0x0961" },
		/*89*/	{ "letter",  "0x0985", Token.UPTO, "0x098C" },
		/*90*/	{ "letter",  "0x098F", Token.UPTO, "0x0990" },
		/*91*/	{ "letter",  "0x0993", Token.UPTO, "0x09A8" },
		/*92*/	{ "letter",  "0x09AA", Token.UPTO, "0x09B0" },
		/*93*/	{ "letter",  "0x09B2" },
		/*94*/	{ "letter",  "0x09B6", Token.UPTO, "0x09B9" },
		/*95*/	{ "letter",  "0x09DC", Token.UPTO, "0x09DD" },
		/*96*/	{ "letter",  "0x09DF", Token.UPTO, "0x09E1" },
		/*97*/	{ "letter",  "0x09F0", Token.UPTO, "0x09F1" },
		/*98*/	{ "letter",  "0x0A05", Token.UPTO, "0x0A0A" },
		/*99*/	{ "letter",  "0x0A0F", Token.UPTO, "0x0A10" },
		/*100*/	{ "letter",  "0x0A13", Token.UPTO, "0x0A28" },
		/*101*/	{ "letter",  "0x0A2A", Token.UPTO, "0x0A30" },
		/*102*/	{ "letter",  "0x0A32", Token.UPTO, "0x0A33" },
		/*103*/	{ "letter",  "0x0A35", Token.UPTO, "0x0A36" },
		/*104*/	{ "letter",  "0x0A38", Token.UPTO, "0x0A39" },
		/*105*/	{ "letter",  "0x0A59", Token.UPTO, "0x0A5C" },
		/*106*/	{ "letter",  "0x0A5E" },
		/*107*/	{ "letter",  "0x0A72", Token.UPTO, "0x0A74" },
		/*108*/	{ "letter",  "0x0A85", Token.UPTO, "0x0A8B" },
		/*109*/	{ "letter",  "0x0A8D" },
		/*110*/	{ "letter",  "0x0A8F", Token.UPTO, "0x0A91" },
		/*111*/	{ "letter",  "0x0A93", Token.UPTO, "0x0AA8" },
		/*112*/	{ "letter",  "0x0AAA", Token.UPTO, "0x0AB0" },
		/*113*/	{ "letter",  "0x0AB2", Token.UPTO, "0x0AB3" },
		/*114*/	{ "letter",  "0x0AB5", Token.UPTO, "0x0AB9" },
		/*115*/	{ "letter",  "0x0ABD" },
		/*116*/	{ "letter",  "0x0AE0" },
		/*117*/	{ "letter",  "0x0B05", Token.UPTO, "0x0B0C" },
		/*118*/	{ "letter",  "0x0B0F", Token.UPTO, "0x0B10" },
		/*119*/	{ "letter",  "0x0B13", Token.UPTO, "0x0B28" },
		/*120*/	{ "letter",  "0x0B2A", Token.UPTO, "0x0B30" },
		/*121*/	{ "letter",  "0x0B32", Token.UPTO, "0x0B33" },
		/*122*/	{ "letter",  "0x0B36", Token.UPTO, "0x0B39" },
		/*123*/	{ "letter",  "0x0B3D" },
		/*124*/	{ "letter",  "0x0B5C", Token.UPTO, "0x0B5D" },
		/*125*/	{ "letter",  "0x0B5F", Token.UPTO, "0x0B61" },
		/*126*/	{ "letter",  "0x0B85", Token.UPTO, "0x0B8A" },
		/*127*/	{ "letter",  "0x0B8E", Token.UPTO, "0x0B90" },
		/*128*/	{ "letter",  "0x0B92", Token.UPTO, "0x0B95" },
		/*129*/	{ "letter",  "0x0B99", Token.UPTO, "0x0B9A" },
		/*130*/	{ "letter",  "0x0B9C" },
		/*131*/	{ "letter",  "0x0B9E", Token.UPTO, "0x0B9F" },
		/*132*/	{ "letter",  "0x0BA3", Token.UPTO, "0x0BA4" },
		/*133*/	{ "letter",  "0x0BA8", Token.UPTO, "0x0BAA" },
		/*134*/	{ "letter",  "0x0BAE", Token.UPTO, "0x0BB5" },
		/*135*/	{ "letter",  "0x0BB7", Token.UPTO, "0x0BB9" },
		/*136*/	{ "letter",  "0x0C05", Token.UPTO, "0x0C0C" },
		/*137*/	{ "letter",  "0x0C0E", Token.UPTO, "0x0C10" },
		/*138*/	{ "letter",  "0x0C12", Token.UPTO, "0x0C28" },
		/*139*/	{ "letter",  "0x0C2A", Token.UPTO, "0x0C33" },
		/*140*/	{ "letter",  "0x0C35", Token.UPTO, "0x0C39" },
		/*141*/	{ "letter",  "0x0C60", Token.UPTO, "0x0C61" },
		/*142*/	{ "letter",  "0x0C85", Token.UPTO, "0x0C8C" },
		/*143*/	{ "letter",  "0x0C8E", Token.UPTO, "0x0C90" },
		/*144*/	{ "letter",  "0x0C92", Token.UPTO, "0x0CA8" },
		/*145*/	{ "letter",  "0x0CAA", Token.UPTO, "0x0CB3" },
		/*146*/	{ "letter",  "0x0CB5", Token.UPTO, "0x0CB9" },
		/*147*/	{ "letter",  "0x0CDE" },
		/*148*/	{ "letter",  "0x0CE0", Token.UPTO, "0x0CE1" },
		/*149*/	{ "letter",  "0x0D05", Token.UPTO, "0x0D0C" },
		/*150*/	{ "letter",  "0x0D0E", Token.UPTO, "0x0D10" },
		/*151*/	{ "letter",  "0x0D12", Token.UPTO, "0x0D28" },
		/*152*/	{ "letter",  "0x0D2A", Token.UPTO, "0x0D39" },
		/*153*/	{ "letter",  "0x0D60", Token.UPTO, "0x0D61" },
		/*154*/	{ "letter",  "0x0E01", Token.UPTO, "0x0E2E" },
		/*155*/	{ "letter",  "0x0E30" },
		/*156*/	{ "letter",  "0x0E32", Token.UPTO, "0x0E33" },
		/*157*/	{ "letter",  "0x0E40", Token.UPTO, "0x0E45" },
		/*158*/	{ "letter",  "0x0E81", Token.UPTO, "0x0E82" },
		/*159*/	{ "letter",  "0x0E84" },
		/*160*/	{ "letter",  "0x0E87", Token.UPTO, "0x0E88" },
		/*161*/	{ "letter",  "0x0E8A" },
		/*162*/	{ "letter",  "0x0E8D" },
		/*163*/	{ "letter",  "0x0E94", Token.UPTO, "0x0E97" },
		/*164*/	{ "letter",  "0x0E99", Token.UPTO, "0x0E9F" },
		/*165*/	{ "letter",  "0x0EA1", Token.UPTO, "0x0EA3" },
		/*166*/	{ "letter",  "0x0EA5" },
		/*167*/	{ "letter",  "0x0EA7" },
		/*168*/	{ "letter",  "0x0EAA", Token.UPTO, "0x0EAB" },
		/*169*/	{ "letter",  "0x0EAD", Token.UPTO, "0x0EAE" },
		/*170*/	{ "letter",  "0x0EB0" },
		/*171*/	{ "letter",  "0x0EB2", Token.UPTO, "0x0EB3" },
		/*172*/	{ "letter",  "0x0EBD" },
		/*173*/	{ "letter",  "0x0EC0", Token.UPTO, "0x0EC4" },
		/*174*/	{ "letter",  "0x0F40", Token.UPTO, "0x0F47" },
		/*175*/	{ "letter",  "0x0F49", Token.UPTO, "0x0F69" },
		/*176*/	{ "letter",  "0x10A0", Token.UPTO, "0x10C5" },
		/*177*/	{ "letter",  "0x10D0", Token.UPTO, "0x10F6" },
		/*178*/	{ "letter",  "0x1100" },
		/*179*/	{ "letter",  "0x1102", Token.UPTO, "0x1103" },
		/*180*/	{ "letter",  "0x1105", Token.UPTO, "0x1107" },
		/*181*/	{ "letter",  "0x1109" },
		/*182*/	{ "letter",  "0x110B", Token.UPTO, "0x110C" },
		/*183*/	{ "letter",  "0x110E", Token.UPTO, "0x1112" },
		/*184*/	{ "letter",  "0x113C" },
		/*185*/	{ "letter",  "0x113E" },
		/*186*/	{ "letter",  "0x1140" },
		/*187*/	{ "letter",  "0x114C" },
		/*188*/	{ "letter",  "0x114E" },
		/*189*/	{ "letter",  "0x1150" },