javascripttokenmaker.java

具有不同语法高亮的编辑器实例

							break;

						case '\'': // Don't need to worry about backslashes as previous char is non-backslash.
							addToken(text, currentTokenStart,i-1, Token.LITERAL_NUMBER_HEXADECIMAL, newStartOffset+currentTokenStart);
							currentTokenStart = i;
							currentTokenType = Token.ERROR_CHAR;
							backslash = false;
							break;

						case '\\':
							addToken(text, currentTokenStart,i-1, Token.LITERAL_NUMBER_HEXADECIMAL, newStartOffset+currentTokenStart);
							addToken(text, i,i, Token.IDENTIFIER, newStartOffset+i);
							currentTokenType = Token.NULL;
							backslash = true;
							break;

						default:

							if (c=='e') {	// Exponent.
								if (numContainsExponent==false) {
									numContainsExponent = true;
								}
								else {
									currentTokenType = Token.ERROR_NUMBER_FORMAT;
								}
								break;
							}
							int indexOf = hexCharacters.indexOf(c);
							if (indexOf>-1) {
								break;	// Still a hexadecimal number.
							}
							indexOf = numberEndChars.indexOf(c);
							if (indexOf>-1) {	// Numbers can end in 'f','F','l','L', etc.
								if (numContainsEndCharacter==true) {
									currentTokenType = Token.ERROR_NUMBER_FORMAT;
								}
								else {
									numContainsEndCharacter = true;
								}
								break;
							}
							indexOf = operators.indexOf(c);
							if (indexOf>-1) {
								addToken(text, currentTokenStart,i-1, Token.LITERAL_NUMBER_HEXADECIMAL, newStartOffset+currentTokenStart);
								currentTokenStart = i;
								currentTokenType = Token.OPERATOR;
								break;
							}
							indexOf = separators.indexOf(c);
							if (indexOf>-1) {
								addToken(text, currentTokenStart,i-1, Token.LITERAL_NUMBER_HEXADECIMAL, newStartOffset+currentTokenStart);
								addToken(text, i,i, Token.SEPARATOR, newStartOffset+i);
								currentTokenType = Token.NULL;
								break;
							}
							indexOf = separators2.indexOf(c);
							if (indexOf>-1) {
								addToken(text, currentTokenStart,i-1, Token.LITERAL_NUMBER_HEXADECIMAL, newStartOffset+currentTokenStart);
								addToken(text, i,i, Token.IDENTIFIER, newStartOffset+i);
								currentTokenType = Token.NULL;
								break;
							}

							// Otherwise, the token is an error.
							currentTokenType = Token.ERROR_NUMBER_FORMAT;

					} // End of switch (c).

					break;

				case Token.COMMENT_MULTILINE:

					// Find either end of MLC or end of the current line.
					while (i < end-1) {
						if (array[i]=='*' && array[i+1]=='/') {
							addToken(text, currentTokenStart,i+1, Token.COMMENT_MULTILINE, newStartOffset+currentTokenStart);
							i = i + 1;
							currentTokenType = Token.NULL;
							backslash = false; // Backslashes can't accumulate before and after a comment...
							break;
						}
						i++;
					}

					break;

				case Token.COMMENT_EOL:
					i = end - 1;
					addToken(text, currentTokenStart,i, Token.COMMENT_EOL, newStartOffset+currentTokenStart);
					// We need to set token type to null so at the bottom we don't add one more token.
					currentTokenType = Token.NULL;
					break;

				// We need this state because comments always start with '/', which is an operator.
				// Note that when we enter this state, the PREVIOUS character was an operator.
				case Token.OPERATOR:

					if (array[i-1]=='/') { // Possibility of comments.

						if (c=='*') {
							currentTokenType = Token.COMMENT_MULTILINE;
							break;
						}

						else if (c=='/') {
							currentTokenType = Token.COMMENT_EOL;
							i = end - 1;	// Since we know the rest of the line is in this token.
						}

						else {
							// We MUST add the token at the previous char now; if we don't and let
							// operators accumulate before we print them, we will mess up syntax
							// highlighting if we get an end-of-line comment.
							addToken(text, currentTokenStart,i-1, Token.OPERATOR, newStartOffset+currentTokenStart);
							currentTokenType = Token.NULL;
							i = i - 1;
						}

					}

					else {

						addToken(text, currentTokenStart,i-1, Token.OPERATOR, newStartOffset+currentTokenStart);

						// Hack to keep code size down...
						i--;
						currentTokenType = Token.NULL;

					}

					break;

				case Token.ERROR_IDENTIFIER:

					switch (c) {

						case ' ':
						case '\t':
							addToken(text, currentTokenStart,i-1, Token.ERROR_IDENTIFIER, newStartOffset+currentTokenStart);
							currentTokenStart = i;
							currentTokenType = Token.WHITESPACE;
							break;

						case '"': // Don't need to worry about backslashes as previous char is non-backslash.
							addToken(text, currentTokenStart,i-1, Token.ERROR_IDENTIFIER, newStartOffset+currentTokenStart);
							currentTokenStart = i;
							currentTokenType = Token.ERROR_STRING_DOUBLE;
							backslash = false;
							break;

						case '\'': // Don't need to worry about backslashes as previous char is non-backslash.
							addToken(text, currentTokenStart,i-1, Token.ERROR_IDENTIFIER, newStartOffset+currentTokenStart);
							currentTokenStart = i;
							currentTokenType = Token.ERROR_CHAR;
							backslash = false;
							break;

						case ';':
							addToken(text, currentTokenStart,i-1, Token.ERROR_IDENTIFIER, newStartOffset+currentTokenStart);
							addToken(text, i,i, Token.IDENTIFIER, newStartOffset+i);
							currentTokenType = Token.NULL;
							break;

						case '\\':
							addToken(text, currentTokenStart,i-1, Token.ERROR_IDENTIFIER, newStartOffset+currentTokenStart);
							addToken(text, i,i, Token.IDENTIFIER, newStartOffset+i);
							currentTokenType = Token.NULL;
							backslash = true; // Must be first backslash in a row since previous character is identifier char.
							break;

						default:
							int indexOf = operators.indexOf(c);
							if (indexOf>-1) {
									addToken(text, currentTokenStart,i-1, Token.ERROR_IDENTIFIER, newStartOffset+currentTokenStart);
									currentTokenStart = i;
									currentTokenType = Token.OPERATOR;
							}
							indexOf = separators.indexOf(c);
							if (indexOf>-1) {
									addToken(text, currentTokenStart,i-1, Token.ERROR_IDENTIFIER, newStartOffset+currentTokenStart);
									addToken(text, i,i, Token.SEPARATOR, newStartOffset+i);
									currentTokenType = Token.NULL;
							}
							indexOf = separators2.indexOf(c);
							if (indexOf>-1) {
									addToken(text, currentTokenStart,i-1, Token.ERROR_IDENTIFIER, newStartOffset+currentTokenStart);
									addToken(text, i,i, Token.IDENTIFIER, newStartOffset+i);
									currentTokenType = Token.NULL;
							}

					} // End of switch (c).

					break;

				case Token.ERROR_NUMBER_FORMAT:

						switch (c) {

							case ' ':
							case '\t':
								addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
								currentTokenStart = i;
								currentTokenType = Token.WHITESPACE;
								break;

							case '"': // Don't need to worry about backslashes as previous char is non-backslash.
								addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
								currentTokenStart = i;
								currentTokenType = Token.ERROR_STRING_DOUBLE;
								backslash = false;
								break;

							case '\'': // Don't need to worry about backslashes as previous char is non-backslash.
								addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
								currentTokenStart = i;
								currentTokenType = Token.ERROR_CHAR;
								backslash = false;
								break;

							case ';':
								addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
								addToken(text, i,i, Token.IDENTIFIER, newStartOffset+i);
								currentTokenType = Token.NULL;
								break;

							case '\\':
								addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
								addToken(text, i,i, Token.IDENTIFIER, newStartOffset+i);
								currentTokenType = Token.NULL;
								backslash = true; // Must be first backslash in a row since previous char is a number char.
								break;

							default:

								// Could be going into hexadecimal.
								int indexOf = hexCharacters.indexOf(c);
								if (indexOf>-1 && (i-currentTokenStart==2 && array[i-1]=='x' && array[i-2]=='0')) {
									currentTokenType = Token.LITERAL_NUMBER_HEXADECIMAL;
									break;
								}

								indexOf = operators.indexOf(c);
								if (indexOf>-1) {
										addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
										currentTokenStart = i;
										currentTokenType = Token.OPERATOR;
								}
								indexOf = separators.indexOf(c);
								if (indexOf>-1) {
										addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
										addToken(text, i,i, Token.SEPARATOR, newStartOffset+i);
										currentTokenType = Token.NULL;
								}
								indexOf = separators2.indexOf(c);
								if (indexOf>-1) {
										addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
										addToken(text, i,i, Token.IDENTIFIER, newStartOffset+i);
										currentTokenType = Token.NULL;
								}

						} // End of switch (c).

						break;

				case Token.ERROR_CHAR:

						if (c=='\\') {
							backslash = !backslash; // Okay because if we got in here, backslash was initially false.
						}
						else {

							if (c=='\'' && !backslash) {
								addToken(text, currentTokenStart,i, Token.LITERAL_CHAR, newStartOffset+currentTokenStart);
								currentTokenType = Token.NULL;
								// backslash is definitely false when we leave.
							}

							backslash = false; // Need to set backslash to false here as a character was typed.

						}
						// Otherwise, we're still an unclosed char...

						break;

				case Token.ERROR_STRING_DOUBLE:

						if (c=='\\') {
							backslash = !backslash; // Okay because if we got in here, backslash was initially false.
						}
						else {
							if (c=='"' && !backslash) {
								addToken(text, currentTokenStart,i, Token.LITERAL_STRING_DOUBLE_QUOTE, newStartOffset+currentTokenStart);
								currentTokenType = Token.NULL;
								// backslash is definitely false when we leave.
							}

							backslash = false; // Need to set backslash to false here as a character was typed.

						}
						// Otherwise, we're still an unclosed string...

						break;

				default:

					System.err.println("Invalid currentTokenType: " + currentTokenType + "; c=='" + c + "'");
					System.exit(0);

			} // End of switch (currentTokenType).

		} // End of for (int i=offset; i<end; i++).

		// Deal with the (possibly there) last token.
		if (currentTokenType != Token.NULL) {
			addToken(text, currentTokenStart,end-1, currentTokenType, newStartOffset+currentTokenStart);
		}
		if (currentTokenType!=Token.COMMENT_MULTILINE) {
			addNullToken();
		}


		// Return the first token in our linked list.
		return firstToken;

	}


/*****************************************************************************/

}