perltokenmaker.java

具有不同语法高亮的编辑器实例

							break;
						}
						i++;
					}
								
					// This only happens if we never found the end of the variable in the loop above.
					if (i==end) {
						addToken(text, currentTokenStart,i-1, Token.VARIABLE, newStartOffset+currentTokenStart);
						currentTokenType = Token.NULL;
					}
								
					break;

				case Token.COMMENT_EOL:
					// If we got here, then the line != "#" only, so check for "#!".
					if (c=='!')
						currentTokenType = Token.PREPROCESSOR;
					i = end - 1;
					addToken(text, currentTokenStart,i, currentTokenType, newStartOffset+currentTokenStart);
					// We need to set token type to null so at the bottom we don't add one more token.
					currentTokenType = Token.NULL;
					
					break;

				case Token.LITERAL_CHAR:

						if (c=='\\') {
							backslash = !backslash; // Okay because if we got in here, backslash was initially false.
						}
						else {
							if (c=='\'' && !backslash) {
								addToken(text, currentTokenStart,i, Token.LITERAL_CHAR, newStartOffset+currentTokenStart);
								currentTokenStart = i + 1;
								currentTokenType = Token.NULL;
								// backslash is definitely false when we leave.
							}

							backslash = false; // Need to set backslash to false here as a character was typed.

						}
						// Otherwise, we're still an unclosed char literal...

						break;

				case Token.LITERAL_BACKQUOTE:

						switch (c) {
								
							case '\\':
								backslash = !backslash;
								break;
								
							case '`':
								if (!backslash) {
									addToken(text, currentTokenStart,i, Token.LITERAL_BACKQUOTE, newStartOffset+currentTokenStart);
									currentTokenType = Token.NULL;
									// backslash is definitely false when we leave.
									break;
								}
								backslash = false;
								break;

							// Variable in the backquote string...

							
							// Variable in the backquote string...
							case '$':
							case '@':
							case '%':

								if (backslash==true) {
									backslash = false;
									break;
								}
							
								// Add the string up-to the variable.
								addToken(text, currentTokenStart,i-1, Token.LITERAL_BACKQUOTE, newStartOffset+currentTokenStart);
								currentTokenType = Token.VARIABLE;
								currentTokenStart = i;
								i++;
								c = array[i];	// Update to point just after the '$' or '@'.
								
								// Note that we first arrive here AFTER the '$' or '@' character.
								if (shellVariables.indexOf(c)>-1) {
									addToken(text, currentTokenStart,i, Token.VARIABLE, newStartOffset+currentTokenStart);
									currentTokenType = Token.LITERAL_BACKQUOTE;
									currentTokenStart = i + 1; // Right after the end of the variable.
									break;
								}

								// Find the end of the variable...
								while (i<end) {
									c = array[i]; // Redundant for the first character...
									if (!RSyntaxUtilities.isLetterOrDigit(c) && c!='_') {
										addToken(text, currentTokenStart,i-1, Token.VARIABLE, newStartOffset+currentTokenStart);
										if (c=='`') { // The only rub.
											addToken(text, i,i, Token.LITERAL_BACKQUOTE, newStartOffset+i);
											currentTokenType = Token.NULL;
											break;
										}
										else {
											currentTokenStart = i;
											currentTokenType = Token.LITERAL_BACKQUOTE;
											i--;
											break;
										}
									}
									i++;
								} // End of while (i<end).
								
								// This only happens if we never found the end of the variable in the loop above.
								// We "trick" this method so that the backquote string token is at the end.
								if (i==end) {
									addToken(text, currentTokenStart,i-1, Token.VARIABLE, newStartOffset+currentTokenStart);
									currentTokenStart = i;
									currentTokenType = Token.LITERAL_BACKQUOTE;
								}
								
								break;
								
							// Otherwise, we're still in an unclosed string...
							default:
								backslash = false; // Need to set backslash to false here as a character was typed.

						} // End of switch (c).
				
						break;

				case Token.LITERAL_STRING_DOUBLE_QUOTE:
				
						switch (c) {
								
							case '\\':
								backslash = !backslash;
								break;
								
							case '"':
								if (!backslash) {
									addToken(text, currentTokenStart,i, Token.LITERAL_STRING_DOUBLE_QUOTE, newStartOffset+currentTokenStart);
									currentTokenType = Token.NULL;
									// backslash is definitely false when we leave.
									break;
								}
								backslash = false;
								break;
							
							// Variable in the double-quoted string...
							case '$':
							case '@':
							case '%':

								if (backslash==true) {
									backslash = false;
									break;
								}
							
								// Add the string up-to the variable.
								addToken(text, currentTokenStart,i-1, Token.LITERAL_STRING_DOUBLE_QUOTE, newStartOffset+currentTokenStart);
								currentTokenType = Token.VARIABLE;
								currentTokenStart = i;
								i++;
								c = array[i];	// Update to point just after the '$' or '@'.
								

								// Note that we first arrive here AFTER the '$' or '@' character.
								if (shellVariables.indexOf(c)>-1) {
									addToken(text, currentTokenStart,i, Token.VARIABLE, newStartOffset+currentTokenStart);
									currentTokenType = Token.LITERAL_STRING_DOUBLE_QUOTE;
									currentTokenStart = i + 1; // Right after the end of the variable.
									break;
								}

								// Find the end of the variable...
								// Increment first to skip the '$'.
								while (i<end) {
									c = array[i]; // Redundant for the first character...
									if (!RSyntaxUtilities.isLetterOrDigit(c) && c!='_') {
										addToken(text, currentTokenStart,i-1, Token.VARIABLE, newStartOffset+currentTokenStart);
										if (c=='`') { // The only rub.
											addToken(text, i,i, Token.LITERAL_STRING_DOUBLE_QUOTE, newStartOffset+i);
											currentTokenType = Token.NULL;
											break;
										}
										else {
											currentTokenStart = i;
											currentTokenType = Token.LITERAL_STRING_DOUBLE_QUOTE;
											i--;
											break;
										}
									}
									i++;
								} // End of while (i<end).
								
								// This only happens if we never found the end of the variable in the loop above.
								// We "trick" this method so that the backquote string token is at the end.
								if (i==end) {
									addToken(text, currentTokenStart,i-1, Token.VARIABLE, newStartOffset+currentTokenStart);
									currentTokenStart = i;
									currentTokenType = Token.LITERAL_STRING_DOUBLE_QUOTE;
								}
								
								break;
								
							// Otherwise, we're still in an unclosed string...
							default:
								backslash = false; // Need to set backslash to false here as a character was typed.

						} // End of switch (c).
				
						break;

				case Token.ERROR_NUMBER_FORMAT:

						switch (c) {

							case ' ':
							case '\t':
								addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
								currentTokenStart = i;
								currentTokenType = Token.WHITESPACE;
								break;

							case '`': // Don't need to worry about backslashes as previous char is space.
								addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
								currentTokenStart = i;
								currentTokenType = Token.LITERAL_BACKQUOTE;
								backslash = false;
								break;

							case '"': // Don't need to worry about backslashes as previous char is non-backslash.
								addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
								currentTokenStart = i;
								currentTokenType = Token.LITERAL_STRING_DOUBLE_QUOTE; //ERROR_UNCLOSED_STRING_DOUBLE;
								backslash = false;
								break;

							case '\'': // Don't need to worry about backslashes as previous char is non-backslash.
								addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
								currentTokenStart = i;
								currentTokenType = Token.ERROR_CHAR;
								backslash = false;
								break;

							case ';':
								addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
								addToken(text, i,i, Token.IDENTIFIER, newStartOffset+i);
								currentTokenType = Token.NULL;
								break;

							case '\\':
								addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
								addToken(text, i,i, Token.IDENTIFIER, newStartOffset+i);
								currentTokenType = Token.NULL;
								backslash = true; // Must be first backslash in a row since previous char is a number char.
								break;

							default:

								// Could be going into hexadecimal.
								int indexOf = hexCharacters.indexOf(c);
								if (indexOf>-1 && (i-currentTokenStart==2 && array[i-1]=='x' && array[i-2]=='0')) {
									currentTokenType = Token.LITERAL_NUMBER_HEXADECIMAL;
									break;
								}

								indexOf = operators.indexOf(c);
								if (indexOf>-1) {
										addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
										currentTokenStart = i;
										currentTokenType = Token.OPERATOR;
								}
								indexOf = separators.indexOf(c);
								if (indexOf>-1) {
										addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
										addToken(text, i,i, Token.SEPARATOR, newStartOffset+i);
										currentTokenType = Token.NULL;
								}
								indexOf = separators2.indexOf(c);
								if (indexOf>-1) {
										addToken(text, currentTokenStart,i-1, Token.ERROR_NUMBER_FORMAT, newStartOffset+currentTokenStart);
										addToken(text, i,i, Token.IDENTIFIER, newStartOffset+i);
										currentTokenType = Token.NULL;
								}

						} // End of switch (c).

						break;

				default:

					System.err.println("Invalid currentTokenType: " + currentTokenType + "; c=='" + c + "'");
					System.exit(0);

			} // End of switch (currentTokenType).

		} // End of for (int i=offset; i<end; i++).

		switch (currentTokenType) {

			// Remember what token type to begin the next line with.
			case Token.LITERAL_BACKQUOTE:
			case Token.LITERAL_STRING_DOUBLE_QUOTE:
			case Token.LITERAL_CHAR:
						addToken(text, currentTokenStart,end-1, currentTokenType, newStartOffset+currentTokenStart);
						break;

			// Do nothing if everything was okay.
			case Token.NULL:
						addNullToken();
						break;

			// All other token types don't continue to the next line...
			default:
						addToken(text, currentTokenStart,end-1, currentTokenType, newStartOffset+currentTokenStart);
						addNullToken();

		}

		// Return the first token in our linked list.
		return firstToken;

	}


/*****************************************************************************/

}