📄 pythontokenmaker.flex
字号:
intpart = ({digit}+)
fraction = ("."{digit}+)
exponent = ([eE][\+\-]?{digit}+)
imagnumber = (({floatnumber}|{intpart})[jJ])
ErrorNumberFormat = ({digit}{NonSeparator}+)
NonSeparator = ([^\t\f\r\n\ \(\)\{\}\[\]\;\,\.\=\>\<\!\~\?\:\+\-\*\/\&\|\^\%\"\']|"#")
LongStringStart1 = ({stringprefix}?\'\'\')
LongStringStart2 = ({stringprefix}?\"\"\")
LineTerminator = (\n)
WhiteSpace = ([ \t\f])
LineComment = ("#".*)
%state LONG_STRING_1
%state LONG_STRING_2
%%
/* Keywords */
<YYINITIAL> "and" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "as" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "assert" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "break" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "class" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "continue" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "def" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "del" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "elif" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "else" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "except" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "exec" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "finally" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "for" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "from" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "global" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "if" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "import" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "in" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "is" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "lambda" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "not" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "or" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "pass" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "print" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "raise" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "return" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "try" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "while" { addToken(Token.RESERVED_WORD); }
<YYINITIAL> "yield" { addToken(Token.RESERVED_WORD); }
/* Data types. */
<YYINITIAL> "char" { addToken(Token.DATA_TYPE); }
<YYINITIAL> "double" { addToken(Token.DATA_TYPE); }
<YYINITIAL> "float" { addToken(Token.DATA_TYPE); }
<YYINITIAL> "int" { addToken(Token.DATA_TYPE); }
<YYINITIAL> "long" { addToken(Token.DATA_TYPE); }
<YYINITIAL> "short" { addToken(Token.DATA_TYPE); }
<YYINITIAL> "signed" { addToken(Token.DATA_TYPE); }
<YYINITIAL> "unsigned" { addToken(Token.DATA_TYPE); }
<YYINITIAL> "void" { addToken(Token.DATA_TYPE); }
/* Standard functions */
<YYINITIAL> "abs" { addToken(Token.FUNCTION); }
<YYINITIAL> "apply" { addToken(Token.FUNCTION); }
<YYINITIAL> "bool" { addToken(Token.FUNCTION); }
<YYINITIAL> "buffer" { addToken(Token.FUNCTION); }
<YYINITIAL> "callable" { addToken(Token.FUNCTION); }
<YYINITIAL> "chr" { addToken(Token.FUNCTION); }
<YYINITIAL> "classmethod" { addToken(Token.FUNCTION); }
<YYINITIAL> "cmp" { addToken(Token.FUNCTION); }
<YYINITIAL> "coerce" { addToken(Token.FUNCTION); }
<YYINITIAL> "compile" { addToken(Token.FUNCTION); }
<YYINITIAL> "complex" { addToken(Token.FUNCTION); }
<YYINITIAL> "delattr" { addToken(Token.FUNCTION); }
<YYINITIAL> "dict" { addToken(Token.FUNCTION); }
<YYINITIAL> "dir" { addToken(Token.FUNCTION); }
<YYINITIAL> "divmod" { addToken(Token.FUNCTION); }
<YYINITIAL> "enumerate" { addToken(Token.FUNCTION); }
<YYINITIAL> "eval" { addToken(Token.FUNCTION); }
<YYINITIAL> "execfile" { addToken(Token.FUNCTION); }
<YYINITIAL> "file" { addToken(Token.FUNCTION); }
<YYINITIAL> "filter" { addToken(Token.FUNCTION); }
<YYINITIAL> "float" { addToken(Token.FUNCTION); }
<YYINITIAL> "getattr" { addToken(Token.FUNCTION); }
<YYINITIAL> "globals" { addToken(Token.FUNCTION); }
<YYINITIAL> "hasattr" { addToken(Token.FUNCTION); }
<YYINITIAL> "hash" { addToken(Token.FUNCTION); }
<YYINITIAL> "hex" { addToken(Token.FUNCTION); }
<YYINITIAL> "id" { addToken(Token.FUNCTION); }
<YYINITIAL> "input" { addToken(Token.FUNCTION); }
<YYINITIAL> "int" { addToken(Token.FUNCTION); }
<YYINITIAL> "intern" { addToken(Token.FUNCTION); }
<YYINITIAL> "isinstance" { addToken(Token.FUNCTION); }
<YYINITIAL> "issubclass" { addToken(Token.FUNCTION); }
<YYINITIAL> "iter" { addToken(Token.FUNCTION); }
<YYINITIAL> "len" { addToken(Token.FUNCTION); }
<YYINITIAL> "list" { addToken(Token.FUNCTION); }
<YYINITIAL> "locals" { addToken(Token.FUNCTION); }
<YYINITIAL> "long" { addToken(Token.FUNCTION); }
<YYINITIAL> "map" { addToken(Token.FUNCTION); }
<YYINITIAL> "max" { addToken(Token.FUNCTION); }
<YYINITIAL> "min" { addToken(Token.FUNCTION); }
<YYINITIAL> "object" { addToken(Token.FUNCTION); }
<YYINITIAL> "oct" { addToken(Token.FUNCTION); }
<YYINITIAL> "open" { addToken(Token.FUNCTION); }
<YYINITIAL> "ord" { addToken(Token.FUNCTION); }
<YYINITIAL> "pow" { addToken(Token.FUNCTION); }
<YYINITIAL> "property" { addToken(Token.FUNCTION); }
<YYINITIAL> "range" { addToken(Token.FUNCTION); }
<YYINITIAL> "raw_input" { addToken(Token.FUNCTION); }
<YYINITIAL> "reduce" { addToken(Token.FUNCTION); }
<YYINITIAL> "reload" { addToken(Token.FUNCTION); }
<YYINITIAL> "repr" { addToken(Token.FUNCTION); }
<YYINITIAL> "round" { addToken(Token.FUNCTION); }
<YYINITIAL> "setattr" { addToken(Token.FUNCTION); }
<YYINITIAL> "slice" { addToken(Token.FUNCTION); }
<YYINITIAL> "staticmethod" { addToken(Token.FUNCTION); }
<YYINITIAL> "str" { addToken(Token.FUNCTION); }
<YYINITIAL> "sum" { addToken(Token.FUNCTION); }
<YYINITIAL> "super" { addToken(Token.FUNCTION); }
<YYINITIAL> "tuple" { addToken(Token.FUNCTION); }
<YYINITIAL> "type" { addToken(Token.FUNCTION); }
<YYINITIAL> "unichr" { addToken(Token.FUNCTION); }
<YYINITIAL> "unicode" { addToken(Token.FUNCTION); }
<YYINITIAL> "vars" { addToken(Token.FUNCTION); }
<YYINITIAL> "xrange" { addToken(Token.FUNCTION); }
<YYINITIAL> "zip" { addToken(Token.FUNCTION); }
<YYINITIAL> {
{LineTerminator} { addNullToken(); return firstToken; }
{identifier} { addToken(Token.IDENTIFIER); }
{WhiteSpace}+ { addToken(Token.WHITESPACE); }
/* String/Character Literals. */
{stringliteral} { addToken(Token.LITERAL_STRING_DOUBLE_QUOTE); }
{LongStringStart1} { yybegin(LONG_STRING_1); addToken(Token.LITERAL_CHAR); }
{LongStringStart2} { yybegin(LONG_STRING_2); addToken(Token.LITERAL_STRING_DOUBLE_QUOTE); }
/* Comment Literals. */
{LineComment} { addToken(Token.COMMENT_EOL); }
/* Separators. */
"(" { addToken(Token.SEPARATOR); }
")" { addToken(Token.SEPARATOR); }
"[" { addToken(Token.SEPARATOR); }
"]" { addToken(Token.SEPARATOR); }
"{" { addToken(Token.SEPARATOR); }
"}" { addToken(Token.SEPARATOR); }
/* Operators. */
"=" { addToken(Token.OPERATOR); }
"+" { addToken(Token.OPERATOR); }
"-" { addToken(Token.OPERATOR); }
"*" { addToken(Token.OPERATOR); }
"/" { addToken(Token.OPERATOR); }
"%" { addToken(Token.OPERATOR); }
"**" { addToken(Token.OPERATOR); }
"~" { addToken(Token.OPERATOR); }
"<" { addToken(Token.OPERATOR); }
">" { addToken(Token.OPERATOR); }
"<<" { addToken(Token.OPERATOR); }
">>" { addToken(Token.OPERATOR); }
"==" { addToken(Token.OPERATOR); }
"+=" { addToken(Token.OPERATOR); }
"-=" { addToken(Token.OPERATOR); }
"*=" { addToken(Token.OPERATOR); }
"/=" { addToken(Token.OPERATOR); }
"%=" { addToken(Token.OPERATOR); }
">>=" { addToken(Token.OPERATOR); }
"<<=" { addToken(Token.OPERATOR); }
"^" { addToken(Token.OPERATOR); }
"&" { addToken(Token.OPERATOR); }
"&&" { addToken(Token.OPERATOR); }
"|" { addToken(Token.OPERATOR); }
"||" { addToken(Token.OPERATOR); }
"?" { addToken(Token.OPERATOR); }
":" { addToken(Token.OPERATOR); }
"," { addToken(Token.OPERATOR); }
"!" { addToken(Token.OPERATOR); }
"++" { addToken(Token.OPERATOR); }
"--" { addToken(Token.OPERATOR); }
"." { addToken(Token.OPERATOR); }
"," { addToken(Token.OPERATOR); }
/* Numbers */
{longinteger}|{integer} { addToken(Token.LITERAL_NUMBER_DECIMAL_INT); }
{floatnumber}|{imagnumber} { addToken(Token.LITERAL_NUMBER_FLOAT); }
{ErrorNumberFormat} { addToken(Token.ERROR_NUMBER_FORMAT); }
/* Other punctuation, we'll highlight it as "identifiers." */
"@" { addToken(Token.IDENTIFIER); }
";" { addToken(Token.IDENTIFIER); }
/* Ended with a line not in a string or comment. */
<<EOF>> { addNullToken(); return firstToken; }
/* Catch any other (unhandled) characters and flag them as bad. */
. { addToken(Token.ERROR_IDENTIFIER); }
}
<LONG_STRING_1> {
[^\']+ { addToken(Token.LITERAL_CHAR); }
"'''" { yybegin(YYINITIAL); addToken(Token.LITERAL_CHAR); }
"'" { addToken(Token.LITERAL_CHAR); }
<<EOF>> {
if (firstToken==null) {
addToken(Token.LITERAL_CHAR);
}
return firstToken;
}
}
<LONG_STRING_2> {
[^\"]+ { addToken(Token.LITERAL_STRING_DOUBLE_QUOTE); }
\"\"\" { yybegin(YYINITIAL); addToken(Token.LITERAL_STRING_DOUBLE_QUOTE); }
\" { addToken(Token.LITERAL_STRING_DOUBLE_QUOTE); }
<<EOF>> {
if (firstToken==null) {
addToken(Token.LITERAL_STRING_DOUBLE_QUOTE);
}
return firstToken;
}
}
⌨️ 快捷键说明
复制代码
Ctrl + C
搜索代码
Ctrl + F
全屏模式
F11
切换主题
Ctrl + Shift + D
显示快捷键
?
增大字号
Ctrl + =
减小字号
Ctrl + -