test_tokenize

来自「mallet是自然语言处理、机器学习领域的一个开源项目。」· 代码 · 共 649 行 · 第 1/2 页

test_tokenize
1,0-1,35:	COMMENT	"# Tests for the 'tokenize' module.\n"
2,0-2,43:	COMMENT	'# Large bits stolen from test_grammar.py. \n'
3,0-3,1:	NL	'\n'
4,0-4,11:	COMMENT	'# Comments\n'
5,0-5,3:	STRING	'"#"'
5,3-5,4:	NEWLINE	'\n'
6,0-6,3:	COMMENT	"#'\n"
7,0-7,3:	COMMENT	'#"\n'
8,0-8,3:	COMMENT	'#\\\n'
9,7-9,9:	COMMENT	'#\n'
10,4-10,10:	COMMENT	'# abc\n'
11,0-12,4:	STRING	"'''#\n#'''"
12,4-12,5:	NEWLINE	'\n'
13,0-13,1:	NL	'\n'
14,0-14,1:	NAME	'x'
14,2-14,3:	OP	'='
14,4-14,5:	NUMBER	'1'
14,7-14,8:	COMMENT	'#'
14,8-14,9:	NEWLINE	'\n'
15,0-15,1:	NL	'\n'
16,0-16,25:	COMMENT	'# Balancing continuation\n'
17,0-17,1:	NL	'\n'
18,0-18,1:	NAME	'a'
18,2-18,3:	OP	'='
18,4-18,5:	OP	'('
18,5-18,6:	NUMBER	'3'
18,6-18,7:	OP	','
18,8-18,9:	NUMBER	'4'
18,9-18,10:	OP	','
18,10-18,11:	NL	'\n'
19,2-19,3:	NUMBER	'5'
19,3-19,4:	OP	','
19,5-19,6:	NUMBER	'6'
19,6-19,7:	OP	')'
19,7-19,8:	NEWLINE	'\n'
20,0-20,1:	NAME	'y'
20,2-20,3:	OP	'='
20,4-20,5:	OP	'['
20,5-20,6:	NUMBER	'3'
20,6-20,7:	OP	','
20,8-20,9:	NUMBER	'4'
20,9-20,10:	OP	','
20,10-20,11:	NL	'\n'
21,2-21,3:	NUMBER	'5'
21,3-21,4:	OP	']'
21,4-21,5:	NEWLINE	'\n'
22,0-22,1:	NAME	'z'
22,2-22,3:	OP	'='
22,4-22,5:	OP	'{'
22,5-22,8:	STRING	"'a'"
22,8-22,9:	OP	':'
22,9-22,10:	NUMBER	'5'
22,10-22,11:	OP	','
22,11-22,12:	NL	'\n'
23,2-23,5:	STRING	"'b'"
23,5-23,6:	OP	':'
23,6-23,7:	NUMBER	'6'
23,7-23,8:	OP	'}'
23,8-23,9:	NEWLINE	'\n'
24,0-24,1:	NAME	'x'
24,2-24,3:	OP	'='
24,4-24,5:	OP	'('
24,5-24,8:	NAME	'len'
24,8-24,9:	OP	'('
24,9-24,10:	OP	'`'
24,10-24,11:	NAME	'y'
24,11-24,12:	OP	'`'
24,12-24,13:	OP	')'
24,14-24,15:	OP	'+'
24,16-24,17:	NUMBER	'5'
24,17-24,18:	OP	'*'
24,18-24,19:	NAME	'x'
24,20-24,21:	OP	'-'
24,22-24,23:	NAME	'a'
24,23-24,24:	OP	'['
24,24-24,25:	NL	'\n'
25,3-25,4:	NUMBER	'3'
25,5-25,6:	OP	']'
25,6-25,7:	NL	'\n'
26,3-26,4:	OP	'-'
26,5-26,6:	NAME	'x'
26,7-26,8:	OP	'+'
26,9-26,12:	NAME	'len'
26,12-26,13:	OP	'('
26,13-26,14:	OP	'{'
26,14-26,15:	NL	'\n'
27,3-27,4:	OP	'}'
27,4-27,5:	NL	'\n'
28,4-28,5:	OP	')'
28,5-28,6:	NL	'\n'
29,2-29,3:	OP	')'
29,3-29,4:	NEWLINE	'\n'
30,0-30,1:	NL	'\n'
31,0-31,37:	COMMENT	'# Backslash means line continuation:\n'
32,0-32,1:	NAME	'x'
32,2-32,3:	OP	'='
32,4-32,5:	NUMBER	'1'
33,0-33,1:	OP	'+'
33,2-33,3:	NUMBER	'1'
33,3-33,4:	NEWLINE	'\n'
34,0-34,1:	NL	'\n'
35,0-35,55:	COMMENT	'# Backslash does not means continuation in comments :\\\n'
36,0-36,1:	NAME	'x'
36,2-36,3:	OP	'='
36,4-36,5:	NUMBER	'0'
36,5-36,6:	NEWLINE	'\n'
37,0-37,1:	NL	'\n'
38,0-38,20:	COMMENT	'# Ordinary integers\n'
39,0-39,4:	NUMBER	'0xff'
39,5-39,7:	OP	'<>'
39,8-39,11:	NUMBER	'255'
39,11-39,12:	NEWLINE	'\n'
40,0-40,4:	NUMBER	'0377'
40,5-40,7:	OP	'<>'
40,8-40,11:	NUMBER	'255'
40,11-40,12:	NEWLINE	'\n'
41,0-41,10:	NUMBER	'2147483647'
41,13-41,15:	OP	'!='
41,16-41,28:	NUMBER	'017777777777'
41,28-41,29:	NEWLINE	'\n'
42,0-42,1:	OP	'-'
42,1-42,11:	NUMBER	'2147483647'
42,11-42,12:	OP	'-'
42,12-42,13:	NUMBER	'1'
42,14-42,16:	OP	'!='
42,17-42,29:	NUMBER	'020000000000'
42,29-42,30:	NEWLINE	'\n'
43,0-43,12:	NUMBER	'037777777777'
43,13-43,15:	OP	'!='
43,16-43,17:	OP	'-'
43,17-43,18:	NUMBER	'1'
43,18-43,19:	NEWLINE	'\n'
44,0-44,10:	NUMBER	'0xffffffff'
44,11-44,13:	OP	'!='
44,14-44,15:	OP	'-'
44,15-44,16:	NUMBER	'1'
44,16-44,17:	NEWLINE	'\n'
45,0-45,1:	NL	'\n'
46,0-46,16:	COMMENT	'# Long integers\n'
47,0-47,1:	NAME	'x'
47,2-47,3:	OP	'='
47,4-47,6:	NUMBER	'0L'
47,6-47,7:	NEWLINE	'\n'
48,0-48,1:	NAME	'x'
48,2-48,3:	OP	'='
48,4-48,6:	NUMBER	'0l'
48,6-48,7:	NEWLINE	'\n'
49,0-49,1:	NAME	'x'
49,2-49,3:	OP	'='
49,4-49,23:	NUMBER	'0xffffffffffffffffL'
49,23-49,24:	NEWLINE	'\n'
50,0-50,1:	NAME	'x'
50,2-50,3:	OP	'='
50,4-50,23:	NUMBER	'0xffffffffffffffffl'
50,23-50,24:	NEWLINE	'\n'
51,0-51,1:	NAME	'x'
51,2-51,3:	OP	'='
51,4-51,23:	NUMBER	'077777777777777777L'
51,23-51,24:	NEWLINE	'\n'
52,0-52,1:	NAME	'x'
52,2-52,3:	OP	'='
52,4-52,23:	NUMBER	'077777777777777777l'
52,23-52,24:	NEWLINE	'\n'
53,0-53,1:	NAME	'x'
53,2-53,3:	OP	'='
53,4-53,35:	NUMBER	'123456789012345678901234567890L'
53,35-53,36:	NEWLINE	'\n'
54,0-54,1:	NAME	'x'
54,2-54,3:	OP	'='
54,4-54,35:	NUMBER	'123456789012345678901234567890l'
54,35-54,36:	NEWLINE	'\n'
55,0-55,1:	NL	'\n'
56,0-56,25:	COMMENT	'# Floating-point numbers\n'
57,0-57,1:	NAME	'x'
57,2-57,3:	OP	'='
57,4-57,8:	NUMBER	'3.14'
57,8-57,9:	NEWLINE	'\n'
58,0-58,1:	NAME	'x'
58,2-58,3:	OP	'='
58,4-58,8:	NUMBER	'314.'
58,8-58,9:	NEWLINE	'\n'
59,0-59,1:	NAME	'x'
59,2-59,3:	OP	'='
59,4-59,9:	NUMBER	'0.314'
59,9-59,10:	NEWLINE	'\n'
60,0-60,18:	COMMENT	'# XXX x = 000.314\n'
61,0-61,1:	NAME	'x'
61,2-61,3:	OP	'='
61,4-61,8:	NUMBER	'.314'
61,8-61,9:	NEWLINE	'\n'
62,0-62,1:	NAME	'x'
62,2-62,3:	OP	'='
62,4-62,8:	NUMBER	'3e14'
62,8-62,9:	NEWLINE	'\n'
63,0-63,1:	NAME	'x'
63,2-63,3:	OP	'='
63,4-63,8:	NUMBER	'3E14'
63,8-63,9:	NEWLINE	'\n'
64,0-64,1:	NAME	'x'
64,2-64,3:	OP	'='
64,4-64,9:	NUMBER	'3e-14'
64,9-64,10:	NEWLINE	'\n'
65,0-65,1:	NAME	'x'
65,2-65,3:	OP	'='
65,4-65,9:	NUMBER	'3e+14'
65,9-65,10:	NEWLINE	'\n'
66,0-66,1:	NAME	'x'
66,2-66,3:	OP	'='
66,4-66,9:	NUMBER	'3.e14'
66,9-66,10:	NEWLINE	'\n'
67,0-67,1:	NAME	'x'
67,2-67,3:	OP	'='
67,4-67,9:	NUMBER	'.3e14'
67,9-67,10:	NEWLINE	'\n'
68,0-68,1:	NAME	'x'
68,2-68,3:	OP	'='
68,4-68,9:	NUMBER	'3.1e4'
68,9-68,10:	NEWLINE	'\n'
69,0-69,1:	NL	'\n'
70,0-70,18:	COMMENT	'# String literals\n'
71,0-71,1:	NAME	'x'
71,2-71,3:	OP	'='
71,4-71,6:	STRING	"''"
71,6-71,7:	OP	';'
71,8-71,9:	NAME	'y'
71,10-71,11:	OP	'='
71,12-71,14:	STRING	'""'
71,14-71,15:	OP	';'
71,15-71,16:	NEWLINE	'\n'
72,0-72,1:	NAME	'x'
72,2-72,3:	OP	'='
72,4-72,8:	STRING	"'\\''"
72,8-72,9:	OP	';'
72,10-72,11:	NAME	'y'
72,12-72,13:	OP	'='
72,14-72,17:	STRING	'"\'"'
72,17-72,18:	OP	';'
72,18-72,19:	NEWLINE	'\n'
73,0-73,1:	NAME	'x'
73,2-73,3:	OP	'='
73,4-73,7:	STRING	'\'"\''
73,7-73,8:	OP	';'
73,9-73,10:	NAME	'y'
73,11-73,12:	OP	'='
73,13-73,17:	STRING	'"\\""'
73,17-73,18:	OP	';'
73,18-73,19:	NEWLINE	'\n'
74,0-74,1:	NAME	'x'
74,2-74,3:	OP	'='
74,4-74,32:	STRING	'"doesn\'t \\"shrink\\" does it"'
74,32-74,33:	NEWLINE	'\n'
75,0-75,1:	NAME	'y'
75,2-75,3:	OP	'='
75,4-75,31:	STRING	'\'doesn\\\'t "shrink" does it\''
75,31-75,32:	NEWLINE	'\n'
76,0-76,1:	NAME	'x'
76,2-76,3:	OP	'='
76,4-76,32:	STRING	'"does \\"shrink\\" doesn\'t it"'
76,32-76,33:	NEWLINE	'\n'
77,0-77,1:	NAME	'y'
77,2-77,3:	OP	'='
77,4-77,31:	STRING	'\'does "shrink" doesn\\\'t it\''
77,31-77,32:	NEWLINE	'\n'
78,0-78,1:	NAME	'x'
78,2-78,3:	OP	'='
78,4-83,3:	STRING	'"""\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n"""'
83,3-83,4:	NEWLINE	'\n'
84,0-84,1:	NAME	'y'
84,2-84,3:	OP	'='
84,4-84,63:	STRING	'\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\''
84,63-84,64:	NEWLINE	'\n'
85,0-85,1:	NAME	'y'
85,2-85,3:	OP	'='
85,4-90,3:	STRING	'\'\'\'\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n\'\'\''
90,3-90,4:	OP	';'
90,4-90,5:	NEWLINE	'\n'
91,0-91,1:	NAME	'y'
91,2-91,3:	OP	'='
91,4-96,1:	STRING	'"\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \'lazy\' dog.\\n\\\n"'
96,1-96,2:	OP	';'
96,2-96,3:	NEWLINE	'\n'
97,0-97,1:	NAME	'y'
97,2-97,3:	OP	'='
97,4-102,1:	STRING	'\'\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \\\'lazy\\\' dog.\\n\\\n\''
102,1-102,2:	OP	';'
102,2-102,3:	NEWLINE	'\n'
103,0-103,1:	NAME	'x'
103,2-103,3:	OP	'='
103,4-103,9:	STRING	"r'\\\\'"
103,10-103,11:	OP	'+'
103,12-103,17:	STRING	"R'\\\\'"
103,17-103,18:	NEWLINE	'\n'
104,0-104,1:	NAME	'x'
104,2-104,3:	OP	'='
104,4-104,9:	STRING	"r'\\''"
104,10-104,11:	OP	'+'
104,12-104,14:	STRING	"''"
104,14-104,15:	NEWLINE	'\n'
105,0-105,1:	NAME	'y'
105,2-105,3:	OP	'='
105,4-107,6:	STRING	"r'''\nfoo bar \\\\\nbaz'''"
107,7-107,8:	OP	'+'
107,9-108,6:	STRING	"R'''\nfoo'''"
108,6-108,7:	NEWLINE	'\n'
109,0-109,1:	NAME	'y'
109,2-109,3:	OP	'='
109,4-111,3:	STRING	'r"""foo\nbar \\\\ baz\n"""'
111,4-111,5:	OP	'+'
111,6-112,3:	STRING	"R'''spam\n'''"
112,3-112,4:	NEWLINE	'\n'
113,0-113,1:	NAME	'x'
113,2-113,3:	OP	'='
113,4-113,10:	STRING	"u'abc'"
113,11-113,12:	OP	'+'
113,13-113,19:	STRING	"U'ABC'"
113,19-113,20:	NEWLINE	'\n'
114,0-114,1:	NAME	'y'
114,2-114,3:	OP	'='
114,4-114,10:	STRING	'u"abc"'
114,11-114,12:	OP	'+'
114,13-114,19:	STRING	'U"ABC"'
114,19-114,20:	NEWLINE	'\n'
115,0-115,1:	NAME	'x'
115,2-115,3:	OP	'='