test_tokenize

来自「mallet是自然语言处理、机器学习领域的一个开源项目。」· 代码 · 共 649 行 · 第 1/2 页

115,4-115,11:	STRING	"ur'abc'"
115,12-115,13:	OP	'+'
115,14-115,21:	STRING	"Ur'ABC'"
115,22-115,23:	OP	'+'
115,24-115,31:	STRING	"uR'ABC'"
115,32-115,33:	OP	'+'
115,34-115,41:	STRING	"UR'ABC'"
115,41-115,42:	NEWLINE	'\n'
116,0-116,1:	NAME	'y'
116,2-116,3:	OP	'='
116,4-116,11:	STRING	'ur"abc"'
116,12-116,13:	OP	'+'
116,14-116,21:	STRING	'Ur"ABC"'
116,22-116,23:	OP	'+'
116,24-116,31:	STRING	'uR"ABC"'
116,32-116,33:	OP	'+'
116,34-116,41:	STRING	'UR"ABC"'
116,41-116,42:	NEWLINE	'\n'
117,0-117,1:	NAME	'x'
117,2-117,3:	OP	'='
117,4-117,10:	STRING	"ur'\\\\'"
117,11-117,12:	OP	'+'
117,13-117,19:	STRING	"UR'\\\\'"
117,19-117,20:	NEWLINE	'\n'
118,0-118,1:	NAME	'x'
118,2-118,3:	OP	'='
118,4-118,10:	STRING	"ur'\\''"
118,11-118,12:	OP	'+'
118,13-118,15:	STRING	"''"
118,15-118,16:	NEWLINE	'\n'
119,0-119,1:	NAME	'y'
119,2-119,3:	OP	'='
119,4-121,6:	STRING	"ur'''\nfoo bar \\\\\nbaz'''"
121,7-121,8:	OP	'+'
121,9-122,6:	STRING	"UR'''\nfoo'''"
122,6-122,7:	NEWLINE	'\n'
123,0-123,1:	NAME	'y'
123,2-123,3:	OP	'='
123,4-125,3:	STRING	'Ur"""foo\nbar \\\\ baz\n"""'
125,4-125,5:	OP	'+'
125,6-126,3:	STRING	"uR'''spam\n'''"
126,3-126,4:	NEWLINE	'\n'
127,0-127,1:	NL	'\n'
128,0-128,14:	COMMENT	'# Indentation\n'
129,0-129,2:	NAME	'if'
129,3-129,4:	NUMBER	'1'
129,4-129,5:	OP	':'
129,5-129,6:	NEWLINE	'\n'
130,0-130,4:	INDENT	'    '
130,4-130,5:	NAME	'x'
130,6-130,7:	OP	'='
130,8-130,9:	NUMBER	'2'
130,9-130,10:	NEWLINE	'\n'
131,0-131,0:	DEDENT	''
131,0-131,2:	NAME	'if'
131,3-131,4:	NUMBER	'1'
131,4-131,5:	OP	':'
131,5-131,6:	NEWLINE	'\n'
132,0-132,8:	INDENT	'        '
132,8-132,9:	NAME	'x'
132,10-132,11:	OP	'='
132,12-132,13:	NUMBER	'2'
132,13-132,14:	NEWLINE	'\n'
133,0-133,0:	DEDENT	''
133,0-133,2:	NAME	'if'
133,3-133,4:	NUMBER	'1'
133,4-133,5:	OP	':'
133,5-133,6:	NEWLINE	'\n'
134,0-134,4:	INDENT	'    '
134,4-134,9:	NAME	'while'
134,10-134,11:	NUMBER	'0'
134,11-134,12:	OP	':'
134,12-134,13:	NEWLINE	'\n'
135,0-135,5:	INDENT	'     '
135,5-135,7:	NAME	'if'
135,8-135,9:	NUMBER	'0'
135,9-135,10:	OP	':'
135,10-135,11:	NEWLINE	'\n'
136,0-136,11:	INDENT	'           '
136,11-136,12:	NAME	'x'
136,13-136,14:	OP	'='
136,15-136,16:	NUMBER	'2'
136,16-136,17:	NEWLINE	'\n'
137,5-137,5:	DEDENT	''
137,5-137,6:	NAME	'x'
137,7-137,8:	OP	'='
137,9-137,10:	NUMBER	'2'
137,10-137,11:	NEWLINE	'\n'
138,0-138,0:	DEDENT	''
138,0-138,0:	DEDENT	''
138,0-138,2:	NAME	'if'
138,3-138,4:	NUMBER	'0'
138,4-138,5:	OP	':'
138,5-138,6:	NEWLINE	'\n'
139,0-139,2:	INDENT	'  '
139,2-139,4:	NAME	'if'
139,5-139,6:	NUMBER	'2'
139,6-139,7:	OP	':'
139,7-139,8:	NEWLINE	'\n'
140,0-140,3:	INDENT	'   '
140,3-140,8:	NAME	'while'
140,9-140,10:	NUMBER	'0'
140,10-140,11:	OP	':'
140,11-140,12:	NEWLINE	'\n'
141,0-141,8:	INDENT	'        '
141,8-141,10:	NAME	'if'
141,11-141,12:	NUMBER	'1'
141,12-141,13:	OP	':'
141,13-141,14:	NEWLINE	'\n'
142,0-142,10:	INDENT	'          '
142,10-142,11:	NAME	'x'
142,12-142,13:	OP	'='
142,14-142,15:	NUMBER	'2'
142,15-142,16:	NEWLINE	'\n'
143,0-143,1:	NL	'\n'
144,0-144,12:	COMMENT	'# Operators\n'
145,0-145,1:	NL	'\n'
146,0-146,0:	DEDENT	''
146,0-146,0:	DEDENT	''
146,0-146,0:	DEDENT	''
146,0-146,0:	DEDENT	''
146,0-146,3:	NAME	'def'
146,4-146,7:	NAME	'd22'
146,7-146,8:	OP	'('
146,8-146,9:	NAME	'a'
146,9-146,10:	OP	','
146,11-146,12:	NAME	'b'
146,12-146,13:	OP	','
146,14-146,15:	NAME	'c'
146,15-146,16:	OP	'='
146,16-146,17:	NUMBER	'1'
146,17-146,18:	OP	','
146,19-146,20:	NAME	'd'
146,20-146,21:	OP	'='
146,21-146,22:	NUMBER	'2'
146,22-146,23:	OP	')'
146,23-146,24:	OP	':'
146,25-146,29:	NAME	'pass'
146,29-146,30:	NEWLINE	'\n'
147,0-147,3:	NAME	'def'
147,4-147,8:	NAME	'd01v'
147,8-147,9:	OP	'('
147,9-147,10:	NAME	'a'
147,10-147,11:	OP	'='
147,11-147,12:	NUMBER	'1'
147,12-147,13:	OP	','
147,14-147,15:	OP	'*'
147,15-147,20:	NAME	'restt'
147,20-147,21:	OP	','
147,22-147,24:	OP	'**'
147,24-147,29:	NAME	'restd'
147,29-147,30:	OP	')'
147,30-147,31:	OP	':'
147,32-147,36:	NAME	'pass'
147,36-147,37:	NEWLINE	'\n'
148,0-148,1:	NL	'\n'
149,0-149,1:	OP	'('
149,1-149,2:	NAME	'x'
149,2-149,3:	OP	','
149,4-149,5:	NAME	'y'
149,5-149,6:	OP	')'
149,7-149,9:	OP	'<>'
149,10-149,11:	OP	'('
149,11-149,12:	OP	'{'
149,12-149,15:	STRING	"'a'"
149,15-149,16:	OP	':'
149,16-149,17:	NUMBER	'1'
149,17-149,18:	OP	'}'
149,18-149,19:	OP	','
149,20-149,21:	OP	'{'
149,21-149,24:	STRING	"'b'"
149,24-149,25:	OP	':'
149,25-149,26:	NUMBER	'2'
149,26-149,27:	OP	'}'
149,27-149,28:	OP	')'
149,28-149,29:	NEWLINE	'\n'
150,0-150,1:	NL	'\n'
151,0-151,13:	COMMENT	'# comparison\n'
152,0-152,2:	NAME	'if'
152,3-152,4:	NUMBER	'1'
152,5-152,6:	OP	'<'
152,7-152,8:	NUMBER	'1'
152,9-152,10:	OP	'>'
152,11-152,12:	NUMBER	'1'
152,13-152,15:	OP	'=='
152,16-152,17:	NUMBER	'1'
152,18-152,20:	OP	'>='
152,21-152,22:	NUMBER	'1'
152,23-152,25:	OP	'<='
152,26-152,27:	NUMBER	'1'
152,28-152,30:	OP	'<>'
152,31-152,32:	NUMBER	'1'
152,33-152,35:	OP	'!='
152,36-152,37:	NUMBER	'1'
152,38-152,40:	NAME	'in'
152,41-152,42:	NUMBER	'1'
152,43-152,46:	NAME	'not'
152,47-152,49:	NAME	'in'
152,50-152,51:	NUMBER	'1'
152,52-152,54:	NAME	'is'
152,55-152,56:	NUMBER	'1'
152,57-152,59:	NAME	'is'
152,60-152,63:	NAME	'not'
152,64-152,65:	NUMBER	'1'
152,65-152,66:	OP	':'
152,67-152,71:	NAME	'pass'
152,71-152,72:	NEWLINE	'\n'
153,0-153,1:	NL	'\n'
154,0-154,9:	COMMENT	'# binary\n'
155,0-155,1:	NAME	'x'
155,2-155,3:	OP	'='
155,4-155,5:	NUMBER	'1'
155,6-155,7:	OP	'&'
155,8-155,9:	NUMBER	'1'
155,9-155,10:	NEWLINE	'\n'
156,0-156,1:	NAME	'x'
156,2-156,3:	OP	'='
156,4-156,5:	NUMBER	'1'
156,6-156,7:	OP	'^'
156,8-156,9:	NUMBER	'1'
156,9-156,10:	NEWLINE	'\n'
157,0-157,1:	NAME	'x'
157,2-157,3:	OP	'='
157,4-157,5:	NUMBER	'1'
157,6-157,7:	OP	'|'
157,8-157,9:	NUMBER	'1'
157,9-157,10:	NEWLINE	'\n'
158,0-158,1:	NL	'\n'
159,0-159,8:	COMMENT	'# shift\n'
160,0-160,1:	NAME	'x'
160,2-160,3:	OP	'='
160,4-160,5:	NUMBER	'1'
160,6-160,8:	OP	'<<'
160,9-160,10:	NUMBER	'1'
160,11-160,13:	OP	'>>'
160,14-160,15:	NUMBER	'1'
160,15-160,16:	NEWLINE	'\n'
161,0-161,1:	NL	'\n'
162,0-162,11:	COMMENT	'# additive\n'
163,0-163,1:	NAME	'x'
163,2-163,3:	OP	'='
163,4-163,5:	NUMBER	'1'
163,6-163,7:	OP	'-'
163,8-163,9:	NUMBER	'1'
163,10-163,11:	OP	'+'
163,12-163,13:	NUMBER	'1'
163,14-163,15:	OP	'-'
163,16-163,17:	NUMBER	'1'
163,18-163,19:	OP	'+'
163,20-163,21:	NUMBER	'1'
163,21-163,22:	NEWLINE	'\n'
164,0-164,1:	NL	'\n'
165,0-165,17:	COMMENT	'# multiplicative\n'
166,0-166,1:	NAME	'x'
166,2-166,3:	OP	'='
166,4-166,5:	NUMBER	'1'
166,6-166,7:	OP	'/'
166,8-166,9:	NUMBER	'1'
166,10-166,11:	OP	'*'
166,12-166,13:	NUMBER	'1'
166,14-166,15:	OP	'%'
166,16-166,17:	NUMBER	'1'
166,17-166,18:	NEWLINE	'\n'
167,0-167,1:	NL	'\n'
168,0-168,8:	COMMENT	'# unary\n'
169,0-169,1:	NAME	'x'
169,2-169,3:	OP	'='
169,4-169,5:	OP	'~'
169,5-169,6:	NUMBER	'1'
169,7-169,8:	OP	'^'
169,9-169,10:	NUMBER	'1'
169,11-169,12:	OP	'&'
169,13-169,14:	NUMBER	'1'
169,15-169,16:	OP	'|'
169,17-169,18:	NUMBER	'1'
169,19-169,20:	OP	'&'
169,21-169,22:	NUMBER	'1'
169,23-169,24:	OP	'^'
169,25-169,26:	OP	'-'
169,26-169,27:	NUMBER	'1'
169,27-169,28:	NEWLINE	'\n'
170,0-170,1:	NAME	'x'
170,2-170,3:	OP	'='
170,4-170,5:	OP	'-'
170,5-170,6:	NUMBER	'1'
170,6-170,7:	OP	'*'
170,7-170,8:	NUMBER	'1'
170,8-170,9:	OP	'/'
170,9-170,10:	NUMBER	'1'
170,11-170,12:	OP	'+'
170,13-170,14:	NUMBER	'1'
170,14-170,15:	OP	'*'
170,15-170,16:	NUMBER	'1'
170,17-170,18:	OP	'-'
170,19-170,20:	OP	'-'
170,20-170,21:	OP	'-'
170,21-170,22:	OP	'-'
170,22-170,23:	NUMBER	'1'
170,23-170,24:	OP	'*'
170,24-170,25:	NUMBER	'1'
170,25-170,26:	NEWLINE	'\n'
171,0-171,1:	NL	'\n'
172,0-172,11:	COMMENT	'# selector\n'
173,0-173,6:	NAME	'import'
173,7-173,10:	NAME	'sys'
173,10-173,11:	OP	','
173,12-173,16:	NAME	'time'
173,16-173,17:	NEWLINE	'\n'
174,0-174,1:	NAME	'x'
174,2-174,3:	OP	'='
174,4-174,7:	NAME	'sys'
174,7-174,8:	OP	'.'
174,8-174,15:	NAME	'modules'
174,15-174,16:	OP	'['
174,16-174,22:	STRING	"'time'"
174,22-174,23:	OP	']'
174,23-174,24:	OP	'.'
174,24-174,28:	NAME	'time'
174,28-174,29:	OP	'('
174,29-174,30:	OP	')'
174,30-174,31:	NEWLINE	'\n'
175,0-175,1:	NL	'\n'
176,0-176,0:	ENDMARKER	''