代码搜索：Tokenize
找到约 228 项符合「Tokenize」的源代码
代码结果 228
www.eeworm.com/read/438718/1823112
py tokenize.py
# Tests for the 'tokenize' module.
# Large bits stolen from test_grammar.py.
# Comments
"#"
#'
#"
#\
#
# abc
'''#
#'''
x = 1 #
# Balancing continuation
a = (3, 4,
5, 6)
y = [3, 4,
www.eeworm.com/read/438717/1823231
test_tokenize
test_tokenize
1,0-1,35: COMMENT "# Tests for the 'tokenize' module.\012"
2,0-2,43: COMMENT '# Large bits stolen from test_grammar.py. \012'
3,0-3,1: NL '\012'
4,0-4,11: COMMENT '# Comments\012'
5,0-5,
www.eeworm.com/read/437033/1836834
c tokenize.c
/* tokenize.c - split input into tokens Author: Kees J. Bot
* 13 Dec 1993
*/
#define nil 0
#include
#include
#include
#include
#include
www.eeworm.com/read/435707/1861820
c tokenize.c
#include "u.h"
#include "lib.h"
static char qsep[] = " \t\r\n";
static char*
qtoken(char *s, char *sep)
{
int quoting;
char *t;
quoting = 0;
t = s; /* s is output string, t is input string */
www.eeworm.com/read/435701/1862045
c tokenize.c
#include
#include
int
getfields(char *str, char **args, int max, int mflag, char *set)
{
Rune r;
int nr, intok, narg;
if(max
www.eeworm.com/read/422060/2045588
c tokenize.c
/*
** 2001 September 15
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiven
www.eeworm.com/read/422060/2045872
c tokenize.c
/*
** 2001 September 15
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiven
www.eeworm.com/read/409921/2223587
h tokenize.h
// tokenize.h
#ifndef TOKENIZE_H
#define TOKENIZE_H
void tokenize ( const std::string& text, std::vector& tokens );
#endif//TOKENIZE_H
www.eeworm.com/read/409921/2223597
cpp tokenize.cpp
// tokenize.cpp
#ifdef _MSC_VER
#pragma warning ( disable : 4786 )
#endif//_MSC_VER
#include
#include
#include
#include "assert.h"
#include "tokenize.h"
#inc
www.eeworm.com/read/409921/2226196
c tokenize.c
/*
** 2001 September 15
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find f