📄 tokenize.htm
字号:
<HTML>
<HEAD>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<TITLE>Parsing a String into Tokens Using a Regular Expression (Java Developers Almanac Example)
</TITLE>
<META CONTENT="Patrick Chan" NAME="AUTHOR">
<META CONTENT="Code Examples from The Java Developers Almanac 1.4" NAME="DESCRIPTION">
<META CONTENT="Addison-Wesley/Patrick Chan" NAME="OWNER">
<META CONTENT="3/20/02" NAME="revision">
<META CONTENT="no-cache" HTTP-EQUIV="Pragma">
<LINK href="/almanac.css" media="screen" type="text/css" rel="stylesheet">
</HEAD>
<BODY>
<TABLE CELLSPACING="0" CELLPADDING="0" BORDER="0">
<TR>
<TD></TD>
</TR>
</TABLE>
<br>
<TABLE CELLSPACING="0" CELLPADDING="0" BORDER="0">
<TR>
<TD></TD>
</TR>
<TR>
<TD rowspan="3"><A HREF="/?l=ex"><IMG BORDER="0" ALIGN="BOTTOM" HSPACE="10" SRC="/egs/almanac14a.jpg"></A></TD><TD VALIGN="top">
<h1>The Java Developers Almanac 1.4</h1>
<br>
Order this book from <a href="/cgi-bin/scripts/redirect.pl?l=ex&url=http://www.amazon.com/exec/obidos/ASIN/0201752808/xeo">Amazon</a>.
</TD>
</TR>
<TR>
<TD align="right" valign="bottom">
<FORM method="get" action="/cgi-bin/search/find.pl">
<INPUT size="25" name="words" type="text"><INPUT value="Search" type="submit">
</FORM>
</TD>
</TR>
</TABLE>
<HR color="#6666cc">
<TABLE CELLSPACING="0" CELLPADDING="0" BORDER="0">
<TR>
<TD valign="top"><script type="text/javascript">
<!--
google_ad_client = "pub-6001183370374757";
google_ad_width = 120;
google_ad_height = 600;
google_ad_format = "120x600_as";
google_ad_channel = "4777242811";
google_ad_type = "text_image";
google_color_border = "FFFFFF";
google_color_bg = "FFFFFF";
google_color_link = "6666CC";
google_color_url = "6666CC";
google_color_text = "000000";
//--></script><script src="http://pagead2.googlesyndication.com/pagead/show_ads.js" type="text/javascript"></script></TD><TD> </TD><TD valign="top">
<DIV ALIGN="LEFT">
<A HREF="/">Home</A>
>
<A HREF="../index.html">List of Packages</A>
>
<A HREF="../java.util.regex/pkg.html">java.util.regex</A><font color="#666666" class="xsmall-font">
[26 examples]
</font>
>
<B><A HREF="../java.util.regex/pkg.html#Tokenizing">Tokenizing</A></B><font color="#666666" class="xsmall-font">
[2 examples]
</font>
</DIV><P>
<h3>e432. Parsing a String into Tokens Using a Regular Expression</h3>
This example implements a tokenizer that uses regular expressions.
The use of this tokenizer is similar to the <code>StringTokenizer</code> class in
that you use it like an iterator to extract the tokens.
<pre> CharSequence inputStr = <font color="#0066ff"><i>"a 1 2 b c 3 4"</i></font>;
String patternStr = <font color="#0066ff"><i>"[a-z]"</i></font>;
// Set to false if only the tokens that match the pattern are to be returned.
// If true, the text between matching tokens are also returned.
boolean returnDelims = <font color="#0066ff"><i>true</i></font>;
// Create the tokenizer
Iterator tokenizer = new RETokenizer(inputStr, patternStr, returnDelims);
// Get the tokens (and delimiters)
for (; tokenizer.hasNext(); ) {
String tokenOrDelim = (String)tokenizer.next();
}
// "", "a", " 1 2 ", "b", " ", "c"
class RETokenizer implements Iterator {
// Holds the original input to search for tokens
private CharSequence input;
// Used to find tokens
private Matcher matcher;
// If true, the String between tokens are returned
private boolean returnDelims;
// The current delimiter value. If non-null, should be returned
// at the next call to next()
private String delim;
// The current matched value. If non-null and delim=null,
// should be returned at the next call to next()
private String match;
// The value of matcher.end() from the last successful match.
private int lastEnd = 0;
// patternStr is a regular expression pattern that identifies tokens.
// If returnDelims delim is false, only those tokens that match the
// pattern are returned. If returnDelims true, the text between
// matching tokens are also returned. If returnDelims is true, the
// tokens are returned in the following sequence - delimiter, token,
// delimiter, token, etc. Tokens can never be empty but delimiters might
// be empty (empty string).
public RETokenizer(CharSequence input, String patternStr, boolean returnDelims) {
// Save values
this.input = input;
this.returnDelims = returnDelims;
// Compile pattern and prepare input
Pattern pattern = Pattern.compile(patternStr);
matcher = pattern.matcher(input);
}
// Returns true if there are more tokens or delimiters.
public boolean hasNext() {
if (matcher == null) {
return false;
}
if (delim != null || match != null) {
return true;
}
if (matcher.find()) {
if (returnDelims) {
delim = input.subSequence(lastEnd, matcher.start()).toString();
}
match = matcher.group();
lastEnd = matcher.end();
} else if (returnDelims && lastEnd < input.length()) {
delim = input.subSequence(lastEnd, input.length()).toString();
lastEnd = input.length();
// Need to remove the matcher since it appears to automatically
// reset itself once it reaches the end.
matcher = null;
}
return delim != null || match != null;
}
// Returns the next token (or delimiter if returnDelims is true).
public Object next() {
String result = null;
if (delim != null) {
result = delim;
delim = null;
} else if (match != null) {
result = match;
match = null;
}
return result;
}
// Returns true if the call to next() will return a token rather
// than a delimiter.
public boolean isNextToken() {
return delim == null && match != null;
}
// Not supported.
public void remove() {
throw new UnsupportedOperationException();
}
}
</pre>
<P><table width="600" CELLSPACING="0" CELLPADDING="2" BORDER="0">
<tr>
<td bgcolor="#6666cc" align="center"><font color="#ffffff">
<b>Related Examples</b></font></td>
</tr>
</table>
e431. <a class="eglink" href="ParseLine.html?l=rel">
Parsing Character-Separated Data with a Regular Expression
</a>
<br>
<table width="600" CELLSPACING="0" CELLPADDING="2" BORDER="0">
<tr>
<td align="left">
<br>
See also:
<a class="eglink" href="/egs/java.util.regex/pkg.html?l=rel#Flags">
Flags
</a>
<a class="eglink" href="/egs/java.util.regex/pkg.html?l=rel#Groups">
Groups
</a>
<a class="eglink" href="/egs/java.util.regex/pkg.html?l=rel#Lines">
Lines
</a>
<a class="eglink" href="/egs/java.util.regex/pkg.html?l=rel#Paragraphs">
Paragraphs
</a>
<a class="eglink" href="/egs/java.util.regex/pkg.html?l=rel#Searching%20and%20Replacing">
Searching and Replacing
</a>
</td>
</tr>
</table>
<br>
<br>
<FONT class="xsmall-font">
© 2002 Addison-Wesley.
</FONT></TD><TD> </TD><TD valign="top"><A href="http://compositesw.com/devzone?ref=javaalmanac"><IMG alt="Click Here" height="600" width="120" border="0" src="/csw_oad_120x600_final.gif"></A></TD>
</TR>
</TABLE>
</BODY>
<HEAD>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<META CONTENT="NO-CACHE" HTTP-EQUIV="PRAGMA">
</HEAD>
</HTML>
⌨️ 快捷键说明
复制代码
Ctrl + C
搜索代码
Ctrl + F
全屏模式
F11
切换主题
Ctrl + Shift + D
显示快捷键
?
增大字号
Ctrl + =
减小字号
Ctrl + -