stringtokenizer.java

ddddddddddddddddddd dddddddddddddd ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd

package abc;

/*
 * A replacement for java.util.StringTokenizer
 * Copyright (C) 2001 Stephen Ostermiller
 * http://ostermiller.org/contact.pl?regarding=Java+Utilities
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * See COPYING.TXT for details.
 */

// package com.Ostermiller.util;


/**
 * The string tokenizer class allows an application to break a string into
 * tokens.
 * More information about this class is available from <a target="_top" href=
 * "http://ostermiller.org/utils/StringTokenizer.html">ostermiller.org</a>.
 * <p>
 * The tokenization method is much simpler than the one used by the
 * <code>StreamTokenizer</code> class. The <code>StringTokenizer</code> methods
 * do not distinguish among identifiers, numbers, and quoted strings, nor do
 * they recognize and skip comments.
 * <p>
 * The set of delimiters (the characters that separate tokens) may be specified
 * either at creation time or on a per-token basis.
 * <p>
 * There are two kinds of delimiters: token delimiters and nontoken delimiters.
 * A token is either one token delimiter character, or a maximal sequence of
 * consecutive characters that are not delimiters.
 * <p>
 * A <code>StringTokenizer</code> object internally maintains a current
 * position within the string to be tokenized. Some operations advance this
 * current position past the characters processed.
 * <p>
 * The implementation is not thread safe; if a <code>StringTokenizer</code>
 * object is intended to be used in multiple threads, an appropriate wrapper
 * must be provided.
 * <p>
 * The following is one example of the use of the tokenizer. It also
 * demonstrates the usefulness of having both token and nontoken delimiters in
 * one <code>StringTokenizer</code>.
 * <p>
 * The code:
 * <blockquote><code>
 * String s = " &nbsp;( &nbsp; aaa  \t &nbsp;* (b+c1 ))";<br>
 * StringTokenizer st = new StringTokenizer(s, " \t\n\r\f", "()+*");<br>
 * while (st.hasMoreTokens()) {<br>
 * &nbsp;&nbsp;&nbsp;&nbsp;System.out.println(st.nextToken());<br>
 * };
 * </code></blockquote>
 * <p>
 * prints the following output:
 * <blockquote>
 * (<br>
 * aaa<br>
 * *<br>
 * (<br>
 * b<br>
 * +<br>
 * c1<br>
 * )<br>
 * )
 * </blockquote>
 * <p>
 * </b>Compatibility with <code>java.util.StringTokenizer</code></b>
 * <p>
 * In the original version of <code>java.util.StringTokenizer</code>, the method
 * <code>nextToken()</code> left the current position after the returned token,
 * and the method <code>hasMoreTokens()</code> moved (as a side effect) the
 * current position before the beginning of the next token. Thus, the code:
 * <blockquote><code>
 * String s = "x=a,b,c";<br>
 * java.util.StringTokenizer st = new java.util.StringTokenizer(s,"=");<br>
 * System.out.println(st.nextToken());<br>
 * while (st.hasMoreTokens()) {<br>
 * &nbsp;&nbsp;&nbsp;&nbsp;System.out.println(st.nextToken(","));<br>
 * };
 * </code></blockquote>
 * <p>
 * prints the following output:
 * <blockquote>
 * x<br>
 * a<br>
 * b<br>
 * c
 * </blockquote>
 * <p>
 * The Java SDK 1.3 implementation removed the undesired side effect of
 * <code>hasMoreTokens</code> method: now, it does not advance current position.
 * However, after these changes the output of the above code was:
 * <blockquote>
 * x<br>
 * =a<br>
 * b<br>
 * c
 * </blockquote>
 * <p>
 * and there was no good way to produce a second token without "=".
 * <p>
 * To solve the problem, this implementation introduces a new method
 * <code>skipDelimiters()</code>. To produce the original output, the above code
 * should be modified as:
 * <blockquote><code>
 * String s = "x=a,b,c";<br>
 * StringTokenizer st = new StringTokenizer(s,"=");<br>
 * System.out.println(st.nextToken());<br>
 * st.skipDelimiters();<br>
 * while (st.hasMoreTokens()) {<br>
 * &nbsp;&nbsp;&nbsp;&nbsp;System.out.println(st.nextToken(","));<br>
 * };
 * </code></blockquote>
 *
 * @author Stephen Ostermiller http://ostermiller.org/contact.pl?regarding=Java+Utilities
 * @since ostermillerutils 1.00.00
 */
public class StringTokenizer implements java.util.Enumeration{

	/**
	 * The string to be tokenized.
	 * The code relies on this to never be null.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	protected String text;

	/**
	 * The length of the text.
	 * Cached for performance.  This should be set whenever the
	 * string we are working with is changed.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	protected int strLength;

	/**
	 * The set of nontoken delimiters.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	protected String nontokenDelims;

	/**
	 * The set of token delimiters.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	protected String tokenDelims;

	/**
	 * One of two variables used to maintain state through
	 * the tokenizing process.
	 * <P>
	 * Represents the position at which we should start looking for
	 * the next token(the position of the character immediately
	 * following the end of the last token, or 0 to start), or
	 * -1 if the entire string has been examined.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	protected int position;

	/**
	 * One of two variables used to maintain state through
	 * the tokenizing process.
	 * <p>
	 * true if and only if is found that an empty token should
	 * be returned or if empty token was the last thing returned.
	 * <p>
	 * If returnEmptyTokens in false, then this variable will
	 * always be false.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	protected boolean emptyReturned;

	/**
	 * Stores the value of the delimiter character with the
	 * highest value. It is used to optimize the detection of delimiter
	 * characters.  The common case will be that the int values of delimiters
	 * will be less than that of most characters in the string (, or space less
	 * than any letter for example).  Given this, we can check easily check
	 * to see if a character is not a delimiter by comparing it to the max
	 * delimiter.  If it is greater than the max delimiter, then it is no
	 * a delimiter otherwise we have to do some more in depth analysis. (ie
	 * search the delimiter string.)  This will reduce the running time of
	 * the algorithm not to depend on the length of the delimiter string
	 * for the common case.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	protected char maxDelimChar;

	/**
	 * Whether empty tokens should be returned.
	 * ie if "" should be returned when text starts with
	 * a delim, has two delims next to each other, or
	 * ends with a delim.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	protected boolean returnEmptyTokens;

	/**
	 * Indicates at which position the delimiters last changed.  This
	 * will effect how null tokens are returned.  Any
	 * time that delimiters are changed, the string will be treated as if
	 * it is being parsed from position zero, ie, null strings are possible
	 * at the very beginning.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	protected int delimsChangedPosition;

	/**
	 * A cache of the token count.  This variable should be -1 if the token
	 * have not yet been counted. It should be greater than or equal to zero
	 * if the tokens have been counted.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	protected int tokenCount;

	/**
	 * Constructs a string tokenizer for the specified string. Both token and
	 * nontoken delimiters are specified.
	 * <p>
	 * The current position is set at the beginning of the string.
	 *
	 * @param text a string to be parsed.
	 * @param nontokenDelims the nontoken delimiters, i.e. the delimiters that only separate
	 *     tokens and are not returned as separate tokens.
	 * @param tokenDelims the token delimiters, i.e. delimiters that both separate tokens,
	 *     and are themselves returned as tokens.
	 * @throws NullPointerException if text is null.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public StringTokenizer(String text, String nontokenDelims, String tokenDelims){
		this(text, nontokenDelims, tokenDelims, false);
	}

	/**
	 * Constructs a string tokenizer for the specified string. Both token and
	 * nontoken delimiters are specified and whether or not empty tokens are returned
	 * is specified.
	 * <p>
	 * Empty tokens are tokens that are between consecutive delimiters.
	 * <p>
	 * It is a primary constructor (i.e. all other constructors are defined in terms
	 * of it.)
	 * <p>
	 * The current position is set at the beginning of the string.
	 *
	 * @param text a string to be parsed.
	 * @param nontokenDelims the nontoken delimiters, i.e. the delimiters that only separate
	 *     tokens and are not returned as separate tokens.
	 * @param tokenDelims the token delimiters, i.e. delimiters that both separate tokens,
	 *     and are themselves returned as tokens.
	 * @param returnEmptyTokens true if empty tokens may be returned; false otherwise.
	 * @throws NullPointerException if text is null.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public StringTokenizer(String text, String nontokenDelims, String tokenDelims, boolean returnEmptyTokens){
		setDelims(nontokenDelims, tokenDelims);
		setText(text);
		setReturnEmptyTokens(returnEmptyTokens);
	}

	/**
	 * Constructs a string tokenizer for the specified string. Either token or
	 * nontoken delimiters are specified.
	 * <p>
	 * Is equivalent to:
	 * <ul>
	 * <li> If the third parameter is <code>false</code> --
	 *      <code>StringTokenizer(text,delims, null)</code>
	 * <li> If the third parameter is <code>true</code> --
	 *      <code>StringTokenizer(text, null ,delims)</code>
	 * </ul>
	 *
	 * @param text a string to be parsed.
	 * @param delims the delimiters.
	 * @param delimsAreTokens
	 *     flag indicating whether the second parameter specifies token or
	 *     nontoken delimiters: <code>false</code> -- the second parameter
	 *     specifies nontoken delimiters, the set of token delimiters is
	 *     empty; <code>true</code> -- the second parameter specifies token
	 *     delimiters, the set of nontoken delimiters is empty.
	 * @throws NullPointerException if text is null.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public StringTokenizer(String text, String delims, boolean delimsAreTokens){
		this(text, (delimsAreTokens ? null : delims), (delimsAreTokens ? delims : null));
	}

	/**
	 * Constructs a string tokenizer for the specified string. The characters in the
	 * <code>nontokenDelims</code> argument are the delimiters for separating
	 * tokens. Delimiter characters themselves will not be treated as tokens.
	 * <p>
	 * Is equivalent to <code>StringTokenizer(text,nontokenDelims, null)</code>.
	 *
	 * @param text a string to be parsed.
	 * @param nontokenDelims the nontoken delimiters.
	 * @throws NullPointerException if text is null.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public StringTokenizer(String text, String nontokenDelims){
		this(text, nontokenDelims, null);
	}

	/**
	 * Constructs a string tokenizer for the specified string. The tokenizer uses
	 * " \t\n\r\f" as a delimiter set of nontoken delimiters, and an empty token
	 * delimiter set.
	 * <p>
	 * Is equivalent to <code>StringTokenizer(text, " \t\n\r\f", null);
	 *
	 * @param text a string to be parsed.
	 * @throws NullPointerException if text is null.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public StringTokenizer(String text){
		this(text, " \t\n\r\f", null);
	}

	/**
	 * Set the text to be tokenized in this StringTokenizer.
	 * <p>
	 * This is useful when for StringTokenizer re-use so that new string tokenizers do no
	 * have to be created for each string you want to tokenizer.
	 * <p>
	 * The string will be tokenized from the beginning of the string.
	 *
	 * @param text a string to be parsed.
	 * @throws NullPointerException if text is null.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public void setText(String text){
		if (text == null){
			throw new NullPointerException();
		}
		this.text = text;
		strLength = text.length();
		emptyReturned = false;
		// set the position to start evaluation to zero
		// unless the string has no length, in which case
		// the entire string has already been examined.
		position = (strLength > 0 ? 0: -1);
		// because the text was changed since the last time the delimiters
		// were changed we need to set the delimiter changed position
		delimsChangedPosition = 0;
		// The token count changes when the text changes
		tokenCount = -1;
	}

	/**
	 * Set the delimiters for this StringTokenizer.