stringtokenizer.java

ddddddddddddddddddd dddddddddddddd ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd

	 * The position must be initialized before this method is used.
	 * (setText does this and it is called from the constructor)
	 *
	 * @param nontokenDelims delimiters that should not be returned as tokens.
	 * @param tokenDelims delimiters that should be returned as tokens.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	private void setDelims(String nontokenDelims, String tokenDelims){
		this.nontokenDelims = nontokenDelims;
		this.tokenDelims = tokenDelims;
		// If we change delimiters, we do not want to start fresh,
		// without returning empty tokens.
		// the delimiter changed position can never be less than
		// zero, unlike position.
		delimsChangedPosition = (position != -1 ? position : strLength);
		// set the max delimiter
		maxDelimChar = 0;
		for (int i=0; nontokenDelims != null && i < nontokenDelims.length(); i++){
			if (maxDelimChar < nontokenDelims.charAt(i)){
				maxDelimChar = nontokenDelims.charAt(i);
			}
		}
		for (int i=0; tokenDelims != null && i < tokenDelims.length(); i++){
			if (maxDelimChar < tokenDelims.charAt(i)){
				maxDelimChar = tokenDelims.charAt(i);
			}
		}
		// Changing the delimiters may change the number of tokens
		tokenCount = -1;
	}


	/**
	 * Tests if there are more tokens available from this tokenizer's string.
	 * If this method returns <tt>true</tt>, then a subsequent call to
	 * <tt>nextToken</tt> with no argument will successfully return a token.
	 * <p>
	 * The current position is not changed.
	 *
	 * @return <code>true</code> if and only if there is at least one token in the
	 *          string after the current position; <code>false</code> otherwise.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public boolean hasMoreTokens(){

		// handle the easy case in which the number
		// of tokens has been counted.
		if (tokenCount == 0){
			return false;
		} else if (tokenCount > 0){
			return true;
		}

		// copy over state variables from the class to local
		// variables so that the state of this object can be
		// restored to the state that it was in before this
		// method was called.
		int savedPosition = position;
		boolean savedEmptyReturned = emptyReturned;

		int workingPosition = position;
		boolean workingEmptyReturned = emptyReturned;
		boolean onToken = advancePosition();
		while(position != workingPosition ||
			emptyReturned != workingEmptyReturned){
			if (onToken){
				// restore object state
				position = savedPosition;
				emptyReturned = savedEmptyReturned;
				return true;
			}
			workingPosition = position;
			workingEmptyReturned = emptyReturned;
			onToken = advancePosition();
		}

		// restore object state
		position = savedPosition;
		emptyReturned = savedEmptyReturned;
		return false;
	}

	/**
	 * Returns the next token from this string tokenizer.
	 * <p>
	 * The current position is set after the token returned.
	 *
	 * @return the next token from this string tokenizer.
	 * @throws NoSuchElementException if there are no more tokens in this tokenizer's string.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public String nextToken(){
		int workingPosition = position;
		boolean workingEmptyReturned = emptyReturned;
		boolean onToken = advancePosition();
		while(position != workingPosition ||
			emptyReturned != workingEmptyReturned){
			if (onToken){
				// returning a token decreases the token count
				tokenCount--;
				return (emptyReturned ? "" : text.substring(workingPosition, (position != -1) ? position : strLength));
			}
			workingPosition = position;
			workingEmptyReturned = emptyReturned;
			onToken = advancePosition();
		}
		throw new java.util.NoSuchElementException();
	}

	/**
	 * Advances the current position so it is before the next token.
	 * <p>
	 * This method skips nontoken delimiters but does not skip
	 * token delimiters.
	 * <p>
	 * This method is useful when switching to the new delimiter sets (see the
	 * second example in the class comment.)
	 *
	 * @return <code>true</code> if there are more tokens, <code>false</code> otherwise.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public boolean skipDelimiters(){
		int workingPosition = position;
		boolean workingEmptyReturned = emptyReturned;
		boolean onToken = advancePosition();

		// skipping delimiters may cause the number of tokens to change
		tokenCount = -1;

		while(position != workingPosition ||
			emptyReturned != workingEmptyReturned){
			if (onToken){
				// restore the state to just as it was before we found
				// this token and return
				position = workingPosition;
				emptyReturned = workingEmptyReturned;
				return true;
			}
			workingPosition = position;
			workingEmptyReturned = emptyReturned;
			onToken = advancePosition();
		}

		// the end of the string was reached
		// without finding any tokens
		return false;
	}

	/**
	 * Calculates the number of times that this tokenizer's <code>nextToken</code>
	 * method can be called before it generates an exception. The current position
	 * is not advanced.
	 *
	 * @return the number of tokens remaining in the string using the current
	 *    delimiter set.
	 *
	 * @see #nextToken()
	 * @since ostermillerutils 1.00.00
	 */
	public int countTokens(){

		// return the cached token count if a cache
		// is available.
		if (this.tokenCount >=0){
			return this.tokenCount;
		}

		int tokenCount = 0;

		// copy over state variables from the class to local
		// variables so that the state of this object can be
		// restored to the state that it was in before this
		// method was called.
		int savedPosition = position;
		boolean savedEmptyReturned = emptyReturned;

		int workingPosition = position;
		boolean workingEmptyReturned = emptyReturned;
		boolean onToken = advancePosition();
		while(position != workingPosition ||
			emptyReturned != workingEmptyReturned){
			if (onToken){
				tokenCount++;
			}
			workingPosition = position;
			workingEmptyReturned = emptyReturned;
			onToken = advancePosition();
		}

		// restore object state
		position = savedPosition;
		emptyReturned = savedEmptyReturned;

		// Save the token count in case this is called again
		// so we wouldn't have to do so much work.
		this.tokenCount = tokenCount;

		return tokenCount;
	}

	/**
	 * Set the delimiters used to this set of (nontoken) delimiters.
	 *
	 * @param delims the new set of nontoken delimiters (the set of token delimiters will be empty).
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public void setDelimiters(String delims){
		setDelims(delims, null);
	}

	/**
	 * Set the delimiters used to this set of delimiters.
	 *
	 * @param delims the new set of delimiters.
	 * @param delimsAreTokens flag indicating whether the first parameter specifies
	 *    token or nontoken delimiters: false -- the first parameter specifies nontoken
	 *    delimiters, the set of token delimiters is empty; true -- the first parameter
	 *    specifies token delimiters, the set of nontoken delimiters is empty.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public void setDelimiters(String delims, boolean delimsAreTokens){
		setDelims((delimsAreTokens ? null : delims), (delimsAreTokens ? delims : null));
	}

	/**
	 * Set the delimiters used to this set of delimiters.
	 *
	 * @param nontokenDelims the new set of nontoken delimiters.
	 * @param tokenDelims the new set of token delimiters.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public void setDelimiters(String nontokenDelims, String tokenDelims){
		setDelims(nontokenDelims, tokenDelims);
	}

	/**
	 * Set the delimiters used to this set of delimiters.
	 *
	 * @param nontokenDelims the new set of nontoken delimiters.
	 * @param tokenDelims the new set of token delimiters.
	 * @param returnEmptyTokens true if empty tokens may be returned; false otherwise.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	public void setDelimiters(String nontokenDelims, String tokenDelims, boolean returnEmptyTokens){
		setDelims(nontokenDelims, tokenDelims);
		setReturnEmptyTokens(returnEmptyTokens);
	}

	/**
	 * Calculates the number of times that this tokenizer's <code>nextToken</code>
	 * method can be called before it generates an exception using the given set of
	 * (nontoken) delimiters.  The delimiters given will be used for future calls to
	 * nextToken() unless new delimiters are given. The current position
	 * is not advanced.
	 *
	 * @param delims the new set of nontoken delimiters (the set of token delimiters will be empty).
	 * @return the number of tokens remaining in the string using the new
	 *    delimiter set.
	 *
	 * @see #countTokens()
	 * @since ostermillerutils 1.00.00
	 */
	public int countTokens(String delims){
		setDelims(delims, null);
		return countTokens();
	}

	/**
	 * Calculates the number of times that this tokenizer's <code>nextToken</code>
	 * method can be called before it generates an exception using the given set of
	 * delimiters.  The delimiters given will be used for future calls to
	 * nextToken() unless new delimiters are given. The current position
	 * is not advanced.
	 *
	 * @param delims the new set of delimiters.
	 * @param delimsAreTokens flag indicating whether the first parameter specifies
	 *    token or nontoken delimiters: false -- the first parameter specifies nontoken
	 *    delimiters, the set of token delimiters is empty; true -- the first parameter
	 *    specifies token delimiters, the set of nontoken delimiters is empty.
	 * @return the number of tokens remaining in the string using the new
	 *    delimiter set.
	 *
	 * @see #countTokens()
	 * @since ostermillerutils 1.00.00
	 */
	public int countTokens(String delims, boolean delimsAreTokens){
		setDelims((delimsAreTokens ? null : delims), (delimsAreTokens ? delims : null));
		return countTokens();
	}

	/**
	 * Calculates the number of times that this tokenizer's <code>nextToken</code>
	 * method can be called before it generates an exception using the given set of
	 * delimiters.  The delimiters given will be used for future calls to
	 * nextToken() unless new delimiters are given. The current position
	 * is not advanced.
	 *
	 * @param nontokenDelims the new set of nontoken delimiters.
	 * @param tokenDelims the new set of token delimiters.
	 * @return the number of tokens remaining in the string using the new
	 *    delimiter set.
	 *
	 * @see #countTokens()
	 * @since ostermillerutils 1.00.00
	 */
	public int countTokens(String nontokenDelims, String tokenDelims){
		setDelims(nontokenDelims, tokenDelims);
		return countTokens();
	}

	/**
	 * Calculates the number of times that this tokenizer's <code>nextToken</code>
	 * method can be called before it generates an exception using the given set of
	 * delimiters.  The delimiters given will be used for future calls to
	 * nextToken() unless new delimiters are given. The current position
	 * is not advanced.
	 *
	 * @param nontokenDelims the new set of nontoken delimiters.
	 * @param tokenDelims the new set of token delimiters.
	 * @param returnEmptyTokens true if empty tokens may be returned; false otherwise.
	 * @return the number of tokens remaining in the string using the new
	 *    delimiter set.
	 *
	 * @see #countTokens()
	 * @since ostermillerutils 1.00.00
	 */
	public int countTokens(String nontokenDelims, String tokenDelims, boolean returnEmptyTokens){
		setDelims(nontokenDelims, tokenDelims);
		setReturnEmptyTokens(returnEmptyTokens);
		return countTokens();
	}

	/**
	 * Advances the state of the tokenizer to the next token or delimiter.  This method only
	 * modifies the class variables position, and emptyReturned.  The type of token that
	 * should be emitted can be deduced by examining the changes to these two variables.
	 * If there are no more tokens, the state of these variables does not change at all.
	 *
	 * @return true if we are at a juncture at which a token may be emitted, false otherwise.
	 *
	 * @since ostermillerutils 1.00.00
	 */
	private boolean advancePosition(){
		// if we are returning empty tokens, we are just starting to tokenizer
		// and there is a delimiter at the beginning of the string or the string
		// is empty we need to indicate that there is an empty token at the beginning.
		// The beginning is defined as where the delimiters were last changed.
		if (returnEmptyTokens && !emptyReturned &&
			(delimsChangedPosition == position ||
			(position == -1 && strLength == delimsChangedPosition))){
			if (strLength == delimsChangedPosition){
				// Case in which the string (since delim change)
				// is empty, but because we are returning empty
				// tokens, a single empty token should be returned.
				emptyReturned = true;
				/*System.out.println("Empty token for empty string.");*/
				return true;
			} else {
				char c = text.charAt(position);
				if (c <= maxDelimChar &&
					(nontokenDelims != null && nontokenDelims.indexOf(c) != -1) ||
					(tokenDelims != null && tokenDelims.indexOf(c) != -1)){
					// There is delimiter at the very start of the string
					// so we must return an empty token at the beginning.
					emptyReturned = true;
					/*System.out.println("Empty token at beginning.");*/