tokenizer.java

java 中 tokenizer 的 extends

import java.util.StringTokenizer;
import java.util.regex.*;
import java.lang.*;

/**
 * Breaks up an input string into tokens for arithmetic expressions.
 */
public class Tokenizer {
  
  // operand symbols
  private static final String opSymbols = "+-*/^";
  
  // misc. symbols
  private static final String allowedSymbols = "()";
  
  // whitespace symbols
  private static final String whitespace = " \t\n\r\f";
  
  private String input;
  private StringTokenizer st;
  
  /**
   * Constructor: takes a String to tokenize.
   * @param in String to tokenize.
   */
  public Tokenizer(String in) {
    // get rid of leading/trailing whitespace characters first
    input = in.trim();
    st = new StringTokenizer(input, (whitespace+allowedSymbols+opSymbols),
                             true);
  }
  
  /**
   * Returns whether the tokenizer has more tokens or not.
   */
  public boolean hasMoreTokens() {
    return st.hasMoreTokens();
  }
  
  /**
   * Returns the next token in the string.
   * Precondition: hasMoreTokens() returns true.
   */
  public CalcToken nextToken() {
    
    // set to null to avoid a compiler warning about unassigned variables
    //  - in reality, this variable is ALWAYS set to something because
    //  of the else clause of the if statement, below.  The Java compiler
    //  just can't see that, unfortunately.
    CalcToken resultToken = null;
    
    // get the next non-whitespace chunk of text
    String text = " ";
    while(whitespace.indexOf(text) >= 0) {
      text = st.nextToken();
    }
    
    // classify the token and prepare the appropriate CalcToken object.
    if (text.equals("+")) {
      resultToken = new AddOp();
    
    } else if (text.equals("-")) {
      resultToken = new SubtractOp();
      
    } else if (text.equals("*")) {
      resultToken = new MultiplyOp();
      
    } else if (text.equals("/")) {
      resultToken = new DivideOp();
    
    } else if (text.equals("^")) {
      resultToken = new ExponentOp();
        
    } else {
      // if it doesn't match anything else, see if the text is formatted like
      //  a number (i.e. a string of numerals, optionally followed by a dot
      //  and another string of numerals)
      Pattern p = Pattern.compile("\\d+([.]\\d+)?");
      Matcher m = p.matcher(text);
      if(m.matches()) {
        resultToken = new ValueToken(Double.valueOf(text));
      
         }
    }    
    return resultToken;
  }  
}