streamtokenizer.java

this gcc-g++-3.3.1.tar.gz is a source file of gcc, you can learn more about gcc through this codes f

		    }
		}
	      else if (ch == TT_EOF)
		{
		  break;
		}
	    }
	  return nextToken(); // Recursive, but not too deep in normal cases
	}
      else
	{
	  if (ch != TT_EOF)
	    in.unread(ch);
	  ch = '/';
	}

    if (ch == TT_EOF)
      ttype = TT_EOF;
    else if (isNumeric(ch))
      {
	boolean isNegative = false;
	if (ch == '-')
	  {
	    // Read ahead to see if this is an ordinary '-' rather than numeric.
	    ch = in.read();
	    if (isNumeric(ch) && ch != '-')
	      {
		isNegative = true;
	      }
	    else
	      {
		if (ch != TT_EOF)
		  in.unread(ch);
		return (ttype = '-');
	      }
	  }

	StringBuffer tokbuf = new StringBuffer();
	tokbuf.append((char) ch);

	int decCount = 0;
	while (isNumeric(ch = in.read()) && ch != '-')
	  if (ch == '.' && decCount++ > 0)
	    break;
	  else
	    tokbuf.append((char) ch);

	if (ch != TT_EOF)
	  in.unread(ch);
	ttype = TT_NUMBER;
	try
	  {
	    nval = Double.valueOf(tokbuf.toString()).doubleValue();
	  }
	catch (NumberFormatException _)
	  {
	    nval = 0.0;
	  }
	if (isNegative)
	  nval = -nval;
      }
    else if (isAlphabetic(ch))
      {
	StringBuffer tokbuf = new StringBuffer();
	tokbuf.append((char) ch);
	while (isAlphabetic(ch = in.read()) || isNumeric(ch))
	  tokbuf.append((char) ch);
	if (ch != TT_EOF)
	  in.unread(ch);
	ttype = TT_WORD;
	sval = tokbuf.toString();
	if (lowerCase)
	  sval = sval.toLowerCase();
      }
    else if (isComment(ch))
      {
	while ((ch = in.read()) != '\n' && ch != '\r' && ch != TT_EOF)
	  ;
	if (ch != TT_EOF)
	  in.unread(ch);
	return nextToken();	// Recursive, but not too deep in normal cases.
      }
    else if (isQuote(ch))
      {
	ttype = ch;
	StringBuffer tokbuf = new StringBuffer();
	while ((ch = in.read()) != ttype && ch != '\n' && ch != '\r' &&
	       ch != TT_EOF)
	  {
	    if (ch == '\\')
	      switch (ch = in.read())
		{
		  case 'a':	ch = 0x7;
		    break;
		  case 'b':	ch = '\b';
		    break;
		  case 'f':	ch = 0xC;
		    break;
		  case 'n':	ch = '\n';
		    break;
		  case 'r':	ch = '\r';
		    break;
		  case 't':	ch = '\t';
		    break;
		  case 'v':	ch = 0xB;
		    break;
		  case '\n':    ch = '\n';
		    break;
                  case '\r':    ch = '\r';
		    break;
		  case '\"':
		  case '\'':
		  case '\\':
		    break;
		  default:
		    int ch1, nextch;
		    if ((nextch = ch1 = ch) >= '0' && ch <= '7')
		      {
		        ch -= '0';
		        if ((nextch = in.read()) >= '0' && nextch <= '7')
			  {
			    ch = ch * 8 + nextch - '0';
			    if ((nextch = in.read()) >= '0' && nextch <= '7' &&
				ch1 >= '0' && ch1 <= '3')
			      {
				ch = ch * 8 + nextch - '0';
				nextch = in.read();
			      }
			  }
		      }

		    if (nextch != TT_EOF)
		      in.unread(nextch);
		}

	    tokbuf.append((char) ch);
	  }

	// Throw away matching quote char.
	if (ch != ttype && ch != TT_EOF)
	  in.unread(ch);

	sval = tokbuf.toString();
      }
    else
      {
	ttype = ch;
      }

    return ttype;
  }

  private void resetChar(int ch)
  {
    whitespace[ch] = alphabetic[ch] = numeric[ch] = quote[ch] = comment[ch] =
      false;
  }

  /**
   * This method makes the specified character an ordinary character.  This
   * means that none of the attributes (whitespace, alphabetic, numeric,
   * quote, or comment) will be set on this character.  This character will
   * parse as its own token.
   *
   * @param c The character to make ordinary, passed as an int
   */
  public void ordinaryChar(int ch)
  {
    if (ch >= 0 && ch <= 255)
      resetChar(ch);
  }

  /**
   * This method makes all the characters in the specified range, range
   * terminators included, ordinary.  This means the none of the attributes
   * (whitespace, alphabetic, numeric, quote, or comment) will be set on
   * any of the characters in the range.  This makes each character in this
   * range parse as its own token.
   *
   * @param low The low end of the range of values to set the whitespace
   *            attribute for
   * @param high The high end of the range of values to set the whitespace
   *            attribute for
   */
  public void ordinaryChars(int low, int hi)
  {
    if (low < 0)
      low = 0;
    if (hi > 255)
      hi = 255;
    for (int i = low; i <= hi; i++)
      resetChar(i);
  }

  /**
   * This method sets the numeric attribute on the characters '0' - '9' and
   * the characters '.' and '-'.
   */
  public void parseNumbers()
  {
    for (int i = 0; i <= 9; i++)
      numeric['0' + i] = true;

    numeric['.'] = true;
    numeric['-'] = true;
  }

  /**
   * Puts the current token back into the StreamTokenizer so
   * <code>nextToken</code> will return the same value on the next call.
   * May cause the lineno method to return an incorrect value
   * if lineno is called before the next call to nextToken.
   */
  public void pushBack()
  {
    pushedBack = true;
  }

  /**
   * This method sets the quote attribute on the specified character.
   *
   * @param c The character to set the quote attribute for, passed as an int.
   */
  public void quoteChar(int ch)
  {
    if (ch >= 0 && ch <= 255)
      quote[ch] = true;
  }

  /**
   * This method removes all attributes (whitespace, alphabetic, numeric,
   * quote, and comment) from all characters.  It is equivalent to calling
   * <code>ordinaryChars(0x00, 0xFF)</code>.
   *
   * @see ordinaryChars
   */
  public void resetSyntax()
  {
    ordinaryChars(0x00, 0xFF);
  }

  /**
   * This method sets a flag that indicates whether or not "C++" language style
   * comments ("//" comments through EOL ) are handled by the parser.
   * If this is <code>true</code> commented out sequences are skipped and
   * ignored by the parser.  This defaults to <code>false</code>.
   *
   * @param flag <code>true</code> to recognized and handle "C++" style
   *             comments, <code>false</code> otherwise
   */
  public void slashSlashComments(boolean flag)
  {
    slashSlash = flag;
  }

  /**
   * This method sets a flag that indicates whether or not "C" language style
   * comments (with nesting not allowed) are handled by the parser.
   * If this is <code>true</code> commented out sequences are skipped and
   * ignored by the parser.  This defaults to <code>false</code>.
   *
   * @param flag <code>true</code> to recognized and handle "C" style comments,
   *             <code>false</code> otherwise
   */
  public void slashStarComments(boolean flag)
  {
    slashStar = flag;
  }

  /**
   * This method returns the current token value as a <code>String</code> in
   * the form "Token[x], line n", where 'n' is the current line numbers and
   * 'x' is determined as follows.
   * <p>
   * <ul>
   * <li>If no token has been read, then 'x' is "NOTHING" and 'n' is 0
   * <li>If <code>ttype</code> is TT_EOF, then 'x' is "EOF"
   * <li>If <code>ttype</code> is TT_EOL, then 'x' is "EOL"
   * <li>If <code>ttype</code> is TT_WORD, then 'x' is <code>sval</code>
   * <li>If <code>ttype</code> is TT_NUMBER, then 'x' is "n=strnval" where
   * 'strnval' is <code>String.valueOf(nval)</code>.
   * <li>If <code>ttype</code> is a quote character, then 'x' is
   * <code>sval</code>
   * <li>For all other cases, 'x' is <code>ttype</code>
   * </ul>
   */
  public String toString()
  {
    String tempstr;
    if (ttype == TT_EOF)
      tempstr = "EOF";
    else if (ttype == TT_EOL)
      tempstr = "EOL";
    else if (ttype == TT_WORD)
      tempstr = sval;
    else if (ttype == TT_NUMBER)
      tempstr = "n=" + nval;
    else if (ttype == TT_NONE)
      tempstr = "NOTHING";
    else // must be an ordinary char.
      tempstr = "\'" + (char) ttype + "\'";

    return "Token[" + tempstr + "], line " + lineno();
  }

  /**
   * This method sets the whitespace attribute for all characters in the
   * specified range, range terminators included.
   *
   * @param low The low end of the range of values to set the whitespace
   *            attribute for
   * @param high The high end of the range of values to set the whitespace
   *             attribute for
   */
  public void whitespaceChars(int low, int hi)
  {
    if (low < 0)
      low = 0;
    if (hi > 255)
      hi = 255;
    for (int i = low; i <= hi; i++)
      {
	resetChar(i);
	whitespace[i] = true;
      }
  }

  /**
   * This method sets the alphabetic attribute for all characters in the
   * specified range, range terminators included.
   *
   * @param low The low end of the range of values to set the alphabetic
   *            attribute for
   * @param high The high end of the range of values to set the alphabetic
   *             attribute for
   */
  public void wordChars(int low, int hi)
  {
    if (low < 0)
      low = 0;
    if (hi > 255)
      hi = 255;
    for (int i = low; i <= hi; i++)
      alphabetic[i] = true;
  }
}