tokenizer.java

hsql是很有名的嵌入式数据库

/* Copyright (c) 1995-2000, The Hypersonic SQL Group.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * Neither the name of the Hypersonic SQL Group nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP,
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * This software consists of voluntary contributions made by many individuals 
 * on behalf of the Hypersonic SQL Group.
 *
 *
 * For work added by the HSQL Development Group:
 *
 * Copyright (c) 2001-2005, The HSQL Development Group
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * Neither the name of the HSQL Development Group nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */


package org.hsqldb;

import java.math.BigDecimal;
import java.util.Locale;

import org.hsqldb.lib.IntValueHashMap;
import org.hsqldb.store.ValuePool;

// fredt@users 20020218 - patch 455785 by hjbusch@users - large DECIMAL inserts
// also Long.MIM_VALUE (bug 473388) inserts - applied to different parts
// fredt@users 20020408 - patch 1.7.0 by fredt - exact integral types
// integral values are cast into the smallest type that can hold them
// fredt@users 20020501 - patch 550970 by boucherb@users - fewer StringBuffers
// fredt@users 20020611 - patch 1.7.0 by fredt - correct statement logging
// changes to the working of getLastPart() to return the correct statement for
// logging in the .script file.
// also restructuring to reduce use of objects and speed up tokenising of
// strings and quoted identifiers
// fredt@users 20021112 - patch 1.7.2 by Nitin Chauhan - use of switch
// rewrite of the majority of multiple if(){}else{} chains with switch(){}
// fredt@users 20030610 - patch 1.7.2 - no StringBuffers

/**
 * Provides the ability to tokenize SQL character sequences.
 *
 * Extensively rewritten and extended in successive versions of HSQLDB.
 *
 * @author Thomas Mueller (Hypersonic SQL Group)
 * @version 1.8.0
 * @since Hypersonic SQL
 */
public class Tokenizer {

    private static final int NO_TYPE   = 0,
                             NAME      = 1,
                             LONG_NAME = 2,
                             SPECIAL   = 3,
                             NUMBER    = 4,
                             FLOAT     = 5,
                             STRING    = 6,
                             LONG      = 7,
                             DECIMAL   = 8,
                             BOOLEAN   = 9,
                             DATE      = 10,
                             TIME      = 11,
                             TIMESTAMP = 12,
                             NULL      = 13;

    // used only internally
    private static final int QUOTED_IDENTIFIER = 14,
                             REMARK_LINE       = 15,
                             REMARK            = 16;
    private String           sCommand;
    private int              iLength;
    private int              iIndex;
    private int              tokenIndex;
    private int              nextTokenIndex;
    private int              beginIndex;
    private int              iType;
    private String           sToken;
    private String           sLongNameFirst = null;
    private int              typeLongNameFirst;

    // getToken() will clear LongNameFirst unless retainFirst is set.
    private boolean retainFirst = false;

//    private String           sLongNameLast;
    // WAIT.  Don't do anything before popping another Token (because the
    // state variables aren't set properly due to a call of wait()).
    private boolean bWait;
    private boolean lastTokenQuotedID;

    // literals that are values
    static IntValueHashMap valueTokens;

    static {
        valueTokens = new IntValueHashMap();

        valueTokens.put(Token.T_NULL, NULL);
        valueTokens.put(Token.T_TRUE, BOOLEAN);
        valueTokens.put(Token.T_FALSE, BOOLEAN);
    }

    public Tokenizer() {}

    public Tokenizer(String s) {

        sCommand = s;
        iLength  = s.length();
        iIndex   = 0;
    }

    public void reset(String s) {

        sCommand          = s;
        iLength           = s.length();
        iIndex            = 0;
        tokenIndex        = 0;
        nextTokenIndex    = 0;
        beginIndex        = 0;
        iType             = NO_TYPE;
        typeLongNameFirst = NO_TYPE;
        sToken            = null;
        sLongNameFirst    = null;

//        sLongNameLast  = null;
        bWait             = false;
        lastTokenQuotedID = false;
        retainFirst       = false;
    }

    /**
     *
     * @throws HsqlException
     */
    void back() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        nextTokenIndex = iIndex;
        iIndex         = tokenIndex;
        bWait          = true;
    }

    /**
     * get the given token or throw
     *
     * for commands and simple unquoted identifiers only
     *
     * @param match
     *
     * @throws HsqlException
     */
    String getThis(String match) throws HsqlException {

        getToken();
        matchThis(match);

        return sToken;
    }

    /**
     * for commands and simple unquoted identifiers only
     */
    void matchThis(String match) throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        if (!sToken.equals(match) || iType == QUOTED_IDENTIFIER
                || iType == LONG_NAME) {
            String token = iType == LONG_NAME ? sLongNameFirst
                                              : sToken;

            throw Trace.error(Trace.UNEXPECTED_TOKEN, Trace.TOKEN_REQUIRED,
                              new Object[] {
                token, match
            });
        }
    }

    void throwUnexpected() throws HsqlException {

        String token = iType == LONG_NAME ? sLongNameFirst
                                          : sToken;

        throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
    }

    /**
     * Used for commands only
     *
     *
     * @param match
     */
    public boolean isGetThis(String match) throws HsqlException {

        getToken();

        if (iType != QUOTED_IDENTIFIER && iType != LONG_NAME
                && sToken.equals(match)) {
            return true;
        }

        back();

        return false;
    }

    /**
     * this methode is called before other wasXXX methods and takes
     * precedence
     */
    boolean wasValue() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        switch (iType) {

            case STRING :
            case NUMBER :
            case LONG :
            case FLOAT :
            case DECIMAL :
            case BOOLEAN :
            case NULL :
                return true;

            default :
                return false;
        }
    }

    boolean wasQuotedIdentifier() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        return lastTokenQuotedID;

        // iType won't help for LONG_NAMEs.
        //return iType == QUOTED_IDENTIFIER;
    }

    boolean wasFirstQuotedIdentifier() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        return (typeLongNameFirst == QUOTED_IDENTIFIER);
    }

    /**
     * Method declaration
     *
     *
     * @return
     */
    boolean wasLongName() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        return iType == LONG_NAME;
    }

    /**
     * Name means all quoted and unquoted identifiers plus any word not in the
     * hKeyword list.
     *
     * @return
     */
    boolean wasSimpleName() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        if (iType == QUOTED_IDENTIFIER && sToken.length() != 0) {
            return true;
        }

        if (iType != NAME) {
            return false;
        }

        return !Token.isKeyword(sToken);
    }

    /**
     * Name means all quoted and unquoted identifiers plus any word not in the
     * hKeyword list.
     *
     * "Aname" is more broad than "Name" in that it includes FULL_NAMEs
     * (i.e., 2-part names).
     *
     * @return true if it's AName
     */
    boolean wasName() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        if (iType == QUOTED_IDENTIFIER) {
            return true;
        }

        if (iType != NAME && iType != LONG_NAME) {
            return false;
        }

        return !Token.isKeyword(sToken);
    }

    /**
     * Return first part of long name
     *
     *
     * @return
     */
    String getLongNameFirst() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        return sLongNameFirst;
    }

    /**
     * Return first part of long name
     *
     *
     * @return
     */