tokenizer.java

一个用java写的开源的数据库系统

/* Copyrights and Licenses
 *
 * This product includes Hypersonic SQL.
 * Originally developed by Thomas Mueller and the Hypersonic SQL Group. 
 *
 * Copyright (c) 1995-2000 by the Hypersonic SQL Group. All rights reserved. 
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met: 
 *     -  Redistributions of source code must retain the above copyright notice, this list of conditions
 *         and the following disclaimer. 
 *     -  Redistributions in binary form must reproduce the above copyright notice, this list of
 *         conditions and the following disclaimer in the documentation and/or other materials
 *         provided with the distribution. 
 *     -  All advertising materials mentioning features or use of this software must display the
 *        following acknowledgment: "This product includes Hypersonic SQL." 
 *     -  Products derived from this software may not be called "Hypersonic SQL" nor may
 *        "Hypersonic SQL" appear in their names without prior written permission of the
 *         Hypersonic SQL Group. 
 *     -  Redistributions of any form whatsoever must retain the following acknowledgment: "This
 *          product includes Hypersonic SQL." 
 * This software is provided "as is" and any expressed or implied warranties, including, but
 * not limited to, the implied warranties of merchantability and fitness for a particular purpose are
 * disclaimed. In no event shall the Hypersonic SQL Group or its contributors be liable for any
 * direct, indirect, incidental, special, exemplary, or consequential damages (including, but
 * not limited to, procurement of substitute goods or services; loss of use, data, or profits;
 * or business interruption). However caused any on any theory of liability, whether in contract,
 * strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this
 * software, even if advised of the possibility of such damage. 
 * This software consists of voluntary contributions made by many individuals on behalf of the
 * Hypersonic SQL Group.
 *
 *
 * For work added by the HSQL Development Group:
 *
 * Copyright (c) 2001-2002, The HSQL Development Group
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer, including earlier
 * license statements (above) and comply with all above license conditions.
 *
 * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution, including earlier
 * license statements (above) and comply with all above license conditions.
 *
 * Neither the name of the HSQL Development Group nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, 
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */


package org.hsqldb;

import java.sql.Types;
import java.sql.SQLException;
import java.math.BigDecimal;
import java.util.Hashtable;

// fredt@users 20020218 - patch 455785 by hjbusch@users - large DECIMAL inserts
// also Long.MIM_VALUE (bug 473388) inserts - applied to different parts
// fredt@users 20020408 - patch 1.7.0 by fredt - exact integral types
// integral values are cast into the smallest type that can hold them
// fredt@users 20020501 - patch 550970 by boucherb@users - fewer StringBuffers
// fredt@users 20020611 - patch 1.7.0 by fredt - correct statement logging
// changes to the working of getLastPart() to return the correct statement for
// logging in the .script file.
// also restructuring to reduce use of objects and speed up tokenising of
// strings and quoted identifiers

/**
 * Tokenizer class declaration
 *
 *
 * @version 1.7.0
 */
class Tokenizer {

    private static final int NAME      = 1,
                             LONG_NAME = 2,
                             SPECIAL   = 3,
                             NUMBER    = 4,
                             FLOAT     = 5,
                             STRING    = 6,
                             LONG      = 7,
                             DECIMAL   = 8;

    // used only internally
    private static final int QUOTED_IDENTIFIER = 9,
                             REMARK_LINE       = 10,
                             REMARK            = 11;
    private String           sCommand;
    private int              iLength;
    private Object           oValue;
    private int              iIndex;
    private int              tokenIndex;
    private int              nextTokenIndex;
    private int              beginIndex;
    private int              iType;
    private String           sToken;
    private String           sLongNameFirst;
    private String           sLongNameLast;
    private boolean          bWait;
    private static Hashtable hKeyword;

    static {
        hKeyword = new Hashtable(67);

        String keyword[] = {
            "AND", "ALL", "AVG", "BY", "BETWEEN", "COUNT", "CASEWHEN",
            "DISTINCT", "EXISTS", "EXCEPT", "FALSE", "FROM", "GROUP", "IF",
            "INTO", "IFNULL", "IS", "IN", "INTERSECT", "INNER", "LEFT",
            "LIKE", "MAX", "MIN", "NULL", "NOT", "ON", "ORDER", "OR", "OUTER",
            "PRIMARY", "SELECT", "SET", "SUM", "TO", "TRUE", "UNIQUE",
            "UNION", "VALUES", "WHERE", "CONVERT", "CAST", "CONCAT", "MINUS",
            "CALL"
        };

        for (int i = 0; i < keyword.length; i++) {
            hKeyword.put(keyword[i], hKeyword);
        }
    }

    /**
     * Constructor declaration
     *
     *
     * @param s
     */
    Tokenizer(String s) {

        sCommand = s;
        iLength  = s.length();
        iIndex   = 0;
    }

    /**
     * Method declaration
     *
     *
     * @throws SQLException
     */
    void back() throws SQLException {

        Trace.doAssert(!bWait, "back");

        nextTokenIndex = iIndex;
        iIndex         = tokenIndex;
        bWait          = true;
    }

    /**
     * Method declaration
     *
     *
     * @param match
     *
     * @throws SQLException
     */
    void getThis(String match) throws SQLException {

        getToken();

        if (!sToken.equals(match)) {
            throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken);
        }
    }

    /**
     * Method declaration
     *
     *
     * @return
     *
     * @throws SQLException
     */
    String getStringToken() throws SQLException {

        getToken();

        if (iType == STRING) {

// fred - no longer including first quote in sToken
            return sToken.toUpperCase();
        } else if (iType == NAME) {
            return sToken;
        } else if (iType == QUOTED_IDENTIFIER) {
            return sToken.toUpperCase();
        }

        throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken);
    }

    /**
     * Method declaration
     *
     *
     * @return
     */
    boolean wasValue() {

        if (iType == STRING || iType == NUMBER || iType == FLOAT
                || iType == LONG || iType == DECIMAL) {
            return true;
        }

        if (sToken.equals("NULL") || sToken.equals("TRUE")
                || sToken.equals("FALSE")) {
            return true;
        }

        return false;
    }

    boolean wasQuotedIdentifier() {
        return iType == QUOTED_IDENTIFIER;
    }

    /**
     * Method declaration
     *
     *
     * @return
     */
    boolean wasLongName() {
        return iType == LONG_NAME;
    }

    /**
     * Method declaration
     *
     *
     * @return
     */
    boolean wasName() {

        if (iType == QUOTED_IDENTIFIER) {
            return true;
        }

        if (iType != NAME) {
            return false;
        }

        return !hKeyword.containsKey(sToken);
    }

    /**
     * Method declaration
     *
     *
     * @return
     */
    String getLongNameFirst() {
        return sLongNameFirst;
    }

    /**
     * Method declaration
     *
     *
     * @return
     */
    String getLongNameLast() {
        return sLongNameLast;
    }

    /**
     * Method declaration
     *
     *
     * @return
     *
     * @throws SQLException
     */
    String getName() throws SQLException {

        getToken();

        if (!wasName()) {
            throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken);
        }

        return sToken;
    }

    /**
     * Method declaration
     *
     *
     * @return
     *
     * @throws SQLException
     */
    String getString() throws SQLException {

        getToken();

        return sToken;
    }

    /**
     * Method declaration
     *
     *
     * @return
     */
    int getType() {

        // todo: make sure it's used only for Values!
        // todo: synchronize iType with hColumn
        switch (iType) {

            case STRING :
                return Types.VARCHAR;

            case NUMBER :
                return Types.INTEGER;

            case LONG :
                return Types.BIGINT;

            case FLOAT :
                return Types.DOUBLE;

            case DECIMAL :
                return Types.DECIMAL;

            default :
                return Types.NULL;
        }
    }

    /**
     * Method declaration
     *
     *
     * @return
     *
     * @throws SQLException
     */
    Object getAsValue() throws SQLException {

        if (!wasValue()) {
            throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken);
        }

        if (iType == STRING) {

            //fredt - no longer returning string with a singlequote as last char
            return sToken;
        }

        // convert NULL to null String if not a String
        // todo: make this more straightforward
        if (sToken.equals("NULL")) {
            return null;
        }

        if (iType == NUMBER) {

            // fredt - this returns unsigned values which are later negated.
            // as a result Integer.MIN_VALUE or Long.MIN_VALUE are promoted
            // to a wider type.
            if (sToken.length() < 10) {
                return new Integer(sToken);
            }

            if (sToken.length() == 10) {
                try {
                    return new Integer(sToken);
                } catch (Exception e1) {
                    iType = LONG;

                    return new Long(sToken);
                }
            }

            if (sToken.length() < 19) {
                iType = LONG;

                return new Long(sToken);
            }

            if (sToken.length() == 19) {
                try {
                    return new Long(sToken);
                } catch (Exception e2) {
                    iType = DECIMAL;

                    return new BigDecimal(sToken);
                }
            }

            iType = DECIMAL;

            return new BigDecimal(sToken);
        } else if (iType == FLOAT) {
            return new Double(sToken);
        } else if (iType == DECIMAL) {
            return new BigDecimal(sToken);
        }

        return sToken;
    }

    /**
     * return the current position to be used for VIEW processing
     *