tokenizer.java

用JAVA实现了,将DB装入内存,内存由DBMS控制,实现简单数据库的创建、数据表的创建、记录插入、查询以及表的删除。

package org.kdb.parser;

import org.kdb.KdbException;

import java.util.HashMap;
import java.util.Locale;

import org.kdb.Trace;

/**

 * Date: 2007-10-29
 * Time: 11:18:16
 */
public class Tokenizer {
    private static final int NO_TYPE = 0,
            NAME = 1,
            LONG_NAME = 2,
            SPECIAL = 3,
            NUMBER = 4,
            FLOAT = 5,
            STRING = 6,
            LONG = 7,
            DECIMAL = 8,
            BOOLEAN = 9,
            DATE = 10,
            TIME = 11,
            TIMESTAMP = 12,
            NULL = 13,
            NAMED_PARAM = 14;

    // used only internally
    private static final int QUOTED_IDENTIFIER = 15,
            REMARK_LINE = 16,
            REMARK = 17;
    private String sCommand;
    private int iLength;
    private int iIndex;
    private int tokenIndex;
    private int nextTokenIndex;
    private int beginIndex;
    private int iType;
    private String sToken;
    private int indexLongNameFirst = -1;
    private String sLongNameFirst = null;
    private int typeLongNameFirst;

    // getToken() will clear LongNameFirst unless retainFirst is set.
    private boolean retainFirst = false;

    //    private String           sLongNameLast;
    // WAIT.  Don't do anything before popping another Token (because the
    // state variables aren't set properly due to a call of wait()).
    private boolean bWait;
    private boolean lastTokenQuotedID;

    // literals that are values
    static HashMap<String, Integer> valueTokens;

    static {
        valueTokens = new HashMap<String, Integer>();

        valueTokens.put(Token.T_NULL, NULL);
        valueTokens.put(Token.T_TRUE, BOOLEAN);
        valueTokens.put(Token.T_FALSE, BOOLEAN);
    }

    public Tokenizer() {
    }

    public Tokenizer(String s) {

        sCommand = s;
        iLength = s.length();
        iIndex = 0;
    }

    public void reset(String s) {

        sCommand = s;
        iLength = s.length();
        iIndex = 0;
        tokenIndex = 0;
        nextTokenIndex = 0;
        beginIndex = 0;
        iType = NO_TYPE;
        typeLongNameFirst = NO_TYPE;
        sToken = null;
        indexLongNameFirst = -1;
        sLongNameFirst = null;

//        sLongNameLast  = null;
        bWait = false;
        lastTokenQuotedID = false;
        retainFirst = false;
    }

    /**
     * @throws KdbException
     */
    void back() throws KdbException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        nextTokenIndex = iIndex;
        iIndex = (indexLongNameFirst != -1) ? indexLongNameFirst : tokenIndex;
        bWait = true;
    }

    /**
     * get the given token or throw
     * <p/>
     * for commands and simple unquoted identifiers only
     *
     * @param match
     * @throws KdbException
     */
    String getThis(String match) throws KdbException {

        getToken();
        matchThis(match);

        return sToken;
    }

    /**
     * for commands and simple unquoted identifiers only
     */
    void matchThis(String match) throws KdbException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        if (!sToken.equals(match) || iType == QUOTED_IDENTIFIER
                || iType == LONG_NAME) {
            String token = iType == LONG_NAME ? sLongNameFirst
                    : sToken;

            throw Trace.error(Trace.UNEXPECTED_TOKEN, Trace.TOKEN_REQUIRED,
                    new Object[]{
                            token, match
                    });
        }
    }

    void throwUnexpected() throws KdbException {

        String token = iType == LONG_NAME ? sLongNameFirst
                : sToken;

        throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
    }

    /**
     * Used for commands only
     *
     * @param match
     */
    public boolean isGetThis(String match) throws KdbException {

        getToken();

        if (iType != QUOTED_IDENTIFIER && iType != LONG_NAME
                && sToken.equals(match)) {
            return true;
        }

        back();

        return false;
    }

    /**
     * this methode is called before other wasXXX methods and takes
     * precedence
     */
    boolean wasValue() throws KdbException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        switch (iType) {

            case STRING:
            case NUMBER:
            case LONG:
            case FLOAT:
            case DECIMAL:
            case BOOLEAN:
            case NULL:
                return true;

            default:
                return false;
        }
    }

    boolean wasQuotedIdentifier() throws KdbException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        return lastTokenQuotedID;

        // iType won't help for LONG_NAMEs.
        //return iType == QUOTED_IDENTIFIER;
    }

    boolean wasFirstQuotedIdentifier() throws KdbException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        return (typeLongNameFirst == QUOTED_IDENTIFIER);
    }

    /**
     * Method declaration
     *
     * @return
     */
    boolean wasLongName() throws KdbException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        return iType == LONG_NAME;
    }

    /**
     * Simple Name means a quoted or unquoted identifier without
     * qualifiers provided it is not in the hKeyword list.
     *
     * @return
     */
    boolean wasSimpleName() throws KdbException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        if (iType == QUOTED_IDENTIFIER && sToken.length() != 0) {
            return true;
        }

        if (iType != NAME) {
            return false;
        }

        return !Token.isKeyword(sToken);
    }

    /**
     * checks whether the previously obtained token was a (named) parameter
     *
     * @return true if the previously obtained token was a (named) parameter
     */
    boolean wasParameter() throws KdbException {

        Trace.doAssert(!bWait, "Querying state when in Wait mode");

        return (iType == NAMED_PARAM);
    }

    /**
     * Name means all quoted and unquoted identifiers plus any word not in the
     * hKeyword list.
     *
     * @return true if it's a name
     */
    boolean wasName() throws KdbException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        if (iType == QUOTED_IDENTIFIER) {
            return true;
        }

        if (iType != NAME && iType != LONG_NAME) {
            return false;
        }

        return !Token.isKeyword(sToken);
    }

    String getLongNamePre() throws KdbException {
        return null;
    }

    /**
     * Return first part of long name
     *
     * @return
     */
    String getLongNameFirst() throws KdbException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        return sLongNameFirst;
    }

    boolean wasSimpleToken() throws KdbException {
        return iType != QUOTED_IDENTIFIER && iType != LONG_NAME
                && iType != STRING && iType != NAMED_PARAM;
    }

    String getSimpleToken() throws KdbException {

        getToken();

        if (!wasSimpleToken()) {
            String token = iType == LONG_NAME ? sLongNameFirst
                    : sToken;

            throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
        }

        return sToken;
    }

    public boolean wasThis(String match) throws KdbException {

        if (sToken.equals(match) && iType != QUOTED_IDENTIFIER
                && iType != LONG_NAME && iType != STRING) {
            return true;
        }

        return false;
    }

    /**
     * getName() is more broad than getSimpleName() in that it includes
     * 2-part names as well
     *
     * @return popped name
     * @throws KdbException if next token is not an AName
     */
    public String getName() throws KdbException {

        getToken();

        if (!wasName()) {
//            throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken);
        }

        return sToken;
    }

    /**
     * Returns a single, unqualified name (identifier)
     *
     * @return name
     * @throws KdbException
     */
    public String getSimpleName() throws KdbException {

        getToken();

        if (!wasSimpleName()) {
            String token = iType == LONG_NAME ? sLongNameFirst
                    : sToken;

            throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
        }

        return sToken;
    }

    /**
     * Return any token.
     *
     * @return
     * @throws KdbException
     */
    public String getString() throws KdbException {

        getToken();

        return sToken;
    }

//    int getInt() throws KdbException {
//
//        long v = getBigint();
//
//        if (v > Integer.MAX_VALUE || v < Integer.MIN_VALUE) {
//            throw Trace.error(Trace.WRONG_DATA_TYPE,
//                              Types.getTypeString(getType()));
//        }
//
//        return (int) v;
//    }
//
//    static BigDecimal LONG_MAX_VALUE_INCREMENT =
//        BigDecimal.getType(Long.MAX_VALUE).add(BigDecimal.getType(1));
//
//    long getBigint() throws KdbException {
//
//        boolean minus = false;
//
//        getToken();
//
//        if (sToken.equals("-")) {
//            minus = true;
//
//            getToken();
//        }
//
//        Object o = getAsValue();
//        int    t = getType();
//
//        switch (t) {
//
//            case Types.INTEGER :
//            case Types.BIGINT :
//                break;
//
//            case Types.DECIMAL :
//
//                // only Long.MAX_VALUE + 1 together with minus is acceptable
//                if (minus && LONG_MAX_VALUE_INCREMENT.equals(o)) {
//                    return Long.MIN_VALUE;
//                }
//            default :
//                throw Trace.error(Trace.WRONG_DATA_TYPE,
//                                  Types.getTypeString(t));
//        }
//
//        long v = ((Number) o).longValue();
//
//        return minus ? -v
//                     : v;
//    }
//
//    Object getInType(int type) throws KdbException {
//
//        getToken();
//
//        Object o = getAsValue();
//        int    t = getType();
//
//        if (t != type) {
//            throw Trace.error(Trace.WRONG_DATA_TYPE, Types.getTypeString(t));
//        }
//
//        return o;
//    }
//
//    /**
//     *
//     *
//     *
//     * @return
//     */
//    public int getType() throws KdbException {
//
//        if (bWait) {
//            Trace.doAssert(false, "Querying state when in Wait mode");
//        }
//
//        // todo: make sure it's used only for Values!
//        // todo: synchronize iType with hColumn
//        switch (iType) {
//
//            case STRING :
//                return Types.VARCHAR;
//
//            case NUMBER :
//                return Types.INTEGER;
//
//            case LONG :
//                return Types.BIGINT;
//
//            case FLOAT :
//                return Types.DOUBLE;
//
//            case DECIMAL :
//                return Types.DECIMAL;
//
//            case BOOLEAN :
//                return Types.BOOLEAN;
//
//            case DATE :
//                return Types.DATE;
//
//            case TIME :
//                return Types.TIME;
//
//            case TIMESTAMP :
//                return Types.TIMESTAMP;
//
//            default :
//                return Types.NULL;
//        }
//    }

    /**
     * Method declaration
     *
     * @return
     * @throws KdbException
     */
//    Object getAsValue() throws KdbException {
//
//        if (!wasValue()) {
//            throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken);
//        }
//
//        switch (iType) {
//
//            case NULL:
//                return null;
//
//            case STRING:
//
//                //fredt - no longer returning string with a singlequote as last char
//                return sToken;
//
//            case LONG:
//                return ValuePool.getLong(Long.parseLong(sToken));
//
//            case NUMBER:
//
//                // fredt - this returns unsigned values which are later negated.
//                // as a result Integer.MIN_VALUE or Long.MIN_VALUE are promoted
//                // to a wider type.
//                if (sToken.length() < 11) {
//                    try {
//                        return ValuePool.getInt(Integer.parseInt(sToken));
//                    } catch (Exception e1) {
//                    }
//                }
//
//                if (sToken.length() < 20) {
//                    try {
//                        iType = LONG;