gtkscanner.java

java jdk 1.4的源码

                            while ((ch = getChar()) != CHAR_EOF) {
                                if (ch == '"') {
                                    inString = false;
                                    break;
                                } else {
                                    if (ch == '\\') {
                                        ch = getChar();
                                        switch(ch) {
                                            case CHAR_EOF:
                                                break;
                                            case '\\':
                                                sb.append('\\');
                                                break;
                                            case 'n':
                                                sb.append('\n');
                                                break;
                                            case 'r':
                                                sb.append('\r');
                                                break;
                                            case 't':
                                                sb.append('\t');
                                                break;
                                            case 'f':
                                                sb.append('\f');
                                                break;
                                            case 'b':
                                                sb.append('\b');
                                                break;
                                            case '0':
                                            case '1':
                                            case '2':
                                            case '3':
                                            case '4':
                                            case '5':
                                            case '6':
                                            case '7':
                                                int i = ch - '0';
                                                int nextCh = peekNextChar();

                                                if (nextCh >= '0' && nextCh <= '7') {
                                                    ch = getChar();
                                                    i = i * 8 + ch - '0';
                                                    nextCh = peekNextChar();
                                                    if (nextCh >= '0' && nextCh <= '7') {
                                                        ch = getChar();
                                                        i = i * 8 + ch - '0';
                                                    }
                                                }

                                                sb.append((char)i);
                                                break;
                                            default:
                                                sb.append((char)ch);
                                                break;
                                        }
                                    } else {
                                        sb.append((char)ch);
                                    }
                                }
                            }

                            ch = CHAR_EOF;
                        }

                        break;
                    case '\'':
                        if (!scanStringSq) {
                            handleOrdinaryChar();
                        } else {
                            token = TOKEN_STRING;
                            inString = true;
                            
                            sb = new StringBuffer();

                            while ((ch = getChar()) != CHAR_EOF) {
                                if (ch == '\'') {
                                    inString = false;
                                    break;
                                } else {
                                    sb.append((char)ch);
                                }
                            }

                            ch = CHAR_EOF;
                        }

                        break;
                    case '$':
                        if (!scanHexDollar) {
                            handleOrdinaryChar();
                        } else {
                            token = TOKEN_HEX;
                            ch = getChar();
                            scanNumber(false);
                        }

                        break;
                    case '.':
                        if (!scanFloat) {
                            handleOrdinaryChar();
                        } else {
                            token = TOKEN_FLOAT;
                            ch = getChar();
                            scanNumber(true);
                        }

                        break;
                    case '0':
                        if (scanOctal) {
                            token = TOKEN_OCTAL;
                        } else {
                            token = TOKEN_INT;
                        }

                        ch = peekNextChar();

                        if (scanHex && (ch == 'x' || ch == 'X')) {
                            token = TOKEN_HEX;
                            getChar();
                            ch = getChar();
                            if (ch == CHAR_EOF) {
                                token = TOKEN_ERROR;
                                value.longVal = ERR_UNEXP_EOF;
                                break;
                            }

                            if (char2int(ch, 16) < 0) {
                                token = TOKEN_ERROR;
                                value.longVal = ERR_DIGIT_RADIX;
                                ch = CHAR_EOF;
                                break;
                            }
                        } else if (scanBinary && (ch == 'b' || ch == 'B')) {
                            token = TOKEN_BINARY;
                            getChar();
                            ch = getChar();
                            if (ch == CHAR_EOF) {
                                token = TOKEN_ERROR;
                                value.longVal = ERR_UNEXP_EOF;
                                break;
                            }

                            if (char2int(ch, 2) < 0) {
                                token = TOKEN_ERROR;
                                value.longVal = ERR_NON_DIGIT_IN_CONST;
                                ch = CHAR_EOF;
                                break;
                            }
                        } else {
                            ch = '0';
                        }

                        // purposely fall through
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                        scanNumber(false);
                        break;
                    default:
                        handleOrdinaryChar();
                        break;
                }
            }
        } while (ch != CHAR_EOF);

        if (inString) {
            token = TOKEN_ERROR;
            value.longVal = ERR_UNEXP_EOF_IN_STRING;
            sb = null;
        }

        if (sb != null) {
            value.stringVal = sb.toString();
            sb = null;
        }

        if (token == TOKEN_IDENTIFIER) {
            if (scanSymbols) {
                int scope = currScope;
                ScannerKey key = lookupSymbol(value.stringVal);

                if (key != null) {
                    value.stringVal = null;
                    token = TOKEN_SYMBOL;
                    value.longVal = key.value;
                }
            }

            if (token == TOKEN_IDENTIFIER && scanIdentifierNULL & value.stringVal.length() == 4) {
                if ("NULL".equals(caseSensitive ? value.stringVal : value.stringVal.toUpperCase())) {
                    token = TOKEN_IDENTIFIER_NULL;
                }
            }
        }

        nextToken = token;
        nextValue.copyFrom(value);
    }

    private void handleOrdinaryChar() throws IOException {
        if (ch != CHAR_EOF) {
            if (char2Token) {
                token = ch;
            } else {
                token = TOKEN_CHAR;
                value.charVal = (char)ch;
            }

            ch = CHAR_EOF;
        }
    }

    private void checkForIdentifier() throws IOException {
        if (ch != CHAR_EOF && identifierNth.indexOf(peekNextChar()) != -1) {
            token = TOKEN_IDENTIFIER;

            sb = new StringBuffer();
            sb.append((char)ch);

            do {
                ch = getChar();
                sb.append((char)ch);
                ch = peekNextChar();
            } while (ch != CHAR_EOF && identifierNth.indexOf(ch) != -1);

            ch = CHAR_EOF;
        } else if (scanIdentifier1Char) {
            token = TOKEN_IDENTIFIER;
            value.stringVal = String.valueOf((char)ch);

            ch = CHAR_EOF;
        }
    }

    private static int char2int(int c, int base) {
        if (c >= '0' && c <= '9') {
            c -= '0';
        } else if (c >= 'A' && c <= 'Z') {
            c -= 'A' - 10;
        } else if (c >= 'a' && c <= 'z') {
            c -= 'a' - 10;
        } else {
            return -1;
        }

        return c < base ? c : -1;
    }

    private void scanNumber(boolean seenDot) throws IOException {
        boolean inNumber = true;

        if (token == TOKEN_NONE) {
            token = TOKEN_INT;
        }
        
        sb = new StringBuffer(seenDot ? "0." : "");
        sb.append((char)ch);

        do {
            boolean isExponent = (token == TOKEN_FLOAT && (ch == 'e' || ch == 'E'));

            ch = peekNextChar();

            if (char2int(ch, 36) >= 0
                    || (scanFloat && ch == '.')
                    || (isExponent && (ch == '+' || ch == '-'))) {
                ch = getChar();

                switch(ch) {
                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                        sb.append((char)ch);
                        break;
                    case '.':
                        if (token != TOKEN_INT && token != TOKEN_OCTAL) {
                            value.longVal = (token == TOKEN_FLOAT ? ERR_FLOAT_MALFORMED : ERR_FLOAT_RADIX);
                            token = TOKEN_ERROR;
                            inNumber = false;
                        } else {
                            token = TOKEN_FLOAT;
                            sb.append((char)ch);
                        }
                        break;
                    case '+':
                    case '-':
                        if (token != TOKEN_FLOAT) {
                            token = TOKEN_ERROR;
                            value.longVal = ERR_NON_DIGIT_IN_CONST;
                            inNumber = false;
                        } else {
                            sb.append((char)ch);
                        }
                        break;
                    case 'E':
                    case 'e':
                        if ((token != TOKEN_HEX && !scanFloat)
                                || (token != TOKEN_HEX
                                    && token != TOKEN_OCTAL
                                    && token != TOKEN_FLOAT
                                    && token != TOKEN_INT)) {
                            token = TOKEN_ERROR;
                            value.longVal = ERR_NON_DIGIT_IN_CONST;
                            inNumber = false;
                        } else {
                            if (token != TOKEN_HEX) {
                                token = TOKEN_FLOAT;
                            }
                            sb.append((char)ch);
                        }
                        break;
                    default:
                        if (token != TOKEN_HEX) {
                            token = TOKEN_ERROR;
                            value.longVal = ERR_NON_DIGIT_IN_CONST;
                        } else {
                            sb.append((char)ch);
                        }
                        break;
                }
            } else {
                inNumber = false;
            }
        } while (inNumber);

        try {
            switch(token) {
                case TOKEN_INT:
                    value.longVal = Long.parseLong(sb.toString(), 10);
                    break;
                case TOKEN_FLOAT:
                    value.doubleVal = Double.parseDouble(sb.toString());
                    break;
                case TOKEN_HEX:
                    value.longVal = Long.parseLong(sb.toString(), 16);
                    break;
                case TOKEN_OCTAL:
                    value.longVal = Long.parseLong(sb.toString(), 8);
                    break;
                case TOKEN_BINARY:
                    value.longVal = Long.parseLong(sb.toString(), 2);
                    break;
            }
        } catch (NumberFormatException nfe) {
            // PENDING(shannonh) - in some cases this could actually be ERR_DIGIT_RADIX
            token = TOKEN_ERROR;
            value.longVal = ERR_NON_DIGIT_IN_CONST;
        }

        sb = null;
        ch = CHAR_EOF;
    }

    public void printMessage(String message, boolean isError) {
        System.err.print(inputName + ":" + currLine + ": ");

        if (isError) {
            System.err.print("error: ");
        }

        System.err.println(message);
    }

    // PENDING(shannonh) - a good implementation of this method is needed
    public void unexpectedToken(int expected, String symbolName, String message, boolean isError) {
        String prefix = "lexical error or unexpected token, expected valid token";

        if (message != null) {
            prefix += " - " + message;
        }

        printMessage(prefix, isError);
    }

}