token.c

来自「开放源码的编译器open watcom 1.6.0版的源代码」· C语言 代码 · 共 1,187 行 · 第 1/3 页

/****************************************************************************
*
*                            Open Watcom Project
*
*    Portions Copyright (c) 1983-2002 Sybase, Inc. All Rights Reserved.
*
*  ========================================================================
*
*    This file contains Original Code and/or Modifications of Original
*    Code as defined in and that are subject to the Sybase Open Watcom
*    Public License version 1.0 (the 'License'). You may not use this file
*    except in compliance with the License. BY USING THIS FILE YOU AGREE TO
*    ALL TERMS AND CONDITIONS OF THE LICENSE. A copy of the License is
*    provided with the Original Code and Modifications, and is also
*    available at www.sybase.com/developer/opensource.
*
*    The Original Code and all software distributed under the License are
*    distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
*    EXPRESS OR IMPLIED, AND SYBASE AND ALL CONTRIBUTORS HEREBY DISCLAIM
*    ALL SUCH WARRANTIES, INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF
*    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR
*    NON-INFRINGEMENT. Please see the License for the specific language
*    governing rights and limitations under the License.
*
*  ========================================================================
*
* Description:  WHEN YOU FIGURE OUT WHAT THIS FILE DOES, PLEASE
*               DESCRIBE IT HERE!
*
****************************************************************************/


#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>

#include "wic.h"

typedef struct {
    int8 linesBefore;
    int32 lineNum;
    int16 colNum;
    char *fileName;
    FILE *file;
    long int filePos;
    TStates state;       /* tokenizer state */
    char  nextChar;      /* 2-char lookahead is only needed to detect '\' */
    char  _nextNextChar; /* followed by newline and get rid of it. */
                         /* _nextNextChar may only be used in getNextChar*/
    int tokNumAfterNewLine; /* Used for #preprocessor directives */
    int prevTokLineNum;
} TokFilePos;

static TokFilePos       tokF[MAX_INCLUDE_NEST];
static int              currTokF = -1;
static int              currTokLen;
static char             currTok[MAX_TOKEN_SIZE];
static int              currLineIsInclude;
static long             orderLineNum = 1;

#define TOK_FILE        (tokF[currTokF].file)
#define CURR_FILE_POS   (tokF[currTokF].filePos)
#define STATE           (tokF[currTokF].state)
#define NEXT_CHAR       (tokF[currTokF].nextChar)
#define _NEXT_NEXT_CHAR (tokF[currTokF]._nextNextChar)
#define LINE_NUM        (tokF[currTokF].lineNum)
#define COL_NUM         (tokF[currTokF].colNum)
#define LINES_BEFORE    (tokF[currTokF].linesBefore)
#define TOK_NUM_AFTER_NEW_LINE    (tokF[currTokF].tokNumAfterNewLine)
#define TOK_FILE_NAME   (tokF[currTokF].fileName)
#define PREV_TOK_LINE_NUM  (tokF[currTokF].prevTokLineNum)

static int getTabSize(int colNum) {
//    assert(colNum > 0);
    return  TAB_SIZE  -  ((colNum-1) % TAB_SIZE);
}

static void getNextChar(void)
{
    static unsigned long counter = 0;
    enum { COUNTER_STEP = 1024 };

    if (counter++ % COUNTER_STEP == 0) {
        dribble();
    }

    if ((char) NEXT_CHAR == (char) EOF){
        printf("INTERNAL ERROR in getNextChar: Attempted to read beyond EOF!\n");
        return;
    }

    if (NEXT_CHAR == '\t'){
        COL_NUM += getTabSize(COL_NUM);
    } else if (NEXT_CHAR != '\n') {
        COL_NUM++;
    } else {
        g_totalNumLines++;
        LINE_NUM++;
        orderLineNum++;
        COL_NUM = 0;
        TOK_NUM_AFTER_NEW_LINE = 0;
    }

    NEXT_CHAR = _NEXT_NEXT_CHAR;
    if ((char) NEXT_CHAR != (char) EOF){
        _NEXT_NEXT_CHAR = fgetc(TOK_FILE);
        if (STATE != TS_COMMENT) {
            while (_NEXT_NEXT_CHAR == '\n' && NEXT_CHAR == '\\'){
                g_totalNumLines++;
                NEXT_CHAR = fgetc(TOK_FILE);
                _NEXT_NEXT_CHAR = fgetc(TOK_FILE);
                LINE_NUM++;
                COL_NUM = 0;
            }
        }
        if ((char) _NEXT_NEXT_CHAR == (char) EOF && NEXT_CHAR == '\\'){
            NEXT_CHAR = (char) EOF;
        }
    }
}

void pushChar(char ch)
{
    if (currTokLen < MAX_TOKEN_SIZE - 1) {
        currTok[currTokLen++] = ch;
        currTok[currTokLen] = 0;
    }
}

void pushGetNextChar(void)
{
    pushChar(NEXT_CHAR);
    getNextChar();
}

void popChars(int num)  /* Delete last 'num' chars from currTok */
{
    currTokLen -= num;
    if (currTokLen < 0)
    {
        assert(0);
        currTokLen = 0;
    }
    currTok[currTokLen] = 0;
}

static FILE *_openIncludeFile(char *fname, char **newAllocName) {
    FILE *file = NULL;
    char fullName[_MAX_PATH];
    pFDReadInd dir = NULL;
    char *fullNameCopy;

    strcpy(fullName, fname);
    rewindCurrSLListPos(g_opt.incPathList);
    do {
        setDefaultExt(fullName, "h");
        if (fileReadable(fullName)) {
            file = wicFopen(fullName, "r");
            if (file != NULL) {
                pFDReadInd newElem;
                fullNameCopy = wicStrdup(fullName);
                newElem = createFDReadInd(fullNameCopy,
                            dir == NULL ? 0 : dir->readOnly);
                if (!addHTableElem(g_fileTable, newElem)) {
                    reportError(ERR_FILE_WAS_INCLUDED, fullNameCopy);
                    wicFclose(file);
                    zapFDReadInd(newElem);
                    file = NULL;
                }
                break;
            }
        }
        if (!getCurrSLListPosElem(g_opt.incPathList, &dir)) {
            strcpy(fullName, fname);
            setDefaultExt(fullName, "h");
            reportError(ERR_INC_FILE_NOT_FOUND, fullName);
            break;
        }
        sprintf(fullName, "%s\\%s", dir->name, fname);
        incCurrSLListPos(g_opt.incPathList);
    } while (1);

    if (file == NULL) {
        *newAllocName = NULL;
    } else {
        *newAllocName = fullNameCopy;
    }
    return file;
}

static void restoreTokFile(void) {
    currTokF--;
    if (currTokF >= 0) {
        g_currFileName = TOK_FILE_NAME;
        g_currLineNum = LINE_NUM;
        g_currColNum = COL_NUM;
        TOK_FILE = wicFopen(TOK_FILE_NAME, "r");
        if (TOK_FILE == NULL) {
            reportError(ERR_OPEN_FILE, TOK_FILE_NAME, strerror(errno));
            reportError(FATAL_DIE);
        }
        if (fseek(TOK_FILE, CURR_FILE_POS, SEEK_SET) != 0) {
            reportError(ERR_COULD_NOT_SEEK, TOK_FILE_NAME, strerror(errno));
            reportError(FATAL_DIE);
        }
    }
}

char* pushTokFile(char *fname)
{
    if (currTokF >= 0) {
        if (TOK_FILE != NULL) {
            CURR_FILE_POS = ftell(TOK_FILE);
            wicFclose(TOK_FILE);
        }
    }
    currTokF++;
    assert(currLineIsInclude == 0);
    currLineIsInclude = 0;

    if (currTokF >= MAX_INCLUDE_NEST) {
        currTokF--;
        reportError(ERR_NEST_INCLUDE);
        return NULL;
    } else {
        assert(fname != NULL);
        TOK_FILE = _openIncludeFile(fname, &TOK_FILE_NAME);
        if (TOK_FILE == NULL) {
            restoreTokFile();
            return NULL;
        } else {
            debugOut("Push file: %s\n", fname);
            g_currFileName = TOK_FILE_NAME;
            STATE = TS_START;
            LINE_NUM = 1;
            PREV_TOK_LINE_NUM = 1;
            COL_NUM = 0;
            g_currLineNum = LINE_NUM;
            g_currColNum = COL_NUM;
            NEXT_CHAR = 0;
            _NEXT_NEXT_CHAR = fgetc(TOK_FILE);
            TOK_NUM_AFTER_NEW_LINE = 0;
            getNextChar();
        }
    }
    return TOK_FILE_NAME;
}

int popTokFile(void) {
    assert(currTokF >= 0);
    wicFclose(TOK_FILE);
    if (currTokF == 0) {
        g_currFileName = NULL;
        return 0;
    } else {
        debugOut("Pop  file: %s\n", TOK_FILE_NAME);
        restoreTokFile();
        return 1;
    }
}

char* initTokenizer(char *fname)
{
    char *retName;
    currLineIsInclude = 0;
    orderLineNum = 1;
    currTokF = -1;
    g_fileTable = createHTable(32,  FDReadIndHashFunc, FDReadIndCmpFunc);
    retName = pushTokFile(fname);
    if (retName == NULL) {
        wicExit(-1);
        return  NULL; // avoid warning
    }
    return retName;
}

void zapTokenizer(void) {
    zapHTable(g_fileTable, zapFDReadInd);
}

static int skipBlank(void)
{
    int count = 0;

    while (NEXT_CHAR == ' ' || NEXT_CHAR == '\t') {
        if (NEXT_CHAR == ' ') {
            count++;
        } else {
            count += getTabSize(COL_NUM);
        }
        getNextChar();
    }

    return count;
}

int scanId(void)  // Returns 0 iff 1st char is 'L' and 2nd is '"'
{
    char firstChar = NEXT_CHAR;
    pushGetNextChar();
    if ((NEXT_CHAR == '"' || NEXT_CHAR == '\'') && firstChar == 'L') {
        return 0;
    }
    while (isalnum(NEXT_CHAR) || NEXT_CHAR == '_') {
        pushGetNextChar();
    }
    return 1;
}

static WicErrors scanComment(pTokData tokData)
{
    while (1) {
        if (NEXT_CHAR == '*') {
            pushGetNextChar();
            if (NEXT_CHAR == '/') {
                STATE = TS_START;
                getNextChar();
                popChars(1);
                goto NormalReturn;
            }
        } else if (NEXT_CHAR == '\n') {
            getNextChar();
            goto NormalReturn;
        } else if ((char) NEXT_CHAR == (char) EOF) {
            STATE = TS_START;
            return RERR_EOF_IN_COMMENT;
        } else {
            pushGetNextChar();
        }
    }

    NormalReturn:
        tokData->code = Y_PRE_COMMENT;
        tokData->repr.string = registerString(wicStrdup(currTok), FREE_STRING);
        return ERR_NONE;
}

static WicErrors scanCPlusPlusComment(pTokData tokData)
{
    char prevChar = 0;
    while (1) {
        if (NEXT_CHAR == '\n') {
            tokData->code = Y_PRE_COMMENT;
            tokData->repr.string = registerString(wicStrdup(currTok),
                                    FREE_STRING);
            if (prevChar != '\\') {
                STATE = TS_START;
            }
            return ERR_NONE;
        } else {
            prevChar = NEXT_CHAR;
            pushGetNextChar();
        }
    }
}

static char scanESCChar(void) {
    int i;
    char n;

    if( NEXT_CHAR >= '0'  &&  NEXT_CHAR <= '7' ) { /* get octal escape sequence */
        n = 0;
        i = 3;
        while( NEXT_CHAR >= '0'  &&  NEXT_CHAR <= '7' ) {
            n = (n << 3) + NEXT_CHAR - '0';
            --i;
            getNextChar();
            if( i == 0 ) break;
        }
    } else if( NEXT_CHAR == 'x' ) {         /* get hex escape sequence */
        getNextChar();
        n = 0;
        i = 0;
        while(isxdigit(NEXT_CHAR)) {
            n = (n << 4) + NEXT_CHAR - '0';
            getNextChar();
            i++;
        }
        if (i == 0) {
            n = 'x';
        }
    } else {
        switch( NEXT_CHAR ) {
        case 'a':
            n = '\a';
            break;
        case 'b':
            n = '\b';
            break;
        case 'f':
            n = '\f';
            break;
        case 'n':
            n = '\n';
            break;