parser.g

SRI international 发布的OAA框架软件

#header <<
#include "charbuf.h"
#include "stringbuffer.h"
#include "libicl.h"
#include "libicl_private.h"
#include "glib.h"
#include <stdio.h>
#ifdef _WINDOWS
#include "oaa-windows.h"
#endif

/* Abstract Syntax Tree fields */
#define AST_FIELDS int isDefined; int type; char* tokenData; size_t tokenLen;
/* Remember column numbers */
#define ZZCOL
/* We define syntax error function */
#define USER_ZZSYN
/* Token buffer size */
/* #define ZZLEXBUFSIZE 10485760*/
#define ZZLEXBUFSIZE 10485760

extern GByteArray* parser_dblQuoteBuf;
extern char* parser_tmpStrIn;
#ifdef NORMAL_GC
extern void* GC_debug_malloc(size_t, const char*, int);
extern char* gc_strdup(char*);
#endif

#ifdef NORMAL_GC
#define zzcr_ast(ast, attr, ttype, text) { \
    if(((ttype) == ICLDATAQ) || \
       ((ttype) == DBLQUOTED)) { \
        char* buf = parser_dblQuoteBuf->data; \
        g_byte_array_free(parser_dblQuoteBuf, FALSE); \
        /* \
        char* buf = (char*)malloc(parser_dblQuoteLen + 1); \
        memcpy(buf, text, parser_dblQuoteLen); \
        buf[parser_dblQuoteLen] = '\0'; \
        */ \
        ast->tokenData = buf; \
        ast->tokenLen = parser_dblQuoteLen; \
    } \
    else { \
        if(zzbufovf) { \
            ast->tokenLen = ZZLEXBUFSIZE; \
            ast->tokenData = (char*)GC_debug_malloc(ZZLEXBUFSIZE + 1, __FILE__, __LINE__); \
            strncpy(ast->tokenData, text, ZZLEXBUFSIZE); \
            fprintf(stderr, "ZZLEXBUFSIZE overflowed (%i)\n", ZZLEXBUFSIZE); \
        } \
        else { \
            ast->tokenData = gc_strdup(text); \
            ast->tokenLen = strlen(text); \
        } \
    } \
    ast->type = (ttype); \
    ast->isDefined = 1; \
    ast->right = NULL; \
    ast->down = NULL; \
    /* printf("zzcr_ast at pointer %p\n", ast); */ \
};

/*
#define zzd_ast(t) { \
  GC_debug_free(t->tokenData); \
  fprintf(stderr, "freed token data at pointer %p\n", t); \
  t->isDefined = 0; \
} \
*/
#undef zzd_ast

#else
#define zzcr_ast(ast, attr, ttype, text) { \
    if(((ttype) == ICLDATAQ) || \
       ((ttype) == DBLQUOTED)) { \
        char* buf = parser_dblQuoteBuf->data; \
        g_byte_array_free(parser_dblQuoteBuf, FALSE); \
        /* \
        char* buf = (char*)malloc(parser_dblQuoteLen + 1); \
        memcpy(buf, text, parser_dblQuoteLen); \
        buf[parser_dblQuoteLen] = '\0'; \
        */ \
        ast->tokenData = buf; \
        ast->tokenLen = parser_dblQuoteLen; \
    } \
    else { \
        if(zzbufovf) { \
            ast->tokenLen = ZZLEXBUFSIZE; \
            ast->tokenData = (char*)malloc(ZZLEXBUFSIZE + 1); \
            strncpy(ast->tokenData, text, ZZLEXBUFSIZE); \
            fprintf(stderr, "ZZLEXBUFSIZE overflowed (%i)\n", ZZLEXBUFSIZE); \
        } \
        else { \
            ast->tokenData = strdup(text); \
            ast->tokenLen = strlen(text); \
        } \
    } \
    ast->type = (ttype); \
    ast->isDefined = 1; \
    ast->right = NULL; \
    ast->down = NULL; \
    /* printf("zzcr_ast at pointer %p\n", ast); */ \
};

/*
#define zzd_ast(t) { \
  free(t->tokenData); \
  t->isDefined = 0; \
} \
*/
#undef zzd_ast

#endif

extern int parser_dblQuoteLen;
extern ICLTerm* parser_getTermFromString(char* str, size_t len);
extern ICLTerm* parser_getTermFromStringDebug(char* str, size_t len);
extern int parser_getNetTermFromBuf(ICLTerm** result, stringbuffer_t* buf);
/*extern void zzsyn(char *text, int tok, char *egroup, SetWordType *eset, int etok, int k, char *bad_text);*/
extern int parser_error;
extern int parser_setDebug(int b);
extern void handleDblQuotedData();
>>

<<

extern int zzSyntaxErrCount; /* defined in err.h */
extern void (*zzerr)(const char*); /* defined in dlgauto.h */
int parser_error = FALSE;
int parser_DEBUG = FALSE;
int parser_dblQuoteLen;
GByteArray* parser_dblQuoteBuf = NULL;
char* parser_tmpStrIn = NULL;
ICLTerm* parser_getTermFromStringReal(char* str, size_t len, int debug);
static stringbuffer_t* currentBuf = NULL;
enum parser_state_vals {parser_dblquote_CONTINUE, parser_dblquote_FOUNDQUOTE};
static int parser_dblquote_state = 0;
static size_t parser_savedIndex = 0;
static int parser_savedIndexSet = FALSE;

/**
 * Binary data isn't handled well by pccts, so we do it ourselves.  This
 * function relies on the fact that we use our own stringbuffer type to
 * buffer intermediate data.
 */
void handleDblQuotedData() {
    struct zzdlg_state dlgState;
    int done = FALSE;
    int nextChar = 0;
    char toAppend[1];

    /* we save the dlg automaton state each time--could do it just once, but it's */
    /* not that expensive */
    zzsave_dlg_state(&dlgState);

    /* if we've been here before without finishing the data, continue where we left off */
    if(parser_savedIndexSet) {
        stringbuffer_setIndex(currentBuf, parser_savedIndex);
    }
    else {
        parser_dblQuoteBuf = g_byte_array_new();
    }

    /* A tiny state machine of two states--CONTINUE or FOUNDQUOTE; we enter FOUNDQUOTE */
    /* when we find a double quote, so that we can check if it is the ending double quote */
    /* or just a quadruple quote (a quoted double quote: "").  If we ever hit EOF, we */
    /* save the index in the currentBuf object and return so that the pccts automaton */
    /* can fail to find the token correctly */
    while(!done) {
        nextChar = (*(dlgState.func_ptr))();
        if(nextChar == EOF) {
            done = TRUE;
            //g_byte_array_free(parser_dblQuoteBuf, TRUE);
            parser_dblQuoteLen = 0;
            zzrestore_dlg_state(&dlgState);
            parser_savedIndex = stringbuffer_getIndex(currentBuf);
            parser_savedIndexSet = TRUE;
            return;
        }
        switch(parser_dblquote_state) {
        case 0:
            if(nextChar == '"') {
                parser_dblquote_state = parser_dblquote_FOUNDQUOTE;
                break;
            }
            else {
                toAppend[0] = nextChar;
                /* printf("Adding char %o\n", nextChar); */
                g_byte_array_append(parser_dblQuoteBuf, toAppend, 1);
            }
            break;
        case 1:
            parser_dblquote_state = parser_dblquote_CONTINUE;
            if(nextChar == '"') {
                /* printf("Adding char %o\n", '"'); */
                g_byte_array_append(parser_dblQuoteBuf, "\"", 1);
            }
            else {
                /* done */
                stringbuffer_rewind(currentBuf, 2);
                done = TRUE;
                parser_dblQuoteLen = parser_dblQuoteBuf->len;
                zzrestore_dlg_state(&dlgState);
                parser_savedIndex = 0;
                parser_savedIndexSet = FALSE;
                return;
            }
            break;
        }
    }
}

void parser_errorWriter(const char* s) {
    int i = 0;
    if(parser_DEBUG) {
        fprintf(stderr, "Error with text: [");
        for(i = 0; s[i] != '\0'; ++i) {
            fprintf(stderr, " %o ", s[i]);
        }
        fprintf(stderr, "\n");
        zzerrstd(s);
    }
}

int parser_setDebug(int b) {
    int ret = parser_DEBUG;
    parser_DEBUG = b;
    return ret;
}

void zzsynstd(char *text, int tok, char *egroup, SetWordType *eset, int etok, int k, char *bad_text) {
    zzSyntaxErrCount++;                             /* MR11 */
	fprintf(stderr, "line %d: syntax error at \"%s\"", zzline, (tok==zzEOF_TOKEN)?"EOF":bad_text);
	if ( !etok && !eset ) {fprintf(stderr, "\n"); return;}
	if ( k==1 ) fprintf(stderr, " missing");
	else
	{
		fprintf(stderr, "; \"%s\" not", bad_text);
		if ( zzset_deg(eset)>1 ) fprintf(stderr, " in");
	}
	if ( zzset_deg(eset)>0 ) zzedecode(eset);
	else fprintf(stderr, " %s", zztokens[etok]);
	if ( strlen(egroup) > 0 ) fprintf(stderr, " in %s", egroup);
	fprintf(stderr, "\n");

}

void zzsyn(char *text, int tok, char *egroup, SetWordType *eset, int etok, int k, char *bad_text) {
    if(parser_DEBUG) {
        fprintf(stderr, "standard error text:\n");
        zzsynstd(text, tok, egroup, eset, etok, k, bad_text);
        fprintf(stderr, "end standard error text\n");
    }
    parser_error = TRUE;
}

void printAst(AST* a) {
    if(a == NULL) {
        return;
    }
    if(a->isDefined != 1) {
        return;
    }
    printf(" [type %i][text %s] ", a->type, a->tokenData);
}

void before(AST* a) {
    (void)a;
    printf(" (");
}

void after(AST* a) {
    (void)a;
    printf(")");
}

int currentBuf_read() {
    return stringbuffer_read(currentBuf);
}

void traverse(AST* tree, 
              void (*func)(AST*, void*),
              void (*pre)(AST*, void*), 
              void (*post)(AST*, void*),
              void* data) {
  while(tree != NULL) {
    if(tree->down != NULL) {
      if(pre != NULL) {
        (*pre)(tree, data);
      }
    }
    if(func != NULL) {
      (*func)(tree, data);
    }
    traverse(tree->down, func, pre, post, data);
    if(tree->down != NULL) {
      if(post != NULL) {
        (*post)(tree, data);
      }
    }
    tree = tree->right;
  }
}

void parser_freeASTData(AST* a, void* dummy) {
  (void)dummy;
  if(a->isDefined == 1) {
#ifdef NORMAL_GC
    GC_debug_free(a->tokenData);
#else
    free(a->tokenData);
#endif
  }
}

void parser_freeTree(AST* root) {
  traverse(root, parser_freeASTData, NULL, NULL, NULL);
}

int astToTerm(AST* root, ICLTerm** result, int debug) {
    int done = FALSE;
    GPtrArray* parentsList = g_ptr_array_new();
    GPtrArray* termList = g_ptr_array_new();
    AST* currentAST = NULL;
    ICLTerm* currentTerm = NULL;
    gint64* intp;
    double* dblp;
    char* endp;
    ICLStructType* st;
    ICLGroupType* gt;
    int parentIndex = 0;

    if(debug || parser_DEBUG) {
        debug = TRUE;
    }

    g_ptr_array_add(parentsList, root);
    while(parentsList->len != 0) {
        currentAST = (AST*)g_ptr_array_index(parentsList, parentsList->len - 1);
        if(currentAST->isDefined != 1) {
            (void)g_ptr_array_remove_index(parentsList, parentIndex);
            continue;
        }
#ifdef NORMAL_GC
        currentTerm = (ICLTerm*)GC_debug_malloc(sizeof(ICLTerm), __FILE__, __LINE__);
        printf("Allocated new term at %p\n", currentTerm);
#else
        currentTerm = (ICLTerm*)malloc(sizeof(ICLTerm));
#endif
        currentTerm->magic_cookie = ICL_MAGIC_COOKIE;
        currentTerm->refCount = 1;
        currentTerm->glibAlloc = 0;
        currentTerm->hadQuotes = 0;
        g_ptr_array_add(termList, currentTerm);
        switch(currentAST->type) {
        case STR:
            currentTerm->iclType = icl_str_type;
            if(currentAST->tokenData[0] == '\'') {
                currentTerm->hadQuotes = 1;
            }
            icl_stRemoveQuotes(currentAST->tokenData);
            icl_stUndoubleQuotes(currentAST->tokenData);
            currentTerm->p = currentAST->tokenData;
            currentTerm->len = strlen(currentAST->tokenData);
            if(debug) {
                printf("Found str: %s\n", currentTerm->p);
            }
            break;
        case VAR:
            currentTerm->iclType = icl_var_type;
            currentTerm->p = currentAST->tokenData;
            currentTerm->len = currentAST->tokenLen;
            if(debug) {
                printf("Found var: %s\n", currentTerm->p);
            }
            break;
        case INT:
            currentTerm->iclType = icl_int_type;
#ifdef NORMAL_GC
            intp = (gint64*)GC_debug_malloc(sizeof(gint64), __FILE__, __LINE__);
#else
            intp = (gint64*)malloc(sizeof(gint64));
#endif
            *intp = strtoll(currentAST->tokenData, &endp, 10);
            if(*endp != '\0') {
                if(debug) {
                    printf("Bad int\n");
                }
                parser_error = TRUE;
                g_ptr_array_free(termList, TRUE);
                g_ptr_array_free(parentsList, TRUE);
                return FALSE;
            }
            free(currentAST->tokenData);
            currentAST->tokenData = NULL;
            currentAST->tokenLen = 0;
            currentTerm->p = intp;
            currentTerm->len = sizeof(gint64);
            if(debug) {
                printf("Found int: %lld\n", *((gint64*)currentTerm->p));
            }
            break;
        case FLOAT:
            currentTerm->iclType = icl_float_type;
#ifdef NORMAL_GC
            dblp = (double*)GC_debug_malloc(sizeof(double), __FILE__, __LINE__);
#else
            dblp = (double*)malloc(sizeof(double));
#endif
            *dblp = strtod(currentAST->tokenData, &endp);
            if(*endp != '\0') {
                if(debug) {
                    printf("Bad float\n");
                }
                parser_error = TRUE;
                g_ptr_array_free(termList, TRUE);
                g_ptr_array_free(parentsList, TRUE);                
                return FALSE;
            }
            free(currentAST->tokenData);
            currentAST->tokenData = NULL;
            currentAST->tokenLen = 0;
            currentTerm->p = dblp;
            currentTerm->len = sizeof(double);
            if(debug) {
                printf("Found float: %lf\n", *((double*)currentTerm->p));
            }
            break;
        case STRUCT:
            currentTerm->iclType = icl_struct_type;
#ifdef NORMAL_GC
            st = (ICLStructType*)GC_debug_malloc(sizeof(ICLStructType), __FILE__, __LINE__);
#else
            st = (ICLStructType*)malloc(sizeof(ICLStructType));
#endif
            st->name = currentAST->tokenData;
            st->args = NULL;
            st->numArgs = 0;
            currentTerm->p = st;
            currentTerm->len = sizeof(st);
            if(debug) {
                printf("Found struct: name at %p: %s\n", st->name, st->name);
            }
            break;
        case LIST:
            free(currentAST->tokenData);
            currentTerm->iclType = icl_list_type;
            currentTerm->p = NULL;
            currentTerm->len = sizeof(ICLListType*);
            if(debug) {
                printf("Found list\n");
            }