exlbrowser.l

这是一个Linux下的集成开发环境

/*

Copyright (c) 2004, Mo DeJong

This file is part of Source-Navigator.

Source-Navigator is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as published
by the Free Software Foundation; either version 2, or (at your option)
any later version.

Source-Navigator is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
General Public License for more details.

You should have received a copy of the GNU General Public License along
with Source-Navigator; see the file COPYING.  If not, write to
the Free Software Foundation, 59 Temple Place - Suite 330, Boston,
MA 02111-1307, USA.



*/

/*
 * exlbrowser.l
 *
 * Copyright (C) 2004 Mo DeJong
 *
 * Description:
 * Lex input file for a simple little example language.
 */

%{

#include <ctype.h>
#include <stdlib.h>
#include <stdio.h>
#include "snptools.h"
#include "lexinput.h"
#include "longstr.h"
#include "srchtbl.h"
#include "tcl.h"

#undef YY_INPUT
#define YY_INPUT(buf,r,ms) (r = sn_encoded_input(buf, ms))

#undef yywrap
#define YY_SKIP_YYWRAP

typedef enum {
    OPEN_PAREN,    /* '(' */
    CLOSE_PAREN,   /* ')' */
    OPEN_BRACE,    /* '{' */
    CLOSE_BRACE,   /* '}' */
    SEMICOLON,     /* ';' */

    ASSIGNMENT_OPERATOR, /* '=' */

    LITERAL,           /* An integer literal like 1 */
    SOMEWORD,          /* a sequence of text */
    KEYWORD,           /* any keyword not handled below */
    DECLARE_KEYWORD, /* The keyword "declare" */
    GLOBAL_KEYWORD, /* The keyword "global" */

    DOUBLE_QUOTED_STRING, /* "I am a double quoted string" */

    UNKNOWN,        /* Token that parser does not know about */
    COMMENT,        /* Emit COMMENT token only when debugging */
} TokenType;

typedef enum {
    VAR_READ,
    VAR_WRITE,
    VAR_READWRITE
} VarAccess;

typedef struct Token {
    TokenType type;
    char * strval; /* String value of token, NULL if single character */
    long start_line;
    int start_column;
    long end_line;
    int end_column;
    struct Token* next;
} Token;

/* Uncomment this to see debug output in token parsing stage */
/*#define TOKEN_DEBUG*/

#ifdef TOKEN_DEBUG
FILE* tokenout = NULL;
#endif

static char * token_dump_file = NULL;
static int highlight_file = 0;

Token* tokens_head = NULL;
Token* tokens_tail = NULL;
int token_index = 0; /* current offset from original token_head */

static char group[] = "exl";

#define MAX_SIZE 512

/* FIXME: Can't pass NULL or "" as current function (core dump) */
static char* current_function = (char *) NULL;

/* line number where highlight starts and ends */
static int  current_function_highlight_line;

/* in "declare fog() {}" column of 'f' in "fog" */
static int  current_function_highlight_column_start;

/* in "declare fog() {}" column of 'g' in "fog" */
static int  current_function_highlight_column_end;

/* line where "declare" appears */
static int  current_function_line_start;

/* line where closing brace of function appears */
static int  current_function_line_end;

/* in "declare fog() {}" column of 'd' in "declare" */
static int  current_function_column_start;

/* in "declare fog() {}" column of '}' in "{}" */
static int  current_function_column_end;

static int  current_function_brace_count;

static Token* current_array_token = NULL;

static int result;

static SearchTable * global_var_table = (SearchTable *) NULL;

/* Stores the contents of a special processing mode over
 * multiple lines/rules.
 */
LongString mode_buff;
long mode_start_line;
int mode_start_column;

#define COMMENT_DUMP 0
#define DQSTRING_DUMP 0
#define MATCH_DUMP 0

static YY_BUFFER_STATE original_buffer;

#if MATCH_DUMP
static void matched_pattern(char * pattern, char * text);
#endif

static char* modestring();

static void FreeGlobalEntry(SearchEntry *entry);

void FreeToken(Token* tok);
Token* pop_head_token();
void free_head_token();
void append_token(TokenType type,
                  char* strval,
                  long start_line,
                  int start_column,
                  long end_line,
                  int end_column);

void append_dqstring_token(char* strval,
                           long start_line,
                           int start_column,
                           long end_line,
                           int end_column);

char * TokenTypeToString(Token *tok);

void emit_function_declaration();
void emit_comment();
void emit_dqstring();
void emit_var_access(Token* tok, VarAccess acc);

int yywrap() { return 1; }

%}

%x COMMENT_MODE
%x EXAMPLE
%x DQSTRING
%x TOKEN

ws		[ \t]
wsn		[ \t\n]
symbol		[a-zA-Z_][a-zA-Z0-9_]*
literal		0|[1-9][0-9]*

someword	{symbol}
token		[a-zA-Z0-9_]+

%%
    /* Start in EXAMPLE mode */
    highlight_file = (int) sn_getopt(SN_OPT_HIGHLIGHT);
    token_dump_file = (char *) sn_getopt(SN_OPT_DUMP_TOKENS);
    BEGIN(EXAMPLE);

<EXAMPLE>"/*"		{ /* A C style multi-line comment, just like this! */
#if MATCH_DUMP
    matched_pattern("/*", yytext);
#endif
    BEGIN(COMMENT_MODE);
    sn_advance_column(2);
    LongStringInit(&mode_buff,0);
    mode_start_line = sn_line();
    mode_start_column = sn_column();
}

<COMMENT_MODE>{
  [^\*\n]* {
    #if MATCH_DUMP
    matched_pattern("[^\\*\\n]*", yytext);
    #endif

    #if COMMENT_DUMP
    fprintf(stderr, "comment(1) \"%s\", %d\n", yytext, yyleng);
    #endif
    mode_buff.append( &mode_buff,
        yytext, yyleng );
    sn_advance_column(yyleng);
  }
  [^\*\n]*\n {
    #if MATCH_DUMP
    matched_pattern("[^\\*\\n]*\\n", yytext);
    #endif

    #if COMMENT_DUMP
    fprintf(stderr, "comment(2) \"%s\", %d\n", yytext, yyleng);
    #endif

    mode_buff.append( &mode_buff,
        yytext, yyleng );
    sn_advance_line();
    sn_reset_column();
  }
  \*+[^\*/\n]* {
    #if MATCH_DUMP
    matched_pattern("\\*+[^\\*/\\n]*", yytext);
    #endif

    #if COMMENT_DUMP
    fprintf(stderr, "comment(3) \"%s\", %d\n", yytext, yyleng);
    #endif

    mode_buff.append( &mode_buff,
        yytext, yyleng );
    sn_advance_column(yyleng);
  }
  \*+[^\*/\n]*\n {
    #if MATCH_DUMP
    matched_pattern("\\*+[^\\*/\\n]*\\n", yytext);
    #endif

    #if COMMENT_DUMP
    fprintf(stderr, "comment(4) \"%s\", %d\n", yytext, yyleng);
    #endif

    mode_buff.append( &mode_buff,
        yytext, yyleng );
    sn_advance_line();
    sn_reset_column();
  }
  "*"+"/" {
    #if MATCH_DUMP
    matched_pattern("\\*+/", yytext);
    #endif

    /* Include multiple '*' characters in the comment */
    if (yyleng > 2) {
        int to_append = yyleng;
        char *comment = (char *) yytext + yyleng - 1;
        assert(*comment == '/');
        comment--;
        assert(*comment == '*');
        *comment = '0';
        to_append -= 2;
        mode_buff.append( &mode_buff,
            yytext, to_append );
    }

    emit_comment();
    sn_advance_column(yyleng);
    BEGIN(EXAMPLE);
  }
  /* Uncomment the next rule if you want to check to make sure
   * the above rules cover all possible input. A warning
   * "rule cannot be matched" should be printed by flex.
   */
  /*. {}*/
}

<EXAMPLE>\"\" {
#if MATCH_DUMP
  matched_pattern("\\\"\\\"", yytext);
#endif
  /* FIXME: Can we pass NULL instead of "" after length issues worked out ? */
  append_dqstring_token("",
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);
  sn_advance_column(yyleng);
}

<EXAMPLE>\" {
#if MATCH_DUMP
  matched_pattern("\\\"", yytext);
#endif

#if DQSTRING_DUMP
  fprintf(stderr, "dqstring started at (%d.%d)\n", sn_line(), sn_column());
#endif
  LongStringInit(&mode_buff,0);
  mode_start_line = sn_line();
  mode_start_column = sn_column();
  sn_advance_column(yyleng);
  BEGIN(DQSTRING);
}

<DQSTRING>{
  [^\"\n]* {
    #if MATCH_DUMP
    matched_pattern("[^\\\"\\n]*", yytext);
    #endif

    #if DQSTRING_DUMP
    fprintf(stderr, "dqstring(1) \"%s\", %d\n", yytext, yyleng);
    #endif
    mode_buff.append( &mode_buff,
        yytext, yyleng );
    sn_advance_column(yyleng);
  }
  [^\"\n]*\n {
    #if MATCH_DUMP
    matched_pattern("[^\\\"\\n]*\\n", yytext);
    #endif

    #if DQSTRING_DUMP
    fprintf(stderr, "dqstring(2) \"%s\", %d\n", yytext, yyleng);
    #endif
    mode_buff.append( &mode_buff,
        yytext, yyleng );
    sn_advance_line();
    sn_reset_column();
  }
  (\\\")+ {
    #if MATCH_DUMP
    matched_pattern("(\\\")+", yytext);
    #endif

    #if DQSTRING_DUMP
    fprintf(stderr, "dqstring(3) \"%s\", %d\n", yytext, yyleng);
    #endif

    mode_buff.append( &mode_buff,
        yytext, yyleng );
    sn_advance_column(yyleng);
  }
  \" {
    #if MATCH_DUMP
    matched_pattern("\\\"", yytext);
    #endif

    sn_advance_column(yyleng);
    emit_dqstring();
    BEGIN(EXAMPLE);
  }
  /* Uncomment the next rule if you want to check to make sure
   * the above rules cover all possible input. A warning
   * "rule cannot be matched" should be printed by flex.
   */
  /*. {}*/
}

<EXAMPLE>"(" {
#if MATCH_DUMP
  matched_pattern("(", yytext);
#endif

  append_token(OPEN_PAREN, NULL,
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);
  sn_advance_column(yyleng);
}

<EXAMPLE>")" {
#if MATCH_DUMP
  matched_pattern(")", yytext);
#endif
  append_token(CLOSE_PAREN, NULL,
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);
  sn_advance_column(yyleng);
}

<EXAMPLE>"{" {
#if MATCH_DUMP
  matched_pattern("{", yytext);
#endif
  append_token(OPEN_BRACE, NULL,
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);
  sn_advance_column(yyleng);
}

<EXAMPLE>"}" {
#if MATCH_DUMP
  matched_pattern("}", yytext);
#endif
  append_token(CLOSE_BRACE, NULL,
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);
  sn_advance_column(yyleng);
}

<EXAMPLE>";" {
#if MATCH_DUMP
  matched_pattern(";", yytext);
#endif
  append_token(SEMICOLON, NULL,
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);
  sn_advance_column(yyleng);
}

<EXAMPLE>"=" {
#if MATCH_DUMP
  matched_pattern("=", yytext);
#endif
  append_token(ASSIGNMENT_OPERATOR, NULL,
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);
  sn_advance_column(yyleng);
}

<EXAMPLE>"declare" {
#if MATCH_DUMP
  matched_pattern("declare", yytext);
#endif

  append_token(DECLARE_KEYWORD, NULL,
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);
  sn_advance_column(yyleng);
}

<EXAMPLE>"global" {
#if MATCH_DUMP
  matched_pattern("global", yytext);
#endif
  append_token(GLOBAL_KEYWORD, NULL,
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);
  sn_advance_column(yyleng);
}

<EXAMPLE>"return" {
#if MATCH_DUMP
  matched_pattern("return", yytext);
#endif
  append_token(KEYWORD, yytext,
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);
  sn_advance_column(yyleng);
}

<EXAMPLE>{literal} {
#if MATCH_DUMP
  matched_pattern("{literal}", yytext);
#endif
  append_token(LITERAL, yytext,
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);
  sn_advance_column(yyleng);
}

<EXAMPLE>{someword} {
#if MATCH_DUMP
  matched_pattern("{someword}", yytext);
#endif

  append_token(SOMEWORD, yytext,
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);
  sn_advance_column(yyleng);
}

<EXAMPLE>{wsn}+ {
  char* x, *y;
  #if MATCH_DUMP
  matched_pattern("{wsn}+", yytext);
  #endif

  for (x=yytext, y=NULL; *x ; x++) {
    if (*x == '\n') {
        y=x+1;
        sn_advance_line();
    }
  }
  if (y == NULL) {
    sn_advance_column(yyleng);
  } else {
    sn_reset_column();
    sn_advance_column(x-y);
  }
}

<EXAMPLE>.		{
#if MATCH_DUMP
  matched_pattern(".", yytext);
#endif

  /* Add an UNKNOWN token for each
   * character that we don't know
   * how to deal with.
   */

  append_token(UNKNOWN, yytext,
          sn_line(),
          sn_column(),
          sn_line(),
          sn_column() + yyleng);

  /*fprintf(stderr, "adding unknown token for \"%s\"\n", yytext);*/

  sn_advance_column(yyleng); /* eat text */
}

<TOKEN>"DECLARE_KEYWORD SOMEWORD OPEN_PAREN CLOSE_PAREN OPEN_BRACE " {
  int parens, noargs;
  LongString abuff;

#ifdef TOKEN_DEBUG
  fprintf(tokenout, "declaration at token %d\n", token_index);
  fprintf(tokenout, "match text was \"%s\"\n", yytext);
#endif
  assert(tokens_head->type == DECLARE_KEYWORD);

  current_function_line_start = tokens_head->start_line;
  current_function_column_start = tokens_head->start_column;

  sn_highlight(SN_HIGH_KEYWORD,
          tokens_head->start_line, tokens_head->start_column,
          tokens_head->end_line, tokens_head->end_column);

  free_head_token(); /* DECLARE_KEYWORD */

  if (current_function != NULL) {
    panic("Can't declare function inside another function");
  }
  current_function = SN_StrDup(tokens_head->strval);

  current_function_highlight_line = tokens_head->start_line;
  current_function_highlight_column_start = tokens_head->start_column;
  current_function_highlight_column_end = tokens_head->end_column;

  free_head_token(); /* SOMEWORD */
  free_head_token(); /* OPEN_PAREN */
  free_head_token(); /* CLOSE_PAREN */
  free_head_token(); /* OPEN_BRACE */

  current_function_brace_count = 1;
}

<TOKEN>"OPEN_BRACE" {
  if (current_function) {
    current_function_brace_count++;
  }
  free_head_token(); /* OPEN_BRACE */
}

<TOKEN>"CLOSE_BRACE" {
  if (current_function && (--current_function_brace_count == 0)) {
#ifdef TOKEN_DEBUG
    fprintf(tokenout, "end of function %s at token %d\n",
        current_function, token_index);
#endif

    current_function_line_end = tokens_head->end_line;
    current_function_column_end = tokens_head->end_column;

    emit_function_declaration();
  }

  free_head_token(); /* CLOSE_BRACE */
}

<TOKEN>"GLOBAL_KEYWORD SOMEWORD SEMICOLON" {
  SearchEntry entry;

#ifdef TOKEN_DEBUG
  fprintf(tokenout, "global statement found at %d\n", token_index);
#endif

  /* FIXME: If global keyword is used outside of a function,
     should we mark it as a keyword? We could mark it as
     a keyword and a syntax error also */
  sn_highlight(SN_HIGH_KEYWORD,
          tokens_head->start_line, tokens_head->start_column,
          tokens_head->end_line, tokens_head->end_column);

  assert(tokens_head->type == GLOBAL_KEYWORD);
  free_head_token(); /* GLOBAL_KEYWORD */

  if (current_function && (global_var_table == NULL)) {
    global_var_table = SearchTableCreate(100, SEARCH_HASH_TABLE, FreeGlobalEntry);