exlbrowser.l

这是一个Linux下的集成开发环境

  }

  /* Insert the variable name into the function's global table */
  assert(tokens_head->type == SOMEWORD);

  if (current_function) {
    entry.key = tokens_head->strval;
    entry.key_len = -1;

    if (global_var_table->search( &global_var_table, entry ) == NULL)
    {
      entry.data = NULL;
      entry.data_len = 0;
      entry.flag = SEARCH_DUP_KEY; /* add copy of entry.key */
      global_var_table->add( &global_var_table, entry );

#ifdef TOKEN_DEBUG
  fprintf(tokenout, "added global \"%s\"\n", entry.key);
#endif
    }

    sn_highlight(SN_HIGH_VAR_GLOBAL,
        tokens_head->start_line, tokens_head->start_column,
        tokens_head->end_line, tokens_head->end_column);
  }

  free_head_token(); /* SOMEWORD */
  free_head_token(); /* SEMICOLON */
}

<TOKEN>"SOMEWORD OPEN_PAREN" {
  char* fname;
  int line;
  int ref_from_scope_type;

#ifdef TOKEN_DEBUG
  fprintf(tokenout, "found function call tokens at %d\n", token_index);
  fprintf(tokenout, "match text was \"%s\"\n", yytext);
#endif

  fname = tokens_head->strval;
#ifdef TOKEN_DEBUG
  fprintf(tokenout, "function name is \"%s\"\n", fname);
#endif
  line = tokens_head->start_line;

  /*
   * Pass SN_GLOBAL_NAMESPACE if the xref is outside of
   * a function, otherwise pass SN_FUNC_DEF.
   */   

  if (current_function == NULL) {
    ref_from_scope_type = SN_GLOBAL_NAMESPACE;
  } else {
    ref_from_scope_type = SN_FUNC_DEF;
  }

  result = sn_insert_xref(SN_REF_TO_FUNCTION,
                 ref_from_scope_type,
                 SN_REF_SCOPE_GLOBAL,
                 NULL,
                 current_function,
                 NULL,
                 NULL,
                 fname,
                 NULL, 
                 sn_current_file(),
                 line,
                 SN_REF_PASS);

  assert(result == 0);

  sn_highlight(SN_HIGH_FUNCTION,
      tokens_head->start_line, tokens_head->start_column,
      tokens_head->end_line, tokens_head->end_column);

  free_head_token(); /* SOMEWORD */
  free_head_token(); /* OPEN_PAREN */
}

<TOKEN>"SOMEWORD ASSIGNMENT_OPERATOR" {
  int offset;

#ifdef TOKEN_DEBUG
  fprintf(tokenout, "variable assignment at token %d\n", token_index);
#endif

  emit_var_access(tokens_head, VAR_WRITE);

  free_head_token(); /* SOMEWORD */
  free_head_token(); /* ASSIGNMENT_OPERATOR */
}

<TOKEN>"SOMEWORD" {
#ifdef TOKEN_DEBUG
  fprintf(tokenout, "variable read at token %d\n", token_index);
#endif

  emit_var_access(tokens_head, VAR_READ);

  free_head_token(); /* SOMEWORD */
}

<TOKEN>{token} {
  enum sn_highlights type;

#ifdef TOKEN_DEBUG
  fprintf(tokenout, "ate token %d %s", token_index,
          TokenTypeToString(tokens_head));
  if (tokens_head->strval) {
      fprintf(tokenout, " \"%s\"", tokens_head->strval);
  }
  fprintf(tokenout, "\n");
#endif

  if (highlight_file) {
      switch (tokens_head->type) {
          case DOUBLE_QUOTED_STRING:
              type = SN_HIGH_STRING;
              break;
          case KEYWORD:
              type = SN_HIGH_KEYWORD;
              break;
          default:
              type = 0;
      }
      if (type != 0) {
          sn_highlight(type,
                  tokens_head->start_line, tokens_head->start_column,
                  tokens_head->end_line, tokens_head->end_column);
      }
  }

  free_head_token(); /* ... */
}

<TOKEN>" "

<TOKEN>.		{
#ifdef TOKEN_DEBUG
    fprintf(tokenout, "matched unknown character \"%s\"\n", yytext);
#endif
}

<TOKEN><<EOF>> {
#ifdef TOKEN_DEBUG
    fprintf(tokenout, "reached EOF in TOKEN buffer\n");
#endif

    /* A function closing brace was not found before we hit EOF */
    if (current_function) {
        current_function_line_end = sn_line();
        current_function_column_end = sn_column();

#ifdef TOKEN_DEBUG
        fprintf(tokenout, "found unfinished function at EOF in %s\n", sn_current_file());
#endif

        emit_function_declaration();
    }

    assert(!tokens_head); /* all tokens were processed */

    yy_delete_buffer( YY_CURRENT_BUFFER );
    yy_switch_to_buffer( original_buffer );
    BEGIN(EXAMPLE);
    yyterminate();
}

<EXAMPLE,COMMENT_MODE,DQSTRING><<EOF>> {
  LongString token_buffer;
  char *base;
  int i;
  Token* tok;
  yy_size_t size;
  YY_BUFFER_STATE yybs;

#ifdef TOKEN_DEBUG
  tokenout = fopen("tokens.out", "a");
  fprintf(tokenout, "reached EOF in lex input buffer in mode %s\n", modestring());
#endif

  /* See if we ran off the end of the lex input buffer in a special mode */
  switch (YY_START) {
    case COMMENT_MODE:
      emit_comment();
      break;
    case DQSTRING:
      emit_dqstring();
      break;
  }

  /* If no tokens were generated, then quit now */
  if (tokens_head == NULL) {
#ifdef TOKEN_DEBUG
    fprintf(tokenout, "no TOKENs generated\n");
#endif
    BEGIN(EXAMPLE);
    yyterminate();
  }

  /*
   * If the -T command line option was passed,
   * dump all tokens to a file, skip the token
   * matching phase and go on to the next file.
   */

  if (token_dump_file) {
    FILE * dump_tokens = fopen(token_dump_file, "a");

    for (i=0, tok = tokens_head ; tok ; tok = tok->next, i++) {
        fprintf(dump_tokens, "%d %s", i, TokenTypeToString(tok));
        if (tok->strval == NULL) {
            fprintf(dump_tokens, " \"\"");
        } else {
            char *x;
            fprintf(dump_tokens, " \"");
            for (x=tok->strval; *x; x++) {
                if (*x == '\n') {
                    fprintf(dump_tokens, "\\n");
                } else if (*x == '\\') {
                    fprintf(dump_tokens, "\\\\");
                } else if (*x == '\"') {
                    fprintf(dump_tokens, "\\\"");
                } else {
                    fprintf(dump_tokens, "%c", *x);
                }
            }
            fprintf(dump_tokens, "\"");
        }
        fprintf(dump_tokens, " %d.%d %d.%d",
            tok->start_line,
            tok->start_column,
            tok->end_line,
            tok->end_column
        );
        fprintf(dump_tokens, "\n");
    }

    fclose(dump_tokens);

    BEGIN(EXAMPLE);
    yyterminate();
  }

  LongStringInit(&token_buffer,0);

  /* Print token info to in memory buffer and then reload
     the input state machine and start out in the TOKEN mode. */

  for (i=0, tok = tokens_head ; tok ; tok = tok->next, i++) {
#ifdef TOKEN_DEBUG
        fprintf(tokenout, "token %d %s", i, TokenTypeToString(tok));
        if (tok->strval) {
            fprintf(tokenout, " \"%s\"", tok->strval);
        }
        fprintf(tokenout, " (%d.%d -> %d.%d)",
            tok->start_line,
            tok->start_column,
            tok->end_line,
            tok->end_column
        );
        fprintf(tokenout, "\n");
#endif
        
        token_buffer.append( &token_buffer, TokenTypeToString(tok), -1);
        token_buffer.append( &token_buffer, " ", -1);
  }

#ifdef TOKEN_DEBUG
  fprintf(tokenout, "token buffer data is \"%s\"\n", token_buffer.buf);
#endif

  original_buffer = YY_CURRENT_BUFFER;
  yy_switch_to_buffer( yy_scan_string(token_buffer.buf) );

  token_buffer.free(&token_buffer);

#ifdef TOKEN_DEBUG
  fprintf(tokenout, "switching to token mode\n");
#endif

  BEGIN(TOKEN);
}

%%

/* Return a string that describes the current mode */

static char* modestring() {
    char* mode = "UNKNOWN";

    switch (YY_START) {
        case INITIAL:
            mode = "INITIAL";
            break;
        case EXAMPLE:
            mode = "EXAMPLE";
            break;
        case COMMENT_MODE:
            mode = "COMMENT_MODE";
            break;
        case DQSTRING:
            mode = "DQSTRING";
            break;
        case TOKEN:
            mode = "TOKEN";
            break;
    }
    return mode;
}

#if MATCH_DUMP

/* Helper method that will print matches as they are made.
 * This method is typically used in the token generation phase.
 */

static void matched_pattern(char * pattern, char * text) {
    char * mode = modestring();

    fprintf(stderr, "Matched \"%s\", with text \"%s\", in mode \"%s\" (%d.%d)\n",
        pattern, text, mode, sn_line(), sn_column());
}

#endif /* MATCH_DUMP */

static void FreeGlobalEntry(SearchEntry *entry) {}


void FreeToken(Token* tok) {
    if (tok->strval != NULL) {
        ckfree(tok->strval);
    }
    ckfree((char *) tok);
}

void append_dqstring_token(char* strval,
                           long start_line,
                           int start_column,
                           long end_line,
                           int end_column) {
    append_token(DOUBLE_QUOTED_STRING,
                     strval,
                     start_line,
                     start_column,
                     end_line,
                     end_column);
}

void append_token(TokenType type,
                  char* strval,
                  long start_line,
                  int start_column,
                  long end_line,
                  int end_column) {
    Token* tok;
    tok = (Token*) ckalloc(sizeof(Token));
    tok->type = type;
    if (strval)
        tok->strval = SN_StrDup(strval);
    else
        tok->strval = NULL;

    tok->start_line = start_line;
    tok->start_column = start_column;

    tok->end_line = end_line;
    tok->end_column = end_column;
    tok->next = NULL;

    /* append to token list */

    if (tokens_tail == NULL) {
        tokens_head = tokens_tail = tok;
    } else {
        tokens_tail->next = tok;
        tokens_tail = tok;
    }
}

Token* pop_head_token() {
    Token* tok;
    assert(tokens_head);
    tok = tokens_head;
    if (tokens_head == tokens_tail) {
        tokens_head = tokens_tail = (Token*) NULL;
    } else {
        tokens_head = tokens_head->next;
    }
    token_index++;
    return tok;
}

void free_head_token() {
    FreeToken(pop_head_token());
}

char * TokenTypeToString(Token *tok) {
    switch(tok->type) {
        case OPEN_PAREN:
            return "OPEN_PAREN";
        case CLOSE_PAREN:
            return "CLOSE_PAREN";
        case OPEN_BRACE:
            return "OPEN_BRACE";
        case CLOSE_BRACE:
            return "CLOSE_BRACE";
        case SEMICOLON:
            return "SEMICOLON";
        case ASSIGNMENT_OPERATOR:
            return "ASSIGNMENT_OPERATOR";
        case LITERAL:
            return "LITERAL";
        case SOMEWORD:
            return "SOMEWORD";
        case KEYWORD:
            return "KEYWORD";
        case DECLARE_KEYWORD:
            return "DECLARE_KEYWORD";
        case GLOBAL_KEYWORD:
            return "GLOBAL_KEYWORD";
        case DOUBLE_QUOTED_STRING:
            return "DOUBLE_QUOTED_STRING";
        case UNKNOWN:
            return "UNKNOWN";
        case COMMENT:
            return "COMMENT";
        default:
            return "TOKEN_NOT_MATCHED";
    }
}

/* Called when the closing brace of a function is found
 * or when we hit EOF without finding the end of the
 * function.
 */

void emit_function_declaration() {
    result = sn_insert_symbol(SN_FUNC_DEF,
            NULL,
            current_function,
            sn_current_file(), 
            current_function_line_start, current_function_column_start,
            current_function_line_end, current_function_column_end,
            0 /* attribute */,
            NULL /* return type */,
            NULL /* argument types */,
            NULL /* argument names */,
            NULL /* comment */,
            current_function_highlight_line,
            current_function_highlight_column_start,
            current_function_highlight_line,
            current_function_highlight_column_end );

    assert(result == 0);

    ckfree(current_function);
    current_function = NULL;

    if (global_var_table) {
      global_var_table->destroy( &global_var_table );
      global_var_table = NULL;
    }
}

void emit_comment() {
    char* comment = mode_buff.buf;

#if COMMENT_DUMP
    fprintf(stderr, "emit comment \"%s\"\n", comment);
#endif

  /* If dumping tokens, emit a special COMMENT token.
   * Otherwise, insert a comment symbol and a highlight.
   */
  if (token_dump_file) {
    append_token(COMMENT, comment,
            mode_start_line,
            mode_start_column - 2,
            sn_line(),
            sn_column() + 2);
  } else {
    sn_insert_comment(
        /* classname */ NULL,
        /* funcname */ NULL,
        sn_current_file(),
        comment,
        mode_start_line,
        mode_start_column);

    sn_highlight(SN_HIGH_COMMENT,
            mode_start_line,
            mode_start_column - 2,
            sn_line(),
            sn_column() + 2);
  }

  mode_buff.free(&mode_buff);
}

void emit_dqstring() {
    char* dqstring = mode_buff.buf;
    char * x;
    char * var;

#if DQSTRING_DUMP
    fprintf(stderr, "creating dqstring token \"%s\"\n", dqstring);
#endif

    append_dqstring_token(dqstring,
            mode_start_line,
            mode_start_column,
            sn_line(),
            sn_column());

    mode_buff.free(&mode_buff);
}

/* This method is invoked when a var read or write operation is found
 * in the token stream.
 */

void emit_var_access(Token *tok, VarAccess acc) {
  char* varname = tok->strval;
  SearchEntry entry;
  int ref_to_symbol_type, ref_to_symbol_scope;
  int ref_from_scope_type;
  int line_start, line_end, column_start, column_end;

  line_start = tok->start_line;
  column_start = tok->start_column;
  line_end = tok->end_line;
  column_end = tok->end_column;

  /*
   * A var is global if not currently in a function,
   * if the variable is in the super global table,
   * or if in the global table.
   */

  entry.key = varname;
  entry.key_len = -1;

  if ((current_function == NULL) ||
      (global_var_table &&
        (global_var_table->search( &global_var_table, entry ) != NULL))) {
    ref_to_symbol_type = SN_REF_TO_GLOB_VAR;
    ref_to_symbol_scope = SN_REF_SCOPE_GLOBAL;
#ifdef TOKEN_DEBUG
    fprintf(tokenout, "global var \"%s\"\n", varname);
#endif
  } else {
    ref_to_symbol_type = SN_REF_TO_LOCAL_VAR;
    ref_to_symbol_scope = SN_REF_SCOPE_LOCAL;
#ifdef TOKEN_DEBUG
    fprintf(tokenout, "local var \"%s\"\n", varname);
#endif
  }

  /*
   * Pass SN_GLOBAL_NAMESPACE if the xref is outside of
   * a function, otherwise pass SN_FUNC_DEF.
   */   

  if (current_function == NULL) {
    ref_from_scope_type = SN_GLOBAL_NAMESPACE;
  } else {
    ref_from_scope_type = SN_FUNC_DEF;
  }

  if ((ref_to_symbol_type == SN_REF_TO_GLOB_VAR) ||
      ((ref_to_symbol_type == SN_REF_TO_LOCAL_VAR) &&
          ((int) sn_getopt(SN_OPT_LOCAL_VARS) != 0))) {

    if ((acc == VAR_READ) || (acc == VAR_READWRITE)) {
        result = sn_insert_xref(ref_to_symbol_type,
                 ref_from_scope_type,
                 ref_to_symbol_scope,
                 NULL,
                 current_function,
                 NULL,
                 NULL,
                 varname,
                 "UNDECLARED",
                 sn_current_file(),
                 line_start,
                 SN_REF_READ);

        assert(result == 0);
    }

    if ((acc == VAR_WRITE) || (acc == VAR_READWRITE)) {
        result = sn_insert_xref(ref_to_symbol_type,
                 ref_from_scope_type,
                 ref_to_symbol_scope,
                 NULL,
                 current_function,
                 NULL,
                 NULL,
                 varname,
                 "UNDECLARED",
                 sn_current_file(),
                 line_start,
                 SN_REF_WRITE);

        assert(result == 0);
    }
  }

  if (ref_to_symbol_type == SN_REF_TO_GLOB_VAR) {
      sn_highlight(SN_HIGH_VAR_GLOBAL,
          line_start, column_start,
          line_end, column_end);
  } else {
      sn_highlight(SN_HIGH_VAR_LOCAL,
          line_start, column_start,
          line_end, column_end);
  }
}

void
reset()
{
  assert(!current_function);
  sn_reset_line();
  sn_reset_column();
  sn_reset_encoding();
}

int
main(int argc, char *argv[])
{
  return sn_main(argc, argv, group, &yyin, yylex, reset);
}