tokens.cpp

UC Library Extensions UnderC comes with a pocket implementation of the standard C++ libraries, wh

/* TOKENS.CPP 
 * The TokenStream class (a C/C++ preprocessor/tokenizer)      
 * UnderC C++ interpreter
 * Steve Donovan, 2001
 * This is GPL'd software, and the usual disclaimers apply.
 * See LICENCE
*/
#include <ctype.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#ifdef _USE_READLINE
#include <readline/readline.h>
#include <readline/history.h>
#endif

#pragma warning(disable:4786)

// disable 'forcing value to bool' (Microsoft specific)
#pragma warning(disable:4800)

#include "tokens.h"
#include "utils.h"
#include <map>

bool tok_dbg = false;  // *TEMPORARRY**

// from SUBST.CPP
char *copy_chars(char* q, char* p);
char *copy_token(char *p, char *tok);
char *massage_subst_string(char *buff, char **args, char *str); 
char *substitute(TokenStream& tok, char *buff, char **actual_args, char *subst);
char *copy_str(char *tok, char *start, char *end);
void insert(char *str, char *ins);
long convert_int(char *buff, int base);

static int mNoIncludePaths = 0;
static string mIncludeDir[MAX_INCLUDE_PATHS];

static char lbuff[LINESIZE];
static char sbuff[STRSIZE];
static char obuff[10];
static char tbuff[STRSIZE];
static char abuff[STRSIZE];

typedef TokenStream& TS;

char *last_line() { return lbuff; }

// note: cmsg is defined in classlib.h as being cout for plain ol console.
void warning(TS tok, string msg, bool is_error = false)
{
   tok.on_error(msg.c_str(),is_error);
}

void fatal_error(TS tok, string msg)
{
    warning(tok,msg,true);
}

//--------------Macro table stuff----------------------

 typedef std::map<string,PMEntry> MacroTable;

 static MacroTable mMacroTable;

 void macro_cleanup()
 {
   mMacroTable.clear();
 }

 static PMEntry macro_lookup(char *name)
 {
   static string nm = "-------------------------------------------";
   nm = name;
   MacroTable::iterator imt = mMacroTable.find(nm);
   return (imt != mMacroTable.end()) ? imt->second : NULL;
 }

 static PMEntry macro_new(const char *name)
 {
   PMEntry pme = new MEntry;
   mMacroTable[name] = pme;
   pme->is_alias = false;  // by default
   return pme;
 }

 bool TokenStream::macro_delete(const char *name)
 { 
   PMEntry pme = mMacroTable[name];
   if (!pme) return false; // wasn't there in the first place!
   delete pme;
   mMacroTable[name] = NULL;
   return true;
 }

 void TokenStream::macro_builtin(const char *name, int id)
 {
    PMEntry pme = macro_new(name); 
    pme->subst = NULL;
    pme->nargs = id;
 }

 void TokenStream::macro_subst(const char *name, const char *subst)
 {
   PMEntry pme = macro_new(name); 
   pme->subst = (char *)subst;
   pme->nargs = 0;
 }


 void TokenStream::quote_str(char *dest, const char *src)
 {
   strcpy(dest,"\"");
   strcat(dest,src); 
   strcat(dest,"\""); 
 }
//-----------------TokenStream class------------------------

TokenStream::TokenStream(const char *fname, UserCommand cmd)
{
    inf = NULL;
    filename = "DUD";
     m_C_str = true;
     m_skip = false;
    in_comment = false; 
    if(fname) open(fname);
    m_cmd = cmd;
    m_prompter = NULL;
    m_line_feed = false;
    m_cwd = "";
}

 static char mPromptBuffer[MAX_PROMPT_SIZE];
 
char* TokenStream::get_prompt_buffer()
{
   return mPromptBuffer;
}

void TokenStream::on_error(const char *msg, bool is_error)
{
  cerr << file() << '(' << lineno() << ") " << msg << endl;
  if (is_error) exit(-1);
}

void TokenStream::set_str(char *str)
{
 *str = 0;  // just in case...
  start = P = start_P = str;
}

TokenStream::~TokenStream()
{
    close();
}

#ifdef _WCON
static WConIstream win;
static istream *con_in = &win;
#else 
static istream *con_in = &cin;
#endif

// *add 1.2.6 exported as uc_include_path
int _uc_include_path(const char *fname, char* buff, int sz)
{
 int knt = 0;
 string path = fname;
 while(! Utils::can_access(path)) {
	 if (knt == mNoIncludePaths) return false;
	 path = mIncludeDir[knt++] + fname;
 }
 strncpy(buff,path.c_str(),sz);
 return true;
}

// *change 1.1.4
// Now supports multiple include paths!
// Normal include will now also check these paths...

bool TokenStream::open(const char *fname, bool system_include)
{
 //..try to allocate and open a file stream
 istream* is = NULL; 
 string path,new_cwd,dir;
 int knt = 0;
 bool true_system_include = system_include;

 while (is == NULL) 
 {
  if (system_include) {
	 path = mIncludeDir[knt++] + fname;
  } else 
     path = fname;

  bool is_relative = Utils::extract_relative_path(path,dir);
  // *change 1.0.0 open() keeps track of working directory
  if (! system_include) {
   if (is_relative && m_cwd.size() != 0) {
     path = m_cwd;
     path += fname; 
   } else
     path = fname;
  } 
  // *fix 1.0.0L 'CON' is of course not a dev name in Linux!
  // *fix 1.2.1   (Eric) Don't create the file if it can't be found!
  if (path == "CON") is = con_in; 
  else is = new ifstream(path.c_str(),IOS_IN_FLAGS); 
  if (!(*is)) {
     delete is; 
     is = NULL;
	 // if we have failed on our first try, keep going...
     system_include = true;	 
  	 // bail out if we have run out of include paths...
	 if (knt == mNoIncludePaths) break;
  }
  // *fix 1.2.8 only append the path part if it isn't absolute
  if (dir.size() != 0) {
	  if (Utils::is_absolute_path(dir)) new_cwd = dir;
	  else  new_cwd = m_cwd + dir;
  }  else new_cwd = "";
 }

 if (is == NULL)
	 fatal_error(*this,"Cannot open " + (true_system_include ? path : fname /*Utils::full_path(path)*/));
  
 return insert_stream(is,path.c_str() /*fname*/,0,new_cwd);
}

static Stack <int,MAX_FILE_DEPTH> mOldSkip;

bool TokenStream::close()
{
    if (inf) {
        if (inf != con_in) delete inf;
        inf = NULL;
    }
    //..pop the filestack!
    if (fstack.empty()) return false;
    FileStruct fs = fstack.pop();
    filename = fs.filename;
    line = fs.lineno;
    inf  = fs.fin;
    m_cwd = fs.cwd;
    if (fs.save_buff != NULL) {
      strcpy(buff,fs.save_buff);      //*LEAK* !!
      P = fs.save_P;
    } else  set_str(buff);

    // any operations that must occur when files are closed...
    // *add 1.2.4 Check that the skip-stack level is the same as it was at entry
    if (mOldSkip.depth() > 0 && mOldSkip.pop() != sstack.depth()) {
        warning(*this,"mismatched #if/#endif");
        set_skip(false);
    }
    on_restore();
    return inf != NULL;
}

void TokenStream::clear()
{
    mOldSkip.clear();
    sstack.clear();
    set_skip(false);
	if (fstack.depth() > 1) close();
	while (fstack.depth() > 1) {
        on_clear(filename.c_str(),line);		
		close();
    }
    set_str(buff); // and clear buffer!
}

bool TokenStream::insert_stream(istream *is, const char *name, int start_line, const string& new_cwd)
{
 if (! is || !(*is) || is->eof()) return false;
//..push our previous state onto the file stack
 FileStruct fs;
 fs.filename = filename;
 fs.fin = inf;
 fs.lineno = line;
 fs.cwd = m_cwd;
 if (start_line > 0) {
   fs.save_buff = strdup(buff);
   fs.save_P = P;
 } else
 fs.save_buff = NULL;
 fstack.push(fs);

 filename = name;
 inf = is;
 line = start_line;
 set_str(buff);
 if (new_cwd.size() > 0) m_cwd = new_cwd;

 // any operations on open - save the initial skip-stack level
 mOldSkip.push(sstack.depth());
 on_open();
 return true;
}

bool TokenStream::is_interactive()
{
  return inf == con_in;   // *change 1.0.0 Should be more efficient
}

int grab_macro_args(TokenStream& tok, char **args)
{
    int nargs = 0;
    char ch = tok.next();
    if (ch != '(') fatal_error(tok,"absurd!");
    ch = tok.next();
    while (ch != ')') {
        if (ch == T_TOKEN) { 
            *args++ = strdup(tok.get_token());
            nargs++;
        }
        else 
        if (ch == T_END) fatal_error(tok,"end of file in macro arguments"); else
        if (ch != ',' && ch != ')') fatal_error(tok,"illegal char in macro argument");
        ch = tok.next();
    }
    *args = NULL;
    return nargs;
}

int grab_actual_args(TokenStream& tok, char **args)
{
    // *fix 1.2.2b (Eric) 1: quoted macro arguments are grabbed verbatim; 2: code cleanup
    char ch, *q = tbuff;
    bool in_quote = false;
    int nargs = 0, level = 1;
    if (tok.next() != '(') return 0;
    while (ch = tok.getch()) {
        if (ch == '"') in_quote = !in_quote;
        if (in_quote) {
            if (ch == '\\' && tok.peek_ahead(0) != 0) {
                // it's escaped - grab two (one below)
                *q++ = ch;
                ch = tok.getch();
            }
        }
        else if (ch == '(') ++level;
        else if (ch == ',' || ch == ')') {
            if (level == 1) {
                // trim off surrounding whitespace
                while (isspace(q[-1])) q--;
                *q = 0;
                q = tbuff;
                while (isspace(q[ 0])) q++;

                // save off the arg and prepare for next time through
                args[nargs++] = strdup(q);
                q = tbuff;

                // either go get the next arg or bail out cause we're done
                if (ch == ',') continue;
                else break;
            }
            if (ch == ')') --level;
        }
        // accumulate the current arg
        *q++ = ch;
    }
    if (ch == 0) fatal_error(tok,"Unterminated macro arg list");
    args[nargs] = NULL;
    return nargs;
}



void expecting_macro_name(TS tok)
{ fatal_error(tok,"expecting macro name"); }

void expecting_string(TS tok)
{ fatal_error(tok,"expecting string"); }

// *add 1.1.0 ensure that the on_hash_cmd() method is always called, however we return!
struct Restore {
  TokenStream& tok;
  Restore(TokenStream& t) : tok(t) { }
 ~Restore() {
    tok.on_hash_cmd();
 }
};

const uchar NO_SKIP = 0,     // prepro is not skipping statements
            SKIP = 1,        // it is skipping normally
			BLOCK_SKIP = 2,  // finished with a #if/elsif/../endif
			NESTED_SKIP = 3; // we are within a block which is skipping

static uchar do_else(TokenStream& tok)
{
   if (tok.skip_stack_empty()) fatal_error(tok,"misplaced #else/#elif");
   uchar skipping = tok.get_skip();

   if (skipping == NO_SKIP || skipping == SKIP)
      tok.set_skip(skipping==SKIP ? NO_SKIP : BLOCK_SKIP);
   return tok.get_skip();
}

bool do_prepro_directive(TokenStream& tok)
{
    Restore after(tok);
    string ppd,path;
    // *add 1.2.6 null directive - also ignores shell script (e.g. #!/bin/ucc -f....)
    char ch = *tok.current();
    if (ch == '\0' || ch == '!') return true;
	// fetch the directive name
    int t = tok.next();    
    if (t != T_TOKEN) {
       fatal_error(tok,"expecting preprocessor directive");
       return true;
    }
    ppd = tok.get_token();
	uchar skipping = tok.get_skip();
    if (ppd == "include") {
		bool is_sys_include = false;
        if (skipping) return true;
        t = tok.next();
        if (t == '<') {
            char *str  = tok.get_upto('>',true);
            if (!str) fatal_error(tok,"expecting '>' in #include");
			is_sys_include = true;
			path = str;
        }
        else path = tok.get_str(tbuff);
        tok.open(path.c_str(),is_sys_include);
    } else 
    if (ppd == "define" || ppd == "alias") {
        char *subst, *sym;
        PMEntry pme;
        int nargs;
        char *args[MAX_MACRO_ARGS];

        if (skipping) return true;
        t = tok.next();
        if (t != T_TOKEN) expecting_macro_name(tok);
        sym = tok.get_str(tbuff);
        pme = macro_lookup(sym);         
        if (pme) warning(tok,"redefining " + string(sym));
        pme = macro_new(sym);
		pme->is_alias = ppd == "alias";
        tok.on_add_macro(sym,pme);
        if (tok.look_ahead() == '(') {
                nargs = grab_macro_args(tok,args);
        } else nargs = 0;
        pme->nargs = nargs;
        subst = tok.get_upto(0,false);
        if (nargs > 0) {
             massage_subst_string(abuff,args,subst);
             pme->subst = strdup(abuff);
        } else  pme->subst = strdup(subst);
    } else
    if (ppd == "ifdef" || ppd == "ifndef" || ppd == "if" || ppd == "elif") {
      if (ppd == "elif") skipping = do_else(tok); // *add 1.2.6 implement #elif
      else tok.push_skip();	  
	  //
	  if (ppd == "if" || ppd == "elif") { // *add 1.2.5 implement #if  
          char* line = tok.get_upto(0,false);
          if (! line) fatal_error(tok,"expecting #if expression");
          if (! skipping) {  // when we are not skipping, then we can change the skip state!
            tok.c_string_mode(true);
            tok.expecting_defined(true);
            int res = tok.eval_const_expr(line);
            tok.c_string_mode(false);
            tok.expecting_defined(false);
		    tok.set_skip(res ? NO_SKIP : SKIP);
          } else
		  if (ppd == "if") tok.set_skip(NESTED_SKIP);
	  } else { // #ifdef, #ifndef
        t = tok.next();
        if (t != T_TOKEN) expecting_macro_name(tok);
        if (! skipping) { 
          bool sym_exists = macro_lookup(tok.get_str(tbuff)) != NULL;
          if(ppd == "ifdef") { if(! sym_exists) tok.set_skip(SKIP); }
                      else   { if(sym_exists)   tok.set_skip(SKIP); }        
        } else tok.set_skip(NESTED_SKIP);
      }
   } else
   if (ppd == "endif") {
      if (tok.skip_stack_empty()) fatal_error(tok,"misplaced #endif");  // throws an exception
      tok.pop_skip();
   } else
   if (ppd == "else") {
       do_else(tok);
   } else
   if (ppd == "undef") {
       if (skipping) return true;
       t = tok.next();
       if (t != T_TOKEN) expecting_macro_name(tok);
       TokenStream::macro_delete(tok.get_str(tbuff));
   } else
       if (ppd == "warning" || ppd == "error") { // *add 1.2.5 #error, 1.2.6 #warning
       if (! skipping) { 
           char* line = tok.get_upto(0,false);
           warning(tok,line, ppd=="error");
       }
   } else { /// an interactive command!
    if (skipping) return true;
    if (!tok.user_cmd(ppd))  return false;  
   }
   return true;
}

void TokenStream::grab_next()
{
// *hack 1.1.4 a cheap & nasty way of looking ahead in the stream....
	inf->getline(lbuff,LINESIZE);
	strcat(buff,lbuff);
}