tokens.cpp

UC Library Extensions UnderC comes with a pocket implementation of the standard C++ libraries, wh

bool TokenStream::look_ahead_more(char ch)
{
 char *p = P;
 while (true) {
   while (*p && isspace(*p)) p++;
   if (*p) {
     return (*p == ch);
   }
   grab_next();
 }
}

bool TokenStream::fetch_line()
{
 int len;
 bool continuation = true;
 *buff = '\0';
 while (true) { // looking for non-blank, non-preprocessor lines
 set_str(buff);
 do {
     //...if we have run out of file, close and try to continue
     if (!inf || inf->eof()) {
         if(!close()){
            set_str(buff);
            return false;  // we have run out of open files
         // restored old file;  can continue
         }
     }
     do_prompt();   // really only need to call this when in interactive mode...
#ifndef _USE_READLINE
     inf->getline(lbuff,LINESIZE); 
#else
         if (inf != con_in) inf->getline(lbuff,LINESIZE);
         else {
            char *pl = readline(get_prompt_buffer());
            if (pl == NULL) return false; // *fix 1.2.2 (Dean) Stopping process with ctrl-D
            strcpy(lbuff,pl);
            free(pl);
            add_history(lbuff);
         }
#endif
// *fix 1.1.0 spaces after \ messes with the preprocessor's little mind!
     char *endp = lbuff + strlen(lbuff) - 1;
	 while (*endp && isspace(*endp)) --endp;
     continuation = (*endp == '\\'); 
//     if (continuation) *endp = '\0';  // lop off '\'
     if (continuation) { *(endp+1) = '\0';  *endp = '\n'; }
     strcat(buff,lbuff);  
	 line++;
 } while (continuation);

 start = P = start_P = buff; 
 while(isspace(*P)) P++;
 if (*P == 0) continue;  // can ignore empty lines!
 if (*P == '#' && ! in_comment) { 
     m_C_str = false;
     P++;
     try {
       if (!do_prepro_directive(*this)) {
          close(); 
          m_C_str = true;
          return false; // bail out!
       }
     } catch(string msg) {
 // *fix 1.2.7 fatal errors must stop parsing, unless in interactive mode.
 // *fix 1.2.8 true, but that isn't the same as 'inf != con_in'!  
 //  is_interactive_mode() is a virtual function overriden by UCTokenStream.
       if (! is_interactive_mode()) return false;
     }
     m_C_str = true;
 }
 else
 if (! m_skip) {
   // ensure that there's a final line feed!    
    if (m_line_feed) { 
      len = strlen(buff);
      buff[len]   = '$';
      buff[len+1] = '\0';
    }
    return true;
 } 
 } // while looking for non-blank, non-preprocessor lines
}

char *TokenStream::get_upto(char ch, bool discard_ch)
// Grab characters from the stream, upto (and optionally including) ch.
// if anything goes wrong we throw a wobbly.
// *this routine (for now) just works for the prepro*
// *fix 1.2.1 (Peter) Strip out C++ line comments if we're grabbing the whole line
{
  bool found_end;
  start_P = P;
  while(*P && *P != ch) P++;
  found_end = *P == ch;
  if (! discard_ch) P++;
  end_P = P;
  if (discard_ch) P++;
  if (found_end) {
      char *str = get_str(tbuff);
      if (ch=='\0') { // special case of grabbing the whole line
      // *fix 1.2.2 (Eric) strip out any C++ comments properly
      // *fix 1.2.4 Discard any \r found at end of line (DOS files in Linux)
          for (char *p = str; *p; ++p) {
              if ((p[0] == '/' && p[1] == '/') || p[0] == '\r') {
                  *p = '\0';
                  break;
              }
	
          }
      }
      return str;
  }
  else return NULL;
}

void TokenStream::discard_line()
{
 *P = 0; skip_whitespace();
}

// *fix 0.9.3 This was ignoring blank lines & generally messing
//  up the template line number diagnostics.  But it's still
//  giving grief - need to handle the case where there's something
//  still in the input buffer (as is usually the case)
void TokenStream::grab_line(char *buff)
{  
  skip_whitespace();
  start_P = P;
  P += strlen(P);
  end_P = P;
  get_str(buff);
  
  //char lbuff[LINESIZE];
  //inf->getline(lbuff,LINESIZE); 
  //strcpy(buff,lbuff);
 
}

void TokenStream::insert_string(char *str)
// stuff characters into the stream!
// for now, it effectively discards what was _in_ the buffer;
// this is fine for its current application, which is to execute
// commands..
{
 set_str(buff);
 strcpy(P,str);
}

bool no_new_line = false;

bool TokenStream::skip_whitespace()
{
 top:
   while(*P && isspace(*P)) P++;
   if (*P == 0) {
     if(no_new_line || !fetch_line()) return false; // EOF will pass through as T_END
     goto top;
   } else
   if (*P == '/') { //...filter out comments at this level!!
       if (*(P+1)=='/') { *P = '\0'; goto top; } 
       else
       if (*(P+1)=='*') { 
           P++; in_comment = true;
           while (true){ 
               if (*P++=='*' && *P=='/') { P++; in_comment = false; goto top; }
               if (*P == 0) if(!fetch_line()) {
                   fatal_error(*this,"unexpected end of file in comment");
                   return false;
               }
           }
       }
   }
   return true;
}

int grab_alias_args(char *ptr, char **args)
{
 int nargs = 0;
 char *tok = strtok(ptr," "); //Utils::quote_strtok(ptr);
 while (tok != NULL) {
	 *args++ = tok;
	 ++nargs;
	 tok = strtok(NULL," "); //Utils::quote_strtok(NULL);
 }
 return nargs;
}

void separate_alias_commands(TokenStream& tok)
{
// approved way to fetch the whole line
 char *line = strdup(tok.get_upto(0,true));  
 char *cmds[10], buff[80];
 int k = 0;
 // break up into individual @-commands (need to do this separately)
 char *cmd = strtok(line,"@");
 while (cmd) { 
   cmds[k++] = cmd;
   cmd = strtok(NULL,"@");
 }

 // insert the commands back into the stream 
 // shifting the current position is necessary to prevent
 // runaway substition of 'cd' etc.
 for(int i = 0; i < k; i++) {
   sprintf(buff,"# %s",cmds[i]);
   tok.insert_string(buff);
   tok.current(tok.current()+1);
   // *fix 1.2.0 NB to switch off C string mode when calling do_prepro_directive()
   tok.c_string_mode(false);
   try {
     do_prepro_directive(tok);   
   } catch(string msg) { }
   tok.c_string_mode(true);
 }
}

void TokenStream::skip_digits()
{
 while(isdigit(*P)) P++;
}

static bool first_token_in(char *buff, char *P)
{
	if (P == buff) return true;
	P--;
	while (P != buff && isspace(*P)) P--;
	return P == buff && isspace(*P);
}

// *change 1.2.2 I've separated out the macro substitution code from next()
// and broken it into the two cases, C macros and aliases, explicitly.

bool TokenStream::macro_attempt_process(char*& p, char *out, char *tok)
{
   PMEntry pme = macro_lookup(tok);
   *out = '\0';
   if (! pme || pme->is_alias) return false;
   else {
       char *old_P = current();
       current(p);
       macro_process(pme,out);
       p = current();
       current(old_P);
   }
   return true;
}

void TokenStream::macro_process(PMEntry pme, char *out)
{
    char *args[MAX_MACRO_ARGS];
    char temp_buff[TT_BUFFSIZE];
    int nargs;
    char *subst;
    if (pme->nargs > 0) {
       if (!pme->subst) { // Builtin macro
            handle_builtin_macro(tbuff,pme->nargs);
            subst = tbuff;
       } else {
		  nargs = grab_actual_args(*this,args);  // regular C-style macro
          if (nargs != pme->nargs) fatal_error(*this,"wrong no. of arguments for this macro");
          substitute(*this,temp_buff, args, pme->subst);
          subst = temp_buff;
       }    
    } else 
       subst = pme->subst;

    if (! out) insert(P,subst);
          else strcpy(out,subst);
}

void TokenStream::alias_process(PMEntry pme)
{
    char *args[MAX_MACRO_ARGS];
    char temp_buff[TT_BUFFSIZE];
    int nargs;
    if (pme->nargs > 0) {		  
		  nargs = grab_alias_args(P,args);
          set_str(buff); 
          if (nargs != pme->nargs) fatal_error(*this,"wrong no. of arguments for this macro");
          substitute(*this,temp_buff, args, pme->subst);
          insert(P,temp_buff);
    } else 
       insert(P,pme->subst);

	 skip_whitespace();
	 if(*P=='@') {
         ++P;
  	     separate_alias_commands(*this);
		 discard_line();
     }  
}

int TokenStream::next()
{
try {  // *fix 1.01 fatal_error() will throw a string!
do_it_again:
  if (! skip_whitespace()) return 0;  // means: finis, end of file, bail out.
  if (iscsymf(*P)) { //--------------------- TOKENS --------------
     start_P = P;
     while (iscsym(*P)) P++;
     end_P = P;
     copy_str(tbuff,start_P,end_P);
     if (m_C_str) { // ie. suppress macro lookup in preprocessor directives
         PMEntry pme;
         // *add 1.2.4 Support for defined(MACRO) in #if directives
         // *fix 1.2.5 'defined MACRO' is also acceptable
         if (m_expecting_defined && strcmp(tbuff,"defined")==0) {
           m_C_str = false;
           int t = next();
           char mname[MAX_IDEN_SIZE];
           bool ok = (t == '(' || t == T_TOKEN);
           if (ok) {  
             if (t == '(') ok = (next() == T_TOKEN);     // skip the '('
             if (ok) {
               get_str(mname);                             // pick up the macro name
               if (t == '(') next();                       // skip the ')'
               insert(P,(char*)(macro_lookup(mname) ? "1" : "0"));  
               m_C_str = true;
               goto do_it_again;
             }
           } 
           m_C_str = true;
           if (! ok) fatal_error(*this,"defined takes one macro argument");
         } else {
          pme = macro_lookup(tbuff);
          if (pme) {
             if (pme->is_alias) {
                 if (! first_token_in(buff,start_P)) return T_TOKEN;
                 else alias_process(pme);
             } else
             macro_process(pme,NULL);
             goto do_it_again;
          }
          // *fix 1.2.7 in #if expressions, all non-macros evaluate as 0
          else if (m_expecting_defined) {
             insert(P,"0");
             goto do_it_again;
          }
         }
     }
     return T_TOKEN;
  } else
  if (isdigit(*P)  ||  *P == '.' && isdigit(*(P+1))) { //------- NUMBERS ------------------
    int ntype = int_type = T_INT;   // until proved otherwise!
     start_P = P;
     if (*P != '.') {
      if (*P == '0') {
        // actual verification of hex or octal constants must happen in lexer
        if (*(P+1) == 'x') {       // hex constant
          while (isalnum(*P)) P++; // a preliminary check!
          ntype = int_type = T_HEX;
        } else 
  	 if (isdigit(*(P+1))) {      // octal constant
         skip_digits();
         ntype = int_type = T_OCT;
	}
	else skip_digits();         // plain zero!
      } else {
        P++;                        // skip first - might be '-'
        skip_digits();
      }
     }
     if (*P == '.') {               // (opt) fractional part
        P++;
        skip_digits();
        ntype = T_DOUBLE;
     }
     if (*P == 'e' || *P == 'E') { // (opt) exponent part
        P++;
        if (*P == '+' || *P == '-') P++;  // (opt) exp sign
        skip_digits();
        ntype = T_DOUBLE;
     }
     if (*P == 'f' || *P == 'F')      { P++; ntype = T_FLOAT; }
     // *fix 1.2.6 long integer constants ending with 'L' are now acceptable 
     else if (*P == 'l' || *P == 'L') { P++; ntype = T_INT;  }
     end_P = P;
     return ntype;
  } else
  if (*P == '\"' || *P == '\'') { //------------CHAR OR STRING CONSTANT-------
     char ch, endch = *P++;  char *p = sbuff;
     start_P = sbuff;
next_string:
     while (*P && *P != endch) {
         if (*P == '\\' && m_C_str) {
             P++;
             switch(*P) {
             case '\\': ch = '\\'; break;
             case 'n':  ch = '\n'; break;
             case 'r':  ch = '\r'; break;
             case 't':  ch = '\t'; break;
             case 'b':  ch = '\b'; break;
             case '\"': ch = '\"'; break;
             case '\'': ch = '\''; break;
             case '0':  { //..collecting OCTAL constant
                char *start_oct = P;
                skip_digits();  
                copy_str(obuff,start_oct,P);
                ch = (char)convert_int(obuff,8);
                P--;  // leave us on last digit
		} break;
            // *fix 1.1.2 We were not letting non-escape sequences through...
			 default: *p++ = '\\'; ch = *P; break; 
             } // switch
             *p++ = ch; P++;
         } else *p++ = *P++;
     } 
     if (! *P) fatal_error(*this,"Unterminated string constant");
     P++;  // skip the endch
     *p = '\0';
     end_P = p;
// *add 1.1.1 adjacent quoted text
// *fix 1.1.4 gave preprocessor directives the heebies
	 if (endch=='\"') {
	 if (m_C_str) {
           skip_whitespace();
           if (*P == '\"') { 
               P++;                         // will be concatenated!
	       goto next_string;            // so go back & keep grabbing...
	   }
         }
       return  T_STRING; 
     }
     else return T_CHAR;
  } else
  return *P++;
 } catch(...) {
   return ' ';
 }
}

int TokenStream::look_ahead(bool skip_wspace)
//*OPT* Can inline this!
{
  if(skip_wspace) skip_whitespace(); 
  return *P;
}

int TokenStream::peek_ahead(int count)
{
 return *(P+count);
}

char *TokenStream::get_string()
{
	return sbuff;
}

char *TokenStream::peek_next_token()
{
// This is a hack and only use it if you are slowly going mad with frustration.
// It will return an empty buffer if there's no more tokens on the current line.
  char *ptr = P;
  while (*ptr && !iscsym(*ptr)) ptr++;
  char *start_p = ptr;
  if (ptr) {
    while (iscsym(*ptr)) ptr++;
  } 
  copy_str(tbuff,start_p,ptr);
  return tbuff;
}

char *TokenStream::get_str(char *tok)
{
 if (tok==NULL) tok = tbuff; 
 copy_str(tok,start_P,end_P);
 return tok;
}

char *TokenStream::get_token()
{  return tbuff; }

double TokenStream::get_float()
{
 char buff[20];
 return atof(get_str(buff));
}

int TokenStream::get_int()
{
 char buff[20];
 return convert_int(get_str(buff),int_type == T_INT ? 10 : 16);
} 

double TokenStream::next_float()
{
 int t;
 do {
  t = next();
  if (t == T_NUMBER || t == T_END) return get_float();
 } while (t != T_END);
 return 0.0;
}

void TokenStream::set_include_dir(const char *s)
{
// *fix 1.2.2 Check for too many include paths
    if (mNoIncludePaths+1 >= MAX_INCLUDE_PATHS) {
        cerr << "Out of room for include paths!" << endl;
        return;
    }
    string path = s;
    Utils::check_path_end(path);
    mIncludeDir[mNoIncludePaths] = path;
    ++mNoIncludePaths;
}