tokens.h

Using the UnderC Tokenizer Class It s often necessary to parse complex text files, where standard

// TOKENS.H
#ifndef __TOKENS_H
#define __TOKENS_H

#undef EXPORT
#ifdef _EXPORTING_DLL
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#ifdef __UNDERC__
#ifndef NULL
#define NULL 0
#endif
#pragma dlink tokens.dll
#endif
#endif

enum TokenType { T_END, T_TOKEN,
       T_INT, T_DOUBLE, T_FLOAT, T_HEX, T_CHAR, T_STRING, T_OCT,
       T_NUMBER = T_DOUBLE,
       T_LAST
 };

const int TT_BUFFSIZE = 512; // 4096;
const int LINESIZE = 512, STRSIZE = 1024;
const int MAX_IDEN_SIZE = 32;

const int C_STRING = 1,
          C_IDEN = 2,
	  C_NUMBER = 4,
	  STRIP_LINEFEEDS = 8,
	  C_MODE = C_STRING | C_IDEN | C_NUMBER;

typedef char* (*SearchFn)(const char*,const char*);

class EXPORT Tokenizer {
private:
    char *m_start_P, *m_P, *m_start, *m_end_P;
    char m_buff[TT_BUFFSIZE];
    void* m_inf;
    TokenType m_int_type;
    int m_flags;
    const char *m_file;
    int m_line;
    const char *m_comment;
    SearchFn m_search_fn;
  public:
   //construction and opening etc
    Tokenizer(const char *str=0);
    ~Tokenizer();
    bool open(const char *fname);
    void close();
    void reset();
    void set_flags(int flags);
    void set_comment(const char* c);
    void set_str(char *str);
    int  line()         { return m_line; }
    const char* file()  { return m_file; }
    void  discard_line();
    char* getline(char* buff = NULL, int sz = 0);
    bool fetch_line(bool skipws = true);
    int  eof();
    char *get_upto(char ch, bool discard_ch);
    bool skip_whitespace();
    void skip_space();
    void skip_digits();
    TokenType next();
    int peek_ahead(int count = 1);
    char *get_str(char *tok=NULL);
    char *get_string();
    char *get_token();
    double get_float();
    double get_number() { return get_float(); }
    int    get_int();
    double next_float();
    bool go_to(const char* str);
    bool go_to_word(const char* str);
    bool next_is(const char* str);
    char getch();

    // used for bookmarking stream position...
    // be careful - only works w/in same line!
    char *current()        { return m_P; }
    void  current(char *p) { m_P = p; }
    // back to start of file
    void  rewind();
    // file position
    long  pos();
    void  set_pos(long p);

};

#ifdef __UNDERC__
#pragma dlink
#endif

#endif