vpreproc.cpp

Verilog Parser in Perl

// -*- C++ -*-
//*************************************************************************
//
// Copyright 2000-2009 by Wilson Snyder.  This program is free software;
// you can redistribute it and/or modify it under the terms of either the GNU
// Lesser General Public License or the Perl Artistic License.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
//*************************************************************************
/// \file
/// \brief Verilog::Preproc: Internal implementation of default preprocessor
///
/// Authors: Wilson Snyder
///
/// Code available from: http://www.veripool.org/verilog-perl
///
//*************************************************************************

#include <cstdio>
#include <fstream>
#include <cstring>
#include <stack>
#include <vector>
#include <map>
#include <cassert>

#include "VPreproc.h"
#include "VPreprocLex.h"

//#undef yyFlexLexer
//#define yyFlexLexer xxFlexLexer
//#include <FlexLexer.h>

//*************************************************************************

class VPreDefRef {
    // One for each pending define substitution
    string	m_name;		// Define last name being defined
    string	m_params;	// Define parameter list for next expansion
    string	m_nextarg;	// String being built for next argument
    int		m_parenLevel;	// Parenthesis counting inside def args

    vector<string> m_args;	// List of define arguments
public:
    string name() const { return m_name; }
    string params() const { return m_params; }
    string nextarg() const { return m_nextarg; }
    void nextarg(const string& value) { m_nextarg = value; }
    int parenLevel() const { return m_parenLevel; }
    vector<string>& args() { return m_args; }
    VPreDefRef(const string& name, const string& params, int pl)
	: m_name(name), m_params(params), m_parenLevel(pl) {}
    ~VPreDefRef() {}
};

//*************************************************************************
/// Data for parsing on/off

class VPreIfEntry {
    // One for each pending ifdef/ifndef
    bool	m_on;		// Current parse for this ifdef level is "on"
    bool	m_everOn;	// Some if term in elsif tree has been on
public:
    bool on() const { return m_on; }
    bool everOn() const { return m_everOn; }
    VPreIfEntry(bool on, bool everOn)
	: m_on(on), m_everOn(everOn || on) {}  // Note everOn includes new state
    ~VPreIfEntry() {}
};

//*************************************************************************
/// Data for a preprocessor instantiation.

struct VPreprocImp : public VPreprocOpaque {
    VPreproc*	m_preprocp;	///< Object we're holding data for
    VFileLine*	m_filelinep;	///< Last token's starting point
    int		m_debug;	///< Debugging level
    VPreprocLex* m_lexp;	///< Current lexer state (NULL = closed)
    stack<VPreprocLex*> m_includeStack;	///< Stack of includers above current m_lexp

    enum ProcState { ps_TOP, ps_DEFNAME, ps_DEFVALUE, ps_DEFPAREN, ps_DEFARG,
		     ps_INCNAME, ps_ERRORNAME };
    ProcState	m_state;	///< Current state of parser
    int		m_stateFor;	///< Token state is parsing for
    int		m_off;		///< If non-zero, ifdef level is turned off, don't dump text
    string	m_lastSym;	///< Last symbol name found.

    // For getRawToken/ `line insertion
    string	m_lineCmt;	///< Line comment(s) to be returned
    bool	m_lineCmtNl;	///< Newline needed before inserting lineCmt
    int		m_lineAdd;	///< Empty lines to return to maintain line count
    bool	m_rawAtBol;	///< Last rawToken left us at beginning of line

    // For defines
    stack<VPreDefRef> m_defRefs; // Pending definine substitution
    stack<VPreIfEntry> m_ifdefStack;	///< Stack of true/false emitting evaluations
    unsigned	m_defDepth;	///< How many `defines deep

    // For getline()
    string	m_lineChars;	///< Characters left for next line

    VPreprocImp(VFileLine* filelinep) {
	m_filelinep = filelinep;
	m_debug = 0;
	m_lexp = NULL;	 // Closed.
	m_state = ps_TOP;
	m_off = 0;
	m_lineChars = "";
	m_lastSym = "";
	m_lineAdd = 0;
	m_lineCmtNl = false;
	m_rawAtBol = true;
	m_defDepth = 0;
    }
    const char* tokenName(int tok);
    int getRawToken();
    int getToken();
    void parseTop();
    void parseUndef();
    string getline();
    bool isEof() const { return (m_lexp==NULL); }
    void open(string filename, VFileLine* filelinep);
    void insertUnreadback(const string& text) { m_lineCmt += text; }
    void insertUnreadbackAtBol(const string& text);
private:
    void error(string msg) { m_filelinep->error(msg); }
    void fatal(string msg) { m_filelinep->fatal(msg); }
    int debug() const { return m_debug; }
    void eof();
    string defineSubst(VPreDefRef* refp);
    void addLineComment(int enter_exit_level);
    string trimWhitespace(const string& strg);

    void parsingOn() { m_off--; assert(m_off>=0); if (!m_off) addLineComment(0); }
    void parsingOff() { m_off++; }
};

//*************************************************************************
// Creation

VPreproc::VPreproc(VFileLine* filelinep) {
    VPreprocImp* idatap = new VPreprocImp(filelinep);
    m_opaquep = idatap;
    idatap->m_preprocp = this;
}

//*************************************************************************
// VPreproc Methods.  Just call the implementation functions.

void VPreproc::comment(string cmt) { }
void VPreproc::open(string filename, VFileLine* filelinep) {
    VPreprocImp* idatap = static_cast<VPreprocImp*>(m_opaquep);
    idatap->open (filename,filelinep);
}
string VPreproc::getline() {
    VPreprocImp* idatap = static_cast<VPreprocImp*>(m_opaquep);
    return idatap->getline();
}
void VPreproc::debug(int level) {
    VPreprocImp* idatap = static_cast<VPreprocImp*>(m_opaquep);
    idatap->m_debug = level;
}
bool VPreproc::isEof() {
    VPreprocImp* idatap = static_cast<VPreprocImp*>(m_opaquep);
    return idatap->isEof();
}
VFileLine* VPreproc::filelinep() {
    VPreprocImp* idatap = static_cast<VPreprocImp*>(m_opaquep);
    return idatap->m_filelinep;
}
void VPreproc::insertUnreadback(string text) {
    VPreprocImp* idatap = static_cast<VPreprocImp*>(m_opaquep);
    return idatap->insertUnreadback(text);
}

//*************************************************************************

// CALLBACK METHODS
// This probably will want to be overridden for given child users of this class.

void VPreproc::include(string filename) {
    open(filename, filelinep());
}
void VPreproc::undef(string define) {
    cout<<"UNDEF "<<define<<endl;
}
bool VPreproc::defExists(string define) {
    return defParams(define)!="";
}
string VPreproc::defParams(string define) {
    return "";
}
void VPreproc::define(string define, string value, string params) {
    error("Defines not implemented: "+define+"\n");
}
string VPreproc::defValue(string define) {
    error("Define not defined: "+define+"\n");
    return define;
}

//**********************************************************************
// Parser Utilities

const char* VPreprocImp::tokenName(int tok) {
    switch (tok) {
    case VP_EOF		: return("EOF");
    case VP_INCLUDE	: return("INCLUDE");
    case VP_IFDEF	: return("IFDEF");
    case VP_IFNDEF	: return("IFNDEF");
    case VP_ENDIF	: return("ENDIF");
    case VP_UNDEF	: return("UNDEF");
    case VP_DEFINE	: return("DEFINE");
    case VP_ELSE	: return("ELSE");
    case VP_ELSIF	: return("ELSIF");
    case VP_LINE	: return("LINE");
    case VP_SYMBOL	: return("SYMBOL");
    case VP_STRING	: return("STRING");
    case VP_DEFVALUE	: return("DEFVALUE");
    case VP_COMMENT	: return("COMMENT");
    case VP_TEXT	: return("TEXT");
    case VP_WHITE	: return("WHITE");
    case VP_DEFREF	: return("DEFREF");
    case VP_DEFARG	: return("DEFARG");
    case VP_ERROR	: return("ERROR");
    default: return("?");
    }
}

string VPreprocImp::trimWhitespace(const string& strg) {
    string out = strg;
    while (out.length()>0 && isspace(out[0])) {
	out.erase(0,1);
    }
    return out;
}

string VPreprocImp::defineSubst(VPreDefRef* refp) {
    // Substitute out defines in a argumented define reference.
    // We could push the define text back into the lexer, but that's slow
    // and would make recursive definitions and parameter handling nasty.
    //
    // Note we parse the definition parameters and value here.  If a
    // parametrized define is used many, many times, we could cache the
    // parsed result.
    if (debug()) {
	cout<<"defineSubstIn  `"<<refp->name()<<" "<<refp->params()<<endl;
	for (unsigned i=0; i<refp->args().size(); i++) {
	    cout<<"defineArg["<<i<<"] = "<<refp->args()[i]<<endl;
	}
    }
    // Grab value
    string value = m_preprocp->defValue(refp->name());
    if (debug()) cout<<"defineValue    '"<<value<<"'"<<endl;

    map<string,string> argValueByName;
    {   // Parse argument list into map
	unsigned numArgs=0;
	string argName;
	for (const char* cp=refp->params().c_str(); *cp; cp++) {
	    if (*cp=='(') {
	    } else if (argName=="" && isspace(*cp)) {
	    } else if (isspace(*cp) || *cp==')' || *cp==',') {
		if (argName!="") {
		    if (refp->args().size() > numArgs) {
			// A call `def( a ) must be equivelent to `def(a ), so trimWhitespace
			// Note other sims don't trim trailing whitespace, so we don't either.
			argValueByName[argName] = trimWhitespace(refp->args()[numArgs]);
		    }
		    numArgs++;
		    //cout << "  arg "<<argName<<endl;
		}
		argName = "";
	    } else if ( isalpha(*cp) || *cp=='_'
			|| (argName!="" && (isdigit(*cp) || *cp=='$'))) {
		argName += *cp;
	    }
	}
	if (refp->args().size() != numArgs) {
	    error("Define passed wrong number of arguments: "+refp->name()+"\n");
	    return " `"+refp->name()+" ";
	}
    }

    string out = "";
    {   // Parse substitution define using arguments
	string argName;
	string prev;
	bool quote = false;
	// Note we go through the loop once more at the NULL end-of-string
	for (const char* cp=value.c_str(); (*cp) || argName!=""; cp=(*cp?cp+1:cp)) {
	    //cout << "CH "<<*cp<<"  an "<<argName<<"\n";
	    if (!quote) {
		if ( isalpha(*cp) || *cp=='_'
		     || (argName!="" && (isdigit(*cp) || *cp=='$'))) {
		    argName += *cp;
		    continue;
		}
	    }
	    if (argName != "") {
		// Found a possible variable substitution
		map<string,string>::iterator iter = argValueByName.find(argName);
		if (iter != argValueByName.end()) {
		    // Substitute
		    string subst = iter->second;
		    out += subst;
		} else {
		    out += argName;
		}
		argName = "";
	    }
	    if (!quote) {
		// Check for `` only after we've detected end-of-argname
		if (cp[0]=='`' && cp[1]=='`') {
		    //out += "";   // `` means to suppress the ``
		    cp++;
		    continue;
		}
		else if (cp[0]=='`' && cp[1]=='"') {
		    out += '"';   // `" means to put out a " without enabling quote mode (sort of)
		    cp++;
		    continue;
		}
		else if (cp[0]=='`' && cp[1]=='\\') {
		    out += '\\';   // `\ means to put out a backslash
		    cp++;
		    continue;
		}
	    }
	    if (cp[0]=='\\' && cp[1]) {
		out += cp[0]; // \{any} Put out literal next character
		out += cp[1];
		cp++;
		continue;
	    }
	    if (*cp=='"') quote=!quote;
	    if (*cp) out += *cp;
	}
    }

    if (debug()) cout<<"defineSubstOut '"<<out<<"'"<<endl;
    return out;
}

//**********************************************************************
// Parser routines

void VPreprocImp::open(string filename, VFileLine* filelinep) {
    // Open a new file, possibly overriding the current one which is active.
    if (filelinep) {
	m_filelinep = filelinep;
    }

    FILE* fp = fopen (filename.c_str(), "r");
    if (!fp) {
	error("File not found: "+filename+"\n");
	return;
    }

    if (m_lexp) {
	// We allow the same include file twice, because occasionally it pops
	// up, with guards preventing a real recursion.
	if (m_includeStack.size()>VPreproc::INCLUDE_DEPTH_MAX) {
	    error("Recursive inclusion of file: "+filename);
	    return;
	}
	// There's already a file active.  Push it to work on the new one.
	m_includeStack.push(m_lexp);
	addLineComment(0);
    }

    m_lexp = new VPreprocLex (fp);
    m_lexp->m_keepComments = m_preprocp->keepComments();
    m_lexp->m_keepWhitespace = m_preprocp->keepWhitespace();
    m_lexp->m_pedantic = m_preprocp->pedantic();
    m_lexp->m_curFilelinep = m_preprocp->filelinep()->create(filename, 1);
    m_filelinep = m_lexp->m_curFilelinep;  // Remember token start location
    addLineComment(1); // Enter

    yy_switch_to_buffer(m_lexp->m_yyState);
}

void VPreprocImp::insertUnreadbackAtBol(const string& text) {
    // Insert insuring we're at the beginning of line, for `line
    // We don't always add a leading newline, as it may result in extra unreadback(newlines).
    if (m_lineCmt == "") { m_lineCmtNl = true; }
    else if (m_lineCmt[m_lineCmt.length()-1]!='\n') {
	insertUnreadback("\n");
    }
    insertUnreadback(text);
}

void VPreprocImp::addLineComment(int enter_exit_level) {
    if (m_preprocp->lineDirectives()) {
	char numbuf[20]; sprintf(numbuf, "%d", m_lexp->m_curFilelinep->lineno());
	char levelbuf[20]; sprintf(levelbuf, "%d", enter_exit_level);
	string cmt = ((string)"`line "+numbuf
		      +" \""+m_lexp->m_curFilelinep->filename()+"\" "
		      +levelbuf+"\n");
	insertUnreadbackAtBol(cmt);
    }
}

void VPreprocImp::eof() {
    // Remove current lexer
    if (debug()) cout<<m_filelinep<<"EOF!\n";
    addLineComment(2);	// Exit
    delete m_lexp;  m_lexp=NULL;
    // Perhaps there's a parent file including us?
    if (!m_includeStack.empty()) {
	// Back to parent.
	m_lexp = m_includeStack.top(); m_includeStack.pop();
	addLineComment(0);
	yy_switch_to_buffer(m_lexp->m_yyState);
    }
}

int VPreprocImp::getRawToken() {
    // Get a token from the file, whatever it may be.
    while (1) {
      next_tok:
	if (m_lineAdd) {
	    m_lineAdd--;
	    m_rawAtBol = true;
	    yytext=(char*)"\n"; yyleng=1;