lexer.py

convert C programs for use in C++ compiler environment

 if second in [ '*', 'huge', 'far', 'near']:
  try:
   if isident(words[2]):
     return TRUE
  except:
   pass
 print "What is \"" + second + "\" ->", line + '?'
 return FALSE




# Get the type of a variable from a valid declaration
# Return a list

def vartypelist(line):
 while commentfound(line):
  line = nocomment(line)

 words =  wstring.split(line, " \t*;[", '*')
 t = []
 # Adding all type keyword but "extern"
 # exiting when encountering the variable's name
 for w in words:
  if w in types:
   if w != "extern": t.append(w)
  else: break
 return t


# Get the type of a variable from a declaration
# Return a string

def vartype(line):
 nline = ""
 words = vartypelist(line)
 for w in words:
   if w in types:
     nline = nline + w + ' '
     continue
   break
 return nline


# Compare the definition and the declaration of two variables
# return TRUE if same ident

def samevar(linedef, linedecl):
  a = getident(linedef)
  b = getident(linedecl)
  #print "sameident?", a, b
  if a is None: return FALSE
  return a == b


# Is the variable declaration changed?
# The name is assumed identical, we compare the types.

def vchanged(cline, hline):
  ctypes  = vartypelist(cline)
  htypes  = vartypelist(hline)
  if ctypes != htypes: return TRUE   # different types/modifiers
  if not isarray(cline):
    if not isarray(hline):
      return FALSE                   # simple var, ok
  # for arrays, test also sizes
  csize = getarraysize(cline)
  hsize = getarraysize(hline)
  #print "sizes=", csize, hsize
  return csize != hsize           # comparing dimensions


# Multiple var definition in line
# Ex:    int i,j;      int i; int j

def multivar(line):
  line = removecomment(line)
  line = wstring.strip(line)
  #print "in multivar", line
  l = wstring.split(line, ",;")
  #print "result=", len(l)
  if len(l) > 1: return TRUE
  if ',' in line:
    print "Error in multi-declaration, string", line
    sys.exit(0)
  return FALSE


# Convert a multiple declaration or definitions
# as a list of declarations
# Ex:   int i,j;    -->   int i; int j;

def splitvardecl(str):
  vlist = splitvardef(str)
  nlist = []
  for v in vlist:
    i = string.find(v, '=')
    if i != -1:
      n = v[:i]
      n = wstring.strip(n) + ';'
    else:
      n = v
    #print v,n
    nlist.append(n)
  return nlist


# Process a var's definition inside C source
# As above, but keep assignments along with variables

def splitvardef(str):
 # split into strings separated by "," or ";" and remove trailing ";"
 # when curly braces openned, a flag is set and separators ignored
 # until curly braces closed

 str = removecomment(str)
 #print str

 words = []
 newword = ""
 flag = FALSE
 str = wstring.strip(str)                     # Remove spaces

 for c in str:
   if (c in ",;") & (flag == FALSE):          # char in delimiter list
     if newword != "": words.append(newword)  # then word ended, add it
     newword = ""                             # clear it
   else:                                      # else char in a word
     newword = newword + c                    # add char to word
     if c == '{': flag = TRUE
     if c == '}': flag = FALSE

 # End of string, either a word or a delimiter sequence remains here
 if newword != "": words.append(newword)

 # get types from the first string and remove it from the list
 first = words[0]
 words.remove(first)

 tlist = wstring.split(first, " \t*=", "*=")
 #print tlist
 flag = FALSE
 commontype = ""
 idfound = ""
 # get type keywords
 for v in tlist:
   if v == "=": break     # right part of an assignment is ignored here
   if v in types:
     if flag == FALSE:
       commontype = commontype + v + ' '
       continue
     print "Error, keyword \'" + v + "\' follows ident \'" + idfound + "\' in", str
     sys.exit(0)
   flag = TRUE
   idfound = v

 newlist = []                            # list of separated vars
 newlist.append(addsemicolon(first))     # first string unchanged
 for n in words:
   if not isdeclaration(n):
     n = commontype + n
   newlist.append(n + ';')  # add type to each other
 return newlist




#--------------------------------------------- Functions

# Is the line a function prototype ?

def isprototype(line):
  global prototype
  prototype = ""
  if line is None:  return FALSE
  # Removing any embedded or trailing comment
  line = removecomment(line)

  # Is the line blank or too short?
  # The shortest seems to be a(), 3 characters long
  if len(line) < 3: return FALSE

  # Is the line a # statement?
  if line[0] == "#": return FALSE

  # Removing any block of statements

  i = string.find(line, "{")
  if i >= 0: line = line[:i]

  # Is the reduced string too short for a declaration?
  if len(line) < 3: return FALSE

  # Irrelevant
  if '=' in line: return FALSE

  # These codes must be found
  if '(' not in line : return FALSE
  if ')' not in line: return FALSE

  prototype = line

  # Is the line starting a struct?
  w = wstring.split(line, " (")
  if w[0] in ['typedef', 'struct', "union" ]: return FALSE

  # A function declaration or a prototype?
  line = wstring.strip(line)
  if line[-1] == ';': return TRUE

  return FALSE


# Is the line the return type of a multiline function interface
# inside a c source?

def typeonly(line):
 # Removing any embedded or trailing comment
 while commentfound(line): line = nocomment(line)
 if len(line) < 3: return FALSE     # Minimum is: int

 words = wstring.split(line, " \t\r\n*;", "*;")  #  * is a word
 if len(words) < 1: return FALSE
 if words[0] == "extern": return FALSE
 for w in words:
  if w not in types: return FALSE
 return TRUE


# --- Is a line a function definition inside a c source?
# If TRUE, prepare the declaration to be inserted in the c header
# and save it in the "shortened" variable
# When this function is called, it is assumed the line is a typed thing

def isfunction(line):
  global shortened

  if line is None:  return FALSE

  # Is the line blank or too short?
  # The shortest seems to be a(), 3 characters long
  if len(line) < 3: return FALSE

  # Removing any embedded or trailing comment
  line = removecomment(line)

  if len(line) < 3: return FALSE   # test again
  if line[0] == "#": return FALSE

  # Removing any block of statements
  i = string.find(line, "{")
  if i >= 0: line = line[:i]

  # test again
  if len(line) < 3: return FALSE

  # Is the last character a ; or a : ?

  if line[-1] == ";": return FALSE
  if line[-1] == ':': return FALSE

  # This code must be found
  if string.find(line, '(') == -1: return FALSE

  # Is the line splitted? Not handled here
  if line[-1] == "\\": return FALSE

  # Is the line starting a construct?

  w = wstring.split(line, " (")
  first = w[0]
  if first in constructs: return FALSE

  # Anything else starts a function definition

  # Is the return type missing? Add the default "int" type

  newreturn = "int "
  if first in reserved:  newreturn = ""     # Add nothing

  shortened = newreturn + line
  #print "function >", line
  return TRUE




# Are the definition and the declaration for the same function?
# I consider only the name of the function anything else may
# change and will be updated



def samefunction(oldline, newline):

  # The function name is the last word before "("

  def typeident(line):
    i = string.find(line, "(")
    words = wstring.split(line[:i], " \t*", "*")
    return words[len(words) - 1]

  oname = typeident(oldline)
  nname = typeident(newline)

  if wstring.iequal(oname, nname): return TRUE
  return FALSE




# --- Is the function's prototype changed?
# I consider the number of words and the types,
# I ignore names inside arguments...

def fchanged(olddec, newdec):

  # Function to get the left part or a declaration
  def left(line):
    i = string.find(line, "(")
    w = wstring.split(line[:i], " \t*", "*")
    return w

  # Comparing lists made of return type and function name
  o = left(olddec)
  n = left(newdec)
  if o != n: return TRUE

  # Function to get the part inside round brackets
  def right(dec):
    i = string.find(dec, "(")
    j = string.find(dec, ")")
    right = dec[i+1:j]
    w = wstring.split(right, " ,\t*", '*')

    # I keep only the types in the list of words
    pw = []
    for n in w:
      if n in reserved: pw.append(n)   # word in list of types
    return pw

  oright = right(olddec)
  nright = right(newdec)
  if oright != nright: return TRUE

  return FALSE


# Get the list of argument inside a function's interface
# return the list, and position of parenthesis in original string

def getargs(line):
  i = string.find(line, '(')
  j = string.find(line, ')')
  if i == -1: return None, 0,0
  if j <= i: return None, 0 ,0
  line = line[i + 1: j]            # Keeping arguments
  if line == None: return None, 0, 0
  alist = wstring.split(line, ',')
  #print "getargs", arglist
  arglist = []
  for w in alist:
   arglist.append(wstring.strip(w))
  return arglist, i, j


# count the number of arguments

def argscount(line):
  l, i, j = getargs(line)
  return len(l)


# Is the function's interface old format?
# (Names of arguments inside parenthesis, declarations on following lines)

def oldformat(line):
  arglist, i, j = getargs(line)
  if arglist == None: return FALSE
  if len(arglist) == 0: return FALSE  # No argument?
  # a void function
  if len(arglist) == 1:
     if "void" in arglist: return FALSE
     if "..." in arglist: return FALSE
  for w in arglist:                # Types present here?
   x = wstring.strip(w)            # Removing leading and trailing
   if ' 'in x: return FALSE        # Multiple words means for type present
  return TRUE


# Get the types and the name from a declaration

def gettypeident(decl):
  l = wstring.split(decl, " *;", "*")
  t = []
  n = ""
  for w in l:
    if w in types: t.append(w)
    else:          n = w
  #print "gettypeident", decl, l,"type", t, "name=", n
  return t, n


# Replace variable's name by typed declaration inside a line
# Typed arguments (found on lines following the interface) are held
# inside the "typedlist" parameter

def argreplace(line, typedlist):
 arglist, left, right = getargs(line)
 if arglist == None: return line

 # Replacing name by declaration in the interface
 for a in typedlist:  # For each declaration in list
   k = string.find(a, ';' )
   if k > -1: a = a[:k]
   t, n = gettypeident(a)             # Get types and the name

   #print "argreplace from", a, "type ", t, "name", n
   #print "searching", a, "in", arglist
   if n in arglist:
     i = arglist.index(n)
     #print "replaced", arglist[i], "by", a
     arglist[i] = a


 # Making the new argument list
 alist = ""
 for a in arglist:
  if alist != "": alist = alist + ', '
  alist = alist + string.strip(a)

 line = line [:left + 1] + alist + line[right:]
 #print "rebuilt", line
 return line



# Replace any declaration, var or func, into the include file

def replace(oline, nline):
  global shortened
  shortened = nline
  # keep a trailing comment
  def getcomment(str):
   i = string.find(str, "/*")
   if i == -1: return ""
   return str[i:]

  comment = getcomment(oline)
  shortened = shortened + comment
  return



# Get the list of files from a makefile
# This version processes a single list of file

def readproject(makefile):
  return wstring.wordfile(makefile)


# Is the string a correct header filename?
# Header from the standard library are not considered here
# they have the format <path>
# other files have the format "path"

def isheader(str):
  if str[0] == '\"':
    if str[-1] == '\"': return TRUE
  if str[0] == '<':
    if str[-1] == '>':  return FALSE
  return FALSE


#-------------------------------------------- Methods


# Change a function's interface to an interface of method

def makemethod(line, classname):
 id = getident(line)
 idcpp = classname + "::" + id       # Make a C++ name from C one

 words = wstring.splitidents(line)      # Make replacements
 nwords = []
 for w in words:                    
   if w == id:
     nwords.append(idcpp)
   else:
     nwords.append(w)

 line = wstring.join(nwords)        # Rebuild the line with changed names
 #print "class", classname, idcpp, line
 return line

# get the name from a class declaration

def getclassname(line):
  words = string.split(line)
  if len(words) < 2:        return ""
  if words[0] != "class":   return ""
  if not isident(words[1]): return ""
  return words[1]