lexer.py

convert C programs for use in C++ compiler environment

# Process struct or enum
# a typedef of struct is out of concern here
# an extern also is it concerns an instance
# (this function may be called when keyword "struct" is found)

structlevel = 0

def openstruct(line):
 if line is None: return FALSE
 line = removecomment(line)
 if len(line) == 0: return FALSE
 temp = string.lower(line)

 words = wstring.split(temp, " \t\r\n")
 if len(words) < 2: return FALSE     # 2 tokens or 1 token and 1 ident
 first  = string.lower(words[0])
 second = string.lower(words[1])
 if first in ["typedef"]: return FALSE
 if first in ["struct", "enum"]:  return TRUE
 """
 if first in ["extern"]:
   if second not in [ "struct"] : return FALSE
   # following code keep instances of struct inside classes
   # but I have decided to put them oustide classes along with decl.
   if ';' in temp:
     if not '{' in temp: return FALSE
     if not '}' in temp: return FALSE
     k = string.rfind(temp, '}')
     if i < k: return FALSE
   return TRUE
 """
 return FALSE


def closestruct(line):
 global structlevel
 for c in line:
  if c == '{':   structlevel = structlevel + 1
  if c == '}':
    structlevel = structlevel - 1
    if structlevel < 0: structlevel = 0
    if structlevel == 0: return FALSE
 return TRUE

# Found a typedef of struct: must be outside class

def typedefstruct(line):
 if line is None: return FALSE
 line = removecomment(line)
 if len(line) == 0: return FALSE
 temp = string.lower(line)

 words = wstring.split(temp, " \t\r\n")
 if len(words) < 2: return FALSE     # 2 tokens or 1 token and 1 ident
 first  = string.lower(words[0])
 second = string.lower(words[1])
 if first not in ["typedef"]: return FALSE
 if second not in ["struct", "enum"]:  return FALSE
 return TRUE



# Is the line a declaration of struct instance
# or an enum, inside header file?

def recordinstance(line):
 line = removecomment(line)
 if line is None: return FALSE
 if len(line) < 18: return FALSE    # Minimum is "extern struct s a;"
 line = wstring.strip(line)
 if line[0] == '#': return FALSE

 words = wstring.split(line, symstr)
 if len(words) < 2: return FALSE
 first  = string.lower(words[0])
 second = string.lower(words[0])
 if first == "enum": return TRUE
 if first != "extern" : return FALSE
 if second != "struct": return FALSE
 if ';' not in line: return FALSE
 if '{' in line:
   i = string.find(line, ';')
   j = string.find(line, '{')
   if j < i: return FALSE
 return TRUE


# a single line typedef

def linetypedef(line):
  if string.find(line, "typedef") != -1:
    #print "typedef >", line
    line = removecomment(line)
    line = wstring.chop(line)
    line = wstring.strip(line)
    i = string.find(line, ';')
    j = string.find(line, '{')
    if i == -1: return FALSE
    if j == -1: return TRUE
    if i < j: return TRUE
  return FALSE


# an instance of struct

def linestruct(line):
  if string.find(line, "struct") != -1:
    line = removecomment(line)
    line = wstring.chop(line)
    line = wstring.strip(line)
    i = string.find(line, ';')
    j = string.find(line, '{')
    if i == -1: return FALSE
    if j == -1: return TRUE
    if i < j: return TRUE
  return FALSE


typelevel = 0

def opentypedef(line):
 if line is None: return FALSE
 if len(line) == 0: return FALSE
 line = string.lower(line)
 if string.find(line, "typedef") == -1: return FALSE
 #print "open typedef >", line
 return TRUE

def closetypedef(line):
 global typelevel
 # single line

 for c in line:
  if c == '{':   typelevel = typelevel + 1
  if c == '}':
    typelevel = typelevel - 1
    if typelevel < 0: typelevel = 0
    if typelevel == 0: return FALSE
 return TRUE


# this test is typedef in this line

def istypedef(line):
  return string.find(line, "typedef") != -1

# test if the typedef is not a struct

def newtype(line):
  if not istypedef(line): return FALSE
  line = removecomment(line)
  i = string.find(line, ';')
  if i == -1: return FALSE
  j = string.find(line, '{')
  if j == -1: return TRUE
  if i < j: return TRUE
  j = string.find(line, '}')
  if j == -1: return FALSE
  if j < i: return TRUE
  return FALSE


# get the ident

def lastword(line):
  words = wstring.split(line, symstr)
  #print "lastword in", words
  l = len(words)
  if l < 1: return None
  last = words[l -1]
  #print "is ident?", last
  if not isident(last): return None
  return last


# Get the name in a typedef: this is the last word

def typename(line):
 i = string.rfind(line, '}')
 if i == -1: i = 0;
 words = wstring.split(line[i:], " };*", "*")
 l = len(words)
 if l == 0: return None
 last = words[l - 1]
 if last in types: return None
 return last


# Get the name of a struct: this is the words following "struct"

def structname(line):
 line = removecomment(line)
 words = wstring.split(line, " {;")
 if len(words) < 2: return None
 first = string.lower(words[0])
 second = string.lower(words[1])
 if first == "extern":
   if len(words) < 3: return None
   first = second
   second = string.lower(words[2])
 if first == "struct": return second
 return None


# Process union

ulevel = 0

def openunion(line):
 if line is None: return FALSE
 if len(line) < 5: return FALSE
 line = string.lower(line)
 return string.find(line, "union")  != -1

def closeunion(line):
 global ulevel
 for c in line:
  if c == '{':   ulevel = ulevel + 1
  if c == '}':
    ulevel = ulevel - 1
    if ulevel < 0: ulevel = 0
    if ulevel == 0: return FALSE
 return TRUE


# Processing a block of any kind

def openbracket(line):
 if line is None:   return FALSE
 if len(line) == 0: return FALSE
 if '{' in line:    return TRUE
 return FALSE


# Test for a start of array assignment


def openarray(line):
 global arraylevel
 arraylevel = 0
 if line is None: return FALSE
 if len(line) == 0: return FALSE
 if '{' in line: return TRUE
 return FALSE

# Called only when an array assignment is opened
# Return TRUE while inside the coumpound block

def closearray(line):
 global arraylevel
 for c in line:
  if c == '{':   arraylevel = arraylevel + 1
  if c == '}':
    arraylevel = arraylevel - 1
    if arraylevel < 0: arraylevel = 0
    if arraylevel == 0: return FALSE
 return TRUE


# Process a class

classlevel = 0

def openclass(line):
 if line is None: return FALSE
 if len(line) == 0: return FALSE
 line = string.lower(line)
 if string.find(line, "class") == -1: return FALSE
 return TRUE

def closeclass(line):
 global classlevel
 for c in line:
  if c == '{':   classlevel = classlevel + 1
  if c == '}':
    classlevel = classlevel - 1
    if classlevel < 0: classlevel = 0
    if classlevel == 0: return FALSE
 return TRUE


#--------------------------------------- Global variables

# Removing extra codes
# From a single var or multi var definition

def getdecl(line):
 while commentfound(line):
  line = nocomment(line)

 line = wstring.strip(line)
 return line


# Is the line an array declaration or definition?
# I test a list of type keywords, then an ident, then "["

def isarray(line):
 words = wstring.split(line, " \r\t\n*()[];{}=", "*[]();{}=")
 if len(words) == 0: return FALSE
 if words[0] not in types: return FALSE

 while len(words) > 0:
  if words[0] in types:
    words.remove(words[0])
    continue
  break

 if len(words) == 0: return FALSE
 if not isident(words[0]): return FALSE

 words.remove(words[0])
 if len(words) == 0: return FALSE
 if words[0] != '[': return FALSE
 return TRUE


# Process the array declaration according to keys of #define statement
# eventually embedded
# Uses the dictionnary of valid define statements
# Indicates if a #define should be included into the header

def processarray(line, defdict, defflag):
 global shortened
 line = removecomment(line)

 i = string.find(line, '=')
 if i != -1: line = line[:i]

 i = string.find(line, '[')
 shortened = line[:i]    # type and name

 sizes = []
 param = ""

 # Extracting each parameter
 #print "process array", line

 while 1:
  i = string.find(line, '[')
  if i == -1: break
  j = string.find(line, ']')
  if j == -1: break
  param = line[i+1:j]        # The parameter inside brackets
  param = wstring.strip(param)
  sizes.append(param)        # Add to list
  line = line[j+1:]            # Go to next
  if len(line) < 3: break

 # Now, searching for embedded define's names
 k = defdict.keys()

 for name in sizes:
  shortened = shortened + '[' + name + ']'
  if len(name) > 0:
    #print "shortened", shortened, "in k?"
    if name in k:                      # This parameter in list?
      defflag[name] = TRUE
      #print "flag changed for", name, defflag[name]

 shortened = shortened + ';'
 return


# Make a list of dimensions

def getarraysize(line):
 line = removecomment(line)
 i = string.find(line, '=')
 if i != -1: line = line[:i]

 # Extracting each parameter
 sizes = []
 while 1:
  i = string.find(line, '[')
  if i == -1: break
  j = string.find(line, ']')
  if j == -1: break
  param = line[i+1:j]          # The parameter inside brackets
  param = wstring.strip(param)
  sizes.append(param)          # Add to list
  line = line[j+1:]            # Go to next
  if len(line) < 3: break

 return sizes

# is the line an instance of struct?

def structinstance(words):
  if words[0] == "struct":
   if isident(words[1]):
     i = 2
     if words[i] == "huge": i += 1
     if words[i] == "*": i += 1
     if len(words) != (i + 1): return FALSE
     if isident(words[i]):
       if words[i + 1] == ";":
         return TRUE
  return FALSE


# This function is used inside header files only
# Is the line a variable declaration?
# Any statement that starts with a modifier or type
# or predefined type, is either a function prototype
# or a variable declaration

vardelimiters = " \\()[]{},;:?+-*=/&~|\"\'<>!\r\n\t`%^"

def isvardecl(line):
 if line is None: return FALSE
 if len(line) < 4: return FALSE    # Minimum is "t a;" if t is a typedef
 line = wstring.strip(line)
 if line[0] == '#': return FALSE

 if isprototype(line): return FALSE

 # some delimiters are kept in the list of words
 words = wstring.split(line, vardelimiters , ";,()*[]")
 #print "(isvardecl word in type?)", words[0]

 if len(words) < 3: return FALSE

 if words[0] in modifiers: return TRUE
 if words[0] in types: return TRUE
 if words[0] in predefined: return TRUE
 return FALSE


# Is the line a global var definition in source (or list of var)?
# Test for array must be previously performed

def isvardef(line):
 #print "928 isvardef", line
 if line is None: return FALSE
 line = removecomment(line)

 if len(line) < 6: return FALSE    # Minimum is int a;
 if line[0] == "#": return FALSE   # Define statement

 if isfunction(line):
   #print "function >>>", line
   return FALSE
 #print "var >>>", line

 # I keep * that is a type keyword
 words = wstring.split(line, vardelimiters, '*;[')
 if len(words) < 3: return FALSE
 #print "words[0]", words[0]
 if words[0] in modifiers:  return TRUE
 if words[0] in predefined: return TRUE
 if words[0] in types:      return TRUE
 if structinstance(words):  return TRUE

 return FALSE


# Is the line a local definition in source (or list of var)?

def islocalvar(line):
 #print "isvardef", line

 if line is None: return FALSE
 line = removecomment(line)

 if len(line) < 6: return FALSE    # Minimum is int a;
 if line[0] == "#": return FALSE   # Define statement

 # I keep separators for further tests and * that is a keyword
 words = wstring.split(line, vardelimiters, '*,;[]()=')
 if len(words) < 3: return FALSE

 # extern declaration not concerned,  * require a type before
 if words[0] in [ "extern", "*" ]: return FALSE

 # modifier concern either a var or a function
 if words[0] in modifiers : return TRUE

 if structinstance(words):  return TRUE
 if linestruct(line): return TRUE

 if words[0] in predefined:
    words.remove(words[0])
    if words[0] == '*':
       words.remove('*')
 else:
   # skipping all type keywords
   if words[0] not in types: return FALSE
   while words[0] in types:
     words.remove(words[0])
     if len(words) == 0: return FALSE

 # Unknown statement, requires at last:  ident and delimiter
 if len(words) < 2: return FALSE
 if not isident(words[0]): return FALSE

 second = words[1]
 if second == ';': return TRUE          # End of declaration
 if second == ',': return TRUE          # Multiple declarations
 if second == '[': return TRUE          # Array
 if second == '=': return TRUE          # Assignent

 if second == '(': return FALSE         # Function

 if isident(second):
  print words[0], "is a type defined by the programmer"
  return TRUE