apibuild.py

xml开源解析代码.版本为libxml2-2.6.29,可支持GB3212.网络消息发送XML时很有用.

	    if token[0] == "name" and token[1] == "int":
		if self.type == "":
		    self.type = tmp[1]
		else:
		    self.type = self.type + " " + tmp[1]
	     
        elif token[0] == "name" and token[1] == "struct":
	    if self.type == "":
	        self.type = token[1]
	    else:
	        self.type = self.type + " " + token[1]
	    token = self.token()
	    nametok = None
	    if token[0] == "name":
	        nametok = token
		token = self.token()
	    if token != None and token[0] == "sep" and token[1] == "{":
		token = self.token()
		token = self.parseStruct(token)
	    elif token != None and token[0] == "op" and token[1] == "*":
	        self.type = self.type + " " + nametok[1] + " *"
		token = self.token()
		while token != None and token[0] == "op" and token[1] == "*":
		    self.type = self.type + " *"
		    token = self.token()
		if token[0] == "name":
		    nametok = token
		    token = self.token()
		else:
		    self.error("struct : expecting name", token)
		    return token
	    elif token != None and token[0] == "name" and nametok != None:
	        self.type = self.type + " " + nametok[1]
		return token

	    if nametok != None:
		self.lexer.push(token)
		token = nametok
	    return token

        elif token[0] == "name" and token[1] == "enum":
	    if self.type == "":
	        self.type = token[1]
	    else:
	        self.type = self.type + " " + token[1]
	    self.enums = []
	    token = self.token()
	    if token != None and token[0] == "sep" and token[1] == "{":
		token = self.token()
		token = self.parseEnumBlock(token)
	    else:
		self.error("parsing enum: expecting '{'", token)
	    enum_type = None
	    if token != None and token[0] != "name":
	        self.lexer.push(token)
	        token = ("name", "enum")
	    else:
	        enum_type = token[1]
	    for enum in self.enums:
		self.index_add(enum[0], self.filename,
			       not self.is_header, "enum",
			       (enum[1], enum[2], enum_type))
	    return token

	elif token[0] == "name":
	    if self.type == "":
	        self.type = token[1]
	    else:
	        self.type = self.type + " " + token[1]
	else:
	    self.error("parsing type %s: expecting a name" % (self.type),
	               token)
	    return token
	token = self.token()
        while token != None and (token[0] == "op" or
	      token[0] == "name" and token[1] == "const"):
	    self.type = self.type + " " + token[1]
	    token = self.token()

	 #
	 # if there is a parenthesis here, this means a function type
	 #
	if token != None and token[0] == "sep" and token[1] == '(':
	    self.type = self.type + token[1]
	    token = self.token()
	    while token != None and token[0] == "op" and token[1] == '*':
	        self.type = self.type + token[1]
		token = self.token()
	    if token == None or token[0] != "name" :
		self.error("parsing function type, name expected", token);
	        return token
	    self.type = self.type + token[1]
	    nametok = token
	    token = self.token()
	    if token != None and token[0] == "sep" and token[1] == ')':
		self.type = self.type + token[1]
		token = self.token()
		if token != None and token[0] == "sep" and token[1] == '(':
		    token = self.token()
		    type = self.type;
		    token = self.parseSignature(token);
		    self.type = type;
		else:
		    self.error("parsing function type, '(' expected", token);
		    return token
	    else:
	        self.error("parsing function type, ')' expected", token);
		return token
	    self.lexer.push(token)
	    token = nametok
	    return token

         #
	 # do some lookahead for arrays
	 #
	if token != None and token[0] == "name":
	    nametok = token
	    token = self.token()
	    if token != None and token[0] == "sep" and token[1] == '[':
	        self.type = self.type + nametok[1]
		while token != None and token[0] == "sep" and token[1] == '[':
		    self.type = self.type + token[1]
		    token = self.token()
		    while token != None and token[0] != 'sep' and \
		          token[1] != ']' and token[1] != ';':
			self.type = self.type + token[1]
			token = self.token()
		if token != None and token[0] == 'sep' and token[1] == ']':
		    self.type = self.type + token[1]
		    token = self.token()
		else:
		    self.error("parsing array type, ']' expected", token);
		    return token
	    elif token != None and token[0] == "sep" and token[1] == ':':
	         # remove :12 in case it's a limited int size
		token = self.token()
		token = self.token()
	    self.lexer.push(token)
	    token = nametok

	return token

     #
     # Parse a signature: '(' has been parsed and we scan the type definition
     #    up to the ')' included
    def parseSignature(self, token):
        signature = []
	if token != None and token[0] == "sep" and token[1] == ')':
	    self.signature = []
	    token = self.token()
	    return token
	while token != None:
	    token = self.parseType(token)
	    if token != None and token[0] == "name":
	        signature.append((self.type, token[1], None))
		token = self.token()
	    elif token != None and token[0] == "sep" and token[1] == ',':
		token = self.token()
		continue
	    elif token != None and token[0] == "sep" and token[1] == ')':
	         # only the type was provided
		if self.type == "...":
		    signature.append((self.type, "...", None))
		else:
		    signature.append((self.type, None, None))
	    if token != None and token[0] == "sep":
	        if token[1] == ',':
		    token = self.token()
		    continue
		elif token[1] == ')':
		    token = self.token()
		    break
	self.signature = signature
	return token

     #
     # Parse a global definition, be it a type, variable or function
     # the extern "C" blocks are a bit nasty and require it to recurse.
     #
    def parseGlobal(self, token):
        static = 0
        if token[1] == 'extern':
	    token = self.token()
	    if token == None:
	        return token
	    if token[0] == 'string':
	        if token[1] == 'C':
		    token = self.token()
		    if token == None:
			return token
		    if token[0] == 'sep' and token[1] == "{":
		        token = self.token()
#			 print 'Entering extern "C line ', self.lineno()
			while token != None and (token[0] != 'sep' or
			      token[1] != "}"):
			    if token[0] == 'name':
				token = self.parseGlobal(token)
			    else:
				self.error(
				 "token %s %s unexpected at the top level" % (
					token[0], token[1]))
				token = self.parseGlobal(token)
#			 print 'Exiting extern "C" line', self.lineno()
			token = self.token()
			return token
		else:
		    return token
	elif token[1] == 'static':
	    static = 1
	    token = self.token()
	    if token == None or  token[0] != 'name':
	        return token

	if token[1] == 'typedef':
	    token = self.token()
	    return self.parseTypedef(token)
	else:
	    token = self.parseType(token)
	    type_orig = self.type
	if token == None or token[0] != "name":
	    return token
	type = type_orig
	self.name = token[1]
	token = self.token()
	while token != None and (token[0] == "sep" or token[0] == "op"):
	    if token[0] == "sep":
		if token[1] == "[":
		    type = type + token[1]
		    token = self.token()
		    while token != None and (token[0] != "sep" or \
		          token[1] != ";"):
			type = type + token[1]
			token = self.token()

	    if token != None and token[0] == "op" and token[1] == "=":
		 #
		 # Skip the initialization of the variable
		 #
		token = self.token()
		if token[0] == 'sep' and token[1] == '{':
		    token = self.token()
		    token = self.parseBlock(token)
		else:
		    self.comment = None
		    while token != None and (token[0] != "sep" or \
			  (token[1] != ';' and token[1] != ',')):
			    token = self.token()
		self.comment = None
		if token == None or token[0] != "sep" or (token[1] != ';' and
		   token[1] != ','):
		    self.error("missing ';' or ',' after value")

	    if token != None and token[0] == "sep":
		if token[1] == ";":
		    self.comment = None
		    token = self.token()
		    if type == "struct":
		        self.index_add(self.name, self.filename,
			     not self.is_header, "struct", self.struct_fields)
		    else:
			self.index_add(self.name, self.filename,
			     not self.is_header, "variable", type)
		    break
		elif token[1] == "(":
		    token = self.token()
		    token = self.parseSignature(token)
		    if token == None:
			return None
		    if token[0] == "sep" and token[1] == ";":
		        d = self.mergeFunctionComment(self.name,
				((type, None), self.signature), 1)
			self.index_add(self.name, self.filename, static,
			                "function", d)
			token = self.token()
		    elif token[0] == "sep" and token[1] == "{":
		        d = self.mergeFunctionComment(self.name,
				((type, None), self.signature), static)
			self.index_add(self.name, self.filename, static,
			                "function", d)
			token = self.token()
			token = self.parseBlock(token);
		elif token[1] == ',':
		    self.comment = None
		    self.index_add(self.name, self.filename, static,
		                    "variable", type)
		    type = type_orig
		    token = self.token()
		    while token != None and token[0] == "sep":
		        type = type + token[1]
			token = self.token()
		    if token != None and token[0] == "name":
		        self.name = token[1]
			token = self.token()
		else:
		    break
		    
	return token

    def parse(self):
        self.warning("Parsing %s" % (self.filename))
        token = self.token()
	while token != None:
            if token[0] == 'name':
	        token = self.parseGlobal(token)
            else:
	        self.error("token %s %s unexpected at the top level" % (
		       token[0], token[1]))
		token = self.parseGlobal(token)
		return
	self.parseTopComment(self.top_comment)
        return self.index
	         

class docBuilder:
    """A documentation builder"""
    def __init__(self, name, directories=['.'], excludes=[]):
        self.name = name
        self.directories = directories
	self.excludes = excludes + ignored_files.keys()
	self.modules = {}
	self.headers = {}
	self.idx = index()
        self.xref = {}
	self.index = {}
	if name == 'libxml2':
	    self.basename = 'libxml'
	else:
	    self.basename = name

    def indexString(self, id, str):
	if str == None:
	    return
	str = string.replace(str, "'", ' ')
	str = string.replace(str, '"', ' ')
	str = string.replace(str, "/", ' ')
	str = string.replace(str, '*', ' ')
	str = string.replace(str, "[", ' ')
	str = string.replace(str, "]", ' ')
	str = string.replace(str, "(", ' ')
	str = string.replace(str, ")", ' ')
	str = string.replace(str, "<", ' ')
	str = string.replace(str, '>', ' ')
	str = string.replace(str, "&", ' ')
	str = string.replace(str, '#', ' ')
	str = string.replace(str, ",", ' ')
	str = string.replace(str, '.', ' ')
	str = string.replace(str, ';', ' ')
	tokens = string.split(str)
	for token in tokens:
	    try:
		c = token[0]
		if string.find(string.letters, c) < 0:
		    pass
		elif len(token) < 3:
		    pass
		else:
		    lower = string.lower(token)
		    # TODO: generalize this a bit
		    if lower == 'and' or lower == 'the':
			pass
		    elif self.xref.has_key(token):
			self.xref[token].append(id)
		    else:
			self.xref[token] = [id]
	    except:
		pass

    def analyze(self):
        print "Project %s : %d headers, %d modules" % (self.name, len(self.headers.keys()), len(self.modules.keys()))
	self.idx.analyze()

    def scanHeaders(self):
	for header in self.headers.keys():
	    parser = CParser(header)
	    idx = parser.parse()
	    self.headers[header] = idx;
	    self.idx.merge(idx)

    def scanModules(self):
	for module in self.modules.keys():
	    parser = CParser(module)
	    idx = parser.parse()
	    # idx.analyze()
	    self.modules[module] = idx
	    self.idx.merge_public(idx)

    def scan(self):
        for directory in self.directories:
	    files = glob.glob(directory + "/*.c")
	    for file in files:
	        skip = 0
		for excl in self.excludes:
		    if string.find(file, excl) != -1:
		        skip = 1;
			break
		if skip == 0:
		    self.modules[file] = None;
	    files = glob.glob(directory + "/*.h")
	    for file in files:
	        skip = 0
		for excl in self.excludes:
		    if string.find(file, excl) != -1:
		        skip = 1;
			break
		if skip == 0:
		    self.headers[file] = None;
	self.scanHeaders()
	self.scanModules()
         
    def modulename_file(self, file):
        module = os.path.basename(file)
	if module[-2:] == '.h':
	    module = module[:-2]
	elif module[-2:] == '.c':
	    module = module[:-2]
	return module

    def serialize_enum(self, output, name):
        id = self.idx.enums[name]
        output.write("    <enum name='%s' file='%s'" % (name,
	             self.modulename_file(id.header)))
	if id.info != None:
	    info = id.info
	    if info[0] != None and info[0] != '':
	        try:
		    val = eval(info[0])
		except: