index.py

xml开源解析代码.版本为libxml2-2.6.29,可支持GB3212.网络消息发送XML时很有用.

#!/usr/bin/python -u
#
# imports the API description and fills up a database with
# name relevance to modules, functions or web pages
#
# Operation needed:
# =================
#
# install mysqld, the python wrappers for mysql and libxml2, start mysqld
# Change the root passwd of mysql:
#    mysqladmin -u root password new_password
# Create the new database xmlsoft
#    mysqladmin -p create xmlsoft
# Create a database user 'veillard' and give him passord access
# change veillard and abcde with the right user name and passwd
#    mysql -p
#    password:
#    mysql> GRANT ALL PRIVILEGES ON xmlsoft TO veillard@localhost
#           IDENTIFIED BY 'abcde' WITH GRANT OPTION;
#
# As the user check the access:
#    mysql -p xmlsoft
#    Enter password:
#    Welcome to the MySQL monitor....
#    mysql> use xmlsoft
#    Database changed
#    mysql> quit
#    Bye
#
# Then run the script in the doc subdir, it will create the symbols and
# word tables and populate them with informations extracted from 
# the libxml2-api.xml API description, and make then accessible read-only
# by nobody@loaclhost the user expected to be Apache's one
#
# On the Apache configuration, make sure you have php support enabled
#

import MySQLdb
import libxml2
import sys
import string
import os

#
# We are not interested in parsing errors here
#
def callback(ctx, str):
    return
libxml2.registerErrorHandler(callback, None)

#
# The dictionnary of tables required and the SQL command needed
# to create them
#
TABLES={
  "symbols" : """CREATE TABLE symbols (
           name varchar(255) BINARY NOT NULL,
	   module varchar(255) BINARY NOT NULL,
           type varchar(25) NOT NULL,
	   descr varchar(255),
	   UNIQUE KEY name (name),
	   KEY module (module))""",
  "words" : """CREATE TABLE words (
           name varchar(50) BINARY NOT NULL,
	   symbol varchar(255) BINARY NOT NULL,
           relevance int,
	   KEY name (name),
	   KEY symbol (symbol),
	   UNIQUE KEY ID (name, symbol))""",
  "wordsHTML" : """CREATE TABLE wordsHTML (
           name varchar(50) BINARY NOT NULL,
	   resource varchar(255) BINARY NOT NULL,
	   section varchar(255),
	   id varchar(50),
           relevance int,
	   KEY name (name),
	   KEY resource (resource),
	   UNIQUE KEY ref (name, resource))""",
  "wordsArchive" : """CREATE TABLE wordsArchive (
           name varchar(50) BINARY NOT NULL,
	   ID int(11) NOT NULL,
           relevance int,
	   KEY name (name),
	   UNIQUE KEY ref (name, ID))""",
  "pages" : """CREATE TABLE pages (
           resource varchar(255) BINARY NOT NULL,
	   title varchar(255) BINARY NOT NULL,
	   UNIQUE KEY name (resource))""",
  "archives" : """CREATE TABLE archives (
           ID int(11) NOT NULL auto_increment,
           resource varchar(255) BINARY NOT NULL,
	   title varchar(255) BINARY NOT NULL,
	   UNIQUE KEY id (ID,resource(255)),
	   INDEX (ID),
	   INDEX (resource))""",
  "Queries" : """CREATE TABLE Queries (
           ID int(11) NOT NULL auto_increment,
	   Value varchar(50) NOT NULL,
	   Count int(11) NOT NULL,
	   UNIQUE KEY id (ID,Value(35)),
	   INDEX (ID))""",
  "AllQueries" : """CREATE TABLE AllQueries (
           ID int(11) NOT NULL auto_increment,
	   Value varchar(50) NOT NULL,
	   Count int(11) NOT NULL,
	   UNIQUE KEY id (ID,Value(35)),
	   INDEX (ID))""",
}

#
# The XML API description file to parse
#
API="libxml2-api.xml"
DB=None

#########################################################################
#									#
#                  MySQL database interfaces				#
#									#
#########################################################################
def createTable(db, name):
    global TABLES

    if db == None:
        return -1
    if name == None:
        return -1
    c = db.cursor()

    ret = c.execute("DROP TABLE IF EXISTS %s" % (name))
    if ret == 1:
        print "Removed table %s" % (name)
    print "Creating table %s" % (name)
    try:
        ret = c.execute(TABLES[name])
    except:
        print "Failed to create table %s" % (name)
	return -1
    return ret

def checkTables(db, verbose = 1):
    global TABLES

    if db == None:
        return -1
    c = db.cursor()
    nbtables = c.execute("show tables")
    if verbose:
	print "Found %d tables" % (nbtables)
    tables = {}
    i = 0
    while i < nbtables:
        l = c.fetchone()
	name = l[0]
	tables[name] = {}
        i = i + 1

    for table in TABLES.keys():
        if not tables.has_key(table):
	    print "table %s missing" % (table)
	    createTable(db, table)
	try:
	    ret = c.execute("SELECT count(*) from %s" % table);
	    row = c.fetchone()
	    if verbose:
		print "Table %s contains %d records" % (table, row[0])
	except:
	    print "Troubles with table %s : repairing" % (table)
	    ret = c.execute("repair table %s" % table);
	    print "repairing returned %d" % (ret)
	    ret = c.execute("SELECT count(*) from %s" % table);
	    row = c.fetchone()
	    print "Table %s contains %d records" % (table, row[0])
    if verbose:
	print "checkTables finished"

    # make sure apache can access the tables read-only
    try:
	ret = c.execute("GRANT SELECT ON xmlsoft.* TO nobody@localhost")
	ret = c.execute("GRANT INSERT,SELECT,UPDATE  ON xmlsoft.Queries TO nobody@localhost")
    except:
        pass
    return 0
    
def openMySQL(db="xmlsoft", passwd=None, verbose = 1):
    global DB

    if passwd == None:
        try:
	    passwd = os.environ["MySQL_PASS"]
	except:
	    print "No password available, set environment MySQL_PASS"
	    sys.exit(1)

    DB = MySQLdb.connect(passwd=passwd, db=db)
    if DB == None:
        return -1
    ret = checkTables(DB, verbose)
    return ret

def updateWord(name, symbol, relevance):
    global DB

    if DB == None:
        openMySQL()
    if DB == None:
        return -1
    if name == None:
        return -1
    if symbol == None:
        return -1

    c = DB.cursor()
    try:
	ret = c.execute(
"""INSERT INTO words (name, symbol, relevance) VALUES ('%s','%s', %d)""" %
		(name, symbol, relevance))
    except:
        try:
	    ret = c.execute(
    """UPDATE words SET relevance = %d where name = '%s' and symbol = '%s'""" %
		    (relevance, name, symbol))
	except:
	    print "Update word (%s, %s, %s) failed command" % (name, symbol, relevance)
	    print "UPDATE words SET relevance = %d where name = '%s' and symbol = '%s'" % (relevance, name, symbol)
	    print sys.exc_type, sys.exc_value
	    return -1
	     
    return ret

def updateSymbol(name, module, type, desc):
    global DB

    updateWord(name, name, 50)
    if DB == None:
        openMySQL()
    if DB == None:
        return -1
    if name == None:
        return -1
    if module == None:
        return -1
    if type == None:
        return -1

    try:
	desc = string.replace(desc, "'", " ")
	l = string.split(desc, ".")
	desc = l[0]
	desc = desc[0:99]
    except:
        desc = ""

    c = DB.cursor()
    try:
	ret = c.execute(
"""INSERT INTO symbols (name, module, type, descr) VALUES ('%s','%s', '%s', '%s')""" %
                    (name, module, type, desc))
    except:
        try:
	    ret = c.execute(
"""UPDATE symbols SET module='%s', type='%s', descr='%s' where name='%s'""" %
                    (module, type, desc, name))
        except:
	    print "Update symbol (%s, %s, %s) failed command" % (name, module, type)
	    print """UPDATE symbols SET module='%s', type='%s', descr='%s' where name='%s'""" % (module, type, desc, name)
	    print sys.exc_type, sys.exc_value
	    return -1
	     
    return ret
        
def addFunction(name, module, desc = ""):
    return updateSymbol(name, module, 'function', desc)

def addMacro(name, module, desc = ""):
    return updateSymbol(name, module, 'macro', desc)

def addEnum(name, module, desc = ""):
    return updateSymbol(name, module, 'enum', desc)

def addStruct(name, module, desc = ""):
    return updateSymbol(name, module, 'struct', desc)

def addConst(name, module, desc = ""):
    return updateSymbol(name, module, 'const', desc)

def addType(name, module, desc = ""):
    return updateSymbol(name, module, 'type', desc)

def addFunctype(name, module, desc = ""):
    return updateSymbol(name, module, 'functype', desc)

def addPage(resource, title):
    global DB

    if DB == None:
        openMySQL()
    if DB == None:
        return -1
    if resource == None:
        return -1

    c = DB.cursor()
    try:
	ret = c.execute(
	    """INSERT INTO pages (resource, title) VALUES ('%s','%s')""" %
                    (resource, title))
    except:
        try:
	    ret = c.execute(
		"""UPDATE pages SET title='%s' WHERE resource='%s'""" %
                    (title, resource))
        except:
	    print "Update symbol (%s, %s, %s) failed command" % (name, module, type)
	    print """UPDATE pages SET title='%s' WHERE resource='%s'""" % (title, resource)
	    print sys.exc_type, sys.exc_value
	    return -1
	     
    return ret

def updateWordHTML(name, resource, desc, id, relevance):
    global DB

    if DB == None:
        openMySQL()
    if DB == None:
        return -1
    if name == None:
        return -1
    if resource == None:
        return -1
    if id == None:
        id = ""
    if desc == None:
        desc = ""
    else:
	try:
	    desc = string.replace(desc, "'", " ")
	    desc = desc[0:99]
	except:
	    desc = ""

    c = DB.cursor()
    try:
	ret = c.execute(
"""INSERT INTO wordsHTML (name, resource, section, id, relevance) VALUES ('%s','%s', '%s', '%s', '%d')""" %
                    (name, resource, desc, id, relevance))
    except:
        try:
	    ret = c.execute(
"""UPDATE wordsHTML SET section='%s', id='%s', relevance='%d' where name='%s' and resource='%s'""" %
                    (desc, id, relevance, name, resource))
        except:
	    print "Update symbol (%s, %s, %d) failed command" % (name, resource, relevance)
	    print """UPDATE wordsHTML SET section='%s', id='%s', relevance='%d' where name='%s' and resource='%s'""" % (desc, id, relevance, name, resource)
	    print sys.exc_type, sys.exc_value
	    return -1
	     
    return ret

def checkXMLMsgArchive(url):
    global DB

    if DB == None:
        openMySQL()
    if DB == None:
        return -1
    if url == None:
        return -1

    c = DB.cursor()
    try:
	ret = c.execute(
	    """SELECT ID FROM archives WHERE resource='%s'""" % (url))
	row = c.fetchone()
	if row == None:
	    return -1
    except:
	return -1
	     
    return row[0]
    
def addXMLMsgArchive(url, title):
    global DB

    if DB == None:
        openMySQL()
    if DB == None:
        return -1
    if url == None:
        return -1
    if title == None:
        title = ""
    else:
	title = string.replace(title, "'", " ")
	title = title[0:99]

    c = DB.cursor()
    try:
        cmd = """INSERT INTO archives (resource, title) VALUES ('%s','%s')""" % (url, title)
        ret = c.execute(cmd)
	cmd = """SELECT ID FROM archives WHERE resource='%s'""" % (url)
        ret = c.execute(cmd)
	row = c.fetchone()
	if row == None:
	    print "addXMLMsgArchive failed to get the ID: %s" % (url)
	    return -1
    except:
        print "addXMLMsgArchive failed command: %s" % (cmd)
	return -1
	     
    return((int)(row[0]))

def updateWordArchive(name, id, relevance):
    global DB

    if DB == None:
        openMySQL()
    if DB == None:
        return -1