parser.py

wxPython的基本示例程序

#----------------------------------------------------------------------------
# Name:         parser.py
# Purpose:      parsing utilities
#
# Author:       Jeff Norton
#
# Created:      8/9/05
# CVS-ID:       $Id: parser.py,v 1.1 2006/04/20 06:25:51 RD Exp $
# Copyright:    (c) 2004-2005 ActiveGrid, Inc.
# License:      wxWindows License
#----------------------------------------------------------------------------

import re
from activegrid.util.lang import *
ifDefPy()
import string
import array
endIfDef()

XPATH_ROOT_VAR = '__rootObj__'
GETOBJECTPARTNAMES  =   ["primaryRef", "ref", "orderings", "limit"]

class Tokenizer(object):
    
    TOKEN_IDENT = 1
    TOKEN_STRING = 2
    TOKEN_OP = 3
    TOKEN_WS = 4
##    TOKEN_PLACEHOLDER = 5
    
    def __init__(self, text, identStart=None, tokenSep=None, ignoreWhitespace=True):
        """
Turn a string into individual tokens.  Three types of tokens are recognized:
    TOKEN_IDENT:   identifiers (those that start with the identStart pattern)
    TOKEN_STRING:  quoted string
    TOKEN_OP:      everything else
Tokens are separated by white space or the tokenSep pattern.  
Constructor parameters:
    text:  The string to tokenize
    identStart:  A regular expression describing characters which start an identifier
                 The default expression accepts letters, "_", and "/".
    tokenSep:    A regular expression describing the characters which end a token 
                 (in addition to whitespace).  The default expression accepts
                 anything except alpha-numerics, "_", "/", and ":".
Usage:
    Invoke getNextToken (or next) to get the next token.  The instance variables
    token, and tokenVal will be populated with the current token type (TOKEN_IDENT,
    TOKEN_STRING, or TOEKN_OP) and value respectively.  nextToken and nextTokenVal
    will also be available for lookahead.   The next method is similar to
    getNextToken but also returns the token value.  A value of None signals end
    of stream.                                           
        """
        self.ignoreWhitespace=ignoreWhitespace
        ifDefPy()
        if (isinstance(text, array.array)):
            text = text.tostring()
        endIfDef()
        self.text = asString(text)
        self.textIndex = 0
        self.textLen = len(self.text)
        self.token = None
        self.tokenVal = None
        self.nextToken = None
        self.nextTokenVal = None
        if (identStart == None):
            identStart = "[a-zA-Z_/]"
        if (tokenSep == None):
            tokenSep = "[^a-zA-Z0-9_/:]"
        self.identStart = re.compile(identStart)
        self.tokenSep = re.compile(tokenSep)
        self.getNextToken() # Prime the pump
        
    def isEscaped(text, index):
        if ((index > 0) and (text[index-1] == '\\') and ((index < 2) or (text[index-2] != '\\'))):
            return True
        return False
    isEscaped = staticmethod(isEscaped)

    def findClosingQuote(text, index, char):
        index = index + 1
        while True:
            endIndex = text.find(char, index)
            if (endIndex < 1):
                return -1
            if (Tokenizer.isEscaped(text, endIndex)):
                index = endIndex+1
            else:
                break
        return endIndex + 1
    findClosingQuote = staticmethod(findClosingQuote)

    def _findClosing(self, char):
        if (self.textIndex >= self.textLen):
            raise Exception("The text \"%s\" has an unmatched string starting at %d" % (self.text, self.textIndex))
        index = Tokenizer.findClosingQuote(self.text, self.textIndex, char)
        if (index < 0):
            raise Exception("The text \"%s\" has an unmatched string starting at %d" % (self.text, self.textIndex-1))
        return index
        
    def next(self):
        self.getNextToken()
        if (self.token == None):
            raise StopIteration()
        return self.tokenVal

    def getNextToken(self):
        self.token = self.nextToken
        self.tokenVal = self.nextTokenVal
        while (self.textIndex < self.textLen):
            c = self.text[self.textIndex]
            if (c not in string.whitespace):
                if (c == '"' or c == "'" or c == '`'):
                    endIndex = self._findClosing(c)
                    self.nextToken = self.TOKEN_STRING
                    self.nextTokenVal = self.text[self.textIndex:endIndex]
                    self.textIndex = endIndex
                    return
                elif (self.identStart.search(c)):
                    endMatch = self.tokenSep.search(self.text, self.textIndex+1)
                    if (endMatch):
                        endIndex = endMatch.start()
                    else:
                        endIndex = self.textLen
                    self.nextToken = self.TOKEN_IDENT
                    self.nextTokenVal = self.text[self.textIndex:endIndex]
                    self.textIndex = endIndex
                    return
                else:
                    self.nextToken = self.TOKEN_OP
                    endIndex = self.textIndex + 1
                    if (c == '<' or c == '>' or c == '!' or c == '='):
                        if ((endIndex < self.textLen) and (self.text[endIndex] == '=')):
                            endIndex += 1
                    elif ((c == '%') and (endIndex < self.textLen)):
                        c = self.text[endIndex]
                        if (c in ['d', 'i', 'o', 'u', 'x', 'X', 'e', 'E', 'f', 'F', 'g', 'G', 'c', 'r', 's', '%']):
                            endIndex += 1
##                            self.nextToken = self.TOKEN_PLACEHOLDER # Should really be this but no one can handle it yet
                    self.nextTokenVal = self.text[self.textIndex:endIndex]
                    self.textIndex = endIndex
                    return
            elif not self.ignoreWhitespace:
                self.nextToken=self.TOKEN_WS
                self.nextTokenVal=""
                while c in string.whitespace:
                    self.nextTokenVal+=c
                    self.textIndex+=1
                    if self.textIndex==len(self.text):
                        break
                    c=self.text[self.textIndex]
                return
            self.textIndex += 1
        self.nextToken = None
        self.nextTokenVal = None

def isXPathNonVar(var):
    """Returns true iff var is a string ("foo" or 'foo') or a number."""
    if (var.startswith("'") and var.endswith("'")) or \
            (var.startswith('"') and var.endswith('"')):
        return True

    # list from XPathToCode, below
    if var.lower() in ["count", "empty", "true", "false", "null", "and", "or", \
            "like", "not"]:
        return True

    try:
        t=int(var)
        return True
    except TypeError, e:
        pass
    except ValueError, e:
        pass

    return False

def xpathToCode(xpaths, convertBracket=True):
    if ((xpaths == None) or (len(xpaths) < 1)):
        return "True"
    if (not isinstance(xpaths, (list, tuple))):
        xpaths = [xpaths]
    result = []
    for xpath in xpaths:
        t = Tokenizer(xpath, "[a-zA-Z0-9_/:\.]", "[^a-zA-Z0-9_/:\.]", ignoreWhitespace=False)
        expr = []
        lastToken=None
        while t.nextToken != None:
            t.getNextToken()
            if (t.token == Tokenizer.TOKEN_WS):
                expr.append(" ")
            elif (t.token == Tokenizer.TOKEN_OP):
                if (t.tokenVal == "="):
                    expr.append("==")
                elif (t.tokenVal == "[" and convertBracket):
                    expr.append("(")
                elif (t.tokenVal == "]" and convertBracket):
                    expr.append(")")
                else:
                    expr.append(t.tokenVal)
            elif (t.token == Tokenizer.TOKEN_IDENT):
                if (t.tokenVal == "and"):
                    expr.append(" and ")
                elif (t.tokenVal == "or"):
                    expr.append(" or ")
                elif (t.tokenVal == "not"):
                    expr.append(" not ")
                elif (t.tokenVal == "like"):
                    # REVIEW [e-mail] 02-Nov-05 --
                    # This is very limited support for like:
                    # typically like queries look like this: "foo like 'blah%'".
                    # So translate this into "foo.startswith(blah)".
                    # We should use a regular expression to support '%'s in
                    # arbitrary places in the string. After 1.1.
                    if t.nextToken and t.nextTokenVal.endswith("%'"):
                        t.getNextToken() # throw away the "like" token
                        last = len(expr) - 1
                        expr[last] = "%s.startswith(%s')"\
                            % (expr[last], t.tokenVal[:-2])
                    else:
                        # old behavior
                        expr.append(t.tokenVal)
                    
                elif (t.tokenVal == "count"):
                    expr.append("len")
                elif (t.tokenVal == 'empty'):
                    expr.append('ctx.isEmptyPath')
                elif (t.tokenVal == 'true'):
                    expr.append(_parseConstantFunction(t, 'True'))
                elif (t.tokenVal == 'false'):
                    expr.append(_parseConstantFunction(t, 'False'))
                elif (t.tokenVal == 'null'):
                    expr.append(_parseConstantFunction(t, 'None'))
                elif (-1!=t.tokenVal.find(':')):
                    serviceDef, args=_parseServiceFunction(t)

                    # XXX handle serviceDef, args being None

                    for i in range(len(args)):
                        args[i]=xpathToCode(args[i], False)
                    jargs="[%s]" % (",".join(args))

                    # XXX should be processmodel.DATASERVICE_PROCESS_NAME, not "dataservice"
                    if serviceDef[0]=='dataservice':
                        expr.append("runtimesupport.invokeDataServiceWrapper(%s, %s, ctx, locals())" % \
                                (serviceDef, jargs))
                    else:
                        expr.append("runtimesupport.invokeServiceWrapper(%s, %s, ctx)" % \
                                (serviceDef, jargs))
                else:
                    if (lastToken==')' or lastToken==']'):
                        wasFunc=True
                    else:
                        wasFunc=False
                    if (t.tokenVal.startswith('/')) and not wasFunc:
                        expr.append(XPATH_ROOT_VAR)
                    expr.append(t.tokenVal.replace('/','.'))
                lastToken=t.tokenVal
            else: 
                expr.append(t.tokenVal)
                

        if (len(expr) == 2 and expr[0]==" "):
            expr = "".join(expr)
            result.append(expr)
        elif (len(expr) > 1):
            expr = "".join(expr)
            result.append("(%s)" % expr)
        elif (len(expr) > 0):
            result.append(expr[0])
        
    return " and ".join(result)

def _parseArgs(t):
    args=[]
    argcon=""

    if t.tokenVal!='(':
        return []
    if t.nextTokenVal==')':
        t.getNextToken()
        return []

    depth=1

    while(depth!=0):
        if not t.nextToken:
            raise Exception("parameters list with no closing ) after token: %s" % t.tokenVal)
        t.getNextToken()

        if t.tokenVal=='(':
            depth+=1
        if t.tokenVal==')':
            depth-=1

        if depth==0 or (depth==1 and t.tokenVal==','):
            args.append(argcon)
            argcon=""
        else:
            argcon+=t.tokenVal
    return args

def _parseServiceFunction(t):
    """Parses what appears to be a service function call into serviceDefs and args lists.

    Returns None, None if the serviceFunction appears to be invalid.
    """
    if t.nextTokenVal!='(':
        return t.tokenVal, None

    serviceDef=t.tokenVal.split(':')
    t.getNextToken()
    args=_parseArgs(t)

    return serviceDef, args
    
def _parseConstantFunction(t, outputValue):
    firstVal = t.tokenVal
    if t.nextTokenVal != '(':
        return firstVal
    t.getNextToken()
    if t.nextTokenVal != ')':
        return "%s%s" % (firstVal, '(')
    t.getNextToken()
    return outputValue

def parseDSPredicate(ctx, str, vars, valueList=None):
    from activegrid.util.utillang import evalCode
    from activegrid.util.utillang import ObjAsDict

    if valueList == None:
        valueList = []
    indexVar=0
    oldIndexVar=0
    sourceStr=str
    inlinedPredicate=[]
    qualifications=[]
    while True:
        oldIndexVar = indexVar
        dollarCurlForm = False
        quoted = False
        indexVar = sourceStr.find("bpws:getVariableData", indexVar)
        if indexVar == -1:
            indexVar = sourceStr.find("${", oldIndexVar)
            if indexVar == -1:
                break
            dollarCurlForm = True
        if indexVar > 0 and sourceStr[indexVar-1] in ('"',"'"):
            quoted = True
        if not dollarCurlForm:
            openParen = sourceStr.find("(", indexVar)
            if openParen == -1:
                break
            closeParen = sourceStr.find(")", openParen)
            if closeParen == -1:
                break
        else:
            openParen = indexVar+1
            closeParen = sourceStr.find("}", openParen)
            if closeParen == -1:
                break
        varRef = sourceStr[openParen+1: closeParen]
        if varRef.startswith('"') or varRef.startswith("'"):
            varRef = varRef[1:]
        if varRef.endswith('"') or varRef.endswith("'"):
            varRef = varRef[:-1]
        if isinstance(vars, dict) or isinstance(vars, ObjAsDict):
            varRefCode = xpathToCode(varRef)
            value = evalCode(varRefCode, vars)
        else:
            value = ctx.evalPath(vars, varRef)
        inlinedPredicate.append(sourceStr[oldIndexVar:indexVar])
        if quoted:
            inlinedPredicate.append("%s" % value)
        else:
            inlinedPredicate.append('%s')
            valueList.append(value)
        indexVar = closeParen+1
    inlinedPredicate.append(sourceStr[oldIndexVar:])
    qualifications.append(''.join(inlinedPredicate))
    return qualifications, valueList