antlr3.py

antlr最新版本V3源代码

        self.decisionNumber = decisionNumber
        self.eot = eot
        self.eof = eof
        self.min = min
        self.max = max
        self.accept = accept
        self.special = special
        self.transition = transition


    def predict(self, input):
        """
        From the input stream, predict what alternative will succeed
	using this DFA (representing the covering regular approximation
	to the underlying CFL).  Return an alternative number 1..n.  Throw
	 an exception upon error.
	"""
        mark = input.mark()
        s = 0 # we always start at s0
        try:
            for _ in xrange(50000):
                #print "***Current state = %d" % s
                
                specialState = self.special[s]
                if specialState >= 0:
                    #print "is special"
                    s = self.specialStateTransition(specialState, input)
                    input.consume()
                    continue

                if self.accept[s] >= 1:
                    #print "accept state for alt %d" % self.accept[s]
                    return self.accept[s]

                # look for a normal char transition
                LA = input.LA(1)
                #print LA, repr(input.LT(1)), input.LT(1).text
                if LA == EOF:
                    c = -1 #0xffff
                else:
                    try:
                        c = ord(LA)
                    except TypeError:
                        # LA is a token type (int), not a char
                        c = LA
                        
                #print "LA = %d (%r)" % (c, unichr(c))
                #print "range = %d..%d" % (self.min[s], self.max[s])

                if c >= self.min[s] and c <= self.max[s]:
                    # move to next state
                    snext = self.transition[s][c-self.min[s]]
                    #print "in range, next state = %d" % snext
                    
                    if snext < 0:
                        #print "not a normal transition"
                        # was in range but not a normal transition
                        # must check EOT, which is like the else clause.
                        # eot[s]>=0 indicates that an EOT edge goes to another
                        # state.
                        if self.eot[s] >= 0: # EOT Transition to accept state?
                            #print "EOT trans to accept state %d" % self.eot[s]
                            
                            s = self.eot[s]
                            input.consume()
                            # TODO: I had this as return accept[eot[s]]
                            # which assumed here that the EOT edge always
                            # went to an accept...faster to do this, but
                            # what about predicated edges coming from EOT
                            # target?
                            continue

                        #print "no viable alt"
                        self.noViableAlt(s, input)
                        return 0

                    s = snext
                    input.consume()
                    continue

                if self.eot[s] >= 0:
                    #print "EOT to %d" % self.eot[s]
                    
                    s = self.eot[s]
                    input.consume()
                    continue

                # EOF Transition to accept state?
                if c == EOF and self.eof[s] >= 0:
                    #print "EOF Transition to accept state %d" \
                    #  % self.accept[self.eof[s]]
                    return self.accept[self.eof[s]]

                # not in range and not EOF/EOT, must be invalid symbol
                self.noViableAlt(s, input)
                return 0

            else:
                raise RuntimeError("DFA bang!")
            
        finally:
            input.rewind(mark)


    def noViableAlt(self, s, input):
        if self.recognizer.backtracking > 0:
            self.recognizer.failed = True
            return

        nvae = NoViableAltException(
            self.getDescription(),
            self.decisionNumber,
            s,
            input
            )

        self.error(nvae)
        raise nvae


    def error(self, nvae):
        """A hook for debugging interface"""
        pass


    def specialStateTransition(self, s, input):
        return -1


    def getDescription(self):
        return "n/a"


    def specialTransition(self, state, symbol):
        return 0


    def unpack(cls, string):
        ret = []
        for i in range(len(string) / 2):
            (n, v) = ord(string[i*2]), ord(string[i*2+1])

            # Is there a bitwise operation to do this?
            if v == 0xFFFF:
                v = -1

            ret += [v] * n

        return ret
    
    unpack = classmethod(unpack)
"""ANTLR3 runtime package"""

# [The "BSD licence"]
# Copyright (c) 2005-2006 Terence Parr
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. The name of the author may not be used to endorse or promote products
#    derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from antlr3.constants import EOF, DEFAULT_CHANNEL, INVALID_TOKEN_TYPE

############################################################################
#
# basic token interface
#
############################################################################

class Token(object):
    ## Get the text of the token

    def getText(self):
        """Using setter/getter methods is deprecated. Use o.text instead."""

        raise NotImplementedError
    
    def setText(self, text):
        """Using setter/getter methods is deprecated. Use o.text instead."""

        raise NotImplementedError
    

    ## Get the type of the token

    def getType(self):
        """Using setter/getter methods is deprecated. Use o.type instead."""

        raise NotImplementedError
    
    def setType(self, ttype):
        """Using setter/getter methods is deprecated. Use o.type instead."""

        raise NotImplementedError
    
    
    ## The line number on which this token was matched; line=1..n

    def getLine(self):
        """Using setter/getter methods is deprecated. Use o.line instead."""

        raise NotImplementedError
    
    def setLine(self, line):
        """Using setter/getter methods is deprecated. Use o.line instead."""

        raise NotImplementedError
    
    
    ## The index of the first character relative to the beginning of the
    ## line 0..n-1 */

    def getCharPositionInLine(self):
        """Using setter/getter methods is deprecated. Use o.charPositionInLine instead."""

        raise NotImplementedError
    
    def setCharPositionInLine(self, pos):
        """Using setter/getter methods is deprecated. Use o.charPositionInLine instead."""

        raise NotImplementedError
    

    ## Get the channel of the token

    def getChannel(self):
        """Using setter/getter methods is deprecated. Use o.channel instead."""

        raise NotImplementedError
    
    def setChannel(self, channel):
        """Using setter/getter methods is deprecated. Use o.channel instead."""

        raise NotImplementedError
    

    ## An index from 0..n-1 of the token object in the input stream.
    ## This must be valid in order to use the ANTLRWorks debugger.

    def getTokenIndex(self):
        """Using setter/getter methods is deprecated. Use o.index instead."""

        raise NotImplementedError
    
    def setTokenIndex(self, index):
        """Using setter/getter methods is deprecated. Use o.index instead."""

        raise NotImplementedError


############################################################################
#
# token implementations
#
# Token
# +- CommonToken
# \- ClassicToken
#
############################################################################

class CommonToken(Token):
    def __init__(self, type=None, channel=DEFAULT_CHANNEL, text=None,
                 input=None, start=None, stop=None, oldToken=None):
        Token.__init__(self)
        
        if oldToken is not None:
            self.type = oldToken.type
            self.line = oldToken.line
            self.charPositionInLine = oldToken.charPositionInLine
            self.channel = oldToken.channel
            self.index = oldToken.index
            self._text = oldToken._text
            self.input = oldToken.input
            self.start = oldToken.start
            self.stop = oldToken.stop
            
        else:
            self.type = type
            self.input = input
            self.charPositionInLine = -1 # set to invalid position
            self.line = 0
            self.channel = channel
            
	    #What token number is this from 0..n-1 tokens; < 0 implies invalid index
            self.index = -1
            
            # We need to be able to change the text once in a while.  If
            # this is non-null, then getText should return this.  Note that
            # start/stop are not affected by changing this.
            self._text = text

            # The char position into the input buffer where this token starts
            self.start = start

            # The char position into the input buffer where this token stops
            # This is the index of the last char, *not* the index after it!
            self.stop = stop


    def getText(self):
        if self._text is not None:
            return self._text

        if self.input is None:
            return None
        
        return self.input.substring(self.start, self.stop)


    def setText(self, text):
        """
        Override the text for this token.  getText() will return this text
        rather than pulling from the buffer.  Note that this does not mean
        that start/stop indexes are not valid.  It means that that input
        was converted to a new string in the token object.
	"""
        self._text = text

    text = property(getText, setText)


    def getType(self):
        return self.type 

    def setType(self, ttype):
        self.type = ttype

    
    def getLine(self):
        return self.line
    
    def setLine(self, line):
        self.line = line


    def getCharPositionInLine(self):
        return self.charPositionInLine
    
    def setCharPositionInLine(self, pos):
        self.charPositionInLine = pos


    def getChannel(self):
        return self.channel
    
    def setChannel(self, channel):
        self.channel = channel
    

    def getTokenIndex(self):
        return self.index
    
    def setTokenIndex(self, index):
        self.index = index


    def __str__(self):
        channelStr = ""
        if self.channel > 0:
            channelStr = ",channel=" + str(self.channel)

        txt = self.text
        if txt is not None:
            txt = txt.replace("\n","\\\\n")
            txt = txt.replace("\r","\\\\r")
            txt = txt.replace("\t","\\\\t")
        else:
            txt = "<no text>"

        return "[@%s,%s:%s=%r,<%s>%s,%s:%s]" % (
            self.index,
            self.start, self.stop,
            txt,
            self.type, channelStr,
            self.line, self.charPositionInLine
            )
    

class ClassicToken(Token):
    """
    A Token object like we'd use in ANTLR 2.x; has an actual string created
    and associated with this object.  These objects are needed for imaginary
    tree nodes that have payload objects.  We need to create a Token object
    that has a string; the tree node will point at this token.  CommonToken
    has indexes into a char stream and hence cannot be used to introduce
    new strings.
    """

    def __init__(self, type=None, text=None, channel=DEFAULT_CHANNEL,
                 oldToken=None
                 ):
        Token.__init__(self)
        
        if oldToken is not None:
            self.text = oldToken.text