antlr3.py

antlr最新版本V3源代码

        
        self.p = 0
        self.line = 1
        self.charPositionInLine = 0
        self._markers = [ ]


    def consume(self):
        if self.p < self.n:
            self.charPositionInLine += 1
            if self.data[self.p] == '\n':
                self.line += 1
                self.charPositionInLine = 0

            self.p += 1


    def LA(self, i):
        if i == 0:
            return 0 # undefined

        if i < 0:
            i += 1 # e.g., translate LA(-1) to use offset i=0; then data[p+0-1]
            if self.p+i-1 < 0:
                return EOF # invalid; no char before first char

        if self.p+i-1 >= self.n:
            return EOF

        return self.data[self.p+i-1]

    LT = LA

    def index(self):
        """
        Return the current input symbol index 0..n where n indicates the
        last symbol has been read.  The index is the index of char to
        be returned from LA(1).
        """
        
        return self.p


    def size(self):
        return self.n


    def markDepth(self):
        return len(self._markers)
    markDepth = property(markDepth)


    def mark(self):
        state = (self.p, self.line, self.charPositionInLine)
        self._markers.append(state)

        return self.markDepth


    def rewind(self, marker=None):
        if marker is None:
            marker = self.markDepth
            
        p, line, charPositionInLine = self._markers[marker-1]

        self.seek(p)
        self.line = line
        self.charPositionInLine = charPositionInLine
        self.release(marker)


    def release(self, marker=None):
        if marker is None:
            marker = self.markDepth
            
        self._markers = self._markers[:marker-1]


    def seek(self, index):
        """
        consume() ahead until p==index; can't just set p=index as we must
        update line and charPositionInLine.
        """
        
        if index <= self.p:
            self.p = index # just jump; don't update stream state (line, ...)
            return

        # seek forward, consume until p hits index
        while self.p < index:
            self.consume()


    def substring(self, start, stop):
        return self.data[start:stop+1]


    def getLine(self):
        """Using setter/getter methods is deprecated. Use o.line instead."""
        return self.line


    def getCharPositionInLine(self):
        """Using setter/getter methods is deprecated. Use o.charPositionInLine instead."""
        return self.charPositionInLine


    def setLine(self, line):
        """Using setter/getter methods is deprecated. Use o.line instead."""
        self.line = line


    def setCharPositionInLine(self, pos):
        """Using setter/getter methods is deprecated. Use o.charPositionInLine instead."""
        self.charPositionInLine = pos


class ANTLRFileStream(ANTLRStringStream):
    """
    This is a char buffer stream that is loaded from a file
    all at once when you construct the object.
    """

    def __init__(self, fileName, encoding=None):
        self.fileName = fileName

        fp = codecs.open(fileName, 'r', encoding)
        try:
            data = fp.read()
        finally:
            fp.close()
            
        ANTLRStringStream.__init__(self, data)


    def getSourceName(self):
        return self.fileName


class ANTLRInputStream(ANTLRStringStream):
    """
    This is a char buffer stream that is loaded from a file like object
    all at once when you construct the object.
    
    All input is consumed from the file, but it is not closed.
    """

    def __init__(self, file, encoding=None):
        if encoding is not None:
            # wrap input in a decoding reader
            reader = codecs.lookup(encoding)[2]
            file = reader(file)

        data = file.read()
            
        ANTLRStringStream.__init__(self, data)


# I guess the ANTLR prefix exists only to avoid a name clash with some Java
# mumbojumbo. A plain "StringStream" looks better to me, which should be
# the preferred name in Python.
StringStream = ANTLRStringStream
FileStream = ANTLRFileStream
InputStream = ANTLRInputStream


############################################################################
#
# Token streams
#   TokenStream
#   +- CommonTokenStream
#   \- TokenRewriteStream
#
############################################################################


class CommonTokenStream(TokenStream):
    """
    The most common stream of tokens is one where every token is buffered up
    and tokens are prefiltered for a certain channel (the parser will only
    see these tokens and cannot change the filter channel number during the
    parse).

    TODO: how to access the full token stream?  How to track all tokens matched per rule?
    """

    def __init__(self, tokenSource=None, channel=DEFAULT_CHANNEL):
        TokenStream.__init__(self)
        
        self.tokenSource = tokenSource

	# Record every single token pulled from the source so we can reproduce
        # chunks of it later.
        self.tokens = []

	# Map<tokentype, channel> to override some Tokens' channel numbers
        self.channelOverrideMap = {}

	# Set<tokentype>; discard any tokens with this type
        self.discardSet = set()

	# Skip tokens on any channel but this one; this is how we skip whitespace...
        self.channel = channel

	# By default, track all incoming tokens
        self.discardOffChannelTokens = False

	# The index into the tokens list of the current token (next token
        # to consume).  p==-1 indicates that the tokens list is empty
        self.p = -1

        # Remember last marked position
        self.lastMarker = None
        

    def setTokenSource(self, tokenSource):
        """Reset this token stream by setting its token source."""
        
        self.tokenSource = tokenSource
        self.tokens = []
        self.p = -1
        self.channel = DEFAULT_CHANNEL


    def fillBuffer(self):
        """
        Load all tokens from the token source and put in tokens.
	This is done upon first LT request because you might want to
        set some token type / channel overrides before filling buffer.
        """
        

        index = 0
        t = self.tokenSource.nextToken()
        while t is not None and t.type != EOF:
            discard = False
            
            if self.discardSet is not None and t.type in self.discardSet:
                discard = True

            elif self.discardOffChannelTokens and t.channel != self.channel:
                discard = True

            # is there a channel override for token type?
            try:
                overrideChannel = self.channelOverrideMap[t.type]
                
            except KeyError:
                # no override for this type
                pass
            
            else:
                if overrideChannel == self.channel:
                    t.channel = overrideChannel
                else:
                    discard = True
            
            if not discard:
                t.index = index
                self.tokens.append(t)
                index += 1

            t = self.tokenSource.nextToken()
       
        # leave p pointing at first token on channel
        self.p = 0
        self.p = self.skipOffTokenChannels(self.p)


    def consume(self):
        """
        Move the input pointer to the next incoming token.  The stream
        must become active with LT(1) available.  consume() simply
        moves the input pointer so that LT(1) points at the next
        input symbol. Consume at least one token.

        Walk past any token not on the channel the parser is listening to.
        """
        
        if self.p < len(self.tokens):
            self.p += 1

            self.p = self.skipOffTokenChannels(self.p) # leave p on valid token


    def skipOffTokenChannels(self, i):
        """
        Given a starting index, return the index of the first on-channel
        token.
        """

        n = len(self.tokens)
        while i < n and self.tokens[i].channel != self.channel:
            i += 1

        return i


    def skipOffTokenChannelsReverse(self, i):
        while i >= 0 and self.tokens[i].channel != self.channel:
            i -= 1

        return i


    def setTokenTypeChannel(self, ttype, channel):
        """
        A simple filter mechanism whereby you can tell this token stream
        to force all tokens of type ttype to be on channel.  For example,
        when interpreting, we cannot exec actions so we need to tell
        the stream to force all WS and NEWLINE to be a different, ignored
        channel.
	"""
        
        self.channelOverrideMap[ttype] = channel


    def discardTokenType(self, ttype):
        self.discardSet.add(ttype)


    def getTokens(self, start=None, stop=None, types=None):
        """
        Given a start and stop index, return a list of all tokens in
        the token type set.  Return None if no tokens were found.  This
        method looks at both on and off channel tokens.
        """

        if self.p == -1:
            self.fillBuffer()

        if stop is None or stop >= len(self.tokens):
            stop = len(self.tokens) - 1
            
        if start is None or stop < 0:
            start = 0

        if start > stop:
            return None

        if isinstance(types, (int, long)):
            # called with a single type, wrap into set
            types = set([types])
            
        filteredTokens = [
            token for token in self.tokens[start:stop]
            if types is None or token.type in types
            ]

        if len(filteredTokens) == 0:
            return None

        return filteredTokens


    def LT(self, k):
        """
        Get the ith token from the current position 1..n where k=1 is the
        first symbol of lookahead.
        """

        if self.p == -1:
            self.fillBuffer()

        if k == 0:
            return None

        if k < 0:
            return self.LB(-k)
                
        if self.p + k - 1 >= len(self.tokens):
            return EOF_TOKEN

        i = self.p
        n = 1
        # find k good tokens
        while n < k:
            # skip off-channel tokens
            i = self.skipOffTokenChannels(i+1) # leave p on valid token
            n += 1
        
        if i >= len(self.tokens):
            return EOF_TOKEN

        return self.tokens[i]


    def LB(self, k):
        """Look backwards k tokens on-channel tokens"""

        if self.p == -1:
            self.fillBuffer()

        if k == 0:
            return None

        if self.p - k < 0:
            return None

        i = self.p
        n = 1
        # find k good tokens looking backwards
        while n <= k:
            # skip off-channel tokens
            i = self.skipOffTokenChannelsReverse(i-1) # leave p on valid token
            n += 1

        if i < 0:
            return None
            
        return self.tokens[i]