tree.py

antlr最新版本V3源代码

    def hasUniqueNavigationNodes(self):
        return self.uniqueNavigationNodes


    def setUniqueNavigationNodes(self, uniqueNavigationNodes):
        self.uniqueNavigationNodes = uniqueNavigationNodes


    def consume(self):
        if self.p == -1:
            self.fillBuffer()
            
        self.p += 1

        
    def LA(self, i):
        return self.adaptor.getType(self.LT(i))


    def mark(self):
        if self.p == -1:
            self.fillBuffer()

        
        self.lastMarker = self.index()
        return self.lastMarker


    def release(self, marker=None):
        # no resources to release

        pass


    def index(self):
        return self.p


    def rewind(self, marker=None):
        if marker is None:
            marker = self.lastMarker
            
        self.seek(marker)


    def seek(self, index):
        if self.p == -1:
            self.fillBuffer()

        self.p = index


    def push(self, index):
        """
        Make stream jump to a new location, saving old location.
        Switch back with pop().  I manage dyanmic array manually
        to avoid creating Integer objects all over the place.
        """

        self.calls.append(self.p) # save current index
        self.seek(index)


    def pop(self):
        """
        Seek back to previous index saved during last push() call.
        Return top of stack (return index).
        """

        ret = self.calls.pop(-1)
        self.seek(ret)
        return ret


    def size(self):
        if self.p == -1:
            self.fillBuffer()

        return len(self.nodes)


    def __str__(self):
        """Used for testing, just return the token type stream"""

        if self.p == -1:
            self.fillBuffer()

        return ' '.join([str(self.adaptor.getType(node))
                         for node in self.nodes
                         ])


    def toString(self, start, stop):
        if start is None or stop is None:
            return None

        if self.p == -1:
            self.fillBuffer()

        #System.out.println("stop: "+stop);
        #if ( start instanceof CommonTree )
        #    System.out.print("toString: "+((CommonTree)start).getToken()+", ");
        #else
        #    System.out.println(start);
        #if ( stop instanceof CommonTree )
        #    System.out.println(((CommonTree)stop).getToken());
        #else
        #    System.out.println(stop);
            
        # if we have the token stream, use that to dump text in order
        if self.tokens is not None:
            beginTokenIndex = self.adaptor.getTokenStartIndex(start)
            endTokenIndex = self.adaptor.getTokenStopIndex(stop)
            
            # if it's a tree, use start/stop index from start node
            # else use token range from start/stop nodes
            if self.adaptor.getType(stop) == UP:
                endTokenIndex = self.adaptor.getTokenStopIndex(start)

            elif self.adaptor.getType(stop) == EOF:
                endTokenIndex = self.size() -2 # don't use EOF

            return self.tokens.toString(beginTokenIndex, endTokenIndex)

        # walk nodes looking for start
        t = None
        for i, t in enumerate(self.nodes):
            if t == start:
                break

        # now walk until we see stop, filling string buffer with text
        buf = []
        t = self.nodes[i]
        while t != stop:
            text = self.adaptor.getText(t)
            if text is None:
                text = " " + self.adaptor.getType(t)

            buf.append(text)
            i += 1
            t = self.nodes[i]

        # include stop node too
        text = self.adaptor.getText(stop)
        if text is None:
            text = " " +self.adaptor.getType(stop)

        buf.append(text)
        
        return ''.join(buf)
    

    ## iterator interface
    def __iter__(self):
        if self.p == -1:
            self.fillBuffer()

        for node in self.nodes:
            yield node


#############################################################################
#
# tree parser
#
#############################################################################

class TreeParser(BaseRecognizer):
    """@brief Baseclass for generated tree parsers.
    
    A parser for a stream of tree nodes.  "tree grammars" result in a subclass
    of this.  All the error reporting and recovery is shared with Parser via
    the BaseRecognizer superclass.
    """

    def __init__(self, input):
        BaseRecognizer.__init__(self)

        self.input = None
        self.setTreeNodeStream(input)


    def reset(self):
        BaseRecognizer.reset(self) # reset all recognizer state variables
        if self.input is not None:
            self.input.seek(0) # rewind the input


    def setTreeNodeStream(self, input):
        """Set the input stream"""

        self.input = input


    def getTreeNodeStream(self):
        return self.input

    
    def matchAny(self, ignore): # ignore stream, copy of this.input
        """
        Match '.' in tree parser has special meaning.  Skip node or
        entire tree if node has children.  If children, scan until
        corresponding UP node.
        """
        
        self.errorRecovery = False
        self.failed = False

        look = self.input.LT(1)
        if self.input.getTreeAdaptor().getChildCount(look) == 0:
            self.input.consume() # not subtree, consume 1 node and return
            return

        # current node is a subtree, skip to corresponding UP.
        # must count nesting level to get right UP
        level = 0
        tokenType = self.input.getTreeAdaptor().getType(look)
        while tokenType != EOF and not (tokenType == UP and level==0):
            self.input.consume()
            look = self.input.LT(1)
            tokenType = self.input.getTreeAdaptor().getType(look)
            if tokenType == DOWN:
                level += 1

            elif tokenType == UP:
                level -= 1

        self.input.consume() # consume UP


    def mismatch(self, input, ttype, follow):
        """
        We have DOWN/UP nodes in the stream that have no line info; override.
        plus we want to alter the exception type.
        """

        mte = MismatchedTreeNodeException(ttype, input)
        self.recoverFromMismatchedToken(input, mte, ttype, follow)


    def getErrorHeader(self, e):
        """
        Prefix error message with the grammar name because message is
        always intended for the programmer because the parser built
        the input tree not the user.
        """

        return self.getGrammarFileName() \
               + ": node from line " \
               + e.line + ":" + e.charPositionInLine


    def getErrorMessage(self, e, tokenNames):
        """
        Tree parsers parse nodes they usually have a token object as
        payload. Set the exception token and do the default behavior.
        """

        if isinstance(self, TreeParser):
            adaptor = e.input.getTreeAdaptor()
            e.token = adaptor.getToken(e.node)
            if e.token is not None: # could be an UP/DOWN node
                e.token = CommonToken(
                    type=adaptor.getType(e.node),
                    text=adaptor.getText(e.node)
                    )

        return BaseRecognizer.getErrorMessage(e, tokenNames)


    def traceIn(self, ruleName, ruleIndex):
        BaseRecognizer.traceIn(self, ruleName, ruleIndex, self.input.LT(1))


    def traceOut(self, ruleName, ruleIndex):
        BaseRecognizer.traceOut(self, ruleName, ruleIndex, self.input.LT(1))


#############################################################################
#
# streams for rule rewriting
#
#############################################################################

class RewriteRuleElementStream(object):
    """@brief Internal helper class.
    
    A generic list of elements tracked in an alternative to be used in
    a -> rewrite rule.  We need to subclass to fill in the next() method,
    which returns either an AST node wrapped around a token payload or
    an existing subtree.

    Once you start next()ing, do not try to add more elements.  It will
    break the cursor tracking I believe.

    @see org.antlr.runtime.tree.RewriteRuleSubtreeStream
    @see org.antlr.runtime.tree.RewriteRuleTokenStream
    
    TODO: add mechanism to detect/puke on modification after reading from
    stream
    """

    def __init__(self, adaptor, elementDescription, elements = None):
        # Cursor 0..n-1.  If singleElement!=null, cursor is 0 until you next(),
        # which bumps it to 1 meaning no more elements.
        self.cursor = 0

        # Track single elements w/o creating a list.  Upon 2nd add, alloc list
        self.singleElement = None

        # The list of tokens or subtrees we are tracking
        self.elements = None

        # The element or stream description; usually has name of the token or
        # rule reference that this list tracks.  Can include rulename too, but
        # the exception would track that info.
        self.elementDescription = elementDescription

        self.adaptor = adaptor

        if isinstance(elements, (list, tuple)):
            # Create a stream, but feed off an existing list
            self.singleElement = None
            self.elements = elements

        else:
            # Create a stream with one element
            self.add(elements)


    def reset(self):
        """
        Reset the condition of this stream so that it appears we have
        not consumed any of its elements.  Elements themselves are untouched.
        """
        
        self.cursor = 0

    def add(self, el):
        if el is None:
            return

        if self.elements is not None: # if in list, just add
            self.elements.append(el)
            return

        if self.singleElement is None: # no elements yet, track w/o list
            self.singleElement = el
            return

        # adding 2nd element, move to list
        self.elements = []
        self.elements.append(self.singleElement)
        self.singleElement = None
        self.elements.append(el)


    def next(self):
        """
        Return the next element in the stream.  If out of elements, throw
        an exception unless size()==1.  If size is 1, then return elements[0].
        
        Return a duplicate node/subtree if stream is out of elements and
        size==1.
        """
        
        if self.cursor >= len(self) and len(self) == 1:
            # if out of elements and size is 1, dup
            el = self._next()
            return self.dup(el)

        # test size above then fetch
        el = self._next()
        return el


    def _next(self):
        """
        do the work of getting the next element, making sure that it's
        a tree node or subtree.  Deal with the optimization of single-
        element list versus list of size > 1.  Throw an exception
        if the stream is empty or we're out of elements and size>1.
        protected so you can override in a subclass if necessary.
        """

        if len(self) == 0:
            raise RewriteEmptyStreamException(self.elementDescription)
            
        if self.cursor >= len(self): # out of elements?
            if len(self) == 1: # if size is 1, it's ok; return and we'll dup 
                return self.singleElement

            # out of elements and size was not 1, so we can't dup
            raise RewriteCardinalityException(self.elementDescription)

        # we have elements
        if self.singleElement is not None:
            self.cursor += 1 # move cursor even for single element list
            return self.toTree(self.singleElement)

        # must have more than one in list, pull from elements
        o = self.toTree(self.elements[self.cursor])
        self.cursor += 1
        return o


    def dup(self, el):
        """
        When constructing trees, sometimes we need to dup a token or AST
        subtree.  Dup'ing a token means just creating another AST node
        around it.  For trees, you must call the adaptor.dupTree().
        """

        raise NotImplementedError
    

    def toTree(self, el):
        """
        Ensure stream emits trees; tokens must be converted to AST nodes.
        AST nodes can be passed through unmolested.
        """

        return el


    def hasNext(self):
        return ( (self.singleElement is not None and self.cursor < 1)
                 or (self.elements is not None and self.cursor < len(self.elements))
                 )

                 
    def size(self):
        if self.singleElement is not None:
            return 1

        if self.elements is not None:
            return len(self.elements)

        return 0

    __len__ = size
    

    def getDescription(self):
        """Deprecated. Directly access elementDescription attribute"""
        
        return self.elementDescription


class RewriteRuleTokenStream(RewriteRuleElementStream):
    """@brief Internal helper class."""
    
    def toTree(self, el):
        return self.adaptor.createWithPayload(el)


    def dup(self, el):
        return self.adaptor.createWithPayload(el)


class RewriteRuleSubtreeStream(RewriteRuleElementStream):
    """@brief Internal helper class."""

    def nextNode(self):
        """
        Treat next element as a single node even if it's a subtree.
        This is used instead of next() when the result has to be a
        tree root node.  Also prevents us from duplicating recently-added
        children; e.g., ^(type ID)+ adds ID to type and then 2nd iteration
        must dup the type node, but ID has been added.

        Referencing a rule result twice is ok; dup entire tree as
        we can't be adding trees; e.g., expr expr. 
        """
        
        el = self._next()

        if self.cursor >= len(self) and len(self) == 1:
            # if out of elements and size is 1, dup just the node
            el = self.adaptor.dupNode(el)

        return el


    def dup(self, el):
        return self.adaptor.dupTree(el)