teststreams.py

antlr最新版本V3源代码

        
        stream.consume()
        marker3 = stream.mark()

        stream.rewind(marker2)
        self.failUnlessEqual(stream.markDepth, 1)
        self.failUnlessEqual(stream.index(), 5)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.LA(1), u'盲')

        
class TestCommonTokenStream(unittest.TestCase):
    """Test case for the StringStream class."""

    def setUp(self):
        """Setup test fixure

        The constructor of CommonTokenStream needs a token source. This
        is a simple mock class providing just the nextToken() method.

        """

        class MockSource(object):
            def __init__(self):
                self.tokens = []

            def nextToken(self):
                try:
                    return self.tokens.pop(0)
                except IndexError:
                    return None
                
        self.source = MockSource()
        
        
    def testInit(self):
        """CommonTokenStream.__init__()"""

        stream = antlr3.CommonTokenStream(self.source)
        self.failUnlessEqual(stream.index(), -1)
        
        
    def testSetTokenSource(self):
        """CommonTokenStream.setTokenSource()"""

        stream = antlr3.CommonTokenStream(None)
        stream.setTokenSource(self.source)
        self.failUnlessEqual(stream.index(), -1)
        self.failUnlessEqual(stream.channel, antlr3.DEFAULT_CHANNEL)


    def testLTEmptySource(self):
        """CommonTokenStream.LT(): EOF (empty source)"""

        stream = antlr3.CommonTokenStream(self.source)

        lt1 = stream.LT(1)
        self.failUnlessEqual(lt1.type, antlr3.EOF)
        

    def testLT1(self):
        """CommonTokenStream.LT(1)"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )
        
        stream = antlr3.CommonTokenStream(self.source)

        lt1 = stream.LT(1)
        self.failUnlessEqual(lt1.type, 12)
        

    def testLT1WithHidden(self):
        """CommonTokenStream.LT(1): with hidden tokens"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )
        
        stream = antlr3.CommonTokenStream(self.source)

        lt1 = stream.LT(1)
        self.failUnlessEqual(lt1.type, 13)
        

    def testLT2BeyondEnd(self):
        """CommonTokenStream.LT(2): beyond end"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=13, channel=antlr3.HIDDEN_CHANNEL)
            )
        
        stream = antlr3.CommonTokenStream(self.source)

        lt1 = stream.LT(2)
        self.failUnlessEqual(lt1.type, antlr3.EOF)
        

    # not yet implemented
    def testLTNegative(self):
        """CommonTokenStream.LT(-1): look back"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )
        
        stream = antlr3.CommonTokenStream(self.source)
        stream.fillBuffer()
        stream.consume()
        
        lt1 = stream.LT(-1)
        self.failUnlessEqual(lt1.type, 12)


    def testLB1(self):
        """CommonTokenStream.LB(1)"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )
        
        stream = antlr3.CommonTokenStream(self.source)
        stream.fillBuffer()
        stream.consume()
        
        self.failUnlessEqual(stream.LB(1).type, 12)
        

    def testLTZero(self):
        """CommonTokenStream.LT(0)"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )
        
        stream = antlr3.CommonTokenStream(self.source)

        lt1 = stream.LT(0)
        self.failUnless(lt1 is None)
        

    def testLBBeyondBegin(self):
        """CommonTokenStream.LB(-1): beyond begin"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )
        
        stream = antlr3.CommonTokenStream(self.source)
        self.failUnless(stream.LB(1) is None)

        stream.consume()
        stream.consume()
        self.failUnless(stream.LB(3) is None)
        

    def testFillBuffer(self):
        """CommonTokenStream.fillBuffer()"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=14)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=antlr3.EOF)
            )
        
        stream = antlr3.CommonTokenStream(self.source)
        stream.fillBuffer()

        self.failUnlessEqual(len(stream.tokens), 3)
        self.failUnlessEqual(stream.tokens[0].type, 12)
        self.failUnlessEqual(stream.tokens[1].type, 13)
        self.failUnlessEqual(stream.tokens[2].type, 14)
        
        
    def testConsume(self):
        """CommonTokenStream.consume()"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=antlr3.EOF)
            )
        
        stream = antlr3.CommonTokenStream(self.source)
        self.failUnlessEqual(stream.LA(1), 12)

        stream.consume()
        self.failUnlessEqual(stream.LA(1), 13)

        stream.consume()
        self.failUnlessEqual(stream.LA(1), antlr3.EOF)

        stream.consume()
        self.failUnlessEqual(stream.LA(1), antlr3.EOF)
        
        
    def testSeek(self):
        """CommonTokenStream.seek()"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=antlr3.EOF)
            )
        
        stream = antlr3.CommonTokenStream(self.source)
        self.failUnlessEqual(stream.LA(1), 12)

        stream.seek(2)
        self.failUnlessEqual(stream.LA(1), antlr3.EOF)

        stream.seek(0)
        self.failUnlessEqual(stream.LA(1), 12)
        
        
    def testMarkRewind(self):
        """CommonTokenStream.mark()/rewind()"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=antlr3.EOF)
            )
        
        stream = antlr3.CommonTokenStream(self.source)
        stream.fillBuffer()
        
        stream.consume()
        marker = stream.mark()
        
        stream.consume()
        stream.rewind(marker)
        
        self.failUnlessEqual(stream.LA(1), 13)


    def testToString(self):
        """CommonTokenStream.toString()"""
        
        self.source.tokens.append(
            antlr3.CommonToken(type=12, text="foo")
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=13, text="bar")
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=14, text="gnurz")
            )
        
        self.source.tokens.append(
            antlr3.CommonToken(type=15, text="blarz")
            )
        
        stream = antlr3.CommonTokenStream(self.source)

        assert stream.toString() == "foobargnurzblarz"
        assert stream.toString(1, 2) == "bargnurz"
        assert stream.toString(stream.tokens[1], stream.tokens[-2]) == "bargnurz"
        

if __name__ == "__main__":
    unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))