teststreams.py

antlr最新版本V3源代码

# -*- coding: utf-8 -*-

import os
import unittest
from StringIO import StringIO
import antlr3


class TestStringStream(unittest.TestCase):
    """Test case for the StringStream class."""

    def testSize(self):
        """StringStream.size()"""

        stream = antlr3.StringStream('foo')

        self.failUnlessEqual(stream.size(), 3)

        
    def testIndex(self):
        """StringStream.index()"""

        stream = antlr3.StringStream('foo')

        self.failUnlessEqual(stream.index(), 0)
        
        
    def testConsume(self):
        """StringStream.consume()"""

        stream = antlr3.StringStream('foo\nbar')

        stream.consume() # f
        self.failUnlessEqual(stream.index(), 1)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.line, 1)

        stream.consume() # o
        self.failUnlessEqual(stream.index(), 2)
        self.failUnlessEqual(stream.charPositionInLine, 2)
        self.failUnlessEqual(stream.line, 1)

        stream.consume() # o
        self.failUnlessEqual(stream.index(), 3)
        self.failUnlessEqual(stream.charPositionInLine, 3)
        self.failUnlessEqual(stream.line, 1)

        stream.consume() # \n
        self.failUnlessEqual(stream.index(), 4)
        self.failUnlessEqual(stream.charPositionInLine, 0)
        self.failUnlessEqual(stream.line, 2)

        stream.consume() # b
        self.failUnlessEqual(stream.index(), 5)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.line, 2)

        stream.consume() # a
        self.failUnlessEqual(stream.index(), 6)
        self.failUnlessEqual(stream.charPositionInLine, 2)
        self.failUnlessEqual(stream.line, 2)

        stream.consume() # r
        self.failUnlessEqual(stream.index(), 7)
        self.failUnlessEqual(stream.charPositionInLine, 3)
        self.failUnlessEqual(stream.line, 2)

        stream.consume() # EOF
        self.failUnlessEqual(stream.index(), 7)
        self.failUnlessEqual(stream.charPositionInLine, 3)
        self.failUnlessEqual(stream.line, 2)

        stream.consume() # EOF
        self.failUnlessEqual(stream.index(), 7)
        self.failUnlessEqual(stream.charPositionInLine, 3)
        self.failUnlessEqual(stream.line, 2)
        
        
    def testReset(self):
        """StringStream.reset()"""

        stream = antlr3.StringStream('foo')

        stream.consume()
        stream.consume()

        stream.reset()
        self.failUnlessEqual(stream.index(), 0)
        self.failUnlessEqual(stream.line, 1)
        self.failUnlessEqual(stream.charPositionInLine, 0)
        self.failUnlessEqual(stream.LA(1), 'f')
        
        
    def testLA(self):
        """StringStream.LA()"""

        stream = antlr3.StringStream('foo')

        self.failUnlessEqual(stream.LA(1), 'f')
        self.failUnlessEqual(stream.LA(2), 'o')
        self.failUnlessEqual(stream.LA(3), 'o')
        
        stream.consume()
        stream.consume()

        self.failUnlessEqual(stream.LA(1), 'o')
        self.failUnlessEqual(stream.LA(2), antlr3.EOF)
        self.failUnlessEqual(stream.LA(3), antlr3.EOF)


    def testSubstring(self):
        """StringStream.substring()"""

        stream = antlr3.StringStream('foobar')
        
        self.failUnlessEqual(stream.substring(0, 0), 'f')
        self.failUnlessEqual(stream.substring(0, 1), 'fo')
        self.failUnlessEqual(stream.substring(0, 5), 'foobar')
        self.failUnlessEqual(stream.substring(3, 5), 'bar')
        
        
    def testSeekForward(self):
        """StringStream.seek(): forward"""

        stream = antlr3.StringStream('foo\nbar')

        stream.seek(4)
        
        self.failUnlessEqual(stream.index(), 4)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 0)
        self.failUnlessEqual(stream.LA(1), 'b')


##     # not yet implemented
##     def testSeekBackward(self):
##         """StringStream.seek(): backward"""

##         stream = antlr3.StringStream('foo\nbar')

##         stream.seek(4)
##         stream.seek(1)
        
##         self.failUnlessEqual(stream.index(), 1)
##         self.failUnlessEqual(stream.line, 1)
##         self.failUnlessEqual(stream.charPositionInLine, 1)
##         self.failUnlessEqual(stream.LA(1), 'o')


    def testMark(self):
        """StringStream.mark()"""

        stream = antlr3.StringStream('foo\nbar')

        stream.seek(4)

        marker = stream.mark()
        self.failUnlessEqual(marker, 1)
        self.failUnlessEqual(stream.markDepth, 1)

        stream.consume()
        marker = stream.mark()
        self.failUnlessEqual(marker, 2)
        self.failUnlessEqual(stream.markDepth, 2)
        

    def testReleaseLast(self):
        """StringStream.release(): last marker"""

        stream = antlr3.StringStream('foo\nbar')

        stream.seek(4)
        marker1 = stream.mark()
        
        stream.consume()
        marker2 = stream.mark()

        stream.release()
        self.failUnlessEqual(stream.markDepth, 1)

        # release same marker again, nothing has changed
        stream.release()
        self.failUnlessEqual(stream.markDepth, 1)
        

    def testReleaseNested(self):
        """StringStream.release(): nested"""

        stream = antlr3.StringStream('foo\nbar')

        stream.seek(4)
        marker1 = stream.mark()
        
        stream.consume()
        marker2 = stream.mark()
        
        stream.consume()
        marker3 = stream.mark()

        stream.release(marker2)
        self.failUnlessEqual(stream.markDepth, 1)
        

    def testRewindLast(self):
        """StringStream.rewind(): last marker"""

        stream = antlr3.StringStream('foo\nbar')

        stream.seek(4)

        marker = stream.mark()
        stream.consume()
        stream.consume()

        stream.rewind()
        self.failUnlessEqual(stream.markDepth, 0)
        self.failUnlessEqual(stream.index(), 4)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 0)
        self.failUnlessEqual(stream.LA(1), 'b')
        

    def testRewindNested(self):
        """StringStream.rewind(): nested"""

        stream = antlr3.StringStream('foo\nbar')

        stream.seek(4)
        marker1 = stream.mark()
        
        stream.consume()
        marker2 = stream.mark()
        
        stream.consume()
        marker3 = stream.mark()

        stream.rewind(marker2)
        self.failUnlessEqual(stream.markDepth, 1)
        self.failUnlessEqual(stream.index(), 5)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.LA(1), 'a')

        
class TestFileStream(unittest.TestCase):
    """Test case for the FileStream class."""


    def testNoEncoding(self):
        path = os.path.join(os.path.dirname(__file__), 'teststreams.input1')
        
        stream = antlr3.FileStream(path)

        stream.seek(4)
        marker1 = stream.mark()
        
        stream.consume()
        marker2 = stream.mark()
        
        stream.consume()
        marker3 = stream.mark()

        stream.rewind(marker2)
        self.failUnlessEqual(stream.markDepth, 1)
        self.failUnlessEqual(stream.index(), 5)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.LA(1), 'a')


    def testEncoded(self):
        path = os.path.join(os.path.dirname(__file__), 'teststreams.input2')
        
        stream = antlr3.FileStream(path, 'utf-8')

        stream.seek(4)
        marker1 = stream.mark()
        
        stream.consume()
        marker2 = stream.mark()
        
        stream.consume()
        marker3 = stream.mark()

        stream.rewind(marker2)
        self.failUnlessEqual(stream.markDepth, 1)
        self.failUnlessEqual(stream.index(), 5)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.LA(1), u'盲')

        

class TestInputStream(unittest.TestCase):
    """Test case for the InputStream class."""

    def testNoEncoding(self):
        file = StringIO('foo\nbar')
        
        stream = antlr3.InputStream(file)

        stream.seek(4)
        marker1 = stream.mark()
        
        stream.consume()
        marker2 = stream.mark()
        
        stream.consume()
        marker3 = stream.mark()

        stream.rewind(marker2)
        self.failUnlessEqual(stream.markDepth, 1)
        self.failUnlessEqual(stream.index(), 5)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.LA(1), 'a')


    def testEncoded(self):
        file = StringIO(u'foo\nb盲r'.encode('utf-8'))
        
        stream = antlr3.InputStream(file, 'utf-8')

        stream.seek(4)
        marker1 = stream.mark()
        
        stream.consume()
        marker2 = stream.mark()