parsetree.py

CNC 的开放码,EMC2 V2.2.8版

        "Return a list of sub-nodes"
        return []
    
    def __repr__(self):
        return str(self)
    
    def update(self, gen):
        if self.accepts_epsilon:
            gen.add_to(self.first, self.follow)

    def output(self, gen, indent):
        "Write out code to _gen_ with _indent_:string indentation"
        gen.write(indent, "assert 0 # Invalid parser node\n")
    
class Terminal(Node):
    """This class stores terminal nodes, which are tokens."""
    def __init__(self, rule, token):
        Node.__init__(self, rule)
        self.token = token
        self.accepts_epsilon = 0

    def __str__(self):
        return self.token

    def update(self, gen):
        Node.update(self, gen)
        if self.first != [self.token]:
            self.first = [self.token]
            gen.changed()

    def output(self, gen, indent):
        gen.write(indent)
        if re.match('[a-zA-Z_][a-zA-Z_0-9]*$', self.token):
            gen.write(self.token, " = ")
        gen.write("self._scan(%s, context=_context)\n" % repr(self.token))
        
class Eval(Node):
    """This class stores evaluation nodes, from {{ ... }} clauses."""
    def __init__(self, rule, expr):
        Node.__init__(self, rule)
        self.expr = expr

    def setup(self, gen):
        Node.setup(self, gen)
        if not self.accepts_epsilon:
            self.accepts_epsilon = 1
            gen.changed()

    def __str__(self):
        return '{{ %s }}' % self.expr.strip()

    def output(self, gen, indent):
        gen.write(indent, self.expr.strip(), '\n')
        
class NonTerminal(Node):
    """This class stores nonterminal nodes, which are rules with arguments."""
    def __init__(self, rule, name, args):
        Node.__init__(self, rule)
        self.name = name
        self.args = args

    def setup(self, gen):
        Node.setup(self, gen)
        try:
            self.target = gen.rules[self.name]
            if self.accepts_epsilon != self.target.accepts_epsilon:
                self.accepts_epsilon = self.target.accepts_epsilon
                gen.changed()
        except KeyError: # Oops, it's nonexistent
            print >>sys.stderr, 'Error: no rule <%s>' % self.name
            self.target = self
            
    def __str__(self):
        return '%s' % self.name

    def update(self, gen):
        Node.update(self, gen)
        gen.equate(self.first, self.target.first)
        gen.equate(self.follow, self.target.follow)

    def output(self, gen, indent):
        gen.write(indent)
        gen.write(self.name, " = ")
        args = self.args
        if args: args += ', '
        args += '_context'
        gen.write("self.", self.name, "(", args, ")\n")
        
class Sequence(Node):
    """This class stores a sequence of nodes (A B C ...)"""
    def __init__(self, rule, *children):
        Node.__init__(self, rule)
        self.children = children

    def setup(self, gen):
        Node.setup(self, gen)
        for c in self.children: c.setup(gen)
        
        if not self.accepts_epsilon:
            # If it's not already accepting epsilon, it might now do so.
            for c in self.children:
                # any non-epsilon means all is non-epsilon
                if not c.accepts_epsilon: break
            else:
                self.accepts_epsilon = 1
                gen.changed()

    def get_children(self):
        return self.children
    
    def __str__(self):
        return '( %s )' % ' '.join(map(str, self.children))

    def update(self, gen):
        Node.update(self, gen)
        for g in self.children:
            g.update(gen)

        empty = 1
        for g_i in range(len(self.children)):
            g = self.children[g_i]

            if empty: gen.add_to(self.first, g.first)
            if not g.accepts_epsilon: empty = 0
            
            if g_i == len(self.children)-1:
                next = self.follow
            else:
                next = self.children[1+g_i].first
            gen.add_to(g.follow, next)

        if self.children:
            gen.add_to(self.follow, self.children[-1].follow)

    def output(self, gen, indent):
        if self.children:
            for c in self.children:
                c.output(gen, indent)
        else:
            # Placeholder for empty sequences, just in case
            gen.write(indent, 'pass\n')
            
class Choice(Node):
    """This class stores a choice between nodes (A | B | C | ...)"""
    def __init__(self, rule, *children):
        Node.__init__(self, rule)
        self.children = children

    def setup(self, gen):
        Node.setup(self, gen)
        for c in self.children: c.setup(gen)
            
        if not self.accepts_epsilon:
            for c in self.children:
                if c.accepts_epsilon:
                    self.accepts_epsilon = 1
                    gen.changed()

    def get_children(self):
        return self.children
    
    def __str__(self):
        return '( %s )' % ' | '.join(map(str, self.children))

    def update(self, gen):
        Node.update(self, gen)
        for g in self.children:
            g.update(gen)

        for g in self.children:
            gen.add_to(self.first, g.first)
            gen.add_to(self.follow, g.follow)
        for g in self.children:
            gen.add_to(g.follow, self.follow)
        if self.accepts_epsilon:
            gen.add_to(self.first, self.follow)

    def output(self, gen, indent):
        test = "if"
        gen.write(indent, "_token = ", gen.peek_call(self.first), "\n")
        tokens_seen = []
        tokens_unseen = self.first[:]
        if gen.has_option('context-insensitive-scanner'):
            # Context insensitive scanners can return ANY token,
            # not only the ones in first.
            tokens_unseen = gen.non_ignored_tokens()
        for c in self.children:
            testset = c.first[:]
            removed = []
            for x in testset:
                if x in tokens_seen:
                    testset.remove(x)
                    removed.append(x)
                if x in tokens_unseen: tokens_unseen.remove(x)
            tokens_seen = tokens_seen + testset
            if removed:
                if not testset:
                    print >>sys.stderr, 'Error in rule', self.rule+':'
                else:
                    print >>sys.stderr, 'Warning in rule', self.rule+':'
                print >>sys.stderr, ' *', self
                print >>sys.stderr, ' * These tokens could be matched by more than one clause:'
                print >>sys.stderr, ' *', ' '.join(removed)
                
            if testset:
                if not tokens_unseen: # context sensitive scanners only!
                    if test == 'if':
                        # if it's the first AND last test, then
                        # we can simply put the code without an if/else
                        c.output(gen, indent)
                    else:
                        gen.write(indent, "else:")
                        t = gen.in_test('', [], testset)
                        if len(t) < 70-len(indent):
                            gen.write(' #', t)
                        gen.write("\n")
                        c.output(gen, indent+INDENT)
                else:
                    gen.write(indent, test, " ",
                              gen.in_test('_token', tokens_unseen, testset),
                              ":\n")
                    c.output(gen, indent+INDENT)
                test = "elif"

        if tokens_unseen:
            gen.write(indent, "else:\n")
            gen.write(indent, INDENT, "raise runtime.SyntaxError(_token[0], ")
            gen.write("'Could not match ", self.rule, "')\n")
        
class Wrapper(Node):
    """This is a base class for nodes that modify a single child."""
    def __init__(self, rule, child):
        Node.__init__(self, rule)
        self.child = child

    def setup(self, gen):
        Node.setup(self, gen)
        self.child.setup(gen)

    def get_children(self):
        return [self.child]
    
    def update(self, gen):
        Node.update(self, gen)
        self.child.update(gen)
        gen.add_to(self.first, self.child.first)
        gen.equate(self.follow, self.child.follow)

class Option(Wrapper):
    """This class represents an optional clause of the form [A]"""
    def setup(self, gen):
        Wrapper.setup(self, gen)
        if not self.accepts_epsilon:
            self.accepts_epsilon = 1
            gen.changed()

    def __str__(self):
        return '[ %s ]' % str(self.child)

    def output(self, gen, indent):
        if self.child.accepts_epsilon:
            print >>sys.stderr, 'Warning in rule', self.rule+': contents may be empty.'
        gen.write(indent, "if %s:\n" %
                  gen.peek_test(self.first, self.child.first))
        self.child.output(gen, indent+INDENT)

        if gen.has_option('context-insensitive-scanner'):
            gen.write(indent, "if %s:\n" %
                    gen.not_peek_test(gen.non_ignored_tokens(), self.follow))
            gen.write(indent+INDENT, "raise runtime.SyntaxError(pos=self._scanner.get_pos(), context=_context, msg='Need one of ' + ', '.join(%s))\n" %
                    repr(self.first))

        
class Plus(Wrapper):
    """This class represents a 1-or-more repetition clause of the form A+"""
    def setup(self, gen):
        Wrapper.setup(self, gen)
        if self.accepts_epsilon != self.child.accepts_epsilon:
            self.accepts_epsilon = self.child.accepts_epsilon
            gen.changed()

    def __str__(self):
        return '%s+' % str(self.child)

    def update(self, gen):
        Wrapper.update(self, gen)
        gen.add_to(self.child.follow, self.child.first)
        
    def output(self, gen, indent):
        if self.child.accepts_epsilon:
            print >>sys.stderr, 'Warning in rule', self.rule+':'
            print >>sys.stderr, ' * The repeated pattern could be empty.  The resulting parser may not work properly.'
        gen.write(indent, "while 1:\n")
        self.child.output(gen, indent+INDENT)
        union = self.first[:]
        gen.add_to(union, self.follow)
        gen.write(indent+INDENT, "if %s: break\n" %
                  gen.not_peek_test(union, self.child.first))

        if gen.has_option('context-insensitive-scanner'):
            gen.write(indent, "if %s:\n" %
                    gen.not_peek_test(gen.non_ignored_tokens(), self.follow))
            gen.write(indent+INDENT, "raise runtime.SyntaxError(pos=self._scanner.get_pos(), context=_context, msg='Need one of ' + ', '.join(%s))\n" %
                    repr(self.first))


class Star(Wrapper):
    """This class represents a 0-or-more repetition clause of the form A*"""
    def setup(self, gen):
        Wrapper.setup(self, gen)
        if not self.accepts_epsilon:
            self.accepts_epsilon = 1
            gen.changed()

    def __str__(self):
        return '%s*' % str(self.child)

    def update(self, gen):
        Wrapper.update(self, gen)
        gen.add_to(self.child.follow, self.child.first)
        
    def output(self, gen, indent):
        if self.child.accepts_epsilon:
            print >>sys.stderr, 'Warning in rule', self.rule+':'
            print >>sys.stderr, ' * The repeated pattern could be empty.  The resulting parser probably will not work properly.'
        gen.write(indent, "while %s:\n" %
                  gen.peek_test(self.follow, self.child.first))
        self.child.output(gen, indent+INDENT)

        # TODO: need to generate tests like this in lots of rules
        if gen.has_option('context-insensitive-scanner'):
            gen.write(indent, "if %s:\n" %
                    gen.not_peek_test(gen.non_ignored_tokens(), self.follow))
            gen.write(indent+INDENT, "raise runtime.SyntaxError(pos=self._scanner.get_pos(), context=_context, msg='Need one of ' + ', '.join(%s))\n" %
                    repr(self.first))