yapps2.py

一个java写的proxy的例子

            for x in testset:
                if x in tokens_seen:
                    testset.remove(x)
                    removed.append(x)
                if x in tokens_unseen: tokens_unseen.remove(x)
            tokens_seen = tokens_seen + testset
            if removed:
                if not testset:
                    print 'Error in rule', self.rule+':', c, 'never matches.'
                else:
                    print 'Warning:', self
                print ' * These tokens are being ignored:', join(removed, ', ')
                print '   due to previous choices using them.'
                
            if testset:
                if not tokens_unseen: # context sensitive scanners only!
                    if test=='if':
                        # if it's the first AND last test, then
                        # we can simply put the code without an if/else
                        c.output(gen, indent)
                    else:
                        gen.write(indent, "else: ")
                        t = gen.in_test('', [], testset)
                        if len(t) < 70-len(indent):
                            gen.write("#", t)
                            gen.write("\n")
                        c.output(gen, indent+INDENT)
                else:
                    gen.write(indent, test, " ",
                              gen.in_test('_token_', tokens_unseen, testset),
                              ":\n")
                    c.output(gen, indent+INDENT)
                test = "elif"

        if gen['context-insensitive-scanner'] and tokens_unseen:
            gen.write(indent, "else:\n")
            gen.write(indent, INDENT, "raise SyntaxError(self._pos, ")
            gen.write("'Could not match ", self.rule, "')\n")

class Wrapper(Node):
    def __init__(self, child):
        Node.__init__(self)
        self.child = child

    def setup(self, gen, rule):
        Node.setup(self, gen, rule)
        self.child.setup(gen, rule)

    def get_children(self):
        return [self.child]
    
    def update(self, gen):
        Node.update(self, gen)
        self.child.update(gen)
        gen.add_to(self.first, self.child.first)
        gen.equate(self.follow, self.child.follow)

class Option(Wrapper):
    def setup(self, gen, rule):
        Wrapper.setup(self, gen, rule)
        if not self.accepts_epsilon:
            self.accepts_epsilon = 1
            gen.changed()

    def __str__(self):
        return '[ %s ]' % str(self.child)

    def output(self, gen, indent):
        if self.child.accepts_epsilon:
            print 'Warning in rule', self.rule+': contents may be empty.'
        gen.write(indent, "if %s:\n" %
                  gen.peek_test(self.first, self.child.first))
        self.child.output(gen, indent+INDENT)
        
class Plus(Wrapper):
    def setup(self, gen, rule):
        Wrapper.setup(self, gen, rule)
        if self.accepts_epsilon != self.child.accepts_epsilon:
            self.accepts_epsilon = self.child.accepts_epsilon
            gen.changed()

    def __str__(self):
        return '%s+' % str(self.child)

    def update(self, gen):
        Wrapper.update(self, gen)
        gen.add_to(self.follow, self.first)

    def output(self, gen, indent):
        if self.child.accepts_epsilon:
            print 'Warning in rule', self.rule+':'
            print ' * The repeated pattern could be empty.  The resulting'
            print '   parser may not work properly.'
        gen.write(indent, "while 1:\n")
        self.child.output(gen, indent+INDENT)
        union = self.first[:]
        gen.add_to(union, self.follow)
        gen.write(indent+INDENT, "if %s: break\n" %
                  gen.not_peek_test(union, self.child.first))

class Star(Plus):
    def setup(self, gen, rule):
        Wrapper.setup(self, gen, rule)
        if not self.accepts_epsilon:
            self.accepts_epsilon = 1
            gen.changed()

    def __str__(self):
        return '%s*' % str(self.child)

    def output(self, gen, indent):
        if self.child.accepts_epsilon:
            print 'Warning in rule', self.rule+':'
            print ' * The repeated pattern could be empty.  The resulting'
            print '   parser probably will not work properly.'
        gen.write(indent, "while %s:\n" %
                  gen.peek_test(self.follow, self.child.first))
        self.child.output(gen, indent+INDENT)

######################################################################
# The remainder of this file is from parsedesc.{g,py}

def append(lst, x):
    "Imperative append"
    lst.append(x)
    return lst

def add_inline_token(tokens, str):
    tokens.insert( 0, (str, eval(str, {}, {})) )
    return Terminal(str)

def cleanup_choice(lst):
    if len(lst) == 0: return Sequence([])
    if len(lst) == 1: return lst[0]
    return apply(Choice, tuple(lst))

def cleanup_sequence(lst):
    if len(lst) == 1: return lst[0]
    return apply(Sequence, tuple(lst))

def cleanup_rep(node, rep):
    if rep == 'star':   return Star(node)
    elif rep == 'plus': return Plus(node)
    else:               return node

def resolve_name(tokens, id, args):
    if id in map(lambda x: x[0], tokens):
	# It's a token
	if args: 
	    print 'Warning: ignoring parameters on TOKEN %s<<%s>>' % (id, args)
        return Terminal(id)
    else:
        # It's a name, so assume it's a nonterminal
        return NonTerminal(id, args)


from string import *
import re
from yappsrt import *

class ParserDescriptionScanner(Scanner):
    def __init__(self, str):
        Scanner.__init__(self,[
            ('"rule"', 'rule'),
            ('"ignore"', 'ignore'),
            ('"token"', 'token'),
            ('"option"', 'option'),
            ('":"', ':'),
            ('"parser"', 'parser'),
            ('[ \011\015\012]+', '[ \011\015\012]+'),
            ('#.*?\015?\012', '#.*?\015?\012'),
            ('END', '$'),
            ('ATTR', '<<.+?>>'),
            ('STMT', '{{.+?}}'),
            ('ID', '[a-zA-Z_][a-zA-Z_0-9]*'),
            ('STR', '[rR]?\'([^\\n\'\\\\]|\\\\.)*\'|[rR]?"([^\\n"\\\\]|\\\\.)*"'),
            ('LP', '\\('),
            ('RP', '\\)'),
            ('LB', '\\['),
            ('RB', '\\]'),
            ('OR', '[|]'),
            ('STAR', '[*]'),
            ('PLUS', '[+]'),
            ], ['[ \011\015\012]+', '#.*?\015?\012'], str)

class ParserDescription(Parser):
    def Parser(self):
        self._scan('"parser"')
        ID = self._scan('ID')
        self._scan('":"')
        Options = self.Options()
        Tokens = self.Tokens()
        Rules = self.Rules(Tokens)
        END = self._scan('END')
        return Generator(ID,Options,Tokens,Rules)

    def Options(self):
        opt = {}
        while self._peek('"option"', '"token"', '"ignore"', 'END', '"rule"') == '"option"':
            self._scan('"option"')
            self._scan('":"')
            Str = self.Str()
            opt[Str] = 1
        return opt

    def Tokens(self):
        tok = []
        while self._peek('"token"', '"ignore"', 'END', '"rule"') in ['"token"', '"ignore"']:
            _token_ = self._peek('"token"', '"ignore"')
            if _token_ == '"token"':
                self._scan('"token"')
                ID = self._scan('ID')
                self._scan('":"')
                Str = self.Str()
                tok.append( (ID,Str) )
            else: # == '"ignore"'
                self._scan('"ignore"')
                self._scan('":"')
                Str = self.Str()
                tok.append( ('#ignore',Str) )
        return tok

    def Rules(self, tokens):
        rul = []
        while self._peek('"rule"', 'END') == '"rule"':
            self._scan('"rule"')
            ID = self._scan('ID')
            OptParam = self.OptParam()
            self._scan('":"')
            ClauseA = self.ClauseA(tokens)
            rul.append( (ID,OptParam,ClauseA) )
        return rul

    def ClauseA(self, tokens):
        ClauseB = self.ClauseB(tokens)
        v = [ClauseB]
        while self._peek('OR', 'RP', 'RB', '"rule"', 'END') == 'OR':
            OR = self._scan('OR')
            ClauseB = self.ClauseB(tokens)
            v.append(ClauseB)
        return cleanup_choice(v)

    def ClauseB(self, tokens):
        v = []
        while self._peek('STR', 'ID', 'LP', 'LB', 'STMT', 'OR', 'RP', 'RB', '"rule"', 'END') in ['STR', 'ID', 'LP', 'LB', 'STMT']:
            ClauseC = self.ClauseC(tokens)
            v.append(ClauseC)
        return cleanup_sequence(v)

    def ClauseC(self, tokens):
        ClauseD = self.ClauseD(tokens)
        _token_ = self._peek('PLUS', 'STAR', 'STR', 'ID', 'LP', 'LB', 'STMT', 'OR', 'RP', 'RB', '"rule"', 'END')
        if _token_ == 'PLUS':
            PLUS = self._scan('PLUS')
            return Plus(ClauseD)
        elif _token_ == 'STAR':
            STAR = self._scan('STAR')
            return Star(ClauseD)
        else: 
            return ClauseD

    def ClauseD(self, tokens):
        _token_ = self._peek('STR', 'ID', 'LP', 'LB', 'STMT')
        if _token_ == 'STR':
            STR = self._scan('STR')
            t = (STR, eval(STR,{},{}))
            if t not in tokens: tokens.insert( 0, t )
            return Terminal(STR)
        elif _token_ == 'ID':
            ID = self._scan('ID')
            OptParam = self.OptParam()
            return resolve_name(tokens, ID, OptParam)
        elif _token_ == 'LP':
            LP = self._scan('LP')
            ClauseA = self.ClauseA(tokens)
            RP = self._scan('RP')
            return ClauseA
        elif _token_ == 'LB':
            LB = self._scan('LB')
            ClauseA = self.ClauseA(tokens)
            RB = self._scan('RB')
            return Option(ClauseA)
        else: # == 'STMT'
            STMT = self._scan('STMT')
            return Eval(STMT[2:-2])

    def OptParam(self):
        if self._peek('ATTR', '":"', 'PLUS', 'STAR', 'STR', 'ID', 'LP', 'LB', 'STMT', 'OR', 'RP', 'RB', '"rule"', 'END') == 'ATTR':
            ATTR = self._scan('ATTR')
            return ATTR[2:-2]
        return ''

    def Str(self):
        STR = self._scan('STR')
        return eval(STR,{},{})





# This replaces the default main routine

yapps_options = [
    ('context-insensitive-scanner', 'context-insensitive-scanner',
     'Scan all tokens (see docs)')
    ]

def generate(inputfilename, outputfilename='', dump=0, **flags):
    """Generate a grammar, given an input filename (X.g)
    and an output filename (defaulting to X.py)."""

    if not outputfilename:
	if inputfilename[-2:]=='.g': outputfilename = inputfilename[:-2]+'.py'
	else: raise "Invalid Filename", outputfilename
        
    print 'Input Grammar:', inputfilename
    print 'Output File:', outputfilename
    
    DIVIDER = '\n%%\n' # This pattern separates the pre/post parsers
    preparser, postparser = None, None # Code before and after the parser desc

    # Read the entire file
    s = open(inputfilename,'r').read()

    # See if there's a separation between the pre-parser and parser
    f = find(s, DIVIDER)
    if f >= 0: preparser, s = s[:f]+'\n\n', s[f+len(DIVIDER):]

    # See if there's a separation between the parser and post-parser
    f = find(s, DIVIDER)
    if f >= 0: s, postparser = s[:f], '\n\n'+s[f+len(DIVIDER):]

    # Create the parser and scanner
    p = ParserDescription(ParserDescriptionScanner(s))
    if not p: return
    
    # Now parse the file
    t = wrap_error_reporter(p, 'Parser')
    if not t: return # Error
    if preparser is not None: t.preparser = preparser
    if postparser is not None: t.postparser = postparser

    # Check the options
    for f in t.options.keys():
        for opt,_,_ in yapps_options:
            if f == opt: break
        else:
            print 'Warning: unrecognized option', f
    # Add command line options to the set
    for f in flags.keys(): t.options[f] = flags[f]
            
    # Generate the output
    if dump:
        t.dump_information()
    else:
        t.output = open(outputfilename, 'w')
        t.generate_output()

if __name__=='__main__':
    import sys, getopt
    optlist, args = getopt.getopt(sys.argv[1:], 'f:', ['dump'])
    if not args or len(args) > 2:
        print 'Usage:'
        print '  python', sys.argv[0], '[flags] input.g [output.py]'
        print 'Flags:'
        print ('  --dump' + ' '*40)[:35] + 'Dump out grammar information'
        for flag, _, doc in yapps_options:
            print ('  -f' + flag + ' '*40)[:35] + doc
    else:
        # Read in the options and create a list of flags
	flags = {}
	for opt in optlist:
	    for flag, name, _ in yapps_options:
		if opt == ('-f', flag):
		    flags[name] = 1
		    break
	    else:
                if opt == ('--dump', ''):
                    flags['dump'] = 1
		else:
                    print 'Warning - unrecognized option:  ', opt[0], opt[1]

        apply(generate, tuple(args), flags)