moduleparser.py

Requirement =====================================================================================

        self.parse_parameter_list(node)
        self.visit(node.code)
        self.context.pop()

    def parse_parameter_list(self, node):
        parameters = []
        special = []
        argnames = list(node.argnames)
        if node.kwargs:
            special.append(make_parameter(argnames[-1], excess_keyword=1))
            argnames.pop()
        if node.varargs:
            special.append(make_parameter(argnames[-1],
                                          excess_positional=1))
            argnames.pop()
        defaults = list(node.defaults)
        defaults = [None] * (len(argnames) - len(defaults)) + defaults
        function_parameters = self.token_parser.function_parameters(
            node.lineno)
        #print >>sys.stderr, function_parameters
        for argname, default in zip(argnames, defaults):
            if type(argname) is TupleType:
                parameter = pynodes.parameter_tuple()
                for tuplearg in argname:
                    parameter.append(make_parameter(tuplearg))
                argname = normalize_parameter_name(argname)
            else:
                parameter = make_parameter(argname)
            if default:
                n_default = pynodes.parameter_default()
                n_default.append(Text(function_parameters[argname]))
                parameter.append(n_default)
            parameters.append(parameter)
        if parameters or special:
            special.reverse()
            parameters.extend(special)
            parameter_list = pynodes.parameter_list()
            parameter_list.extend(parameters)
            self.function.append(parameter_list)


class ClassVisitor(AssignmentVisitor):

    in_class = 0

    def __init__(self, token_parser):
        AssignmentVisitor.__init__(self, token_parser)
        self.bases = []

    def visitClass(self, node):
        if self.in_class:
            self.documentable = None
            # Don't bother with nested class definitions.
            return
        self.in_class = 1
        #import mypdb as pdb
        #pdb.set_trace()
        for base in node.bases:
            self.visit(base)
        self.klass = klass = make_class_section(node.name, self.bases,
                                                doc=node.doc,
                                                lineno=node.lineno)
        self.context.append(klass)
        self.documentable = klass
        self.visit(node.code)
        self.context.pop()

    def visitGetattr(self, node, suffix=None):
        if suffix:
            name = node.attrname + '.' + suffix
        else:
            name = node.attrname
        self.default_visit(node, name)

    def visitName(self, node, suffix=None):
        if suffix:
            name = node.name + '.' + suffix
        else:
            name = node.name
        self.bases.append(name)

    def visitFunction(self, node):
        if node.name == '__init__':
            visitor = InitMethodVisitor(self.token_parser,
                                        function_class=pynodes.method_section)
            compiler.walk(node, visitor, walker=visitor)
        else:
            visitor = FunctionVisitor(self.token_parser,
                                      function_class=pynodes.method_section)
            compiler.walk(node, visitor, walker=visitor)
        self.context[-1].append(visitor.function)


class InitMethodVisitor(FunctionVisitor, AssignmentVisitor): pass


class TokenParser:

    def __init__(self, text):
        self.text = text + '\n\n'
        self.lines = self.text.splitlines(1)
        self.generator = tokenize.generate_tokens(iter(self.lines).next)
        self.next()

    def __iter__(self):
        return self

    def next(self):
        self.token = self.generator.next()
        self.type, self.string, self.start, self.end, self.line = self.token
        return self.token

    def goto_line(self, lineno):
        while self.start[0] < lineno:
            self.next()
        return token

    def rhs(self, lineno):
        """
        Return a whitespace-normalized expression string from the right-hand
        side of an assignment at line `lineno`.
        """
        self.goto_line(lineno)
        while self.string != '=':
            self.next()
        self.stack = None
        while self.type != token.NEWLINE and self.string != ';':
            if self.string == '=' and not self.stack:
                self.tokens = []
                self.stack = []
                self._type = None
                self._string = None
                self._backquote = 0
            else:
                self.note_token()
            self.next()
        self.next()
        text = ''.join(self.tokens)
        return text.strip()

    closers = {')': '(', ']': '[', '}': '{'}
    openers = {'(': 1, '[': 1, '{': 1}
    del_ws_prefix = {'.': 1, '=': 1, ')': 1, ']': 1, '}': 1, ':': 1, ',': 1}
    no_ws_suffix = {'.': 1, '=': 1, '(': 1, '[': 1, '{': 1}

    def note_token(self):
        if self.type == tokenize.NL:
            return
        del_ws = self.del_ws_prefix.has_key(self.string)
        append_ws = not self.no_ws_suffix.has_key(self.string)
        if self.openers.has_key(self.string):
            self.stack.append(self.string)
            if (self._type == token.NAME
                or self.closers.has_key(self._string)):
                del_ws = 1
        elif self.closers.has_key(self.string):
            assert self.stack[-1] == self.closers[self.string]
            self.stack.pop()
        elif self.string == '`':
            if self._backquote:
                del_ws = 1
                assert self.stack[-1] == '`'
                self.stack.pop()
            else:
                append_ws = 0
                self.stack.append('`')
            self._backquote = not self._backquote
        if del_ws and self.tokens and self.tokens[-1] == ' ':
            del self.tokens[-1]
        self.tokens.append(self.string)
        self._type = self.type
        self._string = self.string
        if append_ws:
            self.tokens.append(' ')

    def function_parameters(self, lineno):
        """
        Return a dictionary mapping parameters to defaults
        (whitespace-normalized strings).
        """
        self.goto_line(lineno)
        while self.string != 'def':
            self.next()
        while self.string != '(':
            self.next()
        name = None
        default = None
        parameter_tuple = None
        self.tokens = []
        parameters = {}
        self.stack = [self.string]
        self.next()
        while 1:
            if len(self.stack) == 1:
                if parameter_tuple:
                    # Just encountered ")".
                    #print >>sys.stderr, 'parameter_tuple: %r' % self.tokens
                    name = ''.join(self.tokens).strip()
                    self.tokens = []
                    parameter_tuple = None
                if self.string in (')', ','):
                    if name:
                        if self.tokens:
                            default_text = ''.join(self.tokens).strip()
                        else:
                            default_text = None
                        parameters[name] = default_text
                        self.tokens = []
                        name = None
                        default = None
                    if self.string == ')':
                        break
                elif self.type == token.NAME:
                    if name and default:
                        self.note_token()
                    else:
                        assert name is None, (
                            'token=%r name=%r parameters=%r stack=%r'
                            % (self.token, name, parameters, self.stack))
                        name = self.string
                        #print >>sys.stderr, 'name=%r' % name
                elif self.string == '=':
                    assert name is not None, 'token=%r' % (self.token,)
                    assert default is None, 'token=%r' % (self.token,)
                    assert self.tokens == [], 'token=%r' % (self.token,)
                    default = 1
                    self._type = None
                    self._string = None
                    self._backquote = 0
                elif name:
                    self.note_token()
                elif self.string == '(':
                    parameter_tuple = 1
                    self._type = None
                    self._string = None
                    self._backquote = 0
                    self.note_token()
                else:                   # ignore these tokens:
                    assert (self.string in ('*', '**', '\n') 
                            or self.type == tokenize.COMMENT), (
                        'token=%r' % (self.token,))
            else:
                self.note_token()
            self.next()
        return parameters


def make_docstring(doc, lineno):
    n = pynodes.docstring()
    if lineno:
        # Really, only module docstrings don't have a line
        # (@@: but maybe they should)
        n['lineno'] = lineno
    n.append(Text(doc))
    return n

def append_docstring(node, doc, lineno):
    if doc:
        node.append(make_docstring(doc, lineno))

def make_class_section(name, bases, lineno, doc):
    n = pynodes.class_section()
    n['lineno'] = lineno
    n.append(make_object_name(name))
    for base in bases:
        b = pynodes.class_base()
        b.append(make_object_name(base))
        n.append(b)
    append_docstring(n, doc, lineno)
    return n

def make_object_name(name):
    n = pynodes.object_name()
    n.append(Text(name))
    return n

def make_function_like_section(name, lineno, doc, function_class):
    n = function_class()
    n['lineno'] = lineno
    n.append(make_object_name(name))
    append_docstring(n, doc, lineno)
    return n

def make_import_group(names, lineno, from_name=None):
    n = pynodes.import_group()
    n['lineno'] = lineno
    if from_name:
        n_from = pynodes.import_from()
        n_from.append(Text(from_name))
        n.append(n_from)
    for name, alias in names:
        n_name = pynodes.import_name()
        n_name.append(Text(name))
        if alias:
            n_alias = pynodes.import_alias()
            n_alias.append(Text(alias))
            n_name.append(n_alias)
        n.append(n_name)
    return n

def make_class_attribute(name, lineno):
    n = pynodes.class_attribute()
    n['lineno'] = lineno
    n.append(Text(name))
    return n

def make_attribute(name, lineno):
    n = pynodes.attribute()
    n['lineno'] = lineno
    n.append(make_object_name(name))
    return n

def make_parameter(name, excess_keyword=0, excess_positional=0):
    """
    excess_keyword and excess_positional must be either 1 or 0, and
    not both of them can be 1.
    """
    n = pynodes.parameter()
    n.append(make_object_name(name))
    assert not excess_keyword or not excess_positional
    if excess_keyword:
        n['excess_keyword'] = 1
    if excess_positional:
        n['excess_positional'] = 1
    return n

def trim_docstring(text):
    """
    Trim indentation and blank lines from docstring text & return it.

    See PEP 257.
    """
    if not text:
        return text
    # Convert tabs to spaces (following the normal Python rules)
    # and split into a list of lines:
    lines = text.expandtabs().splitlines()
    # Determine minimum indentation (first line doesn't count):
    indent = sys.maxint
    for line in lines[1:]:
        stripped = line.lstrip()
        if stripped:
            indent = min(indent, len(line) - len(stripped))
    # Remove indentation (first line is special):
    trimmed = [lines[0].strip()]
    if indent < sys.maxint:
        for line in lines[1:]:
            trimmed.append(line[indent:].rstrip())
    # Strip off trailing and leading blank lines:
    while trimmed and not trimmed[-1]:
        trimmed.pop()
    while trimmed and not trimmed[0]:
        trimmed.pop(0)
    # Return a single string:
    return '\n'.join(trimmed)

def normalize_parameter_name(name):
    """
    Converts a tuple like ``('a', ('b', 'c'), 'd')`` into ``'(a, (b, c), d)'``
    """
    if type(name) is TupleType:
        return '(%s)' % ', '.join([normalize_parameter_name(n) for n in name])
    else:
        return name

if __name__ == '__main__':
    import sys
    args = sys.argv[1:]
    if args[0] == '-v':
        filename = args[1]
        module_text = open(filename).read()
        ast = compiler.parse(module_text)
        visitor = compiler.visitor.ExampleASTVisitor()
        compiler.walk(ast, visitor, walker=visitor, verbose=1)
    else:
        filename = args[0]
        content = open(filename).read()
        print parse_module(content, filename).pformat()