parsermodule.c

python s60 1.4.5版本的源代码

    int ok;

    static char *keywords[] = {"ast", NULL};

    if (self == NULL)
        ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:issuite", keywords,
                                         &PyST_Type, &self);
    else
        ok = PyArg_ParseTupleAndKeywords(args, kw, ":issuite", &keywords[1]);

    if (ok) {
        /* Check to see if the ST represents an expression or not. */
        res = (self->st_type == PyST_EXPR) ? Py_False : Py_True;
        Py_INCREF(res);
    }
    return (res);
}


#define PUBLIC_METHOD_TYPE (METH_VARARGS|METH_KEYWORDS)

static PyMethodDef
parser_methods[] = {
    {"compile",         (PyCFunction)parser_compilest,  PUBLIC_METHOD_TYPE,
        "Compile this ST object into a code object."},
    {"isexpr",          (PyCFunction)parser_isexpr,     PUBLIC_METHOD_TYPE,
        "Determines if this ST object was created from an expression."},
    {"issuite",         (PyCFunction)parser_issuite,    PUBLIC_METHOD_TYPE,
        "Determines if this ST object was created from a suite."},
    {"tolist",          (PyCFunction)parser_st2list,    PUBLIC_METHOD_TYPE,
        "Creates a list-tree representation of this ST."},
    {"totuple",         (PyCFunction)parser_st2tuple,   PUBLIC_METHOD_TYPE,
        "Creates a tuple-tree representation of this ST."},

    {NULL, NULL, 0, NULL}
};


static PyObject*
parser_getattr(PyObject *self, char *name)
{
    return (Py_FindMethod(parser_methods, self, name));
}


/*  err_string(char* message)
 *
 *  Sets the error string for an exception of type ParserError.
 *
 */
static void
err_string(char *message)
{
    PyErr_SetString(parser_error, message);
}


/*  PyObject* parser_do_parse(PyObject* args, int type)
 *
 *  Internal function to actually execute the parse and return the result if
 *  successful, or set an exception if not.
 *
 */
static PyObject*
parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type)
{
    char*     string = 0;
    PyObject* res    = 0;

    static char *keywords[] = {"source", NULL};

    if (PyArg_ParseTupleAndKeywords(args, kw, argspec, keywords, &string)) {
        node* n = PyParser_SimpleParseString(string,
                                             (type == PyST_EXPR)
                                             ? eval_input : file_input);

        if (n != 0)
            res = parser_newstobject(n, type);
        else
            err_string("could not parse string");
    }
    return (res);
}


/*  PyObject* parser_expr(PyObject* self, PyObject* args)
 *  PyObject* parser_suite(PyObject* self, PyObject* args)
 *
 *  External interfaces to the parser itself.  Which is called determines if
 *  the parser attempts to recognize an expression ('eval' form) or statement
 *  suite ('exec' form).  The real work is done by parser_do_parse() above.
 *
 */
static PyObject*
parser_expr(PyST_Object *self, PyObject *args, PyObject *kw)
{
    NOTE(ARGUNUSED(self))
    return (parser_do_parse(args, kw, "s:expr", PyST_EXPR));
}


static PyObject*
parser_suite(PyST_Object *self, PyObject *args, PyObject *kw)
{
    NOTE(ARGUNUSED(self))
    return (parser_do_parse(args, kw, "s:suite", PyST_SUITE));
}



/*  This is the messy part of the code.  Conversion from a tuple to an ST
 *  object requires that the input tuple be valid without having to rely on
 *  catching an exception from the compiler.  This is done to allow the
 *  compiler itself to remain fast, since most of its input will come from
 *  the parser directly, and therefore be known to be syntactically correct.
 *  This validation is done to ensure that we don't core dump the compile
 *  phase, returning an exception instead.
 *
 *  Two aspects can be broken out in this code:  creating a node tree from
 *  the tuple passed in, and verifying that it is indeed valid.  It may be
 *  advantageous to expand the number of ST types to include funcdefs and
 *  lambdadefs to take advantage of the optimizer, recognizing those STs
 *  here.  They are not necessary, and not quite as useful in a raw form.
 *  For now, let's get expressions and suites working reliably.
 */


staticforward node* build_node_tree(PyObject *tuple);
staticforward int   validate_expr_tree(node *tree);
staticforward int   validate_file_input(node *tree);


/*  PyObject* parser_tuple2st(PyObject* self, PyObject* args)
 *
 *  This is the public function, called from the Python code.  It receives a
 *  single tuple object from the caller, and creates an ST object if the
 *  tuple can be validated.  It does this by checking the first code of the
 *  tuple, and, if acceptable, builds the internal representation.  If this
 *  step succeeds, the internal representation is validated as fully as
 *  possible with the various validate_*() routines defined below.
 *
 *  This function must be changed if support is to be added for PyST_FRAGMENT
 *  ST objects.
 *
 */
static PyObject*
parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw)
{
    NOTE(ARGUNUSED(self))
    PyObject *st = 0;
    PyObject *tuple;
    node *tree;

    static char *keywords[] = {"sequence", NULL};

    if (!PyArg_ParseTupleAndKeywords(args, kw, "O:sequence2st", keywords,
                                     &tuple))
        return (0);
    if (!PySequence_Check(tuple)) {
        PyErr_SetString(PyExc_ValueError,
                        "sequence2st() requires a single sequence argument");
        return (0);
    }
    /*
     *  Convert the tree to the internal form before checking it.
     */
    tree = build_node_tree(tuple);
    if (tree != 0) {
        int start_sym = TYPE(tree);
        if (start_sym == eval_input) {
            /*  Might be an eval form.  */
            if (validate_expr_tree(tree))
                st = parser_newstobject(tree, PyST_EXPR);
            else
                PyNode_Free(tree);
        }
        else if (start_sym == file_input) {
            /*  This looks like an exec form so far.  */
            if (validate_file_input(tree))
                st = parser_newstobject(tree, PyST_SUITE);
            else
                PyNode_Free(tree);
        }
        else {
            /*  This is a fragment, at best. */
            PyNode_Free(tree);
            err_string("parse tree does not use a valid start symbol");
        }
    }
    /*  Make sure we throw an exception on all errors.  We should never
     *  get this, but we'd do well to be sure something is done.
     */
    if (st == NULL && !PyErr_Occurred())
        err_string("unspecified ST error occurred");

    return st;
}


/*  node* build_node_children()
 *
 *  Iterate across the children of the current non-terminal node and build
 *  their structures.  If successful, return the root of this portion of
 *  the tree, otherwise, 0.  Any required exception will be specified already,
 *  and no memory will have been deallocated.
 *
 */
static node*
build_node_children(PyObject *tuple, node *root, int *line_num)
{
    int len = PyObject_Size(tuple);
    int i, err;

    for (i = 1; i < len; ++i) {
        /* elem must always be a sequence, however simple */
        PyObject* elem = PySequence_GetItem(tuple, i);
        int ok = elem != NULL;
        long  type = 0;
        char *strn = 0;

        if (ok)
            ok = PySequence_Check(elem);
        if (ok) {
            PyObject *temp = PySequence_GetItem(elem, 0);
            if (temp == NULL)
                ok = 0;
            else {
                ok = PyInt_Check(temp);
                if (ok)
                    type = PyInt_AS_LONG(temp);
                Py_DECREF(temp);
            }
        }
        if (!ok) {
            PyErr_SetObject(parser_error,
                            Py_BuildValue("os", elem,
                                          "Illegal node construct."));
            Py_XDECREF(elem);
            return (0);
        }
        if (ISTERMINAL(type)) {
            int len = PyObject_Size(elem);
            PyObject *temp;

            if ((len != 2) && (len != 3)) {
                err_string("terminal nodes must have 2 or 3 entries");
                return 0;
            }
            temp = PySequence_GetItem(elem, 1);
            if (temp == NULL)
                return 0;
            if (!PyString_Check(temp)) {
                PyErr_Format(parser_error,
                             "second item in terminal node must be a string,"
                             " found %s",
                             ((PyTypeObject*)PyObject_Type(temp))->tp_name);
                Py_DECREF(temp);
                return 0;
            }
            if (len == 3) {
                PyObject *o = PySequence_GetItem(elem, 2);
                if (o != NULL) {
                    if (PyInt_Check(o))
                        *line_num = PyInt_AS_LONG(o);
                    else {
                        PyErr_Format(parser_error,
                                     "third item in terminal node must be an"
                                     " integer, found %s",
                                ((PyTypeObject*)PyObject_Type(temp))->tp_name);
                        Py_DECREF(o);
                        Py_DECREF(temp);
                        return 0;
                    }
                    Py_DECREF(o);
                }
            }
            len = PyString_GET_SIZE(temp) + 1;
            strn = (char *)PyMem_MALLOC(len);
            if (strn != NULL)
                (void) memcpy(strn, PyString_AS_STRING(temp), len);
            Py_DECREF(temp);
        }
        else if (!ISNONTERMINAL(type)) {
            /*
             *  It has to be one or the other; this is an error.
             *  Throw an exception.
             */
            PyErr_SetObject(parser_error,
                            Py_BuildValue("os", elem, "unknown node type."));
            Py_XDECREF(elem);
            return (0);
        }
        err = PyNode_AddChild(root, type, strn, *line_num);
        if (err == E_NOMEM) {
            PyMem_DEL(strn);
            return (node *) PyErr_NoMemory();
        }
        if (err == E_OVERFLOW) {
            PyMem_DEL(strn);
            PyErr_SetString(PyExc_ValueError,
                            "unsupported number of child nodes");
            return NULL;
        }

        if (ISNONTERMINAL(type)) {
            node* new_child = CHILD(root, i - 1);

            if (new_child != build_node_children(elem, new_child, line_num)) {
                Py_XDECREF(elem);
                return (0);
            }
        }
        else if (type == NEWLINE) {     /* It's true:  we increment the     */
            ++(*line_num);              /* line number *after* the newline! */
        }
        Py_XDECREF(elem);
    }
    return (root);
}


static node*
build_node_tree(PyObject *tuple)
{
    node* res = 0;
    PyObject *temp = PySequence_GetItem(tuple, 0);
    long num = -1;

    if (temp != NULL)
        num = PyInt_AsLong(temp);
    Py_XDECREF(temp);
    if (ISTERMINAL(num)) {
        /*
         *  The tuple is simple, but it doesn't start with a start symbol.
         *  Throw an exception now and be done with it.
         */
        tuple = Py_BuildValue("os", tuple,
                    "Illegal syntax-tree; cannot start with terminal symbol.");
        PyErr_SetObject(parser_error, tuple);
    }
    else if (ISNONTERMINAL(num)) {
        /*
         *  Not efficient, but that can be handled later.
         */
        int line_num = 0;

        res = PyNode_New(num);
        if (res != NULL) {
            if (res != build_node_children(tuple, res, &line_num)) {
                PyNode_Free(res);
                res = NULL;
            }
        }
    }
    else
        /*  The tuple is illegal -- if the number is neither TERMINAL nor
         *  NONTERMINAL, we can't use it.  Not sure the implementation
         *  allows this condition, but the API doesn't preclude it.
         */
        PyErr_SetObject(parser_error,
                        Py_BuildValue("os", tuple,
                                      "Illegal component tuple."));

    return (res);
}


/*
 *  Validation routines used within the validation section:
 */
staticforward int validate_terminal(node *terminal, int type, char *string);

#define validate_ampersand(ch)  validate_terminal(ch,      AMPER, "&")
#define validate_circumflex(ch) validate_terminal(ch, CIRCUMFLEX, "^")
#define validate_colon(ch)      validate_terminal(ch,      COLON, ":")
#define validate_comma(ch)      validate_terminal(ch,      COMMA, ",")
#define validate_dedent(ch)     validate_terminal(ch,     DEDENT, "")
#define validate_equal(ch)      validate_terminal(ch,      EQUAL, "=")
#define validate_indent(ch)     validate_terminal(ch,     INDENT, (char*)NULL)
#define validate_lparen(ch)     validate_terminal(ch,       LPAR, "(")
#define validate_newline(ch)    validate_terminal(ch,    NEWLINE, (char*)NULL)
#define validate_rparen(ch)     validate_terminal(ch,       RPAR, ")")
#define validate_semi(ch)       validate_terminal(ch,       SEMI, ";")
#define validate_star(ch)       validate_terminal(ch,       STAR, "*")
#define validate_vbar(ch)       validate_terminal(ch,       VBAR, "|")
#define validate_doublestar(ch) validate_terminal(ch, DOUBLESTAR, "**")
#define validate_dot(ch)        validate_terminal(ch,        DOT, ".")
#define validate_name(ch, str)  validate_terminal(ch,       NAME, str)

#define VALIDATER(n)    static int validate_##n(node *tree)

VALIDATER(node);                VALIDATER(small_stmt);
VALIDATER(class);               VALIDATER(node);
VALIDATER(parameters);          VALIDATER(suite);
VALIDATER(testlist);            VALIDATER(varargslist);
VALIDATER(fpdef);               VALIDATER(fplist);
VALIDATER(stmt);                VALIDATER(simple_stmt);
VALIDATER(expr_stmt);           VALIDATER(power);
VALIDATER(print_stmt);          VALIDATER(del_stmt);
VALIDATER(return_stmt);         VALIDATER(list_iter);
VALIDATER(raise_stmt);          VALIDATER(import_stmt);
VALIDATER(global_stmt);         VALIDATER(list_if);
VALIDATER(assert_stmt);         VALIDATER(list_for);
VALIDATER(exec_stmt);           VALIDATER(compound_stmt);
VALIDATER(while);               VALIDATER(for);
VALIDATER(try);                 VALIDATER(except_clause);
VALIDATER(test);                VALIDATER(and_test);
VALIDATER(not_test);            VALIDATER(comparison);
VALIDATER(comp_op);             VALIDATER(expr);
VALIDATER(xor_expr);            VALIDATER(and_expr);
VALIDATER(shift_expr);          VALIDATER(arith_expr);
VALIDATER(term);                VALIDATER(factor);
VALIDATER(atom);                VALIDATER(lambdef);
VALIDATER(trailer);             VALIDATER(subscript);
VALIDATER(subscriptlist);       VALIDATER(sliceop);
VALIDATER(exprlist);            VALIDATER(dictmaker);
VALIDATER(arglist);             VALIDATER(argument);
VALIDATER(listmaker);           VALIDATER(yield_stmt);

#undef VALIDATER

#define is_even(n)      (((n) & 1) == 0)
#define is_odd(n)       (((n) & 1) == 1)


static int
validate_ntype(node *n, int t)
{