Add helper functions for common conversions

src/runtime/python/ffi.c

@@ -28,6 +28,16 @@ PgfExnType handleError(PgfExn err)
 
 // ----------------------------------------------------------------------------
 // conversions
+//
+// You have to remember to call PyMem_Free on these things!
+
+PgfText *
+CString_AsPgfText(const char *s, size_t size) {
+    PgfText *txt = (PgfText *)PyMem_RawMalloc(sizeof(PgfText)+size+1);
+    memcpy(txt->text, s, size+1);
+    txt->size = size;
+    return txt;
+}
 
 PgfText *
 PyUnicode_AsPgfText(PyObject *pystr)
@@ -42,7 +52,7 @@ PyUnicode_AsPgfText(PyObject *pystr)
 
     Py_ssize_t size;
     const char *enc = PyUnicode_AsUTF8AndSize(pystr, &size);
-    PgfText *ptext = malloc(sizeof(PgfText)+size+1);
+    PgfText *ptext = PyMem_RawMalloc(sizeof(PgfText)+size+1);
     memcpy(ptext->text, enc, size+1);
     ptext->size = size;
     return ptext;
@@ -63,7 +73,7 @@ PyList_AsHypos(PyObject *pylist, Py_ssize_t *n_hypos)
     }
     Py_ssize_t n = PyList_Size(pylist);
     *n_hypos = n;
-    PgfTypeHypo *hypos = PyMem_Malloc(sizeof(PgfTypeHypo)*n);
+    PgfTypeHypo *hypos = PyMem_RawMalloc(sizeof(PgfTypeHypo)*n);
 
     for (Py_ssize_t i = 0; i < n; i++) {
         PyObject *tup = PyList_GetItem(pylist, i);
@@ -98,6 +108,59 @@ PyList_AsHypos(PyObject *pylist, Py_ssize_t *n_hypos)
     return hypos;
 }
 
+PyObject *
+PyList_FromHypos(PgfTypeHypo *hypos, const size_t n_hypos)
+{
+    PyObject *pylist = PyList_New(n_hypos);
+    if (pylist == NULL) {
+        return NULL;
+    }
+
+    for (size_t i = 0; i < n_hypos; i++) {
+        PyObject *tup = PyTuple_New(3);
+        PyTuple_SetItem(tup, 0, PyLong_FromLong(hypos[i].bind_type));
+        PyTuple_SetItem(tup, 1, PyUnicode_FromStringAndSize(hypos[i].cid->text, hypos[i].cid->size));
+        PyTuple_SetItem(tup, 2, (PyObject *)hypos[i].type);
+        Py_INCREF(hypos[i].type);
+        PyList_SetItem(pylist, i, tup);
+    }
+    if (PyErr_Occurred()) {
+        Py_DECREF(pylist);
+        return NULL;
+    }
+
+    return pylist;
+}
+
+PgfPrintContext *
+PyList_AsPgfPrintContext(PyObject *pylist)
+{
+    PgfPrintContext *ctxt = NULL;
+    for (Py_ssize_t i = PyList_Size(pylist); i > 0 ; i--) {
+        PyObject *item = PyList_GetItem(pylist, i-1);
+        if (!PyUnicode_Check(item)) {
+            PyErr_SetString(PyExc_TypeError, "variable argument in context must be a string");
+            return NULL;
+        }
+        PgfText *input = PyUnicode_AsPgfText(item);
+
+        // TODO a better way to copy into this->name?
+        PgfPrintContext *this = (PgfPrintContext *)PyMem_RawMalloc(sizeof(PgfPrintContext *) + sizeof(PgfText) + input->size + 1);
+        this->next = ctxt;
+        memcpy(&this->name, input, sizeof(PgfText)+input->size+1);
+        ctxt = this;
+    }
+
+    return ctxt;
+}
+
+void
+FreePgfPrintContext(PgfPrintContext *ctxt)
+{
+    if (ctxt == NULL) return;
+    FreePgfPrintContext(ctxt->next);
+    PyMem_RawFree(ctxt);
+}
 
 // ----------------------------------------------------------------------------
 // unmarshaller
@@ -224,18 +287,8 @@ dtyp(PgfUnmarshaller *this, int n_hypos, PgfTypeHypo *hypos, PgfText *cat, int n
 {
     TypeObject *pytype = (TypeObject *)pgf_TypeType.tp_alloc(&pgf_TypeType, 0);
 
-    pytype->hypos = PyList_New(n_hypos);
+    pytype->hypos = PyList_FromHypos(hypos, n_hypos);
-    for (int i = 0; i < n_hypos; i++) {
-        PyObject *tup = PyTuple_New(3);
-        PyTuple_SetItem(tup, 0, PyLong_FromLong(hypos[i].bind_type));
-        PyTuple_SetItem(tup, 1, PyUnicode_FromStringAndSize(hypos[i].cid->text, hypos[i].cid->size));
-        PyTuple_SetItem(tup, 2, (PyObject *)hypos[i].type);
-        Py_INCREF(hypos[i].type);
-        PyList_SetItem(pytype->hypos, i, tup);
-    }
-
     pytype->cat = PyUnicode_FromStringAndSize(cat->text, cat->size);
-
     pytype->exprs = PyList_New(n_exprs);
     for (int i = 0; i < n_exprs; i++) {
         PyList_SetItem(pytype->exprs, i, (PyObject *)exprs[i]);
@@ -290,7 +343,7 @@ match_lit(PgfMarshaller *this, PgfUnmarshaller *u, PgfLiteral lit)
         // chop up into chunks, always positive
         bool isPos = PyObject_RichCompareBool(pyobj, PyLong_FromLong(0), Py_GE) == 1;
         x = PyNumber_Absolute(PyNumber_Long(pyobj)); // make a copy, ignore sign
-        uintmax_t *i = malloc(sizeof(uintmax_t)*size);
+        uintmax_t *i = PyMem_RawMalloc(sizeof(uintmax_t)*size); // TODO when does this get freed?
         for (int n = size-1; n > 0; n--) {
             PyObject *rem = PyNumber_Remainder(x, intShifter);
             i[n] = PyLong_AsUnsignedLong(rem);

src/runtime/python/ffi.h

@@ -15,10 +15,16 @@ typedef struct {
 PyObject *PGFError;
 PgfExnType handleError(PgfExn err);
 
+PgfText *CString_AsPgfText(const char *s, size_t size);
+
 PgfText *PyUnicode_AsPgfText(PyObject *pystr);
 PyObject *PyUnicode_FromPgfText(PgfText *text);
 
 PgfTypeHypo *PyList_AsHypos(PyObject *pylist, Py_ssize_t *n_hypos);
+PyObject *PyList_FromHypos(PgfTypeHypo *hypos, const size_t n_hypos);
+
+PgfPrintContext *PyList_AsPgfPrintContext(PyObject *pylist);
+void FreePgfPrintContext(PgfPrintContext *ctxt);
 
 PgfUnmarshaller unmarshaller;
 PgfMarshaller marshaller;

src/runtime/python/pypgf.c

@@ -120,14 +120,12 @@ PGF_categoryContext(PGFObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "s#", &s, &size))
         return NULL;
 
-    PgfText *catname = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+    PgfText *catname = CString_AsPgfText(s, size);
-    memcpy(catname->text, s, size+1);
-    catname->size = size;
 
     PgfExn err;
     size_t n_hypos;
     PgfTypeHypo *hypos = pgf_category_context(self->db, self->revision, catname, &n_hypos, &unmarshaller, &err);
-    PyMem_Free(catname);
+    PyMem_RawFree(catname);
     if (handleError(err) != PGF_EXN_NONE) {
         return NULL;
     }
@@ -136,23 +134,7 @@ PGF_categoryContext(PGFObject *self, PyObject *args)
         Py_RETURN_NONE;
     }
 
-    PyObject *contexts = PyList_New(n_hypos);
+    PyObject *contexts = PyList_FromHypos(hypos, n_hypos);
-    if (contexts == NULL) {
-        return NULL;
-    }
-
-    for (size_t i = 0; i < n_hypos; i++) {
-        PyObject *tup = PyTuple_New(3);
-        PyTuple_SetItem(tup, 0, PyLong_FromLong(hypos[i].bind_type));
-        PyTuple_SetItem(tup, 1, PyUnicode_FromStringAndSize(hypos[i].cid->text, hypos[i].cid->size));
-        PyTuple_SetItem(tup, 2, (PyObject *)hypos[i].type);
-        Py_INCREF(hypos[i].type);
-        PyList_SetItem(contexts, i, tup);
-    }
-    if (PyErr_Occurred()) {
-        Py_DECREF(contexts);
-        return NULL;
-    }
 
     return contexts;
 }
@@ -219,19 +201,18 @@ PGF_functionsByCat(PGFObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "s#", &s, &size))
         return NULL;
 
-    PgfText *catname = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+    PgfText *catname = CString_AsPgfText(s, size);
-    memcpy(catname->text, s, size+1);
-    catname->size = size;
 
     PyObject *functions = PyList_New(0);
     if (functions == NULL) {
+        PyMem_RawFree(catname);
         return NULL;
     }
 
     PgfExn err;
     PyPGFClosure clo = { { _collect_funs }, self, functions };
     pgf_iter_functions_by_cat(self->db, self->revision, catname, &clo.fn, &err);
-    PyMem_Free(catname);
+    PyMem_RawFree(catname);
     if (handleError(err) != PGF_EXN_NONE) {
         Py_DECREF(functions);
         return NULL;
@@ -248,13 +229,11 @@ PGF_functionType(PGFObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "s#", &s, &size))
         return NULL;
 
-    PgfText *funname = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+    PgfText *funname = CString_AsPgfText(s, size);
-    memcpy(funname->text, s, size+1);
-    funname->size = size;
 
     PgfExn err;
     PgfType type = pgf_function_type(self->db, self->revision, funname, &unmarshaller, &err);
-    PyMem_Free(funname);
+    PyMem_RawFree(funname);
     if (type == 0) {
         PyErr_Format(PyExc_KeyError, "function '%s' is not defined", s);
         return NULL;
@@ -274,13 +253,11 @@ PGF_functionIsConstructor(PGFObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "s#", &s, &size))
         return NULL;
 
-    PgfText *funname = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+    PgfText *funname = CString_AsPgfText(s, size);
-    memcpy(funname->text, s, size+1);
-    funname->size = size;
 
     PgfExn err;
     int isCon = pgf_function_is_constructor(self->db, self->revision, funname, &err);
-    PyMem_Free(funname);
+    PyMem_RawFree(funname);
     if (handleError(err) != PGF_EXN_NONE) {
         return NULL;
     }
@@ -296,13 +273,11 @@ PGF_functionProbability(PGFObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "s#", &s, &size))
         return NULL;
 
-    PgfText *funname = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+    PgfText *funname = CString_AsPgfText(s, size);
-    memcpy(funname->text, s, size+1);
-    funname->size = size;
 
     PgfExn err;
     prob_t prob = pgf_function_prob(self->db, self->revision, funname, &err);
-    PyMem_Free(funname);
+    PyMem_RawFree(funname);
     if (handleError(err) != PGF_EXN_NONE) {
         return NULL;
     }
@@ -478,16 +453,14 @@ pgf_newNGF(PyObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "s#|s", &s, &size, &fpath))
         return NULL;
 
-    PgfText *absname = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+    PgfText *absname = CString_AsPgfText(s, size);
-    memcpy(absname->text, s, size+1);
-    absname->size = size;
 
     PGFObject *py_pgf = (PGFObject *)pgf_PGFType.tp_alloc(&pgf_PGFType, 0);
 
     // Read the NGF grammar.
     PgfExn err;
     py_pgf->db = pgf_new_ngf(absname, fpath, &py_pgf->revision, &err);
-    PyMem_Free(absname);
+    PyMem_RawFree(absname);
     if (handleError(err) != PGF_EXN_NONE) {
         Py_DECREF(py_pgf);
         return NULL;
@@ -504,12 +477,10 @@ pgf_readExpr(PyObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "s#", &s, &size))
         return NULL;
 
-    PgfText *input = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+    PgfText *input = CString_AsPgfText(s, size);
-    memcpy(input->text, s, size+1);
-    input->size = size;
 
     PgfExpr expr = pgf_read_expr(input, &unmarshaller);
-    PyMem_Free(input);
+    PyMem_RawFree(input);
     if (expr == 0) {
         PyErr_SetString(PGFError, "expression cannot be parsed");
         return NULL;
@@ -522,29 +493,15 @@ static PyObject *
 pgf_showExpr(PyObject *self, PyObject *args)
 {
     PyObject *pylist;
-    PyObject *pyexpr;
+    ExprObject *expr;
-    if (!PyArg_ParseTuple(args, "O!O!", &PyList_Type, &pylist, &pgf_ExprType, &pyexpr))
+    if (!PyArg_ParseTuple(args, "O!O!", &PyList_Type, &pylist, &pgf_ExprType, &expr))
         return NULL;
 
-    PgfPrintContext *ctxt = NULL;
+    PgfPrintContext *ctxt = PyList_AsPgfPrintContext(pylist);
-    for (Py_ssize_t i = PyList_Size(pylist); i > 0 ; i--) {
+    PgfText *s = pgf_print_expr((PgfExpr) expr, ctxt, 0, &marshaller);
-        PyObject *item = PyList_GetItem(pylist, i-1);
+    FreePgfPrintContext(ctxt);
-        if (!PyUnicode_Check(item)) {
-            PyErr_SetString(PyExc_TypeError, "invalid variable argument in showExpr");
-            return NULL;
-        }
-        PgfText *input = PyUnicode_AsPgfText(item);
-
-        // TODO a better way to copy into this->name?
-        PgfPrintContext *this = (PgfPrintContext *)PyMem_Malloc(sizeof(PgfPrintContext *) + sizeof(PgfText) + input->size + 1);
-        this->next = ctxt;
-        memcpy(&this->name, input, sizeof(PgfText) + input->size + 1);
-        ctxt = this;
-    }
-
-    PgfText *s = pgf_print_expr((PgfExpr) pyexpr, ctxt, 0, &marshaller);
     PyObject *str = PyUnicode_FromStringAndSize(s->text, s->size);
-    free(s);
+    PyMem_RawFree(s);
     return str;
 }
 
@@ -556,12 +513,10 @@ pgf_readType(PyObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "s#", &s, &size))
         return NULL;
 
-    PgfText *input = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+    PgfText *input = CString_AsPgfText(s, size);
-    memcpy(input->text, s, size+1);
-    input->size = size;
 
     PgfType type = pgf_read_type(input, &unmarshaller);
-    PyMem_Free(input);
+    PyMem_RawFree(input);
     if (type == 0) {
         PyErr_SetString(PGFError, "type cannot be parsed");
         return NULL;
@@ -574,29 +529,15 @@ static PyObject *
 pgf_showType(PyObject *self, PyObject *args)
 {
     PyObject *pylist;
-    PyObject *pytype;
+    TypeObject *type;
-    if (!PyArg_ParseTuple(args, "O!O!", &PyList_Type, &pylist, &pgf_TypeType, &pytype))
+    if (!PyArg_ParseTuple(args, "O!O!", &PyList_Type, &pylist, &pgf_TypeType, &type))
         return NULL;
 
-    PgfPrintContext *ctxt = NULL;
+    PgfPrintContext *ctxt = PyList_AsPgfPrintContext(pylist);
-    for (Py_ssize_t i = PyList_Size(pylist); i > 0 ; i--) {
+    PgfText *s = pgf_print_type((PgfType) type, ctxt, 0, &marshaller);
-        PyObject *item = PyList_GetItem(pylist, i-1);
+    FreePgfPrintContext(ctxt);
-        if (!PyUnicode_Check(item)) {
-            PyErr_SetString(PyExc_TypeError, "invalid variable argument in showType");
-            return NULL;
-        }
-        PgfText *input = PyUnicode_AsPgfText(item);
-
-        // TODO a better way to copy into this->name?
-        PgfPrintContext *this = (PgfPrintContext *)PyMem_Malloc(sizeof(PgfPrintContext *) + sizeof(PgfText) + input->size + 1);
-        this->next = ctxt;
-        memcpy(&this->name, input, sizeof(PgfText) + input->size + 1);
-        ctxt = this;
-    }
-
-    PgfText *s = pgf_print_type((PgfType) pytype, ctxt, 0, &marshaller);
     PyObject *str = PyUnicode_FromStringAndSize(s->text, s->size);
-    free(s);
+    PyMem_RawFree(s);
     return str;
 }
 

src/runtime/python/transactions.c

@@ -15,9 +15,7 @@ PGF_checkoutBranch(PGFObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "s#", &s, &size))
         return NULL;
 
-    PgfText *name = (PgfText *)PyMem_RawMalloc(sizeof(PgfText)+size+1);
+    PgfText *name = CString_AsPgfText(s, size);
-    memcpy(name->text, s, size+1);
-    name->size = size;
 
     PgfExn err;
     PgfRevision rev = pgf_checkout_revision(self->db, name, &err);
@@ -46,15 +44,13 @@ PGF_newTransaction(PGFObject *self, PyObject *args)
         return NULL;
 
     if (s != NULL) {
-        name = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+        name = CString_AsPgfText(s, size);
-        memcpy(name->text, s, size+1);
-        name->size = size;
     }
 
     PgfExn err;
     PgfRevision rev = pgf_clone_revision(self->db, self->revision, name, &err);
     if (name != NULL) {
-        PyMem_Free(name);
+        PyMem_RawFree(name);
     }
     if (handleError(err) != PGF_EXN_NONE) {
         return NULL;
@@ -94,13 +90,11 @@ Transaction_createFunction(TransactionObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "s#O!nf", &s, &size, &pgf_TypeType, &type, &arity, &prob))
         return NULL;
 
-    PgfText *fname = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+    PgfText *funname = CString_AsPgfText(s, size);
-    memcpy(fname->text, s, size+1);
-    fname->size = size;
 
     PgfExn err;
-    pgf_create_function(self->pgf->db, self->revision, fname, (PgfType) type, arity, prob, &marshaller, &err);
+    pgf_create_function(self->pgf->db, self->revision, funname, (PgfType) type, arity, prob, &marshaller, &err);
-    PyMem_Free(fname);
+    PyMem_RawFree(funname);
     if (handleError(err) != PGF_EXN_NONE) {
         return NULL;
     }
@@ -116,13 +110,11 @@ Transaction_dropFunction(TransactionObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "s#", &s, &size))
         return NULL;
 
-    PgfText *fname = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+    PgfText *funname = CString_AsPgfText(s, size);
-    memcpy(fname->text, s, size+1);
-    fname->size = size;
 
     PgfExn err;
-    pgf_drop_function(self->pgf->db, self->revision, fname, &err);
+    pgf_drop_function(self->pgf->db, self->revision, funname, &err);
-    PyMem_Free(fname);
+    PyMem_RawFree(funname);
     if (handleError(err) != PGF_EXN_NONE) {
         return NULL;
     }
@@ -145,20 +137,19 @@ Transaction_createCategory(TransactionObject *self, PyObject *args)
         PyErr_SetString(PyExc_TypeError, "hypos must be a list");
         return NULL;
     }
-    // Py_INCREF(hypos);
 
-    PgfText *catname = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+    PgfText *catname = CString_AsPgfText(s, size);
-    memcpy(catname->text, s, size+1);
-    catname->size = size;
 
     Py_ssize_t n_hypos;
     PgfTypeHypo *context = PyList_AsHypos(hypos, &n_hypos);
-    if (PyErr_Occurred())
+    if (PyErr_Occurred()) {
+        PyMem_RawFree(catname);
         return NULL;
+    }
 
     PgfExn err;
     pgf_create_category(self->pgf->db, self->revision, catname, n_hypos, context, prob, &marshaller, &err);
-    PyMem_Free(catname);
+    PyMem_RawFree(catname);
     if (handleError(err) != PGF_EXN_NONE) {
         return NULL;
     }
@@ -174,13 +165,11 @@ Transaction_dropCategory(TransactionObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "s#", &s, &size))
         return NULL;
 
-    PgfText *catname = (PgfText *)PyMem_Malloc(sizeof(PgfText)+size+1);
+    PgfText *catname = CString_AsPgfText(s, size);
-    memcpy(catname->text, s, size+1);
-    catname->size = size;
 
     PgfExn err;
     pgf_drop_category(self->pgf->db, self->revision, catname, &err);
-    PyMem_Free(catname);
+    PyMem_RawFree(catname);
     if (handleError(err) != PGF_EXN_NONE) {
         return NULL;
     }