make it possible to bootstrap an .ngf from a cookie

src/runtime/c/pgf/pgf.cxx

@@ -144,6 +144,80 @@ PgfDB *pgf_boot_ngf(const char* pgf_path, const char* ngf_path,
     return NULL;
 }
 
+#if defined(__linux__) || defined(__APPLE__)
+#include <unistd.h>
+
+PGF_API
+PgfDB *pgf_boot_ngf_cookie(void *cookie,
+#if defined(__linux__)
+                           ssize_t (*readfn)(void *, char *, size_t),
+#else
+                           int (*readfn)(void *, const char *, int),
+#endif
+                           const char* ngf_path,
+                           PgfRevision *revision,
+                           PgfProbsCallback *probs_callback,
+                           PgfExn* err)
+{
+    PgfDB *db = NULL;
+    FILE *in = NULL;
+
+    PGF_API_BEGIN {
+#if defined(__linux__)
+        cookie_io_functions_t io_funcs = {
+            readfn,
+            NULL,
+            NULL,
+            NULL
+        };
+
+        FILE *in = fopencookie(cookie, "rb", io_funcs);
+#else
+        FILE *in = fropen(cookie, readfn);
+#endif
+        if (!in) {
+            throw pgf_systemerror(errno);
+        }
+
+        db = new PgfDB(ngf_path, O_CREAT | O_EXCL | O_RDWR,
+#ifndef _WIN32
+                       S_IRUSR | S_IWUSR,
+#else
+                       _S_IREAD | _S_IWRITE,
+#endif
+                       getpagesize());
+
+        {
+            DB_scope scope(db, WRITER_SCOPE);
+
+            db->start_transaction();
+
+            PgfReader rdr(in,probs_callback);
+            ref<PgfPGF> pgf = rdr.read_pgf();
+            fclose(in);
+
+            db->set_transaction_object(pgf.as_object());
+
+            *revision = db->register_revision(pgf.tagged(), PgfDB::get_txn_id());
+            db->commit(pgf.as_object());
+        }
+
+        db->ref_count++;
+        return db;
+    } PGF_API_END
+
+    if (in != NULL)
+        fclose(in);
+
+    if (db != NULL) {
+        delete db;
+        remove(ngf_path);
+    }
+
+    return NULL;
+}
+#endif
+
 PGF_API
 PgfDB *pgf_read_ngf(const char *fpath,
                     PgfRevision *revision,

src/runtime/c/pgf/pgf.h

@@ -261,6 +261,20 @@ PgfDB *pgf_boot_ngf(const char* pgf_path, const char* ngf_path,
                     PgfProbsCallback *probs_callback,
                     PgfExn* err);
 
+#if defined(__linux__) || defined(__APPLE__)
+PGF_API_DECL
+PgfDB *pgf_boot_ngf_cookie(void *cookie,
+#if defined(__linux__)
+                           ssize_t (*readfn)(void *, char *, size_t),
+#else
+                           int (*readfn)(void *, const char *, int),
+#endif
+                           const char* ngf_path,
+                           PgfRevision *revision,
+                           PgfProbsCallback *probs_callback,
+                           PgfExn* err);
+#endif
+
 /* Tries to read the grammar from an already booted NGF file.
  * The function fails if the file does not exist. The default grammar
  * revision is stored in *revision. */

src/runtime/haskell/PGF2.hsc

@@ -154,6 +154,44 @@ bootNGFWithProbs pgf_path mb_probs ngf_path =
     langs <- getConcretes c_db fptr
     return (PGF c_db fptr langs)
 
+#if defined(__linux__) || defined(__APPLE__)
+-- | Similar to 'bootPGF' but instead of reading from a file,
+-- it calls the given callback each time when more data is needed.
+-- This makes it possible to read the file over a socket or
+-- an HTTP connection.
+bootNGF_ :: (Ptr Word8 -> Int -> IO Int) -> FilePath -> IO PGF
+bootNGF_ callback ngf_path = bootNGFWithProbs_ callback Nothing ngf_path
+
+bootNGFWithProbs_ :: (Ptr Word8 -> Int -> IO Int) -> Maybe (Map.Map String Double) -> FilePath -> IO PGF
+bootNGFWithProbs_ callback mb_probs ngf_path =
+  withCString ngf_path $ \c_ngf_path ->
+  alloca $ \p_revision ->
+  withProbsCallback mb_probs $ \c_pcallback ->
+  bracket (newStablePtr callback) freeStablePtr $ \cookie ->
+  mask_ $ do
+    c_db <- withPgfExn "bootNGF" (pgf_boot_ngf_cookie (castStablePtrToPtr cookie) cookie_read_ptr c_ngf_path p_revision c_pcallback)
+    c_revision <- peek p_revision
+    fptr <- newForeignPtrEnv pgf_free_revision c_db c_revision
+    langs <- getConcretes c_db fptr
+    return (PGF c_db fptr langs)
+
+#if defined(__linux__)
+foreign export ccall cookie_read :: Ptr () -> Ptr Word8 -> CSize -> IO CSize
+foreign import ccall "&cookie_read" cookie_read_ptr :: FunPtr (Ptr () -> Ptr Word8 -> CSize -> IO CSize)
+
+cookie_read :: Ptr () -> Ptr Word8 -> CSize -> IO CSize
+#else
+foreign export ccall cookie_read :: Ptr () -> Ptr Word8 -> CInt -> IO CInt
+foreign import ccall "&cookie_read" cookie_read_ptr :: FunPtr (Ptr () -> Ptr Word8 -> CInt -> IO CInt)
+
+cookie_read :: Ptr () -> Ptr Word8 -> CInt -> IO CInt
+#endif
+
+cookie_read cookie buf size = do
+  callback <- deRefStablePtr (castPtrToStablePtr cookie)
+  fmap fromIntegral $ (callback :: Ptr Word8 -> Int -> IO Int) buf (fromIntegral size)
+#endif
+
 withProbsCallback :: Maybe (Map.Map String Double) -> (Ptr PgfProbsCallback -> IO a) -> IO a
 withProbsCallback Nothing      f = f nullPtr
 withProbsCallback (Just probs) f =
@@ -209,6 +247,10 @@ writePGF fpath p mb_langs =
     withLangs clangs (lang:langs) f = withText lang $ \clang -> withLangs (clang:clangs) langs f
 
 #if defined(__linux__) || defined(__APPLE__)
+-- | Similar to 'writePGF' but instead of saving to a file,
+-- it calls the given callback each time when more data is ready.
+-- This makes it possible to send the file over a socket or
+-- an HTTP connection.
 writePGF_ :: (Ptr Word8 -> Int -> IO Int) -> PGF -> Maybe [ConcName] -> IO ()
 writePGF_ callback p mb_langs =
   withForeignPtr (a_revision p) $ \c_revision ->

src/runtime/haskell/PGF2/FFI.hsc

@@ -68,6 +68,12 @@ foreign import ccall "pgf_read_pgf"
 foreign import ccall "pgf_boot_ngf"
   pgf_boot_ngf :: CString -> CString -> Ptr (Ptr PGF) -> Ptr PgfProbsCallback -> Ptr PgfExn -> IO (Ptr PgfDB)
 
+#if defined(__linux__)
+foreign import ccall pgf_boot_ngf_cookie :: Ptr () -> FunPtr (Ptr () -> Ptr Word8 -> CSize -> IO CSize) -> CString -> Ptr (Ptr PGF) -> Ptr PgfProbsCallback -> Ptr PgfExn -> IO (Ptr PgfDB)
+#elif defined(__APPLE__)
+foreign import ccall pgf_boot_ngf_cookie :: Ptr () -> FunPtr (Ptr () -> Ptr Word8 -> CInt -> IO CInt) -> CString -> Ptr (Ptr PGF) -> Ptr PgfProbsCallback -> Ptr PgfExn -> IO (Ptr PgfDB)
+#endif
+
 type ProbsCallback = Ptr PgfProbsCallback -> Ptr PgfText -> IO Double
 
 foreign import ccall "wrapper" wrapProbsCallback :: Wrapper ProbsCallback

src/runtime/python/pypgf.c

@@ -1616,18 +1616,75 @@ pgf_readPGF(PyObject *self, PyObject *args)
     return py_pgf;
 }
 
+#if defined(__linux__) || defined(__APPLE__)
+static
+#if defined(__linux__)
+ssize_t py_readfn(void *cookie, char *mem, size_t size)
+#else
+int py_readfn(void *cookie, char *mem, int size)
+#endif
+{
+    PyObject *source = (PyObject *) cookie;
+
+    PyObject *mv =
+        PyMemoryView_FromMemory(mem, (Py_ssize_t) size, PyBUF_WRITE);
+
+    PyObject *res =
+        PyObject_CallOneArg(source, mv);
+
+    Py_DECREF(mv);
+
+    if (res == NULL) {
+        PyErr_PrintEx(0);
+        errno = EINVAL;
+        return -1;
+    }
+    if (!PyLong_Check(res)) {
+        Py_DECREF(res);
+        errno = EINVAL;
+        return -1;
+    }
+
+    ssize_t n = PyLong_AsSsize_t(res);
+    Py_DECREF(res);
+
+    return n;
+}
+#endif
+
 static PGFObject *
 pgf_bootNGF(PyObject *self, PyObject *args)
 {
-    const char *fpath; // pgf
+#if defined(__linux__) || defined(__APPLE__)
+    PyObject *source;
+    const char *npath; // ngf
+    if (!PyArg_ParseTuple(args, "Os", &source, &npath))
+        return NULL;
+
+    PgfExn err;
+    PGFObject *py_pgf = (PGFObject *)pgf_PGFType.tp_alloc(&pgf_PGFType, 0);
+
+    if (PyUnicode_Check(source)) {        
+        const char *fpath = PyUnicode_AsUTF8(source);
+        py_pgf->db = pgf_boot_ngf(fpath, npath, &py_pgf->revision, NULL, &err);
+    } else if (PyCallable_Check(source)) {
+        py_pgf->db = pgf_boot_ngf_cookie(source, py_readfn, npath, &py_pgf->revision, NULL, &err);
+    } else {
+        Py_DECREF(py_pgf);
+        PyErr_SetString(PyExc_TypeError, "The first argument must be a string or a callable function");
+        return NULL;
+    }
+#else
+    const char *fpath  // pgf
     const char *npath; // ngf
     if (!PyArg_ParseTuple(args, "ss", &fpath, &npath))
         return NULL;
 
-    PGFObject *py_pgf = (PGFObject *)pgf_PGFType.tp_alloc(&pgf_PGFType, 0);
-
     PgfExn err;
+    PGFObject *py_pgf = (PGFObject *)pgf_PGFType.tp_alloc(&pgf_PGFType, 0);
     py_pgf->db = pgf_boot_ngf(fpath, npath, &py_pgf->revision, NULL, &err);
+#endif
+
     if (handleError(err) != PGF_EXN_NONE) {
         Py_DECREF(py_pgf);
         return NULL;