diff --git a/src/runtime/c/pgf/pgf.cxx b/src/runtime/c/pgf/pgf.cxx index 315b44dc8..8766a3fcb 100644 --- a/src/runtime/c/pgf/pgf.cxx +++ b/src/runtime/c/pgf/pgf.cxx @@ -144,6 +144,80 @@ PgfDB *pgf_boot_ngf(const char* pgf_path, const char* ngf_path, return NULL; } +#if defined(__linux__) || defined(__APPLE__) +#include + +PGF_API +PgfDB *pgf_boot_ngf_cookie(void *cookie, +#if defined(__linux__) + ssize_t (*readfn)(void *, char *, size_t), +#else + int (*readfn)(void *, const char *, int), +#endif + const char* ngf_path, + PgfRevision *revision, + PgfProbsCallback *probs_callback, + PgfExn* err) +{ + PgfDB *db = NULL; + FILE *in = NULL; + + PGF_API_BEGIN { +#if defined(__linux__) + cookie_io_functions_t io_funcs = { + readfn, + NULL, + NULL, + NULL + }; + + FILE *in = fopencookie(cookie, "rb", io_funcs); +#else + FILE *in = fropen(cookie, readfn); +#endif + if (!in) { + throw pgf_systemerror(errno); + } + + db = new PgfDB(ngf_path, O_CREAT | O_EXCL | O_RDWR, +#ifndef _WIN32 + S_IRUSR | S_IWUSR, +#else + _S_IREAD | _S_IWRITE, +#endif + getpagesize()); + + { + DB_scope scope(db, WRITER_SCOPE); + + db->start_transaction(); + + PgfReader rdr(in,probs_callback); + ref pgf = rdr.read_pgf(); + fclose(in); + + db->set_transaction_object(pgf.as_object()); + + *revision = db->register_revision(pgf.tagged(), PgfDB::get_txn_id()); + db->commit(pgf.as_object()); + } + + db->ref_count++; + return db; + } PGF_API_END + + if (in != NULL) + fclose(in); + + if (db != NULL) { + delete db; + remove(ngf_path); + } + + return NULL; +} +#endif + PGF_API PgfDB *pgf_read_ngf(const char *fpath, PgfRevision *revision, diff --git a/src/runtime/c/pgf/pgf.h b/src/runtime/c/pgf/pgf.h index 14b9b5a57..01e6fe843 100644 --- a/src/runtime/c/pgf/pgf.h +++ b/src/runtime/c/pgf/pgf.h @@ -261,6 +261,20 @@ PgfDB *pgf_boot_ngf(const char* pgf_path, const char* ngf_path, PgfProbsCallback *probs_callback, PgfExn* err); +#if defined(__linux__) || defined(__APPLE__) +PGF_API_DECL +PgfDB *pgf_boot_ngf_cookie(void *cookie, +#if defined(__linux__) + ssize_t (*readfn)(void *, char *, size_t), +#else + int (*readfn)(void *, const char *, int), +#endif + const char* ngf_path, + PgfRevision *revision, + PgfProbsCallback *probs_callback, + PgfExn* err); +#endif + /* Tries to read the grammar from an already booted NGF file. * The function fails if the file does not exist. The default grammar * revision is stored in *revision. */ diff --git a/src/runtime/haskell/PGF2.hsc b/src/runtime/haskell/PGF2.hsc index b570eb315..6ffa7819c 100644 --- a/src/runtime/haskell/PGF2.hsc +++ b/src/runtime/haskell/PGF2.hsc @@ -154,6 +154,44 @@ bootNGFWithProbs pgf_path mb_probs ngf_path = langs <- getConcretes c_db fptr return (PGF c_db fptr langs) +#if defined(__linux__) || defined(__APPLE__) +-- | Similar to 'bootPGF' but instead of reading from a file, +-- it calls the given callback each time when more data is needed. +-- This makes it possible to read the file over a socket or +-- an HTTP connection. +bootNGF_ :: (Ptr Word8 -> Int -> IO Int) -> FilePath -> IO PGF +bootNGF_ callback ngf_path = bootNGFWithProbs_ callback Nothing ngf_path + +bootNGFWithProbs_ :: (Ptr Word8 -> Int -> IO Int) -> Maybe (Map.Map String Double) -> FilePath -> IO PGF +bootNGFWithProbs_ callback mb_probs ngf_path = + withCString ngf_path $ \c_ngf_path -> + alloca $ \p_revision -> + withProbsCallback mb_probs $ \c_pcallback -> + bracket (newStablePtr callback) freeStablePtr $ \cookie -> + mask_ $ do + c_db <- withPgfExn "bootNGF" (pgf_boot_ngf_cookie (castStablePtrToPtr cookie) cookie_read_ptr c_ngf_path p_revision c_pcallback) + c_revision <- peek p_revision + fptr <- newForeignPtrEnv pgf_free_revision c_db c_revision + langs <- getConcretes c_db fptr + return (PGF c_db fptr langs) + +#if defined(__linux__) +foreign export ccall cookie_read :: Ptr () -> Ptr Word8 -> CSize -> IO CSize +foreign import ccall "&cookie_read" cookie_read_ptr :: FunPtr (Ptr () -> Ptr Word8 -> CSize -> IO CSize) + +cookie_read :: Ptr () -> Ptr Word8 -> CSize -> IO CSize +#else +foreign export ccall cookie_read :: Ptr () -> Ptr Word8 -> CInt -> IO CInt +foreign import ccall "&cookie_read" cookie_read_ptr :: FunPtr (Ptr () -> Ptr Word8 -> CInt -> IO CInt) + +cookie_read :: Ptr () -> Ptr Word8 -> CInt -> IO CInt +#endif + +cookie_read cookie buf size = do + callback <- deRefStablePtr (castPtrToStablePtr cookie) + fmap fromIntegral $ (callback :: Ptr Word8 -> Int -> IO Int) buf (fromIntegral size) +#endif + withProbsCallback :: Maybe (Map.Map String Double) -> (Ptr PgfProbsCallback -> IO a) -> IO a withProbsCallback Nothing f = f nullPtr withProbsCallback (Just probs) f = @@ -209,6 +247,10 @@ writePGF fpath p mb_langs = withLangs clangs (lang:langs) f = withText lang $ \clang -> withLangs (clang:clangs) langs f #if defined(__linux__) || defined(__APPLE__) +-- | Similar to 'writePGF' but instead of saving to a file, +-- it calls the given callback each time when more data is ready. +-- This makes it possible to send the file over a socket or +-- an HTTP connection. writePGF_ :: (Ptr Word8 -> Int -> IO Int) -> PGF -> Maybe [ConcName] -> IO () writePGF_ callback p mb_langs = withForeignPtr (a_revision p) $ \c_revision -> diff --git a/src/runtime/haskell/PGF2/FFI.hsc b/src/runtime/haskell/PGF2/FFI.hsc index a1b478967..f882e984c 100644 --- a/src/runtime/haskell/PGF2/FFI.hsc +++ b/src/runtime/haskell/PGF2/FFI.hsc @@ -68,6 +68,12 @@ foreign import ccall "pgf_read_pgf" foreign import ccall "pgf_boot_ngf" pgf_boot_ngf :: CString -> CString -> Ptr (Ptr PGF) -> Ptr PgfProbsCallback -> Ptr PgfExn -> IO (Ptr PgfDB) +#if defined(__linux__) +foreign import ccall pgf_boot_ngf_cookie :: Ptr () -> FunPtr (Ptr () -> Ptr Word8 -> CSize -> IO CSize) -> CString -> Ptr (Ptr PGF) -> Ptr PgfProbsCallback -> Ptr PgfExn -> IO (Ptr PgfDB) +#elif defined(__APPLE__) +foreign import ccall pgf_boot_ngf_cookie :: Ptr () -> FunPtr (Ptr () -> Ptr Word8 -> CInt -> IO CInt) -> CString -> Ptr (Ptr PGF) -> Ptr PgfProbsCallback -> Ptr PgfExn -> IO (Ptr PgfDB) +#endif + type ProbsCallback = Ptr PgfProbsCallback -> Ptr PgfText -> IO Double foreign import ccall "wrapper" wrapProbsCallback :: Wrapper ProbsCallback diff --git a/src/runtime/python/pypgf.c b/src/runtime/python/pypgf.c index cd0c75978..e0ecc00b3 100644 --- a/src/runtime/python/pypgf.c +++ b/src/runtime/python/pypgf.c @@ -1616,18 +1616,75 @@ pgf_readPGF(PyObject *self, PyObject *args) return py_pgf; } +#if defined(__linux__) || defined(__APPLE__) +static +#if defined(__linux__) +ssize_t py_readfn(void *cookie, char *mem, size_t size) +#else +int py_readfn(void *cookie, char *mem, int size) +#endif +{ + PyObject *source = (PyObject *) cookie; + + PyObject *mv = + PyMemoryView_FromMemory(mem, (Py_ssize_t) size, PyBUF_WRITE); + + PyObject *res = + PyObject_CallOneArg(source, mv); + + Py_DECREF(mv); + + if (res == NULL) { + PyErr_PrintEx(0); + errno = EINVAL; + return -1; + } + if (!PyLong_Check(res)) { + Py_DECREF(res); + errno = EINVAL; + return -1; + } + + ssize_t n = PyLong_AsSsize_t(res); + Py_DECREF(res); + + return n; +} +#endif + static PGFObject * pgf_bootNGF(PyObject *self, PyObject *args) { - const char *fpath; // pgf +#if defined(__linux__) || defined(__APPLE__) + PyObject *source; + const char *npath; // ngf + if (!PyArg_ParseTuple(args, "Os", &source, &npath)) + return NULL; + + PgfExn err; + PGFObject *py_pgf = (PGFObject *)pgf_PGFType.tp_alloc(&pgf_PGFType, 0); + + if (PyUnicode_Check(source)) { + const char *fpath = PyUnicode_AsUTF8(source); + py_pgf->db = pgf_boot_ngf(fpath, npath, &py_pgf->revision, NULL, &err); + } else if (PyCallable_Check(source)) { + py_pgf->db = pgf_boot_ngf_cookie(source, py_readfn, npath, &py_pgf->revision, NULL, &err); + } else { + Py_DECREF(py_pgf); + PyErr_SetString(PyExc_TypeError, "The first argument must be a string or a callable function"); + return NULL; + } +#else + const char *fpath // pgf const char *npath; // ngf if (!PyArg_ParseTuple(args, "ss", &fpath, &npath)) return NULL; - PGFObject *py_pgf = (PGFObject *)pgf_PGFType.tp_alloc(&pgf_PGFType, 0); - PgfExn err; + PGFObject *py_pgf = (PGFObject *)pgf_PGFType.tp_alloc(&pgf_PGFType, 0); py_pgf->db = pgf_boot_ngf(fpath, npath, &py_pgf->revision, NULL, &err); +#endif + if (handleError(err) != PGF_EXN_NONE) { Py_DECREF(py_pgf); return NULL;