msyds/gf-core
1
0
Fork 0
forked from GitHub/gf-core
Code Pull Requests Activity

make it possible to bootstrap an .ngf from a cookie

Browse Source
This commit is contained in:
Krasimir Angelov
2023-09-07 10:07:49 +02:00
parent d110140107
commit f1af1c8be4
5 changed files with 196 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
src/runtime/c/pgf/pgf.cxx
View File

@@ -144,6 +144,80 @@ PgfDB *pgf_boot_ngf(const char* pgf_path, const char* ngf_path,
return NULL;
}
#if defined(__linux__) || defined(__APPLE__)
#include <unistd.h>
PGF_API
PgfDB *pgf_boot_ngf_cookie(void *cookie,
#if defined(__linux__)
ssize_t (*readfn)(void *, char *, size_t),
#else
int (*readfn)(void *, const char *, int),
#endif
const char* ngf_path,
PgfRevision *revision,
PgfProbsCallback *probs_callback,
PgfExn* err)
{
PgfDB *db = NULL;
FILE *in = NULL;
PGF_API_BEGIN {
#if defined(__linux__)
cookie_io_functions_t io_funcs = {
readfn,
NULL,
NULL,
NULL
};
FILE *in = fopencookie(cookie, "rb", io_funcs);
#else
FILE *in = fropen(cookie, readfn);
#endif
if (!in) {
throw pgf_systemerror(errno);
}
db = new PgfDB(ngf_path, O_CREAT | O_EXCL | O_RDWR,
#ifndef _WIN32
S_IRUSR | S_IWUSR,
#else
_S_IREAD | _S_IWRITE,
#endif
getpagesize());
{
DB_scope scope(db, WRITER_SCOPE);
db->start_transaction();
PgfReader rdr(in,probs_callback);
ref<PgfPGF> pgf = rdr.read_pgf();
fclose(in);
db->set_transaction_object(pgf.as_object());
*revision = db->register_revision(pgf.tagged(), PgfDB::get_txn_id());
db->commit(pgf.as_object());
}
db->ref_count++;
return db;
} PGF_API_END
if (in != NULL)
fclose(in);
if (db != NULL) {
delete db;
remove(ngf_path);
}
return NULL;
}
#endif
PGF_API
PgfDB *pgf_read_ngf(const char *fpath,
PgfRevision *revision,

14
src/runtime/c/pgf/pgf.h
View File

@@ -261,6 +261,20 @@ PgfDB *pgf_boot_ngf(const char* pgf_path, const char* ngf_path,
PgfProbsCallback *probs_callback,
PgfExn* err);
#if defined(__linux__) || defined(__APPLE__)
PGF_API_DECL
PgfDB *pgf_boot_ngf_cookie(void *cookie,
#if defined(__linux__)
ssize_t (*readfn)(void *, char *, size_t),
#else
int (*readfn)(void *, const char *, int),
#endif
const char* ngf_path,
PgfRevision *revision,
PgfProbsCallback *probs_callback,
PgfExn* err);
#endif
/* Tries to read the grammar from an already booted NGF file.
* The function fails if the file does not exist. The default grammar
* revision is stored in *revision. */

42
src/runtime/haskell/PGF2.hsc
View File

@@ -154,6 +154,44 @@ bootNGFWithProbs pgf_path mb_probs ngf_path =
langs <- getConcretes c_db fptr
return (PGF c_db fptr langs)
#if defined(__linux__) || defined(__APPLE__)
-- | Similar to 'bootPGF' but instead of reading from a file,
-- it calls the given callback each time when more data is needed.
-- This makes it possible to read the file over a socket or
-- an HTTP connection.
bootNGF_ :: (Ptr Word8 -> Int -> IO Int) -> FilePath -> IO PGF
bootNGF_ callback ngf_path = bootNGFWithProbs_ callback Nothing ngf_path
bootNGFWithProbs_ :: (Ptr Word8 -> Int -> IO Int) -> Maybe (Map.Map String Double) -> FilePath -> IO PGF
bootNGFWithProbs_ callback mb_probs ngf_path =
withCString ngf_path $ \c_ngf_path ->
alloca $ \p_revision ->
withProbsCallback mb_probs $ \c_pcallback ->
bracket (newStablePtr callback) freeStablePtr $ \cookie ->
mask_ $ do
c_db <- withPgfExn "bootNGF" (pgf_boot_ngf_cookie (castStablePtrToPtr cookie) cookie_read_ptr c_ngf_path p_revision c_pcallback)
c_revision <- peek p_revision
fptr <- newForeignPtrEnv pgf_free_revision c_db c_revision
langs <- getConcretes c_db fptr
return (PGF c_db fptr langs)
#if defined(__linux__)
foreign export ccall cookie_read :: Ptr () -> Ptr Word8 -> CSize -> IO CSize
foreign import ccall "&cookie_read" cookie_read_ptr :: FunPtr (Ptr () -> Ptr Word8 -> CSize -> IO CSize)
cookie_read :: Ptr () -> Ptr Word8 -> CSize -> IO CSize
#else
foreign export ccall cookie_read :: Ptr () -> Ptr Word8 -> CInt -> IO CInt
foreign import ccall "&cookie_read" cookie_read_ptr :: FunPtr (Ptr () -> Ptr Word8 -> CInt -> IO CInt)
cookie_read :: Ptr () -> Ptr Word8 -> CInt -> IO CInt
#endif
cookie_read cookie buf size = do
callback <- deRefStablePtr (castPtrToStablePtr cookie)
fmap fromIntegral $ (callback :: Ptr Word8 -> Int -> IO Int) buf (fromIntegral size)
#endif
withProbsCallback :: Maybe (Map.Map String Double) -> (Ptr PgfProbsCallback -> IO a) -> IO a
withProbsCallback Nothing f = f nullPtr
withProbsCallback (Just probs) f =
@@ -209,6 +247,10 @@ writePGF fpath p mb_langs =
withLangs clangs (lang:langs) f = withText lang $ \clang -> withLangs (clang:clangs) langs f
#if defined(__linux__) || defined(__APPLE__)
-- | Similar to 'writePGF' but instead of saving to a file,
-- it calls the given callback each time when more data is ready.
-- This makes it possible to send the file over a socket or
-- an HTTP connection.
writePGF_ :: (Ptr Word8 -> Int -> IO Int) -> PGF -> Maybe [ConcName] -> IO ()
writePGF_ callback p mb_langs =
withForeignPtr (a_revision p) $ \c_revision ->

6
src/runtime/haskell/PGF2/FFI.hsc
View File

@@ -68,6 +68,12 @@ foreign import ccall "pgf_read_pgf"
foreign import ccall "pgf_boot_ngf"
pgf_boot_ngf :: CString -> CString -> Ptr (Ptr PGF) -> Ptr PgfProbsCallback -> Ptr PgfExn -> IO (Ptr PgfDB)
#if defined(__linux__)
foreign import ccall pgf_boot_ngf_cookie :: Ptr () -> FunPtr (Ptr () -> Ptr Word8 -> CSize -> IO CSize) -> CString -> Ptr (Ptr PGF) -> Ptr PgfProbsCallback -> Ptr PgfExn -> IO (Ptr PgfDB)
#elif defined(__APPLE__)
foreign import ccall pgf_boot_ngf_cookie :: Ptr () -> FunPtr (Ptr () -> Ptr Word8 -> CInt -> IO CInt) -> CString -> Ptr (Ptr PGF) -> Ptr PgfProbsCallback -> Ptr PgfExn -> IO (Ptr PgfDB)
#endif
type ProbsCallback = Ptr PgfProbsCallback -> Ptr PgfText -> IO Double
foreign import ccall "wrapper" wrapProbsCallback :: Wrapper ProbsCallback

63
src/runtime/python/pypgf.c
View File

@@ -1616,18 +1616,75 @@ pgf_readPGF(PyObject *self, PyObject *args)
return py_pgf;
}
#if defined(__linux__) || defined(__APPLE__)
static
#if defined(__linux__)
ssize_t py_readfn(void *cookie, char *mem, size_t size)
#else
int py_readfn(void *cookie, char *mem, int size)
#endif
{
PyObject *source = (PyObject *) cookie;
PyObject *mv =
PyMemoryView_FromMemory(mem, (Py_ssize_t) size, PyBUF_WRITE);
PyObject *res =
PyObject_CallOneArg(source, mv);
Py_DECREF(mv);
if (res == NULL) {
PyErr_PrintEx(0);
errno = EINVAL;
return -1;
}
if (!PyLong_Check(res)) {
Py_DECREF(res);
errno = EINVAL;
return -1;
}
ssize_t n = PyLong_AsSsize_t(res);
Py_DECREF(res);
return n;
}
#endif
static PGFObject *
pgf_bootNGF(PyObject *self, PyObject *args)
{
const char *fpath; // pgf
#if defined(__linux__) || defined(__APPLE__)
PyObject *source;
const char *npath; // ngf
if (!PyArg_ParseTuple(args, "Os", &source, &npath))
return NULL;
PgfExn err;
PGFObject *py_pgf = (PGFObject *)pgf_PGFType.tp_alloc(&pgf_PGFType, 0);
if (PyUnicode_Check(source)) {
const char *fpath = PyUnicode_AsUTF8(source);
py_pgf->db = pgf_boot_ngf(fpath, npath, &py_pgf->revision, NULL, &err);
} else if (PyCallable_Check(source)) {
py_pgf->db = pgf_boot_ngf_cookie(source, py_readfn, npath, &py_pgf->revision, NULL, &err);
} else {
Py_DECREF(py_pgf);
PyErr_SetString(PyExc_TypeError, "The first argument must be a string or a callable function");
return NULL;
}
#else
const char *fpath // pgf
const char *npath; // ngf
if (!PyArg_ParseTuple(args, "ss", &fpath, &npath))
return NULL;
PGFObject *py_pgf = (PGFObject *)pgf_PGFType.tp_alloc(&pgf_PGFType, 0);
PgfExn err;
PGFObject *py_pgf = (PGFObject *)pgf_PGFType.tp_alloc(&pgf_PGFType, 0);
py_pgf->db = pgf_boot_ngf(fpath, npath, &py_pgf->revision, NULL, &err);
#endif
if (handleError(err) != PGF_EXN_NONE) {
Py_DECREF(py_pgf);
return NULL;