From 9f2a3de7a3cc0db2db741b7f9766681a95e729ff Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Mon, 8 Nov 2021 12:30:21 +0100 Subject: [PATCH 1/3] Add simpler VSCode extension to editor modes page --- doc/gf-editor-modes.t2t | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/gf-editor-modes.t2t b/doc/gf-editor-modes.t2t index 4c4a62ab6..2e92f31c3 100644 --- a/doc/gf-editor-modes.t2t +++ b/doc/gf-editor-modes.t2t @@ -17,9 +17,10 @@ instructions inside. ==Visual Studio Code== -[Grammatical Framework Language Server https://marketplace.visualstudio.com/items?itemName=anka-213.gf-vscode] by Andreas Källberg. - -This provides syntax highlighting and a client for the Grammatical Framework language server. Follow the installation instructions in the link. +- [Grammatical Framework Language Server https://marketplace.visualstudio.com/items?itemName=anka-213.gf-vscode] by Andreas Källberg. + This provides syntax highlighting and a client for the Grammatical Framework language server. Follow the installation instructions in the link. +- [Grammatical Framework https://marketplace.visualstudio.com/items?itemName=GrammaticalFramework.gf-vscode] is a simpler extension + without any external dependencies which provides only syntax highlighting. ==Eclipse== From ae460e76b6498a76963924a190bc7f5e652f2385 Mon Sep 17 00:00:00 2001 From: Krasimir Angelov Date: Mon, 4 Jul 2022 10:42:34 +0200 Subject: [PATCH 2/3] allow compilation with emscripten --- src/runtime/c/pgf/jit.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/runtime/c/pgf/jit.c b/src/runtime/c/pgf/jit.c index 6c8679523..0d5fa9dc6 100644 --- a/src/runtime/c/pgf/jit.c +++ b/src/runtime/c/pgf/jit.c @@ -8,6 +8,32 @@ //#define PGF_JIT_DEBUG +#ifdef EMSCRIPTEN + +PGF_INTERNAL PgfJitState* +pgf_new_jit(PgfReader* rdr) +{ + return NULL; +} + +PGF_INTERNAL PgfEvalGates* +pgf_jit_gates(PgfReader* rdr) +{ + return NULL; +} + +PGF_INTERNAL void +pgf_jit_predicate(PgfReader* rdr, PgfAbstr* abstr, + PgfAbsCat* abscat) +{ +} + +PGF_INTERNAL void +pgf_jit_done(PgfReader* rdr, PgfAbstr* abstr) +{ +} + +#else struct PgfJitState { jit_state jit; @@ -1329,3 +1355,5 @@ pgf_jit_done(PgfReader* rdr, PgfAbstr* abstr) jit_flush_code(rdr->jit_state->buf, jit_get_ip().ptr); } + +#endif From a42cec21071de72806bedd2fac985fe2a1356db3 Mon Sep 17 00:00:00 2001 From: Krasimir Angelov Date: Sat, 16 Jul 2022 20:29:36 +0200 Subject: [PATCH 3/3] support for BIND tokens in the Python bindings --- src/runtime/c/pgf/parser.c | 44 +++++--- src/runtime/c/pgf/pgf.h | 3 +- src/runtime/haskell-bind/PGF2.hsc | 5 +- src/runtime/python/pypgf.c | 171 +++++++++++++++++++----------- 4 files changed, 143 insertions(+), 80 deletions(-) diff --git a/src/runtime/c/pgf/parser.c b/src/runtime/c/pgf/parser.c index d558908ab..1e47c10a3 100644 --- a/src/runtime/c/pgf/parser.c +++ b/src/runtime/c/pgf/parser.c @@ -44,6 +44,7 @@ typedef struct { PgfParseState *before; PgfParseState *after; PgfToken prefix; + bool prefix_bind; PgfTokenProb* tp; PgfExprEnum en; // enumeration for the generated trees/tokens #ifdef PGF_COUNTS_DEBUG @@ -1009,6 +1010,7 @@ pgf_new_parse_state(PgfParsing* ps, size_t start_offset, (start_offset == end_offset); state->start_offset = start_offset; state->end_offset = end_offset; + state->viterbi_prob = viterbi_prob; state->lexicon_idx = gu_new_buf(PgfLexiconIdxEntry, ps->pool); @@ -1381,20 +1383,30 @@ pgf_parsing_symbol(PgfParsing* ps, PgfItem* item, PgfSymbol sym) break; } case PGF_SYMBOL_BIND: { - if (ps->before->start_offset == ps->before->end_offset && - ps->before->needs_bind) { - PgfParseState* state = - pgf_new_parse_state(ps, ps->before->end_offset, BIND_HARD, - item->inside_prob+item->conts->outside_prob); - if (state != NULL) { - pgf_item_advance(item, ps->pool); - gu_buf_heap_push(state->agenda, pgf_item_prob_order, &item); - } else { - pgf_item_free(ps, item); - } - } else { - pgf_item_free(ps, item); - } + if (!ps->prefix_bind && ps->prefix != NULL && *(ps->sentence + ps->before->end_offset) == 0) { + PgfProductionApply* papp = gu_variant_data(item->prod); + + ps->tp = gu_new(PgfTokenProb, ps->out_pool); + ps->tp->tok = NULL; + ps->tp->cat = item->conts->ccat->cnccat->abscat->name; + ps->tp->fun = papp->fun->absfun->name; + ps->tp->prob = item->inside_prob + item->conts->outside_prob; + } else { + if (ps->before->start_offset == ps->before->end_offset && + ps->before->needs_bind) { + PgfParseState* state = + pgf_new_parse_state(ps, ps->before->end_offset, BIND_HARD, + item->inside_prob+item->conts->outside_prob); + if (state != NULL) { + pgf_item_advance(item, ps->pool); + gu_buf_heap_push(state->agenda, pgf_item_prob_order, &item); + } else { + pgf_item_free(ps, item); + } + } else { + pgf_item_free(ps, item); + } + } break; } case PGF_SYMBOL_SOFT_BIND: @@ -2337,7 +2349,8 @@ pgf_parser_completions_next(GuEnum* self, void* to, GuPool* pool) PGF_API GuEnum* pgf_complete(PgfConcr* concr, PgfType* type, GuString sentence, - GuString prefix, GuExn *err, GuPool* pool) + GuString prefix, bool prefix_bind, + GuExn *err, GuPool* pool) { if (concr->sequences == NULL || concr->cnccats == NULL) { @@ -2377,6 +2390,7 @@ pgf_complete(PgfConcr* concr, PgfType* type, GuString sentence, // Now begin enumerating the completions ps->en.next = pgf_parser_completions_next; ps->prefix = prefix; + ps->prefix_bind = prefix_bind; ps->tp = NULL; return &ps->en; } diff --git a/src/runtime/c/pgf/pgf.h b/src/runtime/c/pgf/pgf.h index 6ff269e00..52aa8420b 100644 --- a/src/runtime/c/pgf/pgf.h +++ b/src/runtime/c/pgf/pgf.h @@ -251,7 +251,8 @@ typedef struct { PGF_API_DECL GuEnum* pgf_complete(PgfConcr* concr, PgfType* type, GuString string, - GuString prefix, GuExn* err, GuPool* pool); + GuString prefix, bool prefix_bind, + GuExn* err, GuPool* pool); typedef struct PgfLiteralCallback PgfLiteralCallback; diff --git a/src/runtime/haskell-bind/PGF2.hsc b/src/runtime/haskell-bind/PGF2.hsc index 38fae67ef..35db9b9dc 100644 --- a/src/runtime/haskell-bind/PGF2.hsc +++ b/src/runtime/haskell-bind/PGF2.hsc @@ -1026,7 +1026,10 @@ complete lang (Type ctype _) sent pfx = touchConcr lang return [] else do - tok <- peekUtf8CString =<< (#peek PgfTokenProb, tok) cmpEntry + p_tok <- (#peek PgfTokenProb, tok) cmpEntry + tok <- if p_tok == nullPtr + then return "&+" + else peekUtf8CString p_tok cat <- peekUtf8CString =<< (#peek PgfTokenProb, cat) cmpEntry fun <- peekUtf8CString =<< (#peek PgfTokenProb, fun) cmpEntry prob <- (#peek PgfTokenProb, prob) cmpEntry diff --git a/src/runtime/python/pypgf.c b/src/runtime/python/pypgf.c index eebaa2781..941986593 100644 --- a/src/runtime/python/pypgf.c +++ b/src/runtime/python/pypgf.c @@ -1155,6 +1155,80 @@ Iter_fetch_expr(IterObject* self) return res; } +typedef struct { + PyObject_HEAD +} BINDObject; + +static PyObject *BIND_instance = NULL; + +static void +BIND_dealloc(PyTypeObject *self) +{ + BIND_instance = NULL; +} + +static PyObject * +BIND_repr(BINDObject *self) +{ + return PyString_FromString("pgf.BIND"); +} + +static PyObject * +BIND_str(BINDObject *self) +{ + return PyString_FromString("&+"); +} + +static PyObject * +BIND_alloc(PyTypeObject *self, Py_ssize_t nitems) +{ + if (BIND_instance == NULL) + BIND_instance = PyType_GenericAlloc(self, nitems); + return BIND_instance; +} + +static PyTypeObject pgf_BINDType = { + PyVarObject_HEAD_INIT(NULL, 0) + //0, /*ob_size*/ + "pgf.BINDType", /*tp_name*/ + sizeof(BINDObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor) BIND_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + (reprfunc) BIND_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + (reprfunc) BIND_str, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "a marker for BIND in a bracketed string", /*tp_doc*/ + 0, /*tp_traverse */ + 0, /*tp_clear */ + 0, /*tp_richcompare */ + 0, /*tp_weaklistoffset */ + 0, /*tp_iter */ + 0, /*tp_iternext */ + 0, /*tp_methods */ + 0, /*tp_members */ + 0, /*tp_getset */ + 0, /*tp_base */ + 0, /*tp_dict */ + 0, /*tp_descr_get */ + 0, /*tp_descr_set */ + 0, /*tp_dictoffset */ + 0, /*tp_init */ + BIND_alloc, /*tp_alloc */ + 0, /*tp_new */ +}; + static PyObject* Iter_fetch_token(IterObject* self) { @@ -1162,7 +1236,9 @@ Iter_fetch_token(IterObject* self) if (tp == NULL) return NULL; - PyObject* py_tok = PyString_FromString(tp->tok); + PyObject* py_tok = + (tp->tok != NULL) ? PyString_FromString(tp->tok) + : pgf_BINDType.tp_alloc(&pgf_BINDType, 0); PyObject* py_cat = PyString_FromString(tp->cat); PyObject* py_fun = PyString_FromString(tp->fun); PyObject* res = Py_BuildValue("(f,O,O,O)", tp->prob, py_tok, py_cat, py_fun); @@ -1599,16 +1675,18 @@ Concr_parse(ConcrObject* self, PyObject *args, PyObject *keywds) static IterObject* Concr_complete(ConcrObject* self, PyObject *args, PyObject *keywds) { - static char *kwlist[] = {"sentence", "cat", "prefix", "n", NULL}; + static char *kwlist[] = {"sentence", "cat", "prefix", "n", NULL}; - const char *sentence = NULL; + PyObject* sentence0 = NULL; + char* sentence = NULL; PyObject* start = NULL; - GuString prefix = ""; - int max_count = -1; - if (!PyArg_ParseTupleAndKeywords(args, keywds, "s|Osi", kwlist, - &sentence, &start, - &prefix, &max_count)) - return NULL; + GuString prefix = ""; + bool prefix_bind = false; + int max_count = -1; + if (!PyArg_ParseTupleAndKeywords(args, keywds, "O|Osi", kwlist, + &sentence0, &start, + &prefix, &max_count)) + return NULL; IterObject* pyres = (IterObject*) pgf_IterType.tp_alloc(&pgf_IterType, 0); @@ -1630,6 +1708,20 @@ Concr_complete(ConcrObject* self, PyObject *args, PyObject *keywds) GuExn* parse_err = gu_new_exn(tmp_pool); + if (PyTuple_Check(sentence0) && + PyTuple_GET_SIZE(sentence0) == 2 && + PyTuple_GET_ITEM(sentence0,1) == pgf_BINDType.tp_alloc(&pgf_BINDType, 0)) + { + sentence0 = PyTuple_GET_ITEM(sentence0,0); + prefix_bind = true; + } + + if (PyUnicode_Check(sentence0)) { + sentence = PyUnicode_AsUTF8(sentence0); + } else { + PyErr_SetString(PyExc_TypeError, "The sentence must be either a string or a tuple of string and pgf.BIND"); + } + PgfType* type; if (start == NULL) { type = pgf_start_cat(self->grammar->pgf, pyres->pool); @@ -1642,7 +1734,7 @@ Concr_complete(ConcrObject* self, PyObject *args, PyObject *keywds) } pyres->res = - pgf_complete(self->concr, type, sentence, prefix, parse_err, pyres->pool); + pgf_complete(self->concr, type, sentence, prefix, prefix_bind, parse_err, pyres->pool); if (!gu_ok(parse_err)) { Py_DECREF(pyres); @@ -2077,58 +2169,6 @@ static PyTypeObject pgf_BracketType = { 0, /*tp_new */ }; -typedef struct { - PyObject_HEAD -} BINDObject; - -static PyObject * -BIND_repr(BINDObject *self) -{ - return PyString_FromString("&+"); -} - -static PyTypeObject pgf_BINDType = { - PyVarObject_HEAD_INIT(NULL, 0) - //0, /*ob_size*/ - "pgf.BIND", /*tp_name*/ - sizeof(BINDObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - 0, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - (reprfunc) BIND_repr, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - "a marker for BIND in a bracketed string", /*tp_doc*/ - 0, /*tp_traverse */ - 0, /*tp_clear */ - 0, /*tp_richcompare */ - 0, /*tp_weaklistoffset */ - 0, /*tp_iter */ - 0, /*tp_iternext */ - 0, /*tp_methods */ - 0, /*tp_members */ - 0, /*tp_getset */ - 0, /*tp_base */ - 0, /*tp_dict */ - 0, /*tp_descr_get */ - 0, /*tp_descr_set */ - 0, /*tp_dictoffset */ - 0, /*tp_init */ - 0, /*tp_alloc */ - 0, /*tp_new */ -}; - typedef struct { PgfLinFuncs* funcs; GuBuf* stack; @@ -2726,6 +2766,11 @@ static PyMethodDef Concr_methods[] = { }, {"complete", (PyCFunction)Concr_complete, METH_VARARGS | METH_KEYWORDS, "Parses a partial string and returns a list with the top n possible next tokens" + "Named arguments:\n" + "- sentence (string or a (string,pgf.BIND) tuple. The later indicates that the sentence ends with a BIND token)\n" + "- cat (string); OPTIONAL, default: the startcat of the grammar\n" + "- prefix (string); OPTIONAL, the prefix of predicted tokens" + "- n (int), max. number of predicted tokens" }, {"parseval", (PyCFunction)Concr_parseval, METH_VARARGS, "Computes precision, recall and exact match for the parser on a given abstract tree" @@ -3670,7 +3715,7 @@ MOD_INIT(pgf) PyModule_AddObject(m, "Bracket", (PyObject *) &pgf_BracketType); Py_INCREF(&pgf_BracketType); - PyModule_AddObject(m, "BIND", (PyObject *) &pgf_BINDType); + PyModule_AddObject(m, "BIND", pgf_BINDType.tp_alloc(&pgf_BINDType, 0)); Py_INCREF(&pgf_BINDType); return MOD_SUCCESS_VAL(m);