From ad0fb899d21323b878c4d304b830937bc59f2a3e Mon Sep 17 00:00:00 2001 From: Krasimir Angelov Date: Sun, 13 Sep 2026 16:10:10 +0200 Subject: [PATCH] for backwards compatibility, the parser run in a non-robust mode by default --- src/compiler/api/GF/Command/Commands.hs | 18 ++--- src/runtime/c/pgf/parser.cxx | 23 ++++--- src/runtime/c/pgf/parser.h | 6 +- src/runtime/c/pgf/pgf.cxx | 19 ++++-- src/runtime/c/pgf/pgf.h | 5 +- src/runtime/haskell/PGF2.hsc | 36 ++++++++-- src/runtime/haskell/PGF2/Collab.hsc | 2 +- src/runtime/haskell/PGF2/FFI.hsc | 4 +- src/runtime/python/ffi.c | 2 + src/runtime/python/ffi.h | 2 + src/runtime/python/pypgf.c | 89 ++++++++++++++++--------- 11 files changed, 139 insertions(+), 67 deletions(-) diff --git a/src/compiler/api/GF/Command/Commands.hs b/src/compiler/api/GF/Command/Commands.hs index d796d4e62..517a4f961 100644 --- a/src/compiler/api/GF/Command/Commands.hs +++ b/src/compiler/api/GF/Command/Commands.hs @@ -358,14 +358,16 @@ pgfCommands = Map.fromList [ "See also the ps command for lexing and character encoding." ], exec = needPGF $ \opts ts pgf -> - return $ - foldr (joinPiped . fromParse1 opts) void - (concat [ - [(s,parse concr (optType pgf opts) s) | - concr <- optLangs pgf opts] - | s <- toStrings ts]), + let parseOp | isOpt "robust" opts = \concr -> ParseOk . robustParse concr (optType pgf opts) + | otherwise = \concr -> parse concr (optType pgf opts) + in return $ + foldr (joinPiped . fromParse1 opts) void + (concat [[(s,parseOp concr s) | + concr <- optLangs pgf opts] + | s <- toStrings ts]), options = [ - ("show_probs", "show the probability of each result") + ("show_probs", "show the probability of each result"), + ("robust", "return a partial result for ungrammatical input") ], flags = [ ("cat","target category of parsing"), @@ -783,7 +785,7 @@ pgfCommands = Map.fromList [ fromParse1 opts (s,po) = case po of ParseOk ts -> fromExprs (isOpt "show_probs" opts) (takeOptNum opts ts) - ParseFailed i t -> pipeMessage $ "The parser failed at token " + ParseFailed i t -> pipeMessage $ "The parser failed at position " ++ show i ++": " ++ show t ParseIncomplete -> pipeMessage "The sentence is not complete" diff --git a/src/runtime/c/pgf/parser.cxx b/src/runtime/c/pgf/parser.cxx index 3101fb94f..4c8e1e73b 100644 --- a/src/runtime/c/pgf/parser.cxx +++ b/src/runtime/c/pgf/parser.cxx @@ -1213,7 +1213,7 @@ void PgfParser::make_chunks(State *state, std::vector &chunks, prob_t pro } } -void PgfParser::prepare(ref start) +bool PgfParser::prepare(ref start, bool robust) { #ifdef DEBUG_PARSER fprintf(stderr, "------------------------------------------\n"); @@ -1235,6 +1235,20 @@ void PgfParser::prepare(ref start) item->outside_prob = 0; state->push_item(item); } + + perform_search(); + + if (queue.size() == 0) { + if (robust) { + std::vector chunks; + make_chunks(current_state, chunks, 0); + return true; + } else { + return false; + } + } + + return true; } void PgfParser::perform_search() @@ -1289,15 +1303,8 @@ PgfExpr PgfParser::fetch(PgfDB *db, prob_t *prob) { DB_scope scope(db, READER_SCOPE); - bool first_fetch = (concr->last_fid == last_fid); - perform_search(); - if (first_fetch && queue.size() == 0) { - std::vector chunks; - make_chunks(current_state, chunks, 0); - } - while (queue.size() > 0) { ExprState *estate = queue.front(); std::pop_heap(queue.begin(), queue.end(), estate_comp); diff --git a/src/runtime/c/pgf/parser.h b/src/runtime/c/pgf/parser.h index 45dc50d7a..b4e1d0868 100644 --- a/src/runtime/c/pgf/parser.h +++ b/src/runtime/c/pgf/parser.h @@ -284,6 +284,8 @@ class PGF_INTERNAL_DECL PgfParser : private PgfAbstractParser, public PgfExprEnu ssize_t delta_byte_pos; ssize_t allocated_size; + void perform_search(); + virtual State *new_state(const PgfTextSpot &start, prob_t viterbi_prob); virtual void symbol_token(Item *item, State *state, ref symks); virtual void symbol_bind(Item *item, State *state, PgfSymbol sym); @@ -319,8 +321,8 @@ public: PgfParser(ref concr, PgfText *sentence, bool case_sensitive, PgfMarshaller *m, PgfUnmarshaller *u); virtual ~PgfParser(); - void prepare(ref start); - void perform_search(); + bool prepare(ref start, bool robust); + size_t get_end_pos() { return current_state->end.pos; } virtual PgfExpr fetch(PgfDB *db, prob_t *prob); diff --git a/src/runtime/c/pgf/pgf.cxx b/src/runtime/c/pgf/pgf.cxx index 3475961ec..0696ab702 100644 --- a/src/runtime/c/pgf/pgf.cxx +++ b/src/runtime/c/pgf/pgf.cxx @@ -2701,8 +2701,8 @@ struct PGF_INTERNAL_DECL PgfLincatUnmarshaller : PgfUnmarshaller { PGF_API PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision, PgfType ty, PgfMarshaller *m, PgfUnmarshaller *u, - PgfText *sentence, - PgfExn * err) + PgfText *sentence, int robust, + PgfExn *err) { PGF_API_BEGIN { DB_scope scope(db, READER_SCOPE); @@ -2717,7 +2717,11 @@ PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision, return 0; PgfParser *parser = new PgfParser(concr, sentence, case_sensitive, m, u); - parser->prepare(lincat_u.lincat); + if (!parser->prepare(lincat_u.lincat, robust)) { + err->type = PGF_EXN_PARSE_ERROR; + err->code = parser->get_end_pos(); + return NULL; + } return parser; } PGF_API_END @@ -2733,7 +2737,7 @@ void pgf_free_expr_enum(PgfExprEnum *en) PGF_API PgfParseChart *pgf_parse_chart(PgfDB *db, PgfConcrRevision revision, PgfType ty, PgfMarshaller *m, PgfUnmarshaller *u, - PgfText *sentence, + PgfText *sentence, int robust, PgfExn * err) { PGF_API_BEGIN { @@ -2749,8 +2753,11 @@ PgfParseChart *pgf_parse_chart(PgfDB *db, PgfConcrRevision revision, return 0; PgfParser *parser = new PgfParser(concr, sentence, case_sensitive, m, u); - parser->prepare(lincat_u.lincat); - parser->perform_search(); + if (!parser->prepare(lincat_u.lincat, robust)) { + err->type = PGF_EXN_PARSE_ERROR; + err->code = parser->get_end_pos(); + return NULL; + } return parser; } PGF_API_END diff --git a/src/runtime/c/pgf/pgf.h b/src/runtime/c/pgf/pgf.h index 156e05d76..e70723049 100644 --- a/src/runtime/c/pgf/pgf.h +++ b/src/runtime/c/pgf/pgf.h @@ -76,6 +76,7 @@ typedef enum { PGF_EXN_SYSTEM_ERROR, PGF_EXN_PGF_ERROR, PGF_EXN_TYPE_ERROR, + PGF_EXN_PARSE_ERROR, PGF_EXN_OTHER_ERROR } PgfExnType; @@ -822,7 +823,7 @@ void pgf_bracketed_linearize_all(PgfDB *db, PgfConcrRevision revision, PGF_API_DECL PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision, PgfType ty, PgfMarshaller *m, PgfUnmarshaller *u, - PgfText *sentence, + PgfText *sentence, int robust, PgfExn * err); PGF_API_DECL @@ -855,7 +856,7 @@ struct PgfParseChart { PGF_API_DECL PgfParseChart *pgf_parse_chart(PgfDB *db, PgfConcrRevision revision, PgfType ty, PgfMarshaller *m, PgfUnmarshaller *u, - PgfText *sentence, + PgfText *sentence, int robust, PgfExn * err); PGF_API_DECL diff --git a/src/runtime/haskell/PGF2.hsc b/src/runtime/haskell/PGF2.hsc index a1a0c6e5d..a1378c5e9 100644 --- a/src/runtime/haskell/PGF2.hsc +++ b/src/runtime/haskell/PGF2.hsc @@ -86,7 +86,7 @@ module PGF2 (-- * PGF printName, alignWords, gizaAlignment, -- ** Parsing - ParseOutput(..), parse, parseWithHeuristics, complete, + ParseOutput(..), parse, robustParse, parseWithHeuristics, complete, -- * Exceptions PGFError(..), @@ -849,11 +849,37 @@ parse :: Concr -> Type -> String -> ParseOutput [(Expr,Float)] parse c ty sent = unsafePerformIO $ withForeignPtr (c_revision c) $ \c_revision_ptr -> - bracket (newStablePtr ty) freeStablePtr $ \c_ty -> - withText sent $ \c_sent -> do - c_enum <- withPgfExn "parse" (pgf_parse (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent) + allocaBytes (#size PgfExn) $ \c_exn -> do + c_enum <- bracket (newStablePtr ty) freeStablePtr $ \c_ty -> + withText sent $ \c_sent -> + pgf_parse (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent 0 c_exn + ex_type <- (#peek PgfExn, type) c_exn :: IO (#type PgfExnType) + case ex_type of + (#const PGF_EXN_NONE) -> do + exprs <- enumerateExprs (c_db c) (c_revision c) c_enum + return (ParseOk exprs) + (#const PGF_EXN_PARSE_ERROR) -> do + pos <- (#peek PgfExn, code) c_exn + case takeWhile (not.isSpace) (drop pos sent) of + [] -> return (ParseIncomplete) + tok -> return (ParseFailed pos tok) + (#const PGF_EXN_PGF_ERROR) -> do + c_msg <- (#peek PgfExn, msg) c_exn + msg <- peekCString c_msg + free c_msg + throwIO (PGFError "parse" msg) + _ -> throwIO (PGFError "parse" "An unidentified error occurred") + +robustParse :: Concr -> Type -> String -> [(Expr,Float)] +robustParse c ty sent = + unsafePerformIO $ + withForeignPtr (c_revision c) $ \c_revision_ptr -> + allocaBytes (#size PgfExn) $ \c_exn -> do + c_enum <- bracket (newStablePtr ty) freeStablePtr $ \c_ty -> + withText sent $ \c_sent -> + withPgfExn "robustParse" (pgf_parse (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent 1) exprs <- enumerateExprs (c_db c) (c_revision c) c_enum - return (ParseOk exprs) + return exprs enumerateExprs c_db c_revision c_enum_ptr = do c_enum <- newForeignPtr pgf_free_expr_enum c_enum_ptr diff --git a/src/runtime/haskell/PGF2/Collab.hsc b/src/runtime/haskell/PGF2/Collab.hsc index 0f3387c49..cd5ff9dfa 100644 --- a/src/runtime/haskell/PGF2/Collab.hsc +++ b/src/runtime/haskell/PGF2/Collab.hsc @@ -35,7 +35,7 @@ parseChart c ty sent = withForeignPtr (c_revision c) $ \c_revision_ptr -> bracket (newStablePtr ty) freeStablePtr $ \c_ty -> withText sent $ \c_sent -> do - c_chart <- withPgfExn "parseChart" (pgf_parse_chart (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent) + c_chart <- withPgfExn "parseChart" (pgf_parse_chart (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent 1) fptr <- newForeignPtr pgf_free_parse_chart c_chart return (ParseChart (c_db c) (c_revision c) fptr) diff --git a/src/runtime/haskell/PGF2/FFI.hsc b/src/runtime/haskell/PGF2/FFI.hsc index 54b2d93b3..67c3c9dd3 100644 --- a/src/runtime/haskell/PGF2/FFI.hsc +++ b/src/runtime/haskell/PGF2/FFI.hsc @@ -283,13 +283,13 @@ foreign import ccall pgf_bracketed_linearize_all :: Ptr PgfDB -> Ptr Concr -> St foreign import ccall pgf_align_words :: Ptr PgfDB -> Ptr Concr -> StablePtr Expr -> Ptr PgfPrintContext -> Ptr PgfMarshaller -> Ptr CSize -> Ptr PgfExn -> IO (Ptr (Ptr PgfAlignmentPhrase)) -foreign import ccall pgf_parse :: Ptr PgfDB -> Ptr Concr -> StablePtr Type -> Ptr PgfMarshaller -> Ptr PgfUnmarshaller -> Ptr PgfText -> Ptr PgfExn -> IO (Ptr PgfExprEnum) +foreign import ccall pgf_parse :: Ptr PgfDB -> Ptr Concr -> StablePtr Type -> Ptr PgfMarshaller -> Ptr PgfUnmarshaller -> Ptr PgfText -> CInt -> Ptr PgfExn -> IO (Ptr PgfExprEnum) foreign import ccall "&pgf_free_expr_enum" pgf_free_expr_enum :: FunPtr (Ptr PgfExprEnum -> IO ()) foreign import ccall "dynamic" callFetch :: Dynamic (Ptr PgfExprEnum -> Ptr PgfDB -> Ptr (#type prob_t) -> IO (StablePtr Expr)) -foreign import ccall pgf_parse_chart :: Ptr PgfDB -> Ptr Concr -> StablePtr Type -> Ptr PgfMarshaller -> Ptr PgfUnmarshaller -> Ptr PgfText -> Ptr PgfExn -> IO (Ptr PgfParseChart) +foreign import ccall pgf_parse_chart :: Ptr PgfDB -> Ptr Concr -> StablePtr Type -> Ptr PgfMarshaller -> Ptr PgfUnmarshaller -> Ptr PgfText -> CInt -> Ptr PgfExn -> IO (Ptr PgfParseChart) foreign import ccall "&pgf_free_parse_chart" pgf_free_parse_chart :: FunPtr (Ptr PgfParseChart -> IO ()) diff --git a/src/runtime/python/ffi.c b/src/runtime/python/ffi.c index 8dce9e33d..e8a52f51e 100644 --- a/src/runtime/python/ffi.c +++ b/src/runtime/python/ffi.c @@ -24,6 +24,8 @@ PgfExnType handleError(PgfExn err) return err.type; } +PyObject *ParseError; + // ---------------------------------------------------------------------------- // conversions diff --git a/src/runtime/python/ffi.h b/src/runtime/python/ffi.h index 07dea8dba..5bdb71f2b 100644 --- a/src/runtime/python/ffi.h +++ b/src/runtime/python/ffi.h @@ -21,6 +21,8 @@ typedef struct { extern PyObject *PGFError; PgfExnType handleError(PgfExn err); +extern PyObject *ParseError; + PgfText *CString_AsPgfText(const char *s, size_t size); PgfText *PyUnicode_AsPgfText(PyObject *pystr); diff --git a/src/runtime/python/pypgf.c b/src/runtime/python/pypgf.c index 12cf8b179..318314b09 100644 --- a/src/runtime/python/pypgf.c +++ b/src/runtime/python/pypgf.c @@ -194,58 +194,84 @@ pgf_type_from_object(PyObject* obj) { static IterObject* Concr_parse(ConcrObject* self, PyObject *args, PyObject *keywds) { - static char *kwlist[] = {"sentence", "cat", "n", NULL}; + static char *kwlist[] = {"sentence", "cat", "n", "robust", NULL}; - const char *sentence = NULL; - size_t sentence_size; + PyObject *sentence = NULL; PyObject* start = NULL; int max_count = -1; - if (!PyArg_ParseTupleAndKeywords(args, keywds, "s#|Oi", kwlist, - &sentence, &sentence_size, - &start, &max_count)) + int robust = 0; + if (!PyArg_ParseTupleAndKeywords(args, keywds, "U|Oip", kwlist, + &sentence, + &start, &max_count, &robust)) return NULL; - IterObject* pyres = (IterObject*) - pgf_IterType.tp_alloc(&pgf_IterType, 0); - if (pyres == NULL) { - return NULL; - } - - pyres->db = self->grammar->db; - pyres->source = (PyObject*) self; - Py_XINCREF(pyres->source); - - pyres->max_count = max_count; - pyres->counter = 0; - PgfExn err; TypeObject *type; if (start == NULL) { type = (TypeObject *) pgf_start_cat(self->grammar->db, self->grammar->revision, &unmarshaller, &err); if (handleError(err) != PGF_EXN_NONE) { - Py_DECREF(pyres); return NULL; } } else { type = pgf_type_from_object(start); } if (type == NULL) { - Py_DECREF(pyres); return NULL; } - PgfText *text_sentence = CString_AsPgfText(sentence, sentence_size); - pyres->res = (PgfExprEnum*) + PgfText *text_sentence = PyUnicode_AsPgfText(sentence); + PgfExprEnum* res = pgf_parse(self->grammar->db, self->concr, (PgfType) type, &marshaller, &unmarshaller, - text_sentence, &err); + text_sentence, robust, &err); FreePgfText(text_sentence); - if (handleError(err) != PGF_EXN_NONE) { - Py_DECREF(pyres); + Py_DECREF(type); + if (err.type == PGF_EXN_PARSE_ERROR) { + PyObject* py_offset = PyLong_FromLong(err.code); + + Py_ssize_t len = PyUnicode_GET_LENGTH(sentence); + + PyObject_SetAttrString(ParseError, "offset", py_offset); + if (err.code == len) + PyObject_SetAttrString(ParseError, "incomplete", Py_True); + else { + Py_ssize_t end = PyUnicode_FindChar(sentence, ' ', err.code, -1, 1); + if (end == -1) + end = len; + else if (end == -2) + return NULL; + PyObject *py_token = PyUnicode_Substring(sentence, err.code, end); + if (py_token == NULL) + return NULL; + + PyObject_SetAttrString(ParseError, "incomplete", Py_False); + PyObject_SetAttrString(ParseError, "token", py_token); + + Py_DECREF(py_token); + Py_DECREF(py_offset); + } + + PyErr_Format(ParseError, "Parse error at position %d", err.code); + return NULL; + } else if (handleError(err) != PGF_EXN_NONE) { return NULL; } + IterObject* pyres = (IterObject*) + pgf_IterType.tp_alloc(&pgf_IterType, 0); + if (pyres == NULL) { + return NULL; + } + + pyres->res = res; + pyres->db = self->grammar->db; + pyres->source = (PyObject*) self; + Py_INCREF(pyres->source); + + pyres->max_count = max_count; + pyres->counter = 0; + return pyres; } @@ -2195,20 +2221,14 @@ pgf_showType(PyObject *self, PyObject *args) { PyObject *pylist; TypeObject *type; -printf("pgf_showType 1\n"); + if (!PyArg_ParseTuple(args, "O!O!", &PyList_Type, &pylist, &pgf_TypeType, &type)) return NULL; -printf("pgf_showType 2\n"); PgfPrintContext *ctxt = PyList_AsPgfPrintContext(pylist); -printf("pgf_showType 3\n"); PgfText *s = pgf_print_type((PgfType) type, ctxt, 0, &marshaller); -printf("pgf_showType 4\n"); FreePgfPrintContext(ctxt); -printf("pgf_showType 5\n"); PyObject *str = PyUnicode_FromStringAndSize(s->text, s->size); -printf("pgf_showType 6\n"); FreePgfText(s); -printf("pgf_showType 7\n"); return str; } @@ -2379,6 +2399,9 @@ MOD_INIT(pgf) PGFError = PyErr_NewException("pgf.PGFError", NULL, NULL); ADD_TYPE_DIRECT("PGFError", PGFError); + ParseError = PyErr_NewException("pgf.ParseError", NULL, NULL); + ADD_TYPE_DIRECT("ParseError", ParseError); + ADD_TYPE("PGF", pgf_PGFType); ADD_TYPE("Concr", pgf_ConcrType); ADD_TYPE("Transaction", pgf_TransactionType);