GitHub/gf-core
1
0
Fork 1
mirror of https://github.com/GrammaticalFramework/gf-core.git synced 2026-09-19 00:36:02 -06:00
Code Issues Packages Projects Releases Wiki Activity

for backwards compatibility, the parser run in a non-robust mode by default

Browse Source
This commit is contained in:
Krasimir Angelov
2026-09-13 16:10:10 +02:00
parent 722b932da8
commit ad0fb899d2
11 changed files with 139 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/compiler/api/GF/Command/Commands.hs
+10 -8
View File
@@ -358,14 +358,16 @@ pgfCommands = Map.fromList [
"See also the ps command for lexing and character encoding."
],
exec = needPGF $ \opts ts pgf ->
return $
foldr (joinPiped . fromParse1 opts) void
(concat [
[(s,parse concr (optType pgf opts) s) |
concr <- optLangs pgf opts]
| s <- toStrings ts]),
let parseOp | isOpt "robust" opts = \concr -> ParseOk . robustParse concr (optType pgf opts)
| otherwise = \concr -> parse concr (optType pgf opts)
in return $
foldr (joinPiped . fromParse1 opts) void
(concat [[(s,parseOp concr s) |
concr <- optLangs pgf opts]
| s <- toStrings ts]),
options = [
("show_probs", "show the probability of each result")
("show_probs", "show the probability of each result"),
("robust", "return a partial result for ungrammatical input")
],
flags = [
("cat","target category of parsing"),
@@ -783,7 +785,7 @@ pgfCommands = Map.fromList [
fromParse1 opts (s,po) =
case po of
ParseOk ts -> fromExprs (isOpt "show_probs" opts) (takeOptNum opts ts)
ParseFailed i t -> pipeMessage $ "The parser failed at token "
ParseFailed i t -> pipeMessage $ "The parser failed at position "
++ show i ++": "
++ show t
ParseIncomplete -> pipeMessage "The sentence is not complete"
src/runtime/c/pgf/parser.cxx
+15 -8
View File
@@ -1213,7 +1213,7 @@ void PgfParser::make_chunks(State *state, std::vector<CCat*> &chunks, prob_t pro
}
}
void PgfParser::prepare(ref<PgfConcrLincat> start)
bool PgfParser::prepare(ref<PgfConcrLincat> start, bool robust)
{
#ifdef DEBUG_PARSER
fprintf(stderr, "------------------------------------------\n");
@@ -1235,6 +1235,20 @@ void PgfParser::prepare(ref<PgfConcrLincat> start)
item->outside_prob = 0;
state->push_item(item);
}
perform_search();
if (queue.size() == 0) {
if (robust) {
std::vector<CCat*> chunks;
make_chunks(current_state, chunks, 0);
return true;
} else {
return false;
}
}
return true;
}
void PgfParser::perform_search()
@@ -1289,15 +1303,8 @@ PgfExpr PgfParser::fetch(PgfDB *db, prob_t *prob)
{
DB_scope scope(db, READER_SCOPE);
bool first_fetch = (concr->last_fid == last_fid);
perform_search();
if (first_fetch && queue.size() == 0) {
std::vector<CCat*> chunks;
make_chunks(current_state, chunks, 0);
}
while (queue.size() > 0) {
ExprState *estate = queue.front();
std::pop_heap(queue.begin(), queue.end(), estate_comp);
src/runtime/c/pgf/parser.h
+4 -2
View File
@@ -284,6 +284,8 @@ class PGF_INTERNAL_DECL PgfParser : private PgfAbstractParser, public PgfExprEnu
ssize_t delta_byte_pos;
ssize_t allocated_size;
void perform_search();
virtual State *new_state(const PgfTextSpot &start, prob_t viterbi_prob);
virtual void symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks);
virtual void symbol_bind(Item *item, State *state, PgfSymbol sym);
@@ -319,8 +321,8 @@ public:
PgfParser(ref<PgfConcr> concr, PgfText *sentence, bool case_sensitive, PgfMarshaller *m, PgfUnmarshaller *u);
virtual ~PgfParser();
void prepare(ref<PgfConcrLincat> start);
void perform_search();
bool prepare(ref<PgfConcrLincat> start, bool robust);
size_t get_end_pos() { return current_state->end.pos; }
virtual PgfExpr fetch(PgfDB *db, prob_t *prob);
src/runtime/c/pgf/pgf.cxx
+13 -6
View File
@@ -2701,8 +2701,8 @@ struct PGF_INTERNAL_DECL PgfLincatUnmarshaller : PgfUnmarshaller {
PGF_API
PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision,
PgfType ty, PgfMarshaller *m, PgfUnmarshaller *u,
PgfText *sentence,
PgfExn * err)
PgfText *sentence, int robust,
PgfExn *err)
{
PGF_API_BEGIN {
DB_scope scope(db, READER_SCOPE);
@@ -2717,7 +2717,11 @@ PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision,
return 0;
PgfParser *parser = new PgfParser(concr, sentence, case_sensitive, m, u);
parser->prepare(lincat_u.lincat);
if (!parser->prepare(lincat_u.lincat, robust)) {
err->type = PGF_EXN_PARSE_ERROR;
err->code = parser->get_end_pos();
return NULL;
}
return parser;
} PGF_API_END
@@ -2733,7 +2737,7 @@ void pgf_free_expr_enum(PgfExprEnum *en)
PGF_API
PgfParseChart *pgf_parse_chart(PgfDB *db, PgfConcrRevision revision,
PgfType ty, PgfMarshaller *m, PgfUnmarshaller *u,
PgfText *sentence,
PgfText *sentence, int robust,
PgfExn * err)
{
PGF_API_BEGIN {
@@ -2749,8 +2753,11 @@ PgfParseChart *pgf_parse_chart(PgfDB *db, PgfConcrRevision revision,
return 0;
PgfParser *parser = new PgfParser(concr, sentence, case_sensitive, m, u);
parser->prepare(lincat_u.lincat);
parser->perform_search();
if (!parser->prepare(lincat_u.lincat, robust)) {
err->type = PGF_EXN_PARSE_ERROR;
err->code = parser->get_end_pos();
return NULL;
}
return parser;
} PGF_API_END
src/runtime/c/pgf/pgf.h
+3 -2
View File
@@ -76,6 +76,7 @@ typedef enum {
PGF_EXN_SYSTEM_ERROR,
PGF_EXN_PGF_ERROR,
PGF_EXN_TYPE_ERROR,
PGF_EXN_PARSE_ERROR,
PGF_EXN_OTHER_ERROR
} PgfExnType;
@@ -822,7 +823,7 @@ void pgf_bracketed_linearize_all(PgfDB *db, PgfConcrRevision revision,
PGF_API_DECL
PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision,
PgfType ty, PgfMarshaller *m, PgfUnmarshaller *u,
PgfText *sentence,
PgfText *sentence, int robust,
PgfExn * err);
PGF_API_DECL
@@ -855,7 +856,7 @@ struct PgfParseChart {
PGF_API_DECL
PgfParseChart *pgf_parse_chart(PgfDB *db, PgfConcrRevision revision,
PgfType ty, PgfMarshaller *m, PgfUnmarshaller *u,
PgfText *sentence,
PgfText *sentence, int robust,
PgfExn * err);
PGF_API_DECL
src/runtime/haskell/PGF2.hsc
+31 -5
View File
@@ -86,7 +86,7 @@ module PGF2 (-- * PGF
printName, alignWords, gizaAlignment,
-- ** Parsing
ParseOutput(..), parse, parseWithHeuristics, complete,
ParseOutput(..), parse, robustParse, parseWithHeuristics, complete,
-- * Exceptions
PGFError(..),
@@ -849,11 +849,37 @@ parse :: Concr -> Type -> String -> ParseOutput [(Expr,Float)]
parse c ty sent =
unsafePerformIO $
withForeignPtr (c_revision c) $ \c_revision_ptr ->
bracket (newStablePtr ty) freeStablePtr $ \c_ty ->
withText sent $ \c_sent -> do
c_enum <- withPgfExn "parse" (pgf_parse (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent)
allocaBytes (#size PgfExn) $ \c_exn -> do
c_enum <- bracket (newStablePtr ty) freeStablePtr $ \c_ty ->
withText sent $ \c_sent ->
pgf_parse (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent 0 c_exn
ex_type <- (#peek PgfExn, type) c_exn :: IO (#type PgfExnType)
case ex_type of
(#const PGF_EXN_NONE) -> do
exprs <- enumerateExprs (c_db c) (c_revision c) c_enum
return (ParseOk exprs)
(#const PGF_EXN_PARSE_ERROR) -> do
pos <- (#peek PgfExn, code) c_exn
case takeWhile (not.isSpace) (drop pos sent) of
[] -> return (ParseIncomplete)
tok -> return (ParseFailed pos tok)
(#const PGF_EXN_PGF_ERROR) -> do
c_msg <- (#peek PgfExn, msg) c_exn
msg <- peekCString c_msg
free c_msg
throwIO (PGFError "parse" msg)
_ -> throwIO (PGFError "parse" "An unidentified error occurred")
robustParse :: Concr -> Type -> String -> [(Expr,Float)]
robustParse c ty sent =
unsafePerformIO $
withForeignPtr (c_revision c) $ \c_revision_ptr ->
allocaBytes (#size PgfExn) $ \c_exn -> do
c_enum <- bracket (newStablePtr ty) freeStablePtr $ \c_ty ->
withText sent $ \c_sent ->
withPgfExn "robustParse" (pgf_parse (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent 1)
exprs <- enumerateExprs (c_db c) (c_revision c) c_enum
return (ParseOk exprs)
return exprs
enumerateExprs c_db c_revision c_enum_ptr = do
c_enum <- newForeignPtr pgf_free_expr_enum c_enum_ptr
src/runtime/haskell/PGF2/Collab.hsc
+1 -1
View File
@@ -35,7 +35,7 @@ parseChart c ty sent =
withForeignPtr (c_revision c) $ \c_revision_ptr ->
bracket (newStablePtr ty) freeStablePtr $ \c_ty ->
withText sent $ \c_sent -> do
c_chart <- withPgfExn "parseChart" (pgf_parse_chart (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent)
c_chart <- withPgfExn "parseChart" (pgf_parse_chart (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent 1)
fptr <- newForeignPtr pgf_free_parse_chart c_chart
return (ParseChart (c_db c) (c_revision c) fptr)
src/runtime/haskell/PGF2/FFI.hsc
+2 -2
View File
@@ -283,13 +283,13 @@ foreign import ccall pgf_bracketed_linearize_all :: Ptr PgfDB -> Ptr Concr -> St
foreign import ccall pgf_align_words :: Ptr PgfDB -> Ptr Concr -> StablePtr Expr -> Ptr PgfPrintContext -> Ptr PgfMarshaller -> Ptr CSize -> Ptr PgfExn -> IO (Ptr (Ptr PgfAlignmentPhrase))
foreign import ccall pgf_parse :: Ptr PgfDB -> Ptr Concr -> StablePtr Type -> Ptr PgfMarshaller -> Ptr PgfUnmarshaller -> Ptr PgfText -> Ptr PgfExn -> IO (Ptr PgfExprEnum)
foreign import ccall pgf_parse :: Ptr PgfDB -> Ptr Concr -> StablePtr Type -> Ptr PgfMarshaller -> Ptr PgfUnmarshaller -> Ptr PgfText -> CInt -> Ptr PgfExn -> IO (Ptr PgfExprEnum)
foreign import ccall "&pgf_free_expr_enum" pgf_free_expr_enum :: FunPtr (Ptr PgfExprEnum -> IO ())
foreign import ccall "dynamic" callFetch :: Dynamic (Ptr PgfExprEnum -> Ptr PgfDB -> Ptr (#type prob_t) -> IO (StablePtr Expr))
foreign import ccall pgf_parse_chart :: Ptr PgfDB -> Ptr Concr -> StablePtr Type -> Ptr PgfMarshaller -> Ptr PgfUnmarshaller -> Ptr PgfText -> Ptr PgfExn -> IO (Ptr PgfParseChart)
foreign import ccall pgf_parse_chart :: Ptr PgfDB -> Ptr Concr -> StablePtr Type -> Ptr PgfMarshaller -> Ptr PgfUnmarshaller -> Ptr PgfText -> CInt -> Ptr PgfExn -> IO (Ptr PgfParseChart)
foreign import ccall "&pgf_free_parse_chart" pgf_free_parse_chart :: FunPtr (Ptr PgfParseChart -> IO ())
src/runtime/python/ffi.c
+2
View File
@@ -24,6 +24,8 @@ PgfExnType handleError(PgfExn err)
return err.type;
}
PyObject *ParseError;
// ----------------------------------------------------------------------------
// conversions
src/runtime/python/ffi.h
+2
View File
@@ -21,6 +21,8 @@ typedef struct {
extern PyObject *PGFError;
PgfExnType handleError(PgfExn err);
extern PyObject *ParseError;
PgfText *CString_AsPgfText(const char *s, size_t size);
PgfText *PyUnicode_AsPgfText(PyObject *pystr);
src/runtime/python/pypgf.c
+56 -33
View File
@@ -194,58 +194,84 @@ pgf_type_from_object(PyObject* obj) {
static IterObject*
Concr_parse(ConcrObject* self, PyObject *args, PyObject *keywds)
{
static char *kwlist[] = {"sentence", "cat", "n", NULL};
static char *kwlist[] = {"sentence", "cat", "n", "robust", NULL};
const char *sentence = NULL;
size_t sentence_size;
PyObject *sentence = NULL;
PyObject* start = NULL;
int max_count = -1;
if (!PyArg_ParseTupleAndKeywords(args, keywds, "s#|Oi", kwlist,
&sentence, &sentence_size,
&start, &max_count))
int robust = 0;
if (!PyArg_ParseTupleAndKeywords(args, keywds, "U|Oip", kwlist,
&sentence,
&start, &max_count, &robust))
return NULL;
IterObject* pyres = (IterObject*)
pgf_IterType.tp_alloc(&pgf_IterType, 0);
if (pyres == NULL) {
return NULL;
}
pyres->db = self->grammar->db;
pyres->source = (PyObject*) self;
Py_XINCREF(pyres->source);
pyres->max_count = max_count;
pyres->counter = 0;
PgfExn err;
TypeObject *type;
if (start == NULL) {
type = (TypeObject *) pgf_start_cat(self->grammar->db, self->grammar->revision, &unmarshaller, &err);
if (handleError(err) != PGF_EXN_NONE) {
Py_DECREF(pyres);
return NULL;
}
} else {
type = pgf_type_from_object(start);
}
if (type == NULL) {
Py_DECREF(pyres);
return NULL;
}
PgfText *text_sentence = CString_AsPgfText(sentence, sentence_size);
pyres->res = (PgfExprEnum*)
PgfText *text_sentence = PyUnicode_AsPgfText(sentence);
PgfExprEnum* res =
pgf_parse(self->grammar->db, self->concr, (PgfType) type,
&marshaller, &unmarshaller,
text_sentence, &err);
text_sentence, robust, &err);
FreePgfText(text_sentence);
if (handleError(err) != PGF_EXN_NONE) {
Py_DECREF(pyres);
Py_DECREF(type);
if (err.type == PGF_EXN_PARSE_ERROR) {
PyObject* py_offset = PyLong_FromLong(err.code);
Py_ssize_t len = PyUnicode_GET_LENGTH(sentence);
PyObject_SetAttrString(ParseError, "offset", py_offset);
if (err.code == len)
PyObject_SetAttrString(ParseError, "incomplete", Py_True);
else {
Py_ssize_t end = PyUnicode_FindChar(sentence, ' ', err.code, -1, 1);
if (end == -1)
end = len;
else if (end == -2)
return NULL;
PyObject *py_token = PyUnicode_Substring(sentence, err.code, end);
if (py_token == NULL)
return NULL;
PyObject_SetAttrString(ParseError, "incomplete", Py_False);
PyObject_SetAttrString(ParseError, "token", py_token);
Py_DECREF(py_token);
Py_DECREF(py_offset);
}
PyErr_Format(ParseError, "Parse error at position %d", err.code);
return NULL;
} else if (handleError(err) != PGF_EXN_NONE) {
return NULL;
}
IterObject* pyres = (IterObject*)
pgf_IterType.tp_alloc(&pgf_IterType, 0);
if (pyres == NULL) {
return NULL;
}
pyres->res = res;
pyres->db = self->grammar->db;
pyres->source = (PyObject*) self;
Py_INCREF(pyres->source);
pyres->max_count = max_count;
pyres->counter = 0;
return pyres;
}
@@ -2195,20 +2221,14 @@ pgf_showType(PyObject *self, PyObject *args)
{
PyObject *pylist;
TypeObject *type;
printf("pgf_showType 1\n");
if (!PyArg_ParseTuple(args, "O!O!", &PyList_Type, &pylist, &pgf_TypeType, &type))
return NULL;
printf("pgf_showType 2\n");
PgfPrintContext *ctxt = PyList_AsPgfPrintContext(pylist);
printf("pgf_showType 3\n");
PgfText *s = pgf_print_type((PgfType) type, ctxt, 0, &marshaller);
printf("pgf_showType 4\n");
FreePgfPrintContext(ctxt);
printf("pgf_showType 5\n");
PyObject *str = PyUnicode_FromStringAndSize(s->text, s->size);
printf("pgf_showType 6\n");
FreePgfText(s);
printf("pgf_showType 7\n");
return str;
}
@@ -2379,6 +2399,9 @@ MOD_INIT(pgf)
PGFError = PyErr_NewException("pgf.PGFError", NULL, NULL);
ADD_TYPE_DIRECT("PGFError", PGFError);
ParseError = PyErr_NewException("pgf.ParseError", NULL, NULL);
ADD_TYPE_DIRECT("ParseError", ParseError);
ADD_TYPE("PGF", pgf_PGFType);
ADD_TYPE("Concr", pgf_ConcrType);
ADD_TYPE("Transaction", pgf_TransactionType);