for backwards compatibility, the parser run in a non-robust mode by default

src/compiler/api/GF/Command/Commands.hs

@@ -358,14 +358,16 @@ pgfCommands = Map.fromList [
        "See also the ps command for lexing and character encoding."
        ],
      exec = needPGF $ \opts ts pgf ->
-              return $
+              let parseOp | isOpt "robust" opts = \concr -> ParseOk . robustParse concr (optType pgf opts)
+                          | otherwise           = \concr -> parse concr (optType pgf opts)
+              in return $
                    foldr (joinPiped . fromParse1 opts) void
-                  (concat [
+                     (concat [[(s,parseOp concr s) |
-                              [(s,parse concr (optType pgf opts) s) |
                                            concr <- optLangs pgf opts]
                               | s <- toStrings ts]),
      options = [
-       ("show_probs", "show the probability of each result")
+       ("show_probs", "show the probability of each result"),
+       ("robust",     "return a partial result for ungrammatical input")
        ],
      flags = [
        ("cat","target category of parsing"),
@@ -783,7 +785,7 @@ pgfCommands = Map.fromList [
    fromParse1 opts (s,po) =
      case po of
        ParseOk ts      -> fromExprs (isOpt "show_probs" opts) (takeOptNum opts ts)
-       ParseFailed i t -> pipeMessage $ "The parser failed at token "
+       ParseFailed i t -> pipeMessage $ "The parser failed at position "
                                         ++ show i ++": "
                                         ++ show t
        ParseIncomplete -> pipeMessage "The sentence is not complete"

src/runtime/c/pgf/parser.cxx

@@ -1213,7 +1213,7 @@ void PgfParser::make_chunks(State *state, std::vector<CCat*> &chunks, prob_t pro
     }
 }
 
-void PgfParser::prepare(ref<PgfConcrLincat> start)
+bool PgfParser::prepare(ref<PgfConcrLincat> start, bool robust)
 {
 #ifdef DEBUG_PARSER
     fprintf(stderr, "------------------------------------------\n");
@@ -1235,6 +1235,20 @@ void PgfParser::prepare(ref<PgfConcrLincat> start)
         item->outside_prob = 0;
         state->push_item(item);
     }
+
+    perform_search();
+
+    if (queue.size() == 0) {
+        if (robust) {
+            std::vector<CCat*> chunks;
+            make_chunks(current_state, chunks, 0);
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    return true;
 }
 
 void PgfParser::perform_search()
@@ -1289,15 +1303,8 @@ PgfExpr PgfParser::fetch(PgfDB *db, prob_t *prob)
 {
     DB_scope scope(db, READER_SCOPE);
 
-    bool first_fetch = (concr->last_fid == last_fid);
-
     perform_search();
 
-    if (first_fetch && queue.size() == 0) {
-        std::vector<CCat*> chunks;
-        make_chunks(current_state, chunks, 0);
-    }
-
     while (queue.size() > 0) {
         ExprState *estate = queue.front();
         std::pop_heap(queue.begin(), queue.end(), estate_comp);

src/runtime/c/pgf/parser.h

@@ -284,6 +284,8 @@ class PGF_INTERNAL_DECL PgfParser : private PgfAbstractParser, public PgfExprEnu
     ssize_t delta_byte_pos;
     ssize_t allocated_size;
 
+    void perform_search();
+
     virtual State *new_state(const PgfTextSpot &start, prob_t viterbi_prob);
     virtual void symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks);
     virtual void symbol_bind(Item *item, State *state, PgfSymbol sym);
@@ -319,8 +321,8 @@ public:
     PgfParser(ref<PgfConcr> concr, PgfText *sentence, bool case_sensitive, PgfMarshaller *m, PgfUnmarshaller *u);
     virtual ~PgfParser();
 
-    void prepare(ref<PgfConcrLincat> start);
+    bool prepare(ref<PgfConcrLincat> start, bool robust);
-    void perform_search();
+    size_t get_end_pos() { return current_state->end.pos; }
 
     virtual PgfExpr fetch(PgfDB *db, prob_t *prob);
 

src/runtime/c/pgf/pgf.cxx

@@ -2701,7 +2701,7 @@ struct PGF_INTERNAL_DECL PgfLincatUnmarshaller : PgfUnmarshaller {
 PGF_API
 PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision,
                        PgfType ty, PgfMarshaller *m, PgfUnmarshaller *u,
-                       PgfText *sentence,
+                       PgfText *sentence, int robust,
                        PgfExn *err)
 {
     PGF_API_BEGIN {
@@ -2717,7 +2717,11 @@ PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision,
             return 0;
 
         PgfParser *parser = new PgfParser(concr, sentence, case_sensitive, m, u);
-        parser->prepare(lincat_u.lincat);
+        if (!parser->prepare(lincat_u.lincat, robust)) {
+            err->type = PGF_EXN_PARSE_ERROR;
+            err->code = parser->get_end_pos();
+            return NULL;
+        }
         return parser;
     } PGF_API_END
 
@@ -2733,7 +2737,7 @@ void pgf_free_expr_enum(PgfExprEnum *en)
 PGF_API
 PgfParseChart *pgf_parse_chart(PgfDB *db, PgfConcrRevision revision,
                                PgfType ty, PgfMarshaller *m, PgfUnmarshaller *u,
-                               PgfText *sentence,
+                               PgfText *sentence, int robust,
                                PgfExn * err)
 {
     PGF_API_BEGIN {
@@ -2749,8 +2753,11 @@ PgfParseChart *pgf_parse_chart(PgfDB *db, PgfConcrRevision revision,
             return 0;
 
         PgfParser *parser = new PgfParser(concr, sentence, case_sensitive, m, u);
-        parser->prepare(lincat_u.lincat);
+        if (!parser->prepare(lincat_u.lincat, robust)) {
-        parser->perform_search();
+            err->type = PGF_EXN_PARSE_ERROR;
+            err->code = parser->get_end_pos();
+            return NULL;
+        }
         return parser;
     } PGF_API_END
 

src/runtime/c/pgf/pgf.h

@@ -76,6 +76,7 @@ typedef enum {
     PGF_EXN_SYSTEM_ERROR,
     PGF_EXN_PGF_ERROR,
     PGF_EXN_TYPE_ERROR,
+    PGF_EXN_PARSE_ERROR,
     PGF_EXN_OTHER_ERROR
 }  PgfExnType;
 
@@ -822,7 +823,7 @@ void pgf_bracketed_linearize_all(PgfDB *db, PgfConcrRevision revision,
 PGF_API_DECL
 PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision,
                        PgfType ty, PgfMarshaller *m, PgfUnmarshaller *u,
-                       PgfText *sentence,
+                       PgfText *sentence, int robust,
                        PgfExn * err);
 
 PGF_API_DECL
@@ -855,7 +856,7 @@ struct PgfParseChart {
 PGF_API_DECL
 PgfParseChart *pgf_parse_chart(PgfDB *db, PgfConcrRevision revision,
                                PgfType ty, PgfMarshaller *m, PgfUnmarshaller *u,
-                               PgfText *sentence,
+                               PgfText *sentence, int robust,
                                PgfExn * err);
 
 PGF_API_DECL

src/runtime/haskell/PGF2.hsc

@@ -86,7 +86,7 @@ module PGF2 (-- * PGF
              printName, alignWords, gizaAlignment,
 
              -- ** Parsing
-             ParseOutput(..), parse, parseWithHeuristics, complete,
+             ParseOutput(..), parse, robustParse, parseWithHeuristics, complete,
 
              -- * Exceptions
              PGFError(..),
@@ -849,11 +849,37 @@ parse :: Concr -> Type -> String -> ParseOutput [(Expr,Float)]
 parse c ty sent =
   unsafePerformIO $
   withForeignPtr (c_revision c) $ \c_revision_ptr ->
-  bracket (newStablePtr ty) freeStablePtr $ \c_ty ->
+  allocaBytes (#size PgfExn) $ \c_exn -> do
-  withText sent $ \c_sent -> do
+    c_enum <- bracket (newStablePtr ty) freeStablePtr $ \c_ty ->
-    c_enum <- withPgfExn "parse" (pgf_parse (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent)
+              withText sent $ \c_sent ->
+                 pgf_parse (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent 0 c_exn
+    ex_type <- (#peek PgfExn, type) c_exn :: IO (#type PgfExnType)
+    case ex_type of
+      (#const PGF_EXN_NONE) -> do
          exprs <- enumerateExprs (c_db c) (c_revision c) c_enum
          return (ParseOk exprs)
+      (#const PGF_EXN_PARSE_ERROR) -> do
+         pos <- (#peek PgfExn, code) c_exn
+         case takeWhile (not.isSpace) (drop pos sent) of
+           []  -> return (ParseIncomplete)
+           tok -> return (ParseFailed pos tok)
+      (#const PGF_EXN_PGF_ERROR) -> do
+         c_msg <- (#peek PgfExn, msg) c_exn
+         msg <- peekCString c_msg
+         free c_msg
+         throwIO (PGFError "parse" msg)
+      _ -> throwIO (PGFError "parse" "An unidentified error occurred")
+
+robustParse :: Concr -> Type -> String -> [(Expr,Float)]
+robustParse c ty sent =
+  unsafePerformIO $
+  withForeignPtr (c_revision c) $ \c_revision_ptr ->
+  allocaBytes (#size PgfExn) $ \c_exn -> do
+    c_enum <- bracket (newStablePtr ty) freeStablePtr $ \c_ty ->
+              withText sent $ \c_sent ->
+              withPgfExn "robustParse" (pgf_parse (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent 1)
+    exprs <- enumerateExprs (c_db c) (c_revision c) c_enum
+    return exprs
 
 enumerateExprs c_db c_revision c_enum_ptr = do
   c_enum <- newForeignPtr pgf_free_expr_enum c_enum_ptr

src/runtime/haskell/PGF2/Collab.hsc

@@ -35,7 +35,7 @@ parseChart c ty sent =
   withForeignPtr (c_revision c) $ \c_revision_ptr ->
   bracket (newStablePtr ty) freeStablePtr $ \c_ty ->
   withText sent $ \c_sent -> do
-    c_chart <- withPgfExn "parseChart" (pgf_parse_chart (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent)
+    c_chart <- withPgfExn "parseChart" (pgf_parse_chart (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent 1)
     fptr <- newForeignPtr pgf_free_parse_chart c_chart
     return (ParseChart (c_db c) (c_revision c) fptr)
 

src/runtime/haskell/PGF2/FFI.hsc

@@ -283,13 +283,13 @@ foreign import ccall pgf_bracketed_linearize_all :: Ptr PgfDB -> Ptr Concr -> St
 
 foreign import ccall pgf_align_words :: Ptr PgfDB -> Ptr Concr -> StablePtr Expr -> Ptr PgfPrintContext -> Ptr PgfMarshaller -> Ptr CSize -> Ptr PgfExn -> IO (Ptr (Ptr PgfAlignmentPhrase))
 
-foreign import ccall pgf_parse :: Ptr PgfDB -> Ptr Concr -> StablePtr Type -> Ptr PgfMarshaller -> Ptr PgfUnmarshaller -> Ptr PgfText -> Ptr PgfExn -> IO (Ptr PgfExprEnum)
+foreign import ccall pgf_parse :: Ptr PgfDB -> Ptr Concr -> StablePtr Type -> Ptr PgfMarshaller -> Ptr PgfUnmarshaller -> Ptr PgfText -> CInt -> Ptr PgfExn -> IO (Ptr PgfExprEnum)
 
 foreign import ccall "&pgf_free_expr_enum" pgf_free_expr_enum :: FunPtr (Ptr PgfExprEnum -> IO ())
 
 foreign import ccall "dynamic" callFetch :: Dynamic (Ptr PgfExprEnum -> Ptr PgfDB -> Ptr (#type prob_t) -> IO (StablePtr Expr))
 
-foreign import ccall pgf_parse_chart :: Ptr PgfDB -> Ptr Concr -> StablePtr Type -> Ptr PgfMarshaller -> Ptr PgfUnmarshaller -> Ptr PgfText -> Ptr PgfExn -> IO (Ptr PgfParseChart)
+foreign import ccall pgf_parse_chart :: Ptr PgfDB -> Ptr Concr -> StablePtr Type -> Ptr PgfMarshaller -> Ptr PgfUnmarshaller -> Ptr PgfText -> CInt -> Ptr PgfExn -> IO (Ptr PgfParseChart)
 
 foreign import ccall "&pgf_free_parse_chart" pgf_free_parse_chart :: FunPtr (Ptr PgfParseChart -> IO ())
 

src/runtime/python/ffi.c

@@ -24,6 +24,8 @@ PgfExnType handleError(PgfExn err)
     return err.type;
 }
 
+PyObject *ParseError;
+
 // ----------------------------------------------------------------------------
 // conversions
 

src/runtime/python/ffi.h

@@ -21,6 +21,8 @@ typedef struct {
 extern PyObject *PGFError;
 PgfExnType handleError(PgfExn err);
 
+extern PyObject *ParseError;
+
 PgfText *CString_AsPgfText(const char *s, size_t size);
 
 PgfText *PyUnicode_AsPgfText(PyObject *pystr);

src/runtime/python/pypgf.c

@@ -194,58 +194,84 @@ pgf_type_from_object(PyObject* obj) {
 static IterObject*
 Concr_parse(ConcrObject* self, PyObject *args, PyObject *keywds)
 {
-	static char *kwlist[] = {"sentence", "cat", "n", NULL};
+	static char *kwlist[] = {"sentence", "cat", "n", "robust", NULL};
 
-	const char *sentence = NULL;
+	PyObject *sentence = NULL;
-    size_t sentence_size;
 	PyObject* start = NULL;
 	int max_count = -1;
-	if (!PyArg_ParseTupleAndKeywords(args, keywds, "s#|Oi", kwlist,
+    int robust = 0;
-                                     &sentence, &sentence_size,
+	if (!PyArg_ParseTupleAndKeywords(args, keywds, "U|Oip", kwlist,
-                                     &start, &max_count))
+                                     &sentence,
+                                     &start, &max_count, &robust))
         return NULL;
 
-	IterObject* pyres = (IterObject*)
-		pgf_IterType.tp_alloc(&pgf_IterType, 0);
-	if (pyres == NULL) {
-		return NULL;
-	}
-
-	pyres->db = self->grammar->db;
-	pyres->source = (PyObject*) self;
-	Py_XINCREF(pyres->source);
-
-	pyres->max_count = max_count;
-	pyres->counter   = 0;
-
     PgfExn err;
 
 	TypeObject *type;
 	if (start == NULL) {
 		type = (TypeObject *) pgf_start_cat(self->grammar->db, self->grammar->revision, &unmarshaller, &err);
         if (handleError(err) != PGF_EXN_NONE) {
-            Py_DECREF(pyres);
             return NULL;
         }
 	} else {
 		type = pgf_type_from_object(start);
 	}
 	if (type == NULL) {
-		Py_DECREF(pyres);
 		return NULL;
 	}
 
-    PgfText *text_sentence = CString_AsPgfText(sentence, sentence_size);
+    PgfText *text_sentence = PyUnicode_AsPgfText(sentence);
-	pyres->res = (PgfExprEnum*)
+	PgfExprEnum* res =
 		pgf_parse(self->grammar->db, self->concr, (PgfType) type,
                   &marshaller, &unmarshaller,
-                  text_sentence, &err);
+                  text_sentence, robust, &err);
     FreePgfText(text_sentence);
-    if (handleError(err) != PGF_EXN_NONE) {
+    Py_DECREF(type);
-        Py_DECREF(pyres);
+    if (err.type == PGF_EXN_PARSE_ERROR) {
+        PyObject* py_offset = PyLong_FromLong(err.code);
+
+        Py_ssize_t len = PyUnicode_GET_LENGTH(sentence);
+
+        PyObject_SetAttrString(ParseError, "offset", py_offset);
+        if (err.code == len)
+            PyObject_SetAttrString(ParseError, "incomplete",  Py_True);
+        else {
+            Py_ssize_t end = PyUnicode_FindChar(sentence, ' ', err.code, -1, 1);
+            if (end == -1)
+                end = len;
+            else if (end == -2)
+                return NULL;
+            PyObject *py_token = PyUnicode_Substring(sentence, err.code, end);
+            if (py_token == NULL)
+                return NULL;
+
+            PyObject_SetAttrString(ParseError, "incomplete",  Py_False);
+            PyObject_SetAttrString(ParseError, "token", py_token);
+
+            Py_DECREF(py_token);
+            Py_DECREF(py_offset);
+        }
+
+        PyErr_Format(ParseError, "Parse error at position %d", err.code);
+        return NULL;
+    } else if (handleError(err) != PGF_EXN_NONE) {
         return NULL;
     }
 
+	IterObject* pyres = (IterObject*)
+		pgf_IterType.tp_alloc(&pgf_IterType, 0);
+	if (pyres == NULL) {
+		return NULL;
+	}
+
+	pyres->res = res;
+    pyres->db = self->grammar->db;
+	pyres->source = (PyObject*) self;
+	Py_INCREF(pyres->source);
+
+	pyres->max_count = max_count;
+	pyres->counter   = 0;
+
 	return pyres;
 }
 
@@ -2195,20 +2221,14 @@ pgf_showType(PyObject *self, PyObject *args)
 {
     PyObject *pylist;
     TypeObject *type;
-printf("pgf_showType 1\n");
+
     if (!PyArg_ParseTuple(args, "O!O!", &PyList_Type, &pylist, &pgf_TypeType, &type))
         return NULL;
-printf("pgf_showType 2\n");
     PgfPrintContext *ctxt = PyList_AsPgfPrintContext(pylist);
-printf("pgf_showType 3\n");
     PgfText *s = pgf_print_type((PgfType) type, ctxt, 0, &marshaller);
-printf("pgf_showType 4\n");
     FreePgfPrintContext(ctxt);
-printf("pgf_showType 5\n");
     PyObject *str = PyUnicode_FromStringAndSize(s->text, s->size);
-printf("pgf_showType 6\n");
     FreePgfText(s);
-printf("pgf_showType 7\n");
     return str;
 }
 
@@ -2379,6 +2399,9 @@ MOD_INIT(pgf)
     PGFError = PyErr_NewException("pgf.PGFError", NULL, NULL);
     ADD_TYPE_DIRECT("PGFError", PGFError);
 
+    ParseError = PyErr_NewException("pgf.ParseError", NULL, NULL);
+    ADD_TYPE_DIRECT("ParseError", ParseError);
+
     ADD_TYPE("PGF", pgf_PGFType);
     ADD_TYPE("Concr", pgf_ConcrType);
     ADD_TYPE("Transaction", pgf_TransactionType);