a new unbiased statistical parser. it is still far from perfect use it on your own risk.

src/runtime/c/pgf/data.h

@@ -125,7 +125,7 @@ struct PgfPGF {
 extern GU_DECLARE_TYPE(PgfPGF, struct);
 
 typedef struct {
-	double prob;
+	float prob;
 	PgfExpr expr;
 } PgfExprProb;
 
@@ -148,6 +148,9 @@ struct PgfCatFun {
 struct PgfCat {
 	// TODO: Add cid here
 	PgfHypos context;
+
+	float meta_prob;
+
 	GuLength n_functions;
 	PgfCatFun functions[]; // XXX: resolve to PgfFunDecl*?
 };
@@ -189,6 +192,7 @@ struct PgfCCat {
 	PgfFunIds* lindefs;
 	size_t n_synprods;
 	PgfProductionSeq prods;
+	float viterbi_prob;
 	int fid;
 };
 

src/runtime/c/pgf/lexer.c

@@ -46,7 +46,7 @@ pgf_lexer_next_token(PgfLexer *lexer, GuExn* err, GuPool *pool)
 			if (gu_exn_is_raised(err))
 				goto stop;
 			
-			if (lexer->ucs == '.' && counter < 3) {
+			if (lexer->ucs == '.' && counter < 4) {
 				// perhaps an abreviation
 				gu_ucs_write(lexer->ucs, wtr, err);
 				if (gu_exn_is_raised(err))

src/runtime/c/pgf/literals.c

@@ -235,6 +235,8 @@ pgf_match_name_lit(PgfConcr* concr, PgfItem* item, PgfToken tok,
 		lit_str->val = gu_string_buf_freeze(sbuf, pool);
 
 		*out_ep = ep;
+	} else {
+		*out_ep = NULL;
 	}
 
 	gu_pool_free(tmp_pool);

src/runtime/c/pgf/parser.h

@@ -15,7 +15,7 @@
  *  @todo HOAS, dependent types...
  */
 
-typedef struct PgfParse PgfParse;
+typedef struct PgfParseState PgfParseState;
 
 /** @}
  * 
@@ -32,8 +32,9 @@ typedef struct PgfParse PgfParse;
  */
 
 /// Begin parsing
-PgfParse*
+PgfParseState*
-pgf_parser_parse(PgfConcr* concr, PgfCId cat, size_t lin_idx, GuPool* pool);
+pgf_parser_init_state(PgfConcr* concr, PgfCId cat, size_t lin_idx, 
+                      GuPool* pool);
 /**<
  * @param parser The parser to use
  *
@@ -48,8 +49,9 @@ pgf_parser_parse(PgfConcr* concr, PgfCId cat, size_t lin_idx, GuPool* pool);
 
 
 /// Feed a token to the parser
-PgfParse*
+PgfParseState*
-pgf_parse_token(PgfParse* parse, PgfToken tok, bool robust, GuPool* pool);
+pgf_parser_next_state(PgfParseState* prev, PgfToken tok,
+                      GuPool* pool);
 /**<
  * @param parse The current parse state
  *
@@ -87,7 +89,7 @@ typedef GuEnum PgfExprEnum;
 
 /// Retrieve the current parses from the parse state.
 PgfExprEnum*
-pgf_parse_result(PgfParse* parse, GuPool* pool);
+pgf_parse_result(PgfParseState* state, GuPool* pool);
 /**<
  * @param parse A parse state
  *
@@ -101,7 +103,7 @@ pgf_parse_result(PgfParse* parse, GuPool* pool);
  */
 
 PgfExpr
-pgf_parse_best_result(PgfParse* parse, GuPool* pool);
+pgf_parse_best_result(PgfParseState* state, GuPool* pool);
 
 
 int

src/runtime/c/pgf/printer.c

@@ -40,7 +40,7 @@ pgf_print_cat(GuMapItor* fn, const void* key, void* value,
 		pgf_print_hypo(hypo, 4, wtr, err);
 	}
 
-    gu_puts(" ;\n", wtr, err);
+    gu_printf(wtr, err, " ;   -- %f\n",cat->meta_prob);
 }
 
 void
@@ -56,7 +56,7 @@ pgf_print_absfun(GuMapItor* fn, const void* key, void* value,
     gu_string_write(name, wtr, err);
     gu_puts(" : ", wtr, err);
     pgf_print_type(fun->type, 0, wtr, err);
-    gu_puts(" ;\n", wtr, err);
+    gu_printf(wtr, err, " ;   -- %f\n", fun->ep.prob);
 }
 static void
 pgf_print_abstract(PgfCId absname, PgfAbstr* abstr, 

src/runtime/c/pgf/reader.c

@@ -34,7 +34,6 @@
 #define GU_LOG_ENABLE
 #include <gu/log.h>
 
-typedef struct PgfIdContext PgfIdContext;
 
 typedef GuMap PgfContsMap;
 
@@ -443,6 +442,7 @@ pgf_read_to_PgfCCatId(GuType* type, PgfReader* rdr, void* to)
         ccat->lindefs = gu_map_get(rdr->curr_lindefs, &fid, PgfFunIds*);
         ccat->n_synprods = 0;
         ccat->prods = gu_null_seq;
+        ccat->viterbi_prob = 0;
         ccat->fid = fid;
 
         gu_map_put(rdr->curr_concr->ccats, &fid, PgfCCat*, ccat);
@@ -465,6 +465,7 @@ pgf_read_to_PgfCCat(GuType* type, PgfReader* rdr, void* to)
     ccat->cnccat = NULL;
     ccat->lindefs = gu_map_get(rdr->curr_lindefs, fidp, PgfFunIds*);
 	ccat->prods = gu_new_seq(PgfProduction, n_prods, rdr->opool);
+	ccat->viterbi_prob = 0;
     ccat->fid = *fidp;
     
     size_t top = 0;
@@ -600,7 +601,7 @@ pgf_read_new_PgfFunDecl(GuType* type, PgfReader* rdr, GuPool* pool, size_t* size
 	}
 
 	absfun->ep.prob = - log(gu_in_f64be(rdr->in, rdr->err));
-	
+
 	PgfExprFun* expr_fun =
 		gu_new_variant(PGF_EXPR_FUN,
 					   PgfExprFun,
@@ -638,11 +639,33 @@ pgf_read_to_PgfFunId(GuType* type, PgfReader* rdr, void* to)
 	*(PgfFunId*) to = gu_list_elems(rdr->curr_concr->cncfuns)[id];
 }
 
+typedef struct {
+	GuMapItor fn;
+	PgfReader* rdr;
+} PgfIndexFn;
+
+static void
+pgf_compute_meta_probs(GuMapItor* fn, const void* key, void* value, GuExn* err)
+{
+	(void) (key && err);
+	
+    PgfCat* cat = *((PgfCat**) value);
+    
+    double mass = 0;
+    for (size_t i = 0; i < cat->n_functions; i++) {
+		mass += cat->functions[i].prob;
+	}
+    cat->meta_prob = - log(fabs(1 - mass));
+}
+
 static void
 pgf_read_to_PgfAbstr(GuType* type, PgfReader* rdr, void* to)
 {
 	rdr->curr_abstr = to;
 	pgf_read_to_struct(type, rdr, to);
+
+	PgfIndexFn clo = { { pgf_compute_meta_probs }, rdr };
+	gu_map_iter(rdr->curr_abstr->cats, &clo.fn, NULL);
 }
 
 static GU_DEFINE_TYPE(PgfLinDefs, GuIntMap, gu_ptr_type(PgfFunIds),
@@ -691,11 +714,6 @@ pgf_ccat_set_cnccat(PgfCCat* ccat)
 	return ccat->cnccat;
 }
 
-typedef struct {
-	GuMapItor fn;
-	PgfReader* rdr;
-} PgfIndexFn;
-
 static void
 pgf_read_ccat_cb(GuMapItor* fn, const void* key, void* value, GuExn* err)
 {
@@ -771,12 +789,6 @@ pgf_read_new_PgfConcr(GuType* type, PgfReader* rdr, GuPool* pool,
 	concr->total_cats = pgf_read_int(rdr);
 	concr->max_fid = concr->total_cats;
 
-	PgfIndexFn clo1 = { { pgf_read_ccat_cb }, rdr };
-	gu_map_iter(concr->ccats, &clo1.fn, NULL);
-	
-	PgfIndexFn clo2 = { { pgf_index_prods }, rdr };
-	gu_map_iter(concr->ccats, &clo2.fn, NULL);
-
     // set the function ids
     int n_funs = gu_list_length(concr->cncfuns);
     for (int funid = 0; funid < n_funs; funid++) {
@@ -788,6 +800,12 @@ pgf_read_new_PgfConcr(GuType* type, PgfReader* rdr, GuPool* pool,
         cncfun->ep = (absfun == NULL) ? NULL : &absfun->ep;
 	}
 
+	PgfIndexFn clo1 = { { pgf_read_ccat_cb }, rdr };
+	gu_map_iter(concr->ccats, &clo1.fn, NULL);
+	
+	PgfIndexFn clo2 = { { pgf_index_prods }, rdr };
+	gu_map_iter(concr->ccats, &clo2.fn, NULL);
+
 	return concr;
 }
 
@@ -821,6 +839,7 @@ pgf_read_new_PgfCncCat(GuType* type, PgfReader* rdr, GuPool* pool,
             ccat->lindefs = gu_map_get(rdr->curr_lindefs, &fid, PgfFunIds*);
             ccat->n_synprods = 0;
             ccat->prods = gu_null_seq;
+            ccat->viterbi_prob = 0;
             ccat->fid = fid;
 
             gu_map_put(rdr->curr_concr->ccats, &fid, PgfCCat*, ccat);

src/runtime/c/utils/pgf-translate.c

@@ -18,6 +18,33 @@
 #include <locale.h>
 #include <time.h>
 
+static void
+print_result(PgfExprProb* ep, PgfConcr* to_concr, 
+             GuWriter* wtr, GuExn* err, GuPool* ppool)
+{
+	// Write out the abstract syntax tree
+	gu_printf(wtr, err, " [%f] ", ep->prob);
+	pgf_print_expr(ep->expr, 0, wtr, err);
+	gu_putc('\n', wtr, err);
+
+	// Enumerate the concrete syntax trees corresponding
+	// to the abstract tree.
+	GuEnum* cts = pgf_lzr_concretize(to_concr, ep->expr, ppool);
+	while (true) {
+		PgfCncTree ctree =
+			gu_next(cts, PgfCncTree, ppool);
+		if (gu_variant_is_null(ctree)) {
+			break;
+		}
+		gu_putc(' ', wtr, err);
+		// Linearize the concrete tree as a simple
+		// sequence of strings.
+		pgf_lzr_linearize_simple(to_concr	, ctree, 0, wtr, err);
+		gu_putc('\n', wtr, err);
+		gu_writer_flush(wtr, err);
+	}
+}
+
 int main(int argc, char* argv[]) {
 	// Set the character locale, so we can produce proper output.
 	setlocale(LC_CTYPE, "");
@@ -32,15 +59,7 @@ int main(int argc, char* argv[]) {
 	}
 	char* filename = argv[1];
 
-	GuString cat;
+	GuString cat = gu_str_string(argv[2], pool);
-	bool robust_mode;
-	if (argv[2][0] == '.') {
-		cat = gu_str_string(argv[2]+1, pool);
-		robust_mode = true;
-	} else {
-		cat = gu_str_string(argv[2], pool);
-		robust_mode = false;
-	}
 
 	GuString from_lang = gu_str_string(argv[3], pool);
 	GuString to_lang = gu_str_string(argv[4], pool);
@@ -83,10 +102,6 @@ int main(int argc, char* argv[]) {
 	pgf_parser_add_literal(from_concr, gu_str_string("Symb", pool),
 	                       &pgf_nerc_literal_callback);
 
-	// Arbitrarily choose linearization index 0. Usually the initial
-	// categories we are interested in only have one field.
-	int lin_idx = 0;
-
 	// Create an output stream for stdout
 	GuOut* out = gu_file_out(stdout, pool);
 
@@ -95,6 +110,11 @@ int main(int argc, char* argv[]) {
 	// Use a writer with hard-coded utf-8 encoding for now.
 	GuWriter* wtr = gu_new_utf8_writer(out, pool);
 
+	// We will keep the latest results in the 'ppool' and
+	// we will iterate over them by using 'result'.
+	GuPool* ppool = NULL;
+	GuEnum* result = NULL;
+
 	// The interactive translation loop.
 	// XXX: This currently reads stdin directly, so it doesn't support
 	// encodings properly. TODO: use a locale reader for input
@@ -109,20 +129,49 @@ int main(int argc, char* argv[]) {
 				status = EXIT_FAILURE;
 			}
 			break;
-		} else if (line[0] == '\0') {
+		} else if (strcmp(line, "") == 0) {
 			// End nicely on empty input
 			break;
+		} else if (strcmp(line, "\n") == 0) {
+			// Empty line -> show the next tree for the last sentence
+
+			if (result != NULL) {
+				clock_t start = clock();
+
+				PgfExprProb* ep = gu_next(result, PgfExprProb*, ppool);
+
+				clock_t end = clock();
+				double cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC;
+				printf("%.2f sec\n", cpu_time_used);
+
+				// The enumerator will return a null variant at the
+				// end of the results.
+				if (ep == NULL) {
+					goto fail_parse;
+				}
+				
+				print_result(ep, to_concr, wtr, err, ppool);
+			}
+			continue;
 		}
+
+		// We release the last results
+		if (ppool != NULL) {
+			gu_pool_free(ppool);
+			ppool  = NULL;
+			result = NULL;
+		}
+		
 		// We create a temporary pool for translating a single
 		// sentence, so our memory usage doesn't increase over time.
-		GuPool* ppool = gu_new_pool();
+		ppool = gu_new_pool();
 
 		clock_t start = clock();
 
 		// Begin parsing a sentence of the specified category
-		PgfParse* parse =
+		PgfParseState* state =
-			pgf_parser_parse(from_concr, cat, lin_idx, pool);
+			pgf_parser_init_state(from_concr, cat, 0, pool);
-		if (parse == NULL) {
+		if (state == NULL) {
 			fprintf(stderr, "Couldn't begin parsing\n");
 			status = EXIT_FAILURE;
 			break;
@@ -133,13 +182,13 @@ int main(int argc, char* argv[]) {
 		PgfLexer *lexer =
 			pgf_new_lexer(rdr, pool);
 
-		// naive tokenization
+		// Tokenization
 		GuExn* lex_err = gu_new_exn(NULL, gu_kind(type), pool);
 		PgfToken tok = pgf_lexer_next_token(lexer, lex_err, pool);
 		while (!gu_exn_is_raised(lex_err)) {
 			// feed the token to get a new parse state
-			parse = pgf_parse_token(parse, tok, robust_mode, ppool);
+			state = pgf_parser_next_state(state, tok, ppool);
-			if (!parse) {
+			if (!state) {
 				gu_puts("Unexpected token: \"", wtr, err);
 				gu_string_write(tok, wtr, err);
 				gu_puts("\"\n", wtr, err);
@@ -149,64 +198,29 @@ int main(int argc, char* argv[]) {
 			tok = pgf_lexer_next_token(lexer, lex_err, pool);
 		}
 
-		if (robust_mode) {
+		// Now begin enumerating the resulting syntax trees
-			PgfExpr expr = pgf_parse_best_result(parse, ppool);
+		result = pgf_parse_result(state, ppool);
 
-			clock_t end = clock();
+		PgfExprProb* ep = gu_next(result, PgfExprProb*, ppool);
 
-			double cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC;
+		clock_t end = clock();
-			printf("%.2f sec\n", cpu_time_used);
+		double cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC;
-			
+		printf("%.2f sec\n", cpu_time_used);
-			if (!gu_variant_is_null(expr)) {
-				gu_putc(' ', wtr, err);
-				// Write out the abstract syntax tree
-				pgf_print_expr(expr, 0, wtr, err);
-				gu_putc('\n', wtr, err);
-			}
-		} else {
-			// Now begin enumerating the resulting syntax trees
-			GuEnum* result = pgf_parse_result(parse, ppool);
 
-			clock_t end = clock();
+		// The enumerator will return a null variant at the
-
+		// end of the results.
-			double cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC;
+		if (ep == NULL) {
-			printf("%.2f sec\n", cpu_time_used);
+			goto fail_parse;
-
-			while (true) {
-				PgfExpr expr = gu_next(result, PgfExpr, ppool);
-
-				// The enumerator will return a null variant at the
-				// end of the results.
-				if (gu_variant_is_null(expr)) {
-					break;
-				}
-				gu_putc(' ', wtr, err);
-				// Write out the abstract syntax tree
-				pgf_print_expr(expr, 0, wtr, err);
-				gu_putc('\n', wtr, err);
-
-				// Enumerate the concrete syntax trees corresponding
-				// to the abstract tree.
-				GuEnum* cts = pgf_lzr_concretize(to_concr, expr, ppool);
-				while (true) {
-					PgfCncTree ctree =
-						gu_next(cts, PgfCncTree, ppool);
-					if (gu_variant_is_null(ctree)) {
-						break;
-					}
-					gu_puts("  ", wtr, err);
-					// Linearize the concrete tree as a simple
-					// sequence of strings.
-					pgf_lzr_linearize_simple(to_concr	, ctree, lin_idx,
-								 wtr, err);
-					gu_putc('\n', wtr, err);
-					gu_writer_flush(wtr, err);
-				}
-			}
 		}
+		
+		print_result(ep, to_concr, wtr, err, ppool);
+
+		continue;
 	fail_parse:
 		// Free all resources allocated during parsing and linearization
 		gu_pool_free(ppool);
+		ppool = NULL;
+		result = NULL;
 	}
 fail_concr:
 fail_read: