change in the API for literals

The API in the C runtime as well as in the Haskell, Python and Java binding
is changed. Now instead of adding the literal callbacks to the concrete syntax
you need to supply them every time when you need to parse. The main reason is:

- referentially transparent API for Haskell
- when we start using memory mapped files we will not be allowed to change
  anything in the grammar data structures. At that point the old API would
  be impossible to use.

src/runtime/c/pgf/data.h

@@ -199,8 +199,6 @@ typedef GuMap PgfCncOverloadMap;
 
 typedef struct PgfItem PgfItem;
 
-typedef GuMap PgfCallbacksMap;
-
 typedef GuVariant PgfSymbol;
 
 typedef enum {
@@ -280,7 +278,6 @@ struct PgfConcr {
     PgfSequences* sequences;
     GuBuf* pre_sequences;
 	PgfCIdMap* cnccats;
-	PgfCallbacksMap* callbacks;
 	int total_cats;
 	
 	GuPool* pool;     // if the language is loaded separately then this is the pool

src/runtime/c/pgf/literals.c

@@ -274,6 +274,19 @@ pgf_new_callbacks_map(PgfConcr* concr, GuPool *pool)
 	return callbacks;
 }
 
+void
+pgf_callbacks_map_add_literal(PgfConcr* concr, PgfCallbacksMap* callbacks,
+                              PgfCId cat, PgfLiteralCallback* callback)
+{
+	PgfCncCat* cnccat =
+		gu_map_get(concr->cnccats, cat, PgfCncCat*);
+	if (cnccat == NULL)
+		return;
+
+	gu_map_put(callbacks, cnccat,
+	           PgfLiteralCallback*, callback);
+}
+
 PgfCCat*
 pgf_literal_cat(PgfConcr* concr, PgfLiteral lit)
 {

src/runtime/c/pgf/literals.h

@@ -3,9 +3,6 @@
 
 #include <pgf/data.h>
 
-PgfCallbacksMap*
-pgf_new_callbacks_map(PgfConcr* concr, GuPool *pool);
-
 // literal for named entities recognition
 extern PgfLiteralCallback pgf_nerc_literal_callback;
 

src/runtime/c/pgf/parser.c

@@ -57,6 +57,7 @@ typedef struct {
     PgfItem* free_item;
 
     prob_t heuristic_factor;
+    PgfCallbacksMap* callbacks;
     prob_t meta_prob;
 	prob_t meta_token_prob;
 } PgfParsing;
@@ -1616,7 +1617,7 @@ pgf_parsing_symbol(PgfParsing* ps, PgfItem* item, PgfSymbol sym)
 				 * literal category so we must call the callback */
 
 				PgfLiteralCallback* callback =
-					gu_map_get(ps->concr->callbacks, 
+					gu_map_get(ps->callbacks,
 					           parg->ccat->cnccat, 
 							   PgfLiteralCallback*);
 
@@ -1861,7 +1862,7 @@ pgf_parsing_set_default_factors(PgfParsing* ps, PgfAbstr* abstr)
 }
 
 static PgfParsing*
-pgf_new_parsing(PgfConcr* concr, GuString sentence,
+pgf_new_parsing(PgfConcr* concr, GuString sentence, PgfCallbacksMap* callbacks,
                 GuPool* pool, GuPool* out_pool)
 {
 	PgfParsing* ps = gu_new(PgfParsing, pool);
@@ -1884,6 +1885,7 @@ pgf_new_parsing(PgfConcr* concr, GuString sentence,
 	ps->tp = NULL;
 	ps->free_item = NULL;
 	ps->heuristic_factor = 0;
+	ps->callbacks = callbacks;
 	ps->meta_prob = INFINITY;
 	ps->meta_token_prob = INFINITY;
 
@@ -2087,9 +2089,9 @@ pgf_parse_result_is_new(PgfExprState* st)
 // TODO: s/CId/Cat, add the cid to Cat, make Cat the key to CncCat
 static PgfParsing*
 pgf_parsing_init(PgfConcr* concr, PgfCId cat, size_t lin_idx, 
-                 GuString sentence, double heuristic_factor,
-                 GuExn* err,
-                 GuPool* pool, GuPool* out_pool)
+                 GuString sentence,
+                 double heuristic_factor, PgfCallbacksMap* callbacks,
+                 GuExn* err, GuPool* pool, GuPool* out_pool)
 {
 	PgfCncCat* cnccat =
 		gu_map_get(concr->cnccats, cat, PgfCncCat*);
@@ -2102,7 +2104,7 @@ pgf_parsing_init(PgfConcr* concr, PgfCId cat, size_t lin_idx,
 	gu_assert(lin_idx < cnccat->n_lins);
 
 	PgfParsing* ps =
-		pgf_new_parsing(concr, sentence, pool, out_pool);
+		pgf_new_parsing(concr, sentence, callbacks, pool, out_pool);
 
 	if (heuristic_factor >= 0) {
 		ps->heuristic_factor = heuristic_factor;
@@ -2312,12 +2314,14 @@ pgf_parse(PgfConcr* concr, PgfCId cat, GuString sentence,
           GuExn* err, 
           GuPool* pool, GuPool* out_pool)
 {
-    return pgf_parse_with_heuristics(concr, cat, sentence, -1.0, err, pool, out_pool);
+	PgfCallbacksMap* callbacks = pgf_new_callbacks_map(concr, out_pool); 
+    return pgf_parse_with_heuristics(concr, cat, sentence, -1.0, callbacks, err, pool, out_pool);
 }
 
 GuEnum*
 pgf_parse_with_heuristics(PgfConcr* concr, PgfCId cat, GuString sentence,
                           double heuristics,
+                          PgfCallbacksMap* callbacks,
                           GuExn* err,
                           GuPool* pool, GuPool* out_pool)
 {
@@ -2333,7 +2337,7 @@ pgf_parse_with_heuristics(PgfConcr* concr, PgfCId cat, GuString sentence,
 
 	// Begin parsing a sentence with the specified category
 	PgfParsing* ps =
-		pgf_parsing_init(concr, cat, 0, sentence, heuristics, err, pool, out_pool);
+		pgf_parsing_init(concr, cat, 0, sentence, heuristics, callbacks, err, pool, out_pool);
 	if (ps == NULL) {
 		return NULL;
 	}
@@ -2393,8 +2397,10 @@ pgf_complete(PgfConcr* concr, PgfCId cat, GuString sentence,
 	}
 
 	// Begin parsing a sentence with the specified category
+	PgfCallbacksMap* callbacks =
+		pgf_new_callbacks_map(concr, pool);
 	PgfParsing* ps =
-		pgf_parsing_init(concr, cat, 0, sentence, -1.0, err, pool, pool);
+		pgf_parsing_init(concr, cat, 0, sentence, -1.0, callbacks, err, pool, pool);
 	if (ps == NULL) {
 		return NULL;
 	}

src/runtime/c/pgf/pgf.c

@@ -32,6 +32,16 @@ pgf_read(const char* fpath,
 	return pgf;
 }
 
+PgfPGF*
+pgf_read_in(GuIn* in,
+            GuPool* pool, GuPool* tmp_pool, GuExn* err)
+{
+	PgfReader* rdr = pgf_new_reader(in, pool, tmp_pool, err);
+	PgfPGF* pgf = pgf_read_pgf(rdr);
+	pgf_reader_done(rdr, pgf);
+	return pgf;
+}
+
 GuString
 pgf_abstract_name(PgfPGF* pgf)
 {
@@ -171,38 +181,6 @@ pgf_has_linearization(PgfConcr* concr, PgfCId id)
 	return (overl_table != NULL);
 }
 
-GuPool*
-pgf_concr_get_pool(PgfConcr* concr)
-{
-	GuPool* pool = concr->pool;
-	if (pool == NULL)
-		pool = gu_container(concr->abstr, PgfPGF, abstract)->pool;
-	return pool;
-}
-
-void
-pgf_concr_add_literal(PgfConcr *concr, PgfCId cat,
-                      PgfLiteralCallback* callback,
-                      GuExn* err)
-{
-	if (concr->cnccats == NULL ||
-	    concr->callbacks == NULL) {
-		GuExnData* err_data = gu_raise(err, PgfExn);
-		if (err_data) {
-			err_data->data = "The concrete syntax is not loaded";
-			return;
-		}
-	}
-
-	PgfCncCat* cnccat =
-		gu_map_get(concr->cnccats, cat, PgfCncCat*);
-	if (cnccat == NULL)
-		return;
-
-	gu_map_put(concr->callbacks, cnccat,
-	           PgfLiteralCallback*, callback);
-}
-
 PgfExprProb*
 pgf_fun_get_ep(void* value)
 {

src/runtime/c/pgf/pgf.h

@@ -25,6 +25,10 @@ PgfPGF*
 pgf_read(const char* fpath,
          GuPool* pool, GuExn* err);
 
+PgfPGF*
+pgf_read_in(GuIn* in,
+            GuPool* pool, GuPool* tmp_pool, GuExn* err);
+
 void
 pgf_concrete_load(PgfConcr* concr, GuIn* in, GuExn* err);
 
@@ -114,9 +118,12 @@ GuEnum*
 pgf_lookup_word_prefix(PgfConcr *concr, GuString prefix,
                        GuPool* pool, GuExn* err);
 
+typedef GuMap PgfCallbacksMap;
+
 PgfExprEnum*
 pgf_parse_with_heuristics(PgfConcr* concr, PgfCId cat, 
                           GuString sentence, double heuristics,
+                          PgfCallbacksMap* callbacks,
                           GuExn* err,
                           GuPool* pool, GuPool* out_pool);
 
@@ -130,9 +137,6 @@ GuEnum*
 pgf_complete(PgfConcr* concr, PgfCId cat, GuString string, 
              GuString prefix, GuExn* err, GuPool* pool);
 
-GuPool*
-pgf_concr_get_pool(PgfConcr* concr);
-
 typedef struct PgfLiteralCallback PgfLiteralCallback;
 
 struct PgfLiteralCallback {
@@ -146,10 +150,12 @@ struct PgfLiteralCallback {
 	                      GuPool *out_pool);
 };
 
+PgfCallbacksMap*
+pgf_new_callbacks_map(PgfConcr* concr, GuPool *pool);
+
 void
-pgf_concr_add_literal(PgfConcr *concr, PgfCId cat,
-                      PgfLiteralCallback* callback,
-                      GuExn* err);
+pgf_callbacks_map_add_literal(PgfConcr* concr, PgfCallbacksMap* callbacks,
+                              PgfCId cat, PgfLiteralCallback* callback);
 
 void
 pgf_print(PgfPGF* pgf, GuOut* out, GuExn* err); 

src/runtime/c/pgf/reader.c

@@ -1,6 +1,5 @@
 #include "data.h"
 #include "expr.h"
-#include "literals.h"
 #include "evaluator.h"
 #include "reader.h"
 
@@ -1148,7 +1147,6 @@ pgf_read_concrete_content(PgfReader* rdr, PgfConcr* concr)
 	pgf_read_linrefs(rdr, concr);
 	pgf_read_ccats(rdr, concr);
 	concr->cnccats = pgf_read_cnccats(rdr, concr->abstr, concr);
-	concr->callbacks = pgf_new_callbacks_map(concr, rdr->opool); 
 	concr->total_cats = pgf_read_int(rdr);
 
 	GuMapItor clo1 = { pgf_read_ccat_cb };
@@ -1166,7 +1164,6 @@ pgf_read_concrete_init_header(PgfConcr* concr)
 	concr->fun_indices = NULL;
 	concr->coerce_idx = NULL;
 	concr->cnccats = NULL;
-	concr->callbacks = NULL;
 	concr->total_cats = 0;
 }
 

src/runtime/c/utils/pgf-parse.c

@@ -58,10 +58,6 @@ int main(int argc, char* argv[]) {
     goto fail;
   }
 
-  /* // Register a callback for the literal category Symbol */
-  /* pgf_parser_add_literal(from_concr, "Symb", */
-  /*                        &pgf_nerc_literal_callback); */
-
   clock_t end = clock();
   double cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC;
 
@@ -109,7 +105,8 @@ int main(int argc, char* argv[]) {
     clock_t start = clock();
 
     GuExn* parse_err = gu_new_exn(ppool);
-    GuEnum* result = pgf_parse_with_heuristics(concr, cat, line, heuristics, parse_err, ppool, ppool);
+    PgfCallbacksMap* callbacks = pgf_new_callbacks_map(concr, ppool);
+    GuEnum* result = pgf_parse_with_heuristics(concr, cat, line, heuristics, callbacks, parse_err, ppool, ppool);
 
     PgfExprProb* ep = NULL;
     if (gu_ok(parse_err))

src/runtime/c/utils/pgf-translate.c

@@ -87,8 +87,10 @@ int main(int argc, char* argv[]) {
 	}
 	
 	// Register a callback for the literal category Symbol
-	pgf_concr_add_literal(from_concr, "Symb",
-	                      &pgf_nerc_literal_callback, err);
+	PgfCallbacksMap* callbacks =
+		pgf_new_callbacks_map(from_concr, pool);
+	pgf_callbacks_map_add_literal(from_concr, callbacks,
+	                              "PN", &pgf_nerc_literal_callback);
 
 	// Create an output stream for stdout
 	GuOut* out = gu_file_out(stdout, pool);