change in the API for literals

The API in the C runtime as well as in the Haskell, Python and Java binding
is changed. Now instead of adding the literal callbacks to the concrete syntax
you need to supply them every time when you need to parse. The main reason is:

- referentially transparent API for Haskell
- when we start using memory mapped files we will not be allowed to change
  anything in the grammar data structures. At that point the old API would
  be impossible to use.

src/runtime/c/pgf/data.h

@@ -199,8 +199,6 @@ typedef GuMap PgfCncOverloadMap;
 
 typedef struct PgfItem PgfItem;
 
-typedef GuMap PgfCallbacksMap;
-
 typedef GuVariant PgfSymbol;
 
 typedef enum {
@@ -280,7 +278,6 @@ struct PgfConcr {
     PgfSequences* sequences;
     GuBuf* pre_sequences;
 	PgfCIdMap* cnccats;
-	PgfCallbacksMap* callbacks;
 	int total_cats;
 	
 	GuPool* pool;     // if the language is loaded separately then this is the pool

src/runtime/c/pgf/literals.c

@@ -274,6 +274,19 @@ pgf_new_callbacks_map(PgfConcr* concr, GuPool *pool)
 	return callbacks;
 }
 
+void
+pgf_callbacks_map_add_literal(PgfConcr* concr, PgfCallbacksMap* callbacks,
+                              PgfCId cat, PgfLiteralCallback* callback)
+{
+	PgfCncCat* cnccat =
+		gu_map_get(concr->cnccats, cat, PgfCncCat*);
+	if (cnccat == NULL)
+		return;
+
+	gu_map_put(callbacks, cnccat,
+	           PgfLiteralCallback*, callback);
+}
+
 PgfCCat*
 pgf_literal_cat(PgfConcr* concr, PgfLiteral lit)
 {

src/runtime/c/pgf/literals.h

@@ -3,9 +3,6 @@
 
 #include <pgf/data.h>
 
-PgfCallbacksMap*
-pgf_new_callbacks_map(PgfConcr* concr, GuPool *pool);
-
 // literal for named entities recognition
 extern PgfLiteralCallback pgf_nerc_literal_callback;
 

src/runtime/c/pgf/parser.c

@@ -57,6 +57,7 @@ typedef struct {
     PgfItem* free_item;
 
     prob_t heuristic_factor;
+    PgfCallbacksMap* callbacks;
     prob_t meta_prob;
 	prob_t meta_token_prob;
 } PgfParsing;
@@ -1616,7 +1617,7 @@ pgf_parsing_symbol(PgfParsing* ps, PgfItem* item, PgfSymbol sym)
 				 * literal category so we must call the callback */
 
 				PgfLiteralCallback* callback =
-					gu_map_get(ps->concr->callbacks, 
+					gu_map_get(ps->callbacks,
 					           parg->ccat->cnccat, 
 							   PgfLiteralCallback*);
 
@@ -1861,7 +1862,7 @@ pgf_parsing_set_default_factors(PgfParsing* ps, PgfAbstr* abstr)
 }
 
 static PgfParsing*
-pgf_new_parsing(PgfConcr* concr, GuString sentence,
+pgf_new_parsing(PgfConcr* concr, GuString sentence, PgfCallbacksMap* callbacks,
                 GuPool* pool, GuPool* out_pool)
 {
 	PgfParsing* ps = gu_new(PgfParsing, pool);
@@ -1884,6 +1885,7 @@ pgf_new_parsing(PgfConcr* concr, GuString sentence,
 	ps->tp = NULL;
 	ps->free_item = NULL;
 	ps->heuristic_factor = 0;
+	ps->callbacks = callbacks;
 	ps->meta_prob = INFINITY;
 	ps->meta_token_prob = INFINITY;
 
@@ -2087,9 +2089,9 @@ pgf_parse_result_is_new(PgfExprState* st)
 // TODO: s/CId/Cat, add the cid to Cat, make Cat the key to CncCat
 static PgfParsing*
 pgf_parsing_init(PgfConcr* concr, PgfCId cat, size_t lin_idx, 
-                 GuString sentence, double heuristic_factor,
+                 GuString sentence,
-                 GuExn* err,
+                 double heuristic_factor, PgfCallbacksMap* callbacks,
-                 GuPool* pool, GuPool* out_pool)
+                 GuExn* err, GuPool* pool, GuPool* out_pool)
 {
 	PgfCncCat* cnccat =
 		gu_map_get(concr->cnccats, cat, PgfCncCat*);
@@ -2102,7 +2104,7 @@ pgf_parsing_init(PgfConcr* concr, PgfCId cat, size_t lin_idx,
 	gu_assert(lin_idx < cnccat->n_lins);
 
 	PgfParsing* ps =
-		pgf_new_parsing(concr, sentence, pool, out_pool);
+		pgf_new_parsing(concr, sentence, callbacks, pool, out_pool);
 
 	if (heuristic_factor >= 0) {
 		ps->heuristic_factor = heuristic_factor;
@@ -2312,12 +2314,14 @@ pgf_parse(PgfConcr* concr, PgfCId cat, GuString sentence,
           GuExn* err, 
           GuPool* pool, GuPool* out_pool)
 {
-    return pgf_parse_with_heuristics(concr, cat, sentence, -1.0, err, pool, out_pool);
+	PgfCallbacksMap* callbacks = pgf_new_callbacks_map(concr, out_pool); 
+    return pgf_parse_with_heuristics(concr, cat, sentence, -1.0, callbacks, err, pool, out_pool);
 }
 
 GuEnum*
 pgf_parse_with_heuristics(PgfConcr* concr, PgfCId cat, GuString sentence,
                           double heuristics,
+                          PgfCallbacksMap* callbacks,
                           GuExn* err,
                           GuPool* pool, GuPool* out_pool)
 {
@@ -2333,7 +2337,7 @@ pgf_parse_with_heuristics(PgfConcr* concr, PgfCId cat, GuString sentence,
 
 	// Begin parsing a sentence with the specified category
 	PgfParsing* ps =
-		pgf_parsing_init(concr, cat, 0, sentence, heuristics, err, pool, out_pool);
+		pgf_parsing_init(concr, cat, 0, sentence, heuristics, callbacks, err, pool, out_pool);
 	if (ps == NULL) {
 		return NULL;
 	}
@@ -2393,8 +2397,10 @@ pgf_complete(PgfConcr* concr, PgfCId cat, GuString sentence,
 	}
 
 	// Begin parsing a sentence with the specified category
+	PgfCallbacksMap* callbacks =
+		pgf_new_callbacks_map(concr, pool);
 	PgfParsing* ps =
-		pgf_parsing_init(concr, cat, 0, sentence, -1.0, err, pool, pool);
+		pgf_parsing_init(concr, cat, 0, sentence, -1.0, callbacks, err, pool, pool);
 	if (ps == NULL) {
 		return NULL;
 	}

src/runtime/c/pgf/pgf.c

@@ -32,6 +32,16 @@ pgf_read(const char* fpath,
 	return pgf;
 }
 
+PgfPGF*
+pgf_read_in(GuIn* in,
+            GuPool* pool, GuPool* tmp_pool, GuExn* err)
+{
+	PgfReader* rdr = pgf_new_reader(in, pool, tmp_pool, err);
+	PgfPGF* pgf = pgf_read_pgf(rdr);
+	pgf_reader_done(rdr, pgf);
+	return pgf;
+}
+
 GuString
 pgf_abstract_name(PgfPGF* pgf)
 {
@@ -171,38 +181,6 @@ pgf_has_linearization(PgfConcr* concr, PgfCId id)
 	return (overl_table != NULL);
 }
 
-GuPool*
-pgf_concr_get_pool(PgfConcr* concr)
-{
-	GuPool* pool = concr->pool;
-	if (pool == NULL)
-		pool = gu_container(concr->abstr, PgfPGF, abstract)->pool;
-	return pool;
-}
-
-void
-pgf_concr_add_literal(PgfConcr *concr, PgfCId cat,
-                      PgfLiteralCallback* callback,
-                      GuExn* err)
-{
-	if (concr->cnccats == NULL ||
-	    concr->callbacks == NULL) {
-		GuExnData* err_data = gu_raise(err, PgfExn);
-		if (err_data) {
-			err_data->data = "The concrete syntax is not loaded";
-			return;
-		}
-	}
-
-	PgfCncCat* cnccat =
-		gu_map_get(concr->cnccats, cat, PgfCncCat*);
-	if (cnccat == NULL)
-		return;
-
-	gu_map_put(concr->callbacks, cnccat,
-	           PgfLiteralCallback*, callback);
-}
-
 PgfExprProb*
 pgf_fun_get_ep(void* value)
 {

src/runtime/c/pgf/pgf.h

@@ -25,6 +25,10 @@ PgfPGF*
 pgf_read(const char* fpath,
          GuPool* pool, GuExn* err);
 
+PgfPGF*
+pgf_read_in(GuIn* in,
+            GuPool* pool, GuPool* tmp_pool, GuExn* err);
+
 void
 pgf_concrete_load(PgfConcr* concr, GuIn* in, GuExn* err);
 
@@ -114,9 +118,12 @@ GuEnum*
 pgf_lookup_word_prefix(PgfConcr *concr, GuString prefix,
                        GuPool* pool, GuExn* err);
 
+typedef GuMap PgfCallbacksMap;
+
 PgfExprEnum*
 pgf_parse_with_heuristics(PgfConcr* concr, PgfCId cat, 
                           GuString sentence, double heuristics,
+                          PgfCallbacksMap* callbacks,
                           GuExn* err,
                           GuPool* pool, GuPool* out_pool);
 
@@ -130,9 +137,6 @@ GuEnum*
 pgf_complete(PgfConcr* concr, PgfCId cat, GuString string, 
              GuString prefix, GuExn* err, GuPool* pool);
 
-GuPool*
-pgf_concr_get_pool(PgfConcr* concr);
-
 typedef struct PgfLiteralCallback PgfLiteralCallback;
 
 struct PgfLiteralCallback {
@@ -146,10 +150,12 @@ struct PgfLiteralCallback {
 	                      GuPool *out_pool);
 };
 
+PgfCallbacksMap*
+pgf_new_callbacks_map(PgfConcr* concr, GuPool *pool);
+
 void
-pgf_concr_add_literal(PgfConcr *concr, PgfCId cat,
+pgf_callbacks_map_add_literal(PgfConcr* concr, PgfCallbacksMap* callbacks,
-                      PgfLiteralCallback* callback,
+                              PgfCId cat, PgfLiteralCallback* callback);
-                      GuExn* err);
 
 void
 pgf_print(PgfPGF* pgf, GuOut* out, GuExn* err); 

src/runtime/c/pgf/reader.c

@@ -1,6 +1,5 @@
 #include "data.h"
 #include "expr.h"
-#include "literals.h"
 #include "evaluator.h"
 #include "reader.h"
 
@@ -1148,7 +1147,6 @@ pgf_read_concrete_content(PgfReader* rdr, PgfConcr* concr)
 	pgf_read_linrefs(rdr, concr);
 	pgf_read_ccats(rdr, concr);
 	concr->cnccats = pgf_read_cnccats(rdr, concr->abstr, concr);
-	concr->callbacks = pgf_new_callbacks_map(concr, rdr->opool); 
 	concr->total_cats = pgf_read_int(rdr);
 
 	GuMapItor clo1 = { pgf_read_ccat_cb };
@@ -1166,7 +1164,6 @@ pgf_read_concrete_init_header(PgfConcr* concr)
 	concr->fun_indices = NULL;
 	concr->coerce_idx = NULL;
 	concr->cnccats = NULL;
-	concr->callbacks = NULL;
 	concr->total_cats = 0;
 }
 

src/runtime/c/utils/pgf-parse.c

@@ -58,10 +58,6 @@ int main(int argc, char* argv[]) {
     goto fail;
   }
 
-  /* // Register a callback for the literal category Symbol */
-  /* pgf_parser_add_literal(from_concr, "Symb", */
-  /*                        &pgf_nerc_literal_callback); */
-
   clock_t end = clock();
   double cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC;
 
@@ -109,7 +105,8 @@ int main(int argc, char* argv[]) {
     clock_t start = clock();
 
     GuExn* parse_err = gu_new_exn(ppool);
-    GuEnum* result = pgf_parse_with_heuristics(concr, cat, line, heuristics, parse_err, ppool, ppool);
+    PgfCallbacksMap* callbacks = pgf_new_callbacks_map(concr, ppool);
+    GuEnum* result = pgf_parse_with_heuristics(concr, cat, line, heuristics, callbacks, parse_err, ppool, ppool);
 
     PgfExprProb* ep = NULL;
     if (gu_ok(parse_err))

src/runtime/c/utils/pgf-translate.c

@@ -87,8 +87,10 @@ int main(int argc, char* argv[]) {
 	}
 	
 	// Register a callback for the literal category Symbol
-	pgf_concr_add_literal(from_concr, "Symb",
+	PgfCallbacksMap* callbacks =
-	                      &pgf_nerc_literal_callback, err);
+		pgf_new_callbacks_map(from_concr, pool);
+	pgf_callbacks_map_add_literal(from_concr, callbacks,
+	                              "PN", &pgf_nerc_literal_callback);
 
 	// Create an output stream for stdout
 	GuOut* out = gu_file_out(stdout, pool);

src/runtime/haskell-bind/PGF2.hsc

@@ -26,6 +26,7 @@ module PGF2 (-- * PGF
 
 import Prelude hiding (fromEnum)
 import Control.Exception(Exception,throwIO)
+import Control.Monad(forM_)
 import System.IO.Unsafe(unsafePerformIO,unsafeInterleaveIO)
 import PGF2.FFI
 
@@ -231,14 +232,18 @@ getAnalysis ref self c_lemma c_anal prob exn = do
   writeIORef ref ((lemma, anal, prob):ans)
 
 parse :: Concr -> String -> String -> Either String [(Expr,Float)]
-parse lang cat sent =
+parse lang cat sent = parse_with_heuristics lang cat sent (-1.0) []
+
+parse_with_heuristics :: Concr -> String -> String -> Double -> [(String, Int -> String -> Int -> Maybe (Expr,Float,Int))] -> Either String [(Expr,Float)]
+parse_with_heuristics lang cat sent heuristic callbacks =
   unsafePerformIO $
     do parsePl <- gu_new_pool
        exprPl  <- gu_new_pool
        exn     <- gu_new_exn parsePl
        enum    <- withCString cat $ \cat ->
-                    withCString sent $ \sent ->
+                    withCString sent $ \sent -> do
-                      pgf_parse (concr lang) cat sent exn parsePl exprPl
+                      callbacks_map <- mkCallbacksMap (concr lang) callbacks parsePl
+                      pgf_parse_with_heuristics (concr lang) cat sent heuristic callbacks_map exn parsePl exprPl
        failed  <- gu_exn_is_raised exn
        if failed
          then do is_parse_error <- gu_exn_caught exn gu_exn_type_PgfParseError
@@ -263,28 +268,17 @@ parse lang cat sent =
                  exprs    <- fromPgfExprEnum enum parseFPl (lang,exprFPl)
                  return (Right exprs)
 
-addLiteral :: Concr -> String -> (Int -> String -> Int -> Maybe (Expr,Float,Int)) -> IO ()
+mkCallbacksMap :: Ptr PgfConcr -> [(String, Int -> String -> Int -> Maybe (Expr,Float,Int))] -> Ptr GuPool -> IO (Ptr PgfCallbacksMap)
-addLiteral lang cat match =
+mkCallbacksMap concr callbacks pool = do
-  withCString cat $ \ccat ->
+  callbacks_map <- pgf_new_callbacks_map concr pool
-  withGuPool  $ \tmp_pool -> do
+  forM_ callbacks $ \(cat,match) ->
-    callback <- hspgf_new_literal_callback (concr lang)
+    withCString cat $ \ccat -> do
-    match    <- wrapLiteralMatchCallback match_callback
+      match    <- wrapLiteralMatchCallback (match_callback match)
       predict  <- wrapLiteralPredictCallback predict_callback
-    (#poke PgfLiteralCallback, match)   callback match
+      hspgf_callbacks_map_add_literal concr callbacks_map ccat match predict pool
-    (#poke PgfLiteralCallback, predict) callback predict
+  return callbacks_map
-    exn      <- gu_new_exn tmp_pool
-    pgf_concr_add_literal (concr lang) ccat callback exn
-    failed <- gu_exn_is_raised exn
-    if failed
-      then do is_exn <- gu_exn_caught exn gu_exn_type_PgfExn
-              if is_exn
-                then do c_msg <- (#peek GuExn, data.data) exn
-                        msg <- peekCString c_msg
-                        throwIO (PGFError msg)
-                else throwIO (PGFError "The literal cannot be added")
-      else return ()
   where
-    match_callback _ clin_idx csentence poffset out_pool = do
+    match_callback match _ clin_idx csentence poffset out_pool = do
       sentence <- peekCString csentence
       coffset  <- peek poffset
       offset <- alloca $ \pcsentence -> do

src/runtime/haskell-bind/PGF2/FFI.hs

@@ -91,7 +91,7 @@ data PgfFullFormEntry
 data PgfMorphoCallback
 data PgfPrintContext
 data PgfType
-data PgfLiteralCallback
+data PgfCallbacksMap
 
 foreign import ccall "pgf/pgf.h pgf_read"
   pgf_read :: CString -> Ptr GuPool -> Ptr GuExn -> IO (Ptr PgfPGF)
@@ -138,8 +138,8 @@ foreign import ccall "pgf/pgf.h pgf_print_name"
 foreign import ccall "pgf/pgf.h pgf_linearize"
   pgf_linearize :: Ptr PgfConcr -> PgfExpr -> Ptr GuOut -> Ptr GuExn -> IO ()
 
-foreign import ccall "pgf/pgf.h pgf_parse"
+foreign import ccall "pgf/pgf.h pgf_parse_with_heuristics"
-  pgf_parse :: Ptr PgfConcr -> CString -> CString -> Ptr GuExn -> Ptr GuPool -> Ptr GuPool -> IO (Ptr GuEnum)
+  pgf_parse_with_heuristics :: Ptr PgfConcr -> CString -> CString -> Double -> Ptr PgfCallbacksMap -> Ptr GuExn -> Ptr GuPool -> Ptr GuPool -> IO (Ptr GuEnum)
 
 type LiteralMatchCallback = Ptr () -> CInt -> CString -> Ptr CInt -> Ptr GuPool -> IO (Ptr PgfExprProb)
 
@@ -151,11 +151,11 @@ type LiteralPredictCallback = Ptr () -> CInt -> CString -> Ptr GuPool -> IO (Ptr
 foreign import ccall "wrapper"
   wrapLiteralPredictCallback :: LiteralPredictCallback -> IO (FunPtr LiteralPredictCallback)
 
-foreign import ccall 
+foreign import ccall "pgf/pgf.h pgf_new_callbacks_map"
-  hspgf_new_literal_callback :: Ptr PgfConcr -> IO (Ptr PgfLiteralCallback)
+  pgf_new_callbacks_map :: Ptr PgfConcr -> Ptr GuPool -> IO (Ptr PgfCallbacksMap)
 
-foreign import ccall "pgf/pgf.h pgf_concr_add_literal"
+foreign import ccall
-  pgf_concr_add_literal :: Ptr PgfConcr -> CString -> Ptr PgfLiteralCallback -> Ptr GuExn -> IO ()
+  hspgf_callbacks_map_add_literal :: Ptr PgfConcr -> Ptr PgfCallbacksMap -> CString -> FunPtr LiteralMatchCallback -> FunPtr LiteralPredictCallback -> Ptr GuPool -> IO ()
 
 foreign import ccall "pgf/pgf.h pgf_lookup_morpho"
   pgf_lookup_morpho :: Ptr PgfConcr -> CString -> Ptr PgfMorphoCallback -> Ptr GuExn -> IO ()

src/runtime/haskell-bind/utils.c

@@ -17,13 +17,15 @@ hspgf_literal_callback_fin(GuFinalizer* self)
 		hs_free_fun_ptr((HsFunPtr) callback->callback.predict);
 }
 
-PgfLiteralCallback*
+void
-hspgf_new_literal_callback(PgfConcr* concr) {
+hspgf_callbacks_map_add_literal(PgfConcr* concr, PgfCallbacksMap* callbacks,
-	GuPool* pool = pgf_concr_get_pool(concr);
+                                PgfCId cat, HsFunPtr match, HsFunPtr predict,
+                                GuPool* pool)
+{
 	HSPgfLiteralCallback* callback = gu_new(HSPgfLiteralCallback, pool);
-	callback->callback.match   = NULL;
+	callback->callback.match   = (void*) match;
-	callback->callback.predict = NULL;
+	callback->callback.predict = (void*) predict;
 	callback->fin.fn = hspgf_literal_callback_fin;
 	gu_pool_finally(pool, &callback->fin);
-	return &callback->callback;
+	pgf_callbacks_map_add_literal(concr, callbacks, cat, &callback->callback);
 }

src/runtime/java/Test.java

@@ -20,7 +20,6 @@ public class Test {
 		System.out.println(gr.getAbstractName());
 		for (Map.Entry<String,Concr> entry : gr.getLanguages().entrySet()) {
 			System.out.println(entry.getKey()+" "+entry.getValue()+" "+entry.getValue().getName());
-			entry.getValue().addLiteral("PN", new NercLiteralCallback(gr,entry.getValue()));
 		}
 
 		Concr eng = gr.getLanguages().get("SimpleEng");

src/runtime/java/jpgf.c

@@ -1,5 +1,4 @@
 #include <pgf/pgf.h>
-#include <pgf/reader.h>
 #include <pgf/linearizer.h>
 #include <gu/mem.h>
 #include <gu/exn.h>
@@ -223,10 +222,7 @@ Java_org_grammaticalframework_pgf_PGF_readPGF__Ljava_io_InputStream_2(JNIEnv *en
 	GuExn* err = gu_exn(tmp_pool);
 
 	// Read the PGF grammar.
-	PgfReader* rdr = pgf_new_reader(in, pool, tmp_pool, err);
+	PgfPGF* pgf = pgf_read_in(in, pool, tmp_pool, err);
-	PgfPGF* pgf = pgf_read_pgf(rdr);
-	pgf_reader_done(rdr, pgf);
-
 	if (!gu_ok(err)) {
 		throw_string_exception(env, "org/grammaticalframework/pgf/PGFError", "The grammar cannot be loaded");
 		gu_pool_free(pool);
@@ -371,11 +367,169 @@ Java_org_grammaticalframework_pgf_Concr_unload(JNIEnv* env, jobject self)
 	pgf_concrete_unload(get_ref(env, self));
 }
 
-JNIEXPORT jobject JNICALL 
+JNIEXPORT jlong JNICALL 
-Java_org_grammaticalframework_pgf_Parser_parse
+Java_org_grammaticalframework_pgf_Parser_newCallbacksMap
-  (JNIEnv* env, jclass clazz, jobject concr, jstring jstartCat, jstring js)
+  (JNIEnv* env, jclass clazz, jobject jconcr, jobject jpool)
 {
-	GuPool* pool = gu_new_pool();
+	return p2l(pgf_new_callbacks_map(get_ref(env, jconcr), get_ref(env, jpool)));
+}
+
+typedef struct {
+	PgfLiteralCallback callback;
+	jobject jcallback;
+	jmethodID match_methodId;
+	jmethodID predict_methodId;
+	GuFinalizer fin;
+} JPgfLiteralCallback;
+
+typedef struct {
+	GuEnum en;
+	jobject jiterator;
+	GuFinalizer fin;
+} JPgfTokenProbEnum;
+
+static PgfExprProb*
+jpgf_literal_callback_match(PgfLiteralCallback* self,
+                            size_t lin_idx,
+                            GuString sentence, size_t* poffset,
+                            GuPool *out_pool)
+{
+	JPgfLiteralCallback* callback = gu_container(self, JPgfLiteralCallback, callback);
+
+	JNIEnv *env;
+    (*cachedJVM)->AttachCurrentThread(cachedJVM, &env, NULL);
+
+	jstring jsentence = gu2j_string(env, sentence);
+	size_t  joffset   = gu2j_string_offset(sentence, *poffset);
+	jobject result = (*env)->CallObjectMethod(env, callback->jcallback, callback->match_methodId, lin_idx, jsentence, joffset);
+	if (result == NULL)
+		return NULL;
+
+	jclass result_class = (*env)->GetObjectClass(env, result);
+	
+	jfieldID epId = (*env)->GetFieldID(env, result_class, "ep", "Lorg/grammaticalframework/pgf/ExprProb;");
+	jobject jep = (*env)->GetObjectField(env, result, epId);
+	jclass ep_class = (*env)->GetObjectClass(env, jep);
+	jfieldID exprId = (*env)->GetFieldID(env, ep_class, "expr", "Lorg/grammaticalframework/pgf/Expr;");
+	jobject jexpr = (*env)->GetObjectField(env, jep, exprId);
+	jfieldID probId = (*env)->GetFieldID(env, ep_class, "prob", "D");
+	double prob = (*env)->GetDoubleField(env, jep, probId);
+
+	jfieldID offsetId = (*env)->GetFieldID(env, result_class, "offset", "I");
+	*poffset = j2gu_string_offset(sentence, (*env)->GetIntField(env, result, offsetId));
+
+	PgfExprProb* ep = gu_new(PgfExprProb, out_pool);
+	ep->expr = gu_variant_from_ptr(get_ref(env, jexpr));
+	ep->prob = prob;
+
+	
+	{
+		// This is an uggly hack. We first show the expression ep->expr
+		// and then we read it back but in out_pool. The whole purpose
+		// of this is to copy the expression from the temporary pool
+		// that was created in the Java binding to the parser pool.
+		// There should be a real copying function or even better
+		// there must be a way to avoid copying at all.
+
+		GuPool* tmp_pool = gu_local_pool();
+
+		GuExn* err = gu_exn(tmp_pool);
+		GuStringBuf* sbuf = gu_string_buf(tmp_pool);
+		GuOut* out = gu_string_buf_out(sbuf);
+
+		pgf_print_expr(ep->expr, NULL, 0, out, err);
+
+		GuString str = gu_string_buf_freeze(sbuf, tmp_pool);
+		GuIn* in = gu_data_in((uint8_t*) str, strlen(str), tmp_pool);
+
+		ep->expr = pgf_read_expr(in, out_pool, err);
+		if (!gu_ok(err) || gu_variant_is_null(ep->expr)) {
+			throw_string_exception(env, "org/grammaticalframework/pgf/PGFError", "The expression cannot be parsed");
+			gu_pool_free(tmp_pool);
+			return NULL;
+		}
+
+		gu_pool_free(tmp_pool);
+	}
+
+	return ep;
+}
+
+static void 
+jpgf_token_prob_enum_fin(GuFinalizer* self)
+{
+	JPgfTokenProbEnum* en = gu_container(self, JPgfTokenProbEnum, fin);
+	
+	JNIEnv *env;
+    (*cachedJVM)->AttachCurrentThread(cachedJVM, &env, NULL);
+
+	(*env)->DeleteGlobalRef(env, en->jiterator);
+}
+
+static GuEnum*
+jpgf_literal_callback_predict(PgfLiteralCallback* self,
+	                          size_t lin_idx,
+	                          GuString prefix,
+	                          GuPool *out_pool)
+{
+	JPgfLiteralCallback* callback = gu_container(self, JPgfLiteralCallback, callback);
+
+	JNIEnv *env;
+    (*cachedJVM)->AttachCurrentThread(cachedJVM, &env, NULL);
+
+	jstring jprefix = gu2j_string(env, prefix);
+	jobject jiterator = (*env)->CallObjectMethod(env, callback->jcallback, callback->predict_methodId, lin_idx, jprefix);
+	if (jiterator == NULL)
+		return NULL;
+
+	JPgfTokenProbEnum* en = gu_new(JPgfTokenProbEnum, out_pool);
+	en->en.next = NULL;
+	en->jiterator = (*env)->NewGlobalRef(env, jiterator);
+	en->fin.fn = jpgf_token_prob_enum_fin;
+
+	gu_pool_finally(out_pool, &en->fin);
+
+	return &en->en;
+}
+
+static void 
+jpgf_literal_callback_fin(GuFinalizer* self)
+{
+	JPgfLiteralCallback* callback = gu_container(self, JPgfLiteralCallback, fin);
+	
+	JNIEnv *env;
+    (*cachedJVM)->AttachCurrentThread(cachedJVM, &env, NULL);
+
+	(*env)->DeleteGlobalRef(env, callback->jcallback);
+}
+
+JNIEXPORT void JNICALL Java_org_grammaticalframework_pgf_Parser_addLiteralCallback
+  (JNIEnv* env, jclass clazz, jobject jconcr, jlong callbacksRef, jstring jcat, jobject jcallback, jobject jpool)
+{
+	PgfConcr* concr = get_ref(env, jconcr);
+	GuPool* pool = get_ref(env, jpool);
+
+	JPgfLiteralCallback* callback = gu_new(JPgfLiteralCallback, pool);
+	callback->callback.match   = jpgf_literal_callback_match;
+	callback->callback.predict = jpgf_literal_callback_predict;
+	callback->jcallback = (*env)->NewGlobalRef(env, jcallback);
+	callback->fin.fn = jpgf_literal_callback_fin;
+
+	jclass callback_class = (*env)->GetObjectClass(env, jcallback);
+	callback->match_methodId = (*env)->GetMethodID(env, callback_class, "match", "(ILjava/lang/String;I)Lorg/grammaticalframework/pgf/LiteralCallback$CallbackResult;");
+	callback->predict_methodId = (*env)->GetMethodID(env, callback_class, "predict", "(ILjava/lang/String;)Ljava/util/Iterator;");
+
+	gu_pool_finally(pool, &callback->fin);
+	
+	pgf_callbacks_map_add_literal(concr, l2p(callbacksRef),
+                                  j2gu_string(env, jcat, pool), &callback->callback);
+}
+
+JNIEXPORT jobject JNICALL 
+Java_org_grammaticalframework_pgf_Parser_parseWithHeuristics
+  (JNIEnv* env, jclass clazz, jobject jconcr, jstring jstartCat, jstring js, jdouble heuristics, jlong callbacksRef, jobject jpool)
+{
+	GuPool* pool = get_ref(env, jpool);
 	GuPool* out_pool = gu_new_pool();
 
     GuString startCat = j2gu_string(env, jstartCat, pool);
@@ -383,7 +537,7 @@ Java_org_grammaticalframework_pgf_Parser_parse
     GuExn* parse_err = gu_new_exn(pool);
 
 	GuEnum* res =
-		pgf_parse(get_ref(env, concr), startCat, s, parse_err, pool, out_pool);
+		pgf_parse_with_heuristics(get_ref(env, jconcr), startCat, s, heuristics, l2p(callbacksRef), parse_err, pool, out_pool);
 
 	if (!gu_ok(parse_err)) {
 		if (gu_exn_caught(parse_err, PgfExn)) {
@@ -394,17 +548,16 @@ Java_org_grammaticalframework_pgf_Parser_parse
 			throw_string_exception(env, "org/grammaticalframework/pgf/ParseError", tok);
 		}
 
-		gu_pool_free(pool);
 		gu_pool_free(out_pool);
 		return NULL;
 	}
 
-	jfieldID refId = (*env)->GetFieldID(env, (*env)->GetObjectClass(env, concr), "gr", "Lorg/grammaticalframework/pgf/PGF;");
+	jfieldID refId = (*env)->GetFieldID(env, (*env)->GetObjectClass(env, jconcr), "gr", "Lorg/grammaticalframework/pgf/PGF;");
-	jobject jpgf = (*env)->GetObjectField(env, concr, refId);
+	jobject jpgf = (*env)->GetObjectField(env, jconcr, refId);
 
 	jclass expiter_class = (*env)->FindClass(env, "org/grammaticalframework/pgf/ExprIterator");
-	jmethodID constrId = (*env)->GetMethodID(env, expiter_class, "<init>", "(Lorg/grammaticalframework/pgf/PGF;JJJ)V");
+	jmethodID constrId = (*env)->GetMethodID(env, expiter_class, "<init>", "(Lorg/grammaticalframework/pgf/PGF;Lorg/grammaticalframework/pgf/Pool;JJ)V");
-	jobject jexpiter = (*env)->NewObject(env, expiter_class, constrId, jpgf, p2l(pool), p2l(out_pool), p2l(res));
+	jobject jexpiter = (*env)->NewObject(env, expiter_class, constrId, jpgf, jpool, p2l(out_pool), p2l(res));
 
 	return jexpiter;
 }
@@ -878,169 +1031,6 @@ Java_org_grammaticalframework_pgf_Concr_hasLinearization(JNIEnv* env, jobject se
 	return res;
 }
 
-typedef struct {
-	PgfLiteralCallback callback;
-	jobject jcallback;
-	jmethodID match_methodId;
-	jmethodID predict_methodId;
-	GuFinalizer fin;
-} JPgfLiteralCallback;
-
-typedef struct {
-	GuEnum en;
-	jobject jiterator;
-	GuFinalizer fin;
-} JPgfTokenProbEnum;
-
-static PgfExprProb*
-jpgf_literal_callback_match(PgfLiteralCallback* self,
-                            size_t lin_idx,
-                            GuString sentence, size_t* poffset,
-                            GuPool *out_pool)
-{
-	JPgfLiteralCallback* callback = gu_container(self, JPgfLiteralCallback, callback);
-
-	JNIEnv *env;
-    (*cachedJVM)->AttachCurrentThread(cachedJVM, &env, NULL);
-
-	jstring jsentence = gu2j_string(env, sentence);
-	size_t  joffset   = gu2j_string_offset(sentence, *poffset);
-	jobject result = (*env)->CallObjectMethod(env, callback->jcallback, callback->match_methodId, lin_idx, jsentence, joffset);
-	if (result == NULL)
-		return NULL;
-
-	jclass result_class = (*env)->GetObjectClass(env, result);
-	
-	jfieldID epId = (*env)->GetFieldID(env, result_class, "ep", "Lorg/grammaticalframework/pgf/ExprProb;");
-	jobject jep = (*env)->GetObjectField(env, result, epId);
-	jclass ep_class = (*env)->GetObjectClass(env, jep);
-	jfieldID exprId = (*env)->GetFieldID(env, ep_class, "expr", "Lorg/grammaticalframework/pgf/Expr;");
-	jobject jexpr = (*env)->GetObjectField(env, jep, exprId);
-	jfieldID probId = (*env)->GetFieldID(env, ep_class, "prob", "D");
-	double prob = (*env)->GetDoubleField(env, jep, probId);
-
-	jfieldID offsetId = (*env)->GetFieldID(env, result_class, "offset", "I");
-	*poffset = j2gu_string_offset(sentence, (*env)->GetIntField(env, result, offsetId));
-
-	PgfExprProb* ep = gu_new(PgfExprProb, out_pool);
-	ep->expr = gu_variant_from_ptr(get_ref(env, jexpr));
-	ep->prob = prob;
-
-	
-	{
-		// This is an uggly hack. We first show the expression ep->expr
-		// and then we read it back but in out_pool. The whole purpose
-		// of this is to copy the expression from the temporary pool
-		// that was created in the Java binding to the parser pool.
-		// There should be a real copying function or even better
-		// there must be a way to avoid copying at all.
-
-		GuPool* tmp_pool = gu_local_pool();
-
-		GuExn* err = gu_exn(tmp_pool);
-		GuStringBuf* sbuf = gu_string_buf(tmp_pool);
-		GuOut* out = gu_string_buf_out(sbuf);
-
-		pgf_print_expr(ep->expr, NULL, 0, out, err);
-
-		GuString str = gu_string_buf_freeze(sbuf, tmp_pool);
-		GuIn* in = gu_data_in((uint8_t*) str, strlen(str), tmp_pool);
-
-		ep->expr = pgf_read_expr(in, out_pool, err);
-		if (!gu_ok(err) || gu_variant_is_null(ep->expr)) {
-			throw_string_exception(env, "org/grammaticalframework/pgf/PGFError", "The expression cannot be parsed");
-			gu_pool_free(tmp_pool);
-			return NULL;
-		}
-
-		gu_pool_free(tmp_pool);
-	}
-
-	return ep;
-}
-
-static void 
-jpgf_token_prob_enum_fin(GuFinalizer* self)
-{
-	JPgfTokenProbEnum* en = gu_container(self, JPgfTokenProbEnum, fin);
-	
-	JNIEnv *env;
-    (*cachedJVM)->AttachCurrentThread(cachedJVM, &env, NULL);
-
-	(*env)->DeleteGlobalRef(env, en->jiterator);
-}
-
-static GuEnum*
-jpgf_literal_callback_predict(PgfLiteralCallback* self,
-	                          size_t lin_idx,
-	                          GuString prefix,
-	                          GuPool *out_pool)
-{
-	JPgfLiteralCallback* callback = gu_container(self, JPgfLiteralCallback, callback);
-
-	JNIEnv *env;
-    (*cachedJVM)->AttachCurrentThread(cachedJVM, &env, NULL);
-
-	jstring jprefix = gu2j_string(env, prefix);
-	jobject jiterator = (*env)->CallObjectMethod(env, callback->jcallback, callback->predict_methodId, lin_idx, jprefix);
-	if (jiterator == NULL)
-		return NULL;
-
-	JPgfTokenProbEnum* en = gu_new(JPgfTokenProbEnum, out_pool);
-	en->en.next = NULL;
-	en->jiterator = (*env)->NewGlobalRef(env, jiterator);
-	en->fin.fn = jpgf_token_prob_enum_fin;
-
-	gu_pool_finally(out_pool, &en->fin);
-
-	return &en->en;
-}
-
-static void 
-jpgf_literal_callback_fin(GuFinalizer* self)
-{
-	JPgfLiteralCallback* callback = gu_container(self, JPgfLiteralCallback, fin);
-	
-	JNIEnv *env;
-    (*cachedJVM)->AttachCurrentThread(cachedJVM, &env, NULL);
-
-	(*env)->DeleteGlobalRef(env, callback->jcallback);
-}
-
-JNIEXPORT void JNICALL 
-Java_org_grammaticalframework_pgf_Concr_addLiteral(JNIEnv* env, jobject self, jstring jcat, jobject jcallback)
-{
-	PgfConcr* concr = get_ref(env, self);
-	GuPool* pool = pgf_concr_get_pool(concr);
-	
-	JPgfLiteralCallback* callback = gu_new(JPgfLiteralCallback, pool);
-	callback->callback.match   = jpgf_literal_callback_match;
-	callback->callback.predict = jpgf_literal_callback_predict;
-	callback->jcallback = (*env)->NewGlobalRef(env, jcallback);
-	callback->fin.fn = jpgf_literal_callback_fin;
-
-	jclass callback_class = (*env)->GetObjectClass(env, jcallback);
-	callback->match_methodId = (*env)->GetMethodID(env, callback_class, "match", "(ILjava/lang/String;I)Lorg/grammaticalframework/pgf/LiteralCallback$CallbackResult;");
-	callback->predict_methodId = (*env)->GetMethodID(env, callback_class, "predict", "(ILjava/lang/String;)Ljava/util/Iterator;");
-
-	gu_pool_finally(pool, &callback->fin);
-	
-	GuPool* tmp_pool = gu_local_pool();
-	GuExn* err = gu_exn(tmp_pool);
-	pgf_concr_add_literal(concr, j2gu_string(env, jcat, tmp_pool), &callback->callback, err);
-
-	if (!gu_ok(err)) {
-		if (gu_exn_caught(err, PgfExn)) {
-			GuString msg = (GuString) gu_exn_caught_data(err);
-			throw_string_exception(env, "org/grammaticalframework/pgf/PGFError", msg);
-		} else {
-			throw_string_exception(env, "org/grammaticalframework/pgf/PGFError", "The literal cannot be added");
-		}
-	}
-
-	gu_pool_free(tmp_pool);
-}
-
 JNIEXPORT jlong JNICALL
 Java_org_grammaticalframework_pgf_Pool_alloc(JNIEnv* env, jclass clazz)
 {

src/runtime/java/org/grammaticalframework/pgf/Concr.java

@@ -8,16 +8,11 @@ public class Concr {
 	public native String getName();
 
 	public Iterable<ExprProb> parse(String startCat, String s) throws ParseError {
-		return new Parser(this, startCat, s);
+		return new Parser(this, startCat, s, -1, null);
 	}
 
-	public Expr parseBest(String startCat, String s) throws ParseError {
+	public Iterable<ExprProb> parseWithHeuristics(String startCat, String s, double heuristics, Map<String,LiteralCallback> callbacks) throws ParseError {
-		Iterator<ExprProb> iter = Parser.parse(this, startCat, s);
+		return new Parser(this, startCat, s, heuristics, callbacks);
-		if (iter.hasNext()) {
-			return iter.next().getExpr();
-		} else {
-			return null;
-		}
 	}
 
 	public Iterable<TokenProb> complete(String startCat, String s, String prefix) throws ParseError {
@@ -44,8 +39,6 @@ public class Concr {
 
 	public native void unload();
 
-	public native void addLiteral(String cat, LiteralCallback callback);
-
 	//////////////////////////////////////////////////////////////////
 	// private stuff
 	

src/runtime/java/org/grammaticalframework/pgf/ExprIterator.java

@@ -9,9 +9,9 @@ class ExprIterator implements Iterator<ExprProb> {
 	private ExprProb ep;
 	private boolean fetched;
 
-	public ExprIterator(PGF gr, long pool, long out_pool, long ref) {
+	public ExprIterator(PGF gr, Pool pool, long out_pool, long ref) {
 		this.gr       = gr;
-		this.pool     = new Pool(pool);
+		this.pool     = pool;
 		this.out_pool = new Pool(out_pool);
 		this.ref      = ref;
 		this.ep       = null;

src/runtime/java/org/grammaticalframework/pgf/Parser.java

@@ -6,13 +6,19 @@ class Parser implements Iterable<ExprProb> {
 	private Concr concr;
 	private String s;
 	private String startCat;
+	private double heuristics;
+	private Map<String,LiteralCallback> callbacks;
 	private ExprIterator iter;
 
-	public Parser(Concr concr, String startCat, String s) throws ParseError {
+	public Parser(Concr concr, String startCat, String s,
+			      double heuristics,
+			      Map<String,LiteralCallback> callbacks) throws ParseError {
 		this.concr    = concr;
 		this.startCat = startCat;
 		this.s        = s;
-		this.iter     = parse(concr, startCat, s);
+		this.heuristics = heuristics;
+		this.callbacks = callbacks;
+		this.iter     = doParse();
 	}
 
 	public Iterator<ExprProb> iterator() {
@@ -20,7 +26,7 @@ class Parser implements Iterable<ExprProb> {
 			// If someone has asked for a second iterator over
 			// the same parse results then we have to parse again.
 			try {
-				return parse(concr, startCat, s);
+				return doParse();
 			} catch (ParseError e) {
 				return null;
 			}
@@ -31,5 +37,19 @@ class Parser implements Iterable<ExprProb> {
 		}
 	}
 
-	static native ExprIterator parse(Concr concr, String startCat, String s) throws ParseError;
+	private ExprIterator doParse() throws ParseError
+	{
+		Pool pool = new Pool();
+		long callbacksRef = newCallbacksMap(concr, pool);
+		for (Map.Entry<String, LiteralCallback> entry : callbacks.entrySet()) {
+			addLiteralCallback(concr, callbacksRef, 
+					           entry.getKey(), entry.getValue(),
+					           pool);
+		} 
+		return parseWithHeuristics(concr, startCat, s, heuristics, callbacksRef, pool);
+	}
+
+	static native long newCallbacksMap(Concr concr, Pool pool);
+	static native void addLiteralCallback(Concr concr, long callbacksRef, String cat, LiteralCallback callback, Pool pool);
+	static native ExprIterator parseWithHeuristics(Concr concr, String startCat, String s, double heuristics, long callbacksRef, Pool pool) throws ParseError;
 }

src/runtime/python/pypgf.c

@@ -1194,17 +1194,135 @@ void pypgf_container_descructor(PyObject *capsule)
 
 #endif
 
+typedef struct {
+	PgfLiteralCallback callback;
+	PyObject* pycallback;
+	GuFinalizer fin;
+} PyPgfLiteralCallback;
+
+static PgfExprProb*
+pypgf_literal_callback_match(PgfLiteralCallback* self,
+                             size_t lin_idx,
+                             GuString sentence, size_t* poffset,
+                             GuPool *out_pool)
+{
+	PyPgfLiteralCallback* callback = 
+		gu_container(self, PyPgfLiteralCallback, callback);
+
+	PyObject* result =
+		PyObject_CallFunction(callback->pycallback, "isi", 
+		                      lin_idx, sentence, *poffset);
+	if (result == NULL || result == Py_None)
+		return NULL;
+
+	PgfExprProb* ep = gu_new(PgfExprProb, out_pool);
+
+	ExprObject* pyexpr;
+	if (!PyArg_ParseTuple(result, "Ofi", &pyexpr, &ep->prob, poffset))
+        return NULL;
+	ep->expr = pyexpr->expr;
+	
+	{
+		// This is an uggly hack. We first show the expression ep->expr
+		// and then we read it back but in out_pool. The whole purpose
+		// of this is to copy the expression from the temporary pool
+		// that was created in the Java binding to the parser pool.
+		// There should be a real copying function or even better
+		// there must be a way to avoid copying at all.
+
+		GuPool* tmp_pool = gu_local_pool();
+
+		GuExn* err = gu_exn(tmp_pool);
+		GuStringBuf* sbuf = gu_string_buf(tmp_pool);
+		GuOut* out = gu_string_buf_out(sbuf);
+
+		pgf_print_expr(ep->expr, NULL, 0, out, err);
+
+		GuString str = gu_string_buf_freeze(sbuf, tmp_pool);
+		GuIn* in = gu_data_in((uint8_t*) str, strlen(str), tmp_pool);
+
+		ep->expr = pgf_read_expr(in, out_pool, err);
+		if (!gu_ok(err) || gu_variant_is_null(ep->expr)) {
+			PyErr_SetString(PGFError, "The expression cannot be parsed");
+			gu_pool_free(tmp_pool);
+			return NULL;
+		}
+
+		gu_pool_free(tmp_pool);
+	}
+
+	Py_DECREF(pyexpr);
+
+	return ep;
+}
+
+static GuEnum*
+pypgf_literal_callback_predict(PgfLiteralCallback* self,
+	                           size_t lin_idx,
+	                           GuString prefix,
+	                           GuPool *out_pool)
+{
+	return NULL;
+}
+
+static void 
+pypgf_literal_callback_fin(GuFinalizer* self)
+{
+	PyPgfLiteralCallback* callback = 
+		gu_container(self, PyPgfLiteralCallback, fin);
+
+	Py_XDECREF(callback->pycallback);
+}
+
+static PgfCallbacksMap*
+pypgf_new_callbacks_map(PgfConcr* concr, PyObject *py_callbacks,
+                        GuPool* pool)
+{
+	PgfCallbacksMap* callbacks =
+		pgf_new_callbacks_map(concr, pool);
+
+	if (py_callbacks == NULL)
+		return callbacks;
+
+	size_t n_callbacks = PyList_Size(py_callbacks);
+	for (size_t i = 0; i < n_callbacks; i++) {
+		PyObject* item =
+			PyList_GetItem(py_callbacks, i);
+
+		PyObject* pycallback = NULL;
+		const char* cat = NULL;
+		if (!PyArg_ParseTuple(item, "sO", &cat, &pycallback))
+			return NULL;
+
+		PyPgfLiteralCallback* callback = gu_new(PyPgfLiteralCallback, pool);
+		callback->callback.match   = pypgf_literal_callback_match;
+		callback->callback.predict = pypgf_literal_callback_predict;
+		callback->pycallback = pycallback;
+		callback->fin.fn = pypgf_literal_callback_fin;
+
+		gu_pool_finally(pool, &callback->fin);
+
+		pgf_callbacks_map_add_literal(concr, callbacks,
+		                              cat, &callback->callback);
+	}
+	
+	return callbacks;
+}
+
 static IterObject*
 Concr_parse(ConcrObject* self, PyObject *args, PyObject *keywds)
 {
-	static char *kwlist[] = {"sentence", "cat", "n", "heuristics", NULL};
+	static char *kwlist[] = {"sentence", "cat", "n", "heuristics", "callbacks", NULL};
 
 	const char *sentence = NULL;
 	PgfCId catname = pgf_start_cat(self->grammar->pgf);
 	int max_count = -1;
 	double heuristics = -1;
-    if (!PyArg_ParseTupleAndKeywords(args, keywds, "s|sid", kwlist,
+	PyObject* py_callbacks = NULL;
-                                     &sentence, &catname, &max_count, &heuristics))
+    if (!PyArg_ParseTupleAndKeywords(args, keywds, "s|sidO!", kwlist,
+                                     &sentence, &catname, &max_count,
+                                     &heuristics,
+                                     &PyList_Type, &py_callbacks))
         return NULL;
 
 	IterObject* pyres = (IterObject*) 
@@ -1229,9 +1347,13 @@ Concr_parse(ConcrObject* self, PyObject *args, PyObject *keywds)
 
 	GuExn* parse_err = gu_exn(pyres->pool);
 
+	PgfCallbacksMap* callbacks =
+		pypgf_new_callbacks_map(self->concr, py_callbacks, pyres->pool);
+	if (callbacks == NULL)
+		return NULL;
 	pyres->res =
 		pgf_parse_with_heuristics(self->concr, catname, sentence, 
-		                          heuristics, parse_err,
+		                          heuristics, callbacks, parse_err,
 		                          pyres->pool, out_pool);
 
 	if (!gu_ok(parse_err)) {
@@ -1333,120 +1455,6 @@ Concr_parseval(ConcrObject* self, PyObject *args) {
     return Py_BuildValue("ddd", precision, recall, exact);
 }
 
-typedef struct {
-	PgfLiteralCallback callback;
-	PyObject* pycallback;
-	GuFinalizer fin;
-} PyPgfLiteralCallback;
-
-static PgfExprProb*
-pypgf_literal_callback_match(PgfLiteralCallback* self,
-                             size_t lin_idx,
-                             GuString sentence, size_t* poffset,
-                             GuPool *out_pool)
-{
-	PyPgfLiteralCallback* callback = 
-		gu_container(self, PyPgfLiteralCallback, callback);
-
-	PyObject* result =
-		PyObject_CallFunction(callback->pycallback, "isi", 
-		                      lin_idx, sentence, *poffset);
-	if (result == NULL || result == Py_None)
-		return NULL;
-
-	PgfExprProb* ep = gu_new(PgfExprProb, out_pool);
-
-	ExprObject* pyexpr;
-	if (!PyArg_ParseTuple(result, "Ofi", &pyexpr, &ep->prob, poffset))
-        return NULL;
-	ep->expr = pyexpr->expr;
-	
-	{
-		// This is an uggly hack. We first show the expression ep->expr
-		// and then we read it back but in out_pool. The whole purpose
-		// of this is to copy the expression from the temporary pool
-		// that was created in the Java binding to the parser pool.
-		// There should be a real copying function or even better
-		// there must be a way to avoid copying at all.
-
-		GuPool* tmp_pool = gu_local_pool();
-
-		GuExn* err = gu_exn(tmp_pool);
-		GuStringBuf* sbuf = gu_string_buf(tmp_pool);
-		GuOut* out = gu_string_buf_out(sbuf);
-
-		pgf_print_expr(ep->expr, NULL, 0, out, err);
-
-		GuString str = gu_string_buf_freeze(sbuf, tmp_pool);
-		GuIn* in = gu_data_in((uint8_t*) str, strlen(str), tmp_pool);
-
-		ep->expr = pgf_read_expr(in, out_pool, err);
-		if (!gu_ok(err) || gu_variant_is_null(ep->expr)) {
-			PyErr_SetString(PGFError, "The expression cannot be parsed");
-			gu_pool_free(tmp_pool);
-			return NULL;
-		}
-
-		gu_pool_free(tmp_pool);
-	}
-
-	Py_DECREF(pyexpr);
-
-	return ep;
-}
-
-static GuEnum*
-pypgf_literal_callback_predict(PgfLiteralCallback* self,
-	                           size_t lin_idx,
-	                           GuString prefix,
-	                           GuPool *out_pool)
-{
-	return NULL;
-}
-
-static void 
-pypgf_literal_callback_fin(GuFinalizer* self)
-{
-	PyPgfLiteralCallback* callback = 
-		gu_container(self, PyPgfLiteralCallback, fin);
-
-	Py_XDECREF(callback->pycallback);
-}
-
-static PyObject*
-Concr_addLiteral(ConcrObject* self, PyObject *args) {
-	PyObject* pycallback = NULL;
-	const char* cat = NULL;
-	if (!PyArg_ParseTuple(args, "sO", &cat, &pycallback))
-        return NULL;
-
-	GuPool* pool = pgf_concr_get_pool(self->concr);
-
-	PyPgfLiteralCallback* callback = gu_new(PyPgfLiteralCallback, pool);
-	callback->callback.match   = pypgf_literal_callback_match;
-	callback->callback.predict = pypgf_literal_callback_predict;
-	callback->pycallback = pycallback;
-	callback->fin.fn = pypgf_literal_callback_fin;
-
-	gu_pool_finally(pool, &callback->fin);
-
-	GuPool* tmp_pool = gu_local_pool();
-	GuExn* err = gu_exn(tmp_pool);
-	pgf_concr_add_literal(self->concr, cat, &callback->callback, err);
-
-	if (!gu_ok(err)) {
-		if (gu_exn_caught(err, PgfExn)) {
-			GuString msg = (GuString) gu_exn_caught_data(err);
-			PyErr_SetString(PGFError, msg);
-		} else {
-			PyErr_SetString(PGFError, "The literal cannot be added");
-		}
-	}
-
-	gu_pool_free(tmp_pool);
-	Py_RETURN_NONE;
-}
-
 static PyObject*
 Concr_linearize(ConcrObject* self, PyObject *args)
 {
@@ -2027,7 +2035,8 @@ static PyMethodDef Concr_methods[] = {
      "- sentence (string) or tokens (list of strings)\n"
      "- cat (string); OPTIONAL, default: the startcat of the grammar\n"
      "- n (int), max. trees; OPTIONAL, default: extract all trees\n"
-     "- heuristics (double >= 0.0); OPTIONAL, default: taken from the flags in the grammar"
+     "- heuristics (double >= 0.0); OPTIONAL, default: taken from the flags in the grammar\n"
+     "- callbacks (list of category and callback); OPTIONAL, default: built-in callbacks only for Int, String and Float"
     },
     {"complete", (PyCFunction)Concr_complete, METH_VARARGS | METH_KEYWORDS,
      "Parses a partial string and returns a list with the top n possible next tokens"
@@ -2035,9 +2044,6 @@ static PyMethodDef Concr_methods[] = {
     {"parseval", (PyCFunction)Concr_parseval, METH_VARARGS,
      "Computes precision, recall and exact match for the parser on a given abstract tree"
     },
-    {"addLiteral", (PyCFunction)Concr_addLiteral, METH_VARARGS,
-     "adds callbacks for custom literals in the grammar"
-    },
     {"linearize", (PyCFunction)Concr_linearize, METH_VARARGS,
      "Takes an abstract tree and linearizes it to a string"
     },

src/ui/android/AndroidManifest.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
     package="org.grammaticalframework.ui.android"
-    android:versionCode="5"
+    android:versionCode="6"
     android:versionName="1.1"
     android:installLocation="auto" >
 

src/ui/android/src/org/grammaticalframework/ui/android/Translator.java

@@ -6,24 +6,9 @@ import android.util.Log;
 import android.util.Pair;
 import android.view.inputmethod.CompletionInfo;
 
-import org.grammaticalframework.pgf.Concr;
+import org.grammaticalframework.pgf.*;
-import org.grammaticalframework.pgf.Expr;
+import java.io.*;
-import org.grammaticalframework.pgf.ExprProb;
+import java.util.*;
-import org.grammaticalframework.pgf.FullFormEntry;
-import org.grammaticalframework.pgf.MorphoAnalysis;
-import org.grammaticalframework.pgf.NercLiteralCallback;
-import org.grammaticalframework.pgf.UnknownLiteralCallback;
-import org.grammaticalframework.pgf.PGF;
-import org.grammaticalframework.pgf.ParseError;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.PriorityQueue;
 
 public class Translator {
 
@@ -256,7 +241,8 @@ public class Translator {
     	String lowerinput = input.toLowerCase() ;  // also consider lower-cased versions of the word
 
         try {
-        	Expr expr = sourceLang.parseBest("Chunk", input) ; // try parse as chunk
+        	Iterator<ExprProb> iter = sourceLang.parse("Chunk", input).iterator(); // try parse as chunk
+        	Expr expr = iter.next().getExpr();
             output = targetLang.linearize(expr);
             return output ;
         } catch (ParseError e) {                               	  // if this fails
@@ -301,8 +287,12 @@ public class Translator {
             Concr sourceLang = getSourceConcr();
             Concr targetLang = getTargetConcr();
 
+            Map<String,LiteralCallback> callbacks = new HashMap<String,LiteralCallback>();
+            callbacks.put("PN", new NercLiteralCallback(mGrammarLoader.getGrammar(), sourceLang));
+            callbacks.put("Symb", new UnknownLiteralCallback(sourceLang));
+
             int count = NUM_ALT_TRANSLATIONS;
-            for (ExprProb ep : sourceLang.parse(getGrammar().getStartCat(), input)) {
+            for (ExprProb ep : sourceLang.parseWithHeuristics(getGrammar().getStartCat(), input, -1, callbacks)) {
             	if (count-- <= 0)
             		break;
             	exprs.add(ep);
@@ -487,8 +477,6 @@ public class Translator {
 		        long t1 = System.currentTimeMillis();
 		        mConcr = mGrammarLoader.getGrammar().getLanguages().get(mLanguage.getConcrete());
 		        mConcr.load(in);
-		        mConcr.addLiteral("PN", new NercLiteralCallback(mGrammarLoader.getGrammar(), mConcr));
-		        mConcr.addLiteral("Symb", new UnknownLiteralCallback(mConcr));
 		        long t2 = System.currentTimeMillis();
 		        Log.d(TAG, name + " loaded ("+(t2-t1)+" ms)");
 		    } catch (FileNotFoundException e) {