change in the API for literals

The API in the C runtime as well as in the Haskell, Python and Java binding
is changed. Now instead of adding the literal callbacks to the concrete syntax
you need to supply them every time when you need to parse. The main reason is:

- referentially transparent API for Haskell
- when we start using memory mapped files we will not be allowed to change
  anything in the grammar data structures. At that point the old API would
  be impossible to use.

9 files changed, 56 insertions, 63 deletions

diff --git a/src/runtime/c/pgf/data.h b/src/runtime/c/pgf/data.h
index a3554ef45..6b62cfe4e 100644
--- a/src/runtime/c/pgf/data.h
+++ b/src/runtime/c/pgf/data.h
@@ -199,8 +199,6 @@ typedef GuMap PgfCncOverloadMap;
 
 typedef struct PgfItem PgfItem;
 
-typedef GuMap PgfCallbacksMap;
-
 typedef GuVariant PgfSymbol;
 
 typedef enum {
@@ -280,7 +278,6 @@ struct PgfConcr {
     PgfSequences* sequences;
     GuBuf* pre_sequences;
 	PgfCIdMap* cnccats;
-	PgfCallbacksMap* callbacks;
 	int total_cats;
 	
 	GuPool* pool;     // if the language is loaded separately then this is the pool
diff --git a/src/runtime/c/pgf/literals.c b/src/runtime/c/pgf/literals.c
index 51a426031..1376c8beb 100644
--- a/src/runtime/c/pgf/literals.c
+++ b/src/runtime/c/pgf/literals.c
@@ -274,6 +274,19 @@ pgf_new_callbacks_map(PgfConcr* concr, GuPool *pool)
 	return callbacks;
 }
 
+void
+pgf_callbacks_map_add_literal(PgfConcr* concr, PgfCallbacksMap* callbacks,
+                              PgfCId cat, PgfLiteralCallback* callback)
+{
+	PgfCncCat* cnccat =
+		gu_map_get(concr->cnccats, cat, PgfCncCat*);
+	if (cnccat == NULL)
+		return;
+
+	gu_map_put(callbacks, cnccat,
+	           PgfLiteralCallback*, callback);
+}
+
 PgfCCat*
 pgf_literal_cat(PgfConcr* concr, PgfLiteral lit)
 {
diff --git a/src/runtime/c/pgf/literals.h b/src/runtime/c/pgf/literals.h
index 88f9304a1..bf071c202 100644
--- a/src/runtime/c/pgf/literals.h
+++ b/src/runtime/c/pgf/literals.h
@@ -3,9 +3,6 @@
 
 #include <pgf/data.h>
 
-PgfCallbacksMap*
-pgf_new_callbacks_map(PgfConcr* concr, GuPool *pool);
-
 // literal for named entities recognition
 extern PgfLiteralCallback pgf_nerc_literal_callback;
 
diff --git a/src/runtime/c/pgf/parser.c b/src/runtime/c/pgf/parser.c
index 2ff98d772..bbdcd3cf6 100644
--- a/src/runtime/c/pgf/parser.c
+++ b/src/runtime/c/pgf/parser.c
@@ -57,6 +57,7 @@ typedef struct {
     PgfItem* free_item;
 
     prob_t heuristic_factor;
+    PgfCallbacksMap* callbacks;
     prob_t meta_prob;
 	prob_t meta_token_prob;
 } PgfParsing;
@@ -1616,7 +1617,7 @@ pgf_parsing_symbol(PgfParsing* ps, PgfItem* item, PgfSymbol sym)
 				 * literal category so we must call the callback */
 
 				PgfLiteralCallback* callback =
-					gu_map_get(ps->concr->callbacks, 
+					gu_map_get(ps->callbacks,
 					           parg->ccat->cnccat, 
 							   PgfLiteralCallback*);
 
@@ -1861,7 +1862,7 @@ pgf_parsing_set_default_factors(PgfParsing* ps, PgfAbstr* abstr)
 }
 
 static PgfParsing*
-pgf_new_parsing(PgfConcr* concr, GuString sentence,
+pgf_new_parsing(PgfConcr* concr, GuString sentence, PgfCallbacksMap* callbacks,
                 GuPool* pool, GuPool* out_pool)
 {
 	PgfParsing* ps = gu_new(PgfParsing, pool);
@@ -1884,6 +1885,7 @@ pgf_new_parsing(PgfConcr* concr, GuString sentence,
 	ps->tp = NULL;
 	ps->free_item = NULL;
 	ps->heuristic_factor = 0;
+	ps->callbacks = callbacks;
 	ps->meta_prob = INFINITY;
 	ps->meta_token_prob = INFINITY;
 
@@ -2087,9 +2089,9 @@ pgf_parse_result_is_new(PgfExprState* st)
 // TODO: s/CId/Cat, add the cid to Cat, make Cat the key to CncCat
 static PgfParsing*
 pgf_parsing_init(PgfConcr* concr, PgfCId cat, size_t lin_idx, 
-                 GuString sentence, double heuristic_factor,
-                 GuExn* err,
-                 GuPool* pool, GuPool* out_pool)
+                 GuString sentence,
+                 double heuristic_factor, PgfCallbacksMap* callbacks,
+                 GuExn* err, GuPool* pool, GuPool* out_pool)
 {
 	PgfCncCat* cnccat =
 		gu_map_get(concr->cnccats, cat, PgfCncCat*);
@@ -2102,7 +2104,7 @@ pgf_parsing_init(PgfConcr* concr, PgfCId cat, size_t lin_idx,
 	gu_assert(lin_idx < cnccat->n_lins);
 
 	PgfParsing* ps =
-		pgf_new_parsing(concr, sentence, pool, out_pool);
+		pgf_new_parsing(concr, sentence, callbacks, pool, out_pool);
 
 	if (heuristic_factor >= 0) {
 		ps->heuristic_factor = heuristic_factor;
@@ -2312,12 +2314,14 @@ pgf_parse(PgfConcr* concr, PgfCId cat, GuString sentence,
           GuExn* err, 
           GuPool* pool, GuPool* out_pool)
 {
-    return pgf_parse_with_heuristics(concr, cat, sentence, -1.0, err, pool, out_pool);
+	PgfCallbacksMap* callbacks = pgf_new_callbacks_map(concr, out_pool); 
+    return pgf_parse_with_heuristics(concr, cat, sentence, -1.0, callbacks, err, pool, out_pool);
 }
 
 GuEnum*
 pgf_parse_with_heuristics(PgfConcr* concr, PgfCId cat, GuString sentence,
                           double heuristics,
+                          PgfCallbacksMap* callbacks,
                           GuExn* err,
                           GuPool* pool, GuPool* out_pool)
 {
@@ -2333,7 +2337,7 @@ pgf_parse_with_heuristics(PgfConcr* concr, PgfCId cat, GuString sentence,
 
 	// Begin parsing a sentence with the specified category
 	PgfParsing* ps =
-		pgf_parsing_init(concr, cat, 0, sentence, heuristics, err, pool, out_pool);
+		pgf_parsing_init(concr, cat, 0, sentence, heuristics, callbacks, err, pool, out_pool);
 	if (ps == NULL) {
 		return NULL;
 	}
@@ -2393,8 +2397,10 @@ pgf_complete(PgfConcr* concr, PgfCId cat, GuString sentence,
 	}
 
 	// Begin parsing a sentence with the specified category
+	PgfCallbacksMap* callbacks =
+		pgf_new_callbacks_map(concr, pool);
 	PgfParsing* ps =
-		pgf_parsing_init(concr, cat, 0, sentence, -1.0, err, pool, pool);
+		pgf_parsing_init(concr, cat, 0, sentence, -1.0, callbacks, err, pool, pool);
 	if (ps == NULL) {
 		return NULL;
 	}
diff --git a/src/runtime/c/pgf/pgf.c b/src/runtime/c/pgf/pgf.c
index 0412099e1..4fc909acf 100644
--- a/src/runtime/c/pgf/pgf.c
+++ b/src/runtime/c/pgf/pgf.c
@@ -32,6 +32,16 @@ pgf_read(const char* fpath,
 	return pgf;
 }
 
+PgfPGF*
+pgf_read_in(GuIn* in,
+            GuPool* pool, GuPool* tmp_pool, GuExn* err)
+{
+	PgfReader* rdr = pgf_new_reader(in, pool, tmp_pool, err);
+	PgfPGF* pgf = pgf_read_pgf(rdr);
+	pgf_reader_done(rdr, pgf);
+	return pgf;
+}
+
 GuString
 pgf_abstract_name(PgfPGF* pgf)
 {
@@ -171,38 +181,6 @@ pgf_has_linearization(PgfConcr* concr, PgfCId id)
 	return (overl_table != NULL);
 }
 
-GuPool*
-pgf_concr_get_pool(PgfConcr* concr)
-{
-	GuPool* pool = concr->pool;
-	if (pool == NULL)
-		pool = gu_container(concr->abstr, PgfPGF, abstract)->pool;
-	return pool;
-}
-
-void
-pgf_concr_add_literal(PgfConcr *concr, PgfCId cat,
-                      PgfLiteralCallback* callback,
-                      GuExn* err)
-{
-	if (concr->cnccats == NULL ||
-	    concr->callbacks == NULL) {
-		GuExnData* err_data = gu_raise(err, PgfExn);
-		if (err_data) {
-			err_data->data = "The concrete syntax is not loaded";
-			return;
-		}
-	}
-
-	PgfCncCat* cnccat =
-		gu_map_get(concr->cnccats, cat, PgfCncCat*);
-	if (cnccat == NULL)
-		return;
-
-	gu_map_put(concr->callbacks, cnccat,
-	           PgfLiteralCallback*, callback);
-}
-
 PgfExprProb*
 pgf_fun_get_ep(void* value)
 {
diff --git a/src/runtime/c/pgf/pgf.h b/src/runtime/c/pgf/pgf.h
index 49f522278..e542e4213 100644
--- a/src/runtime/c/pgf/pgf.h
+++ b/src/runtime/c/pgf/pgf.h
@@ -25,6 +25,10 @@ PgfPGF*
 pgf_read(const char* fpath,
          GuPool* pool, GuExn* err);
 
+PgfPGF*
+pgf_read_in(GuIn* in,
+            GuPool* pool, GuPool* tmp_pool, GuExn* err);
+
 void
 pgf_concrete_load(PgfConcr* concr, GuIn* in, GuExn* err);
 
@@ -114,9 +118,12 @@ GuEnum*
 pgf_lookup_word_prefix(PgfConcr *concr, GuString prefix,
                        GuPool* pool, GuExn* err);
 
+typedef GuMap PgfCallbacksMap;
+
 PgfExprEnum*
 pgf_parse_with_heuristics(PgfConcr* concr, PgfCId cat, 
                           GuString sentence, double heuristics,
+                          PgfCallbacksMap* callbacks,
                           GuExn* err,
                           GuPool* pool, GuPool* out_pool);
 
@@ -130,9 +137,6 @@ GuEnum*
 pgf_complete(PgfConcr* concr, PgfCId cat, GuString string, 
              GuString prefix, GuExn* err, GuPool* pool);
 
-GuPool*
-pgf_concr_get_pool(PgfConcr* concr);
-
 typedef struct PgfLiteralCallback PgfLiteralCallback;
 
 struct PgfLiteralCallback {
@@ -146,10 +150,12 @@ struct PgfLiteralCallback {
 	                      GuPool *out_pool);
 };
 
+PgfCallbacksMap*
+pgf_new_callbacks_map(PgfConcr* concr, GuPool *pool);
+
 void
-pgf_concr_add_literal(PgfConcr *concr, PgfCId cat,
-                      PgfLiteralCallback* callback,
-                      GuExn* err);
+pgf_callbacks_map_add_literal(PgfConcr* concr, PgfCallbacksMap* callbacks,
+                              PgfCId cat, PgfLiteralCallback* callback);
 
 void
 pgf_print(PgfPGF* pgf, GuOut* out, GuExn* err); 
diff --git a/src/runtime/c/pgf/reader.c b/src/runtime/c/pgf/reader.c
index 8314fef5f..1c250f559 100644
--- a/src/runtime/c/pgf/reader.c
+++ b/src/runtime/c/pgf/reader.c
@@ -1,6 +1,5 @@
 #include "data.h"
 #include "expr.h"
-#include "literals.h"
 #include "evaluator.h"
 #include "reader.h"
 
@@ -1148,7 +1147,6 @@ pgf_read_concrete_content(PgfReader* rdr, PgfConcr* concr)
 	pgf_read_linrefs(rdr, concr);
 	pgf_read_ccats(rdr, concr);
 	concr->cnccats = pgf_read_cnccats(rdr, concr->abstr, concr);
-	concr->callbacks = pgf_new_callbacks_map(concr, rdr->opool); 
 	concr->total_cats = pgf_read_int(rdr);
 
 	GuMapItor clo1 = { pgf_read_ccat_cb };
@@ -1166,7 +1164,6 @@ pgf_read_concrete_init_header(PgfConcr* concr)
 	concr->fun_indices = NULL;
 	concr->coerce_idx = NULL;
 	concr->cnccats = NULL;
-	concr->callbacks = NULL;
 	concr->total_cats = 0;
 }
 
diff --git a/src/runtime/c/utils/pgf-parse.c b/src/runtime/c/utils/pgf-parse.c
index eb417d5e7..088fe409d 100644
--- a/src/runtime/c/utils/pgf-parse.c
+++ b/src/runtime/c/utils/pgf-parse.c
@@ -58,10 +58,6 @@ int main(int argc, char* argv[]) {
     goto fail;
   }
 
-  /* // Register a callback for the literal category Symbol */
-  /* pgf_parser_add_literal(from_concr, "Symb", */
-  /*                        &pgf_nerc_literal_callback); */
-
   clock_t end = clock();
   double cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC;
 
@@ -109,7 +105,8 @@ int main(int argc, char* argv[]) {
     clock_t start = clock();
 
     GuExn* parse_err = gu_new_exn(ppool);
-    GuEnum* result = pgf_parse_with_heuristics(concr, cat, line, heuristics, parse_err, ppool, ppool);
+    PgfCallbacksMap* callbacks = pgf_new_callbacks_map(concr, ppool);
+    GuEnum* result = pgf_parse_with_heuristics(concr, cat, line, heuristics, callbacks, parse_err, ppool, ppool);
 
     PgfExprProb* ep = NULL;
     if (gu_ok(parse_err))
diff --git a/src/runtime/c/utils/pgf-translate.c b/src/runtime/c/utils/pgf-translate.c
index 0f3b297aa..065853215 100644
--- a/src/runtime/c/utils/pgf-translate.c
+++ b/src/runtime/c/utils/pgf-translate.c
@@ -87,8 +87,10 @@ int main(int argc, char* argv[]) {
 	}
 	
 	// Register a callback for the literal category Symbol
-	pgf_concr_add_literal(from_concr, "Symb",
-	                      &pgf_nerc_literal_callback, err);
+	PgfCallbacksMap* callbacks =
+		pgf_new_callbacks_map(from_concr, pool);
+	pgf_callbacks_map_add_literal(from_concr, callbacks,
+	                              "PN", &pgf_nerc_literal_callback);
 
 	// Create an output stream for stdout
 	GuOut* out = gu_file_out(stdout, pool);