From 7ba27229b314057104aeab8f46a7247fbb20352e Mon Sep 17 00:00:00 2001
From: "kr.angelov" <kr.angelov@gmail.com>
Date: Mon, 6 May 2013 15:28:04 +0000
Subject: the statistical parser is now using two memory pools: one for parsing
 and one for the output trees. This means that the memory for parsing can be
 released as soon as the needed abstract trees are retrieved, while the trees
 themselves are retained in the separate output pool

---
 src/runtime/python/pypgf.c | 44 ++++++++++++++++++++++++++++++++------------
 1 file changed, 32 insertions(+), 12 deletions(-)

(limited to 'src/runtime/python/pypgf.c')

diff --git a/src/runtime/python/pypgf.c b/src/runtime/python/pypgf.c
index f9428f96d..de5ead10f 100644
--- a/src/runtime/python/pypgf.c
+++ b/src/runtime/python/pypgf.c
@@ -435,6 +435,7 @@ Expr_getattro(ExprObject *self, PyObject *attr_name) {
 typedef struct IterObject {
     PyObject_HEAD
     PGFObject* grammar;
+    PyObject* container;
     GuPool* pool;
     int max_count;
     int counter;
@@ -454,8 +455,8 @@ Iter_fetch_expr(IterObject* self)
 		return NULL;
 	pyexpr->pool   = NULL;
 	pyexpr->expr   = ep->expr;
-	pyexpr->master = (PyObject*) self;
-	Py_INCREF(self);
+	pyexpr->master = self->container;
+	Py_INCREF(self->container);
 
 	PyObject* res = Py_BuildValue("(f,O)", ep->prob, pyexpr);
 	Py_DECREF(pyexpr);
@@ -483,6 +484,7 @@ Iter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     IterObject* self = (IterObject *)type->tp_alloc(type, 0);
     if (self != NULL) {
 		self->grammar = NULL;
+		self->container = NULL;
 		self->pool = NULL;
 		self->max_count = -1;
 		self->counter   = 0;
@@ -499,6 +501,8 @@ Iter_dealloc(IterObject* self)
 		gu_pool_free(self->pool);
 
 	Py_XDECREF(self->grammar);
+	
+	Py_XDECREF(self->container);
 
     self->ob_type->tp_free((PyObject*)self);
 }
@@ -661,6 +665,14 @@ pypgf_new_python_lexer(PyObject* pylexer, GuPool* pool)
 	return ((PgfLexer*) lexer);
 }
 
+#define PGF_CONTAINER_NAME "pgf.Container"
+
+void pypgf_container_descructor(PyObject *capsule)
+{
+	GuPool* pool = PyCapsule_GetPointer(capsule, PGF_CONTAINER_NAME);
+	gu_pool_free(pool);
+}
+
 static IterObject*
 Concr_parse(ConcrObject* self, PyObject *args, PyObject *keywds)
 {
@@ -698,28 +710,35 @@ Concr_parse(ConcrObject* self, PyObject *args, PyObject *keywds)
 	pyres->grammar = self->grammar;
 	Py_XINCREF(pyres->grammar);
 
-	pyres->pool = gu_new_pool();
+	GuPool* out_pool = gu_new_pool();
+	
+	PyObject* py_pool =
+		PyCapsule_New(out_pool, PGF_CONTAINER_NAME, 
+		              pypgf_container_descructor);
+	pyres->container = PyTuple_Pack(2, pyres->grammar, py_pool);
+	Py_DECREF(py_pool);
+
+	pyres->pool      = gu_new_pool();
 	pyres->max_count = max_count;
 	pyres->counter   = 0;
 	pyres->fetch     = Iter_fetch_expr;
 
-	GuPool *tmp_pool = gu_local_pool();
     GuString catname =
-		(catname_s == NULL) ? pgf_start_cat(self->grammar->pgf, tmp_pool)
-		                    : gu_str_string(catname_s, tmp_pool);
+		(catname_s == NULL) ? pgf_start_cat(self->grammar->pgf, pyres->pool)
+		                    : gu_str_string(catname_s, pyres->pool);
 
 	PgfLexer *lexer = NULL;
 	if (buf != NULL) {
-		GuIn* in = gu_data_in(buf, len, tmp_pool);
-		GuReader* rdr = gu_new_utf8_reader(in, tmp_pool);
-		lexer = pgf_new_simple_lexer(rdr, tmp_pool);
+		GuIn* in = gu_data_in(buf, len, pyres->pool);
+		GuReader* rdr = gu_new_utf8_reader(in, pyres->pool);
+		lexer = pgf_new_simple_lexer(rdr, pyres->pool);
 	} 
 	if (py_lexer != NULL) {
-		lexer = pypgf_new_python_lexer(py_lexer, tmp_pool);
+		lexer = pypgf_new_python_lexer(py_lexer, pyres->pool);
 	}
 
 	pyres->res =
-		pgf_parse(self->concr, catname, lexer, pyres->pool);
+		pgf_parse(self->concr, catname, lexer, pyres->pool, out_pool);
 
 	if (pyres->res == NULL) {
 		Py_DECREF(pyres);
@@ -740,7 +759,6 @@ Concr_parse(ConcrObject* self, PyObject *args, PyObject *keywds)
 	}
 
 	Py_XDECREF(py_lexer);
-	gu_pool_free(tmp_pool);
 
 	return pyres;
 }
@@ -784,6 +802,8 @@ Concr_getCompletions(ConcrObject* self, PyObject *args, PyObject *keywds)
 
 	pyres->grammar = self->grammar;
 	Py_XINCREF(pyres->grammar);
+	
+	pyres->container = NULL;
 
 	pyres->pool = gu_new_pool();
 	pyres->max_count = max_count;
-- 
cgit v1.2.3