This patch removes Gregoire's parse_tokens function in the python binding and adds another implementation which builds on the existing API for lexers in the C runtime. Now it is possible to write incremental Lexers in Python

1 files changed, 5 insertions, 2 deletions

diff --git a/src/runtime/c/pgf/lexer.h b/src/runtime/c/pgf/lexer.h
index 6f01d4d10..f89629cea 100644
--- a/src/runtime/c/pgf/lexer.h
+++ b/src/runtime/c/pgf/lexer.h
@@ -6,10 +6,13 @@
 /// A single lexical token			      
 typedef GuString PgfToken;
 
-typedef struct PgfLexer PgfLexer;
+typedef struct {
+	PgfToken (*read_token)();
+	PgfToken tok;
+} PgfLexer;
 
 PgfLexer*
-pgf_new_lexer(GuReader *rdr, GuPool *pool);
+pgf_new_simple_lexer(GuReader *rdr, GuPool *pool);
 
 PgfToken
 pgf_lexer_read_token(PgfLexer *lexer, GuExn* err);