Adding a basic lexicon-based tokenizer and the asociated command in gf shell

1 files changed, 4 insertions, 0 deletions

diff --git a/src/runtime/haskell/PGF.hs b/src/runtime/haskell/PGF.hs
index 42ef8aaff..8530d9a71 100644
--- a/src/runtime/haskell/PGF.hs
+++ b/src/runtime/haskell/PGF.hs
@@ -109,6 +109,9 @@ module PGF(
            -- ** Morphological Analysis
            Lemma, Analysis, Morpho,
            lookupMorpho, buildMorpho, fullFormLexicon,
+           
+           -- ** Tokenizing
+           mkTokenizer,
 
            -- ** Visualizations
            graphvizAbstractTree,
@@ -141,6 +144,7 @@ import PGF.Expr (Tree)
 import PGF.Morphology
 import PGF.Data
 import PGF.Binary
+import PGF.Tokenizer
 import qualified PGF.Forest as Forest
 import qualified PGF.Parse as Parse