From 94326929b144913642121bef8f8ecc98feb992e7 Mon Sep 17 00:00:00 2001
From: aarne <unknown>
Date: Mon, 3 Nov 2003 16:27:55 +0000
Subject: Fixed several things, e.g. tokenizer.

---
 src/GF/UseGrammar/Tokenize.hs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'src/GF/UseGrammar/Tokenize.hs')

diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs
index dd0879931..ac28276f5 100644
--- a/src/GF/UseGrammar/Tokenize.hs
+++ b/src/GF/UseGrammar/Tokenize.hs
@@ -21,7 +21,11 @@ tokVars :: String -> [CFTok]
 tokVars = map mkCFTokVar . words
 
 mkCFTok :: String -> CFTok
-mkCFTok s = tS s ---- if (isLiteral s) then (mkLit s) else (tS s)
+mkCFTok s = case s of
+  '"' :cs@(_:_) -> tL $ init cs
+  '\'':cs@(_:_) -> tL $ init cs --- 's Gravenhage
+  _:_ | all isDigit s -> tI s
+  _ -> tS s
 
 mkCFTokVar :: String -> CFTok
 mkCFTokVar s = case s of
-- 
cgit v1.2.3