From 51c9afa5c8025971d8080fa271a0e8486b9effde Mon Sep 17 00:00:00 2001
From: aarne <unknown>
Date: Wed, 25 Aug 2004 15:17:13 +0000
Subject: codevars

---
 src/GF/UseGrammar/Custom.hs   | 2 +-
 src/GF/UseGrammar/Tokenize.hs | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

(limited to 'src/GF')

diff --git a/src/GF/UseGrammar/Custom.hs b/src/GF/UseGrammar/Custom.hs
index e7989de30..9df79e983 100644
--- a/src/GF/UseGrammar/Custom.hs
+++ b/src/GF/UseGrammar/Custom.hs
@@ -320,7 +320,7 @@ customTokenizer =
   ,(strCI "vars",      const $ tokVars)
   ,(strCI "chars",     const $ map (tS . singleton))
   ,(strCI "code",      const $ lexHaskell)
-  ,(strCI "codevars",  const $ (mkTokVars lexHaskell))
+  ,(strCI "codevars",  lexHaskellVar . stateIsWord)
   ,(strCI "text",      const $ lexText)
   ,(strCI "unglue",    \gr -> map tS . decomposeWords (stateMorpho gr))
   ,(strCI "codelit",   lexHaskellLiteral . stateIsWord)
diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs
index 77c6222ac..7ae3463a0 100644
--- a/src/GF/UseGrammar/Tokenize.hs
+++ b/src/GF/UseGrammar/Tokenize.hs
@@ -134,9 +134,18 @@ unknown2string isKnown = map mkOne where
   mkOne t@(TC s) = if isKnown s then t else mkTL s
   mkOne t        = t
 
+unknown2var :: (String -> Bool) -> [CFTok] -> [CFTok]
+unknown2var isKnown = map mkOne where
+  mkOne t@(TS "??") = if isKnown "??" then t else tM "??"
+  mkOne t@(TS s) = if isKnown s then t else tV s
+  mkOne t@(TC s) = if isKnown s then t else tV s
+  mkOne t        = t
+
 lexTextLiteral    isKnown = unknown2string (eitherUpper isKnown) . lexText
 lexHaskellLiteral isKnown = unknown2string isKnown . lexHaskell
 
+lexHaskellVar     isKnown = unknown2var isKnown . lexHaskell
+
 eitherUpper isKnown w@(c:cs) = isKnown (toLower c : cs) || isKnown (toUpper c : cs)
 eitherUpper isKnown w = isKnown w
 
-- 
cgit v1.2.3