codevars

1 files changed, 9 insertions, 0 deletions

diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs
index 77c6222ac..7ae3463a0 100644
--- a/src/GF/UseGrammar/Tokenize.hs
+++ b/src/GF/UseGrammar/Tokenize.hs
@@ -134,9 +134,18 @@ unknown2string isKnown = map mkOne where
   mkOne t@(TC s) = if isKnown s then t else mkTL s
   mkOne t        = t
 
+unknown2var :: (String -> Bool) -> [CFTok] -> [CFTok]
+unknown2var isKnown = map mkOne where
+  mkOne t@(TS "??") = if isKnown "??" then t else tM "??"
+  mkOne t@(TS s) = if isKnown s then t else tV s
+  mkOne t@(TC s) = if isKnown s then t else tV s
+  mkOne t        = t
+
 lexTextLiteral    isKnown = unknown2string (eitherUpper isKnown) . lexText
 lexHaskellLiteral isKnown = unknown2string isKnown . lexHaskell
 
+lexHaskellVar     isKnown = unknown2var isKnown . lexHaskell
+
 eitherUpper isKnown w@(c:cs) = isKnown (toLower c : cs) || isKnown (toUpper c : cs)
 eitherUpper isKnown w = isKnown w