diff options
Diffstat (limited to 'src/GF')
| -rw-r--r-- | src/GF/UseGrammar/Custom.hs | 2 | ||||
| -rw-r--r-- | src/GF/UseGrammar/Tokenize.hs | 9 |
2 files changed, 10 insertions, 1 deletions
diff --git a/src/GF/UseGrammar/Custom.hs b/src/GF/UseGrammar/Custom.hs index e7989de30..9df79e983 100644 --- a/src/GF/UseGrammar/Custom.hs +++ b/src/GF/UseGrammar/Custom.hs @@ -320,7 +320,7 @@ customTokenizer = ,(strCI "vars", const $ tokVars) ,(strCI "chars", const $ map (tS . singleton)) ,(strCI "code", const $ lexHaskell) - ,(strCI "codevars", const $ (mkTokVars lexHaskell)) + ,(strCI "codevars", lexHaskellVar . stateIsWord) ,(strCI "text", const $ lexText) ,(strCI "unglue", \gr -> map tS . decomposeWords (stateMorpho gr)) ,(strCI "codelit", lexHaskellLiteral . stateIsWord) diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs index 77c6222ac..7ae3463a0 100644 --- a/src/GF/UseGrammar/Tokenize.hs +++ b/src/GF/UseGrammar/Tokenize.hs @@ -134,9 +134,18 @@ unknown2string isKnown = map mkOne where mkOne t@(TC s) = if isKnown s then t else mkTL s mkOne t = t +unknown2var :: (String -> Bool) -> [CFTok] -> [CFTok] +unknown2var isKnown = map mkOne where + mkOne t@(TS "??") = if isKnown "??" then t else tM "??" + mkOne t@(TS s) = if isKnown s then t else tV s + mkOne t@(TC s) = if isKnown s then t else tV s + mkOne t = t + lexTextLiteral isKnown = unknown2string (eitherUpper isKnown) . lexText lexHaskellLiteral isKnown = unknown2string isKnown . lexHaskell +lexHaskellVar isKnown = unknown2var isKnown . lexHaskell + eitherUpper isKnown w@(c:cs) = isKnown (toLower c : cs) || isKnown (toUpper c : cs) eitherUpper isKnown w = isKnown w |