diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/GF/UseGrammar/Tokenize.hs | 9 |
1 files changed, 8 insertions, 1 deletions
diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs index 7ae3463a0..d53fb0597 100644 --- a/src/GF/UseGrammar/Tokenize.hs +++ b/src/GF/UseGrammar/Tokenize.hs @@ -130,13 +130,20 @@ reservedAnsiCWords = words $ unknown2string :: (String -> Bool) -> [CFTok] -> [CFTok] unknown2string isKnown = map mkOne where - mkOne t@(TS s) = if isKnown s then t else mkTL s + mkOne t@(TS s) + | isKnown s = t + | all isDigit s = tI s + | otherwise = tV s mkOne t@(TC s) = if isKnown s then t else mkTL s mkOne t = t unknown2var :: (String -> Bool) -> [CFTok] -> [CFTok] unknown2var isKnown = map mkOne where mkOne t@(TS "??") = if isKnown "??" then t else tM "??" + mkOne t@(TS s) + | isKnown s = t + | all isDigit s = tI s + | otherwise = tV s mkOne t@(TS s) = if isKnown s then t else tV s mkOne t@(TC s) = if isKnown s then t else tV s mkOne t = t |