diff options
| author | aarne <unknown> | 2004-09-17 22:02:35 +0000 |
|---|---|---|
| committer | aarne <unknown> | 2004-09-17 22:02:35 +0000 |
| commit | 6ec3a53d3cd1666696430d25e1d0c746f3c7dde8 (patch) | |
| tree | ee8ad62e4fb72a9bee1a953f3e35bb723bc870c5 /src | |
| parent | ecc132dccfc7617ed413f21ee37539475081f8ec (diff) | |
C compiler
Diffstat (limited to 'src')
| -rw-r--r-- | src/GF/UseGrammar/Tokenize.hs | 9 |
1 files changed, 8 insertions, 1 deletions
diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs index 7ae3463a0..d53fb0597 100644 --- a/src/GF/UseGrammar/Tokenize.hs +++ b/src/GF/UseGrammar/Tokenize.hs @@ -130,13 +130,20 @@ reservedAnsiCWords = words $ unknown2string :: (String -> Bool) -> [CFTok] -> [CFTok] unknown2string isKnown = map mkOne where - mkOne t@(TS s) = if isKnown s then t else mkTL s + mkOne t@(TS s) + | isKnown s = t + | all isDigit s = tI s + | otherwise = tV s mkOne t@(TC s) = if isKnown s then t else mkTL s mkOne t = t unknown2var :: (String -> Bool) -> [CFTok] -> [CFTok] unknown2var isKnown = map mkOne where mkOne t@(TS "??") = if isKnown "??" then t else tM "??" + mkOne t@(TS s) + | isKnown s = t + | all isDigit s = tI s + | otherwise = tV s mkOne t@(TS s) = if isKnown s then t else tV s mkOne t@(TC s) = if isKnown s then t else tV s mkOne t = t |
