diff options
Diffstat (limited to 'src/GF/UseGrammar/Tokenize.hs')
| -rw-r--r-- | src/GF/UseGrammar/Tokenize.hs | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs index 22d70a9b1..77c6222ac 100644 --- a/src/GF/UseGrammar/Tokenize.hs +++ b/src/GF/UseGrammar/Tokenize.hs @@ -35,6 +35,11 @@ mkCFTokVar s = case s of '$':xs@(_:_) -> if last s == '$' then tV (init xs) else tS s _ -> tS s +mkTokVars :: (String -> [CFTok]) -> String -> [CFTok] +mkTokVars tok = map tv . tok where + tv (TS s) = mkCFTokVar s + tv t = t + mkLit :: String -> CFTok mkLit s = if (all isDigit s) then (tI s) else (tL s) |