diff options
Diffstat (limited to 'src/GF/UseGrammar/Tokenize.hs')
| -rw-r--r-- | src/GF/UseGrammar/Tokenize.hs | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs index dd0879931..ac28276f5 100644 --- a/src/GF/UseGrammar/Tokenize.hs +++ b/src/GF/UseGrammar/Tokenize.hs @@ -21,7 +21,11 @@ tokVars :: String -> [CFTok] tokVars = map mkCFTokVar . words mkCFTok :: String -> CFTok -mkCFTok s = tS s ---- if (isLiteral s) then (mkLit s) else (tS s) +mkCFTok s = case s of + '"' :cs@(_:_) -> tL $ init cs + '\'':cs@(_:_) -> tL $ init cs --- 's Gravenhage + _:_ | all isDigit s -> tI s + _ -> tS s mkCFTokVar :: String -> CFTok mkCFTokVar s = case s of |