diff options
| author | aarne <unknown> | 2003-11-03 16:27:55 +0000 |
|---|---|---|
| committer | aarne <unknown> | 2003-11-03 16:27:55 +0000 |
| commit | 94326929b144913642121bef8f8ecc98feb992e7 (patch) | |
| tree | 07d59cc33cbef2ac79c6f3f573b9718c51322e7a /src/GF/UseGrammar/Tokenize.hs | |
| parent | 2728e6e7ceec92c7f781368b4a523b37e5dee3b9 (diff) | |
Fixed several things, e.g. tokenizer.
Diffstat (limited to 'src/GF/UseGrammar/Tokenize.hs')
| -rw-r--r-- | src/GF/UseGrammar/Tokenize.hs | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs index dd0879931..ac28276f5 100644 --- a/src/GF/UseGrammar/Tokenize.hs +++ b/src/GF/UseGrammar/Tokenize.hs @@ -21,7 +21,11 @@ tokVars :: String -> [CFTok] tokVars = map mkCFTokVar . words mkCFTok :: String -> CFTok -mkCFTok s = tS s ---- if (isLiteral s) then (mkLit s) else (tS s) +mkCFTok s = case s of + '"' :cs@(_:_) -> tL $ init cs + '\'':cs@(_:_) -> tL $ init cs --- 's Gravenhage + _:_ | all isDigit s -> tI s + _ -> tS s mkCFTokVar :: String -> CFTok mkCFTokVar s = case s of |