diff options
| author | aarne <aarne@cs.chalmers.se> | 2005-12-21 19:46:48 +0000 |
|---|---|---|
| committer | aarne <aarne@cs.chalmers.se> | 2005-12-21 19:46:48 +0000 |
| commit | 89ec5b808b56eb408f0200aa38f64c25f59aff07 (patch) | |
| tree | cc1b2678da9009fcf89ab1785275ba1e2ee58ca8 /src/GF/UseGrammar/Tokenize.hs | |
| parent | f4c5fcf44ad9cfa79435ce76fa40e5f57d645cce (diff) | |
parsing escaped strings from command line fixed
Diffstat (limited to 'src/GF/UseGrammar/Tokenize.hs')
| -rw-r--r-- | src/GF/UseGrammar/Tokenize.hs | 8 |
1 files changed, 5 insertions, 3 deletions
diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs index 91f7f0c61..d16fdf32f 100644 --- a/src/GF/UseGrammar/Tokenize.hs +++ b/src/GF/UseGrammar/Tokenize.hs @@ -22,7 +22,8 @@ module GF.UseGrammar.Tokenize ( tokWords, lexText, lexC2M, lexC2M', lexTextLiteral, - lexIgnore + lexIgnore, + wordsLits ) where import GF.Data.Operations @@ -39,7 +40,7 @@ tokWords :: String -> [CFTok] tokWords = map tS . words tokLits :: String -> [CFTok] -tokLits = map mkCFTok . mergeStr . words where +tokLits = map mkCFTok . mergeStr . wordsLits where mergeStr ss = case ss of w@(c:cs):rest | elem c "\'\"" && c /= last w -> getStr [w] rest w :rest -> w : mergeStr rest @@ -50,7 +51,7 @@ tokLits = map mkCFTok . mergeStr . words where [] -> reverse v tokVars :: String -> [CFTok] -tokVars = map mkCFTokVar . words +tokVars = map mkCFTokVar . wordsLits isFloat s = case s of c:cs | isDigit c -> isFloat cs @@ -208,3 +209,4 @@ lexIgnore isKnown = concatMap mkOne where | isKnown s = [t] | otherwise = [] mkOne t = [t] + |