diff options
| author | aarne <aarne@cs.chalmers.se> | 2006-06-20 08:38:44 +0000 |
|---|---|---|
| committer | aarne <aarne@cs.chalmers.se> | 2006-06-20 08:38:44 +0000 |
| commit | 402a113b567a96eef61946552b68df6ac6eb6712 (patch) | |
| tree | 6446bbaf195800c7e0ec71d30cba113c9bcf1f74 /src/GF/UseGrammar/Tokenize.hs | |
| parent | cb168e92e23d0f620b75f6119a4cb298360e6f21 (diff) | |
made -fcfg default parser; added lexer textvars
Diffstat (limited to 'src/GF/UseGrammar/Tokenize.hs')
| -rw-r--r-- | src/GF/UseGrammar/Tokenize.hs | 12 |
1 files changed, 11 insertions, 1 deletions
diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs index d16fdf32f..9f1ab5449 100644 --- a/src/GF/UseGrammar/Tokenize.hs +++ b/src/GF/UseGrammar/Tokenize.hs @@ -20,6 +20,7 @@ module GF.UseGrammar.Tokenize ( tokWords, lexHaskellLiteral, lexHaskellVar, lexText, + lexTextVar, lexC2M, lexC2M', lexTextLiteral, lexIgnore, @@ -58,6 +59,10 @@ isFloat s = case s of '.':cs@(_:_) -> all isDigit cs _ -> False +isString s = case s of + c:cs@(_:_) -> (c == '\'' && d == '\'') || (c == '"' && d == '"') where d = last cs + _ -> False + mkCFTok :: String -> CFTok mkCFTok s = case s of @@ -86,6 +91,7 @@ mkLit s | all isDigit s = tI s | otherwise = tL s +-- obsolete mkTL :: String -> CFTok mkTL s | isFloat s = tF s @@ -104,6 +110,7 @@ lexText :: String -> [CFTok] lexText = uncap . lx where lx s = case s of + '?':'?':cs -> tS "??" : lx cs p : cs | isMPunct p -> tS [p] : uncap (lx cs) p : cs | isPunct p -> tS [p] : lx cs s : cs | isSpace s -> lx cs @@ -177,7 +184,7 @@ unknown2string isKnown = map mkOne where | isFloat s = tF s | all isDigit s = tI s | otherwise = tL s - mkOne t@(TC s) = if isKnown s then t else mkTL s + mkOne t@(TC s) = if isKnown s then t else mkLit s mkOne t = t unknown2var :: (String -> Bool) -> [CFTok] -> [CFTok] @@ -186,6 +193,7 @@ unknown2var isKnown = map mkOne where mkOne t@(TS s) | isKnown s = t | isFloat s = tF s + | isString s = tL (init (tail s)) | all isDigit s = tI s | otherwise = tV s mkOne t@(TC s) = if isKnown s then t else tV s @@ -197,6 +205,8 @@ lexTextLiteral isKnown = unknown2string (eitherUpper isKnown) . lexText lexHaskellLiteral isKnown = unknown2string isKnown . lexHaskell lexHaskellVar isKnown = unknown2var isKnown . lexHaskell +lexTextVar isKnown = unknown2var (eitherUpper isKnown) . lexText + eitherUpper isKnown w@(c:cs) = isKnown (toLower c : cs) || isKnown (toUpper c : cs) eitherUpper isKnown w = isKnown w |