diff options
| author | aarne <unknown> | 2005-09-29 12:20:08 +0000 |
|---|---|---|
| committer | aarne <unknown> | 2005-09-29 12:20:08 +0000 |
| commit | 1db8b90811b280a61acb3dc2961fe1ef45c6a2fb (patch) | |
| tree | ec4b1524f493cb2b8dc50c253018c8f4b1aa84bb /src/GF/UseGrammar/Tokenize.hs | |
| parent | 34c0e7925922a353397802987ef61511504589f0 (diff) | |
lexer=ignore
Diffstat (limited to 'src/GF/UseGrammar/Tokenize.hs')
| -rw-r--r-- | src/GF/UseGrammar/Tokenize.hs | 15 |
1 files changed, 12 insertions, 3 deletions
diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs index 6a8119ac0..bfc0e53bb 100644 --- a/src/GF/UseGrammar/Tokenize.hs +++ b/src/GF/UseGrammar/Tokenize.hs @@ -5,9 +5,9 @@ -- Stability : (stable) -- Portability : (portable) -- --- > CVS $Date: 2005/04/21 16:23:52 $ --- > CVS $Author: bringert $ --- > CVS $Revision: 1.13 $ +-- > CVS $Date: 2005/09/29 13:20:08 $ +-- > CVS $Author: aarne $ +-- > CVS $Revision: 1.14 $ -- -- lexers = tokenizers, to prepare input for GF grammars. AR 4\/1\/2002. -- an entry for each is included in 'Custom.customTokenizer' @@ -22,6 +22,7 @@ module GF.UseGrammar.Tokenize ( tokWords, lexText, lexC2M, lexC2M', lexTextLiteral, + lexIgnore ) where import GF.Data.Operations @@ -184,3 +185,11 @@ lexHaskellVar isKnown = unknown2var isKnown . lexHaskell eitherUpper isKnown w@(c:cs) = isKnown (toLower c : cs) || isKnown (toUpper c : cs) eitherUpper isKnown w = isKnown w +-- ignore unknown tokens (e.g. keyword spotting) + +lexIgnore :: (String -> Bool) -> [CFTok] -> [CFTok] +lexIgnore isKnown = concatMap mkOne where + mkOne t@(TS s) + | isKnown s = [t] + | otherwise = [] + mkOne t = [t] |