diff options
| author | aarne <unknown> | 2004-08-25 15:17:13 +0000 |
|---|---|---|
| committer | aarne <unknown> | 2004-08-25 15:17:13 +0000 |
| commit | 51c9afa5c8025971d8080fa271a0e8486b9effde (patch) | |
| tree | f5bf73caee6a25347d72385f16cc5dcf8fd12967 /src | |
| parent | 4af78854b77949ebc839c43281774976b5575309 (diff) | |
codevars
Diffstat (limited to 'src')
| -rw-r--r-- | src/GF/UseGrammar/Custom.hs | 2 | ||||
| -rw-r--r-- | src/GF/UseGrammar/Tokenize.hs | 9 | ||||
| -rw-r--r-- | src/HelpFile | 2 | ||||
| -rw-r--r-- | src/HelpFile.hs | 2 |
4 files changed, 12 insertions, 3 deletions
diff --git a/src/GF/UseGrammar/Custom.hs b/src/GF/UseGrammar/Custom.hs index e7989de30..9df79e983 100644 --- a/src/GF/UseGrammar/Custom.hs +++ b/src/GF/UseGrammar/Custom.hs @@ -320,7 +320,7 @@ customTokenizer = ,(strCI "vars", const $ tokVars) ,(strCI "chars", const $ map (tS . singleton)) ,(strCI "code", const $ lexHaskell) - ,(strCI "codevars", const $ (mkTokVars lexHaskell)) + ,(strCI "codevars", lexHaskellVar . stateIsWord) ,(strCI "text", const $ lexText) ,(strCI "unglue", \gr -> map tS . decomposeWords (stateMorpho gr)) ,(strCI "codelit", lexHaskellLiteral . stateIsWord) diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs index 77c6222ac..7ae3463a0 100644 --- a/src/GF/UseGrammar/Tokenize.hs +++ b/src/GF/UseGrammar/Tokenize.hs @@ -134,9 +134,18 @@ unknown2string isKnown = map mkOne where mkOne t@(TC s) = if isKnown s then t else mkTL s mkOne t = t +unknown2var :: (String -> Bool) -> [CFTok] -> [CFTok] +unknown2var isKnown = map mkOne where + mkOne t@(TS "??") = if isKnown "??" then t else tM "??" + mkOne t@(TS s) = if isKnown s then t else tV s + mkOne t@(TC s) = if isKnown s then t else tV s + mkOne t = t + lexTextLiteral isKnown = unknown2string (eitherUpper isKnown) . lexText lexHaskellLiteral isKnown = unknown2string isKnown . lexHaskell +lexHaskellVar isKnown = unknown2var isKnown . lexHaskell + eitherUpper isKnown w@(c:cs) = isKnown (toLower c : cs) || isKnown (toUpper c : cs) eitherUpper isKnown w = isKnown w diff --git a/src/HelpFile b/src/HelpFile index 3e2b04335..85b39c8d9 100644 --- a/src/HelpFile +++ b/src/HelpFile @@ -418,7 +418,7 @@ q, quit: q -lexer=vars like words, but "x","x_...","$...$" as vars, "?..." as meta -lexer=chars each character is a token -lexer=code use Haskell's lex - -lexer=codevars like code, but "x", "x_..." as vars, "??" as meta + -lexer=codevars like code, but treat unknown words as variables, ?? as meta -lexer=text with conventions on punctuation and capital letters -lexer=codelit like code, but treat unknown words as string literals -lexer=textlit like text, but treat unknown words as string literals diff --git a/src/HelpFile.hs b/src/HelpFile.hs index 66be956fa..3cc6b2ada 100644 --- a/src/HelpFile.hs +++ b/src/HelpFile.hs @@ -431,7 +431,7 @@ txtHelpFile = "\n -lexer=vars like words, but \"x\",\"x_...\",\"$...$\" as vars, \"?...\" as meta" ++ "\n -lexer=chars each character is a token" ++ "\n -lexer=code use Haskell's lex" ++ - "\n -lexer=codevars like code, but \"x\", \"x_...\" as vars, \"??\" as meta" ++ + "\n -lexer=codevars like code, but treat unknown words as variables, ?? as meta " ++ "\n -lexer=text with conventions on punctuation and capital letters" ++ "\n -lexer=codelit like code, but treat unknown words as string literals" ++ "\n -lexer=textlit like text, but treat unknown words as string literals" ++ |