From 3f57151cc346be0fbf0726d4953f3529ea45e7f4 Mon Sep 17 00:00:00 2001 From: hallgren Date: Tue, 26 Nov 2013 16:12:03 +0000 Subject: Represent identifiers as UTF-8-encoded ByteStrings This was a fairly simple change thanks to previous work on making the Ident type abstract and the fact that PGF.CId already uses UTF-8-encoded ByteStrings. One potential pitfall is that Data.ByteString.UTF8 uses the same type for ByteStrings as Data.ByteString. I renamed ident2bs to ident2utf8 and bsCId to utf8CId, to make it clearer that they work with UTF-8-encoded ByteStrings. Since both the compiler input and identifiers are now UTF-8-encoded ByteStrings, the lexer now creates identifiers without copying any characters. **END OF DESCRIPTION*** Place the long patch description above the ***END OF DESCRIPTION*** marker. The first line of this file will be the patch name. This patch contains the following changes: M ./src/compiler/GF/Compile/CheckGrammar.hs -3 +3 M ./src/compiler/GF/Compile/GrammarToPGF.hs -2 +2 M ./src/compiler/GF/Grammar/Binary.hs -5 +1 M ./src/compiler/GF/Grammar/Lexer.x -11 +13 M ./src/compiler/GF/Infra/Ident.hs -19 +36 M ./src/runtime/haskell/PGF.hs -1 +1 M ./src/runtime/haskell/PGF/CId.hs -2 +3 --- src/compiler/GF/Compile/CheckGrammar.hs | 6 +++--- src/compiler/GF/Compile/GrammarToPGF.hs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'src/compiler/GF/Compile') diff --git a/src/compiler/GF/Compile/CheckGrammar.hs b/src/compiler/GF/Compile/CheckGrammar.hs index 568686f92..d924e413e 100644 --- a/src/compiler/GF/Compile/CheckGrammar.hs +++ b/src/compiler/GF/Compile/CheckGrammar.hs @@ -298,9 +298,9 @@ checkInfo opts sgr (m,mo) c info = do -- | for grammars obtained otherwise than by parsing ---- update!! checkReservedId :: Ident -> Check () -checkReservedId x - | isReservedWord (ident2bs x) = checkWarn (text "reserved word used as identifier:" <+> ppIdent x) - | otherwise = return () +checkReservedId x = + when (isReservedWord x) $ + checkWarn (text "reserved word used as identifier:" <+> ppIdent x) -- auxiliaries diff --git a/src/compiler/GF/Compile/GrammarToPGF.hs b/src/compiler/GF/Compile/GrammarToPGF.hs index 25db4bac7..5917b709c 100644 --- a/src/compiler/GF/Compile/GrammarToPGF.hs +++ b/src/compiler/GF/Compile/GrammarToPGF.hs @@ -5,7 +5,7 @@ module GF.Compile.GrammarToPGF (mkCanon2pgf) where import GF.Compile.GeneratePMCFG import GF.Compile.GenerateBC -import PGF(CId,mkCId,bsCId) +import PGF(CId,mkCId,utf8CId) import PGF.Data(fidInt,fidFloat,fidString,fidVar) import PGF.Optimize(updateProductionIndices) --import qualified PGF.Macros as CM @@ -103,7 +103,7 @@ mkCanon2pgf opts gr am = do return (seqs, ((m,id), info) : is) i2i :: Ident -> CId -i2i = bsCId . ident2bs +i2i = utf8CId . ident2utf8 mkType :: [Ident] -> A.Type -> C.Type mkType scope t = -- cgit v1.2.3