diff options
| author | hallgren <hallgren@chalmers.se> | 2013-11-26 16:12:03 +0000 |
|---|---|---|
| committer | hallgren <hallgren@chalmers.se> | 2013-11-26 16:12:03 +0000 |
| commit | 3f57151cc346be0fbf0726d4953f3529ea45e7f4 (patch) | |
| tree | 6106269ff6ea7abb0c27d15cafdd818bb91b6bf7 /src/runtime | |
| parent | 9d7fdf7c9a525a3b5659a566f76d26d151dcd664 (diff) | |
Represent identifiers as UTF-8-encoded ByteStrings
This was a fairly simple change thanks to previous work on making the Ident
type abstract and the fact that PGF.CId already uses UTF-8-encoded
ByteStrings.
One potential pitfall is that Data.ByteString.UTF8 uses the same type for
ByteStrings as Data.ByteString. I renamed ident2bs to ident2utf8 and
bsCId to utf8CId, to make it clearer that they work with UTF-8-encoded
ByteStrings.
Since both the compiler input and identifiers are now UTF-8-encoded
ByteStrings, the lexer now creates identifiers without copying any characters.
**END OF DESCRIPTION***
Place the long patch description above the ***END OF DESCRIPTION*** marker.
The first line of this file will be the patch name.
This patch contains the following changes:
M ./src/compiler/GF/Compile/CheckGrammar.hs -3 +3
M ./src/compiler/GF/Compile/GrammarToPGF.hs -2 +2
M ./src/compiler/GF/Grammar/Binary.hs -5 +1
M ./src/compiler/GF/Grammar/Lexer.x -11 +13
M ./src/compiler/GF/Infra/Ident.hs -19 +36
M ./src/runtime/haskell/PGF.hs -1 +1
M ./src/runtime/haskell/PGF/CId.hs -2 +3
Diffstat (limited to 'src/runtime')
| -rw-r--r-- | src/runtime/haskell/PGF.hs | 2 | ||||
| -rw-r--r-- | src/runtime/haskell/PGF/CId.hs | 5 |
2 files changed, 4 insertions, 3 deletions
diff --git a/src/runtime/haskell/PGF.hs b/src/runtime/haskell/PGF.hs index 0e3c79f40..c1d903f4f 100644 --- a/src/runtime/haskell/PGF.hs +++ b/src/runtime/haskell/PGF.hs @@ -22,7 +22,7 @@ module PGF( CId, mkCId, wildCId, showCId, readCId, -- extra - ppCId, pIdent, bsCId, + ppCId, pIdent, utf8CId, -- * Languages Language, diff --git a/src/runtime/haskell/PGF/CId.hs b/src/runtime/haskell/PGF/CId.hs index 0594d9fc1..785642cdf 100644 --- a/src/runtime/haskell/PGF/CId.hs +++ b/src/runtime/haskell/PGF/CId.hs @@ -3,7 +3,7 @@ module PGF.CId (CId(..), readCId, showCId, -- utils - bsCId, pCId, pIdent, ppCId) where + utf8CId, pCId, pIdent, ppCId) where import Control.Monad import qualified Data.ByteString.Char8 as BS @@ -24,7 +24,8 @@ wildCId = CId (BS.singleton '_') mkCId :: String -> CId mkCId s = CId (UTF8.fromString s) -bsCId = CId +-- | Creates an identifier from a UTF-8-encoded 'ByteString' +utf8CId = CId -- | Reads an identifier from 'String'. The function returns 'Nothing' if the string is not valid identifier. readCId :: String -> Maybe CId |