Adding an option to the gf compiler to add an index to pgf files

This is gonna be used by the android library to skip unused part of a
pgf file

2 files changed, 41 insertions, 0 deletions

diff --git a/src/compiler/GF/Index.hs b/src/compiler/GF/Index.hs
new file mode 100644
index 000000000..a685f09c2
--- /dev/null
+++ b/src/compiler/GF/Index.hs
@@ -0,0 +1,36 @@
+{--
+This module provide a function for indexing a pgf. 
+
+It reads the pgf and add a global flag, called "index", containing a string 
+with concrete names and size in bytes separated by a column.
+ex : "DisambPhrasebookEng:18778 PhrasebookBul:49971 PhrasebookCat:32738..."
+--}
+module GF.Index (addIndex) where
+
+import PGF
+import PGF.Data
+import PGF.Binary
+import Data.Binary
+import Data.ByteString.Lazy (readFile,length)
+import qualified Data.Map as Map
+import Data.Map (toAscList)
+import Data.List (intercalate)
+import qualified Data.ByteString.Lazy as BS
+
+addIndex  :: PGF -> PGF
+addIndex pgf = pgf {gflags = flags}  
+  where flags = Map.insert (mkCId "index") (LStr $ showIndex index) (gflags pgf)
+        index = getIndex pgf
+
+
+showIndex :: [(String,Int)] -> String
+showIndex = intercalate " " . map f
+  where f (name,size) = name ++ ":" ++ show size
+
+getsize :: Binary a => a -> Int
+getsize x = let bs = encode x in fromIntegral $ Data.ByteString.Lazy.length bs
+
+getIndex :: PGF -> [(String,Int)]
+getIndex pgf = cncindex
+  where cncindex = map f $ Data.Map.toAscList $ concretes pgf
+        f (cncname,cnc) = (show cncname, getsize cnc)
diff --git a/src/compiler/GF/Infra/Option.hs b/src/compiler/GF/Infra/Option.hs
index fecfa273f..9c8925f3d 100644
--- a/src/compiler/GF/Infra/Option.hs
+++ b/src/compiler/GF/Infra/Option.hs
@@ -167,6 +167,7 @@ data Flags = Flags {
       optEncoding        :: String,
       optOptimizations   :: Set Optimization,
       optOptimizePGF     :: Bool,
+      optMkIndexPGF     :: Bool,
       optCFGTransforms   :: Set CFGTransform,
       optLibraryPath     :: [FilePath],
       optStartCat        :: Maybe String,
@@ -268,6 +269,7 @@ defaultFlags = Flags {
       optEncoding        = "latin1",
       optOptimizations   = Set.fromList [OptStem,OptCSE,OptExpand,OptParametrize],
       optOptimizePGF     = False,
+      optMkIndexPGF     = False,
       optCFGTransforms   = Set.fromList [CFGRemoveCycles, CFGBottomUpFilter, 
                                          CFGTopDownFilter, CFGMergeIdentical],
       optLibraryPath     = [],
@@ -360,6 +362,8 @@ optDescr =
                 "Select an optimization package. OPT = all | values | parametrize | none",
      Option [] ["optimize-pgf"] (NoArg (optimize_pgf True))
                 "Enable or disable global grammar optimization. This could significantly reduce the size of the final PGF file",
+     Option [] ["mk-index"] (NoArg (mkIndex True))
+                "Add an index to the pgf file",
      Option [] ["stem"] (onOff (toggleOptimize OptStem) True) "Perform stem-suffix analysis (default on).",
      Option [] ["cse"] (onOff (toggleOptimize OptCSE) True) "Perform common sub-expression elimination (default on).",
      Option [] ["cfg"] (ReqArg cfgTransform "TRANS") "Enable or disable specific CFG transformations. TRANS = merge, no-merge, bottomup, no-bottomup, ...",
@@ -421,6 +425,7 @@ optDescr =
                          Nothing -> fail $ "Unknown optimization package: " ++ x
                          
        optimize_pgf x = set $ \o -> o { optOptimizePGF = x }
+       mkIndex x = set $ \o -> o { optMkIndexPGF = x }
 
        toggleOptimize x b = set $ setOptimization' x b