Adding an option to the gf compiler to add an index to pgf files

This is gonna be used by the android library to skip unused part of a
pgf file

gf.cabal

@@ -145,6 +145,7 @@ executable gf
     GF.Compile.PGFtoProlog
     GF.Compile.PGFtoJS
     GF.Compile
+    GF.Index
     GF.Quiz
     PGF
     PGF.CId

src/compiler/GF/Index.hs

@@ -0,0 +1,36 @@
+{--
+This module provide a function for indexing a pgf. 
+
+It reads the pgf and add a global flag, called "index", containing a string 
+with concrete names and size in bytes separated by a column.
+ex : "DisambPhrasebookEng:18778 PhrasebookBul:49971 PhrasebookCat:32738..."
+--}
+module GF.Index (addIndex) where
+
+import PGF
+import PGF.Data
+import PGF.Binary
+import Data.Binary
+import Data.ByteString.Lazy (readFile,length)
+import qualified Data.Map as Map
+import Data.Map (toAscList)
+import Data.List (intercalate)
+import qualified Data.ByteString.Lazy as BS
+
+addIndex  :: PGF -> PGF
+addIndex pgf = pgf {gflags = flags}  
+  where flags = Map.insert (mkCId "index") (LStr $ showIndex index) (gflags pgf)
+        index = getIndex pgf
+
+
+showIndex :: [(String,Int)] -> String
+showIndex = intercalate " " . map f
+  where f (name,size) = name ++ ":" ++ show size
+
+getsize :: Binary a => a -> Int
+getsize x = let bs = encode x in fromIntegral $ Data.ByteString.Lazy.length bs
+
+getIndex :: PGF -> [(String,Int)]
+getIndex pgf = cncindex
+  where cncindex = map f $ Data.Map.toAscList $ concretes pgf
+        f (cncname,cnc) = (show cncname, getsize cnc)

src/compiler/GF/Infra/Option.hs

@@ -167,6 +167,7 @@ data Flags = Flags {
       optEncoding        :: String,
       optOptimizations   :: Set Optimization,
       optOptimizePGF     :: Bool,
+      optMkIndexPGF     :: Bool,
       optCFGTransforms   :: Set CFGTransform,
       optLibraryPath     :: [FilePath],
       optStartCat        :: Maybe String,
@@ -268,6 +269,7 @@ defaultFlags = Flags {
       optEncoding        = "latin1",
       optOptimizations   = Set.fromList [OptStem,OptCSE,OptExpand,OptParametrize],
       optOptimizePGF     = False,
+      optMkIndexPGF     = False,
       optCFGTransforms   = Set.fromList [CFGRemoveCycles, CFGBottomUpFilter, 
                                          CFGTopDownFilter, CFGMergeIdentical],
       optLibraryPath     = [],
@@ -360,6 +362,8 @@ optDescr =
                 "Select an optimization package. OPT = all | values | parametrize | none",
      Option [] ["optimize-pgf"] (NoArg (optimize_pgf True))
                 "Enable or disable global grammar optimization. This could significantly reduce the size of the final PGF file",
+     Option [] ["mk-index"] (NoArg (mkIndex True))
+                "Add an index to the pgf file",
      Option [] ["stem"] (onOff (toggleOptimize OptStem) True) "Perform stem-suffix analysis (default on).",
      Option [] ["cse"] (onOff (toggleOptimize OptCSE) True) "Perform common sub-expression elimination (default on).",
      Option [] ["cfg"] (ReqArg cfgTransform "TRANS") "Enable or disable specific CFG transformations. TRANS = merge, no-merge, bottomup, no-bottomup, ...",
@@ -421,6 +425,7 @@ optDescr =
                          Nothing -> fail $ "Unknown optimization package: " ++ x
                          
        optimize_pgf x = set $ \o -> o { optOptimizePGF = x }
+       mkIndex x = set $ \o -> o { optMkIndexPGF = x }
 
        toggleOptimize x b = set $ setOptimization' x b
 

src/compiler/GFC.hs

@@ -5,6 +5,7 @@ import PGF
 import PGF.CId
 import PGF.Data
 import PGF.Optimize
+import GF.Index
 import GF.Compile
 import GF.Compile.Export
 
@@ -59,7 +60,8 @@ unionPGFFiles :: Options -> [FilePath] -> IOE ()
 unionPGFFiles opts fs = 
     do pgfs <- mapM readPGFVerbose fs
        let pgf0 = foldl1 unionPGF pgfs
-           pgf = if flag optOptimizePGF opts then optimizePGF pgf0 else pgf0
+           pgf1 = if flag optOptimizePGF opts then optimizePGF pgf0 else pgf0
+           pgf = if flag optMkIndexPGF opts then addIndex pgf1 else pgf1
            pgfFile = grammarName opts pgf <.> "pgf"
        if pgfFile `elem` fs 
          then putStrLnE $ "Refusing to overwrite " ++ pgfFile