From d84c5ef1715c3e4aed4098ee9c847e2dcc86cba4 Mon Sep 17 00:00:00 2001 From: hallgren Date: Mon, 25 Aug 2014 09:56:00 +0000 Subject: Experimental: parallel batch compilation of grammars On my laptop these changes speed up the full build of the RGL and example grammars with 'cabal build' from ~95s to ~43s and the zero build from ~18s to ~5s. The main change is the introduction of the module GF.CompileInParallel that replaces GF.Compile and the function GF.Compile.ReadFiles.getAllFiles. At present, it is activated with the new -j flag, and it is only used when combined with --make or --batch. In addition, to get parallel computations, you need to add GHC run-time flags, e.g., +RTS -N -A20M -RTS, to the command line. The Setup.hs script has been modified to pass the appropriate flags to GF for parallel compilation when compiling the RGL and example grammars, but you need a recent version of Cabal for this to work (probably >=1.20). Some additonal refactoring were made during this work. A new monad is used to avoid warnings/error messages from different modules to be intertwined when compiling in parallel, so some functios that were hardiwred to the IO or IOE monads have been lifted to work in arbitrary monads that are instances in the appropriate classes. --- src/compiler/GFC.hs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'src/compiler/GFC.hs') diff --git a/src/compiler/GFC.hs b/src/compiler/GFC.hs index 4b1034faa..6f909b511 100644 --- a/src/compiler/GFC.hs +++ b/src/compiler/GFC.hs @@ -4,7 +4,8 @@ module GFC (mainGFC, writePGF) where import PGF import PGF.Internal(concretes,optimizePGF,unionPGF) import PGF.Internal(putSplitAbs,encodeFile,runPut) -import GF.Compile +import GF.Compile as S(batchCompile,link,srcAbsName) +import qualified GF.CompileInParallel as P(batchCompile) import GF.Compile.Export import GF.Compile.CFGtoPGF import GF.Compile.GetGrammar @@ -41,7 +42,7 @@ mainGFC opts fs = do compileSourceFiles :: Options -> [FilePath] -> IOE () compileSourceFiles opts fs = - do cnc_gr@(cnc,t_src,gr) <- batchCompile opts fs + do (t_src,~cnc_grs@(~(cnc,gr):_)) <- batchCompile opts fs unless (flag optStopAfterPhase opts == Compile) $ do let abs = showIdent (srcAbsName gr cnc) pgfFile = outputPath opts (grammarName' opts abs<.>"pgf") @@ -50,9 +51,15 @@ compileSourceFiles opts fs = else return Nothing if t_pgf >= Just t_src then putIfVerb opts $ pgfFile ++ " is up-to-date." - else do pgf <- link opts cnc_gr + else do pgfs <- mapM (link opts) + [(cnc,t_src,gr)|(cnc,gr)<-cnc_grs] + let pgf = foldl1 unionPGF pgfs writePGF opts pgf writeOutputs opts pgf + where + batchCompile = maybe batchCompile' P.batchCompile (flag optJobs opts) + batchCompile' opts fs = do (cnc,t,gr) <- S.batchCompile opts fs + return (t,[(cnc,gr)]) compileCFFiles :: Options -> [FilePath] -> IOE () compileCFFiles opts fs = do -- cgit v1.2.3