langs option to Make.hs ; utf8 exception in linearization from the coding flag in grammar

2 files changed, 16 insertions, 3 deletions

diff --git a/src-3.0/GF/Command/Commands.hs b/src-3.0/GF/Command/Commands.hs
index 5514a0596..68e2c5526 100644
--- a/src-3.0/GF/Command/Commands.hs
+++ b/src-3.0/GF/Command/Commands.hs
@@ -497,7 +497,11 @@ allCommands pgf = Map.fromList [
      (abstractName pgf ++ ": " ++ showTree t) :
      [lang ++ ": " ++ linear opts lang t | lang <- optLangs opts]
 
-   unlex opts lang = stringOps opts
+   unlex opts lang = stringOps (exceptUTF8 opts) where
+     exceptUTF8 = if isUTF8 then filter ((/="to_UTF8") . prOpt) else id
+     isUTF8 = case lookFlag pgf lang "coding" of
+       Just "utf8" -> True
+       _ -> False
 
    optRestricted opts = restrictPGF (hasLin pgf (mkCId (optLang opts))) pgf
 
@@ -570,3 +574,5 @@ morphologyQuiz pgf ig cat = do
 infinity :: Int
 infinity = 256
 
+lookFlag :: PGF -> String -> String -> Maybe String
+lookFlag pgf lang flag = lookConcrFlag pgf (mkCId lang) (mkCId flag)
diff --git a/src-3.0/GF/Grammar/AppPredefined.hs b/src-3.0/GF/Grammar/AppPredefined.hs
index 452050ac8..cfb6baf1d 100644
--- a/src-3.0/GF/Grammar/AppPredefined.hs
+++ b/src-3.0/GF/Grammar/AppPredefined.hs
@@ -106,10 +106,17 @@ appPredefined t = case t of
   _ -> retb t
                   ---- should really check the absence of arg variables
  where
-   retb t = return (t,True)  -- no further computing needed
-   retf t = return (t,False) -- must be computed further
+   retb t = return (retc t,True)  -- no further computing needed
+   retf t = return (retc t,False) -- must be computed further
+   retc t = case t of
+     K [] -> t
+     K s  -> foldr1 C (map K (words s))
+     _    -> t
    norm t = case t of
      Empty -> K []
+     C u v -> case (norm u,norm v) of
+       (K x,K y) -> K (x +++ y)
+       _ -> t
      _ -> t
    fi = fromInteger