Merge with master and drop the Haskell runtime completely

src/compiler/GF/Compile/CheckGrammar.hs

@@ -146,11 +146,17 @@ checkCompleteGrammar opts cwd gr (am,abs) (cm,cnc) = checkInModule cwd cnc NoLoc
                                            return $ updateTree (c,CncFun (Just linty) d mn mf) js
                              _       -> do checkWarn ("function" <+> c <+> "is not in abstract")
                                            return js
-       CncCat _ _ _ _ _ -> case lookupOrigInfo gr (am,c) of
-                             Ok _ -> return $ updateTree i js
-                             _    -> do checkWarn ("category" <+> c <+> "is not in abstract")
-                                        return js
-       _                -> return $ updateTree i js
+       CncCat {} ->
+         case lookupOrigInfo gr (am,c) of
+           Ok (_,AbsCat _) -> return $ updateTree i js
+           {- -- This might be too pedantic:
+           Ok (_,AbsFun {}) ->
+                   checkError ("lincat:"<+>c<+>"is a fun, not a cat")
+           -}
+           _ -> do checkWarn ("category" <+> c <+> "is not in abstract")
+                   return js
+
+       _ -> return $ updateTree i js
 
 
 -- | General Principle: only Just-values are checked. 

src/compiler/GF/Compile/ConcreteToHaskell.hs

@@ -1,365 +1,351 @@
 -- | Translate concrete syntax to Haskell
 module GF.Compile.ConcreteToHaskell(concretes2haskell,concrete2haskell) where
-import Data.List(sort,sortBy)
-import Data.Function(on)
+import Data.List(isPrefixOf,sort,sortOn)
 import qualified Data.Map as M
 import qualified Data.Set as S
-import GF.Data.ErrM
-import GF.Data.Utilities(mapSnd)
 import GF.Text.Pretty
-import GF.Grammar.Grammar
-import GF.Grammar.Lookup(lookupFunType,lookupOrigInfo,allOrigInfos)--,allParamValues
-import GF.Grammar.Macros(typeForm,collectOp,collectPattOp,mkAbs,mkApp)
-import GF.Grammar.Lockfield(isLockLabel)
-import GF.Grammar.Predef(cPredef,cInts)
-import GF.Compile.Compute.Predef(predef)
-import GF.Compile.Compute.Value(Predefined(..))
-import GF.Infra.Ident(Ident,identS,prefixIdent) --,moduleNameS
+--import GF.Grammar.Predef(cPredef,cInts)
+--import GF.Compile.Compute.Predef(predef)
+--import GF.Compile.Compute.Value(Predefined(..))
+import GF.Infra.Ident(Ident,identS,identW,prefixIdent)
 import GF.Infra.Option
-import GF.Compile.Compute.ConcreteNew(normalForm,resourceValues)
-import GF.Haskell
-import Debug.Trace
+import GF.Haskell as H
+import GF.Grammar.Canonical as C
+import GF.Compile.GrammarToCanonical
+import Debug.Trace(trace)
 
 -- | Generate Haskell code for the all concrete syntaxes associated with
 -- the named abstract syntax in given the grammar.
 concretes2haskell opts absname gr =
-  [(cncname,concrete2haskell opts gr cenv absname cnc cncmod)
-     | let cenv = resourceValues opts gr,
-       cnc<-allConcretes gr absname,
-       let cncname = render cnc ++ ".hs" :: FilePath
-           Ok cncmod = lookupModule gr cnc
+  [(filename,render80 $ concrete2haskell opts abstr cncmod)
+     | let Grammar abstr cncs = grammar2canonical opts absname gr,
+       cncmod<-cncs,
+       let ModId name = concName cncmod
+           filename = name ++ ".hs" :: FilePath
   ]
 
 -- | Generate Haskell code for the given concrete module.
 -- The only options that make a difference are
 -- @-haskell=noprefix@ and @-haskell=variants@.
-concrete2haskell opts gr cenv absname cnc modinfo =
-    renderStyle style{lineLength=80,ribbonsPerLine=1} $
-      haskPreamble va absname cnc $$ vcat (
-      nl:Comment "--- Parameter types ---":
-      neededParamTypes S.empty (params defs) ++
-      nl:Comment "--- Type signatures for linearization functions ---":
-      map signature (S.toList allcats)++
-      nl:Comment "--- Linearization functions for empty categories ---":
-      emptydefs ++
-      nl:Comment "--- Linearization types and linearization functions ---":
-      map ppDef defs ++
-      nl:Comment "--- Type classes for projection functions ---":
-      map labelClass (S.toList labels) ++
-      nl:Comment "--- Record types ---":
-      concatMap recordType recs)
+concrete2haskell opts
+                 abstr@(Abstract _ _ cats funs)
+                 modinfo@(Concrete cnc absname _ ps lcs lns) =
+  haskPreamble absname cnc $$
+  vcat (
+    nl:Comment "--- Parameter types ---":
+    map paramDef ps ++
+    nl:Comment "--- Type signatures for linearization functions ---":
+    map signature cats ++
+    nl:Comment "--- Linearization functions for empty categories ---":
+    emptydefs ++
+    nl:Comment "--- Linearization types ---":
+    map lincatDef lcs ++
+    nl:Comment "--- Linearization functions ---":
+    lindefs ++
+    nl:Comment "--- Type classes for projection functions ---":
+    map labelClass (S.toList labels) ++
+    nl:Comment "--- Record types ---":
+    concatMap recordType recs)
   where
     nl = Comment ""
+    recs = S.toList (S.difference (records (lcs,lns)) common_records)
+
     labels = S.difference (S.unions (map S.fromList recs)) common_labels
-    recs = S.toList (S.difference (records rhss) common_records)
     common_records = S.fromList [[label_s]]
     common_labels = S.fromList [label_s]
-    label_s = ident2label (identS "s")
+    label_s = LabelId "s"
 
-    rhss = map (either snd (snd.snd)) defs
-    defs = sortBy (compare `on` either (const Nothing) (Just . fst)) .
-           concatMap (toHaskell gId gr absname cenv) . 
-           M.toList $
-           jments modinfo
-
---  signature c = "lin"<>c<+>"::"<+>"A."<>gId c<+>"->"<+>"Lin"<>c
---  signature c = "--lin"<>c<+>":: (Applicative f,Monad f) =>"<+>"A."<>gId c<+>"->"<+>"f Lin"<>c
-    signature c = TypeSig lf (Fun abs (pure lin))
+    signature (CatDef c _) = TypeSig lf (Fun abs (pure lin))
       where
         abs = tcon0 (prefixIdent "A." (gId c))
         lin = tcon0 lc
-        lf = prefixIdent "lin" c
-        lc = prefixIdent "Lin" c
+        lf = linfunName c
+        lc = lincatName c
 
     emptydefs = map emptydef (S.toList emptyCats)
-    emptydef c = Eqn (prefixIdent "lin" c,[WildP]) (Const "undefined")
+    emptydef c = Eqn (linfunName c,[WildP]) (Const "undefined")
 
-    emptyCats = allcats `S.difference` cats
-    cats = S.fromList [c|Right (c,_)<-defs]
-    allcats = S.fromList [c|((_,c),AbsCat (Just _))<-allOrigInfos gr absname]
+    emptyCats = allcats `S.difference` linfuncats
+      where
+     --funcats = S.fromList [c | FunDef f (C.Type _ (TypeApp c _))<-funs]
+       allcats = S.fromList [c | CatDef c _<-cats]
+      
+    gId :: ToIdent i => i -> Ident
+    gId = (if haskellOption opts HaskellNoPrefix then id else prefixIdent "G")
+          . toIdent
 
-    params = S.toList . S.unions . map params1
-    params1 (Left (_,rhs)) = paramTypes gr rhs
-    params1 (Right (_,(_,rhs))) = tableTypes gr [rhs]
-
-    ppDef (Left (lhs,rhs)) = lhs (convType va gId rhs)
-    ppDef (Right (_,(lhs,rhs))) = lhs (convert va gId gr rhs)
-
-    gId :: Ident -> Ident
-    gId = if haskellOption opts HaskellNoPrefix then id else prefixIdent "G"
     va = haskellOption opts HaskellVariants
     pure = if va then ListT else id
 
-    neededParamTypes have [] = []
-    neededParamTypes have (q:qs) =
-        if q `S.member` have
-        then neededParamTypes have qs
-        else let ((got,need),def) = paramType va gId gr q
-             in def++neededParamTypes (S.union got have) (S.toList need++qs)
-
-haskPreamble :: Bool -> ModuleName -> ModuleName -> Doc
-haskPreamble va absname cncname =
-  "{-# LANGUAGE MultiParamTypeClasses, FunctionalDependencies, FlexibleInstances, LambdaCase #-}" $$
-  "module" <+> cncname <+> "where" $$
-  "import Prelude hiding (Ordering(..))" $$
-  "import Control.Applicative((<$>),(<*>))" $$
-  "import PGF.Haskell" $$
-  "import qualified" <+> absname <+> "as A" $$
-  "" $$
-  "--- Standard definitions ---" $$
-  "linString (A.GString s) ="<+>pure "R_s [TK s]" $$
-  "linInt (A.GInt i) ="<+>pure "R_s [TK (show i)]" $$
-  "linFloat (A.GFloat x) ="<+>pure "R_s [TK (show x)]" $$
-  "" $$
-  "----------------------------------------------------" $$
-  "-- Automatic translation from GF to Haskell follows" $$
-  "----------------------------------------------------"
-  where
-    pure = if va then brackets else pp
-
-toHaskell gId gr absname cenv (name,jment) =
-  case jment of
-    CncCat (Just (L loc typ)) _ _ pprn _ ->
-        [Left (tsyn0 (prefixIdent "Lin" name),nf loc typ)]
-    CncFun (Just r@(cat,ctx,lincat)) (Just (L loc def)) pprn _ ->
---      trace (render (name<+>hcat[parens (x<>"::"<>t)|(_,x,t)<-ctx]<+>"::"<+>cat)) $
-        [Right (cat,(Eqn (prefixIdent "lin" cat,lhs),coerce [] lincat rhs))]
+    haskPreamble :: ModId -> ModId -> Doc
+    haskPreamble absname cncname =
+      "{-# LANGUAGE MultiParamTypeClasses, FunctionalDependencies, FlexibleInstances, LambdaCase #-}" $$
+      "module" <+> cncname <+> "where" $$
+      "import Prelude hiding (Ordering(..))" $$
+      "import Control.Applicative((<$>),(<*>))" $$
+      "import PGF.Haskell" $$
+      "import qualified" <+> absname <+> "as A" $$
+      "" $$
+      "--- Standard definitions ---" $$
+      "linString (A.GString s) ="<+>pure "R_s [TK s]" $$
+      "linInt (A.GInt i) ="<+>pure "R_s [TK (show i)]" $$
+      "linFloat (A.GFloat x) ="<+>pure "R_s [TK (show x)]" $$
+      "" $$
+      "----------------------------------------------------" $$
+      "-- Automatic translation from GF to Haskell follows" $$
+      "----------------------------------------------------"
       where
-        Ok abstype = lookupFunType gr absname name
-        (absctx,_abscat,_absargs) = typeForm abstype
+        pure = if va then brackets else pp
 
-        e' = unAbs (length params) $
-             nf loc (mkAbs params (mkApp def (map Vr args)))
-        params = [(b,prefixIdent "g" x)|(b,x,_)<-ctx]
-        args = map snd params
-        abs_args = map (prefixIdent "abs_") args
-        lhs = [ConP (aId name) (map VarP abs_args)]
-        rhs = foldr letlin e' (zip args absctx)
-        letlin (a,(_,_,at)) =
-           Let (a,(Just (con ("Lin"++render at)),(App (con ("lin"++render at)) (con ("abs_"++render a)))))
-    AnyInd _ m  -> case lookupOrigInfo gr (m,name) of
-                     Ok (m,jment) -> toHaskell gId gr absname cenv (name,jment)
-                     _ -> []
-    _ -> []
-  where
-    nf loc = normalForm cenv (L loc name)
-    aId n = prefixIdent "A." (gId n)
+    paramDef pd =
+      case pd of
+        ParamAliasDef p t -> H.Type (conap0 (gId p)) (convLinType t)
+        ParamDef p pvs -> Data (conap0 (gId p)) (map paramCon pvs) derive
+          where
+            paramCon (Param c cs) = ConAp (gId c) (map (tcon0.gId) cs)
+            derive = ["Eq","Ord","Show"]
 
-    unAbs 0 t = t
-    unAbs n (Abs _ _ t) = unAbs (n-1) t
-    unAbs _ t = t
+    convLinType = ppT
+      where
+        ppT t =
+          case t of
+            FloatType -> tcon0 (identS "Float")
+            IntType -> tcon0 (identS "Int")
+            ParamType (ParamTypeId p) -> tcon0 (gId p)
+            RecordType rs -> tcon (rcon' ls) (map ppT ts)
+              where (ls,ts) = unzip $ sortOn fst [(l,t)|RecordRow l t<-rs]
+            StrType -> tcon0 (identS "Str")
+            TableType pt lt -> Fun (ppT pt) (ppT lt)
+--          TupleType lts -> 
+
+    lincatDef (LincatDef c t) = tsyn0 (lincatName c) (convLinType t)
+
+    linfuncats = S.fromList linfuncatl
+    (linfuncatl,lindefs) = unzip (linDefs lns)
+
+    linDefs = map eqn . sortOn fst . map linDef
+      where eqn (cat,(f,(ps,rhs))) = (cat,Eqn (f,ps) rhs)
+
+    linDef (LinDef f xs rhs0) = 
+        (cat,(linfunName cat,(lhs,rhs)))
+      where
+        lhs = [ConP (aId f) (map VarP abs_args)]
+        aId f = prefixIdent "A." (gId f)
+
+        [lincat] = [lincat | LincatDef c lincat<-lcs,c==cat]
+        [C.Type absctx (TypeApp cat _)] = [t | FunDef f' t<-funs, f'==f]
+
+        abs_args = map abs_arg args
+        abs_arg = prefixIdent "abs_"
+        args = map (prefixIdent "g" . toIdent) xs
+
+        rhs = lets (zipWith letlin args absctx)
+                   (convert vs (coerce env lincat rhs0))
+          where
+            vs = [(VarValueId (Unqual x),a)|(VarId x,a)<-zip xs args]
+            env= [(VarValueId (Unqual x),lc)|(VarId x,lc)<-zip xs (map arglincat absctx)]
+            
+        letlin a (TypeBinding _ (C.Type _ (TypeApp acat _))) =
+          (a,Ap (Var (linfunName acat)) (Var (abs_arg a)))
+
+        arglincat (TypeBinding _ (C.Type _ (TypeApp acat _))) = lincat
+          where
+            [lincat] = [lincat | LincatDef c lincat<-lcs,c==acat]
+
+    convert = convert' va
+
+    convert' va vs = ppT
+      where
+        ppT0 = convert' False vs
+        ppTv vs' = convert' va vs'
+
+        pure = if va then single else id
+
+        ppT t =
+          case t of
+            TableValue ty cs -> pure (table cs)
+            Selection t p -> select (ppT t) (ppT p)
+            ConcatValue t1 t2 -> concat (ppT t1) (ppT t2)
+            RecordValue r -> aps (rcon ls) (map ppT ts)
+              where (ls,ts) = unzip $ sortOn fst [(l,t)|RecordRow l t<-r]
+            PredefValue p -> single (Var (toIdent p)) -- hmm
+            Projection t l -> ap (proj l) (ppT t)
+            VariantValue [] -> empty
+            VariantValue ts@(_:_) -> variants ts
+            VarValue x -> maybe (Var (gId x)) (pure . Var) $ lookup x vs
+            PreValue vs t' -> pure (alts t' vs)
+            ParamConstant (Param c vs) -> aps (Var (pId c)) (map ppT vs)
+            ErrorValue s -> ap (Const "error") (Const (show s)) -- !!
+            LiteralValue l -> ppL l
+            _ -> error ("convert "++show t)
+
+        ppL l =
+          case l of
+            FloatConstant x -> pure (lit x)
+            IntConstant n -> pure (lit n)
+            StrConstant s -> pure (token s)
+
+        pId p@(ParamId s) =
+          if "to_R_" `isPrefixOf` unqual s then toIdent p else gId p -- !! a hack
+          
+        table cs =
+            if all (null.patVars) ps
+            then lets ds (LambdaCase [(ppP p,t')|(p,t')<-zip ps ts'])
+            else LambdaCase (map ppCase cs)
+          where
+            (ds,ts') = dedup ts
+            (ps,ts) = unzip [(p,t)|TableRow p t<-cs]
+        ppCase (TableRow p t) = (ppP p,ppTv (patVars p++vs) t)
+{-
+        ppPredef n =
+          case predef n of
+            Ok BIND       -> single (c "BIND")
+            Ok SOFT_BIND  -> single (c "SOFT_BIND")
+            Ok SOFT_SPACE -> single (c "SOFT_SPACE")
+            Ok CAPIT      -> single (c "CAPIT")
+            Ok ALL_CAPIT  -> single (c "ALL_CAPIT")
+            _ -> Var n
+-}
+        ppP p =
+          case p of
+            ParamPattern (Param c ps) -> ConP (gId c) (map ppP ps)
+            RecordPattern r -> ConP (rcon' ls) (map ppP ps)
+              where (ls,ps) = unzip $ sortOn fst [(l,p)|RecordRow l p<-r]
+            WildPattern -> WildP
+
+        token s = single (c "TK" `Ap` lit s)
+
+        alts t' vs = single (c "TP" `Ap` List (map alt vs) `Ap` ppT0 t')
+          where
+            alt (s,t) = Pair (List (pre s)) (ppT0 t)
+            pre s = map lit s
+
+        c = Const
+        lit s = c (show s) -- hmm
+        concat = if va then concat' else plusplus
+          where
+            concat' (List [List ts1]) (List [List ts2]) = List [List (ts1++ts2)]
+            concat' t1 t2 = Op t1 "+++" t2
+
+        pure' = single -- forcing the list monad
+
+        select = if va then select' else Ap
+        select' (List [t]) (List [p]) = Op t "!" p
+        select' (List [t]) p = Op t "!$" p
+        select' t p = Op t "!*" p
+
+        ap = if va then ap' else Ap
+          where
+            ap' (List [f]) x = fmap f x
+            ap' f x = Op f "<*>" x
+            fmap f (List [x]) = pure' (Ap f x)
+            fmap f x = Op f "<$>" x
+
+    --  join = if va then join' else id
+        join' (List [x]) = x
+        join' x = c "concat" `Ap` x
+
+        empty = if va then List [] else c "error" `Ap` c (show "empty variant")
+        variants = if va then \ ts -> join' (List (map ppT ts))
+                         else \ (t:_) -> ppT t
+
+        aps f [] = f
+        aps f (a:as) = aps (ap f a) as
+
+        dedup ts =
+            if M.null dups
+            then ([],map ppT ts)
+            else ([(ev i,ppT t)|(i,t)<-defs],zipWith entry ts is)
+          where
+            entry t i = maybe (ppT t) (Var . ev) (M.lookup i dups)
+            ev i = identS ("e'"++show i)
+
+            defs = [(i1,t)|(t,i1:_:_)<-ms]
+            dups = M.fromList [(i2,i1)|(_,i1:is@(_:_))<-ms,i2<-i1:is]
+            ms = M.toList m
+            m = fmap sort (M.fromListWith (++) (zip ts [[i]|i<-is]))
+            is = [0..]::[Int]
 
 
-con = Cn . identS
+--con = Cn . identS
 
-tableTypes gr ts = S.unions (map tabtys ts)
-  where
-    tabtys t =
-      case t of
-        V t cc -> S.union (paramTypes gr t) (tableTypes gr cc)
-        T (TTyped t) cs -> S.union (paramTypes gr t) (tableTypes gr (map snd cs))
-        _ -> collectOp tabtys t
+class Records t where
+  records :: t -> S.Set [LabelId]
 
-paramTypes gr t =
-  case t of
-    RecType fs -> S.unions (map (paramTypes gr.snd) fs)
-    Table t1 t2 -> S.union (paramTypes gr t1) (paramTypes gr t2)
-    App tf ta -> S.union (paramTypes gr tf) (paramTypes gr ta)
-    Sort _ -> S.empty
-    EInt _ -> S.empty
-    Q q -> lookup q
-    QC q -> lookup q
-    FV ts -> S.unions (map (paramTypes gr) ts)
-    _ -> ignore
-  where
-    lookup q = case lookupOrigInfo gr q of
-                 Ok (_,ResOper  _ (Just (L _ t))) ->
-                                       S.insert q (paramTypes gr t)
-                 Ok (_,ResParam {}) -> S.singleton q
-                 _ -> ignore
+instance Records t => Records [t] where
+  records = S.unions . map records
 
-    ignore = trace ("Ignore: "++show t) S.empty
-
-
-
-records ts = S.unions (map recs ts)
-  where
-    recs t =
-      case t of
-        R r -> S.insert (labels r) (records (map (snd.snd) r))
-        RecType r -> S.insert (labels r) (records (map snd r))
-        _ -> collectOp recs t
-
-    labels = sort . filter (not . isLockLabel) . map fst
+instance (Records t1,Records t2) => Records (t1,t2) where
+  records (t1,t2) = S.union (records t1) (records t2)
+
+instance Records LincatDef where
+  records (LincatDef _ lt) = records lt
+
+instance Records LinDef where
+  records (LinDef _ _ lv) = records lv
+
+instance Records LinType where
+  records t =
+    case t of
+      RecordType r -> rowRecords r
+      TableType pt lt -> records (pt,lt)
+      TupleType ts -> records ts
+      _ -> S.empty
+
+rowRecords r = S.insert (sort ls) (records ts)
+  where (ls,ts) = unzip [(l,t)|RecordRow l t<-r]
+
+instance Records LinValue where
+  records v =
+    case v of
+      ConcatValue v1 v2 -> records (v1,v2)
+      ParamConstant (Param c vs) -> records vs
+      RecordValue r -> rowRecords r
+      TableValue t r -> records (t,r)
+      TupleValue vs -> records vs
+      VariantValue vs -> records vs
+      PreValue alts d -> records (map snd alts,d)
+      Projection v l -> records v
+      Selection v1 v2 -> records (v1,v2)
+      _ -> S.empty
+
+instance Records rhs => Records (TableRow rhs) where
+  records (TableRow _ v) = records v
 
 
+-- | Record subtyping is converted into explicit coercions in Haskell
 coerce env ty t =
   case (ty,t) of 
-    (_,Let d t) -> Let d (coerce (extend env d) ty t)
-    (_,FV ts) -> FV (map (coerce env ty) ts)
-    (Table ti tv,V _ ts) -> V ti (map (coerce env tv) ts)
-    (Table ti tv,T (TTyped _) cs) -> T (TTyped ti) (mapSnd (coerce env tv) cs)
-    (RecType rt,R r) ->
-      R [(l,(Just ft,coerce env ft f))|(l,(_,f))<-r,Just ft<-[lookup l rt]]
-    (RecType rt,Vr x)->
+    (_,VariantValue ts) -> VariantValue (map (coerce env ty) ts)
+    (TableType ti tv,TableValue _ cs) ->
+      TableValue ti [TableRow p (coerce env tv t)|TableRow p t<-cs]
+    (RecordType rt,RecordValue r) ->
+      RecordValue [RecordRow l (coerce env ft f) |
+                     RecordRow l f<-r,ft<-[ft|RecordRow l' ft<-rt,l'==l]]
+    (RecordType rt,VarValue x)->
       case lookup x env of
         Just ty' | ty'/=ty -> -- better to compare to normal form of ty'
-          --trace ("coerce "++render ty'++" to "++render ty) $
-          App (to_rcon (map fst rt)) t
-        _ -> trace ("no coerce to "++render ty) t
+                            --trace ("coerce "++render ty'++" to "++render ty) $
+                            app (to_rcon rt) [t]
+                 | otherwise -> t -- types match, no coercion needed
+        _ -> trace (render ("missing type to coerce"<+>x<+>"to"<+>render ty
+                            $$ "in" <+> map fst env))
+                   t
     _ -> t
   where
-    extend env (x,(Just ty,rhs)) = (x,ty):env
-    extend env _ = env
+    app f ts = ParamConstant (Param f ts) -- !! a hack
+    to_rcon = ParamId . Unqual . to_rcon' . labels
 
-convert va gId gr = convert' va gId [] gr
+patVars p = []
 
-convert' va gId vs gr = ppT
-  where
-    ppT0 = convert' False gId vs gr
-    ppTv vs' = convert' va gId vs' gr
+labels r = [l|RecordRow l _<-r]
 
-    ppT t =
-      case t of
-         -- Only for 'let' inserted on the top-level by this converter:
-        Let (x,(_,xt)) t -> let1 x (ppT0 xt) (ppT t)
---      Abs b x t -> ...
-        V ty ts -> pure (c "table" `Ap` dedup ts)
-        T (TTyped ty) cs -> pure (LambdaCase (map ppCase cs))
-        S t p -> select (ppT t) (ppT p)
-        C t1 t2 -> concat (ppT t1) (ppT t2)
-        App f a -> ap (ppT f) (ppT a)
-        R r -> aps (ppT (rcon (map fst r))) (fields r)
-        P t l -> ap (ppT (proj l)) (ppT t)
-        FV [] -> empty
-        Vr x -> if x `elem` vs then pure (Var x) else Var x
-        Cn x -> pure (Var x)
-        Con c -> pure (Var (gId c))
-        Sort k -> pure (Var k)
-        EInt n -> pure (lit n)
-        Q (m,n) -> if m==cPredef then pure (ppPredef n) else Var (qual m n)
-        QC (m,n) -> pure (Var (gId (qual m n)))
-        K s -> pure (token s)
-        Empty -> pure (List [])
-        FV ts@(_:_) -> variants ts
-        Alts t' vs -> pure (alts t' vs)
-
-    ppCase (p,t) = (ppP p,ppTv (patVars p++vs) t)
-
-    ppPredef n =
-      case predef n of
-        Ok BIND       -> single (c "BIND")
-        Ok SOFT_BIND  -> single (c "SOFT_BIND")
-        Ok SOFT_SPACE -> single (c "SOFT_SPACE")
-        Ok CAPIT      -> single (c "CAPIT")
-        Ok ALL_CAPIT  -> single (c "ALL_CAPIT")
-        _ -> Var n
-
-    ppP p =
-      case p of
-        PC c ps -> ConP (gId c) (map ppP ps)
-        PP (_,c) ps -> ConP (gId c) (map ppP ps)
-        PR r -> ConP (rcon' (map fst r)) (map (ppP.snd) (filter (not.isLockLabel.fst) r))
-        PW -> WildP
-        PV x -> VarP x
-        PString s -> Lit (show s) -- !!
-        PInt i -> Lit (show i)
-        PFloat x -> Lit (show x)
-        PT _ p -> ppP p
-        PAs x p -> AsP x (ppP p)
-        
-    token s = single (c "TK" `Ap` lit s)
-
-    alts t' vs = single (c "TP" `Ap` List (map alt vs) `Ap` ppT0 t')
-      where
-        alt (t,p) = Pair (List (pre p)) (ppT0 t)
-
-        pre (K s) = [lit s]
-        pre (Strs ts) = concatMap pre ts
-        pre (EPatt p) = pat p
-        pre t = error $ "pre "++show t
-
-        pat (PString s) = [lit s]
-        pat (PAlt p1 p2) = pat p1++pat p2
-        pat p = error $ "pat "++show p
-
-    fields = map (ppT.snd.snd) . sort . filter (not.isLockLabel.fst)
-
-    c = Const
-    lit s = c (show s) -- hmm
-    concat = if va then concat' else plusplus
-      where
-        concat' (List [List ts1]) (List [List ts2]) = List [List (ts1++ts2)]
-        concat' t1 t2 = Op t1 "+++" t2
-    pure = if va then single else id
-    pure' = single -- forcing the list monad
-
-    select = if va then select' else Ap
-    select' (List [t]) (List [p]) = Op t "!" p
-    select' (List [t]) p = Op t "!$" p
-    select' t p = Op t "!*" p
-
-    ap = if va then ap' else Ap
-      where
-        ap' (List [f]) x = fmap f x
-        ap' f x = Op f "<*>" x
-        fmap f (List [x]) = pure' (Ap f x)
-        fmap f x = Op f "<$>" x
-
---  join = if va then join' else id
-    join' (List [x]) = x
-    join' x = c "concat" `Ap` x
-
-    empty = if va then List [] else c "error" `Ap` c (show "empty variant")
-    variants = if va then \ ts -> join' (List (map ppT ts))
-                     else \ (t:_) -> ppT t
-
-    aps f [] = f
-    aps f (a:as) = aps (ap f a) as
-
-    dedup ts =
-        if M.null dups
-        then List (map ppT ts)
-        else Lets [(ev i,ppT t)|(i,t)<-defs] (List (zipWith entry ts is))
-      where
-        entry t i = maybe (ppT t) (Var . ev) (M.lookup i dups)
-        ev i = identS ("e'"++show i)
-
-        defs = [(i1,t)|(t,i1:_:_)<-ms]
-        dups = M.fromList [(i2,i1)|(_,i1:is@(_:_))<-ms,i2<-i1:is]
-        ms = M.toList m
-        m = fmap sort (M.fromListWith (++) (zip ts [[i]|i<-is]))
-        is = [0..]::[Int]
-
-patVars p =
-  case p of
-    PV x -> [x]
-    PAs x p -> x:patVars p
-    _ -> collectPattOp patVars p
-
-convType va gId = ppT
-  where
-    ppT t =
-      case t of
-        Table ti tv -> Fun (ppT ti) (if va then ListT (ppT tv) else ppT tv)
-        RecType rt -> tcon (rcon' (map fst rt)) (fields rt)
-        App tf ta -> TAp (ppT tf) (ppT ta)
-        FV [] -> tcon0 (identS "({-empty variant-})")
-        Sort k -> tcon0 k
-        EInt n -> tcon0 (identS ("({-"++show n++"-})")) -- type level numeric literal
-        FV (t:ts) -> ppT t -- !!
-        QC (m,n) -> tcon0 (gId (qual m n))
-        Q (m,n) -> tcon0 (gId (qual m n))
-        _ -> error $ "Missing case in convType for: "++show t
-
-    fields = map (ppT.snd) . sort . filter (not.isLockLabel.fst)
-
-proj = con . proj'
-proj' l = "proj_"++render l
-rcon = con . rcon_name
+proj = Var . identS . proj'
+proj' (LabelId l) = "proj_"++l
+rcon = Var . rcon'
 rcon' = identS . rcon_name
-rcon_name ls = "R"++concat (sort ['_':render l|l<-ls,not (isLockLabel l)])
-to_rcon = con . to_rcon'
+rcon_name ls = "R"++concat (sort ['_':l|LabelId l<-ls])
+
 to_rcon' = ("to_"++) . rcon_name
 
 recordType ls =
@@ -400,31 +386,6 @@ labelClass l =
     r = identS "r"
     a = identS "a"
 
-paramType va gId gr q@(_,n) =
-    case lookupOrigInfo gr q of
-      Ok (m,ResParam (Just (L _ ps)) _)
-       {- - | m/=cPredef && m/=moduleNameS "Prelude"-} ->
-         ((S.singleton (m,n),argTypes ps),
-          [Data (conap0 name) (map (param m) ps)["Eq","Ord","Show"],
-           Instance [] (TId (identS "EnumAll") `TAp` TId name)
-              [(lhs0 "enumAll",foldr1 plusplus (map (enumParam m) ps))]]
-         )
-       where name = gId (qual m n)
-      Ok (m,ResOper  _ (Just (L _ t)))
-        | m==cPredef && n==cInts ->
-           ((S.singleton (m,n),S.empty),
-            [Type (ConAp (gId (qual m n)) [identS "n"]) (TId (identS "Int"))])
-        | otherwise ->
-           ((S.singleton (m,n),paramTypes gr t),
-            [Type (conap0 (gId (qual m n))) (convType va gId t)])
-      _ -> ((S.empty,S.empty),[])
-  where
-    param m (n,ctx) = ConAp (gId (qual m n)) [convType va gId t|(_,_,t)<-ctx]
-    argTypes = S.unions . map argTypes1
-    argTypes1 (n,ctx) = S.unions [paramTypes gr t|(_,_,t)<-ctx]
-
-    enumParam m (n,ctx) = enumCon (gId (qual m n)) (length ctx)
-
 enumCon name arity =
     if arity==0
     then single (Var name)
@@ -433,5 +394,23 @@ enumCon name arity =
     ap (List [f]) a = Op f "<$>" a
     ap f a = Op f "<*>" a
 
-qual :: ModuleName -> Ident -> Ident
-qual m = prefixIdent (render m++"_")
+lincatName,linfunName :: CatId -> Ident
+lincatName c = prefixIdent "Lin" (toIdent c)
+linfunName c = prefixIdent "lin" (toIdent c)
+
+class ToIdent i where toIdent :: i -> Ident
+
+instance ToIdent ParamId where toIdent (ParamId q) = qIdentS q
+instance ToIdent PredefId where toIdent (PredefId s) = identS s
+instance ToIdent CatId   where toIdent (CatId s) = identS s
+instance ToIdent C.FunId where toIdent (FunId s) = identS s
+instance ToIdent VarValueId where toIdent (VarValueId q) = qIdentS q
+
+qIdentS = identS . unqual
+
+unqual (Qual (ModId m) n) = m++"_"++n
+unqual (Unqual n) = n
+
+instance ToIdent VarId where
+  toIdent Anonymous = identW
+  toIdent (VarId s) = identS s

src/compiler/GF/Compile/Export.hs

@@ -1,11 +1,10 @@
 module GF.Compile.Export where
 
-import PGF
+import PGF2
 import GF.Compile.PGFtoHaskell
+--import GF.Compile.PGFtoAbstract
 import GF.Compile.PGFtoJava
-import GF.Compile.PGFtoProlog
-import GF.Compile.PGFtoJS
-import GF.Compile.PGFtoPython
+import GF.Compile.PGFtoJSON
 import GF.Infra.Option
 --import GF.Speech.CFG
 import GF.Speech.PGFToCFG
@@ -19,6 +18,7 @@ import GF.Speech.SLF
 import GF.Speech.PrRegExp
 
 import Data.Maybe
+import qualified Data.Map as Map
 import System.FilePath
 import GF.Text.Pretty
 
@@ -33,11 +33,11 @@ exportPGF :: Options
 exportPGF opts fmt pgf = 
     case fmt of
       FmtPGFPretty    -> multi "txt" (showPGF)
-      FmtJavaScript   -> multi "js"  pgf2js
-      FmtPython       -> multi "py"  pgf2python
+      FmtCanonicalGF  -> [] -- canon "gf" (render80 . abstract2canonical)
+      FmtCanonicalJson-> []
+      FmtJSON         -> multi "json"  pgf2json
       FmtHaskell      -> multi "hs"  (grammar2haskell opts name)
       FmtJava         -> multi "java" (grammar2java opts name)
-      FmtProlog       -> multi "pl"  grammar2prolog
       FmtBNF          -> single "bnf"   bnfPrinter
       FmtEBNF         -> single "ebnf"  (ebnfPrinter opts)
       FmtSRGS_XML     -> single "grxml" (srgsXmlPrinter opts)
@@ -51,17 +51,13 @@ exportPGF opts fmt pgf =
       FmtRegExp       -> single "rexp" regexpPrinter
       FmtFA           -> single "dot"  slfGraphvizPrinter
  where
-   name = fromMaybe (showCId (abstractName pgf)) (flag optName opts)
+   name = fromMaybe (abstractName pgf) (flag optName opts)
 
    multi :: String -> (PGF -> String) -> [(FilePath,String)]
    multi ext pr = [(name <.> ext, pr pgf)]
 
-   single :: String -> (PGF -> CId -> String) -> [(FilePath,String)]
-   single ext pr = [(showCId cnc <.> ext, pr pgf cnc) | cnc <- languages pgf]
+-- canon ext pr = [("canonical"</>name<.>ext,pr pgf)]
+
+   single :: String -> (PGF -> Concr -> String) -> [(FilePath,String)]
+   single ext pr = [(concreteName cnc <.> ext, pr pgf cnc) | cnc <- Map.elems (languages pgf)]
 
--- | Get the name of the concrete syntax to generate output from.
--- FIXME: there should be an option to change this.
-outputConcr :: PGF -> CId
-outputConcr pgf = case languages pgf of
-                    []     -> error "No concrete syntax."
-                    cnc:_  -> cnc

src/compiler/GF/Compile/GrammarToCanonical.hs

@@ -0,0 +1,389 @@
+-- | Translate grammars to Canonical form
+-- (a common intermediate representation to simplify export to other formats)
+module GF.Compile.GrammarToCanonical(
+       grammar2canonical,abstract2canonical,concretes2canonical,
+       projection,selection
+       ) where
+import Data.List(nub,partition)
+import qualified Data.Map as M
+import qualified Data.Set as S
+import GF.Data.ErrM
+import GF.Text.Pretty
+import GF.Grammar.Grammar
+import GF.Grammar.Lookup(lookupOrigInfo,allOrigInfos,allParamValues)
+import GF.Grammar.Macros(typeForm,collectOp,collectPattOp,mkAbs,mkApp,term2patt)
+import GF.Grammar.Lockfield(isLockLabel)
+import GF.Grammar.Predef(cPredef,cInts)
+import GF.Compile.Compute.Predef(predef)
+import GF.Compile.Compute.Value(Predefined(..))
+import GF.Infra.Ident(ModuleName(..),Ident,prefixIdent,showIdent,isWildIdent)
+import GF.Infra.Option(optionsPGF)
+import PGF.Internal(Literal(..))
+import GF.Compile.Compute.ConcreteNew(normalForm,resourceValues)
+import GF.Grammar.Canonical as C
+import Debug.Trace
+
+-- | Generate Canonical code for the named abstract syntax and all associated
+-- concrete syntaxes
+grammar2canonical opts absname gr =
+   Grammar (abstract2canonical absname gr)
+           (map snd (concretes2canonical opts absname gr))
+
+-- | Generate Canonical code for the named abstract syntax
+abstract2canonical absname gr =
+    Abstract (modId absname) (convFlags gr absname) cats funs
+  where
+    cats = [CatDef (gId c) (convCtx ctx) | ((_,c),AbsCat ctx) <- adefs]
+
+    funs = [FunDef (gId f) (convType ty) |
+            ((_,f),AbsFun (Just (L _ ty)) ma mdef _) <- adefs]
+
+    adefs = allOrigInfos gr absname
+
+    convCtx = maybe [] (map convHypo . unLoc)
+    convHypo (bt,name,t) =
+      case typeForm t of
+        ([],(_,cat),[]) -> gId cat -- !!
+
+    convType t =
+      case typeForm t of
+        (hyps,(_,cat),args) -> Type bs (TypeApp (gId cat) as)
+          where
+            bs = map convHypo' hyps
+            as = map convType args
+
+    convHypo' (bt,name,t) = TypeBinding (gId name) (convType t)
+
+
+-- | Generate Canonical code for the all concrete syntaxes associated with
+-- the named abstract syntax in given the grammar.
+concretes2canonical opts absname gr =
+  [(cncname,concrete2canonical gr cenv absname cnc cncmod)
+     | let cenv = resourceValues opts gr,
+       cnc<-allConcretes gr absname,
+       let cncname = "canonical/"++render cnc ++ ".gf" :: FilePath
+           Ok cncmod = lookupModule gr cnc
+  ]
+
+-- | Generate Canonical GF for the given concrete module.
+concrete2canonical gr cenv absname cnc modinfo =
+  Concrete (modId cnc) (modId absname) (convFlags gr cnc)
+      (neededParamTypes S.empty (params defs))
+      [lincat|(_,Left lincat)<-defs]
+      [lin|(_,Right lin)<-defs]
+  where
+    defs = concatMap (toCanonical gr absname cenv) . 
+           M.toList $
+           jments modinfo
+
+    params = S.toList . S.unions . map fst
+
+    neededParamTypes have [] = []
+    neededParamTypes have (q:qs) =
+        if q `S.member` have
+        then neededParamTypes have qs
+        else let ((got,need),def) = paramType gr q
+             in def++neededParamTypes (S.union got have) (S.toList need++qs)
+
+toCanonical gr absname cenv (name,jment) =
+  case jment of
+    CncCat (Just (L loc typ)) _ _ pprn _ ->
+        [(pts,Left (LincatDef (gId name) (convType ntyp)))]
+      where
+        pts = paramTypes gr ntyp
+        ntyp = nf loc typ
+    CncFun (Just r@(cat,ctx,lincat)) (Just (L loc def)) pprn _ ->
+        [(tts,Right (LinDef (gId name) (map gId args) (convert gr e')))]
+      where
+        tts = tableTypes gr [e']
+
+        e' = unAbs (length params) $
+             nf loc (mkAbs params (mkApp def (map Vr args)))
+        params = [(b,x)|(b,x,_)<-ctx]
+        args = map snd params
+
+    AnyInd _ m  -> case lookupOrigInfo gr (m,name) of
+                     Ok (m,jment) -> toCanonical gr absname cenv (name,jment)
+                     _ -> []
+    _ -> []
+  where
+    nf loc = normalForm cenv (L loc name)
+--  aId n = prefixIdent "A." (gId n)
+
+    unAbs 0 t = t
+    unAbs n (Abs _ _ t) = unAbs (n-1) t
+    unAbs _ t = t
+
+tableTypes gr ts = S.unions (map tabtys ts)
+  where
+    tabtys t =
+      case t of
+        V t cc -> S.union (paramTypes gr t) (tableTypes gr cc)
+        T (TTyped t) cs -> S.union (paramTypes gr t) (tableTypes gr (map snd cs))
+        _ -> collectOp tabtys t
+
+paramTypes gr t =
+  case t of
+    RecType fs -> S.unions (map (paramTypes gr.snd) fs)
+    Table t1 t2 -> S.union (paramTypes gr t1) (paramTypes gr t2)
+    App tf ta -> S.union (paramTypes gr tf) (paramTypes gr ta)
+    Sort _ -> S.empty
+    EInt _ -> S.empty
+    Q q -> lookup q
+    QC q -> lookup q
+    FV ts -> S.unions (map (paramTypes gr) ts)
+    _ -> ignore
+  where
+    lookup q = case lookupOrigInfo gr q of
+                 Ok (_,ResOper  _ (Just (L _ t))) ->
+                                       S.insert q (paramTypes gr t)
+                 Ok (_,ResParam {}) -> S.singleton q
+                 _ -> ignore
+
+    ignore = trace ("Ignore: "++show t) S.empty
+
+
+convert gr = convert' gr []
+
+convert' gr vs = ppT
+  where
+    ppT0 = convert' gr vs
+    ppTv vs' = convert' gr vs'
+
+    ppT t =
+      case t of
+--      Abs b x t -> ...
+--      V ty ts -> VTableValue (convType ty) (map ppT ts)
+        V ty ts -> TableValue (convType ty) [TableRow (ppP p) (ppT t)|(p,t)<-zip ps ts]
+          where
+            Ok pts = allParamValues gr ty
+            Ok ps = mapM term2patt pts
+        T (TTyped ty) cs -> TableValue (convType ty) (map ppCase cs)
+        S t p -> selection (ppT t) (ppT p)
+        C t1 t2 -> concatValue (ppT t1) (ppT t2)
+        App f a -> ap (ppT f) (ppT a)
+        R r -> RecordValue (fields r)
+        P t l -> projection (ppT t) (lblId l)
+        Vr x -> VarValue (gId x)
+        Cn x -> VarValue (gId x) -- hmm
+        Con c -> ParamConstant (Param (gId c) [])
+        Sort k -> VarValue (gId k)
+        EInt n -> LiteralValue (IntConstant n)
+        Q (m,n) -> if m==cPredef then ppPredef n else VarValue ((gQId m n))
+        QC (m,n) -> ParamConstant (Param ((gQId m n)) [])
+        K s -> LiteralValue (StrConstant s)
+        Empty -> LiteralValue (StrConstant "")
+        FV ts -> VariantValue (map ppT ts)
+        Alts t' vs -> alts vs (ppT t')
+        _ -> error $ "convert' "++show t
+
+    ppCase (p,t) = TableRow (ppP p) (ppTv (patVars p++vs) t)
+
+    ppPredef n =
+      case predef n of
+        Ok BIND       -> p "BIND"
+        Ok SOFT_BIND  -> p "SOFT_BIND"
+        Ok SOFT_SPACE -> p "SOFT_SPACE"
+        Ok CAPIT      -> p "CAPIT"
+        Ok ALL_CAPIT  -> p "ALL_CAPIT"
+        _ -> VarValue (gQId cPredef n) -- hmm
+      where
+       p = PredefValue . PredefId
+      
+    ppP p =
+      case p of
+        PC c ps -> ParamPattern (Param (gId c) (map ppP ps))
+        PP (m,c) ps -> ParamPattern (Param ((gQId m c)) (map ppP ps))
+        PR r -> RecordPattern (fields r) {-
+        PW -> WildPattern
+        PV x -> VarP x
+        PString s -> Lit (show s) -- !!
+        PInt i -> Lit (show i)
+        PFloat x -> Lit (show x)
+        PT _ p -> ppP p
+        PAs x p -> AsP x (ppP p) -}
+      where
+        fields = map field . filter (not.isLockLabel.fst)
+        field (l,p) = RecordRow (lblId l) (ppP p)
+
+--  patToParam p = case ppP p of ParamPattern pv -> pv
+
+--  token s = single (c "TK" `Ap` lit s)
+
+    alts vs = PreValue (map alt vs)
+      where
+        alt (t,p) = (pre p,ppT0 t)
+
+        pre (K s) = [s]
+        pre Empty = [""]  -- Empty == K ""
+        pre (Strs ts) = concatMap pre ts
+        pre (EPatt p) = pat p
+        pre t = error $ "pre "++show t
+
+        pat (PString s) = [s]
+        pat (PAlt p1 p2) = pat p1++pat p2
+        pat (PSeq p1 p2) = [s1++s2 | s1<-pat p1, s2<-pat p2]
+        pat p = error $ "pat "++show p
+
+    fields = map field . filter (not.isLockLabel.fst)
+    field (l,(_,t)) = RecordRow (lblId l) (ppT t)
+  --c = Const
+  --c = VarValue . VarValueId
+  --lit s = c (show s) -- hmm
+
+    ap f a = case f of
+               ParamConstant (Param p ps) ->
+                 ParamConstant (Param p (ps++[a]))
+               _ -> error $ "convert' ap: "++render (ppA f <+> ppA a)
+
+concatValue v1 v2 =
+  case (v1,v2) of
+    (LiteralValue (StrConstant ""),_) -> v2
+    (_,LiteralValue (StrConstant "")) -> v1
+    _ -> ConcatValue v1 v2
+
+-- | Smart constructor for projections
+projection r l = maybe (Projection r l) id (proj r l)
+
+proj r l =
+  case r of
+    RecordValue r -> case [v|RecordRow l' v<-r,l'==l] of
+                          [v] -> Just v
+                          _ -> Nothing
+    _ -> Nothing
+
+-- | Smart constructor for selections
+selection t v =
+  -- Note: impossible cases can become possible after grammar transformation
+  case t of
+    TableValue tt r ->
+        case nub [rv|TableRow _ rv<-keep] of
+          [rv] -> rv
+          _ -> Selection (TableValue tt r') v
+      where
+        -- Don't introduce wildcard patterns, true to the canonical format,
+        -- annotate (or eliminate) rhs in impossible rows
+        r' = map trunc r
+        trunc r@(TableRow p e) = if mightMatchRow v r
+                                 then r
+                                 else TableRow p (impossible e)
+        {-
+        -- Creates smaller tables, but introduces wildcard patterns
+        r' = if null discard
+             then r
+             else keep++[TableRow WildPattern impossible]
+        -}
+        (keep,discard) = partition (mightMatchRow v) r
+    _ -> Selection t v
+
+impossible = CommentedValue "impossible"
+
+mightMatchRow v (TableRow p _) =
+  case p of
+    WildPattern -> True
+    _ -> mightMatch v p
+
+mightMatch v p =
+  case v of
+    ConcatValue _ _ -> False
+    ParamConstant (Param c1 pvs) ->
+      case p of
+        ParamPattern (Param c2 pps) -> c1==c2 && length pvs==length pps &&
+                                       and [mightMatch v p|(v,p)<-zip pvs pps]
+        _ -> False
+    RecordValue rv ->
+      case p of
+        RecordPattern rp ->
+          and [maybe False (flip mightMatch p) (proj v l) | RecordRow l p<-rp]
+        _ -> False
+    _ -> True
+
+patVars p =
+  case p of
+    PV x -> [x]
+    PAs x p -> x:patVars p
+    _ -> collectPattOp patVars p
+
+convType = ppT
+  where
+    ppT t =
+      case t of
+        Table ti tv -> TableType (ppT ti) (ppT tv)
+        RecType rt -> RecordType (convFields rt)
+--      App tf ta -> TAp (ppT tf) (ppT ta)
+--      FV [] -> tcon0 (identS "({-empty variant-})")
+        Sort k -> convSort k
+--      EInt n -> tcon0 (identS ("({-"++show n++"-})")) -- type level numeric literal
+        FV (t:ts) -> ppT t -- !!
+        QC (m,n) -> ParamType (ParamTypeId ((gQId m n)))
+        Q (m,n) -> ParamType (ParamTypeId ((gQId m n)))
+        _ -> error $ "Missing case in convType for: "++show t
+
+    convFields = map convField . filter (not.isLockLabel.fst)
+    convField (l,r) = RecordRow (lblId l) (ppT r)
+
+    convSort k = case showIdent k of
+                   "Float" -> FloatType
+                   "Int" -> IntType
+                   "Str" -> StrType
+                   _ -> error ("convSort "++show k)
+
+toParamType t = case convType t of
+                  ParamType pt -> pt
+                  _ -> error ("toParamType "++show t)
+
+toParamId t = case toParamType t of
+                   ParamTypeId p -> p
+
+paramType gr q@(_,n) =
+    case lookupOrigInfo gr q of
+      Ok (m,ResParam (Just (L _ ps)) _)
+       {- - | m/=cPredef && m/=moduleNameS "Prelude"-} ->
+         ((S.singleton (m,n),argTypes ps),
+          [ParamDef name (map (param m) ps)]
+         )
+       where name = (gQId m n)
+      Ok (m,ResOper  _ (Just (L _ t)))
+        | m==cPredef && n==cInts ->
+           ((S.empty,S.empty),[]) {-
+           ((S.singleton (m,n),S.empty),
+            [Type (ConAp ((gQId m n)) [identS "n"]) (TId (identS "Int"))])-}
+        | otherwise ->
+           ((S.singleton (m,n),paramTypes gr t),
+            [ParamAliasDef ((gQId m n)) (convType t)])
+      _ -> ((S.empty,S.empty),[])
+  where
+    param m (n,ctx) = Param ((gQId m n)) [toParamId t|(_,_,t)<-ctx]
+    argTypes = S.unions . map argTypes1
+    argTypes1 (n,ctx) = S.unions [paramTypes gr t|(_,_,t)<-ctx]
+
+lblId = LabelId . render -- hmm
+modId (MN m) = ModId (showIdent m)
+
+class FromIdent i where gId :: Ident -> i
+
+instance FromIdent VarId where
+  gId i = if isWildIdent i then Anonymous else VarId (showIdent i)
+
+instance FromIdent C.FunId where gId = C.FunId . showIdent
+instance FromIdent CatId where gId = CatId . showIdent
+instance FromIdent ParamId where gId = ParamId . unqual
+instance FromIdent VarValueId where gId = VarValueId . unqual
+
+class FromIdent i => QualIdent i where gQId :: ModuleName -> Ident -> i
+
+instance QualIdent ParamId    where gQId m n = ParamId (qual m n)
+instance QualIdent VarValueId where gQId m n = VarValueId (qual m n)
+
+qual m n = Qual (modId m) (showIdent n)
+unqual n = Unqual (showIdent n)
+
+convFlags gr mn =
+  Flags [(n,convLit v) |
+         (n,v)<-err (const []) (optionsPGF.mflags) (lookupModule gr mn)]
+  where
+    convLit l =
+      case l of
+        LStr s -> Str s
+        LInt i -> C.Int i
+        LFlt d -> Flt d

src/compiler/GF/Compile/GrammarToPGF.hs

@@ -1,12 +1,12 @@
-{-# LANGUAGE ImplicitParams, BangPatterns, FlexibleContexts #-}
+{-# LANGUAGE ImplicitParams, BangPatterns, FlexibleContexts, MagicHash #-}
 module GF.Compile.GrammarToPGF (grammar2PGF) where
 
 import GF.Compile.GeneratePMCFG
 import GF.Compile.GenerateBC
 import GF.Compile.OptimizePGF
 
-import PGF(CId,mkCId,Type,Hypo,Expr)
-import PGF.Internal
+import PGF2 hiding (mkType)
+import PGF2.Internal
 import GF.Grammar.Predef
 import GF.Grammar.Grammar hiding (Production)
 import qualified GF.Grammar.Lookup as Look
@@ -19,18 +19,22 @@ import GF.Infra.UseIO (IOE)
 import GF.Data.Operations
 
 import Data.List
+import Data.Char
 import qualified Data.Set as Set
 import qualified Data.Map as Map
 import qualified Data.IntMap as IntMap
 import Data.Array.IArray
 import Data.Maybe(fromMaybe)
 
-grammar2PGF :: Options -> SourceGrammar -> ModuleName -> Map.Map CId Double -> IO PGF
+import GHC.Prim
+import GHC.Base(getTag)
+
+grammar2PGF :: Options -> SourceGrammar -> ModuleName -> Map.Map PGF2.Fun Double -> IO PGF
 grammar2PGF opts gr am probs = do
   cnc_infos <- getConcreteInfos gr am
   return $
     build (let gflags   = if flag optSplitPGF opts 
-                            then [(mkCId "split", LStr "true")]
+                            then [("split", LStr "true")]
                             else []
                (an,abs) = mkAbstr am probs
                cncs     = map (mkConcr opts abs) cnc_infos
@@ -39,21 +43,21 @@ grammar2PGF opts gr am probs = do
     cenv = resourceValues opts gr
     aflags = err (const noOptions) mflags (lookupModule gr am)
 
-    mkAbstr :: (?builder :: Builder s) => ModuleName -> Map.Map CId Double -> (CId, B s AbstrInfo)
+    mkAbstr :: (?builder :: Builder s) => ModuleName -> Map.Map PGF2.Fun Double -> (AbsName, B s AbstrInfo)
     mkAbstr am probs = (mi2i am, newAbstr flags cats funs)
       where
         adefs =
             [((cPredefAbs,c), AbsCat (Just (L NoLoc []))) | c <- [cFloat,cInt,cString]] ++ 
             Look.allOrigInfos gr am
 
-        flags = [(mkCId f,x) | (f,x) <- optionsPGF aflags]
+        flags = optionsPGF aflags
 
         toLogProb = realToFrac . negate . log
 
         cats = [(c', snd (mkContext [] cont), toLogProb (fromMaybe 0 (Map.lookup c' probs))) |
                                    ((m,c),AbsCat (Just (L _ cont))) <- adefs, let c' = i2i c]
 
-        funs = [(f', mkType [] ty, arity, {-mkDef gr arity mdef,-} toLogProb (fromMaybe 0 (Map.lookup f' funs_probs))) |
+        funs = [(f', mkType [] ty, arity, toLogProb (fromMaybe 0 (Map.lookup f' funs_probs))) |
                                    ((m,f),AbsFun (Just (L _ ty)) ma mdef _) <- adefs,
                                    let arity = mkArity ma mdef ty,
                                    let f' = i2i f]
@@ -72,7 +76,10 @@ grammar2PGF opts gr am probs = do
 
     mkConcr opts abs (cm,ex_seqs,cdefs) =
       let cflags = err (const noOptions) mflags (lookupModule gr cm)
-          flags  = [(mkCId f,x) | (f,x) <- optionsPGF cflags]
+          ciCmp | flag optCaseSensitive cflags = compare
+                | otherwise                    = compareCaseInsensitive
+
+          flags = optionsPGF aflags
 
           seqs = (mkSetArray . Set.fromList . concat) $
                        (elems (ex_seqs :: Array SeqId [Symbol]) : [maybe [] elems (mseqs mi) | (m,mi) <- allExtends gr cm])
@@ -80,11 +87,11 @@ grammar2PGF opts gr am probs = do
           !(!fid_cnt1,!cnccats) = genCncCats gr am cm cdefs
           cnccat_ranges = Map.fromList (map (\(cid,s,e,_) -> (cid,(s,e))) cnccats)
           !(!fid_cnt2,!productions,!lindefs,!linrefs,!cncfuns)
-                                = genCncFuns gr am cm ex_seqs seqs cdefs fid_cnt1 cnccat_ranges
+                                = genCncFuns gr am cm ex_seqs ciCmp seqs cdefs fid_cnt1 cnccat_ranges
 
           printnames = genPrintNames cdefs
 
-          startCat = mkCId (fromMaybe "S" (flag optStartCat aflags))
+          startCat = (fromMaybe "S" (flag optStartCat aflags))
 
           (lindefs',linrefs',productions',cncfuns',sequences',cnccats') =
                (if flag optOptimizePGF opts then optimizePGF startCat else id)
@@ -118,16 +125,13 @@ grammar2PGF opts gr am probs = do
           (seqs,infos) <- addMissingPMCFGs cm seqs is
           return (seqs, ((m,id), info) : infos)
 
-mkSetArray set = listArray (0,Set.size set-1) (Set.toList set)
-mkMapArray map = array (0,Map.size map-1) [(k,v) | (v,k) <- Map.toList map]
+i2i :: Ident -> String
+i2i = showIdent
 
-i2i :: Ident -> CId
-i2i = mkCId . showIdent
-
-mi2i :: ModuleName -> CId
+mi2i :: ModuleName -> String
 mi2i (MN i) = i2i i
 
-mkType :: (?builder :: Builder s) => [Ident] -> A.Type -> B s PGF.Type
+mkType :: (?builder :: Builder s) => [Ident] -> A.Type -> B s PGF2.Type
 mkType scope t =
   case GM.typeForm t of
     (hyps,(_,cat),args) -> let (scope',hyps') = mkContext scope hyps
@@ -164,7 +168,7 @@ mkPatt scope p =
                    in (scope',C.PImplArg p')
     A.PTilde t  -> (  scope,C.PTilde (mkExp scope t))
 -}
-mkContext :: (?builder :: Builder s) => [Ident] -> A.Context -> ([Ident],[B s PGF.Hypo])
+mkContext :: (?builder :: Builder s) => [Ident] -> A.Context -> ([Ident],[B s PGF2.Hypo])
 mkContext scope hyps = mapAccumL (\scope (bt,x,ty) -> let ty' = mkType scope ty
                                                       in if x == identW
                                                            then (  scope,hypo bt (i2i x) ty')
@@ -206,16 +210,17 @@ genCncFuns :: Grammar
            -> ModuleName
            -> ModuleName
            -> Array SeqId [Symbol]
+           -> ([Symbol] -> [Symbol] -> Ordering)
            -> Array SeqId [Symbol]
            -> [(QIdent, Info)]
            -> FId
-           -> Map.Map CId (Int,Int)
+           -> Map.Map PGF2.Cat (Int,Int)
            -> (FId,
                [(FId, [Production])],
                [(FId, [FunId])],
                [(FId, [FunId])],
-               [(CId,[SeqId])])
-genCncFuns gr am cm ex_seqs seqs cdefs fid_cnt cnccat_ranges =
+               [(PGF2.Fun,[SeqId])])
+genCncFuns gr am cm ex_seqs ciCmp seqs cdefs fid_cnt cnccat_ranges =
   let (fid_cnt1,funs_cnt1,funs1,lindefs,linrefs) = mkCncCats cdefs fid_cnt  0 [] IntMap.empty IntMap.empty
       (fid_cnt2,funs_cnt2,funs2,prods0)          = mkCncFuns cdefs fid_cnt1 funs_cnt1 funs1 lindefs Map.empty IntMap.empty
       prods                                      = [(fid,Set.toList prodSet) | (fid,prodSet) <- IntMap.toList prods0]
@@ -304,7 +309,7 @@ genCncFuns gr am cm ex_seqs seqs cdefs fid_cnt cnccat_ranges =
         newIndex mseqs i = binSearch (mseqs ! i) seqs (bounds seqs)
 
         binSearch v arr (i,j)
-          | i <= j    = case compare v (arr ! k) of
+          | i <= j    = case ciCmp v (arr ! k) of
                           LT -> binSearch v arr (i,k-1)
                           EQ -> k
                           GT -> binSearch v arr (k+1,j)
@@ -323,3 +328,121 @@ genPrintNames cdefs =
     flatten (K s)      = s
     flatten (Alts x _) = flatten x
     flatten (C x y)    = flatten x +++ flatten y
+
+mkArray    lst = listArray (0,length lst-1) lst
+mkMapArray map = array (0,Map.size map-1) [(v,k) | (k,v) <- Map.toList map]
+mkSetArray set = listArray (0,Set.size set-1) (Set.toList set)
+
+-- The following is a version of Data.List.sortBy which together
+-- with the sorting also eliminates duplicate values 
+sortNubBy cmp = mergeAll . sequences
+  where
+    sequences (a:b:xs) =
+      case cmp a b of
+        GT -> descending b [a]  xs
+        EQ -> sequences (b:xs)
+        LT -> ascending  b (a:) xs
+    sequences xs = [xs]
+
+    descending a as []     = [a:as]
+    descending a as (b:bs) =
+      case cmp a b of
+        GT -> descending b (a:as) bs
+        EQ -> descending a as bs
+        LT -> (a:as) : sequences (b:bs)
+
+    ascending a as []     = let !x = as [a]
+                            in [x]
+    ascending a as (b:bs) =
+      case cmp a b of
+        GT -> let !x = as [a]
+              in x : sequences (b:bs)
+        EQ -> ascending a as bs
+        LT -> ascending b (\ys -> as (a:ys)) bs
+
+    mergeAll [x] = x
+    mergeAll xs  = mergeAll (mergePairs xs)
+
+    mergePairs (a:b:xs) = let !x = merge a b
+                          in x : mergePairs xs
+    mergePairs xs       = xs
+
+    merge as@(a:as') bs@(b:bs') =
+      case cmp a b of
+        GT -> b:merge as  bs'
+        EQ -> a:merge as' bs'
+        LT -> a:merge as' bs
+    merge [] bs         = bs
+    merge as []         = as
+
+-- The following function does case-insensitive comparison of sequences.
+-- This is used to allow case-insensitive parsing, while
+-- the linearizer still has access to the original cases.
+
+compareCaseInsensitive []     []     = EQ
+compareCaseInsensitive []     _      = LT
+compareCaseInsensitive _      []     = GT
+compareCaseInsensitive (x:xs) (y:ys) =
+  case compareSym x y of
+    EQ -> compareCaseInsensitive xs ys
+    x  -> x
+  where
+    compareSym s1 s2 =
+      case s1 of
+        SymCat d1 r1
+          -> case s2 of
+               SymCat d2 r2
+                 -> case compare d1 d2 of
+                      EQ -> r1 `compare` r2
+                      x  -> x
+               _ -> LT
+        SymLit d1 r1
+          -> case s2 of
+               SymCat {} -> GT
+               SymLit d2 r2
+                 -> case compare d1 d2 of
+                      EQ -> r1 `compare` r2
+                      x  -> x
+               _ -> LT
+        SymVar d1 r1
+          -> if tagToEnum# (getTag s2 ># 2#)
+               then LT
+               else case s2 of
+                      SymVar d2 r2
+                        -> case compare d1 d2 of
+                             EQ -> r1 `compare` r2
+                             x  -> x
+                      _ -> GT
+        SymKS t1
+          -> if tagToEnum# (getTag s2 ># 3#)
+               then LT
+               else case s2 of
+                      SymKS t2 -> t1 `compareToken` t2
+                      _          -> GT
+        SymKP a1 b1
+          -> if tagToEnum# (getTag s2 ># 4#)
+               then LT
+               else case s2 of
+                      SymKP a2 b2
+                        -> case compare a1 a2 of
+                             EQ -> b1 `compare` b2
+                             x  -> x
+                      _ -> GT
+        _ -> let t1 = getTag s1
+                 t2 = getTag s2
+             in if tagToEnum# (t1 <# t2) 
+                  then LT
+                  else if tagToEnum# (t1 ==# t2)
+                         then EQ
+                         else GT
+  
+    compareToken []     []     = EQ
+    compareToken []     _      = LT
+    compareToken _      []     = GT
+    compareToken (x:xs) (y:ys)
+      | x == y    = compareToken xs ys
+      | otherwise = case compare (toLower x) (toLower y) of
+                      EQ -> case compareToken xs ys of
+                              EQ -> compare x y
+                              x  -> x
+                      x  -> x

src/compiler/GF/Compile/PGFtoJS.hs

@@ -1,88 +0,0 @@
-module GF.Compile.PGFtoJS (pgf2js) where
-
-import PGF
-import PGF.Internal
-import qualified GF.JavaScript.AbsJS as JS
-import qualified GF.JavaScript.PrintJS as JS
-import Data.Map (Map)
-import qualified Data.Set as Set
-import qualified Data.Map as Map
-import qualified Data.IntMap as IntMap
-
-pgf2js :: PGF -> String
-pgf2js pgf =
-  JS.printTree $ JS.Program [JS.ElStmt $ JS.SDeclOrExpr $ JS.Decl [JS.DInit (JS.Ident n) grammar]]
- where
-   n  = showCId $ abstractName pgf
-   start = showType [] $ startCat pgf
-   grammar = new "GFGrammar" [js_abstract, js_concrete]
-   js_abstract = abstract2js start pgf
-   js_concrete = JS.EObj $ map (concrete2js pgf) (languages pgf)
-
-abstract2js :: String -> PGF -> JS.Expr
-abstract2js start pgf = new "GFAbstract" [JS.EStr start, JS.EObj [absdef2js f ty | f <- functions pgf, Just ty <- [functionType pgf f]]]
-
-absdef2js :: CId -> Type -> JS.Property
-absdef2js f typ =
-  let (hypos,cat,_) = unType typ
-      args          = [cat | (_,_,typ) <- hypos, let (hypos,cat,_) = unType typ]
-  in JS.Prop (JS.IdentPropName (JS.Ident (showCId f))) (new "Type" [JS.EArray [JS.EStr (showCId x) | x <- args], JS.EStr (showCId cat)])
-
-lit2js (LStr s) = JS.EStr s
-lit2js (LInt n) = JS.EInt n
-lit2js (LFlt d) = JS.EDbl d
-
-concrete2js :: PGF -> Language -> JS.Property
-concrete2js pgf lang =
-  JS.Prop l (new "GFConcrete" [mapToJSObj (lit2js) $ concrFlags cnc,
-                               JS.EObj [JS.Prop (JS.IntPropName cat) (JS.EArray (map frule2js (concrProductions cnc cat))) | cat <- [0..concrTotalCats cnc]],
-                               JS.EArray [ffun2js (concrFunction cnc funid) | funid <- [0..concrTotalFuns cnc]],
-                               JS.EArray [seq2js (concrSequence cnc seqid) | seqid <- [0..concrTotalSeqs cnc]],
-                               JS.EObj $ map cats (concrCategories cnc),
-                               JS.EInt (concrTotalCats cnc)])
-  where
-   cnc = lookConcr pgf lang
-   l  = JS.IdentPropName (JS.Ident (showCId lang))
-
-   litslins = [JS.Prop (JS.StringPropName    "Int") (JS.EFun [children] [JS.SReturn $ new "Arr" [JS.EIndex (JS.EVar children) (JS.EInt 0)]]), 
-               JS.Prop (JS.StringPropName  "Float") (JS.EFun [children] [JS.SReturn $ new "Arr" [JS.EIndex (JS.EVar children) (JS.EInt 0)]]),
-               JS.Prop (JS.StringPropName "String") (JS.EFun [children] [JS.SReturn $ new "Arr" [JS.EIndex (JS.EVar children) (JS.EInt 0)]])]
-
-   cats (c,start,end,_) = JS.Prop (JS.IdentPropName (JS.Ident (showCId c))) (JS.EObj [JS.Prop (JS.IdentPropName (JS.Ident "s")) (JS.EInt start)
-                                                                                     ,JS.Prop (JS.IdentPropName (JS.Ident "e")) (JS.EInt end)])
-
-children :: JS.Ident
-children = JS.Ident "cs"
-
-frule2js :: Production -> JS.Expr
-frule2js (PApply funid args) = new "Apply"  [JS.EInt funid, JS.EArray (map farg2js args)]
-frule2js (PCoerce arg)       = new "Coerce" [JS.EInt arg]
-
-farg2js (PArg hypos fid) = new "PArg" (map (JS.EInt . snd) hypos ++ [JS.EInt fid])
-
-ffun2js (f,lins) = new "CncFun" [JS.EStr (showCId f), JS.EArray (map JS.EInt lins)]
-
-seq2js :: [Symbol] -> JS.Expr
-seq2js seq = JS.EArray [sym2js s | s <- seq]
-
-sym2js :: Symbol -> JS.Expr
-sym2js (SymCat n l)    = new "SymCat" [JS.EInt n, JS.EInt l]
-sym2js (SymLit n l)    = new "SymLit" [JS.EInt n, JS.EInt l]
-sym2js (SymVar n l)    = new "SymVar" [JS.EInt n, JS.EInt l]
-sym2js (SymKS t)       = new "SymKS"  [JS.EStr t]
-sym2js (SymKP ts alts) = new "SymKP"  [JS.EArray (map sym2js ts), JS.EArray (map alt2js alts)]
-sym2js SymBIND         = new "SymKS"  [JS.EStr "&+"]
-sym2js SymSOFT_BIND    = new "SymKS"  [JS.EStr "&+"]
-sym2js SymSOFT_SPACE   = new "SymKS"  [JS.EStr "&+"]
-sym2js SymCAPIT        = new "SymKS"  [JS.EStr "&|"]
-sym2js SymALL_CAPIT    = new "SymKS"  [JS.EStr "&|"]
-sym2js SymNE           = new "SymNE"  []
-
-alt2js (ps,ts) = new "Alt" [JS.EArray (map sym2js ps), JS.EArray (map JS.EStr ts)]
-
-new :: String -> [JS.Expr] -> JS.Expr
-new f xs = JS.ENew (JS.Ident f) xs
-
-mapToJSObj :: (a -> JS.Expr) -> Map CId a -> JS.Expr
-mapToJSObj f m = JS.EObj [ JS.Prop (JS.IdentPropName (JS.Ident (showCId k))) (f v) | (k,v) <- Map.toList m ]
-

src/compiler/GF/Compile/PGFtoJSON.hs

@@ -0,0 +1,156 @@
+module GF.Compile.PGFtoJSON (pgf2json) where
+
+import PGF (showCId)
+import qualified PGF.Internal as M
+import PGF.Internal (
+  Abstr,
+  CId,
+  CncCat(..),
+  CncFun(..),
+  Concr,
+  DotPos,
+  Equation(..),
+  Literal(..),
+  PArg(..),
+  PGF,
+  Production(..),
+  Symbol(..),
+  Type,
+  absname,
+  abstract,
+  cflags,
+  cnccats,
+  cncfuns,
+  concretes,
+  funs,
+  productions,
+  sequences,
+  totalCats
+  )
+
+import qualified Text.JSON as JSON
+import Text.JSON (JSValue(..))
+
+import qualified Data.Array.IArray as Array
+import Data.Map (Map)
+import qualified Data.Set as Set
+import qualified Data.Map as Map
+import qualified Data.IntMap as IntMap
+
+pgf2json :: PGF -> String
+pgf2json pgf =
+  JSON.encode $ JSON.makeObj
+    [ ("abstract", json_abstract)
+    , ("concretes", json_concretes)
+    ]
+ where
+   n  = showCId $ absname pgf
+   as = abstract pgf
+   cs = Map.assocs (concretes pgf)
+   start = showCId $ M.lookStartCat pgf
+   json_abstract = abstract2json n start as
+   json_concretes = JSON.makeObj $ map concrete2json cs
+
+abstract2json :: String -> String -> Abstr -> JSValue
+abstract2json name start ds =
+  JSON.makeObj
+    [ ("name", mkJSStr name)
+    , ("startcat", mkJSStr start)
+    , ("funs", JSON.makeObj $ map absdef2json (Map.assocs (funs ds)))
+    ]
+
+absdef2json :: (CId,(Type,Int,Maybe ([Equation],[[M.Instr]]),Double)) -> (String,JSValue)
+absdef2json (f,(typ,_,_,_)) = (showCId f,sig)
+  where
+    (args,cat) = M.catSkeleton typ
+    sig = JSON.makeObj
+      [ ("args", JSArray $ map (mkJSStr.showCId) args)
+      , ("cat", mkJSStr $ showCId cat)
+      ]
+
+lit2json :: Literal -> JSValue
+lit2json (LStr s) = mkJSStr s
+lit2json (LInt n) = mkJSInt n
+lit2json (LFlt d) = JSRational True (toRational d)
+
+concrete2json :: (CId,Concr) -> (String,JSValue)
+concrete2json (c,cnc) = (showCId c,obj)
+  where
+    obj = JSON.makeObj
+      [ ("flags", JSON.makeObj [ (showCId k, lit2json v) | (k,v) <- Map.toList (cflags cnc) ])
+      , ("productions", JSON.makeObj [ (show cat, JSArray (map frule2json (Set.toList set))) | (cat,set) <- IntMap.toList (productions cnc)])
+      , ("functions", JSArray (map ffun2json (Array.elems (cncfuns cnc))))
+      , ("sequences", JSArray (map seq2json (Array.elems (sequences cnc))))
+      , ("categories", JSON.makeObj $ map cats2json (Map.assocs (cnccats cnc)))
+      , ("totalfids", mkJSInt (totalCats cnc))
+      ]
+
+cats2json :: (CId, CncCat) -> (String,JSValue)
+cats2json (c,CncCat start end _) = (showCId c, ixs)
+  where
+    ixs = JSON.makeObj
+      [ ("start", mkJSInt start)
+      , ("end", mkJSInt end)
+      ]
+
+frule2json :: Production -> JSValue
+frule2json (PApply fid args) =
+  JSON.makeObj
+    [ ("type", mkJSStr "Apply")
+    , ("fid", mkJSInt fid)
+    , ("args", JSArray (map farg2json args))
+    ]
+frule2json (PCoerce arg) =
+  JSON.makeObj
+    [ ("type", mkJSStr "Coerce")
+    , ("arg", mkJSInt arg)
+    ]
+
+farg2json :: PArg -> JSValue
+farg2json (PArg hypos fid) =
+  JSON.makeObj
+    [ ("type", mkJSStr "PArg")
+    , ("hypos", JSArray $ map (mkJSInt . snd) hypos)
+    , ("fid", mkJSInt fid)
+    ]
+
+ffun2json :: CncFun -> JSValue
+ffun2json (CncFun f lins) =
+  JSON.makeObj
+    [ ("name", mkJSStr $ showCId f)
+    , ("lins", JSArray (map mkJSInt (Array.elems lins)))
+    ]
+
+seq2json :: Array.Array DotPos Symbol -> JSValue
+seq2json seq = JSArray [sym2json s | s <- Array.elems seq]
+
+sym2json :: Symbol -> JSValue
+sym2json (SymCat n l)    = new "SymCat" [mkJSInt n, mkJSInt l]
+sym2json (SymLit n l)    = new "SymLit" [mkJSInt n, mkJSInt l]
+sym2json (SymVar n l)    = new "SymVar" [mkJSInt n, mkJSInt l]
+sym2json (SymKS t)       = new "SymKS"  [mkJSStr t]
+sym2json (SymKP ts alts) = new "SymKP"  [JSArray (map sym2json ts), JSArray (map alt2json alts)]
+sym2json SymBIND         = new "SymKS"  [mkJSStr "&+"]
+sym2json SymSOFT_BIND    = new "SymKS"  [mkJSStr "&+"]
+sym2json SymSOFT_SPACE   = new "SymKS"  [mkJSStr "&+"]
+sym2json SymCAPIT        = new "SymKS"  [mkJSStr "&|"]
+sym2json SymALL_CAPIT    = new "SymKS"  [mkJSStr "&|"]
+sym2json SymNE           = new "SymNE"  []
+
+alt2json :: ([Symbol],[String]) -> JSValue
+alt2json (ps,ts) = new "Alt" [JSArray (map sym2json ps), JSArray (map mkJSStr ts)]
+
+new :: String -> [JSValue] -> JSValue
+new f xs =
+  JSON.makeObj
+    [ ("type", mkJSStr f)
+    , ("args", JSArray xs)
+    ]
+
+-- | Make JSON value from string
+mkJSStr :: String -> JSValue
+mkJSStr = JSString . JSON.toJSString
+
+-- | Make JSON value from integer
+mkJSInt :: Integral a => a -> JSValue
+mkJSInt = JSRational False . toRational

src/compiler/GF/Compile/PGFtoProlog.hs

@@ -1,186 +0,0 @@
-----------------------------------------------------------------------
--- |
--- Module      : PGFtoProlog
--- Maintainer  : Peter Ljunglöf
---
--- exports a GF grammar into a Prolog module
------------------------------------------------------------------------------
-
-module GF.Compile.PGFtoProlog (grammar2prolog) where
-
-import PGF
-import PGF.Internal
-
-import GF.Data.Operations
-
-import qualified Data.Array.IArray as Array
-import qualified Data.Set as Set
-import qualified Data.Map as Map
-import qualified Data.IntMap as IntMap
-import Data.Char (isAlphaNum, isAscii, isAsciiLower, isAsciiUpper, ord)
-import Data.List (isPrefixOf, mapAccumL)
-
-grammar2prolog :: PGF -> String
-grammar2prolog pgf
-    = ("%% This file was automatically generated by GF" +++++
-       ":- style_check(-singleton)." +++++
-       plFacts wildCId "abstract" 1 "(?AbstractName)"
-                   [[plp name]] ++++
-       plFacts wildCId "concrete" 2 "(?AbstractName, ?ConcreteName)" 
-                   [[plp name, plp cncname] | 
-                    cncname <- languages pgf] ++++
-       plFacts wildCId "flag" 2 "(?Flag, ?Value): global flags" 
-                   [[plp f, plp v] | 
-                    (f, v) <- Map.assocs (globalFlags pgf)] ++++
-       plAbstract name pgf ++++
-       unlines [plConcrete name (lookConcr pgf name) | name <- languages pgf]
-      )
-    where name = abstractName pgf
-
-----------------------------------------------------------------------
--- abstract syntax
-
-plAbstract :: CId -> PGF -> String
-plAbstract name pgf
-    = (plHeader "Abstract syntax" ++++
-       plFacts name "flag" 2 "(?Flag, ?Value): flags for abstract syntax"
-                   [[plp f, plp v] | 
-                    (f, v) <- Map.assocs (abstrFlags pgf)] ++++
-       plFacts name "cat" 2 "(?Type, ?[X:Type,...])"
-                   [[plType cat, []] | cat <- categories pgf] ++++
-       plFacts name "fun" 3 "(?Fun, ?Type, ?[X:Type,...])"
-                   [[plp fun, plType cat, plHypos hypos] |
-                    fun <- functions pgf, Just typ <- [functionType pgf fun],
-                    let (hypos,cat,_) = unType typ]
-      )
-    where plType cat = plTerm (plp cat) []
-          plHypos hypos = plList [plOper ":" (plp x) (plp ty) | (_, x, ty) <- hypos]
-
-----------------------------------------------------------------------
--- concrete syntax
-
-plConcrete :: CId -> Concr -> String
-plConcrete name cnc
-    = (plHeader ("Concrete syntax: " ++ plp name) ++++
-       plFacts name "flag" 2 "(?Flag, ?Value): flags for concrete syntax"
-                   [[plp f, plp v] | 
-                    (f, v) <- Map.assocs (concrFlags cnc)] ++++
-       plFacts name "prod" 3 "(?CncCat, ?CncFun, ?[CncCat])"
-                   [[plCat cat, fun, plTerm "c" (map plCat args)] |
-                    cat <- [0..concrTotalCats cnc-1],
-                    (fun, args) <- map plProduction (concrProductions cnc cat)] ++++
-       plFacts name "cncfun" 3 "(?CncFun, ?[Seq,...], ?AbsFun)"
-                   [[plFun funid, plTerm "s" (map plSeq lins), plp absfun] |
-                    funid <- [0..concrTotalFuns cnc-1], let (absfun,lins) = concrFunction cnc funid] ++++
-       plFacts name "seq" 2 "(?Seq, ?[Term])"
-                   [[plSeq seqid, plp (concrSequence cnc seqid)] |
-                    seqid <- [0..concrTotalSeqs cnc-1]] ++++
-       plFacts name "cnccat" 2 "(?AbsCat, ?[CnCCat])" 
-                   [[plp cat, plList (map plCat [start..end])] |
-                    (cat,start,end,_) <- concrCategories cnc]
-      )
-    where plProduction (PCoerce arg)       = ("-", [arg])
-          plProduction (PApply funid args) = (plFun funid, [fid | PArg hypos fid <- args])
-
-----------------------------------------------------------------------
--- prolog-printing pgf datatypes
-
-instance PLPrint Type where
-    plp ty
-      | null hypos = result
-      | otherwise  = plOper " -> " plHypos result
-      where (hypos,cat,_) = unType ty
-            result = plTerm (plp cat) []
-            plHypos = plList [plOper ":" (plp x) (plp ty) | (_,x,ty) <- hypos]
-
-instance PLPrint CId where
-    plp cid | isLogicalVariable str || cid == wildCId = plVar str
-            | otherwise = plAtom str
-        where str = showCId cid
-
-instance PLPrint Literal where
-    plp (LStr s) = plp s
-    plp (LInt n) = plp (show n)
-    plp (LFlt f) = plp (show f)
-
-instance PLPrint Symbol where
-    plp (SymCat n l)    = plOper ":" (show n) (show l)
-    plp (SymLit n l)    = plTerm "lit" [show n, show l]
-    plp (SymVar n l)    = plTerm "var" [show n, show l]
-    plp (SymKS t)       = plAtom t
-    plp (SymKP ts alts) = plTerm "pre" [plList (map plp ts), plList (map plAlt alts)]
-        where plAlt (ps,ts) = plOper "/" (plList (map plp ps)) (plList (map plAtom ts))
-
-class PLPrint a where
-    plp :: a -> String
-    plps :: [a] -> String
-    plps = plList . map plp
-
-instance PLPrint Char where
-    plp  c = plAtom [c]
-    plps s = plAtom s
-
-instance PLPrint a => PLPrint [a] where
-    plp = plps
-
-----------------------------------------------------------------------
--- other prolog-printing functions
-
-plCat :: Int -> String
-plCat n = plAtom ('c' : show n)
-
-plFun :: Int -> String
-plFun n = plAtom ('f' : show n)
-
-plSeq :: Int -> String
-plSeq n = plAtom ('s' : show n)
-
-plHeader :: String -> String
-plHeader hdr = "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n%% " ++ hdr ++ "\n"
-
-plFacts :: CId -> String -> Int -> String -> [[String]] -> String
-plFacts mod pred arity comment facts = "%% " ++ pred ++ comment ++++ clauses
-    where clauses = (if facts == [] then ":- dynamic " ++ pred ++ "/" ++ show arity ++ ".\n"
-                     else unlines [mod' ++ plTerm pred args ++ "." | args <- facts])
-          mod' = if mod == wildCId then "" else plp mod ++ ": "
-
-plTerm :: String -> [String] -> String
-plTerm fun args = plAtom fun ++ prParenth (prTList ", " args)
-
-plList :: [String] -> String
-plList xs = prBracket (prTList "," xs)
-
-plOper :: String -> String -> String -> String
-plOper op a b = prParenth (a ++ op ++ b)
-
-plVar :: String -> String
-plVar = varPrefix  . concatMap changeNonAlphaNum 
-    where varPrefix var@(c:_) | isAsciiUpper c || c=='_' = var
-                              | otherwise = "_" ++ var
-          changeNonAlphaNum c | isAlphaNumUnderscore c = [c]
-                              | otherwise = "_" ++ show (ord c) ++ "_"
-
-plAtom :: String -> String
-plAtom "" = "''"
-plAtom atom@(c:cs) | isAsciiLower c && all isAlphaNumUnderscore cs 
-                     || c == '\'' && cs /= "" && last cs == '\''   = atom
-                   | otherwise = "'" ++ changeQuote atom ++ "'"
-    where changeQuote ('\'':cs) = '\\' : '\'' : changeQuote cs
-          changeQuote ('\\':cs) = '\\' : '\\' : changeQuote cs
-          changeQuote (c:cs) = c : changeQuote cs
-          changeQuote "" = ""
-
-isAlphaNumUnderscore :: Char -> Bool
-isAlphaNumUnderscore c = (isAscii c && isAlphaNum c) || c == '_'
-
-----------------------------------------------------------------------
--- prolog variables 
-
-createLogicalVariable :: Int -> CId
-createLogicalVariable n = mkCId (logicalVariablePrefix ++ show n)
-
-isLogicalVariable :: String -> Bool
-isLogicalVariable = isPrefixOf logicalVariablePrefix 
-
-logicalVariablePrefix :: String 
-logicalVariablePrefix = "X"

src/compiler/GF/Compile/PGFtoPython.hs

@@ -1,114 +0,0 @@
-----------------------------------------------------------------------
--- |
--- Module      : PGFtoPython
--- Maintainer  : Peter Ljunglöf
---
--- exports a GF grammar into a Python module
------------------------------------------------------------------------------
-
-{-# LANGUAGE FlexibleContexts #-}
-module GF.Compile.PGFtoPython (pgf2python) where
-
-import PGF
-import PGF.Internal
-import qualified Data.Map as Map
-import GF.Data.Operations
-
-pgf2python :: PGF -> String
-pgf2python pgf = ("# -*- coding: utf-8 -*-" ++++
-                  "# This file was automatically generated by GF" +++++
-                  showCId name +++ "=" +++ 
-                  pyDict 1 pyStr id [
-                              ("flags", pyDict 2 pyCId pyLiteral (Map.assocs (globalFlags pgf))),
-                              ("abstract", pyDict 2 pyStr id [
-                                              ("name", pyCId name), 
-                                              ("start", pyCId start),
-                                              ("flags", pyDict 3 pyCId pyLiteral (Map.assocs (abstrFlags pgf))),
-                                              ("funs", pyDict 3 pyCId pyAbsdef [(f,ty) | f <- functions pgf, Just ty <- [functionType pgf f]])
-                                             ]),
-                              ("concretes", pyDict 2 pyCId pyConcrete [(lang,lookConcr pgf lang) | lang <- languages pgf])
-                             ] ++ "\n")
-    where
-      name  = abstractName pgf
-      (_,start,_) = unType (startCat pgf)
---      cncs = concretes pgf
-
-pyAbsdef :: Type -> String
-pyAbsdef typ = pyTuple 0 id [pyCId cat, pyList 0 pyCId args]
-    where (hypos,cat,_) = unType typ
-          args = [cat | (_,_,typ) <- hypos, let (_,cat,_) = unType typ]
-
-pyLiteral :: Literal -> String
-pyLiteral (LStr s) = pyStr s
-pyLiteral (LInt n) = show n
-pyLiteral (LFlt d) = show d
-
-pyConcrete :: Concr -> String
-pyConcrete cnc = pyDict 3 pyStr id [
-                  ("flags", pyDict 0 pyCId pyLiteral (Map.assocs (concrFlags cnc))),
-                  ("productions", pyDict 4 pyCat pyProds [(fid,concrProductions cnc fid) | fid <- [0..concrTotalCats cnc-1]]),
-                  ("cncfuns", pyDict 4 pyFun pyCncFun [(funid,concrFunction cnc funid) | funid <- [0..concrTotalFuns cnc-1]]),
-                  ("sequences",  pyDict 4 pySeq pySymbols [(seqid,concrSequence cnc seqid) | seqid <- [0..concrTotalSeqs cnc-1]]),
-                  ("cnccats", pyDict 4 pyCId pyCncCat [(cat,(s,e,lbls)) | (cat,s,e,lbls) <- concrCategories cnc]),
-                  ("size",  show (concrTotalCats cnc))
-                 ]
-    where pyProds prods = pyList 5 pyProduction prods
-          pyCncCat (start,end,_) = pyList 0 pyCat [start..end]
-          pyCncFun (f,lins) = pyTuple 0 id [pyList 0 pySeq lins, pyCId f]
-          pySymbols syms = pyList 0 pySymbol syms
-
-pyProduction :: Production -> String
-pyProduction (PCoerce arg)       = pyTuple 0 id [pyStr "", pyList 0 pyCat [arg]]
-pyProduction (PApply funid args) = pyTuple 0 id [pyFun funid, pyList 0 pyPArg args]
-    where pyPArg (PArg [] fid) = pyCat fid
-          pyPArg (PArg hypos fid) = pyTuple 0 pyCat (fid : map snd hypos)
-
-pySymbol :: Symbol -> String
-pySymbol (SymCat n l)    = pyTuple 0 show [n, l]
-pySymbol (SymLit n l)    = pyDict 0 pyStr id [("lit", pyTuple 0 show [n, l])]
-pySymbol (SymVar n l)    = pyDict 0 pyStr id [("var", pyTuple 0 show [n, l])]
-pySymbol (SymKS t)       = pyStr t
-pySymbol (SymKP ts alts) = pyDict 0 pyStr id [("pre", pyList 0 pySymbol ts), ("alts", pyList 0 alt2py alts)]
-    where alt2py (ps,ts) = pyTuple 0 (pyList 0 pyStr) [map pySymbol ps, ts]
-pySymbol SymBIND         = pyStr "&+"
-pySymbol SymSOFT_BIND    = pyStr "&+"
-pySymbol SymSOFT_SPACE   = pyStr "&+"
-pySymbol SymCAPIT        = pyStr "&|"
-pySymbol SymALL_CAPIT    = pyStr "&|"
-pySymbol SymNE           = pyDict 0 pyStr id [("nonExist", pyTuple 0 id [])]
-
-----------------------------------------------------------------------
--- python helpers 
-
-pyDict :: Int -> (k -> String) -> (v -> String) -> [(k, v)] -> String
-pyDict n pk pv [] = "{}"
-pyDict n pk pv kvlist = prCurly (pyIndent n ++ prTList ("," ++ pyIndent n) (map pyKV kvlist) ++ pyIndent n)
-    where pyKV (k, v) = pk k ++ ":" ++ pv v
-
-pyList :: Int -> (v -> String) -> [v] -> String
-pyList n pv [] = "[]"
-pyList n pv xs = prBracket (pyIndent n ++ prTList ("," ++ pyIndent n) (map pv xs) ++ pyIndent n)
-
-pyTuple :: Int -> (v -> String) -> [v] -> String
-pyTuple n pv [] = "()"
-pyTuple n pv [x] = prParenth (pyIndent n ++ pv x ++ "," ++ pyIndent n)
-pyTuple n pv xs = prParenth (pyIndent n ++ prTList ("," ++ pyIndent n) (map pv xs) ++ pyIndent n)
-
-pyCat :: Int -> String
-pyCat n = pyStr ('C' : show n)
-
-pyFun :: Int -> String
-pyFun n = pyStr ('F' : show n)
-
-pySeq :: Int -> String
-pySeq n = pyStr ('S' : show n)
-
-pyStr :: String -> String
-pyStr s = 'u' : prQuotedString s
-
-pyCId :: CId -> String
-pyCId = pyStr . showCId
-
-pyIndent :: Int -> String
-pyIndent n | n > 0 = "\n" ++ replicate n ' '
-           | otherwise = ""

src/compiler/GF/Compile/TypeCheck/RConcrete.hs

@@ -359,12 +359,13 @@ getOverload gr g mt ot = case appForm ot of
                       nest 2 (showTypes pre)
          return (mkApp fun tts, val)
        ([],[]) -> do
-         checkError $ "no overload instance of" <+> ppTerm Unqualified 0 f $$
-                      "for" $$
+         checkError $ "no overload instance of" <+> ppTerm Qualified 0 f $$
+                      maybe empty (\x -> "with value type" <+> ppType x) mt $$
+                      "for argument list" $$
                       nest 2 stysError $$
-                      "among" $$
-                      nest 2 (vcat stypsError) $$
-                      maybe empty (\x -> "with value type" <+> ppType x) mt
+                      "among alternatives" $$
+                      nest 2 (vcat stypsError) 
+
 
        (vfs1,vfs2) -> case (noProds vfs1,noProds vfs2) of
          ([(val,fun)],_) -> do

src/compiler/GF/Compile/pgf.schema.json

@@ -0,0 +1,232 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "http://grammaticalframework.org/pgf.schema.json",
+  "type": "object",
+  "title": "PGF JSON Schema",
+  "required": [
+    "abstract",
+    "concretes"
+  ],
+  "properties": {
+    "abstract": {
+      "type": "object",
+      "required": [
+        "name",
+        "startcat",
+        "funs"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "startcat": {
+          "type": "string"
+        },
+        "funs": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "object",
+            "required": [
+              "args",
+              "cat"
+            ],
+            "properties": {
+              "args": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              "cat": {
+                "type": "string"
+              }
+            }
+          }
+        }
+      }
+    },
+    "concretes": {
+      "type": "object",
+      "additionalProperties": {
+        "required": [
+          "flags",
+          "productions",
+          "functions",
+          "sequences",
+          "categories",
+          "totalfids"
+        ],
+        "properties": {
+          "flags": {
+            "type": "object",
+            "additionalProperties": {
+              "type": ["string", "number"]
+            }
+          },
+          "productions": {
+            "type": "object",
+            "additionalProperties": {
+              "type": "array",
+              "items": {
+                "oneOf": [
+                  {
+                    "$ref": "#/definitions/apply"
+                  },
+                  {
+                    "$ref": "#/definitions/coerce"
+                  }
+                ]
+              }
+            }
+          },
+          "functions": {
+            "type": "array",
+            "items": {
+              "title": "CncFun",
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "lins": {
+                  "type": "array",
+                  "items": {
+                    "type": "integer"
+                  }
+                }
+              }
+            }
+          },
+          "sequences": {
+            "type": "array",
+            "items": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/sym"
+              }
+            }
+          },
+          "categories": {
+            "type": "object",
+            "additionalProperties": {
+              "title": "CncCat",
+              "type": "object",
+              "required": [
+                "start",
+                "end"
+              ],
+              "properties": {
+                "start": {
+                  "type": "integer"
+                },
+                "end": {
+                  "type": "integer"
+                }
+              }
+            }
+          },
+          "totalfids": {
+            "type": "integer"
+          }
+        }
+      }
+    }
+  },
+  "definitions": {
+    "apply": {
+      "required": [
+        "type",
+        "fid",
+        "args"
+      ],
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": ["Apply"]
+        },
+        "fid": {
+          "type": "integer"
+        },
+        "args": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/parg"
+          }
+        }
+      }
+    },
+    "coerce": {
+      "required": [
+        "type",
+        "arg"
+      ],
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": ["Coerce"]
+        },
+        "arg": {
+          "type": "integer"
+        }
+      }
+    },
+    "parg": {
+      "required": [
+        "type",
+        "hypos",
+        "fid"
+      ],
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": ["PArg"]
+        },
+        "hypos": {
+          "type": "array",
+          "items": {
+            "type": "integer"
+          }
+        },
+        "fid": {
+          "type": "integer"
+        }
+      }
+    },
+    "sym": {
+      "title": "Sym",
+      "required": [
+        "type",
+        "args"
+      ],
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": [
+            "SymCat",
+            "SymLit",
+            "SymVar",
+            "SymKS",
+            "SymKP",
+            "SymNE"
+          ]
+        },
+        "args": {
+          "type": "array",
+          "items": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "integer"
+              },
+              {
+                "$ref": "#/definitions/sym"
+              }
+            ]
+          }
+        }
+      }
+    }
+  }
+}

src/compiler/GF/Compiler.hs

@@ -1,11 +1,12 @@
 module GF.Compiler (mainGFC, linkGrammars, writeGrammar, writeOutputs) where
 
-import PGF
-import PGF.Internal(unionPGF,writePGF,writeConcr)
+import PGF2
+import PGF2.Internal(unionPGF,writePGF,writeConcr)
 import GF.Compile as S(batchCompile,link,srcAbsName)
 import GF.CompileInParallel as P(parallelBatchCompile)
 import GF.Compile.Export
 import GF.Compile.ConcreteToHaskell(concretes2haskell)
+import GF.Compile.GrammarToCanonical--(concretes2canonical)
 import GF.Compile.CFGtoPGF
 import GF.Compile.GetGrammar
 import GF.Grammar.BNFC
@@ -16,12 +17,13 @@ import GF.Infra.UseIO
 import GF.Infra.Option
 import GF.Data.ErrM
 import GF.System.Directory
-import GF.Text.Pretty(render)
+import GF.Text.Pretty(render,render80)
 
 import Data.Maybe
 import qualified Data.Map as Map
 import qualified Data.Set as Set
 import qualified Data.ByteString.Lazy as BSL
+import GF.Grammar.CanonicalJSON (encodeJSON)
 import System.FilePath
 import Control.Monad(when,unless,forM_)
 
@@ -46,7 +48,7 @@ mainGFC opts fs = do
 compileSourceFiles :: Options -> [FilePath] -> IOE ()
 compileSourceFiles opts fs = 
     do output <- batchCompile opts fs
-       cncs2haskell output
+       exportCanonical output
        unless (flag optStopAfterPhase opts == Compile) $
            linkGrammars opts output
   where
@@ -54,15 +56,35 @@ compileSourceFiles opts fs =
     batchCompile' opts fs = do (t,cnc_gr) <- S.batchCompile opts fs
                                return (t,[cnc_gr])
 
-    cncs2haskell output =
-      when (FmtHaskell `elem` flag optOutputFormats opts &&
-            haskellOption opts HaskellConcrete) $
-        mapM_ cnc2haskell (snd output)
+    exportCanonical (_time, canonical) =
+      do when (FmtHaskell `elem` ofmts && haskellOption opts HaskellConcrete) $
+           mapM_ cnc2haskell canonical
+         when (FmtCanonicalGF `elem` ofmts) $
+           do createDirectoryIfMissing False "canonical"
+              mapM_ abs2canonical canonical
+              mapM_ cnc2canonical canonical
+         when (FmtCanonicalJson `elem` ofmts) $ mapM_ grammar2json canonical
+      where
+        ofmts = flag optOutputFormats opts
 
     cnc2haskell (cnc,gr) =
-        mapM_ writeHs $ concretes2haskell opts (srcAbsName gr cnc) gr
+      do mapM_ writeExport $ concretes2haskell opts (srcAbsName gr cnc) gr
 
-    writeHs (path,s) = writing opts path $ writeUTF8File path s
+    abs2canonical (cnc,gr) =
+        writeExport ("canonical/"++render absname++".gf",render80 canAbs)
+      where
+        absname = srcAbsName gr cnc
+        canAbs = abstract2canonical absname gr
+
+    cnc2canonical (cnc,gr) =
+      mapM_ (writeExport.fmap render80) $
+            concretes2canonical opts (srcAbsName gr cnc) gr
+
+    grammar2json (cnc,gr) = encodeJSON (render absname ++ ".json") gr_canon
+      where absname = srcAbsName gr cnc
+            gr_canon = grammar2canonical opts absname gr
+
+    writeExport (path,s) = writing opts path $ writeUTF8File path s
 
 
 -- | Create a @.pgf@ file (and possibly files in other formats, if specified
@@ -113,7 +135,7 @@ unionPGFFiles opts fs =
     doIt =
       do pgfs <- mapM readPGFVerbose fs
          let pgf = foldl1 (\one two -> fromMaybe two (unionPGF one two)) pgfs
-             pgfFile = outputPath opts (grammarName opts pgf <.> "pgf")
+         let pgfFile = outputPath opts (grammarName opts pgf <.> "pgf")
          if pgfFile `elem` fs
            then putStrLnE $ "Refusing to overwrite " ++ pgfFile
            else writeGrammar opts pgf
@@ -135,7 +157,7 @@ writeOutputs opts pgf = do
 -- A split PGF file is output if the @-split-pgf@ option is used.
 writeGrammar :: Options -> PGF -> IOE ()
 writeGrammar opts pgf =
-    if flag optSplitPGF opts then writeSplitPGF else writeNormalPGF
+  if flag optSplitPGF opts then writeSplitPGF else writeNormalPGF
   where
     writeNormalPGF =
        do let outfile = outputPath opts (grammarName opts pgf <.> "pgf")
@@ -144,9 +166,9 @@ writeGrammar opts pgf =
     writeSplitPGF =
       do let outfile = outputPath opts (grammarName opts pgf <.> "pgf")
          writing opts outfile $ writePGF outfile pgf
-         forM_ (languages pgf) $ \lang -> do
-           let outfile = outputPath opts (showCId lang <.> "pgf_c")
-           writing opts outfile (writeConcr outfile pgf lang)
+         forM_ (Map.toList (languages pgf)) $ \(concrname,concr) -> do
+           let outfile = outputPath opts (concrname <.> "pgf_c")
+           writing opts outfile (writeConcr outfile concr)
 
 
 writeOutput :: Options -> FilePath-> String -> IOE ()
@@ -156,7 +178,7 @@ writeOutput opts file str = writing opts path $ writeUTF8File path str
 -- * Useful helper functions
 
 grammarName :: Options -> PGF -> String
-grammarName opts pgf = grammarName' opts (showCId (abstractName pgf))
+grammarName opts pgf = grammarName' opts (abstractName pgf)
 grammarName' opts abs = fromMaybe abs (flag optName opts)
 
 outputJustPGF opts = null (flag optOutputFormats opts) && not (flag optSplitPGF opts)

src/compiler/GF/Grammar/Canonical.hs

@@ -0,0 +1,313 @@
+-- |
+-- Module    :  GF.Grammar.Canonical
+-- Stability :  provisional
+--
+-- Abstract syntax for canonical GF grammars, i.e. what's left after
+-- high-level constructions such as functors and opers have been eliminated
+-- by partial evaluation. This is intended as a common intermediate
+-- representation to simplify export to other formats.
+
+{-# LANGUAGE DeriveTraversable #-}
+module GF.Grammar.Canonical where
+import Prelude hiding ((<>))
+import GF.Text.Pretty
+
+-- | A Complete grammar
+data Grammar = Grammar Abstract [Concrete] deriving Show
+
+--------------------------------------------------------------------------------
+-- ** Abstract Syntax
+
+-- | Abstract Syntax
+data Abstract = Abstract ModId Flags [CatDef] [FunDef] deriving Show
+abstrName (Abstract mn _ _ _) = mn
+
+data CatDef   = CatDef CatId [CatId]        deriving Show
+data FunDef   = FunDef FunId Type           deriving Show
+data Type     = Type [TypeBinding] TypeApp  deriving Show
+data TypeApp  = TypeApp CatId [Type]        deriving Show
+
+data TypeBinding = TypeBinding VarId Type   deriving Show
+
+--------------------------------------------------------------------------------
+-- ** Concreate syntax
+
+-- | Concrete Syntax
+data Concrete  = Concrete ModId ModId Flags [ParamDef] [LincatDef] [LinDef]
+                 deriving Show
+concName (Concrete cnc _ _ _ _ _) = cnc
+
+data ParamDef  = ParamDef ParamId [ParamValueDef]
+               | ParamAliasDef ParamId LinType
+               deriving Show
+data LincatDef = LincatDef CatId LinType  deriving Show
+data LinDef    = LinDef FunId [VarId] LinValue  deriving Show
+
+-- | Linearization type, RHS of @lincat@
+data LinType = FloatType 
+             | IntType 
+             | ParamType ParamType
+             | RecordType [RecordRowType]
+             | StrType 
+             | TableType LinType LinType 
+             | TupleType [LinType]
+              deriving (Eq,Ord,Show)
+
+newtype ParamType = ParamTypeId ParamId deriving (Eq,Ord,Show)
+
+-- | Linearization value, RHS of @lin@
+data LinValue = ConcatValue LinValue LinValue
+              | LiteralValue LinLiteral
+              | ErrorValue String
+              | ParamConstant ParamValue 
+              | PredefValue PredefId
+              | RecordValue [RecordRowValue]
+              | TableValue LinType [TableRowValue]
+---           | VTableValue LinType [LinValue]
+              | TupleValue [LinValue]
+              | VariantValue [LinValue]
+              | VarValue VarValueId
+              | PreValue [([String], LinValue)] LinValue
+              | Projection LinValue LabelId
+              | Selection LinValue LinValue
+              | CommentedValue String LinValue
+              deriving (Eq,Ord,Show)
+
+data LinLiteral = FloatConstant Float 
+                | IntConstant Int 
+                | StrConstant String 
+                deriving (Eq,Ord,Show)
+
+data LinPattern = ParamPattern ParamPattern
+                | RecordPattern [RecordRow LinPattern]
+                | TuplePattern [LinPattern]
+                | WildPattern
+                deriving (Eq,Ord,Show)
+
+type ParamValue = Param LinValue
+type ParamPattern = Param LinPattern
+type ParamValueDef = Param ParamId
+
+data Param arg = Param ParamId [arg]
+                 deriving (Eq,Ord,Show,Functor,Foldable,Traversable)
+
+type RecordRowType  = RecordRow LinType
+type RecordRowValue = RecordRow LinValue
+type TableRowValue  = TableRow LinValue
+
+data RecordRow rhs = RecordRow LabelId    rhs
+                     deriving (Eq,Ord,Show,Functor,Foldable,Traversable)
+data TableRow  rhs = TableRow  LinPattern rhs
+                     deriving (Eq,Ord,Show,Functor,Foldable,Traversable)
+
+-- *** Identifiers in Concrete Syntax
+
+newtype PredefId = PredefId Id        deriving (Eq,Ord,Show)
+newtype LabelId  = LabelId Id         deriving (Eq,Ord,Show)
+data VarValueId  = VarValueId QualId  deriving (Eq,Ord,Show)
+
+-- | Name of param type or param value 
+newtype ParamId = ParamId QualId  deriving (Eq,Ord,Show)
+
+--------------------------------------------------------------------------------
+-- ** Used in both Abstract and Concrete Syntax
+
+newtype ModId = ModId Id  deriving (Eq,Ord,Show)
+
+newtype CatId = CatId Id  deriving (Eq,Ord,Show)
+newtype FunId = FunId Id  deriving (Eq,Show)
+
+data VarId = Anonymous | VarId Id  deriving Show
+
+newtype Flags = Flags [(FlagName,FlagValue)] deriving Show
+type FlagName = Id
+data FlagValue = Str String | Int Int | Flt Double deriving Show
+
+
+-- *** Identifiers
+
+type Id = String
+data QualId = Qual ModId Id | Unqual Id  deriving (Eq,Ord,Show)
+
+--------------------------------------------------------------------------------
+-- ** Pretty printing
+
+instance Pretty Grammar where
+  pp (Grammar abs cncs) = abs $+$ vcat cncs
+
+instance Pretty Abstract where
+  pp (Abstract m flags cats funs) =
+    "abstract" <+> m <+> "=" <+> "{" $$
+       flags $$
+       "cat" <+> fsep cats $$
+       "fun" <+> vcat funs $$
+       "}"
+
+instance Pretty CatDef where
+  pp (CatDef c cs) = hsep (c:cs)<>";"
+
+instance Pretty FunDef where
+  pp (FunDef f ty) = f <+> ":" <+> ty <>";"
+
+instance Pretty Type where
+  pp (Type bs ty) = sep (punctuate " ->" (map pp bs ++ [pp ty]))
+
+instance PPA Type where
+  ppA (Type [] (TypeApp c [])) = pp c
+  ppA t = parens t
+
+instance Pretty TypeBinding where
+  pp (TypeBinding Anonymous (Type [] tapp)) = pp tapp
+  pp (TypeBinding Anonymous ty) = parens ty
+  pp (TypeBinding (VarId x) ty) = parens (x<+>":"<+>ty)
+
+instance Pretty TypeApp where
+ pp (TypeApp c targs) = c<+>hsep (map ppA targs)
+
+instance Pretty VarId where
+  pp Anonymous = pp "_"
+  pp (VarId x) = pp x
+
+--------------------------------------------------------------------------------
+
+instance Pretty Concrete where
+  pp (Concrete cncid absid flags params lincats lins) =
+      "concrete" <+> cncid <+> "of" <+> absid <+> "=" <+> "{" $$
+      vcat params $$
+      section "lincat" lincats $$
+      section "lin" lins $$
+      "}"
+    where
+      section name [] = empty
+      section name ds = name <+> vcat (map (<> ";") ds)
+
+instance Pretty ParamDef where
+  pp (ParamDef p pvs) = hang ("param"<+> p <+> "=") 4 (punctuate " |" pvs)<>";"
+  pp (ParamAliasDef p t) = hang ("oper"<+> p <+> "=") 4 t<>";"
+
+instance PPA arg => Pretty (Param arg) where
+  pp (Param p ps) = pp p<+>sep (map ppA ps)
+
+instance PPA arg => PPA (Param arg) where
+  ppA (Param p []) = pp p
+  ppA pv = parens pv
+
+instance Pretty LincatDef where
+  pp (LincatDef c lt) = hang (c <+> "=") 4 lt
+
+instance Pretty LinType where
+ pp lt = case lt of
+           FloatType -> pp "Float"
+           IntType -> pp "Int"
+           ParamType pt -> pp pt
+           RecordType rs -> block rs
+           StrType -> pp "Str"
+           TableType pt lt -> sep [pt <+> "=>",pp lt]
+           TupleType lts -> "<"<>punctuate "," lts<>">"
+
+instance RhsSeparator LinType  where rhsSep _ = pp ":"
+
+instance Pretty ParamType where
+  pp (ParamTypeId p) = pp p
+
+instance Pretty LinDef where
+  pp (LinDef f xs lv) = hang (f<+>hsep xs<+>"=") 4 lv
+
+instance Pretty LinValue where
+  pp lv = case lv of
+            ConcatValue v1 v2 -> sep [v1 <+> "++",pp v2]
+            ErrorValue s -> "Predef.error"<+>doubleQuotes s
+            ParamConstant pv -> pp pv
+            Projection lv l -> ppA lv<>"."<>l
+            Selection tv pv -> ppA tv<>"!"<>ppA pv
+            VariantValue vs -> "variants"<+>block vs
+            CommentedValue s v -> "{-" <+> s <+> "-}" $$ v
+            _ -> ppA lv
+
+instance PPA LinValue where
+  ppA lv = case lv of
+             LiteralValue l -> ppA l
+             ParamConstant pv -> ppA pv
+             PredefValue p -> ppA p
+             RecordValue [] -> pp "<>"
+             RecordValue rvs -> block rvs
+             PreValue alts def ->
+               "pre"<+>block (map alt alts++["_"<+>"=>"<+>def])
+               where
+                 alt (ss,lv) = hang (hcat (punctuate "|" (map doubleQuotes ss)))
+                                    2 ("=>"<+>lv)
+             TableValue _ tvs -> "table"<+>block tvs
+--           VTableValue t ts -> "table"<+>t<+>brackets (semiSep ts)
+             TupleValue lvs -> "<"<>punctuate "," lvs<>">"
+             VarValue v -> pp v
+             _ -> parens lv
+
+instance Pretty LinLiteral where pp = ppA
+
+instance PPA LinLiteral where
+  ppA l = case l of
+            FloatConstant f -> pp f
+            IntConstant n -> pp n
+            StrConstant s -> doubleQuotes s -- hmm
+  
+instance RhsSeparator LinValue where rhsSep _ = pp "="
+
+instance Pretty LinPattern where
+  pp p =
+    case p of
+      ParamPattern pv -> pp pv
+      _ -> ppA p
+
+instance PPA LinPattern where
+  ppA p =
+    case p of
+      ParamPattern pv -> ppA pv
+      RecordPattern r -> block r
+      TuplePattern ps -> "<"<>punctuate "," ps<>">"
+      WildPattern     -> pp "_"                
+      _ -> parens p
+
+instance RhsSeparator LinPattern where rhsSep _ = pp "="
+
+instance RhsSeparator rhs => Pretty (RecordRow rhs) where
+  pp (RecordRow l v) = hang (l<+>rhsSep v) 2 v
+
+instance Pretty rhs => Pretty (TableRow rhs) where
+  pp (TableRow l v) = hang (l<+>"=>") 2 v
+
+--------------------------------------------------------------------------------
+instance Pretty ModId where pp (ModId s) = pp s
+instance Pretty CatId where pp (CatId s) = pp s
+instance Pretty FunId where pp (FunId s) = pp s
+instance Pretty LabelId where pp (LabelId s) = pp s
+instance Pretty PredefId where pp = ppA
+instance PPA    PredefId where ppA (PredefId s) = "Predef."<>s
+instance Pretty ParamId where pp = ppA
+instance PPA    ParamId where ppA (ParamId s) = pp s
+instance Pretty VarValueId where pp (VarValueId s) = pp s
+
+instance Pretty QualId where pp = ppA
+
+instance PPA QualId where
+  ppA (Qual m n) = m<>"_"<>n -- hmm
+  ppA (Unqual n) = pp n
+
+instance Pretty Flags where
+  pp (Flags []) = empty
+  pp (Flags flags) = "flags" <+> vcat (map ppFlag flags)
+    where
+      ppFlag (name,value) = name <+> "=" <+> value <>";"
+
+instance Pretty FlagValue where
+  pp (Str s) = pp s
+  pp (Int i) = pp i
+  pp (Flt d) = pp d
+
+--------------------------------------------------------------------------------
+-- | Pretty print atomically (i.e. wrap it in parentheses if necessary)
+class Pretty a => PPA a where ppA :: a -> Doc
+
+class Pretty rhs => RhsSeparator rhs where rhsSep :: rhs -> Doc
+
+semiSep xs = punctuate ";" xs
+block xs = braces (semiSep xs)

src/compiler/GF/Grammar/CanonicalJSON.hs

@@ -0,0 +1,289 @@
+module GF.Grammar.CanonicalJSON (
+  encodeJSON
+  ) where
+
+import Text.JSON
+import Control.Applicative ((<|>))
+import Data.Ratio (denominator, numerator)
+import GF.Grammar.Canonical
+
+
+encodeJSON :: FilePath -> Grammar -> IO ()
+encodeJSON fpath g = writeFile fpath (encode g)
+
+
+-- in general we encode grammars using JSON objects/records,
+-- except for newtypes/coercions/direct values
+
+-- the top-level definitions use normal record labels,
+-- but recursive types/values/ids use labels staring with a "."
+
+instance JSON Grammar where
+  showJSON (Grammar abs cncs) = makeObj [("abstract", showJSON abs), ("concretes", showJSON cncs)]
+
+  readJSON o = Grammar <$> o!"abstract" <*> o!"concretes"
+
+
+--------------------------------------------------------------------------------
+-- ** Abstract Syntax
+
+instance JSON Abstract where
+  showJSON (Abstract absid flags cats funs) 
+    = makeObj [("abs", showJSON absid),
+               ("flags", showJSON flags),
+               ("cats", showJSON cats),
+               ("funs", showJSON funs)]
+
+  readJSON o = Abstract
+    <$> o!"abs"
+    <*>(o!"flags" <|> return (Flags []))
+    <*> o!"cats"
+    <*> o!"funs"
+
+instance JSON CatDef where
+  -- non-dependent categories are encoded as simple strings:
+  showJSON (CatDef c []) = showJSON c
+  showJSON (CatDef c cs) = makeObj [("cat", showJSON c), ("args", showJSON cs)]
+
+  readJSON o = CatDef <$> readJSON o <*> return []
+    <|>        CatDef <$> o!"cat" <*> o!"args"
+
+instance JSON FunDef where
+  showJSON (FunDef f ty) = makeObj [("fun", showJSON f), ("type", showJSON ty)]
+
+  readJSON o = FunDef <$> o!"fun" <*> o!"type"
+
+instance JSON Type where
+  showJSON (Type bs ty) = makeObj [(".args", showJSON bs), (".result", showJSON ty)]
+
+  readJSON o = Type <$> o!".args" <*> o!".result"
+
+instance JSON TypeApp where
+  -- non-dependent categories are encoded as simple strings:
+  showJSON (TypeApp c [])   = showJSON c
+  showJSON (TypeApp c args) = makeObj [(".cat", showJSON c), (".args", showJSON args)]
+
+  readJSON o = TypeApp <$> readJSON o <*> return []
+    <|>        TypeApp <$> o!".cat" <*> o!".args"
+
+instance JSON TypeBinding where
+  -- non-dependent categories are encoded as simple strings:
+  showJSON (TypeBinding Anonymous (Type [] (TypeApp c []))) = showJSON c
+  showJSON (TypeBinding x ty) = makeObj [(".var", showJSON x), (".type", showJSON ty)]
+
+  readJSON o = do c <- readJSON o
+                  return (TypeBinding Anonymous (Type [] (TypeApp c [])))
+           <|> TypeBinding <$> o!".var" <*> o!".type"
+
+
+--------------------------------------------------------------------------------
+-- ** Concrete syntax
+
+instance JSON Concrete where
+  showJSON (Concrete cncid absid flags params lincats lins) 
+    = makeObj [("cnc", showJSON cncid),
+               ("abs", showJSON absid),
+               ("flags", showJSON flags),
+               ("params", showJSON params),
+               ("lincats", showJSON lincats),
+               ("lins", showJSON lins)]
+
+  readJSON o = Concrete
+    <$> o!"cnc"
+    <*> o!"abs"
+    <*>(o!"flags" <|> return (Flags []))
+    <*> o!"params"
+    <*> o!"lincats"
+    <*> o!"lins"
+
+instance JSON ParamDef where
+  showJSON (ParamDef      p pvs) = makeObj [("param", showJSON p), ("values", showJSON pvs)]
+  showJSON (ParamAliasDef p t)   = makeObj [("param", showJSON p), ("alias", showJSON t)]
+
+  readJSON o = ParamDef      <$> o!"param" <*> o!"values"
+    <|>        ParamAliasDef <$> o!"param" <*> o!"alias"
+
+instance JSON LincatDef where
+  showJSON (LincatDef c lt) = makeObj [("cat", showJSON c), ("lintype", showJSON lt)]
+
+  readJSON o = LincatDef <$> o!"cat" <*> o!"lintype"
+
+instance JSON LinDef where
+  showJSON (LinDef f xs lv) = makeObj [("fun", showJSON f), ("args", showJSON xs), ("lin", showJSON lv)]
+
+  readJSON o = LinDef <$> o!"fun" <*> o!"args" <*> o!"lin"
+
+instance JSON LinType where
+  -- the basic types (Str, Float, Int) are encoded as strings:
+  showJSON (StrType)         = showJSON "Str"
+  showJSON (FloatType)       = showJSON "Float"
+  showJSON (IntType)         = showJSON "Int"
+  -- parameters are also encoded as strings:
+  showJSON (ParamType pt)    = showJSON pt
+  -- tables/tuples are encoded as JSON objects:
+  showJSON (TableType pt lt) = makeObj [(".tblarg", showJSON pt), (".tblval", showJSON lt)]
+  showJSON (TupleType lts)   = makeObj [(".tuple",  showJSON lts)]
+  -- records are encoded as records:
+  showJSON (RecordType rows) = showJSON rows
+
+  readJSON o = do "Str"   <- readJSON o; return StrType
+    <|>        do "Float" <- readJSON o; return FloatType
+    <|>        do "Int"   <- readJSON o; return IntType
+    <|>        do  ptype  <- readJSON o; return (ParamType ptype)
+    <|>        TableType  <$> o!".tblarg" <*> o!".tblval"
+    <|>        TupleType  <$> o!".tuple"
+    <|>        RecordType <$> readJSON o
+
+instance JSON LinValue where
+  showJSON (LiteralValue  l ) = showJSON l
+  -- most values are encoded as JSON objects:
+  showJSON (ParamConstant pv) = makeObj [(".param",    showJSON pv)]
+  showJSON (PredefValue   p ) = makeObj [(".predef",   showJSON p)]
+  showJSON (TableValue t tvs) = makeObj [(".tblarg",   showJSON t), (".tblrows", showJSON tvs)]
+  showJSON (TupleValue   lvs) = makeObj [(".tuple",    showJSON lvs)]
+  showJSON (VarValue      v ) = makeObj [(".var",      showJSON v)]
+  showJSON (ErrorValue    s ) = makeObj [(".error",    showJSON s)]
+  showJSON (Projection lv l ) = makeObj [(".project",  showJSON lv), (".label", showJSON l)]
+  showJSON (Selection  tv pv) = makeObj [(".select",   showJSON tv), (".key", showJSON pv)]
+  showJSON (VariantValue  vs) = makeObj [(".variants", showJSON vs)]
+  showJSON (PreValue pre def) = makeObj [(".pre",      showJSON pre),(".default", showJSON def)]
+  -- records are encoded directly as JSON records:
+  showJSON (RecordValue rows) = showJSON rows
+  -- concatenation is encoded as a JSON array:
+  showJSON v@(ConcatValue _ _) = showJSON (flatten v [])
+    where flatten (ConcatValue v v') = flatten v . flatten v'
+          flatten  v                 = (v :)
+
+  readJSON o = LiteralValue  <$> readJSON o
+    <|>        ParamConstant <$> o!".param"
+    <|>        PredefValue   <$> o!".predef"
+    <|>        TableValue    <$> o!".tblarg"  <*> o!".tblrows"
+    <|>        TupleValue    <$> o!".tuple"
+    <|>        VarValue      <$> o!".var"
+    <|>        ErrorValue    <$> o!".error"
+    <|>        Projection    <$> o!".project" <*> o!".label"
+    <|>        Selection     <$> o!".select"  <*> o!".key"
+    <|>        VariantValue  <$> o!".variants"
+    <|>        PreValue      <$> o!".pre"     <*> o!".default"
+    <|>        RecordValue   <$> readJSON o
+    <|>        do vs <- readJSON o :: Result [LinValue]
+                  return (foldr1 ConcatValue vs)
+
+instance JSON LinLiteral where
+  -- basic values (Str, Float, Int) are encoded as JSON strings/numbers:
+  showJSON (StrConstant   s) = showJSON s
+  showJSON (FloatConstant f) = showJSON f
+  showJSON (IntConstant   n) = showJSON n
+
+  readJSON = readBasicJSON StrConstant IntConstant FloatConstant
+
+instance JSON LinPattern where
+  -- wildcards and patterns without arguments are encoded as strings:
+  showJSON (WildPattern) = showJSON "_"
+  showJSON (ParamPattern (Param p [])) = showJSON p
+  -- complex patterns are encoded as JSON objects:
+  showJSON (ParamPattern pv) = showJSON pv
+  -- and records as records:
+  showJSON (RecordPattern r) = showJSON r
+
+  readJSON o = do "_" <- readJSON o; return WildPattern
+    <|>        do  p  <- readJSON o; return (ParamPattern (Param p []))
+    <|>        ParamPattern  <$> readJSON o
+    <|>        RecordPattern <$> readJSON o
+
+instance JSON arg => JSON (Param arg) where
+  -- parameters without arguments are encoded as strings:
+  showJSON (Param p [])   = showJSON p
+  showJSON (Param p args) = makeObj [(".paramid", showJSON p), (".args", showJSON args)]
+
+  readJSON o = Param <$> readJSON o <*> return []
+    <|>        Param <$> o!".paramid" <*> o!".args"
+
+instance JSON a => JSON (RecordRow a) where
+  -- record rows and lists of record rows are both encoded as JSON records (i.e., objects)
+  showJSON  row  = showJSONs [row]
+  showJSONs rows = makeObj (map toRow rows)
+    where toRow (RecordRow (LabelId lbl) val) = (lbl, showJSON val)
+
+  readJSON  obj = head <$> readJSONs obj
+  readJSONs obj = mapM fromRow (assocsJSObject obj)
+    where fromRow (lbl, jsvalue) = do value <- readJSON jsvalue
+                                      return (RecordRow (LabelId lbl) value)
+
+instance JSON rhs => JSON (TableRow rhs) where
+  showJSON (TableRow l v) = makeObj [(".pattern", showJSON l), (".value", showJSON v)]
+
+  readJSON o = TableRow <$> o!".pattern" <*> o!".value"
+
+
+-- *** Identifiers in Concrete Syntax
+
+instance JSON PredefId   where showJSON (PredefId    s) = showJSON s ; readJSON = fmap PredefId    . readJSON 
+instance JSON LabelId    where showJSON (LabelId     s) = showJSON s ; readJSON = fmap LabelId     . readJSON 
+instance JSON VarValueId where showJSON (VarValueId  s) = showJSON s ; readJSON = fmap VarValueId  . readJSON 
+instance JSON ParamId    where showJSON (ParamId     s) = showJSON s ; readJSON = fmap ParamId     . readJSON 
+instance JSON ParamType  where showJSON (ParamTypeId s) = showJSON s ; readJSON = fmap ParamTypeId . readJSON 
+
+
+--------------------------------------------------------------------------------
+-- ** Used in both Abstract and Concrete Syntax
+
+instance JSON ModId where showJSON (ModId s) = showJSON s ; readJSON = fmap ModId . readJSON 
+instance JSON CatId where showJSON (CatId s) = showJSON s ; readJSON = fmap CatId . readJSON 
+instance JSON FunId where showJSON (FunId s) = showJSON s ; readJSON = fmap FunId . readJSON 
+
+instance JSON VarId where
+  -- the anonymous variable is the underscore:
+  showJSON Anonymous = showJSON "_"
+  showJSON (VarId x) = showJSON x
+
+  readJSON o = do "_" <- readJSON o; return Anonymous
+    <|>        VarId <$> readJSON o
+
+instance JSON QualId where
+  showJSON (Qual (ModId m) n) = showJSON (m++"."++n)
+  showJSON (Unqual n) = showJSON n
+
+  readJSON o = do qualid <- readJSON o
+                  let (mod, id) = span (/= '.') qualid
+                  return $ if null mod then Unqual id else Qual (ModId mod) id
+
+instance JSON Flags where
+  -- flags are encoded directly as JSON records (i.e., objects):
+  showJSON (Flags fs) = makeObj [(f, showJSON v) | (f, v) <- fs]
+
+  readJSON obj = Flags <$> mapM fromRow (assocsJSObject obj)
+    where fromRow (lbl, jsvalue) = do value <- readJSON jsvalue
+                                      return (lbl, value)
+
+instance JSON FlagValue where
+  -- flag values are encoded as basic JSON types:
+  showJSON (Str s) = showJSON s
+  showJSON (Int i) = showJSON i
+  showJSON (Flt f) = showJSON f
+
+  readJSON = readBasicJSON Str Int Flt
+
+
+--------------------------------------------------------------------------------
+-- ** Convenience functions
+
+(!) :: JSON a => JSValue -> String -> Result a
+obj ! key = maybe (fail $ "CanonicalJSON.(!): Could not find key: " ++ show key)
+            readJSON
+            (lookup key (assocsJSObject obj))
+
+assocsJSObject :: JSValue -> [(String, JSValue)]
+assocsJSObject (JSObject o) = fromJSObject o
+assocsJSObject (JSArray  _) = fail $ "CanonicalJSON.assocsJSObject: Expected a JSON object, found an Array"
+assocsJSObject  jsvalue     = fail $ "CanonicalJSON.assocsJSObject: Expected a JSON object, found " ++ show jsvalue
+
+
+readBasicJSON :: (JSON int, Integral int, JSON flt, RealFloat flt) =>
+                 (String -> v) -> (int -> v) -> (flt -> v) -> JSValue -> Result v
+readBasicJSON str int flt o
+  =   str        <$> readJSON o
+  <|> int_or_flt <$> readJSON o
+  where int_or_flt f | f == fromIntegral n = int n
+                     | otherwise           = flt f
+          where n = round f

src/compiler/GF/Grammar/Printer.hs

@@ -208,7 +208,7 @@ ppTerm q d (S x y)     = case x of
 ppTerm q d (ExtR x y)  = prec d 3 (ppTerm q 3 x <+> "**" <+> ppTerm q 4 y)
 ppTerm q d (App x y)   = prec d 4 (ppTerm q 4 x <+> ppTerm q 5 y)
 ppTerm q d (V e es)    = hang "table" 2 (sep [ppTerm q 6 e,brackets (fsep (punctuate ';' (map (ppTerm q 0) es)))])
-ppTerm q d (FV es)     = "variants" <+> braces (fsep (punctuate ';' (map (ppTerm q 0) es)))
+ppTerm q d (FV es)     = prec d 4 ("variants" <+> braces (fsep (punctuate ';' (map (ppTerm q 0) es))))
 ppTerm q d (AdHocOverload es)     = "overload" <+> braces (fsep (punctuate ';' (map (ppTerm q 0) es)))
 ppTerm q d (Alts e xs) = prec d 4 ("pre" <+> braces (ppTerm q 0 e <> ';' <+> fsep (punctuate ';' (map (ppAltern q) xs))))
 ppTerm q d (Strs es)   = "strs" <+> braces (fsep (punctuate ';' (map (ppTerm q 0) es)))

src/compiler/GF/Haskell.hs

@@ -40,6 +40,9 @@ tvar = TId
 tcon0 = TId
 tcon c = foldl TAp (TId c)
 
+lets [] e = e
+lets ds e = Lets ds e
+
 let1 x xe e = Lets [(x,xe)] e
 single x = List [x]
 
@@ -113,7 +116,8 @@ instance Pretty Exp where
           Op e1 op e2 -> hang (ppB e1<+>op) 2 (ppB e2)
           Lets bs e -> sep ["let"<+>vcat [hang (x<+>"=") 2 xe|(x,xe)<-bs],
                             "in" <+>e]
-          LambdaCase alts -> hang "\\case" 4 (vcat [p<+>"->"<+>e|(p,e)<-alts])
+          LambdaCase alts ->
+              hang "\\case" 2 (vcat [hang (p<+>"->") 2 e|(p,e)<-alts])
           _ -> ppB e
 
       ppB e = case flatAp e of f:as -> hang (ppA f) 2 (sep (map ppA as))

src/compiler/GF/Infra/Option.hs

@@ -83,7 +83,10 @@ data Phase = Preproc | Convert | Compile | Link
   deriving (Show,Eq,Ord)
 
 data OutputFormat = FmtPGFPretty
+                  | FmtCanonicalGF
+                  | FmtCanonicalJson
                   | FmtJavaScript 
+                  | FmtJSON
                   | FmtPython 
                   | FmtHaskell 
                   | FmtJava
@@ -318,7 +321,8 @@ optDescr =
      Option [] ["gfo-dir"] (ReqArg gfoDir "DIR") "Directory to put .gfo files in (default = '.').",
      Option ['f'] ["output-format"] (ReqArg outFmt "FMT") 
         (unlines ["Output format. FMT can be one of:",
-                  "Multiple concrete: pgf (default), js, pgf_pretty, prolog, python, ...", -- gar,
+                  "Canonical GF grammar: canonical_gf, canonical_json, (and haskell with option --haskell=concrete)",
+                  "Multiple concrete: pgf (default), json, js, pgf_pretty, prolog, python, ...", -- gar,
                   "Single concrete only: bnf, ebnf, fa, gsl, jsgf, regexp, slf, srgs_xml, srgs_abnf, vxml, ....", -- cf, lbnf,
                   "Abstract only: haskell, ..."]), -- prolog_abs,
      Option [] ["sisr"] (ReqArg sisrFmt "FMT") 
@@ -366,8 +370,6 @@ optDescr =
                 "Enable or disable global grammar optimization. This could significantly reduce the size of the final PGF file",
      Option [] ["split-pgf"] (NoArg (splitPGF True))
                 "Split the PGF into one file per language. This allows the runtime to load only individual languages",
-     Option [] ["stem"] (onOff (toggleOptimize OptStem) True) "Perform stem-suffix analysis (default on).",
-     Option [] ["cse"] (onOff (toggleOptimize OptCSE) True) "Perform common sub-expression elimination (default on).",
      Option [] ["cfg"] (ReqArg cfgTransform "TRANS") "Enable or disable specific CFG transformations. TRANS = merge, no-merge, bottomup, no-bottomup, ...",
      Option [] ["heuristic_search_factor"] (ReqArg (readDouble (\d o -> o { optHeuristicFactor = Just d })) "FACTOR") "Set the heuristic search factor for statistical parsing",
      Option [] ["case_sensitive"] (onOff (\v -> set $ \o -> o{optCaseSensitive=v}) True) "Set the parser in case-sensitive/insensitive mode [sensitive by default]",
@@ -441,8 +443,6 @@ optDescr =
        optimize_pgf x = set $ \o -> o { optOptimizePGF = x }
        splitPGF x = set $ \o -> o { optSplitPGF = x }
 
-       toggleOptimize x b = set $ setOptimization' x b
-
        cfgTransform x = let (x', b) = case x of
                                         'n':'o':'-':rest -> (rest, False)
                                         _                -> (x, True)
@@ -465,7 +465,10 @@ outputFormats = map fst outputFormatsExpl
 outputFormatsExpl :: [((String,OutputFormat),String)]
 outputFormatsExpl = 
     [(("pgf_pretty",   FmtPGFPretty),"human-readable pgf"),
+     (("canonical_gf", FmtCanonicalGF),"Canonical GF source files"),
+     (("canonical_json", FmtCanonicalJson),"Canonical JSON source files"),
      (("js",           FmtJavaScript),"JavaScript (whole grammar)"),
+     (("json",         FmtJSON),"JSON (whole grammar)"),
      (("python",       FmtPython),"Python (whole grammar)"),
      (("haskell",      FmtHaskell),"Haskell (abstract syntax)"),
      (("java",         FmtJava),"Java (abstract syntax)"),

src/compiler/GF/Main.hs

@@ -20,7 +20,7 @@ import GF.System.Console (setConsoleEncoding)
 -- Run @gf --help@ for usage info.
 main :: IO ()
 main = do
-  setConsoleEncoding
+  --setConsoleEncoding
   uncurry mainOpts =<< getOptions
 
 -- | Get and parse GF command line arguments. Fix relative paths.

src/compiler/GF/Speech/PGFToCFG.hs

@@ -6,8 +6,8 @@
 ----------------------------------------------------------------------
 module GF.Speech.PGFToCFG (bnfPrinter, pgfToCFG) where
 
-import PGF
-import PGF.Internal
+import PGF2
+import PGF2.Internal
 import GF.Grammar.CFG hiding (Symbol)
 
 import Data.Map (Map)
@@ -16,28 +16,25 @@ import qualified Data.IntMap as IntMap
 import Data.Set (Set)
 import qualified Data.Set as Set
 
-bnfPrinter :: PGF -> CId -> String
+bnfPrinter :: PGF -> Concr -> String
 bnfPrinter = toBNF id
 
-toBNF :: (CFG -> CFG) -> PGF -> CId -> String
+toBNF :: (CFG -> CFG) -> PGF -> Concr -> String
 toBNF f pgf cnc = prCFG $ f $ pgfToCFG pgf cnc
 
 type Profile = [Int]
 
-pgfToCFG :: PGF 
-          -> CId   -- ^ Concrete syntax name
-          -> CFG
-pgfToCFG pgf lang = mkCFG (showCId start_cat) extCats (startRules ++ concatMap ruleToCFRule rules)
+pgfToCFG :: PGF -> Concr -> CFG
+pgfToCFG pgf cnc = mkCFG start_cat extCats (startRules ++ concatMap ruleToCFRule rules)
   where
     (_,start_cat,_) = unType (startCat pgf)
-    cnc = lookConcr pgf lang
 
     rules :: [(FId,Production)]
     rules = [(fcat,prod) | fcat <- [0..concrTotalCats cnc],
                            prod <- concrProductions cnc fcat]
 
     fcatCats :: Map FId Cat
-    fcatCats = Map.fromList [(fc, showCId c ++ "_" ++ show i) 
+    fcatCats = Map.fromList [(fc, c ++ "_" ++ show i) 
                                  | (c,s,e,lbls) <- concrCategories cnc,
                                    (fc,i) <- zip [s..e] [1..]]
 
@@ -64,7 +61,7 @@ pgfToCFG pgf lang = mkCFG (showCId start_cat) extCats (startRules ++ concatMap r
     extCats = Set.fromList $ map ruleLhs startRules
 
     startRules :: [CFRule]
-    startRules = [Rule (showCId c) [NonTerminal (fcatToCat fc r)] (CFRes 0) 
+    startRules = [Rule c [NonTerminal (fcatToCat fc r)] (CFRes 0) 
                       | (c,s,e,lbls) <- concrCategories cnc,
                         fc <- [s..e], not (isPredefFId fc),
                         r <- [0..catLinArity fc-1]]
@@ -113,7 +110,7 @@ pgfToCFG pgf lang = mkCFG (showCId start_cat) extCats (startRules ++ concatMap r
             where Just (hypos,_,_) = fmap unType (functionType pgf f)
                   argTypes = [cat | (_,_,ty) <- hypos, let (_,cat,_) = unType ty]
 
-        profileToTerm :: CId -> Profile -> CFTerm
+        profileToTerm :: Fun -> Profile -> CFTerm
         profileToTerm t [] = CFMeta t
         profileToTerm _ xs = CFRes (last xs) -- FIXME: unify
     ruleToCFRule (c,PCoerce c') =

src/compiler/GF/System/Directory.hs

@@ -8,13 +8,13 @@ import System.Directory as D
                doesDirectoryExist,doesFileExist,getModificationTime,
                getCurrentDirectory,getDirectoryContents,getPermissions,
                removeFile,renameFile)
-import Data.Time.Compat
+--import Data.Time.Compat
 
 canonicalizePath path = liftIO $ D.canonicalizePath path
 createDirectoryIfMissing b = liftIO . D.createDirectoryIfMissing b
 doesDirectoryExist path = liftIO $ D.doesDirectoryExist path
 doesFileExist path = liftIO $ D.doesFileExist path
-getModificationTime path = liftIO $ fmap toUTCTime (D.getModificationTime path)
+getModificationTime path = liftIO $ {-fmap toUTCTime-} (D.getModificationTime path)
 getDirectoryContents path = liftIO $ D.getDirectoryContents path
 
 getCurrentDirectory :: MonadIO io => io FilePath

src/compiler/GF/Text/Pretty.hs

@@ -20,6 +20,7 @@ instance Pretty a => Pretty [a] where
   ppList = fsep . map pp -- hmm
 
 render x = PP.render (pp x)
+render80 x = renderStyle style{lineLength=80,ribbonsPerLine=1} x
 renderStyle s x = PP.renderStyle s (pp x)
 
 infixl 5 $$,$+$