added Predef.SOFT_BIND. This special token allows zero or more spaces between ordinary tokens. It is also used in the English RGL to attach the commas to the previous word.

src/runtime/haskell/PGF/Binary.hs

@@ -152,7 +152,8 @@ instance Binary Symbol where
   put (SymKS ts)         = putWord8 3 >> put ts
   put (SymKP d vs)       = putWord8 4 >> put (d,vs)
   put SymBIND            = putWord8 5
-  put SymNE              = putWord8 6
+  put SymSOFT_BIND       = putWord8 6
+  put SymNE              = putWord8 7
   get = do tag <- getWord8
            case tag of
              0 -> liftM2 SymCat get get
@@ -161,7 +162,8 @@ instance Binary Symbol where
              3 -> liftM  SymKS  get
              4 -> liftM2 (\d vs -> SymKP d vs) get get
              5 -> return SymBIND
-             6 -> return SymNE
+             6 -> return SymSOFT_BIND
+             7 -> return SymNE
              _ -> decodingError
 
 instance Binary PArg where

src/runtime/haskell/PGF/Data.hs

@@ -62,6 +62,7 @@ data Symbol
   | SymKS Token
   | SymKP [Symbol] [([Symbol],[String])]
   | SymBIND                         -- the special BIND token
+  | SymSOFT_BIND                    -- the special SOFT_BIND token
   | SymNE                           -- non exist (this should be last constructor to simplify the binary search in the runtime)
   deriving (Eq,Ord,Show)
 data Production

src/runtime/haskell/PGF/Macros.hs

@@ -160,6 +160,7 @@ data BracketedTokn
   | LeafKS Token
   | LeafNE
   | LeafBIND
+  | LeafSOFT_BIND
   | LeafKP [BracketedTokn] [([BracketedTokn],[String])]
   deriving Eq
 
@@ -222,6 +223,7 @@ computeSeq filter seq args = concatMap compute seq
     compute (SymKS t)         = [LeafKS t]
     compute SymNE             = [LeafNE]
     compute SymBIND           = [LeafKS "&+"]
+    compute SymSOFT_BIND      = []
     compute (SymKP syms alts) = [LeafKP (concatMap compute syms) [(concatMap compute syms,cs) | (syms,cs) <- alts]]
 
     getArg d r

src/runtime/haskell/PGF/Optimize.hs

@@ -228,6 +228,7 @@ splitLexicalRules cnc p_prods =
                                                   [seq2prefix (syms1 ++ syms) | (syms1,ps) <- alts])
         seq2prefix (SymNE           :syms) = TrieMap.empty
         seq2prefix (SymBIND         :syms) = TrieMap.fromList [wf ["&+"]]
+        seq2prefix (SymSOFT_BIND    :syms) = TrieMap.fromList [wf []]
 
 updateConcrete abs cnc = 
   let p_prods0      = filterProductions IntMap.empty IntSet.empty (productions cnc)

src/runtime/haskell/PGF/Parse.hs

@@ -309,10 +309,12 @@ process flit ftok cnc (item@(Active j ppos funid seqid args key0):items) acc cha
       	SymNE      -> process flit ftok cnc items acc chart
       	SymBIND    -> let !acc' = ftok_ ["&+"] (Active j (ppos+1) funid seqid args key0) acc
       	              in process flit ftok cnc items acc' chart
+      	SymSOFT_BIND->process flit ftok cnc ((Active j (ppos+1) funid seqid args key0):items) acc chart
       	SymKP syms vars
-      	           -> let to_tok (SymKS t) = [t]
-      	                  to_tok SymBIND   = ["&+"]
-      	                  to_tok _         = []
+      	           -> let to_tok (SymKS t)    = [t]
+      	                  to_tok SymBIND      = ["&+"]
+      	                  to_tok SymSOFT_BIND = []
+      	                  to_tok _            = []
 
       	                  !acc' = foldl (\acc syms -> ftok_ (concatMap to_tok syms) (Active j (ppos+1) funid seqid args key0) acc) acc
                                         (syms:[syms' | (syms',_) <- vars])

src/runtime/haskell/PGF/Printer.hs

@@ -92,6 +92,7 @@ ppSymbol (SymVar d r) = char '<' <> int d <> comma <> char '$' <> int r <> char
 ppSymbol (SymKS t)    = doubleQuotes (text t)
 ppSymbol SymNE        = text "nonExist"
 ppSymbol SymBIND      = text "BIND"
+ppSymbol SymSOFT_BIND = text "SOFT_BIND"
 ppSymbol (SymKP syms alts) = text "pre" <+> braces (hsep (punctuate semi (hsep (map ppSymbol syms) : map ppAlt alts)))
 
 ppAlt (syms,ps) = hsep (map ppSymbol syms) <+> char '/' <+> hsep (map (doubleQuotes . text) ps)