added Predef.SOFT_BIND. This special token allows zero or more spaces between ordinary tokens. It is also used in the English RGL to attach the commas to the previous word.

17 files changed, 69 insertions, 8 deletions

diff --git a/src/compiler/GF/Compile/Compute/AppPredefined.hs b/src/compiler/GF/Compile/Compute/AppPredefined.hs
index a597726c9..0d23f8cb6 100644
--- a/src/compiler/GF/Compile/Compute/AppPredefined.hs
+++ b/src/compiler/GF/Compile/Compute/AppPredefined.hs
@@ -86,6 +86,8 @@ primitives = Map.fromList
                                          [] typeStr []))) Nothing)
   , (cBIND     , ResOper (Just (noLoc (mkProd -- Str
                                          [] typeStr []))) Nothing)
+  , (cSOFT_BIND, ResOper (Just (noLoc (mkProd -- Str
+                                         [] typeStr []))) Nothing)
   ]
   where
     fun from to = oper (mkFunType from to)
diff --git a/src/compiler/GF/Compile/Compute/Predef.hs b/src/compiler/GF/Compile/Compute/Predef.hs
index 8c793502a..8c5e5c5f7 100644
--- a/src/compiler/GF/Compile/Compute/Predef.hs
+++ b/src/compiler/GF/Compile/Compute/Predef.hs
@@ -78,7 +78,8 @@ predefList =
      (cError,Error),
      -- Canonical values:
      (cPBool,PBool),(cPFalse,PFalse),(cPTrue,PTrue),(cInt,Int),
-     (cInts,Ints),(cNonExist,NonExist),(cBIND,BIND)]
+     (cInts,Ints),(cNonExist,NonExist)
+     ,(cBIND,BIND),(cSOFT_BIND,SOFT_BIND)]
     --- add more functions!!!
 
 delta f vs =
@@ -107,6 +108,7 @@ delta f vs =
       PTrue   -> canonical
       NonExist-> canonical
       BIND    -> canonical
+      SOFT_BIND->canonical
   where
     canonical = delay
     delay = return (VApp f vs) -- wrong number of arguments
diff --git a/src/compiler/GF/Compile/Compute/Value.hs b/src/compiler/GF/Compile/Compute/Value.hs
index 21a3eb20e..8f47ad184 100644
--- a/src/compiler/GF/Compile/Compute/Value.hs
+++ b/src/compiler/GF/Compile/Compute/Value.hs
@@ -51,5 +51,6 @@ data Predefined = Drop | Take | Tk | Dp | EqStr | Occur | Occurs | ToUpper
              {- | Show | Read | ToStr | MapStr | EqVal -}
                 | Error
                 -- Canonical values below:
-                | PBool | PFalse | PTrue | Int | Ints | NonExist | BIND
+                | PBool | PFalse | PTrue | Int | Ints | NonExist 
+                | BIND | SOFT_BIND
                 deriving (Show,Eq,Ord,Ix,Bounded,Enum)
diff --git a/src/compiler/GF/Compile/GeneratePMCFG.hs b/src/compiler/GF/Compile/GeneratePMCFG.hs
index 99df9af3e..457853150 100644
--- a/src/compiler/GF/Compile/GeneratePMCFG.hs
+++ b/src/compiler/GF/Compile/GeneratePMCFG.hs
@@ -414,6 +414,8 @@ convertTerm opts sel ctype (Q (m,f))
     f == cNonExist                      = return (CStr [SymNE])
   | m == cPredef &&
     f == cBIND                          = return (CStr [SymBIND])
+  | m == cPredef &&
+    f == cSOFT_BIND                     = return (CStr [SymSOFT_BIND])
 
 convertTerm opts sel@(CProj l _) ctype (ExtR t1 t2@(R rs2))
                     | l `elem` map fst rs2 = convertTerm opts sel ctype t2
diff --git a/src/compiler/GF/Compile/PGFtoJS.hs b/src/compiler/GF/Compile/PGFtoJS.hs
index 1730d62c4..707739025 100644
--- a/src/compiler/GF/Compile/PGFtoJS.hs
+++ b/src/compiler/GF/Compile/PGFtoJS.hs
@@ -89,6 +89,7 @@ sym2js (SymKS t)       = new "SymKS"  [JS.EStr t]
 sym2js (SymKP ts alts) = new "SymKP"  [JS.EArray (map sym2js ts), JS.EArray (map alt2js alts)]
 sym2js SymNE           = new "SymNE"  []
 sym2js SymBIND         = new "SymKS"  [JS.EStr "&+"]
+sym2js SymSOFT_BIND    = new "SymKS"  [JS.EStr "&+"]
 
 alt2js (ps,ts) = new "Alt" [JS.EArray (map sym2js ps), JS.EArray (map JS.EStr ts)]
 
diff --git a/src/compiler/GF/Grammar/Predef.hs b/src/compiler/GF/Grammar/Predef.hs
index b814dd110..8ffd17b74 100644
--- a/src/compiler/GF/Grammar/Predef.hs
+++ b/src/compiler/GF/Grammar/Predef.hs
@@ -31,6 +31,7 @@ cOverload = identS "overload"
 cUndefinedType = identS "UndefinedType"
 cNonExist = identS "nonExist"
 cBIND = identS "BIND"
+cSOFT_BIND = identS "SOFT_BIND"
 
 isPredefCat :: Ident -> Bool
 isPredefCat c = elem c [cInt,cString,cFloat]
diff --git a/src/runtime/c/pgf/data.h b/src/runtime/c/pgf/data.h
index 9bc73dd0a..131201132 100644
--- a/src/runtime/c/pgf/data.h
+++ b/src/runtime/c/pgf/data.h
@@ -167,6 +167,7 @@ typedef enum {
 	PGF_SYMBOL_KS,
 	PGF_SYMBOL_KP,
 	PGF_SYMBOL_BIND,
+	PGF_SYMBOL_SOFT_BIND,
 	PGF_SYMBOL_NE
 } PgfSymbolTag;
 
diff --git a/src/runtime/c/pgf/linearizer.c b/src/runtime/c/pgf/linearizer.c
index 698bab939..732bce75a 100644
--- a/src/runtime/c/pgf/linearizer.c
+++ b/src/runtime/c/pgf/linearizer.c
@@ -723,7 +723,8 @@ pgf_lzr_linearize_symbols(PgfConcr* concr, PgfCncTreeApp* fapp,
 			}
 			break;
 		}
-		case PGF_SYMBOL_BIND: {
+		case PGF_SYMBOL_BIND:
+		case PGF_SYMBOL_SOFT_BIND: {
 			if ((*fnsp)->symbol_bind) {
 				(*fnsp)->symbol_bind(fnsp);
 			}
diff --git a/src/runtime/c/pgf/parser.c b/src/runtime/c/pgf/parser.c
index 46cfe5614..66d77175d 100644
--- a/src/runtime/c/pgf/parser.c
+++ b/src/runtime/c/pgf/parser.c
@@ -145,6 +145,7 @@ pgf_prev_extern_sym(PgfSymbol sym)
 	case PGF_SYMBOL_VAR:
 		return *((PgfSymbol*) (((PgfSymbolVar*) i.data)+1));
 	case PGF_SYMBOL_BIND:
+	case PGF_SYMBOL_SOFT_BIND:
 		return *((PgfSymbol*) (((PgfSymbolBIND*) i.data)+1));
 	case PGF_SYMBOL_NE:
 		return *((PgfSymbol*) (((PgfSymbolNE*) i.data)+1));
@@ -1137,6 +1138,10 @@ pgf_symbols_cmp(GuString* psent, size_t sent_len, BIND_TYPE* pbind, PgfSymbols*
 			*pbind = BIND_HARD;
 			break;
 		}
+		case PGF_SYMBOL_SOFT_BIND: {
+			*pbind = BIND_SOFT;
+			break;
+		}
 		case PGF_SYMBOL_NE: {
 			return -2;
 		}
@@ -1635,6 +1640,31 @@ pgf_parsing_symbol(PgfParsing* ps, PgfItem* item, PgfSymbol sym)
 		}
 		break;
 	}
+	case PGF_SYMBOL_SOFT_BIND: {
+		if (ps->before->start_offset == ps->before->end_offset) {
+			if (ps->before->needs_bind) {
+				PgfParseState* state =
+					pgf_new_parse_state(ps, ps->before->end_offset, BIND_HARD);
+				if (state != NULL) {
+					if (state->next == NULL) {
+						state->viterbi_prob = 
+							item->inside_prob+item->conts->outside_prob;
+					}
+
+					pgf_item_advance(item, ps->pool);
+					gu_buf_heap_push(state->agenda, pgf_item_prob_order, &item);
+				} else {
+					pgf_item_free(ps, item);
+				}
+			} else {
+				pgf_item_free(ps, item);
+			}
+		} else {
+			pgf_item_advance(item, ps->pool);
+			gu_buf_heap_push(ps->before->agenda, pgf_item_prob_order, &item);
+		}
+		break;
+	}
 	default:
 		gu_impossible();
 	}
diff --git a/src/runtime/c/pgf/printer.c b/src/runtime/c/pgf/printer.c
index 78c2b74db..8b737266e 100644
--- a/src/runtime/c/pgf/printer.c
+++ b/src/runtime/c/pgf/printer.c
@@ -280,6 +280,10 @@ pgf_print_symbol(PgfSymbol sym, GuOut *out, GuExn *err)
 		gu_puts("BIND", out, err);
 		break;
 	}
+	case PGF_SYMBOL_SOFT_BIND: {
+		gu_puts("SOFT_BIND", out, err);
+		break;
+	}
 	default:
 		gu_impossible();
 	}
diff --git a/src/runtime/c/pgf/reader.c b/src/runtime/c/pgf/reader.c
index 477adc069..c40781e26 100644
--- a/src/runtime/c/pgf/reader.c
+++ b/src/runtime/c/pgf/reader.c
@@ -723,6 +723,13 @@ pgf_read_symbol(PgfReader* rdr)
 		gu_return_on_exn(rdr->err, gu_null_variant);
 		break;
 	}
+	case PGF_SYMBOL_SOFT_BIND: {
+		gu_new_variant(PGF_SYMBOL_SOFT_BIND,
+		               PgfSymbolBIND,
+		               &sym, rdr->opool);
+		gu_return_on_exn(rdr->err, gu_null_variant);
+		break;
+	}
 	default:
 		pgf_read_tag_error(rdr);
 	}
diff --git a/src/runtime/haskell/PGF/Binary.hs b/src/runtime/haskell/PGF/Binary.hs
index 2debcf12d..bf30e4506 100644
--- a/src/runtime/haskell/PGF/Binary.hs
+++ b/src/runtime/haskell/PGF/Binary.hs
@@ -152,7 +152,8 @@ instance Binary Symbol where
   put (SymKS ts)         = putWord8 3 >> put ts

   put (SymKP d vs)       = putWord8 4 >> put (d,vs)

   put SymBIND            = putWord8 5

-  put SymNE              = putWord8 6

+  put SymSOFT_BIND       = putWord8 6

+  put SymNE              = putWord8 7

   get = do tag <- getWord8

            case tag of

              0 -> liftM2 SymCat get get

@@ -161,7 +162,8 @@ instance Binary Symbol where
              3 -> liftM  SymKS  get

              4 -> liftM2 (\d vs -> SymKP d vs) get get

              5 -> return SymBIND

-             6 -> return SymNE

+             6 -> return SymSOFT_BIND

+             7 -> return SymNE

              _ -> decodingError

 

 instance Binary PArg where

diff --git a/src/runtime/haskell/PGF/Data.hs b/src/runtime/haskell/PGF/Data.hs
index f5797739f..3222867d2 100644
--- a/src/runtime/haskell/PGF/Data.hs
+++ b/src/runtime/haskell/PGF/Data.hs
@@ -62,6 +62,7 @@ data Symbol
   | SymKS Token
   | SymKP [Symbol] [([Symbol],[String])]
   | SymBIND                         -- the special BIND token
+  | SymSOFT_BIND                    -- the special SOFT_BIND token
   | SymNE                           -- non exist (this should be last constructor to simplify the binary search in the runtime)
   deriving (Eq,Ord,Show)
 data Production
diff --git a/src/runtime/haskell/PGF/Macros.hs b/src/runtime/haskell/PGF/Macros.hs
index ce75b1c91..0e73180d5 100644
--- a/src/runtime/haskell/PGF/Macros.hs
+++ b/src/runtime/haskell/PGF/Macros.hs
@@ -160,6 +160,7 @@ data BracketedTokn
   | LeafKS Token
   | LeafNE
   | LeafBIND
+  | LeafSOFT_BIND
   | LeafKP [BracketedTokn] [([BracketedTokn],[String])]
   deriving Eq
 
@@ -222,6 +223,7 @@ computeSeq filter seq args = concatMap compute seq
     compute (SymKS t)         = [LeafKS t]
     compute SymNE             = [LeafNE]
     compute SymBIND           = [LeafKS "&+"]
+    compute SymSOFT_BIND      = []
     compute (SymKP syms alts) = [LeafKP (concatMap compute syms) [(concatMap compute syms,cs) | (syms,cs) <- alts]]
 
     getArg d r
diff --git a/src/runtime/haskell/PGF/Optimize.hs b/src/runtime/haskell/PGF/Optimize.hs
index bfc12e097..ab831704f 100644
--- a/src/runtime/haskell/PGF/Optimize.hs
+++ b/src/runtime/haskell/PGF/Optimize.hs
@@ -228,6 +228,7 @@ splitLexicalRules cnc p_prods =
                                                   [seq2prefix (syms1 ++ syms) | (syms1,ps) <- alts])
         seq2prefix (SymNE           :syms) = TrieMap.empty
         seq2prefix (SymBIND         :syms) = TrieMap.fromList [wf ["&+"]]
+        seq2prefix (SymSOFT_BIND    :syms) = TrieMap.fromList [wf []]
 
 updateConcrete abs cnc = 
   let p_prods0      = filterProductions IntMap.empty IntSet.empty (productions cnc)
diff --git a/src/runtime/haskell/PGF/Parse.hs b/src/runtime/haskell/PGF/Parse.hs
index 9c69940ef..3f5dc313a 100644
--- a/src/runtime/haskell/PGF/Parse.hs
+++ b/src/runtime/haskell/PGF/Parse.hs
@@ -309,10 +309,12 @@ process flit ftok cnc (item@(Active j ppos funid seqid args key0):items) acc cha
       	SymNE      -> process flit ftok cnc items acc chart

       	SymBIND    -> let !acc' = ftok_ ["&+"] (Active j (ppos+1) funid seqid args key0) acc

       	              in process flit ftok cnc items acc' chart

+      	SymSOFT_BIND->process flit ftok cnc ((Active j (ppos+1) funid seqid args key0):items) acc chart

       	SymKP syms vars

-      	           -> let to_tok (SymKS t) = [t]

-      	                  to_tok SymBIND   = ["&+"]

-      	                  to_tok _         = []

+      	           -> let to_tok (SymKS t)    = [t]

+      	                  to_tok SymBIND      = ["&+"]

+      	                  to_tok SymSOFT_BIND = []

+      	                  to_tok _            = []

 

       	                  !acc' = foldl (\acc syms -> ftok_ (concatMap to_tok syms) (Active j (ppos+1) funid seqid args key0) acc) acc

                                         (syms:[syms' | (syms',_) <- vars])

diff --git a/src/runtime/haskell/PGF/Printer.hs b/src/runtime/haskell/PGF/Printer.hs
index d3a5ea1d9..4945667f4 100644
--- a/src/runtime/haskell/PGF/Printer.hs
+++ b/src/runtime/haskell/PGF/Printer.hs
@@ -92,6 +92,7 @@ ppSymbol (SymVar d r) = char '<' <> int d <> comma <> char '$' <> int r <> char
 ppSymbol (SymKS t)    = doubleQuotes (text t)
 ppSymbol SymNE        = text "nonExist"
 ppSymbol SymBIND      = text "BIND"
+ppSymbol SymSOFT_BIND = text "SOFT_BIND"
 ppSymbol (SymKP syms alts) = text "pre" <+> braces (hsep (punctuate semi (hsep (map ppSymbol syms) : map ppAlt alts)))
 
 ppAlt (syms,ps) = hsep (map ppSymbol syms) <+> char '/' <+> hsep (map (doubleQuotes . text) ps)