msyds/gf-core
1
0
Fork 0
forked from GitHub/gf-core
Code Pull Requests Activity

added Predef.SOFT_BIND. This special token allows zero or more spaces between ordinary tokens. It is also used in the English RGL to attach the commas to the previous word.

Browse Source
This commit is contained in:
kr.angelov
2013-11-12 09:54:57 +00:00
parent b111b476a7
commit 0095119ec0
17 changed files with 69 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/compiler/GF/Compile/Compute/AppPredefined.hs
View File

@@ -86,6 +86,8 @@ primitives = Map.fromList
[] typeStr []))) Nothing)
, (cBIND , ResOper (Just (noLoc (mkProd -- Str
[] typeStr []))) Nothing)
, (cSOFT_BIND, ResOper (Just (noLoc (mkProd -- Str
[] typeStr []))) Nothing)
]
where
fun from to = oper (mkFunType from to)

4
src/compiler/GF/Compile/Compute/Predef.hs
View File

@@ -78,7 +78,8 @@ predefList =
(cError,Error),
-- Canonical values:
(cPBool,PBool),(cPFalse,PFalse),(cPTrue,PTrue),(cInt,Int),
(cInts,Ints),(cNonExist,NonExist),(cBIND,BIND)]
(cInts,Ints),(cNonExist,NonExist)
,(cBIND,BIND),(cSOFT_BIND,SOFT_BIND)]
--- add more functions!!!
delta f vs =
@@ -107,6 +108,7 @@ delta f vs =
PTrue -> canonical
NonExist-> canonical
BIND -> canonical
SOFT_BIND->canonical
where
canonical = delay
delay = return (VApp f vs) -- wrong number of arguments

3
src/compiler/GF/Compile/Compute/Value.hs
View File

@@ -51,5 +51,6 @@ data Predefined = Drop | Take | Tk | Dp | EqStr | Occur | Occurs | ToUpper
{- | Show | Read | ToStr | MapStr | EqVal -}
| Error
-- Canonical values below:
| PBool | PFalse | PTrue | Int | Ints | NonExist | BIND
| PBool | PFalse | PTrue | Int | Ints | NonExist
| BIND | SOFT_BIND
deriving (Show,Eq,Ord,Ix,Bounded,Enum)

2
src/compiler/GF/Compile/GeneratePMCFG.hs
View File

@@ -414,6 +414,8 @@ convertTerm opts sel ctype (Q (m,f))
f == cNonExist = return (CStr [SymNE])
| m == cPredef &&
f == cBIND = return (CStr [SymBIND])
| m == cPredef &&
f == cSOFT_BIND = return (CStr [SymSOFT_BIND])
convertTerm opts sel@(CProj l _) ctype (ExtR t1 t2@(R rs2))
| l `elem` map fst rs2 = convertTerm opts sel ctype t2

1
src/compiler/GF/Compile/PGFtoJS.hs
View File

@@ -89,6 +89,7 @@ sym2js (SymKS t) = new "SymKS" [JS.EStr t]
sym2js (SymKP ts alts) = new "SymKP" [JS.EArray (map sym2js ts), JS.EArray (map alt2js alts)]
sym2js SymNE = new "SymNE" []
sym2js SymBIND = new "SymKS" [JS.EStr "&+"]
sym2js SymSOFT_BIND = new "SymKS" [JS.EStr "&+"]
alt2js (ps,ts) = new "Alt" [JS.EArray (map sym2js ps), JS.EArray (map JS.EStr ts)]

1
src/compiler/GF/Grammar/Predef.hs
View File

@@ -31,6 +31,7 @@ cOverload = identS "overload"
cUndefinedType = identS "UndefinedType"
cNonExist = identS "nonExist"
cBIND = identS "BIND"
cSOFT_BIND = identS "SOFT_BIND"
isPredefCat :: Ident -> Bool
isPredefCat c = elem c [cInt,cString,cFloat]

1
src/runtime/c/pgf/data.h
View File

@@ -167,6 +167,7 @@ typedef enum {
PGF_SYMBOL_KS,
PGF_SYMBOL_KP,
PGF_SYMBOL_BIND,
PGF_SYMBOL_SOFT_BIND,
PGF_SYMBOL_NE
} PgfSymbolTag;

3
src/runtime/c/pgf/linearizer.c
View File

@@ -723,7 +723,8 @@ pgf_lzr_linearize_symbols(PgfConcr* concr, PgfCncTreeApp* fapp,
}
break;
}
case PGF_SYMBOL_BIND: {
case PGF_SYMBOL_BIND:
case PGF_SYMBOL_SOFT_BIND: {
if ((*fnsp)->symbol_bind) {
(*fnsp)->symbol_bind(fnsp);
}

30
src/runtime/c/pgf/parser.c
View File

@@ -145,6 +145,7 @@ pgf_prev_extern_sym(PgfSymbol sym)
case PGF_SYMBOL_VAR:
return *((PgfSymbol*) (((PgfSymbolVar*) i.data)+1));
case PGF_SYMBOL_BIND:
case PGF_SYMBOL_SOFT_BIND:
return *((PgfSymbol*) (((PgfSymbolBIND*) i.data)+1));
case PGF_SYMBOL_NE:
return *((PgfSymbol*) (((PgfSymbolNE*) i.data)+1));
@@ -1137,6 +1138,10 @@ pgf_symbols_cmp(GuString* psent, size_t sent_len, BIND_TYPE* pbind, PgfSymbols*
*pbind = BIND_HARD;
break;
}
case PGF_SYMBOL_SOFT_BIND: {
*pbind = BIND_SOFT;
break;
}
case PGF_SYMBOL_NE: {
return -2;
}
@@ -1635,6 +1640,31 @@ pgf_parsing_symbol(PgfParsing* ps, PgfItem* item, PgfSymbol sym)
}
break;
}
case PGF_SYMBOL_SOFT_BIND: {
if (ps->before->start_offset == ps->before->end_offset) {
if (ps->before->needs_bind) {
PgfParseState* state =
pgf_new_parse_state(ps, ps->before->end_offset, BIND_HARD);
if (state != NULL) {
if (state->next == NULL) {
state->viterbi_prob =
item->inside_prob+item->conts->outside_prob;
}
pgf_item_advance(item, ps->pool);
gu_buf_heap_push(state->agenda, pgf_item_prob_order, &item);
} else {
pgf_item_free(ps, item);
}
} else {
pgf_item_free(ps, item);
}
} else {
pgf_item_advance(item, ps->pool);
gu_buf_heap_push(ps->before->agenda, pgf_item_prob_order, &item);
}
break;
}
default:
gu_impossible();
}

4
src/runtime/c/pgf/printer.c
View File

@@ -280,6 +280,10 @@ pgf_print_symbol(PgfSymbol sym, GuOut *out, GuExn *err)
gu_puts("BIND", out, err);
break;
}
case PGF_SYMBOL_SOFT_BIND: {
gu_puts("SOFT_BIND", out, err);
break;
}
default:
gu_impossible();
}

7
src/runtime/c/pgf/reader.c
View File

@@ -723,6 +723,13 @@ pgf_read_symbol(PgfReader* rdr)
gu_return_on_exn(rdr->err, gu_null_variant);
break;
}
case PGF_SYMBOL_SOFT_BIND: {
gu_new_variant(PGF_SYMBOL_SOFT_BIND,
PgfSymbolBIND,
&sym, rdr->opool);
gu_return_on_exn(rdr->err, gu_null_variant);
break;
}
default:
pgf_read_tag_error(rdr);
}

6
src/runtime/haskell/PGF/Binary.hs
View File

@@ -152,7 +152,8 @@ instance Binary Symbol where
put (SymKS ts) = putWord8 3 >> put ts
put (SymKP d vs) = putWord8 4 >> put (d,vs)
put SymBIND = putWord8 5
put SymNE = putWord8 6
put SymSOFT_BIND = putWord8 6
put SymNE = putWord8 7
get = do tag <- getWord8
case tag of
0 -> liftM2 SymCat get get
@@ -161,7 +162,8 @@ instance Binary Symbol where
3 -> liftM SymKS get
4 -> liftM2 (\d vs -> SymKP d vs) get get
5 -> return SymBIND
6 -> return SymNE
6 -> return SymSOFT_BIND
7 -> return SymNE
_ -> decodingError
instance Binary PArg where

1
src/runtime/haskell/PGF/Data.hs
View File

@@ -62,6 +62,7 @@ data Symbol
| SymKS Token
| SymKP [Symbol] [([Symbol],[String])]
| SymBIND -- the special BIND token
| SymSOFT_BIND -- the special SOFT_BIND token
| SymNE -- non exist (this should be last constructor to simplify the binary search in the runtime)
deriving (Eq,Ord,Show)
data Production

2
src/runtime/haskell/PGF/Macros.hs
View File

@@ -160,6 +160,7 @@ data BracketedTokn
| LeafKS Token
| LeafNE
| LeafBIND
| LeafSOFT_BIND
| LeafKP [BracketedTokn] [([BracketedTokn],[String])]
deriving Eq
@@ -222,6 +223,7 @@ computeSeq filter seq args = concatMap compute seq
compute (SymKS t) = [LeafKS t]
compute SymNE = [LeafNE]
compute SymBIND = [LeafKS "&+"]
compute SymSOFT_BIND = []
compute (SymKP syms alts) = [LeafKP (concatMap compute syms) [(concatMap compute syms,cs) | (syms,cs) <- alts]]
getArg d r

1
src/runtime/haskell/PGF/Optimize.hs
View File

@@ -228,6 +228,7 @@ splitLexicalRules cnc p_prods =
[seq2prefix (syms1 ++ syms) | (syms1,ps) <- alts])
seq2prefix (SymNE :syms) = TrieMap.empty
seq2prefix (SymBIND :syms) = TrieMap.fromList [wf ["&+"]]
seq2prefix (SymSOFT_BIND :syms) = TrieMap.fromList [wf []]
updateConcrete abs cnc =
let p_prods0 = filterProductions IntMap.empty IntSet.empty (productions cnc)

8
src/runtime/haskell/PGF/Parse.hs
View File

@@ -309,10 +309,12 @@ process flit ftok cnc (item@(Active j ppos funid seqid args key0):items) acc cha
SymNE -> process flit ftok cnc items acc chart
SymBIND -> let !acc' = ftok_ ["&+"] (Active j (ppos+1) funid seqid args key0) acc
in process flit ftok cnc items acc' chart
SymSOFT_BIND->process flit ftok cnc ((Active j (ppos+1) funid seqid args key0):items) acc chart
SymKP syms vars
-> let to_tok (SymKS t) = [t]
to_tok SymBIND = ["&+"]
to_tok _ = []
-> let to_tok (SymKS t) = [t]
to_tok SymBIND = ["&+"]
to_tok SymSOFT_BIND = []
to_tok _ = []
!acc' = foldl (\acc syms -> ftok_ (concatMap to_tok syms) (Active j (ppos+1) funid seqid args key0) acc) acc
(syms:[syms' | (syms',_) <- vars])

1
src/runtime/haskell/PGF/Printer.hs
View File

@@ -92,6 +92,7 @@ ppSymbol (SymVar d r) = char '<' <> int d <> comma <> char '$' <> int r <> char
ppSymbol (SymKS t) = doubleQuotes (text t)
ppSymbol SymNE = text "nonExist"
ppSymbol SymBIND = text "BIND"
ppSymbol SymSOFT_BIND = text "SOFT_BIND"
ppSymbol (SymKP syms alts) = text "pre" <+> braces (hsep (punctuate semi (hsep (map ppSymbol syms) : map ppAlt alts)))
ppAlt (syms,ps) = hsep (map ppSymbol syms) <+> char '/' <+> hsep (map (doubleQuotes . text) ps)