diff --git a/src/runtime/c/pgf/parser.cxx b/src/runtime/c/pgf/parser.cxx index 3e0d75426..603642add 100644 --- a/src/runtime/c/pgf/parser.cxx +++ b/src/runtime/c/pgf/parser.cxx @@ -1444,66 +1444,73 @@ void PgfParser::suspend(Cont *cont,Item *item,bool do_predict,ref auto &suspended = cont->suspended[value_i][lin_idx_i]; suspended.push_back(item); - size_t n_suspended = suspended.size(); - if (cont->ccat == NULL) { - if (n_suspended == 1) { - std::function)> f = - [this,item,cont](ref arg) { + if (suspended.size() == 1) { + std::function)> f = + [this,item,cont](ref arg) { - ref pitem = arg->items[0]; + ref pitem = arg->items[0]; - PgfSymbol sym = item->rule->syms[item->dot]; - auto sym_cat = ref::untagged(sym); - size_t *values1 = CLONE_VALUES(item->rule, &item->vars[0]); - size_t *values2 = CLONE_VALUES(pitem->rule, &pitem->vars[0]); - if (!instantiate(item->rule, values1, item->rule->args[sym_cat->d], - pitem->rule, values2, pitem->rule->res)) { - return; - } - if (!instantiate(item->rule, values1, ref::from_ptr(&sym_cat->r), - pitem->rule, values2, pitem->rule->lin_idx)) { - return; - } + PgfSymbol sym = item->rule->syms[item->dot]; + auto sym_cat = ref::untagged(sym); + size_t *values1 = CLONE_VALUES(item->rule, &item->vars[0]); + size_t *values2 = CLONE_VALUES(pitem->rule, &pitem->vars[0]); + if (!instantiate(item->rule, values1, item->rule->args[sym_cat->d], + pitem->rule, values2, pitem->rule->res)) { + return; + } + if (!instantiate(item->rule, values1, ref::from_ptr(&sym_cat->r), + pitem->rule, values2, pitem->rule->lin_idx)) { + return; + } - CCat *&arg_ccat = epsilons[arg->fid]; - if (arg_ccat == NULL) { - arg_ccat = new CCat; - arg_ccat->fid = arg->fid; - arg_ccat->epsilon = arg; - arg_ccat->cont = NULL; - arg_ccat->state = NULL; - arg_ccat->lin_idx = arg->lin_idx; - arg_ccat->value = arg->value; - arg_ccat->covered = true; - arg_ccat->viterbi_prob = arg->viterbi_prob; - } + CCat *&arg_ccat = epsilons[arg->fid]; + if (arg_ccat == NULL) { + arg_ccat = new CCat; + arg_ccat->fid = arg->fid; + arg_ccat->epsilon = arg; + arg_ccat->cont = NULL; + arg_ccat->state = NULL; + arg_ccat->lin_idx = arg->lin_idx; + arg_ccat->value = arg->value; + arg_ccat->covered = true; + arg_ccat->viterbi_prob = arg->viterbi_prob; + } - cont->state->completed[cont][arg_ccat->value][arg_ccat->lin_idx] = arg_ccat; - }; - epsilontable_iter(concr->epsilontable,cont->lincat,f); - } + cont->state->completed[cont][arg_ccat->value][arg_ccat->lin_idx] = arg_ccat; + }; - if (!cont->state->did_bu_predict) { - cont->state->did_bu_predict = true; - prob_t viterbi_prob = item->inside_prob+item->outside_prob; - if (cont->state->needs_bind) { - bu_predict(concr->phrasetable4, cont->state, viterbi_prob); - } else { - bu_predict(concr->phrasetable1, cont->state, viterbi_prob, 1, sentence->size); + if (cont->ccat == NULL) { + epsilontable_iter(concr->epsilontable,cont->lincat,0,f); + + if (!cont->state->did_bu_predict) { + cont->state->did_bu_predict = true; + prob_t viterbi_prob = item->inside_prob+item->outside_prob; + if (cont->state->needs_bind) { + bu_predict(concr->phrasetable4, cont->state, viterbi_prob); + } else { + bu_predict(concr->phrasetable1, cont->state, viterbi_prob, 1, sentence->size); + } } - } - } else { - if (do_predict && n_suspended == 1) { - if (cont->ccat->fid <= concr->last_fid) { - for (size_t i = 0; i < cont->ccat->epsilon->n_items; i++) { - ref pitem = cont->ccat->epsilon->items[i]; - td_epsilon(cont->state,cont,pitem,item,symcat); - } - } else { - for (Production *prod : cont->ccat->prods) { - td_predict(cont->state,cont,prod,item,symcat); + } else if (cont->ccat->fid <= concr->last_fid) { + epsilontable_iter(concr->epsilontable,cont->lincat,cont->ccat->fid,f); + + if (do_predict) { + size_t n_items; + phrasetable_lookup(concr->phrasetable3, + cont->ccat->epsilon, + &n_items); + + if (n_items == 0) { + for (size_t i = 0; i < cont->ccat->epsilon->n_items; i++) { + ref pitem = cont->ccat->epsilon->items[i]; + td_epsilon(cont->state,cont,pitem,item,symcat); + } } } + } else { + for (Production *prod : cont->ccat->prods) { + td_predict(cont->state,cont,prod,item,symcat); + } } } @@ -1838,6 +1845,7 @@ void PgfParseTableMaker::final_item(State *state, CCat *ccat, Item *item, interv epsilontable = epsilontable_insert(epsilontable, ccat->cont->lincat, + (ccat->cont->ccat != NULL) ? ccat->cont->ccat->fid : 0, ccat->value, ccat->lin_idx, ccat->fid, ccat->viterbi_prob, pitem, diff --git a/src/runtime/c/pgf/phrasetable.cxx b/src/runtime/c/pgf/phrasetable.cxx index 303f37fce..f4b2fd39a 100644 --- a/src/runtime/c/pgf/phrasetable.cxx +++ b/src/runtime/c/pgf/phrasetable.cxx @@ -576,7 +576,7 @@ vector> phrasetable_lookup(PgfPhrasetable phrasetabl PGF_INTERNAL PgfEpsilontable epsilontable_insert(PgfEpsilontable table, - ref lincat, + ref lincat, PgfMetaId prev_fid, interval_t value, interval_t lin_idx, PgfMetaId fid, prob_t viterbi_prob, ref item, @@ -587,6 +587,7 @@ PgfEpsilontable epsilontable_insert(PgfEpsilontable table, items[0] = item; PgfEpsilontable new_table = Node::new_node({.lincat=lincat, + .prev_fid=prev_fid, .fid=fid, .value=value, .lin_idx=lin_idx, @@ -604,12 +605,12 @@ PgfEpsilontable epsilontable_insert(PgfEpsilontable table, if (cmp < 0) { PgfEpsilontable left = epsilontable_insert(table->left, - lincat, value, lin_idx, fid, viterbi_prob, item, pepsilon); + lincat, prev_fid, value, lin_idx, fid, viterbi_prob, item, pepsilon); table = Node::upd_node(table,left,table->right); return Node::balanceL(table); } else if (cmp > 0) { PgfEpsilontable right = epsilontable_insert(table->right, - lincat, value, lin_idx, fid, viterbi_prob, item, pepsilon); + lincat, prev_fid, value, lin_idx, fid, viterbi_prob, item, pepsilon); table = Node::upd_node(table, table->left, right); return Node::balanceR(table); } else { @@ -665,20 +666,24 @@ ref epsilontable_get(PgfEpsilontable table, } PGF_INTERNAL -void epsilontable_iter(PgfEpsilontable table, ref lincat, std::function arg)> &f) +void epsilontable_iter(PgfEpsilontable table, + ref lincat, PgfMetaId prev_fid, + std::function arg)> &f) { if (table == 0) return; int cmp = textcmp(&lincat->name, &table->value.lincat->name); if (cmp < 0) - epsilontable_iter(table->left, lincat, f); + epsilontable_iter(table->left, lincat, prev_fid, f); else if (cmp > 0) - epsilontable_iter(table->right, lincat, f); + epsilontable_iter(table->right, lincat, prev_fid, f); else { - epsilontable_iter(table->left, lincat, f); - f(ref::from_ptr(&table->value)); - epsilontable_iter(table->right, lincat, f); + epsilontable_iter(table->left, lincat, prev_fid, f); + if (table->value.prev_fid == prev_fid) { + f(ref::from_ptr(&table->value)); + } + epsilontable_iter(table->right, lincat, prev_fid, f); } } diff --git a/src/runtime/c/pgf/phrasetable.h b/src/runtime/c/pgf/phrasetable.h index 727f3668c..1ca33ab33 100644 --- a/src/runtime/c/pgf/phrasetable.h +++ b/src/runtime/c/pgf/phrasetable.h @@ -50,7 +50,7 @@ struct PGF_INTERNAL_DECL PgfItem { struct PGF_INTERNAL_DECL PgfCCat { ref lincat; - PgfMetaId fid; + PgfMetaId prev_fid, fid; interval_t value, lin_idx; prob_t viterbi_prob; @@ -127,7 +127,7 @@ typedef ref> PgfEpsilontable; // The new category is mutable within the current transaction PGF_INTERNAL_DECL PgfEpsilontable epsilontable_insert(PgfEpsilontable table, - ref lincat, + ref lincat, PgfMetaId prev_fid, interval_t value, interval_t lin_idx, PgfMetaId fid, prob_t viterbi_prob, ref item, @@ -143,7 +143,9 @@ ref epsilontable_get(PgfEpsilontable table, PgfText *name, PgfMetaId fid); PGF_INTERNAL -void epsilontable_iter(PgfEpsilontable table, ref lincat, std::function arg)> &f); +void epsilontable_iter(PgfEpsilontable table, + ref lincat, PgfMetaId prev_fid, + std::function arg)> &f); PGF_INTERNAL_DECL void epsilontable_release(PgfEpsilontable table);