From c41b75a9cb8566f11aaca5b6b42b9694d8a209f9 Mon Sep 17 00:00:00 2001 From: Krasimir Angelov Date: Wed, 12 Aug 2026 10:31:03 +0200 Subject: [PATCH] split the phrasetable to five parts and better serve the parser --- src/runtime/c/pgf/data.cxx | 6 +- src/runtime/c/pgf/data.h | 20 +- src/runtime/c/pgf/parser.cxx | 532 ++++++++++++++++-------------- src/runtime/c/pgf/parser.h | 25 +- src/runtime/c/pgf/pgf.cxx | 16 +- src/runtime/c/pgf/phrasetable.cxx | 518 ++++++++++------------------- src/runtime/c/pgf/phrasetable.h | 96 ++++-- src/runtime/c/pgf/printer.cxx | 14 - src/runtime/c/pgf/reader.cxx | 6 +- 9 files changed, 571 insertions(+), 662 deletions(-) diff --git a/src/runtime/c/pgf/data.cxx b/src/runtime/c/pgf/data.cxx index fda0f80b1..5c7492bd5 100644 --- a/src/runtime/c/pgf/data.cxx +++ b/src/runtime/c/pgf/data.cxx @@ -41,7 +41,11 @@ void PgfConcr::release(ref concr) namespace_release(concr->lins); namespace_release(concr->lincats); namespace_release(concr->printnames); - phrasetable_release(concr->phrasetable); + phrasetable_release(concr->phrasetable1); + phrasetable_release(concr->phrasetable2); + phrasetable_release(concr->phrasetable3); + phrasetable_release(concr->phrasetable4); + epsilontable_release(concr->epsilontable); PgfDB::free(concr, concr->name.size+1); } diff --git a/src/runtime/c/pgf/data.h b/src/runtime/c/pgf/data.h index cdb6b68d0..f3870fefb 100644 --- a/src/runtime/c/pgf/data.h +++ b/src/runtime/c/pgf/data.h @@ -252,20 +252,6 @@ struct PGF_INTERNAL_DECL PgfConcrLin { static void release(ref lin); }; -struct PGF_INTERNAL_DECL PgfSymbolACat { - static const uint8_t tag = 11; - PgfText name; -}; - -struct PGF_INTERNAL_DECL PgfSymbolCCat { - static const uint8_t tag = 12; - ref lincat; - interval_t value; - interval_t lin_idx; - prob_t viterbi_prob; - PgfMetaId fid; -}; - struct PGF_INTERNAL_DECL PgfConcrPrintname { ref printname; PgfText name; @@ -281,7 +267,11 @@ struct PGF_INTERNAL_DECL PgfConcr { Namespace cflags; Namespace lins; Namespace lincats; - PgfPhrasetable phrasetable; + PgfPhrasetable phrasetable1; // suspended on token + PgfPhrasetable phrasetable2; // suspended on lincat + PgfPhrasetable phrasetable3; // suspended on ccat + PgfPhrasetable phrasetable4; // suspended on bind + PgfEpsilontable epsilontable; Namespace printnames; PgfMetaId last_fid; diff --git a/src/runtime/c/pgf/parser.cxx b/src/runtime/c/pgf/parser.cxx index e4b12f036..f333b3f3e 100644 --- a/src/runtime/c/pgf/parser.cxx +++ b/src/runtime/c/pgf/parser.cxx @@ -11,26 +11,44 @@ PgfAbstractParser::PgfAbstractParser(ref concr) this->concr = concr; this->current_state = NULL; - this->initial_fid = concr->last_fid; this->last_fid = concr->last_fid; } void PgfAbstractParser::get_info(CCat *ccat, ref *prule, size_t **pvalues) { - if (ccat->fid > initial_fid) { + if (ccat->epsilon == 0) { Production *prod = ccat->prods[0]; *prule = prod->rule; *pvalues = &prod->vars[0]; } else { - size_t n_items; - vector> items = - phrasetable_lookup(concr->phrasetable, ccat->epsilons, &n_items); - ref pitem = items[0]; + ref pitem = ccat->epsilon->items[0]; *prule = pitem->rule; *pvalues = &pitem->vars[0]; } } +PgfAbstractParser::CCat *PgfAbstractParser::get_epsilon_ccat(PgfText *name, PgfMetaId fid) +{ + if (fid == 0) + return NULL; + + CCat *&ccat = epsilons[fid]; + if (ccat == NULL) { + ref arg = epsilontable_get(concr->epsilontable, + name, fid); + ccat = new CCat; + ccat->fid = arg->fid; + ccat->epsilon = arg; + ccat->cont = NULL; + ccat->state = NULL; + ccat->lin_idx = arg->lin_idx; + ccat->value = arg->value; + ccat->covered = true; + ccat->viterbi_prob = arg->viterbi_prob; + } + return ccat; +} + PgfAbstractParser::CCat::~CCat() { for (Production *prod : prods) { @@ -97,7 +115,7 @@ void PgfAbstractParser::process(Item *item, State *state) PGF_INTERNAL_DECL int text_symbol_cmp(PgfTextSpot *spot, const uint8_t *end, - PgfSymbol sym, bool case_sensitive); + ref sym, bool case_sensitive); void PgfAbstractParser::symbol(Item *item, State *state, PgfSymbol sym) { @@ -137,15 +155,16 @@ void PgfAbstractParser::symbol(Item *item, State *state, PgfSymbol sym) auto &suspended = cont->suspended[value_i][lin_idx_i]; suspended.push_back(item); - suspend(cont,item,n_suspended1,suspended.size()); + suspend(cont,item,n_suspended1 == 0,suspended.size(),symcat); } } else { interval_t value_i = item->interval(item->rule->args[symcat->d]); interval_t lin_idx_i = item->interval(ref::from_ptr(&symcat->r)); + // the following prevents infinite loops with epsilons bool found = false; CCat *prev_ccat = ccat; - while (prev_ccat != NULL && prev_ccat->fid > initial_fid && prev_ccat->cont->state == state) { + while (prev_ccat != NULL && prev_ccat->epsilon == 0 && prev_ccat->cont->state == state) { if (prev_ccat->value == value_i && prev_ccat->lin_idx == lin_idx_i) { found = true; break; @@ -162,8 +181,8 @@ void PgfAbstractParser::symbol(Item *item, State *state, PgfSymbol sym) if (cont == NULL) { cont = new Cont; cont->ccat = ccat; - if (ccat->fid <= initial_fid) - cont->lincat = ref::untagged(ccat->epsilons)->lincat; + if (ccat->epsilon != 0) + cont->lincat = ccat->epsilon->lincat; else cont->lincat = ccat->cont->lincat; cont->state = state; @@ -185,44 +204,13 @@ found:; auto &suspended = cont->suspended[value_i][lin_idx_i]; suspended.push_back(item); - if (!subsumed && suspended.size() == 1) { - if (ccat->fid <= initial_fid) { - size_t n_items = 0; - vector> items = - phrasetable_lookup(concr->phrasetable, ccat->epsilons, &n_items); - - for (size_t i = 0; i < n_items; i++) { - ref pitem = items[i]; - td_epsilon(state,cont,pitem,item,symcat); - } - } else { - for (Production *prod : ccat->prods) { - td_predict(state,cont,prod,item,symcat); - } - } - } else { - State *next = state; - while (next != NULL) { - auto it1 = next->completed.find(cont); - if (it1 != next->completed.end()) { - auto *it2 = it1->second.lookup(ccat->value); - if (it2 != NULL) { - auto *it3 = it2->lookup(lin_idx_i); - if (it3 != NULL) { - CCat *arg = *it3; - Item *new_item = new (item) Item; - combine(next, new_item, arg); - } - } - } - next = next->next; - } - } + suspend(cont,item,!subsumed,suspended.size(),symcat); } break; } case PgfSymbolKS::tag: { - symbol_token(item, state, sym); + auto symks = ref::untagged(sym); + symbol_token(item, state, symks); break; } case PgfSymbolKP::tag: { @@ -282,6 +270,7 @@ void PgfAbstractParser::complete(Item *item, State *state) if (ccat == NULL) { ccat = new CCat; ccat->fid = (++last_fid); + ccat->epsilon = 0; ccat->cont = item->cont; ccat->state = state; ccat->lin_idx = lin_idx; @@ -331,8 +320,7 @@ void PgfAbstractParser::complete(Item *item, State *state) final_item(state, ccat, item, res, lin_idx); if (ccat->prods.size() == 1) { - if (ccat->cont->ccat == NULL) - bu_predict(state, ccat); + bu_predict(state, ccat); for (auto it1 : ccat->cont->suspended.overlaps(ccat->value)) { for (auto it2 : it1.second.overlaps(ccat->lin_idx)) { @@ -552,22 +540,9 @@ void PgfAbstractParser::td_epsilon(State *state, Cont *cont, ref pitem, } for (size_t i = 0; i < pitem->args.size(); i++) { - ref arg = pitem->args[i]; - - if (arg != 0) { - CCat *&arg_ccat = epsilons[arg->lincat][arg->value][arg->lin_idx]; - if (arg_ccat == NULL) { - arg_ccat = new CCat; - arg_ccat->fid = arg->fid; - arg_ccat->epsilons = arg.tagged(); - arg_ccat->state = NULL; - arg_ccat->lin_idx = arg->lin_idx; - arg_ccat->value = arg->value; - arg_ccat->covered = true; - arg_ccat->viterbi_prob = arg->viterbi_prob; - } - item->args[i] = arg_ccat; - item->inside_prob += arg_ccat->viterbi_prob; + if (pitem->args[i] != 0) { + item->args[i] = get_epsilon_ccat(&lin->absfun->type->hypos[i].type->name,pitem->args[i]); + item->inside_prob += item->args[i]->viterbi_prob; } if (!item->instantiate(item->rule->args[i], pitem->rule, &pitem->vars[0], pitem->rule->args[i])) { @@ -676,7 +651,7 @@ void PgfAbstractParser::print_item(Item *item, State *state) { PgfPrinter printer(NULL,0,NULL); - printer.nprintf(32, "[%zd-%zd; ", item->cont ? item->cont->state->end.pos : 0, state->start.pos); + printer.nprintf(32, "[%zd-%zd; ", (item->cont && item->cont->state) ? item->cont->state->end.pos : 0, state->start.pos); if (item->vars.size() > 0) { printer.lvar_ranges(item->rule->ranges, &item->vars[0]); @@ -832,7 +807,7 @@ PgfParser::~PgfParser() for (auto it1 : state->completed) { for (auto it2 : it1.second) { for (auto it3 : it2.second) { - if (it3.second->fid <= initial_fid) + if (it3.second->epsilon != 0) continue; for (ExprState *estate : it3.second->pending) { @@ -850,21 +825,17 @@ PgfParser::~PgfParser() } for (auto it1 : epsilons) { - for (auto it2 : it1.second) { - for (auto it3 : it2.second) { - for (ExprState *estate : it3.second->pending) { - if (estate->expr != 0) - u->free_ref(estate->expr); - } - for (ExprProb &ep : it3.second->exprs) { - u->free_ref(ep.expr); - } - } + for (ExprState *estate : it1.second->pending) { + if (estate->expr != 0) + u->free_ref(estate->expr); + } + for (ExprProb &ep : it1.second->exprs) { + u->free_ref(ep.expr); } } } -void PgfParser::bu_predict(PgfPhrasetable phrasetable, +void PgfParser::bu_predict(PgfPhrasetable phrasetable, State *state, ptrdiff_t min, ptrdiff_t max) { @@ -872,7 +843,7 @@ void PgfParser::bu_predict(PgfPhrasetable phrasetable, return; PgfTextSpot current = state->end; - int cmp = text_symbol_cmp(¤t,end,phrasetable->value.sym,case_sensitive); + int cmp = text_symbol_cmp(¤t,end,phrasetable->value.key,case_sensitive); if (cmp < 0) { bu_predict(phrasetable->left,state,min,max); } else if (cmp > 0) { @@ -906,51 +877,51 @@ void PgfParser::bu_predict(PgfPhrasetable phrasetable, } } -void PgfParser::bu_predict(PgfPhrasetable phrasetable, +void PgfParser::bu_predict(PgfPhrasetable phrasetable, State *state) { - if (phrasetable == 0) - return; + size_t n_items = 0; + vector> items = + phrasetable_lookup(concr->phrasetable4, + ref(0), + &n_items); - PgfTextSpot current = state->end; - int cmp; - uint8_t tag = ref::get_tag(phrasetable->value.sym); - cmp = ((int) PgfSymbolBIND::tag) - ((int) tag); - if (cmp < 0) { - bu_predict(phrasetable->left,state); - } else if (cmp > 0) { - bu_predict(phrasetable->right,state); - } else { - State *next_state = state->next; - if (next_state == NULL || state->end.pos != next_state->start.pos) { - next_state = new State; - next_state->start = state->end; - next_state->end = state->end; - next_state->next = state->next; - next_state->needs_bind = false; - state->next = next_state; - } + State *next_state = state->next; + if (next_state == NULL || state->end.pos != next_state->start.pos) { + next_state = new State; + next_state->start = state->end; + next_state->end = state->end; + next_state->next = state->next; + next_state->needs_bind = false; + state->next = next_state; + } - for (size_t i = 0; i < phrasetable->value.n_items; i++) { - std::map, bool> visited; - //if (!td_reachable(state, phrasetable->items[i], visited)) - // continue; - Item *item = bu_item(state, phrasetable->value.items[i]); - item->dot++; - next_state->push_item(item); - } + for (size_t i = 0; i < n_items; i++) { + //std::map, bool> visited; + //if (!td_reachable(state, phrasetable->items[i], visited)) + // continue; + Item *item = bu_item(state, items[i]); + item->dot++; + next_state->push_item(item); } } void PgfParser::bu_predict(State *state, CCat *ccat) { size_t n_items = 0; - vector> items = - phrasetable_lookup(concr->phrasetable, - ccat->cont->lincat, - &n_items); + vector> items = 0; + if (ccat->cont->ccat == NULL) { + items = phrasetable_lookup(concr->phrasetable2, + ccat->cont->lincat, + &n_items); + } else if (ccat->cont->ccat->epsilon != 0) { + items = phrasetable_lookup(concr->phrasetable3, + ccat->cont->ccat->epsilon, + &n_items); + } + for (size_t i = 0; i < n_items; i++) { - std::map, bool> visited; + //std::map, bool> visited; //if (!td_reachable(ccat->cont->state, items[i], visited)) // continue; auto new_item = bu_item(ccat->cont->state, items[i]); @@ -976,7 +947,7 @@ bool PgfParser::td_reachable(State *state, ref pitem, size_t n_items = 0; vector> items = - phrasetable_lookup(concr->phrasetable, + phrasetable_lookup(concr->phrasetable2, lin->lincat, &n_items); for (size_t i = 0; i < n_items; i++) { @@ -997,12 +968,26 @@ PgfAbstractParser::Item *PgfParser::bu_item(State *state, ref pitem) case PgfConcrLin::tag: { auto lin = ref::untagged(pitem->rule->container); - Cont *&cont = state->conts1[lin->lincat]; - if (cont == NULL) { - cont = new Cont; - cont->ccat = NULL; - cont->lincat = lin->lincat; - cont->state = state; + Cont *cont; + if (pitem->res == 0) { + Cont *&tmp = state->conts1[lin->lincat]; + if (tmp == NULL) { + tmp = new Cont; + tmp->ccat = NULL; + tmp->lincat = lin->lincat; + tmp->state = state; + } + cont = tmp; + } else { + CCat *ccat = get_epsilon_ccat(&lin->lincat->name,pitem->res); + Cont *&tmp = state->conts2[ccat]; + if (tmp == NULL) { + tmp = new Cont; + tmp->ccat = ccat; + tmp->lincat = lin->lincat; + tmp->state = state; + } + cont = tmp; } item = new(pitem->rule) Item; @@ -1014,6 +999,14 @@ PgfAbstractParser::Item *PgfParser::bu_item(State *state, ref pitem) item->rule = pitem->rule; item->inside_prob = lin->absfun->prob; item->outside_prob = 0; + + for (size_t i = 0; i < pitem->args.size(); i++) { + item->args[i] = 0; + if (pitem->args[i] != 0) { + item->args[i] = get_epsilon_ccat(&lin->absfun->type->hypos[i].type->name,pitem->args[i]); + item->inside_prob += item->args[i]->viterbi_prob; + } + } break; } case PgfConcrLincat::tag: { @@ -1036,6 +1029,7 @@ PgfAbstractParser::Item *PgfParser::bu_item(State *state, ref pitem) item->rule = pitem->rule; item->inside_prob = 0; item->outside_prob = 0; + item->args[0] = 0; break; } } @@ -1050,29 +1044,6 @@ PgfAbstractParser::Item *PgfParser::bu_item(State *state, ref pitem) } memcpy(&item->vars[0], &pitem->vars[0], sizeof(size_t) * item->vars.size()); - - for (size_t i = 0; i < pitem->args.size(); i++) { - ref arg = pitem->args[i]; - - item->args[i] = 0; - - if (arg != 0) { - CCat *&arg_ccat = epsilons[arg->lincat][arg->value][arg->lin_idx]; - if (arg_ccat == NULL) { - arg_ccat = new CCat; - arg_ccat->fid = arg->fid; - arg_ccat->epsilons = arg.tagged(); - arg_ccat->state = NULL; - arg_ccat->lin_idx = arg->lin_idx; - arg_ccat->value = arg->value; - arg_ccat->covered = true; - arg_ccat->viterbi_prob = arg->viterbi_prob; - } - item->args[i] = arg_ccat; - item->inside_prob += arg_ccat->viterbi_prob; - } - } - return item; } @@ -1136,7 +1107,7 @@ PgfExpr PgfParser::fetch(PgfDB *db, prob_t *prob) { DB_scope scope(db, READER_SCOPE); - bool first_fetch = (initial_fid == last_fid); + bool first_fetch = (concr->last_fid == last_fid); for (;;) { State *state = current_state; @@ -1225,13 +1196,9 @@ PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob) ccat->pending.push_back(estate); if (ccat->pending.size() == 1) { - if (ccat->fid <= initial_fid) { - size_t n_items = 0; - vector> items = - phrasetable_lookup(concr->phrasetable, ccat->epsilons, &n_items); - - for (size_t i = 0; i < n_items; i++) { - ref pitem = items[i]; + if (ccat->epsilon != 0) { + for (size_t i = 0; i < ccat->epsilon->n_items; i++) { + ref pitem = ccat->epsilon->items[i]; auto lin = ref::untagged(pitem->rule->container); @@ -1246,22 +1213,10 @@ PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob) new_estate->hash = new_estate->hash * 101 + lin->name.text[i]; } for (size_t i = 0; i < new_estate->n_args; i++) { - ref arg = pitem->args[i]; new_estate->args[i] = NULL; - if (arg != 0) { - CCat *&arg_ccat = epsilons[arg->lincat][arg->value][arg->lin_idx]; - if (arg_ccat == NULL) { - arg_ccat = new CCat; - arg_ccat->fid = arg->fid; - arg_ccat->epsilons = arg.tagged(); - arg_ccat->state = NULL; - arg_ccat->lin_idx = arg->lin_idx; - arg_ccat->value = arg->value; - arg_ccat->covered = true; - arg_ccat->viterbi_prob = arg->viterbi_prob; - } - new_estate->args[i] = arg_ccat; - new_estate->prob += arg_ccat->viterbi_prob; + if (pitem->args[i] != 0) { + new_estate->args[i] = get_epsilon_ccat(&lin->absfun->type->hypos[i].type->name,pitem->args[i]); + new_estate->prob += new_estate->args[i]->viterbi_prob; } } queue.push_back(new_estate); @@ -1373,10 +1328,10 @@ PgfAbstractParser::State *PgfParser::new_state(const PgfTextSpot &start) return state; } -void PgfParser::symbol_token(Item *item, State *state, PgfSymbol sym) +void PgfParser::symbol_token(Item *item, State *state, ref symks) { PgfTextSpot next = state->end; - if (text_symbol_cmp(&next,end,sym,case_sensitive) != 0) + if (text_symbol_cmp(&next,end,symks,case_sensitive) != 0) return; State *next_state = new_state(next); @@ -1409,68 +1364,101 @@ void PgfParser::symbol_bind(Item *item, State *state, PgfSymbol sym) } } -void PgfParser::suspend(Cont *cont,Item *item,size_t n_suspended1,size_t n_suspended) +void PgfParser::suspend(Cont *cont,Item *item,bool do_predict,size_t n_suspended,ref symcat) { - if (n_suspended == 1) { - std::function,size_t,vector>)> f = - [this,item,cont](ref symcf, size_t n_items, vector> items) { + if (cont->ccat == NULL) { + if (n_suspended == 1) { + std::function)> f = + [this,item,cont](ref arg) { - ref xitem = items[0]; + ref xitem = arg->items[0]; - Item *new_item = new (item) Item; - PgfSymbol sym = new_item->rule->syms[new_item->dot]; - auto sym_cat = ref::untagged(sym); - if (!new_item->instantiate(new_item->rule->args[sym_cat->d],xitem->rule,&xitem->vars[0],xitem->rule->res)) { - delete new_item; - return; - } - if (!new_item->instantiate(ref::from_ptr(&sym_cat->r),xitem->rule,&xitem->vars[0],xitem->rule->lin_idx)) { - delete new_item; - return; - } - - CCat *&arg_ccat = epsilons[symcf->lincat][symcf->value][symcf->lin_idx]; - if (arg_ccat == NULL) { - arg_ccat = new CCat; - arg_ccat->fid = symcf->fid; - arg_ccat->epsilons = symcf.tagged(); - arg_ccat->state = NULL; - arg_ccat->lin_idx = symcf->lin_idx; - arg_ccat->value = symcf->value; - arg_ccat->covered = true; - arg_ccat->viterbi_prob = symcf->viterbi_prob; - } - - cont->state->completed[cont][symcf->value][symcf->lin_idx] = arg_ccat; - - new_item->dot++; - new_item->args[sym_cat->d] = arg_ccat; - new_item->inside_prob += arg_ccat->viterbi_prob; - - cont->state->push_item(new_item); - }; - phrasetable_iter(concr->phrasetable,cont->lincat,f); - } - - State *state = cont->state; - while (state != NULL) { - auto it1 = state->completed.find(cont); - if (it1 != state->completed.end()) { - for (auto it2 : it1->second) { - for (auto it3 : it2.second) { Item *new_item = new (item) Item; - combine(state, new_item, it3.second); + PgfSymbol sym = new_item->rule->syms[new_item->dot]; + auto sym_cat = ref::untagged(sym); + if (!new_item->instantiate(new_item->rule->args[sym_cat->d],xitem->rule,&xitem->vars[0],xitem->rule->res)) { + delete new_item; + return; + } + if (!new_item->instantiate(ref::from_ptr(&sym_cat->r),xitem->rule,&xitem->vars[0],xitem->rule->lin_idx)) { + delete new_item; + return; + } + + CCat *&arg_ccat = epsilons[arg->fid]; + if (arg_ccat == NULL) { + arg_ccat = new CCat; + arg_ccat->fid = arg->fid; + arg_ccat->epsilon = arg; + arg_ccat->state = NULL; + arg_ccat->lin_idx = arg->lin_idx; + arg_ccat->value = arg->value; + arg_ccat->covered = true; + arg_ccat->viterbi_prob = arg->viterbi_prob; + } + + cont->state->completed[cont][arg_ccat->value][arg_ccat->lin_idx] = arg_ccat; + + new_item->dot++; + new_item->args[sym_cat->d] = arg_ccat; + new_item->inside_prob += arg_ccat->viterbi_prob; + + cont->state->push_item(new_item); + }; + epsilontable_iter(concr->epsilontable,cont->lincat,f); + } + + State *state = cont->state; + while (state != NULL) { + auto it1 = state->completed.find(cont); + if (it1 != state->completed.end()) { + for (auto it2 : it1->second) { + for (auto it3 : it2.second) { + Item *new_item = new (item) Item; + combine(state, new_item, it3.second); + } } } + state = state->next; } - state = state->next; - } - if (n_suspended1 == 0) { - if (cont->state->needs_bind) { - bu_predict(concr->phrasetable, cont->state); + if (do_predict) { + if (cont->state->needs_bind) { + bu_predict(concr->phrasetable4, cont->state); + } else { + bu_predict(concr->phrasetable1, cont->state, 1, sentence->size); + } + } + } else { + if (do_predict && n_suspended == 1) { + if (cont->ccat->epsilon != 0) { + for (size_t i = 0; i < cont->ccat->epsilon->n_items; i++) { + ref pitem = cont->ccat->epsilon->items[i]; + td_epsilon(cont->state,cont,pitem,item,symcat); + } + } else { + for (Production *prod : cont->ccat->prods) { + td_predict(cont->state,cont,prod,item,symcat); + } + } } else { - bu_predict(concr->phrasetable, cont->state, 1, sentence->size); + State *next = cont->state; + while (next != NULL) { + auto it1 = next->completed.find(cont); + if (it1 != next->completed.end()) { + auto *it2 = it1->second.lookup(cont->ccat->value); + if (it2 != NULL) { + interval_t lin_idx_i = item->interval(ref::from_ptr(&symcat->r)); + auto *it3 = it2->lookup(lin_idx_i); + if (it3 != NULL) { + CCat *arg = *it3; + Item *new_item = new (item) Item; + combine(next, new_item, arg); + } + } + } + next = next->next; + } } } } @@ -1561,9 +1549,10 @@ PgfParseTableMaker::PgfParseTableMaker(ref concr) ref PgfParseTableMaker::clone_item(Item *item) { size_t ex_size = - sizeof(ref) * item->args.size() + - sizeof(size_t) * item->vars.size(); + sizeof(PgfMetaId) * item->args.size() + + sizeof(size_t) * item->vars.size(); auto pitem = PgfDB::malloc(ex_size); + pitem->res = (item->cont->ccat == NULL) ? 0 : item->cont->ccat->fid; pitem->pre_alt = item->pre_alt; pitem->pre_dot = item->pre_dot; pitem->dot = item->dot; @@ -1571,16 +1560,7 @@ ref PgfParseTableMaker::clone_item(Item *item) memcpy(&pitem->vars[0],&item->vars[0],sizeof(size_t) * item->vars.size()); for (size_t i = 0; i < item->args.size(); i++) { - ref symcf = 0; - if (item->args[i] != NULL) { - symcf = PgfDB::malloc(); - symcf->lincat = item->args[i]->cont->lincat; - symcf->value = item->args[i]->value; - symcf->lin_idx = item->args[i]->lin_idx; - symcf->fid = item->args[i]->fid; - symcf->viterbi_prob = item->args[i]->viterbi_prob; - } - pitem->args[i] = symcf; + pitem->args[i] = (item->args[i] == NULL) ? 0 : item->args[i]->fid; } return pitem; @@ -1591,19 +1571,19 @@ PgfAbstractParser::State *PgfParseTableMaker::new_state(const PgfTextSpot &start return current_state; } -void PgfParseTableMaker::symbol_token(Item *item, State *state, PgfSymbol sym) +void PgfParseTableMaker::symbol_token(Item *item, State *state, ref symks) { auto pitem = clone_item(item); - auto phrasetable = phrasetable_insert(concr->phrasetable,sym,pitem); - concr->phrasetable = phrasetable; + auto phrasetable1 = phrasetable_insert(concr->phrasetable1,symks,pitem); + concr->phrasetable1 = phrasetable1; delete item; } void PgfParseTableMaker::symbol_bind(Item *item, State *state, PgfSymbol sym) { auto pitem = clone_item(item); - auto phrasetable = phrasetable_insert(concr->phrasetable,ref(0).tagged(),pitem); - concr->phrasetable = phrasetable; + auto phrasetable4 = phrasetable_insert(concr->phrasetable4,ref(0),pitem); + concr->phrasetable4 = phrasetable4; if (ref::get_tag(sym) == PgfSymbolBIND::tag) { delete item; @@ -1613,33 +1593,77 @@ void PgfParseTableMaker::symbol_bind(Item *item, State *state, PgfSymbol sym) } } -void PgfParseTableMaker::suspend(Cont *cont,Item *item,size_t n_suspended1,size_t n_suspended) +void PgfParseTableMaker::suspend(Cont *cont,Item *item,bool do_predict,size_t n_suspended,ref symcat) { - for (auto it1 : cont->state->completed[cont]) { - for (auto it2 : it1.second) { - CCat *ccat = it2.second; - if (ccat != NULL) { - Item *new_item = new (item) Item; - combine(cont->state,new_item,ccat); + if (cont->ccat == NULL) { + for (auto it1 : cont->state->completed[cont]) { + for (auto it2 : it1.second) { + CCat *ccat = it2.second; + if (ccat != NULL) { + Item *new_item = new (item) Item; + combine(cont->state,new_item,ccat); + } } } - } - auto pitem = clone_item(item); - auto acat = ref::from_ptr((PgfSymbolACat*) &cont->lincat->name); - auto phrasetable = phrasetable_insert(concr->phrasetable,acat.tagged(),pitem); - concr->phrasetable = phrasetable; + auto pitem = clone_item(item); + auto phrasetable2 = phrasetable_insert(concr->phrasetable2,cont->lincat,pitem); + concr->phrasetable2 = phrasetable2; + } else { + if (do_predict && n_suspended == 1) { + if (cont->ccat->epsilon != 0) { + for (size_t i = 0; i < cont->ccat->epsilon->n_items; i++) { + ref pitem = cont->ccat->epsilon->items[i]; + td_epsilon(cont->state,cont,pitem,item,symcat); + } + } else { + for (Production *prod : cont->ccat->prods) { + td_predict(cont->state,cont,prod,item,symcat); + } + } + } else { + State *next = cont->state; + while (next != NULL) { + auto it1 = next->completed.find(cont); + if (it1 != next->completed.end()) { + auto *it2 = it1->second.lookup(cont->ccat->value); + if (it2 != NULL) { + interval_t lin_idx_i = item->interval(ref::from_ptr(&symcat->r)); + auto *it3 = it2->lookup(lin_idx_i); + if (it3 != NULL) { + CCat *arg = *it3; + Item *new_item = new (item) Item; + combine(next, new_item, arg); + } + } + } + next = next->next; + } + } + + auto pitem = clone_item(item); + auto phrasetable3 = phrasetable_insert(concr->phrasetable3,cont->ccat->epsilon,pitem); + concr->phrasetable3 = phrasetable3; + } } void PgfParseTableMaker::final_item(State *state, CCat *ccat, Item *item, interval_t value, interval_t lin_idx) { auto pitem = clone_item(item); - - PgfPhrasetable phrasetable = concr->phrasetable; - phrasetable = phrasetable_insert(phrasetable, - item->cont->lincat, value, lin_idx, ccat->fid, ccat->viterbi_prob, - pitem); - concr->phrasetable = phrasetable; + + if (ccat->epsilon == 0) { + PgfEpsilontable epsilontable = concr->epsilontable; + epsilontable = + epsilontable_insert(epsilontable, + ccat->cont->lincat, + ccat->value, ccat->lin_idx, + ccat->fid, ccat->viterbi_prob, + pitem, + &ccat->epsilon); + concr->epsilontable = epsilontable; + } else { + epsilontable_add(ccat->epsilon, pitem); + } } void PgfParseTableMaker::bu_predict(State *state, CCat *ccat) diff --git a/src/runtime/c/pgf/parser.h b/src/runtime/c/pgf/parser.h index 72403733a..c9b8c6cef 100644 --- a/src/runtime/c/pgf/parser.h +++ b/src/runtime/c/pgf/parser.h @@ -82,10 +82,8 @@ protected: struct CCat { PgfMetaId fid; - union { - object epsilons; - Cont *cont; - }; + ref epsilon; + Cont *cont; State *state; interval_t value; interval_t lin_idx; @@ -231,7 +229,7 @@ protected: }; State *current_state; - std::map,interval_map>> epsilons; + std::map epsilons; PgfMetaId initial_fid, last_fid; void process(Item *item, State *state); @@ -239,9 +237,9 @@ protected: void complete(Item *item, State *state); virtual State *new_state(const PgfTextSpot &start)=0; - virtual void symbol_token(Item *item, State *state, PgfSymbol sym)=0; + virtual void symbol_token(Item *item, State *state, ref symks)=0; virtual void symbol_bind(Item *item, State *state, PgfSymbol sym)=0; - virtual void suspend(Cont *cont, Item *item, size_t n_suspended1, size_t n_suspended)=0; + virtual void suspend(Cont *cont, Item *item, bool do_predict, size_t n_suspended,ref symcat)=0; virtual void final_item(State *state,CCat *ccat,Item *item,interval_t value,interval_t lin_idx)=0; virtual void bu_predict(State *state, CCat *ccat)=0; @@ -250,6 +248,7 @@ protected: void combine(State *state, Item *item, CCat *ccat); void get_info(CCat *ccat, ref *rule, size_t **pvalues); + CCat *get_epsilon_ccat(PgfText *name, PgfMetaId fid); static void print_item(Item *item, State *state); @@ -271,14 +270,14 @@ class PGF_INTERNAL_DECL PgfParser : private PgfAbstractParser, public PgfExprEnu bool case_sensitive; virtual State *new_state(const PgfTextSpot &start); - virtual void symbol_token(Item *item, State *state, PgfSymbol sym); + virtual void symbol_token(Item *item, State *state, ref symks); virtual void symbol_bind(Item *item, State *state, PgfSymbol sym); - virtual void suspend(Cont *cont,Item *item,size_t n_suspended1,size_t n_suspended); + virtual void suspend(Cont *cont,Item *item,bool do_predict,size_t n_suspended,ref symcat); virtual void final_item(State *state,CCat *ccat,Item *item,interval_t value,interval_t lin_idx); virtual void bu_predict(State *state, CCat *ccat); - void bu_predict(PgfPhrasetable phrasetable, State *state); - void bu_predict(PgfPhrasetable phrasetable, State *state, ptrdiff_t min, ptrdiff_t max); + void bu_predict(PgfPhrasetable phrasetable, State *state); + void bu_predict(PgfPhrasetable phrasetable, State *state, ptrdiff_t min, ptrdiff_t max); void make_chunks(State *state, std::vector &chunks, prob_t prob); PgfExpr process_expr(ExprState *estate, prob_t *prob); @@ -313,9 +312,9 @@ class PGF_INTERNAL_DECL PgfParseTableMaker : private PgfAbstractParser { private: virtual State *new_state(const PgfTextSpot &start); - virtual void symbol_token(Item *item, State *state, PgfSymbol sym); + virtual void symbol_token(Item *item, State *state, ref symks); virtual void symbol_bind(Item *item, State *state, PgfSymbol sym); - virtual void suspend(Cont *cont, Item *item, size_t n_suspended1, size_t n_suspended); + virtual void suspend(Cont *cont, Item *item, bool do_predict, size_t n_suspended,ref symcat); virtual void final_item(State *state, CCat *ccat,Item *item,interval_t value,interval_t lin_idx); virtual void bu_predict(State *state, CCat *ccat); diff --git a/src/runtime/c/pgf/pgf.cxx b/src/runtime/c/pgf/pgf.cxx index 9a238f030..e71c913eb 100644 --- a/src/runtime/c/pgf/pgf.cxx +++ b/src/runtime/c/pgf/pgf.cxx @@ -1022,7 +1022,7 @@ void pgf_lookup_morpho(PgfDB *db, PgfConcrRevision cnc_revision, PgfMorphoScanner scanner(callback); size_t n_items; - phrasetable_lookup(concr->phrasetable, + phrasetable_lookup(concr->phrasetable1, sentence, case_sensitive, &scanner, err); } PGF_API_END @@ -1072,7 +1072,7 @@ void pgf_lookup_cohorts(PgfDB *db, PgfConcrRevision cnc_revision, bool case_sensitive = pgf_is_case_sensitive(concr); PgfCohortsScanner scanner(callback); - phrasetable_lookup_cohorts(concr->phrasetable, + phrasetable_lookup_cohorts(concr->phrasetable1, sentence, case_sensitive, &scanner, err); } PGF_API_END @@ -1485,7 +1485,11 @@ ref clone_concrete(ref pgf, ref concr) clone->cflags = concr->cflags; clone->lins = concr->lins; clone->lincats = concr->lincats; - clone->phrasetable = concr->phrasetable; + clone->phrasetable1 = concr->phrasetable1; + clone->phrasetable2 = concr->phrasetable2; + clone->phrasetable3 = concr->phrasetable3; + clone->phrasetable4 = concr->phrasetable4; + clone->epsilontable = concr->epsilontable; clone->printnames = concr->printnames; clone->last_fid = concr->last_fid; memcpy(&clone->name, &concr->name, sizeof(PgfText)+concr->name.size+1); @@ -1664,7 +1668,11 @@ PgfConcrRevision pgf_create_concrete(PgfDB *db, PgfRevision revision, concr->cflags = 0; concr->lins = 0; concr->lincats = 0; - concr->phrasetable = 0; + concr->phrasetable1 = 0; + concr->phrasetable2 = 0; + concr->phrasetable3 = 0; + concr->phrasetable4 = 0; + concr->epsilontable = 0; concr->printnames = 0; concr->last_fid = 0; memcpy(&concr->name, name, sizeof(PgfText)+name->size+1); diff --git a/src/runtime/c/pgf/phrasetable.cxx b/src/runtime/c/pgf/phrasetable.cxx index d57a714fa..76b583d3c 100644 --- a/src/runtime/c/pgf/phrasetable.cxx +++ b/src/runtime/c/pgf/phrasetable.cxx @@ -31,169 +31,12 @@ int lparam_cmp(PgfLParam *p1, PgfLParam *p2) return 0; } -static -int sequence_cmp(vector seq1, vector seq2); - -static -void symbol_cmp(PgfSymbol sym1, PgfSymbol sym2, int res[2]) -{ - uint8_t t1 = ref::get_tag(sym1); - uint8_t t2 = ref::get_tag(sym2); - - if (t1 != t2) { - res[0] = (res[1] = ((int) t1) - ((int) t2)); - return; - } - - switch (t1) { - case PgfSymbolCat::tag: { - auto sym_cat1 = ref::untagged(sym1); - auto sym_cat2 = ref::untagged(sym2); - if (sym_cat1->d < sym_cat2->d) - res[0] = (res[1] = -1); - else if (sym_cat1->d > sym_cat2->d) - res[0] = (res[1] = 1); - else - res[0] = (res[1] = lparam_cmp(&sym_cat1->r, &sym_cat2->r)); - break; - } - case PgfSymbolLit::tag: { - auto sym_lit1 = ref::untagged(sym1); - auto sym_lit2 = ref::untagged(sym2); - if (sym_lit1->d < sym_lit2->d) - res[0] = (res[1] = -1); - else if (sym_lit1->d > sym_lit2->d) - res[0] = (res[1] = 1); - else - res[0] = (res[1] = lparam_cmp(&sym_lit1->r, &sym_lit2->r)); - break; - } - case PgfSymbolVar::tag: { - auto sym_var1 = ref::untagged(sym1); - auto sym_var2 = ref::untagged(sym2); - if (sym_var1->d < sym_var2->d) - res[0] = (res[1] = -1); - else if (sym_var1->d > sym_var2->d) - res[0] = (res[1] = 1); - else if (sym_var1->r < sym_var2->r) - res[0] = (res[1] = -1); - else if (sym_var1->r > sym_var2->r) - res[0] = (res[1] = 1); - break; - } - case PgfSymbolKS::tag: { - auto sym_ks1 = ref::untagged(sym1); - auto sym_ks2 = ref::untagged(sym2); - texticmp(&sym_ks1->token,&sym_ks2->token,res); - break; - } - case PgfSymbolKP::tag: { - auto sym_kp1 = ref::untagged(sym1); - auto sym_kp2 = ref::untagged(sym2); - res[0] = (res[1] = sequence_cmp(sym_kp1->default_form, sym_kp2->default_form)); - if (res[0] != 0) - return; - - for (size_t i = 0; ; i++) { - if (i >= sym_kp1->alts.size()) { - res[0] = (res[1] = -(i < sym_kp2->alts.size())); - return; - } - if (i >= sym_kp2->alts.size()) { - res[0] = (res[1] = 1); - return; - } - - res[0] = (res[1] = sequence_cmp(sym_kp1->alts[i].form, sym_kp2->alts[i].form)); - if (res[0] != 0) - return; - - vector> prefixes1 = sym_kp1->alts[i].prefixes; - vector> prefixes2 = sym_kp2->alts[i].prefixes; - for (size_t j = 0; ; j++) { - if (j >= prefixes1.size()) { - res[0] = (res[1] = -(j < prefixes2.size())); - return; - } - if (j >= prefixes2.size()) { - res[0] = (res[1] = 1); - return; - } - - res[0] = (res[1] = textcmp(&*prefixes1[j], &*prefixes2[j])); - if (res[0] != 0) - return; - } - } - } - case PgfSymbolBIND::tag: - case PgfSymbolSOFTBIND::tag: - case PgfSymbolNE::tag: - case PgfSymbolSOFTSPACE::tag: - case PgfSymbolCAPIT::tag: - case PgfSymbolALLCAPIT::tag: - break; - case PgfSymbolACat::tag: { - auto sym_acat1 = ref::untagged(sym1); - auto sym_acat2 = ref::untagged(sym2); - res[0] = (res[1] = textcmp(&sym_acat1->name,&sym_acat2->name)); - return; - } - case PgfSymbolCCat::tag: { - auto sym_ccat1 = ref::untagged(sym1); - auto sym_ccat2 = ref::untagged(sym2); - res[0] = (res[1] = textcmp(&sym_ccat1->lincat->name,&sym_ccat2->lincat->name)); - if (res[0] != 0) - return; - if (sym_ccat1->value < sym_ccat2->value) - res[0] = (res[1] = -1); - else if (sym_ccat1->value > sym_ccat2->value) - res[0] = (res[1] = 1); - if (sym_ccat1->lin_idx < sym_ccat2->lin_idx) - res[0] = (res[1] = -1); - else if (sym_ccat1->lin_idx > sym_ccat2->lin_idx) - res[0] = (res[1] = 1); - else - res[0] = (res[1] = 0); - return; - } - default: - throw pgf_error("Unknown symbol tag"); - } -} - -static -int sequence_cmp(vector seq1, vector seq2) -{ - int res[2] = {0,0}; - for (size_t i = 0; ; i++) { - if (i >= seq1.size()) { - if (i < seq2.size()) - return -1; - return res[1]; - } - if (i >= seq2.size()) - return 1; - - symbol_cmp(seq1[i], seq2[i], res); - if (res[0] != 0) - return res[0]; - } - - return 0; -} - PGF_INTERNAL int text_symbol_cmp(PgfTextSpot *spot, const uint8_t *end, - PgfSymbol sym, bool case_sensitive) + ref sym_ks, bool case_sensitive) { - uint8_t tag = ref::get_tag(sym); - if (PgfSymbolKS::tag != tag) - return ((int) PgfSymbolKS::tag) - ((int) tag); - int res1 = 0; - auto sym_ks = ref::untagged(sym); const uint8_t *s2 = (uint8_t *) &sym_ks->token.text; const uint8_t *e2 = s2+sym_ks->token.size; @@ -326,134 +169,14 @@ bool text_item_match(PgfTextSpot *spot, const uint8_t *end, PGF_INTERNAL_DECL size_t get_next_padovan(size_t min); -static -int symbol_cmp(ref lincat, interval_t value, interval_t lin_idx, PgfSymbol sym) -{ - uint8_t tag = ref::get_tag(sym); - if (PgfSymbolCCat::tag != tag) - return ((int) PgfSymbolCCat::tag) - ((int) tag); - - auto symcf = ref::untagged(sym); - int res = textcmp(&lincat->name, &symcf->lincat->name); - if (res != 0) - return res; - if (value < symcf->value) - return -1; - else if (value > symcf->value) - return 1; - else if (lin_idx < symcf->lin_idx) - return -1; - else if (lin_idx > symcf->lin_idx) - return 1; - else - return 0; -} - -static -int symbol_cmp(PgfSymbol sym1, PgfSymbol sym2) -{ - uint8_t tag1 = ref::get_tag(sym1); - uint8_t tag2 = ref::get_tag(sym2); - if (tag1 != tag2) - return ((int) tag1) - ((int) tag2); - - switch (tag1) { - case PgfSymbolKS::tag: { - auto symks1 = ref::untagged(sym1); - auto symks2 = ref::untagged(sym2); - int res[2] = {0,0}; - texticmp(&symks1->token, &symks2->token, res); - if (res[0] != 0) - return res[0]; - return res[1]; - } - case PgfSymbolACat::tag: { - auto symcf1 = ref::untagged(sym1); - auto symcf2 = ref::untagged(sym2); - return textcmp(&symcf1->name, &symcf2->name); - } - case PgfSymbolCCat::tag: { - auto symcf1 = ref::untagged(sym1); - auto symcf2 = ref::untagged(sym2); - int res = textcmp(&symcf1->lincat->name, &symcf2->lincat->name); - if (res != 0) - return res; - if (symcf1->value < symcf2->value) - return -1; - else if (symcf1->value > symcf2->value) - return 1; - else if (symcf1->lin_idx < symcf2->lin_idx) - return -1; - else if (symcf1->lin_idx > symcf2->lin_idx) - return 1; - else - return 0; - } - default: - return 0; - } -} - +template PGF_INTERNAL -void phrasetable_iter(PgfPhrasetable table, ref lincat, std::function arg,size_t,vector>)> &f) -{ - if (table == 0) - return; - - int cmp = 0; - ref symcf = 0; - uint8_t tag = ref::get_tag(table->value.sym); - if (PgfSymbolCCat::tag != tag) { - cmp = ((int) PgfSymbolCCat::tag) - ((int) tag); - } else { - symcf = ref::untagged(table->value.sym); - cmp = textcmp(&lincat->name, &symcf->lincat->name); - } - - if (cmp < 0) - phrasetable_iter(table->left, lincat, f); - else if (cmp > 0) - phrasetable_iter(table->right, lincat, f); - else { - phrasetable_iter(table->left, lincat, f); - f(symcf,table->value.n_items,table->value.items); - phrasetable_iter(table->right, lincat, f); - } -} - -PGF_INTERNAL -vector> phrasetable_lookup(PgfPhrasetable table, PgfSymbol sym, size_t *n_items) -{ - while (table != 0) { - int cmp = symbol_cmp(sym,table->value.sym); - if (cmp < 0) - table = table->left; - else if (cmp > 0) - table = table->right; - else { - *n_items = table->value.n_items; - return table->value.items; - } - } - - *n_items = 0; - return 0; -} - -PGF_INTERNAL -vector> phrasetable_lookup(PgfPhrasetable phrasetable, - ref lincat, +vector> phrasetable_lookup(PgfPhrasetable phrasetable, + ref key, size_t *n_items) { while (phrasetable != 0) { - int cmp; - uint8_t tag = ref::get_tag(phrasetable->value.sym); - if (PgfSymbolACat::tag != tag) { - cmp = ((int) PgfSymbolACat::tag) - ((int) tag); - } else { - auto symcf = ref::untagged(phrasetable->value.sym); - cmp = textcmp(&lincat->name, &symcf->name); - } + int cmp = compare_key(key, phrasetable->value.key); if (cmp < 0) phrasetable = phrasetable->left; else if (cmp > 0) @@ -469,7 +192,7 @@ vector> phrasetable_lookup(PgfPhrasetable phrasetable, } PGF_INTERNAL -void phrasetable_lookup(PgfPhrasetable table, +void phrasetable_lookup(PgfPhrasetable table, PgfText *sentence, bool case_sensitive, PgfPhraseScanner *scanner, PgfExn* err) @@ -481,7 +204,7 @@ void phrasetable_lookup(PgfPhrasetable table, spot.pos = 0; spot.ptr = (uint8_t *) sentence->text; const uint8_t *end = spot.ptr+sentence->size; - int cmp = text_symbol_cmp(&spot,end,table->value.sym,case_sensitive); + int cmp = text_symbol_cmp(&spot,end,table->value.key,case_sensitive); if (cmp < 0) { phrasetable_lookup(table->left,sentence,case_sensitive,scanner,err); } else if (cmp > 0) { @@ -587,14 +310,14 @@ void finish_skipping(PgfCohortsState *state) { static void phrasetable_lookup_prefixes(PgfCohortsState *state, - PgfPhrasetable table, + PgfPhrasetable table, ptrdiff_t min, ptrdiff_t max) { if (table == 0) return; PgfTextSpot current = state->spot; - int cmp = text_symbol_cmp(¤t,state->end,table->value.sym,state->case_sensitive); + int cmp = text_symbol_cmp(¤t,state->end,table->value.key,state->case_sensitive); if (cmp < 0) { phrasetable_lookup_prefixes(state,table->left,min,max); } else if (cmp > 0) { @@ -661,7 +384,7 @@ void phrasetable_lookup_prefixes(PgfCohortsState *state, } PGF_INTERNAL -void phrasetable_lookup_cohorts(PgfPhrasetable table, +void phrasetable_lookup_cohorts(PgfPhrasetable table, PgfText *sentence, bool case_sensitive, PgfPhraseScanner *scanner, PgfExn* err) @@ -748,29 +471,29 @@ void phrasetable_lookup_cohorts(PgfPhrasetable table, } } +template PGF_INTERNAL -PgfPhrasetable phrasetable_insert(PgfPhrasetable table, - PgfSymbol sym, - ref item) +PgfPhrasetable phrasetable_insert(PgfPhrasetable table, + ref key, ref item) { if (table == 0) { auto items = vector>::alloc(1); items[0] = item; - return Node::new_node({.sym=sym,.n_items=1,.items=items}); + return Node>::new_node({.key=key,.n_items=1,.items=items}); } - int cmp = symbol_cmp(sym,table->value.sym); + int cmp = compare_key(key, table->value.key); if (cmp < 0) { - PgfPhrasetable left = phrasetable_insert(table->left, sym, item); - table = Node::upd_node(table,left,table->right); - return Node::balanceL(table); + PgfPhrasetable left = phrasetable_insert(table->left, key, item); + table = Node>::upd_node(table,left,table->right); + return Node>::balanceL(table); } else if (cmp > 0) { - PgfPhrasetable right = phrasetable_insert(table->right, sym, item); - table = Node::upd_node(table, table->left, right); - return Node::balanceR(table); + PgfPhrasetable right = phrasetable_insert(table->right, key, item); + table = Node>::upd_node(table, table->left, right); + return Node>::balanceR(table); } else { - PgfPhrasetable new_table = - Node::upd_node(table, table->left, table->right); + PgfPhrasetable new_table = + Node>::upd_node(table, table->left, table->right); auto items = new_table->value.items; if (new_table->value.n_items >= items.size()) { @@ -784,62 +507,187 @@ PgfPhrasetable phrasetable_insert(PgfPhrasetable table, } } +static +int compare_key(ref symks1, ref symks2) { + int res[2] = {0,0}; + texticmp(&symks1->token, &symks2->token, res); + if (res[0] != 0) + return res[0]; + return res[1]; +} + +template +PgfPhrasetable phrasetable_insert(PgfPhrasetable table, + ref key, + ref item); + +static +int compare_key(ref lincat1, ref lincat2) { + return textcmp(&lincat1->name, &lincat2->name); +} + +template PGF_INTERNAL -PgfPhrasetable phrasetable_insert(PgfPhrasetable table, - ref lincat, - interval_t value, interval_t lin_idx, - PgfMetaId fid, prob_t viterbi_prob, - ref item) +PgfPhrasetable phrasetable_insert(PgfPhrasetable table, + ref key, + ref item); + +template +PGF_INTERNAL +vector> phrasetable_lookup(PgfPhrasetable phrasetable, + ref key, + size_t *n_items); + +static +int compare_key(ref ccat1, ref ccat2) { + return ((int) ccat1->fid) - ((int) ccat2->fid); +} + +template +PgfPhrasetable phrasetable_insert(PgfPhrasetable table, + ref key, + ref item); + +template +PGF_INTERNAL +vector> phrasetable_lookup(PgfPhrasetable phrasetable, + ref key, + size_t *n_items); + +static +int compare_key(ref symbind1, ref symbind2) { + return 0; +} + +template +PgfPhrasetable phrasetable_insert(PgfPhrasetable table, + ref key, + ref item); + +template +PGF_INTERNAL +vector> phrasetable_lookup(PgfPhrasetable phrasetable, + ref key, + size_t *n_items); + +PGF_INTERNAL +PgfEpsilontable epsilontable_insert(PgfEpsilontable table, + ref lincat, + interval_t value, interval_t lin_idx, + PgfMetaId fid, prob_t viterbi_prob, + ref item, + ref *pepsilon) { if (table == 0) { - ref symcf = PgfDB::malloc(); - symcf->lincat = lincat; - symcf->value = value; - symcf->lin_idx = lin_idx; - symcf->fid = fid; - symcf->viterbi_prob = viterbi_prob; auto items = vector>::alloc(1); items[0] = item; - return Node::new_node({.sym=symcf.tagged(),.n_items=1,.items=items}); + PgfEpsilontable new_table = + Node::new_node({.lincat=lincat, + .fid=fid, + .value=value, + .lin_idx=lin_idx, + .viterbi_prob=viterbi_prob, + .n_items=1, + .items=items}); + *pepsilon = ref::from_ptr(&new_table->value); + return new_table; } - int cmp = symbol_cmp(lincat,value,lin_idx,table->value.sym); - if (cmp < 0) { - PgfPhrasetable left = phrasetable_insert(table->left, - lincat, value, lin_idx, fid, viterbi_prob, item); - table = Node::upd_node(table,left,table->right); - return Node::balanceL(table); - } else if (cmp > 0) { - PgfPhrasetable right = phrasetable_insert(table->right, - lincat, value, lin_idx, fid, viterbi_prob, item); - table = Node::upd_node(table, table->left, right); - return Node::balanceR(table); - } else { - PgfPhrasetable new_table = - Node::upd_node(table, table->left, table->right); + int cmp = textcmp(&lincat->name, &table->value.lincat->name); + if (cmp == 0) { + cmp = ((int)fid) - ((int)table->value.fid); + } - auto items = new_table->value.items; - if (new_table->value.n_items >= items.size()) { - size_t new_len = get_next_padovan(new_table->value.n_items+1); - items = items.realloc(new_len, new_table->txn_id); + if (cmp < 0) { + PgfEpsilontable left = epsilontable_insert(table->left, + lincat, value, lin_idx, fid, viterbi_prob, item, pepsilon); + table = Node::upd_node(table,left,table->right); + return Node::balanceL(table); + } else if (cmp > 0) { + PgfEpsilontable right = epsilontable_insert(table->right, + lincat, value, lin_idx, fid, viterbi_prob, item, pepsilon); + table = Node::upd_node(table, table->left, right); + return Node::balanceR(table); + } else { + PgfEpsilontable new_table = + Node::upd_node(table, table->left, table->right); + + auto items = table->value.items; + if (table->value.n_items >= items.size()) { + size_t new_len = get_next_padovan(table->value.n_items+1); + items = items.realloc(new_len, table->txn_id); } - items[new_table->value.n_items] = item; + items[table->value.n_items] = item; new_table->value.n_items++; new_table->value.items = items; + *pepsilon = ref::from_ptr(&new_table->value); return new_table; } } PGF_INTERNAL -void phrasetable_release(PgfPhrasetable table) +void epsilontable_add(ref epsilon,ref item) +{ + auto items = epsilon->items; + if (epsilon->n_items >= items.size()) { + size_t new_len = get_next_padovan(epsilon->n_items+1); + items = items.realloc(new_len, PgfDB::get_txn_id()); + } + items[epsilon->n_items] = item; + epsilon->n_items++; + epsilon->items = items; +} + +PGF_INTERNAL +ref epsilontable_get(PgfEpsilontable table, + PgfText *name, PgfMetaId fid) +{ + if (table == 0) { + return 0; + } + + int cmp = textcmp(name, &table->value.lincat->name); + if (cmp == 0) { + cmp = ((int)fid) - ((int)table->value.fid); + } + + if (cmp < 0) { + return epsilontable_get(table->left,name,fid); + } else if (cmp > 0) { + return epsilontable_get(table->right,name,fid); + } else { + return ref::from_ptr(&table->value); + } +} + +PGF_INTERNAL +void epsilontable_iter(PgfEpsilontable table, ref lincat, std::function arg)> &f) { if (table == 0) return; - phrasetable_release(table->left); - phrasetable_release(table->right); + + int cmp = textcmp(&lincat->name, &table->value.lincat->name); + if (cmp < 0) + epsilontable_iter(table->left, lincat, f); + else if (cmp > 0) + epsilontable_iter(table->right, lincat, f); + else { + epsilontable_iter(table->left, lincat, f); + f(ref::from_ptr(&table->value)); + epsilontable_iter(table->right, lincat, f); + } +} + +PGF_INTERNAL +void epsilontable_release(PgfEpsilontable table) +{ + if (table == 0) + return; + epsilontable_release(table->left); + epsilontable_release(table->right); for (size_t i = 0; i < table->value.n_items; i++) { PgfItem::release(table->value.items[i]); } vector>::release(table->value.items); - Node::release(table); + Node::release(table); } diff --git a/src/runtime/c/pgf/phrasetable.h b/src/runtime/c/pgf/phrasetable.h index a952774ab..3d8cb4c86 100644 --- a/src/runtime/c/pgf/phrasetable.h +++ b/src/runtime/c/pgf/phrasetable.h @@ -10,10 +10,12 @@ struct PGF_INTERNAL_DECL PgfTextSpot { }; struct PGF_INTERNAL_DECL PgfItem { + PgfMetaId res; + struct { size_t &operator[](int i) { PgfItem *item = containerof(PgfItem,vars,this); - return ((size_t*) (((ref*) (item+1))+item->rule->args.size()))[i]; + return ((size_t*) (((PgfMetaId*) (item+1))+item->rule->args.size()))[i]; } size_t size() { PgfItem *item = containerof(PgfItem,vars,this); @@ -22,9 +24,9 @@ struct PGF_INTERNAL_DECL PgfItem { } vars; struct { - ref &operator[](int i) { + PgfMetaId &operator[](int i) { PgfItem *item = containerof(PgfItem,args,this); - return ((ref*) (item+1))[i]; + return ((PgfMetaId*) (item+1))[i]; } size_t size() { PgfItem *item = containerof(PgfItem,args,this); @@ -35,8 +37,8 @@ struct PGF_INTERNAL_DECL PgfItem { static void release(ref item) { size_t ex_size = - sizeof(ref) * item->args.size() + - sizeof(size_t) * item->vars.size(); + sizeof(PgfMetaId) * item->args.size() + + sizeof(size_t) * item->vars.size(); PgfDB::free(item, ex_size); } @@ -46,8 +48,11 @@ struct PGF_INTERNAL_DECL PgfItem { ref rule; }; -struct PGF_INTERNAL_DECL PgfPhrasetableValue { - PgfSymbol sym; +struct PGF_INTERNAL_DECL PgfCCat { + ref lincat; + PgfMetaId fid; + interval_t value, lin_idx; + prob_t viterbi_prob; // Here n_items tells us how many actual items there are in // the vector items. On the other hand, items.size() tells us @@ -56,27 +61,29 @@ struct PGF_INTERNAL_DECL PgfPhrasetableValue { vector> items; }; -typedef ref> PgfPhrasetable; +template +struct PGF_INTERNAL_DECL PgfPhrasetableValue { + ref key; -PgfPhrasetable phrasetable_insert(PgfPhrasetable table, - PgfSymbol sym, - ref item); + // Here n_items tells us how many actual items there are in + // the vector items. On the other hand, items.size() tells us + // how big buffer we have allocated. + size_t n_items; + vector> items; +}; -PgfPhrasetable phrasetable_insert(PgfPhrasetable table, - ref lincat, - interval_t value, interval_t lin_idx, - PgfMetaId fid, prob_t viterbi_prob, - ref item); +template +using PgfPhrasetable = ref>>; +template PGF_INTERNAL_DECL -void phrasetable_iter(PgfPhrasetable phrasetable,ref lincat,std::function symcf,size_t,vector>)> &f); +PgfPhrasetable phrasetable_insert(PgfPhrasetable table, + ref key, ref item); +template PGF_INTERNAL_DECL -vector> phrasetable_lookup(PgfPhrasetable phrasetable, PgfSymbol sym, size_t *n_items); - -PGF_INTERNAL_DECL -vector> phrasetable_lookup(PgfPhrasetable phrasetable, - ref lincat, +vector> phrasetable_lookup(PgfPhrasetable phrasetable, + ref key, size_t *n_items); class PGF_INTERNAL_DECL PgfPhraseScanner { @@ -88,18 +95,57 @@ public: }; PGF_INTERNAL_DECL -void phrasetable_lookup(PgfPhrasetable phrasetable, +void phrasetable_lookup(PgfPhrasetable phrasetable, PgfText *sentence, bool case_sensitive, PgfPhraseScanner *scanner, PgfExn* err); PGF_INTERNAL_DECL -void phrasetable_lookup_cohorts(PgfPhrasetable phrasetable, +void phrasetable_lookup_cohorts(PgfPhrasetable phrasetable, PgfText *sentence, bool case_sensitive, PgfPhraseScanner *scanner, PgfExn* err); +template +void phrasetable_release(PgfPhrasetable table) +{ + if (table == 0) + return; + phrasetable_release(table->left); + phrasetable_release(table->right); + for (size_t i = 0; i < table->value.n_items; i++) { + PgfItem::release(table->value.items[i]); + } + vector>::release(table->value.items); + Node>::release(table); +} + + +typedef ref> PgfEpsilontable; + +// Creates a new epsilon category with its first item. +// The new category is mutable within the current transaction PGF_INTERNAL_DECL -void phrasetable_release(PgfPhrasetable table); +PgfEpsilontable epsilontable_insert(PgfEpsilontable table, + ref lincat, + interval_t value, interval_t lin_idx, + PgfMetaId fid, prob_t viterbi_prob, + ref item, + ref *pepsilon); + +// Adds a new item to an existing epsilon category. The category +// must have been created by epsilontable_insert in the current transaction. +PGF_INTERNAL_DECL +void epsilontable_add(ref epsilon, ref item); + +PGF_INTERNAL_DECL +ref epsilontable_get(PgfEpsilontable table, + PgfText *name, PgfMetaId fid); + +PGF_INTERNAL +void epsilontable_iter(PgfEpsilontable table, ref lincat, std::function arg)> &f); + +PGF_INTERNAL_DECL +void epsilontable_release(PgfEpsilontable table); #endif diff --git a/src/runtime/c/pgf/printer.cxx b/src/runtime/c/pgf/printer.cxx index 8b740c5ac..74277a460 100644 --- a/src/runtime/c/pgf/printer.cxx +++ b/src/runtime/c/pgf/printer.cxx @@ -578,20 +578,6 @@ void PgfPrinter::symbol(PgfSymbol sym) case PgfSymbolALLCAPIT::tag: puts("ALL_CAPIT"); break; - case PgfSymbolACat::tag: { - auto symcf = ref::untagged(sym); - efun(&symcf->name); - break; - } - case PgfSymbolCCat::tag: { - auto symcf = ref::untagged(sym); - efun(&symcf->lincat->name); - nprintf(64,"(%zu-%zu,%zu-%zu)",symcf->value.first - ,symcf->value.second - ,symcf->lin_idx.first - ,symcf->lin_idx.second); - break; - } } } diff --git a/src/runtime/c/pgf/reader.cxx b/src/runtime/c/pgf/reader.cxx index f8ec65453..2cd4fca59 100644 --- a/src/runtime/c/pgf/reader.cxx +++ b/src/runtime/c/pgf/reader.cxx @@ -697,7 +697,11 @@ ref PgfReader::read_printname() ref PgfReader::read_concrete() { concrete = read_name(&PgfConcr::name); - concrete->phrasetable = 0; + concrete->phrasetable1 = 0; + concrete->phrasetable2 = 0; + concrete->phrasetable3 = 0; + concrete->phrasetable4 = 0; + concrete->epsilontable = 0; concrete->last_fid = 0; auto cflags = read_namespace(&PgfReader::read_flag);