GitHub/gf-core
1
0
Fork 1
mirror of https://github.com/GrammaticalFramework/gf-core.git synced 2026-08-19 10:46:22 -06:00
Code Issues Packages Projects Releases Wiki Activity

split the phrasetable to five parts and better serve the parser

Browse Source
This commit is contained in:
Krasimir Angelov
2026-08-12 10:31:03 +02:00
parent 9ac7ea1df9
commit c41b75a9cb
9 changed files with 571 additions and 662 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/runtime/c/pgf/data.cxx
+5 -1
View File
@@ -41,7 +41,11 @@ void PgfConcr::release(ref<PgfConcr> concr)
namespace_release(concr->lins);
namespace_release(concr->lincats);
namespace_release(concr->printnames);
phrasetable_release(concr->phrasetable);
phrasetable_release(concr->phrasetable1);
phrasetable_release(concr->phrasetable2);
phrasetable_release(concr->phrasetable3);
phrasetable_release(concr->phrasetable4);
epsilontable_release(concr->epsilontable);
PgfDB::free(concr, concr->name.size+1);
}
src/runtime/c/pgf/data.h
+5 -15
View File
@@ -252,20 +252,6 @@ struct PGF_INTERNAL_DECL PgfConcrLin {
static void release(ref<PgfConcrLin> lin);
};
struct PGF_INTERNAL_DECL PgfSymbolACat {
static const uint8_t tag = 11;
PgfText name;
};
struct PGF_INTERNAL_DECL PgfSymbolCCat {
static const uint8_t tag = 12;
ref<PgfConcrLincat> lincat;
interval_t value;
interval_t lin_idx;
prob_t viterbi_prob;
PgfMetaId fid;
};
struct PGF_INTERNAL_DECL PgfConcrPrintname {
ref<PgfText> printname;
PgfText name;
@@ -281,7 +267,11 @@ struct PGF_INTERNAL_DECL PgfConcr {
Namespace<PgfFlag> cflags;
Namespace<PgfConcrLin> lins;
Namespace<PgfConcrLincat> lincats;
PgfPhrasetable phrasetable;
PgfPhrasetable<PgfSymbolKS> phrasetable1; // suspended on token
PgfPhrasetable<PgfConcrLincat> phrasetable2; // suspended on lincat
PgfPhrasetable<PgfCCat> phrasetable3; // suspended on ccat
PgfPhrasetable<PgfSymbolBIND> phrasetable4; // suspended on bind
PgfEpsilontable epsilontable;
Namespace<PgfConcrPrintname> printnames;
PgfMetaId last_fid;
src/runtime/c/pgf/parser.cxx
+278 -254
View File
@@ -11,26 +11,44 @@ PgfAbstractParser::PgfAbstractParser(ref<PgfConcr> concr)
this->concr = concr;
this->current_state = NULL;
this->initial_fid = concr->last_fid;
this->last_fid = concr->last_fid;
}
void PgfAbstractParser::get_info(CCat *ccat, ref<PgfConcrRule> *prule, size_t **pvalues)
{
if (ccat->fid > initial_fid) {
if (ccat->epsilon == 0) {
Production *prod = ccat->prods[0];
*prule = prod->rule;
*pvalues = &prod->vars[0];
} else {
size_t n_items;
vector<ref<PgfItem>> items =
phrasetable_lookup(concr->phrasetable, ccat->epsilons, &n_items);
ref<PgfItem> pitem = items[0];
ref<PgfItem> pitem = ccat->epsilon->items[0];
*prule = pitem->rule;
*pvalues = &pitem->vars[0];
}
}
PgfAbstractParser::CCat *PgfAbstractParser::get_epsilon_ccat(PgfText *name, PgfMetaId fid)
{
if (fid == 0)
return NULL;
CCat *&ccat = epsilons[fid];
if (ccat == NULL) {
ref<PgfCCat> arg = epsilontable_get(concr->epsilontable,
name, fid);
ccat = new CCat;
ccat->fid = arg->fid;
ccat->epsilon = arg;
ccat->cont = NULL;
ccat->state = NULL;
ccat->lin_idx = arg->lin_idx;
ccat->value = arg->value;
ccat->covered = true;
ccat->viterbi_prob = arg->viterbi_prob;
}
return ccat;
}
PgfAbstractParser::CCat::~CCat()
{
for (Production *prod : prods) {
@@ -97,7 +115,7 @@ void PgfAbstractParser::process(Item *item, State *state)
PGF_INTERNAL_DECL
int text_symbol_cmp(PgfTextSpot *spot, const uint8_t *end,
PgfSymbol sym, bool case_sensitive);
ref<PgfSymbolKS> sym, bool case_sensitive);
void PgfAbstractParser::symbol(Item *item, State *state, PgfSymbol sym)
{
@@ -137,15 +155,16 @@ void PgfAbstractParser::symbol(Item *item, State *state, PgfSymbol sym)
auto &suspended = cont->suspended[value_i][lin_idx_i];
suspended.push_back(item);
suspend(cont,item,n_suspended1,suspended.size());
suspend(cont,item,n_suspended1 == 0,suspended.size(),symcat);
}
} else {
interval_t value_i = item->interval(item->rule->args[symcat->d]);
interval_t lin_idx_i = item->interval(ref<PgfLParam>::from_ptr(&symcat->r));
// the following prevents infinite loops with epsilons
bool found = false;
CCat *prev_ccat = ccat;
while (prev_ccat != NULL && prev_ccat->fid > initial_fid && prev_ccat->cont->state == state) {
while (prev_ccat != NULL && prev_ccat->epsilon == 0 && prev_ccat->cont->state == state) {
if (prev_ccat->value == value_i && prev_ccat->lin_idx == lin_idx_i) {
found = true;
break;
@@ -162,8 +181,8 @@ void PgfAbstractParser::symbol(Item *item, State *state, PgfSymbol sym)
if (cont == NULL) {
cont = new Cont;
cont->ccat = ccat;
if (ccat->fid <= initial_fid)
cont->lincat = ref<PgfSymbolCCat>::untagged(ccat->epsilons)->lincat;
if (ccat->epsilon != 0)
cont->lincat = ccat->epsilon->lincat;
else
cont->lincat = ccat->cont->lincat;
cont->state = state;
@@ -185,44 +204,13 @@ found:;
auto &suspended = cont->suspended[value_i][lin_idx_i];
suspended.push_back(item);
if (!subsumed && suspended.size() == 1) {
if (ccat->fid <= initial_fid) {
size_t n_items = 0;
vector<ref<PgfItem>> items =
phrasetable_lookup(concr->phrasetable, ccat->epsilons, &n_items);
for (size_t i = 0; i < n_items; i++) {
ref<PgfItem> pitem = items[i];
td_epsilon(state,cont,pitem,item,symcat);
}
} else {
for (Production *prod : ccat->prods) {
td_predict(state,cont,prod,item,symcat);
}
}
} else {
State *next = state;
while (next != NULL) {
auto it1 = next->completed.find(cont);
if (it1 != next->completed.end()) {
auto *it2 = it1->second.lookup(ccat->value);
if (it2 != NULL) {
auto *it3 = it2->lookup(lin_idx_i);
if (it3 != NULL) {
CCat *arg = *it3;
Item *new_item = new (item) Item;
combine(next, new_item, arg);
}
}
}
next = next->next;
}
}
suspend(cont,item,!subsumed,suspended.size(),symcat);
}
break;
}
case PgfSymbolKS::tag: {
symbol_token(item, state, sym);
auto symks = ref<PgfSymbolKS>::untagged(sym);
symbol_token(item, state, symks);
break;
}
case PgfSymbolKP::tag: {
@@ -282,6 +270,7 @@ void PgfAbstractParser::complete(Item *item, State *state)
if (ccat == NULL) {
ccat = new CCat;
ccat->fid = (++last_fid);
ccat->epsilon = 0;
ccat->cont = item->cont;
ccat->state = state;
ccat->lin_idx = lin_idx;
@@ -331,8 +320,7 @@ void PgfAbstractParser::complete(Item *item, State *state)
final_item(state, ccat, item, res, lin_idx);
if (ccat->prods.size() == 1) {
if (ccat->cont->ccat == NULL)
bu_predict(state, ccat);
bu_predict(state, ccat);
for (auto it1 : ccat->cont->suspended.overlaps(ccat->value)) {
for (auto it2 : it1.second.overlaps(ccat->lin_idx)) {
@@ -552,22 +540,9 @@ void PgfAbstractParser::td_epsilon(State *state, Cont *cont, ref<PgfItem> pitem,
}
for (size_t i = 0; i < pitem->args.size(); i++) {
ref<PgfSymbolCCat> arg = pitem->args[i];
if (arg != 0) {
CCat *&arg_ccat = epsilons[arg->lincat][arg->value][arg->lin_idx];
if (arg_ccat == NULL) {
arg_ccat = new CCat;
arg_ccat->fid = arg->fid;
arg_ccat->epsilons = arg.tagged();
arg_ccat->state = NULL;
arg_ccat->lin_idx = arg->lin_idx;
arg_ccat->value = arg->value;
arg_ccat->covered = true;
arg_ccat->viterbi_prob = arg->viterbi_prob;
}
item->args[i] = arg_ccat;
item->inside_prob += arg_ccat->viterbi_prob;
if (pitem->args[i] != 0) {
item->args[i] = get_epsilon_ccat(&lin->absfun->type->hypos[i].type->name,pitem->args[i]);
item->inside_prob += item->args[i]->viterbi_prob;
}
if (!item->instantiate(item->rule->args[i], pitem->rule, &pitem->vars[0], pitem->rule->args[i])) {
@@ -676,7 +651,7 @@ void PgfAbstractParser::print_item(Item *item, State *state)
{
PgfPrinter printer(NULL,0,NULL);
printer.nprintf(32, "[%zd-%zd; ", item->cont ? item->cont->state->end.pos : 0, state->start.pos);
printer.nprintf(32, "[%zd-%zd; ", (item->cont && item->cont->state) ? item->cont->state->end.pos : 0, state->start.pos);
if (item->vars.size() > 0) {
printer.lvar_ranges(item->rule->ranges, &item->vars[0]);
@@ -832,7 +807,7 @@ PgfParser::~PgfParser()
for (auto it1 : state->completed) {
for (auto it2 : it1.second) {
for (auto it3 : it2.second) {
if (it3.second->fid <= initial_fid)
if (it3.second->epsilon != 0)
continue;
for (ExprState *estate : it3.second->pending) {
@@ -850,21 +825,17 @@ PgfParser::~PgfParser()
}
for (auto it1 : epsilons) {
for (auto it2 : it1.second) {
for (auto it3 : it2.second) {
for (ExprState *estate : it3.second->pending) {
if (estate->expr != 0)
u->free_ref(estate->expr);
}
for (ExprProb &ep : it3.second->exprs) {
u->free_ref(ep.expr);
}
}
for (ExprState *estate : it1.second->pending) {
if (estate->expr != 0)
u->free_ref(estate->expr);
}
for (ExprProb &ep : it1.second->exprs) {
u->free_ref(ep.expr);
}
}
}
void PgfParser::bu_predict(PgfPhrasetable phrasetable,
void PgfParser::bu_predict(PgfPhrasetable<PgfSymbolKS> phrasetable,
State *state,
ptrdiff_t min, ptrdiff_t max)
{
@@ -872,7 +843,7 @@ void PgfParser::bu_predict(PgfPhrasetable phrasetable,
return;
PgfTextSpot current = state->end;
int cmp = text_symbol_cmp(&current,end,phrasetable->value.sym,case_sensitive);
int cmp = text_symbol_cmp(&current,end,phrasetable->value.key,case_sensitive);
if (cmp < 0) {
bu_predict(phrasetable->left,state,min,max);
} else if (cmp > 0) {
@@ -906,51 +877,51 @@ void PgfParser::bu_predict(PgfPhrasetable phrasetable,
}
}
void PgfParser::bu_predict(PgfPhrasetable phrasetable,
void PgfParser::bu_predict(PgfPhrasetable<PgfSymbolBIND> phrasetable,
State *state)
{
if (phrasetable == 0)
return;
size_t n_items = 0;
vector<ref<PgfItem>> items =
phrasetable_lookup(concr->phrasetable4,
ref<PgfSymbolBIND>(0),
&n_items);
PgfTextSpot current = state->end;
int cmp;
uint8_t tag = ref<PgfSymbol>::get_tag(phrasetable->value.sym);
cmp = ((int) PgfSymbolBIND::tag) - ((int) tag);
if (cmp < 0) {
bu_predict(phrasetable->left,state);
} else if (cmp > 0) {
bu_predict(phrasetable->right,state);
} else {
State *next_state = state->next;
if (next_state == NULL || state->end.pos != next_state->start.pos) {
next_state = new State;
next_state->start = state->end;
next_state->end = state->end;
next_state->next = state->next;
next_state->needs_bind = false;
state->next = next_state;
}
State *next_state = state->next;
if (next_state == NULL || state->end.pos != next_state->start.pos) {
next_state = new State;
next_state->start = state->end;
next_state->end = state->end;
next_state->next = state->next;
next_state->needs_bind = false;
state->next = next_state;
}
for (size_t i = 0; i < phrasetable->value.n_items; i++) {
std::map<ref<PgfConcrLincat>, bool> visited;
//if (!td_reachable(state, phrasetable->items[i], visited))
// continue;
Item *item = bu_item(state, phrasetable->value.items[i]);
item->dot++;
next_state->push_item(item);
}
for (size_t i = 0; i < n_items; i++) {
//std::map<ref<PgfConcrLincat>, bool> visited;
//if (!td_reachable(state, phrasetable->items[i], visited))
// continue;
Item *item = bu_item(state, items[i]);
item->dot++;
next_state->push_item(item);
}
}
void PgfParser::bu_predict(State *state, CCat *ccat)
{
size_t n_items = 0;
vector<ref<PgfItem>> items =
phrasetable_lookup(concr->phrasetable,
ccat->cont->lincat,
&n_items);
vector<ref<PgfItem>> items = 0;
if (ccat->cont->ccat == NULL) {
items = phrasetable_lookup(concr->phrasetable2,
ccat->cont->lincat,
&n_items);
} else if (ccat->cont->ccat->epsilon != 0) {
items = phrasetable_lookup(concr->phrasetable3,
ccat->cont->ccat->epsilon,
&n_items);
}
for (size_t i = 0; i < n_items; i++) {
std::map<ref<PgfConcrLincat>, bool> visited;
//std::map<ref<PgfConcrLincat>, bool> visited;
//if (!td_reachable(ccat->cont->state, items[i], visited))
// continue;
auto new_item = bu_item(ccat->cont->state, items[i]);
@@ -976,7 +947,7 @@ bool PgfParser::td_reachable(State *state, ref<PgfItem> pitem,
size_t n_items = 0;
vector<ref<PgfItem>> items =
phrasetable_lookup(concr->phrasetable,
phrasetable_lookup(concr->phrasetable2,
lin->lincat,
&n_items);
for (size_t i = 0; i < n_items; i++) {
@@ -997,12 +968,26 @@ PgfAbstractParser::Item *PgfParser::bu_item(State *state, ref<PgfItem> pitem)
case PgfConcrLin::tag: {
auto lin = ref<PgfConcrLin>::untagged(pitem->rule->container);
Cont *&cont = state->conts1[lin->lincat];
if (cont == NULL) {
cont = new Cont;
cont->ccat = NULL;
cont->lincat = lin->lincat;
cont->state = state;
Cont *cont;
if (pitem->res == 0) {
Cont *&tmp = state->conts1[lin->lincat];
if (tmp == NULL) {
tmp = new Cont;
tmp->ccat = NULL;
tmp->lincat = lin->lincat;
tmp->state = state;
}
cont = tmp;
} else {
CCat *ccat = get_epsilon_ccat(&lin->lincat->name,pitem->res);
Cont *&tmp = state->conts2[ccat];
if (tmp == NULL) {
tmp = new Cont;
tmp->ccat = ccat;
tmp->lincat = lin->lincat;
tmp->state = state;
}
cont = tmp;
}
item = new(pitem->rule) Item;
@@ -1014,6 +999,14 @@ PgfAbstractParser::Item *PgfParser::bu_item(State *state, ref<PgfItem> pitem)
item->rule = pitem->rule;
item->inside_prob = lin->absfun->prob;
item->outside_prob = 0;
for (size_t i = 0; i < pitem->args.size(); i++) {
item->args[i] = 0;
if (pitem->args[i] != 0) {
item->args[i] = get_epsilon_ccat(&lin->absfun->type->hypos[i].type->name,pitem->args[i]);
item->inside_prob += item->args[i]->viterbi_prob;
}
}
break;
}
case PgfConcrLincat::tag: {
@@ -1036,6 +1029,7 @@ PgfAbstractParser::Item *PgfParser::bu_item(State *state, ref<PgfItem> pitem)
item->rule = pitem->rule;
item->inside_prob = 0;
item->outside_prob = 0;
item->args[0] = 0;
break;
}
}
@@ -1050,29 +1044,6 @@ PgfAbstractParser::Item *PgfParser::bu_item(State *state, ref<PgfItem> pitem)
}
memcpy(&item->vars[0], &pitem->vars[0], sizeof(size_t) * item->vars.size());
for (size_t i = 0; i < pitem->args.size(); i++) {
ref<PgfSymbolCCat> arg = pitem->args[i];
item->args[i] = 0;
if (arg != 0) {
CCat *&arg_ccat = epsilons[arg->lincat][arg->value][arg->lin_idx];
if (arg_ccat == NULL) {
arg_ccat = new CCat;
arg_ccat->fid = arg->fid;
arg_ccat->epsilons = arg.tagged();
arg_ccat->state = NULL;
arg_ccat->lin_idx = arg->lin_idx;
arg_ccat->value = arg->value;
arg_ccat->covered = true;
arg_ccat->viterbi_prob = arg->viterbi_prob;
}
item->args[i] = arg_ccat;
item->inside_prob += arg_ccat->viterbi_prob;
}
}
return item;
}
@@ -1136,7 +1107,7 @@ PgfExpr PgfParser::fetch(PgfDB *db, prob_t *prob)
{
DB_scope scope(db, READER_SCOPE);
bool first_fetch = (initial_fid == last_fid);
bool first_fetch = (concr->last_fid == last_fid);
for (;;) {
State *state = current_state;
@@ -1225,13 +1196,9 @@ PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob)
ccat->pending.push_back(estate);
if (ccat->pending.size() == 1) {
if (ccat->fid <= initial_fid) {
size_t n_items = 0;
vector<ref<PgfItem>> items =
phrasetable_lookup(concr->phrasetable, ccat->epsilons, &n_items);
for (size_t i = 0; i < n_items; i++) {
ref<PgfItem> pitem = items[i];
if (ccat->epsilon != 0) {
for (size_t i = 0; i < ccat->epsilon->n_items; i++) {
ref<PgfItem> pitem = ccat->epsilon->items[i];
auto lin = ref<PgfConcrLin>::untagged(pitem->rule->container);
@@ -1246,22 +1213,10 @@ PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob)
new_estate->hash = new_estate->hash * 101 + lin->name.text[i];
}
for (size_t i = 0; i < new_estate->n_args; i++) {
ref<PgfSymbolCCat> arg = pitem->args[i];
new_estate->args[i] = NULL;
if (arg != 0) {
CCat *&arg_ccat = epsilons[arg->lincat][arg->value][arg->lin_idx];
if (arg_ccat == NULL) {
arg_ccat = new CCat;
arg_ccat->fid = arg->fid;
arg_ccat->epsilons = arg.tagged();
arg_ccat->state = NULL;
arg_ccat->lin_idx = arg->lin_idx;
arg_ccat->value = arg->value;
arg_ccat->covered = true;
arg_ccat->viterbi_prob = arg->viterbi_prob;
}
new_estate->args[i] = arg_ccat;
new_estate->prob += arg_ccat->viterbi_prob;
if (pitem->args[i] != 0) {
new_estate->args[i] = get_epsilon_ccat(&lin->absfun->type->hypos[i].type->name,pitem->args[i]);
new_estate->prob += new_estate->args[i]->viterbi_prob;
}
}
queue.push_back(new_estate);
@@ -1373,10 +1328,10 @@ PgfAbstractParser::State *PgfParser::new_state(const PgfTextSpot &start)
return state;
}
void PgfParser::symbol_token(Item *item, State *state, PgfSymbol sym)
void PgfParser::symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks)
{
PgfTextSpot next = state->end;
if (text_symbol_cmp(&next,end,sym,case_sensitive) != 0)
if (text_symbol_cmp(&next,end,symks,case_sensitive) != 0)
return;
State *next_state = new_state(next);
@@ -1409,68 +1364,101 @@ void PgfParser::symbol_bind(Item *item, State *state, PgfSymbol sym)
}
}
void PgfParser::suspend(Cont *cont,Item *item,size_t n_suspended1,size_t n_suspended)
void PgfParser::suspend(Cont *cont,Item *item,bool do_predict,size_t n_suspended,ref<PgfSymbolCat> symcat)
{
if (n_suspended == 1) {
std::function<void(ref<PgfSymbolCCat>,size_t,vector<ref<PgfItem>>)> f =
[this,item,cont](ref<PgfSymbolCCat> symcf, size_t n_items, vector<ref<PgfItem>> items) {
if (cont->ccat == NULL) {
if (n_suspended == 1) {
std::function<void(ref<PgfCCat>)> f =
[this,item,cont](ref<PgfCCat> arg) {
ref<PgfItem> xitem = items[0];
ref<PgfItem> xitem = arg->items[0];
Item *new_item = new (item) Item;
PgfSymbol sym = new_item->rule->syms[new_item->dot];
auto sym_cat = ref<PgfSymbolCat>::untagged(sym);
if (!new_item->instantiate(new_item->rule->args[sym_cat->d],xitem->rule,&xitem->vars[0],xitem->rule->res)) {
delete new_item;
return;
}
if (!new_item->instantiate(ref<PgfLParam>::from_ptr(&sym_cat->r),xitem->rule,&xitem->vars[0],xitem->rule->lin_idx)) {
delete new_item;
return;
}
CCat *&arg_ccat = epsilons[symcf->lincat][symcf->value][symcf->lin_idx];
if (arg_ccat == NULL) {
arg_ccat = new CCat;
arg_ccat->fid = symcf->fid;
arg_ccat->epsilons = symcf.tagged();
arg_ccat->state = NULL;
arg_ccat->lin_idx = symcf->lin_idx;
arg_ccat->value = symcf->value;
arg_ccat->covered = true;
arg_ccat->viterbi_prob = symcf->viterbi_prob;
}
cont->state->completed[cont][symcf->value][symcf->lin_idx] = arg_ccat;
new_item->dot++;
new_item->args[sym_cat->d] = arg_ccat;
new_item->inside_prob += arg_ccat->viterbi_prob;
cont->state->push_item(new_item);
};
phrasetable_iter(concr->phrasetable,cont->lincat,f);
}
State *state = cont->state;
while (state != NULL) {
auto it1 = state->completed.find(cont);
if (it1 != state->completed.end()) {
for (auto it2 : it1->second) {
for (auto it3 : it2.second) {
Item *new_item = new (item) Item;
combine(state, new_item, it3.second);
PgfSymbol sym = new_item->rule->syms[new_item->dot];
auto sym_cat = ref<PgfSymbolCat>::untagged(sym);
if (!new_item->instantiate(new_item->rule->args[sym_cat->d],xitem->rule,&xitem->vars[0],xitem->rule->res)) {
delete new_item;
return;
}
if (!new_item->instantiate(ref<PgfLParam>::from_ptr(&sym_cat->r),xitem->rule,&xitem->vars[0],xitem->rule->lin_idx)) {
delete new_item;
return;
}
CCat *&arg_ccat = epsilons[arg->fid];
if (arg_ccat == NULL) {
arg_ccat = new CCat;
arg_ccat->fid = arg->fid;
arg_ccat->epsilon = arg;
arg_ccat->state = NULL;
arg_ccat->lin_idx = arg->lin_idx;
arg_ccat->value = arg->value;
arg_ccat->covered = true;
arg_ccat->viterbi_prob = arg->viterbi_prob;
}
cont->state->completed[cont][arg_ccat->value][arg_ccat->lin_idx] = arg_ccat;
new_item->dot++;
new_item->args[sym_cat->d] = arg_ccat;
new_item->inside_prob += arg_ccat->viterbi_prob;
cont->state->push_item(new_item);
};
epsilontable_iter(concr->epsilontable,cont->lincat,f);
}
State *state = cont->state;
while (state != NULL) {
auto it1 = state->completed.find(cont);
if (it1 != state->completed.end()) {
for (auto it2 : it1->second) {
for (auto it3 : it2.second) {
Item *new_item = new (item) Item;
combine(state, new_item, it3.second);
}
}
}
state = state->next;
}
state = state->next;
}
if (n_suspended1 == 0) {
if (cont->state->needs_bind) {
bu_predict(concr->phrasetable, cont->state);
if (do_predict) {
if (cont->state->needs_bind) {
bu_predict(concr->phrasetable4, cont->state);
} else {
bu_predict(concr->phrasetable1, cont->state, 1, sentence->size);
}
}
} else {
if (do_predict && n_suspended == 1) {
if (cont->ccat->epsilon != 0) {
for (size_t i = 0; i < cont->ccat->epsilon->n_items; i++) {
ref<PgfItem> pitem = cont->ccat->epsilon->items[i];
td_epsilon(cont->state,cont,pitem,item,symcat);
}
} else {
for (Production *prod : cont->ccat->prods) {
td_predict(cont->state,cont,prod,item,symcat);
}
}
} else {
bu_predict(concr->phrasetable, cont->state, 1, sentence->size);
State *next = cont->state;
while (next != NULL) {
auto it1 = next->completed.find(cont);
if (it1 != next->completed.end()) {
auto *it2 = it1->second.lookup(cont->ccat->value);
if (it2 != NULL) {
interval_t lin_idx_i = item->interval(ref<PgfLParam>::from_ptr(&symcat->r));
auto *it3 = it2->lookup(lin_idx_i);
if (it3 != NULL) {
CCat *arg = *it3;
Item *new_item = new (item) Item;
combine(next, new_item, arg);
}
}
}
next = next->next;
}
}
}
}
@@ -1561,9 +1549,10 @@ PgfParseTableMaker::PgfParseTableMaker(ref<PgfConcr> concr)
ref<PgfItem> PgfParseTableMaker::clone_item(Item *item)
{
size_t ex_size =
sizeof(ref<PgfSymbolCCat>) * item->args.size() +
sizeof(size_t) * item->vars.size();
sizeof(PgfMetaId) * item->args.size() +
sizeof(size_t) * item->vars.size();
auto pitem = PgfDB::malloc<PgfItem>(ex_size);
pitem->res = (item->cont->ccat == NULL) ? 0 : item->cont->ccat->fid;
pitem->pre_alt = item->pre_alt;
pitem->pre_dot = item->pre_dot;
pitem->dot = item->dot;
@@ -1571,16 +1560,7 @@ ref<PgfItem> PgfParseTableMaker::clone_item(Item *item)
memcpy(&pitem->vars[0],&item->vars[0],sizeof(size_t) * item->vars.size());
for (size_t i = 0; i < item->args.size(); i++) {
ref<PgfSymbolCCat> symcf = 0;
if (item->args[i] != NULL) {
symcf = PgfDB::malloc<PgfSymbolCCat>();
symcf->lincat = item->args[i]->cont->lincat;
symcf->value = item->args[i]->value;
symcf->lin_idx = item->args[i]->lin_idx;
symcf->fid = item->args[i]->fid;
symcf->viterbi_prob = item->args[i]->viterbi_prob;
}
pitem->args[i] = symcf;
pitem->args[i] = (item->args[i] == NULL) ? 0 : item->args[i]->fid;
}
return pitem;
@@ -1591,19 +1571,19 @@ PgfAbstractParser::State *PgfParseTableMaker::new_state(const PgfTextSpot &start
return current_state;
}
void PgfParseTableMaker::symbol_token(Item *item, State *state, PgfSymbol sym)
void PgfParseTableMaker::symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks)
{
auto pitem = clone_item(item);
auto phrasetable = phrasetable_insert(concr->phrasetable,sym,pitem);
concr->phrasetable = phrasetable;
auto phrasetable1 = phrasetable_insert(concr->phrasetable1,symks,pitem);
concr->phrasetable1 = phrasetable1;
delete item;
}
void PgfParseTableMaker::symbol_bind(Item *item, State *state, PgfSymbol sym)
{
auto pitem = clone_item(item);
auto phrasetable = phrasetable_insert(concr->phrasetable,ref<PgfSymbolBIND>(0).tagged(),pitem);
concr->phrasetable = phrasetable;
auto phrasetable4 = phrasetable_insert(concr->phrasetable4,ref<PgfSymbolBIND>(0),pitem);
concr->phrasetable4 = phrasetable4;
if (ref<PgfSymbol>::get_tag(sym) == PgfSymbolBIND::tag) {
delete item;
@@ -1613,33 +1593,77 @@ void PgfParseTableMaker::symbol_bind(Item *item, State *state, PgfSymbol sym)
}
}
void PgfParseTableMaker::suspend(Cont *cont,Item *item,size_t n_suspended1,size_t n_suspended)
void PgfParseTableMaker::suspend(Cont *cont,Item *item,bool do_predict,size_t n_suspended,ref<PgfSymbolCat> symcat)
{
for (auto it1 : cont->state->completed[cont]) {
for (auto it2 : it1.second) {
CCat *ccat = it2.second;
if (ccat != NULL) {
Item *new_item = new (item) Item;
combine(cont->state,new_item,ccat);
if (cont->ccat == NULL) {
for (auto it1 : cont->state->completed[cont]) {
for (auto it2 : it1.second) {
CCat *ccat = it2.second;
if (ccat != NULL) {
Item *new_item = new (item) Item;
combine(cont->state,new_item,ccat);
}
}
}
}
auto pitem = clone_item(item);
auto acat = ref<PgfSymbolACat>::from_ptr((PgfSymbolACat*) &cont->lincat->name);
auto phrasetable = phrasetable_insert(concr->phrasetable,acat.tagged(),pitem);
concr->phrasetable = phrasetable;
auto pitem = clone_item(item);
auto phrasetable2 = phrasetable_insert(concr->phrasetable2,cont->lincat,pitem);
concr->phrasetable2 = phrasetable2;
} else {
if (do_predict && n_suspended == 1) {
if (cont->ccat->epsilon != 0) {
for (size_t i = 0; i < cont->ccat->epsilon->n_items; i++) {
ref<PgfItem> pitem = cont->ccat->epsilon->items[i];
td_epsilon(cont->state,cont,pitem,item,symcat);
}
} else {
for (Production *prod : cont->ccat->prods) {
td_predict(cont->state,cont,prod,item,symcat);
}
}
} else {
State *next = cont->state;
while (next != NULL) {
auto it1 = next->completed.find(cont);
if (it1 != next->completed.end()) {
auto *it2 = it1->second.lookup(cont->ccat->value);
if (it2 != NULL) {
interval_t lin_idx_i = item->interval(ref<PgfLParam>::from_ptr(&symcat->r));
auto *it3 = it2->lookup(lin_idx_i);
if (it3 != NULL) {
CCat *arg = *it3;
Item *new_item = new (item) Item;
combine(next, new_item, arg);
}
}
}
next = next->next;
}
}
auto pitem = clone_item(item);
auto phrasetable3 = phrasetable_insert(concr->phrasetable3,cont->ccat->epsilon,pitem);
concr->phrasetable3 = phrasetable3;
}
}
void PgfParseTableMaker::final_item(State *state, CCat *ccat, Item *item, interval_t value, interval_t lin_idx)
{
auto pitem = clone_item(item);
PgfPhrasetable phrasetable = concr->phrasetable;
phrasetable = phrasetable_insert(phrasetable,
item->cont->lincat, value, lin_idx, ccat->fid, ccat->viterbi_prob,
pitem);
concr->phrasetable = phrasetable;
if (ccat->epsilon == 0) {
PgfEpsilontable epsilontable = concr->epsilontable;
epsilontable =
epsilontable_insert(epsilontable,
ccat->cont->lincat,
ccat->value, ccat->lin_idx,
ccat->fid, ccat->viterbi_prob,
pitem,
&ccat->epsilon);
concr->epsilontable = epsilontable;
} else {
epsilontable_add(ccat->epsilon, pitem);
}
}
void PgfParseTableMaker::bu_predict(State *state, CCat *ccat)
src/runtime/c/pgf/parser.h
+12 -13
View File
@@ -82,10 +82,8 @@ protected:
struct CCat {
PgfMetaId fid;
union {
object epsilons;
Cont *cont;
};
ref<PgfCCat> epsilon;
Cont *cont;
State *state;
interval_t value;
interval_t lin_idx;
@@ -231,7 +229,7 @@ protected:
};
State *current_state;
std::map<ref<PgfConcrLincat>,interval_map<interval_map<CCat*>>> epsilons;
std::map<PgfMetaId,CCat*> epsilons;
PgfMetaId initial_fid, last_fid;
void process(Item *item, State *state);
@@ -239,9 +237,9 @@ protected:
void complete(Item *item, State *state);
virtual State *new_state(const PgfTextSpot &start)=0;
virtual void symbol_token(Item *item, State *state, PgfSymbol sym)=0;
virtual void symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks)=0;
virtual void symbol_bind(Item *item, State *state, PgfSymbol sym)=0;
virtual void suspend(Cont *cont, Item *item, size_t n_suspended1, size_t n_suspended)=0;
virtual void suspend(Cont *cont, Item *item, bool do_predict, size_t n_suspended,ref<PgfSymbolCat> symcat)=0;
virtual void final_item(State *state,CCat *ccat,Item *item,interval_t value,interval_t lin_idx)=0;
virtual void bu_predict(State *state, CCat *ccat)=0;
@@ -250,6 +248,7 @@ protected:
void combine(State *state, Item *item, CCat *ccat);
void get_info(CCat *ccat, ref<PgfConcrRule> *rule, size_t **pvalues);
CCat *get_epsilon_ccat(PgfText *name, PgfMetaId fid);
static
void print_item(Item *item, State *state);
@@ -271,14 +270,14 @@ class PGF_INTERNAL_DECL PgfParser : private PgfAbstractParser, public PgfExprEnu
bool case_sensitive;
virtual State *new_state(const PgfTextSpot &start);
virtual void symbol_token(Item *item, State *state, PgfSymbol sym);
virtual void symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks);
virtual void symbol_bind(Item *item, State *state, PgfSymbol sym);
virtual void suspend(Cont *cont,Item *item,size_t n_suspended1,size_t n_suspended);
virtual void suspend(Cont *cont,Item *item,bool do_predict,size_t n_suspended,ref<PgfSymbolCat> symcat);
virtual void final_item(State *state,CCat *ccat,Item *item,interval_t value,interval_t lin_idx);
virtual void bu_predict(State *state, CCat *ccat);
void bu_predict(PgfPhrasetable phrasetable, State *state);
void bu_predict(PgfPhrasetable phrasetable, State *state, ptrdiff_t min, ptrdiff_t max);
void bu_predict(PgfPhrasetable<PgfSymbolBIND> phrasetable, State *state);
void bu_predict(PgfPhrasetable<PgfSymbolKS> phrasetable, State *state, ptrdiff_t min, ptrdiff_t max);
void make_chunks(State *state, std::vector<CCat*> &chunks, prob_t prob);
PgfExpr process_expr(ExprState *estate, prob_t *prob);
@@ -313,9 +312,9 @@ class PGF_INTERNAL_DECL PgfParseTableMaker : private PgfAbstractParser
{
private:
virtual State *new_state(const PgfTextSpot &start);
virtual void symbol_token(Item *item, State *state, PgfSymbol sym);
virtual void symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks);
virtual void symbol_bind(Item *item, State *state, PgfSymbol sym);
virtual void suspend(Cont *cont, Item *item, size_t n_suspended1, size_t n_suspended);
virtual void suspend(Cont *cont, Item *item, bool do_predict, size_t n_suspended,ref<PgfSymbolCat> symcat);
virtual void final_item(State *state, CCat *ccat,Item *item,interval_t value,interval_t lin_idx);
virtual void bu_predict(State *state, CCat *ccat);
src/runtime/c/pgf/pgf.cxx
+12 -4
View File
@@ -1022,7 +1022,7 @@ void pgf_lookup_morpho(PgfDB *db, PgfConcrRevision cnc_revision,
PgfMorphoScanner scanner(callback);
size_t n_items;
phrasetable_lookup(concr->phrasetable,
phrasetable_lookup(concr->phrasetable1,
sentence, case_sensitive,
&scanner, err);
} PGF_API_END
@@ -1072,7 +1072,7 @@ void pgf_lookup_cohorts(PgfDB *db, PgfConcrRevision cnc_revision,
bool case_sensitive = pgf_is_case_sensitive(concr);
PgfCohortsScanner scanner(callback);
phrasetable_lookup_cohorts(concr->phrasetable,
phrasetable_lookup_cohorts(concr->phrasetable1,
sentence, case_sensitive,
&scanner, err);
} PGF_API_END
@@ -1485,7 +1485,11 @@ ref<PgfConcr> clone_concrete(ref<PgfPGF> pgf, ref<PgfConcr> concr)
clone->cflags = concr->cflags;
clone->lins = concr->lins;
clone->lincats = concr->lincats;
clone->phrasetable = concr->phrasetable;
clone->phrasetable1 = concr->phrasetable1;
clone->phrasetable2 = concr->phrasetable2;
clone->phrasetable3 = concr->phrasetable3;
clone->phrasetable4 = concr->phrasetable4;
clone->epsilontable = concr->epsilontable;
clone->printnames = concr->printnames;
clone->last_fid = concr->last_fid;
memcpy(&clone->name, &concr->name, sizeof(PgfText)+concr->name.size+1);
@@ -1664,7 +1668,11 @@ PgfConcrRevision pgf_create_concrete(PgfDB *db, PgfRevision revision,
concr->cflags = 0;
concr->lins = 0;
concr->lincats = 0;
concr->phrasetable = 0;
concr->phrasetable1 = 0;
concr->phrasetable2 = 0;
concr->phrasetable3 = 0;
concr->phrasetable4 = 0;
concr->epsilontable = 0;
concr->printnames = 0;
concr->last_fid = 0;
memcpy(&concr->name, name, sizeof(PgfText)+name->size+1);
src/runtime/c/pgf/phrasetable.cxx
+183 -335
View File
@@ -31,169 +31,12 @@ int lparam_cmp(PgfLParam *p1, PgfLParam *p2)
return 0;
}
static
int sequence_cmp(vector<PgfSymbol> seq1, vector<PgfSymbol> seq2);
static
void symbol_cmp(PgfSymbol sym1, PgfSymbol sym2, int res[2])
{
uint8_t t1 = ref<PgfSymbol>::get_tag(sym1);
uint8_t t2 = ref<PgfSymbol>::get_tag(sym2);
if (t1 != t2) {
res[0] = (res[1] = ((int) t1) - ((int) t2));
return;
}
switch (t1) {
case PgfSymbolCat::tag: {
auto sym_cat1 = ref<PgfSymbolCat>::untagged(sym1);
auto sym_cat2 = ref<PgfSymbolCat>::untagged(sym2);
if (sym_cat1->d < sym_cat2->d)
res[0] = (res[1] = -1);
else if (sym_cat1->d > sym_cat2->d)
res[0] = (res[1] = 1);
else
res[0] = (res[1] = lparam_cmp(&sym_cat1->r, &sym_cat2->r));
break;
}
case PgfSymbolLit::tag: {
auto sym_lit1 = ref<PgfSymbolLit>::untagged(sym1);
auto sym_lit2 = ref<PgfSymbolLit>::untagged(sym2);
if (sym_lit1->d < sym_lit2->d)
res[0] = (res[1] = -1);
else if (sym_lit1->d > sym_lit2->d)
res[0] = (res[1] = 1);
else
res[0] = (res[1] = lparam_cmp(&sym_lit1->r, &sym_lit2->r));
break;
}
case PgfSymbolVar::tag: {
auto sym_var1 = ref<PgfSymbolVar>::untagged(sym1);
auto sym_var2 = ref<PgfSymbolVar>::untagged(sym2);
if (sym_var1->d < sym_var2->d)
res[0] = (res[1] = -1);
else if (sym_var1->d > sym_var2->d)
res[0] = (res[1] = 1);
else if (sym_var1->r < sym_var2->r)
res[0] = (res[1] = -1);
else if (sym_var1->r > sym_var2->r)
res[0] = (res[1] = 1);
break;
}
case PgfSymbolKS::tag: {
auto sym_ks1 = ref<PgfSymbolKS>::untagged(sym1);
auto sym_ks2 = ref<PgfSymbolKS>::untagged(sym2);
texticmp(&sym_ks1->token,&sym_ks2->token,res);
break;
}
case PgfSymbolKP::tag: {
auto sym_kp1 = ref<PgfSymbolKP>::untagged(sym1);
auto sym_kp2 = ref<PgfSymbolKP>::untagged(sym2);
res[0] = (res[1] = sequence_cmp(sym_kp1->default_form, sym_kp2->default_form));
if (res[0] != 0)
return;
for (size_t i = 0; ; i++) {
if (i >= sym_kp1->alts.size()) {
res[0] = (res[1] = -(i < sym_kp2->alts.size()));
return;
}
if (i >= sym_kp2->alts.size()) {
res[0] = (res[1] = 1);
return;
}
res[0] = (res[1] = sequence_cmp(sym_kp1->alts[i].form, sym_kp2->alts[i].form));
if (res[0] != 0)
return;
vector<ref<PgfText>> prefixes1 = sym_kp1->alts[i].prefixes;
vector<ref<PgfText>> prefixes2 = sym_kp2->alts[i].prefixes;
for (size_t j = 0; ; j++) {
if (j >= prefixes1.size()) {
res[0] = (res[1] = -(j < prefixes2.size()));
return;
}
if (j >= prefixes2.size()) {
res[0] = (res[1] = 1);
return;
}
res[0] = (res[1] = textcmp(&*prefixes1[j], &*prefixes2[j]));
if (res[0] != 0)
return;
}
}
}
case PgfSymbolBIND::tag:
case PgfSymbolSOFTBIND::tag:
case PgfSymbolNE::tag:
case PgfSymbolSOFTSPACE::tag:
case PgfSymbolCAPIT::tag:
case PgfSymbolALLCAPIT::tag:
break;
case PgfSymbolACat::tag: {
auto sym_acat1 = ref<PgfSymbolACat>::untagged(sym1);
auto sym_acat2 = ref<PgfSymbolACat>::untagged(sym2);
res[0] = (res[1] = textcmp(&sym_acat1->name,&sym_acat2->name));
return;
}
case PgfSymbolCCat::tag: {
auto sym_ccat1 = ref<PgfSymbolCCat>::untagged(sym1);
auto sym_ccat2 = ref<PgfSymbolCCat>::untagged(sym2);
res[0] = (res[1] = textcmp(&sym_ccat1->lincat->name,&sym_ccat2->lincat->name));
if (res[0] != 0)
return;
if (sym_ccat1->value < sym_ccat2->value)
res[0] = (res[1] = -1);
else if (sym_ccat1->value > sym_ccat2->value)
res[0] = (res[1] = 1);
if (sym_ccat1->lin_idx < sym_ccat2->lin_idx)
res[0] = (res[1] = -1);
else if (sym_ccat1->lin_idx > sym_ccat2->lin_idx)
res[0] = (res[1] = 1);
else
res[0] = (res[1] = 0);
return;
}
default:
throw pgf_error("Unknown symbol tag");
}
}
static
int sequence_cmp(vector<PgfSymbol> seq1, vector<PgfSymbol> seq2)
{
int res[2] = {0,0};
for (size_t i = 0; ; i++) {
if (i >= seq1.size()) {
if (i < seq2.size())
return -1;
return res[1];
}
if (i >= seq2.size())
return 1;
symbol_cmp(seq1[i], seq2[i], res);
if (res[0] != 0)
return res[0];
}
return 0;
}
PGF_INTERNAL
int text_symbol_cmp(PgfTextSpot *spot, const uint8_t *end,
PgfSymbol sym, bool case_sensitive)
ref<PgfSymbolKS> sym_ks, bool case_sensitive)
{
uint8_t tag = ref<PgfSymbol>::get_tag(sym);
if (PgfSymbolKS::tag != tag)
return ((int) PgfSymbolKS::tag) - ((int) tag);
int res1 = 0;
auto sym_ks = ref<PgfSymbolKS>::untagged(sym);
const uint8_t *s2 = (uint8_t *) &sym_ks->token.text;
const uint8_t *e2 = s2+sym_ks->token.size;
@@ -326,134 +169,14 @@ bool text_item_match(PgfTextSpot *spot, const uint8_t *end,
PGF_INTERNAL_DECL
size_t get_next_padovan(size_t min);
static
int symbol_cmp(ref<PgfConcrLincat> lincat, interval_t value, interval_t lin_idx, PgfSymbol sym)
{
uint8_t tag = ref<PgfSymbol>::get_tag(sym);
if (PgfSymbolCCat::tag != tag)
return ((int) PgfSymbolCCat::tag) - ((int) tag);
auto symcf = ref<PgfSymbolCCat>::untagged(sym);
int res = textcmp(&lincat->name, &symcf->lincat->name);
if (res != 0)
return res;
if (value < symcf->value)
return -1;
else if (value > symcf->value)
return 1;
else if (lin_idx < symcf->lin_idx)
return -1;
else if (lin_idx > symcf->lin_idx)
return 1;
else
return 0;
}
static
int symbol_cmp(PgfSymbol sym1, PgfSymbol sym2)
{
uint8_t tag1 = ref<PgfSymbol>::get_tag(sym1);
uint8_t tag2 = ref<PgfSymbol>::get_tag(sym2);
if (tag1 != tag2)
return ((int) tag1) - ((int) tag2);
switch (tag1) {
case PgfSymbolKS::tag: {
auto symks1 = ref<PgfSymbolKS>::untagged(sym1);
auto symks2 = ref<PgfSymbolKS>::untagged(sym2);
int res[2] = {0,0};
texticmp(&symks1->token, &symks2->token, res);
if (res[0] != 0)
return res[0];
return res[1];
}
case PgfSymbolACat::tag: {
auto symcf1 = ref<PgfSymbolACat>::untagged(sym1);
auto symcf2 = ref<PgfSymbolACat>::untagged(sym2);
return textcmp(&symcf1->name, &symcf2->name);
}
case PgfSymbolCCat::tag: {
auto symcf1 = ref<PgfSymbolCCat>::untagged(sym1);
auto symcf2 = ref<PgfSymbolCCat>::untagged(sym2);
int res = textcmp(&symcf1->lincat->name, &symcf2->lincat->name);
if (res != 0)
return res;
if (symcf1->value < symcf2->value)
return -1;
else if (symcf1->value > symcf2->value)
return 1;
else if (symcf1->lin_idx < symcf2->lin_idx)
return -1;
else if (symcf1->lin_idx > symcf2->lin_idx)
return 1;
else
return 0;
}
default:
return 0;
}
}
template<class K>
PGF_INTERNAL
void phrasetable_iter(PgfPhrasetable table, ref<PgfConcrLincat> lincat, std::function<void(ref<PgfSymbolCCat> arg,size_t,vector<ref<PgfItem>>)> &f)
{
if (table == 0)
return;
int cmp = 0;
ref<PgfSymbolCCat> symcf = 0;
uint8_t tag = ref<PgfSymbol>::get_tag(table->value.sym);
if (PgfSymbolCCat::tag != tag) {
cmp = ((int) PgfSymbolCCat::tag) - ((int) tag);
} else {
symcf = ref<PgfSymbolCCat>::untagged(table->value.sym);
cmp = textcmp(&lincat->name, &symcf->lincat->name);
}
if (cmp < 0)
phrasetable_iter(table->left, lincat, f);
else if (cmp > 0)
phrasetable_iter(table->right, lincat, f);
else {
phrasetable_iter(table->left, lincat, f);
f(symcf,table->value.n_items,table->value.items);
phrasetable_iter(table->right, lincat, f);
}
}
PGF_INTERNAL
vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable table, PgfSymbol sym, size_t *n_items)
{
while (table != 0) {
int cmp = symbol_cmp(sym,table->value.sym);
if (cmp < 0)
table = table->left;
else if (cmp > 0)
table = table->right;
else {
*n_items = table->value.n_items;
return table->value.items;
}
}
*n_items = 0;
return 0;
}
PGF_INTERNAL
vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable phrasetable,
ref<PgfConcrLincat> lincat,
vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable<K> phrasetable,
ref<K> key,
size_t *n_items)
{
while (phrasetable != 0) {
int cmp;
uint8_t tag = ref<PgfSymbol>::get_tag(phrasetable->value.sym);
if (PgfSymbolACat::tag != tag) {
cmp = ((int) PgfSymbolACat::tag) - ((int) tag);
} else {
auto symcf = ref<PgfSymbolACat>::untagged(phrasetable->value.sym);
cmp = textcmp(&lincat->name, &symcf->name);
}
int cmp = compare_key(key, phrasetable->value.key);
if (cmp < 0)
phrasetable = phrasetable->left;
else if (cmp > 0)
@@ -469,7 +192,7 @@ vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable phrasetable,
}
PGF_INTERNAL
void phrasetable_lookup(PgfPhrasetable table,
void phrasetable_lookup(PgfPhrasetable<PgfSymbolKS> table,
PgfText *sentence,
bool case_sensitive,
PgfPhraseScanner *scanner, PgfExn* err)
@@ -481,7 +204,7 @@ void phrasetable_lookup(PgfPhrasetable table,
spot.pos = 0;
spot.ptr = (uint8_t *) sentence->text;
const uint8_t *end = spot.ptr+sentence->size;
int cmp = text_symbol_cmp(&spot,end,table->value.sym,case_sensitive);
int cmp = text_symbol_cmp(&spot,end,table->value.key,case_sensitive);
if (cmp < 0) {
phrasetable_lookup(table->left,sentence,case_sensitive,scanner,err);
} else if (cmp > 0) {
@@ -587,14 +310,14 @@ void finish_skipping(PgfCohortsState *state) {
static
void phrasetable_lookup_prefixes(PgfCohortsState *state,
PgfPhrasetable table,
PgfPhrasetable<PgfSymbolKS> table,
ptrdiff_t min, ptrdiff_t max)
{
if (table == 0)
return;
PgfTextSpot current = state->spot;
int cmp = text_symbol_cmp(&current,state->end,table->value.sym,state->case_sensitive);
int cmp = text_symbol_cmp(&current,state->end,table->value.key,state->case_sensitive);
if (cmp < 0) {
phrasetable_lookup_prefixes(state,table->left,min,max);
} else if (cmp > 0) {
@@ -661,7 +384,7 @@ void phrasetable_lookup_prefixes(PgfCohortsState *state,
}
PGF_INTERNAL
void phrasetable_lookup_cohorts(PgfPhrasetable table,
void phrasetable_lookup_cohorts(PgfPhrasetable<PgfSymbolKS> table,
PgfText *sentence,
bool case_sensitive,
PgfPhraseScanner *scanner, PgfExn* err)
@@ -748,29 +471,29 @@ void phrasetable_lookup_cohorts(PgfPhrasetable table,
}
}
template<class K>
PGF_INTERNAL
PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
PgfSymbol sym,
ref<PgfItem> item)
PgfPhrasetable<K> phrasetable_insert(PgfPhrasetable<K> table,
ref<K> key, ref<PgfItem> item)
{
if (table == 0) {
auto items = vector<ref<PgfItem>>::alloc(1);
items[0] = item;
return Node<PgfPhrasetableValue>::new_node({.sym=sym,.n_items=1,.items=items});
return Node<PgfPhrasetableValue<K>>::new_node({.key=key,.n_items=1,.items=items});
}
int cmp = symbol_cmp(sym,table->value.sym);
int cmp = compare_key(key, table->value.key);
if (cmp < 0) {
PgfPhrasetable left = phrasetable_insert(table->left, sym, item);
table = Node<PgfPhrasetableValue>::upd_node(table,left,table->right);
return Node<PgfPhrasetableValue>::balanceL(table);
PgfPhrasetable<K> left = phrasetable_insert(table->left, key, item);
table = Node<PgfPhrasetableValue<K>>::upd_node(table,left,table->right);
return Node<PgfPhrasetableValue<K>>::balanceL(table);
} else if (cmp > 0) {
PgfPhrasetable right = phrasetable_insert(table->right, sym, item);
table = Node<PgfPhrasetableValue>::upd_node(table, table->left, right);
return Node<PgfPhrasetableValue>::balanceR(table);
PgfPhrasetable<K> right = phrasetable_insert(table->right, key, item);
table = Node<PgfPhrasetableValue<K>>::upd_node(table, table->left, right);
return Node<PgfPhrasetableValue<K>>::balanceR(table);
} else {
PgfPhrasetable new_table =
Node<PgfPhrasetableValue>::upd_node(table, table->left, table->right);
PgfPhrasetable<K> new_table =
Node<PgfPhrasetableValue<K>>::upd_node(table, table->left, table->right);
auto items = new_table->value.items;
if (new_table->value.n_items >= items.size()) {
@@ -784,62 +507,187 @@ PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
}
}
static
int compare_key(ref<PgfSymbolKS> symks1, ref<PgfSymbolKS> symks2) {
int res[2] = {0,0};
texticmp(&symks1->token, &symks2->token, res);
if (res[0] != 0)
return res[0];
return res[1];
}
template
PgfPhrasetable<PgfSymbolKS> phrasetable_insert<PgfSymbolKS>(PgfPhrasetable<PgfSymbolKS> table,
ref<PgfSymbolKS> key,
ref<PgfItem> item);
static
int compare_key(ref<PgfConcrLincat> lincat1, ref<PgfConcrLincat> lincat2) {
return textcmp(&lincat1->name, &lincat2->name);
}
template
PGF_INTERNAL
PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
ref<PgfConcrLincat> lincat,
interval_t value, interval_t lin_idx,
PgfMetaId fid, prob_t viterbi_prob,
ref<PgfItem> item)
PgfPhrasetable<PgfConcrLincat> phrasetable_insert<PgfConcrLincat>(PgfPhrasetable<PgfConcrLincat> table,
ref<PgfConcrLincat> key,
ref<PgfItem> item);
template
PGF_INTERNAL
vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable<PgfConcrLincat> phrasetable,
ref<PgfConcrLincat> key,
size_t *n_items);
static
int compare_key(ref<PgfCCat> ccat1, ref<PgfCCat> ccat2) {
return ((int) ccat1->fid) - ((int) ccat2->fid);
}
template
PgfPhrasetable<PgfCCat> phrasetable_insert<PgfCCat>(PgfPhrasetable<PgfCCat> table,
ref<PgfCCat> key,
ref<PgfItem> item);
template
PGF_INTERNAL
vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable<PgfCCat> phrasetable,
ref<PgfCCat> key,
size_t *n_items);
static
int compare_key(ref<PgfSymbolBIND> symbind1, ref<PgfSymbolBIND> symbind2) {
return 0;
}
template
PgfPhrasetable<PgfSymbolBIND> phrasetable_insert<PgfSymbolBIND>(PgfPhrasetable<PgfSymbolBIND> table,
ref<PgfSymbolBIND> key,
ref<PgfItem> item);
template
PGF_INTERNAL
vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable<PgfSymbolBIND> phrasetable,
ref<PgfSymbolBIND> key,
size_t *n_items);
PGF_INTERNAL
PgfEpsilontable epsilontable_insert(PgfEpsilontable table,
ref<PgfConcrLincat> lincat,
interval_t value, interval_t lin_idx,
PgfMetaId fid, prob_t viterbi_prob,
ref<PgfItem> item,
ref<PgfCCat> *pepsilon)
{
if (table == 0) {
ref<PgfSymbolCCat> symcf = PgfDB::malloc<PgfSymbolCCat>();
symcf->lincat = lincat;
symcf->value = value;
symcf->lin_idx = lin_idx;
symcf->fid = fid;
symcf->viterbi_prob = viterbi_prob;
auto items = vector<ref<PgfItem>>::alloc(1);
items[0] = item;
return Node<PgfPhrasetableValue>::new_node({.sym=symcf.tagged(),.n_items=1,.items=items});
PgfEpsilontable new_table =
Node<PgfCCat>::new_node({.lincat=lincat,
.fid=fid,
.value=value,
.lin_idx=lin_idx,
.viterbi_prob=viterbi_prob,
.n_items=1,
.items=items});
*pepsilon = ref<PgfCCat>::from_ptr(&new_table->value);
return new_table;
}
int cmp = symbol_cmp(lincat,value,lin_idx,table->value.sym);
if (cmp < 0) {
PgfPhrasetable left = phrasetable_insert(table->left,
lincat, value, lin_idx, fid, viterbi_prob, item);
table = Node<PgfPhrasetableValue>::upd_node(table,left,table->right);
return Node<PgfPhrasetableValue>::balanceL(table);
} else if (cmp > 0) {
PgfPhrasetable right = phrasetable_insert(table->right,
lincat, value, lin_idx, fid, viterbi_prob, item);
table = Node<PgfPhrasetableValue>::upd_node(table, table->left, right);
return Node<PgfPhrasetableValue>::balanceR(table);
} else {
PgfPhrasetable new_table =
Node<PgfPhrasetableValue>::upd_node(table, table->left, table->right);
int cmp = textcmp(&lincat->name, &table->value.lincat->name);
if (cmp == 0) {
cmp = ((int)fid) - ((int)table->value.fid);
}
auto items = new_table->value.items;
if (new_table->value.n_items >= items.size()) {
size_t new_len = get_next_padovan(new_table->value.n_items+1);
items = items.realloc(new_len, new_table->txn_id);
if (cmp < 0) {
PgfEpsilontable left = epsilontable_insert(table->left,
lincat, value, lin_idx, fid, viterbi_prob, item, pepsilon);
table = Node<PgfCCat>::upd_node(table,left,table->right);
return Node<PgfCCat>::balanceL(table);
} else if (cmp > 0) {
PgfEpsilontable right = epsilontable_insert(table->right,
lincat, value, lin_idx, fid, viterbi_prob, item, pepsilon);
table = Node<PgfCCat>::upd_node(table, table->left, right);
return Node<PgfCCat>::balanceR(table);
} else {
PgfEpsilontable new_table =
Node<PgfCCat>::upd_node(table, table->left, table->right);
auto items = table->value.items;
if (table->value.n_items >= items.size()) {
size_t new_len = get_next_padovan(table->value.n_items+1);
items = items.realloc(new_len, table->txn_id);
}
items[new_table->value.n_items] = item;
items[table->value.n_items] = item;
new_table->value.n_items++;
new_table->value.items = items;
*pepsilon = ref<PgfCCat>::from_ptr(&new_table->value);
return new_table;
}
}
PGF_INTERNAL
void phrasetable_release(PgfPhrasetable table)
void epsilontable_add(ref<PgfCCat> epsilon,ref<PgfItem> item)
{
auto items = epsilon->items;
if (epsilon->n_items >= items.size()) {
size_t new_len = get_next_padovan(epsilon->n_items+1);
items = items.realloc(new_len, PgfDB::get_txn_id());
}
items[epsilon->n_items] = item;
epsilon->n_items++;
epsilon->items = items;
}
PGF_INTERNAL
ref<PgfCCat> epsilontable_get(PgfEpsilontable table,
PgfText *name, PgfMetaId fid)
{
if (table == 0) {
return 0;
}
int cmp = textcmp(name, &table->value.lincat->name);
if (cmp == 0) {
cmp = ((int)fid) - ((int)table->value.fid);
}
if (cmp < 0) {
return epsilontable_get(table->left,name,fid);
} else if (cmp > 0) {
return epsilontable_get(table->right,name,fid);
} else {
return ref<PgfCCat>::from_ptr(&table->value);
}
}
PGF_INTERNAL
void epsilontable_iter(PgfEpsilontable table, ref<PgfConcrLincat> lincat, std::function<void(ref<PgfCCat> arg)> &f)
{
if (table == 0)
return;
phrasetable_release(table->left);
phrasetable_release(table->right);
int cmp = textcmp(&lincat->name, &table->value.lincat->name);
if (cmp < 0)
epsilontable_iter(table->left, lincat, f);
else if (cmp > 0)
epsilontable_iter(table->right, lincat, f);
else {
epsilontable_iter(table->left, lincat, f);
f(ref<PgfCCat>::from_ptr(&table->value));
epsilontable_iter(table->right, lincat, f);
}
}
PGF_INTERNAL
void epsilontable_release(PgfEpsilontable table)
{
if (table == 0)
return;
epsilontable_release(table->left);
epsilontable_release(table->right);
for (size_t i = 0; i < table->value.n_items; i++) {
PgfItem::release(table->value.items[i]);
}
vector<ref<PgfItem>>::release(table->value.items);
Node<PgfPhrasetableValue>::release(table);
Node<PgfCCat>::release(table);
}
src/runtime/c/pgf/phrasetable.h
+71 -25
View File
@@ -10,10 +10,12 @@ struct PGF_INTERNAL_DECL PgfTextSpot {
};
struct PGF_INTERNAL_DECL PgfItem {
PgfMetaId res;
struct {
size_t &operator[](int i) {
PgfItem *item = containerof(PgfItem,vars,this);
return ((size_t*) (((ref<PgfSymbolCCat>*) (item+1))+item->rule->args.size()))[i];
return ((size_t*) (((PgfMetaId*) (item+1))+item->rule->args.size()))[i];
}
size_t size() {
PgfItem *item = containerof(PgfItem,vars,this);
@@ -22,9 +24,9 @@ struct PGF_INTERNAL_DECL PgfItem {
} vars;
struct {
ref<PgfSymbolCCat> &operator[](int i) {
PgfMetaId &operator[](int i) {
PgfItem *item = containerof(PgfItem,args,this);
return ((ref<PgfSymbolCCat>*) (item+1))[i];
return ((PgfMetaId*) (item+1))[i];
}
size_t size() {
PgfItem *item = containerof(PgfItem,args,this);
@@ -35,8 +37,8 @@ struct PGF_INTERNAL_DECL PgfItem {
static
void release(ref<PgfItem> item) {
size_t ex_size =
sizeof(ref<PgfSymbolCCat>) * item->args.size() +
sizeof(size_t) * item->vars.size();
sizeof(PgfMetaId) * item->args.size() +
sizeof(size_t) * item->vars.size();
PgfDB::free(item, ex_size);
}
@@ -46,8 +48,11 @@ struct PGF_INTERNAL_DECL PgfItem {
ref<PgfConcrRule> rule;
};
struct PGF_INTERNAL_DECL PgfPhrasetableValue {
PgfSymbol sym;
struct PGF_INTERNAL_DECL PgfCCat {
ref<PgfConcrLincat> lincat;
PgfMetaId fid;
interval_t value, lin_idx;
prob_t viterbi_prob;
// Here n_items tells us how many actual items there are in
// the vector items. On the other hand, items.size() tells us
@@ -56,27 +61,29 @@ struct PGF_INTERNAL_DECL PgfPhrasetableValue {
vector<ref<PgfItem>> items;
};
typedef ref<Node<PgfPhrasetableValue>> PgfPhrasetable;
template<class K>
struct PGF_INTERNAL_DECL PgfPhrasetableValue {
ref<K> key;
PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
PgfSymbol sym,
ref<PgfItem> item);
// Here n_items tells us how many actual items there are in
// the vector items. On the other hand, items.size() tells us
// how big buffer we have allocated.
size_t n_items;
vector<ref<PgfItem>> items;
};
PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
ref<PgfConcrLincat> lincat,
interval_t value, interval_t lin_idx,
PgfMetaId fid, prob_t viterbi_prob,
ref<PgfItem> item);
template <class K>
using PgfPhrasetable = ref<Node<PgfPhrasetableValue<K>>>;
template<class K>
PGF_INTERNAL_DECL
void phrasetable_iter(PgfPhrasetable phrasetable,ref<PgfConcrLincat> lincat,std::function<void(ref<PgfSymbolCCat> symcf,size_t,vector<ref<PgfItem>>)> &f);
PgfPhrasetable<K> phrasetable_insert(PgfPhrasetable<K> table,
ref<K> key, ref<PgfItem> item);
template<class K>
PGF_INTERNAL_DECL
vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable phrasetable, PgfSymbol sym, size_t *n_items);
PGF_INTERNAL_DECL
vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable phrasetable,
ref<PgfConcrLincat> lincat,
vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable<K> phrasetable,
ref<K> key,
size_t *n_items);
class PGF_INTERNAL_DECL PgfPhraseScanner {
@@ -88,18 +95,57 @@ public:
};
PGF_INTERNAL_DECL
void phrasetable_lookup(PgfPhrasetable phrasetable,
void phrasetable_lookup(PgfPhrasetable<PgfSymbolKS> phrasetable,
PgfText *sentence,
bool case_sensitive,
PgfPhraseScanner *scanner, PgfExn* err);
PGF_INTERNAL_DECL
void phrasetable_lookup_cohorts(PgfPhrasetable phrasetable,
void phrasetable_lookup_cohorts(PgfPhrasetable<PgfSymbolKS> phrasetable,
PgfText *sentence,
bool case_sensitive,
PgfPhraseScanner *scanner, PgfExn* err);
template <class V>
void phrasetable_release(PgfPhrasetable<V> table)
{
if (table == 0)
return;
phrasetable_release(table->left);
phrasetable_release(table->right);
for (size_t i = 0; i < table->value.n_items; i++) {
PgfItem::release(table->value.items[i]);
}
vector<ref<PgfItem>>::release(table->value.items);
Node<PgfPhrasetableValue<V>>::release(table);
}
typedef ref<Node<PgfCCat>> PgfEpsilontable;
// Creates a new epsilon category with its first item.
// The new category is mutable within the current transaction
PGF_INTERNAL_DECL
void phrasetable_release(PgfPhrasetable table);
PgfEpsilontable epsilontable_insert(PgfEpsilontable table,
ref<PgfConcrLincat> lincat,
interval_t value, interval_t lin_idx,
PgfMetaId fid, prob_t viterbi_prob,
ref<PgfItem> item,
ref<PgfCCat> *pepsilon);
// Adds a new item to an existing epsilon category. The category
// must have been created by epsilontable_insert in the current transaction.
PGF_INTERNAL_DECL
void epsilontable_add(ref<PgfCCat> epsilon, ref<PgfItem> item);
PGF_INTERNAL_DECL
ref<PgfCCat> epsilontable_get(PgfEpsilontable table,
PgfText *name, PgfMetaId fid);
PGF_INTERNAL
void epsilontable_iter(PgfEpsilontable table, ref<PgfConcrLincat> lincat, std::function<void(ref<PgfCCat> arg)> &f);
PGF_INTERNAL_DECL
void epsilontable_release(PgfEpsilontable table);
#endif
src/runtime/c/pgf/printer.cxx
-14
View File
@@ -578,20 +578,6 @@ void PgfPrinter::symbol(PgfSymbol sym)
case PgfSymbolALLCAPIT::tag:
puts("ALL_CAPIT");
break;
case PgfSymbolACat::tag: {
auto symcf = ref<PgfSymbolACat>::untagged(sym);
efun(&symcf->name);
break;
}
case PgfSymbolCCat::tag: {
auto symcf = ref<PgfSymbolCCat>::untagged(sym);
efun(&symcf->lincat->name);
nprintf(64,"(%zu-%zu,%zu-%zu)",symcf->value.first
,symcf->value.second
,symcf->lin_idx.first
,symcf->lin_idx.second);
break;
}
}
}
src/runtime/c/pgf/reader.cxx
+5 -1
View File
@@ -697,7 +697,11 @@ ref<PgfConcrPrintname> PgfReader::read_printname()
ref<PgfConcr> PgfReader::read_concrete()
{
concrete = read_name(&PgfConcr::name);
concrete->phrasetable = 0;
concrete->phrasetable1 = 0;
concrete->phrasetable2 = 0;
concrete->phrasetable3 = 0;
concrete->phrasetable4 = 0;
concrete->epsilontable = 0;
concrete->last_fid = 0;
auto cflags = read_namespace<PgfFlag>(&PgfReader::read_flag);