forked from GitHub/gf-core
1898 lines
62 KiB
C++
1898 lines
62 KiB
C++
#include "data.h"
|
|
#include "printer.h"
|
|
#include "parser.h"
|
|
#include <math.h>
|
|
|
|
//#define DEBUG_PARSER
|
|
//#define DEBUG_EXPRS
|
|
|
|
PgfAbstractParser::PgfAbstractParser(ref<PgfConcr> concr)
|
|
{
|
|
this->concr = concr;
|
|
|
|
this->current_state = NULL;
|
|
this->last_fid = concr->last_fid;
|
|
}
|
|
|
|
void PgfAbstractParser::get_info(CCat *ccat, ref<PgfConcrRule> *prule, size_t **pvalues)
|
|
{
|
|
if (ccat->fid <= concr->last_fid) {
|
|
ref<PgfItem> pitem = ccat->epsilon->items[0];
|
|
*prule = pitem->rule;
|
|
*pvalues = &pitem->vars[0];
|
|
} else {
|
|
Production *prod = ccat->prods[0];
|
|
*prule = prod->rule;
|
|
*pvalues = &prod->vars[0];
|
|
}
|
|
}
|
|
|
|
PgfAbstractParser::CCat *PgfAbstractParser::get_epsilon_ccat(PgfText *name, PgfMetaId fid)
|
|
{
|
|
if (fid == 0)
|
|
return NULL;
|
|
|
|
CCat *&ccat = epsilons[fid];
|
|
if (ccat == NULL) {
|
|
ref<PgfCCat> arg = epsilontable_get(concr->epsilontable,
|
|
name, fid);
|
|
ccat = new CCat;
|
|
ccat->fid = arg->fid;
|
|
ccat->epsilon = arg;
|
|
ccat->cont = NULL;
|
|
ccat->state = NULL;
|
|
ccat->lin_idx = arg->lin_idx;
|
|
ccat->value = arg->value;
|
|
ccat->covered = true;
|
|
ccat->viterbi_prob = arg->viterbi_prob;
|
|
}
|
|
return ccat;
|
|
}
|
|
|
|
PgfAbstractParser::CCat::~CCat()
|
|
{
|
|
for (Production *prod : prods) {
|
|
delete prod;
|
|
}
|
|
for (ExprState *estate : pending) {
|
|
delete estate;
|
|
}
|
|
}
|
|
|
|
PgfAbstractParser::Cont::~Cont()
|
|
{
|
|
for (auto it1 : suspended) {
|
|
for (auto it2 : it1.second) {
|
|
for (Item *item : it2.second) {
|
|
delete item;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
PgfAbstractParser::~PgfAbstractParser()
|
|
{
|
|
State *state = current_state;
|
|
while (state != NULL) {
|
|
for (auto it1 : state->completed) {
|
|
/* for (auto it2 : it1) {
|
|
for (auto it3 : it2) {
|
|
delete it3;
|
|
}
|
|
}*/
|
|
}
|
|
for (auto it : state->conts1) {
|
|
delete it.second;
|
|
}
|
|
for (auto it : state->conts2) {
|
|
delete it.second;
|
|
}
|
|
|
|
State *next = state->next;
|
|
delete state;
|
|
state = next;
|
|
}
|
|
}
|
|
|
|
void PgfAbstractParser::process(Item *item, State *state)
|
|
{
|
|
#ifdef DEBUG_PARSER
|
|
print_item(item,state);
|
|
#endif
|
|
|
|
if (item->dot < item->syms.size()) {
|
|
symbol(item,state,item->syms[item->dot]);
|
|
} else if (item->pre_alt > 0) {
|
|
item->dot = item->pre_dot+1;
|
|
item->pre_alt = 0;
|
|
item->pre_dot = 0;
|
|
item->syms = item->rule->syms.as_vector();
|
|
process(item,state);
|
|
} else {
|
|
complete(item,state);
|
|
}
|
|
}
|
|
|
|
PGF_INTERNAL_DECL
|
|
int text_symbol_cmp(PgfText *sentence, PgfTextSpot *spot,
|
|
ref<PgfSymbolKS> sym, bool case_sensitive);
|
|
|
|
void PgfAbstractParser::symbol(Item *item, State *state, PgfSymbol sym)
|
|
{
|
|
switch (ref<PgfSymbol>::get_tag(sym)) {
|
|
case PgfSymbolCat::tag: {
|
|
auto symcat = ref<PgfSymbolCat>::untagged(sym);
|
|
|
|
CCat *ccat = item->args[symcat->d];
|
|
if (ccat == NULL) {
|
|
ref<PgfConcrLincat> lincat = 0;
|
|
switch (ref<object>::get_tag(item->rule->container)) {
|
|
case PgfConcrLin::tag: {
|
|
auto lin = ref<PgfConcrLin>::untagged(item->rule->container);
|
|
lincat =
|
|
namespace_lookup(concr->lincats,
|
|
&lin->absfun->type->hypos[symcat->d].type->name);
|
|
break;
|
|
}
|
|
case PgfConcrLincat::tag: {
|
|
lincat = ref<PgfConcrLincat>::untagged(item->rule->container);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (lincat != 0) {
|
|
size_t n_suspended1 = state->conts1.size();
|
|
Cont *&cont = state->conts1[lincat];
|
|
if (cont == NULL) {
|
|
cont = new Cont;
|
|
cont->ccat = NULL;
|
|
cont->lincat = lincat;
|
|
cont->state = state;
|
|
}
|
|
|
|
interval_t value_i = interval(item->rule, &item->vars[0], item->rule->args[symcat->d]);
|
|
interval_t lin_idx_i = interval(item->rule, &item->vars[0], ref<PgfLParam>::from_ptr(&symcat->r));
|
|
|
|
suspend(cont,item,n_suspended1 == 0,symcat,value_i,lin_idx_i);
|
|
}
|
|
} else {
|
|
interval_t value_i = interval(item->rule, &item->vars[0], item->rule->args[symcat->d]);
|
|
interval_t lin_idx_i = interval(item->rule, &item->vars[0], ref<PgfLParam>::from_ptr(&symcat->r));
|
|
|
|
// the following prevents infinite loops with epsilons
|
|
bool found = false;
|
|
CCat *prev_ccat = ccat;
|
|
while (prev_ccat != NULL && prev_ccat->cont != NULL && prev_ccat->cont->state == state) {
|
|
if (prev_ccat->value == value_i && prev_ccat->lin_idx == lin_idx_i) {
|
|
found = true;
|
|
break;
|
|
}
|
|
prev_ccat = prev_ccat->cont->ccat;
|
|
}
|
|
if (found) {
|
|
item->dot++;
|
|
state->push_item(item);
|
|
break;
|
|
}
|
|
|
|
Cont *&cont = state->conts2[ccat];
|
|
if (cont == NULL) {
|
|
cont = new Cont;
|
|
cont->ccat = ccat;
|
|
if (ccat->fid <= concr->last_fid)
|
|
cont->lincat = ccat->epsilon->lincat;
|
|
else
|
|
cont->lincat = ccat->cont->lincat;
|
|
cont->state = state;
|
|
}
|
|
|
|
bool subsumed = false;
|
|
for (auto it1 : cont->suspended.overlaps(value_i)) {
|
|
if (it1.first.first <= value_i.first && it1.first.second >= value_i.second) {
|
|
for (auto it2 : it1.second.overlaps(lin_idx_i)) {
|
|
if (it2.first.first <= lin_idx_i.first && it2.first.second >= lin_idx_i.second) {
|
|
subsumed = true;
|
|
goto found;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
found:
|
|
suspend(cont,item,!subsumed,symcat,value_i,lin_idx_i);
|
|
}
|
|
break;
|
|
}
|
|
case PgfSymbolKS::tag: {
|
|
auto symks = ref<PgfSymbolKS>::untagged(sym);
|
|
symbol_token(item, state, symks);
|
|
break;
|
|
}
|
|
case PgfSymbolKP::tag: {
|
|
auto symkp = ref<PgfSymbolKP>::untagged(sym);
|
|
|
|
Item *new_item = new(item) Item;
|
|
new_item->pre_alt = 1;
|
|
new_item->pre_dot = item->dot;
|
|
new_item->dot = 0;
|
|
new_item->syms = symkp->default_form;
|
|
new_item->rule = item->rule;
|
|
new_item->inside_prob = item->inside_prob;
|
|
new_item->outside_prob = item->outside_prob;
|
|
process(new_item, state);
|
|
|
|
for (size_t i = 0; i < symkp->alts.size(); i++) {
|
|
Item *new_item = new(item) Item;
|
|
new_item->pre_alt = i+2;
|
|
new_item->pre_dot = item->dot;
|
|
new_item->dot = 0;
|
|
new_item->syms = symkp->alts[i].form;
|
|
new_item->rule = item->rule;
|
|
new_item->inside_prob = item->inside_prob;
|
|
new_item->outside_prob = item->outside_prob;
|
|
process(new_item, state);
|
|
}
|
|
|
|
delete item;
|
|
break;
|
|
}
|
|
case PgfSymbolBIND::tag:
|
|
case PgfSymbolSOFTBIND::tag:
|
|
case PgfSymbolSOFTSPACE::tag: {
|
|
symbol_bind(item, state, sym);
|
|
break;
|
|
}
|
|
case PgfSymbolNE::tag:
|
|
delete item;
|
|
break;
|
|
case PgfSymbolCAPIT::tag:
|
|
case PgfSymbolALLCAPIT::tag:
|
|
item->dot++;
|
|
process(item, state);
|
|
break;
|
|
}
|
|
}
|
|
|
|
void PgfAbstractParser::complete(Item *item, State *state)
|
|
{
|
|
switch (ref<object>::get_tag(item->rule->container)) {
|
|
case PgfConcrLin::tag: {
|
|
auto lin = ref<PgfConcrLin>::untagged(item->rule->container);
|
|
|
|
interval_t res = interval(item->rule, &item->vars[0], item->rule->res);
|
|
interval_t lin_idx = interval(item->rule, &item->vars[0], item->rule->lin_idx);
|
|
CCat *&ccat = state->completed[item->cont][res][lin_idx];
|
|
if (ccat == NULL) {
|
|
ccat = new CCat;
|
|
ccat->fid = (++last_fid);
|
|
ccat->epsilon = 0;
|
|
ccat->cont = item->cont;
|
|
ccat->state = state;
|
|
ccat->lin_idx = lin_idx;
|
|
ccat->value = res;
|
|
ccat->covered = false;
|
|
ccat->viterbi_prob = item->inside_prob;
|
|
|
|
#ifdef DEBUG_PARSER
|
|
{
|
|
PgfPrinter printer(NULL,0,NULL);
|
|
if (item->rule->ranges.size() > 0) {
|
|
printer.lvar_ranges(item->rule->ranges, &item->vars[0]);
|
|
printer.puts(" ");
|
|
}
|
|
printer.nprintf(64,"complete [%zd-%zd; ",item->cont->state->end.pos,state->start.pos);
|
|
if (ccat->cont->ccat == NULL) {
|
|
printer.efun(&ccat->cont->lincat->name);
|
|
printer.puts("(");
|
|
printer.lparam(item->rule->res);
|
|
printer.puts(")");
|
|
} else {
|
|
printer.emeta(ccat->cont->ccat->fid);
|
|
}
|
|
printer.puts("; ");
|
|
printer.lparam(item->rule->lin_idx);
|
|
printer.puts("; ");
|
|
printer.emeta(ccat->fid);
|
|
printer.puts("]");
|
|
PgfText *text = printer.get_text();
|
|
fprintf(stderr, "%s\n", text->text);
|
|
free(text);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
auto prod = new(item) Production;
|
|
prod->rule = item->rule;
|
|
for (size_t i = 0; i < prod->args.size(); i++) {
|
|
if (prod->args[i] != NULL && prod->args[i] != ccat)
|
|
prod->args[i]->covered = true;
|
|
}
|
|
ccat->prods.push_back(prod);
|
|
|
|
#ifdef DEBUG_PARSER
|
|
print_prod(ccat, prod);
|
|
#endif
|
|
final_item(state, ccat, item, res, lin_idx);
|
|
|
|
if (ccat->cont == NULL)
|
|
break;
|
|
|
|
if (ccat->prods.size() == 1) {
|
|
bu_predict(state, item->outside_prob, ccat);
|
|
|
|
for (auto it1 : ccat->cont->suspended.overlaps(ccat->value)) {
|
|
for (auto it2 : it1.second.overlaps(ccat->lin_idx)) {
|
|
size_t n_items = it2.second.size();
|
|
for (size_t i = 0; i < n_items; i++) {
|
|
Item *new_item = new (it2.second[i]) Item;
|
|
combine(state,new_item,ccat);
|
|
};
|
|
}
|
|
}
|
|
} else {
|
|
State *next = state;
|
|
while (next != NULL) {
|
|
Cont *cont = next->conts2[ccat];
|
|
if (cont != NULL) {
|
|
for (auto it1 : cont->suspended) {
|
|
for (auto it2 : it1.second) {
|
|
Item *item = it2.second[0];
|
|
auto symcat = ref<PgfSymbolCat>::untagged(item->syms[item->dot]);
|
|
td_predict(next,cont,prod,item,symcat);
|
|
}
|
|
}
|
|
}
|
|
next = next->next;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
case PgfConcrLincat::tag: {
|
|
auto lincat = ref<PgfConcrLincat>::untagged(item->rule->container);
|
|
interval_t zero = {0,0};
|
|
final_item(state, NULL, item, zero, zero);
|
|
break;
|
|
}
|
|
}
|
|
|
|
delete item;
|
|
}
|
|
|
|
#define ZERO_VALUES(rule) \
|
|
((size_t*) memset(alloca(rule->ranges.size()*sizeof(size_t)), 0, rule->ranges.size()*sizeof(size_t)))
|
|
#define CLONE_VALUES(rule,values) \
|
|
((size_t*) memcpy(alloca(rule->ranges.size()*sizeof(size_t)), values, rule->ranges.size()*sizeof(size_t)))
|
|
|
|
bool PgfAbstractParser::instantiate(ref<PgfConcrRule> rule1, size_t *values1, ref<PgfLParam> lparam1,
|
|
ref<PgfConcrRule> rule2, size_t *values2, ref<PgfLParam> lparam2)
|
|
{
|
|
size_t i01 = lparam1->i0;
|
|
size_t i02 = lparam2->i0;
|
|
|
|
size_t i1 = 0, i2 = 0;
|
|
while (i1 < lparam1->n_terms || i2 < lparam2->n_terms) {
|
|
size_t scale1 = 0;
|
|
term t1 = {0,0};
|
|
if (i1 < lparam1->n_terms) {
|
|
t1 = lparam1->terms[i1];
|
|
if (values1[t1.var] > 0) {
|
|
i01 += t1.factor * (values1[t1.var]-1);
|
|
i1++;
|
|
continue;
|
|
}
|
|
scale1 = t1.factor * rule1->ranges[t1.var];
|
|
}
|
|
|
|
size_t scale2 = 0;
|
|
term t2 = {0,0};
|
|
if (i2 < lparam2->n_terms) {
|
|
t2 = lparam2->terms[i2];
|
|
if (values2[t2.var] > 0) {
|
|
i02 += t2.factor * (values2[t2.var]-1);
|
|
i2++;
|
|
continue;
|
|
}
|
|
scale2 = t2.factor * rule2->ranges[t2.var];
|
|
}
|
|
|
|
if (scale1 > scale2) {
|
|
ssize_t min = (i02 / t1.factor - i01 / t1.factor);
|
|
ssize_t max = min;
|
|
if (min < 0)
|
|
return false;
|
|
while (i2 < lparam2->n_terms) {
|
|
t2 = lparam2->terms[i2];
|
|
size_t f = t2.factor / t1.factor;
|
|
if (f == 0)
|
|
break;
|
|
|
|
if (values2[t2.var] == 0) {
|
|
max += f * (rule2->ranges[t2.var]-1);
|
|
} else {
|
|
min += f * (values2[t2.var]-1);
|
|
max += f * (values2[t2.var]-1);
|
|
}
|
|
i2++;
|
|
}
|
|
|
|
if (min >= rule1->ranges[t1.var])
|
|
return false;
|
|
|
|
if (min == max) {
|
|
if (values1[t1.var] == 0)
|
|
values1[t1.var] = min+1;
|
|
else if (values1[t1.var] != min+1)
|
|
return false;
|
|
}
|
|
|
|
i01 %= t1.factor;
|
|
i02 %= t1.factor;
|
|
i1++;
|
|
} else {
|
|
size_t min = (i01 / t2.factor - i02 / t2.factor);
|
|
size_t max = min;
|
|
if (min < 0)
|
|
return false;
|
|
while (i1 < lparam1->n_terms) {
|
|
t1 = lparam1->terms[i1];
|
|
size_t f = t1.factor / t2.factor;
|
|
if (f == 0)
|
|
break;
|
|
|
|
if (values1[t1.var] == 0) {
|
|
max += f * (rule1->ranges[t1.var]-1);
|
|
} else {
|
|
min += f * (values1[t1.var]-1);
|
|
max += f * (values1[t1.var]-1);
|
|
}
|
|
i1++;
|
|
}
|
|
|
|
if (min >= rule2->ranges[t2.var])
|
|
return false;
|
|
|
|
if (min == max) {
|
|
if (values2[t2.var] == 0) {
|
|
values2[t2.var] = min+1;
|
|
} else if (values2[t2.var] != min+1)
|
|
return false;
|
|
}
|
|
|
|
i01 %= t2.factor;
|
|
i02 %= t2.factor;
|
|
i2++;
|
|
}
|
|
}
|
|
|
|
return (i01 == i02);
|
|
}
|
|
|
|
interval_t PgfAbstractParser::interval(ref<PgfConcrRule> rule, size_t *values, ref<PgfLParam> lparam)
|
|
{
|
|
interval_t interval;
|
|
interval.first = lparam->i0;
|
|
interval.second = interval.first;
|
|
for (size_t i = 0; i < lparam->n_terms; i++) {
|
|
size_t var = lparam->terms[i].var;
|
|
if (values[var] == 0) {
|
|
interval.second += lparam->terms[i].factor * (rule->ranges[var]-1);
|
|
} else {
|
|
size_t value = lparam->terms[i].factor * (values[var]-1);
|
|
interval.first += value;
|
|
interval.second += value;
|
|
}
|
|
}
|
|
return interval;
|
|
}
|
|
|
|
void PgfAbstractParser::combine(State *state, Item *item, CCat *ccat)
|
|
{
|
|
PgfSymbol sym = item->rule->syms[item->dot];
|
|
auto sym_cat = ref<PgfSymbolCat>::untagged(sym);
|
|
|
|
ref<PgfConcrRule> rule;
|
|
size_t *values;
|
|
get_info(ccat, &rule,&values);
|
|
values = CLONE_VALUES(rule, values);
|
|
|
|
if (!instantiate(item->rule, &item->vars[0], item->rule->args[sym_cat->d],
|
|
rule, values, rule->res)) {
|
|
delete item;
|
|
return;
|
|
}
|
|
if (!instantiate(item->rule, &item->vars[0], ref<PgfLParam>::from_ptr(&sym_cat->r),
|
|
rule, values, rule->lin_idx)) {
|
|
delete item;
|
|
return;
|
|
}
|
|
|
|
item->dot++;
|
|
if (item->args[sym_cat->d] != NULL) {
|
|
item->inside_prob -= item->args[sym_cat->d]->viterbi_prob;
|
|
}
|
|
item->args[sym_cat->d] = ccat;
|
|
item->inside_prob += ccat->viterbi_prob;
|
|
state->push_item(item);
|
|
}
|
|
|
|
bool PgfAbstractParser::ItemComparator::operator()(Item *item1, Item *item2) const
|
|
{
|
|
if (item1->rule.as_object() < item2->rule.as_object())
|
|
return true;
|
|
else if (item1->rule.as_object() > item2->rule.as_object())
|
|
return false;
|
|
|
|
for (size_t j = 0; j < item1->args.size(); j++) {
|
|
if (item1->args[j] < item2->args[j])
|
|
return true;
|
|
else if (item1->args[j] > item2->args[j])
|
|
return false;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
void PgfAbstractParser::td_epsilon(State *state, Cont *cont, ref<PgfItem> pitem, Item *xitem, ref<PgfSymbolCat> symcat)
|
|
{
|
|
switch (ref<object>::get_tag(pitem->rule->container)) {
|
|
case PgfConcrLin::tag: {
|
|
auto lin = ref<PgfConcrLin>::untagged(pitem->rule->container);
|
|
|
|
for (ref<PgfConcrRule> rule : lin->rules) {
|
|
size_t *values1 = ZERO_VALUES(rule);
|
|
size_t *values2 = CLONE_VALUES(xitem->rule, &xitem->vars[0]);
|
|
if (!instantiate(rule, values1, rule->res,
|
|
xitem->rule, values2, xitem->rule->args[symcat->d])) {
|
|
continue;
|
|
}
|
|
if (!instantiate(rule, values1, rule->lin_idx,
|
|
xitem->rule, values2, ref<PgfLParam>::from_ptr(&symcat->r))) {
|
|
continue;
|
|
}
|
|
|
|
size_t *values3 = CLONE_VALUES(pitem->rule, &pitem->vars[0]);
|
|
for (size_t i = 0; i < pitem->args.size(); i++) {
|
|
if (!instantiate(rule, values1, rule->args[i],
|
|
pitem->rule, values3, pitem->rule->args[i])) {
|
|
goto next;
|
|
}
|
|
}
|
|
|
|
{
|
|
Item *item = new (rule) Item;
|
|
item->cont = cont;
|
|
item->dot = 0;
|
|
item->pre_alt = 0;
|
|
item->pre_dot = 0;
|
|
item->syms = rule->syms.as_vector();
|
|
item->rule = rule;
|
|
item->inside_prob = lin->absfun->prob;
|
|
item->outside_prob = xitem->outside_prob+xitem->inside_prob-xitem->args[symcat->d]->viterbi_prob;
|
|
|
|
size_t *values3 = CLONE_VALUES(pitem->rule, &pitem->vars[0]);
|
|
for (size_t i = 0; i < pitem->args.size(); i++) {
|
|
if (pitem->args[i] != 0) {
|
|
if (!instantiate(rule, &item->vars[0], rule->args[i],
|
|
pitem->rule, values3, pitem->rule->args[i])) {
|
|
delete item;
|
|
goto next;
|
|
}
|
|
|
|
item->args[i] = get_epsilon_ccat(&lin->absfun->type->hypos[i].type->name,pitem->args[i]);
|
|
item->inside_prob += item->args[i]->viterbi_prob;
|
|
}
|
|
}
|
|
|
|
auto res = cont->predicted.insert(item);
|
|
if (res.second) {
|
|
state->push_item(item);
|
|
} else {
|
|
delete item;
|
|
}
|
|
}
|
|
next:;
|
|
}
|
|
}
|
|
default:;
|
|
// should not happend
|
|
}
|
|
}
|
|
|
|
void PgfAbstractParser::td_predict(State *state, Cont *cont, Production *prod, Item *xitem, ref<PgfSymbolCat> symcat)
|
|
{
|
|
switch (ref<object>::get_tag(prod->rule->container)) {
|
|
case PgfConcrLin::tag: {
|
|
auto lin = ref<PgfConcrLin>::untagged(prod->rule->container);
|
|
|
|
for (ref<PgfConcrRule> rule : lin->rules) {
|
|
size_t *values1 = ZERO_VALUES(rule);
|
|
size_t *values2 = CLONE_VALUES(xitem->rule, &xitem->vars[0]);
|
|
if (!instantiate(rule, values1, rule->res,
|
|
xitem->rule, values2, xitem->rule->args[symcat->d])) {
|
|
continue;
|
|
}
|
|
|
|
if (!instantiate(rule, values1, rule->lin_idx,
|
|
xitem->rule, values2, ref<PgfLParam>::from_ptr(&symcat->r))) {
|
|
continue;
|
|
}
|
|
|
|
size_t *values3 = CLONE_VALUES(prod->rule, &prod->vars[0]);
|
|
for (size_t i = 0; i < rule->args.size(); i++) {
|
|
if (!instantiate(rule, values1, rule->args[i],
|
|
prod->rule, values3, prod->rule->args[i])) {
|
|
goto next;
|
|
}
|
|
}
|
|
|
|
{
|
|
Item *item = new (rule) Item;
|
|
item->cont = cont;
|
|
item->dot = 0;
|
|
item->pre_alt = 0;
|
|
item->pre_dot = 0;
|
|
item->syms = rule->syms.as_vector();
|
|
item->rule = rule;
|
|
item->inside_prob = lin->absfun->prob;
|
|
item->outside_prob = xitem->outside_prob+xitem->inside_prob-xitem->args[symcat->d]->viterbi_prob;
|
|
|
|
size_t *values3 = CLONE_VALUES(prod->rule, &prod->vars[0]);
|
|
for (size_t i = 0; i < rule->args.size(); i++) {
|
|
if (!instantiate(rule, &item->vars[0], rule->args[i],
|
|
prod->rule, values3, prod->rule->args[i])) {
|
|
delete item;
|
|
goto next;
|
|
}
|
|
|
|
item->args[i] = prod->args[i];
|
|
if (item->args[i] != NULL) {
|
|
item->inside_prob += item->args[i]->viterbi_prob;
|
|
}
|
|
}
|
|
|
|
auto res = cont->predicted.insert(item);
|
|
if (res.second) {
|
|
state->push_item(item);
|
|
} else {
|
|
delete item;
|
|
}
|
|
}
|
|
|
|
next:;
|
|
}
|
|
}
|
|
default:;
|
|
// should not happend
|
|
}
|
|
}
|
|
|
|
#ifdef DEBUG_PARSER
|
|
static
|
|
void print_symbols(PgfPrinter &printer, PgfConcrRule *rule, vector<PgfSymbol> syms, size_t pre_alt, size_t pre_dot, size_t dot)
|
|
{
|
|
for (size_t i = 0; i < syms.size(); i++) {
|
|
if (pre_alt == 0 && dot == i) {
|
|
printer.puts(" . ");
|
|
printer.symbol(syms[i]);
|
|
} else if (pre_alt > 0 && pre_dot == i) {
|
|
auto sym_kp = ref<PgfSymbolKP>::untagged(rule->syms[pre_dot]);
|
|
|
|
printer.puts("pre {");
|
|
|
|
if (pre_alt == 1)
|
|
print_symbols(printer, rule, syms, 0, 0, dot);
|
|
else
|
|
printer.symbols(sym_kp->default_form);
|
|
|
|
for (size_t i = 0; i < sym_kp->alts.size(); i++) {
|
|
printer.puts("; ");
|
|
if (pre_alt-2 == i)
|
|
print_symbols(printer, rule, syms, 0, 0, dot);
|
|
else
|
|
printer.symbols(sym_kp->alts[i].form);
|
|
printer.puts(" /");
|
|
for (size_t j = 0; j < sym_kp->alts[i].prefixes.size(); j++) {
|
|
printer.puts(" ");
|
|
printer.lstr(sym_kp->alts[i].prefixes[j]);
|
|
}
|
|
}
|
|
|
|
printer.puts("}");
|
|
} else {
|
|
printer.symbol(syms[i]);
|
|
}
|
|
}
|
|
if (pre_alt == 0 && dot >= syms.size())
|
|
printer.puts(" . ");
|
|
}
|
|
|
|
void PgfAbstractParser::print_item(Item *item, State *state)
|
|
{
|
|
PgfPrinter printer(NULL,0,NULL);
|
|
|
|
printer.nprintf(32, "[%zd-%zd; ", (item->cont && item->cont->state) ? item->cont->state->end.pos : 0, state->start.pos);
|
|
|
|
if (item->vars.size() > 0) {
|
|
printer.lvar_ranges(item->rule->ranges, &item->vars[0]);
|
|
printer.puts(" ");
|
|
}
|
|
|
|
if (item->cont) {
|
|
if (item->cont->ccat == NULL) {
|
|
printer.efun(&item->cont->lincat->name);
|
|
} else {
|
|
printer.emeta(item->cont->ccat->fid);
|
|
}
|
|
printer.puts("(");
|
|
printer.lparam(item->rule->res);
|
|
printer.puts(")");
|
|
}
|
|
printer.puts(" -> ");
|
|
|
|
switch (ref<object>::get_tag(item->rule->container)) {
|
|
case PgfConcrLin::tag: {
|
|
auto lin = ref<PgfConcrLin>::untagged(item->rule->container);
|
|
printer.efun(&lin->name);
|
|
|
|
printer.puts("[");
|
|
for (size_t i = 0; i < item->args.size(); i++) {
|
|
if (i > 0)
|
|
printer.puts(",");
|
|
|
|
CCat *ccat = item->args[i];
|
|
if (ccat == NULL) {
|
|
printer.efun(&lin->absfun->type->hypos[i].type->name);
|
|
} else {
|
|
printer.emeta(ccat->fid);
|
|
}
|
|
printer.puts("(");
|
|
printer.lparam(item->rule->args[i]);
|
|
printer.puts(")");
|
|
}
|
|
printer.puts("]; ");
|
|
break;
|
|
}
|
|
case PgfConcrLincat::tag: {
|
|
auto lincat = ref<PgfConcrLincat>::untagged(item->rule->container);
|
|
printer.puts("linref ");
|
|
printer.efun(&lincat->name);
|
|
|
|
printer.puts("[");
|
|
CCat *ccat = item->args[0];
|
|
if (ccat == NULL) {
|
|
printer.efun(&lincat->name);
|
|
printer.puts("(");
|
|
printer.lparam(item->rule->args[0]);
|
|
printer.puts(")");
|
|
} else {
|
|
printer.emeta(ccat->fid);
|
|
}
|
|
printer.puts("]; ");
|
|
break;
|
|
}
|
|
}
|
|
|
|
printer.lparam(item->rule->lin_idx);
|
|
printer.puts(" : ");
|
|
print_symbols(printer, item->rule, item->syms, item->pre_alt, item->pre_dot, item->dot);
|
|
printer.nprintf(40,"; %f+%f=%f]", item->inside_prob, item->outside_prob, item->inside_prob+item->outside_prob);
|
|
|
|
PgfText *text = printer.get_text();
|
|
fprintf(stderr, "%s\n", text->text);
|
|
free(text);
|
|
}
|
|
|
|
void PgfAbstractParser::print_prod(CCat *ccat, Production *prod)
|
|
{
|
|
PgfPrinter printer(NULL,0,NULL);
|
|
|
|
if (prod->vars.size() > 0) {
|
|
printer.lvar_ranges(prod->rule->ranges, &prod->vars[0]);
|
|
printer.puts(" ");
|
|
}
|
|
|
|
printer.emeta(ccat->fid);
|
|
printer.puts("(");
|
|
printer.lparam(prod->rule->res);
|
|
printer.puts(")");
|
|
|
|
printer.puts(" -> ");
|
|
|
|
switch (ref<object>::get_tag(prod->rule->container)) {
|
|
case PgfConcrLin::tag: {
|
|
auto lin = ref<PgfConcrLin>::untagged(prod->rule->container);
|
|
printer.efun(&lin->name);
|
|
|
|
printer.puts("[");
|
|
for (size_t i = 0; i < prod->args.size(); i++) {
|
|
if (i > 0)
|
|
printer.puts(",");
|
|
|
|
CCat *ccat = prod->args[i];
|
|
if (ccat == NULL) {
|
|
printer.efun(&lin->absfun->type->hypos[i].type->name);
|
|
} else {
|
|
printer.emeta(ccat->fid);
|
|
}
|
|
printer.puts("(");
|
|
printer.lparam(prod->rule->args[i]);
|
|
printer.puts(")");
|
|
}
|
|
printer.puts("]");
|
|
break;
|
|
}
|
|
case PgfConcrLincat::tag: {
|
|
auto lincat = ref<PgfConcrLincat>::untagged(prod->rule->container);
|
|
printer.puts("linref ");
|
|
printer.efun(&lincat->name);
|
|
|
|
printer.puts("[");
|
|
CCat *ccat = prod->args[0];
|
|
if (ccat == NULL) {
|
|
printer.efun(&lincat->name);
|
|
printer.puts("(");
|
|
printer.lparam(prod->rule->args[0]);
|
|
printer.puts(")");
|
|
} else {
|
|
printer.emeta(ccat->fid);
|
|
}
|
|
printer.puts("]");
|
|
break;
|
|
}
|
|
}
|
|
|
|
PgfText *text = printer.get_text();
|
|
fprintf(stderr, "%s\n", text->text);
|
|
free(text);
|
|
}
|
|
#endif
|
|
|
|
PgfParser::PgfParser(ref<PgfConcr> concr, PgfText *sentence, bool case_sensitive, PgfMarshaller *m, PgfUnmarshaller *u)
|
|
: PgfAbstractParser(concr)
|
|
{
|
|
this->m = m;
|
|
this->u = u;
|
|
this->sentence = textdup(sentence);
|
|
this->case_sensitive = case_sensitive;
|
|
this->prev_state = NULL;
|
|
this->update_old_pos = 0;
|
|
this->update_new_pos = 0;
|
|
this->delta_byte_pos = 0;
|
|
this->allocated_size = sizeof(PgfText)+sentence->size+1;
|
|
}
|
|
|
|
PgfParser::~PgfParser()
|
|
{
|
|
free(sentence);
|
|
|
|
State *state = current_state;
|
|
while (state != NULL) {
|
|
for (auto it1 : state->completed) {
|
|
for (auto it2 : it1.second) {
|
|
for (auto it3 : it2.second) {
|
|
if (it3.second->epsilon != 0)
|
|
continue;
|
|
|
|
for (ExprState *estate : it3.second->pending) {
|
|
if (estate->expr != 0)
|
|
u->free_ref(estate->expr);
|
|
}
|
|
for (ExprProb &ep : it3.second->exprs) {
|
|
u->free_ref(ep.expr);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
state = state->next;
|
|
}
|
|
|
|
for (auto it1 : epsilons) {
|
|
for (ExprState *estate : it1.second->pending) {
|
|
if (estate->expr != 0)
|
|
u->free_ref(estate->expr);
|
|
}
|
|
for (ExprProb &ep : it1.second->exprs) {
|
|
u->free_ref(ep.expr);
|
|
}
|
|
}
|
|
}
|
|
|
|
void PgfParser::bu_predict(PgfPhrasetable<PgfSymbolKS> phrasetable,
|
|
State *state, prob_t outside_prob,
|
|
ptrdiff_t min, ptrdiff_t max)
|
|
{
|
|
if (phrasetable == 0)
|
|
return;
|
|
|
|
PgfTextSpot current = state->end;
|
|
int cmp = text_symbol_cmp(sentence,¤t,phrasetable->value.key,case_sensitive);
|
|
if (cmp < 0) {
|
|
bu_predict(phrasetable->left,state,outside_prob,min,max);
|
|
} else if (cmp > 0) {
|
|
ptrdiff_t len = current.byte_pos - state->end.byte_pos;
|
|
|
|
if (min <= len-1)
|
|
bu_predict(phrasetable->left,state,outside_prob,min,len-1);
|
|
|
|
if (len <= max)
|
|
bu_predict(phrasetable->right,state,outside_prob,len,max);
|
|
} else {
|
|
ptrdiff_t len = current.byte_pos - state->end.byte_pos;
|
|
|
|
if (min <= len)
|
|
bu_predict(phrasetable->left,state,outside_prob,min,len);
|
|
|
|
if (len > 0) {
|
|
for (size_t i = 0; i < phrasetable->value.n_items; i++) {
|
|
//std::map<ref<PgfConcrLincat>, bool> visited;
|
|
//if (!td_reachable(state, phrasetable->items[i], visited))
|
|
// continue;
|
|
Item *item = bu_item(state, outside_prob, phrasetable->value.items[i]);
|
|
item->dot++;
|
|
|
|
State *next_state = new_state(current,item->outside_prob+item->inside_prob);
|
|
next_state->push_item(item);
|
|
}
|
|
}
|
|
|
|
if (len <= max)
|
|
bu_predict(phrasetable->right,state,outside_prob,len,max);
|
|
}
|
|
}
|
|
|
|
void PgfParser::bu_predict(PgfPhrasetable<PgfSymbolBIND> phrasetable,
|
|
State *state, prob_t outside_prob)
|
|
{
|
|
size_t n_items = 0;
|
|
vector<ref<PgfItem>> items =
|
|
phrasetable_lookup(concr->phrasetable4,
|
|
ref<PgfSymbolBIND>(0),
|
|
&n_items);
|
|
|
|
State *next_state = state->next;
|
|
if (next_state == NULL || state->end.pos != next_state->start.pos) {
|
|
next_state = new State;
|
|
next_state->start = state->end;
|
|
next_state->end = state->end;
|
|
next_state->next = state->next;
|
|
next_state->needs_bind = false;
|
|
next_state->did_bu_predict = false;
|
|
next_state->viterbi_prob = state->viterbi_prob;
|
|
state->next = next_state;
|
|
}
|
|
|
|
for (size_t i = 0; i < n_items; i++) {
|
|
//std::map<ref<PgfConcrLincat>, bool> visited;
|
|
//if (!td_reachable(state, phrasetable->items[i], visited))
|
|
// continue;
|
|
Item *item = bu_item(state, outside_prob, items[i]);
|
|
item->dot++;
|
|
next_state->push_item(item);
|
|
}
|
|
}
|
|
|
|
void PgfParser::bu_predict(State *state, prob_t outside_prob, CCat *ccat)
|
|
{
|
|
size_t n_items = 0;
|
|
vector<ref<PgfItem>> items = 0;
|
|
if (ccat->cont->ccat == NULL) {
|
|
items = phrasetable_lookup(concr->phrasetable2,
|
|
ccat->cont->lincat,
|
|
&n_items);
|
|
} else if (ccat->cont->ccat->epsilon != 0) {
|
|
items = phrasetable_lookup(concr->phrasetable3,
|
|
ccat->cont->ccat->epsilon,
|
|
&n_items);
|
|
}
|
|
|
|
for (size_t i = 0; i < n_items; i++) {
|
|
//std::map<ref<PgfConcrLincat>, bool> visited;
|
|
//if (!td_reachable(ccat->cont->state, items[i], visited))
|
|
// continue;
|
|
auto new_item = bu_item(ccat->cont->state, outside_prob, items[i]);
|
|
combine(state,new_item,ccat);
|
|
}
|
|
}
|
|
|
|
bool PgfParser::td_reachable(State *state, ref<PgfItem> pitem,
|
|
std::map<ref<PgfConcrLincat>, bool> &visited)
|
|
{
|
|
switch (ref<object>::get_tag(pitem->rule->container)) {
|
|
case PgfConcrLin::tag: {
|
|
auto lin = ref<PgfConcrLin>::untagged(pitem->rule->container);
|
|
|
|
if (visited[lin->lincat])
|
|
return false;
|
|
visited[lin->lincat] = true;
|
|
|
|
auto it = state->conts1.find(lin->lincat);
|
|
if (it != state->conts1.end()) {
|
|
return true;
|
|
}
|
|
|
|
size_t n_items = 0;
|
|
vector<ref<PgfItem>> items =
|
|
phrasetable_lookup(concr->phrasetable2,
|
|
lin->lincat,
|
|
&n_items);
|
|
for (size_t i = 0; i < n_items; i++) {
|
|
if (td_reachable(state, items[i], visited))
|
|
return true;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
PgfAbstractParser::Item *PgfParser::bu_item(State *state, prob_t outside_prob, ref<PgfItem> pitem)
|
|
{
|
|
Item *item = NULL;
|
|
|
|
switch (ref<object>::get_tag(pitem->rule->container)) {
|
|
case PgfConcrLin::tag: {
|
|
auto lin = ref<PgfConcrLin>::untagged(pitem->rule->container);
|
|
|
|
Cont *cont;
|
|
if (pitem->res == 0) {
|
|
Cont *&tmp = state->conts1[lin->lincat];
|
|
if (tmp == NULL) {
|
|
tmp = new Cont;
|
|
tmp->ccat = NULL;
|
|
tmp->lincat = lin->lincat;
|
|
tmp->state = state;
|
|
}
|
|
cont = tmp;
|
|
} else {
|
|
CCat *ccat = get_epsilon_ccat(&lin->lincat->name,pitem->res);
|
|
Cont *&tmp = state->conts2[ccat];
|
|
if (tmp == NULL) {
|
|
tmp = new Cont;
|
|
tmp->ccat = ccat;
|
|
tmp->lincat = lin->lincat;
|
|
tmp->state = state;
|
|
}
|
|
cont = tmp;
|
|
}
|
|
|
|
item = new(pitem->rule) Item;
|
|
item->cont = cont;
|
|
item->pre_alt = pitem->pre_alt;
|
|
item->pre_dot = pitem->pre_dot;
|
|
item->dot = pitem->dot;
|
|
item->syms = pitem->rule->syms.as_vector();
|
|
item->rule = pitem->rule;
|
|
item->inside_prob = lin->absfun->prob;
|
|
item->outside_prob = outside_prob;
|
|
|
|
for (size_t i = 0; i < pitem->args.size(); i++) {
|
|
item->args[i] = 0;
|
|
if (pitem->args[i] != 0) {
|
|
item->args[i] = get_epsilon_ccat(&lin->absfun->type->hypos[i].type->name,pitem->args[i]);
|
|
item->inside_prob += item->args[i]->viterbi_prob;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
case PgfConcrLincat::tag: {
|
|
auto lincat = ref<PgfConcrLincat>::untagged(pitem->rule->container);
|
|
|
|
Cont *&cont = state->conts1[0];
|
|
if (cont == NULL) {
|
|
cont = new Cont;
|
|
cont->ccat = NULL;
|
|
cont->lincat = 0;
|
|
cont->state = state;
|
|
}
|
|
|
|
item = new(pitem->rule) Item;
|
|
item->cont = cont;
|
|
item->pre_alt = pitem->pre_alt;
|
|
item->pre_dot = pitem->pre_dot;
|
|
item->dot = pitem->dot;
|
|
item->syms = pitem->rule->syms.as_vector();
|
|
item->rule = pitem->rule;
|
|
item->inside_prob = 0;
|
|
item->outside_prob = 0;
|
|
item->args[0] = 0;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (item->pre_alt > 0) {
|
|
auto symkp = ref<PgfSymbolKP>::untagged(item->syms[item->pre_dot]);
|
|
|
|
if (item->pre_alt == 1)
|
|
item->syms = symkp->default_form;
|
|
else
|
|
item->syms = symkp->alts[item->pre_alt-2].form;
|
|
}
|
|
|
|
memcpy(&item->vars[0], &pitem->vars[0], sizeof(size_t) * item->vars.size());
|
|
return item;
|
|
}
|
|
|
|
void PgfParser::make_chunks(State *state, std::vector<CCat*> &chunks, prob_t prob)
|
|
{
|
|
if (state->completed.size() == 0) {
|
|
ExprState *estate = new(chunks.size()) ExprState;
|
|
estate->expr = u->emeta(0);
|
|
estate->prob = prob;
|
|
estate->hash = '?';
|
|
estate->res = NULL;
|
|
estate->index = 0;
|
|
estate->n_args = chunks.size();
|
|
for (size_t i = 0; i < estate->n_args; i++) {
|
|
estate->args[i] = chunks[estate->n_args-i-1];
|
|
estate->prob += estate->args[i]->viterbi_prob;
|
|
}
|
|
queue.push_back(estate);
|
|
std::push_heap(queue.begin(), queue.end(), estate_comp);
|
|
}
|
|
|
|
for (auto it1 : state->completed) {
|
|
for (auto it2 : it1.second) {
|
|
for (auto it3 : it2.second) {
|
|
CCat *ccat = it3.second;
|
|
if (!ccat->covered && ccat->cont->state != state) {
|
|
chunks.push_back(ccat);
|
|
make_chunks(ccat->cont->state, chunks, prob+ccat->cont->lincat->abscat->prob);
|
|
chunks.pop_back();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void PgfParser::prepare(ref<PgfConcrLincat> start)
|
|
{
|
|
#ifdef DEBUG_PARSER
|
|
fprintf(stderr, "------------------------------------------\n");
|
|
#endif
|
|
|
|
PgfTextSpot start_spot = {0, 0};
|
|
State *state = new_state(start_spot, 0);
|
|
|
|
for (size_t i = start->n_lindefs; i < start->rules.size(); i++) {
|
|
ref<PgfConcrRule> rule = start->rules[i];
|
|
Item *item = new(rule) Item;
|
|
item->cont = NULL;
|
|
item->dot = 0;
|
|
item->pre_alt = 0;
|
|
item->pre_dot = 0;
|
|
item->syms = rule->syms.as_vector();
|
|
item->rule = rule;
|
|
item->inside_prob = 0;
|
|
item->outside_prob = 0;
|
|
state->push_item(item);
|
|
}
|
|
}
|
|
|
|
void PgfParser::perform_search()
|
|
{
|
|
for (;;) {
|
|
State *state = current_state;
|
|
prob_t min_prob = INFINITY;
|
|
State *min_state = NULL;
|
|
if (queue.size() > 0) {
|
|
min_prob = queue.front()->prob;
|
|
}
|
|
|
|
while (state != NULL) {
|
|
if (state->queue.size() > 0) {
|
|
Item *item = state->queue.front();
|
|
prob_t delta = current_state->viterbi_prob - state->viterbi_prob;
|
|
prob_t prob = item->outside_prob + item->inside_prob + delta;
|
|
if (min_prob > prob) {
|
|
min_prob = prob;
|
|
min_state = state;
|
|
}
|
|
}
|
|
state = state->next;
|
|
}
|
|
|
|
if (min_state == NULL)
|
|
break;
|
|
|
|
State *prev = current_state;
|
|
current_state = NULL;
|
|
while (current_state != min_state) {
|
|
State *next = prev->next;
|
|
prev->next = current_state;
|
|
current_state = prev;
|
|
prev = next;
|
|
}
|
|
|
|
Item *item = current_state->pop_item();
|
|
process(item,current_state);
|
|
|
|
while (current_state != NULL) {
|
|
State *next = current_state->next;
|
|
current_state->next = prev;
|
|
prev = current_state;
|
|
current_state = next;
|
|
}
|
|
current_state = prev;
|
|
}
|
|
}
|
|
|
|
PgfExpr PgfParser::fetch(PgfDB *db, prob_t *prob)
|
|
{
|
|
DB_scope scope(db, READER_SCOPE);
|
|
|
|
bool first_fetch = (concr->last_fid == last_fid);
|
|
|
|
perform_search();
|
|
|
|
if (first_fetch && queue.size() == 0) {
|
|
std::vector<CCat*> chunks;
|
|
make_chunks(current_state, chunks, 0);
|
|
}
|
|
|
|
while (queue.size() > 0) {
|
|
ExprState *estate = queue.front();
|
|
std::pop_heap(queue.begin(), queue.end(), estate_comp);
|
|
queue.pop_back();
|
|
|
|
#ifdef DEBUG_EXPRS
|
|
print_expr_state(m, estate);
|
|
#endif
|
|
|
|
PgfExpr expr = process_expr(estate, prob);
|
|
if (expr != 0)
|
|
return expr;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob)
|
|
{
|
|
if (estate->index < estate->n_args) {
|
|
CCat *ccat = estate->args[estate->index];
|
|
|
|
if (ccat == NULL) {
|
|
ExprState *app_state = new(estate->n_args) ExprState;
|
|
app_state->expr = estate->expr ? u->eapp(estate->expr, u->emeta(0)) : u->emeta(0);
|
|
app_state->prob = estate->prob;
|
|
app_state->hash = estate->hash * 101 + '?';
|
|
app_state->res = estate->res;
|
|
app_state->index = estate->index+1;
|
|
app_state->n_args = estate->n_args;
|
|
for (size_t i = 0; i < app_state->n_args; i++) {
|
|
app_state->args[i] = estate->args[i];
|
|
}
|
|
queue.push_back(app_state);
|
|
std::push_heap(queue.begin(), queue.end(), estate_comp);
|
|
} else {
|
|
ccat->pending.push_back(estate);
|
|
|
|
if (ccat->pending.size() == 1) {
|
|
if (ccat->epsilon != 0) {
|
|
for (size_t i = 0; i < ccat->epsilon->n_items; i++) {
|
|
ref<PgfItem> pitem = ccat->epsilon->items[i];
|
|
|
|
auto lin = ref<PgfConcrLin>::untagged(pitem->rule->container);
|
|
|
|
ExprState *new_estate = new(pitem->args.size()) ExprState;
|
|
new_estate->expr = u->efun(&lin->name);
|
|
new_estate->prob = estate->prob-ccat->viterbi_prob+lin->absfun->prob;
|
|
new_estate->hash = 0;
|
|
new_estate->res = ccat;
|
|
new_estate->index = 0;
|
|
new_estate->n_args = pitem->args.size();
|
|
for (size_t i = 0; i < lin->name.size; i++) {
|
|
new_estate->hash = new_estate->hash * 101 + lin->name.text[i];
|
|
}
|
|
for (size_t i = 0; i < new_estate->n_args; i++) {
|
|
new_estate->args[i] = NULL;
|
|
if (pitem->args[i] != 0) {
|
|
new_estate->args[i] = get_epsilon_ccat(&lin->absfun->type->hypos[i].type->name,pitem->args[i]);
|
|
new_estate->prob += new_estate->args[i]->viterbi_prob;
|
|
}
|
|
}
|
|
queue.push_back(new_estate);
|
|
std::push_heap(queue.begin(), queue.end(), estate_comp);
|
|
}
|
|
} else {
|
|
for (Production *prod : ccat->prods) {
|
|
auto lin = ref<PgfConcrLin>::untagged(prod->rule->container);
|
|
|
|
ExprState *new_estate = new(prod->args.size()) ExprState;
|
|
new_estate->expr = u->efun(&lin->name);
|
|
new_estate->prob = estate->prob-ccat->viterbi_prob+lin->absfun->prob;
|
|
new_estate->hash = 0;
|
|
new_estate->res = ccat;
|
|
new_estate->index = 0;
|
|
new_estate->n_args = prod->args.size();
|
|
for (size_t i = 0; i < lin->name.size; i++) {
|
|
new_estate->hash = new_estate->hash * 101 + lin->name.text[i];
|
|
}
|
|
for (size_t i = 0; i < new_estate->n_args; i++) {
|
|
new_estate->args[i] = prod->args[i];
|
|
if (prod->args[i] != NULL) {
|
|
new_estate->prob += prod->args[i]->viterbi_prob;
|
|
}
|
|
}
|
|
queue.push_back(new_estate);
|
|
std::push_heap(queue.begin(), queue.end(), estate_comp);
|
|
}
|
|
}
|
|
} else {
|
|
for (ExprProb ep : ccat->exprs) {
|
|
ExprState *app_state = new(estate->n_args) ExprState;
|
|
app_state->expr = estate->expr ? u->eapp(estate->expr, ep.expr) : ep.expr;
|
|
app_state->prob = estate->prob-ccat->viterbi_prob+ep.prob;
|
|
app_state->hash = estate->hash * 31 + ep.hash;
|
|
app_state->res = estate->res;
|
|
app_state->index = estate->index+1;
|
|
app_state->n_args= estate->n_args;
|
|
for (size_t i = 0; i < app_state->n_args; i++) {
|
|
app_state->args[i] = estate->args[i];
|
|
}
|
|
queue.push_back(app_state);
|
|
std::push_heap(queue.begin(), queue.end(), estate_comp);
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
if (estate->res == NULL) {
|
|
*prob = estate->prob;
|
|
return estate->expr;
|
|
}
|
|
|
|
prob_t prob = estate->prob - (estate->res->pending[0]->prob-estate->res->viterbi_prob);
|
|
for (size_t i = estate->res->exprs.size(); i > 0; i--) {
|
|
ExprProb &ep = estate->res->exprs[i-1];
|
|
if (ep.prob != prob)
|
|
break;
|
|
if (ep.hash == estate->hash)
|
|
return 0;
|
|
}
|
|
|
|
estate->res->exprs.emplace_back(estate->expr, prob, estate->hash);
|
|
for (ExprState *parent : estate->res->pending) {
|
|
ExprState *app_state = new(parent->n_args) ExprState;
|
|
app_state->expr = parent->expr ? u->eapp(parent->expr, estate->expr) : estate->expr;
|
|
app_state->prob = parent->prob-estate->res->viterbi_prob+prob;
|
|
app_state->hash = parent->hash * 31 + estate->hash;
|
|
app_state->res = parent->res;
|
|
app_state->index = parent->index+1;
|
|
app_state->n_args= parent->n_args;
|
|
for (size_t i = 0; i < app_state->n_args; i++) {
|
|
app_state->args[i] = parent->args[i];
|
|
}
|
|
queue.push_back(app_state);
|
|
std::push_heap(queue.begin(), queue.end(), estate_comp);
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
PgfAbstractParser::State *PgfParser::new_state(const PgfTextSpot &start, prob_t viterbi_prob)
|
|
{
|
|
State **prev = ¤t_state;
|
|
State *state = current_state;
|
|
while (state != NULL && state->start.byte_pos <= start.byte_pos) {
|
|
if (state->start.byte_pos == start.byte_pos)
|
|
return state;
|
|
prev = &state->next;
|
|
state = state->next;
|
|
}
|
|
|
|
state = new State;
|
|
state->start = start;
|
|
state->end = start;
|
|
state->did_bu_predict = false;
|
|
state->viterbi_prob = viterbi_prob;
|
|
state->next = *prev;
|
|
*prev = state;
|
|
|
|
while (state->end.byte_pos < sentence->size) {
|
|
const uint8_t *ptr = (uint8_t *) &sentence->text[state->end.byte_pos];
|
|
uint32_t ucs = pgf_utf8_decode(&ptr);
|
|
if (!pgf_utf8_is_space(ucs))
|
|
break;
|
|
state->end.pos++;
|
|
state->end.byte_pos = ptr-(uint8_t *) sentence->text;
|
|
}
|
|
|
|
state->needs_bind = (state->start.pos > 0 && state->start.pos == state->end.pos);
|
|
|
|
return state;
|
|
}
|
|
|
|
void PgfParser::symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks)
|
|
{
|
|
PgfTextSpot next = state->end;
|
|
if (text_symbol_cmp(sentence,&next,symks,case_sensitive) != 0)
|
|
return;
|
|
|
|
State *next_state = new_state(next, item->inside_prob+item->outside_prob);
|
|
|
|
item->dot++;
|
|
process(item, next_state);
|
|
}
|
|
|
|
void PgfParser::symbol_bind(Item *item, State *state, PgfSymbol sym)
|
|
{
|
|
if (state->needs_bind) {
|
|
State *next_state = state->next;
|
|
if (next_state == NULL || state->end.pos != next_state->start.pos) {
|
|
next_state = new State;
|
|
next_state->start = state->end;
|
|
next_state->end = state->end;
|
|
next_state->next = state->next;
|
|
next_state->needs_bind = false;
|
|
next_state->did_bu_predict = false;
|
|
next_state->viterbi_prob = state->viterbi_prob;
|
|
state->next = next_state;
|
|
}
|
|
item->dot++;
|
|
next_state->push_item(item);
|
|
} else {
|
|
if (ref<PgfSymbol>::get_tag(sym) == PgfSymbolBIND::tag) {
|
|
delete item;
|
|
} else {
|
|
item->dot++;
|
|
process(item, state);
|
|
}
|
|
}
|
|
}
|
|
|
|
void PgfParser::suspend(Cont *cont,Item *item,bool do_predict,ref<PgfSymbolCat> symcat,interval_t value_i,interval_t lin_idx_i)
|
|
{
|
|
auto &suspended = cont->suspended[value_i][lin_idx_i];
|
|
suspended.push_back(item);
|
|
|
|
if (suspended.size() == 1) {
|
|
std::function<void(ref<PgfCCat>)> f =
|
|
[this,item,cont](ref<PgfCCat> arg) {
|
|
|
|
ref<PgfItem> pitem = arg->items[0];
|
|
|
|
PgfSymbol sym = item->rule->syms[item->dot];
|
|
auto sym_cat = ref<PgfSymbolCat>::untagged(sym);
|
|
size_t *values1 = CLONE_VALUES(item->rule, &item->vars[0]);
|
|
size_t *values2 = CLONE_VALUES(pitem->rule, &pitem->vars[0]);
|
|
if (!instantiate(item->rule, values1, item->rule->args[sym_cat->d],
|
|
pitem->rule, values2, pitem->rule->res)) {
|
|
return;
|
|
}
|
|
if (!instantiate(item->rule, values1, ref<PgfLParam>::from_ptr(&sym_cat->r),
|
|
pitem->rule, values2, pitem->rule->lin_idx)) {
|
|
return;
|
|
}
|
|
|
|
CCat *&arg_ccat = epsilons[arg->fid];
|
|
if (arg_ccat == NULL) {
|
|
arg_ccat = new CCat;
|
|
arg_ccat->fid = arg->fid;
|
|
arg_ccat->epsilon = arg;
|
|
arg_ccat->cont = NULL;
|
|
arg_ccat->state = NULL;
|
|
arg_ccat->lin_idx = arg->lin_idx;
|
|
arg_ccat->value = arg->value;
|
|
arg_ccat->covered = true;
|
|
arg_ccat->viterbi_prob = arg->viterbi_prob;
|
|
}
|
|
|
|
cont->state->completed[cont][arg_ccat->value][arg_ccat->lin_idx] = arg_ccat;
|
|
};
|
|
|
|
if (cont->ccat == NULL) {
|
|
epsilontable_iter(concr->epsilontable,cont->lincat,0,f);
|
|
|
|
if (!cont->state->did_bu_predict) {
|
|
cont->state->did_bu_predict = true;
|
|
prob_t viterbi_prob = item->inside_prob+item->outside_prob;
|
|
if (cont->state->needs_bind) {
|
|
bu_predict(concr->phrasetable4, cont->state, viterbi_prob);
|
|
} else {
|
|
bu_predict(concr->phrasetable1, cont->state, viterbi_prob, 1, sentence->size);
|
|
}
|
|
}
|
|
} else if (cont->ccat->fid <= concr->last_fid) {
|
|
epsilontable_iter(concr->epsilontable,cont->lincat,cont->ccat->fid,f);
|
|
|
|
if (do_predict) {
|
|
size_t n_items;
|
|
phrasetable_lookup(concr->phrasetable3,
|
|
cont->ccat->epsilon,
|
|
&n_items);
|
|
|
|
if (n_items == 0) {
|
|
for (size_t i = 0; i < cont->ccat->epsilon->n_items; i++) {
|
|
ref<PgfItem> pitem = cont->ccat->epsilon->items[i];
|
|
td_epsilon(cont->state,cont,pitem,item,symcat);
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
for (Production *prod : cont->ccat->prods) {
|
|
td_predict(cont->state,cont,prod,item,symcat);
|
|
}
|
|
}
|
|
}
|
|
|
|
State *state = cont->state;
|
|
while (state != NULL) {
|
|
auto it1 = state->completed.find(cont);
|
|
if (it1 != state->completed.end()) {
|
|
for (auto it2 : it1->second.overlaps(value_i)) {
|
|
for (auto it3 : it2.second.overlaps(lin_idx_i)) {
|
|
Item *new_item = new (item) Item;
|
|
combine(state, new_item, it3.second);
|
|
}
|
|
}
|
|
}
|
|
state = state->next;
|
|
}
|
|
}
|
|
|
|
void PgfParser::final_item(State *state, CCat *ccat, Item *item, interval_t value, interval_t lin_idx)
|
|
{
|
|
if (item->cont == NULL && state->end.byte_pos == sentence->size) {
|
|
ExprState *estate = new(item->args.size()) ExprState;
|
|
estate->expr = 0;
|
|
estate->prob = 0;
|
|
estate->hash = 0;
|
|
estate->res = ccat;
|
|
estate->index = 0;
|
|
estate->n_args = item->args.size();
|
|
for (size_t i = 0; i < estate->n_args; i++) {
|
|
estate->args[i] = item->args[i];
|
|
estate->prob += estate->args[i]->viterbi_prob;
|
|
}
|
|
queue.push_back(estate);
|
|
std::push_heap(queue.begin(), queue.end(), estate_comp);
|
|
} else if (ccat != NULL && ccat->pending.size() > 0) {
|
|
auto lin = ref<PgfConcrLin>::untagged(item->rule->container);
|
|
ExprState *estate = new(item->args.size()) ExprState;
|
|
estate->expr = u->efun(&lin->name);
|
|
estate->prob = ccat->pending[0]->prob-ccat->viterbi_prob+lin->absfun->prob;
|
|
estate->hash = 0;
|
|
estate->res = ccat;
|
|
estate->index = 0;
|
|
estate->n_args = item->args.size();
|
|
for (size_t i = 0; i < lin->name.size; i++) {
|
|
estate->hash = estate->hash * 101 + lin->name.text[i];
|
|
}
|
|
for (size_t i = 0; i < estate->n_args; i++) {
|
|
estate->args[i] = item->args[i];
|
|
if (estate->args[i] != NULL) {
|
|
estate->prob += estate->args[i]->viterbi_prob;
|
|
}
|
|
}
|
|
queue.push_back(estate);
|
|
std::push_heap(queue.begin(), queue.end(), estate_comp);
|
|
}
|
|
}
|
|
|
|
#ifdef DEBUG_EXPRS
|
|
void PgfParser::print_expr_state_left(PgfPrinter *printer, PgfMarshaller *m, ExprState *estate)
|
|
{
|
|
if (estate->res && estate->res->pending.size() > 0) {
|
|
ExprState *parent = estate->res->pending[0];
|
|
print_expr_state_left(printer, m, parent);
|
|
printer->puts(" (");
|
|
}
|
|
|
|
if (estate->expr)
|
|
m->match_expr(printer, estate->expr);
|
|
else
|
|
printer->puts("::");
|
|
}
|
|
|
|
void PgfParser::print_expr_state_right(PgfPrinter *printer, ExprState *estate)
|
|
{
|
|
for (size_t i = estate->index+1; i < estate->n_args; i++) {
|
|
printer->puts(" ");
|
|
if (estate->args[i] != NULL)
|
|
printer->emeta(estate->args[i]->fid);
|
|
else
|
|
printer->puts("?");
|
|
}
|
|
|
|
if (estate->res && estate->res->pending.size() > 0) {
|
|
printer->puts(")");
|
|
ExprState *parent = estate->res->pending[0];
|
|
print_expr_state_right(printer, parent);
|
|
}
|
|
}
|
|
|
|
void PgfParser::print_expr_state(PgfMarshaller *m, ExprState *estate)
|
|
{
|
|
PgfPrinter printer(NULL,0,m);
|
|
printer.nprintf(64,"[%f] ",estate->prob);
|
|
print_expr_state_left(&printer, m, estate);
|
|
printer.puts(" . ");
|
|
|
|
if (estate->index < estate->n_args) {
|
|
if (estate->args[estate->index] != NULL)
|
|
printer.emeta(estate->args[estate->index]->fid);
|
|
else
|
|
printer.puts("?");
|
|
}
|
|
|
|
print_expr_state_right(&printer, estate);
|
|
|
|
PgfText *text = printer.get_text();
|
|
fprintf(stderr, "%s\n", text->text);
|
|
free(text);
|
|
}
|
|
#endif
|
|
|
|
PgfText *PgfParser::get_text() {
|
|
return sentence;
|
|
}
|
|
|
|
void PgfParser::start()
|
|
{
|
|
prev_state = current_state;
|
|
current_state = NULL;
|
|
while (prev_state != NULL) {
|
|
State *next = prev_state->next;
|
|
prev_state->next = current_state;
|
|
current_state = prev_state;
|
|
prev_state = next;
|
|
}
|
|
update_old_pos = 0;
|
|
update_new_pos = 0;
|
|
delta_byte_pos = 0;
|
|
}
|
|
|
|
bool PgfParser::skip(size_t i)
|
|
{
|
|
update_old_pos += i;
|
|
update_new_pos += i;
|
|
if (update_new_pos > sentence->size)
|
|
return false;
|
|
|
|
while (current_state != NULL && current_state->start.pos < update_old_pos) {
|
|
State *next = current_state->next;
|
|
current_state->next = prev_state;
|
|
current_state->start.pos += update_new_pos-update_old_pos;
|
|
current_state->start.byte_pos += delta_byte_pos;
|
|
current_state->end.pos += update_new_pos-update_old_pos;
|
|
current_state->end.byte_pos += delta_byte_pos;
|
|
prev_state = current_state;
|
|
current_state = next;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
bool PgfParser::change(size_t i, PgfText *change)
|
|
{
|
|
update_old_pos += i;
|
|
if (update_new_pos+i > sentence->size)
|
|
return false;
|
|
|
|
while (current_state != NULL && current_state->end.pos <= update_old_pos) {
|
|
State *next = current_state->next;
|
|
delete current_state;
|
|
current_state = next;
|
|
}
|
|
|
|
size_t new_size = sentence->size + change->size - i;
|
|
size_t new_allocated_size = sizeof(PgfText)+new_size+1;
|
|
if (allocated_size < new_allocated_size) {
|
|
PgfText *new_sentence = (PgfText *) realloc(sentence, new_allocated_size);
|
|
if (new_sentence == NULL)
|
|
return false;
|
|
sentence = new_sentence;
|
|
allocated_size = new_allocated_size;
|
|
}
|
|
|
|
size_t pos;
|
|
const uint8_t *ptr1;
|
|
if (prev_state!=NULL) {
|
|
ptr1 = (uint8_t *) &sentence->text[prev_state->start.byte_pos];
|
|
pos = prev_state->start.pos;
|
|
} else {
|
|
ptr1 = (uint8_t *) sentence->text;
|
|
pos = 0;
|
|
}
|
|
while (pos < update_new_pos) {
|
|
pgf_utf8_decode(&ptr1); pos++;
|
|
}
|
|
|
|
const uint8_t *ptr2 = ptr1;
|
|
while (pos < update_new_pos+i) {
|
|
pgf_utf8_decode(&ptr2); pos++;
|
|
}
|
|
|
|
memcpy((void*) (ptr1+change->size), ptr2, (uint8_t*)sentence->text+sentence->size-ptr2+1);
|
|
memcpy((void*) ptr1, change->text, change->size);
|
|
|
|
delta_byte_pos += (ptr1+change->size)-ptr2;
|
|
ptr2 = ptr1+change->size;
|
|
while (ptr1 < ptr2) {
|
|
pgf_utf8_decode(&ptr1);
|
|
update_new_pos++;
|
|
}
|
|
|
|
sentence->size = new_size;
|
|
return true;
|
|
}
|
|
|
|
void PgfParser::done()
|
|
{
|
|
while (current_state != NULL) {
|
|
State *next = current_state->next;
|
|
current_state->next = prev_state;
|
|
current_state->start.pos += update_new_pos-update_old_pos;
|
|
current_state->start.byte_pos += delta_byte_pos;
|
|
current_state->end.pos += update_new_pos-update_old_pos;
|
|
current_state->end.byte_pos += delta_byte_pos;
|
|
prev_state = current_state;
|
|
current_state = next;
|
|
}
|
|
current_state = prev_state;
|
|
prev_state = NULL;
|
|
update_old_pos = 0;
|
|
update_new_pos = 0;
|
|
delta_byte_pos = 0;
|
|
}
|
|
|
|
PgfParseTableMaker::PgfParseTableMaker(ref<PgfConcr> concr)
|
|
: PgfAbstractParser(concr)
|
|
{
|
|
current_state = new State;
|
|
current_state->start.pos = 0;
|
|
current_state->start.byte_pos = 0;
|
|
current_state->needs_bind = false;
|
|
current_state->did_bu_predict = false;
|
|
current_state->end = current_state->start;
|
|
current_state->viterbi_prob = 0;
|
|
current_state->next = NULL;
|
|
}
|
|
|
|
ref<PgfItem> PgfParseTableMaker::clone_item(Item *item)
|
|
{
|
|
size_t ex_size =
|
|
sizeof(PgfMetaId) * item->args.size() +
|
|
sizeof(size_t) * item->vars.size();
|
|
auto pitem = PgfDB::malloc<PgfItem>(ex_size);
|
|
pitem->res = (item->cont->ccat == NULL) ? 0 : item->cont->ccat->fid;
|
|
pitem->pre_alt = item->pre_alt;
|
|
pitem->pre_dot = item->pre_dot;
|
|
pitem->dot = item->dot;
|
|
pitem->rule = item->rule;
|
|
memcpy(&pitem->vars[0],&item->vars[0],sizeof(size_t) * item->vars.size());
|
|
|
|
for (size_t i = 0; i < item->args.size(); i++) {
|
|
pitem->args[i] = (item->args[i] == NULL) ? 0 : item->args[i]->fid;
|
|
}
|
|
|
|
return pitem;
|
|
}
|
|
|
|
PgfAbstractParser::State *PgfParseTableMaker::new_state(const PgfTextSpot &start, prob_t viterbi_prob)
|
|
{
|
|
return current_state;
|
|
}
|
|
|
|
void PgfParseTableMaker::symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks)
|
|
{
|
|
auto pitem = clone_item(item);
|
|
auto phrasetable1 = phrasetable_insert(concr->phrasetable1,symks,pitem);
|
|
concr->phrasetable1 = phrasetable1;
|
|
delete item;
|
|
}
|
|
|
|
void PgfParseTableMaker::symbol_bind(Item *item, State *state, PgfSymbol sym)
|
|
{
|
|
auto pitem = clone_item(item);
|
|
auto phrasetable4 = phrasetable_insert(concr->phrasetable4,ref<PgfSymbolBIND>(0),pitem);
|
|
concr->phrasetable4 = phrasetable4;
|
|
|
|
if (ref<PgfSymbol>::get_tag(sym) == PgfSymbolBIND::tag) {
|
|
delete item;
|
|
} else {
|
|
item->dot++;
|
|
process(item,state);
|
|
}
|
|
}
|
|
|
|
void PgfParseTableMaker::suspend(Cont *cont,Item *item,bool do_predict,ref<PgfSymbolCat> symcat,interval_t value_i,interval_t lin_idx_i)
|
|
{
|
|
auto &suspended = cont->suspended[value_i][lin_idx_i];
|
|
suspended.push_back(item);
|
|
|
|
size_t n_suspended = suspended.size();
|
|
if (cont->ccat == NULL) {
|
|
auto pitem = clone_item(item);
|
|
auto phrasetable2 = phrasetable_insert(concr->phrasetable2,cont->lincat,pitem);
|
|
concr->phrasetable2 = phrasetable2;
|
|
} else {
|
|
if (do_predict && n_suspended == 1) {
|
|
if (cont->ccat->fid <= concr->last_fid) {
|
|
for (size_t i = 0; i < cont->ccat->epsilon->n_items; i++) {
|
|
ref<PgfItem> pitem = cont->ccat->epsilon->items[i];
|
|
td_epsilon(cont->state,cont,pitem,item,symcat);
|
|
}
|
|
} else {
|
|
for (Production *prod : cont->ccat->prods) {
|
|
td_predict(cont->state,cont,prod,item,symcat);
|
|
}
|
|
}
|
|
}
|
|
|
|
auto pitem = clone_item(item);
|
|
auto phrasetable3 = phrasetable_insert(concr->phrasetable3,cont->ccat->epsilon,pitem);
|
|
concr->phrasetable3 = phrasetable3;
|
|
}
|
|
|
|
auto it1 = cont->state->completed.find(cont);
|
|
if (it1 != cont->state->completed.end()) {
|
|
for (auto it2 : it1->second.overlaps(value_i)) {
|
|
for (auto it3 : it2.second.overlaps(lin_idx_i)) {
|
|
CCat *arg = it3.second;
|
|
Item *new_item = new (item) Item;
|
|
combine(cont->state, new_item, arg);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void PgfParseTableMaker::final_item(State *state, CCat *ccat, Item *item, interval_t value, interval_t lin_idx)
|
|
{
|
|
auto pitem = clone_item(item);
|
|
|
|
if (ccat->epsilon == 0) {
|
|
PgfEpsilontable epsilontable = concr->epsilontable;
|
|
epsilontable =
|
|
epsilontable_insert(epsilontable,
|
|
ccat->cont->lincat,
|
|
(ccat->cont->ccat != NULL) ? ccat->cont->ccat->fid : 0,
|
|
ccat->value, ccat->lin_idx,
|
|
ccat->fid, ccat->viterbi_prob,
|
|
pitem,
|
|
&ccat->epsilon);
|
|
concr->epsilontable = epsilontable;
|
|
} else {
|
|
epsilontable_add(ccat->epsilon, pitem);
|
|
}
|
|
}
|
|
|
|
void PgfParseTableMaker::bu_predict(State *state, prob_t outside_prob, CCat *ccat)
|
|
{
|
|
}
|
|
|
|
void PgfParseTableMaker::insert_rule(ref<PgfConcrRule> rule)
|
|
{
|
|
switch (ref<object>::get_tag(rule->container)) {
|
|
case PgfConcrLin::tag: {
|
|
auto lin = ref<PgfConcrLin>::untagged(rule->container);
|
|
|
|
Cont *&cont = current_state->conts1[lin->lincat];
|
|
if (cont == NULL) {
|
|
cont = new Cont;
|
|
cont->ccat = NULL;
|
|
cont->lincat = lin->lincat;
|
|
cont->state = current_state;
|
|
}
|
|
|
|
Item *item = new(rule) Item;
|
|
item->cont = cont;
|
|
item->dot = 0;
|
|
item->pre_alt = 0;
|
|
item->pre_dot = 0;
|
|
item->syms = rule->syms.as_vector();
|
|
item->rule = rule;
|
|
item->inside_prob = lin->absfun->prob;
|
|
item->outside_prob = 0;
|
|
current_state->push_item(item);
|
|
}
|
|
}
|
|
}
|
|
|
|
void PgfParseTableMaker::prepare()
|
|
{
|
|
while (current_state->has_items()) {
|
|
Item *item = current_state->pop_item();
|
|
process(item,current_state);
|
|
}
|
|
}
|