msyds/gf-core
1
0
Fork 0
forked from GitHub/gf-core
Code Pull Requests Activity
Files
a307ed6c75c2dbf116170f0dc1904e6725ac74f3
gf-core/src/runtime/c/pgf/parser.c

2129 lines
56 KiB
C
Raw Blame History

#include <pgf/parser.h>
#include <gu/seq.h>
#include <gu/assert.h>
#include <gu/log.h>
#include <gu/file.h>
#include <math.h>
//#define PGF_PARSER_DEBUG
typedef GuBuf PgfItemBuf;
static GU_DEFINE_TYPE(PgfItemBuf, abstract, _);
typedef GuList(PgfItemBuf*) PgfItemBufs;
static GU_DEFINE_TYPE(PgfItemBufs, abstract, _);
typedef GuMap PgfContsMap;
GU_DEFINE_TYPE(PgfContsMap, GuMap,
gu_type(PgfCCat), NULL,
gu_ptr_type(PgfItemBufs), &gu_null_struct);
typedef GuMap PgfGenCatMap;
GU_DEFINE_TYPE(PgfGenCatMap, GuMap,
gu_type(PgfItemBuf), NULL,
gu_ptr_type(PgfCCat), &gu_null_struct);
typedef GuBuf PgfCCatBuf;
typedef struct {
PgfConcr* concr;
PgfCCat* completed;
PgfItem* target;
PgfExpr meta_var;
PgfProduction meta_prod;
int max_fid;
int item_count;
} PgfParsing;
typedef struct {
PgfToken tok;
PgfItemBuf* lexicon_idx;
} PgfTokenState;
struct PgfParseState {
GuPool* pool;
PgfParseState* next;
PgfItemBuf* agenda;
PgfContsMap* conts_map;
PgfGenCatMap* generated_cats;
int offset;
PgfParsing* ps;
PgfTokenState* ts;
};
typedef struct PgfExprState PgfExprState;
struct PgfExprState {
PgfExprState* cont;
PgfExpr expr;
PgfPArgs args;
size_t arg_idx;
};
typedef struct {
PgfExprState *st;
prob_t prob;
} PgfExprQState;
typedef struct PgfParseResult PgfParseResult;
struct PgfParseResult {
GuPool* tmp_pool;
PgfParseState* state;
GuBuf* pqueue;
PgfExprEnum en;
};
typedef struct PgfItemBase PgfItemBase;
struct PgfItemBase {
PgfItemBuf* conts;
PgfCCat* ccat;
PgfProduction prod;
unsigned short lin_idx;
};
struct PgfItem {
#ifdef PGF_PARSER_DEBUG
int start,end;
#endif
PgfItemBase* base;
PgfPArgs args;
PgfSymbol curr_sym;
uint16_t seq_idx;
uint8_t tok_idx;
uint8_t alt;
prob_t inside_prob;
prob_t outside_prob;
};
typedef struct {
int fid;
int lin_idx;
} PgfCFCat;
static GU_DEFINE_TYPE(PgfCFCat, struct,
GU_MEMBER(PgfCFCat, fid, int),
GU_MEMBER(PgfCFCat, lin_idx, int));
extern GuHasher pgf_cfcat_hasher;
GU_DEFINE_TYPE(PgfEpsilonIdx, GuMap,
gu_type(PgfCFCat), &pgf_cfcat_hasher,
gu_ptr_type(PgfCCat), &gu_null_struct);
// GuString -> PgfItemBuf*
GU_DEFINE_TYPE(PgfTransitions, GuStringMap,
gu_ptr_type(PgfItemBuf), &gu_null_struct);
static PgfSymbol
pgf_prev_extern_sym(PgfSymbol sym)
{
GuVariantInfo i = gu_variant_open(sym);
switch (i.tag) {
case PGF_SYMBOL_CAT:
return *((PgfSymbol*) (((PgfSymbolCat*) i.data)+1));
case PGF_SYMBOL_KP:
return *((PgfSymbol*) (((PgfSymbolKP*) i.data)+1));
case PGF_SYMBOL_KS:
return *((PgfSymbol*) (((PgfSymbolKS*) i.data)+1));
case PGF_SYMBOL_LIT:
return *((PgfSymbol*) (((PgfSymbolLit*) i.data)+1));
case PGF_SYMBOL_VAR:
return *((PgfSymbol*) (((PgfSymbolVar*) i.data)+1));
default:
gu_impossible();
return gu_null_variant;
}
}
int
pgf_item_lin_idx(PgfItem* item) {
return item->base->lin_idx;
}
int
pgf_item_sequence_length(PgfItem* item)
{
GuVariantInfo i = gu_variant_open(item->base->prod);
switch (i.tag) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papp = i.data;
return gu_seq_length(papp->fun->lins[item->base->lin_idx]);
}
case PGF_PRODUCTION_COERCE: {
return 1;
}
case PGF_PRODUCTION_EXTERN: {
PgfProductionExtern* pext = i.data;
PgfSequence seq;
if (!gu_seq_is_null(pext->lins) &&
!gu_seq_is_null(seq = gu_seq_get(pext->lins,PgfSequence,item->base->lin_idx))) {
return gu_seq_length(seq);
} else {
int seq_len = 0;
PgfSymbol sym = item->curr_sym;
while (!gu_variant_is_null(sym)) {
seq_len++;
sym = pgf_prev_extern_sym(sym);
}
return seq_len;
}
}
case PGF_PRODUCTION_META: {
PgfProductionMeta* pmeta = i.data;
PgfSequence seq;
if (!gu_seq_is_null(pmeta->lins) &&
!gu_seq_is_null(seq = gu_seq_get(pmeta->lins,PgfSequence,item->base->lin_idx))) {
return gu_seq_length(seq);
} else {
int seq_len = 0;
PgfSymbol sym = item->curr_sym;
while (!gu_variant_is_null(sym)) {
seq_len++;
sym = pgf_prev_extern_sym(sym);
}
return seq_len;
}
}
default:
gu_impossible();
return 0;
}
}
static PgfSequence
pgf_extern_seq_get(PgfItem* item, GuPool* pool)
{
int seq_len = pgf_item_sequence_length(item);
PgfSequence seq =
gu_new_seq(PgfSymbol, seq_len, pool);
PgfSymbol sym = item->curr_sym;
while (!gu_variant_is_null(sym)) {
gu_seq_set(seq, PgfSymbol, --seq_len, sym);
sym = pgf_prev_extern_sym(sym);
}
return seq;
}
void
pgf_item_sequence(PgfItem* item,
int* lin_idx, PgfSequence* seq,
GuPool* pool) {
*lin_idx = item->base->lin_idx;
GuVariantInfo i = gu_variant_open(item->base->prod);
switch (i.tag) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papp = i.data;
*seq = papp->fun->lins[item->base->lin_idx];
break;
}
case PGF_PRODUCTION_COERCE: {
PgfSymbol sym =
gu_new_variant_i(pool, PGF_SYMBOL_CAT,
PgfSymbolCat,
.d = 0, .r = item->base->lin_idx);
*seq = gu_new_seq(PgfSequence, 1, pool);
gu_seq_set(*seq, PgfSymbol, 0, sym);
break;
}
case PGF_PRODUCTION_EXTERN: {
PgfProductionExtern* pext = i.data;
if (gu_seq_is_null(pext->lins) ||
gu_seq_is_null(*seq = gu_seq_get(pext->lins, PgfSequence, item->base->lin_idx))) {
*seq = pgf_extern_seq_get(item, pool);
}
break;
}
case PGF_PRODUCTION_META: {
PgfProductionMeta* pmeta = i.data;
if (gu_seq_is_null(pmeta->lins) ||
gu_seq_is_null(*seq = gu_seq_get(pmeta->lins,PgfSequence,item->base->lin_idx))) {
*seq = pgf_extern_seq_get(item, pool);
}
break;
}
default:
gu_impossible();
}
}
#ifdef PGF_PARSER_DEBUG
static void
pgf_print_production_args(PgfPArgs args,
GuWriter* wtr, GuExn* err)
{
size_t n_args = gu_seq_length(args);
for (size_t j = 0; j < n_args; j++) {
if (j > 0)
gu_putc(',',wtr,err);
PgfPArg arg = gu_seq_get(args, PgfPArg, j);
if (arg.hypos != NULL &&
gu_list_length(arg.hypos) > 0) {
size_t n_hypos = gu_list_length(arg.hypos);
for (size_t k = 0; k < n_hypos; k++) {
PgfCCat *hypo = gu_list_index(arg.hypos, k);
gu_printf(wtr,err,"C%d ",hypo->fid);
}
gu_printf(wtr,err,"-> ");
}
gu_printf(wtr,err,"C%d",arg.ccat->fid);
}
}
static void
pgf_print_production(int fid, PgfProduction prod,
GuWriter *wtr, GuExn* err, GuPool* pool)
{
gu_printf(wtr,err,"C%d -> ",fid);
GuVariantInfo i = gu_variant_open(prod);
switch (i.tag) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papp = i.data;
gu_printf(wtr,err,"F%d(",papp->fun->funid);
pgf_print_expr(papp->fun->ep->expr, 0, wtr, err);
gu_printf(wtr,err,")[");
pgf_print_production_args(papp->args,wtr,err);
gu_printf(wtr,err,"]\n");
break;
}
case PGF_PRODUCTION_COERCE: {
PgfProductionCoerce* pcoerce = i.data;
gu_printf(wtr,err,"_[C%d]\n",pcoerce->coerce->fid);
break;
}
case PGF_PRODUCTION_EXTERN: {
PgfProductionExtern* pext = i.data;
gu_printf(wtr,err,"<extern>(");
pgf_print_expr(pext->ep->expr, 0, wtr, err);
gu_printf(wtr,err,")[]\n");
break;
}
case PGF_PRODUCTION_META: {
PgfProductionMeta* pmeta = i.data;
gu_printf(wtr,err,"<meta>[");
pgf_print_production_args(pmeta->args,wtr,err);
gu_printf(wtr,err,"]\n");
break;
}
default:
gu_impossible();
}
}
void
pgf_print_symbol(PgfSymbol sym, GuWriter *wtr, GuExn *err);
static void
pgf_print_item_seq(PgfItem *item,
GuWriter* wtr, GuExn* err, GuPool* pool)
{
int lin_idx;
PgfSequence seq;
pgf_item_sequence(item, &lin_idx, &seq, pool);
gu_printf(wtr, err, "%d : ",lin_idx);
size_t index;
for (index = 0; index < gu_seq_length(seq); index++) {
if (item->seq_idx == index)
gu_printf(wtr, err, " . ");
PgfSymbol *sym = gu_seq_index(seq, PgfSymbol, index);
pgf_print_symbol(*sym, wtr, err);
}
if (item->seq_idx == index)
gu_printf(wtr, err, " .");
}
static void
pgf_print_item(PgfItem* item, GuWriter* wtr, GuExn* err, GuPool* pool)
{
gu_printf(wtr, err, "[%d-%d; C%d -> ",
item->start,
item->end,
item->base->ccat->fid);
GuVariantInfo i = gu_variant_open(item->base->prod);
switch (i.tag) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papp = i.data;
PgfCncFun* fun = papp->fun;
gu_printf(wtr, err, "F%d(", fun->funid);
pgf_print_expr(fun->ep->expr, 0, wtr, err);
gu_printf(wtr, err, ")[");
pgf_print_production_args(item->args, wtr, err);
gu_printf(wtr, err, "]; ");
break;
}
case PGF_PRODUCTION_COERCE: {
gu_printf(wtr, err, "_[C%d]; ",
gu_seq_index(item->args, PgfPArg, 0)->ccat->fid);
break;
}
case PGF_PRODUCTION_EXTERN: {
PgfProductionExtern* pext = i.data;
gu_printf(wtr, err, "<extern>");
if (pext->ep != NULL) {
gu_printf(wtr, err, "(");
pgf_print_expr(pext->ep->expr, 0, wtr, err);
gu_printf(wtr, err, ")");
}
gu_printf(wtr, err, "[");
pgf_print_production_args(item->args, wtr, err);
gu_printf(wtr, err, "]; ");
break;
}
case PGF_PRODUCTION_META: {
gu_printf(wtr, err, "<meta>[");
pgf_print_production_args(item->args, wtr, err);
gu_printf(wtr, err, "]; ");
break;
}
default:
gu_impossible();
}
pgf_print_item_seq(item, wtr, err, pool);
gu_printf(wtr, err, "; %f+%f=%f]\n",
item->inside_prob,
item->outside_prob,
item->inside_prob+item->outside_prob);
}
#endif
static int
cmp_item_prob(GuOrder* self, const void* a, const void* b)
{
PgfItem *item1 = *((PgfItem **) a);
PgfItem *item2 = *((PgfItem **) b);
prob_t prob1 = item1->inside_prob + item1->outside_prob;
prob_t prob2 = item2->inside_prob + item2->outside_prob;
if (prob1 < prob2)
return -1;
else if (prob1 > prob2)
return 1;
else
return 0;
}
static GuOrder
pgf_item_prob_order = { cmp_item_prob };
static void
pgf_parsing_add_transition(PgfParseState* before, PgfParseState* after,
PgfToken tok, PgfItem* item)
{
if (gu_string_eq(tok, after->ts->tok)) {
if (after->next == NULL)
after->ps->target = item;
gu_buf_heap_push(after->agenda, &pgf_item_prob_order, &item);
}
}
static PgfItemBufs*
pgf_parsing_get_contss(PgfContsMap* conts_map, PgfCCat* cat, GuPool *tmp_pool)
{
PgfItemBufs* contss = gu_map_get(conts_map, cat, PgfItemBufs*);
if (!contss) {
size_t n_lins = cat->cnccat->n_lins;
contss = gu_new_list(PgfItemBufs, tmp_pool, n_lins);
for (size_t i = 0; i < n_lins; i++) {
gu_list_index(contss, i) = NULL;
}
gu_map_put(conts_map, cat, PgfItemBufs*, contss);
}
return contss;
}
static PgfItemBuf*
pgf_parsing_get_conts(PgfContsMap* conts_map,
PgfCCat* ccat, size_t lin_idx,
GuPool *pool, GuPool *tmp_pool)
{
gu_require(lin_idx < ccat->cnccat->n_lins);
PgfItemBufs* contss =
pgf_parsing_get_contss(conts_map, ccat, tmp_pool);
PgfItemBuf* conts = gu_list_index(contss, lin_idx);
if (!conts) {
conts = gu_new_buf(PgfItem*, pool);
gu_list_index(contss, lin_idx) = conts;
}
return conts;
}
static PgfCCat*
pgf_parsing_create_completed(PgfParseState* state, PgfItemBuf* conts,
prob_t viterbi_prob, PgfCncCat* cnccat)
{
PgfCCat* cat = gu_new(PgfCCat, state->pool);
cat->cnccat = cnccat;
cat->viterbi_prob = viterbi_prob;
cat->fid = state->ps->max_fid++;
cat->prods = gu_buf_seq(gu_new_buf(PgfProduction, state->pool));
cat->n_synprods = 0;
gu_map_put(state->generated_cats, conts, PgfCCat*, cat);
return cat;
}
static PgfCCat*
pgf_parsing_get_completed(PgfParseState* state, PgfItemBuf* conts)
{
return gu_map_get(state->generated_cats, conts, PgfCCat*);
}
static void
pgf_item_set_curr_symbol(PgfItem* item, GuPool* pool)
{
GuVariantInfo i = gu_variant_open(item->base->prod);
switch (i.tag) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papp = i.data;
PgfCncFun* fun = papp->fun;
gu_assert(item->base->lin_idx < fun->n_lins);
PgfSequence seq = fun->lins[item->base->lin_idx];
gu_assert(item->seq_idx <= gu_seq_length(seq));
if (item->seq_idx == gu_seq_length(seq)) {
item->curr_sym = gu_null_variant;
} else {
item->curr_sym = gu_seq_get(seq, PgfSymbol, item->seq_idx);
}
break;
}
case PGF_PRODUCTION_COERCE: {
gu_assert(item->seq_idx <= 1);
if (item->seq_idx == 1) {
item->curr_sym = gu_null_variant;
} else {
item->curr_sym = gu_new_variant_i(pool, PGF_SYMBOL_CAT,
PgfSymbolCat,
.d = 0, .r = item->base->lin_idx);
}
break;
}
case PGF_PRODUCTION_EXTERN: {
break;
}
case PGF_PRODUCTION_META: {
break;
}
default:
gu_impossible();
}
}
static PgfItem*
pgf_new_item(int pos, PgfCCat* ccat, size_t lin_idx,
PgfProduction prod, PgfItemBuf* conts,
prob_t delta_prob,
GuPool* pool)
{
PgfItemBase* base = gu_new(PgfItemBase, pool);
base->ccat = ccat;
base->lin_idx = lin_idx;
base->prod = prod;
base->conts = conts;
PgfItem* item = gu_new(PgfItem, pool);
GuVariantInfo pi = gu_variant_open(prod);
switch (pi.tag) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papp = pi.data;
item->args = papp->args;
item->inside_prob = papp->fun->ep->prob;
int n_args = gu_seq_length(item->args);
for (int i = 0; i < n_args; i++) {
PgfPArg *arg = gu_seq_index(item->args, PgfPArg, i);
item->inside_prob += arg->ccat->viterbi_prob;
}
break;
}
case PGF_PRODUCTION_COERCE: {
PgfProductionCoerce* pcoerce = pi.data;
item->args = gu_new_seq(PgfPArg, 1, pool);
PgfPArg* parg = gu_seq_index(item->args, PgfPArg, 0);
parg->hypos = NULL;
parg->ccat = pcoerce->coerce;
item->inside_prob = pcoerce->coerce->viterbi_prob;
break;
}
case PGF_PRODUCTION_EXTERN: {
PgfProductionExtern* pext = pi.data;
item->args = gu_empty_seq();
item->inside_prob = pext->ep ? pext->ep->prob : 0;
int n_args = gu_seq_length(item->args);
for (int i = 0; i < n_args; i++) {
PgfPArg *arg = gu_seq_index(item->args, PgfPArg, i);
item->inside_prob += arg->ccat->viterbi_prob;
}
break;
}
case PGF_PRODUCTION_META: {
PgfProductionMeta* pmeta = pi.data;
item->args = pmeta->args;
item->inside_prob = pmeta->ep ? pmeta->ep->prob : 0;
int n_args = gu_seq_length(item->args);
for (int i = 0; i < n_args; i++) {
PgfPArg *arg = gu_seq_index(item->args, PgfPArg, i);
item->inside_prob += arg->ccat->viterbi_prob;
}
break;
}
default:
gu_impossible();
}
#ifdef PGF_PARSER_DEBUG
item->start = pos;
item->end = pos;
#endif
item->base = base;
item->curr_sym = gu_null_variant;
item->seq_idx = 0;
item->tok_idx = 0;
item->alt = 0;
item->outside_prob = 0;
if (gu_buf_length(conts) > 0) {
PgfItem* best_cont = gu_buf_get(conts, PgfItem*, 0);
if (best_cont != NULL)
item->outside_prob =
best_cont->inside_prob-ccat->viterbi_prob+
best_cont->outside_prob;
}
item->outside_prob += delta_prob;
pgf_item_set_curr_symbol(item, pool);
return item;
}
static PgfItemBase*
pgf_item_base_copy(PgfItemBase* base, GuPool* pool)
{
PgfItemBase* copy = gu_new(PgfItemBase, pool);
memcpy(copy, base, sizeof(PgfItemBase));
return copy;
}
static PgfItem*
pgf_item_copy(PgfItem* item, GuPool* pool)
{
PgfItem* copy = gu_new(PgfItem, pool);
memcpy(copy, item, sizeof(PgfItem));
return copy;
}
static PgfItem*
pgf_item_update_arg(PgfItem* item, size_t d, PgfCCat *new_ccat,
GuPool* pool)
{
PgfCCat *old_ccat =
gu_seq_index(item->args, PgfPArg, d)->ccat;
PgfItem* new_item = pgf_item_copy(item, pool);
size_t nargs = gu_seq_length(item->args);
new_item->args = gu_new_seq(PgfPArg, nargs, pool);
memcpy(gu_seq_data(new_item->args), gu_seq_data(item->args),
nargs * sizeof(PgfPArg));
gu_seq_set(new_item->args, PgfPArg, d,
((PgfPArg) { .hypos = NULL, .ccat = new_ccat }));
new_item->inside_prob +=
new_ccat->viterbi_prob - old_ccat->viterbi_prob;
return new_item;
}
static void
pgf_item_advance(PgfItem* item, GuPool* pool)
{
item->seq_idx++;
pgf_item_set_curr_symbol(item, pool);
}
static void
pgf_parsing_combine(PgfParseState* before, PgfParseState* after,
PgfItem* cont, PgfCCat* cat, int lin_idx)
{
if (cont == NULL) {
if (after == NULL)
before->ps->completed = cat;
return;
}
bool extend = false;
GuVariantInfo i = gu_variant_open(cont->base->prod);
if (i.tag == PGF_PRODUCTION_META) {
PgfProductionMeta* pmeta = i.data;
if (gu_seq_is_null(pmeta->lins) ||
gu_seq_is_null(gu_seq_get(pmeta->lins,PgfSequence,cont->base->lin_idx))) {
extend = true;
}
}
PgfItem* item = NULL;
if (!extend) {
switch (gu_variant_tag(cont->curr_sym)) {
case PGF_SYMBOL_CAT: {
PgfSymbolCat* scat = gu_variant_data(cont->curr_sym);
item = pgf_item_update_arg(cont, scat->d, cat, before->pool);
break;
}
case PGF_SYMBOL_LIT: {
PgfSymbolLit* slit = gu_variant_data(cont->curr_sym);
item = pgf_item_update_arg(cont, slit->d, cat, before->pool);
break;
}
default:
gu_impossible();
}
} else {
item = pgf_item_copy(cont, before->pool);
size_t nargs = gu_seq_length(cont->args);
item->args = gu_new_seq(PgfPArg, nargs+1, before->pool);
memcpy(gu_seq_data(item->args), gu_seq_data(cont->args),
nargs * sizeof(PgfPArg));
gu_seq_set(item->args, PgfPArg, nargs,
((PgfPArg) { .hypos = NULL, .ccat = cat }));
PgfCIdMap* meta_child_probs =
item->base->ccat->cnccat->abscat->meta_child_probs;
item->inside_prob +=
cat->viterbi_prob+
gu_map_get(meta_child_probs, cat->cnccat->abscat, prob_t);
PgfSymbol prev = item->curr_sym;
PgfSymbolCat* scat = (PgfSymbolCat*)
gu_alloc_variant(PGF_SYMBOL_CAT,
sizeof(PgfSymbolCat)+sizeof(PgfSymbol),
gu_alignof(PgfSymbolCat),
&item->curr_sym, before->pool);
*((PgfSymbol*)(scat+1)) = prev;
scat->d = nargs;
scat->r = lin_idx;
}
#ifdef PGF_PARSER_DEBUG
item->end = before->offset;
#endif
pgf_item_advance(item, before->pool);
gu_buf_heap_push(before->agenda, &pgf_item_prob_order, &item);
}
static void
pgf_parsing_production(PgfParseState* state,
PgfCCat* ccat, size_t lin_idx,
PgfProduction prod, PgfItemBuf* conts,
prob_t delta_prob)
{
PgfItem* item =
pgf_new_item(state->offset, ccat, lin_idx, prod, conts, delta_prob, state->pool);
state->ps->item_count++;
gu_buf_heap_push(state->agenda, &pgf_item_prob_order, &item);
}
static PgfProduction
pgf_parsing_new_production(PgfItem* item, PgfExprProb *ep, GuPool *pool)
{
GuVariantInfo i = gu_variant_open(item->base->prod);
PgfProduction prod = gu_null_variant;
switch (i.tag) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papp = i.data;
PgfProductionApply* new_papp =
gu_new_variant(PGF_PRODUCTION_APPLY,
PgfProductionApply,
&prod, pool);
new_papp->fun = papp->fun;
new_papp->args = item->args;
break;
}
case PGF_PRODUCTION_COERCE: {
PgfProductionCoerce* new_pcoerce =
gu_new_variant(PGF_PRODUCTION_COERCE,
PgfProductionCoerce,
&prod, pool);
PgfPArg* parg = gu_seq_index(item->args, PgfPArg, 0);
gu_assert(!parg->hypos || !parg->hypos->len);
new_pcoerce->coerce = parg->ccat;
break;
}
case PGF_PRODUCTION_EXTERN: {
PgfProductionExtern* pext = i.data;
if (gu_seq_is_null(pext->lins) ||
gu_seq_is_null(gu_seq_get(pext->lins,PgfSequence,item->base->lin_idx))) {
PgfSequence seq =
pgf_extern_seq_get(item, pool);
size_t n_lins = item->base->ccat->cnccat->n_lins;
PgfProductionExtern* new_pext = (PgfProductionExtern*)
gu_new_variant(PGF_PRODUCTION_EXTERN,
PgfProductionExtern,
&prod, pool);
new_pext->callback = pext->callback;
new_pext->ep = ep;
new_pext->lins = gu_new_seq(PgfSequence, n_lins, pool);
if (gu_seq_is_null(pext->lins)) {
for (size_t i = 0; i < n_lins; i++) {
gu_seq_set(new_pext->lins,PgfSequence,i,
gu_null_seq);
}
} else {
for (size_t i = 0; i < n_lins; i++) {
gu_seq_set(new_pext->lins,PgfSequence,i,
gu_seq_get(pext->lins,PgfSequence,i));
}
}
gu_seq_set(new_pext->lins,PgfSequence,item->base->lin_idx,seq);
} else {
prod = item->base->prod;
}
break;
}
case PGF_PRODUCTION_META: {
PgfProductionMeta* pmeta = i.data;
PgfProductionMeta* new_pmeta =
gu_new_variant(PGF_PRODUCTION_META,
PgfProductionMeta,
&prod, pool);
new_pmeta->ep = ep;
new_pmeta->lins = pmeta->lins;
new_pmeta->args = item->args;
if (gu_seq_is_null(pmeta->lins) ||
gu_seq_is_null(gu_seq_get(pmeta->lins,PgfSequence,item->base->lin_idx))) {
PgfSequence seq =
pgf_extern_seq_get(item, pool);
size_t n_lins = item->base->ccat->cnccat->n_lins;
new_pmeta->lins = gu_new_seq(PgfSequence, n_lins, pool);
if (gu_seq_is_null(pmeta->lins)) {
for (size_t i = 0; i < n_lins; i++) {
gu_seq_set(new_pmeta->lins,PgfSequence,i,
gu_null_seq);
}
} else {
for (size_t i = 0; i < n_lins; i++) {
gu_seq_set(new_pmeta->lins,PgfSequence,i,
gu_seq_get(pmeta->lins,PgfSequence,i));
}
}
gu_seq_set(new_pmeta->lins,PgfSequence,item->base->lin_idx,seq);
}
break;
}
default:
gu_impossible();
}
return prod;
}
static void
pgf_parsing_complete(PgfParseState* before, PgfParseState* after,
PgfItem* item, PgfExprProb *ep)
{
PgfProduction prod =
pgf_parsing_new_production(item, ep, before->pool);
PgfItemBuf* conts = item->base->conts;
PgfCCat* tmp_cat = pgf_parsing_get_completed(before, conts);
PgfCCat* cat = tmp_cat;
if (cat == NULL) {
cat = pgf_parsing_create_completed(before, conts,
item->inside_prob, item->base->ccat->cnccat);
}
GuBuf* prodbuf = gu_seq_buf(cat->prods);
gu_buf_push(prodbuf, PgfProduction, prod);
cat->n_synprods++;
#ifdef PGF_PARSER_DEBUG
GuPool* tmp_pool = gu_new_pool();
GuOut* out = gu_file_out(stderr, tmp_pool);
GuWriter* wtr = gu_new_utf8_writer(out, tmp_pool);
GuExn* err = gu_exn(NULL, type, tmp_pool);
if (tmp_cat == NULL)
gu_printf(wtr, err, "[%d-%d; C%d; %d; C%d]\n",
item->start,
item->end,
item->base->ccat->fid,
item->base->lin_idx,
cat->fid);
pgf_print_production(cat->fid, prod, wtr, err, tmp_pool);
gu_pool_free(tmp_pool);
#endif
if (tmp_cat != NULL) {
PgfItemBufs* contss =
pgf_parsing_get_contss(before->conts_map, cat, before->pool);
size_t n_contss = gu_list_length(contss);
for (size_t i = 0; i < n_contss; i++) {
PgfItemBuf* conts2 = gu_list_index(contss, i);
/* If there are continuations for
* linearization index i, then (cat, i) has
* already been predicted. Add the new
* production immediately to the agenda,
* i.e. process it. */
if (conts2) {
pgf_parsing_production(before, cat, i,
prod, conts2, 0);
}
}
// The category has already been created. If it has also been
// predicted already, then process a new item for this production.
PgfParseState* state = after;
while (state != NULL) {
PgfItemBufs* contss =
pgf_parsing_get_contss(state->conts_map, cat, state->pool);
size_t n_contss = gu_list_length(contss);
for (size_t i = 0; i < n_contss; i++) {
PgfItemBuf* conts2 = gu_list_index(contss, i);
/* If there are continuations for
* linearization index i, then (cat, i) has
* already been predicted. Add the new
* production immediately to the agenda,
* i.e. process it. */
if (conts2) {
pgf_parsing_production(state, cat, i,
prod, conts2, 0);
}
}
state = state->next;
}
} else {
size_t n_conts = gu_buf_length(conts);
for (size_t i = 0; i < n_conts; i++) {
PgfItem* cont = gu_buf_get(conts, PgfItem*, i);
pgf_parsing_combine(before, after, cont, cat, item->base->lin_idx);
}
}
}
static void
pgf_parsing_td_predict(PgfParseState* before, PgfParseState* after,
PgfItem* item, PgfCCat* ccat, size_t lin_idx,
prob_t delta_prob)
{
gu_enter("-> cat: %d", ccat->fid);
if (gu_seq_is_null(ccat->prods)) {
// Empty category
return;
}
PgfItemBuf* conts =
pgf_parsing_get_conts(before->conts_map, ccat, lin_idx,
before->pool, before->pool);
gu_buf_push(conts, PgfItem*, item);
if (gu_buf_length(conts) == 1) {
/* First time we encounter this linearization
* of this category at the current position,
* so predict it. */
// Top-down prediction for syntactic rules
PgfProductionSeq prods = ccat->prods;
for (size_t i = 0; i < ccat->n_synprods; i++) {
PgfProduction prod =
gu_seq_get(prods, PgfProduction, i);
pgf_parsing_production(before, ccat, lin_idx, prod, conts, delta_prob);
}
if (ccat->cnccat->abscat->meta_prob != INFINITY &&
ccat->fid < before->ps->concr->total_cats) {
// Top-down prediction for meta rules
PgfItem *item =
pgf_new_item(before->offset, ccat, lin_idx, before->ps->meta_prod, conts, 0, before->pool);
before->ps->item_count++;
item->inside_prob =
ccat->cnccat->abscat->meta_prob;
gu_buf_heap_push(before->agenda, &pgf_item_prob_order, &item);
}
// Bottom-up prediction for lexical rules
if (after != NULL && after->ts->lexicon_idx != NULL) {
size_t n_items = gu_buf_length(after->ts->lexicon_idx);
for (size_t i = 0; i < n_items; i++) {
PgfItem* new_item = gu_buf_get(after->ts->lexicon_idx, PgfItem*, i);
if (new_item->base->ccat == ccat &&
new_item->base->lin_idx == lin_idx &&
gu_seq_length(new_item->args) == 0) {
pgf_parsing_production(before, ccat, lin_idx,
new_item->base->prod, conts, 0);
}
}
}
// Bottom-up prediction for epsilon rules
PgfCFCat cfc = {ccat->fid, lin_idx};
PgfCCat* eps_ccat = gu_map_get(before->ps->concr->epsilon_idx,
&cfc, PgfCCat*);
if (eps_ccat != NULL) {
size_t n_prods = gu_seq_length(eps_ccat->prods);
for (size_t i = 0; i < n_prods; i++) {
PgfProduction prod =
gu_seq_get(eps_ccat->prods, PgfProduction, i);
GuVariantInfo i = gu_variant_open(prod);
switch (i.tag) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papp = i.data;
if (gu_seq_length(papp->args) == 0) {
pgf_parsing_production(before, ccat, lin_idx,
prod, conts, 0);
}
break;
}
}
}
}
} else {
/* If it has already been completed, combine. */
PgfCCat* completed =
pgf_parsing_get_completed(before, conts);
if (completed) {
pgf_parsing_combine(before, after, item, completed, lin_idx);
}
PgfParseState* state = after;
while (state != NULL) {
PgfCCat* completed =
pgf_parsing_get_completed(state, conts);
if (completed) {
pgf_parsing_combine(state, state->next, item, completed, lin_idx);
}
state = state->next;
}
}
gu_exit("<-");
}
typedef struct {
GuMapItor fn;
PgfParseState* before;
PgfParseState* after;
PgfItem* meta_item;
} PgfMetaPredictFn;
static void
pgf_parsing_meta_predict(GuMapItor* fn, const void* key, void* value, GuExn* err)
{
(void) (err);
PgfCat* abscat = (PgfCat*) key;
prob_t meta_prob = *((prob_t*) value);
PgfMetaPredictFn* clo = (PgfMetaPredictFn*) fn;
PgfParseState* before = clo->before;
PgfParseState* after = clo->after;
PgfItem* meta_item = clo->meta_item;
PgfCncCat* cnccat =
gu_map_get(before->ps->concr->cnccats, &abscat->name, PgfCncCat*);
if (cnccat == NULL)
return;
size_t n_cats = gu_list_length(cnccat->cats);
for (size_t i = 0; i < n_cats; i++) {
PgfCCat* ccat = gu_list_index(cnccat->cats, i);
for (size_t lin_idx = 0; lin_idx < cnccat->n_lins; lin_idx++) {
pgf_parsing_td_predict(before, after,
meta_item, ccat, lin_idx, meta_prob);
}
}
}
static prob_t
pgf_parsing_bu_predict(PgfParseState* before, PgfParseState* after,
PgfItemBuf* index, PgfItem* meta_item,
PgfItemBuf* agenda)
{
prob_t prob = INFINITY;
PgfMetaChildMap* meta_child_probs =
meta_item->base->ccat->cnccat->abscat->meta_child_probs;
if (meta_child_probs == NULL)
return prob;
if (!gu_map_has(before->generated_cats, index)) {
gu_map_put(before->generated_cats, index, PgfCCat*, NULL);
size_t n_items = gu_buf_length(index);
for (size_t i = 0; i < n_items; i++) {
PgfItem *item = gu_buf_get(index, PgfItem*, i);
prob_t meta_prob =
meta_item->inside_prob+
meta_item->outside_prob+
gu_map_get(meta_child_probs, item->base->ccat->cnccat->abscat, prob_t);
PgfItemBuf* conts =
pgf_parsing_get_conts(before->conts_map,
item->base->ccat, item->base->lin_idx,
before->pool, before->pool);
if (gu_buf_length(conts) == 0) {
prob_t outside_prob =
pgf_parsing_bu_predict(before, after,
item->base->conts, meta_item,
conts);
if (outside_prob > meta_prob)
outside_prob = meta_prob;
for (size_t j = i; j < n_items; j++) {
PgfItem *item_ = gu_buf_get(index, PgfItem*, j);
if (item->base->conts == item_->base->conts) {
PgfItem* copy = pgf_item_copy(item_, after->pool);
copy->base = pgf_item_base_copy(item_->base, after->pool);
copy->base->conts = conts;
copy->outside_prob = outside_prob;
#ifdef PGF_PARSER_DEBUG
copy->start = before->offset;
copy->end = (agenda == NULL)
? after->offset
: before->offset;
#endif
if (agenda == NULL)
pgf_parsing_add_transition(before, after, after->ts->tok, copy);
else
gu_buf_push(agenda, PgfItem*, copy);
prob_t item_prob =
copy->inside_prob+copy->outside_prob;
if (prob > item_prob)
prob = item_prob;
}
}
} else {
size_t n_items = gu_buf_length(conts);
for (size_t i = 0; i < n_items; i++) {
PgfItem *item = gu_buf_get(conts, PgfItem*, i);
prob_t item_prob =
item->inside_prob+item->outside_prob;
if (prob > item_prob)
prob = item_prob;
}
prob += item->inside_prob;
/* If it has already been completed, combine. */
/*PgfCCat* completed =
pgf_parsing_get_completed(before, conts);
if (completed) {
pgf_parsing_combine(before, after, meta_item, completed, item->base->lin_idx);
}*/
PgfParseState* state = after;
while (state != NULL) {
PgfCCat* completed =
pgf_parsing_get_completed(state, conts);
if (completed) {
pgf_parsing_combine(state, state->next, meta_item, completed, item->base->lin_idx);
}
state = state->next;
}
}
if (meta_prob != INFINITY)
gu_buf_push(conts, PgfItem*, meta_item);
}
}
return prob;
}
static void
pgf_parsing_symbol(PgfParseState* before, PgfParseState* after,
PgfItem* item, PgfSymbol sym) {
switch (gu_variant_tag(sym)) {
case PGF_SYMBOL_CAT: {
PgfSymbolCat* scat = gu_variant_data(sym);
PgfPArg* parg = gu_seq_index(item->args, PgfPArg, scat->d);
gu_assert(!parg->hypos || !parg->hypos->len);
pgf_parsing_td_predict(before, after, item, parg->ccat, scat->r, 0);
break;
}
case PGF_SYMBOL_KS: {
if (after != NULL) {
PgfSymbolKS* sks = gu_variant_data(sym);
gu_assert(item->tok_idx < gu_seq_length(sks->tokens));
PgfToken tok =
gu_seq_get(sks->tokens, PgfToken, item->tok_idx++);
if (item->tok_idx == gu_seq_length(sks->tokens)) {
item->tok_idx = 0;
pgf_item_advance(item, after->pool);
}
#ifdef PGF_PARSER_DEBUG
item->end++;
#endif
pgf_parsing_add_transition(before, after, tok, item);
}
break;
}
case PGF_SYMBOL_KP: {
if (after != NULL) {
PgfSymbolKP* skp = gu_variant_data(sym);
size_t idx = item->tok_idx;
uint8_t alt = item->alt;
gu_assert(idx < gu_seq_length(skp->default_form));
if (idx == 0) {
PgfToken tok;
PgfItem* new_item;
tok = gu_seq_get(skp->default_form, PgfToken, 0);
new_item = pgf_item_copy(item, after->pool);
new_item->tok_idx++;
#ifdef PGF_PARSER_DEBUG
new_item->end++;
#endif
if (new_item->tok_idx == gu_seq_length(skp->default_form)) {
new_item->tok_idx = 0;
pgf_item_advance(new_item, after->pool);
}
pgf_parsing_add_transition(before, after, tok, new_item);
for (size_t i = 0; i < skp->n_forms; i++) {
// XXX: do nubbing properly
PgfTokens toks = skp->forms[i].form;
PgfTokens toks2 = skp->default_form;
bool skip = pgf_tokens_equal(toks, toks2);
for (size_t j = 0; j < i; j++) {
PgfTokens toks2 = skp->forms[j].form;
skip |= pgf_tokens_equal(toks, toks2);
}
if (!skip) {
tok = gu_seq_get(toks, PgfToken, 0);
new_item = pgf_item_copy(item, after->pool);
new_item->tok_idx++;
#ifdef PGF_PARSER_DEBUG
new_item->end++;
#endif
new_item->alt = i;
if (new_item->tok_idx == gu_seq_length(toks)) {
new_item->tok_idx = 0;
pgf_item_advance(new_item, after->pool);
}
pgf_parsing_add_transition(before, after, tok, new_item);
}
}
} else if (alt == 0) {
PgfToken tok =
gu_seq_get(skp->default_form, PgfToken, idx);
item->tok_idx++;
#ifdef PGF_PARSER_DEBUG
item->end++;
#endif
if (item->tok_idx == gu_seq_length(skp->default_form)) {
item->tok_idx = 0;
pgf_item_advance(item, after->pool);
}
pgf_parsing_add_transition(before, after, tok, item);
} else {
gu_assert(alt <= skp->n_forms);
PgfTokens toks = skp->forms[alt - 1].form;
PgfToken tok = gu_seq_get(toks, PgfToken, idx);
item->tok_idx++;
#ifdef PGF_PARSER_DEBUG
item->end++;
#endif
if (item->tok_idx == gu_seq_length(toks)) {
item->tok_idx = 0;
pgf_item_advance(item, after->pool);
}
pgf_parsing_add_transition(before, after, tok, item);
}
}
break;
}
case PGF_SYMBOL_LIT: {
if (after != NULL) {
PgfSymbolLit* slit = gu_variant_data(sym);
PgfPArg* parg = gu_seq_index(item->args, PgfPArg, slit->d);
gu_assert(!parg->hypos || !parg->hypos->len);
if (parg->ccat->fid > 0 &&
parg->ccat->fid >= before->ps->concr->total_cats)
pgf_parsing_td_predict(before, after, item, parg->ccat, slit->r, 0);
else {
PgfItemBuf* conts =
pgf_parsing_get_conts(before->conts_map,
parg->ccat, slit->r,
before->pool, before->pool);
gu_buf_push(conts, PgfItem*, item);
if (gu_buf_length(conts) == 1) {
/* This is the first time when we encounter this
* literal category so we must call the callback */
PgfLiteralCallback* callback =
gu_map_get(before->ps->concr->callbacks,
parg->ccat->cnccat,
PgfLiteralCallback*);
if (callback != NULL) {
PgfProduction prod;
PgfProductionExtern* pext =
gu_new_variant(PGF_PRODUCTION_EXTERN,
PgfProductionExtern,
&prod, before->pool);
pext->callback = callback;
pext->ep = NULL;
pext->lins = gu_null_seq;
pgf_parsing_production(before, parg->ccat, slit->r,
prod, conts, 0);
}
} else {
/* If it has already been completed, combine. */
PgfCCat* completed =
pgf_parsing_get_completed(before, conts);
if (completed) {
pgf_parsing_combine(before, after, item, completed, slit->r);
}
PgfParseState* state = after;
while (state != NULL) {
PgfCCat* completed =
pgf_parsing_get_completed(state, conts);
if (completed) {
pgf_parsing_combine(state, state->next, item, completed, slit->r);
}
state = state->next;
}
}
}
}
break;
}
case PGF_SYMBOL_VAR:
// XXX TODO proper support
break;
default:
gu_impossible();
}
}
static void
pgf_parsing_item(PgfParseState* before, PgfParseState* after, PgfItem* item)
{
#ifdef PGF_PARSER_DEBUG
GuPool* tmp_pool = gu_new_pool();
GuOut* out = gu_file_out(stderr, tmp_pool);
GuWriter* wtr = gu_new_utf8_writer(out, tmp_pool);
GuExn* err = gu_exn(NULL, type, tmp_pool);
pgf_print_item(item, wtr, err, tmp_pool);
gu_pool_free(tmp_pool);
#endif
GuVariantInfo i = gu_variant_open(item->base->prod);
switch (i.tag) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papp = i.data;
PgfCncFun* fun = papp->fun;
PgfSequence seq = fun->lins[item->base->lin_idx];
if (item->seq_idx == gu_seq_length(seq)) {
pgf_parsing_complete(before, after, item, NULL);
} else {
PgfSymbol sym =
gu_seq_get(seq, PgfSymbol, item->seq_idx);
pgf_parsing_symbol(before, after, item, sym);
}
break;
}
case PGF_PRODUCTION_COERCE: {
PgfProductionCoerce* pcoerce = i.data;
switch (item->seq_idx) {
case 0:
pgf_parsing_td_predict(before, after, item,
pcoerce->coerce,
item->base->lin_idx, 0);
break;
case 1:
pgf_parsing_complete(before, after, item, NULL);
break;
default:
gu_impossible();
}
break;
}
case PGF_PRODUCTION_EXTERN: {
PgfProductionExtern* pext = i.data;
PgfSequence seq;
if (!gu_seq_is_null(pext->lins) &&
!gu_seq_is_null(seq = gu_seq_get(pext->lins,PgfSequence,item->base->lin_idx))) {
if (item->seq_idx == gu_seq_length(seq)) {
pgf_parsing_complete(before, after, item, NULL);
} else {
PgfSymbol sym =
gu_seq_get(seq, PgfSymbol, item->seq_idx);
pgf_parsing_symbol(before, after, item, sym);
}
} else {
PgfToken tok = (after != NULL) ? after->ts->tok
: gu_empty_string;
PgfExprProb *ep = NULL;
bool accepted =
pext->callback->match(before->ps->concr, item,
tok,
&ep, before->pool);
if (ep != NULL)
pgf_parsing_complete(before, after, item, ep);
if (accepted && after != NULL) {
PgfSymbol prev = item->curr_sym;
PgfSymbolKS* sks = (PgfSymbolKS*)
gu_alloc_variant(PGF_SYMBOL_KS,
sizeof(PgfSymbolKS)+sizeof(PgfSymbol),
gu_alignof(PgfSymbolKS),
&item->curr_sym, after->pool);
*((PgfSymbol*)(sks+1)) = prev;
sks->tokens = gu_new_seq(PgfToken, 1, after->pool);
gu_seq_set(sks->tokens, PgfToken, 0, tok);
item->seq_idx++;
#ifdef PGF_PARSER_DEBUG
item->end++;
#endif
pgf_parsing_add_transition(before, after, tok, item);
}
}
break;
}
case PGF_PRODUCTION_META: {
PgfProductionMeta* pmeta = i.data;
PgfSequence seq;
if (!gu_seq_is_null(pmeta->lins) &&
!gu_seq_is_null(seq = gu_seq_get(pmeta->lins,PgfSequence,item->base->lin_idx))) {
if (item->seq_idx == gu_seq_length(seq)) {
pgf_parsing_complete(before, after, item, NULL);
} else {
PgfSymbol sym =
gu_seq_get(seq, PgfSymbol, item->seq_idx);
pgf_parsing_symbol(before, after, item, sym);
}
} else {
if (!gu_variant_is_null(item->curr_sym)) {
PgfExprProb *ep = gu_new(PgfExprProb, before->pool);
ep->expr = before->ps->meta_var;
ep->prob = item->inside_prob;
size_t n_args = gu_seq_length(item->args);
for (size_t i = 0; i < n_args; i++) {
PgfPArg* arg = gu_seq_index(item->args, PgfPArg, i);
ep->prob -= arg->ccat->viterbi_prob;
}
pgf_parsing_complete(before, after, item, ep);
}
if (after != NULL) {
if (after->ts->lexicon_idx == NULL) {
prob_t meta_token_prob =
item->base->ccat->cnccat->abscat->meta_token_prob;
if (meta_token_prob == INFINITY)
break;
item->inside_prob += meta_token_prob;
PgfSymbol prev = item->curr_sym;
PgfSymbolKS* sks = (PgfSymbolKS*)
gu_alloc_variant(PGF_SYMBOL_KS,
sizeof(PgfSymbolKS)+sizeof(PgfSymbol),
gu_alignof(PgfSymbolKS),
&item->curr_sym, after->pool);
*((PgfSymbol*)(sks+1)) = prev;
sks->tokens = gu_new_seq(PgfToken, 1, after->pool);
gu_seq_set(sks->tokens, PgfToken, 0, after->ts->tok);
item->seq_idx++;
#ifdef PGF_PARSER_DEBUG
item->end++;
#endif
pgf_parsing_add_transition(before, after, after->ts->tok, item);
} else {
PgfCIdMap* meta_child_probs =
item->base->ccat->cnccat->abscat->meta_child_probs;
if (meta_child_probs != NULL) {
PgfMetaPredictFn clo = { { pgf_parsing_meta_predict }, before, after, item };
gu_map_iter(meta_child_probs, &clo.fn, NULL);
}
/*
fprintf(stderr, "------------------------------------\n");
pgf_parsing_bu_predict(before, after,
after->ts->lexicon_idx, item,
NULL);
fprintf(stderr, "------------------------------------\n");*/
}
}
}
break;
}
default:
gu_impossible();
}
}
static void
pgf_parsing_proceed(PgfParseState* state, void** output) {
while (*output == NULL) {
if (state->ps->item_count > state->ps->concr->item_quota)
break;
prob_t best_prob = INFINITY;
PgfParseState* before = NULL;
PgfParseState* st = state;
while (st != NULL) {
if (gu_buf_length(st->agenda) > 0) {
PgfItem* item = gu_buf_get(st->agenda, PgfItem*, 0);
prob_t item_prob = item->inside_prob+item->outside_prob;
if (item_prob < best_prob) {
best_prob = item_prob;
before = st;
}
}
st = st->next;
}
if (before == NULL)
break;
PgfParseState* after = NULL;
st = state;
while (st != before) {
PgfParseState* tmp = st->next;
st->next = after;
after = st;
st = tmp;
}
PgfItem* item;
gu_buf_heap_pop(before->agenda, &pgf_item_prob_order, &item);
pgf_parsing_item(before, after, item);
while (after != NULL) {
PgfParseState* tmp = after->next;
after->next = before;
before = after;
after = tmp;
}
state = before;
}
}
static PgfParsing*
pgf_new_parsing(PgfConcr* concr, GuPool* pool)
{
PgfParsing* ps = gu_new(PgfParsing, pool);
ps->concr = concr;
ps->completed = NULL;
ps->target = NULL;
ps->max_fid = concr->max_fid;
ps->item_count = 0;
PgfExprMeta *expr_meta =
gu_new_variant(PGF_EXPR_META,
PgfExprMeta,
&ps->meta_var, pool);
expr_meta->id = 0;
PgfProductionMeta* pmeta =
gu_new_variant(PGF_PRODUCTION_META,
PgfProductionMeta,
&ps->meta_prod, pool);
pmeta->ep = NULL;
pmeta->lins = gu_null_seq;
pmeta->args = gu_new_seq(PgfPArg, 0, pool);
return ps;
}
static PgfParseState*
pgf_new_parse_state(PgfParsing* ps,
PgfParseState* next,
PgfTokenState* ts, int offset,
GuPool* pool)
{
PgfParseState* state = gu_new(PgfParseState, pool);
state->pool = pool;
state->next = next;
state->agenda = gu_new_buf(PgfItem*, pool);
state->generated_cats = gu_map_type_new(PgfGenCatMap, pool);
state->conts_map = gu_map_type_new(PgfContsMap, pool);
state->offset = offset;
state->ps = ps;
state->ts = ts;
return state;
}
static PgfTokenState*
pgf_new_token_state(PgfConcr *concr, PgfToken tok, GuPool* pool)
{
PgfTokenState* ts = gu_new(PgfTokenState, pool);
ts->tok = tok;
ts->lexicon_idx = gu_map_get(concr->lexicon_idx, &tok, GuBuf*);
return ts;
}
PgfParseState*
pgf_parser_next_state(PgfParseState* prev, PgfToken tok, GuPool* pool)
{
PgfTokenState* ts =
pgf_new_token_state(prev->ps->concr,tok,pool);
PgfParseState* state =
pgf_new_parse_state(prev->ps, prev,
ts, prev->offset+1,
pool);
state->ps->target = NULL;
pgf_parsing_proceed(state, (void**) &state->ps->target);
if (state->ps->target != NULL) {
return state;
}
return NULL;
}
static int
cmp_expr_qstate(GuOrder* self, const void* a, const void* b)
{
PgfExprQState *s1 = (PgfExprQState *) a;
PgfExprQState *s2 = (PgfExprQState *) b;
if (s1->prob < s2->prob)
return -1;
else if (s1->prob > s2->prob)
return 1;
else
return 0;
}
static GuOrder
pgf_expr_qstate_order = { cmp_expr_qstate };
static void
pgf_result_cat_init(PgfParseResult* pr,
PgfExprState* cont, prob_t cont_prob, PgfCCat* ccat)
{
// Checking for loops in the chart
if (cont != NULL) {
PgfExprState* st = cont->cont;
while (st != NULL) {
PgfPArg* arg = gu_seq_index(st->args, PgfPArg, st->arg_idx);
if (arg->ccat == ccat)
return;
st = st->cont;
}
}
if (gu_seq_is_null(ccat->prods))
return;
// Generation
size_t n_prods = gu_seq_length(ccat->prods);
for (size_t i = 0; i < n_prods; i++) {
PgfProduction prod =
gu_seq_get(ccat->prods, PgfProduction, i);
GuVariantInfo pi = gu_variant_open(prod);
switch (pi.tag) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papp = pi.data;
PgfExprState *st = gu_new(PgfExprState, pr->tmp_pool);
st->cont = cont;
st->expr = papp->fun->ep->expr;
st->args = papp->args;
st->arg_idx = 0;
PgfExprQState q = {st, cont_prob + papp->fun->ep->prob};
size_t n_args = gu_seq_length(st->args);
for (size_t k = 0; k < n_args; k++) {
PgfPArg* parg = gu_seq_index(st->args, PgfPArg, k);
q.prob += parg->ccat->viterbi_prob;
}
gu_buf_heap_push(pr->pqueue, &pgf_expr_qstate_order, &q);
break;
}
case PGF_PRODUCTION_COERCE: {
PgfProductionCoerce* pcoerce = pi.data;
pgf_result_cat_init(pr, cont, cont_prob, pcoerce->coerce);
break;
}
case PGF_PRODUCTION_EXTERN: {
PgfProductionExtern* pext = pi.data;
PgfExprState *st = gu_new(PgfExprState, pr->tmp_pool);
st->cont = cont;
st->expr = pext->ep->expr;
st->args = gu_empty_seq();
st->arg_idx = 0;
PgfExprQState q = {st, cont_prob + pext->ep->prob};
gu_buf_heap_push(pr->pqueue, &pgf_expr_qstate_order, &q);
break;
}
case PGF_PRODUCTION_META: {
PgfProductionMeta* pmeta = pi.data;
PgfExprState *st = gu_new(PgfExprState, pr->tmp_pool);
st->cont = cont;
st->expr = pmeta->ep->expr;
st->args = pmeta->args;
st->arg_idx = 0;
PgfExprQState q = {st, cont_prob + pmeta->ep->prob};
size_t n_args = gu_seq_length(st->args);
for (size_t k = 0; k < n_args; k++) {
PgfPArg* parg = gu_seq_index(st->args, PgfPArg, k);
q.prob += parg->ccat->viterbi_prob;
}
gu_buf_heap_push(pr->pqueue, &pgf_expr_qstate_order, &q);
break;
}
default:
gu_impossible();
}
}
}
static PgfExprProb*
pgf_parse_result_next(PgfParseResult* pr, GuPool* pool)
{
if (pr->pqueue == NULL)
return NULL;
for (;;) {
if (pr->state->ps->completed == NULL) {
pgf_parsing_proceed(pr->state,
(void**) &pr->state->ps->completed);
if (pr->state->ps->completed == NULL)
return NULL;
pgf_result_cat_init(pr, NULL, 0, pr->state->ps->completed);
}
while (gu_buf_length(pr->pqueue) > 0) {
PgfExprQState q;
gu_buf_heap_pop(pr->pqueue, &pgf_expr_qstate_order, &q);
if (q.st->arg_idx < gu_seq_length(q.st->args)) {
PgfPArg* arg = gu_seq_index(q.st->args, PgfPArg, q.st->arg_idx);
prob_t cont_prob = q.prob - arg->ccat->viterbi_prob;
if (arg->ccat->fid < pr->state->ps->concr->total_cats) {
q.st->expr =
gu_new_variant_i(pool, PGF_EXPR_APP,
PgfExprApp,
.fun = q.st->expr,
.arg = pr->state->ps->meta_var);
q.st->arg_idx++;
gu_buf_heap_push(pr->pqueue, &pgf_expr_qstate_order, &q);
} else {
pgf_result_cat_init(pr, q.st, cont_prob, arg->ccat);
}
} else {
if (q.st->cont == NULL) {
PgfExprProb* ep = gu_new(PgfExprProb, pool);
ep->expr = q.st->expr;
ep->prob = q.prob;
return ep;
}
PgfExprState* st = gu_new(PgfExprState, pr->tmp_pool);
st->cont = q.st->cont->cont;
st->expr =
gu_new_variant_i(pool, PGF_EXPR_APP,
PgfExprApp,
.fun = q.st->cont->expr, .arg = q.st->expr);
st->args = q.st->cont->args;
st->arg_idx = q.st->cont->arg_idx+1;
PgfExprQState q2 = {st, q.prob};
gu_buf_heap_push(pr->pqueue, &pgf_expr_qstate_order, &q2);
}
}
pr->state->ps->completed = NULL;
}
gu_pool_free(pr->tmp_pool);
pr->tmp_pool = NULL;
pr->pqueue = NULL;
return NULL;
}
static void
pgf_parse_result_enum_next(GuEnum* self, void* to, GuPool* pool)
{
PgfParseResult* pr = gu_container(self, PgfParseResult, en);
*(PgfExprProb**)to = pgf_parse_result_next(pr, pool);
}
PgfExprEnum*
pgf_parse_result(PgfParseState* state, GuPool* pool)
{
GuPool* tmp_pool = gu_new_pool();
GuBuf* pqueue = gu_new_buf(PgfExprQState, tmp_pool);
PgfExprEnum* en =
&gu_new_i(pool, PgfParseResult,
.tmp_pool = tmp_pool,
.state = state,
.pqueue = pqueue,
.en.next = pgf_parse_result_enum_next)->en;
return en;
}
// TODO: s/CId/Cat, add the cid to Cat, make Cat the key to CncCat
PgfParseState*
pgf_parser_init_state(PgfConcr* concr, PgfCId cat, size_t lin_idx, GuPool* pool)
{
PgfCncCat* cnccat =
gu_map_get(concr->cnccats, &cat, PgfCncCat*);
if (!cnccat) {
// error ...
gu_impossible();
}
gu_assert(lin_idx < cnccat->n_lins);
PgfParsing* ps =
pgf_new_parsing(concr, pool);
PgfParseState* state =
pgf_new_parse_state(ps, NULL, NULL, 0, pool);
PgfItemBuf* conts = gu_new_buf(PgfItem*, pool);
gu_buf_push(conts, PgfItem*, NULL);
size_t n_ccats = gu_list_length(cnccat->cats);
for (size_t i = 0; i < n_ccats; i++) {
PgfCCat* ccat = gu_list_index(cnccat->cats, i);
if (ccat != NULL) {
if (gu_seq_is_null(ccat->prods)) {
// Empty category
continue;
}
PgfProductionSeq prods = ccat->prods;
size_t n_prods = gu_seq_length(prods);
for (size_t i = 0; i < n_prods; i++) {
PgfProduction prod =
gu_seq_get(prods, PgfProduction, i);
PgfItem* item =
pgf_new_item(0, ccat, lin_idx, prod, conts, 0, pool);
ps->item_count++;
gu_buf_heap_push(state->agenda, &pgf_item_prob_order, &item);
}
PgfItem *item =
pgf_new_item(0, ccat, lin_idx, ps->meta_prod, conts, 0, pool);
ps->item_count++;
item->inside_prob =
ccat->cnccat->abscat->meta_prob;
gu_buf_heap_push(state->agenda, &pgf_item_prob_order, &item);
}
}
return state;
}
void
pgf_parser_add_literal(PgfConcr *concr, PgfCId cat,
PgfLiteralCallback* callback)
{
PgfCncCat* cnccat =
gu_map_get(concr->cnccats, &cat, PgfCncCat*);
if (cnccat == NULL)
return;
gu_map_put(concr->callbacks, cnccat,
PgfLiteralCallback*, callback);
}
static void
pgf_parser_bu_add_entry(PgfConcr* concr, PgfTokens tokens,
PgfItem* item,
GuPool *pool)
{
PgfToken tok = gu_seq_get(tokens, PgfToken, 0);
GuBuf* items = gu_map_get(concr->lexicon_idx, &tok, GuBuf*);
if (items == NULL) {
items = gu_new_buf(PgfItemBase*, pool);
gu_map_put(concr->lexicon_idx, &tok, GuBuf*, items);
}
gu_buf_push(items, PgfItem*, item);
}
void
pgf_parser_bu_item(PgfConcr* concr, PgfItem* item,
PgfContsMap* conts_map,
GuPool *pool, GuPool *tmp_pool)
{
GuVariantInfo i = gu_variant_open(item->base->prod);
switch (i.tag) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papp = i.data;
PgfSequence seq = papp->fun->lins[item->base->lin_idx];
if (item->seq_idx < gu_seq_length(seq)) {
GuVariantInfo i = gu_variant_open(item->curr_sym);
switch (i.tag) {
case PGF_SYMBOL_CAT: {
PgfSymbolCat* scat = i.data;
// Place the item in the continuation map
PgfPArg* parg =
gu_seq_index(item->args, PgfPArg, scat->d);
PgfItemBuf* conts_ =
pgf_parsing_get_conts(conts_map,
parg->ccat, scat->r,
pool, tmp_pool);
gu_buf_push(conts_, PgfItem*, item);
// If the current category has epsilon rules
// then we must do the same for a new item where
// the dot is moved with one position.
PgfCFCat cfc = {parg->ccat->fid, scat->r};
PgfCCat* eps_ccat =
gu_map_get(concr->epsilon_idx, &cfc, PgfCCat*);
if (eps_ccat != NULL) {
PgfItem* new_item =
pgf_item_update_arg(item, scat->d, eps_ccat, pool);
pgf_item_advance(new_item, pool);
pgf_parser_bu_item(concr, new_item, conts_map,
pool, tmp_pool);
}
break;
}
case PGF_SYMBOL_KS: {
PgfSymbolKS* sks = i.data;
item->tok_idx++;
if (item->tok_idx == gu_seq_length(sks->tokens)) {
item->tok_idx = 0;
pgf_item_advance(item, pool);
}
pgf_parser_bu_add_entry(concr, sks->tokens,
item, pool);
break;
}
case PGF_SYMBOL_KP: {
PgfSymbolKP* skp = i.data;
item->tok_idx++;
if (item->tok_idx == gu_seq_length(skp->default_form)) {
item->tok_idx = 0;
pgf_item_advance(item, pool);
}
pgf_parser_bu_add_entry(concr, skp->default_form,
item, pool);
for (size_t i = 0; i < skp->n_forms; i++) {
pgf_parser_bu_add_entry(concr, skp->forms[i].form,
item, pool);
}
break;
}
case PGF_SYMBOL_LIT:
// Nothing to be done here
break;
case PGF_SYMBOL_VAR:
// XXX TODO proper support
break;
default:
gu_impossible();
}
} else {
PgfCFCat cfc = {item->base->ccat->fid, item->base->lin_idx};
PgfCCat* tmp_ccat =
gu_map_get(concr->epsilon_idx, &cfc, PgfCCat*);
PgfCCat* eps_ccat = tmp_ccat;
if (eps_ccat == NULL) {
eps_ccat = gu_new(PgfCCat, pool);
eps_ccat->cnccat = item->base->ccat->cnccat;
eps_ccat->viterbi_prob = INFINITY;
eps_ccat->fid = concr->max_fid++;
eps_ccat->prods =
gu_buf_seq(gu_new_buf(PgfProduction, pool));
eps_ccat->n_synprods = 0;
gu_map_put(concr->epsilon_idx, &cfc, PgfCCat*, eps_ccat);
}
PgfProduction prod =
pgf_parsing_new_production(item, NULL, pool);
GuBuf* prodbuf = gu_seq_buf(eps_ccat->prods);
gu_buf_push(prodbuf, PgfProduction, prod);
eps_ccat->n_synprods++;
if (eps_ccat->viterbi_prob > item->inside_prob)
eps_ccat->viterbi_prob = item->inside_prob;
if (tmp_ccat == NULL) {
size_t n_items = gu_buf_length(item->base->conts);
for (size_t i = 0; i < n_items; i++) {
PgfItem* cont =
gu_buf_get(item->base->conts, PgfItem*, i);
gu_assert(gu_variant_tag(cont->curr_sym) == PGF_SYMBOL_CAT);
PgfSymbolCat* scat = gu_variant_data(cont->curr_sym);
PgfItem* new_item =
pgf_item_update_arg(cont, scat->d, eps_ccat, pool);
pgf_item_advance(new_item, pool);
pgf_parser_bu_item(concr, new_item, conts_map,
pool, tmp_pool);
}
}
}
}
break;
case PGF_PRODUCTION_COERCE: {
PgfPArg* parg =
gu_seq_index(item->args, PgfPArg, 0);
PgfItemBuf* conts_ =
pgf_parsing_get_conts(conts_map,
parg->ccat, item->base->lin_idx,
pool, tmp_pool);
gu_buf_push(conts_, PgfItem*, item);
// If the argument category has epsilon rules
// then the result category has epsilon rules too.
PgfCFCat cfc = {parg->ccat->fid, item->base->lin_idx};
PgfCCat* eps_arg_ccat =
gu_map_get(concr->epsilon_idx, &cfc, PgfCCat*);
if (eps_arg_ccat != NULL) {
PgfCFCat cfc = {item->base->ccat->fid, item->base->lin_idx};
PgfCCat* tmp_ccat =
gu_map_get(concr->epsilon_idx, &cfc, PgfCCat*);
PgfCCat* eps_res_ccat = tmp_ccat;
if (eps_res_ccat == NULL) {
eps_res_ccat = gu_new(PgfCCat, pool);
eps_res_ccat->cnccat = item->base->ccat->cnccat;
eps_res_ccat->fid = concr->max_fid++;
eps_res_ccat->prods =
gu_buf_seq(gu_new_buf(PgfProduction, pool));
eps_res_ccat->n_synprods = 0;
gu_map_put(concr->epsilon_idx, &cfc, PgfCCat*, eps_res_ccat);
}
PgfProduction prod;
PgfProductionCoerce* new_pcoerce =
gu_new_variant(PGF_PRODUCTION_COERCE,
PgfProductionCoerce,
&prod, pool);
new_pcoerce->coerce = eps_arg_ccat;
GuBuf* prodbuf = gu_seq_buf(eps_res_ccat->prods);
gu_buf_push(prodbuf, PgfProduction, prod);
eps_res_ccat->n_synprods++;
if (tmp_ccat == NULL) {
size_t n_items = gu_buf_length(item->base->conts);
for (size_t i = 0; i < n_items; i++) {
PgfItem* cont =
gu_buf_get(item->base->conts, PgfItem*, i);
gu_assert(gu_variant_tag(cont->curr_sym) == PGF_SYMBOL_CAT);
PgfSymbolCat* scat = gu_variant_data(cont->curr_sym);
PgfItem* new_item =
pgf_item_update_arg(cont, scat->d, eps_res_ccat, pool);
pgf_item_advance(new_item, pool);
pgf_parser_bu_item(concr, new_item, conts_map,
pool, tmp_pool);
}
}
}
break;
}
default:
gu_impossible();
}
}
void
pgf_parser_bu_index(PgfConcr* concr, PgfCCat* ccat, PgfProduction prod,
PgfContsMap* conts_map,
GuPool *pool, GuPool *tmp_pool)
{
for (size_t lin_idx = 0; lin_idx < ccat->cnccat->n_lins; lin_idx++) {
PgfItemBuf* conts =
pgf_parsing_get_conts(conts_map, ccat, lin_idx,
pool, tmp_pool);
PgfItem* item =
pgf_new_item(0, ccat, lin_idx, prod, conts, 0, pool);
pgf_parser_bu_item(concr, item, conts_map, pool, tmp_pool);
}
}
bool
pgf_cfcat_eq_fn(GuEquality* self, const void* a, const void* b)
{
PgfCFCat *x = (PgfCFCat *) a;
PgfCFCat *y = (PgfCFCat *) b;
return (x->fid == y->fid && x->lin_idx == y->lin_idx);
}
GuHash
pgf_cfcat_hash_fn(GuHasher* self, const void* a)
{
PgfCFCat *x = (PgfCFCat *) a;
return ((x->fid << 16) ^ x->lin_idx);
}
GuHasher pgf_cfcat_hasher = {
{ pgf_cfcat_eq_fn },
pgf_cfcat_hash_fn
};
View Git Blame Copy Permalink