From cb6bace896ba8d6a57198e0cfa395f1c62191ee6 Mon Sep 17 00:00:00 2001 From: Krasimir Angelov Date: Thu, 13 Nov 2025 11:17:16 +0100 Subject: [PATCH] Diophantine grammars in the runtime --- src/runtime/c/pgf/data.cxx | 55 +- src/runtime/c/pgf/data.h | 173 +- src/runtime/c/pgf/linearizer.cxx | 383 ++- src/runtime/c/pgf/linearizer.h | 65 +- src/runtime/c/pgf/parser.cxx | 3486 ++++++++------------- src/runtime/c/pgf/parser.h | 433 ++- src/runtime/c/pgf/pgf.cxx | 576 ++-- src/runtime/c/pgf/pgf.h | 61 +- src/runtime/c/pgf/phrasetable.cxx | 906 +++--- src/runtime/c/pgf/phrasetable.h | 204 +- src/runtime/c/pgf/printer.cxx | 92 +- src/runtime/c/pgf/printer.h | 4 +- src/runtime/c/pgf/reader.cxx | 124 +- src/runtime/c/pgf/reader.h | 13 +- src/runtime/c/pgf/writer.cxx | 73 +- src/runtime/c/pgf/writer.h | 12 +- src/runtime/haskell/PGF2.hsc | 54 +- src/runtime/haskell/PGF2/FFI.hsc | 19 +- src/runtime/haskell/PGF2/Transactions.hsc | 72 +- 19 files changed, 3003 insertions(+), 3802 deletions(-) diff --git a/src/runtime/c/pgf/data.cxx b/src/runtime/c/pgf/data.cxx index 982db8de2..0799b8a32 100644 --- a/src/runtime/c/pgf/data.cxx +++ b/src/runtime/c/pgf/data.cxx @@ -40,7 +40,6 @@ void PgfConcr::release(ref concr) namespace_release(concr->cflags); namespace_release(concr->lins); namespace_release(concr->lincats); - phrasetable_release(concr->phrasetable); namespace_release(concr->printnames); PgfDB::free(concr, concr->name.size+1); } @@ -52,17 +51,10 @@ void PgfConcrLincat::release(ref lincat) } vector>::release(lincat->fields); - for (size_t i = 0; i < lincat->args.size(); i++) { - PgfLParam::release(lincat->args[i].param); + for (ref rule : lincat->rules) { + PgfConcrRule::release(rule); } - vector::release(lincat->args); - - for (ref res : lincat->res) { - PgfPResult::release(res); - } - vector>::release(lincat->res); - - vector>::release(lincat->seqs); + vector>::release(lincat->rules); PgfDB::free(lincat, lincat->name.size+1); } @@ -79,9 +71,9 @@ void PgfPResult::release(ref res) PgfDB::free(res, res->param.n_terms*sizeof(res->param.terms[0])); } -void PgfSequence::release(ref seq) +static void symbols_release(vector syms) { - for (PgfSymbol sym : seq->syms) { + for (PgfSymbol sym : syms) { switch (ref::get_tag(sym)) { case PgfSymbolCat::tag: { auto sym_cat = ref::untagged(sym); @@ -103,9 +95,11 @@ void PgfSequence::release(ref seq) } case PgfSymbolKP::tag: { auto sym_kp = ref::untagged(sym); - PgfSequence::release(sym_kp->default_form); + symbols_release(sym_kp->default_form); + vector::release(sym_kp->default_form); for (size_t i = 0; i < sym_kp->alts.size(); i++) { - PgfSequence::release(sym_kp->alts[i].form); + symbols_release(sym_kp->alts[i].form); + vector::release(sym_kp->alts[i].form); for (size_t j = 0; j < sym_kp->alts[i].prefixes.size(); j++) { text_db_release(sym_kp->alts[i].prefixes[j]); } @@ -124,22 +118,31 @@ void PgfSequence::release(ref seq) throw pgf_error("Unknown symbol tag"); } } - inline_vector::release(&PgfSequence::syms, seq); +} + +void PgfConcrRule::release(ref rule) +{ + vector::release(rule->vars); + + PgfLParam::release(rule->res); + + for (ref arg : rule->args) { + PgfLParam::release(arg); + } + vector>::release(rule->args); + + PgfLParam::release(rule->lin_idx); + + symbols_release(rule->syms.as_vector()); + inline_vector::release(&PgfConcrRule::syms, rule); } void PgfConcrLin::release(ref lin) { - for (size_t i = 0; i < lin->args.size(); i++) { - PgfLParam::release(lin->args[i].param); + for (ref rule : lin->rules) { + PgfConcrRule::release(rule); } - vector::release(lin->args); - - for (ref res : lin->res) { - PgfPResult::release(res); - } - vector>::release(lin->res); - - vector>::release(lin->seqs); + vector>::release(lin->rules); PgfDB::free(lin, lin->name.size+1); } diff --git a/src/runtime/c/pgf/data.h b/src/runtime/c/pgf/data.h index 99ee8a427..a96e45eaf 100644 --- a/src/runtime/c/pgf/data.h +++ b/src/runtime/c/pgf/data.h @@ -87,7 +87,6 @@ struct PgfConcr; #include "text.h" #include "vector.h" #include "namespace.h" -#include "phrasetable.h" #include "probspace.h" #include "expr.h" @@ -155,12 +154,6 @@ struct PGF_INTERNAL_DECL PgfPResult { typedef object PgfSymbol; -struct PGF_INTERNAL_DECL PgfSequence { - inline_vector syms; - - static void release(ref seq); -}; - struct PGF_INTERNAL_DECL PgfSequenceBackref { object container; size_t seq_index; @@ -189,7 +182,7 @@ struct PGF_INTERNAL_DECL PgfSymbolKS { }; struct PGF_INTERNAL_DECL PgfAlternative { - ref form; + vector form; /**< The form of this variant as a list of tokens. */ vector> prefixes; @@ -199,7 +192,7 @@ struct PGF_INTERNAL_DECL PgfAlternative { struct PGF_INTERNAL_DECL PgfSymbolKP { static const uint8_t tag = 4; - ref default_form; + vector default_form; inline_vector alts; }; @@ -227,15 +220,24 @@ struct PGF_INTERNAL_DECL PgfSymbolALLCAPIT { static const uint8_t tag = 10; }; +struct PGF_INTERNAL_DECL PgfConcrRule { + vector vars; + ref res; + object container; + vector> args; + ref lin_idx; + inline_vector syms; + + static void release(ref seq); +}; + struct PGF_INTERNAL_DECL PgfConcrLincat { static const uint8_t tag = 0; ref abscat; size_t n_lindefs; - vector args; - vector> res; - vector> seqs; + vector> rules; vector> fields; PgfText name; @@ -249,15 +251,25 @@ struct PGF_INTERNAL_DECL PgfConcrLin { ref absfun; ref lincat; - vector args; - vector> res; - vector> seqs; + vector> rules; PgfText name; static void release(ref lin); }; +struct PGF_INTERNAL_DECL PgfSymbolACat { + static const uint8_t tag = 11; + PgfText name; +}; + +struct PGF_INTERNAL_DECL PgfSymbolCCat { + static const uint8_t tag = 12; + ref lincat; + size_t value; + size_t lin_idx; +}; + struct PGF_INTERNAL_DECL PgfConcrPrintname { ref printname; PgfText name; @@ -267,134 +279,7 @@ struct PGF_INTERNAL_DECL PgfConcrPrintname { #define containerof(T,field,p) (T*) (((char*) p)-offsetof(T,field)) -struct PGF_INTERNAL_DECL PgfLCEdge { - struct { - ref lincat; - struct { - size_t i0; - term& operator[](int i) { - PgfLCEdge *edge = containerof(PgfLCEdge,from.value,this); - return edge->terms[i]; - } - size_t size() { - PgfLCEdge *edge = containerof(PgfLCEdge,from.value,this); - return edge->from.lin_idx.n_offset; - } - } value; - struct { - size_t i0; - size_t n_offset; - term& operator[](int i) { - PgfLCEdge *edge = containerof(PgfLCEdge,from.lin_idx,this); - return edge->terms[n_offset+i]; - } - size_t size() { - PgfLCEdge *edge = containerof(PgfLCEdge,from.lin_idx,this); - return edge->to.value.n_offset-n_offset; - } - } lin_idx; - } from; - - struct { - ref lincat; - struct { - size_t i0; - size_t n_offset; - term& operator[](int i) { - PgfLCEdge *edge = containerof(PgfLCEdge,to.value,this); - return edge->terms[n_offset+i]; - } - size_t size() { - PgfLCEdge *edge = containerof(PgfLCEdge,to.value,this); - return edge->to.lin_idx.n_offset-n_offset; - } - } value; - struct { - size_t i0; - size_t n_offset; - term& operator[](int i) { - PgfLCEdge *edge = containerof(PgfLCEdge,to.lin_idx,this); - return edge->terms[n_offset+i]; - } - size_t size() { - PgfLCEdge *edge = containerof(PgfLCEdge,to.lin_idx,this); - return edge->n_terms-n_offset; - } - } lin_idx; - } to; - - struct { - size_t n_vars; - PgfVariableRange& operator[](int i) { - PgfLCEdge *edge = containerof(PgfLCEdge,vars,this); - return ((PgfVariableRange*)(((term*) (edge+1))+edge->n_terms))[i]; - } - size_t size() { - return n_vars; - } - } vars; - - size_t n_terms; - term terms[]; - - static ref alloc(size_t n_terms1, size_t n_terms2, size_t n_terms3, size_t n_terms4, size_t n_vars) { - auto edge = PgfDB::malloc((n_terms1+n_terms2+n_terms3+n_terms4)*sizeof(term)+n_vars*sizeof(PgfVariableRange)); - edge->from.lin_idx.n_offset = n_terms1; - edge->to.value.n_offset = n_terms1+n_terms2; - edge->to.lin_idx.n_offset = n_terms1+n_terms2+n_terms3; - edge->n_terms = n_terms1+n_terms2+n_terms3+n_terms4; - edge->vars.n_vars = n_vars; - return edge; - } -}; - -struct PGF_INTERNAL_DECL PgfLRShift { - size_t next_state; - ref lincat; - size_t r; -}; - -struct PGF_INTERNAL_DECL PgfLRShiftKS { - size_t next_state; - ref seq; - size_t sym_idx; -}; - -struct PgfLRReduceArg; - -struct PGF_INTERNAL_DECL PgfLRProduction { - ref lin; - size_t index; - vector> args; -}; - -struct PGF_INTERNAL_DECL PgfLRReduceArg { - static const uint8_t tag = 2; - - size_t id; - size_t n_prods; - PgfLRProduction prods[]; -}; - -struct PGF_INTERNAL_DECL PgfLRReduce { - object lin_obj; - size_t seq_idx; - size_t depth; - - struct Arg { - ref arg; - size_t stk_idx; - }; - - vector args; -}; - -struct PGF_INTERNAL_DECL PgfLRState { - vector shifts; - vector tokens; - size_t next_bind_state; - vector reductions; -}; +#include "phrasetable.h" struct PGF_INTERNAL_DECL PgfConcr { Namespace cflags; @@ -403,8 +288,6 @@ struct PGF_INTERNAL_DECL PgfConcr { PgfPhrasetable phrasetable; Namespace printnames; - vector lrtable; - PgfText name; static void release(ref pgf); diff --git a/src/runtime/c/pgf/linearizer.cxx b/src/runtime/c/pgf/linearizer.cxx index 8591bc718..ee02d237f 100644 --- a/src/runtime/c/pgf/linearizer.cxx +++ b/src/runtime/c/pgf/linearizer.cxx @@ -2,6 +2,59 @@ #include "printer.h" #include "linearizer.h" +bool PgfLinearizer::Item::instantiate(ref lparam,size_t value) +{ + if (value < lparam->i0) + return false; + value -= lparam->i0; + + for (size_t j = 0; j < lparam->n_terms; j++) { + term t = lparam->terms[j]; + for (size_t k = 0; k < vars.size(); k++) { + if (rule->vars[k].var == t.var) { + if (vars[k] > 0) { + if (value < vars[k]-1) + return false; + value -= vars[k]-1; + } + break; + } + } + } + + for (size_t j = 0; j < lparam->n_terms; j++) { + term t = lparam->terms[j]; + for (size_t k = 0; k < vars.size(); k++) { + if (rule->vars[k].var == t.var) { + if (vars[k] == 0) { + size_t v_val = value / t.factor; + if (v_val >= rule->vars[k].range) + return false; + vars[k] = v_val + 1; + value %= t.factor; + } + break; + } + } + } + + return (value == 0); +} + +size_t PgfLinearizer::Item::eval(ref lparam) +{ + size_t value = lparam->i0; + for (size_t i = 0; i < lparam->n_terms; i++) { + for (size_t j = 0; j < rule->vars.size(); j++) { + if (lparam->terms[i].var == rule->vars[j].var) { + value += lparam->terms[i].factor * (vars[j]-1); + break; + } + } + } + return value; +} + PgfLinearizer::TreeNode::TreeNode(PgfLinearizer *linearizer) { this->next = linearizer->prev; @@ -11,8 +64,6 @@ PgfLinearizer::TreeNode::TreeNode(PgfLinearizer *linearizer) this->fid = 0; this->value = 0; - this->var_count = 0; - this->var_values= NULL; this->n_hoas_vars = 0; this->hoas_vars = NULL; @@ -20,7 +71,7 @@ PgfLinearizer::TreeNode::TreeNode(PgfLinearizer *linearizer) linearizer->prev = this; } -void PgfLinearizer::TreeNode::linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, PgfLParam *r) +void PgfLinearizer::TreeNode::linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, size_t r) { TreeNode *arg = args; while (d > 0) { @@ -31,8 +82,7 @@ void PgfLinearizer::TreeNode::linearize_arg(PgfLinearizationOutputIface *out, Pg } if (arg == 0) throw pgf_error("Missing argument"); - size_t lindex = eval_param(r); - arg->linearize(out, linearizer, lindex); + arg->linearize(out, linearizer, r); } void PgfLinearizer::TreeNode::linearize_var(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, size_t r) @@ -52,20 +102,22 @@ void PgfLinearizer::TreeNode::linearize_var(PgfLinearizationOutputIface *out, Pg out->symbol_token(linearizer->printer.get_text()); } -void PgfLinearizer::TreeNode::linearize_seq(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, ref seq) +void PgfLinearizer::TreeNode::linearize_item(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, Item *item) { - for (size_t i = 0; i < seq->syms.size(); i++) { - PgfSymbol sym = seq->syms[i]; + for (size_t i = 0; i < item->rule->syms.size(); i++) { + PgfSymbol sym = item->rule->syms[i]; switch (ref::get_tag(sym)) { case PgfSymbolCat::tag: { auto sym_cat = ref::untagged(sym); - linearize_arg(out, linearizer, sym_cat->d, &sym_cat->r); + size_t r = item->eval(ref::from_ptr(&sym_cat->r)); + linearize_arg(out, linearizer, sym_cat->d, r); break; } case PgfSymbolLit::tag: { auto sym_lit = ref::untagged(sym); - linearize_arg(out, linearizer, sym_lit->d, &sym_lit->r); + size_t r = item->eval(ref::from_ptr(&sym_lit->r)); + linearize_arg(out, linearizer, sym_lit->d, r); break; } case PgfSymbolVar::tag: { @@ -169,113 +221,68 @@ void PgfLinearizer::TreeNode::linearize_seq(PgfLinearizationOutputIface *out, Pg } } -size_t PgfLinearizer::TreeNode::eval_param(PgfLParam *param) -{ - size_t value = param->i0; - for (size_t j = 0; j < param->n_terms; j++) { - size_t factor = param->terms[j].factor; - size_t var = param->terms[j].var; - - if (var < var_count && var_values[var] != (size_t) -1) { - value += factor * var_values[var]; - } else { - throw pgf_error("Unbound variable in resolving a linearization"); - } - } - return value; -} - PgfLinearizer::TreeLinNode::TreeLinNode(PgfLinearizer *linearizer, ref lin) : TreeNode(linearizer) { - this->lin = lin; - this->lin_index = 0; + this->lin = lin; + this->rule_index = 0; + this->items = new Item*[lin->lincat->fields.size()](); } bool PgfLinearizer::TreeLinNode::resolve(PgfLinearizer *linearizer) { vector hypos = lin->absfun->type->hypos; - size_t n_args = lin->args.size() / lin->res.size(); - while (lin_index < lin->res.size()) { - size_t offset = lin_index*n_args; - - ref pres = lin->res[lin_index]; - - // Unbind all variables - for (size_t j = 0; j < var_count; j++) { - var_values[j] = (size_t) -1; - } + while (rule_index < lin->rules.size()) { + Item *item = new (lin->rules[rule_index]) Item(); int i = 0; TreeNode *arg = args; while (arg != NULL) { - ref parg = lin->args.elem(offset+i); arg->check_category(linearizer, &hypos[i].type->name); - if (arg->value < parg->param->i0) + if (!item->instantiate(item->rule->args[i], arg->value)) break; - size_t value = arg->value - parg->param->i0; - for (size_t j = 0; j < parg->param->n_terms; j++) { - size_t factor = parg->param->terms[j].factor; - size_t var = parg->param->terms[j].var; - size_t var_value; - - if (var < var_count && var_values[var] != (size_t) -1) { - // The variable already has a value - var_value = var_values[var]; - } else { - // The variable is not assigned yet - var_value = value / factor; - - // find the range for the variable - size_t range = 0; - for (size_t k = 0; k < pres->vars.size(); k++) { - ref var_range = pres->vars.elem(k); - if (var_range->var == var) { - range = var_range->range; - break; - } - } - if (range == 0) - throw pgf_error("Unknown variable in resolving a linearization"); - - if (var_value >= range) - break; - - // Assign the variable; - if (var >= var_count) { - var_values = (size_t*) - realloc(var_values, (var+1)*sizeof(size_t)); - while (var_count < var) { - var_values[var_count++] = (size_t) -1; - } - var_count++; - } - var_values[var] = var_value; - } - - value -= var_value * factor; - } - - if (value != 0) - break; - - arg = arg->next_arg; - i++; + arg = arg->next_arg; i++; } - lin_index++; + size_t max_value = 1; + for (size_t i = 0; i < item->vars.size(); i++) { + if (item->vars[i] == 0) + max_value *= item->rule->vars[i].range; + } - if (arg == NULL) { - value = eval_param(&pres->param); - return true; + for (size_t value = 0; value < max_value; value++) { + Item *new_item = new (item) Item; + + size_t v = value; + for (size_t i = 0; i < new_item->vars.size(); i++) { + if (new_item->vars[i] == 0) { + size_t range = new_item->rule->vars[i].range; + new_item->vars[i] = (v % range)+1; + v = v / range; + } + } + + size_t lin_idx = new_item->eval(new_item->rule->lin_idx); + items[lin_idx] = new_item; + + this->value = new_item->eval(new_item->rule->res); + } + delete item; + + rule_index++; + } + + for (size_t i = 0; i < lin->lincat->fields.size(); i++) { + if (items[i] == NULL) { + rule_index = 0; + return false; } } - lin_index = 0; - return false; + return true; } void PgfLinearizer::TreeLinNode::check_category(PgfLinearizer *linearizer, PgfText *cat) @@ -302,9 +309,7 @@ void PgfLinearizer::TreeLinNode::linearize(PgfLinearizationOutputIface *out, Pgf linearizer->pre_stack->bracket_stack = bracket; } - size_t n_seqs = lin->seqs.size() / lin->res.size(); - ref seq = lin->seqs[(lin_index-1)*n_seqs + lindex]; - linearize_seq(out, linearizer, seq); + linearize_item(out, linearizer, items[lindex]); if (linearizer->pre_stack == NULL) out->end_phrase(cat, fid, field, &lin->name); @@ -325,11 +330,21 @@ ref PgfLinearizer::TreeLinNode::get_lincat(PgfLinearizer *linear return namespace_lookup(linearizer->concr->lincats, &lin->absfun->type->name); } +PgfLinearizer::TreeLinNode::~TreeLinNode() +{ + size_t n_fields = lin->lincat->fields.size(); + for (size_t i = 0; i < n_fields; i++) { + delete items[i]; + } + delete[] items; +}; + PgfLinearizer::TreeLindefNode::TreeLindefNode(PgfLinearizer *linearizer, PgfText *fun, PgfText *literal) : TreeNode(linearizer) { this->lincat = 0; - this->lin_index = 0; + this->rule_index= 0; + this->items = NULL; this->fun = fun; this->literal = literal; @@ -355,17 +370,46 @@ PgfLinearizer::TreeLindefNode::TreeLindefNode(PgfLinearizer *linearizer, PgfText bool PgfLinearizer::TreeLindefNode::resolve(PgfLinearizer *linearizer) { - if (lincat == 0) { - return (lin_index = !lin_index); - } else { - ref pres = lincat->res[lin_index]; - value = eval_param(&pres->param); - lin_index++; - if (lin_index <= lincat->n_lindefs) - return true; - lin_index = 0; - return false; +/* while (rule_index < lincat->n_lindefs2) { + ref rule = lincat->rules[rule_index]; + Item *item = new (rule) Item(); + + size_t max_value = 1; + for (size_t i = 0; i < item->vars.size(); i++) { + if (item->vars[i] == 0) + max_value *= item->rule->vars[i].range; + } + + for (size_t value = 0; value < max_value; value++) { + size_t v = value; + for (size_t i = 0; i < item->vars.size(); i++) { + if (item->vars[i] == 0) { + size_t range = item->rule->vars[i].range; + item->vars[i] = v % range; + v = v / range; + } + } + + Item *new_item = new (item) Item; + + size_t lin_idx = item->eval(new_item->rule->lin_idx); + items[lin_idx] = new_item; + + this->value = item->eval(new_item->rule->res); + } + delete item; + + rule_index++; } + + for (size_t i = 0; i < lincat->fields.size(); i++) { + if (items[i] == NULL) { + rule_index = 0; + return false; + } + } +*/ + return true; } void PgfLinearizer::TreeLindefNode::check_category(PgfLinearizer *linearizer, PgfText *cat) @@ -373,6 +417,7 @@ void PgfLinearizer::TreeLindefNode::check_category(PgfLinearizer *linearizer, Pg lincat = namespace_lookup(linearizer->concr->lincats, cat); if (lincat == 0) throw pgf_error("Cannot find a lincat for a category"); + this->items = new Item*[lincat->fields.size()](); } void PgfLinearizer::TreeLindefNode::linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, PgfLParam *r) @@ -389,7 +434,7 @@ void PgfLinearizer::TreeLindefNode::linearize_arg(PgfLinearizationOutputIface *o void PgfLinearizer::TreeLindefNode::linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex) { - if (lincat != 0) { +/* if (lincat != 0) { PgfText *field = &*lincat->fields[lindex]; if (linearizer->pre_stack == NULL) out->begin_phrase(&lincat->name, fid, field, fun); @@ -404,8 +449,8 @@ void PgfLinearizer::TreeLindefNode::linearize(PgfLinearizationOutputIface *out, linearizer->pre_stack->bracket_stack = bracket; } - ref seq = lincat->seqs[(lin_index-1)*lincat->fields.size() + lindex]; - linearize_seq(out, linearizer, seq); + ref seq = lincat->seqs[(rule_index-1)*lincat->fields.size() + lindex]; +// linearize_seq(out, linearizer, seq); if (linearizer->pre_stack == NULL) out->end_phrase(&lincat->name, fid, field, fun); @@ -421,7 +466,7 @@ void PgfLinearizer::TreeLindefNode::linearize(PgfLinearizationOutputIface *out, } } else { linearize_arg(out, linearizer, 0, NULL); - } + }*/ } ref PgfLinearizer::TreeLindefNode::get_lincat(PgfLinearizer *linearizer) @@ -429,11 +474,26 @@ ref PgfLinearizer::TreeLindefNode::get_lincat(PgfLinearizer *lin return lincat; } +PgfLinearizer::TreeLindefNode::~TreeLindefNode() +{ + if (lincat) { + size_t n_fields = lincat->fields.size(); + for (size_t i = 0; i < n_fields; i++) { + delete items[i]; + } + delete[] items; + } + + free(fun); + free(literal); +}; + PgfLinearizer::TreeLinrefNode::TreeLinrefNode(PgfLinearizer *linearizer, TreeNode *root) : TreeNode(linearizer) { args = root; - lin_index=0; + rule_index=0; + item = NULL; } bool PgfLinearizer::TreeLinrefNode::resolve(PgfLinearizer *linearizer) @@ -441,81 +501,53 @@ bool PgfLinearizer::TreeLinrefNode::resolve(PgfLinearizer *linearizer) TreeNode *root = args; ref lincat = root->get_lincat(linearizer); if (lincat == 0) - return (lin_index = !lin_index); + return (rule_index = !rule_index); - while (lincat->n_lindefs+lin_index < lincat->res.size()) { - // Unbind all variables - for (size_t j = 0; j < var_count; j++) { - var_values[j] = (size_t) -1; + while (rule_index < lincat->rules.size()) { + Item *item = new (lincat->rules[lincat->n_lindefs+rule_index]) Item(); + + if (!item->instantiate(item->rule->args[0], root->value)) { + rule_index++; + continue; } - ref pres = lincat->res[lincat->n_lindefs+lin_index]; - ref parg = lincat->args.elem(lincat->n_lindefs+lin_index); + size_t max_value = 1; + for (size_t i = 0; i < item->vars.size(); i++) { + if (item->vars[i] == 0) + max_value *= item->rule->vars[i].range; + } - if (root->value < parg->param->i0) - break; - - size_t value = root->value - parg->param->i0; - for (size_t j = 0; j < parg->param->n_terms; j++) { - size_t factor = parg->param->terms[j].factor; - size_t var = parg->param->terms[j].var; - size_t var_value; - - if (var < var_count && var_values[var] != (size_t) -1) { - // The variable already has a value - var_value = var_values[var]; - } else { - // The variable is not assigned yet - var_value = value / factor; - - // find the range for the variable - size_t range = 0; - for (size_t k = 0; k < pres->vars.size(); k++) { - ref var_range = pres->vars.elem(k); - if (var_range->var == var) { - range = var_range->range; - break; - } + for (size_t value = 0; value < max_value; value++) { + size_t v = value; + for (size_t i = 0; i < item->vars.size(); i++) { + if (item->vars[i] == 0) { + size_t range = item->rule->vars[i].range; + item->vars[i] = v % range; + v = v / range; } - if (range == 0) - throw pgf_error("Unknown variable in resolving a linearization"); - - if (var_value >= range) - break; - - // Assign the variable; - if (var >= var_count) { - var_values = (size_t*) - realloc(var_values, (var+1)*sizeof(size_t)); - while (var_count < var) { - var_values[var_count++] = (size_t) -1; - } - var_count++; - } - var_values[var] = var_value; } - value -= var_value * factor; + this->item = new (item) Item; + this->value = item->eval(this->item->rule->res); } + delete item; - lin_index++; - if (value == 0) { - value = eval_param(&pres->param); - return true; - } + break; } - lin_index = 0; - return false; + if (item == NULL) { + rule_index = 0; + return false; + } + + return true; } void PgfLinearizer::TreeLinrefNode::linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex) { ref lincat = args->get_lincat(linearizer); if (lincat != 0) { - size_t i = lincat->n_lindefs*lincat->fields.size() + (lin_index-1); - ref seq = lincat->seqs[i]; - linearize_seq(out, linearizer, seq); + linearize_item(out, linearizer, item); } else { args->linearize(out, linearizer, lindex); } @@ -526,6 +558,11 @@ ref PgfLinearizer::TreeLinrefNode::get_lincat(PgfLinearizer *lin return 0; } +PgfLinearizer::TreeLinrefNode::~TreeLinrefNode() +{ + delete item; +} + PgfLinearizer::TreeLitNode::TreeLitNode(PgfLinearizer *linearizer, ref lincat, PgfText *lit) : TreeNode(linearizer) { @@ -663,14 +700,14 @@ void PgfLinearizer::flush_pre_stack(PgfLinearizationOutputIface *out, PgfText *t ref alt = pre->sym_kp->alts.elem(i); for (ref prefix : alt->prefixes) { if (cmp(token, &(*prefix))) { - pre->node->linearize_seq(out, this, alt->form); +// pre->node->linearize_seq(out, this, alt->form); goto done; } } } } - pre->node->linearize_seq(out, this, pre->sym_kp->default_form); +// pre->node->linearize_seq(out, this, pre->sym_kp->default_form); done: if (pre->bracket_stack != NULL) diff --git a/src/runtime/c/pgf/linearizer.h b/src/runtime/c/pgf/linearizer.h index f72224f0d..af54030ef 100644 --- a/src/runtime/c/pgf/linearizer.h +++ b/src/runtime/c/pgf/linearizer.h @@ -26,6 +26,49 @@ class PGF_INTERNAL_DECL PgfLinearizer : public PgfUnmarshaller { ref concr; PgfMarshaller *m; + struct Item { + ref rule; + + struct { + size_t &operator[](int i) { + Item *item = containerof(Item,vars,this); + return ((size_t*) (item+1))[i]; + } + size_t size() { + Item *item = containerof(Item,vars,this); + return item->rule->vars.size(); + } + } vars; + + void *operator new(size_t sz, ref rule) + { + size_t sz2 = rule->vars.size()*sizeof(size_t); + Item *new_item = (Item *) malloc(sz+sz2); + memset(new_item, 0, sz+sz2); + new_item->rule = rule; + return new_item; + } + + void *operator new(size_t sz, Item *item) + { + size_t sz2 = item->vars.size()*sizeof(size_t); + Item *new_item = (Item *) malloc(sz+sz2); + memcpy(new_item, item, sz+sz2); + return new_item; + } + + void operator delete(void *p) + { + free(p); + } + + Item() { + } + + bool instantiate(ref lparam,size_t value); + size_t eval(ref lparam); + }; + struct TreeNode { TreeNode *next; TreeNode *next_arg; @@ -34,8 +77,6 @@ class PGF_INTERNAL_DECL PgfLinearizer : public PgfUnmarshaller { int fid; size_t value; - size_t var_count; - size_t *var_values; size_t n_hoas_vars; PgfText **hoas_vars; @@ -43,29 +84,31 @@ class PGF_INTERNAL_DECL PgfLinearizer : public PgfUnmarshaller { TreeNode(PgfLinearizer *linearizer); virtual bool resolve(PgfLinearizer *linearizer) { return true; }; virtual void check_category(PgfLinearizer *linearizer, PgfText *cat)=0; - virtual void linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, PgfLParam *r); + virtual void linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, size_t r); virtual void linearize_var(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, size_t r); - virtual void linearize_seq(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, ref seq); + virtual void linearize_item(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, Item *item); virtual void linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex)=0; - size_t eval_param(PgfLParam *param); virtual ref get_lincat(PgfLinearizer *linearizer)=0; - virtual ~TreeNode() { free(var_values); free(hoas_vars); }; + virtual ~TreeNode() { free(hoas_vars); }; }; struct TreeLinNode : public TreeNode { ref lin; - size_t lin_index; + size_t rule_index; + Item **items; TreeLinNode(PgfLinearizer *linearizer, ref lin); virtual bool resolve(PgfLinearizer *linearizer); virtual void check_category(PgfLinearizer *linearizer, PgfText *cat); virtual void linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex); virtual ref get_lincat(PgfLinearizer *linearizer); + virtual ~TreeLinNode(); }; struct TreeLindefNode : public TreeNode { ref lincat; - size_t lin_index; + size_t rule_index; + Item **items; PgfText *fun; PgfText *literal; @@ -75,17 +118,19 @@ class PGF_INTERNAL_DECL PgfLinearizer : public PgfUnmarshaller { virtual void linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, PgfLParam *r); virtual void linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex); virtual ref get_lincat(PgfLinearizer *linearizer); - ~TreeLindefNode() { free(fun); free(literal); }; + ~TreeLindefNode(); }; struct TreeLinrefNode : public TreeNode { - size_t lin_index; + size_t rule_index; + Item *item; TreeLinrefNode(PgfLinearizer *linearizer, TreeNode *root); virtual bool resolve(PgfLinearizer *linearizer); virtual void check_category(PgfLinearizer *linearizer, PgfText *cat) {}; virtual void linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex); virtual ref get_lincat(PgfLinearizer *linearizer); + ~TreeLinrefNode(); }; struct TreeLitNode : public TreeNode { diff --git a/src/runtime/c/pgf/parser.cxx b/src/runtime/c/pgf/parser.cxx index 261d53b3d..894d69e25 100644 --- a/src/runtime/c/pgf/parser.cxx +++ b/src/runtime/c/pgf/parser.cxx @@ -1,2362 +1,1480 @@ #include "data.h" #include "printer.h" #include "parser.h" -#include -//#define DEBUG_STATE_CREATION -//#define DEBUG_AUTOMATON -//#define DEBUG_PARSER -//#define DEBUG_GENERATOR +// #define DEBUG_PARSER +// #define DEBUG_EXPRS -struct PgfLRTableMaker::CCat { - CCat *parent; - size_t lin_idx; - ref lincat; +PgfAbstractParser::PgfAbstractParser(ref concr) +{ + this->concr = concr; - size_t id; - bool productive; // true if it has non epsilon rules - std::vector items; // productive items - std::vector suspended; // items that can progress on epsilon - std::vector prods; // epsilon productions - - ref persistant; - - CCat(size_t id, CCat *parent, size_t lin_idx) { - this->parent = parent; - this->lin_idx = lin_idx; - this->lincat = (parent != NULL) ? parent->lincat : 0; - this->id = id; - this->productive = false; - this->persistant = 0; - } - - ref persist(); - - void suspend_item(Item *item); - void register_item(Item *item); - - ~CCat(); -}; - -#define container(T,field,p) ((T*) (((char*) p) - offsetof(T, field))) - -struct PgfLRTableMaker::Production { - ref lin; - size_t index; - - struct { - // After the Production there is an array of arguments - size_t count; - CCat *&operator [](int i) { - return ((CCat **) (container(Production,args,this)+1))[i]; - } - } args; - - struct { - // After the array of arguments there is an array of variables - size_t count; - size_t &operator [](int i) { - Production *prod = container(Production,vals,this); - return ((size_t *) (((CCat**) (prod+1)) + prod->args.count))[i]; - } - } vals; - - void *operator new(size_t size, Item *item); - - Production() { - // If there is no constructor, GCC will zero the object, - // while it has already been initialized in the new operator. - } - - void operator delete(void *p) { - free(p); - } -}; - -struct PgfLRTableMaker::Item { - size_t ref_cnt; // how many CCat:s contain the item? - CCat* ccat; - object lin_obj; - ref seq; - size_t seq_idx; - size_t sym_idx; - size_t stk_size; - - struct Arg { - CCat *ccat; - size_t stk_idx; - }; - - struct { - // After the Item there is an array of arguments - size_t count; - Arg &operator [](int i) const { - return ((Arg*) (container(Item,args,this)+1))[i]; - } - } args; - - struct { - // After the array of arguments there is an array of variables - size_t count; - size_t &operator [](int i) const { - Item *item = container(Item,vals,this); - return ((size_t *) (((Arg*) (item+1)) + item->args.count))[i]; - } - } vals; - - void *operator new(size_t size, CCat* ccat, ref lin, size_t seq_idx); - void *operator new(size_t size, ref lincat, size_t index); - void *operator new(size_t size, CCat* ccat, Production *prod, size_t lin_idx); - void *operator new(size_t size, Item *item, CCat *ccat); - void *operator new(size_t size, Item *item, size_t lin_idx); - void *operator new(size_t size, Item *item); - - Item() { - // If there is no constructor, GCC will zero the object, - // while it has already been initialized in the new operator. - } - - void operator delete(void *p) { - if (((Item *) p)->ref_cnt == 0) - free(p); - } -}; - -struct PgfLRTableMaker::CompareItem : std::less { - bool operator() (const Item *item1, const Item *item2) const { - if (item1->lin_obj < item2->lin_obj) - return true; - else if (item1->lin_obj > item2->lin_obj) - return false; - - if (item1->seq_idx < item2->seq_idx) - return true; - else if (item1->seq_idx > item2->seq_idx) - return false; - - if (item1->sym_idx < item2->sym_idx) - return true; - else if (item1->sym_idx > item2->sym_idx) - return false; - - for (size_t i = 0; i < item1->args.count; i++) { - if (item1->args[i].ccat < item2->args[i].ccat) - return true; - else if (item1->args[i].ccat > item2->args[i].ccat) - return false; - if (item1->args[i].stk_idx < item2->args[i].stk_idx) - return true; - else if (item1->args[i].stk_idx > item2->args[i].stk_idx) - return false; - } - - return false; - } -}; - -const PgfLRTableMaker::CompareItem PgfLRTableMaker::compare_item; - -ref PgfLRTableMaker::CCat::persist() { - if (persistant != 0) - return persistant; - - size_t n_prods = prods.size(); - persistant = PgfDB::malloc(n_prods*sizeof(PgfLRReduce)); - persistant->n_prods = n_prods; - for (size_t i = 0; i < n_prods; i++) { - Production *prod = prods[i]; - persistant->prods[i].lin = prod->lin; - persistant->prods[i].index = prod->index; - auto children = vector>::alloc(prod->args.count); - for (size_t j = 0; j < prod->args.count; j++) { - if (prod->args[j] == NULL) { - children[j] = 0; - } else { - ref child_arg = prod->args[j]->persist(); - children[j] = child_arg; - } - } - persistant->prods[i].args = children; - } - - return persistant; + this->first_state = NULL; + this->current_state = NULL; + this->last_fid = 0; } -void PgfLRTableMaker::CCat::suspend_item(Item *item) { - suspended.push_back(item); - if (item != NULL) - item->ref_cnt++; -} - -void PgfLRTableMaker::CCat::register_item(Item *item) { - items.push_back(item); item->ref_cnt++; -} - -PgfLRTableMaker::CCat::~CCat() { - for (Item *item : items) { - item->ref_cnt--; - delete item; - } - for (Item *item : suspended) { - item->ref_cnt--; - delete item; - } +PgfAbstractParser::CCat::~CCat() +{ for (Production *prod : prods) { delete prod; } + for (ExprState *estate : pending) { + delete estate; + } } -void *PgfLRTableMaker::Production::operator new(size_t size, Item *item) { - ref lin = ref::untagged(item->lin_obj); - - size_t n_fields = lin->seqs.size() / lin->res.size(); - size_t ex_size = sizeof(CCat*)*item->args.count+sizeof(size_t)*item->vals.count; - - Production *prod = (Production *) malloc(size+ex_size); - prod->lin = lin; - prod->index = item->seq_idx / n_fields; - prod->args.count = item->args.count; - prod->vals.count = item->vals.count; - - for (size_t i = 0; i < item->args.count; i++) { - prod->args[i] = item->args[i].ccat; - } - for (size_t i = 0; i < item->vals.count; i++) { - prod->vals[i] = item->vals[i]; - } - - return prod; -} - -void *PgfLRTableMaker::Item::operator new(size_t size, CCat* ccat, ref lin, size_t seq_idx) { - size_t n_args = lin->absfun->type->hypos.size(); - size_t n_fields = lin->seqs.size() / lin->res.size(); - ref res = lin->res[seq_idx / n_fields]; - size_t n_vars = res->vars.size(); - size_t ex_size = sizeof(Arg)*n_args+sizeof(size_t)*n_vars; - - Item *item = (Item *) malloc(size+ex_size); - item->ref_cnt = 0; - item->ccat = ccat; - item->lin_obj = lin.tagged(); - item->seq = lin->seqs[seq_idx]; - item->seq_idx = seq_idx; - item->sym_idx = 0; - item->stk_size = 0; - item->args.count = n_args; - item->vals.count = n_vars; - memset(item+1, 0, ex_size); - - return item; -} - -void *PgfLRTableMaker::Item::operator new(size_t size, ref lincat, size_t index) { - size_t n_args = 1; - ref res = lincat->res[lincat->n_lindefs+index]; - size_t n_vars = res->vars.size(); - size_t ex_size = sizeof(Arg)*n_args+sizeof(size_t)*n_vars; - - size_t seq_idx = - lincat->n_lindefs*lincat->fields.size() + index; - - Item *item = (Item *) malloc(size+ex_size); - item->ref_cnt = 0; - item->ccat = NULL; - item->lin_obj = lincat.tagged(); - item->seq = lincat->seqs[seq_idx]; - item->seq_idx = seq_idx; - item->sym_idx = 0; - item->stk_size = 0; - item->args.count = n_args; - item->vals.count = n_vars; - memset(item+1, 0, ex_size); - - return item; -} - -void *PgfLRTableMaker::Item::operator new(size_t size, CCat* ccat, Production *prod, size_t lin_idx) { - size_t n_fields = prod->lin->seqs.size() / prod->lin->res.size(); - ref res = prod->lin->res[prod->index]; - size_t ex_size = sizeof(Arg)*prod->args.count+sizeof(size_t)*prod->vals.count; - - Item *item = (Item *) malloc(size+ex_size); - item->ref_cnt = 0; - item->ccat = ccat; - item->lin_obj = prod->lin.tagged(); - item->seq_idx = prod->index*n_fields+lin_idx; - item->seq = prod->lin->seqs[item->seq_idx]; - item->sym_idx = 0; - item->stk_size = 0; - item->args.count = prod->args.count; - item->vals.count = prod->vals.count; - - for (size_t i = 0; i < item->args.count; i++) { - item->args[i].ccat = prod->args[i]; - item->args[i].stk_idx = 0; - } - for (size_t i = 0; i < item->vals.count; i++) { - item->vals[i] = prod->vals[i]; - } - - return item; -} - -void *PgfLRTableMaker::Item::operator new(size_t size, Item *item, CCat *ccat) { - size_t ex_size = sizeof(Arg)*item->args.count+sizeof(size_t)*item->vals.count; - - Item *new_item = (Item *) malloc(size+ex_size); - new_item->ref_cnt = 0; - new_item->ccat = item->ccat; - new_item->lin_obj = item->lin_obj; - new_item->seq = item->seq; - new_item->seq_idx = item->seq_idx; - new_item->sym_idx = item->sym_idx+1; - new_item->stk_size = item->stk_size; - new_item->args.count = item->args.count; - new_item->vals.count = item->vals.count; - memcpy(new_item+1,item+1,ex_size); - - ref scat = - ref::untagged(item->seq->syms[item->sym_idx]); - new_item->args[scat->d].ccat = ccat; - - return new_item; -} - -void *PgfLRTableMaker::Item::operator new(size_t size, Item *item, size_t lin_idx) { - size_t ex_size = sizeof(Arg)*item->args.count+sizeof(size_t)*item->vals.count; - - Item *new_item = (Item *) malloc(size+ex_size); - new_item->ref_cnt = 0; - new_item->ccat = item->ccat; - new_item->lin_obj = item->lin_obj; - new_item->seq = item->seq; - new_item->seq_idx = item->seq_idx; - new_item->sym_idx = item->sym_idx+1; - new_item->stk_size = item->stk_size; - new_item->args.count = item->args.count; - new_item->vals.count = item->vals.count; - memcpy(new_item+1,item+1,ex_size); - - ref scat = - ref::untagged(item->seq->syms[item->sym_idx]); - new_item->args[scat->d].stk_idx = ++new_item->stk_size; - - return new_item; -} - -void *PgfLRTableMaker::Item::operator new(size_t size, Item *item) { - size_t ex_size = sizeof(Arg)*item->args.count+sizeof(size_t)*item->vals.count; - - Item *new_item = (Item *) malloc(size+ex_size); - memcpy(new_item,item,size+ex_size); - new_item->ref_cnt = 0; - - return new_item; -} - -bool PgfLRTableMaker::CompareKey3::operator() (const Key3& k1, const Key3& k2) const { - size_t i = k1.second; - size_t j = k2.second; - for (;;) { - if (i >= k1.first->syms.size() || ref::get_tag(k1.first->syms[i]) != PgfSymbolKS::tag) - return (j < k2.first->syms.size() && ref::get_tag(k2.first->syms[j]) == PgfSymbolKS::tag); - - if (j >= k2.first->syms.size() || ref::get_tag(k2.first->syms[j]) != PgfSymbolKS::tag) - return false; - - auto symks1 = ref::untagged(k1.first->syms[i]); - auto symks2 = ref::untagged(k2.first->syms[j]); - - int res[2] = {0,0}; - texticmp(&symks1->token, &symks2->token, res); - if (res[0] < 0) - return true; - if (res[0] > 0) - return false; - - i++; j++; - } - - return false; -} - -struct PgfLRTableMaker::State { - size_t id; - std::vector items; // The seed items for this state - std::vector completed; // Completed items that will become reductions - std::map ccats1; - std::map ccats2; - std::map tokens; - State *bind_state; - - State() { - this->id = 0; - this->bind_state = NULL; - } - - ~State() { - for (Item *item : items) { - item->ref_cnt--; - delete item; - } - - for (Item *item : completed) { - item->ref_cnt--; - delete item; - } - } - - void push_item(Item *item) { - items.push_back(item); item->ref_cnt++; - push_heap(items.begin(), items.end(), compare_item); - } - - Item *pop_item() { - Item *item = items.back(); items.pop_back(); - item->ref_cnt--; - return item; - } -}; - -PgfLRTableMaker::PgfLRTableMaker(ref abstr, ref concr) +PgfAbstractParser::Cont::~Cont() { - this->abstr = abstr; - this->concr = concr; - this->ccat_id = 0; - this->state_id = 0; + for (Item *item : suspended) { + delete item; + } +} - PgfText *startcat = (PgfText *) - alloca(sizeof(PgfText)+9); - startcat->size = 8; - strcpy(startcat->text, "startcat"); - - ref flag = - namespace_lookup(abstr->aflags, startcat); - - ref lincat = 0; - if (flag != 0) { - switch (ref::get_tag(flag->value)) { - case PgfLiteralStr::tag: { - auto lstr = ref::untagged(flag->value); - - State *state = new State(); - - lincat = - namespace_lookup(concr->lincats, &lstr->val); - - MD5Context ctxt; - - for (size_t i = 0; i < lincat->res.size()-lincat->n_lindefs; i++) { - Item *item = new(lincat, i) Item; - - ctxt.update(item->lin_obj); - ctxt.update(item->seq_idx); - ctxt.update(item->sym_idx); - ctxt.update(item->args[0].ccat); - ctxt.update(item->args[0].stk_idx); - for (size_t i = 0; i < item->vals.count; i++) { - ctxt.update(item->vals[i]); +PgfAbstractParser::~PgfAbstractParser() +{ + State *state = first_state; + while (state != NULL) { + for (auto it1 : state->completed) { + for (auto it2 : it1.second) { + for (auto it3 : it2.second) { + delete it3.second; } - - state->push_item(item); } - - MD5Digest digest; - ctxt.finalize(&digest); - - states[digest] = state; - todo.push(state); } + for (auto it : state->conts1) { + delete it.second; } - } -} - -PgfLRTableMaker::~PgfLRTableMaker() -{ - for (auto p : states) { - delete p.second; - } - - for (auto p : ccats1) { - delete p.second; - } - - for (auto p : ccats2) { - delete p.second; - } -} - -#if defined(DEBUG_STATE_CREATION) || defined(DEBUG_AUTOMATON) -void PgfLRTableMaker::print_production(CCat *ccat, Production *prod) -{ - PgfPrinter printer(NULL, 0, NULL); - - ref res = *vector_elem(prod->lin->res, prod->index); - if (res->vars != 0) { - printer.lvar_ranges(res->vars, &prod->vals[0]); - printer.puts(" "); - } - - ref type = prod->lin->absfun->type; - printer.nprintf(37, "?%zu -> ", ccat->id); - printer.puts(&prod->lin->name); - printer.nprintf(37, "/%zu[", prod->index); - PgfDBMarshaller m; - size_t args_start = type->hypos->len * prod->index; - for (size_t i = 0; i < type->hypos->len; i++) { - if (i > 0) - printer.puts(","); - - if (prod->args[i] == NULL) { - ref arg = vector_elem(prod->lin->args, args_start + i); - m.match_type(&printer, vector_elem(type->hypos, i)->type.as_object()); - printer.puts("("); - printer.lparam(arg->param); - printer.puts(")"); - } else { - printer.nprintf(32, "?%zu", prod->args[i]->id); - } - } - printer.puts("]\n"); - - PgfText *text = printer.get_text(); - fputs(text->text, stderr); - free(text); -} - -void PgfLRTableMaker::print_item(Item *item) -{ - PgfPrinter printer(NULL, 0, NULL); - - switch (ref::get_tag(item->lin_obj)) { - case PgfConcrLin::tag: { - auto lin = - ref::untagged(item->lin_obj); - - size_t index = item->seq_idx / lin->lincat->fields->len; - size_t r = item->seq_idx % lin->lincat->fields->len; - ref res = *vector_elem(lin->res, index); - if (res->vars != 0) { - printer.lvar_ranges(res->vars, &item->vals[0]); - printer.puts(" "); - } - - if (item->ccat->parent == NULL) { - printer.puts(&item->ccat->lincat->name); - printer.puts("("); - printer.lparam(ref::from_ptr(&res->param)); - printer.puts(") -> "); - } else { - printer.nprintf(32,"?%zu -> ",item->ccat->parent->id); - } - - printer.puts(&lin->name); - printer.nprintf(32, "/%zd[", index); - PgfDBMarshaller m; - ref type = lin->absfun->type; - size_t args_start = type->hypos->len * index; - for (size_t i = 0; i < type->hypos->len; i++) { - if (i > 0) - printer.puts(","); - - if (item->args[i].ccat == NULL) { - ref arg = vector_elem(lin->args, args_start + i); - m.match_type(&printer, vector_elem(type->hypos, i)->type.as_object()); - printer.puts("("); - printer.lparam(arg->param); - printer.puts(")"); - } else { - printer.nprintf(32, "?%zu", item->args[i].ccat->id); + for (auto it1 : state->conts2) { + for (auto it2 : it1.second) { + delete it2.second; } - if (item->args[i].stk_idx > 0) - printer.nprintf(32, "$%zd", item->args[i].stk_idx); - } - printer.nprintf(32, "]; %zu : ", r); - break; - } - case PgfConcrLincat::tag: { - auto lincat = - ref::untagged(item->lin_obj); - - size_t index = item->seq_idx - lincat->n_lindefs*lincat->fields->len; - ref res = *vector_elem(lincat->res, lincat->n_lindefs+index); - if (res->vars != 0) { - printer.lvar_ranges(res->vars, &item->vals[0]); - printer.puts(" "); } - printer.puts("linref "); - printer.puts(&lincat->name); - printer.nprintf(32, "/%zd[", index); - if (item->args[0].ccat == NULL) { - printer.puts(&lincat->name); - printer.puts("("); - printer.lparam(vector_elem(lincat->args, lincat->n_lindefs+index)->param); - printer.puts(")"); - } else { - printer.nprintf(32, "?%zu", item->args[0].ccat->id); - } - if (item->args[0].stk_idx > 0) - printer.nprintf(32, "$%zd", item->args[0].stk_idx); - printer.puts("]; 0 : "); - break; + State *next = state->next; + delete state; + state = next; } - } - - if (item->sym_idx == 0) - printer.puts(". "); - - for (size_t i = 0; i < item->seq->syms.len; i++) { - PgfSymbol sym = item->seq->syms.data[i]; - printer.symbol(sym); - - if (i+1 == item->sym_idx) - printer.puts(" . "); - } - printer.puts("\n"); - - PgfText *text = printer.get_text(); - fputs(text->text, stderr); - free(text); } + +void PgfAbstractParser::process(Item *item, const PgfTextSpot &spot, bool bind) +{ +#ifdef DEBUG_PARSER + print_item(item,spot); #endif -void PgfLRTableMaker::process(State *state, Fold fold, Item *item) -{ -#if defined(DEBUG_STATE_CREATION) - if (fold == PROBE) - fprintf(stderr, "PROBE "); - else if (fold == INIT) - fprintf(stderr, "INIT "); - else if (fold == REPEAT) - fprintf(stderr, "REPEAT "); - print_item(item); -#endif - - if (item->sym_idx < item->seq->syms.size()) { - PgfSymbol sym = item->seq->syms[item->sym_idx]; - symbol(state, fold, item, sym); + if (item->dot < item->syms.size()) { + symbol(item,spot,bind,item->syms[item->dot]); + } else if (item->pre_alt > 0) { + item->dot = item->pre_dot+1; + item->pre_alt = 0; + item->pre_dot = 0; + item->syms = item->rule->syms.as_vector(); + process(item,spot,bind); } else { - complete(state, fold, item); + complete(item,spot,bind); } } -void PgfLRTableMaker::symbol(State *state, Fold fold, Item *item, PgfSymbol sym) +PGF_INTERNAL_DECL +int text_symbol_cmp(PgfTextSpot *spot, const uint8_t *end, + PgfSymbol sym, bool case_sensitive); + +void PgfAbstractParser::symbol(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym) { switch (ref::get_tag(sym)) { case PgfSymbolCat::tag: { auto symcat = ref::untagged(sym); - switch (ref::get_tag(item->lin_obj)) { - case PgfConcrLin::tag: { - auto lin = - ref::untagged(item->lin_obj); - ref res = lin->res[item->seq_idx / lin->lincat->fields.size()]; - auto arg = item->args[symcat->d]; - if (arg.ccat != NULL) { - predict(state, fold, item, arg.ccat, res->vars, &symcat->r); - } else { - ref hypo = lin->absfun->type->hypos.elem(symcat->d); - predict(state, fold, item, ref::from_ptr(&hypo->type->name), res->vars, &symcat->r); + State *state = new_state(spot); + + CCat *ccat = item->args[symcat->d]; + if (ccat == NULL) { + ref lincat = 0; + switch (ref::get_tag(item->rule->container)) { + case PgfConcrLin::tag: { + auto lin = ref::untagged(item->rule->container); + lincat = + namespace_lookup(concr->lincats, + &lin->absfun->type->hypos[symcat->d].type->name); + break; } - break; - } - case PgfConcrLincat::tag: { - auto lincat = - ref::untagged(item->lin_obj); - ref res = - lincat->res[lincat->n_lindefs + item->seq_idx - lincat->n_lindefs*lincat->fields.size()]; - auto arg = item->args[symcat->d]; - if (arg.ccat != NULL) { - predict(state, fold, item, arg.ccat, res->vars, &symcat->r); - } else { - predict(state, fold, item, ref::from_ptr(&lincat->name), res->vars, &symcat->r); + case PgfConcrLincat::tag: { + lincat = ref::untagged(item->rule->container); + break; + } + } + + if (lincat != 0) { + suspend(state,lincat,item); + } + } else { + size_t max_value = 1; + for (size_t i = 0; i < symcat->r.n_terms; i++) { + size_t var = symcat->r.terms[i].var; + for (size_t j = 0; j < item->vars.size(); j++) { + if (item->rule->vars[j].var == var && item->vars[j] == 0) { + max_value *= item->rule->vars[j].range; + break; + } + } + } + + for (size_t value = 0; value < max_value; value++) { + Item *new_item = new (item) Item; + + size_t value_ = value; + size_t lin_idx = symcat->r.i0; + for (size_t i = 0; i < symcat->r.n_terms; i++) { + size_t var = symcat->r.terms[i].var; + for (size_t j = 0; j < new_item->vars.size(); j++) { + if (new_item->rule->vars[j].var == var) { + if (new_item->vars[j] == 0) { + size_t range = new_item->rule->vars[j].range; + new_item->vars[j] = (value_ % range) + 1; + value_ = value_ / range; + } + lin_idx += symcat->r.terms[i].factor * (new_item->vars[j]-1); + break; + } + } + } + + Cont *&cont = state->conts2[ccat][lin_idx]; + if (cont == NULL) { + cont = new Cont; + cont->ccat = ccat; + cont->lincat = ccat->cont->lincat; + cont->state = state; + } + + cont->suspended.push_back(item); + + if (cont->suspended.size() == 1) { + for (Production *prod : cont->ccat->prods) { + td_predict(state,cont,prod,lin_idx); + } + } else { + State *next = state; + while (next != NULL) { + + auto it1 = next->completed.find(cont); + if (it1 != next->completed.end()) { + auto it2 = it1->second.find(ccat->value); + if (it2 != it1->second.end()) { + auto it3 = it2->second.find(lin_idx); + if (it3 != it2->second.end()) { + CCat *arg = it3->second; + Item *new_item = new (item) Item; + combine(next, new_item, arg); + } + } + } + next = next->next; + } + } } - break; - } } break; } case PgfSymbolKS::tag: { - auto symks = ref::untagged(sym); - - size_t sym_idx_2 = item->sym_idx+1; - while (sym_idx_2 < item->seq->syms.size()) { - if (ref::get_tag(item->seq->syms[sym_idx_2]) != PgfSymbolKS::tag) - break; - sym_idx_2++; - } - - if (fold == PROBE) { - item->ccat->productive = true; - if (item->sym_idx > 0 || sym_idx_2 < item->seq->syms.size()) { - item->ccat->register_item(item); - } - } else { - auto &next_state = state->tokens[Key3(item->seq,item->sym_idx)]; - if (next_state == NULL) { - next_state = new State; - } - item = new (item) Item; - item->sym_idx = sym_idx_2; - item->stk_size++; - next_state->push_item(item); - } + symbol_token(item, spot, bind, sym); break; } case PgfSymbolKP::tag: { - if (fold == PROBE) { - item->ccat->productive = true; - item->ccat->register_item(item); - } else { - auto symkp = ref::untagged(sym); - Item *new_item1 = NULL; - Item *new_item2 = NULL; - for (size_t i = 0; i < symkp->alts.size(); i++) { - ref form = symkp->alts[i].form; - if (form->syms.size() == 0) { - if (!new_item1) { - new_item1 = new (item) Item; - new_item1->sym_idx++; - } - process(state, fold, new_item1); - } else { - auto &next_state = state->tokens[Key3(form,0)]; - if (next_state == NULL) { - next_state = new State; - } - if (!new_item2) { - new_item2 = new (item) Item; - new_item2->sym_idx++; - new_item2->stk_size++; - } - next_state->push_item(new_item2); - } - } + auto symkp = ref::untagged(sym); - ref form = symkp->default_form; - if (form->syms.size() == 0) { - if (!new_item1) { - new_item1 = new (item) Item; - new_item1->sym_idx++; - } - process(state, fold, new_item1); - } else { - auto &next_state = state->tokens[Key3(form,0)]; - if (next_state == NULL) { - next_state = new State; - } - if (!new_item2) { - new_item2 = new (item) Item; - new_item2->sym_idx++; - new_item2->stk_size++; - } - next_state->push_item(new_item2); - } + Item *new_item = new(item) Item; + new_item->pre_alt = 1; + new_item->pre_dot = item->dot; + new_item->dot = 0; + new_item->syms = symkp->default_form; + new_item->rule = item->rule; + process(new_item, spot, bind); - // If the items are not owned by anyone, we must delete them - if (new_item1 != NULL) - delete new_item1; - if (new_item2 != NULL) - delete new_item2; + for (size_t i = 0; i < symkp->alts.size(); i++) { + Item *new_item = new(item) Item; + new_item->pre_alt = i+2; + new_item->pre_dot = item->dot; + new_item->dot = 0; + new_item->syms = symkp->alts[i].form; + new_item->rule = item->rule; + process(new_item, spot, bind); } + + // delete item; + break; } case PgfSymbolBIND::tag: { - if (fold == PROBE) { - item->ccat->productive = true; - item->ccat->register_item(item); - } else { - if (state->bind_state == NULL) { - state->bind_state = new State; - } - item = new (item) Item; - item->sym_idx++; - item->stk_size++; - state->bind_state->push_item(item); - } + symbol_bind(item, spot, sym); break; } case PgfSymbolSOFTBIND::tag: case PgfSymbolSOFTSPACE::tag: { - if (fold == PROBE) { - item->ccat->productive = true; - item->ccat->register_item(item); - } else { - // SOFT_BIND && SOFT_SPACE also allow a space - Item *new_item = new (item) Item(); - new_item->sym_idx++; - process(state,fold,new_item); - delete new_item; - - // Now we handle the case where there is no space. - if (state->bind_state == NULL) { - state->bind_state = new State; - } - new_item = new (item) Item(); - new_item->stk_size++; - state->bind_state->push_item(new_item); - delete new_item; - } + item->dot++; + process(item, spot, true); + process(item, spot, false); break; } case PgfSymbolCAPIT::tag: - case PgfSymbolALLCAPIT::tag: { - // We just ignore CAPIT && ALLCAPIT during parsing - item = new (item) Item(); item->sym_idx++; - process(state,fold,item); - delete item; + case PgfSymbolALLCAPIT::tag: + item->dot++; + process(item, spot, bind); + break; + } +} + +void PgfAbstractParser::complete(Item *item, const PgfTextSpot &spot, bool bind) +{ + State *state = new_state(spot); + + switch (ref::get_tag(item->rule->container)) { + case PgfConcrLin::tag: { + auto lin = ref::untagged(item->rule->container); + + size_t max_value = 1; + + size_t n_inst_vars = 0; + size_t *inst_vars = (size_t*) + alloca(sizeof(size_t)*item->vars.size()); + + // Compute which variables to assign to get determinate + // values of res and lin_idx + for (size_t i = 0; i < item->vars.size(); i++) { + if (item->vars[i] != 0) + continue; + + size_t var = item->rule->vars[i].var; + for (size_t j = 0; j < item->rule->res->n_terms; j++) { + if (item->rule->res->terms[j].var == var) { + goto found; + } + } + for (size_t j = 0; j < item->rule->lin_idx->n_terms; j++) { + if (item->rule->lin_idx->terms[j].var == var) { + goto found; + } + } + + continue; + + found: + inst_vars[n_inst_vars++] = i; + max_value *= item->rule->vars[i].range; + } + + // Go through all possible assignments and create a production + for (size_t value = 0; value < max_value; value++) { + size_t value_ = value; + for (size_t i = 0; i < n_inst_vars; i++) { + size_t var = inst_vars[i]; + size_t range = item->rule->vars[var].range; + item->vars[var] = (value_ % range) + 1; + value_ = value_ / range; + } + + size_t res = item->rule->res->i0; + for (size_t i = 0; i < item->rule->res->n_terms; i++) { + term t = item->rule->res->terms[i]; + for (size_t j = 0; j < item->vars.size(); j++) { + if (t.var == item->rule->vars[j].var) { + res += t.factor * (item->vars[j]-1); + break; + } + } + } + size_t lin_idx = item->rule->lin_idx->i0; + for (size_t i = 0; i < item->rule->lin_idx->n_terms; i++) { + term t = item->rule->lin_idx->terms[i]; + for (size_t j = 0; j < item->vars.size(); j++) { + if (t.var == item->rule->vars[j].var) { + lin_idx += t.factor * (item->vars[j]-1); + break; + } + } + } + + CCat *&ccat = state->completed[item->cont][res][lin_idx]; + if (ccat == NULL) { + ccat = new CCat; + ccat->fid = (++last_fid); + ccat->cont = item->cont; + ccat->state = state; + ccat->lin_idx = lin_idx; + ccat->value = res; + ccat->covered = false; + +#ifdef DEBUG_PARSER + { + PgfPrinter printer(NULL,0,NULL); + printer.nprintf(64,"[%zd-%zd; ",item->cont->state->end.pos,state->start.pos); + if (ccat->cont->ccat == NULL) { + printer.efun(&ccat->cont->lincat->name); + printer.nprintf(64,"(%zd)",ccat->value); + } else { + printer.emeta(ccat->cont->ccat->fid); + } + printer.nprintf(64,"; %zd; ",ccat->lin_idx); + printer.emeta(ccat->fid); + printer.puts("]"); + PgfText *text = printer.get_text(); + fprintf(stderr, "%s\n", text->text); + free(text); + } +#endif + } + + auto prod = new(item) Production; + prod->rule = item->rule; + for (size_t i = 0; i < prod->args.size(); i++) { + if (prod->args[i] != NULL && prod->args[i] != ccat) + prod->args[i]->covered = true; + } + ccat->prods.push_back(prod); + +#ifdef DEBUG_PARSER + print_prod(ccat, prod); +#endif + final_item(state, item, res, lin_idx); + + if (ccat->prods.size() == 1) { + if (ccat->cont->ccat == NULL) + bu_predict(concr->phrasetable, state, ccat); + size_t n_items = item->cont->suspended.size(); + for (size_t i = 0; i < n_items; i++) { + Item *new_item = new (item->cont->suspended[i]) Item; + combine(state,new_item,ccat); + }; + } else { + State *next = state; + while (next != NULL) { + for (auto it : next->conts2[ccat]) { + size_t lin_idx = it.first; + Cont *cont = it.second; + if (cont != NULL) { + size_t n_items = cont->suspended.size(); + for (size_t i = 0; i < n_items; i++) { + td_predict(next,cont,prod,lin_idx); + } + } + } + next = next->next; + } + } + } + break; + } + case PgfConcrLincat::tag: { + auto lincat = ref::untagged(item->rule->container); + final_item(state, item, 0, 0); break; } } } -struct PGF_INTERNAL_DECL PgfVariableValue { - size_t range; - size_t factor; - size_t value; - size_t j; -}; - -template -void PgfLRTableMaker::predict(State *state, Fold fold, Item *item, T cat, - vector vars, PgfLParam *r) +bool PgfAbstractParser::Item::instantiate(ref lparam,size_t value) { - size_t index = r->i0; - size_t n_terms = 0; - - PgfVariableValue *values = (PgfVariableValue *) - alloca(sizeof(PgfVariableValue)*r->n_terms); - for (size_t i = 0; i < r->n_terms; i++) - { - size_t var = r->terms[i].var; - for (size_t j = 0; j < vars.size(); j++) - { - ref range = vars.elem(j); - if (range->var == var) { - if (item->vals[j] == 0) { - values[n_terms].range = range->range; - values[n_terms].factor = r->terms[i].factor; - values[n_terms].value = 0; - values[n_terms].j = j; - n_terms++; - } else { - index += (item->vals[j]-1) * r->terms[i].factor; - } - break; - } - } - } - - for (;;) { - Item *new_item = new (item) Item(); - for (size_t i = 0; i < n_terms; i++) { - new_item->vals[values[i].j] = values[i].value+1; - } - - predict(state, fold, new_item, cat, index); - - delete new_item; - - size_t i = n_terms; - while (i > 0) { - i--; - values[i].value++; - if (values[i].value < values[i].range) { - index += values[i].factor; - i++; - break; - } - - index -= (values[i].value-1) * values[i].factor; - values[i].value = 0; - } - - if (i == 0) { - break; - } - } -} - -void PgfLRTableMaker::predict(State *state, Fold fold, Item *item, ref cat, size_t lin_idx) -{ - CCat *&ccat = ccats1[Key0(cat,lin_idx)]; - CCat *tmp = ccat; - if (tmp == NULL) { - ccat = new CCat(++ccat_id, NULL, lin_idx); - } - if (fold == PROBE) { - ccat->suspend_item(item); - } - if (tmp == NULL) { - std::function)> f = - [this,ccat](ref fun) { - predict(fun, ccat); - return true; - }; - probspace_iter(abstr->funs_by_cat, cat, f, false); - } else if (fold == PROBE && ccat->prods.size() > 0) { - Item *new_item = new(item,ccat) Item; - process(state,fold,new_item); - delete new_item; - } - - if (fold == PROBE) { - if (item->ccat != NULL && ccat->productive) { - item->ccat->productive = true; - item->ccat->register_item(item); - } - } else { - if (ccat->productive) { - auto &next_state = state->ccats1[Key1(ccat->lincat,lin_idx)]; - if (next_state == NULL) { - next_state = new State; - } - next_state->push_item(new(item,lin_idx) Item); - - if (next_state->items.size() == 1) { - for (size_t i = 0; i < ccat->items.size(); i++) { - process(state, REPEAT, ccat->items[i]); - } - } - } - - if (fold == INIT && ccat->prods.size() > 0) { - Item *new_item = new (item, ccat) Item; - process(state, fold, new_item); - delete new_item; - } - } -} - -void PgfLRTableMaker::predict(State *state, Fold fold, Item *item, CCat *ccat, size_t lin_idx) -{ - CCat *&new_ccat = ccats2[Key2(ccat,lin_idx)]; - CCat *tmp = new_ccat; - if (new_ccat == NULL) { - new_ccat = new CCat(++ccat_id, ccat, lin_idx); - } - if (fold == PROBE) { - new_ccat->suspend_item(item); - } - if (tmp == NULL) { - size_t n_prods = ccat->prods.size(); - for (size_t i = 0; i < n_prods; i++) { - Production *prod = ccat->prods[i]; - Item *item = new(new_ccat, prod, lin_idx) Item; - process(NULL, PROBE, item); - delete item; - } - } - - if (fold == PROBE) { - if (item->ccat != NULL && new_ccat->productive) { - item->ccat->productive = true; - item->ccat->register_item(item); - } - } else { - if (new_ccat->productive) { - auto &next_state = state->ccats2[Key2(new_ccat,lin_idx)]; - if (next_state == NULL) { - next_state = new State; - } - next_state->push_item(new(item,lin_idx) Item); - - if (next_state->items.size() == 1) { - for (size_t i = 0; i < new_ccat->items.size(); i++) { - process(state, REPEAT, new_ccat->items[i]); - } - } - } - if (fold == INIT && new_ccat->prods.size() > 0) { - Item *new_item = new (item, new_ccat) Item; - process(state, fold, new_item); - delete new_item; - } - } -} - -void PgfLRTableMaker::predict(ref absfun, CCat *ccat) -{ - ref lin = - namespace_lookup(concr->lins, &absfun->name); - - if (lin != 0) { - ccat->lincat = lin->lincat; - - size_t n_fields = lin->seqs.size() / lin->res.size(); - for (size_t i = 0; i < lin->res.size(); i++) { - size_t seq_idx = n_fields * i + ccat->lin_idx; - Item *item = new(ccat, lin, seq_idx) Item; - process(NULL, PROBE, item); - delete item; - } - } -} - -void PgfLRTableMaker::complete(State *state, Fold fold, Item *item) -{ - if (fold == PROBE) { - Production *prod = new(item) Production; - item->ccat->prods.push_back(prod); - -#if defined(DEBUG_STATE_CREATION) || defined(DEBUG_AUTOMATON) - print_production(item->ccat, prod); -#endif - - if (item->ccat->prods.size() == 1) { - // If this is the first epsilon production, - // resume the suspended items. - - // We don't use an iterator here since the vector suspended, - // may get updated in the recursion. - size_t n_susp = item->ccat->suspended.size(); - for (size_t i = 0; i < n_susp; i++) { - Item *susp = item->ccat->suspended[i]; - if (susp != NULL) { - Item *new_item = new (susp, item->ccat) Item; - process(state, PROBE, new_item); - delete new_item; - } - } - } - } else { - state->completed.push_back(item); item->ref_cnt++; - -#if defined(DEBUG_AUTOMATON) - fprintf(stderr, "reduce "); - print_item(item); -#endif - } -} - -void PgfLRTableMaker::internalize_state(State *&state) -{ - MD5Context ctxt; - auto begin = state->items.begin(); - auto end = state->items.end(); - while (begin != end) { - Item *item = *(--end); - ctxt.update(item->lin_obj); - ctxt.update(item->seq_idx); - ctxt.update(item->sym_idx); - for (size_t i = 0; i < item->args.count; i++) { - ctxt.update(item->args[i].ccat); - ctxt.update(item->args[i].stk_idx); - } - for (size_t i = 0; i < item->vals.count; i++) { - ctxt.update(item->vals[i]); - } - - pop_heap(begin,end,compare_item); - } - - MD5Digest digest; - ctxt.finalize(&digest); - - State *&next_state = states[digest]; - if (next_state == NULL) { - next_state = state; - next_state->id = ++state_id; - todo.push(next_state); - } else { - delete state; - state = next_state; - } -} - -vector PgfLRTableMaker::make() -{ - while (!todo.empty()) { - State *state = todo.front(); todo.pop(); - -#if defined(DEBUG_AUTOMATON) || defined(DEBUG_STATE_CREATION) - fprintf(stderr, "--------------- state %ld ---------------\n", state->id); -#endif - - while (!state->items.empty()) { - Item *item = state->pop_item(); - -#if defined(DEBUG_AUTOMATON) && !defined(DEBUG_STATE_CREATION) - // The order in which we process the items should not matter, - // For debugging however it is useful to see them in the same order. - pop_heap(state->items.begin(),state->items.end(),compare_item); - print_item(item); -#endif - - process(state, INIT, item); - - delete item; - } - - for (auto &i : state->ccats1) { - internalize_state(i.second); -#if defined(DEBUG_AUTOMATON) - fprintf(stderr, "%s.%zu: state %ld\n", - i.first.first->name.text, i.first.second, i.second->id); -#endif - } - for (auto &i : state->ccats2) { - internalize_state(i.second); -#if defined(DEBUG_AUTOMATON) - fprintf(stderr, "%s.%zu: state %ld\n", - i.first.first->lincat->name.text, i.first.second, i.second->id); -#endif - } - for (auto &i : state->tokens) { - internalize_state(i.second); -#if defined(DEBUG_AUTOMATON) - PgfPrinter printer(NULL, 0, NULL); - size_t sym_idx = i.first.second; - ref seq = i.first.first; - while (sym_idx < seq->syms.len) { - PgfSymbol sym = seq->syms.data[sym_idx]; - if (ref::get_tag(sym) != PgfSymbolKS::tag) - break; - printer.symbol(sym); - sym_idx++; - } - printer.nprintf(64, ": state %ld\n", i.second->id); - - PgfText *text = printer.get_text(); - fputs(text->text, stderr); - free(text); -#endif - } - if (state->bind_state != NULL) { - internalize_state(state->bind_state); -#if defined(DEBUG_AUTOMATON) - fprintf(stderr, "BIND: state %ld\n", state->bind_state->id); -#endif - } - - } - - vector lrtable = vector::alloc(states.size()); - for (auto v : states) { - State *state = v.second; - - size_t index = 0; - auto shifts = vector::alloc(state->ccats1.size()+state->ccats2.size()); - for (auto i : state->ccats1) { - ref shift = shifts.elem(index++); - shift->lincat = i.first.first; - shift->r = i.first.second; - shift->next_state = i.second->id; - } - for (auto i : state->ccats2) { - ref shift = shifts.elem(index++); - shift->lincat = i.first.first->lincat; - shift->r = i.first.second; - shift->next_state = i.second->id; - } - - vector tokens = 0; - if (state->tokens.size() > 0) { - size_t index = 0; - tokens = vector::alloc(state->tokens.size()); - for (auto i : state->tokens) { - ref shift = tokens.elem(index++); - shift->seq = i.first.first; - shift->sym_idx = i.first.second; - shift->next_state = i.second->id; - } - } - - size_t next_bind_state = 0; - if (state->bind_state != NULL) { - next_bind_state = state->bind_state->id; - } - - auto reductions = vector::alloc(state->completed.size()); - for (size_t i = 0; i < state->completed.size(); i++) { - Item *item = state->completed[i]; - ref reduction = reductions.elem(i); - reduction->lin_obj = item->lin_obj; - reduction->seq_idx = item->seq_idx; - reduction->depth = item->stk_size; - - auto args = vector::alloc(item->args.count); - for (size_t j = 0; j < item->args.count; j++) { - ref arg = 0; - if (item->args[j].ccat != NULL) { - arg = item->args[j].ccat->persist(); - } - args[j].arg = arg; - args[j].stk_idx = item->args[j].stk_idx; - } - reduction->args = args; - } - - ref lrstate = lrtable.elem(state->id); - lrstate->shifts = shifts; - lrstate->tokens = tokens; - lrstate->next_bind_state = next_bind_state; - lrstate->reductions = reductions; - } - return lrtable; -} - -PgfLCTableMaker::PgfLCTableMaker(ref abstr, ref concr) -{ - this->abstr = abstr; - this->concr = concr; -} - -PgfLCTableMaker::~PgfLCTableMaker() -{ -} - -static bool edge_match(ref edge1, ref edge2) -{ - size_t sz1 = sizeof(PgfLCEdge) + sizeof(term)*edge1->n_terms + sizeof(PgfVariableRange)*edge1->vars.size(); - size_t sz2 = sizeof(PgfLCEdge) + sizeof(term)*edge2->n_terms + sizeof(PgfVariableRange)*edge2->vars.size(); - - if (sz1 != sz2) + if (value < lparam->i0) return false; - return (memcmp(&*edge1,&*edge2,sz1) == 0); -} + value -= lparam->i0; -int comp (const void * elem1, const void * elem2) -{ - int f = *((int*)elem1); - int s = *((int*)elem2); - if (f > s) return 1; - if (f < s) return -1; - return 0; -} - -void PgfLCTableMaker::rename(ref edge) -{ - size_t next_var = 0; - std::map subst; - for (size_t i = 0; i < edge->n_terms; i++) { - auto it = subst.find(edge->terms[i].var); - if (it == subst.end()) { - subst[edge->terms[i].var] = next_var; - edge->terms[i].var = next_var++; - } else { - edge->terms[i].var = it->second; - } - } - - for (size_t i = 0; i < edge->vars.size(); i++) { - edge->vars[i].var = subst[edge->vars[i].var]; - } - qsort (&edge->vars[0], edge->vars.size(), sizeof(PgfVariableRange), comp); -} - -void PgfLCTableMaker::add_edge(ref edge) -{ - bool found = false; - for (ref xedge : forwards[edge->from.lincat]) { - if (edge_match(edge,xedge)) { - found = true; - break; - } - } - - if (!found) { - print_edge(edge); - forwards[edge->from.lincat].push_back(edge); - backwards[edge->to.lincat].push_back(edge); - update_closure(edge); - } -} - -void PgfLCTableMaker::update_closure(ref edge) -{ - auto &incoming = backwards[edge->from.lincat]; - size_t n_incoming = incoming.size(); - for (size_t i = 0; i < n_incoming; i++) { - ref xedge = compute_unifier(incoming[i],edge); - if (xedge != 0) { - rename(xedge); - add_edge(xedge); - } - } - - auto &outgoing = forwards[edge->to.lincat]; - size_t n_outgoing = outgoing.size(); - for (size_t i = 0; i < n_outgoing; i++) { - ref xedge = compute_unifier(edge,outgoing[i]); - if (xedge != 0) { - rename(xedge); - add_edge(xedge); - } - } -} - -typedef std::pair> Param; -typedef std::map Subst; - -template -bool unifier_helper1(Subst &subst1, V &vars1, T &to, - Subst &subst2, V &vars2, F &from) -{ - size_t i01t = to.i0; - size_t i02f = from.i0; - - size_t i = 0, j = 0; - while (i < to.size() && j < from.size()) { - size_t factor1 = to[i].factor; - size_t range1 = 0; - for (size_t k = 0; k < vars1.size(); k++) { - if (vars1[k].var == to[i].var) { - range1 = vars1[k].range; + for (size_t j = 0; j < lparam->n_terms; j++) { + term t = lparam->terms[j]; + for (size_t k = 0; k < vars.size(); k++) { + if (rule->vars[k].var == t.var) { + if (vars[k] > 0) { + if (value < vars[k]-1) + return false; + value -= vars[k]-1; + } break; } } - size_t value1 = factor1*range1; + } - size_t factor2 = from[j].factor; - size_t range2 = 0; - for (size_t k = 0; k < vars2.size(); k++) { - if (vars2[k].var == from[j].var) { - range2 = vars2[k].range; + for (size_t j = 0; j < lparam->n_terms; j++) { + term t = lparam->terms[j]; + for (size_t k = 0; k < vars.size(); k++) { + if (rule->vars[k].var == t.var) { + if (vars[k] == 0) { + size_t v_val = value / t.factor; + if (v_val >= rule->vars[k].range) + return false; + vars[k] = v_val + 1; + value %= t.factor; + } break; } } - size_t value2 = factor2*range2; + } - if (value1 > value2) { - size_t x = i02f / factor1; - if (x >= range1) - return false; - auto &s = subst1[to[i].var]; - s.first = i02f / factor1; - s.second.clear(); - i02f %= factor1; - while (j < from.size() && factor2 % factor1 == 0) { - size_t factor = factor2 / factor1; - s.second.emplace_back(); - s.second.back().factor=factor; - s.second.back().var=subst2[from[j].var].second[0].var; + return (value == 0); +} + +bool PgfAbstractParser::Item::instantiate(ref lparam,ref value,Item *other) +{ + size_t i = 0; + size_t i0_lparam = lparam->i0; + + size_t j = 0; + size_t i0_value = value->i0; + + while (i < lparam->n_terms && j < value->n_terms) { + size_t max_lparam = 0, k_lparam = 0; + while (i < lparam->n_terms) { + for (k_lparam = 0; k_lparam < this->rule->vars.size(); k_lparam++) { + if (this->rule->vars[k_lparam].var == lparam->terms[i].var) { + break; + } + } + if (this->vars[k_lparam] > 0) { + i0_lparam += lparam->terms[i].factor * (this->vars[k_lparam]-1); + i++; + } else { + max_lparam = lparam->terms[i].factor * this->rule->vars[k_lparam].range; + break; + } + } + + size_t max_value = 0, k_value = 0; + while (j < value->n_terms) { + for (k_value = 0; k_value < other->rule->vars.size(); k_value++) { + if (other->rule->vars[k_value].var == value->terms[j].var) { + break; + } + } + if (other->vars[k_value] > 0) { + i0_lparam += value->terms[j].factor * (other->vars[k_value]-1); j++; - factor2 = from[j].factor; + } else { + max_value = value->terms[j].factor * other->rule->vars[k_value].range; + break; } + } + + if (max_lparam > max_value) { + this->vars[k_lparam] = i0_value / this->rule->vars[k_lparam].range; + i0_value = i0_value % this->rule->vars[k_lparam].range; i++; } else { - size_t x = i01t / factor2; - if (x >= range2) - return false; - auto &s = subst2[from[j].var]; - s.first = i01t / factor2; - s.second.clear(); - i01t %= factor2; - while (i < to.size() && factor1 % factor2 == 0) { - size_t factor = factor1 / factor2; - s.second.emplace_back(); - s.second.back().factor=factor; - s.second.back().var=subst1[to[i].var].second[0].var; - i++; - factor1 = to[i].factor; - } + //other->vars[k_value] = i0_lparam / other->rule->vars[k_value].range; + i0_lparam = i0_lparam % other->rule->vars[k_value].range; j++; } } - while (i < to.size()) { - auto &s = subst1[to[i].var]; - size_t factor1 = to[i].factor; - s.first = i02f / factor1; - s.second.clear(); - i02f %= factor1; - i++; - } - - while (j < from.size()) { - auto &s = subst2[from[j].var]; - size_t factor2 = from[j].factor; - s.first = i01t / factor2; - s.second.clear(); - i01t %= factor2; - j++; - } - - return (i01t == i02f); + return (i0_lparam == i0_value); } -template -void unifier_helper2(Subst &subst, std::map &vars, std::map &ranges, A &v, Param &p) +void PgfAbstractParser::bu_predict(PgfPhrasetable phrasetable, + State *state, CCat *ccat) { - for (size_t i = 0; i < v.size(); i++) { - auto &s = subst[v[i].var]; - size_t factor = v[i].factor; - p.first += factor * s.first; - for (term &t : s.second) { - p.second.emplace_back(); - p.second.back().factor = factor * t.factor; - p.second.back().var = t.var; - vars[t.var] = ranges[t.var]; + if (phrasetable == 0) { + return; + } + + int cmp; + uint8_t tag = ref::get_tag(phrasetable->sym); + if (PgfSymbolACat::tag != tag) { + cmp = ((int) PgfSymbolACat::tag) - ((int) tag); + } else { + auto symcf = ref::untagged(phrasetable->sym); + cmp = textcmp(&ccat->cont->lincat->name, &symcf->name); + } + if (cmp < 0) { + bu_predict(phrasetable->left,state,ccat); + } else if (cmp > 0) { + bu_predict(phrasetable->right,state,ccat); + } else { + for (size_t i = 0; i < phrasetable->n_items; i++) { + auto new_item = bu_item(ccat->cont->state, phrasetable->items[i]); + combine(state,new_item,ccat); } } } -ref PgfLCTableMaker::compute_unifier(ref edge1, ref edge2) +PgfAbstractParser::Item *PgfAbstractParser::bu_item(State *state, ref pitem) { - std::map>> subst1, subst2; - std::map vars, ranges; + Item *item = NULL; - size_t next_var = 0; - for (size_t i = 0; i < edge1->vars.size(); i++) { - ranges[next_var] = edge1->vars[i].range; + switch (ref::get_tag(pitem->rule->container)) { + case PgfConcrLin::tag: { + auto lin = ref::untagged(pitem->rule->container); - auto &s = subst1[edge1->vars[i].var]; - s.second.emplace_back(); - s.second.back().factor = 1; - s.second.back().var = next_var++; + Cont *&cont = state->conts1[lin->lincat]; + if (cont == NULL) { + cont = new Cont; + cont->ccat = NULL; + cont->lincat = lin->lincat; + cont->state = state; + } + + item = new(pitem->rule) Item; + item->cont = cont; + item->pre_alt = pitem->pre_alt; + item->pre_dot = pitem->pre_dot; + item->dot = pitem->dot; + item->syms = pitem->rule->syms.as_vector(); + item->rule = pitem->rule; + break; } - for (size_t i = 0; i < edge2->vars.size(); i++) { - ranges[next_var] = edge2->vars[i].range; + case PgfConcrLincat::tag: { + auto lincat = ref::untagged(pitem->rule->container); - auto &s = subst2[edge2->vars[i].var]; - s.second.emplace_back(); - s.second.back().factor = 1; - s.second.back().var = next_var++; + Cont *&cont = state->conts1[0]; + if (cont == NULL) { + cont = new Cont; + cont->ccat = NULL; + cont->lincat = 0; + cont->state = state; + } + + item = new(pitem->rule) Item; + item->cont = cont; + item->pre_alt = pitem->pre_alt; + item->pre_dot = pitem->pre_dot; + item->dot = pitem->dot; + item->syms = pitem->rule->syms.as_vector(); + item->rule = pitem->rule; + break; + } } - if (!unifier_helper1(subst1, edge1->vars, edge1->to.value, - subst2, edge2->vars, edge2->from.value)) - return 0; - if (!unifier_helper1(subst1, edge1->vars, edge1->to.lin_idx, - subst2, edge2->vars, edge2->from.lin_idx)) - return 0; + if (item->pre_alt > 0) { + auto symkp = ref::untagged(item->syms[item->pre_dot]); - Param p1fv,p1fi,p2tv,p2ti; - p1fv.first = edge1->from.value.i0; - p1fi.first = edge1->from.lin_idx.i0; - p2tv.first = edge2->to.value.i0; - p2ti.first = edge2->to.lin_idx.i0; + if (item->pre_alt == 1) + item->syms = symkp->default_form; + else + item->syms = symkp->alts[item->pre_alt-2].form; + } - unifier_helper2(subst1, vars, ranges, edge1->from.value, p1fv); - unifier_helper2(subst1, vars, ranges, edge1->from.lin_idx, p1fi); - unifier_helper2(subst2, vars, ranges, edge2->to.value, p2tv); - unifier_helper2(subst2, vars, ranges, edge2->to.lin_idx, p2ti); + memcpy(&item->vars[0], &pitem->vars[0], sizeof(size_t) * item->vars.size()); - ref edge = PgfLCEdge::alloc(p1fv.second.size(),p1fi.second.size(),p2tv.second.size(),p2ti.second.size(),vars.size()); - edge->from.lincat = edge1->from.lincat; - edge->from.value.i0 = p1fv.first; - for (size_t i = 0; i < p1fv.second.size(); i++) { - edge->from.value[i] = p1fv.second[i]; + for (size_t i = 0; i < pitem->args.size(); i++) { + ref arg = pitem->args[i]; + + item->args[i] = 0; + + if (arg != 0) { + Cont *&arg_cont = state->conts1[arg->lincat]; + if (arg_cont == NULL) { + arg_cont = new Cont; + arg_cont->ccat = NULL; + arg_cont->lincat = arg->lincat; + arg_cont->state = state; + } + item->args[i] = + td_epsilon(state, arg_cont, arg); + } } - edge->from.lin_idx.i0 = p1fi.first; - for (size_t i = 0; i < p1fi.second.size(); i++) { - edge->from.lin_idx[i] = p1fi.second[i]; - } - edge->to.lincat = edge2->to.lincat; - edge->to.value.i0 = p2tv.first; - for (size_t i = 0; i < p2tv.second.size(); i++) { - edge->to.value[i] = p2tv.second[i]; - } - edge->to.lin_idx.i0 = p2ti.first; - for (size_t i = 0; i < p2ti.second.size(); i++) { - edge->to.lin_idx[i] = p2ti.second[i]; - } - size_t i = 0; - for (auto it : vars) { - edge->vars[i].var = it.first; - edge->vars[i].range = it.second; - i++; - } -/* - if (strcmp(edge->to.lincat->name.text, "VP") == 0 && edge->to.value.i0 == 2 && edge->to.value.size() == 2) { - print_edge(edge1); - print_edge(edge2); - fprintf(stderr,"------------------\n"); - print_edge(edge); - fprintf(stderr,"\n"); - } -*/ - return edge; + + return item; } -void PgfLCTableMaker::print_edge(ref edge) +void PgfAbstractParser::combine(State *state, Item *item, CCat *ccat) { - PgfPrinter printer(NULL, 0, NULL); + PgfSymbol sym = item->rule->syms[item->dot]; + auto sym_cat = ref::untagged(sym); - if (edge->vars.size() > 0) { - printer.puts("{"); - for (size_t i = 0; i < edge->vars.size(); i++) { + if (!item->instantiate(item->rule->args[sym_cat->d],ccat->value)) { + // delete item; + return; + } + if (!item->instantiate(ref::from_ptr(&sym_cat->r),ccat->lin_idx)) { + // delete item; + return; + } + item->dot++; + item->args[sym_cat->d] = ccat; + + process(item, state->start, false); +} + +#ifdef DEBUG_PARSER +static +void print_symbols(PgfPrinter &printer, PgfConcrRule *rule, vector syms, size_t pre_alt, size_t pre_dot, size_t dot) +{ + for (size_t i = 0; i < syms.size(); i++) { + if (pre_alt == 0 && dot == i) { + printer.puts(" . "); + printer.symbol(syms[i]); + } else if (pre_alt > 0 && pre_dot == i) { + auto sym_kp = ref::untagged(rule->syms[pre_dot]); + + printer.puts("pre {"); + + if (pre_alt == 1) + print_symbols(printer, rule, syms, 0, 0, dot); + else + printer.symbols(sym_kp->default_form); + + for (size_t i = 0; i < sym_kp->alts.size(); i++) { + printer.puts("; "); + if (pre_alt-2 == i) + print_symbols(printer, rule, syms, 0, 0, dot); + else + printer.symbols(sym_kp->alts[i].form); + printer.puts(" /"); + for (size_t j = 0; j < sym_kp->alts[i].prefixes.size(); j++) { + printer.puts(" "); + printer.lstr(sym_kp->alts[i].prefixes[j]); + } + } + + printer.puts("}"); + } else { + printer.symbol(syms[i]); + } + } + if (pre_alt == 0 && dot >= syms.size()) + printer.puts(" . "); +} + +void PgfAbstractParser::print_item(Item *item, const PgfTextSpot &spot) +{ + PgfPrinter printer(NULL,0,NULL); + + printer.nprintf(32, "[%zd-%zd; ", item->cont ? item->cont->state->end.pos : 0, spot.pos); + + if (item->vars.size() > 0) { + printer.lvar_ranges(item->rule->vars, &item->vars[0]); + printer.puts(" "); + } + + if (item->cont) { + if (item->cont->ccat == NULL) { + printer.efun(&item->cont->lincat->name); + printer.puts("("); + printer.lparam(item->rule->res); + printer.puts(")"); + } else { + printer.emeta(item->cont->ccat->fid); + } + } + printer.puts(" -> "); + + switch (ref::get_tag(item->rule->container)) { + case PgfConcrLin::tag: { + auto lin = ref::untagged(item->rule->container); + printer.efun(&lin->name); + + printer.puts("["); + for (size_t i = 0; i < item->args.size(); i++) { if (i > 0) printer.puts(","); - printer.lvar(edge->vars[i].var); - printer.nprintf(32,"<%zu",edge->vars[i].range); - } - printer.puts("} "); - } - printer.efun(&edge->from.lincat->name); - printer.puts("("); - if (edge->from.value.i0 != 0 || edge->from.value.size() == 0) - printer.nprintf(32,"%ld",edge->from.value.i0); - for (size_t i = 0; i < edge->from.value.size(); i++) { - if (edge->from.value.i0 != 0 || i > 0) - printer.puts("+"); - if (edge->from.value[i].factor != 1) { - printer.nprintf(32,"%ld",edge->from.value[i].factor); - printer.puts("*"); - } - printer.lvar(edge->from.value[i].var); - } - printer.puts(","); - if (edge->from.lin_idx.i0 != 0 || edge->from.lin_idx.size() == 0) - printer.nprintf(32,"%ld",edge->from.lin_idx.i0); - for (size_t i = 0; i < edge->from.lin_idx.size(); i++) { - if (edge->from.lin_idx.i0 != 0 || i > 0) - printer.puts("+"); - if (edge->from.lin_idx[i].factor != 1) { - printer.nprintf(32,"%ld",edge->from.lin_idx[i].factor); - printer.puts("*"); - } - printer.lvar(edge->from.lin_idx[i].var); - } - printer.puts(") -> "); - - printer.efun(&edge->to.lincat->name); - printer.puts("("); - if (edge->to.value.i0 != 0 || edge->to.value.size() == 0) - printer.nprintf(32,"%ld",edge->to.value.i0); - for (size_t i = 0; i < edge->to.value.size(); i++) { - if (edge->to.value.i0 != 0 || i > 0) - printer.puts("+"); - if (edge->to.value[i].factor != 1) { - printer.nprintf(32,"%ld",edge->to.value[i].factor); - printer.puts("*"); - } - printer.lvar(edge->to.value[i].var); - } - printer.puts(","); - if (edge->to.lin_idx.i0 != 0 || edge->to.lin_idx.size() == 0) - printer.nprintf(32,"%ld",edge->to.lin_idx.i0); - for (size_t i = 0; i < edge->to.lin_idx.size(); i++) { - if (edge->to.lin_idx.i0 != 0 || i > 0) - printer.puts("+"); - if (edge->to.lin_idx[i].factor != 1) { - printer.nprintf(32,"%ld",edge->to.lin_idx[i].factor); - printer.puts("*"); - } - printer.lvar(edge->to.lin_idx[i].var); - } - printer.puts(")\n"); - - PgfText *text = printer.get_text(); - fputs(text->text, stderr); - free(text); -} - -vector PgfLCTableMaker::make() -{ - std::function)> f = - [this](ref lin) { - for (size_t seq_idx = 0; seq_idx < lin->seqs.size(); seq_idx++) { - size_t index = seq_idx / (lin->seqs.size() / lin->res.size()); - size_t n_args = (lin->args.size() / lin->res.size()); - ref res = lin->res[index]; - ref seq = lin->seqs[seq_idx]; - - if (seq->syms.size() > 0) { - PgfSymbol sym = seq->syms[0]; - switch (ref::get_tag(sym)) { - case PgfSymbolCat::tag: { - auto sym_cat = ref::untagged(sym); - size_t arg_idx = n_args * index + sym_cat->d; - ref arg = ref::from_ptr(&lin->args[arg_idx]); - - std::set vars; - for (size_t i = 0; i < res->param.n_terms; i++) { - vars.insert(res->param.terms[i].var); - } - for (size_t i = 0; i < arg->param->n_terms; i++) { - vars.insert(arg->param->terms[i].var); - } - for (size_t i = 0; i < sym_cat->r.n_terms; i++) { - vars.insert(sym_cat->r.terms[i].var); - } - - ref edge = - PgfLCEdge::alloc(res->param.n_terms,0,arg->param->n_terms,sym_cat->r.n_terms,vars.size()); - edge->from.lincat = lin->lincat; - edge->from.value.i0 = res->param.i0; - for (size_t i = 0; i < res->param.n_terms; i++) { - edge->from.value[i] = res->param.terms[i]; - } - edge->from.lin_idx.i0 = seq_idx % (lin->seqs.size() / lin->res.size()); - edge->to.lincat = - namespace_lookup(concr->lincats, &lin->absfun->type->hypos[sym_cat->d].type->name); - edge->to.value.i0 = arg->param->i0; - for (size_t i = 0; i < arg->param->n_terms; i++) { - edge->to.value[i] = arg->param->terms[i]; - } - edge->to.lin_idx.i0 = sym_cat->r.i0; - for (size_t i = 0; i < sym_cat->r.n_terms; i++) { - edge->to.lin_idx[i] = sym_cat->r.terms[i]; - } - size_t i = 0; - for (size_t var : vars) { - edge->vars[i].var = var; - for (size_t k = 0; k < res->vars.size(); k++) { - if (res->vars[k].var == var) { - edge->vars[i].range = res->vars[k].range; - break; - } - } - i++; - } - - rename(edge); - add_edge(edge); - } - break; - } - } + CCat *ccat = item->args[i]; + if (ccat == NULL) { + printer.efun(&lin->absfun->type->hypos[i].type->name); + printer.puts("("); + printer.lparam(item->rule->args[i]); + printer.puts(")"); + } else { + printer.emeta(ccat->fid); } - return true; - }; - namespace_iter(concr->lins, f); - -/* for (auto it : forwards) { - for (ref edge : it.second) { - print_edge(edge); } + printer.puts("]; "); + break; } -*/ - return 0; -} + case PgfConcrLincat::tag: { + auto lincat = ref::untagged(item->rule->container); + printer.puts("linref "); + printer.efun(&lincat->name); -struct PgfParser::Choice { - int fid; - std::vector prods; - std::vector states; - std::vector exprs; - - Choice(int fid) { - this->fid = fid; - } - - ~Choice(); -}; - -struct PgfParser::Production { - ref lin; - size_t index; - size_t n_args; - Choice *args[]; - - void *operator new(size_t size, ref lin, size_t index) { - size_t n_args = lin->args.size() / lin->res.size(); - Production *prod = (Production *) - malloc(size+sizeof(Choice*)*n_args); - prod->lin = lin; - prod->index = index; - prod->n_args = n_args; - for (size_t i = 0; i < n_args; i++) { - prod->args[i] = NULL; - } - return prod; - } - - Production() { - // If there is no constructor, GCC will zero the object, - // while it has already been initialized in the new operator. - } - - bool operator==(const Production& other) const { - if (lin != other.lin || index != other.index) - return false; - - for (size_t i = 0; i < n_args; i++) { - if (args[i] != other.args[i]) - return false; - } - - return true; - } - - void operator delete(void *p) { - free(p); - } -}; - -struct PgfParser::StackNode { - Stage *stage; - size_t state_id; - Choice *choice; - std::vector parents; - - StackNode(Stage *stage, size_t state_id) { - this->stage = stage; - this->state_id = state_id; - this->choice = NULL; - } -}; - -struct PgfParser::Stage { - Stage *next; - PgfTextSpot start; - PgfTextSpot end; - std::vector nodes; - - Stage(PgfTextSpot spot) { - next = NULL; - start = spot; - end = spot; - } - - ~Stage() { - for (StackNode *node : nodes) { - delete node; - } - } -}; - -struct PgfParser::ExprState { - prob_t prob; - - Choice *choice; - Production *prod; - size_t n_args; - PgfExpr expr; -}; - -struct PgfParser::ExprInstance { - PgfExpr expr; - prob_t prob; - - ExprInstance(PgfExpr expr, prob_t prob) { - this->expr = expr; - this->prob = prob; - } -}; - -PgfParser::Choice::~Choice() { - while (states.size() > 0) { - ExprState *state = states.back(); states.pop_back(); - delete state; - } - - for (Production *prod : prods) { - delete prod; - } -} - -#if defined(DEBUG_STATE_CREATION) || defined(DEBUG_AUTOMATON) || defined(DEBUG_PARSER) -void PgfParser::print_prod(Choice *choice, Production *prod) -{ - PgfPrinter printer(NULL, 0, m); - - printer.nprintf(32, "?%d -> ", choice->fid); - - ref type = prod->lin->absfun->type; - printer.puts(&prod->lin->name); - printer.nprintf(32,"/%zd[", prod->index); - PgfDBMarshaller m; - for (size_t i = 0; i < prod->n_args; i++) { - Choice *choice = prod->args[i]; - if (i > 0) - printer.puts(","); - if (choice == NULL) { - m.match_type(&printer, vector_elem(type->hypos, i)->type.as_object()); + printer.puts("["); + CCat *ccat = item->args[0]; + if (ccat == NULL) { + printer.efun(&lincat->name); + printer.puts("("); + printer.lparam(item->rule->args[0]); + printer.puts(")"); } else { - printer.nprintf(32, "?%d", choice->fid); + printer.emeta(ccat->fid); } + printer.puts("]; "); + break; } - printer.puts("]\n"); + } + + printer.lparam(item->rule->lin_idx); + printer.puts(" : "); + print_symbols(printer, item->rule, item->syms, item->pre_alt, item->pre_dot, item->dot); + printer.puts("]"); PgfText *text = printer.get_text(); - fputs(text->text, stderr); + fprintf(stderr, "%s\n", text->text); free(text); } -void PgfParser::print_transition(StackNode *source, StackNode *target, Stage *stage, ref shift) +void PgfAbstractParser::print_prod(CCat *ccat, Production *prod) { - PgfPrinter printer(NULL, 0, m); - printer.nprintf(64, "state %ld --- ", source->state_id); - if (target->choice != 0) { - printer.nprintf(32, "?%d", target->choice->fid); - } else if (shift != 0) { - size_t sym_idx = shift->sym_idx; - ref seq = shift->seq; - while (sym_idx < seq->syms.len) { - PgfSymbol sym = seq->syms.data[sym_idx]; - if (ref::get_tag(sym) != PgfSymbolKS::tag) - break; - printer.symbol(sym); - sym_idx++; - } - } else { - printer.puts("BIND"); + PgfPrinter printer(NULL,0,NULL); + + if (prod->vars.size() > 0) { + printer.lvar_ranges(prod->rule->vars, &prod->vars[0]); + printer.puts(" "); } - printer.nprintf(80, " ---> state %ld (position %zu-%zu, nodes %zu)\n", - target->state_id, - stage->start.pos, stage->end.pos, stage->nodes.size()); + + printer.emeta(ccat->fid); + printer.puts(" -> "); + + switch (ref::get_tag(prod->rule->container)) { + case PgfConcrLin::tag: { + auto lin = ref::untagged(prod->rule->container); + printer.efun(&lin->name); + + printer.puts("["); + for (size_t i = 0; i < prod->args.size(); i++) { + if (i > 0) + printer.puts(","); + + CCat *ccat = prod->args[i]; + if (ccat == NULL) { + printer.efun(&lin->absfun->type->hypos[i].type->name); + printer.puts("("); + printer.lparam(prod->rule->args[i]); + printer.puts(")"); + } else { + printer.emeta(ccat->fid); + } + } + printer.puts("]"); + break; + } + case PgfConcrLincat::tag: { + auto lincat = ref::untagged(prod->rule->container); + printer.puts("linref "); + printer.efun(&lincat->name); + + printer.puts("["); + CCat *ccat = prod->args[0]; + if (ccat == NULL) { + printer.efun(&lincat->name); + printer.puts("("); + printer.lparam(prod->rule->args[0]); + printer.puts(")"); + } else { + printer.emeta(ccat->fid); + } + printer.puts("]"); + break; + } + } + PgfText *text = printer.get_text(); - fputs(text->text, stderr); + fprintf(stderr, "%s\n", text->text); free(text); } #endif -PgfParser::PgfParser(ref concr, ref start, PgfText *sentence, bool case_sensitive, PgfMarshaller *m, PgfUnmarshaller *u) +PgfParser::PgfParser(ref concr, PgfText *sentence, bool case_sensitive, PgfMarshaller *m, PgfUnmarshaller *u) + : PgfAbstractParser(concr) { - this->concr = concr; - this->sentence = sentence; - this->case_sensitive = case_sensitive; this->m = m; this->u = u; - this->last_fid = 0; - this->top_choice = NULL; - this->top_choice_index = 0; - - PgfTextSpot spot; - spot.pos = 0; - spot.ptr = (uint8_t*) sentence->text; - - this->before = new Stage(spot); - this->after = NULL; - this->ahead = NULL; - - StackNode *node = new StackNode(before, 0); - this->before->nodes.push_back(node); + this->sentence = sentence; + this->end = (uint8_t *) (sentence->text+sentence->size); + this->case_sensitive = case_sensitive; } -bool PgfParser::shift(StackNode *parent, ref lincat, size_t r, Production *prod, - Stage *before, Stage *after) +PgfParser::~PgfParser() { - vector shifts = concr->lrtable[parent->state_id].shifts; - for (size_t i = 0; i < shifts.size(); i++) { - ref shift = shifts.elem(i); - if (lincat == shift->lincat && r == shift->r) { - StackNode *node = NULL; - for (StackNode *n : after->nodes) { - if (n->stage == before && n->state_id == shift->next_state) { - node = n; - break; - } - } - if (node == NULL) { - node = new StackNode(before, shift->next_state); - node->choice = new Choice(++last_fid); - dynamic.push_back(node->choice); - after->nodes.push_back(node); - } - - bool added = true; - for (Production *other : node->choice->prods) { - if (*prod == *other) { - added = false; - break; - } - } - - if (added) { - node->choice->prods.push_back(prod); -#ifdef DEBUG_PARSER - print_prod(node->choice, prod); -#endif - } - - if (std::find(node->parents.begin(), node->parents.end(), parent) == node->parents.end()) { - node->parents.push_back(parent); -#ifdef DEBUG_PARSER - print_transition(parent,node,after,0); -#endif - } - - return added; - } - } - - return false; -} - -void PgfParser::shift(StackNode *parent, Stage *before) -{ - vector shifts = concr->lrtable[parent->state_id].tokens; - if (shifts != 0) { - const uint8_t *sent_end = (const uint8_t *) &sentence->text[sentence->size]; - for (size_t i = 0; i < shifts.size(); i++) { - ref shift = shifts.elem(i); - PgfTextSpot spot = before->end; - size_t sym_idx = shift->sym_idx; - int cmp = - text_sequence_cmp(&spot, sent_end, - shift->seq, &sym_idx, - case_sensitive, SM_PARTIAL); - if (cmp == 0) { - start_matches(&spot, NULL); - - StackNode *node = NULL; - for (StackNode *n : after->nodes) { - if (n->stage == before && n->state_id == shift->next_state) { - node = n; - break; + State *state = first_state; + while (state != NULL) { + for (auto it1 : state->completed) { + for (auto it2 : it1.second) { + for (auto it3 : it2.second) { + for (ExprState *estate : it3.second->pending) { + if (estate->expr != 0) + u->free_ref(estate->expr); + } + for (ExprProb &ep : it3.second->exprs) { + u->free_ref(ep.expr); } } - if (node == NULL) { - node = new StackNode(before, shift->next_state); - node->choice = NULL; - after->nodes.push_back(node); - } - - if (std::find(node->parents.begin(), node->parents.end(), parent) == node->parents.end()) { - node->parents.push_back(parent); -#ifdef DEBUG_PARSER - print_transition(parent,node,after,shift); -#endif - } - - end_matches(&spot, NULL); } } + + state = state->next; } } -void PgfParser::shift(StackNode *parent, Stage *before, Stage *after) +void PgfParser::bu_predict(PgfPhrasetable phrasetable, + State *state, + ptrdiff_t min, ptrdiff_t max) { - size_t next_bind_state = concr->lrtable[parent->state_id].next_bind_state; - if (next_bind_state != 0) { - StackNode *node = NULL; - for (StackNode *n : after->nodes) { - if (n->stage == before && n->state_id == next_bind_state) { - node = n; - break; - } - } - if (node == NULL) { - node = new StackNode(before, next_bind_state); - node->choice = NULL; - after->nodes.push_back(node); - } - - if (std::find(node->parents.begin(), node->parents.end(), parent) == node->parents.end()) { - node->parents.push_back(parent); -#ifdef DEBUG_PARSER - print_transition(parent,node,after,0); -#endif - } - } -} - -PgfParser::Choice *PgfParser::intersect_choice(Choice *choice1, Choice *choice2, intersection_map &im) -{ - if (choice1 == NULL) - return choice2; - if (choice2 == NULL) - return choice1; - if (choice1 == choice2) - return choice1; - - std::pair key(choice1,choice2); - auto it = im.find(key); - if (it != im.end()) { - return it->second; - } - - Choice *choice = new Choice(++last_fid); - dynamic.push_back(choice); - im[key] = choice; - for (Production *prod1 : choice1->prods) { - for (Production *prod2 : choice2->prods) { - if (prod1->lin == prod2->lin && prod1->index == prod2->index) { - Production *prod = new(prod1->lin,prod1->index) Production(); - choice->prods.push_back(prod); - - for (size_t i = 0; i < prod->n_args; i++) { - Choice *arg = intersect_choice(prod1->args[i],prod2->args[i],im); - if (arg == NULL) { - //delete choice; - return NULL; - } - prod->args[i] = arg; - } - -#ifdef DEBUG_PARSER - print_prod(choice, prod); -#endif - } - } - } - - return choice; -} - -void PgfParser::reduce(StackNode *parent, ref lin, ref red, - size_t n, std::vector &args, - Stage *before, Stage *after) -{ - if (n == 0) { - ref lincat = lin->lincat; - - size_t index = red->seq_idx / lincat->fields.size(); - size_t r = red->seq_idx % lincat->fields.size(); - Production *prod = new(lin,index) Production(); - - for (size_t i = 0; i < prod->n_args; i++) { - auto arg = red->args.elem(i); - - if (arg->stk_idx > 0) { - Choice *choice = args[red->depth-arg->stk_idx]; - if (choice != NULL) { - intersection_map im; - choice = intersect_choice(choice, prod->args[i], im); - if (choice == NULL) { - //delete prod; - return; - } - } - prod->args[i] = choice; - } - if (arg->arg != 0) { - Choice *choice = retrieve_choice(arg->arg); - if (choice != NULL) { - intersection_map im; - choice = intersect_choice(choice, prod->args[i], im); - if (choice == NULL) { - //delete prod; - return; - } - } - prod->args[i] = choice; - } - } - - if (!shift(parent, lincat, r, prod, before, after)) { - delete prod; - } + if (phrasetable == 0) return; - } - args.push_back(parent->choice); - for (auto node : parent->parents) { - reduce(node, lin, red, n-1, args, parent->stage, after); - } - args.pop_back(); -} - -PgfParser::Choice *PgfParser::retrieve_choice(ref arg) -{ - if (arg == 0) - return NULL; - - Choice *&tmp = persistant[arg.tagged()]; - Choice *choice = tmp; - if (choice == NULL) { - tmp = new Choice(++last_fid); choice = tmp; - for (size_t i = 0; i < arg->n_prods; i++) { - Production *prod = new(arg->prods[i].lin, arg->prods[i].index) Production(); - for (size_t j = 0; j < prod->n_args; j++) { - auto child = arg->prods[i].args[j]; - prod->args[j] = retrieve_choice(child); - } - choice->prods.push_back(prod); -#ifdef DEBUG_PARSER - print_prod(choice, prod); -#endif - } - } - - return choice; -} - -void PgfParser::complete(StackNode *parent, ref lincat, size_t r, - size_t n, std::vector &args) -{ - if (n == 0) { - top_choice = args[0]; - return; - } - - args.push_back(parent->choice); - for (auto node : parent->parents) { - complete(node, lincat, r, n-1, args); - } - args.pop_back(); -} - -void PgfParser::reduce_all(StackNode *node) -{ - vector reductions = concr->lrtable[node->state_id].reductions; - for (size_t j = 0; j < reductions.size(); j++) { - ref red = reductions.elem(j); - switch (ref::get_tag(red->lin_obj)) { - case PgfConcrLin::tag: { - auto lin = - ref::untagged(red->lin_obj); - std::vector args; - reduce(node, lin, red, red->depth, args, before, before); - break; - } - case PgfConcrLincat::tag: { - auto lincat = - ref::untagged(red->lin_obj); - std::vector args; - if (before->end.pos == sentence->size) { - complete(node, lincat, red->seq_idx % lincat->fields.size(), red->depth, args); - } - } - } - } -} - -void PgfParser::space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err) -{ -#ifdef DEBUG_PARSER - fprintf(stderr, "------------------ position %zu-%zu ------------------\n", - start->pos, end->pos); -#endif - - while (ahead != NULL && ahead->start.pos <= start->pos) { - Stage *tmp = ahead->next; - ahead->next = before; - before = ahead; - ahead = tmp; - } - - before->end = *end; - - if (before->next != NULL && before->start.pos==before->end.pos) { - after = new Stage(*end); - after->next = before; - size_t i = 0; - while (i < before->nodes.size()) { - StackNode *node = before->nodes[i++]; - reduce_all(node); - shift(node, before, after); - } - before = after; - } - - size_t i = 0; - while (i < before->nodes.size()) { - StackNode *node = before->nodes[i++]; - reduce_all(node); - shift(node, before); - } -} - -void PgfParser::start_matches(PgfTextSpot *end, PgfExn* err) -{ - Stage **last = &ahead; after = *last; - while (after != NULL && after->start.pos < end->pos) { - last = &after->next; after = *last; - } - - if (after == NULL) { - *last = new Stage(*end); - after = *last; - } -} - -void PgfParser::match(ref lin, size_t seq_index, PgfExn* err) -{ - size_t index = seq_index / lin->lincat->fields.size(); - size_t r = seq_index % lin->lincat->fields.size(); - - for (StackNode *parent : before->nodes) { - Production *prod = new(lin,index) Production(); - if (!shift(parent, lin->lincat, r, prod, before, after)) { - delete prod; - } - } -} - -void PgfParser::end_matches(PgfTextSpot *end, PgfExn* err) -{ -} - -bool PgfParser::CompareExprState::operator() (const ExprState *state1, const ExprState *state2) const { - return state1->prob > state2->prob; -} - -void PgfParser::prepare() -{ - if (top_choice != NULL) - predict_expr_states(top_choice, 0); -} - -void PgfParser::predict_expr_states(Choice *choice, prob_t outside_prob) -{ - for (Production *prod : choice->prods) { - ExprState *state = new ExprState; - state->choice = choice; - state->prod = prod; - state->n_args = 0; - state->expr = u->efun(&prod->lin->name); - state->prob = outside_prob+prod->lin->absfun->prob; - exprs.push_back(state->expr); - queue.push(state); - } -} - -#ifdef DEBUG_GENERATOR -void PgfParser::print_expr_state_before(PgfPrinter *printer, ExprState *state) -{ - if (state->choice->states.size() > 0) { - ExprState *parent = state->choice->states[0]; - print_expr_state_before(printer, parent); - printer->puts(" ["); - } - m->match_expr(printer, state->expr); -} - -void PgfParser::print_expr_state_after(PgfPrinter *printer, ExprState *state) -{ - for (size_t i = state->n_args+1; i < state->prod->n_args; i++) { - if (state->prod->args[i] == NULL) - printer->puts(" ?"); - else - printer->nprintf(32, " ?%d", state->prod->args[i]->fid); - } - - if (state->choice->states.size() > 0) { - printer->puts("]"); - ExprState *parent = state->choice->states[0]; - print_expr_state_after(printer, parent); - } -} - -void PgfParser::print_expr_state(ExprState *state) -{ - PgfPrinter printer(NULL, 0, m); - - printer.nprintf(16, "[%f] ", state->prob); - print_expr_state_before(&printer, state); - if (state->n_args < state->prod->n_args) { - Choice *choice = state->prod->args[state->n_args]; - if (choice == NULL) - printer.puts(" ?"); - else - printer.nprintf(32, " ?%d", state->prod->args[state->n_args]->fid); - } - print_expr_state_after(&printer, state); - printer.puts("\n"); - - PgfText *text = printer.get_text(); - fputs(text->text, stderr); - free(text); -} -#endif - -bool PgfParser::process_expr_state(ExprState *state) -{ - if (state->n_args >= state->prod->n_args) { - complete_expr_state(state); - return true; - } - - Choice *choice = state->prod->args[state->n_args]; - if (choice == NULL) { - PgfExpr meta = u->emeta(0); - PgfExpr app = u->eapp(state->expr, meta); - exprs.push_back(app); - u->free_ref(meta); - state->expr = app; - state->n_args++; - queue.push(state); + PgfTextSpot current = state->end; + int cmp; + if (state->needs_bind) { + uint8_t tag = ref::get_tag(phrasetable->sym); + cmp = ((int) PgfSymbolBIND::tag) - ((int) tag); } else { - choice->states.push_back(state); + cmp = text_symbol_cmp(¤t,end,phrasetable->sym,case_sensitive); + } + if (cmp < 0) { + bu_predict(phrasetable->left,state,min,max); + } else if (cmp > 0) { + ptrdiff_t len = current.ptr - state->end.ptr; - if (choice->states.size() == 1) { - predict_expr_states(choice, state->prob); - } else { - for (ExprInstance p : choice->exprs) { - combine_expr_state(state,p); + if (min <= len-1) + bu_predict(phrasetable->left,state,min,len-1); + + if (len <= max) + bu_predict(phrasetable->right,state,len,max); + } else { + ptrdiff_t len = current.ptr - state->end.ptr; + + if (min <= len) + bu_predict(phrasetable->left,state,min,len); + + if (len > 0) { + for (size_t i = 0; i < phrasetable->n_items; i++) { + Item *item = bu_item(state, phrasetable->items[i]); + item->dot++; + if (item != NULL) + process(item, current, false); + } + } + + if (len <= max) + bu_predict(phrasetable->right,state,len,max); + } +} + +void PgfParser::make_chunks(State *state, std::vector &chunks, prob_t prob) +{ + if (state->completed.size() == 0) { + ExprState *estate = new(chunks.size()) ExprState; + estate->expr = u->emeta(0); + estate->prob = prob; + estate->hash = '?'; + estate->res = NULL; + estate->index = 0; + estate->n_args = chunks.size(); + for (size_t i = 0; i < estate->n_args; i++) { + estate->args[i] = chunks[estate->n_args-i-1]; + } + queue.push_back(estate); + std::push_heap(queue.begin(), queue.end(), estate_comp); + } + + for (auto it1 : state->completed) { + for (auto it2 : it1.second) { + for (auto it3 : it2.second) { + CCat *ccat = it3.second; + if (!ccat->covered && ccat->cont->state != state) { + chunks.push_back(ccat); + make_chunks(ccat->cont->state, chunks, prob+ccat->cont->lincat->abscat->prob); + chunks.pop_back(); + } } } } - - return false; } -void PgfParser::complete_expr_state(ExprState *state) +void PgfParser::prepare(ref start) { - Choice *choice = state->choice; + PgfTextSpot start_spot = {0, (uint8_t *) sentence->text}; + State *state = new_state(start_spot); + state->needs_bind = false; + current_state = state; - prob_t outside_prob; - if (choice == top_choice) - outside_prob = 0; - else - outside_prob = choice->states[0]->prob; - - prob_t inside_prob = state->prob-outside_prob; - choice->exprs.emplace_back(state->expr,inside_prob); - for (ExprState *state : choice->states) { - combine_expr_state(state,choice->exprs.back()); + for (size_t i = start->n_lindefs; i < start->rules.size(); i++) { + ref rule = start->rules[i]; + Item *item = new(rule) Item; + item->cont = NULL; + item->dot = 0; + item->pre_alt = 0; + item->pre_dot = 0; + item->syms = rule->syms.as_vector(); + item->rule = rule; + process(item, start_spot, false); } -} -void PgfParser::combine_expr_state(ExprState *state, ExprInstance &inst) -{ - PgfExpr app = u->eapp(state->expr, inst.expr); - exprs.push_back(app); + while (current_state != NULL) { + bu_predict(concr->phrasetable, current_state, 1, sentence->size); + state = current_state; + current_state = current_state->next; + } - ExprState *app_state = new ExprState(); - app_state->prob = state->prob + inst.prob; - app_state->choice = state->choice; - app_state->prod = state->prod; - app_state->n_args = state->n_args+1; - app_state->expr = app; - queue.push(app_state); + if (queue.size() == 0) { + std::vector chunks; + make_chunks(state, chunks, 0); + } } PgfExpr PgfParser::fetch(PgfDB *db, prob_t *prob) { DB_scope scope(db, READER_SCOPE); - if (top_choice == NULL) - return 0; + while (queue.size() > 0) { + ExprState *estate = queue.front(); + std::pop_heap(queue.begin(), queue.end(), estate_comp); + queue.pop_back(); - for (;;) { - if (top_choice_index < top_choice->exprs.size()) { - auto inst = top_choice->exprs[top_choice_index++]; - *prob = inst.prob; - return inst.expr; - } - - if (queue.empty()) - return 0; - - ExprState *state = queue.top(); queue.pop(); -#ifdef DEBUG_GENERATOR - print_expr_state(state); +#ifdef DEBUG_EXPRS + print_expr_state(m, estate); #endif - if (process_expr_state(state)) { - delete state; - } + PgfExpr expr = process_expr(estate, prob); + if (expr != 0) + return expr; } - return 0; } -PgfParser::~PgfParser() +PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob) { - while (before != NULL) { - Stage *tmp = before; - before = before->next; - delete tmp; + if (estate->index < estate->n_args) { + CCat *ccat = estate->args[estate->index]; + + if (ccat == NULL) { + ExprState *app_state = new(estate->n_args) ExprState; + app_state->expr = estate->expr ? u->eapp(estate->expr, u->emeta(0)) : u->emeta(0); + app_state->prob = estate->prob; + app_state->hash = estate->hash * 101 + '?'; + app_state->res = estate->res; + app_state->index = estate->index+1; + app_state->n_args = estate->n_args; + for (size_t i = 0; i < app_state->n_args; i++) { + app_state->args[i] = estate->args[i]; + } + queue.push_back(app_state); + std::push_heap(queue.begin(), queue.end(), estate_comp); + } else { + ccat->pending.push_back(estate); + + if (ccat->pending.size() == 1) { + for (Production *prod : ccat->prods) { + auto lin = ref::untagged(prod->rule->container); + + ExprState *new_estate = new(prod->args.size()) ExprState; + new_estate->expr = u->efun(&lin->name); + new_estate->prob = estate->prob+lin->absfun->prob; + new_estate->hash = 0; + new_estate->res = ccat; + new_estate->index = 0; + new_estate->n_args = prod->args.size(); + for (size_t i = 0; i < lin->name.size; i++) { + new_estate->hash = new_estate->hash * 101 + lin->name.text[i]; + } + for (size_t i = 0; i < new_estate->n_args; i++) { + new_estate->args[i] = prod->args[i]; + } + queue.push_back(new_estate); + std::push_heap(queue.begin(), queue.end(), estate_comp); + } + } else { + for (ExprProb ep : ccat->exprs) { + ExprState *app_state = new(estate->n_args) ExprState; + app_state->expr = estate->expr ? u->eapp(estate->expr, ep.expr) : ep.expr; + app_state->prob = estate->prob+ep.prob; + app_state->hash = estate->hash * 31 + ep.hash; + app_state->res = estate->res; + app_state->index = estate->index+1; + app_state->n_args= estate->n_args; + for (size_t i = 0; i < app_state->n_args; i++) { + app_state->args[i] = estate->args[i]; + } + queue.push_back(app_state); + std::push_heap(queue.begin(), queue.end(), estate_comp); + } + } + } + } else { + if (estate->res == NULL) { + *prob = estate->prob; + return estate->expr; + } + + prob_t prob = estate->prob - estate->res->pending[0]->prob; + for (size_t i = estate->res->exprs.size(); i > 0; i--) { + ExprProb &ep = estate->res->exprs[i-1]; + if (ep.prob != prob) + break; + if (ep.hash == estate->hash) + return 0; + } + + estate->res->exprs.emplace_back(estate->expr, prob, estate->hash); + for (ExprState *parent : estate->res->pending) { + ExprState *app_state = new(parent->n_args) ExprState; + app_state->expr = parent->expr ? u->eapp(parent->expr, estate->expr) : estate->expr; + app_state->prob = parent->prob+estate->prob; + app_state->hash = parent->hash * 31 + estate->hash; + app_state->res = parent->res; + app_state->index = parent->index+1; + app_state->n_args= parent->n_args; + for (size_t i = 0; i < app_state->n_args; i++) { + app_state->args[i] = parent->args[i]; + } + queue.push_back(app_state); + std::push_heap(queue.begin(), queue.end(), estate_comp); + } + } + return 0; +} + +PgfAbstractParser::State *PgfParser::new_state(const PgfTextSpot &start) +{ + State **prev = &first_state; + State *state = current_state; + while (state != NULL && state->start.ptr <= start.ptr) { + if (state->start.ptr == start.ptr) + return state; + prev = &state->next; + state = state->next; } - while (ahead != NULL) { - Stage *tmp = ahead; - ahead = ahead->next; - delete tmp; + state = new State; + state->start = start; + state->end = start; + state->next = *prev; + *prev = state; + + while (state->end.ptr < end) { + const uint8_t *ptr = state->end.ptr; + uint32_t ucs = pgf_utf8_decode(&ptr); + if (!pgf_utf8_is_space(ucs)) + break; + state->end.pos++; + state->end.ptr = ptr; } - for (auto it : persistant) { - delete it.second; + state->needs_bind = (state->start.pos == state->end.pos); + + return state; +} + +void PgfParser::symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym) +{ + PgfTextSpot next = spot; + + const uint8_t *start = next.ptr; + for (;;) { + const uint8_t *ptr = next.ptr; + uint32_t ucs = pgf_utf8_decode(&ptr); + if (!pgf_utf8_is_space(ucs)) + break; + next.ptr = ptr; + next.pos++; } - for (Choice *choice : dynamic) { - delete choice; + if (bind != (spot.ptr == next.ptr)) + return; + + if (text_symbol_cmp(&next,end,sym,case_sensitive) != 0) + return; + + item->dot++; + process(item, next, false); +} + +void PgfParser::symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym) +{ + item->dot++; + process(item, spot, true); +} + +PgfAbstractParser::CCat *PgfAbstractParser::td_epsilon(State *state, Cont *cont, ref arg) +{ + CCat *&ccat = state->completed[cont][arg->value][arg->lin_idx]; + if (ccat == NULL) { + ccat = new CCat; + ccat->fid = (++last_fid); + ccat->cont = cont; + ccat->state = state; + ccat->lin_idx = arg->lin_idx; + ccat->value = arg->value; + ccat->covered = true; + +#ifdef DEBUG_PARSER + { + PgfPrinter printer(NULL,0,NULL); + printer.nprintf(64,"[%zd-%zd; ",cont->state->end.pos,state->start.pos); + printer.efun(&ccat->cont->lincat->name); + printer.nprintf(64,"(%zd); %zd; ",ccat->value,ccat->lin_idx); + printer.emeta(ccat->fid); + printer.puts("]"); + PgfText *text = printer.get_text(); + fprintf(stderr, "%s\n", text->text); + free(text); + } +#endif + + size_t n_items = 0; + vector> items = + phrasetable_lookup(concr->phrasetable, arg.tagged(), &n_items); + + for (size_t i = 0; i < n_items; i++) { + ref pitem = items[i]; + + Production *prod = new (pitem) Production; + prod->rule = pitem->rule; + memcpy(&prod->vars[0], &pitem->vars[0], sizeof(size_t) * prod->vars.size()); + + for (size_t j = 0; j < pitem->args.size(); j++) { + ref arg = pitem->args[j]; + + prod->args[j] = 0; + + if (arg != 0) { + Cont *&arg_cont = state->conts1[arg->lincat]; + if (arg_cont == NULL) { + arg_cont = new Cont; + arg_cont->ccat = NULL; + arg_cont->lincat = arg->lincat; + arg_cont->state = state; + } + prod->args[j] = + td_epsilon(state, arg_cont, arg); + } + } + +#ifdef DEBUG_PARSER + print_prod(ccat, prod); +#endif + ccat->prods.push_back(prod); + } } - for (PgfExpr expr : exprs) { - u->free_ref(expr); + return ccat; +} + +PgfAbstractParser::CCat *PgfAbstractParser::td_epsilon(State *state, Cont *cont, ref arg, + size_t n_items, vector> items) +{ + CCat *&ccat = state->completed[cont][arg->value][arg->lin_idx]; + if (ccat == NULL) { + ccat = new CCat; + ccat->fid = (++last_fid); + ccat->cont = cont; + ccat->state = state; + ccat->lin_idx = arg->lin_idx; + ccat->value = arg->value; + ccat->covered = true; + +#ifdef DEBUG_PARSER + { + PgfPrinter printer(NULL,0,NULL); + printer.nprintf(64,"[%zd-%zd; ",cont->state->end.pos,state->start.pos); + printer.efun(&ccat->cont->lincat->name); + printer.nprintf(64,"(%zd); %zd; ",ccat->value,ccat->lin_idx); + printer.emeta(ccat->fid); + printer.puts("]"); + PgfText *text = printer.get_text(); + fprintf(stderr, "%s\n", text->text); + free(text); + } +#endif + + for (size_t i = 0; i < n_items; i++) { + ref pitem = items[i]; + + Production *prod = new (pitem) Production; + prod->rule = pitem->rule; + memcpy(&prod->vars[0], &pitem->vars[0], sizeof(size_t) * prod->vars.size()); + + for (size_t j = 0; j < pitem->args.size(); j++) { + ref arg = pitem->args[j]; + + prod->args[j] = 0; + + if (arg != 0) { + Cont *&arg_cont = state->conts1[arg->lincat]; + if (arg_cont == NULL) { + arg_cont = new Cont; + arg_cont->ccat = NULL; + arg_cont->lincat = arg->lincat; + arg_cont->state = state; + } + prod->args[j] = + td_epsilon(state, arg_cont, arg); + } + } + +#ifdef DEBUG_PARSER + print_prod(ccat, prod); +#endif + ccat->prods.push_back(prod); + } } - while (!queue.empty()) { - ExprState *state = queue.top(); queue.pop(); - delete state; + return ccat; +} + +void PgfAbstractParser::td_predict(State *state, Cont *cont, Production *prod, size_t lin_idx) +{ + switch (ref::get_tag(prod->rule->container)) { + case PgfConcrLin::tag: { + auto lin = ref::untagged(prod->rule->container); + + for (ref rule : lin->rules) { + Item *item = new (rule) Item; + item->cont = cont; + item->dot = 0; + item->pre_alt = 0; + item->pre_dot = 0; + item->syms = rule->syms.as_vector(); + item->rule = rule; + + if (!item->instantiate(item->rule->res, cont->ccat->value)) { + // delete item; + continue; + } + + if (!item->instantiate(item->rule->lin_idx, lin_idx)) { + // delete item; + continue; + } + + for (size_t i = 0; i < item->args.size(); i++) { + if (prod->args[i] != NULL) { + if (!item->instantiate(item->rule->args[i], prod->args[i]->value)) { + // delete item; + goto next; + } + } else { + /*if (!item->instantiate(item->rule->args[i], prod->args[i]->value)) { + delete item; + continue; + }*/ + } + item->args[i] = prod->args[i]; + } + + process(item, state->start, false); + next:; + } + } + default:; + // should not happend + } +} + +void PgfParser::suspend(State *state,ref lincat,Item *item) +{ + Cont *&cont = state->conts1[lincat]; + if (cont == NULL) { + cont = new Cont; + cont->ccat = NULL; + cont->lincat = lincat; + cont->state = state; + } + + cont->suspended.push_back(item); + + if (cont->suspended.size() == 1) { + std::function,size_t,vector>)> f = + [this,state,item,cont](ref symcf, size_t n_items, vector> items) { + + Item *new_item = new (item) Item; + PgfSymbol sym = new_item->rule->syms[new_item->dot]; + auto sym_cat = ref::untagged(sym); + if (!new_item->instantiate(new_item->rule->args[sym_cat->d],symcf->value)) + return; + if (!new_item->instantiate(ref::from_ptr(&sym_cat->r),symcf->lin_idx)) + return; + + new_item->dot++; + new_item->args[sym_cat->d] = + td_epsilon(state,cont,symcf,n_items,items); + + process(new_item, state->start, false); + }; + phrasetable_iter(concr->phrasetable,lincat,f); + } +} + +void PgfParser::final_item(State *state, Item *item, size_t value, size_t lin_idx) +{ + if (item->cont == NULL && state->end.ptr == end) { + ExprState *estate = new(item->args.size()) ExprState; + estate->expr = 0; + estate->prob = 0; + estate->hash = 0; + estate->res = NULL; + estate->index = 0; + estate->n_args = item->args.size(); + for (size_t i = 0; i < estate->n_args; i++) { + estate->args[i] = item->args[i]; + } + queue.push_back(estate); + std::push_heap(queue.begin(), queue.end(), estate_comp); + } +} + +#ifdef DEBUG_EXPRS +void PgfParser::print_expr_state_left(PgfPrinter *printer, PgfMarshaller *m, ExprState *estate) +{ + if (estate->res && estate->res->pending.size() > 0) { + ExprState *parent = estate->res->pending[0]; + print_expr_state_left(printer, m, parent); + printer->puts(" ("); + } + + if (estate->expr) + m->match_expr(printer, estate->expr); + else + printer->puts("::"); +} + +void PgfParser::print_expr_state_right(PgfPrinter *printer, PgfMarshaller *m, ExprState *estate) +{ + for (size_t i = estate->index+1; i < estate->n_args; i++) { + printer->puts(" "); + if (estate->args[i] != NULL) + printer->emeta(estate->args[i]->fid); + else + printer->puts("?"); + } + + if (estate->res && estate->res->pending.size() > 0) { + printer->puts(")"); + ExprState *parent = estate->res->pending[0]; + print_expr_state_right(printer, m, parent); + } +} + +void PgfParser::print_expr_state(PgfMarshaller *m, ExprState *estate) +{ + PgfPrinter printer(NULL,0,m); + printer.nprintf(64,"[%f] ",estate->prob); + print_expr_state_left(&printer, m, estate); + printer.puts(" ."); + print_expr_state_right(&printer, m, estate); + + PgfText *text = printer.get_text(); + fprintf(stderr, "%s\n", text->text); + free(text); +} +#endif + +PgfParseTableMaker::PgfParseTableMaker(ref concr) + : PgfAbstractParser(concr) +{ + first_state = new State; + first_state->start.pos = 0; + first_state->start.ptr = NULL; + first_state->end = first_state->start; + first_state->next = NULL; + current_state = first_state; +} + +ref PgfParseTableMaker::clone_item(Item *item) +{ + size_t ex_size = + sizeof(ref) * item->args.size() + + sizeof(size_t) * item->vars.size(); + auto pitem = PgfDB::malloc(ex_size); + pitem->pre_alt = item->pre_alt; + pitem->pre_dot = item->pre_dot; + pitem->dot = item->dot; + pitem->rule = item->rule; + memcpy(&pitem->vars[0],&item->vars[0],sizeof(size_t) * item->vars.size()); + + for (size_t i = 0; i < item->args.size(); i++) { + ref symcf = 0; + if (item->args[i] != NULL) { + symcf = PgfDB::malloc(); + symcf->lincat = item->args[i]->cont->lincat; + symcf->value = item->args[i]->value; + symcf->lin_idx = item->args[i]->lin_idx; + } + pitem->args[i] = symcf; + } + + return pitem; +} + +PgfAbstractParser::State *PgfParseTableMaker::new_state(const PgfTextSpot &start) +{ + return this->first_state; +} + +void PgfParseTableMaker::symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym) +{ + auto pitem = clone_item(item); + auto phrasetable = phrasetable_insert(concr->phrasetable,sym,pitem); + concr->phrasetable = phrasetable; +} + +void PgfParseTableMaker::symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym) +{ + auto pitem = clone_item(item); + auto phrasetable = phrasetable_insert(concr->phrasetable,sym,pitem); + concr->phrasetable = phrasetable; +} + +void PgfParseTableMaker::suspend(State *state,ref lincat,Item *item) +{ + Cont *&cont = state->conts1[lincat]; + if (cont == NULL) { + cont = new Cont; + cont->ccat = NULL; + cont->lincat = lincat; + cont->state = state; + } + + cont->suspended.push_back(item); + + for (auto it1 : state->completed[cont]) { + for (auto it2 : it1.second) { + CCat *ccat = it2.second; + if (ccat != NULL) { + Item *new_item = new (item) Item; + combine(state,new_item,ccat); + } + } + } + + auto pitem = clone_item(item); + auto acat = ref::from_ptr((PgfSymbolACat*) &lincat->name); + auto phrasetable = phrasetable_insert(concr->phrasetable,acat.tagged(),pitem); + concr->phrasetable = phrasetable; +} + +void PgfParseTableMaker::final_item(State *state, Item *item, size_t value, size_t lin_idx) +{ + auto pitem = clone_item(item); + + PgfPhrasetable phrasetable = concr->phrasetable; + phrasetable = phrasetable_insert(phrasetable, + item->cont->lincat, value, lin_idx, + pitem); + concr->phrasetable = phrasetable; +} + +void PgfParseTableMaker::bu_predict(PgfPhrasetable phrasetable, State *state, CCat *ccat) +{ +} + +void PgfParseTableMaker::insert_rule(ref rule) +{ + switch (ref::get_tag(rule->container)) { + case PgfConcrLin::tag: { + auto lin = ref::untagged(rule->container); + + Cont *&cont = first_state->conts1[lin->lincat]; + if (cont == NULL) { + cont = new Cont; + cont->ccat = NULL; + cont->lincat = lin->lincat; + cont->state = first_state; + } + + Item *item = new(rule) Item; + item->cont = cont; + item->dot = 0; + item->pre_alt = 0; + item->pre_dot = 0; + item->syms = rule->syms.as_vector(); + item->rule = rule; + return process(item, first_state->end, false); + } } } diff --git a/src/runtime/c/pgf/parser.h b/src/runtime/c/pgf/parser.h index c0bbd4773..1bbb2aa49 100644 --- a/src/runtime/c/pgf/parser.h +++ b/src/runtime/c/pgf/parser.h @@ -1,191 +1,294 @@ #ifndef LR_TABLE_H #define LR_TABLE_H -#include "md5.h" - -class PGF_INTERNAL_DECL PgfLRTableMaker -{ - struct CCat; - struct Production; - struct Item; - struct State; - - struct CompareItem; - static const CompareItem compare_item; - - typedef std::pair,size_t> Key0; - - struct PGF_INTERNAL_DECL CompareKey0 : std::less { - bool operator() (const Key0& k1, const Key0& k2) const { - int cmp = textcmp(k1.first,k2.first); - if (cmp < 0) - return true; - else if (cmp > 0) - return false; - - return (k1.second < k2.second); - } - }; - - typedef std::pair,size_t> Key1; - - struct PGF_INTERNAL_DECL CompareKey1 : std::less { - bool operator() (const Key1& k1, const Key1& k2) const { - if (k1.first < k2.first) - return true; - else if (k1.first > k2.first) - return false; - - return (k1.second < k2.second); - } - }; - - typedef std::pair Key2; - - struct PGF_INTERNAL_DECL CompareKey2 : std::less { - bool operator() (const Key2& k1, const Key2& k2) const { - if (k1.first < k2.first) - return true; - else if (k1.first > k2.first) - return false; - - return (k1.second < k2.second); - } - }; - - typedef std::pair,size_t> Key3; - - struct PGF_INTERNAL_DECL CompareKey3 : std::less { - bool operator() (const Key3& k1, const Key3& k2) const; - }; - - ref abstr; - ref concr; - - size_t ccat_id; - size_t state_id; - - std::queue todo; - std::map states; - std::map ccats1; - std::map ccats2; - - // The Threefold Way of building an automaton - typedef enum { INIT, PROBE, REPEAT } Fold; - - void process(State *state, Fold fold, Item *item); - void symbol(State *state, Fold fold, Item *item, PgfSymbol sym); - - template - void predict(State *state, Fold fold, Item *item, T cat, - vector vars, PgfLParam *r); - void predict(State *state, Fold fold, Item *item, ref cat, size_t lin_idx); - void predict(State *state, Fold fold, Item *item, CCat *ccat, size_t lin_idx); - void predict(ref absfun, CCat *ccat); - void complete(State *state, Fold fold, Item *item); - - void print_production(CCat *ccat, Production *prod); - void print_item(Item *item); - - void internalize_state(State *&state); - -public: - PgfLRTableMaker(ref abstr, ref concr); - vector make(); - ~PgfLRTableMaker(); -}; - -class PGF_INTERNAL_DECL PgfLCTableMaker -{ - ref abstr; - ref concr; - - - std::map,std::vector>> forwards; - std::map,std::vector>> backwards; - - ref compute_unifier(ref edge1, ref edge2); - void update_closure(ref edge); - void rename(ref edge); - void add_edge(ref edge); - void print_edge(ref edge); - -public: - PgfLCTableMaker(ref abstr, ref concr); - vector make(); - ~PgfLCTableMaker(); -}; - class PgfPrinter; -class PGF_INTERNAL_DECL PgfParser : public PgfPhraseScanner, public PgfExprEnum +class PGF_INTERNAL_DECL PgfAbstractParser { - ref concr; - PgfText *sentence; - bool case_sensitive; - PgfMarshaller *m; - PgfUnmarshaller *u; + typedef size_t hash_t; - struct Choice; - struct Production; - struct StackNode; - struct Stage; +protected: + ref concr; + + struct CCat; + struct Cont; + struct Item; + struct State; struct ExprState; - struct ExprInstance; - struct CompareExprState : std::less { - bool operator() (const ExprState *state1, const ExprState *state2) const; + + struct Production { + ref rule; + + struct { + size_t &operator[](int i) { + Production *prod = containerof(Production,vars,this); + return ((size_t*) (((CCat**) (prod+1))+prod->args.size()))[i]; + } + size_t size() { + Production *prod = containerof(Production,vars,this); + return prod->rule->vars.size(); + } + } vars; + + struct { + CCat *&operator[](int i) { + Production *prod = containerof(Production,args,this); + return ((CCat**) (prod+1))[i]; + } + size_t size() { + Production *prod = containerof(Production,args,this); + return (prod->rule->args != 0) ? prod->rule->args.size() : 0; + } + } args; + + void *operator new(size_t sz, Item *item) + { + size_t sz2 = item->args.size()*sizeof(CCat*) + + item->vars.size()*sizeof(size_t); + Production *prod = (Production *) malloc(sz+sz2); + memcpy(prod+1, item+1, sz2); + return prod; + } + + void *operator new(size_t sz, ref pitem) + { + size_t sz2 = pitem->args.size()*sizeof(CCat*) + + pitem->vars.size()*sizeof(size_t); + Production *prod = (Production *) malloc(sz+sz2); + memset(prod+1,0,sz2); + return prod; + } + + void operator delete(void *p) + { + free(p); + } + + Production() { + } }; - Stage *before, *after, *ahead; - std::priority_queue, CompareExprState> queue; - int last_fid; + struct ExprProb { + PgfExpr expr; + prob_t prob; + hash_t hash; + + ExprProb(PgfExpr expr, prob_t prob, hash_t hash) { + this->expr = expr; + this->prob = prob; + this->hash = hash; + } + }; - std::vector dynamic; - std::map persistant; + struct CCat { + PgfMetaId fid; + Cont *cont; + State *state; + size_t value; + size_t lin_idx; + bool covered; + std::vector prods; + std::vector pending; + std::vector exprs; - std::vector exprs; + ~CCat(); + }; - Choice *top_choice; - size_t top_choice_index; + struct State { + PgfTextSpot start, end; + bool needs_bind; + std::map,Cont*> conts1; + std::map> conts2; + std::map>> completed; + State *next; + }; - bool shift(StackNode *parent, ref lincat, size_t r, Production *prod, - Stage *before, Stage *after); - void shift(StackNode *parent, Stage *before); - void shift(StackNode *parent, Stage *before, Stage *after); - void reduce(StackNode *parent, ref lin, ref red, - size_t n, std::vector &args, - Stage *before, Stage *after); - Choice *retrieve_choice(ref arg); - void complete(StackNode *parent, ref lincat, size_t r, - size_t n, std::vector &args); - void reduce_all(StackNode *state); - void print_prod(Choice *choice, Production *prod); - void print_transition(StackNode *source, StackNode *target, Stage *stage, ref shift); + struct Cont { + CCat *ccat; + ref lincat; + State *state; + std::vector suspended; - typedef std::map,Choice*> intersection_map; + ~Cont(); + }; - Choice *intersect_choice(Choice *choice1, Choice *choice2, intersection_map &im); + struct Item { + Cont *cont; + uint16_t pre_alt; + uint16_t pre_dot; + uint16_t dot; + vector syms; + ref rule; - void print_expr_state_before(PgfPrinter *printer, ExprState *state); - void print_expr_state_after(PgfPrinter *printer, ExprState *state); - void print_expr_state(ExprState *state); + struct { + size_t &operator[](int i) { + Item *item = containerof(Item,vars,this); + return ((size_t*) (((CCat**) (item+1))+item->args.size()))[i]; + } + size_t size() { + Item *item = containerof(Item,vars,this); + return item->rule->vars.size(); + } + } vars; - void predict_expr_states(Choice *choice, prob_t outside_prob); - bool process_expr_state(ExprState *state); - void complete_expr_state(ExprState *state); - void combine_expr_state(ExprState *state, ExprInstance &inst); + struct { + CCat *&operator[](int i) { + Item *item = containerof(Item,args,this); + return ((CCat**) (item+1))[i]; + } + size_t size() { + Item *item = containerof(Item,args,this); + return (item->rule->args != 0) ? item->rule->args.size() : 0; + } + } args; + + void *operator new(size_t sz, ref rule) + { + size_t sz2 = rule->args.size()*sizeof(CCat*) + + rule->vars.size()*sizeof(size_t); + Item *new_item = (Item *) malloc(sz+sz2); + memset(new_item+1, 0, sz2); + return new_item; + } + + void *operator new(size_t sz, Item *item) + { + size_t sz2 = item->args.size()*sizeof(CCat*) + + item->vars.size()*sizeof(size_t); + Item *new_item = (Item *) malloc(sz+sz2); + memcpy(new_item, item, sz+sz2); + return new_item; + } + + void operator delete(void *p) + { + free(p); + } + + Item() { + } + + bool instantiate(ref lparam,size_t value); + bool instantiate(ref lparam,ref value,Item *other); + }; + + struct ExprState { + PgfExpr expr; + prob_t prob; + hash_t hash; + + CCat *res; + + size_t index; + size_t n_args; + CCat *args[]; + + void *operator new(size_t sz, size_t n_args) + { + ExprState *estate = (ExprState *) + malloc(sz+n_args*sizeof(CCat*)); + return estate; + } + + void operator delete(void *p) + { + free(p); + } + + ExprState() { + } + }; + + State *first_state, *current_state; + PgfMetaId last_fid; + + void process(Item *item, const PgfTextSpot &spot, bool bind); + void symbol(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym); + void complete(Item *item, const PgfTextSpot &spot, bool bind); + + virtual State *new_state(const PgfTextSpot &start)=0; + virtual void symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym)=0; + virtual void symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym)=0; + virtual void suspend(State *state,ref lincat, Item *item)=0; + virtual void final_item(State *state,Item *item,size_t value,size_t lin_idx)=0; + + virtual void bu_predict(PgfPhrasetable phrasetable, State *state, CCat *ccat); + Item *bu_item(State *state, ref pitem); + CCat *td_epsilon(State *state, Cont *cont, ref arg); + CCat *td_epsilon(State *state, Cont *cont, ref arg, + size_t n_items, vector> items); + void td_predict(State *state, Cont *cont, Production *prod, size_t lin_idx); + void combine(State *state, Item *item, CCat *ccat); + + static + void print_item(Item *item, const PgfTextSpot &spot); + + static + void print_prod(CCat *ccat, Production *prod); public: - PgfParser(ref concr, ref start, PgfText *sentence, bool case_sensitive, PgfMarshaller *m, PgfUnmarshaller *u); + PgfAbstractParser(ref concr); + virtual ~PgfAbstractParser(); +}; + +class PGF_INTERNAL_DECL PgfParser : private PgfAbstractParser, public PgfExprEnum +{ + PgfMarshaller *m; + PgfUnmarshaller *u; + PgfText *sentence; + uint8_t *end; + bool case_sensitive; + + virtual State *new_state(const PgfTextSpot &start); + virtual void symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym); + virtual void symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym); + virtual void suspend(State *state,ref lincat, Item *item); + virtual void final_item(State *state,Item *item,size_t value,size_t lin_idx); + + void bu_predict(PgfPhrasetable phrasetable, State *state, ptrdiff_t min, ptrdiff_t max); + void make_chunks(State *state, std::vector &chunks, prob_t prob); + PgfExpr process_expr(ExprState *estate, prob_t *prob); + + static + void print_expr_state_left(PgfPrinter *printer, PgfMarshaller *m, ExprState *estate); + static + void print_expr_state_right(PgfPrinter *printer, PgfMarshaller *m, ExprState *estate); + static + void print_expr_state(PgfMarshaller *m, ExprState *estate); + + struct ExprStateComparator : std::less { + bool operator()(ExprState *estate1, ExprState *estate2) { + return estate1->prob > estate2->prob; + } + } estate_comp; + + std::vector queue; + +public: + PgfParser(ref concr, PgfText *sentence, bool case_sensitive, PgfMarshaller *m, PgfUnmarshaller *u); virtual ~PgfParser(); - virtual void space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err); - virtual void start_matches(PgfTextSpot *end, PgfExn* err); - virtual void match(ref lin, size_t seq_index, PgfExn* err); - virtual void end_matches(PgfTextSpot *end, PgfExn* err); - - void prepare(); + void prepare(ref start); PgfExpr fetch(PgfDB *db, prob_t *prob); }; + +class PGF_INTERNAL_DECL PgfParseTableMaker : private PgfAbstractParser +{ +private: + virtual State *new_state(const PgfTextSpot &start); + virtual void symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym); + virtual void symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym); + virtual void suspend(State *state,ref lincat,Item *item); + virtual void final_item(State *state,Item *item,size_t value,size_t lin_idx); + virtual void bu_predict(PgfPhrasetable phrasetable, State *state, CCat *ccat); + + static + ref clone_item(Item *item); + +public: + PgfParseTableMaker(ref concr); + void insert_rule(ref rule); +}; + #endif diff --git a/src/runtime/c/pgf/pgf.cxx b/src/runtime/c/pgf/pgf.cxx index 6a332a295..0f2eacb2d 100644 --- a/src/runtime/c/pgf/pgf.cxx +++ b/src/runtime/c/pgf/pgf.cxx @@ -985,10 +985,10 @@ public: { } - virtual void match(ref lin, size_t seq_index, PgfExn* err) + virtual void match(ref lin, size_t lin_idx, PgfExn* err) { ref field = - lin->lincat->fields[seq_index % lin->lincat->fields.size()]; + lin->lincat->fields[lin_idx]; callback->fn(callback, &lin->absfun->name, field, lin->lincat->abscat->prob+lin->absfun->prob, err); } @@ -1012,6 +1012,8 @@ void pgf_lookup_morpho(PgfDB *db, PgfConcrRevision cnc_revision, bool case_sensitive = pgf_is_case_sensitive(concr); PgfMorphoScanner scanner(callback); + size_t n_items; + phrasetable_lookup(concr->phrasetable, sentence, case_sensitive, &scanner, err); @@ -1069,24 +1071,22 @@ void pgf_lookup_cohorts(PgfDB *db, PgfConcrRevision cnc_revision, } PGF_API -PgfPhrasetableIds *pgf_iter_sequences(PgfDB *db, PgfConcrRevision cnc_revision, - PgfSequenceItor *itor, - PgfMorphoCallback *callback, - PgfExn *err) +void pgf_iter_sequences(PgfDB *db, PgfConcrRevision cnc_revision, + PgfSequenceItor *itor, + PgfMorphoCallback *callback, + PgfExn *err) { PGF_API_BEGIN { DB_scope scope(db, READER_SCOPE); ref concr = db->revision2concr(cnc_revision); - PgfPhrasetableIds *seq_ids = new PgfPhrasetableIds(); +/* PgfPhrasetableIds *seq_ids = new PgfPhrasetableIds(); seq_ids->start(concr); phrasetable_iter(concr, concr->phrasetable, itor, callback, seq_ids, err); - return seq_ids; + return seq_ids; */ } PGF_API_END - - return NULL; } PGF_API @@ -1095,7 +1095,7 @@ void pgf_get_lincat_counts_internal(object o, size_t *counts) ref lincat = o; counts[0] = lincat->fields.size(); counts[1] = lincat->n_lindefs; - counts[2] = lincat->res.size() - lincat->n_lindefs; + counts[2] = lincat->rules.size() - lincat->n_lindefs; } PGF_API @@ -1106,59 +1106,50 @@ PgfText *pgf_get_lincat_field_internal(object o, size_t i) } PGF_API -size_t pgf_get_lin_get_prod_count(object o) +size_t pgf_get_lin_rules_count(object o) { ref lin = o; - return lin->res.size(); + return lin->rules.size(); } PGF_API -PgfText *pgf_print_lindef_internal(PgfPhrasetableIds *seq_ids, object o, size_t i) +PgfText *pgf_print_lindef_internal(object o, size_t i) { ref lincat = o; PgfInternalMarshaller m; PgfPrinter printer(NULL,0,&m); - ref res = lincat->res[i]; - if (res->vars != 0) { - printer.lvar_ranges(res->vars, NULL); - printer.puts(" . "); + ref rule = lincat->rules[i]; + if (rule->vars != 0) { + printer.lvar_ranges(rule->vars, NULL); + printer.puts(" "); } - printer.efun(&lincat->name); printer.puts("("); - printer.lparam(ref::from_ptr(&res->param)); + printer.lparam(rule->res); printer.puts(") -> "); printer.efun(&lincat->name); - printer.puts("[String(0)] = ["); - - size_t n_seqs = lincat->fields.size(); - for (size_t j = 0; j < n_seqs; j++) { - if (j > 0) - printer.puts(","); - - ref seq = lincat->seqs[i*n_seqs + j]; - printer.seq_id(seq_ids, seq); - } - - printer.puts("]"); - + printer.puts("[String(0)]; "); + printer.lparam(rule->lin_idx); + printer.puts(" : "); + printer.symbols(rule->syms.as_vector()); return printer.get_text(); } PGF_API -PgfText *pgf_print_linref_internal(PgfPhrasetableIds *seq_ids, object o, size_t i) +PgfText *pgf_print_linref_internal(object o, size_t i) { ref lincat = o; PgfInternalMarshaller m; PgfPrinter printer(NULL,0,&m); - ref res = lincat->res[lincat->n_lindefs+i]; - if (res->vars != 0) { - printer.lvar_ranges(res->vars, NULL); - printer.puts(" . "); + ref rule = lincat->rules[lincat->n_lindefs+i]; + + if (rule->vars != 0) { + printer.lvar_ranges(rule->vars, NULL); + printer.puts(" "); } printer.puts("String(0) -> "); @@ -1166,74 +1157,52 @@ PgfText *pgf_print_linref_internal(PgfPhrasetableIds *seq_ids, object o, size_t printer.puts("["); printer.efun(&lincat->name); printer.puts("("); - printer.lparam(lincat->args[lincat->n_lindefs+i].param); - printer.puts(")] = ["); + printer.lparam(rule->args[0]); + printer.puts(")]; "); - size_t n_seqs = lincat->fields.size(); - ref seq = lincat->seqs[lincat->n_lindefs*n_seqs+i]; - printer.seq_id(seq_ids, seq); - - printer.puts("]"); + printer.lparam(rule->lin_idx); + printer.puts(" : "); + printer.symbols(rule->syms.as_vector()); return printer.get_text(); } PGF_API -PgfText *pgf_print_lin_internal(PgfPhrasetableIds *seq_ids, object o, size_t i) +PgfText *pgf_print_lin_internal(object o, size_t i) { ref lin = o; PgfInternalMarshaller m; PgfPrinter printer(NULL,0,&m); - ref res = lin->res[i]; + ref rule = lin->rules[i]; ref ty = lin->absfun->type; - if (res->vars != 0) { - printer.lvar_ranges(res->vars, NULL); - printer.puts(" . "); + if (rule->vars != 0) { + printer.lvar_ranges(rule->vars, NULL); + printer.puts(" "); } printer.efun(&ty->name); printer.puts("("); - printer.lparam(ref::from_ptr(&res->param)); + printer.lparam(rule->res); printer.puts(") -> "); printer.efun(&lin->name); printer.puts("["); - size_t n_args = lin->args.size() / lin->res.size(); - for (size_t j = 0; j < n_args; j++) { + for (size_t j = 0; j < rule->args.size(); j++) { if (j > 0) printer.puts(","); - printer.parg(ty->hypos.elem(j)->type, - lin->args.elem(i*n_args + j)); + printer.efun(&ty->hypos.elem(j)->type->name); + printer.puts("("); + printer.lparam(rule->args[j]); + printer.puts(")"); } - printer.puts("] = ["); + printer.puts("]; "); - size_t n_seqs = lin->seqs.size() / lin->res.size(); - for (size_t j = 0; j < n_seqs; j++) { - if (j > 0) - printer.puts(","); - - ref seq = lin->seqs[i*n_seqs + j]; - printer.seq_id(seq_ids, seq); - } - - printer.puts("]"); - - return printer.get_text(); -} - -PGF_API -PgfText *pgf_print_sequence_internal(size_t seq_id, object o) -{ - ref seq = o; - - PgfInternalMarshaller m; - PgfPrinter printer(NULL,0,&m); - - printer.nprintf(10,"S%zu = ", seq_id); - printer.sequence(seq); + printer.lparam(rule->lin_idx); + printer.puts(" : "); + printer.symbols(rule->syms.as_vector()); return printer.get_text(); } @@ -1241,14 +1210,14 @@ PgfText *pgf_print_sequence_internal(size_t seq_id, object o) PGF_API PgfText *pgf_sequence_get_text_internal(object o) { - ref seq = o; + ref rule = o; PgfPrinter printer(NULL,0,NULL); - for (size_t i = 0; i < seq->syms.size(); i++) { + for (size_t i = 0; i < rule->syms.size(); i++) { if (i > 0) printer.puts(" "); - PgfSymbol sym = seq->syms[i]; + PgfSymbol sym = rule->syms[i]; switch (ref::get_tag(sym)) { case PgfSymbolKS::tag: { auto sym_ks = ref::untagged(sym); @@ -1263,12 +1232,6 @@ PgfText *pgf_sequence_get_text_internal(object o) return printer.get_text(); } -PGF_API_DECL -void pgf_release_phrasetable_ids(PgfPhrasetableIds *seq_ids) -{ - delete seq_ids; -} - PGF_API PgfExpr pgf_check_expr(PgfDB *db, PgfRevision revision, PgfExpr e, PgfType ty, @@ -1535,13 +1498,6 @@ void drop_lin(ref concr, PgfText *name) Namespace lins = namespace_delete(concr->lins, name, &lin); if (lin != 0) { - object container = lin.tagged(); - for (size_t i = 0; i < lin->seqs.size(); i++) { - ref seq = lin->seqs[i]; - PgfPhrasetable phrasetable = - phrasetable_delete(concr->phrasetable,container,i,seq); - concr->phrasetable = phrasetable; - } PgfConcrLin::release(lin); } concr->lins = lins; @@ -1764,48 +1720,46 @@ void pgf_drop_concrete(PgfDB *db, PgfRevision revision, class PGF_INTERNAL PgfLinBuilder : public PgfLinBuilderIface { ref concr; - - vector args; - vector> res; - vector> seqs; + vector> rules; object container; // what are we building? ref container_lincat; size_t var_index; size_t arg_index; - size_t res_index; - size_t seq_index; size_t sym_index; size_t alt_index; size_t n_lindefs; size_t n_linrefs; - ref seq; + size_t n_args; + size_t rule_index; + + vector syms; size_t pre_sym_index; + PgfParseTableMaker tm; + const char *builder_error_msg = "Detected incorrect use of the linearization builder"; public: - PgfLinBuilder(ref concr) + PgfLinBuilder(ref concr) : tm(concr) { this->concr = concr; - this->args = 0; - this->res = 0; - this->seqs = 0; + this->rules = 0; this->var_index = 0; this->arg_index = 0; - this->res_index = 0; - this->seq_index = 0; this->sym_index = (size_t) -1; this->alt_index = (size_t) -1; this->n_lindefs = 0; this->n_linrefs = 0; - this->seq = 0; + this->n_args = 0; + this->rule_index = 0; + this->syms = 0; this->pre_sym_index = (size_t) -1; } @@ -1814,20 +1768,24 @@ public: size_t n_lindefs, size_t n_linrefs, PgfBuildLinIface *build, PgfExn *err) { - size_t n_prods = n_lindefs+n_linrefs; - this->args = vector::alloc(n_prods); - this->res = vector>::alloc(n_prods); - this->seqs = vector>::alloc(n_lindefs*n_fields+n_linrefs); + this->var_index = 0; + this->arg_index = 0; + this->sym_index = (size_t) -1; + this->alt_index = (size_t) -1; this->n_lindefs = n_lindefs; this->n_linrefs = n_linrefs; + this->n_args = 1; + this->rule_index = 0; + this->syms = 0; + this->pre_sym_index = (size_t) -1; + + this->rules = vector>::alloc(n_lindefs+n_linrefs); ref lincat = PgfDB::malloc(abscat->name.size+1); memcpy(&lincat->name, &abscat->name, sizeof(PgfText)+abscat->name.size+1); lincat->abscat = abscat; - lincat->args = args; - lincat->res = res; - lincat->seqs = seqs; lincat->n_lindefs = n_lindefs; + lincat->rules= this->rules; vector> db_fields = vector>::alloc(n_fields); for (size_t i = 0; i < n_fields; i++) { @@ -1840,7 +1798,7 @@ public: this->container_lincat = 0; build->build(this, err); - if (err->type == PGF_EXN_NONE && res_index != res.size()) { + if (err->type == PGF_EXN_NONE && rule_index != rules.size()) { err->type = PGF_EXN_PGF_ERROR; err->msg = builder_error_msg; } @@ -1852,7 +1810,7 @@ public: return lincat; } - ref build(ref absfun, size_t n_prods, + ref build(ref absfun, size_t n_rules, PgfBuildLinIface *build, PgfExn *err) { ref lincat = @@ -1861,24 +1819,32 @@ public: throw pgf_error("Missing linearization category"); } - this->args = vector::alloc(n_prods*absfun->type->hypos.size()); - this->res = vector>::alloc(n_prods); - this->seqs = vector>::alloc(n_prods*lincat->fields.size()); - this->n_lindefs = n_prods; + this->var_index = 0; + this->arg_index = 0; + this->sym_index = (size_t) -1; + this->alt_index = (size_t) -1; + this->n_lindefs = n_rules; + this->n_linrefs = n_linrefs; + this->n_args = 1; + this->rule_index = 0; + this->syms = 0; + this->pre_sym_index = (size_t) -1; + + this->rules = vector>::alloc(n_rules); ref lin = PgfDB::malloc(absfun->name.size+1); memcpy(&lin->name, &absfun->name, sizeof(PgfText)+absfun->name.size+1); lin->absfun = absfun; lin->lincat = lincat; - lin->args = args; - lin->res = res; - lin->seqs = seqs; + lin->rules = this->rules; this->container = lin.tagged(); this->container_lincat = lincat; + this->n_args = absfun->type->hypos.size(); + build->build(this, err); - if (err->type == PGF_EXN_NONE && res_index != res.size()) { + if (err->type == PGF_EXN_NONE && rule_index != rules.size()) { err->type = PGF_EXN_PGF_ERROR; err->msg = builder_error_msg; } @@ -1890,26 +1856,43 @@ public: return lin; } - void start_production(PgfExn *err) + void start_rule(size_t n_vars, size_t n_syms, PgfExn *err) { if (err->type != PGF_EXN_NONE) return; PGF_API_BEGIN { - if (res_index >= res.size()) + if (rule_index >= rules.size()) throw pgf_error(builder_error_msg); + + vector vars = + (n_vars > 0) ? vector::alloc(n_vars) : 0; + vector> args = + (n_args > 0) ? vector>::alloc(n_args) : 0; + + ref rule = inline_vector::alloc(&PgfConcrRule::syms, n_syms); + rule->vars = vars; + rule->res = 0; + rule->container = container; + rule->args = args; + rule->lin_idx = 0; + rules[rule_index] = rule; + var_index = 0; - res[res_index] = 0; + arg_index = 0; + sym_index = 0; + + syms = rule->syms.as_vector(); } PGF_API_END } - void add_argument(size_t n_hypos, size_t i0, size_t n_terms, size_t *terms, PgfExn *err) + void add_argument(size_t i0, size_t n_terms, size_t *terms, PgfExn *err) { if (err->type != PGF_EXN_NONE) return; PGF_API_BEGIN { - if (arg_index >= args.size()) + if (rule_index >= rules.size() || rules[rule_index]->args == 0 || arg_index >= rules[rule_index]->args.size()) throw pgf_error(builder_error_msg); ref param = PgfDB::malloc(n_terms*2*sizeof(size_t)); @@ -1921,37 +1904,53 @@ public: param->terms[i].var = terms[2*i+1]; } - ref parg = args.elem(arg_index); - parg->param = param; + rules[rule_index]->args[arg_index] = param; arg_index++; } PGF_API_END } - void set_result(size_t n_vars, size_t i0, size_t n_terms, size_t *terms, PgfExn *err) + void set_result(size_t i0, size_t n_terms, size_t *terms, PgfExn *err) { if (err->type != PGF_EXN_NONE) return; PGF_API_BEGIN { - if (res_index >= res.size()) + if (rule_index >= rules.size() || rules[rule_index]->res != 0) throw pgf_error(builder_error_msg); - vector vars = - (n_vars > 0) ? vector::alloc(n_vars) - : 0; - - ref res_elem = PgfDB::malloc(n_terms*2*sizeof(size_t)); - res_elem->vars = vars; - res_elem->param.i0 = i0; - res_elem->param.n_terms = n_terms; + ref res = PgfDB::malloc(n_terms*2*sizeof(size_t)); + res->i0 = i0; + res->n_terms = n_terms; for (size_t i = 0; i < n_terms; i++) { - res_elem->param.terms[i].factor = terms[2*i]; - res_elem->param.terms[i].var = terms[2*i+1]; + res->terms[i].factor = terms[2*i]; + res->terms[i].var = terms[2*i+1]; } - res[res_index] = res_elem; + rules[rule_index]->res = res; + } PGF_API_END + } + + void set_lin_idx(size_t i0, size_t n_terms, size_t *terms, PgfExn *err) + { + if (err->type != PGF_EXN_NONE) + return; + + PGF_API_BEGIN { + if (rule_index >= rules.size() || rules[rule_index]->lin_idx != 0) + throw pgf_error(builder_error_msg); + + ref lin_idx = PgfDB::malloc(n_terms*2*sizeof(size_t)); + lin_idx->i0 = i0; + lin_idx->n_terms = n_terms; + + for (size_t i = 0; i < n_terms; i++) { + lin_idx->terms[i].factor = terms[2*i]; + lin_idx->terms[i].var = terms[2*i+1]; + } + + rules[rule_index]->lin_idx = lin_idx; } PGF_API_END } @@ -1961,16 +1960,16 @@ public: return; PGF_API_BEGIN { - if (res_index >= res.size()) + if (rule_index >= rules.size()) throw pgf_error(builder_error_msg); - ref res_elem = res[res_index]; + ref rule = rules[rule_index]; - if (res_elem->vars == 0 || var_index >= res_elem->vars.size()) + if (rule->vars == 0 || var_index >= rule->vars.size()) throw pgf_error(builder_error_msg); ref var_range = - res_elem->vars.elem(var_index); + rule->vars.elem(var_index); var_range->var = var; var_range->range = range; @@ -1978,29 +1977,16 @@ public: } PGF_API_END } - void start_sequence(size_t n_syms, PgfExn *err) - { - if (err->type != PGF_EXN_NONE) - return; - - PGF_API_BEGIN { - if (seq_index >= seqs.size()) - throw pgf_error(builder_error_msg); - - seq = inline_vector::alloc(&PgfSequence::syms, n_syms); - - seqs[seq_index] = seq; - sym_index = 0; - } PGF_API_END - } - void add_symcat(size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err) { if (err->type != PGF_EXN_NONE) return; PGF_API_BEGIN { - if (seq == 0 || sym_index == (size_t) -1 || sym_index >= seq->syms.size()) + if (syms == 0 || sym_index == (size_t) -1 || sym_index >= syms.size()) + throw pgf_error(builder_error_msg); + + if (d > n_args) throw pgf_error(builder_error_msg); ref symcat = PgfDB::malloc(n_terms*2*sizeof(size_t)); @@ -2013,7 +1999,7 @@ public: symcat->r.terms[i].var = terms[2*i+1]; } - seq->syms[sym_index] = symcat.tagged(); + syms[sym_index] = symcat.tagged(); sym_index++; } PGF_API_END } @@ -2024,7 +2010,10 @@ public: return; PGF_API_BEGIN { - if (seq == 0 || sym_index == (size_t) -1 || sym_index >= seq->syms.size()) + if (syms == 0 || sym_index == (size_t) -1 || sym_index >= syms.size()) + throw pgf_error(builder_error_msg); + + if (d > n_args) throw pgf_error(builder_error_msg); ref symlit = PgfDB::malloc(n_terms*2*sizeof(size_t)); @@ -2037,7 +2026,7 @@ public: symlit->r.terms[i].var = terms[2*i+1]; } - seq->syms[sym_index] = symlit.tagged(); + syms[sym_index] = symlit.tagged(); sym_index++; } PGF_API_END } @@ -2048,14 +2037,17 @@ public: return; PGF_API_BEGIN { - if (seq == 0 || sym_index == (size_t) -1 || sym_index >= seq->syms.size()) + if (syms == 0 || sym_index == (size_t) -1 || sym_index >= syms.size()) + throw pgf_error(builder_error_msg); + + if (d > n_args) throw pgf_error(builder_error_msg); ref symvar = PgfDB::malloc(); symvar->d = d; symvar->r = r; - seq->syms[sym_index] = symvar.tagged(); + syms[sym_index] = symvar.tagged(); sym_index++; } PGF_API_END } @@ -2066,13 +2058,13 @@ public: return; PGF_API_BEGIN { - if (seq == 0 || sym_index == (size_t) -1 || sym_index >= seq->syms.size()) + if (syms == 0 || sym_index == (size_t) -1 || sym_index >= syms.size()) throw pgf_error(builder_error_msg); ref symtok = PgfDB::malloc(token->size+1); memcpy(&symtok->token, token, sizeof(PgfText)+token->size+1); - seq->syms[sym_index] = symtok.tagged(); + syms[sym_index] = symtok.tagged(); sym_index++; } PGF_API_END } @@ -2083,18 +2075,18 @@ public: return; PGF_API_BEGIN { - if (seq == 0 || sym_index == (size_t) -1 || sym_index >= seq->syms.size() || pre_sym_index != (size_t) -1) + if (syms == 0 || sym_index == (size_t) -1 || sym_index >= syms.size() || pre_sym_index != (size_t) -1) throw pgf_error(builder_error_msg); - ref def = inline_vector::alloc(&PgfSequence::syms,n_syms); + vector def = vector::alloc(n_syms); ref symkp = inline_vector::alloc(&PgfSymbolKP::alts,n_alts); symkp->default_form = def; - seq->syms[sym_index] = symkp.tagged(); + syms[sym_index] = symkp.tagged(); pre_sym_index = sym_index; - seq = def; + syms = def; sym_index = 0; alt_index = 0; } PGF_API_END @@ -2109,7 +2101,7 @@ public: if (pre_sym_index == (size_t) -1) throw pgf_error(builder_error_msg); - ref form = inline_vector::alloc(&PgfSequence::syms, n_syms); + vector form = vector::alloc(n_syms); vector> prefixes = vector>::alloc(n_prefs); for (size_t i = 0; i < n_prefs; i++) { @@ -2117,14 +2109,14 @@ public: prefixes[i] = pref; } - seq = seqs[seq_index]; - ref symkp = ref::untagged(seq->syms[pre_sym_index]); + syms = rules[rule_index]->syms.as_vector(); + ref symkp = ref::untagged(syms[pre_sym_index]); ref alt = symkp->alts.elem(alt_index); alt->form = form; alt->prefixes = prefixes; - seq = form; + syms = form; sym_index = 0; } PGF_API_END } @@ -2138,8 +2130,8 @@ public: if (pre_sym_index == (size_t) -1) throw pgf_error(builder_error_msg); - seq = seqs[seq_index]; - ref symkp = ref::untagged(seq->syms[pre_sym_index]); + syms = rules[rule_index]->syms.as_vector(); + ref symkp = ref::untagged(syms[pre_sym_index]); if (alt_index >= symkp->alts.size()) throw pgf_error(builder_error_msg); @@ -2156,7 +2148,7 @@ public: if (pre_sym_index == (size_t) -1) throw pgf_error(builder_error_msg); - seq = seqs[seq_index]; + syms = rules[rule_index]->syms.as_vector(); sym_index = pre_sym_index+1; alt_index = 0; pre_sym_index = (size_t) -1; @@ -2169,10 +2161,10 @@ public: return; PGF_API_BEGIN { - if (seq == 0 || sym_index == (size_t) -1 || sym_index >= seq->syms.size()) + if (syms == 0 || sym_index == (size_t) -1 || sym_index >= syms.size()) throw pgf_error(builder_error_msg); - seq->syms[sym_index] = ref(0).tagged(); + syms[sym_index] = ref(0).tagged(); sym_index++; } PGF_API_END } @@ -2183,10 +2175,10 @@ public: return; PGF_API_BEGIN { - if (seq == 0 || sym_index == (size_t) -1 || sym_index >= seq->syms.size()) + if (syms == 0 || sym_index == (size_t) -1 || sym_index >= syms.size()) throw pgf_error(builder_error_msg); - seq->syms[sym_index] = ref(0).tagged(); + syms[sym_index] = ref(0).tagged(); sym_index++; } PGF_API_END } @@ -2197,10 +2189,10 @@ public: return; PGF_API_BEGIN { - if (seq == 0 || sym_index == (size_t) -1 || sym_index >= seq->syms.size()) + if (syms == 0 || sym_index == (size_t) -1 || sym_index >= syms.size()) throw pgf_error(builder_error_msg); - seq->syms[sym_index] = ref(0).tagged(); + syms[sym_index] = ref(0).tagged(); sym_index++; } PGF_API_END } @@ -2211,10 +2203,10 @@ public: return; PGF_API_BEGIN { - if (seq == 0 || sym_index == (size_t) -1 || sym_index >= seq->syms.size()) + if (syms == 0 || sym_index == (size_t) -1 || sym_index >= syms.size()) throw pgf_error(builder_error_msg); - seq->syms[sym_index] = ref(0).tagged(); + syms[sym_index] = ref(0).tagged(); sym_index++; } PGF_API_END } @@ -2225,10 +2217,10 @@ public: return; PGF_API_BEGIN { - if (seq == 0 || sym_index == (size_t) -1 || sym_index >= seq->syms.size()) + if (syms == 0 || sym_index == (size_t) -1 || sym_index >= syms.size()) throw pgf_error(builder_error_msg); - seq->syms[sym_index] = ref(0).tagged(); + syms[sym_index] = ref(0).tagged(); sym_index++; } PGF_API_END } @@ -2239,79 +2231,38 @@ public: return; PGF_API_BEGIN { - if (seq == 0 || sym_index == (size_t) -1 || sym_index >= seq->syms.size()) + if (syms == 0 || sym_index == (size_t) -1 || sym_index >= syms.size()) throw pgf_error(builder_error_msg); - seq->syms[sym_index] = ref(0).tagged(); + syms[sym_index] = ref(0).tagged(); sym_index++; } PGF_API_END } - object end_sequence(PgfExn *err) - { - if (err->type != PGF_EXN_NONE) - return 0; - - ref entry = 0; - - PGF_API_BEGIN { - if (seq == 0 || sym_index != seq->syms.size()) - throw pgf_error(builder_error_msg); - - PgfPhrasetable phrasetable = - phrasetable_internalize(concr->phrasetable, - seq, container_lincat, container, seq_index, - &entry); - concr->phrasetable = phrasetable; - seqs[seq_index] = entry->seq; - - sym_index = (size_t) -1; - seq = 0; - seq_index++; - } PGF_API_END - - return entry.as_object(); - } - - void add_sequence_id(object seq_id, PgfExn *err) + void end_rule(PgfExn *err) { if (err->type != PGF_EXN_NONE) return; PGF_API_BEGIN { - if (seq_index >= seqs.size()) + if (rule_index >= rules.size()) throw pgf_error(builder_error_msg); - ref entry = seq_id; - phrasetable_add_backref(entry,PgfDB::get_txn_id(),container,seq_index); - - seqs[seq_index] = entry->seq; - - seq_index++; - } PGF_API_END - } - - void end_production(PgfExn *err) - { - if (err->type != PGF_EXN_NONE) - return; - - PGF_API_BEGIN { - size_t n_args = (args.size()/res.size()); - if (arg_index != (res_index+1)*n_args) + ref rule = rules[rule_index]; + if (rule->res == 0) throw pgf_error(builder_error_msg); - if (res[res_index] == 0) + if (arg_index < n_args) throw pgf_error(builder_error_msg); - size_t n_seqs = ((seqs.size()-n_linrefs)/(res.size()-n_linrefs)); - size_t exp_index = - (res_index < n_lindefs) ? (res_index+1)*n_seqs - : n_seqs * n_lindefs + (res_index-n_lindefs+1) ; - if (seq_index != exp_index) - throw pgf_error(builder_error_msg); + if ((ref::get_tag(rule->container) == PgfConcrLin::tag) || + (rule_index < n_lindefs)) { + /*PgfParseIndex parse_index = + parse_index_insert(concr->parse_index, rule); + concr->parse_index = parse_index;*/ + } - res_index++; + rule_index++; } PGF_API_END } }; @@ -2375,16 +2326,6 @@ void pgf_drop_lincat(PgfDB *db, }; probspace_iter(pgf->abstract.funs_by_cat, name, f, true); - // Remove the sequences comprizing the lindef and linref - object container = lincat.tagged(); - PgfPhrasetable phrasetable = concr->phrasetable; - for (size_t i = 0; i < lincat->seqs.size(); i++) { - ref seq = lincat->seqs[i]; - phrasetable = - phrasetable_delete(phrasetable,container,i,seq); - } - concr->phrasetable = phrasetable; - // Finaly remove the lincat object itself. PgfConcrLincat::release(lincat); } @@ -2395,11 +2336,11 @@ void pgf_drop_lincat(PgfDB *db, PGF_API void pgf_create_lin(PgfDB *db, PgfRevision revision, PgfConcrRevision cnc_revision, - PgfText *name, size_t n_prods, + PgfText *name, size_t n_rules, PgfBuildLinIface *build, PgfExn *err) { - if (n_prods == 0) + if (n_rules == 0) return; PGF_API_BEGIN { @@ -2415,7 +2356,7 @@ void pgf_create_lin(PgfDB *db, } ref lin = - PgfLinBuilder(concr).build(absfun, n_prods, build, err); + PgfLinBuilder(concr).build(absfun, n_rules, build, err); if (lin != 0) { Namespace lins = namespace_insert(concr->lins, lin); @@ -2430,7 +2371,7 @@ void pgf_create_lin(PgfDB *db, PGF_API void pgf_alter_lin(PgfDB *db, PgfRevision revision, PgfConcrRevision cnc_revision, - PgfText *name, size_t n_prods, + PgfText *name, size_t n_rules, PgfBuildLinIface *build, PgfExn *err) { @@ -2447,21 +2388,13 @@ void pgf_alter_lin(PgfDB *db, } ref lin = - PgfLinBuilder(concr).build(absfun, n_prods, build, err); + PgfLinBuilder(concr).build(absfun, n_rules, build, err); if (lin != 0) { ref old_lin; Namespace lins = namespace_replace(concr->lins, lin, &old_lin); concr->lins = lins; if (old_lin != 0) { - object container = old_lin.tagged(); - PgfPhrasetable phrasetable = concr->phrasetable; - for (size_t i = 0; i < old_lin->seqs.size(); i++) { - ref seq = old_lin->seqs[i]; - phrasetable = - phrasetable_delete(phrasetable,container,i,seq); - } - concr->phrasetable = phrasetable; PgfConcrLin::release(old_lin); } } @@ -2757,11 +2690,8 @@ PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision, if (lincat_u.lincat == 0) return 0; - PgfParser *parser = new PgfParser(concr, lincat_u.lincat, sentence, case_sensitive, m, u); - phrasetable_lookup_cohorts(concr->phrasetable, - sentence, case_sensitive, - parser, err); - parser->prepare(); + PgfParser *parser = new PgfParser(concr, sentence, case_sensitive, m, u); + parser->prepare(lincat_u.lincat); return parser; } PGF_API_END @@ -3122,91 +3052,3 @@ pgf_align_words(PgfDB *db, PgfConcrRevision revision, return NULL; } - -PGF_API PgfText * -pgf_graphviz_lr_automaton(PgfDB *db, PgfConcrRevision revision, - PgfExn *err) -{ - PGF_API_BEGIN { - DB_scope scope(db, READER_SCOPE); - - ref concr = db->revision2concr(revision); - - PgfPrinter printer(NULL,0,NULL); - - printer.puts("digraph {\n"); - for (size_t i = 0; i < concr->lrtable.size(); i++) { - ref state = concr->lrtable.elem(i); - - printer.nprintf(16, " s%zu [label=\"", i); - for (size_t j = 0; j < state->reductions.size(); j++) { - ref reduce = state->reductions.elem(j); - - switch (ref::get_tag(reduce->lin_obj)) { - case PgfConcrLin::tag: { - auto lin = - ref::untagged(reduce->lin_obj); - printer.efun(&lin->name); - break; - } - case PgfConcrLincat::tag: { - auto lincat = - ref::untagged(reduce->lin_obj); - printer.puts("linref "); - printer.efun(&lincat->name); - break; - } - } - - printer.puts("["); - for (size_t i = 0; i < reduce->args.size(); i++) { - ref arg = reduce->args.elem(i); - if (i > 0) - printer.puts(","); - if (arg->arg == 0 && arg->stk_idx == 0) { - printer.nprintf(32,"?"); - } else { - if (arg->arg != 0) - printer.nprintf(32,"?%zd",arg->arg->id); - if (arg->stk_idx != 0) - printer.nprintf(32,"$%zd",arg->stk_idx); - } - } - printer.nprintf(32,"] %zd\n",reduce->depth); - } - printer.puts("\""); - if (i == 0) printer.puts(",penwidth=3"); - printer.nprintf(16, "]\n"); - - for (size_t j = 0; j < state->shifts.size(); j++) { - ref shift = state->shifts.elem(j); - printer.nprintf(16, " s%zu -> s%zu [label=\"", i, shift->next_state); - printer.efun(&shift->lincat->name); - printer.nprintf(16, ".%zu\"];\n", shift->r); - } - - for (size_t j = 0; j < state->tokens.size(); j++) { - ref shift = state->tokens.elem(j); - printer.nprintf(16, " s%zu -> s%zu [label=\"", i, shift->next_state); - size_t sym_idx = shift->sym_idx; - while (sym_idx < shift->seq->syms.size()) { - if (ref::get_tag(shift->seq->syms[sym_idx]) != PgfSymbolKS::tag) - break; - if (sym_idx > shift->sym_idx) - printer.puts(" "); - auto symks = ref::untagged(shift->seq->syms[sym_idx]); - printer.puts("\\\""); - printer.put_esc_str(&symks->token); - printer.puts("\\\""); - sym_idx++; - } - printer.puts("\"];\n"); - } - } - printer.puts("}"); - - return printer.get_text(); - } PGF_API_END - - return NULL; -} diff --git a/src/runtime/c/pgf/pgf.h b/src/runtime/c/pgf/pgf.h index 3c41c6e26..8bc2ddede 100644 --- a/src/runtime/c/pgf/pgf.h +++ b/src/runtime/c/pgf/pgf.h @@ -461,8 +461,6 @@ PGF_API_DECL void pgf_iter_lins(PgfDB *db, PgfConcrRevision cnc_revision, PgfItor *itor, PgfExn *err); -typedef struct PgfPhrasetableIds PgfPhrasetableIds; - typedef struct PgfSequenceItor PgfSequenceItor; struct PgfSequenceItor { int (*fn)(PgfSequenceItor* self, size_t seq_id, object value, @@ -493,10 +491,10 @@ void pgf_lookup_cohorts(PgfDB *db, PgfConcrRevision cnc_revision, PgfCohortsCallback* callback, PgfExn* err); PGF_API_DECL -PgfPhrasetableIds *pgf_iter_sequences(PgfDB *db, PgfConcrRevision cnc_revision, - PgfSequenceItor *itor, - PgfMorphoCallback *callback, - PgfExn *err); +void pgf_iter_sequences(PgfDB *db, PgfConcrRevision cnc_revision, + PgfSequenceItor *itor, + PgfMorphoCallback *callback, + PgfExn *err); PGF_API_DECL void pgf_get_lincat_counts_internal(object o, size_t *counts); @@ -505,26 +503,20 @@ PGF_API_DECL PgfText *pgf_get_lincat_field_internal(object o, size_t i); PGF_API_DECL -size_t pgf_get_lin_get_prod_count(object o); +size_t pgf_get_lin_rules_count(object o); PGF_API_DECL -PgfText *pgf_print_lindef_internal(PgfPhrasetableIds *seq_ids, object o, size_t i); +PgfText *pgf_print_lindef_internal(object o, size_t i); PGF_API_DECL -PgfText *pgf_print_linref_internal(PgfPhrasetableIds *seq_ids, object o, size_t i); +PgfText *pgf_print_linref_internal(object o, size_t i); PGF_API_DECL -PgfText *pgf_print_lin_internal(PgfPhrasetableIds *seq_ids, object o, size_t i); - -PGF_API_DECL -PgfText *pgf_print_sequence_internal(size_t seq_id, object o); +PgfText *pgf_print_lin_internal(object o, size_t i); PGF_API_DECL PgfText *pgf_sequence_get_text_internal(object o); -PGF_API_DECL -void pgf_release_phrasetable_ids(PgfPhrasetableIds *seq_ids); - PGF_API_DECL PgfExpr pgf_check_expr(PgfDB *db, PgfRevision revision, PgfExpr e, PgfType ty, @@ -635,11 +627,11 @@ void pgf_drop_concrete(PgfDB *db, PgfRevision revision, #ifdef __cplusplus struct PgfLinBuilderIface { - virtual void start_production(PgfExn *err)=0; - virtual void add_argument(size_t n_hypos, size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0; - virtual void set_result(size_t n_vars, size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0; + virtual void start_rule(size_t n_vars, size_t n_syms, PgfExn *err)=0; + virtual void add_argument(size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0; + virtual void set_result(size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0; + virtual void set_lin_idx(size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0; virtual void add_variable(size_t var, size_t range, PgfExn *err)=0; - virtual void start_sequence(size_t n_syms, PgfExn *err)=0; virtual void add_symcat(size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0; virtual void add_symlit(size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0; virtual void add_symvar(size_t d, size_t r, PgfExn *err)=0; @@ -654,9 +646,7 @@ struct PgfLinBuilderIface { virtual void add_symsoftspace(PgfExn *err)=0; virtual void add_symcapit(PgfExn *err)=0; virtual void add_symallcapit(PgfExn *err)=0; - virtual object end_sequence(PgfExn *err)=0; - virtual void add_sequence_id(object seq_id, PgfExn *err)=0; - virtual void end_production(PgfExn *err)=0; + virtual void end_rule(PgfExn *err)=0; }; struct PgfBuildLinIface { @@ -666,11 +656,11 @@ struct PgfBuildLinIface { typedef struct PgfLinBuilderIface PgfLinBuilderIface; typedef struct { - void (*start_production)(PgfLinBuilderIface *this, PgfExn *err); - void (*add_argument)(PgfLinBuilderIface *this, size_t n_hypos, size_t i0, size_t n_terms, size_t *terms, PgfExn *err); - void (*set_result)(PgfLinBuilderIface *this, size_t n_vars, size_t i0, size_t n_terms, size_t *terms, PgfExn *err); + void (*start_rule)(PgfLinBuilderIface *this, size_t n_vars, size_t n_syms, PgfExn *err); + void (*add_argument)(PgfLinBuilderIface *this, size_t i0, size_t n_terms, size_t *terms, PgfExn *err); + void (*set_result)(PgfLinBuilderIface *this, size_t i0, size_t n_terms, size_t *terms, PgfExn *err); + void (*set_lin_idx)(PgfLinBuilderIface *this, size_t i0, size_t n_terms, size_t *terms, PgfExn *err); void (*add_variable)(PgfLinBuilderIface *this, size_t var, size_t range, PgfExn *err); - void (*start_sequence)(PgfLinBuilderIface *this, size_t n_syms, PgfExn *err); void (*add_symcat)(PgfLinBuilderIface *this, size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err); void (*add_symlit)(PgfLinBuilderIface *this, size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err); void (*add_symvar)(PgfLinBuilderIface *this, size_t d, size_t r, PgfExn *err); @@ -685,9 +675,7 @@ typedef struct { void (*add_symsoftspace)(PgfLinBuilderIface *this, PgfExn *err); void (*add_symcapit)(PgfLinBuilderIface *this, PgfExn *err); void (*add_symallcapit)(PgfLinBuilderIface *this, PgfExn *err); - object (*end_sequence)(PgfLinBuilderIface *this, PgfExn *err); - void (*add_sequence_id)(PgfLinBuilderIface *this, object seq_id, PgfExn *err); - void (*end_production)(PgfLinBuilderIface *this, PgfExn *err); + void (*end_rule)(PgfLinBuilderIface *this, PgfExn *err); } PgfLinBuilderIfaceVtbl; struct PgfLinBuilderIface { @@ -720,10 +708,17 @@ void pgf_drop_lincat(PgfDB *db, PGF_API_DECL void pgf_create_lin(PgfDB *db, PgfRevision revision, PgfConcrRevision cnc_revision, - PgfText *name, size_t n_prods, + PgfText *name, size_t n_rules, PgfBuildLinIface *build, PgfExn *err); +PGF_API_DECL +void pgf_alter_lin(PgfDB *db, + PgfRevision revision, PgfConcrRevision cnc_revision, + PgfText *name, size_t n_rules, + PgfBuildLinIface *build, + PgfExn *err); + PGF_API_DECL void pgf_drop_lin(PgfDB *db, PgfRevision revision, PgfConcrRevision cnc_revision, @@ -916,8 +911,4 @@ pgf_align_words(PgfDB *db, PgfConcrRevision revision, size_t *n_phrases /* out */, PgfExn* err); -PGF_API PgfText * -pgf_graphviz_lr_automaton(PgfDB *db, PgfConcrRevision revision, - PgfExn *err); - #endif // PGF_H_ diff --git a/src/runtime/c/pgf/phrasetable.cxx b/src/runtime/c/pgf/phrasetable.cxx index 49e7c95c7..86a8cfffb 100644 --- a/src/runtime/c/pgf/phrasetable.cxx +++ b/src/runtime/c/pgf/phrasetable.cxx @@ -1,77 +1,8 @@ #include "data.h" +#include "printer.h" #include -PgfPhrasetableIds::PgfPhrasetableIds() -{ - next_id = 0; - n_pairs = 0; - pairs = NULL; - chains = NULL; -} - -void PgfPhrasetableIds::start(ref concr) -{ - next_id = 0; - n_pairs = phrasetable_size(concr->phrasetable); - size_t mem_size = sizeof(SeqIdPair)*n_pairs; - pairs = (SeqIdPair*) malloc(mem_size); - if (pairs == NULL) - throw pgf_systemerror(ENOMEM); - memset(pairs, 0, mem_size); -} - -size_t PgfPhrasetableIds::add(ref seq) -{ - size_t index = (seq.as_object() >> 4) % n_pairs; - if (pairs[index].seq == 0) { - pairs[index].seq = seq; - pairs[index].seq_id = next_id++; - return pairs[index].seq_id; - } else { - SeqIdChain *chain = - (SeqIdChain*) malloc(sizeof(SeqIdChain)); - if (chain == NULL) - throw pgf_systemerror(ENOMEM); - chain->next = chains; - chain->chain = pairs[index].chain; - chain->seq = seq; - chain->seq_id = next_id++; - pairs[index].chain = chain; - chains = chain; - return chain->seq_id; - } -} - -size_t PgfPhrasetableIds::get(ref seq) -{ - size_t index = (seq.as_object() >> 4) % n_pairs; - if (pairs[index].seq == seq) { - return pairs[index].seq_id; - } else { - SeqIdChain *chain = pairs[index].chain; - while (chain != NULL) { - if (chain->seq == seq) - return chain->seq_id; - chain = chain->chain; - } - throw pgf_error("Can't find sequence id"); - } -} - -void PgfPhrasetableIds::end() -{ - next_id = 0; - n_pairs = 0; - - while (chains != NULL) { - SeqIdChain *next = chains->next; - free(chains); - chains = next; - } - - free(pairs); - pairs = NULL; -} +// #define DEBUG_PARSE_INDEX static int lparam_cmp(PgfLParam *p1, PgfLParam *p2) @@ -101,7 +32,7 @@ int lparam_cmp(PgfLParam *p1, PgfLParam *p2) } static -int sequence_cmp(ref seq1, ref seq2); +int sequence_cmp(vector seq1, vector seq2); static void symbol_cmp(PgfSymbol sym1, PgfSymbol sym2, int res[2]) @@ -202,25 +133,49 @@ void symbol_cmp(PgfSymbol sym1, PgfSymbol sym2, int res[2]) case PgfSymbolCAPIT::tag: case PgfSymbolALLCAPIT::tag: break; + case PgfSymbolACat::tag: { + auto sym_acat1 = ref::untagged(sym1); + auto sym_acat2 = ref::untagged(sym2); + res[0] = (res[1] = textcmp(&sym_acat1->name,&sym_acat2->name)); + return; + } + case PgfSymbolCCat::tag: { + auto sym_ccat1 = ref::untagged(sym1); + auto sym_ccat2 = ref::untagged(sym2); + res[0] = (res[1] = textcmp(&sym_ccat1->lincat->name,&sym_ccat2->lincat->name)); + if (res[0] != 0) + return; + if (sym_ccat1->value < sym_ccat2->value) + res[0] = (res[1] = -1); + else if (sym_ccat1->value > sym_ccat2->value) + res[0] = (res[1] = 1); + if (sym_ccat1->lin_idx < sym_ccat2->lin_idx) + res[0] = (res[1] = -1); + else if (sym_ccat1->lin_idx > sym_ccat2->lin_idx) + res[0] = (res[1] = 1); + else + res[0] = (res[1] = 0); + return; + } default: throw pgf_error("Unknown symbol tag"); } } static -int sequence_cmp(ref seq1, ref seq2) +int sequence_cmp(vector seq1, vector seq2) { int res[2] = {0,0}; for (size_t i = 0; ; i++) { - if (i >= seq1->syms.size()) { - if (i < seq2->syms.size()) + if (i >= seq1.size()) { + if (i < seq2.size()) return -1; return res[1]; } - if (i >= seq2->syms.size()) + if (i >= seq2.size()) return 1; - symbol_cmp(seq1->syms[i], seq2->syms[i], res); + symbol_cmp(seq1[i], seq2[i], res); if (res[0] != 0) return res[0]; } @@ -229,67 +184,33 @@ int sequence_cmp(ref seq1, ref seq2) } PGF_INTERNAL -int text_sequence_cmp(PgfTextSpot *spot, const uint8_t *end, - ref seq, size_t *p_i, - bool case_sensitive, SeqMatch sm) +int text_symbol_cmp(PgfTextSpot *spot, const uint8_t *end, + PgfSymbol sym, bool case_sensitive) { - int res1 = 0; + uint8_t tag = ref::get_tag(sym); + if (PgfSymbolKS::tag != tag) + return ((int) PgfSymbolKS::tag) - ((int) tag); - const uint8_t *s2 = NULL; - const uint8_t *e2 = NULL; + int res1 = 0; - uint8_t t = 0xff; - if (*p_i < seq->syms.size()) { - t = ref::get_tag(seq->syms[*p_i]); - } - - size_t count = 0; + auto sym_ks = ref::untagged(sym); + const uint8_t *s2 = (uint8_t *) &sym_ks->token.text; + const uint8_t *e2 = s2+sym_ks->token.size; for (;;) { if (spot->ptr >= end) { - if (s2 < e2 || t == PgfSymbolKS::tag) + if (s2 < e2) return -1; return case_sensitive ? res1 : 0; } - if (s2 >= e2 && t != PgfSymbolKS::tag) { - return (sm == SM_FULL_MATCH) ? 1 : 0; + if (s2 >= e2) { + return case_sensitive ? res1 : 0; } uint32_t ucs1 = pgf_utf8_decode(&spot->ptr); spot->pos++; uint32_t ucs1i = pgf_utf8_to_upper(ucs1); - if (s2 >= e2) { - if (s2 != NULL) { - if (pgf_utf8_is_space(ucs1)) { - count++; - continue; - } - - if (count == 0) { - return (((int) ucs1) - ' '); - } else { - count = 0; - } - } - - if (t != PgfSymbolKS::tag) { - if (sm == SM_PARTIAL) - return 0; - return ((int) PgfSymbolKS::tag) - ((int) t); - } - - auto sym_ks = ref::untagged(seq->syms[*p_i]); - s2 = (uint8_t *) &sym_ks->token.text; - e2 = s2+sym_ks->token.size; - - (*p_i)++; - t = 0xff; - if (*p_i < seq->syms.size()) { - t = ref::get_tag(seq->syms[*p_i]); - } - } - uint32_t ucs2 = pgf_utf8_decode(&s2); uint32_t ucs2i = pgf_utf8_to_upper(ucs2); @@ -309,179 +230,429 @@ int text_sequence_cmp(PgfTextSpot *spot, const uint8_t *end, } } +static +bool text_symbols_match(PgfTextSpot *spot, const uint8_t *end, + vector syms, size_t dot, bool *bind, + bool case_sensitive) +{ + while (dot < syms.size()) { + PgfSymbol sym = syms[dot]; + switch (ref::get_tag(sym)) { + case PgfSymbolKS::tag: { + const uint8_t *start = spot->ptr; + for (;;) { + const uint8_t *ptr = spot->ptr; + uint32_t ucs = pgf_utf8_decode(&ptr); + if (!pgf_utf8_is_space(ucs)) + break; + spot->ptr = ptr; + spot->pos++; + } + + if (*bind != (start == spot->ptr)) + return false; + + if (text_symbol_cmp(spot,end,sym,case_sensitive) != 0) + return false; + + break; + } + case PgfSymbolKP::tag: { + auto symkp = ref::untagged(syms[dot]); + + PgfTextSpot current = *spot; + if (text_symbols_match(¤t, end, symkp->default_form, 0, bind, case_sensitive)) { + goto matched; + } + + for (size_t i = 0; i < symkp->alts.size(); i++) { + current = *spot; + if (text_symbols_match(¤t, end, symkp->alts[i].form, 0, bind, case_sensitive)) { + goto matched; + } + } + + return false; + + matched: + *spot = current; + break; + } + case PgfSymbolBIND::tag: { + *bind = true; + break; + } + case PgfSymbolSOFTBIND::tag: + case PgfSymbolSOFTSPACE::tag: { + *bind = true; + break; + } + case PgfSymbolCAPIT::tag: + case PgfSymbolALLCAPIT::tag: + // skip + break; + default: + return false; + } + + dot++; + } + + return true; +} + +static +bool text_item_match(PgfTextSpot *spot, const uint8_t *end, + ref item, + bool case_sensitive) +{ + bool bind = false; + size_t dot = item->dot; + vector syms = item->rule->syms.as_vector(); + if (item->pre_alt > 0) { + auto symkp = ref::untagged(syms[item->pre_dot]); + if (item->pre_alt == 1) { + if (!text_symbols_match(spot, end, symkp->default_form, item->dot, &bind, case_sensitive)) + return false; + } else { + if (!text_symbols_match(spot, end, symkp->alts[item->pre_alt-2].form, item->dot, &bind, case_sensitive)) + return false; + } + dot = item->pre_dot+1; + } + return text_symbols_match(spot, end, syms, dot, &bind, case_sensitive); +} + PGF_INTERNAL_DECL size_t get_next_padovan(size_t min); -PGF_INTERNAL_DECL -void phrasetable_add_backref(ref entry, txn_t txn_id, - object container, - size_t seq_index) +static +int symbol_cmp(ref lincat, size_t value, size_t lin_idx, PgfSymbol sym) { - vector backrefs = entry->backrefs; + uint8_t tag = ref::get_tag(sym); + if (PgfSymbolCCat::tag != tag) + return ((int) PgfSymbolCCat::tag) - ((int) tag); - size_t len = (backrefs != 0) ? backrefs.size() : 0; - if (entry->n_backrefs >= len) { - size_t new_len = get_next_padovan(entry->n_backrefs+1); - backrefs = backrefs.realloc(new_len, txn_id); - } - backrefs[entry->n_backrefs].container = container; - backrefs[entry->n_backrefs].seq_index = seq_index; - - entry->n_backrefs++; - entry->backrefs = backrefs; -} - -PGF_INTERNAL -PgfPhrasetable phrasetable_internalize(PgfPhrasetable table, - ref seq, - ref lincat, - object container, - size_t seq_index, - ref *pentry) -{ - if (table == 0) { - PgfPhrasetableEntry entry; - entry.seq = seq; - entry.n_backrefs = 1; - entry.backrefs = vector::alloc(1); - entry.backrefs[0].container = container; - entry.backrefs[0].seq_index = seq_index; - PgfPhrasetable new_table = Node::new_node(entry); - *pentry = ref::from_ptr(&new_table->value); - return new_table; - } - - int cmp = sequence_cmp(seq,table->value.seq); - if (cmp < 0) { - PgfPhrasetable left = phrasetable_internalize(table->left, - seq, - lincat, - container, - seq_index, - pentry); - table = Node::upd_node(table,left,table->right); - return Node::balanceL(table); - } else if (cmp > 0) { - PgfPhrasetable right = phrasetable_internalize(table->right, - seq, - lincat, - container, - seq_index, - pentry); - table = Node::upd_node(table, table->left, right); - return Node::balanceR(table); - } else { - PgfSequence::release(seq); - - PgfPhrasetable new_table = - Node::upd_node(table, table->left, table->right); - *pentry = ref::from_ptr(&new_table->value); - phrasetable_add_backref(*pentry,table->txn_id,container,seq_index); - return new_table; - } -} - -PGF_INTERNAL -ref phrasetable_relink(PgfPhrasetable table, - object container, - size_t seq_index, - size_t seq_id) -{ - while (table != 0) { - size_t left_sz = (table->left==0) ? 0 : table->left->sz; - if (seq_id < left_sz) - table = table->left; - else if (seq_id == left_sz) { - auto entry = ref::from_ptr(&table->value); - phrasetable_add_backref(entry,table->txn_id,container,seq_index); - return table->value.seq; - } else { - table = table->right; - seq_id -= left_sz+1; - } - } - return 0; -} - -PGF_INTERNAL -PgfPhrasetable phrasetable_delete(PgfPhrasetable table, - object container, - size_t seq_index, - ref seq) -{ - if (table == 0) + auto symcf = ref::untagged(sym); + int res = textcmp(&lincat->name, &symcf->lincat->name); + if (res != 0) + return res; + if (value < symcf->value) + return -1; + else if (value > symcf->value) + return 1; + else if (lin_idx < symcf->lin_idx) + return -1; + else if (lin_idx > symcf->lin_idx) + return 1; + else return 0; +} - int cmp = sequence_cmp(seq,table->value.seq); - if (cmp < 0) { - PgfPhrasetable left = phrasetable_delete(table->left, - container, seq_index, - seq); - table = Node::upd_node(table,left,table->right); - return Node::balanceR(table); - } else if (cmp > 0) { - PgfPhrasetable right = phrasetable_delete(table->right, - container, seq_index, - seq); - table = Node::upd_node(table,table->left,right); - return Node::balanceL(table); - } else { - size_t len = table->value.backrefs.size(); - size_t n_backrefs = table->value.n_backrefs; - if (n_backrefs > 1) { - vector backrefs = - table->value.backrefs.realloc(n_backrefs,table->txn_id); - size_t i = 0; - while (i < n_backrefs) { - ref backref = backrefs.elem(i); - if (backref->container == container && - backref->seq_index == seq_index) { - break; - } - i++; - } - i++; - while (i < n_backrefs) { - backrefs[i-1] = table->value.backrefs[i]; - i++; - } - n_backrefs--; +static +int symbol_cmp(PgfSymbol sym1, PgfSymbol sym2) +{ + uint8_t tag1 = ref::get_tag(sym1); + uint8_t tag2 = ref::get_tag(sym2); + if (tag1 != tag2) + return ((int) tag1) - ((int) tag2); - PgfPhrasetable new_table = - Node::upd_node(table, table->left, table->right); - new_table->value.n_backrefs = n_backrefs; - new_table->value.backrefs = backrefs; - return new_table; + switch (tag1) { + case PgfSymbolKS::tag: { + auto symks1 = ref::untagged(sym1); + auto symks2 = ref::untagged(sym2); + int res[2] = {0,0}; + texticmp(&symks1->token, &symks2->token, res); + if (res[0] != 0) + return res[0]; + return res[1]; + } + case PgfSymbolACat::tag: { + auto symcf1 = ref::untagged(sym1); + auto symcf2 = ref::untagged(sym2); + return textcmp(&symcf1->name, &symcf2->name); + } + case PgfSymbolCCat::tag: { + auto symcf1 = ref::untagged(sym1); + auto symcf2 = ref::untagged(sym2); + int res = textcmp(&symcf1->lincat->name, &symcf2->lincat->name); + if (res != 0) + return res; + if (symcf1->value < symcf2->value) + return -1; + else if (symcf1->value > symcf2->value) + return 1; + else if (symcf1->lin_idx < symcf2->lin_idx) + return -1; + else if (symcf1->lin_idx > symcf2->lin_idx) + return 1; + else + return 0; + } + default: + return 0; + } +} + +ref PgfPhrasetableNode::new_node(PgfSymbol sym, size_t n_items) +{ + auto items = vector>::alloc(n_items); + + auto node = PgfDB::malloc(); + node->sym = sym; + node->n_items = 0; + node->items = items; + node->txn_id = PgfDB::get_txn_id(); + node->sz = 1; + node->left = 0; + node->right = 0; + + return node; +} + +PgfPhrasetable PgfPhrasetableNode::upd_node(PgfPhrasetable node, PgfPhrasetable left, PgfPhrasetable right) +{ + if (node->txn_id != PgfDB::get_txn_id()) { + PgfPhrasetable new_node = PgfDB::malloc(); + new_node->sym = node->sym; + new_node->n_items = node->n_items; + new_node->items = node->items; + new_node->txn_id = PgfDB::get_txn_id(); + release(node); + node = new_node; + } + + node->sz = 1+PgfPhrasetableNode::size(left)+PgfPhrasetableNode::size(right); + node->left = left; + node->right = right; + + return node; +} + +PgfPhrasetable PgfPhrasetableNode::balanceL(PgfPhrasetable node) +{ + if (node->right == 0) { + if (node->left == 0) { + return node; } else { - PgfSequence::release(table->value.seq); - vector::release(table->value.backrefs); - if (table->left == 0) { - Node::release(table); - return table->right; - } else if (table->right == 0) { - Node::release(table); - return table->left; - } else if (table->left->sz > table->right->sz) { - PgfPhrasetable node; - PgfPhrasetable left = Node::pop_last(table->left, &node); - node = Node::upd_node(node, left, table->right); - Node::release(table); - return Node::balanceR(node); + if (node->left->left == 0) { + if (node->left->right == 0) { + return node; + } else { + PgfPhrasetable left_right = node->left->right; + PgfPhrasetable left = upd_node(node->left,0,0); + PgfPhrasetable right = upd_node(node,0,0); + return upd_node(left_right, + left, + right); + } } else { - PgfPhrasetable node; - PgfPhrasetable right = Node::pop_first(table->right, &node); - node = Node::upd_node(node, table->left, right); - Node::release(table); - return Node::balanceL(node); + if (node->left->right == 0) { + PgfPhrasetable left = node->left; + PgfPhrasetable right = upd_node(node,0,0); + return upd_node(left, + left->left, + right); + } else { + if (node->left->right->sz < RATIO * node->left->left->sz) { + PgfPhrasetable left = node->left; + PgfPhrasetable right = + upd_node(node, + left->right, + 0); + return upd_node(left, + left->left, + right); + } else { + PgfPhrasetable left_right = node->left->right; + PgfPhrasetable left = + upd_node(node->left, + node->left->left, + left_right->left); + PgfPhrasetable right = + upd_node(node, + left_right->right, + 0); + return upd_node(left_right, + left, + right); + } + } + } + } + } else { + if (node->left == 0) { + return node; + } else { + if (node->left->sz > DELTA*node->right->sz) { + if (node->left->right->sz < RATIO*node->left->left->sz) { + PgfPhrasetable left = node->left; + PgfPhrasetable right = + upd_node(node, + left->right, + node->right); + return upd_node(left, + left->left, + right); + } else { + PgfPhrasetable left_right = node->left->right; + PgfPhrasetable left = + upd_node(node->left, + node->left->left, + left_right->left); + PgfPhrasetable right = + upd_node(node, + left_right->right, + node->right); + return upd_node(left_right, + left, + right); + } + } else { + return node; } } } } -PGF_INTERNAL -size_t phrasetable_size(PgfPhrasetable table) +PgfPhrasetable PgfPhrasetableNode::balanceR(PgfPhrasetable node) { - return Node::size(table); + if (node->left == 0) { + if (node->right == 0) { + return node; + } else { + if (node->right->left == 0) { + if (node->right->right == 0) { + return node; + } else { + PgfPhrasetable right = node->right; + PgfPhrasetable left = + upd_node(node, + 0, + 0); + return upd_node(right, + left, + right->right); + } + } else { + if (node->right->right == 0) { + PgfPhrasetable right_left = node->right->left; + PgfPhrasetable right = + upd_node(node->right,0,0); + PgfPhrasetable left = + upd_node(node,0,0); + return upd_node(right_left, + left, + right); + } else { + if (node->right->left->sz < RATIO * node->right->right->sz) { + PgfPhrasetable right = node->right; + PgfPhrasetable left = + upd_node(node, + 0, + right->left); + return upd_node(right, + left, + right->right); + } else { + PgfPhrasetable right_left = node->right->left; + PgfPhrasetable right = + upd_node(node->right, + right_left->right, + node->right->right); + PgfPhrasetable left = + upd_node(node, + 0, + right_left->left); + return upd_node(right_left, + left, + right); + } + } + } + } + } else { + if (node->right == 0) { + return node; + } else { + if (node->right->sz > DELTA*node->left->sz) { + if (node->right->left->sz < RATIO*node->right->right->sz) { + PgfPhrasetable right = node->right; + PgfPhrasetable left = + upd_node(node, + node->left, + right->left); + return upd_node(right, + left, + right->right); + } else { + PgfPhrasetable right_left = node->right->left; + PgfPhrasetable right = + upd_node(node->right, + right_left->right, + node->right->right); + PgfPhrasetable left = + upd_node(node, + node->left, + right_left->left); + return upd_node(right_left, + left, + right); + } + } else { + return node; + } + } + } +} + +void PgfPhrasetableNode::release(ref node) +{ + PgfDB::free(node); +} + +void phrasetable_iter(PgfPhrasetable table, ref lincat, std::function arg,size_t,vector>)> &f) +{ + if (table == 0) + return; + + int cmp = 0; + ref symcf = 0; + uint8_t tag = ref::get_tag(table->sym); + if (PgfSymbolCCat::tag != tag) { + cmp = ((int) PgfSymbolCCat::tag) - ((int) tag); + } else { + symcf = ref::untagged(table->sym); + cmp = textcmp(&lincat->name, &symcf->lincat->name); + } + + if (cmp < 0) + phrasetable_iter(table->left, lincat, f); + else if (cmp > 0) + phrasetable_iter(table->right, lincat, f); + else { + phrasetable_iter(table->left, lincat, f); + f(symcf,table->n_items,table->items); + phrasetable_iter(table->right, lincat, f); + } +} + +vector> phrasetable_lookup(PgfPhrasetable table, PgfSymbol sym, size_t *n_items) +{ + while (table != 0) { + int cmp = symbol_cmp(sym,table->sym); + if (cmp < 0) + table = table->left; + else if (cmp > 0) + table = table->right; + else { + *n_items = table->n_items; + return table->items; + } + } + + *n_items = 0; + return 0; } PGF_INTERNAL @@ -493,27 +664,34 @@ void phrasetable_lookup(PgfPhrasetable table, if (table == 0) return; - PgfTextSpot current; - current.pos = 0; - current.ptr = (uint8_t *) sentence->text; - const uint8_t *end = current.ptr+sentence->size; - size_t sym_idx = 0; - int cmp = text_sequence_cmp(¤t,end,table->value.seq,&sym_idx,case_sensitive,SM_FULL_MATCH); + PgfTextSpot spot; + spot.pos = 0; + spot.ptr = (uint8_t *) sentence->text; + const uint8_t *end = spot.ptr+sentence->size; + int cmp = text_symbol_cmp(&spot,end,table->sym,case_sensitive); if (cmp < 0) { phrasetable_lookup(table->left,sentence,case_sensitive,scanner,err); } else if (cmp > 0) { phrasetable_lookup(table->right,sentence,case_sensitive,scanner,err); } else { - auto backrefs = table->value.backrefs; - for (size_t i = 0; i < table->value.n_backrefs; i++) { - PgfSequenceBackref backref = backrefs[i]; - switch (ref::get_tag(backref.container)) { + if (!case_sensitive) { + phrasetable_lookup(table->left,sentence,case_sensitive,scanner,err); + if (err->type != PGF_EXN_NONE) + return; + } + + for (size_t i = 0; i < table->n_items; i++) { + ref item = table->items[i]; + switch (ref::get_tag(item->rule->container)) { case PgfConcrLin::tag: { - ref lin = ref::untagged(backref.container); + ref lin = ref::untagged(item->rule->container); if (lin->absfun->type->hypos.size() == 0) { - scanner->match(lin, backref.seq_index, err); - if (err->type != PGF_EXN_NONE) - return; + PgfTextSpot current = spot; + if (text_item_match(¤t, end, item, case_sensitive) && current.ptr == end) { + scanner->match(lin, item->rule->lin_idx->i0, err); + if (err->type != PGF_EXN_NONE) + return; + } } break; } @@ -525,10 +703,7 @@ void phrasetable_lookup(PgfPhrasetable table, } if (!case_sensitive) { - phrasetable_lookup(table->left,sentence,false,scanner,err); - if (err->type != PGF_EXN_NONE) - return; - phrasetable_lookup(table->right,sentence,false,scanner,err); + phrasetable_lookup(table->right,sentence,case_sensitive,scanner,err); if (err->type != PGF_EXN_NONE) return; } @@ -606,8 +781,7 @@ void phrasetable_lookup_prefixes(PgfCohortsState *state, return; PgfTextSpot current = state->spot; - size_t sym_idx = 0; - int cmp = text_sequence_cmp(¤t,state->end,table->value.seq,&sym_idx,state->case_sensitive,SM_PREFIX); + int cmp = text_symbol_cmp(¤t,state->end,table->sym,state->case_sensitive); if (cmp < 0) { phrasetable_lookup_prefixes(state,table->left,min,max); } else if (cmp > 0) { @@ -628,8 +802,7 @@ void phrasetable_lookup_prefixes(PgfCohortsState *state, if (min <= len) phrasetable_lookup_prefixes(state,table->left,min,len); - auto backrefs = table->value.backrefs; - if (len > 0 && backrefs != 0) { + if (len > 0) { if (state->last.pos != current.pos) { if (state->last.pos > 0) { state->scanner->end_matches(&state->last, @@ -647,14 +820,14 @@ void phrasetable_lookup_prefixes(PgfCohortsState *state, } state->queue.push(current); - for (size_t i = 0; i < table->value.n_backrefs; i++) { - PgfSequenceBackref backref = backrefs[i]; - switch (ref::get_tag(backref.container)) { + for (size_t i = 0; i < table->n_items; i++) { + auto rule = table->items[i]->rule; + switch (ref::get_tag(rule->container)) { case PgfConcrLin::tag: { - ref lin = ref::untagged(backref.container); + ref lin = ref::untagged(rule->container); if (lin->absfun->type->hypos.size() == 0) { state->scanner->match(lin, - backref.seq_index, + rule->lin_idx->i0, state->err); if (state->err->type != PGF_EXN_NONE) return; @@ -762,62 +935,81 @@ void phrasetable_lookup_cohorts(PgfPhrasetable table, } } -PGF_INTERNAL -void phrasetable_iter(PgfConcr *concr, - PgfPhrasetable table, - PgfSequenceItor* itor, - PgfMorphoCallback *callback, - PgfPhrasetableIds *seq_ids, PgfExn *err) +PgfPhrasetable phrasetable_insert(PgfPhrasetable table, + PgfSymbol sym, + ref item) { - if (table == 0) - return; + if (table == 0) { + PgfPhrasetable new_table = PgfPhrasetableNode::new_node(sym,1); + new_table->n_items = 1; + new_table->items[0] = item; + return new_table; + } - phrasetable_iter(concr, table->left, itor, callback, seq_ids, err); - if (err->type != PGF_EXN_NONE) - return; + int cmp = symbol_cmp(sym,table->sym); + if (cmp < 0) { + PgfPhrasetable left = phrasetable_insert(table->left, sym, item); + table = PgfPhrasetableNode::upd_node(table,left,table->right); + return PgfPhrasetableNode::balanceL(table); + } else if (cmp > 0) { + PgfPhrasetable right = phrasetable_insert(table->right, sym, item); + table = PgfPhrasetableNode::upd_node(table, table->left, right); + return PgfPhrasetableNode::balanceR(table); + } else { + PgfPhrasetable new_table = + PgfPhrasetableNode::upd_node(table, table->left, table->right); - size_t seq_id = seq_ids->add(table->value.seq); - int res = itor->fn(itor, seq_id, table->value.seq.as_object(), err); - if (err->type != PGF_EXN_NONE) - return; - - if (table->value.backrefs != 0 && res == 0 && callback != 0) { - for (size_t i = 0; i < table->value.n_backrefs; i++) { - PgfSequenceBackref backref = table->value.backrefs[i]; - switch (ref::get_tag(backref.container)) { - case PgfConcrLin::tag: { - ref lin = ref::untagged(backref.container); - ref lincat = - namespace_lookup(concr->lincats, &lin->absfun->type->name); - if (lincat != 0) { - ref field = - lincat->fields[backref.seq_index % lincat->fields.size()]; - - callback->fn(callback, &lin->absfun->name, &*field, lincat->abscat->prob+lin->absfun->prob, err); - if (err->type != PGF_EXN_NONE) - return; - } - break; - } - case PgfConcrLincat::tag: { - //ignore - break; - } - } + auto items = new_table->items; + if (new_table->n_items >= items.size()) { + size_t new_len = get_next_padovan(new_table->n_items+1); + items = items.realloc(new_len, new_table->txn_id); } + items[new_table->n_items] = item; + new_table->n_items++; + new_table->items = items; + return new_table; } - - phrasetable_iter(concr, table->right, itor, callback, seq_ids, err); - if (err->type != PGF_EXN_NONE) - return; } -PGF_INTERNAL -void phrasetable_release(PgfPhrasetable table) +PgfPhrasetable phrasetable_insert(PgfPhrasetable table, + ref lincat, + size_t value, size_t lin_idx, + ref item) { - if (table == 0) - return; - phrasetable_release(table->left); - phrasetable_release(table->right); - Node::release(table); + if (table == 0) { + ref symcf = PgfDB::malloc(); + symcf->lincat = lincat; + symcf->value = value; + symcf->lin_idx = lin_idx; + PgfPhrasetable new_table = PgfPhrasetableNode::new_node(symcf.tagged(),1); + new_table->n_items = 1; + new_table->items[0] = item; + return new_table; + } + + int cmp = symbol_cmp(lincat,value,lin_idx,table->sym); + if (cmp < 0) { + PgfPhrasetable left = phrasetable_insert(table->left, + lincat, value, lin_idx, item); + table = PgfPhrasetableNode::upd_node(table,left,table->right); + return PgfPhrasetableNode::balanceL(table); + } else if (cmp > 0) { + PgfPhrasetable right = phrasetable_insert(table->right, + lincat, value, lin_idx, item); + table = PgfPhrasetableNode::upd_node(table, table->left, right); + return PgfPhrasetableNode::balanceR(table); + } else { + PgfPhrasetable new_table = + PgfPhrasetableNode::upd_node(table, table->left, table->right); + + auto items = new_table->items; + if (new_table->n_items >= items.size()) { + size_t new_len = get_next_padovan(new_table->n_items+1); + items = items.realloc(new_len, new_table->txn_id); + } + items[new_table->n_items] = item; + new_table->n_items++; + new_table->items = items; + return new_table; + } } diff --git a/src/runtime/c/pgf/phrasetable.h b/src/runtime/c/pgf/phrasetable.h index eabd74abd..894fda150 100644 --- a/src/runtime/c/pgf/phrasetable.h +++ b/src/runtime/c/pgf/phrasetable.h @@ -1,138 +1,122 @@ #ifndef PHRASETABLE_H #define PHRASETABLE_H -struct PgfSequence; -struct PgfSequenceBackref; - -struct PGF_INTERNAL_DECL PgfPhrasetableEntry { - ref seq; - - // Here n_backrefs tells us how many actual backrefs there are in - // the vector backrefs. On the other hand, backrefs->len tells us - // how big buffer we have allocated. - size_t n_backrefs; - vector backrefs; -}; - -struct PgfSequenceItor; -typedef ref> PgfPhrasetable; - - -#if __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wattributes" -#endif - -struct PgfPhrasetableIds { -public: - PGF_INTERNAL_DECL PgfPhrasetableIds(); - PGF_INTERNAL_DECL ~PgfPhrasetableIds() { end(); } - - PGF_INTERNAL_DECL void start(ref concr); - PGF_INTERNAL_DECL size_t add(ref seq); - PGF_INTERNAL_DECL size_t get(ref seq); - PGF_INTERNAL_DECL void end(); - -private: - size_t next_id; - - struct PGF_INTERNAL_DECL SeqIdChain; - - struct PGF_INTERNAL_DECL SeqIdPair { - SeqIdChain *chain; - ref seq; - size_t seq_id; - }; - - struct PGF_INTERNAL_DECL SeqIdChain : public SeqIdPair { - SeqIdChain *next; - }; - - size_t n_pairs; - SeqIdPair *pairs; - SeqIdChain *chains; -}; - -#if __GNUC__ -#pragma GCC diagnostic pop -#endif - -struct PgfConcrLincat; - -PGF_INTERNAL_DECL -PgfPhrasetable phrasetable_internalize(PgfPhrasetable table, - ref seq, - ref lincat, - object container, - size_t seq_index, - ref *pentry); - -PGF_INTERNAL_DECL -ref phrasetable_relink(PgfPhrasetable table, - object container, - size_t seq_index, - size_t seq_id); - -PGF_INTERNAL_DECL -PgfPhrasetable phrasetable_delete(PgfPhrasetable table, - object container, - size_t seq_index, - ref seq); - -PGF_INTERNAL_DECL -size_t phrasetable_size(PgfPhrasetable table); - struct PgfConcrLin; +struct PgfConcrLincat; struct PGF_INTERNAL_DECL PgfTextSpot { size_t pos; // position in Unicode characters const uint8_t *ptr; // pointer into the spot location }; +struct PGF_INTERNAL_DECL PgfItem { + struct { + size_t &operator[](int i) { + PgfItem *item = containerof(PgfItem,vars,this); + return ((size_t*) (((ref*) (item+1))+item->rule->args.size()))[i]; + } + size_t size() { + PgfItem *item = containerof(PgfItem,vars,this); + return (item->rule->vars != 0) ? item->rule->vars.size() : 0; + } + } vars; + + struct { + ref &operator[](int i) { + PgfItem *item = containerof(PgfItem,args,this); + return ((ref*) (item+1))[i]; + } + size_t size() { + PgfItem *item = containerof(PgfItem,args,this); + return item->rule->args.size(); + } + } args; + + uint16_t pre_alt; + uint16_t pre_dot; + uint16_t dot; + ref rule; +}; + +struct PgfPhrasetableNode; +typedef ref PgfPhrasetable; + +struct PGF_INTERNAL_DECL PgfPhrasetableNode { + const static size_t DELTA = 3; + const static size_t RATIO = 2; + +public: + PgfSymbol sym; + + // Here n_items tells us how many actual items there are in + // the vector items. On the other hand, items.size() tells us + // how big buffer we have allocated. + size_t n_items; + vector> items; + + txn_t txn_id; + + size_t sz; + ref left; + ref right; + + static + ref new_node(PgfSymbol sym, size_t n_items); + + static + ref upd_node(ref node, ref left, ref right); + + static + ref balanceL(ref node); + + static + ref balanceR(ref node); + + static + size_t size(ref node) + { + if (node == 0) + return 0; + return node->sz; + } + + static + void release(ref node); +}; + +PgfPhrasetable phrasetable_insert(PgfPhrasetable table, + PgfSymbol sym, + ref item); + +PgfPhrasetable phrasetable_insert(PgfPhrasetable table, + ref lincat, + size_t value, size_t lin_idx, + ref item); + +PGF_INTERNAL_DECL +void phrasetable_iter(PgfPhrasetable phrasetable,ref lincat,std::function symcf,size_t,vector>)> &f); + +PGF_INTERNAL_DECL +vector> phrasetable_lookup(PgfPhrasetable phrasetable, PgfSymbol sym, size_t *n_items); + class PGF_INTERNAL_DECL PgfPhraseScanner { public: virtual void space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err)=0; virtual void start_matches(PgfTextSpot *spot, PgfExn* err)=0; - virtual void match(ref lin, size_t seq_index, PgfExn* err)=0; + virtual void match(ref lin, size_t lin_idx, PgfExn* err)=0; virtual void end_matches(PgfTextSpot *spot, PgfExn* err)=0; }; PGF_INTERNAL_DECL -void phrasetable_lookup(PgfPhrasetable table, +void phrasetable_lookup(PgfPhrasetable phrasetable, PgfText *sentence, bool case_sensitive, PgfPhraseScanner *scanner, PgfExn* err); PGF_INTERNAL_DECL -void phrasetable_lookup_cohorts(PgfPhrasetable table, +void phrasetable_lookup_cohorts(PgfPhrasetable phrasetable, PgfText *sentence, bool case_sensitive, PgfPhraseScanner *scanner, PgfExn* err); -PGF_INTERNAL_DECL -void phrasetable_iter(PgfConcr *concr, - PgfPhrasetable table, - PgfSequenceItor* itor, - PgfMorphoCallback *callback, - PgfPhrasetableIds *seq_ids, PgfExn *err); - -PGF_INTERNAL_DECL -void phrasetable_release(PgfPhrasetable table); - -// The following are used internally in the parser - -enum SeqMatch { SM_FULL_MATCH, SM_PREFIX, SM_PARTIAL }; - -PGF_INTERNAL_DECL -int text_sequence_cmp(PgfTextSpot *spot, const uint8_t *end, - ref seq, size_t *p_i, - bool case_sensitive, SeqMatch sm); - -// The following is used internally in the grammar builder - -PGF_INTERNAL_DECL -void phrasetable_add_backref(ref entry, txn_t txn_id, - object container, - size_t seq_index); - #endif diff --git a/src/runtime/c/pgf/printer.cxx b/src/runtime/c/pgf/printer.cxx index 0add283eb..444aceefe 100644 --- a/src/runtime/c/pgf/printer.cxx +++ b/src/runtime/c/pgf/printer.cxx @@ -545,11 +545,11 @@ void PgfPrinter::symbol(PgfSymbol sym) auto sym_kp = ref::untagged(sym); puts("pre {"); - sequence(sym_kp->default_form); + symbols(sym_kp->default_form); for (size_t i = 0; i < sym_kp->alts.size(); i++) { puts("; "); - sequence(sym_kp->alts[i].form); + symbols(sym_kp->alts[i].form); puts(" /"); for (size_t j = 0; j < sym_kp->alts[i].prefixes.size(); j++) { puts(" "); @@ -578,22 +578,100 @@ void PgfPrinter::symbol(PgfSymbol sym) case PgfSymbolALLCAPIT::tag: puts("ALL_CAPIT"); break; + case PgfSymbolACat::tag: { + auto symcf = ref::untagged(sym); + efun(&symcf->name); + break; + } + case PgfSymbolCCat::tag: { + auto symcf = ref::untagged(sym); + efun(&symcf->lincat->name); + nprintf(64,"(%zu,%zu)",symcf->value,symcf->lin_idx); + break; + } } } -void PgfPrinter::sequence(ref seq) +void PgfPrinter::symbols(vector syms) { - for (size_t i = 0; i < seq->syms.size(); i++) { + for (size_t i = 0; i < syms.size(); i++) { if (i > 0) puts(" "); - symbol(seq->syms[i]); + symbol(syms[i]); } } -void PgfPrinter::seq_id(PgfPhrasetableIds *seq_ids, ref seq) +void PgfPrinter::item(ref item) { - nprintf(5, "S%zu", seq_ids->get(seq)); + switch (ref::get_tag(item->rule->container)) { + case PgfConcrLincat::tag: { + ref lincat = ref::untagged(item->rule->container); + + if (item->rule->vars != 0) { + lvar_ranges(item->rule->vars, &item->vars[0]); + puts(" "); + } + + puts("String("); + lparam(item->rule->res); + puts(") -> "); + + efun(&lincat->name); + puts("["); + efun(&lincat->name); + puts("("); + lparam(item->rule->args[0]); + puts(")]; "); + break; + } + case PgfConcrLin::tag: { + ref lin = ref::untagged(item->rule->container); + ref ty = lin->absfun->type; + + if (item->rule->vars != 0) { + lvar_ranges(item->rule->vars, &item->vars[0]); + puts(" "); + } + + efun(&ty->name); + puts("("); + lparam(item->rule->res); + puts(") -> "); + + efun(&lin->name); + puts("["); + for (size_t i = 0; i < item->rule->args.size(); i++) { + if (i > 0) + puts(","); + if (item->args[i] == 0) { + efun(&ty->hypos.elem(i)->type->name); + puts("("); + lparam(item->rule->args[i]); + puts(")"); + } else { + emeta(0); + } + } + puts("]; "); + break; + } + } + + lparam(item->rule->lin_idx); + puts(" : "); + + for (size_t i = 0; i < item->rule->syms.size(); i++) { + if (i > 0) + puts(" "); + + if (item->pre_alt == 0 && item->dot == i) + puts(". "); + else if (item->pre_alt > 0 && item->pre_dot == i) + puts(". "); + + symbol(item->rule->syms[i]); + } } void PgfPrinter::free_ref(object x) diff --git a/src/runtime/c/pgf/printer.h b/src/runtime/c/pgf/printer.h index 9cd209605..54bcfd753 100644 --- a/src/runtime/c/pgf/printer.h +++ b/src/runtime/c/pgf/printer.h @@ -79,9 +79,9 @@ public: void lvar(size_t var); void lparam(ref lparam); void lvar_ranges(vector vars, size_t *values); - void seq_id(PgfPhrasetableIds *seq_ids, ref seq); void symbol(PgfSymbol sym); - void sequence(ref seq); + void symbols(vector syms); + void item(ref item); virtual PgfExpr eabs(PgfBindType btype, PgfText *name, PgfExpr body); virtual PgfExpr eapp(PgfExpr fun, PgfExpr arg); diff --git a/src/runtime/c/pgf/reader.cxx b/src/runtime/c/pgf/reader.cxx index aa9df3bdb..53b08e0f0 100644 --- a/src/runtime/c/pgf/reader.cxx +++ b/src/runtime/c/pgf/reader.cxx @@ -10,6 +10,7 @@ PgfReader::PgfReader(FILE *in,PgfProbsCallback *probs_callback) this->probs_callback = probs_callback; this->abstract = 0; this->concrete = 0; + this->container = 0; } uint8_t PgfReader::read_uint8() @@ -161,6 +162,21 @@ ref PgfReader::read_vector(inline_vector C::* field, void (PgfReader::*rea return loc; } +template +vector PgfReader::read_null_vector(void (PgfReader::*read_value)(ref val)) +{ + size_t len = read_len(); + if (len == 0) { + return 0; + } else { + vector vec = vector::alloc(len); + for (size_t i = 0; i < len; i++) { + (this->*read_value)(vec.elem(i)); + } + return vec; + } +} + template vector PgfReader::read_vector(void (PgfReader::*read_value)(ref val)) { @@ -572,14 +588,14 @@ PgfSymbol PgfReader::read_symbol() ref sym_kp = inline_vector::alloc(&PgfSymbolKP::alts,n_alts); for (size_t i = 0; i < n_alts; i++) { - auto form = read_seq(); + auto form = read_vector(&PgfReader::read_symbol2); auto prefixes = read_vector(&PgfReader::read_text2); sym_kp->alts[i].form = form; sym_kp->alts[i].prefixes = prefixes; } - auto default_form = read_seq(); + auto default_form = read_vector(&PgfReader::read_symbol2); sym_kp->default_form = default_form; sym = sym_kp.tagged(); @@ -616,80 +632,50 @@ PgfSymbol PgfReader::read_symbol() return sym; } -ref PgfReader::read_seq() +ref PgfReader::read_rule() { - size_t n_syms = read_len(); + size_t n_syms = read_len(); + ref rule = inline_vector::alloc(&PgfConcrRule::syms, n_syms); - ref seq = inline_vector::alloc(&PgfSequence::syms, n_syms); + vector vars = read_null_vector(&PgfReader::read_variable_range); + ref res = read_lparam(); + vector> args = read_null_vector(&PgfReader::read_lparam); + ref lin_idx = read_lparam(); + + rule->vars = vars; + rule->res = res; + rule->container = container; + rule->args = args; + rule->lin_idx = lin_idx; for (size_t i = 0; i < n_syms; i++) { PgfSymbol sym = read_symbol(); - seq->syms[i] = sym; + rule->syms[i] = sym; } - return seq; -} - -vector> PgfReader::read_seq_ids(object container) -{ - size_t len = read_len(); - vector> vec = vector>::alloc(len); - for (size_t i = 0; i < len; i++) { - size_t seq_id = read_len(); - ref seq = phrasetable_relink(concrete->phrasetable, - container, i, - seq_id); - if (seq == 0) { - throw pgf_error("Invalid sequence id"); - } - vec[i] = seq; - } - return vec; -} - -PgfPhrasetable PgfReader::read_phrasetable(size_t len) -{ - if (len == 0) - return 0; - - PgfPhrasetableEntry value; - - size_t half = len/2; - PgfPhrasetable left = read_phrasetable(half); - value.seq = read_seq(); - value.n_backrefs = 0; - value.backrefs = 0; - PgfPhrasetable right = read_phrasetable(len-half-1); - - PgfPhrasetable table = Node::new_node(value); - table->sz = 1+Node::size(left)+Node::size(right); - table->left = left; - table->right = right; - return table; -} - -PgfPhrasetable PgfReader::read_phrasetable() -{ - size_t len = read_len(); - return read_phrasetable(len); + return rule; } ref PgfReader::read_lincat() { ref lincat = read_name(&PgfConcrLincat::name); + container = lincat.tagged(); + auto fields = read_lincat_fields(lincat); auto n_lindefs = read_len(); - auto args = read_vector(&PgfReader::read_parg); - auto res = read_vector(&PgfReader::read_presult2); - auto seqs = read_seq_ids(lincat.tagged()); + auto rules = read_vector(&PgfReader::read_rule2); + + container = 0; + + for (size_t i = n_lindefs; i < rules.size(); i++) { + table_maker->insert_rule(rules[i]); + } lincat->abscat = namespace_lookup(abstract->cats, &lincat->name); lincat->fields = fields; lincat->n_lindefs = n_lindefs; - lincat->args = args; - lincat->res = res; - lincat->seqs = seqs; + lincat->rules = rules; return lincat; } @@ -715,13 +701,16 @@ ref PgfReader::read_lin() if (lin->lincat == 0) throw pgf_error("Found a lin which uses a category without a lincat"); - auto args = read_vector(&PgfReader::read_parg); - auto res = read_vector(&PgfReader::read_presult2); - auto seqs = read_seq_ids(lin.tagged()); + container = lin.tagged(); - lin->args = args; - lin->res = res; - lin->seqs = seqs; + auto rules = read_vector(&PgfReader::read_rule2); + lin->rules = rules; + + container = 0; + + for (size_t i = 0; i < rules.size(); i++) { + table_maker->insert_rule(rules[i]); + } return lin; } @@ -740,8 +729,8 @@ ref PgfReader::read_concrete() auto cflags = read_namespace(&PgfReader::read_flag); concrete->cflags = cflags; - auto phrasetable = read_phrasetable(); - concrete->phrasetable = phrasetable; + PgfParseTableMaker tm(concrete); + this->table_maker = &tm; auto lincats = read_namespace(&PgfReader::read_lincat); concrete->lincats = lincats; @@ -749,12 +738,11 @@ ref PgfReader::read_concrete() auto lins = read_namespace(&PgfReader::read_lin); concrete->lins = lins; + this->table_maker = NULL; + auto printnames = read_namespace(&PgfReader::read_printname); concrete->printnames = printnames; - //PgfLRTableMaker maker(abstract, concrete); - //concrete->lrtable = maker.make(); - return concrete; } diff --git a/src/runtime/c/pgf/reader.h b/src/runtime/c/pgf/reader.h index 74902a6a7..2167e7214 100644 --- a/src/runtime/c/pgf/reader.h +++ b/src/runtime/c/pgf/reader.h @@ -51,6 +51,9 @@ public: template ref read_vector(inline_vector C::* field, void (PgfReader::*read_value)(ref val)); + template + vector read_null_vector(void (PgfReader::*read_value)(ref val)); + template vector read_vector(void (PgfReader::*read_value)(ref val)); @@ -70,6 +73,7 @@ public: void read_abstract(ref abstract); void merge_abstract(ref abstract); + ref read_rule(); ref read_lincat(); vector> read_lincat_fields(ref lincat); ref read_lparam(); @@ -77,10 +81,6 @@ public: void read_parg(ref parg); ref read_presult(); PgfSymbol read_symbol(); - ref read_seq(); - vector> read_seq_ids(object container); - PgfPhrasetable read_phrasetable(size_t len); - PgfPhrasetable read_phrasetable(); ref read_lin(); ref read_printname(); @@ -94,6 +94,9 @@ private: PgfProbsCallback *probs_callback; ref abstract; ref concrete; + object container; + + class PgfParseTableMaker *table_maker; object read_name_internal(size_t struct_size); object read_text_internal(size_t struct_size); @@ -101,6 +104,8 @@ private: void read_text2(ref> r) { auto text = read_text(); *r = text; } void read_lparam(ref> r) { auto lparam = read_lparam(); *r = lparam; } void read_presult2(ref> r) { auto res = read_presult(); *r = res; } + void read_rule2(ref> r) { auto rule = read_rule(); *r = rule; } + void read_symbol2(ref r) { auto sym = read_symbol(); *r = sym; } template ref read_symbol_idx(); diff --git a/src/runtime/c/pgf/writer.cxx b/src/runtime/c/pgf/writer.cxx index dfd881995..cfc05b1d6 100644 --- a/src/runtime/c/pgf/writer.cxx +++ b/src/runtime/c/pgf/writer.cxx @@ -144,6 +144,19 @@ void PgfWriter::write_vector(vector vec, void (PgfWriter::*write_value)(ref +void PgfWriter::write_null_vector(vector vec, void (PgfWriter::*write_value)(ref val)) +{ + if (vec == 0) { + write_len(0); + } else { + write_len(vec.size()); + for (size_t i = 0; i < vec.size(); i++) { + (this->*write_value)(vec.elem(i)); + } + } +} + void PgfWriter::write_literal(PgfLiteral literal) { auto tag = ref::get_tag(literal); @@ -293,18 +306,19 @@ void PgfWriter::write_lparam(ref lparam) } } -void PgfWriter::write_parg(ref parg) +void PgfWriter::write_rule(ref rule) { - write_lparam(parg->param); -} + write_len(rule->syms.size()); -void PgfWriter::write_presult(ref pres) -{ - if (pres->vars != 0) - write_vector(pres->vars, &PgfWriter::write_variable_range); - else - write_len(0); - write_lparam(ref::from_ptr(&pres->param)); + write_null_vector(rule->vars, &PgfWriter::write_variable_range); + write_lparam(rule->res); + write_null_vector(rule->args, &PgfWriter::write_lparam); + + write_lparam(rule->lin_idx); + + for (PgfSymbol sym : rule->syms) { + write_symbol(sym); + } } void PgfWriter::write_symbol(PgfSymbol sym) @@ -341,10 +355,10 @@ void PgfWriter::write_symbol(PgfSymbol sym) write_len(sym_kp->alts.size()); for (size_t i = 0; i < sym_kp->alts.size(); i++) { ref alt = sym_kp->alts.elem(i); - write_vector(alt->form->syms.as_vector(), &PgfWriter::write_symbol); + write_vector(alt->form, &PgfWriter::write_symbol); write_vector(alt->prefixes, &PgfWriter::write_text); } - write_vector(sym_kp->default_form->syms.as_vector(), &PgfWriter::write_symbol); + write_vector(sym_kp->default_form, &PgfWriter::write_symbol); break; } case PgfSymbolBIND::tag: @@ -359,36 +373,12 @@ void PgfWriter::write_symbol(PgfSymbol sym) } } -void PgfWriter::write_seq(ref seq) -{ - seq_ids.add(seq); - write_vector(seq->syms.as_vector(), &PgfWriter::write_symbol); -} - -void PgfWriter::write_phrasetable(PgfPhrasetable table) -{ - write_len(phrasetable_size(table)); - write_phrasetable_helper(table); -} - -void PgfWriter::write_phrasetable_helper(PgfPhrasetable table) -{ - if (table == 0) - return; - - write_phrasetable_helper(table->left); - write_seq(table->value.seq); - write_phrasetable_helper(table->right); -} - void PgfWriter::write_lincat(ref lincat) { write_name(&lincat->name); write_vector(lincat->fields, &PgfWriter::write_lincat_field); write_len(lincat->n_lindefs); - write_vector(lincat->args, &PgfWriter::write_parg); - write_vector(lincat->res, &PgfWriter::write_presult); - write_vector(lincat->seqs, &PgfWriter::write_seq_id); + write_vector(lincat->rules, &PgfWriter::write_rule); } void PgfWriter::write_lincat_field(ref> field) @@ -399,9 +389,7 @@ void PgfWriter::write_lincat_field(ref> field) void PgfWriter::write_lin(ref lin) { write_name(&lin->name); - write_vector(lin->args, &PgfWriter::write_parg); - write_vector(lin->res, &PgfWriter::write_presult); - write_vector(lin->seqs, &PgfWriter::write_seq_id); + write_vector(lin->rules, &PgfWriter::write_rule); } void PgfWriter::write_printname(ref printname) @@ -428,16 +416,11 @@ void PgfWriter::write_concrete(ref concr) } } - seq_ids.start(concr); - write_name(&concr->name); write_namespace(concr->cflags, &PgfWriter::write_flag); - write_phrasetable(concr->phrasetable); write_namespace(concr->lincats, &PgfWriter::write_lincat); write_namespace(concr->lins, &PgfWriter::write_lin); write_namespace(concr->printnames, &PgfWriter::write_printname); - - seq_ids.end(); } void PgfWriter::write_pgf(ref pgf) diff --git a/src/runtime/c/pgf/writer.h b/src/runtime/c/pgf/writer.h index 4625f41c1..5bb8bb881 100644 --- a/src/runtime/c/pgf/writer.h +++ b/src/runtime/c/pgf/writer.h @@ -24,6 +24,8 @@ public: template void write_vector(vector vec, void (PgfWriter::*write_value)(ref val)); + template + void write_null_vector(vector vec, void (PgfWriter::*write_value)(ref val)); void write_literal(PgfLiteral literal); void write_expr(PgfExpr expr); @@ -42,12 +44,7 @@ public: void write_lincat_field(ref> field); void write_variable_range(ref var); void write_lparam(ref lparam); - void write_parg(ref linarg); - void write_presult(ref linres); void write_symbol(PgfSymbol sym); - void write_seq(ref seq); - void write_seq_id(ref> r) { write_len(seq_ids.get(*r)); }; - void write_phrasetable(PgfPhrasetable table); void write_lin(ref lin); void write_printname(ref printname); @@ -58,18 +55,17 @@ public: private: template void write_namespace_helper(Namespace nmsp, void (PgfWriter::*write_value)(ref)); - void write_phrasetable_helper(PgfPhrasetable table); void write_text(ref> r) { write_text(&(**r)); }; void write_lparam(ref> r) { write_lparam(*r); }; + void write_rule(ref rule); void write_symbol(ref r) { write_symbol(*r); }; - void write_presult(ref> r) { write_presult(*r); }; + void write_rule(ref> r) { write_rule(*r); }; FILE *out; PgfText **langs; ref abstract; - PgfPhrasetableIds seq_ids; }; #endif diff --git a/src/runtime/haskell/PGF2.hsc b/src/runtime/haskell/PGF2.hsc index 6870a725f..e81030fcf 100644 --- a/src/runtime/haskell/PGF2.hsc +++ b/src/runtime/haskell/PGF2.hsc @@ -73,7 +73,7 @@ module PGF2 (-- * PGF graphvizAbstractTree, graphvizParseTree, Labels, getDepLabels, graphvizDependencyTree, conlls2latexDoc, getCncDepLabels, - graphvizWordAlignment, graphvizLRAutomaton, + graphvizWordAlignment, -- * Concrete syntax ConcName,Concr,languages,language,concreteName,languageCode,concreteFlag, @@ -363,19 +363,14 @@ showPGF p = modifyIORef ref (\doc -> doc $$ text def) ppConcr name c = unsafePerformIO $ do - (seq_ids,doc3) <- prepareSequences c -- run first to update all seq_id - doc1 <- ppLincats seq_ids c - doc2 <- ppLins seq_ids c - pgf_release_phrasetable_ids seq_ids + doc1 <- ppLincats c + doc2 <- ppLins c return (text "concrete" <+> text name <+> char '{' $$ nest 2 (doc1 $$ - doc2 $$ - (text "sequences" <+> char '{' $$ - nest 2 doc3 $$ - char '}')) $$ + doc2) $$ char '}') - ppLincats seq_ids c = do + ppLincats c = do ref <- newIORef empty (allocaBytes (#size PgfItor) $ \itor -> bracket (wrapItorCallback (getLincats ref)) freeHaskellFunPtr $ \fptr -> @@ -402,15 +397,15 @@ showPGF p = char ']') modifyIORef ref $ (\doc -> doc $$ def) forM_ (init [0..n_lindefs]) $ \i -> do - def <- bracket (pgf_print_lindef_internal seq_ids val i) free $ \c_text -> do + def <- bracket (pgf_print_lindef_internal val i) free $ \c_text -> do fmap text (peekText c_text) modifyIORef ref (\doc -> doc $$ text "lindef" <+> def) forM_ (init [0..n_linrefs]) $ \i -> do - def <- bracket (pgf_print_linref_internal seq_ids val i) free $ \c_text -> do + def <- bracket (pgf_print_linref_internal val i) free $ \c_text -> do fmap text (peekText c_text) modifyIORef ref $ (\doc -> doc $$ text "linref" <+> def) - ppLins seq_ids c = do + ppLins c = do ref <- newIORef empty (allocaBytes (#size PgfItor) $ \itor -> bracket (wrapItorCallback (getLins ref)) freeHaskellFunPtr $ \fptr -> @@ -421,30 +416,13 @@ showPGF p = where getLins :: IORef Doc -> ItorCallback getLins ref itor key val exn = do - n_prods <- pgf_get_lin_get_prod_count val + n_prods <- pgf_get_lin_rules_count val forM_ (init [0..n_prods]) $ \i -> do - def <- bracket (pgf_print_lin_internal seq_ids val i) free $ \c_text -> do + def <- bracket (pgf_print_lin_internal val i) free $ \c_text -> do fmap text (peekText c_text) modifyIORef ref (\doc -> doc $$ text "lin" <+> def) return () - prepareSequences c = do - ref <- newIORef empty - seq_ids <- (allocaBytes (#size PgfSequenceItor) $ \itor -> - bracket (wrapSequenceItorCallback (getSequences ref)) freeHaskellFunPtr $ \fptr -> - withForeignPtr (c_revision c) $ \c_revision -> do - (#poke PgfSequenceItor, fn) itor fptr - withPgfExn "showPGF" (pgf_iter_sequences (a_db p) c_revision itor nullPtr)) - doc <- readIORef ref - return (seq_ids, doc) - where - getSequences :: IORef Doc -> SequenceItorCallback - getSequences ref itor seq_id val exn = do - def <- bracket (pgf_print_sequence_internal seq_id val) free $ \c_text -> do - fmap text (peekText c_text) - modifyIORef ref $ (\doc -> doc $$ def) - return 0 - -- | The abstract language name is the name of the top-level -- abstract module abstractName :: PGF -> AbsName @@ -830,8 +808,7 @@ fullFormLexicon c = unsafePerformIO $ do withForeignPtr (c_revision c) $ \c_revision -> do (#poke PgfSequenceItor, fn) itor1 fptr1 (#poke PgfMorphoCallback, fn) itor2 fptr2 - seq_ids <- withPgfExn "fullFormLexicon" (pgf_iter_sequences (c_db c) c_revision itor1 itor2) - pgf_release_phrasetable_ids seq_ids) + withPgfExn "fullFormLexicon" (pgf_iter_sequences (c_db c) c_revision itor1 itor2)) fmap (reverse2 []) (readIORef ref) where getSequences ref _ seq_id val exn = do @@ -1484,15 +1461,6 @@ graphvizDependencyTree -> String -- ^ Rendered output in the specified format graphvizDependencyTree format debug mlab mclab concr t = error "TODO: graphvizDependencyTree" -graphvizLRAutomaton :: Concr -> String -graphvizLRAutomaton c = - unsafePerformIO $ - withForeignPtr (c_revision c) $ \c_revision -> - bracket (withPgfExn "graphvizLRAutomaton" (pgf_graphviz_lr_automaton (c_db c) c_revision)) free $ \c_text -> - if c_text == nullPtr - then return "" - else peekText c_text - ---------------------- should be a separate module? -- visualization with latex output. AR Nov 2015 diff --git a/src/runtime/haskell/PGF2/FFI.hsc b/src/runtime/haskell/PGF2/FFI.hsc index 2030846bd..1ad73a18f 100644 --- a/src/runtime/haskell/PGF2/FFI.hsc +++ b/src/runtime/haskell/PGF2/FFI.hsc @@ -48,7 +48,6 @@ data PgfSequenceItor data PgfProbsCallback data PgfMorphoCallback data PgfCohortsCallback -data PgfPhrasetableIds data PgfExprEnum data PgfAlignmentPhrase @@ -150,26 +149,22 @@ foreign import ccall "wrapper" wrapCohortsCallback :: Wrapper CohortsCallback foreign import ccall pgf_lookup_cohorts :: Ptr PgfDB -> Ptr Concr -> Ptr PgfText -> Ptr PgfCohortsCallback -> Ptr PgfExn -> IO () -foreign import ccall pgf_iter_sequences :: Ptr PgfDB -> Ptr Concr -> Ptr PgfSequenceItor -> Ptr PgfMorphoCallback -> Ptr PgfExn -> IO (Ptr PgfPhrasetableIds) +foreign import ccall pgf_iter_sequences :: Ptr PgfDB -> Ptr Concr -> Ptr PgfSequenceItor -> Ptr PgfMorphoCallback -> Ptr PgfExn -> IO () foreign import ccall pgf_get_lincat_counts_internal :: Ptr () -> Ptr CSize -> IO () foreign import ccall pgf_get_lincat_field_internal :: Ptr () -> CSize -> IO (Ptr PgfText) -foreign import ccall pgf_print_lindef_internal :: Ptr PgfPhrasetableIds -> Ptr () -> CSize -> IO (Ptr PgfText) +foreign import ccall pgf_print_lindef_internal :: Ptr () -> CSize -> IO (Ptr PgfText) -foreign import ccall pgf_print_linref_internal :: Ptr PgfPhrasetableIds -> Ptr () -> CSize -> IO (Ptr PgfText) +foreign import ccall pgf_print_linref_internal :: Ptr () -> CSize -> IO (Ptr PgfText) -foreign import ccall pgf_get_lin_get_prod_count :: Ptr () -> IO CSize +foreign import ccall pgf_get_lin_rules_count :: Ptr () -> IO CSize -foreign import ccall pgf_print_lin_internal :: Ptr PgfPhrasetableIds -> Ptr () -> CSize -> IO (Ptr PgfText) - -foreign import ccall pgf_print_sequence_internal :: CSize -> Ptr () -> IO (Ptr PgfText) +foreign import ccall pgf_print_lin_internal :: Ptr () -> CSize -> IO (Ptr PgfText) foreign import ccall pgf_sequence_get_text_internal :: Ptr () -> IO (Ptr PgfText) -foreign import ccall pgf_release_phrasetable_ids :: Ptr PgfPhrasetableIds -> IO () - type ItorCallback = Ptr PgfItor -> Ptr PgfText -> Ptr () -> Ptr PgfExn -> IO () foreign import ccall "wrapper" wrapItorCallback :: Wrapper ItorCallback @@ -244,7 +239,7 @@ foreign import ccall "dynamic" callLinBuilder1 :: Dynamic (Ptr PgfLinBuilderIfac foreign import ccall "dynamic" callLinBuilder2 :: Dynamic (Ptr PgfLinBuilderIface -> CSize -> CSize -> Ptr PgfExn -> IO ()) -foreign import ccall "dynamic" callLinBuilder3 :: Dynamic (Ptr PgfLinBuilderIface -> CSize -> CSize -> CSize -> Ptr CSize -> Ptr PgfExn -> IO ()) +foreign import ccall "dynamic" callLinBuilder3 :: Dynamic (Ptr PgfLinBuilderIface -> CSize -> CSize -> Ptr CSize -> Ptr PgfExn -> IO ()) foreign import ccall "dynamic" callLinBuilder4 :: Dynamic (Ptr PgfLinBuilderIface -> CSize -> CSize -> CSize -> Ptr CSize -> Ptr PgfExn -> IO ()) @@ -318,8 +313,6 @@ foreign import ccall pgf_graphviz_parse_tree :: Ptr PgfDB -> Ptr Concr -> Stable foreign import ccall pgf_graphviz_word_alignment :: Ptr PgfDB -> Ptr (Ptr Concr) -> CSize -> StablePtr Expr -> Ptr PgfPrintContext -> Ptr PgfMarshaller -> Ptr PgfGraphvizOptions -> Ptr PgfExn -> IO (Ptr PgfText) -foreign import ccall pgf_graphviz_lr_automaton :: Ptr PgfDB -> Ptr Concr -> Ptr PgfExn -> IO (Ptr PgfText) - ----------------------------------------------------------------------- -- Texts diff --git a/src/runtime/haskell/PGF2/Transactions.hsc b/src/runtime/haskell/PGF2/Transactions.hsc index 51aca6082..d3a197a3b 100644 --- a/src/runtime/haskell/PGF2/Transactions.hsc +++ b/src/runtime/haskell/PGF2/Transactions.hsc @@ -1,3 +1,4 @@ +{-# LANGUAGE ScopedTypeVariables #-} module PGF2.Transactions ( -- transactions TxnID @@ -18,15 +19,14 @@ module PGF2.Transactions , setAbstractFlag -- concrete syntax - , Token, SeqId, LIndex, LVar, LParam(..) - , PArg(..), Symbol(..), Production(..) + , Token, LIndex, LVar, LParam(..) + , PArg(..), Symbol(..), Rule(..) , createConcrete , alterConcrete , dropConcrete , mergePGF , setConcreteFlag - , SeqTable , createLincat , dropLincat , createLin, alterLin @@ -251,21 +251,23 @@ data Symbol | SymALL_CAPIT -- the special ALL_CAPIT token deriving (Eq,Ord,Show) +type Quantifiers = [(LVar,Int)] +data Rule = Rule Quantifiers LParam [LParam] LParam [Symbol] + deriving (Eq,Show) + data PArg = PArg [(LIndex,LIndex)] {-# UNPACK #-} !LParam deriving (Eq,Show) data Production = Production [(LVar,LIndex)] [PArg] LParam [SeqId] deriving (Eq,Show) -type SeqTable = Seq.Seq (Either [Symbol] SeqId) - -createLincat :: Cat -> [String] -> [Production] -> [Production] -> SeqTable -> Transaction Concr SeqTable -createLincat name fields lindefs linrefs seqtbl = Transaction $ \c_db c_abstr c_revision c_exn -> +createLincat :: Cat -> [String] -> [Rule] -> [Rule] -> Transaction Concr () +createLincat name fields lindefs linrefs = Transaction $ \c_db c_abstr c_revision c_exn -> let n_fields = length fields in withText name $ \c_name -> allocaBytes (n_fields*(#size PgfText*)) $ \c_fields -> withTexts c_fields 0 fields $ - withBuildLinIface (lindefs++linrefs) seqtbl $ \c_build -> + withBuildLinIface (lindefs++linrefs) $ \c_build -> pgf_create_lincat c_db c_abstr c_revision c_name (fromIntegral n_fields) c_fields (fromIntegral (length lindefs)) (fromIntegral (length linrefs)) @@ -282,27 +284,25 @@ dropLincat name = Transaction $ \c_db c_abstr c_revision c_exn -> withText name $ \c_name -> pgf_drop_lincat c_db c_abstr c_revision c_name c_exn -createLin :: Fun -> [Production] -> SeqTable -> Transaction Concr SeqTable -createLin name prods seqtbl = Transaction $ \c_db c_abstr c_revision c_exn -> +createLin :: Fun -> [Rule] -> Transaction Concr () +createLin name rules = Transaction $ \c_db c_abstr c_revision c_exn -> withText name $ \c_name -> - withBuildLinIface prods seqtbl $ \c_build -> - pgf_create_lin c_db c_abstr c_revision c_name (fromIntegral (length prods)) c_build c_exn + withBuildLinIface rules $ \c_build -> + pgf_create_lin c_db c_abstr c_revision c_name (fromIntegral (length rules)) c_build c_exn -alterLin :: Fun -> [Production] -> SeqTable -> Transaction Concr SeqTable -alterLin name prods seqtbl = Transaction $ \c_db c_abstr c_revision c_exn -> +alterLin :: Fun -> [Rule] -> Transaction Concr () +alterLin name rules = Transaction $ \c_db c_abstr c_revision c_exn -> withText name $ \c_name -> - withBuildLinIface prods seqtbl $ \c_build -> - pgf_alter_lin c_db c_abstr c_revision c_name (fromIntegral (length prods)) c_build c_exn + withBuildLinIface rules $ \c_build -> + pgf_alter_lin c_db c_abstr c_revision c_name (fromIntegral (length rules)) c_build c_exn -withBuildLinIface prods seqtbl f = do - ref <- newIORef seqtbl +withBuildLinIface rules f = do (allocaBytes (#size PgfBuildLinIface) $ \c_build -> allocaBytes (#size PgfBuildLinIfaceVtbl) $ \vtbl -> - bracket (wrapLinBuild (build ref)) freeHaskellFunPtr $ \c_callback -> do + bracket (wrapLinBuild build) freeHaskellFunPtr $ \c_callback -> do (#poke PgfBuildLinIface, vtbl) c_build vtbl (#poke PgfBuildLinIfaceVtbl, build) vtbl c_callback f c_build) - readIORef ref where forM_ [] c_exn f = return () forM_ (x:xs) c_exn f = do @@ -311,31 +311,23 @@ withBuildLinIface prods seqtbl f = do then f x >> forM_ xs c_exn f else return () - build ref _ c_builder c_exn = do + build _ c_builder c_exn = do vtbl <- (#peek PgfLinBuilderIface, vtbl) c_builder - forM_ prods c_exn $ \(Production vars args res seqids) -> do - fun <- (#peek PgfLinBuilderIfaceVtbl, start_production) vtbl - callLinBuilder0 fun c_builder c_exn + forM_ rules c_exn $ \(Rule vars res args lin_idx seq) -> do + fun <- (#peek PgfLinBuilderIfaceVtbl, start_rule) vtbl + callLinBuilder2 fun c_builder (fromIntegral (length vars)) (fromIntegral (length seq)) c_exn fun <- (#peek PgfLinBuilderIfaceVtbl, add_argument) vtbl - forM_ args c_exn $ \(PArg hypos param) -> - callLParam (callLinBuilder3 fun c_builder (fromIntegral (length hypos))) param c_exn - fun <- (#peek PgfLinBuilderIfaceVtbl, set_result) vtbl - callLParam (callLinBuilder3 fun c_builder (fromIntegral (length vars))) res c_exn + forM_ args c_exn $ \arg -> + callLParam (callLinBuilder3 fun c_builder) arg c_exn + fun <- (#peek PgfLinBuilderIfaceVtbl, set_result) vtbl + callLParam (callLinBuilder3 fun c_builder) res c_exn + fun <- (#peek PgfLinBuilderIfaceVtbl, set_lin_idx) vtbl + callLParam (callLinBuilder3 fun c_builder) lin_idx c_exn fun <- (#peek PgfLinBuilderIfaceVtbl, add_variable) vtbl forM_ vars c_exn $ \(v,r) -> callLinBuilder2 fun c_builder (fromIntegral v) (fromIntegral r) c_exn - fun <- (#peek PgfLinBuilderIfaceVtbl, add_sequence_id) vtbl - seqtbl <- readIORef ref - forM_ seqids c_exn $ \seqid -> - case Seq.index seqtbl seqid of - Left syms -> do fun <- (#peek PgfLinBuilderIfaceVtbl, start_sequence) vtbl - callLinBuilder1 fun c_builder (fromIntegral (length syms)) c_exn - forM_ syms c_exn (addSymbol c_builder vtbl c_exn) - fun <- (#peek PgfLinBuilderIfaceVtbl, end_sequence) vtbl - seqid' <- callLinBuilder7 fun c_builder c_exn - writeIORef ref $! Seq.update seqid (Right (fromIntegral seqid')) seqtbl - Right seqid -> do callLinBuilder1 fun c_builder (fromIntegral seqid) c_exn - fun <- (#peek PgfLinBuilderIfaceVtbl, end_production) vtbl + forM_ seq c_exn (addSymbol c_builder vtbl c_exn) + fun <- (#peek PgfLinBuilderIfaceVtbl, end_rule) vtbl callLinBuilder0 fun c_builder c_exn addSymbol c_builder vtbl c_exn (SymCat d r) = do