an interval based parser

This commit is contained in:
Krasimir Angelov
2026-01-12 14:36:41 +01:00
parent fbfb54c9b2
commit 21f4c009ab
9 changed files with 1095 additions and 593 deletions
+5 -2
View File
@@ -89,6 +89,7 @@ struct PgfConcr;
#include "namespace.h"
#include "probspace.h"
#include "expr.h"
#include "intervalmap.h"
struct PGF_INTERNAL_DECL PgfFlag {
PgfLiteral value;
@@ -266,8 +267,9 @@ struct PGF_INTERNAL_DECL PgfSymbolACat {
struct PGF_INTERNAL_DECL PgfSymbolCCat {
static const uint8_t tag = 12;
ref<PgfConcrLincat> lincat;
size_t value;
size_t lin_idx;
interval_t value;
interval_t lin_idx;
PgfMetaId fid;
};
struct PGF_INTERNAL_DECL PgfConcrPrintname {
@@ -287,6 +289,7 @@ struct PGF_INTERNAL_DECL PgfConcr {
Namespace<PgfConcrLincat> lincats;
PgfPhrasetable phrasetable;
Namespace<PgfConcrPrintname> printnames;
PgfMetaId last_fid;
PgfText name;
+388
View File
@@ -0,0 +1,388 @@
#ifndef INTERVAL_MAP_H
#define INTERVAL_MAP_H
typedef std::pair<size_t,size_t> interval_t;
template<class V>
class PGF_INTERNAL_DECL interval_map {
const static size_t DELTA = 3;
const static size_t RATIO = 2;
struct Node {
size_t sz;
size_t start, end, max;
V value;
Node *left;
Node *right;
Node(size_t start, size_t end)
{
this->sz = 1;
this->start = start;
this->end = end;
this->max = end;
this->left = NULL;
this->right = NULL;
memset(&value, 0, sizeof(value));
}
};
Node *root;
static
Node *insert(Node *node, size_t start, size_t end, Node **target)
{
if (node == NULL) {
node = new Node(start, end);
*target = node;
return node;
}
int cmp;
if (node->start < start)
cmp = -1;
else if (node->start > start)
cmp = 1;
else if (node->end < end)
cmp = -1;
else if (node->end > end)
cmp = 1;
else
cmp = 0;
if (cmp < 0) {
Node *left = insert(node->left, start, end, target);
node = upd_node(node,left,node->right);
return balanceL(node);
} else if (cmp > 0) {
Node *right = insert(node->right, start, end, target);
node = upd_node(node,node->left,right);
return balanceR(node);
} else {
*target = node;
return node;
}
}
static
V *lookup(Node *node, size_t start, size_t end)
{
if (node == NULL) {
return NULL;
}
int cmp;
if (node->start < start)
cmp = -1;
else if (node->start > start)
cmp = 1;
else if (node->end < end)
cmp = -1;
else if (node->end > end)
cmp = 1;
else
cmp = 0;
if (cmp < 0) {
return lookup(node->left, start, end);
} else if (cmp > 0) {
return lookup(node->right, start, end);
} else {
return &node->value;
}
}
static size_t size(Node *node)
{
if (node == 0)
return 0;
return node->sz;
}
static
Node *upd_node(Node *node, Node *left, Node *right)
{
node->sz = 1+size(left)+size(right);
node->max = std::max((left == NULL) ? node->end : left->max,
(right == NULL) ? node->end : right->max);
node->left = left;
node->right = right;
return node;
}
static
Node *balanceL(Node *node)
{
if (node->right == NULL) {
if (node->left == NULL) {
return node;
} else {
if (node->left->left == NULL) {
if (node->left->right == NULL) {
return node;
} else {
Node *left_right = node->left->right;
Node *left = upd_node(node->left,NULL,NULL);
Node *right = upd_node(node,NULL,NULL);
return upd_node(left_right,
left,
right);
}
} else {
if (node->left->right == 0) {
Node *left = node->left;
Node *right = upd_node(node,NULL,NULL);
return upd_node(left,
left->left,
right);
} else {
if (node->left->right->sz < RATIO * node->left->left->sz) {
Node *left = node->left;
Node *right =
upd_node(node,
left->right,
NULL);
return upd_node(left,
left->left,
right);
} else {
Node *left_right = node->left->right;
Node *left =
upd_node(node->left,
node->left->left,
left_right->left);
Node *right =
upd_node(node,
left_right->right,
NULL);
return upd_node(left_right,
left,
right);
}
}
}
}
} else {
if (node->left == NULL) {
return node;
} else {
if (node->left->sz > DELTA*node->right->sz) {
if (node->left->right->sz < RATIO*node->left->left->sz) {
Node *left = node->left;
Node *right =
upd_node(node,
left->right,
node->right);
return upd_node(left,
left->left,
right);
} else {
Node *left_right = node->left->right;
Node *left =
upd_node(node->left,
node->left->left,
left_right->left);
Node *right =
upd_node(node,
left_right->right,
node->right);
return upd_node(left_right,
left,
right);
}
} else {
return node;
}
}
}
}
static
Node *balanceR(Node *node)
{
if (node->left == NULL) {
if (node->right == NULL) {
return node;
} else {
if (node->right->left == NULL) {
if (node->right->right == NULL) {
return node;
} else {
Node *right = node->right;
Node *left =
upd_node(node,
NULL,
NULL);
return upd_node(right,
left,
right->right);
}
} else {
if (node->right->right == NULL) {
Node *right_left = node->right->left;
Node *right =
upd_node(node->right,NULL,NULL);
Node *left =
upd_node(node,NULL,NULL);
return upd_node(right_left,
left,
right);
} else {
if (node->right->left->sz < RATIO * node->right->right->sz) {
Node *right = node->right;
Node *left =
upd_node(node,
NULL,
right->left);
return upd_node(right,
left,
right->right);
} else {
Node *right_left = node->right->left;
Node *right =
upd_node(node->right,
right_left->right,
node->right->right);
Node *left =
upd_node(node,
NULL,
right_left->left);
return upd_node(right_left,
left,
right);
}
}
}
}
} else {
if (node->right == NULL) {
return node;
} else {
if (node->right->sz > DELTA*node->left->sz) {
if (node->right->left->sz < RATIO*node->right->right->sz) {
Node *right = node->right;
Node *left =
upd_node(node,
node->left,
right->left);
return upd_node(right,
left,
right->right);
} else {
Node *right_left = node->right->left;
Node *right =
upd_node(node->right,
right_left->right,
node->right->right);
Node *left =
upd_node(node,
node->left,
right_left->left);
return upd_node(right_left,
left,
right);
}
} else {
return node;
}
}
}
}
public:
interval_map() {
root = NULL;
}
V &operator[](interval_t interval)
{
Node *node;
this->root = insert(this->root, interval.first, interval.second, &node);
return node->value;
}
V *lookup(interval_t interval)
{
return lookup(this->root, interval.first, interval.second);
}
size_t size()
{
return size(root);
}
class iterator {
struct Parent {
Node *node;
Parent *next;
};
Parent *spine;
public:
iterator() {
spine = NULL;
}
iterator(Node *node) {
spine = NULL;
while (node != NULL) {
Parent *parent = new Parent;
parent->node = node;
parent->next = spine;
spine = parent;
node = node->left;
}
}
bool operator ==(const iterator other) const {
return this->spine == other.spine;
}
bool operator !=(const iterator other) const {
return this->spine != other.spine;
}
std::pair<interval_t,V&> operator *() const {
return std::pair<interval_t,V&>
(interval_t(spine->node->start,spine->node->end)
,spine->node->value
);
}
void operator ++() {
Parent *parent = spine->next;
Node *node = spine->node->right;
delete spine;
spine = parent;
while (node != NULL) {
parent = new Parent;
parent->node = node;
parent->next = spine;
spine = parent;
node = node->left;
}
}
~iterator() {
while (spine != NULL) {
Parent *parent = spine->next;
delete spine;
spine = parent;
}
}
};
iterator begin() const {
return iterator(root);
}
iterator end() const {
return iterator();
}
};
#endif
File diff suppressed because it is too large Load Diff
+36 -26
View File
@@ -20,22 +20,22 @@ protected:
ref<PgfConcrRule> rule;
struct {
size_t &operator[](int i) {
size_t &operator[](int i) const {
Production *prod = containerof(Production,vars,this);
return ((size_t*) (((CCat**) (prod+1))+prod->args.size()))[i];
}
size_t size() {
size_t size() const {
Production *prod = containerof(Production,vars,this);
return prod->rule->vars.size();
}
} vars;
struct {
CCat *&operator[](int i) {
CCat *&operator[](int i) const {
Production *prod = containerof(Production,args,this);
return ((CCat**) (prod+1))[i];
}
size_t size() {
size_t size() const {
Production *prod = containerof(Production,args,this);
return (prod->rule->args != 0) ? prod->rule->args.size() : 0;
}
@@ -82,10 +82,13 @@ protected:
struct CCat {
PgfMetaId fid;
Cont *cont;
union {
object epsilons;
Cont *cont;
};
State *state;
size_t value;
size_t lin_idx;
interval_t value;
interval_t lin_idx;
bool covered;
std::vector<Production*> prods;
std::vector<ExprState*> pending;
@@ -98,8 +101,9 @@ protected:
PgfTextSpot start, end;
bool needs_bind;
std::map<ref<PgfConcrLincat>,Cont*> conts1;
std::map<CCat*,std::map<size_t,Cont*>> conts2;
std::map<Cont*,std::map<size_t,std::map<size_t,CCat*>>> completed;
std::map<CCat*,interval_map<Cont*>> conts2;
std::map<Cont*,interval_map<interval_map<CCat*>>> completed;
State *next;
};
@@ -121,22 +125,22 @@ protected:
ref<PgfConcrRule> rule;
struct {
size_t &operator[](int i) {
size_t &operator[](int i) const {
Item *item = containerof(Item,vars,this);
return ((size_t*) (((CCat**) (item+1))+item->args.size()))[i];
}
size_t size() {
size_t size() const {
Item *item = containerof(Item,vars,this);
return item->rule->vars.size();
}
} vars;
struct {
CCat *&operator[](int i) {
CCat *&operator[](int i) const {
Item *item = containerof(Item,args,this);
return ((CCat**) (item+1))[i];
}
size_t size() {
size_t size() const {
Item *item = containerof(Item,args,this);
return (item->rule->args != 0) ? item->rule->args.size() : 0;
}
@@ -168,8 +172,9 @@ protected:
Item() {
}
bool instantiate(ref<PgfLParam> lparam,size_t value);
bool instantiate(ref<PgfLParam> lparam,ref<PgfLParam> value,Item *other);
interval_t interval(ref<PgfLParam> lparam) const;
bool instantiate(ref<PgfLParam> lparam1,
PgfConcrRule *rule, size_t *values, ref<PgfLParam> lparam2);
};
struct ExprState {
@@ -200,7 +205,8 @@ protected:
};
State *first_state, *current_state;
PgfMetaId last_fid;
std::map<ref<PgfConcrLincat>,interval_map<interval_map<CCat*>>> epsilons;
PgfMetaId initial_fid, last_fid;
void process(Item *item, const PgfTextSpot &spot, bool bind);
void symbol(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym);
@@ -210,16 +216,15 @@ protected:
virtual void symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym)=0;
virtual void symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym)=0;
virtual void suspend(State *state,ref<PgfConcrLincat> lincat, Item *item)=0;
virtual void final_item(State *state,Item *item,size_t value,size_t lin_idx)=0;
virtual void final_item(State *state,CCat *ccat,Item *item,interval_t value,interval_t lin_idx)=0;
virtual void bu_predict(State *state, CCat *ccat)=0;
virtual void bu_predict(PgfPhrasetable phrasetable, State *state, CCat *ccat);
Item *bu_item(State *state, ref<PgfItem> pitem);
CCat *td_epsilon(State *state, Cont *cont, ref<PgfSymbolCCat> arg);
CCat *td_epsilon(State *state, Cont *cont, ref<PgfSymbolCCat> arg,
size_t n_items, vector<ref<PgfItem>> items);
void td_predict(State *state, Cont *cont, Production *prod, size_t lin_idx);
void td_epsilon(State *state, Cont *cont, ref<PgfItem> pitem, Item *xitem, ref<PgfLParam> value, ref<PgfLParam> lin_idx);
void td_predict(State *state, Cont *cont, Production *prod, Item *xitem, ref<PgfLParam> value, ref<PgfLParam> lin_idx);
void combine(State *state, Item *item, CCat *ccat);
void get_info(CCat *ccat, ref<PgfConcrRule> *rule, size_t **pvalues);
static
void print_item(Item *item, const PgfTextSpot &spot);
@@ -243,12 +248,16 @@ class PGF_INTERNAL_DECL PgfParser : private PgfAbstractParser, public PgfExprEnu
virtual void symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym);
virtual void symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym);
virtual void suspend(State *state,ref<PgfConcrLincat> lincat, Item *item);
virtual void final_item(State *state,Item *item,size_t value,size_t lin_idx);
virtual void final_item(State *state,CCat *ccat,Item *item,interval_t value,interval_t lin_idx);
virtual void bu_predict(State *state, CCat *ccat);
void bu_predict(PgfPhrasetable phrasetable, State *state, ptrdiff_t min, ptrdiff_t max);
void make_chunks(State *state, std::vector<CCat*> &chunks, prob_t prob);
PgfExpr process_expr(ExprState *estate, prob_t *prob);
bool td_reachable(State *state, ref<PgfItem> pitem, std::map<ref<PgfConcrLincat>, bool> &visited);
Item *bu_item(State *state, ref<PgfItem> pitem);
static
void print_expr_state_left(PgfPrinter *printer, PgfMarshaller *m, ExprState *estate);
static
@@ -280,8 +289,8 @@ private:
virtual void symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym);
virtual void symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym);
virtual void suspend(State *state,ref<PgfConcrLincat> lincat,Item *item);
virtual void final_item(State *state,Item *item,size_t value,size_t lin_idx);
virtual void bu_predict(PgfPhrasetable phrasetable, State *state, CCat *ccat);
virtual void final_item(State *state,CCat *ccat,Item *item,interval_t value,interval_t lin_idx);
virtual void bu_predict(State *state, CCat *ccat);
static
ref<PgfItem> clone_item(Item *item);
@@ -289,6 +298,7 @@ private:
public:
PgfParseTableMaker(ref<PgfConcr> concr);
void insert_rule(ref<PgfConcrRule> rule);
PgfMetaId get_last_fid() { return last_fid; };
};
#endif
+2
View File
@@ -1479,6 +1479,7 @@ ref<PgfConcr> clone_concrete(ref<PgfPGF> pgf, ref<PgfConcr> concr)
clone->lincats = concr->lincats;
clone->phrasetable = concr->phrasetable;
clone->printnames = concr->printnames;
clone->last_fid = concr->last_fid;
memcpy(&clone->name, &concr->name, sizeof(PgfText)+concr->name.size+1);
ref<PgfConcr> old_concr;
@@ -1656,6 +1657,7 @@ PgfConcrRevision pgf_create_concrete(PgfDB *db, PgfRevision revision,
concr->lincats = 0;
concr->phrasetable = 0;
concr->printnames = 0;
concr->last_fid = 0;
memcpy(&concr->name, name, sizeof(PgfText)+name->size+1);
Namespace<PgfConcr> concrs =
+33 -4
View File
@@ -327,7 +327,7 @@ PGF_INTERNAL_DECL
size_t get_next_padovan(size_t min);
static
int symbol_cmp(ref<PgfConcrLincat> lincat, size_t value, size_t lin_idx, PgfSymbol sym)
int symbol_cmp(ref<PgfConcrLincat> lincat, interval_t value, interval_t lin_idx, PgfSymbol sym)
{
uint8_t tag = ref<PgfSymbol>::get_tag(sym);
if (PgfSymbolCCat::tag != tag)
@@ -655,6 +655,33 @@ vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable table, PgfSymbol sym, siz
return 0;
}
vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable phrasetable,
ref<PgfConcrLincat> lincat,
size_t *n_items)
{
while (phrasetable != 0) {
int cmp;
uint8_t tag = ref<PgfSymbol>::get_tag(phrasetable->sym);
if (PgfSymbolACat::tag != tag) {
cmp = ((int) PgfSymbolACat::tag) - ((int) tag);
} else {
auto symcf = ref<PgfSymbolACat>::untagged(phrasetable->sym);
cmp = textcmp(&lincat->name, &symcf->name);
}
if (cmp < 0)
phrasetable = phrasetable->left;
else if (cmp > 0)
phrasetable = phrasetable->right;
else {
*n_items = phrasetable->n_items;
return phrasetable->items;
}
}
*n_items = 0;
return 0;
}
PGF_INTERNAL
void phrasetable_lookup(PgfPhrasetable table,
PgfText *sentence,
@@ -973,7 +1000,8 @@ PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
ref<PgfConcrLincat> lincat,
size_t value, size_t lin_idx,
interval_t value, interval_t lin_idx,
PgfMetaId fid,
ref<PgfItem> item)
{
if (table == 0) {
@@ -981,6 +1009,7 @@ PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
symcf->lincat = lincat;
symcf->value = value;
symcf->lin_idx = lin_idx;
symcf->fid = fid;
PgfPhrasetable new_table = PgfPhrasetableNode::new_node(symcf.tagged(),1);
new_table->n_items = 1;
new_table->items[0] = item;
@@ -990,12 +1019,12 @@ PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
int cmp = symbol_cmp(lincat,value,lin_idx,table->sym);
if (cmp < 0) {
PgfPhrasetable left = phrasetable_insert(table->left,
lincat, value, lin_idx, item);
lincat, value, lin_idx, fid, item);
table = PgfPhrasetableNode::upd_node(table,left,table->right);
return PgfPhrasetableNode::balanceL(table);
} else if (cmp > 0) {
PgfPhrasetable right = phrasetable_insert(table->right,
lincat, value, lin_idx, item);
lincat, value, lin_idx, fid, item);
table = PgfPhrasetableNode::upd_node(table, table->left, right);
return PgfPhrasetableNode::balanceR(table);
} else {
+7 -1
View File
@@ -90,7 +90,8 @@ PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
ref<PgfConcrLincat> lincat,
size_t value, size_t lin_idx,
interval_t value, interval_t lin_idx,
PgfMetaId fid,
ref<PgfItem> item);
PGF_INTERNAL_DECL
@@ -99,6 +100,11 @@ void phrasetable_iter(PgfPhrasetable phrasetable,ref<PgfConcrLincat> lincat,std:
PGF_INTERNAL_DECL
vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable phrasetable, PgfSymbol sym, size_t *n_items);
PGF_INTERNAL_DECL
vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable phrasetable,
ref<PgfConcrLincat> lincat,
size_t *n_items);
class PGF_INTERNAL_DECL PgfPhraseScanner {
public:
virtual void space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err)=0;
+4 -1
View File
@@ -586,7 +586,10 @@ void PgfPrinter::symbol(PgfSymbol sym)
case PgfSymbolCCat::tag: {
auto symcf = ref<PgfSymbolCCat>::untagged(sym);
efun(&symcf->lincat->name);
nprintf(64,"(%zu,%zu)",symcf->value,symcf->lin_idx);
nprintf(64,"(%zu-%zu,%zu-%zu)",symcf->value.first
,symcf->value.second
,symcf->lin_idx.first
,symcf->lin_idx.second);
break;
}
}
+2
View File
@@ -726,6 +726,7 @@ ref<PgfConcr> PgfReader::read_concrete()
{
concrete = read_name(&PgfConcr::name);
concrete->phrasetable = 0;
concrete->last_fid = 0;
auto cflags = read_namespace<PgfFlag>(&PgfReader::read_flag);
concrete->cflags = cflags;
@@ -739,6 +740,7 @@ ref<PgfConcr> PgfReader::read_concrete()
auto lins = read_namespace<PgfConcrLin>(&PgfReader::read_lin);
concrete->lins = lins;
concrete->last_fid = tm.get_last_fid();
this->table_maker = NULL;
auto printnames = read_namespace<PgfConcrPrintname>(&PgfReader::read_printname);