msyds/gf-core
1
0
Fork 0
forked from GitHub/gf-core
Code Pull Requests Activity

much faster grammar loading and dynamic updates

Browse Source
This commit is contained in:
Krasimir Angelov
2024-03-21 16:42:41 +01:00
parent 614f4b2dc9
commit d48a8d06c1
7 changed files with 70 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/runtime/c/pgf/db.cxx
View File

@@ -143,6 +143,9 @@ static inline ssize_t get_mmap_size(size_t init_size, size_t page_size)
return mmap_size;
}
PGF_INTERNAL_DECL
size_t get_next_padovan(size_t min);
static inline ssize_t get_mremap_size(size_t file_size, size_t block_size, size_t free_size, size_t page_size)
{
size_t n_pages =

8
src/runtime/c/pgf/pgf.cxx
View File

@@ -2264,13 +2264,7 @@ public:
throw pgf_error(builder_error_msg);
ref<PgfPhrasetableEntry> entry = seq_id;
size_t len = entry->backrefs->len;
ref<Vector<PgfSequenceBackref>> backrefs =
vector_resize<PgfSequenceBackref>(entry->backrefs, len+1, PgfDB::get_txn_id());
backrefs->data[len].container = container;
backrefs->data[len].seq_index = seq_index;
entry->backrefs = backrefs;
phrasetable_add_backref(entry,PgfDB::get_txn_id(),container,seq_index);
*vector_elem(seqs, seq_index) = entry->seq;

128
src/runtime/c/pgf/phrasetable.cxx
View File

@@ -342,48 +342,27 @@ int backref_cmp(ref<PgfSequenceBackref> backref, ref<PgfConcrLincat> lincat, siz
return cmp;
}
static
ref<Vector<PgfSequenceBackref>> phrasetable_update_backrefs(PgfPhrasetable table,
ref<PgfConcrLincat> lincat,
object container,
size_t seq_index)
PGF_INTERNAL_DECL
size_t get_next_padovan(size_t min);
PGF_INTERNAL_DECL
void phrasetable_add_backref(ref<PgfPhrasetableEntry> entry, txn_t txn_id,
object container,
size_t seq_index)
{
size_t len = (table->value.backrefs != 0)
? table->value.backrefs->len
: 0;
ref<Vector<PgfSequenceBackref>> backrefs = entry->backrefs;
ref<Vector<PgfSequenceBackref>> backrefs =
vector_resize<PgfSequenceBackref>(table->value.backrefs, len+1, table->txn_id);
ssize_t i = 0;
ssize_t j = len-1;
if (table->value.seq->syms.len == 0 && len > 0) {
// The backrefs for the epsilon sequence are sorted by lincat and r
size_t r = (lincat!=0) ? (seq_index % lincat->fields->len) : 0;
while (i <= j) {
ssize_t k = (i + j) / 2;
ref<PgfSequenceBackref> backref = vector_elem(backrefs, k);
int cmp = backref_cmp(backref, lincat, r);
if (cmp < 0) {
while (j >= k) {
backrefs->data[j+1] = backrefs->data[j];
j--;
}
} else if (cmp > 0) {
i = k+1;
} else {
while (j > k) {
backrefs->data[j+1] = backrefs->data[j];
j--;
}
break;
}
}
size_t len = (backrefs != 0) ? backrefs->len : 0;
if (entry->n_backrefs >= len) {
size_t new_len = get_next_padovan(entry->n_backrefs+1);
backrefs = PgfDB::realloc<Vector<PgfSequenceBackref>>(backrefs,len*sizeof(PgfSequenceBackref),new_len*sizeof(PgfSequenceBackref),txn_id);
backrefs->len = new_len;
}
backrefs->data[j+1].container = container;
backrefs->data[j+1].seq_index = seq_index;
return backrefs;
backrefs->data[entry->n_backrefs].container = container;
backrefs->data[entry->n_backrefs].seq_index = seq_index;
entry->n_backrefs++;
entry->backrefs = backrefs;
}
PGF_INTERNAL
@@ -397,6 +376,7 @@ PgfPhrasetable phrasetable_internalize(PgfPhrasetable table,
if (table == 0) {
PgfPhrasetableEntry entry;
entry.seq = seq;
entry.n_backrefs = 1;
entry.backrefs = vector_new<PgfSequenceBackref>(1);
entry.backrefs->data[0].container = container;
entry.backrefs->data[0].seq_index = seq_index;
@@ -427,24 +407,16 @@ PgfPhrasetable phrasetable_internalize(PgfPhrasetable table,
} else {
PgfSequence::release(seq);
size_t len = (table->value.backrefs)
? table->value.backrefs->len
: 0;
ref<Vector<PgfSequenceBackref>> backrefs =
phrasetable_update_backrefs(table,lincat,container,seq_index);
PgfPhrasetable new_table =
Node<PgfPhrasetableEntry>::upd_node(table, table->left, table->right);
new_table->value.backrefs = backrefs;
*pentry = ref<PgfPhrasetableEntry>::from_ptr(&new_table->value);
phrasetable_add_backref(*pentry,table->txn_id,container,seq_index);
return new_table;
}
}
PGF_INTERNAL
ref<PgfSequence> phrasetable_relink(PgfPhrasetable table,
ref<PgfConcrLincat> lincat,
object container,
size_t seq_index,
size_t seq_id)
@@ -454,14 +426,8 @@ ref<PgfSequence> phrasetable_relink(PgfPhrasetable table,
if (seq_id < left_sz)
table = table->left;
else if (seq_id == left_sz) {
size_t len = (table->value.backrefs == 0)
? 0
: table->value.backrefs->len;
ref<Vector<PgfSequenceBackref>> backrefs =
phrasetable_update_backrefs(table,lincat,container,seq_index);
table->value.backrefs = backrefs;
auto entry = ref<PgfPhrasetableEntry>::from_ptr(&table->value);
phrasetable_add_backref(entry,table->txn_id,container,seq_index);
return table->value.seq;
} else {
table = table->right;
@@ -471,6 +437,7 @@ ref<PgfSequence> phrasetable_relink(PgfPhrasetable table,
return 0;
}
PGF_INTERNAL
PgfPhrasetable phrasetable_delete(PgfPhrasetable table,
object container,
size_t seq_index,
@@ -494,11 +461,12 @@ PgfPhrasetable phrasetable_delete(PgfPhrasetable table,
return Node<PgfPhrasetableEntry>::balanceL(table);
} else {
size_t len = table->value.backrefs->len;
if (len > 1) {
size_t n_backrefs = table->value.n_backrefs;
if (n_backrefs > 1) {
ref<Vector<PgfSequenceBackref>> backrefs =
vector_resize<PgfSequenceBackref>(table->value.backrefs, len-1, table->txn_id);
PgfDB::realloc<Vector<PgfSequenceBackref>>(table->value.backrefs,len*sizeof(PgfSequenceBackref),n_backrefs*sizeof(PgfSequenceBackref),table->txn_id);
size_t i = 0;
while (i < len-1) {
while (i < n_backrefs) {
ref<PgfSequenceBackref> backref =
vector_elem(backrefs, i);
if (backref->container == container &&
@@ -508,14 +476,16 @@ PgfPhrasetable phrasetable_delete(PgfPhrasetable table,
i++;
}
i++;
while (i < len) {
while (i < n_backrefs) {
*vector_elem(backrefs, i-1) = *vector_elem(table->value.backrefs, i);
i++;
}
n_backrefs--;
PgfPhrasetable new_table =
Node<PgfPhrasetableEntry>::upd_node(table, table->left, table->right);
new_table->value.backrefs = backrefs;
new_table->value.n_backrefs = n_backrefs;
new_table->value.backrefs = backrefs;
return new_table;
} else {
PgfSequence::release(table->value.seq);
@@ -570,24 +540,22 @@ void phrasetable_lookup(PgfPhrasetable table,
phrasetable_lookup(table->right,sentence,case_sensitive,scanner,err);
} else {
auto backrefs = table->value.backrefs;
if (backrefs != 0) {
for (size_t i = 0; i < backrefs->len; i++) {
PgfSequenceBackref backref = *vector_elem<PgfSequenceBackref>(backrefs,i);
switch (ref<PgfConcrLin>::get_tag(backref.container)) {
case PgfConcrLin::tag: {
ref<PgfConcrLin> lin = ref<PgfConcrLin>::untagged(backref.container);
if (lin->absfun->type->hypos->len == 0) {
scanner->match(lin, backref.seq_index, err);
if (err->type != PGF_EXN_NONE)
return;
}
break;
}
case PgfConcrLincat::tag: {
//ignore
break;
}
for (size_t i = 0; i < table->value.n_backrefs; i++) {
PgfSequenceBackref backref = *vector_elem<PgfSequenceBackref>(backrefs,i);
switch (ref<PgfConcrLin>::get_tag(backref.container)) {
case PgfConcrLin::tag: {
ref<PgfConcrLin> lin = ref<PgfConcrLin>::untagged(backref.container);
if (lin->absfun->type->hypos->len == 0) {
scanner->match(lin, backref.seq_index, err);
if (err->type != PGF_EXN_NONE)
return;
}
break;
}
case PgfConcrLincat::tag: {
//ignore
break;
}
}
}
@@ -714,7 +682,7 @@ void phrasetable_lookup_prefixes(PgfCohortsState *state,
}
state->queue.push(current);
for (size_t i = 0; i < backrefs->len; i++) {
for (size_t i = 0; i < table->value.n_backrefs; i++) {
PgfSequenceBackref backref = *vector_elem<PgfSequenceBackref>(backrefs,i);
switch (ref<PgfConcrLin>::get_tag(backref.container)) {
case PgfConcrLin::tag: {
@@ -849,7 +817,7 @@ void phrasetable_iter(PgfConcr *concr,
return;
if (table->value.backrefs != 0 && res == 0 && callback != 0) {
for (size_t i = 0; i < table->value.backrefs->len; i++) {
for (size_t i = 0; i < table->value.n_backrefs; i++) {
PgfSequenceBackref backref = *vector_elem<PgfSequenceBackref>(table->value.backrefs,i);
switch (ref<PgfConcrLin>::get_tag(backref.container)) {
case PgfConcrLin::tag: {

13
src/runtime/c/pgf/phrasetable.h
View File

@@ -6,6 +6,11 @@ struct PgfSequenceBackref;
struct PGF_INTERNAL_DECL PgfPhrasetableEntry {
ref<PgfSequence> seq;
// Here n_backrefs tells us how many actual backrefs there are in
// the vector backrefs. On the other hand, backrefs->len tells us
// how big buffer we have allocated.
size_t n_backrefs;
ref<Vector<PgfSequenceBackref>> backrefs;
};
@@ -64,7 +69,6 @@ PgfPhrasetable phrasetable_internalize(PgfPhrasetable table,
PGF_INTERNAL_DECL
ref<PgfSequence> phrasetable_relink(PgfPhrasetable table,
ref<PgfConcrLincat> lincat,
object container,
size_t seq_index,
size_t seq_id);
@@ -124,4 +128,11 @@ int text_sequence_cmp(PgfTextSpot *spot, const uint8_t *end,
ref<PgfSequence> seq, size_t *p_i,
bool case_sensitive, SeqMatch sm);
// The following is used internally in the grammar builder
PGF_INTERNAL_DECL
void phrasetable_add_backref(ref<PgfPhrasetableEntry> entry, txn_t txn_id,
object container,
size_t seq_index);
#endif

9
src/runtime/c/pgf/reader.cxx
View File

@@ -632,14 +632,14 @@ ref<PgfSequence> PgfReader::read_seq()
return seq;
}
ref<Vector<ref<PgfSequence>>> PgfReader::read_seq_ids(ref<PgfConcrLincat> lincat, object container)
ref<Vector<ref<PgfSequence>>> PgfReader::read_seq_ids(object container)
{
size_t len = read_len();
ref<Vector<ref<PgfSequence>>> vec = vector_new<ref<PgfSequence>>(len);
for (size_t i = 0; i < len; i++) {
size_t seq_id = read_len();
ref<PgfSequence> seq = phrasetable_relink(concrete->phrasetable,
lincat, container, i,
container, i,
seq_id);
if (seq == 0) {
throw pgf_error("Invalid sequence id");
@@ -659,6 +659,7 @@ PgfPhrasetable PgfReader::read_phrasetable(size_t len)
size_t half = len/2;
PgfPhrasetable left = read_phrasetable(half);
value.seq = read_seq();
value.n_backrefs = 0;
value.backrefs = 0;
PgfPhrasetable right = read_phrasetable(len-half-1);
@@ -683,7 +684,7 @@ ref<PgfConcrLincat> PgfReader::read_lincat()
auto n_lindefs = read_len();
auto args = read_vector(&PgfReader::read_parg);
auto res = read_vector(&PgfReader::read_presult2);
auto seqs = read_seq_ids(0, lincat.tagged());
auto seqs = read_seq_ids(lincat.tagged());
lincat->abscat = namespace_lookup(abstract->cats, &lincat->name);
lincat->fields = fields;
@@ -718,7 +719,7 @@ ref<PgfConcrLin> PgfReader::read_lin()
auto args = read_vector(&PgfReader::read_parg);
auto res = read_vector(&PgfReader::read_presult2);
auto seqs = read_seq_ids(lin->lincat, lin.tagged());
auto seqs = read_seq_ids(lin.tagged());
lin->args = args;
lin->res = res;

2
src/runtime/c/pgf/reader.h
View File

@@ -78,7 +78,7 @@ public:
ref<PgfPResult> read_presult();
PgfSymbol read_symbol();
ref<PgfSequence> read_seq();
ref<Vector<ref<PgfSequence>>> read_seq_ids(ref<PgfConcrLincat> lincat, object container);
ref<Vector<ref<PgfSequence>>> read_seq_ids(object container);
PgfPhrasetable read_phrasetable(size_t len);
PgfPhrasetable read_phrasetable();
ref<PgfConcrLin> read_lin();

19
src/runtime/c/pgf/vector.h
View File

@@ -28,25 +28,6 @@ ref<C> vector_new(Vector<A> C::* field, size_t len)
return res;
}
PGF_INTERNAL_DECL size_t
get_next_padovan(size_t min);
/* Resize a vector by changing its length. If there is no enough space
* the implementation will create a copy, but whenever possible it will
* return the reference to the original vector. A copy is created also
* if txn_id is different from the current transaction. In this way
* it is safe to change the length. */
template <class A> inline PGF_INTERNAL
ref<Vector<A>> vector_resize(ref<Vector<A>> vec, size_t len, txn_t txn_id)
{
size_t new_len = get_next_padovan(len);
size_t old_len = get_next_padovan(vec->len);
vec = PgfDB::realloc<Vector<A>>(vec,old_len*sizeof(A),new_len*sizeof(A),txn_id);
vec->len = len;
return vec;
}
template <class A> inline PGF_INTERNAL
ref<A> vector_elem(ref<Vector<A>> v, size_t index)
{