GitHub/gf-core
1
0
Fork 1
mirror of https://github.com/GrammaticalFramework/gf-core.git synced 2026-09-17 07:46:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

a higher-level API allows for a more efficient implementation

Browse Source
This commit is contained in:
Krasimir Angelov
2026-08-20 11:49:19 +02:00
parent f1f39d67c7
commit 17a308972a
10 changed files with 264 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/runtime/c/pgf/parser.cxx
+122 -32
View File
@@ -114,7 +114,7 @@ void PgfAbstractParser::process(Item *item, State *state)
} }
PGF_INTERNAL_DECL PGF_INTERNAL_DECL
int text_symbol_cmp(PgfTextSpot *spot, const uint8_t *end, int text_symbol_cmp(PgfText *sentence, PgfTextSpot *spot,
ref<PgfSymbolKS> sym, bool case_sensitive); ref<PgfSymbolKS> sym, bool case_sensitive);
void PgfAbstractParser::symbol(Item *item, State *state, PgfSymbol sym) void PgfAbstractParser::symbol(Item *item, State *state, PgfSymbol sym)
@@ -857,8 +857,12 @@ PgfParser::PgfParser(ref<PgfConcr> concr, PgfText *sentence, bool case_sensitive
this->m = m; this->m = m;
this->u = u; this->u = u;
this->sentence = textdup(sentence); this->sentence = textdup(sentence);
this->end = (uint8_t *) (this->sentence->text+this->sentence->size);
this->case_sensitive = case_sensitive; this->case_sensitive = case_sensitive;
this->prev_state = NULL;
this->update_old_pos = 0;
this->update_new_pos = 0;
this->delta_byte_pos = 0;
this->allocated_size = sizeof(PgfText)+sentence->size+1;
} }
PgfParser::~PgfParser() PgfParser::~PgfParser()
@@ -906,11 +910,11 @@ void PgfParser::bu_predict(PgfPhrasetable<PgfSymbolKS> phrasetable,
return; return;
PgfTextSpot current = state->end; PgfTextSpot current = state->end;
int cmp = text_symbol_cmp(&current,end,phrasetable->value.key,case_sensitive); int cmp = text_symbol_cmp(sentence,&current,phrasetable->value.key,case_sensitive);
if (cmp < 0) { if (cmp < 0) {
bu_predict(phrasetable->left,state,outside_prob,min,max); bu_predict(phrasetable->left,state,outside_prob,min,max);
} else if (cmp > 0) { } else if (cmp > 0) {
ptrdiff_t len = current.ptr - state->end.ptr; ptrdiff_t len = current.byte_pos - state->end.byte_pos;
if (min <= len-1) if (min <= len-1)
bu_predict(phrasetable->left,state,outside_prob,min,len-1); bu_predict(phrasetable->left,state,outside_prob,min,len-1);
@@ -918,7 +922,7 @@ void PgfParser::bu_predict(PgfPhrasetable<PgfSymbolKS> phrasetable,
if (len <= max) if (len <= max)
bu_predict(phrasetable->right,state,outside_prob,len,max); bu_predict(phrasetable->right,state,outside_prob,len,max);
} else { } else {
ptrdiff_t len = current.ptr - state->end.ptr; ptrdiff_t len = current.byte_pos - state->end.byte_pos;
if (min <= len) if (min <= len)
bu_predict(phrasetable->left,state,outside_prob,min,len); bu_predict(phrasetable->left,state,outside_prob,min,len);
@@ -1150,7 +1154,7 @@ void PgfParser::prepare(ref<PgfConcrLincat> start)
fprintf(stderr, "------------------------------------------\n"); fprintf(stderr, "------------------------------------------\n");
#endif #endif
PgfTextSpot start_spot = {0, (uint8_t *) sentence->text}; PgfTextSpot start_spot = {0, 0};
State *state = new_state(start_spot, 0); State *state = new_state(start_spot, 0);
for (size_t i = start->n_lindefs; i < start->rules.size(); i++) { for (size_t i = start->n_lindefs; i < start->rules.size(); i++) {
@@ -1168,12 +1172,8 @@ void PgfParser::prepare(ref<PgfConcrLincat> start)
} }
} }
PgfExpr PgfParser::fetch(PgfDB *db, prob_t *prob) void PgfParser::perform_search()
{ {
DB_scope scope(db, READER_SCOPE);
bool first_fetch = (concr->last_fid == last_fid);
for (;;) { for (;;) {
State *state = current_state; State *state = current_state;
prob_t min_prob = INFINITY; prob_t min_prob = INFINITY;
@@ -1218,6 +1218,15 @@ PgfExpr PgfParser::fetch(PgfDB *db, prob_t *prob)
} }
current_state = prev; current_state = prev;
} }
}
PgfExpr PgfParser::fetch(PgfDB *db, prob_t *prob)
{
DB_scope scope(db, READER_SCOPE);
bool first_fetch = (concr->last_fid == last_fid);
perform_search();
if (first_fetch && queue.size() == 0) { if (first_fetch && queue.size() == 0) {
std::vector<CCat*> chunks; std::vector<CCat*> chunks;
@@ -1367,8 +1376,8 @@ PgfAbstractParser::State *PgfParser::new_state(const PgfTextSpot &start, prob_t
{ {
State **prev = &current_state; State **prev = &current_state;
State *state = current_state; State *state = current_state;
while (state != NULL && state->start.ptr <= start.ptr) { while (state != NULL && state->start.byte_pos <= start.byte_pos) {
if (state->start.ptr == start.ptr) if (state->start.byte_pos == start.byte_pos)
return state; return state;
prev = &state->next; prev = &state->next;
state = state->next; state = state->next;
@@ -1381,13 +1390,13 @@ PgfAbstractParser::State *PgfParser::new_state(const PgfTextSpot &start, prob_t
state->next = *prev; state->next = *prev;
*prev = state; *prev = state;
while (state->end.ptr < end) { while (state->end.byte_pos < sentence->size) {
const uint8_t *ptr = state->end.ptr; const uint8_t *ptr = (uint8_t *) &sentence->text[state->end.byte_pos];
uint32_t ucs = pgf_utf8_decode(&ptr); uint32_t ucs = pgf_utf8_decode(&ptr);
if (!pgf_utf8_is_space(ucs)) if (!pgf_utf8_is_space(ucs))
break; break;
state->end.pos++; state->end.pos++;
state->end.ptr = ptr; state->end.byte_pos = ptr-(uint8_t *) sentence->text;
} }
state->needs_bind = (state->start.pos > 0 && state->start.pos == state->end.pos); state->needs_bind = (state->start.pos > 0 && state->start.pos == state->end.pos);
@@ -1398,7 +1407,7 @@ PgfAbstractParser::State *PgfParser::new_state(const PgfTextSpot &start, prob_t
void PgfParser::symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks) void PgfParser::symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks)
{ {
PgfTextSpot next = state->end; PgfTextSpot next = state->end;
if (text_symbol_cmp(&next,end,symks,case_sensitive) != 0) if (text_symbol_cmp(sentence,&next,symks,case_sensitive) != 0)
return; return;
State *next_state = new_state(next, item->inside_prob+item->outside_prob); State *next_state = new_state(next, item->inside_prob+item->outside_prob);
@@ -1516,7 +1525,7 @@ void PgfParser::suspend(Cont *cont,Item *item,bool do_predict,ref<PgfSymbolCat>
void PgfParser::final_item(State *state, CCat *ccat, Item *item, interval_t value, interval_t lin_idx) void PgfParser::final_item(State *state, CCat *ccat, Item *item, interval_t value, interval_t lin_idx)
{ {
if (item->cont == NULL && state->end.ptr == end) { if (item->cont == NULL && state->end.byte_pos == sentence->size) {
ExprState *estate = new(item->args.size()) ExprState; ExprState *estate = new(item->args.size()) ExprState;
estate->expr = 0; estate->expr = 0;
estate->prob = 0; estate->prob = 0;
@@ -1609,40 +1618,121 @@ PgfText *PgfParser::get_text() {
return sentence; return sentence;
} }
bool PgfParser::change(size_t start, size_t end, PgfText *change) void PgfParser::start()
{ {
if (start > sentence->size || end > sentence->size || start > end) prev_state = current_state;
current_state = NULL;
while (prev_state != NULL) {
State *next = prev_state->next;
prev_state->next = current_state;
current_state = prev_state;
prev_state = next;
}
update_old_pos = 0;
update_new_pos = 0;
delta_byte_pos = 0;
}
bool PgfParser::skip(size_t i)
{
update_old_pos += i;
update_new_pos += i;
if (update_new_pos > sentence->size)
return false; return false;
size_t new_size = sentence->size + change->size - (end-start); while (current_state != NULL && current_state->start.pos < update_old_pos) {
State *next = current_state->next;
if (sentence->size < new_size) { current_state->next = prev_state;
PgfText *new_sentence = (PgfText *) realloc(sentence, sizeof(PgfText)+new_size+1); current_state->start.pos += update_new_pos-update_old_pos;
if (new_sentence == NULL) current_state->start.byte_pos += delta_byte_pos;
return false; current_state->end.pos += update_new_pos-update_old_pos;
sentence = new_sentence; current_state->end.byte_pos += delta_byte_pos;
prev_state = current_state;
current_state = next;
} }
memcpy(&sentence->text[start+change->size], &sentence->text[end], sentence->size-end); return true;
memcpy(&sentence->text[start], change->text, change->size); }
if (sentence->size > new_size) { bool PgfParser::change(size_t i, PgfText *change)
PgfText *new_sentence = (PgfText *) realloc(sentence, sizeof(PgfText)+new_size+1); {
update_old_pos += i;
if (update_new_pos+i > sentence->size)
return false;
while (current_state != NULL && current_state->end.pos <= update_old_pos) {
State *next = current_state->next;
delete current_state;
current_state = next;
}
size_t new_size = sentence->size + change->size - i;
size_t new_allocated_size = sizeof(PgfText)+new_size+1;
if (allocated_size < new_allocated_size) {
PgfText *new_sentence = (PgfText *) realloc(sentence, new_allocated_size);
if (new_sentence == NULL) if (new_sentence == NULL)
return false; return false;
sentence = new_sentence; sentence = new_sentence;
allocated_size = new_allocated_size;
}
const uint8_t *ptr1 = (uint8_t *)
&sentence->text[prev_state->start.byte_pos];
size_t pos = prev_state->start.pos;
while (pos < update_new_pos) {
pgf_utf8_decode(&ptr1); pos++;
}
const uint8_t *ptr2 = ptr1;
while (pos < update_new_pos+i) {
pgf_utf8_decode(&ptr2); pos++;
}
memcpy((void*) (ptr1+change->size), ptr2, (uint8_t*)sentence->text+sentence->size-ptr2+1);
memcpy((void*) ptr1, change->text, change->size);
delta_byte_pos += (ptr1+change->size)-ptr2;
ptr2 = ptr1+change->size;
while (ptr1 < ptr2) {
pgf_utf8_decode(&ptr1);
update_new_pos++;
} }
sentence->size = new_size; sentence->size = new_size;
return true; return true;
} }
void PgfParser::done()
{
while (current_state != NULL) {
State *next = current_state->next;
current_state->next = prev_state;
current_state->start.pos += update_new_pos-update_old_pos;
current_state->start.byte_pos += delta_byte_pos;
current_state->end.pos += update_new_pos-update_old_pos;
current_state->end.byte_pos += delta_byte_pos;
prev_state = current_state;
current_state = next;
}
current_state = prev_state;
prev_state = NULL;
update_old_pos = 0;
update_new_pos = 0;
delta_byte_pos = 0;
for (State *state = current_state; state != NULL; state=state->next) {
fprintf(stderr, "(%zd:%zd-%zd:%zd) ", state->start.pos, state->start.byte_pos
, state->end.pos, state->end.byte_pos);
}
fprintf(stderr,"\n");
}
PgfParseTableMaker::PgfParseTableMaker(ref<PgfConcr> concr) PgfParseTableMaker::PgfParseTableMaker(ref<PgfConcr> concr)
: PgfAbstractParser(concr) : PgfAbstractParser(concr)
{ {
current_state = new State; current_state = new State;
current_state->start.pos = 0; current_state->start.pos = 0;
current_state->start.ptr = NULL; current_state->start.byte_pos = 0;
current_state->end = current_state->start; current_state->end = current_state->start;
current_state->viterbi_prob = 0; current_state->viterbi_prob = 0;
current_state->next = NULL; current_state->next = NULL;
src/runtime/c/pgf/parser.h
+11 -2
View File
@@ -275,9 +275,14 @@ class PGF_INTERNAL_DECL PgfParser : private PgfAbstractParser, public PgfExprEnu
PgfMarshaller *m; PgfMarshaller *m;
PgfUnmarshaller *u; PgfUnmarshaller *u;
PgfText *sentence; PgfText *sentence;
uint8_t *end;
bool case_sensitive; bool case_sensitive;
// The following are used only during chart updates.
State *prev_state;
size_t update_old_pos, update_new_pos;
ssize_t delta_byte_pos;
ssize_t allocated_size;
virtual State *new_state(const PgfTextSpot &start, prob_t viterbi_prob); virtual State *new_state(const PgfTextSpot &start, prob_t viterbi_prob);
virtual void symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks); virtual void symbol_token(Item *item, State *state, ref<PgfSymbolKS> symks);
virtual void symbol_bind(Item *item, State *state, PgfSymbol sym); virtual void symbol_bind(Item *item, State *state, PgfSymbol sym);
@@ -313,11 +318,15 @@ public:
virtual ~PgfParser(); virtual ~PgfParser();
void prepare(ref<PgfConcrLincat> start); void prepare(ref<PgfConcrLincat> start);
void perform_search();
virtual PgfExpr fetch(PgfDB *db, prob_t *prob); virtual PgfExpr fetch(PgfDB *db, prob_t *prob);
virtual PgfText *get_text(); virtual PgfText *get_text();
virtual bool change(size_t start, size_t end, PgfText *change); virtual void start();
virtual bool skip(size_t i);
virtual bool change(size_t i, PgfText *change);
virtual void done();
}; };
class PGF_INTERNAL_DECL PgfParseTableMaker : private PgfAbstractParser class PGF_INTERNAL_DECL PgfParseTableMaker : private PgfAbstractParser
src/runtime/c/pgf/pgf.cxx
+1
View File
@@ -2777,6 +2777,7 @@ PgfParseChart *pgf_parse_chart(PgfDB *db, PgfConcrRevision revision,
PgfParser *parser = new PgfParser(concr, sentence, case_sensitive, m, u); PgfParser *parser = new PgfParser(concr, sentence, case_sensitive, m, u);
parser->prepare(lincat_u.lincat); parser->prepare(lincat_u.lincat);
parser->perform_search();
return parser; return parser;
} PGF_API_END } PGF_API_END
src/runtime/c/pgf/pgf.h
+8 -2
View File
@@ -833,7 +833,10 @@ void pgf_free_expr_enum(PgfExprEnum *en);
#ifdef __cplusplus #ifdef __cplusplus
struct PgfParseChart { struct PgfParseChart {
virtual PgfText *get_text()=0; virtual PgfText *get_text()=0;
virtual bool change(size_t start, size_t end, PgfText *change)=0; virtual void start()=0;
virtual bool skip(size_t i)=0;
virtual bool change(size_t i, PgfText *change)=0;
virtual void done()=0;
virtual ~PgfParseChart() {}; virtual ~PgfParseChart() {};
}; };
#else #else
@@ -841,7 +844,10 @@ typedef struct PgfParseChart PgfParseChart;
typedef struct PgfParseChartVtbl PgfParseChartVtbl; typedef struct PgfParseChartVtbl PgfParseChartVtbl;
struct PgfParseChartVtbl { struct PgfParseChartVtbl {
PgfText *(*get_text)(PgfParseChart *this); PgfText *(*get_text)(PgfParseChart *this);
int (*change)(PgfParseChart *this, size_t start, size_t end, PgfText *change); void (*start)(PgfParseChart *this);
int (*skip)(PgfParseChart *this, size_t i);
int (*change)(PgfParseChart *this, size_t i, PgfText *change);
void (*done)(PgfParseChart *this);
}; };
struct PgfParseChart { struct PgfParseChart {
PgfParseChartVtbl *vtbl; PgfParseChartVtbl *vtbl;
src/runtime/c/pgf/phrasetable.cxx
+39 -35
View File
@@ -32,16 +32,19 @@ int lparam_cmp(PgfLParam *p1, PgfLParam *p2)
} }
PGF_INTERNAL PGF_INTERNAL
int text_symbol_cmp(PgfTextSpot *spot, const uint8_t *end, int text_symbol_cmp(PgfText *sentence, PgfTextSpot *spot,
ref<PgfSymbolKS> sym_ks, bool case_sensitive) ref<PgfSymbolKS> sym_ks, bool case_sensitive)
{ {
int res1 = 0; int res1 = 0;
const uint8_t *s1 = (uint8_t *) &sentence->text[spot->byte_pos];
const uint8_t *e1 = (uint8_t *) &sentence->text[sentence->size];
const uint8_t *s2 = (uint8_t *) &sym_ks->token.text; const uint8_t *s2 = (uint8_t *) &sym_ks->token.text;
const uint8_t *e2 = s2+sym_ks->token.size; const uint8_t *e2 = s2+sym_ks->token.size;
for (;;) { for (;;) {
if (spot->ptr >= end) { if (s1 >= e1) {
if (s2 < e2) if (s2 < e2)
return -1; return -1;
return case_sensitive ? res1 : 0; return case_sensitive ? res1 : 0;
@@ -51,8 +54,10 @@ int text_symbol_cmp(PgfTextSpot *spot, const uint8_t *end,
return case_sensitive ? res1 : 0; return case_sensitive ? res1 : 0;
} }
uint32_t ucs1 = pgf_utf8_decode(&spot->ptr); spot->pos++; uint32_t ucs1 = pgf_utf8_decode(&s1);
uint32_t ucs1i = pgf_utf8_to_upper(ucs1); uint32_t ucs1i = pgf_utf8_to_upper(ucs1);
spot->pos++;
spot->byte_pos = s1-(uint8_t*) sentence->text;
uint32_t ucs2 = pgf_utf8_decode(&s2); uint32_t ucs2 = pgf_utf8_decode(&s2);
uint32_t ucs2i = pgf_utf8_to_upper(ucs2); uint32_t ucs2i = pgf_utf8_to_upper(ucs2);
@@ -74,7 +79,7 @@ int text_symbol_cmp(PgfTextSpot *spot, const uint8_t *end,
} }
static static
bool text_symbols_match(PgfTextSpot *spot, const uint8_t *end, bool text_symbols_match(PgfText *sentence, PgfTextSpot *spot,
vector<PgfSymbol> syms, size_t dot, bool *bind, vector<PgfSymbol> syms, size_t dot, bool *bind,
bool case_sensitive) bool case_sensitive)
{ {
@@ -82,20 +87,20 @@ bool text_symbols_match(PgfTextSpot *spot, const uint8_t *end,
PgfSymbol sym = syms[dot]; PgfSymbol sym = syms[dot];
switch (ref<PgfSymbol>::get_tag(sym)) { switch (ref<PgfSymbol>::get_tag(sym)) {
case PgfSymbolKS::tag: { case PgfSymbolKS::tag: {
const uint8_t *start = spot->ptr; size_t start = spot->byte_pos;
for (;;) { for (;;) {
const uint8_t *ptr = spot->ptr; const uint8_t *ptr = (uint8_t *) sentence->text+spot->byte_pos;
uint32_t ucs = pgf_utf8_decode(&ptr); uint32_t ucs = pgf_utf8_decode(&ptr);
if (!pgf_utf8_is_space(ucs)) if (!pgf_utf8_is_space(ucs))
break; break;
spot->ptr = ptr; spot->byte_pos = ptr-(uint8_t *)sentence->text;
spot->pos++; spot->pos++;
} }
if (*bind != (start == spot->ptr)) if (*bind != (start == spot->byte_pos))
return false; return false;
if (text_symbol_cmp(spot,end,sym,case_sensitive) != 0) if (text_symbol_cmp(sentence,spot,sym,case_sensitive) != 0)
return false; return false;
break; break;
@@ -104,13 +109,13 @@ bool text_symbols_match(PgfTextSpot *spot, const uint8_t *end,
auto symkp = ref<PgfSymbolKP>::untagged(syms[dot]); auto symkp = ref<PgfSymbolKP>::untagged(syms[dot]);
PgfTextSpot current = *spot; PgfTextSpot current = *spot;
if (text_symbols_match(&current, end, symkp->default_form, 0, bind, case_sensitive)) { if (text_symbols_match(sentence, &current, symkp->default_form, 0, bind, case_sensitive)) {
goto matched; goto matched;
} }
for (size_t i = 0; i < symkp->alts.size(); i++) { for (size_t i = 0; i < symkp->alts.size(); i++) {
current = *spot; current = *spot;
if (text_symbols_match(&current, end, symkp->alts[i].form, 0, bind, case_sensitive)) { if (text_symbols_match(sentence, &current, symkp->alts[i].form, 0, bind, case_sensitive)) {
goto matched; goto matched;
} }
} }
@@ -145,7 +150,7 @@ bool text_symbols_match(PgfTextSpot *spot, const uint8_t *end,
} }
static static
bool text_item_match(PgfTextSpot *spot, const uint8_t *end, bool text_item_match(PgfText *sentence, PgfTextSpot *spot,
ref<PgfItem> item, ref<PgfItem> item,
bool case_sensitive) bool case_sensitive)
{ {
@@ -155,15 +160,15 @@ bool text_item_match(PgfTextSpot *spot, const uint8_t *end,
if (item->pre_alt > 0) { if (item->pre_alt > 0) {
auto symkp = ref<PgfSymbolKP>::untagged(syms[item->pre_dot]); auto symkp = ref<PgfSymbolKP>::untagged(syms[item->pre_dot]);
if (item->pre_alt == 1) { if (item->pre_alt == 1) {
if (!text_symbols_match(spot, end, symkp->default_form, item->dot, &bind, case_sensitive)) if (!text_symbols_match(sentence, spot, symkp->default_form, item->dot, &bind, case_sensitive))
return false; return false;
} else { } else {
if (!text_symbols_match(spot, end, symkp->alts[item->pre_alt-2].form, item->dot, &bind, case_sensitive)) if (!text_symbols_match(sentence, spot, symkp->alts[item->pre_alt-2].form, item->dot, &bind, case_sensitive))
return false; return false;
} }
dot = item->pre_dot+1; dot = item->pre_dot+1;
} }
return text_symbols_match(spot, end, syms, dot, &bind, case_sensitive); return text_symbols_match(sentence, spot, syms, dot, &bind, case_sensitive);
} }
PGF_INTERNAL_DECL PGF_INTERNAL_DECL
@@ -202,9 +207,8 @@ void phrasetable_lookup(PgfPhrasetable<PgfSymbolKS> table,
PgfTextSpot spot; PgfTextSpot spot;
spot.pos = 0; spot.pos = 0;
spot.ptr = (uint8_t *) sentence->text; spot.byte_pos = 0;
const uint8_t *end = spot.ptr+sentence->size; int cmp = text_symbol_cmp(sentence,&spot,table->value.key,case_sensitive);
int cmp = text_symbol_cmp(&spot,end,table->value.key,case_sensitive);
if (cmp < 0) { if (cmp < 0) {
phrasetable_lookup(table->left,sentence,case_sensitive,scanner,err); phrasetable_lookup(table->left,sentence,case_sensitive,scanner,err);
} else if (cmp > 0) { } else if (cmp > 0) {
@@ -223,7 +227,7 @@ void phrasetable_lookup(PgfPhrasetable<PgfSymbolKS> table,
ref<PgfConcrLin> lin = ref<PgfConcrLin>::untagged(item->rule->container); ref<PgfConcrLin> lin = ref<PgfConcrLin>::untagged(item->rule->container);
if (lin->absfun->type->hypos.size() == 0) { if (lin->absfun->type->hypos.size() == 0) {
PgfTextSpot current = spot; PgfTextSpot current = spot;
if (text_item_match(&current, end, item, case_sensitive) && current.ptr == end) { if (text_item_match(sentence, &current, item, case_sensitive) && current.byte_pos == sentence->size) {
scanner->match(lin, item->rule->lin_idx->i0, err); scanner->match(lin, item->rule->lin_idx->i0, err);
if (err->type != PGF_EXN_NONE) if (err->type != PGF_EXN_NONE)
return; return;
@@ -255,12 +259,12 @@ struct PGF_INTERNAL_DECL PgfCohortsState {
} }
}; };
PgfText *sentence;
PgfTextSpot spot; PgfTextSpot spot;
std::priority_queue<PgfTextSpot, std::vector<PgfTextSpot>, PgfTextSpotComparator> queue; std::priority_queue<PgfTextSpot, std::vector<PgfTextSpot>, PgfTextSpotComparator> queue;
PgfTextSpot last; PgfTextSpot last;
bool skipping; bool skipping;
const uint8_t *end; // pointer into the end of the sentence
bool case_sensitive; bool case_sensitive;
PgfPhraseScanner *scanner; PgfPhraseScanner *scanner;
@@ -303,7 +307,7 @@ void finish_skipping(PgfCohortsState *state) {
state->err); state->err);
*/ */
state->last.pos = 0; state->last.pos = 0;
state->last.ptr = NULL; state->last.byte_pos = 0;
state->skipping = false; state->skipping = false;
} }
} }
@@ -311,17 +315,17 @@ void finish_skipping(PgfCohortsState *state) {
static static
void phrasetable_lookup_prefixes(PgfCohortsState *state, void phrasetable_lookup_prefixes(PgfCohortsState *state,
PgfPhrasetable<PgfSymbolKS> table, PgfPhrasetable<PgfSymbolKS> table,
ptrdiff_t min, ptrdiff_t max) size_t min, size_t max)
{ {
if (table == 0) if (table == 0)
return; return;
PgfTextSpot current = state->spot; PgfTextSpot current = state->spot;
int cmp = text_symbol_cmp(&current,state->end,table->value.key,state->case_sensitive); int cmp = text_symbol_cmp(state->sentence,&current,table->value.key,state->case_sensitive);
if (cmp < 0) { if (cmp < 0) {
phrasetable_lookup_prefixes(state,table->left,min,max); phrasetable_lookup_prefixes(state,table->left,min,max);
} else if (cmp > 0) { } else if (cmp > 0) {
ptrdiff_t len = current.ptr - state->spot.ptr; size_t len = current.byte_pos - state->spot.byte_pos;
if (min <= len-1) if (min <= len-1)
phrasetable_lookup_prefixes(state,table->left,min,len-1); phrasetable_lookup_prefixes(state,table->left,min,len-1);
@@ -329,7 +333,7 @@ void phrasetable_lookup_prefixes(PgfCohortsState *state,
if (len <= max) if (len <= max)
phrasetable_lookup_prefixes(state,table->right,len,max); phrasetable_lookup_prefixes(state,table->right,len,max);
} else { } else {
ptrdiff_t len = current.ptr - state->spot.ptr; size_t len = current.byte_pos - state->spot.byte_pos;
finish_skipping(state); finish_skipping(state);
if (state->err->type != PGF_EXN_NONE) if (state->err->type != PGF_EXN_NONE)
@@ -391,15 +395,15 @@ void phrasetable_lookup_cohorts(PgfPhrasetable<PgfSymbolKS> table,
{ {
PgfTextSpot spot; PgfTextSpot spot;
spot.pos = 0; spot.pos = 0;
spot.ptr = (uint8_t *) sentence->text; spot.byte_pos = 0;
PgfCohortsState state; PgfCohortsState state;
state.spot.pos = -1; state.spot.pos = -1;
state.spot.ptr = NULL; state.spot.byte_pos = -1;
state.sentence = sentence;
state.queue.push(spot); state.queue.push(spot);
state.last = spot; state.last = spot;
state.skipping = false; state.skipping = false;
state.end = (uint8_t *) &sentence->text[sentence->size];
state.case_sensitive = case_sensitive; state.case_sensitive = case_sensitive;
state.scanner = scanner; state.scanner = scanner;
state.err = err; state.err = err;
@@ -412,20 +416,20 @@ void phrasetable_lookup_cohorts(PgfPhrasetable<PgfSymbolKS> table,
state.spot = spot; state.spot = spot;
// skip leading spaces // skip leading spaces
while (state.spot.ptr < state.end) { while (state.spot.byte_pos < state.sentence->size) {
const uint8_t *ptr = state.spot.ptr; const uint8_t *ptr = (uint8_t *) &sentence->text[state.spot.byte_pos];
uint32_t ucs = pgf_utf8_decode(&ptr); uint32_t ucs = pgf_utf8_decode(&ptr);
if (!pgf_utf8_is_space(ucs)) if (!pgf_utf8_is_space(ucs))
break; break;
state.spot.pos++; state.spot.pos++;
state.spot.ptr = ptr; state.spot.byte_pos = ptr-(uint8_t *) sentence->text;
} }
state.scanner->space(&spot,&state.spot,state.err); state.scanner->space(&spot,&state.spot,state.err);
if (state.err->type != PGF_EXN_NONE) if (state.err->type != PGF_EXN_NONE)
return; return;
while (state.spot.ptr < state.end) { while (state.spot.byte_pos < state.sentence->size) {
phrasetable_lookup_prefixes(&state, table, 1, sentence->size); phrasetable_lookup_prefixes(&state, table, 1, sentence->size);
if (state.err->type != PGF_EXN_NONE) if (state.err->type != PGF_EXN_NONE)
return; return;
@@ -438,7 +442,7 @@ void phrasetable_lookup_cohorts(PgfPhrasetable<PgfSymbolKS> table,
if (state.err->type != PGF_EXN_NONE) if (state.err->type != PGF_EXN_NONE)
return; return;
state.last.pos = 0; state.last.pos = 0;
state.last.ptr = (uint8_t*) sentence->text; state.last.byte_pos = 0;
break; break;
} else { } else {
// No matches were found, try the next position // No matches were found, try the next position
@@ -451,14 +455,14 @@ void phrasetable_lookup_cohorts(PgfPhrasetable<PgfSymbolKS> table,
state.skipping = true; state.skipping = true;
} }
const uint8_t *ptr = state.spot.ptr; const uint8_t *ptr = (uint8_t *) &state.sentence->text[state.spot.byte_pos];
uint32_t ucs = pgf_utf8_decode(&ptr); uint32_t ucs = pgf_utf8_decode(&ptr);
if (pgf_utf8_is_space(ucs)) { if (pgf_utf8_is_space(ucs)) {
state.queue.push(state.spot); state.queue.push(state.spot);
break; break;
} }
state.spot.pos++; state.spot.pos++;
state.spot.ptr = ptr; state.spot.byte_pos = ptr-(uint8_t *) state.sentence->text;
} }
} }
src/runtime/c/pgf/phrasetable.h
+1 -1
View File
@@ -6,7 +6,7 @@ struct PgfConcrLincat;
struct PGF_INTERNAL_DECL PgfTextSpot { struct PGF_INTERNAL_DECL PgfTextSpot {
size_t pos; // position in Unicode characters size_t pos; // position in Unicode characters
const uint8_t *ptr; // pointer into the spot location size_t byte_pos; // position in number of bytes
}; };
struct PGF_INTERNAL_DECL PgfItem { struct PGF_INTERNAL_DECL PgfItem {
src/runtime/haskell/PGF2.hsc
-31
View File
@@ -87,7 +87,6 @@ module PGF2 (-- * PGF
-- ** Parsing -- ** Parsing
ParseOutput(..), parse, parseWithHeuristics, complete, ParseOutput(..), parse, parseWithHeuristics, complete,
ParseChart, parseChart, getParseChartText, changeParseChartText,
-- * Exceptions -- * Exceptions
PGFError(..), PGFError(..),
@@ -873,36 +872,6 @@ enumerateExprs c_db c_revision c_enum_ptr = do
rest <- unsafeInterleaveIO (fetchLazy c_fetch c_enum) rest <- unsafeInterleaveIO (fetchLazy c_fetch c_enum)
return ((expr,prob) : rest) return ((expr,prob) : rest)
data ParseChart = ParseChart {ch_db :: Ptr PgfDB,
ch_revision :: ForeignPtr Concr,
ch_chart :: ForeignPtr PgfParseChart
}
parseChart :: Concr -> Type -> String -> IO ParseChart
parseChart c ty sent =
withForeignPtr (c_revision c) $ \c_revision_ptr ->
bracket (newStablePtr ty) freeStablePtr $ \c_ty ->
withText sent $ \c_sent -> do
c_chart <- withPgfExn "parseChart" (pgf_parse_chart (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent)
fptr <- newForeignPtr pgf_free_parse_chart c_chart
return (ParseChart (c_db c) (c_revision c) fptr)
getParseChartText :: ParseChart -> IO String
getParseChartText (ParseChart c_db c_revision c_chart) =
withForeignPtr c_chart $ \c_chart_ptr -> do
c_get_text <- (#peek PgfParseChartVtbl, get_text) =<< (#peek PgfParseChart, vtbl) c_chart_ptr
c_text <- callGetText c_get_text c_chart_ptr
peekText c_text
changeParseChartText :: ParseChart -> CSize -> CSize -> String -> IO Bool
changeParseChartText (ParseChart c_db c_revision c_chart) start end text =
withForeignPtr c_revision $ \_ ->
withForeignPtr c_chart $ \c_chart_ptr ->
withText text $ \c_text -> do
c_change <- (#peek PgfParseChartVtbl, change) =<< (#peek PgfParseChart, vtbl) c_chart_ptr
c_res <- callChange c_change c_chart_ptr start end c_text
return (c_res /= 0)
parseWithHeuristics :: Concr -- ^ the language with which we parse parseWithHeuristics :: Concr -- ^ the language with which we parse
-> Type -- ^ the start category -> Type -- ^ the start category
-> String -- ^ the input sentence -> String -- ^ the input sentence
src/runtime/haskell/PGF2/Collab.hsc
+74
View File
@@ -0,0 +1,74 @@
-------------------------------------------------
-- |
-- Module : PGF2.Colab
-- Maintainer : Krasimir Angelov
-- Stability : stable
-- Portability : portable
--
-- This module is a server API for implementing
-- collaborative editing, where the edited document
-- is continuously parsed.
-------------------------------------------------
module PGF2.Collab
( ParseChart, parseChart
, getParseChartText
, Change(..), changeParseChartText
) where
import PGF2.Expr
import PGF2.FFI
import Foreign
import Foreign.C
import Control.Exception(bracket)
#include <pgf/pgf.h>
data ParseChart = ParseChart {ch_db :: Ptr PgfDB,
ch_revision :: ForeignPtr Concr,
ch_chart :: ForeignPtr PgfParseChart
}
parseChart :: Concr -> Type -> String -> IO ParseChart
parseChart c ty sent =
withForeignPtr (c_revision c) $ \c_revision_ptr ->
bracket (newStablePtr ty) freeStablePtr $ \c_ty ->
withText sent $ \c_sent -> do
c_chart <- withPgfExn "parseChart" (pgf_parse_chart (c_db c) c_revision_ptr c_ty marshaller unmarshaller c_sent)
fptr <- newForeignPtr pgf_free_parse_chart c_chart
return (ParseChart (c_db c) (c_revision c) fptr)
getParseChartText :: ParseChart -> IO String
getParseChartText (ParseChart c_db c_revision c_chart) =
withForeignPtr c_chart $ \c_chart_ptr -> do
c_get_text <- (#peek PgfParseChartVtbl, get_text) =<< (#peek PgfParseChart, vtbl) c_chart_ptr
c_text <- callGetText c_get_text c_chart_ptr
peekText c_text
data Change = Skip {-# UNPACK #-} !CSize | Change {-# UNPACK #-} !CSize String deriving Show
changeParseChartText :: ParseChart -> [Change] -> IO Bool
changeParseChartText (ParseChart c_db c_revision c_chart) changes =
withForeignPtr c_revision $ \_ ->
withForeignPtr c_chart $ \c_chart_ptr -> do
c_start <- (#peek PgfParseChartVtbl, start) =<< (#peek PgfParseChart, vtbl) c_chart_ptr
callStart c_start c_chart_ptr
res <- apply c_chart_ptr changes
c_done <- (#peek PgfParseChartVtbl, done) =<< (#peek PgfParseChart, vtbl) c_chart_ptr
callDone c_done c_chart_ptr
return res
where
apply c_chart_ptr [] = return True
apply c_chart_ptr (Skip i :changes) = do
c_skip <- (#peek PgfParseChartVtbl, skip) =<< (#peek PgfParseChart, vtbl) c_chart_ptr
c_res <- callSkip c_skip c_chart_ptr i
if c_res == 0
then return False
else apply c_chart_ptr changes
apply c_chart_ptr (Change i text:changes) = do
c_change <- (#peek PgfParseChartVtbl, change) =<< (#peek PgfParseChart, vtbl) c_chart_ptr
c_res <- withText text (callChange c_change c_chart_ptr i)
if c_res == 0
then return False
else apply c_chart_ptr changes
src/runtime/haskell/PGF2/FFI.hsc
+7 -1
View File
@@ -295,7 +295,13 @@ foreign import ccall "&pgf_free_parse_chart" pgf_free_parse_chart :: FunPtr (Ptr
foreign import ccall "dynamic" callGetText :: Dynamic (Ptr PgfParseChart -> IO (Ptr PgfText)) foreign import ccall "dynamic" callGetText :: Dynamic (Ptr PgfParseChart -> IO (Ptr PgfText))
foreign import ccall "dynamic" callChange :: Dynamic (Ptr PgfParseChart -> CSize -> CSize -> Ptr PgfText -> IO CInt) foreign import ccall "dynamic" callStart :: Dynamic (Ptr PgfParseChart -> IO CInt)
foreign import ccall "dynamic" callSkip :: Dynamic (Ptr PgfParseChart -> CSize -> IO CInt)
foreign import ccall "dynamic" callChange :: Dynamic (Ptr PgfParseChart -> CSize -> Ptr PgfText -> IO CInt)
foreign import ccall "dynamic" callDone :: Dynamic (Ptr PgfParseChart -> IO CInt)
foreign import ccall "wrapper" wrapSymbol0 :: Wrapper (Ptr PgfLinearizationOutputIface -> IO ()) foreign import ccall "wrapper" wrapSymbol0 :: Wrapper (Ptr PgfLinearizationOutputIface -> IO ())
src/runtime/haskell/pgf2.cabal
+1
View File
@@ -23,6 +23,7 @@ library
exposed-modules: exposed-modules:
PGF2, PGF2,
PGF2.Transactions, PGF2.Transactions,
PGF2.Collab,
PGF2.ByteCode, PGF2.ByteCode,
-- backwards compatibility API: -- backwards compatibility API:
PGF PGF