msyds/gf-core
1
0
Fork 0
forked from GitHub/gf-core
Code Pull Requests Activity

use TextSpots for correct extraction of unknown words

Browse Source
This commit is contained in:
Krasimir Angelov
2022-09-30 11:58:09 +02:00
parent 6b63c2f779
commit 855fa7ebf3
5 changed files with 56 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/runtime/c/pgf/parser.cxx
View File

@@ -83,7 +83,7 @@ public:
}
public:
size_t start, end;
PgfTextSpot start, end;
State *prev, *next;
prob_t viterbi_prob;
@@ -495,10 +495,10 @@ public:
prev = prev->prev;
}
size_t size = state->start-prev->end;
size_t size = state->start.ptr-prev->end.ptr;
PgfText *token = (PgfText *) alloca(sizeof(PgfText)+size+1);
token->size = size;
memcpy(token->text,parser->sentence->text+prev->end,size);
memcpy(token->text,prev->end.ptr,size);
token->text[size] = 0;
PgfExpr expr = u->elit(u->lstr(token));
@@ -638,19 +638,19 @@ PgfParser::PgfParser(ref<PgfConcr> concr, ref<PgfConcrLincat> start, PgfText *se
this->m = m;
}
void PgfParser::space(size_t start, size_t end, PgfExn* err)
void PgfParser::space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err)
{
State *prev = NULL;
State *next = before;
while (next != NULL && next->start < start) {
while (next != NULL && next->start.pos < start->pos) {
prev = next;
next = next->next;
}
if (next == NULL || next->start != start) {
if (next == NULL || next->start.pos != start->pos) {
before = new State();
before->start = start;
before->end = end;
before->start = *start;
before->end = *end;
before->prev = prev;
before->next = next;
before->viterbi_prob = prev ? prev->viterbi_prob : 0;
@@ -659,23 +659,23 @@ void PgfParser::space(size_t start, size_t end, PgfExn* err)
if (next != NULL) next->prev = before;
} else {
before = next;
before->end = end;
before->end = *end;
}
}
void PgfParser::start_matches(size_t end, PgfExn* err)
void PgfParser::start_matches(PgfTextSpot *end, PgfExn* err)
{
State *prev = NULL;
State *next = before;
while (next != NULL && next->start < end) {
while (next != NULL && next->start.pos < end->pos) {
prev = next;
next = next->next;
}
if (next == NULL || next->start != end) {
if (next == NULL || next->start.pos != end->pos) {
after = new State();
after->start = end;
after->end = end;
after->start = *end;
after->end = *end;
after->prev = prev;
after->next = next;
after->viterbi_prob = prev ? prev->viterbi_prob : 0;
@@ -696,7 +696,7 @@ void PgfParser::match(ref<PgfConcrLin> lin, size_t seq_index, PgfExn* err)
after->queue.push(new(0) ParseItem(conts, lin, seq_index));
}
void PgfParser::end_matches(size_t end, PgfExn* err)
void PgfParser::end_matches(PgfTextSpot *end, PgfExn* err)
{
while (!after->queue.empty()) {
Item *item = after->queue.top();

6
src/runtime/c/pgf/parser.h
View File

@@ -5,10 +5,10 @@ class PGF_INTERNAL_DECL PgfParser : public PgfPhraseScanner, public PgfExprEnum
public:
PgfParser(ref<PgfConcr> concr, ref<PgfConcrLincat> start, PgfText *sentence, PgfMarshaller *m);
void space(size_t start, size_t end, PgfExn* err);
void start_matches(size_t end, PgfExn* err);
void space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err);
void start_matches(PgfTextSpot *end, PgfExn* err);
void match(ref<PgfConcrLin> lin, size_t seq_index, PgfExn* err);
void end_matches(size_t end, PgfExn* err);
void end_matches(PgfTextSpot *end, PgfExn* err);
void prepare();
PgfExpr fetch(PgfDB *db, PgfUnmarshaller *u, prob_t *prob);

16
src/runtime/c/pgf/pgf.cxx
View File

@@ -822,11 +822,11 @@ public:
this->callback = callback;
}
virtual void space(size_t start, size_t end, PgfExn* err)
virtual void space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err)
{
}
virtual void start_matches(size_t end, PgfExn* err)
virtual void start_matches(PgfTextSpot *end, PgfExn* err)
{
}
@@ -837,7 +837,7 @@ public:
callback->fn(callback, &lin->absfun->name, &(*field->name), lin->lincat->abscat->prob+lin->absfun->prob, err);
}
virtual void end_matches(size_t end, PgfExn* err)
virtual void end_matches(PgfTextSpot *end, PgfExn* err)
{
}
@@ -869,12 +869,12 @@ public:
this->callback = callback;
}
virtual void space(size_t start, size_t end, PgfExn* err)
virtual void space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err)
{
match_start = end;
match_start = end->pos;
}
virtual void start_matches(size_t match_end, PgfExn* err)
virtual void start_matches(PgfTextSpot *end, PgfExn* err)
{
}
@@ -885,9 +885,9 @@ public:
callback->morpho.fn(&callback->morpho, &lin->absfun->name, &(*field->name), lin->lincat->abscat->prob+lin->absfun->prob, err);
}
virtual void end_matches(size_t match_end, PgfExn* err)
virtual void end_matches(PgfTextSpot *end, PgfExn* err)
{
callback->fn(callback, match_start, match_end, err);
callback->fn(callback, match_start, end->pos, err);
}
private:

47
src/runtime/c/pgf/phrasetable.cxx
View File

@@ -228,11 +228,6 @@ int sequence_cmp(ref<PgfSequence> seq1, ref<PgfSequence> seq2)
return 0;
}
struct PGF_INTERNAL_DECL PgfTextSpot {
size_t pos; // position in Unicode characters
const uint8_t *ptr; // pointer into the spot location
};
static
int text_sequence_cmp(PgfTextSpot *spot, const uint8_t *end,
ref<PgfSequence> seq,
@@ -535,7 +530,7 @@ struct PGF_INTERNAL_DECL PgfCohortsState {
PgfTextSpot spot;
std::priority_queue<PgfTextSpot, std::vector<PgfTextSpot>, PgfTextSpotComparator> queue;
size_t last_pos;
PgfTextSpot last;
bool skipping;
const uint8_t *end; // pointer into the end of the sentence
@@ -552,34 +547,35 @@ void finish_skipping(PgfCohortsState *state) {
if (spot.pos >= state->spot.pos)
break;
if (spot.pos != state->last_pos) {
if (state->last_pos > 0) {
state->scanner->space(spot.pos, spot.pos,
if (spot.pos != state->last.pos) {
if (state->last.pos > 0) {
state->scanner->space(&spot, &spot,
state->err);
if (state->err->type != PGF_EXN_NONE)
return;
}
state->scanner->start_matches(state->spot.pos,
state->scanner->start_matches(&state->spot,
state->err);
if (state->err->type != PGF_EXN_NONE)
return;
state->scanner->end_matches(state->spot.pos,
state->scanner->end_matches(&state->spot,
state->err);
if (state->err->type != PGF_EXN_NONE)
return;
state->last_pos = spot.pos;
state->last = spot;
}
state->queue.pop();
}
state->scanner->space(state->spot.pos, state->spot.pos,
state->scanner->space(&state->spot, &state->spot,
state->err);
state->last_pos = 0;
state->last.pos = 0;
state->last.ptr = NULL;
state->skipping = false;
}
}
@@ -616,20 +612,20 @@ void phrasetable_lookup_prefixes(PgfCohortsState *state,
auto backrefs = table->value.backrefs;
if (len > 0 && backrefs != 0) {
if (state->last_pos != current.pos) {
if (state->last_pos > 0) {
state->scanner->end_matches(state->last_pos,
if (state->last.pos != current.pos) {
if (state->last.pos > 0) {
state->scanner->end_matches(&state->last,
state->err);
if (state->err->type != PGF_EXN_NONE)
return;
}
state->scanner->start_matches(current.pos,
state->scanner->start_matches(&current,
state->err);
if (state->err->type != PGF_EXN_NONE)
return;
state->last_pos = current.pos;
state->last = current;
}
state->queue.push(current);
@@ -668,13 +664,13 @@ void phrasetable_lookup_cohorts(PgfPhrasetable table,
{
PgfTextSpot spot;
spot.pos = 0;
spot.ptr = (uint8_t *) &sentence->text[0];
spot.ptr = (uint8_t *) sentence->text;
PgfCohortsState state;
state.spot.pos = -1;
state.spot.ptr = NULL;
state.queue.push(spot);
state.last_pos = 0;
state.last = spot;
state.skipping = false;
state.end = (uint8_t *) &sentence->text[sentence->size];
state.case_sensitive = case_sensitive;
@@ -698,7 +694,7 @@ void phrasetable_lookup_cohorts(PgfPhrasetable table,
state.spot.ptr = ptr;
}
state.scanner->space(spot.pos,state.spot.pos,state.err);
state.scanner->space(&spot,&state.spot,state.err);
if (state.err->type != PGF_EXN_NONE)
return;
@@ -707,14 +703,15 @@ void phrasetable_lookup_cohorts(PgfPhrasetable table,
if (state.err->type != PGF_EXN_NONE)
return;
if (state.last_pos > 0) {
if (state.last.pos > 0) {
// We found at least one match.
// The last range is yet to be reported.
state.scanner->end_matches(state.last_pos,
state.scanner->end_matches(&state.last,
state.err);
if (state.err->type != PGF_EXN_NONE)
return;
state.last_pos = 0;
state.last.pos = 0;
state.last.ptr = (uint8_t*) sentence->text;
break;
} else {
// No matches were found, try the next position

11
src/runtime/c/pgf/phrasetable.h
View File

@@ -71,12 +71,17 @@ size_t phrasetable_size(PgfPhrasetable table);
class PgfConcrLin;
struct PGF_INTERNAL_DECL PgfTextSpot {
size_t pos; // position in Unicode characters
const uint8_t *ptr; // pointer into the spot location
};
class PGF_INTERNAL_DECL PgfPhraseScanner {
public:
virtual void space(size_t start, size_t end, PgfExn* err)=0;
virtual void start_matches(size_t pos, PgfExn* err)=0;
virtual void space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err)=0;
virtual void start_matches(PgfTextSpot *spot, PgfExn* err)=0;
virtual void match(ref<PgfConcrLin> lin, size_t seq_index, PgfExn* err)=0;
virtual void end_matches(size_t pos, PgfExn* err)=0;
virtual void end_matches(PgfTextSpot *spot, PgfExn* err)=0;
};
PGF_INTERNAL_DECL