msyds/gf-core
1
0
Fork 0
forked from GitHub/gf-core
Code Pull Requests Activity

progress on the parser

Browse Source
This commit is contained in:
Krasimir Angelov
2023-06-02 11:52:47 +02:00
parent 58b1a9a535
commit 1a17234a9c
3 changed files with 105 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
src/runtime/c/pgf/parser.cxx
View File

@@ -1,11 +1,12 @@
#include "data.h" #include "data.h"
#include "printer.h" #include "printer.h"
#include "parser.h" #include "parser.h"
#include <algorithm>
//#define DEBUG_STATE_CREATION //#define DEBUG_STATE_CREATION
//#define DEBUG_AUTOMATON //#define DEBUG_AUTOMATON
//#define DEBUG_PARSER #define DEBUG_PARSER
//#define DEBUG_GENERATOR #define DEBUG_GENERATOR
struct PgfLRTableMaker::Item { struct PgfLRTableMaker::Item {
object lin_obj; object lin_obj;
@@ -459,11 +460,11 @@ ref<PgfLRTable> PgfLRTableMaker::make()
struct PgfParser::Choice { struct PgfParser::Choice {
int fid; int fid;
std::vector<Production*> prods; std::vector<Production*> prods;
Result* res; std::vector<ExprState*> states;
std::vector<ExprInstance> exprs;
Choice(int fid) { Choice(int fid) {
this->fid = fid; this->fid = fid;
this->res = NULL;
} }
}; };
@@ -495,23 +496,25 @@ struct PgfParser::Production {
}; };
struct PgfParser::StackNode { struct PgfParser::StackNode {
Stage *stage;
size_t state_id; size_t state_id;
Choice *choice; Choice *choice;
std::vector<StackNode*> parents; std::vector<StackNode*> parents;
StackNode(size_t state_id) { StackNode(Stage *stage, size_t state_id) {
this->stage = stage;
this->state_id = state_id; this->state_id = state_id;
this->choice = NULL; this->choice = NULL;
} }
}; };
struct PgfParser::ParseState { struct PgfParser::Stage {
ParseState *next; Stage *next;
PgfTextSpot start; PgfTextSpot start;
PgfTextSpot end; PgfTextSpot end;
std::vector<StackNode*> stacks; std::vector<StackNode*> nodes;
ParseState(PgfTextSpot spot) { Stage(PgfTextSpot spot) {
next = NULL; next = NULL;
start = spot; start = spot;
end = spot; end = spot;
@@ -519,9 +522,9 @@ struct PgfParser::ParseState {
}; };
struct PgfParser::ExprState { struct PgfParser::ExprState {
Result *res;
prob_t prob; prob_t prob;
Choice *choice;
Production *prod; Production *prod;
size_t n_args; size_t n_args;
PgfExpr expr; PgfExpr expr;
@@ -537,11 +540,6 @@ struct PgfParser::ExprInstance {
} }
}; };
struct PgfParser::Result {
std::vector<ExprState*> states;
std::vector<ExprInstance> exprs;
};
#if defined(DEBUG_STATE_CREATION) || defined(DEBUG_AUTOMATON) || defined(DEBUG_PARSER) #if defined(DEBUG_STATE_CREATION) || defined(DEBUG_AUTOMATON) || defined(DEBUG_PARSER)
void PgfParser::print_prod(Choice *choice, Production *prod) void PgfParser::print_prod(Choice *choice, Production *prod)
{ {
@@ -570,12 +568,12 @@ void PgfParser::print_prod(Choice *choice, Production *prod)
free(text); free(text);
} }
void PgfParser::print_transition(StackNode *source, StackNode *target, ParseState *state) void PgfParser::print_transition(StackNode *source, StackNode *target, Stage *stage)
{ {
fprintf(stderr, "state %ld --- ?%d ---> state %ld (position %zu-%zu, count %zu)\n", fprintf(stderr, "state %ld --- ?%d ---> state %ld (position %zu-%zu, nodes %zu)\n",
source->state_id, target->choice->fid, target->state_id, source->state_id, target->choice->fid, target->state_id,
state->start.pos, state->end.pos, stage->start.pos, stage->end.pos,
state->stacks.size()); stage->nodes.size());
} }
#endif #endif
@@ -586,8 +584,8 @@ PgfParser::PgfParser(ref<PgfConcr> concr, ref<PgfConcrLincat> start, PgfText *se
this->m = m; this->m = m;
this->u = u; this->u = u;
this->last_fid = 0; this->last_fid = 0;
this->top_res = NULL; this->top_choice = NULL;
this->top_res_index = 0; this->top_choice_index = 0;
PgfTextSpot spot; PgfTextSpot spot;
spot.pos = 0; spot.pos = 0;
@@ -595,38 +593,46 @@ PgfParser::PgfParser(ref<PgfConcr> concr, ref<PgfConcrLincat> start, PgfText *se
this->before = NULL; this->before = NULL;
this->after = NULL; this->after = NULL;
this->ahead = new ParseState(spot); this->ahead = new Stage(spot);
StackNode *node = new StackNode(0); StackNode *node = new StackNode(ahead, 0);
this->ahead->stacks.push_back(node); this->ahead->nodes.push_back(node);
} }
void PgfParser::shift(StackNode *parent, ref<PgfConcrLincat> lincat, size_t r, Production *prod, void PgfParser::shift(StackNode *parent, ref<PgfConcrLincat> lincat, size_t r, Production *prod,
ParseState *state) Stage *before, Stage *after)
{ {
ref<Vector<PgfLRShift>> shifts = vector_elem(concr->lrtable,parent->state_id)->shifts; ref<Vector<PgfLRShift>> shifts = vector_elem(concr->lrtable,parent->state_id)->shifts;
for (size_t i = 0; i < shifts->len; i++) { for (size_t i = 0; i < shifts->len; i++) {
ref<PgfLRShift> shift = vector_elem(shifts,i); ref<PgfLRShift> shift = vector_elem(shifts,i);
if (lincat == shift->lincat && r == shift->r) { if (lincat == shift->lincat && r == shift->r) {
StackNode *node = NULL; StackNode *node = NULL;
for (StackNode *n : state->stacks) { for (StackNode *n : after->nodes) {
if (n->state_id == shift->next_state) { if (n->stage == before && n->state_id == shift->next_state) {
node = n; node = n;
break; break;
} }
} }
if (node == NULL) { if (node == NULL) {
node = new StackNode(shift->next_state); node = new StackNode(before, shift->next_state);
node->choice = new Choice(++last_fid); node->choice = new Choice(++last_fid);
node->parents.push_back(parent); after->nodes.push_back(node);
state->stacks.push_back(node);
} }
node->choice->prods.push_back(prod);
if (std::find(node->choice->prods.begin(), node->choice->prods.end(), prod) == node->choice->prods.end()) {
node->choice->prods.push_back(prod);
#ifdef DEBUG_PARSER #ifdef DEBUG_PARSER
print_prod(node->choice, prod); print_prod(node->choice, prod);
print_transition(parent,node,state);
#endif #endif
}
if (std::find(node->parents.begin(), node->parents.end(), parent) == node->parents.end()) {
node->parents.push_back(parent);
#ifdef DEBUG_PARSER
print_transition(parent,node,after);
#endif
}
break; break;
} }
} }
@@ -671,7 +677,8 @@ PgfParser::Choice *PgfParser::intersect_choice(Choice *choice1, Choice *choice2,
} }
void PgfParser::reduce(StackNode *parent, ref<PgfConcrLin> lin, size_t seq_index, void PgfParser::reduce(StackNode *parent, ref<PgfConcrLin> lin, size_t seq_index,
size_t n, std::vector<Choice*> &args) size_t n, std::vector<Choice*> &args,
Stage *before, Stage *after)
{ {
if (n == 0) { if (n == 0) {
ref<PgfConcrLincat> lincat = lin->lincat; ref<PgfConcrLincat> lincat = lin->lincat;
@@ -699,13 +706,13 @@ void PgfParser::reduce(StackNode *parent, ref<PgfConcrLin> lin, size_t seq_index
} }
} }
shift(parent, lincat, r, prod, before); shift(parent, lincat, r, prod, before, after);
return; return;
} }
args.push_back(parent->choice); args.push_back(parent->choice);
for (auto node : parent->parents) { for (auto node : parent->parents) {
reduce(node, lin, seq_index, n-1, args); reduce(node, lin, seq_index, n-1, args, parent->stage, after);
} }
args.pop_back(); args.pop_back();
} }
@@ -714,11 +721,7 @@ void PgfParser::complete(StackNode *parent, ref<PgfConcrLincat> lincat, size_t s
size_t n, std::vector<Choice*> &args) size_t n, std::vector<Choice*> &args)
{ {
if (n == 0) { if (n == 0) {
Choice *choice = args[0]; top_choice = args[0];
if (top_res == NULL)
top_res = new Result();
choice->res = top_res;
predict_expr_states(choice, 0);
return; return;
} }
@@ -736,18 +739,17 @@ void PgfParser::reduce_all(StackNode *node)
ref<PgfLRShift> shift = vector_elem(shifts,j); ref<PgfLRShift> shift = vector_elem(shifts,j);
if (shift->is_epsilon) { if (shift->is_epsilon) {
StackNode *new_node = NULL; StackNode *new_node = NULL;
for (StackNode *n : before->stacks) { for (StackNode *n : before->nodes) {
if (n->state_id == shift->next_state) { if (n->stage == before && n->state_id == shift->next_state) {
new_node = n; new_node = n;
break; break;
} }
} }
if (new_node == NULL) { if (new_node == NULL) {
new_node = new StackNode(shift->next_state); new_node = new StackNode(before, shift->next_state);
new_node->choice = new Choice(++last_fid); new_node->choice = new Choice(++last_fid);
new_node->parents.push_back(node); before->nodes.push_back(new_node);
before->stacks.push_back(new_node);
std::function<void(ref<PgfConcrLin>, size_t seq_index)> f = std::function<void(ref<PgfConcrLin>, size_t seq_index)> f =
[this,new_node](ref<PgfConcrLin> lin, size_t seq_index) { [this,new_node](ref<PgfConcrLin> lin, size_t seq_index) {
@@ -761,9 +763,12 @@ void PgfParser::reduce_all(StackNode *node)
shift->lincat, shift->r, f); shift->lincat, shift->r, f);
} }
if (std::find(new_node->parents.begin(), new_node->parents.end(), node) == new_node->parents.end()) {
new_node->parents.push_back(node);
#ifdef DEBUG_PARSER #ifdef DEBUG_PARSER
print_transition(node,new_node,before); print_transition(node,new_node,before);
#endif #endif
}
} }
} }
@@ -776,7 +781,7 @@ void PgfParser::reduce_all(StackNode *node)
ref<PgfConcrLin>::untagged(red->lin_obj); ref<PgfConcrLin>::untagged(red->lin_obj);
ref<PgfSequence> seq = *vector_elem(lin->seqs,red->seq_index); ref<PgfSequence> seq = *vector_elem(lin->seqs,red->seq_index);
std::vector<Choice*> args; std::vector<Choice*> args;
reduce(node, lin, red->seq_index, seq->syms.len, args); reduce(node, lin, red->seq_index, seq->syms.len, args, before, before);
break; break;
} }
case PgfConcrLincat::tag: { case PgfConcrLincat::tag: {
@@ -800,7 +805,7 @@ void PgfParser::space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err)
#endif #endif
while (ahead != NULL && ahead->start.pos <= start->pos) { while (ahead != NULL && ahead->start.pos <= start->pos) {
ParseState *tmp = ahead->next; Stage *tmp = ahead->next;
ahead->next = before; ahead->next = before;
before = ahead; before = ahead;
ahead = tmp; ahead = tmp;
@@ -809,21 +814,21 @@ void PgfParser::space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err)
before->end = *end; before->end = *end;
size_t i = 0; size_t i = 0;
while (i < before->stacks.size()) { while (i < before->nodes.size()) {
StackNode *node = before->stacks[i++]; StackNode *node = before->nodes[i++];
reduce_all(node); reduce_all(node);
} }
} }
void PgfParser::start_matches(PgfTextSpot *end, PgfExn* err) void PgfParser::start_matches(PgfTextSpot *end, PgfExn* err)
{ {
ParseState **last = &ahead; after = *last; Stage **last = &ahead; after = *last;
while (after != NULL && after->start.pos < end->pos) { while (after != NULL && after->start.pos < end->pos) {
last = &after->next; after = *last; last = &after->next; after = *last;
} }
if (after == NULL) { if (after == NULL) {
*last = new ParseState(*end); *last = new Stage(*end);
after = *last; after = *last;
} }
} }
@@ -834,8 +839,8 @@ void PgfParser::match(ref<PgfConcrLin> lin, size_t seq_index, PgfExn* err)
Production *prod = new(lin) Production(); Production *prod = new(lin) Production();
for (StackNode *parent : before->stacks) { for (StackNode *parent : before->nodes) {
shift(parent, lin->lincat, r, prod, after); shift(parent, lin->lincat, r, prod, before, after);
} }
} }
@@ -847,11 +852,17 @@ bool PgfParser::CompareExprState::operator() (const ExprState *state1, const Exp
return state1->prob > state2->prob; return state1->prob > state2->prob;
} }
void PgfParser::prepare()
{
if (top_choice != NULL)
predict_expr_states(top_choice, 0);
}
void PgfParser::predict_expr_states(Choice *choice, prob_t outside_prob) void PgfParser::predict_expr_states(Choice *choice, prob_t outside_prob)
{ {
for (Production *prod : choice->prods) { for (Production *prod : choice->prods) {
ExprState *state = new ExprState; ExprState *state = new ExprState;
state->res = choice->res; state->choice = choice;
state->prod = prod; state->prod = prod;
state->n_args = 0; state->n_args = 0;
state->expr = u->efun(&prod->lin->name); state->expr = u->efun(&prod->lin->name);
@@ -863,8 +874,8 @@ void PgfParser::predict_expr_states(Choice *choice, prob_t outside_prob)
#ifdef DEBUG_GENERATOR #ifdef DEBUG_GENERATOR
void PgfParser::print_expr_state_before(PgfPrinter *printer, ExprState *state) void PgfParser::print_expr_state_before(PgfPrinter *printer, ExprState *state)
{ {
if (state->res->states.size() > 0) { if (state->choice->states.size() > 0) {
ExprState *parent = state->res->states[0]; ExprState *parent = state->choice->states[0];
print_expr_state_before(printer, parent); print_expr_state_before(printer, parent);
printer->puts(" ["); printer->puts(" [");
} }
@@ -874,12 +885,12 @@ void PgfParser::print_expr_state_before(PgfPrinter *printer, ExprState *state)
void PgfParser::print_expr_state_after(PgfPrinter *printer, ExprState *state) void PgfParser::print_expr_state_after(PgfPrinter *printer, ExprState *state)
{ {
for (size_t i = state->n_args+1; i < state->prod->n_args; i++) { for (size_t i = state->n_args+1; i < state->prod->n_args; i++) {
printer->puts(" ?"); printer->nprintf(32, " ?%d", state->prod->args[i]->fid);
} }
if (state->res->states.size() > 0) { if (state->choice->states.size() > 0) {
printer->puts("]"); printer->puts("]");
ExprState *parent = state->res->states[0]; ExprState *parent = state->choice->states[0];
print_expr_state_after(printer, parent); print_expr_state_after(printer, parent);
} }
} }
@@ -890,8 +901,13 @@ void PgfParser::print_expr_state(ExprState *state)
printer.nprintf(16, "[%f] ", state->prob); printer.nprintf(16, "[%f] ", state->prob);
print_expr_state_before(&printer, state); print_expr_state_before(&printer, state);
if (state->prod->n_args > 0) if (state->n_args < state->prod->n_args) {
printer.puts(" ?"); Choice *choice = state->prod->args[state->n_args];
if (choice == NULL)
printer.puts(" ?");
else
printer.nprintf(32, " ?%d", state->prod->args[state->n_args]->fid);
}
print_expr_state_after(&printer, state); print_expr_state_after(&printer, state);
printer.puts("\n"); printer.puts("\n");
@@ -916,16 +932,12 @@ bool PgfParser::process_expr_state(ExprState *state)
u->free_ref(meta); u->free_ref(meta);
state->expr = app; state->expr = app;
} else { } else {
Result *tmp = choice->res; choice->states.push_back(state);
if (choice->res == NULL) {
choice->res = new Result();
}
choice->res->states.push_back(state);
if (tmp == NULL) { if (choice->states.size() == 1) {
predict_expr_states(choice, state->prob); predict_expr_states(choice, state->prob);
} else { } else {
for (ExprInstance p : choice->res->exprs) { for (ExprInstance p : choice->exprs) {
combine_expr_state(state,p); combine_expr_state(state,p);
} }
} }
@@ -936,18 +948,18 @@ bool PgfParser::process_expr_state(ExprState *state)
void PgfParser::complete_expr_state(ExprState *state) void PgfParser::complete_expr_state(ExprState *state)
{ {
Result *res = state->res; Choice *choice = state->choice;
prob_t outside_prob; prob_t outside_prob;
if (res == top_res) if (choice == top_choice)
outside_prob = 0; outside_prob = 0;
else else
outside_prob = res->states[0]->prob; outside_prob = choice->states[0]->prob;
prob_t inside_prob = state->prob-outside_prob; prob_t inside_prob = state->prob-outside_prob;
res->exprs.emplace_back(state->expr,inside_prob); choice->exprs.emplace_back(state->expr,inside_prob);
for (ExprState *state : res->states) { for (ExprState *state : choice->states) {
combine_expr_state(state,res->exprs.back()); combine_expr_state(state,choice->exprs.back());
} }
} }
@@ -956,8 +968,8 @@ void PgfParser::combine_expr_state(ExprState *state, ExprInstance &inst)
PgfExpr app = u->eapp(state->expr, inst.expr); PgfExpr app = u->eapp(state->expr, inst.expr);
ExprState *app_state = new ExprState(); ExprState *app_state = new ExprState();
app_state->res = state->res;
app_state->prob = state->prob + inst.prob; app_state->prob = state->prob + inst.prob;
app_state->choice = state->choice;
app_state->prod = state->prod; app_state->prod = state->prod;
app_state->n_args = state->n_args+1; app_state->n_args = state->n_args+1;
app_state->expr = app; app_state->expr = app;
@@ -973,12 +985,12 @@ PgfExpr PgfParser::fetch(PgfDB *db, prob_t *prob)
{ {
DB_scope scope(db, READER_SCOPE); DB_scope scope(db, READER_SCOPE);
if (top_res == NULL) if (top_choice == NULL)
return 0; return 0;
for (;;) { for (;;) {
if (top_res_index < top_res->exprs.size()) { if (top_choice_index < top_choice->exprs.size()) {
auto inst = top_res->exprs[top_res_index++]; auto inst = top_choice->exprs[top_choice_index++];
*prob = inst.prob; *prob = inst.prob;
return inst.expr; return inst.expr;
} }

18
src/runtime/c/pgf/parser.h
View File

@@ -63,30 +63,30 @@ class PGF_INTERNAL_DECL PgfParser : public PgfPhraseScanner, public PgfExprEnum
struct Choice; struct Choice;
struct Production; struct Production;
struct StackNode; struct StackNode;
struct ParseState; struct Stage;
struct ExprState; struct ExprState;
struct ExprInstance; struct ExprInstance;
struct Result;
struct CompareExprState : std::less<ExprState*> { struct CompareExprState : std::less<ExprState*> {
bool operator() (const ExprState *state1, const ExprState *state2) const; bool operator() (const ExprState *state1, const ExprState *state2) const;
}; };
ParseState *before, *after, *ahead; Stage *before, *after, *ahead;
std::priority_queue<ExprState*, std::vector<ExprState*>, CompareExprState> queue; std::priority_queue<ExprState*, std::vector<ExprState*>, CompareExprState> queue;
int last_fid; int last_fid;
Result *top_res; Choice *top_choice;
size_t top_res_index; size_t top_choice_index;
void shift(StackNode *parent, ref<PgfConcrLincat> lincat, size_t r, Production *prod, void shift(StackNode *parent, ref<PgfConcrLincat> lincat, size_t r, Production *prod,
ParseState *state); Stage *before, Stage *after);
void reduce(StackNode *parent, ref<PgfConcrLin> lin, size_t seq_index, void reduce(StackNode *parent, ref<PgfConcrLin> lin, size_t seq_index,
size_t n, std::vector<Choice*> &args); size_t n, std::vector<Choice*> &args,
Stage *before, Stage *after);
void complete(StackNode *parent, ref<PgfConcrLincat> lincat, size_t seq_index, void complete(StackNode *parent, ref<PgfConcrLincat> lincat, size_t seq_index,
size_t n, std::vector<Choice*> &args); size_t n, std::vector<Choice*> &args);
void reduce_all(StackNode *state); void reduce_all(StackNode *state);
void print_prod(Choice *choice, Production *prod); void print_prod(Choice *choice, Production *prod);
void print_transition(StackNode *source, StackNode *target, ParseState *state); void print_transition(StackNode *source, StackNode *target, Stage *stage);
typedef std::map<std::pair<Choice*,Choice*>,Choice*> intersection_map; typedef std::map<std::pair<Choice*,Choice*>,Choice*> intersection_map;
@@ -110,6 +110,8 @@ public:
virtual void match(ref<PgfConcrLin> lin, size_t seq_index, PgfExn* err); virtual void match(ref<PgfConcrLin> lin, size_t seq_index, PgfExn* err);
virtual void end_matches(PgfTextSpot *end, PgfExn* err); virtual void end_matches(PgfTextSpot *end, PgfExn* err);
void prepare();
PgfExpr fetch(PgfDB *db, prob_t *prob); PgfExpr fetch(PgfDB *db, prob_t *prob);
}; };
#endif #endif

1
src/runtime/c/pgf/pgf.cxx
View File

@@ -2656,6 +2656,7 @@ PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision,
phrasetable_lookup_cohorts(concr->phrasetable, phrasetable_lookup_cohorts(concr->phrasetable,
sentence, case_sensitive, sentence, case_sensitive,
parser, err); parser, err);
parser->prepare();
return parser; return parser;
} PGF_API_END } PGF_API_END