GitHub/gf-core
1
0
Fork 1
mirror of https://github.com/GrammaticalFramework/gf-core.git synced 2026-04-23 03:32:51 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix the robust parser for unknown tokens

Browse Source
This commit is contained in:
kr.angelov
2013-11-01 13:05:17 +00:00
parent ffa143f1d2
commit d7cd5ba195
1 changed files with 15 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/runtime/c/pgf/parser.c
View File

@@ -1359,16 +1359,23 @@ static void
pgf_parsing_meta_scan(PgfParsing* ps, pgf_parsing_meta_scan(PgfParsing* ps,
PgfItem* meta_item, prob_t meta_prob) PgfItem* meta_item, prob_t meta_prob)
{ {
/* PgfToken tok = after->ts->fn->get_token(after->ts); PgfItem* item = pgf_item_copy(meta_item, ps);
item->inside_prob += meta_prob;
if (*tok == 0) { size_t offset = ps->before->end_offset;
PgfItem* item = pgf_item_copy(meta_item, before->ps); while (ps->sentence[offset] != 0 &&
item->inside_prob += meta_prob; !gu_is_space(ps->sentence[offset])) {
offset++;
}
size_t len = offset - ps->before->end_offset;
char* tok = gu_malloc(ps->pool, len+1);
memcpy(tok, ps->sentence+ps->before->end_offset, len);
tok[len] = 0;
pgf_add_extern_tok(item, tok, ps->pool); pgf_add_extern_tok(&item->curr_sym, tok, ps->pool);
gu_buf_heap_push(before->agenda, pgf_item_prob_order, &item); gu_buf_heap_push(ps->before->agenda, pgf_item_prob_order, &item);
}*/
} }
typedef struct { typedef struct {
@@ -1679,7 +1686,7 @@ pgf_parsing_item(PgfParsing* ps, PgfItem* item)
if (ps->before->meta_item != NULL) if (ps->before->meta_item != NULL)
break; break;
ps->before->meta_item = item; ps->before->meta_item = item;
if (ps->before->end_offset == strlen(ps->sentence)) { if (ps->before->end_offset == strlen(ps->sentence)) {
PgfExprProb *ep = gu_new(PgfExprProb, ps->pool); PgfExprProb *ep = gu_new(PgfExprProb, ps->pool);
ep->expr = ps->meta_var; ep->expr = ps->meta_var;