msyds/gf-core
1
0
Fork 0
forked from GitHub/gf-core
Code Pull Requests Activity

a major refactoring in the robust parser: bottom-up filtering and garbage collection for the chart

Browse Source
This commit is contained in:
kr.angelov
2012-10-25 14:42:53 +00:00
parent 619c341ee9
commit d185938952
8 changed files with 850 additions and 604 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/runtime/c/gu/map.c
View File

@@ -251,6 +251,12 @@ gu_map_find_key(GuMap* map, const void* key)
return NULL;
}
bool
gu_map_has(GuMap* ht, const void* key)
{
size_t idx;
return gu_map_lookup(ht, key, &idx);
}
void*
gu_map_insert(GuMap* map, const void* key)

8
src/runtime/c/gu/map.h
View File

@@ -52,12 +52,8 @@ gu_map_find(GuMap* ht, const void* key);
const void*
gu_map_find_key(GuMap* ht, const void* key);
static inline bool
gu_map_has(GuMap* ht, const void* key)
{
return gu_map_find_key(ht, key) != NULL;
}
bool
gu_map_has(GuMap* ht, const void* key);
void*
gu_map_insert(GuMap* ht, const void* key);

21
src/runtime/c/pgf/data.h
View File

@@ -193,6 +193,8 @@ struct PgfAlternative {
* form. */
};
typedef struct PgfItemConts PgfItemConts;
struct PgfCCat {
PgfCncCat* cnccat;
PgfFunIds* lindefs;
@@ -200,6 +202,7 @@ struct PgfCCat {
PgfProductionSeq prods;
float viterbi_prob;
int fid;
PgfItemConts* conts;
};
extern PgfCCat pgf_ccat_string, pgf_ccat_int, pgf_ccat_float, pgf_ccat_var;
@@ -213,11 +216,14 @@ extern GU_DECLARE_TYPE(PgfFunIndices, GuStringMap);
typedef GuMap PgfCoerceIdx;
extern GU_DECLARE_TYPE(PgfCoerceIdx, GuMap);
typedef GuStringMap PgfTransitions;
extern GU_DECLARE_TYPE(PgfTransitions, GuStringMap);
typedef GuMap PgfProductionIdx;
extern GU_DECLARE_TYPE(PgfProductionIdx, GuMap);
typedef GuMap PgfEpsilonIdx;
extern GU_DECLARE_TYPE(PgfEpsilonIdx, GuMap);
typedef GuMap PgfLeftcornerCatIdx;
extern GU_DECLARE_TYPE(PgfLeftcornerCatIdx, GuMap);
typedef GuMap PgfLeftcornerTokIdx;
extern GU_DECLARE_TYPE(PgfLeftcornerTokIdx, GuMap);
typedef struct PgfLiteralCallback PgfLiteralCallback;
extern GU_DECLARE_TYPE(PgfLiteralCallback, struct);
@@ -238,15 +244,14 @@ struct PgfConcr {
GuMap* ccats;
PgfFunIndices* fun_indices;
PgfCoerceIdx* coerce_idx;
PgfTransitions* lexicon_idx;
PgfEpsilonIdx* epsilon_idx;
PgfProductionIdx* epsilon_idx;
PgfLeftcornerCatIdx* leftcorner_cat_idx;
PgfLeftcornerTokIdx* leftcorner_tok_idx;
PgfCncFuns* cncfuns;
PgfSequences* sequences;
PgfCIdMap* cnccats;
PgfCallbacksMap* callbacks;
int total_cats;
int max_fid;
int item_quota;
};
extern GU_DECLARE_TYPE(PgfConcr, struct);

99
src/runtime/c/pgf/linearize.c
View File

@@ -151,45 +151,72 @@ pgf_lzr_add_infer_entry(
};
gu_buf_push(entries, PgfLinInferEntry, entry);
}
typedef struct {
GuMapItor fn;
PgfConcr* concr;
GuPool* pool;
} PgfLzrIndexFn;
static void
pgf_lzr_index_iter(GuMapItor* fn, const void* key, void* value, GuExn* err)
{
(void) (key && err);
PgfLzrIndexFn* clo = (PgfLzrIndexFn*) fn;
PgfCCat* ccat = *((PgfCCat**) value);
PgfConcr *concr = clo->concr;
GuPool *pool = clo->pool;
if (gu_seq_is_null(ccat->prods))
return;
size_t n_prods = gu_seq_length(ccat->prods);
for (size_t i = 0; i < n_prods; i++) {
PgfProduction prod = gu_seq_get(ccat->prods, PgfProduction, i);
void* data = gu_variant_data(prod);
switch (gu_variant_tag(prod)) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papply = data;
PgfInferMap* infer =
gu_map_get(concr->fun_indices, &papply->fun->name,
PgfInferMap*);
gu_debug("index: %s -> %d", papply->fun->name, ccat->fid);
if (!infer) {
infer = gu_map_type_new(PgfInferMap, pool);
gu_map_put(concr->fun_indices,
&papply->fun->name, PgfInferMap*, infer);
}
pgf_lzr_add_infer_entry(infer, ccat, papply, pool);
break;
}
case PGF_PRODUCTION_COERCE: {
PgfProductionCoerce* pcoerce = data;
PgfCCatBuf* cats = gu_map_get(concr->coerce_idx, pcoerce->coerce,
PgfCCatBuf*);
if (!cats) {
cats = gu_new_buf(PgfCCat*, pool);
gu_map_put(concr->coerce_idx,
pcoerce->coerce, PgfCCatBuf*, cats);
}
gu_debug("coerce_idx: %d -> %d", pcoerce->coerce->fid, ccat->fid);
gu_buf_push(cats, PgfCCat*, ccat);
break;
}
default:
// Display warning?
break;
}
}
}
void
pgf_lzr_index(PgfConcr* concr, PgfCCat* ccat, PgfProduction prod,
GuPool *pool)
pgf_lzr_index(PgfConcr* concr, GuPool *pool)
{
void* data = gu_variant_data(prod);
switch (gu_variant_tag(prod)) {
case PGF_PRODUCTION_APPLY: {
PgfProductionApply* papply = data;
PgfInferMap* infer =
gu_map_get(concr->fun_indices, &papply->fun->name,
PgfInferMap*);
gu_debug("index: %s -> %d", papply->fun->name, ccat->fid);
if (!infer) {
infer = gu_map_type_new(PgfInferMap, pool);
gu_map_put(concr->fun_indices,
&papply->fun->name, PgfInferMap*, infer);
}
pgf_lzr_add_infer_entry(infer, ccat, papply, pool);
break;
}
case PGF_PRODUCTION_COERCE: {
PgfProductionCoerce* pcoerce = data;
PgfCCatBuf* cats = gu_map_get(concr->coerce_idx, pcoerce->coerce,
PgfCCatBuf*);
if (!cats) {
cats = gu_new_buf(PgfCCat*, pool);
gu_map_put(concr->coerce_idx,
pcoerce->coerce, PgfCCatBuf*, cats);
}
gu_debug("coerce_idx: %d -> %d", pcoerce->coerce->fid, ccat->fid);
gu_buf_push(cats, PgfCCat*, ccat);
break;
}
default:
// Display warning?
break;
}
PgfLzrIndexFn clo = { { pgf_lzr_index_iter }, concr, pool };
gu_map_iter(concr->ccats, &clo.fn, NULL);
}
typedef struct PgfLzn PgfLzn;

1242
src/runtime/c/pgf/parser.c
View File

File diff suppressed because it is too large Load Diff

7
src/runtime/c/pgf/pgf.h
View File

@@ -74,13 +74,6 @@ pgf_load_meta_child_probs(PgfPGF*, const char* fpath, GuPool* pool);
typedef struct PgfConcr PgfConcr;
void
pgf_set_item_quota(PgfConcr* concr, int quota);
int
pgf_get_item_quota(PgfConcr* concr);
#include <gu/type.h>
extern GU_DECLARE_TYPE(PgfPGF, struct);

69
src/runtime/c/pgf/reader.c
View File

@@ -36,8 +36,6 @@
#include <gu/log.h>
typedef GuMap PgfContsMap;
//
// PgfReader
//
@@ -53,7 +51,6 @@ struct PgfReader {
PgfAbstr* curr_abstr;
PgfConcr* curr_concr;
GuMap* curr_lindefs;
PgfContsMap* curr_conts_map; // used temporary for building the bu index for the parser
GuTypeMap* read_to_map;
GuTypeMap* read_new_map;
void* curr_key;
@@ -445,6 +442,7 @@ pgf_read_to_PgfCCatId(GuType* type, PgfReader* rdr, void* to)
ccat->prods = gu_null_seq;
ccat->viterbi_prob = 0;
ccat->fid = fid;
ccat->conts = NULL;
gu_map_put(rdr->curr_concr->ccats, &fid, PgfCCat*, ccat);
}
@@ -468,6 +466,7 @@ pgf_read_to_PgfCCat(GuType* type, PgfReader* rdr, void* to)
ccat->prods = gu_new_seq(PgfProduction, n_prods, rdr->opool);
ccat->viterbi_prob = 0;
ccat->fid = *fidp;
ccat->conts = NULL;
size_t top = 0;
size_t bot = n_prods-1;
@@ -716,6 +715,9 @@ pgf_ccat_set_cnccat(PgfCCat* ccat)
return ccat->cnccat;
}
extern float
pgf_ccat_set_viterbi_prob(PgfCCat* ccat);
static void
pgf_read_ccat_cb(GuMapItor* fn, const void* key, void* value, GuExn* err)
{
@@ -723,41 +725,14 @@ pgf_read_ccat_cb(GuMapItor* fn, const void* key, void* value, GuExn* err)
PgfCCat* ccat = *((PgfCCat**) value);
pgf_ccat_set_cnccat(ccat);
// pgf_ccat_set_viterbi_prob(ccat);
}
extern GU_DECLARE_TYPE(PgfContsMap, GuMap);
void
pgf_parser_index(PgfConcr* concr, GuPool *pool);
void
pgf_parser_bu_index(PgfConcr* concr, PgfCCat* cat, PgfProduction prod,
PgfContsMap* conts_map,
GuPool *pool, GuPool *tmp_pool);
void
pgf_lzr_index(PgfConcr* concr, PgfCCat* cat, PgfProduction prod,
GuPool *pool);
static void
pgf_index_prods(GuMapItor* fn, const void* key, void* value, GuExn* err)
{
(void) (key && err);
PgfIndexFn* clo = (PgfIndexFn*) fn;
PgfCCat* ccat = *((PgfCCat**) value);
PgfReader *rdr = clo->rdr;
if (gu_seq_is_null(ccat->prods))
return;
size_t n_prods = gu_seq_length(ccat->prods);
for (size_t i = 0; i < n_prods; i++) {
PgfProduction prod = gu_seq_get(ccat->prods, PgfProduction, i);
pgf_parser_bu_index(rdr->curr_concr, ccat, prod,
rdr->curr_conts_map,
rdr->opool, rdr->tmp_pool);
pgf_lzr_index(rdr->curr_concr, ccat, prod, rdr->opool);
}
}
pgf_lzr_index(PgfConcr* concr, GuPool *pool);
static void*
pgf_read_new_PgfConcr(GuType* type, PgfReader* rdr, GuPool* pool,
@@ -777,20 +752,18 @@ pgf_read_new_PgfConcr(GuType* type, PgfReader* rdr, GuPool* pool,
GuMapType* lindefs_t = gu_type_cast(gu_type(PgfLinDefs), GuMap);
rdr->curr_lindefs = gu_map_type_make(lindefs_t, rdr->tmp_pool);
pgf_read_into_map(lindefs_t, rdr, rdr->curr_lindefs);
rdr->curr_conts_map = gu_map_type_new(PgfContsMap, rdr->tmp_pool);
GuMapType* ccats_t = gu_type_cast(gu_type(PgfCCatMap), GuMap);
concr->ccats =
gu_new_int_map(PgfCCat*, &gu_null_struct, pool);
concr->fun_indices = gu_map_type_new(PgfFunIndices, pool);
concr->coerce_idx = gu_map_type_new(PgfCoerceIdx, pool);
concr->lexicon_idx = gu_map_type_new(PgfTransitions,pool);
concr->epsilon_idx = gu_map_type_new(PgfEpsilonIdx, pool);
concr->epsilon_idx = gu_map_type_new(PgfProductionIdx, pool);
concr->leftcorner_cat_idx = gu_map_type_new(PgfLeftcornerCatIdx,pool);
concr->leftcorner_tok_idx = gu_map_type_new(PgfLeftcornerTokIdx,pool);
pgf_read_into_map(ccats_t, rdr, concr->ccats);
concr->cnccats = pgf_read_new(rdr, gu_type(PgfCncCatMap), pool, NULL);
concr->callbacks = pgf_new_callbacks_map(concr, pool);
concr->total_cats = pgf_read_int(rdr);
concr->max_fid = concr->total_cats;
concr->item_quota = 2000000;
// set the function ids
int n_funs = gu_list_length(concr->cncfuns);
@@ -805,9 +778,9 @@ pgf_read_new_PgfConcr(GuType* type, PgfReader* rdr, GuPool* pool,
PgfIndexFn clo1 = { { pgf_read_ccat_cb }, rdr };
gu_map_iter(concr->ccats, &clo1.fn, NULL);
PgfIndexFn clo2 = { { pgf_index_prods }, rdr };
gu_map_iter(concr->ccats, &clo2.fn, NULL);
pgf_parser_index(concr, pool);
pgf_lzr_index(concr, pool);
return concr;
}
@@ -844,6 +817,7 @@ pgf_read_new_PgfCncCat(GuType* type, PgfReader* rdr, GuPool* pool,
ccat->prods = gu_null_seq;
ccat->viterbi_prob = 0;
ccat->fid = fid;
ccat->conts = NULL;
gu_map_put(rdr->curr_concr->ccats, &fid, PgfCCat*, ccat);
}
@@ -922,7 +896,6 @@ pgf_new_reader(GuIn* in, GuPool* opool, GuPool* tmp_pool, GuExn* err)
rdr->curr_abstr = NULL;
rdr->curr_concr = NULL;
rdr->curr_lindefs = NULL;
rdr->curr_conts_map = NULL;
rdr->read_to_map = gu_new_type_map(&pgf_read_to_table, tmp_pool);
rdr->read_new_map = gu_new_type_map(&pgf_read_new_table, tmp_pool);
return rdr;
@@ -989,13 +962,3 @@ pgf_load_meta_child_probs(PgfPGF* pgf, const char* fpath, GuPool* pool)
fclose(fp);
return true;
}
void
pgf_set_item_quota(PgfConcr* concr, int quota) {
concr->item_quota = quota > 0 ? quota : 0;
}
int
pgf_get_item_quota(PgfConcr* concr) {
return concr->item_quota;
}

2
src/runtime/c/utils/pgf-translate.c
View File

@@ -87,7 +87,7 @@ int main(int argc, char* argv[]) {
goto fail_read;
}
if (!pgf_load_meta_child_probs(pgf, "../../../examples/PennTreebank/ParseEngAbs2.probs", pool)) {
if (!pgf_load_meta_child_probs(pgf, "../../../treebanks/PennTreebank/ParseEngAbs3.probs", pool)) {
fprintf(stderr, "Loading meta child probs failed\n");
status = EXIT_FAILURE;
goto fail_read;