incremental chart construction

2026-08-19 10:46:22 -06:00 · 2026-08-04 10:39:42 +02:00
parent ec7354ca1c
commit cccce4d064
8 changed files with 328 additions and 145 deletions
@@ -262,6 +262,7 @@ struct PGF_INTERNAL_DECL PgfSymbolCCat {
    ref<PgfConcrLincat> lincat;
    interval_t value;
    interval_t lin_idx;
+    prob_t viterbi_prob;
    PgfMetaId fid;
 };

@@ -12,11 +12,11 @@ class PGF_INTERNAL_DECL interval_map {
        size_t sz;
        size_t start, end, max;

-        V value;
-
        Node *left;
        Node *right;

+        V value;
+
        Node(size_t start, size_t end)
        {
            this->sz = 1;
@@ -1,6 +1,7 @@
 #include "data.h"
 #include "printer.h"
 #include "parser.h"
+#include <math.h>

 //#define DEBUG_PARSER
 //#define DEBUG_EXPRS
@@ -9,7 +10,6 @@ PgfAbstractParser::PgfAbstractParser(ref<PgfConcr> concr)
 {
    this->concr = concr;

-    this->first_state = NULL;
    this->current_state = NULL;
    this->initial_fid = concr->last_fid;
    this->last_fid = concr->last_fid;
@@ -54,7 +54,7 @@ PgfAbstractParser::Cont::~Cont()

 PgfAbstractParser::~PgfAbstractParser()
 {
-    State *state = first_state;
+    State *state = current_state;
    while (state != NULL) {
        for (auto it1 : state->completed) {
     /*       for (auto it2 : it1) {
@@ -76,22 +76,22 @@ PgfAbstractParser::~PgfAbstractParser()
    }
 }

-void PgfAbstractParser::process(Item *item, const PgfTextSpot &spot, bool bind)
+void PgfAbstractParser::process(Item *item, State *state)
 {
 #ifdef DEBUG_PARSER
-    print_item(item,spot);
+    print_item(item,state);
 #endif

    if (item->dot < item->syms.size()) {
-        symbol(item,spot,bind,item->syms[item->dot]);
+        symbol(item,state,item->syms[item->dot]);
    } else if (item->pre_alt > 0) {
        item->dot     = item->pre_dot+1;
        item->pre_alt = 0;
        item->pre_dot = 0;
        item->syms    = item->rule->syms.as_vector();
-        process(item,spot,bind);
+        process(item,state);
    } else {
-        complete(item,spot,bind);
+        complete(item,state);
    }
 }

@@ -99,14 +99,12 @@ PGF_INTERNAL_DECL
 int text_symbol_cmp(PgfTextSpot *spot, const uint8_t *end,
                    PgfSymbol sym, bool case_sensitive);

-void PgfAbstractParser::symbol(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym)
+void PgfAbstractParser::symbol(Item *item, State *state, PgfSymbol sym)
 {
    switch (ref<PgfSymbol>::get_tag(sym)) {
    case PgfSymbolCat::tag: {
        auto symcat = ref<PgfSymbolCat>::untagged(sym);

-        State *state = new_state(spot);
-
        CCat *ccat = item->args[symcat->d];
        if (ccat == NULL) {
            ref<PgfConcrLincat> lincat = 0;
@@ -125,6 +123,7 @@ void PgfAbstractParser::symbol(Item *item, const PgfTextSpot &spot, bool bind, P
            }

            if (lincat != 0) {
+                size_t n_suspended1 = state->conts1.size();
                Cont *&cont = state->conts1[lincat];
                if (cont == NULL) {
                    cont = new Cont;
@@ -138,7 +137,7 @@ void PgfAbstractParser::symbol(Item *item, const PgfTextSpot &spot, bool bind, P
                auto &suspended = cont->suspended[value_i][lin_idx_i];
                suspended.push_back(item);

-                suspend(cont,item,suspended.size());
+                suspend(cont,item,n_suspended1,suspended.size());
            }
        } else {
            Cont *&cont = state->conts2[ccat];
@@ -154,10 +153,24 @@ void PgfAbstractParser::symbol(Item *item, const PgfTextSpot &spot, bool bind, P

            interval_t value_i   = item->interval(item->rule->args[symcat->d]);
            interval_t lin_idx_i = item->interval(ref<PgfLParam>::from_ptr(&symcat->r));
+
+            bool subsumed = false;
+            for (auto it1 : cont->suspended.overlaps(value_i)) {
+                if (it1.first.first <= value_i.first && it1.first.second >= value_i.second) {
+                    for (auto it2 : it1.second.overlaps(lin_idx_i)) {
+                        if (it2.first.first <= lin_idx_i.first && it2.first.second >= lin_idx_i.second) {
+                            subsumed = true;
+                            goto found;
+                        }
+                    }
+                }
+            }
+found:;
+
            auto &suspended = cont->suspended[value_i][lin_idx_i];
            suspended.push_back(item);

-            if (suspended.size() == 1) {
+            if (!subsumed && suspended.size() == 1) {
                if (ccat->fid <= initial_fid) {
                    size_t n_items = 0;
                    vector<ref<PgfItem>> items =
@@ -165,11 +178,11 @@ void PgfAbstractParser::symbol(Item *item, const PgfTextSpot &spot, bool bind, P

                    for (size_t i = 0; i < n_items; i++) {
                        ref<PgfItem> pitem = items[i];
-                        td_epsilon(state,cont,pitem,item,item->rule->args[symcat->d],ref<PgfLParam>::from_ptr(&symcat->r));
+                        td_epsilon(state,cont,pitem,item,symcat);
                    }
                } else {
                    for (Production *prod : ccat->prods) {
-                        td_predict(state,cont,prod,item,item->rule->args[symcat->d],ref<PgfLParam>::from_ptr(&symcat->r));
+                        td_predict(state,cont,prod,item,symcat);
                    }
                }
            } else {
@@ -194,7 +207,7 @@ void PgfAbstractParser::symbol(Item *item, const PgfTextSpot &spot, bool bind, P
        break;
    }
    case PgfSymbolKS::tag: {
-        symbol_token(item, spot, bind, sym);
+        symbol_token(item, state, sym);
        break;
    }
    case PgfSymbolKP::tag: {
@@ -206,7 +219,9 @@ void PgfAbstractParser::symbol(Item *item, const PgfTextSpot &spot, bool bind, P
        new_item->dot     = 0;
        new_item->syms    = symkp->default_form;
        new_item->rule    = item->rule;
-        process(new_item, spot, bind);
+        new_item->inside_prob  = item->inside_prob;
+        new_item->outside_prob = item->outside_prob;
+        process(new_item, state);

        for (size_t i = 0; i < symkp->alts.size(); i++) {
            Item *new_item = new(item) Item;
@@ -215,22 +230,18 @@ void PgfAbstractParser::symbol(Item *item, const PgfTextSpot &spot, bool bind, P
            new_item->dot     = 0;
            new_item->syms    = symkp->alts[i].form;
            new_item->rule    = item->rule;
-            process(new_item, spot, bind);
+            new_item->inside_prob  = item->inside_prob;
+            new_item->outside_prob = item->outside_prob;
+            process(new_item, state);
        }

        delete item;
        break;
    }
-    case PgfSymbolBIND::tag: {
-        symbol_bind(item, spot, sym);
-        break;
-    }
+    case PgfSymbolBIND::tag:
    case PgfSymbolSOFTBIND::tag:
    case PgfSymbolSOFTSPACE::tag: {
-        item->dot++;
-        process(new (item) Item, spot, true);
-        process(new (item) Item, spot, false);
-        delete item;
+        symbol_bind(item, state, sym);
        break;
    }
    case PgfSymbolNE::tag:
@@ -239,15 +250,13 @@ void PgfAbstractParser::symbol(Item *item, const PgfTextSpot &spot, bool bind, P
    case PgfSymbolCAPIT::tag:
    case PgfSymbolALLCAPIT::tag:
        item->dot++;
-        process(item, spot, bind);
+        process(item, state);
        break;
    }
 }

-void PgfAbstractParser::complete(Item *item, const PgfTextSpot &spot, bool bind)
+void PgfAbstractParser::complete(Item *item, State *state)
 {
-    State *state = new_state(spot);
-
    switch (ref<object>::get_tag(item->rule->container)) {
    case PgfConcrLin::tag: {
        auto lin = ref<PgfConcrLin>::untagged(item->rule->container);
@@ -263,6 +272,7 @@ void PgfAbstractParser::complete(Item *item, const PgfTextSpot &spot, bool bind)
            ccat->lin_idx = lin_idx;
            ccat->value = res;
            ccat->covered = false;
+            ccat->viterbi_prob = item->inside_prob;

 #ifdef DEBUG_PARSER
            {
@@ -327,7 +337,7 @@ void PgfAbstractParser::complete(Item *item, const PgfTextSpot &spot, bool bind)
                        for (auto it2 : it1.second) {
                            Item *item = it2.second[0];
                            auto symcat = ref<PgfSymbolCat>::untagged(item->syms[item->dot]);
-                            td_predict(next,cont,prod,item,item->rule->args[symcat->d],ref<PgfLParam>::from_ptr(&symcat->r));
+                            td_predict(next,cont,prod,item,symcat);
                        }
                    }
                }
@@ -492,11 +502,15 @@ void PgfAbstractParser::combine(State *state, Item *item, CCat *ccat)
    }

    item->dot++;
+    if (item->args[sym_cat->d] != NULL) {
+        item->inside_prob -= item->args[sym_cat->d]->viterbi_prob;
+    }
    item->args[sym_cat->d] = ccat;
-    process(item, state->start, false);
+    item->inside_prob += ccat->viterbi_prob;
+    state->push_item(item);
 }

-void PgfAbstractParser::td_epsilon(State *state, Cont *cont, ref<PgfItem> pitem, Item *xitem, ref<PgfLParam> value, ref<PgfLParam> lin_idx)
+void PgfAbstractParser::td_epsilon(State *state, Cont *cont, ref<PgfItem> pitem, Item *xitem, ref<PgfSymbolCat> symcat)
 {
    switch (ref<object>::get_tag(pitem->rule->container)) {
    case PgfConcrLin::tag: {
@@ -510,12 +524,14 @@ void PgfAbstractParser::td_epsilon(State *state, Cont *cont, ref<PgfItem> pitem,
            item->pre_dot = 0;
            item->syms    = rule->syms.as_vector();
            item->rule    = rule;
+            item->inside_prob = lin->absfun->prob;
+            item->outside_prob = xitem->outside_prob+xitem->inside_prob-xitem->args[symcat->d]->viterbi_prob;

-            if (!item->instantiate(item->rule->res, xitem->rule, &xitem->vars[0], value)) {
+            if (!item->instantiate(item->rule->res, xitem->rule, &xitem->vars[0], xitem->rule->args[symcat->d])) {
                delete item;
                continue;
            }
-            if (!item->instantiate(item->rule->lin_idx, xitem->rule, &xitem->vars[0], lin_idx)) {
+            if (!item->instantiate(item->rule->lin_idx, xitem->rule, &xitem->vars[0], ref<PgfLParam>::from_ptr(&symcat->r))) {
                delete item;
                continue;
            }
@@ -533,8 +549,10 @@ void PgfAbstractParser::td_epsilon(State *state, Cont *cont, ref<PgfItem> pitem,
                        arg_ccat->lin_idx = arg->lin_idx;
                        arg_ccat->value = arg->value;
                        arg_ccat->covered = true;
+                        arg_ccat->viterbi_prob = arg->viterbi_prob;
                    }
                    item->args[i] = arg_ccat;
+                    item->inside_prob += arg_ccat->viterbi_prob;
                }

                if (!item->instantiate(item->rule->args[i], pitem->rule, &pitem->vars[0], pitem->rule->args[i])) {
@@ -543,7 +561,7 @@ void PgfAbstractParser::td_epsilon(State *state, Cont *cont, ref<PgfItem> pitem,
                }
            }

-            process(item, state->start, false);
+            state->push_item(item);
 next:;
        }
    }
@@ -552,7 +570,7 @@ next:;
    }
 }

-void PgfAbstractParser::td_predict(State *state, Cont *cont, Production *prod, Item *xitem, ref<PgfLParam> value, ref<PgfLParam> lin_idx)
+void PgfAbstractParser::td_predict(State *state, Cont *cont, Production *prod, Item *xitem, ref<PgfSymbolCat> symcat)
 {
    switch (ref<object>::get_tag(prod->rule->container)) {
    case PgfConcrLin::tag: {
@@ -566,13 +584,15 @@ void PgfAbstractParser::td_predict(State *state, Cont *cont, Production *prod, I
            item->pre_dot = 0;
            item->syms    = rule->syms.as_vector();
            item->rule    = rule;
+            item->inside_prob = lin->absfun->prob;
+            item->outside_prob = xitem->outside_prob+xitem->inside_prob-xitem->args[symcat->d]->viterbi_prob;

-            if (!item->instantiate(item->rule->res, xitem->rule, &xitem->vars[0], value)) {
+            if (!item->instantiate(item->rule->res, xitem->rule, &xitem->vars[0], xitem->rule->args[symcat->d])) {
                delete item;
                continue;
            }

-            if (!item->instantiate(item->rule->lin_idx, xitem->rule, &xitem->vars[0], lin_idx)) {
+            if (!item->instantiate(item->rule->lin_idx, xitem->rule, &xitem->vars[0], ref<PgfLParam>::from_ptr(&symcat->r))) {
                delete item;
                continue;
            }
@@ -583,9 +603,12 @@ void PgfAbstractParser::td_predict(State *state, Cont *cont, Production *prod, I
                    goto next;
                }
                item->args[i] = prod->args[i];
+                if (item->args[i] != NULL) {
+                    item->inside_prob += item->args[i]->viterbi_prob;
+                }
            }

-            process(item, state->start, false);
+            state->push_item(item);
 next:;
        }
    }
@@ -634,11 +657,11 @@ void print_symbols(PgfPrinter &printer, PgfConcrRule *rule, vector<PgfSymbol> sy
        printer.puts(" . ");
 }

-void PgfAbstractParser::print_item(Item *item, const PgfTextSpot &spot)
+void PgfAbstractParser::print_item(Item *item, State *state)
 {
    PgfPrinter printer(NULL,0,NULL);

-    printer.nprintf(32, "[%zd-%zd; ", item->cont ? item->cont->state->end.pos : 0, spot.pos);
+    printer.nprintf(32, "[%zd-%zd; ", item->cont ? item->cont->state->end.pos : 0, state->start.pos);

    if (item->vars.size() > 0) {
        printer.lvar_ranges(item->rule->ranges, &item->vars[0]);
@@ -703,7 +726,7 @@ void PgfAbstractParser::print_item(Item *item, const PgfTextSpot &spot)
    printer.lparam(item->rule->lin_idx);
    printer.puts(" : ");
    print_symbols(printer, item->rule, item->syms, item->pre_alt, item->pre_dot, item->dot);
-    printer.puts("]");
+    printer.nprintf(40,"; %f+%f=%f]", item->inside_prob, item->outside_prob, item->inside_prob+item->outside_prob);

    PgfText *text = printer.get_text();
    fprintf(stderr, "%s\n", text->text);
@@ -780,14 +803,16 @@ PgfParser::PgfParser(ref<PgfConcr> concr, PgfText *sentence, bool case_sensitive
 {
    this->m = m;
    this->u = u;
-    this->sentence = sentence;
-    this->end = (uint8_t *) (sentence->text+sentence->size);
+    this->sentence = textdup(sentence);
+    this->end = (uint8_t *) (this->sentence->text+this->sentence->size);
    this->case_sensitive = case_sensitive;
 }

 PgfParser::~PgfParser()
 {
-    State *state = first_state;
+    free(sentence);
+
+    State *state = current_state;
    while (state != NULL) {
        for (auto it1 : state->completed) {
            for (auto it2 : it1.second) {
@@ -832,13 +857,7 @@ void PgfParser::bu_predict(PgfPhrasetable phrasetable,
        return;

    PgfTextSpot current = state->end;
-    int cmp;
-    if (state->needs_bind) {
-        uint8_t tag = ref<PgfSymbol>::get_tag(phrasetable->sym);
-        cmp = ((int) PgfSymbolBIND::tag) - ((int) tag);
-    } else {
-        cmp = text_symbol_cmp(&current,end,phrasetable->sym,case_sensitive);
-    }
+    int cmp = text_symbol_cmp(&current,end,phrasetable->sym,case_sensitive);
    if (cmp < 0) {
        bu_predict(phrasetable->left,state,min,max);
    } else if (cmp > 0) {
@@ -856,16 +875,14 @@ void PgfParser::bu_predict(PgfPhrasetable phrasetable,
            bu_predict(phrasetable->left,state,min,len);

        if (len > 0) {
-            if (*current.ptr != ' ' && *current.ptr != 0)
-                return;
-
+            State *next_state = new_state(current);
            for (size_t i = 0; i < phrasetable->n_items; i++) {
                std::map<ref<PgfConcrLincat>, bool> visited;
                //if (!td_reachable(state, phrasetable->items[i], visited))
                //    continue;
                Item *item = bu_item(state, phrasetable->items[i]);
                item->dot++;
-                process(item, current, false);
+                next_state->push_item(item);
            }
        }

@@ -874,6 +891,42 @@ void PgfParser::bu_predict(PgfPhrasetable phrasetable,
     }
 }

+void PgfParser::bu_predict(PgfPhrasetable phrasetable,
+                           State *state)
+{
+    if (phrasetable == 0)
+        return;
+
+    PgfTextSpot current = state->end;
+    int cmp;
+    uint8_t tag = ref<PgfSymbol>::get_tag(phrasetable->sym);
+    cmp = ((int) PgfSymbolBIND::tag) - ((int) tag);
+    if (cmp < 0) {
+        bu_predict(phrasetable->left,state);
+    } else if (cmp > 0) {
+        bu_predict(phrasetable->right,state);
+    } else {
+        State *next_state = state->next;
+        if (next_state == NULL || state->end.pos != next_state->start.pos) {
+            next_state = new State;
+            next_state->start = state->end;
+            next_state->end   = state->end;
+            next_state->next  = state->next;
+            next_state->needs_bind = false;
+            state->next = next_state;
+        }
+
+        for (size_t i = 0; i < phrasetable->n_items; i++) {
+            std::map<ref<PgfConcrLincat>, bool> visited;
+            //if (!td_reachable(state, phrasetable->items[i], visited))
+            //    continue;
+            Item *item = bu_item(state, phrasetable->items[i]);
+            item->dot++;
+            next_state->push_item(item);
+        }
+    }
+}
+
 void PgfParser::bu_predict(State *state, CCat *ccat)
 {
    size_t n_items = 0;
@@ -944,6 +997,8 @@ PgfAbstractParser::Item *PgfParser::bu_item(State *state, ref<PgfItem> pitem)
        item->dot     = pitem->dot;
        item->syms    = pitem->rule->syms.as_vector();
        item->rule    = pitem->rule;
+        item->inside_prob = lin->absfun->prob;
+        item->outside_prob = 0;
        break;
    }
    case PgfConcrLincat::tag: {
@@ -964,6 +1019,8 @@ PgfAbstractParser::Item *PgfParser::bu_item(State *state, ref<PgfItem> pitem)
        item->dot     = pitem->dot;
        item->syms    = pitem->rule->syms.as_vector();
        item->rule    = pitem->rule;
+        item->inside_prob = 0;
+        item->outside_prob = 0;
        break;
    }
    }
@@ -994,8 +1051,10 @@ PgfAbstractParser::Item *PgfParser::bu_item(State *state, ref<PgfItem> pitem)
                arg_ccat->lin_idx = arg->lin_idx;
                arg_ccat->value = arg->value;
                arg_ccat->covered = true;
+                arg_ccat->viterbi_prob = arg->viterbi_prob;
            }
            item->args[i] = arg_ccat;
+            item->inside_prob += arg_ccat->viterbi_prob;
        }
    }

@@ -1014,6 +1073,7 @@ void PgfParser::make_chunks(State *state, std::vector<CCat*> &chunks, prob_t pro
        estate->n_args = chunks.size();
        for (size_t i = 0; i < estate->n_args; i++) {
            estate->args[i] = chunks[estate->n_args-i-1];
+            estate->prob += estate->args[i]->viterbi_prob;
        }
        queue.push_back(estate);
        std::push_heap(queue.begin(), queue.end(), estate_comp);
@@ -1041,8 +1101,6 @@ void PgfParser::prepare(ref<PgfConcrLincat> start)

    PgfTextSpot start_spot = {0, (uint8_t *) sentence->text};
    State *state = new_state(start_spot);
-    state->needs_bind = false;
-    current_state = state;

    for (size_t i = start->n_lindefs; i < start->rules.size(); i++) {
        ref<PgfConcrRule> rule = start->rules[i];
@@ -1053,18 +1111,9 @@ void PgfParser::prepare(ref<PgfConcrLincat> start)
        item->pre_dot = 0;
        item->syms    = rule->syms.as_vector();
        item->rule    = rule;
-        process(item, start_spot, false);
-    }
-
-    while (current_state != NULL) {
-        bu_predict(concr->phrasetable, current_state, 1, sentence->size);
-        state = current_state;
-        current_state = current_state->next;
-    }
-
-    if (queue.size() == 0) {
-        std::vector<CCat*> chunks;
-        make_chunks(state, chunks, 0);
+        item->inside_prob  = 0;
+        item->outside_prob = 0;
+        state->push_item(item);
    }
 }

@@ -1072,6 +1121,57 @@ PgfExpr PgfParser::fetch(PgfDB *db, prob_t *prob)
 {
    DB_scope scope(db, READER_SCOPE);

+    bool first_fetch = (initial_fid == last_fid);
+
+    for (;;) {
+        State *state = current_state;
+        prob_t min_prob  = INFINITY;
+        State *min_state = NULL;
+        if (queue.size() > 0) {
+            min_prob = queue.front()->prob;
+        }
+
+        while (state != NULL) {
+            if (state->queue.size() > 0) {
+                Item *item = state->queue.front();
+                prob_t prob = item->outside_prob + item->inside_prob;
+                if (min_prob > prob) {
+                    min_prob  = prob;
+                    min_state = state;
+                }
+            }
+            state = state->next;
+        }
+
+        if (min_state == NULL)
+            break;
+
+        State *prev = current_state;
+        current_state = NULL;
+        while (current_state != min_state) {
+            State *next = prev->next;
+            prev->next = current_state;
+            current_state = prev;
+            prev = next;
+        }
+
+        Item *item = current_state->pop_item();
+        process(item,current_state);
+
+        while (current_state != NULL) {
+            State *next = current_state->next;
+            current_state->next = prev;
+            prev = current_state;
+            current_state = next;
+        }
+        current_state = prev;
+    }
+
+    if (first_fetch && queue.size() == 0) {
+        std::vector<CCat*> chunks;
+        make_chunks(current_state, chunks, 0);
+    }
+
    while (queue.size() > 0) {
        ExprState *estate = queue.front();
        std::pop_heap(queue.begin(), queue.end(), estate_comp);
@@ -1122,7 +1222,7 @@ PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob)

                        ExprState *new_estate = new(pitem->args.size()) ExprState;
                        new_estate->expr   = u->efun(&lin->name);
-                        new_estate->prob   = estate->prob+lin->absfun->prob;
+                        new_estate->prob   = estate->prob-ccat->viterbi_prob+lin->absfun->prob;
                        new_estate->hash   = 0;
                        new_estate->res    = ccat;
                        new_estate->index  = 0;
@@ -1143,8 +1243,10 @@ PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob)
                                    arg_ccat->lin_idx = arg->lin_idx;
                                    arg_ccat->value = arg->value;
                                    arg_ccat->covered = true;
+                                    arg_ccat->viterbi_prob = arg->viterbi_prob;
                                }
                                new_estate->args[i] = arg_ccat;
+                                new_estate->prob += arg_ccat->viterbi_prob;
                            }
                        }
                        queue.push_back(new_estate);
@@ -1156,7 +1258,7 @@ PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob)

                        ExprState *new_estate = new(prod->args.size()) ExprState;
                        new_estate->expr   = u->efun(&lin->name);
-                        new_estate->prob   = estate->prob+lin->absfun->prob;
+                        new_estate->prob   = estate->prob-ccat->viterbi_prob+lin->absfun->prob;
                        new_estate->hash   = 0;
                        new_estate->res    = ccat;
                        new_estate->index  = 0;
@@ -1166,6 +1268,9 @@ PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob)
                        }
                        for (size_t i = 0; i < new_estate->n_args; i++) {
                            new_estate->args[i] = prod->args[i];
+                            if (prod->args[i] != NULL) {
+                                new_estate->prob += prod->args[i]->viterbi_prob;
+                            }
                        }
                        queue.push_back(new_estate);
                        std::push_heap(queue.begin(), queue.end(), estate_comp);
@@ -1175,7 +1280,7 @@ PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob)
                for (ExprProb ep : ccat->exprs) {
                    ExprState *app_state = new(estate->n_args) ExprState;
                    app_state->expr  = estate->expr ? u->eapp(estate->expr, ep.expr) : ep.expr;
-                    app_state->prob  = estate->prob+ep.prob;
+                    app_state->prob  = estate->prob-ccat->viterbi_prob+ep.prob;
                    app_state->hash  = estate->hash * 31 + ep.hash;
                    app_state->res   = estate->res;
                    app_state->index = estate->index+1;
@@ -1194,7 +1299,7 @@ PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob)
            return estate->expr;
        }

-        prob_t prob = estate->prob - estate->res->pending[0]->prob;
+        prob_t prob = estate->prob - (estate->res->pending[0]->prob-estate->res->viterbi_prob);
        for (size_t i = estate->res->exprs.size(); i > 0; i--) {
            ExprProb &ep = estate->res->exprs[i-1];
            if (ep.prob != prob)
@@ -1207,7 +1312,7 @@ PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob)
        for (ExprState *parent : estate->res->pending) {
            ExprState *app_state = new(parent->n_args) ExprState;
            app_state->expr  = parent->expr ? u->eapp(parent->expr, estate->expr) : estate->expr;
-            app_state->prob  = parent->prob+prob;
+            app_state->prob  = parent->prob-estate->res->viterbi_prob+prob;
            app_state->hash  = parent->hash * 31 + estate->hash;
            app_state->res   = parent->res;
            app_state->index = parent->index+1;
@@ -1224,7 +1329,7 @@ PgfExpr PgfParser::process_expr(ExprState *estate, prob_t *prob)

 PgfAbstractParser::State *PgfParser::new_state(const PgfTextSpot &start)
 {
-    State **prev = &first_state;
+    State **prev = &current_state;
    State *state = current_state;
    while (state != NULL && state->start.ptr <= start.ptr) {
        if (state->start.ptr == start.ptr)
@@ -1248,42 +1353,48 @@ PgfAbstractParser::State *PgfParser::new_state(const PgfTextSpot &start)
        state->end.ptr = ptr;
    }

-    state->needs_bind = (state->start.pos == state->end.pos);
+    state->needs_bind = (state->start.pos > 0 && state->start.pos == state->end.pos);

    return state;
 }

-void PgfParser::symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym)
+void PgfParser::symbol_token(Item *item, State *state, PgfSymbol sym)
 {
-    PgfTextSpot next = spot;
-
-    const uint8_t *start = next.ptr;
-    for (;;) {
-        const uint8_t *ptr = next.ptr;
-        uint32_t ucs = pgf_utf8_decode(&ptr);
-        if (!pgf_utf8_is_space(ucs))
-            break;
-        next.ptr = ptr;
-        next.pos++;
-    }
-
-    if (bind != (spot.ptr == next.ptr))
-        return;
-
+    PgfTextSpot next = state->end;
    if (text_symbol_cmp(&next,end,sym,case_sensitive) != 0)
        return;

+    State *next_state = new_state(next);
+
    item->dot++;
-    process(item, next, false);
+    process(item, next_state);
 }

-void PgfParser::symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym)
+void PgfParser::symbol_bind(Item *item, State *state, PgfSymbol sym)
 {
-    item->dot++;
-    process(item, spot, true);
+    if (state->needs_bind) {
+        State *next_state = state->next;
+        if (next_state == NULL || state->end.pos != next_state->start.pos) {
+            next_state = new State;
+            next_state->start = state->end;
+            next_state->end   = state->end;
+            next_state->next  = state->next;
+            next_state->needs_bind = false;
+            state->next = next_state;
+        }
+        item->dot++;
+        next_state->push_item(item);
+    } else {
+        if (ref<PgfSymbol>::get_tag(sym) == PgfSymbolBIND::tag) {
+            delete item;
+        } else {
+            item->dot++;
+            process(item, state);
+        }
+    }
 }

-void PgfParser::suspend(Cont *cont,Item *item,size_t n_suspended)
+void PgfParser::suspend(Cont *cont,Item *item,size_t n_suspended1,size_t n_suspended)
 {
    if (n_suspended == 1) {
        std::function<void(ref<PgfSymbolCCat>,size_t,vector<ref<PgfItem>>)> f =
@@ -1312,26 +1423,40 @@ void PgfParser::suspend(Cont *cont,Item *item,size_t n_suspended)
                    arg_ccat->lin_idx = symcf->lin_idx;
                    arg_ccat->value = symcf->value;
                    arg_ccat->covered = true;
+                    arg_ccat->viterbi_prob = symcf->viterbi_prob;
                }

                cont->state->completed[cont][symcf->value][symcf->lin_idx] = arg_ccat;

                new_item->dot++;
                new_item->args[sym_cat->d] = arg_ccat;
+                new_item->inside_prob += arg_ccat->viterbi_prob;

-                process(new_item, cont->state->start, false);
+                cont->state->push_item(new_item);
            };
        phrasetable_iter(concr->phrasetable,cont->lincat,f);
-    } else {
-        auto it1 = cont->state->completed.find(cont);
-        if (it1 != cont->state->completed.end()) {
+    }
+
+    State *state = cont->state;
+    while (state != NULL) {
+        auto it1 = state->completed.find(cont);
+        if (it1 != state->completed.end()) {
            for (auto it2 : it1->second) {
                for (auto it3 : it2.second) {
                    Item *new_item = new (item) Item;
-                    combine(cont->state, new_item, it3.second);
+                    combine(state, new_item, it3.second);
                }
            }
        }
+        state = state->next;
+    }
+
+    if (n_suspended1 == 0) {
+        if (cont->state->needs_bind) {
+            bu_predict(concr->phrasetable, cont->state);
+        } else {
+            bu_predict(concr->phrasetable, cont->state, 1, sentence->size);
+        }
    }
 }

@@ -1347,6 +1472,7 @@ void PgfParser::final_item(State *state, CCat *ccat, Item *item, interval_t valu
        estate->n_args = item->args.size();
        for (size_t i = 0; i < estate->n_args; i++) {
            estate->args[i] = item->args[i];
+            estate->prob += estate->args[i]->viterbi_prob;
        }
        queue.push_back(estate);
        std::push_heap(queue.begin(), queue.end(), estate_comp);
@@ -1390,7 +1516,15 @@ void PgfParser::print_expr_state(PgfMarshaller *m, ExprState *estate)
    PgfPrinter printer(NULL,0,m);
    printer.nprintf(64,"[%f] ",estate->prob);
    print_expr_state_left(&printer, m, estate);
-    printer.puts(" .");
+    printer.puts(" . ");
+
+    if (estate->index < estate->n_args) {
+        if (estate->args[estate->index] != NULL)
+            printer.emeta(estate->args[estate->index]->fid);
+        else
+            printer.puts("?");
+    }
+
    print_expr_state_right(&printer, estate);

    PgfText *text = printer.get_text();
@@ -1402,12 +1536,11 @@ void PgfParser::print_expr_state(PgfMarshaller *m, ExprState *estate)
 PgfParseTableMaker::PgfParseTableMaker(ref<PgfConcr> concr)
    : PgfAbstractParser(concr)
 {
-    first_state = new State;
-    first_state->start.pos = 0;
-    first_state->start.ptr = NULL;
-    first_state->end       = first_state->start;
-    first_state->next      = NULL;
-    current_state = first_state;
+    current_state = new State;
+    current_state->start.pos = 0;
+    current_state->start.ptr = NULL;
+    current_state->end       = current_state->start;
+    current_state->next      = NULL;
 }

 ref<PgfItem> PgfParseTableMaker::clone_item(Item *item)
@@ -1421,7 +1554,7 @@ ref<PgfItem> PgfParseTableMaker::clone_item(Item *item)
    pitem->dot     = item->dot;
    pitem->rule    = item->rule;
    memcpy(&pitem->vars[0],&item->vars[0],sizeof(size_t) * item->vars.size());
-    
+
    for (size_t i = 0; i < item->args.size(); i++) {
        ref<PgfSymbolCCat> symcf = 0;
        if (item->args[i] != NULL) {
@@ -1430,6 +1563,7 @@ ref<PgfItem> PgfParseTableMaker::clone_item(Item *item)
            symcf->value   = item->args[i]->value;
            symcf->lin_idx = item->args[i]->lin_idx;
            symcf->fid     = item->args[i]->fid;
+            symcf->viterbi_prob = item->args[i]->viterbi_prob;
        }
        pitem->args[i] = symcf;
    }
@@ -1439,10 +1573,10 @@ ref<PgfItem> PgfParseTableMaker::clone_item(Item *item)

 PgfAbstractParser::State *PgfParseTableMaker::new_state(const PgfTextSpot &start)
 {
-    return this->first_state;
+    return current_state;
 }

-void PgfParseTableMaker::symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym)
+void PgfParseTableMaker::symbol_token(Item *item, State *state, PgfSymbol sym)
 {
    auto pitem = clone_item(item);
    auto phrasetable = phrasetable_insert(concr->phrasetable,sym,pitem);
@@ -1450,15 +1584,21 @@ void PgfParseTableMaker::symbol_token(Item *item, const PgfTextSpot &spot, bool
    delete item;
 }

-void PgfParseTableMaker::symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym)
+void PgfParseTableMaker::symbol_bind(Item *item, State *state, PgfSymbol sym)
 {
    auto pitem = clone_item(item);
-    auto phrasetable = phrasetable_insert(concr->phrasetable,sym,pitem);
+    auto phrasetable = phrasetable_insert(concr->phrasetable,ref<PgfSymbolBIND>(0).tagged(),pitem);
    concr->phrasetable = phrasetable;
-    delete item;
+
+    if (ref<PgfSymbol>::get_tag(sym) == PgfSymbolBIND::tag) {
+        delete item;
+    } else {
+        item->dot++;
+        process(item,state);
+    }
 }

-void PgfParseTableMaker::suspend(Cont *cont,Item *item,size_t n_suspended)
+void PgfParseTableMaker::suspend(Cont *cont,Item *item,size_t n_suspended1,size_t n_suspended)
 {
    // collect the cats first, since calling combine in
    // the loop will change the search index
@@ -1488,7 +1628,7 @@ void PgfParseTableMaker::final_item(State *state, CCat *ccat, Item *item, interv
    
    PgfPhrasetable phrasetable = concr->phrasetable;
    phrasetable = phrasetable_insert(phrasetable,
-                                     item->cont->lincat, value, lin_idx, ccat->fid,
+                                     item->cont->lincat, value, lin_idx, ccat->fid, ccat->viterbi_prob,
                                     pitem);
    concr->phrasetable = phrasetable;
 }
@@ -1503,12 +1643,12 @@ void PgfParseTableMaker::insert_rule(ref<PgfConcrRule> rule)
    case PgfConcrLin::tag: {
        auto lin = ref<PgfConcrLin>::untagged(rule->container);

-        Cont *&cont = first_state->conts1[lin->lincat];
+        Cont *&cont = current_state->conts1[lin->lincat];
        if (cont == NULL) {
            cont = new Cont;
            cont->ccat = NULL;
            cont->lincat = lin->lincat;
-            cont->state = first_state;
+            cont->state = current_state;
        }

        Item *item = new(rule) Item;
@@ -1518,7 +1658,17 @@ void PgfParseTableMaker::insert_rule(ref<PgfConcrRule> rule)
        item->pre_dot = 0;
        item->syms    = rule->syms.as_vector();
        item->rule    = rule;
-        return process(item, first_state->end, false);
+        item->inside_prob = lin->absfun->prob;
+        item->outside_prob = 0;
+        current_state->push_item(item);
    }
    }
 }
+
+void PgfParseTableMaker::prepare()
+{
+    while (current_state->has_items()) {
+        Item *item = current_state->pop_item();
+        process(item,current_state);
+    }
+}
@@ -89,6 +89,7 @@ protected:
        State *state;
        interval_t value;
        interval_t lin_idx;
+        prob_t viterbi_prob;
        bool covered;
        std::vector<Production*> prods;
        std::vector<ExprState*> pending;
@@ -103,8 +104,25 @@ protected:
        std::map<ref<PgfConcrLincat>,Cont*> conts1;
        std::map<CCat*,Cont*> conts2;
        std::map<Cont*,interval_map<interval_map<CCat*>>> completed;
-        
+        std::vector<Item*> queue;
+
        State *next;
+
+        bool has_items() {
+            return queue.size() > 0;
+        }
+
+        void push_item(Item *item) {
+            queue.push_back(item);
+            std::push_heap(queue.begin(), queue.end(), item_comp);
+        }
+
+        Item *pop_item() {
+            Item *item = queue.front();
+            std::pop_heap(queue.begin(), queue.end(), item_comp);
+            queue.pop_back();
+            return item;
+        }
    };

    struct Cont {
@@ -123,6 +141,8 @@ protected:
        uint16_t dot;
        vector<PgfSymbol> syms;
        ref<PgfConcrRule> rule;
+        prob_t inside_prob;
+        prob_t outside_prob;

        struct {
            size_t &operator[](int i) const {
@@ -177,6 +197,12 @@ protected:
                         PgfConcrRule *rule, size_t *values, ref<PgfLParam> lparam2);
    };

+    static struct ItemComparator : std::less<Item*> {
+        bool operator()(Item *item1, Item *item2) {
+            return item1->inside_prob+item1->outside_prob > item2->inside_prob+item2->outside_prob;
+        }
+    } item_comp;
+
    struct ExprState {
        PgfExpr expr;
        prob_t prob;
@@ -204,29 +230,29 @@ protected:
        }
    };

-    State *first_state, *current_state;
+    State *current_state;
    std::map<ref<PgfConcrLincat>,interval_map<interval_map<CCat*>>> epsilons;
    PgfMetaId initial_fid, last_fid;

-    void process(Item *item, const PgfTextSpot &spot, bool bind);
-    void symbol(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym);
-    void complete(Item *item, const PgfTextSpot &spot, bool bind);
+    void process(Item *item, State *state);
+    void symbol(Item *item, State *state, PgfSymbol sym);
+    void complete(Item *item, State *state);

    virtual State *new_state(const PgfTextSpot &start)=0;
-    virtual void symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym)=0;
-    virtual void symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym)=0;
-    virtual void suspend(Cont *cont, Item *item, size_t n_suspended)=0;
+    virtual void symbol_token(Item *item, State *state, PgfSymbol sym)=0;
+    virtual void symbol_bind(Item *item, State *state, PgfSymbol sym)=0;
+    virtual void suspend(Cont *cont, Item *item, size_t n_suspended1, size_t n_suspended)=0;
    virtual void final_item(State *state,CCat *ccat,Item *item,interval_t value,interval_t lin_idx)=0;
    virtual void bu_predict(State *state, CCat *ccat)=0;

-    void td_epsilon(State *state, Cont *cont, ref<PgfItem> pitem, Item *xitem, ref<PgfLParam> value, ref<PgfLParam> lin_idx);
-    void td_predict(State *state, Cont *cont, Production *prod, Item *xitem, ref<PgfLParam> value, ref<PgfLParam> lin_idx);
+    void td_epsilon(State *state, Cont *cont, ref<PgfItem> pitem, Item *xitem, ref<PgfSymbolCat> symcat);
+    void td_predict(State *state, Cont *cont, Production *prod, Item *xitem, ref<PgfSymbolCat> symcat);
    void combine(State *state, Item *item, CCat *ccat);

    void get_info(CCat *ccat, ref<PgfConcrRule> *rule, size_t **pvalues);

    static
-    void print_item(Item *item, const PgfTextSpot &spot);
+    void print_item(Item *item, State *state);

    static
    void print_prod(CCat *ccat, Production *prod);
@@ -245,12 +271,13 @@ class PGF_INTERNAL_DECL PgfParser : private PgfAbstractParser, public PgfExprEnu
    bool case_sensitive;

    virtual State *new_state(const PgfTextSpot &start);
-    virtual void symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym);
-    virtual void symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym);
-    virtual void suspend(Cont *cont,Item *item,size_t n_suspended);
+    virtual void symbol_token(Item *item, State *state, PgfSymbol sym);
+    virtual void symbol_bind(Item *item, State *state, PgfSymbol sym);
+    virtual void suspend(Cont *cont,Item *item,size_t n_suspended1,size_t n_suspended);
    virtual void final_item(State *state,CCat *ccat,Item *item,interval_t value,interval_t lin_idx);
    virtual void bu_predict(State *state, CCat *ccat);

+    void bu_predict(PgfPhrasetable phrasetable, State *state);
    void bu_predict(PgfPhrasetable phrasetable, State *state, ptrdiff_t min, ptrdiff_t max);
    void make_chunks(State *state, std::vector<CCat*> &chunks, prob_t prob);
    PgfExpr process_expr(ExprState *estate, prob_t *prob);
@@ -265,7 +292,7 @@ class PGF_INTERNAL_DECL PgfParser : private PgfAbstractParser, public PgfExprEnu
    static
    void print_expr_state(PgfMarshaller *m, ExprState *estate);

-    struct ExprStateComparator : std::less<ExprState*> {
+    static struct ExprStateComparator : std::less<ExprState*> {
        bool operator()(ExprState *estate1, ExprState *estate2) {
            return estate1->prob > estate2->prob;
        }
@@ -286,9 +313,9 @@ class PGF_INTERNAL_DECL PgfParseTableMaker : private PgfAbstractParser
 {
 private:
    virtual State *new_state(const PgfTextSpot &start);
-    virtual void symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym);
-    virtual void symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym);
-    virtual void suspend(Cont *cont, Item *item, size_t n_suspended);
+    virtual void symbol_token(Item *item, State *state, PgfSymbol sym);
+    virtual void symbol_bind(Item *item, State *state, PgfSymbol sym);
+    virtual void suspend(Cont *cont, Item *item, size_t n_suspended1, size_t n_suspended);
    virtual void final_item(State *state, CCat *ccat,Item *item,interval_t value,interval_t lin_idx);
    virtual void bu_predict(State *state, CCat *ccat);

@@ -298,6 +325,7 @@ private:
 public:
    PgfParseTableMaker(ref<PgfConcr> concr);
    void insert_rule(ref<PgfConcrRule> rule);
+    void prepare();
    PgfMetaId get_last_fid() { return last_fid; };
 };

@@ -1726,6 +1726,7 @@ void pgf_free_parse_table(PgfDB *db,
    ref<PgfPGF> pgf = db->revision2pgf(revision);
    ref<PgfConcr> concr = db->revision2concr(cnc_revision);

+    table_maker->prepare();
    concr->last_fid = table_maker->get_last_fid();
    delete table_maker;
 }
@@ -1001,7 +1001,7 @@ PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
 PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
                                  ref<PgfConcrLincat> lincat,
                                  interval_t value, interval_t lin_idx,
-                                  PgfMetaId fid,
+                                  PgfMetaId fid, prob_t viterbi_prob,
                                  ref<PgfItem> item)
 {
    if (table == 0) {
@@ -1010,6 +1010,7 @@ PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
        symcf->value   = value;
        symcf->lin_idx = lin_idx;
        symcf->fid = fid;
+        symcf->viterbi_prob = viterbi_prob;
        PgfPhrasetable new_table = PgfPhrasetableNode::new_node(symcf.tagged(),1);
        new_table->n_items = 1;
        new_table->items[0] = item;
@@ -1019,12 +1020,12 @@ PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
    int cmp = symbol_cmp(lincat,value,lin_idx,table->sym);
    if (cmp < 0) {
        PgfPhrasetable left = phrasetable_insert(table->left,
-                                                lincat, value, lin_idx, fid, item);
+                                                lincat, value, lin_idx, fid, viterbi_prob, item);
        table = PgfPhrasetableNode::upd_node(table,left,table->right);
        return PgfPhrasetableNode::balanceL(table);
    } else if (cmp > 0) {
        PgfPhrasetable right = phrasetable_insert(table->right,
-                                                  lincat, value, lin_idx, fid, item);
+                                                  lincat, value, lin_idx, fid, viterbi_prob, item);
        table = PgfPhrasetableNode::upd_node(table, table->left, right);
        return PgfPhrasetableNode::balanceR(table);
    } else {
@@ -91,7 +91,7 @@ PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
 PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
                                  ref<PgfConcrLincat> lincat,
                                  interval_t value, interval_t lin_idx,
-                                  PgfMetaId fid,
+                                  PgfMetaId fid, prob_t viterbi_prob,
                                  ref<PgfItem> item);

 PGF_INTERNAL_DECL
@@ -712,6 +712,8 @@ ref<PgfConcr> PgfReader::read_concrete()
 	auto lins = read_namespace<PgfConcrLin>(&PgfReader::read_lin);
 	concrete->lins = lins;

+    tm.prepare();
+
    concrete->last_fid = tm.get_last_fid();
    this->table_maker = NULL;