Diophantine grammars in the runtime

src/runtime/c/pgf/data.cxx

@@ -40,7 +40,6 @@ void PgfConcr::release(ref<PgfConcr> concr)
     namespace_release(concr->cflags);
     namespace_release(concr->lins);
     namespace_release(concr->lincats);
-    phrasetable_release(concr->phrasetable);
     namespace_release(concr->printnames);
     PgfDB::free(concr, concr->name.size+1);
 }
@@ -52,17 +51,10 @@ void PgfConcrLincat::release(ref<PgfConcrLincat> lincat)
     }
     vector<ref<PgfText>>::release(lincat->fields);
 
-	for (size_t i = 0; i < lincat->args.size(); i++) {
+    for (ref<PgfConcrRule> rule : lincat->rules) {
-        PgfLParam::release(lincat->args[i].param);
+        PgfConcrRule::release(rule);
     }
-    vector<PgfPArg>::release(lincat->args);
+    vector<ref<PgfConcrRule>>::release(lincat->rules);
-
-    for (ref<PgfPResult> res : lincat->res) {
-        PgfPResult::release(res);
-    }
-    vector<ref<PgfPResult>>::release(lincat->res);
-
-    vector<ref<PgfSequence>>::release(lincat->seqs);
 
     PgfDB::free(lincat, lincat->name.size+1);
 }
@@ -79,9 +71,9 @@ void PgfPResult::release(ref<PgfPResult> res)
     PgfDB::free(res, res->param.n_terms*sizeof(res->param.terms[0]));
 }
 
-void PgfSequence::release(ref<PgfSequence> seq)
+static void symbols_release(vector<PgfSymbol> syms)
 {
-	for (PgfSymbol sym : seq->syms) {
+	for (PgfSymbol sym : syms) {
         switch (ref<PgfSymbol>::get_tag(sym)) {
         case PgfSymbolCat::tag: {
             auto sym_cat = ref<PgfSymbolCat>::untagged(sym);
@@ -103,9 +95,11 @@ void PgfSequence::release(ref<PgfSequence> seq)
         }
         case PgfSymbolKP::tag: {
             auto sym_kp = ref<PgfSymbolKP>::untagged(sym);
-            PgfSequence::release(sym_kp->default_form);
+            symbols_release(sym_kp->default_form);
+            vector<PgfSymbol>::release(sym_kp->default_form);
             for (size_t i = 0; i < sym_kp->alts.size(); i++) {
-                PgfSequence::release(sym_kp->alts[i].form);
+                symbols_release(sym_kp->alts[i].form);
+                vector<PgfSymbol>::release(sym_kp->alts[i].form);
                 for (size_t j = 0; j < sym_kp->alts[i].prefixes.size(); j++) {
                     text_db_release(sym_kp->alts[i].prefixes[j]);
                 }
@@ -124,22 +118,31 @@ void PgfSequence::release(ref<PgfSequence> seq)
             throw pgf_error("Unknown symbol tag");
         }
     }
-    inline_vector<PgfSymbol>::release(&PgfSequence::syms, seq);
+}
+
+void PgfConcrRule::release(ref<PgfConcrRule> rule)
+{
+    vector<PgfVariableRange>::release(rule->vars);
+
+    PgfLParam::release(rule->res);
+
+    for (ref<PgfLParam> arg : rule->args) {
+        PgfLParam::release(arg);
+    }
+    vector<ref<PgfLParam>>::release(rule->args);
+
+    PgfLParam::release(rule->lin_idx);
+
+    symbols_release(rule->syms.as_vector());
+    inline_vector<PgfSymbol>::release(&PgfConcrRule::syms, rule);
 }
 
 void PgfConcrLin::release(ref<PgfConcrLin> lin)
 {
-    for (size_t i = 0; i < lin->args.size(); i++) {
+    for (ref<PgfConcrRule> rule : lin->rules) {
-        PgfLParam::release(lin->args[i].param);
+        PgfConcrRule::release(rule);
     }
-    vector<PgfPArg>::release(lin->args);
+    vector<ref<PgfConcrRule>>::release(lin->rules);
-
-    for (ref<PgfPResult> res : lin->res) {
-        PgfPResult::release(res);
-    }
-    vector<ref<PgfPResult>>::release(lin->res);
-
-    vector<ref<PgfSequence>>::release(lin->seqs);
 
     PgfDB::free(lin, lin->name.size+1);
 }

src/runtime/c/pgf/data.h

@@ -87,7 +87,6 @@ struct PgfConcr;
 #include "text.h"
 #include "vector.h"
 #include "namespace.h"
-#include "phrasetable.h"
 #include "probspace.h"
 #include "expr.h"
 
@@ -155,12 +154,6 @@ struct PGF_INTERNAL_DECL PgfPResult {
 
 typedef object PgfSymbol;
 
-struct PGF_INTERNAL_DECL PgfSequence {
-	inline_vector<PgfSymbol> syms;
-
-    static void release(ref<PgfSequence> seq);
-};
-
 struct PGF_INTERNAL_DECL PgfSequenceBackref {
     object container;
     size_t seq_index;
@@ -189,7 +182,7 @@ struct PGF_INTERNAL_DECL PgfSymbolKS {
 };
 
 struct PGF_INTERNAL_DECL PgfAlternative {
-	ref<PgfSequence> form;
+	vector<PgfSymbol> form;
 	/**< The form of this variant as a list of tokens. */
 
 	vector<ref<PgfText>> prefixes;
@@ -199,7 +192,7 @@ struct PGF_INTERNAL_DECL PgfAlternative {
 
 struct PGF_INTERNAL_DECL PgfSymbolKP {
     static const uint8_t tag = 4;
-    ref<PgfSequence> default_form;
+    vector<PgfSymbol> default_form;
     inline_vector<PgfAlternative> alts;
 };
 
@@ -227,15 +220,24 @@ struct PGF_INTERNAL_DECL PgfSymbolALLCAPIT {
     static const uint8_t tag = 10;
 };
 
+struct PGF_INTERNAL_DECL PgfConcrRule {
+    vector<PgfVariableRange> vars;
+    ref<PgfLParam> res;
+    object container;
+    vector<ref<PgfLParam>> args;
+    ref<PgfLParam> lin_idx;
+    inline_vector<PgfSymbol> syms;
+
+    static void release(ref<PgfConcrRule> seq);
+};
+
 struct PGF_INTERNAL_DECL PgfConcrLincat {
     static const uint8_t tag = 0;
 
     ref<PgfAbsCat> abscat;
 
     size_t n_lindefs;
-    vector<PgfPArg> args;
+    vector<ref<PgfConcrRule>> rules;
-    vector<ref<PgfPResult>> res;
-    vector<ref<PgfSequence>> seqs;
     vector<ref<PgfText>> fields;
 
     PgfText name;
@@ -249,15 +251,25 @@ struct PGF_INTERNAL_DECL PgfConcrLin {
     ref<PgfAbsFun> absfun;
     ref<PgfConcrLincat> lincat;
 
-    vector<PgfPArg> args;
+    vector<ref<PgfConcrRule>> rules;
-    vector<ref<PgfPResult>> res;
-    vector<ref<PgfSequence>> seqs;
 
     PgfText name;
 
     static void release(ref<PgfConcrLin> lin);
 };
 
+struct PGF_INTERNAL_DECL PgfSymbolACat {
+    static const uint8_t tag = 11;
+    PgfText name;
+};
+
+struct PGF_INTERNAL_DECL PgfSymbolCCat {
+    static const uint8_t tag = 12;
+    ref<PgfConcrLincat> lincat;
+    size_t value;
+    size_t lin_idx;
+};
+
 struct PGF_INTERNAL_DECL PgfConcrPrintname {
     ref<PgfText> printname;
     PgfText name;
@@ -267,134 +279,7 @@ struct PGF_INTERNAL_DECL PgfConcrPrintname {
 
 #define containerof(T,field,p) (T*) (((char*) p)-offsetof(T,field))
 
-struct PGF_INTERNAL_DECL PgfLCEdge {
+#include "phrasetable.h"
-    struct {
-        ref<PgfConcrLincat> lincat;
-        struct {
-            size_t i0;
-            term& operator[](int i) {
-                PgfLCEdge *edge = containerof(PgfLCEdge,from.value,this);
-                return edge->terms[i];
-            }
-            size_t size() {
-                PgfLCEdge *edge = containerof(PgfLCEdge,from.value,this);
-                return edge->from.lin_idx.n_offset;
-            }
-        } value;
-        struct {
-            size_t i0;
-            size_t n_offset;
-            term& operator[](int i) {
-                PgfLCEdge *edge = containerof(PgfLCEdge,from.lin_idx,this);
-                return edge->terms[n_offset+i];
-            }
-            size_t size() {
-                PgfLCEdge *edge = containerof(PgfLCEdge,from.lin_idx,this);
-                return edge->to.value.n_offset-n_offset;
-            }
-        } lin_idx;
-    } from;
-
-    struct {
-        ref<PgfConcrLincat> lincat;
-        struct {
-            size_t i0;
-            size_t n_offset;
-            term& operator[](int i) {
-                PgfLCEdge *edge = containerof(PgfLCEdge,to.value,this);
-                return edge->terms[n_offset+i];
-            }
-            size_t size() {
-                PgfLCEdge *edge = containerof(PgfLCEdge,to.value,this);
-                return edge->to.lin_idx.n_offset-n_offset;
-            }
-        } value;
-        struct {
-            size_t i0;
-            size_t n_offset;
-            term& operator[](int i) {
-                PgfLCEdge *edge = containerof(PgfLCEdge,to.lin_idx,this);
-                return edge->terms[n_offset+i];
-            }
-            size_t size() {
-                PgfLCEdge *edge = containerof(PgfLCEdge,to.lin_idx,this);
-                return edge->n_terms-n_offset;
-            }
-        } lin_idx;
-    } to;
-
-    struct {
-        size_t n_vars;
-        PgfVariableRange& operator[](int i) {
-            PgfLCEdge *edge = containerof(PgfLCEdge,vars,this);
-            return ((PgfVariableRange*)(((term*) (edge+1))+edge->n_terms))[i];
-        }
-        size_t size() {
-            return n_vars;
-        }
-    } vars;
-
-    size_t n_terms;
-    term terms[];
-
-    static ref<PgfLCEdge> alloc(size_t n_terms1, size_t n_terms2, size_t n_terms3, size_t n_terms4, size_t n_vars) {
-        auto edge = PgfDB::malloc<PgfLCEdge>((n_terms1+n_terms2+n_terms3+n_terms4)*sizeof(term)+n_vars*sizeof(PgfVariableRange));
-        edge->from.lin_idx.n_offset = n_terms1;
-        edge->to.value.n_offset = n_terms1+n_terms2;
-        edge->to.lin_idx.n_offset = n_terms1+n_terms2+n_terms3;
-        edge->n_terms = n_terms1+n_terms2+n_terms3+n_terms4;
-        edge->vars.n_vars = n_vars;
-        return edge;
-    }
-};
-
-struct PGF_INTERNAL_DECL PgfLRShift {
-    size_t next_state;
-    ref<PgfConcrLincat> lincat;
-    size_t r;
-};
-
-struct PGF_INTERNAL_DECL PgfLRShiftKS {
-    size_t next_state;
-    ref<PgfSequence> seq;
-    size_t sym_idx;
-};
-
-struct PgfLRReduceArg;
-
-struct PGF_INTERNAL_DECL PgfLRProduction {
-    ref<PgfConcrLin> lin;
-    size_t index;
-    vector<ref<PgfLRReduceArg>> args;
-};
-
-struct PGF_INTERNAL_DECL PgfLRReduceArg {
-    static const uint8_t tag = 2;
-
-    size_t id;
-    size_t n_prods;
-    PgfLRProduction prods[];
-};
-
-struct PGF_INTERNAL_DECL PgfLRReduce {
-    object lin_obj;
-    size_t seq_idx;
-    size_t depth;
-
-    struct Arg {
-        ref<PgfLRReduceArg> arg;
-        size_t stk_idx;
-    };
-
-    vector<Arg> args;
-};
-
-struct PGF_INTERNAL_DECL PgfLRState {
-    vector<PgfLRShift> shifts;
-    vector<PgfLRShiftKS> tokens;
-    size_t next_bind_state;
-    vector<PgfLRReduce> reductions;
-};
 
 struct PGF_INTERNAL_DECL PgfConcr {
     Namespace<PgfFlag> cflags;
@@ -403,8 +288,6 @@ struct PGF_INTERNAL_DECL PgfConcr {
     PgfPhrasetable phrasetable;
     Namespace<PgfConcrPrintname> printnames;
 
-    vector<PgfLRState> lrtable;
-
     PgfText name;
 
     static void release(ref<PgfConcr> pgf);

src/runtime/c/pgf/linearizer.cxx

@@ -2,6 +2,59 @@
 #include "printer.h"
 #include "linearizer.h"
 
+bool PgfLinearizer::Item::instantiate(ref<PgfLParam> lparam,size_t value)
+{
+    if (value < lparam->i0)
+        return false;
+    value -= lparam->i0;
+
+    for (size_t j = 0; j < lparam->n_terms; j++) {
+        term t = lparam->terms[j];
+        for (size_t k = 0; k < vars.size(); k++) {
+            if (rule->vars[k].var == t.var) {
+                if (vars[k] > 0) {
+                    if (value < vars[k]-1)
+                        return false;
+                    value -= vars[k]-1;
+                }
+                break;
+            }
+        }
+    }
+
+    for (size_t j = 0; j < lparam->n_terms; j++) {
+        term t = lparam->terms[j];
+        for (size_t k = 0; k < vars.size(); k++) {
+            if (rule->vars[k].var == t.var) {
+                if (vars[k] == 0) {
+                    size_t v_val = value / t.factor;
+                    if (v_val >= rule->vars[k].range)
+                        return false;
+                    vars[k] = v_val + 1;
+                    value %= t.factor;
+                }
+                break;
+            }
+        }
+    }
+
+    return (value == 0);
+}
+
+size_t PgfLinearizer::Item::eval(ref<PgfLParam> lparam)
+{
+    size_t value = lparam->i0;
+    for (size_t i = 0; i < lparam->n_terms; i++) {
+        for (size_t j = 0; j < rule->vars.size(); j++) {
+            if (lparam->terms[i].var == rule->vars[j].var) {
+                value += lparam->terms[i].factor * (vars[j]-1);
+                break;
+            }
+        }
+    }
+    return value;
+}
+
 PgfLinearizer::TreeNode::TreeNode(PgfLinearizer *linearizer)
 {
     this->next     = linearizer->prev;
@@ -11,8 +64,6 @@ PgfLinearizer::TreeNode::TreeNode(PgfLinearizer *linearizer)
     this->fid       = 0;
 
     this->value     = 0;
-    this->var_count = 0;
-    this->var_values= NULL;
 
     this->n_hoas_vars = 0;
     this->hoas_vars   = NULL;
@@ -20,7 +71,7 @@ PgfLinearizer::TreeNode::TreeNode(PgfLinearizer *linearizer)
     linearizer->prev = this;
 }
 
-void PgfLinearizer::TreeNode::linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, PgfLParam *r)
+void PgfLinearizer::TreeNode::linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, size_t r)
 {
     TreeNode *arg = args;
     while (d > 0) {
@@ -31,8 +82,7 @@ void PgfLinearizer::TreeNode::linearize_arg(PgfLinearizationOutputIface *out, Pg
     }
     if (arg == 0)
         throw pgf_error("Missing argument");
-    size_t lindex = eval_param(r);
+    arg->linearize(out, linearizer, r);
-    arg->linearize(out, linearizer, lindex);
 }
 
 void PgfLinearizer::TreeNode::linearize_var(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, size_t r)
@@ -52,20 +102,22 @@ void PgfLinearizer::TreeNode::linearize_var(PgfLinearizationOutputIface *out, Pg
     out->symbol_token(linearizer->printer.get_text());
 }
 
-void PgfLinearizer::TreeNode::linearize_seq(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, ref<PgfSequence> seq)
+void PgfLinearizer::TreeNode::linearize_item(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, Item *item)
 {
-    for (size_t i = 0; i < seq->syms.size(); i++) {
+    for (size_t i = 0; i < item->rule->syms.size(); i++) {
-        PgfSymbol sym = seq->syms[i];
+        PgfSymbol sym = item->rule->syms[i];
 
         switch (ref<PgfSymbol>::get_tag(sym)) {
         case PgfSymbolCat::tag: {
             auto sym_cat = ref<PgfSymbolCat>::untagged(sym);
-            linearize_arg(out, linearizer, sym_cat->d, &sym_cat->r);
+            size_t r = item->eval(ref<PgfLParam>::from_ptr(&sym_cat->r));
+            linearize_arg(out, linearizer, sym_cat->d, r);
             break;
         }
         case PgfSymbolLit::tag: {
             auto sym_lit = ref<PgfSymbolLit>::untagged(sym);
-            linearize_arg(out, linearizer, sym_lit->d, &sym_lit->r);
+            size_t r = item->eval(ref<PgfLParam>::from_ptr(&sym_lit->r));
+            linearize_arg(out, linearizer, sym_lit->d, r);
             break;
         }
         case PgfSymbolVar::tag: {
@@ -169,113 +221,68 @@ void PgfLinearizer::TreeNode::linearize_seq(PgfLinearizationOutputIface *out, Pg
     }
 }
 
-size_t PgfLinearizer::TreeNode::eval_param(PgfLParam *param)
-{
-    size_t value = param->i0;
-    for (size_t j = 0; j < param->n_terms; j++) {
-        size_t factor = param->terms[j].factor;
-        size_t var    = param->terms[j].var;
-
-        if (var < var_count && var_values[var] != (size_t) -1) {
-            value += factor * var_values[var];
-        } else {
-            throw pgf_error("Unbound variable in resolving a linearization");
-        }
-    }
-    return value;
-}
-
 PgfLinearizer::TreeLinNode::TreeLinNode(PgfLinearizer *linearizer, ref<PgfConcrLin> lin)
   : TreeNode(linearizer)
 {
-    this->lin       = lin;
+    this->lin        = lin;
-    this->lin_index = 0;
+    this->rule_index = 0;
+    this->items = new Item*[lin->lincat->fields.size()]();
 }
 
 bool PgfLinearizer::TreeLinNode::resolve(PgfLinearizer *linearizer)
 {
     vector<PgfHypo> hypos = lin->absfun->type->hypos;
-    size_t n_args = lin->args.size() / lin->res.size();
 
-    while (lin_index < lin->res.size()) {
+    while (rule_index < lin->rules.size()) {
-        size_t offset = lin_index*n_args;
+        Item *item = new (lin->rules[rule_index]) Item();
-
-        ref<PgfPResult> pres = lin->res[lin_index];
-
-        // Unbind all variables
-        for (size_t j = 0; j < var_count; j++) {
-            var_values[j] = (size_t) -1;
-        }
 
         int i = 0;
         TreeNode *arg = args;
         while (arg != NULL) {
-            ref<PgfPArg> parg = lin->args.elem(offset+i);
             arg->check_category(linearizer, &hypos[i].type->name);
 
-            if (arg->value < parg->param->i0)
+            if (!item->instantiate(item->rule->args[i], arg->value))
                 break;
 
-            size_t value = arg->value - parg->param->i0;
+            arg = arg->next_arg;  i++;
-            for (size_t j = 0; j < parg->param->n_terms; j++) {
-                size_t factor    = parg->param->terms[j].factor;
-                size_t var       = parg->param->terms[j].var;
-                size_t var_value;
-
-                if (var < var_count && var_values[var] != (size_t) -1) {
-                    // The variable already has a value
-                    var_value = var_values[var];
-                } else {
-                    // The variable is not assigned yet
-                    var_value = value / factor;
-
-                    // find the range for the variable
-                    size_t range = 0;
-                    for (size_t k = 0; k < pres->vars.size(); k++) {
-                        ref<PgfVariableRange> var_range = pres->vars.elem(k);
-                        if (var_range->var == var) {
-                            range = var_range->range;
-                            break;
-                        }
-                    }
-                    if (range == 0)
-                        throw pgf_error("Unknown variable in resolving a linearization");
-
-                    if (var_value >= range)
-                        break;
-
-                    // Assign the variable;
-                    if (var >= var_count) {
-                        var_values = (size_t*)
-                            realloc(var_values, (var+1)*sizeof(size_t));
-                        while (var_count < var) {
-                            var_values[var_count++] = (size_t) -1;
-                        }
-                        var_count++;
-                    }
-                    var_values[var] = var_value;
-                }
-
-                value -= var_value * factor;
-            }
-
-            if (value != 0)
-                break;
-
-            arg = arg->next_arg;
-            i++;
         }
 
-        lin_index++;
+        size_t max_value = 1;
+        for (size_t i = 0; i < item->vars.size(); i++) {
+            if (item->vars[i] == 0)
+                max_value *= item->rule->vars[i].range;
+        }
 
-        if (arg == NULL) {
+        for (size_t value = 0; value < max_value; value++) {
-            value = eval_param(&pres->param);
+            Item *new_item = new (item) Item;
-            return true;
+
+            size_t v = value;
+            for (size_t i = 0; i < new_item->vars.size(); i++) {
+                if (new_item->vars[i] == 0) {
+                    size_t range = new_item->rule->vars[i].range;
+                    new_item->vars[i] = (v % range)+1;
+                    v = v / range;
+                }
+            }
+
+            size_t lin_idx = new_item->eval(new_item->rule->lin_idx);
+            items[lin_idx] = new_item;
+
+            this->value = new_item->eval(new_item->rule->res);
+        }
+        delete item;
+
+        rule_index++;
+    }
+
+    for (size_t i = 0; i < lin->lincat->fields.size(); i++) {
+        if (items[i] == NULL) {
+            rule_index = 0;
+            return false;
         }
     }
 
-    lin_index = 0;
+    return true;
-    return false;
 }
 
 void PgfLinearizer::TreeLinNode::check_category(PgfLinearizer *linearizer, PgfText *cat)
@@ -302,9 +309,7 @@ void PgfLinearizer::TreeLinNode::linearize(PgfLinearizationOutputIface *out, Pgf
         linearizer->pre_stack->bracket_stack = bracket;
     }
 
-    size_t n_seqs = lin->seqs.size() / lin->res.size();
+    linearize_item(out, linearizer, items[lindex]);
-    ref<PgfSequence> seq = lin->seqs[(lin_index-1)*n_seqs + lindex];
-    linearize_seq(out, linearizer, seq);
 
     if (linearizer->pre_stack == NULL)
         out->end_phrase(cat, fid, field, &lin->name);
@@ -325,11 +330,21 @@ ref<PgfConcrLincat> PgfLinearizer::TreeLinNode::get_lincat(PgfLinearizer *linear
     return namespace_lookup(linearizer->concr->lincats, &lin->absfun->type->name);
 }
 
+PgfLinearizer::TreeLinNode::~TreeLinNode()
+{
+    size_t n_fields = lin->lincat->fields.size();
+    for (size_t i = 0; i < n_fields; i++) {
+        delete items[i];
+    }
+    delete[] items;
+};
+
 PgfLinearizer::TreeLindefNode::TreeLindefNode(PgfLinearizer *linearizer, PgfText *fun, PgfText *literal)
   : TreeNode(linearizer)
 {
     this->lincat    = 0;
-    this->lin_index = 0;
+    this->rule_index= 0;
+    this->items     = NULL;
     this->fun       = fun;
     this->literal   = literal;
 
@@ -355,17 +370,46 @@ PgfLinearizer::TreeLindefNode::TreeLindefNode(PgfLinearizer *linearizer, PgfText
 
 bool PgfLinearizer::TreeLindefNode::resolve(PgfLinearizer *linearizer)
 {
-    if (lincat == 0) {
+/*    while (rule_index < lincat->n_lindefs2) {
-        return (lin_index = !lin_index);
+        ref<PgfConcrRule> rule = lincat->rules[rule_index];
-    } else {
+        Item *item = new (rule) Item();
-        ref<PgfPResult> pres = lincat->res[lin_index];
+
-        value = eval_param(&pres->param);
+        size_t max_value = 1;
-        lin_index++;
+        for (size_t i = 0; i < item->vars.size(); i++) {
-        if (lin_index <= lincat->n_lindefs)
+            if (item->vars[i] == 0)
-            return true;
+                max_value *= item->rule->vars[i].range;
-        lin_index = 0;
+        }
-        return false;
+
+        for (size_t value = 0; value < max_value; value++) {
+            size_t v = value;
+            for (size_t i = 0; i < item->vars.size(); i++) {
+                if (item->vars[i] == 0) {
+                    size_t range = item->rule->vars[i].range;
+                    item->vars[i] = v % range;
+                    v = v / range;
+                }
+            }
+
+            Item *new_item = new (item) Item;
+
+            size_t lin_idx = item->eval(new_item->rule->lin_idx);
+            items[lin_idx] = new_item;
+
+            this->value = item->eval(new_item->rule->res);
+        }
+        delete item;
+
+        rule_index++;
     }
+
+    for (size_t i = 0; i < lincat->fields.size(); i++) {
+        if (items[i] == NULL) {
+            rule_index = 0;
+            return false;
+        }
+    }
+*/
+    return true;
 }
 
 void PgfLinearizer::TreeLindefNode::check_category(PgfLinearizer *linearizer, PgfText *cat)
@@ -373,6 +417,7 @@ void PgfLinearizer::TreeLindefNode::check_category(PgfLinearizer *linearizer, Pg
     lincat = namespace_lookup(linearizer->concr->lincats, cat);
     if (lincat == 0)
         throw pgf_error("Cannot find a lincat for a category");
+    this->items = new Item*[lincat->fields.size()]();
 }
 
 void PgfLinearizer::TreeLindefNode::linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, PgfLParam *r)
@@ -389,7 +434,7 @@ void PgfLinearizer::TreeLindefNode::linearize_arg(PgfLinearizationOutputIface *o
 
 void PgfLinearizer::TreeLindefNode::linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex)
 {
-    if (lincat != 0) {
+/*    if (lincat != 0) {
         PgfText *field = &*lincat->fields[lindex];
         if (linearizer->pre_stack == NULL)
             out->begin_phrase(&lincat->name, fid, field, fun);
@@ -404,8 +449,8 @@ void PgfLinearizer::TreeLindefNode::linearize(PgfLinearizationOutputIface *out,
             linearizer->pre_stack->bracket_stack = bracket;
         }
 
-        ref<PgfSequence> seq = lincat->seqs[(lin_index-1)*lincat->fields.size() + lindex];
+        ref<PgfSequence> seq = lincat->seqs[(rule_index-1)*lincat->fields.size() + lindex];
-        linearize_seq(out, linearizer, seq);
+//        linearize_seq(out, linearizer, seq);
 
         if (linearizer->pre_stack == NULL)
             out->end_phrase(&lincat->name, fid, field, fun);
@@ -421,7 +466,7 @@ void PgfLinearizer::TreeLindefNode::linearize(PgfLinearizationOutputIface *out,
         }
     } else {
         linearize_arg(out, linearizer, 0, NULL);
-    }
+    }*/
 }
 
 ref<PgfConcrLincat> PgfLinearizer::TreeLindefNode::get_lincat(PgfLinearizer *linearizer)
@@ -429,11 +474,26 @@ ref<PgfConcrLincat> PgfLinearizer::TreeLindefNode::get_lincat(PgfLinearizer *lin
     return lincat;
 }
 
+PgfLinearizer::TreeLindefNode::~TreeLindefNode()
+{
+    if (lincat) {
+        size_t n_fields = lincat->fields.size();
+        for (size_t i = 0; i < n_fields; i++) {
+            delete items[i];
+        }
+        delete[] items;
+    }
+
+    free(fun);
+    free(literal);
+};
+
 PgfLinearizer::TreeLinrefNode::TreeLinrefNode(PgfLinearizer *linearizer, TreeNode *root)
   : TreeNode(linearizer)
 {
     args = root;
-    lin_index=0;
+    rule_index=0;
+    item = NULL;
 }
 
 bool PgfLinearizer::TreeLinrefNode::resolve(PgfLinearizer *linearizer)
@@ -441,81 +501,53 @@ bool PgfLinearizer::TreeLinrefNode::resolve(PgfLinearizer *linearizer)
     TreeNode *root = args;
     ref<PgfConcrLincat> lincat = root->get_lincat(linearizer);
     if (lincat == 0)
-        return (lin_index = !lin_index);
+        return (rule_index = !rule_index);
 
-    while (lincat->n_lindefs+lin_index < lincat->res.size()) {
+    while (rule_index < lincat->rules.size()) {
-        // Unbind all variables
+        Item *item = new (lincat->rules[lincat->n_lindefs+rule_index]) Item();
-        for (size_t j = 0; j < var_count; j++) {
+
-            var_values[j] = (size_t) -1;
+        if (!item->instantiate(item->rule->args[0], root->value)) {
+            rule_index++;
+            continue;
         }
 
-        ref<PgfPResult> pres = lincat->res[lincat->n_lindefs+lin_index];
+        size_t max_value = 1;
-        ref<PgfPArg> parg = lincat->args.elem(lincat->n_lindefs+lin_index);
+        for (size_t i = 0; i < item->vars.size(); i++) {
+            if (item->vars[i] == 0)
+                max_value *= item->rule->vars[i].range;
+        }
 
-        if (root->value < parg->param->i0)
+        for (size_t value = 0; value < max_value; value++) {
-            break;
+            size_t v = value;
-
+            for (size_t i = 0; i < item->vars.size(); i++) {
-        size_t value = root->value - parg->param->i0;
+                if (item->vars[i] == 0) {
-        for (size_t j = 0; j < parg->param->n_terms; j++) {
+                    size_t range = item->rule->vars[i].range;
-            size_t factor    = parg->param->terms[j].factor;
+                    item->vars[i] = v % range;
-            size_t var       = parg->param->terms[j].var;
+                    v = v / range;
-            size_t var_value;
-
-            if (var < var_count && var_values[var] != (size_t) -1) {
-                // The variable already has a value
-                var_value = var_values[var];
-            } else {
-                // The variable is not assigned yet
-                var_value = value / factor;
-
-                // find the range for the variable
-                size_t range = 0;
-                for (size_t k = 0; k < pres->vars.size(); k++) {
-                    ref<PgfVariableRange> var_range = pres->vars.elem(k);
-                    if (var_range->var == var) {
-                        range = var_range->range;
-                        break;
-                    }
                 }
-                if (range == 0)
-                    throw pgf_error("Unknown variable in resolving a linearization");
-
-                if (var_value >= range)
-                    break;
-
-                // Assign the variable;
-                if (var >= var_count) {
-                    var_values = (size_t*)
-                        realloc(var_values, (var+1)*sizeof(size_t));
-                    while (var_count < var) {
-                        var_values[var_count++] = (size_t) -1;
-                    }
-                    var_count++;
-                }
-                var_values[var] = var_value;
             }
 
-            value -= var_value * factor;
+            this->item = new (item) Item;
+            this->value = item->eval(this->item->rule->res);
         }
+        delete item;
 
-        lin_index++;
+        break;
-        if (value == 0) {
-            value = eval_param(&pres->param);
-            return true;
-        }
     }
 
-    lin_index = 0;
+    if (item == NULL) {
-    return false;
+        rule_index = 0;
+        return false;
+    }
+
+    return true;
 }
 
 void PgfLinearizer::TreeLinrefNode::linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex)
 {
     ref<PgfConcrLincat> lincat = args->get_lincat(linearizer);
     if (lincat != 0) {
-        size_t i = lincat->n_lindefs*lincat->fields.size() + (lin_index-1);
+        linearize_item(out, linearizer, item);
-        ref<PgfSequence> seq = lincat->seqs[i];
-        linearize_seq(out, linearizer, seq);
     } else {
         args->linearize(out, linearizer, lindex);
     }
@@ -526,6 +558,11 @@ ref<PgfConcrLincat> PgfLinearizer::TreeLinrefNode::get_lincat(PgfLinearizer *lin
     return 0;
 }
 
+PgfLinearizer::TreeLinrefNode::~TreeLinrefNode()
+{
+    delete item;
+}
+
 PgfLinearizer::TreeLitNode::TreeLitNode(PgfLinearizer *linearizer, ref<PgfConcrLincat> lincat, PgfText *lit)
   : TreeNode(linearizer)
 {
@@ -663,14 +700,14 @@ void PgfLinearizer::flush_pre_stack(PgfLinearizationOutputIface *out, PgfText *t
                 ref<PgfAlternative> alt = pre->sym_kp->alts.elem(i);
                 for (ref<PgfText> prefix : alt->prefixes) {
                     if (cmp(token, &(*prefix))) {
-                        pre->node->linearize_seq(out, this, alt->form);
+//                        pre->node->linearize_seq(out, this, alt->form);
                         goto done;
                     }
                 }
             }
         }
 
-        pre->node->linearize_seq(out, this, pre->sym_kp->default_form);
+//        pre->node->linearize_seq(out, this, pre->sym_kp->default_form);
 
     done:
         if (pre->bracket_stack != NULL)

src/runtime/c/pgf/linearizer.h

@@ -26,6 +26,49 @@ class PGF_INTERNAL_DECL PgfLinearizer : public PgfUnmarshaller {
     ref<PgfConcr> concr;
     PgfMarshaller *m;
 
+    struct Item {
+        ref<PgfConcrRule> rule;
+
+        struct {
+            size_t &operator[](int i) {
+                Item *item = containerof(Item,vars,this);
+                return ((size_t*) (item+1))[i];
+            }
+            size_t size() {
+                Item *item = containerof(Item,vars,this);
+                return item->rule->vars.size();
+            }
+        } vars;
+
+        void *operator new(size_t sz, ref<PgfConcrRule> rule)
+        {
+            size_t sz2 = rule->vars.size()*sizeof(size_t);
+            Item *new_item = (Item *) malloc(sz+sz2);
+            memset(new_item, 0, sz+sz2);
+            new_item->rule = rule;
+            return new_item;
+        }
+
+        void *operator new(size_t sz, Item *item)
+        {
+            size_t sz2 = item->vars.size()*sizeof(size_t);
+            Item *new_item = (Item *) malloc(sz+sz2);
+            memcpy(new_item, item, sz+sz2);
+            return new_item;
+        }
+
+        void operator delete(void *p)
+        {
+            free(p);
+        }
+        
+        Item() {
+        }
+
+        bool instantiate(ref<PgfLParam> lparam,size_t value);
+        size_t eval(ref<PgfLParam> lparam);
+    };
+
     struct TreeNode {
         TreeNode *next;
         TreeNode *next_arg;
@@ -34,8 +77,6 @@ class PGF_INTERNAL_DECL PgfLinearizer : public PgfUnmarshaller {
         int fid;
 
         size_t value;
-        size_t var_count;
-        size_t *var_values;
 
         size_t n_hoas_vars;
         PgfText **hoas_vars;
@@ -43,29 +84,31 @@ class PGF_INTERNAL_DECL PgfLinearizer : public PgfUnmarshaller {
         TreeNode(PgfLinearizer *linearizer);
         virtual bool resolve(PgfLinearizer *linearizer) { return true; };
         virtual void check_category(PgfLinearizer *linearizer, PgfText *cat)=0;
-        virtual void linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, PgfLParam *r);
+        virtual void linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, size_t r);
         virtual void linearize_var(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, size_t r);
-        virtual void linearize_seq(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, ref<PgfSequence> seq);
+        virtual void linearize_item(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, Item *item);
         virtual void linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex)=0;
-        size_t eval_param(PgfLParam *param);
         virtual ref<PgfConcrLincat> get_lincat(PgfLinearizer *linearizer)=0;
-        virtual ~TreeNode() { free(var_values); free(hoas_vars); };
+        virtual ~TreeNode() { free(hoas_vars); };
     };
 
     struct TreeLinNode : public TreeNode {
         ref<PgfConcrLin> lin;
-        size_t lin_index;
+        size_t rule_index;
+        Item **items;
 
         TreeLinNode(PgfLinearizer *linearizer, ref<PgfConcrLin> lin);
         virtual bool resolve(PgfLinearizer *linearizer);
         virtual void check_category(PgfLinearizer *linearizer, PgfText *cat);
         virtual void linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex);
         virtual ref<PgfConcrLincat> get_lincat(PgfLinearizer *linearizer);
+        virtual ~TreeLinNode();
     };
 
     struct TreeLindefNode : public TreeNode {
         ref<PgfConcrLincat> lincat;
-        size_t lin_index;
+        size_t rule_index;
+        Item **items;
         PgfText *fun;
         PgfText *literal;
 
@@ -75,17 +118,19 @@ class PGF_INTERNAL_DECL PgfLinearizer : public PgfUnmarshaller {
         virtual void linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, PgfLParam *r);
         virtual void linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex);
         virtual ref<PgfConcrLincat> get_lincat(PgfLinearizer *linearizer);
-        ~TreeLindefNode() { free(fun); free(literal); };
+        ~TreeLindefNode();
     };
 
     struct TreeLinrefNode : public TreeNode {
-        size_t lin_index;
+        size_t rule_index;
+        Item *item;
 
         TreeLinrefNode(PgfLinearizer *linearizer, TreeNode *root);
         virtual bool resolve(PgfLinearizer *linearizer);
         virtual void check_category(PgfLinearizer *linearizer, PgfText *cat) {};
         virtual void linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex);
         virtual ref<PgfConcrLincat> get_lincat(PgfLinearizer *linearizer);
+        ~TreeLinrefNode();
     };
 
     struct TreeLitNode : public TreeNode {

src/runtime/c/pgf/parser.h

@@ -1,191 +1,294 @@
 #ifndef LR_TABLE_H
 #define LR_TABLE_H
 
-#include "md5.h"
-
-class PGF_INTERNAL_DECL PgfLRTableMaker
-{
-    struct CCat;
-    struct Production;
-    struct Item;
-    struct State;
-
-    struct CompareItem;
-    static const CompareItem compare_item;
-
-    typedef std::pair<ref<PgfText>,size_t> Key0;
-
-    struct PGF_INTERNAL_DECL CompareKey0 : std::less<Key0> {
-        bool operator() (const Key0& k1, const Key0& k2) const {
-            int cmp = textcmp(k1.first,k2.first);
-            if (cmp < 0)
-                return true;
-            else if (cmp > 0)
-                return false;
-
-            return (k1.second < k2.second);
-        }
-    };
-
-    typedef std::pair<ref<PgfConcrLincat>,size_t> Key1;
-
-    struct PGF_INTERNAL_DECL CompareKey1 : std::less<Key1> {
-        bool operator() (const Key1& k1, const Key1& k2) const {
-            if (k1.first < k2.first)
-                return true;
-            else if (k1.first > k2.first)
-                return false;
-
-            return (k1.second < k2.second);
-        }
-    };
-
-    typedef std::pair<CCat*,size_t> Key2;
-
-    struct PGF_INTERNAL_DECL CompareKey2 : std::less<Key2> {
-        bool operator() (const Key2& k1, const Key2& k2) const {
-            if (k1.first < k2.first)
-                return true;
-            else if (k1.first > k2.first)
-                return false;
-
-            return (k1.second < k2.second);
-        }
-    };
-
-    typedef std::pair<ref<PgfSequence>,size_t> Key3;
-
-    struct PGF_INTERNAL_DECL CompareKey3 : std::less<Key3> {
-        bool operator() (const Key3& k1, const Key3& k2) const;
-    };
-
-    ref<PgfAbstr> abstr;
-    ref<PgfConcr> concr;
-
-    size_t ccat_id;
-    size_t state_id;
-
-    std::queue<State*> todo;
-    std::map<MD5Digest,State*> states;
-    std::map<Key0,CCat*,CompareKey0> ccats1;
-    std::map<Key2,CCat*,CompareKey2> ccats2;
-
-    // The Threefold Way of building an automaton
-    typedef enum { INIT, PROBE, REPEAT } Fold;
-
-    void process(State *state, Fold fold, Item *item);
-    void symbol(State *state, Fold fold, Item *item, PgfSymbol sym);
-
-    template<class T>
-    void predict(State *state, Fold fold, Item *item, T cat,
-                 vector<PgfVariableRange> vars, PgfLParam *r);
-    void predict(State *state, Fold fold, Item *item, ref<PgfText> cat, size_t lin_idx);
-    void predict(State *state, Fold fold, Item *item, CCat *ccat, size_t lin_idx);
-    void predict(ref<PgfAbsFun> absfun, CCat *ccat);
-    void complete(State *state, Fold fold, Item *item);
-
-    void print_production(CCat *ccat, Production *prod);
-    void print_item(Item *item);
-
-    void internalize_state(State *&state);
-
-public:
-    PgfLRTableMaker(ref<PgfAbstr> abstr, ref<PgfConcr> concr);
-    vector<PgfLRState> make();
-    ~PgfLRTableMaker();
-};
-
-class PGF_INTERNAL_DECL PgfLCTableMaker
-{
-    ref<PgfAbstr> abstr;
-    ref<PgfConcr> concr;
-
-
-    std::map<ref<PgfConcrLincat>,std::vector<ref<PgfLCEdge>>> forwards;
-    std::map<ref<PgfConcrLincat>,std::vector<ref<PgfLCEdge>>> backwards;
-
-    ref<PgfLCEdge> compute_unifier(ref<PgfLCEdge> edge1, ref<PgfLCEdge> edge2);
-    void update_closure(ref<PgfLCEdge> edge);
-    void rename(ref<PgfLCEdge> edge);
-    void add_edge(ref<PgfLCEdge> edge);
-    void print_edge(ref<PgfLCEdge> edge);
-
-public:
-    PgfLCTableMaker(ref<PgfAbstr> abstr, ref<PgfConcr> concr);
-    vector<PgfLRState> make();
-    ~PgfLCTableMaker();
-};
-
 class PgfPrinter;
 
-class PGF_INTERNAL_DECL PgfParser : public PgfPhraseScanner, public PgfExprEnum
+class PGF_INTERNAL_DECL PgfAbstractParser
 {
-    ref<PgfConcr> concr;
+    typedef size_t hash_t;
-    PgfText *sentence;
-    bool case_sensitive;
-    PgfMarshaller *m;
-    PgfUnmarshaller *u;
 
-    struct Choice;
+protected:
-    struct Production;
+    ref<PgfConcr> concr;
-    struct StackNode;
+
-    struct Stage;
+    struct CCat;
+    struct Cont;
+    struct Item;
+    struct State;
     struct ExprState;
-    struct ExprInstance;
+
-    struct CompareExprState : std::less<ExprState*> {
+    struct Production {
-        bool operator() (const ExprState *state1, const ExprState *state2) const;
+        ref<PgfConcrRule> rule;
+
+        struct {
+            size_t &operator[](int i) {
+                Production *prod = containerof(Production,vars,this);
+                return ((size_t*) (((CCat**) (prod+1))+prod->args.size()))[i];
+            }
+            size_t size() {
+                Production *prod = containerof(Production,vars,this);
+                return prod->rule->vars.size();
+            }
+        } vars;
+
+        struct {
+            CCat *&operator[](int i) {
+                Production *prod = containerof(Production,args,this);
+                return ((CCat**) (prod+1))[i];
+            }
+            size_t size() {
+                Production *prod = containerof(Production,args,this);
+                return (prod->rule->args != 0) ? prod->rule->args.size() : 0;
+            }
+        } args;
+
+        void *operator new(size_t sz, Item *item)
+        {
+            size_t sz2 = item->args.size()*sizeof(CCat*)
+                       + item->vars.size()*sizeof(size_t);
+            Production *prod = (Production *) malloc(sz+sz2);
+            memcpy(prod+1, item+1, sz2);
+            return prod;
+        }
+
+        void *operator new(size_t sz, ref<PgfItem> pitem)
+        {
+            size_t sz2 = pitem->args.size()*sizeof(CCat*)
+                       + pitem->vars.size()*sizeof(size_t);
+            Production *prod = (Production *) malloc(sz+sz2);
+            memset(prod+1,0,sz2);
+            return prod;
+        }
+
+        void operator delete(void *p)
+        {
+            free(p);
+        }
+
+        Production() {
+        }
     };
 
-    Stage *before, *after, *ahead;
+    struct ExprProb {
-    std::priority_queue<ExprState*, std::vector<ExprState*>, CompareExprState> queue;
+        PgfExpr expr;
-    int last_fid;
+        prob_t prob;
+        hash_t hash;
         
-    std::vector<Choice*> dynamic;
+        ExprProb(PgfExpr expr, prob_t prob, hash_t hash) {
-    std::map<object,Choice*> persistant;
+            this->expr = expr;
+            this->prob = prob;
+            this->hash = hash;
+        }
+    };
 
-    std::vector<PgfExpr> exprs;
+    struct CCat {
+        PgfMetaId fid;
+        Cont *cont;
+        State *state;
+        size_t value;
+        size_t lin_idx;
+        bool covered;
+        std::vector<Production*> prods;
+        std::vector<ExprState*> pending;
+        std::vector<ExprProb> exprs;
 
-    Choice *top_choice;
+        ~CCat();
-    size_t top_choice_index;
+    };
 
-    bool shift(StackNode *parent, ref<PgfConcrLincat> lincat, size_t r, Production *prod,
+    struct State {
-               Stage *before, Stage *after);
+        PgfTextSpot start, end;
-    void shift(StackNode *parent, Stage *before);
+        bool needs_bind;
-    void shift(StackNode *parent, Stage *before, Stage *after);
+        std::map<ref<PgfConcrLincat>,Cont*> conts1;
-    void reduce(StackNode *parent, ref<PgfConcrLin> lin, ref<PgfLRReduce> red,
+        std::map<CCat*,std::map<size_t,Cont*>> conts2;
-                size_t n, std::vector<Choice*> &args,
+        std::map<Cont*,std::map<size_t,std::map<size_t,CCat*>>> completed;
-                Stage *before, Stage *after);
+        State *next;
-    Choice *retrieve_choice(ref<PgfLRReduceArg> arg);
+    };
-    void complete(StackNode *parent, ref<PgfConcrLincat> lincat, size_t r,
-                  size_t n, std::vector<Choice*> &args);
-    void reduce_all(StackNode *state);
-    void print_prod(Choice *choice, Production *prod);
-    void print_transition(StackNode *source, StackNode *target, Stage *stage, ref<PgfLRShiftKS> shift);
 
-    typedef std::map<std::pair<Choice*,Choice*>,Choice*> intersection_map;
+    struct Cont {
+        CCat *ccat;
+        ref<PgfConcrLincat> lincat;
+        State *state;
+        std::vector<Item*> suspended;
 
-    Choice *intersect_choice(Choice *choice1, Choice *choice2, intersection_map &im);
+        ~Cont();
+    };
 
-    void print_expr_state_before(PgfPrinter *printer, ExprState *state);
+    struct Item {
-    void print_expr_state_after(PgfPrinter *printer, ExprState *state);
+        Cont *cont;
-    void print_expr_state(ExprState *state);
+        uint16_t pre_alt;
+        uint16_t pre_dot;
+        uint16_t dot;
+        vector<PgfSymbol> syms;
+        ref<PgfConcrRule> rule;
 
-    void predict_expr_states(Choice *choice, prob_t outside_prob);
+        struct {
-    bool process_expr_state(ExprState *state);
+            size_t &operator[](int i) {
-    void complete_expr_state(ExprState *state);
+                Item *item = containerof(Item,vars,this);
-    void combine_expr_state(ExprState *state, ExprInstance &inst);
+                return ((size_t*) (((CCat**) (item+1))+item->args.size()))[i];
+            }
+            size_t size() {
+                Item *item = containerof(Item,vars,this);
+                return item->rule->vars.size();
+            }
+        } vars;
+
+        struct {
+            CCat *&operator[](int i) {
+                Item *item = containerof(Item,args,this);
+                return ((CCat**) (item+1))[i];
+            }
+            size_t size() {
+                Item *item = containerof(Item,args,this);
+                return (item->rule->args != 0) ? item->rule->args.size() : 0;
+            }
+        } args;
+
+        void *operator new(size_t sz, ref<PgfConcrRule> rule)
+        {
+            size_t sz2 = rule->args.size()*sizeof(CCat*)
+                       + rule->vars.size()*sizeof(size_t);
+            Item *new_item = (Item *) malloc(sz+sz2);
+            memset(new_item+1, 0, sz2);
+            return new_item;
+        }
+
+        void *operator new(size_t sz, Item *item)
+        {
+            size_t sz2 = item->args.size()*sizeof(CCat*)
+                       + item->vars.size()*sizeof(size_t);
+            Item *new_item = (Item *) malloc(sz+sz2);
+            memcpy(new_item, item, sz+sz2);
+            return new_item;
+        }
+
+        void operator delete(void *p)
+        {
+            free(p);
+        }
+        
+        Item() {
+        }
+
+        bool instantiate(ref<PgfLParam> lparam,size_t value);
+        bool instantiate(ref<PgfLParam> lparam,ref<PgfLParam> value,Item *other);
+    };
+
+    struct ExprState {
+        PgfExpr expr;
+        prob_t prob;
+        hash_t hash;
+
+        CCat *res;
+
+        size_t index;
+        size_t n_args;
+        CCat *args[];
+
+        void *operator new(size_t sz, size_t n_args)
+        {
+            ExprState *estate = (ExprState *)
+                malloc(sz+n_args*sizeof(CCat*));
+            return estate;
+        }
+        
+        void operator delete(void *p)
+        {
+            free(p);
+        }
+
+        ExprState() {
+        }
+    };
+
+    State *first_state, *current_state;
+    PgfMetaId last_fid;
+
+    void process(Item *item, const PgfTextSpot &spot, bool bind);
+    void symbol(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym);
+    void complete(Item *item, const PgfTextSpot &spot, bool bind);
+
+    virtual State *new_state(const PgfTextSpot &start)=0;
+    virtual void symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym)=0;
+    virtual void symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym)=0;
+    virtual void suspend(State *state,ref<PgfConcrLincat> lincat, Item *item)=0;
+    virtual void final_item(State *state,Item *item,size_t value,size_t lin_idx)=0;
+
+    virtual void bu_predict(PgfPhrasetable phrasetable, State *state, CCat *ccat);
+    Item *bu_item(State *state, ref<PgfItem> pitem);
+    CCat *td_epsilon(State *state, Cont *cont, ref<PgfSymbolCCat> arg);
+    CCat *td_epsilon(State *state, Cont *cont, ref<PgfSymbolCCat> arg,
+                     size_t n_items, vector<ref<PgfItem>> items);
+    void td_predict(State *state, Cont *cont, Production *prod, size_t lin_idx);
+    void combine(State *state, Item *item, CCat *ccat);
+
+    static
+    void print_item(Item *item, const PgfTextSpot &spot);
+
+    static
+    void print_prod(CCat *ccat, Production *prod);
 
 public:
-    PgfParser(ref<PgfConcr> concr, ref<PgfConcrLincat> start, PgfText *sentence, bool case_sensitive, PgfMarshaller *m, PgfUnmarshaller *u);
+    PgfAbstractParser(ref<PgfConcr> concr);
+    virtual ~PgfAbstractParser();
+};
+
+class PGF_INTERNAL_DECL PgfParser : private PgfAbstractParser, public PgfExprEnum
+{
+    PgfMarshaller *m;
+    PgfUnmarshaller *u;
+    PgfText *sentence;
+    uint8_t *end;
+    bool case_sensitive;
+
+    virtual State *new_state(const PgfTextSpot &start);
+    virtual void symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym);
+    virtual void symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym);
+    virtual void suspend(State *state,ref<PgfConcrLincat> lincat, Item *item);
+    virtual void final_item(State *state,Item *item,size_t value,size_t lin_idx);
+
+    void bu_predict(PgfPhrasetable phrasetable, State *state, ptrdiff_t min, ptrdiff_t max);
+    void make_chunks(State *state, std::vector<CCat*> &chunks, prob_t prob);
+    PgfExpr process_expr(ExprState *estate, prob_t *prob);
+
+    static
+    void print_expr_state_left(PgfPrinter *printer, PgfMarshaller *m, ExprState *estate);
+    static
+    void print_expr_state_right(PgfPrinter *printer, PgfMarshaller *m, ExprState *estate);
+    static
+    void print_expr_state(PgfMarshaller *m, ExprState *estate);
+
+    struct ExprStateComparator : std::less<ExprState*> {
+        bool operator()(ExprState *estate1, ExprState *estate2) {
+            return estate1->prob > estate2->prob;
+        }
+    } estate_comp;
+
+    std::vector<ExprState*> queue;
+
+public:
+    PgfParser(ref<PgfConcr> concr, PgfText *sentence, bool case_sensitive, PgfMarshaller *m, PgfUnmarshaller *u);
     virtual ~PgfParser();
 
-	virtual void space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err);
+    void prepare(ref<PgfConcrLincat> start);
-    virtual void start_matches(PgfTextSpot *end, PgfExn* err);
-    virtual void match(ref<PgfConcrLin> lin, size_t seq_index, PgfExn* err);
-	virtual void end_matches(PgfTextSpot *end, PgfExn* err);
-
-    void prepare();
 
     PgfExpr fetch(PgfDB *db, prob_t *prob);
 };
+
+class PGF_INTERNAL_DECL PgfParseTableMaker : private PgfAbstractParser
+{
+private:
+    virtual State *new_state(const PgfTextSpot &start);
+    virtual void symbol_token(Item *item, const PgfTextSpot &spot, bool bind, PgfSymbol sym);
+    virtual void symbol_bind(Item *item, const PgfTextSpot &spot, PgfSymbol sym);
+    virtual void suspend(State *state,ref<PgfConcrLincat> lincat,Item *item);
+    virtual void final_item(State *state,Item *item,size_t value,size_t lin_idx);
+    virtual void bu_predict(PgfPhrasetable phrasetable, State *state, CCat *ccat);
+
+    static
+    ref<PgfItem> clone_item(Item *item);
+
+public:
+    PgfParseTableMaker(ref<PgfConcr> concr);
+    void insert_rule(ref<PgfConcrRule> rule);
+};
+
 #endif

src/runtime/c/pgf/pgf.h

@@ -461,8 +461,6 @@ PGF_API_DECL
 void pgf_iter_lins(PgfDB *db, PgfConcrRevision cnc_revision,
                    PgfItor *itor, PgfExn *err);
 
-typedef struct PgfPhrasetableIds PgfPhrasetableIds;
-
 typedef struct PgfSequenceItor PgfSequenceItor;
 struct PgfSequenceItor {
 	int (*fn)(PgfSequenceItor* self, size_t seq_id, object value,
@@ -493,10 +491,10 @@ void pgf_lookup_cohorts(PgfDB *db, PgfConcrRevision cnc_revision,
                         PgfCohortsCallback* callback, PgfExn* err);
 
 PGF_API_DECL
-PgfPhrasetableIds *pgf_iter_sequences(PgfDB *db, PgfConcrRevision cnc_revision,
+void pgf_iter_sequences(PgfDB *db, PgfConcrRevision cnc_revision,
-                                      PgfSequenceItor *itor,
+                        PgfSequenceItor *itor,
-                                      PgfMorphoCallback *callback,
+                        PgfMorphoCallback *callback,
-                                      PgfExn *err);
+                        PgfExn *err);
 
 PGF_API_DECL
 void pgf_get_lincat_counts_internal(object o, size_t *counts);
@@ -505,26 +503,20 @@ PGF_API_DECL
 PgfText *pgf_get_lincat_field_internal(object o, size_t i);
 
 PGF_API_DECL
-size_t pgf_get_lin_get_prod_count(object o);
+size_t pgf_get_lin_rules_count(object o);
 
 PGF_API_DECL
-PgfText *pgf_print_lindef_internal(PgfPhrasetableIds *seq_ids, object o, size_t i);
+PgfText *pgf_print_lindef_internal(object o, size_t i);
 
 PGF_API_DECL
-PgfText *pgf_print_linref_internal(PgfPhrasetableIds *seq_ids, object o, size_t i);
+PgfText *pgf_print_linref_internal(object o, size_t i);
 
 PGF_API_DECL
-PgfText *pgf_print_lin_internal(PgfPhrasetableIds *seq_ids, object o, size_t i);
+PgfText *pgf_print_lin_internal(object o, size_t i);
-
-PGF_API_DECL
-PgfText *pgf_print_sequence_internal(size_t seq_id, object o);
 
 PGF_API_DECL
 PgfText *pgf_sequence_get_text_internal(object o);
 
-PGF_API_DECL
-void pgf_release_phrasetable_ids(PgfPhrasetableIds *seq_ids);
-
 PGF_API_DECL
 PgfExpr pgf_check_expr(PgfDB *db, PgfRevision revision,
                        PgfExpr e, PgfType ty,
@@ -635,11 +627,11 @@ void pgf_drop_concrete(PgfDB *db, PgfRevision revision,
 
 #ifdef __cplusplus
 struct PgfLinBuilderIface {
-    virtual void start_production(PgfExn *err)=0;
+    virtual void start_rule(size_t n_vars, size_t n_syms, PgfExn *err)=0;
-    virtual void add_argument(size_t n_hypos, size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0;
+    virtual void add_argument(size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0;
-    virtual void set_result(size_t n_vars, size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0;
+    virtual void set_result(size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0;
+    virtual void set_lin_idx(size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0;
     virtual void add_variable(size_t var, size_t range, PgfExn *err)=0;
-    virtual void start_sequence(size_t n_syms, PgfExn *err)=0;
     virtual void add_symcat(size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0;
     virtual void add_symlit(size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0;
     virtual void add_symvar(size_t d, size_t r, PgfExn *err)=0;
@@ -654,9 +646,7 @@ struct PgfLinBuilderIface {
     virtual void add_symsoftspace(PgfExn *err)=0;
     virtual void add_symcapit(PgfExn *err)=0;
     virtual void add_symallcapit(PgfExn *err)=0;
-    virtual object end_sequence(PgfExn *err)=0;
+    virtual void end_rule(PgfExn *err)=0;
-    virtual void add_sequence_id(object seq_id, PgfExn *err)=0;
-    virtual void end_production(PgfExn *err)=0;
 };
 
 struct PgfBuildLinIface {
@@ -666,11 +656,11 @@ struct PgfBuildLinIface {
 typedef struct PgfLinBuilderIface PgfLinBuilderIface;
 
 typedef struct {
-    void (*start_production)(PgfLinBuilderIface *this, PgfExn *err);
+    void (*start_rule)(PgfLinBuilderIface *this, size_t n_vars, size_t n_syms, PgfExn *err);
-    void (*add_argument)(PgfLinBuilderIface *this, size_t n_hypos, size_t i0, size_t n_terms, size_t *terms, PgfExn *err);
+    void (*add_argument)(PgfLinBuilderIface *this, size_t i0, size_t n_terms, size_t *terms, PgfExn *err);
-    void (*set_result)(PgfLinBuilderIface *this, size_t n_vars, size_t i0, size_t n_terms, size_t *terms, PgfExn *err);
+    void (*set_result)(PgfLinBuilderIface *this, size_t i0, size_t n_terms, size_t *terms, PgfExn *err);
+    void (*set_lin_idx)(PgfLinBuilderIface *this, size_t i0, size_t n_terms, size_t *terms, PgfExn *err);
     void (*add_variable)(PgfLinBuilderIface *this, size_t var, size_t range, PgfExn *err);
-    void (*start_sequence)(PgfLinBuilderIface *this, size_t n_syms, PgfExn *err);
     void (*add_symcat)(PgfLinBuilderIface *this, size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err);
     void (*add_symlit)(PgfLinBuilderIface *this, size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err);
     void (*add_symvar)(PgfLinBuilderIface *this, size_t d, size_t r, PgfExn *err);
@@ -685,9 +675,7 @@ typedef struct {
     void (*add_symsoftspace)(PgfLinBuilderIface *this, PgfExn *err);
     void (*add_symcapit)(PgfLinBuilderIface *this, PgfExn *err);
     void (*add_symallcapit)(PgfLinBuilderIface *this, PgfExn *err);
-    object (*end_sequence)(PgfLinBuilderIface *this, PgfExn *err);
+    void (*end_rule)(PgfLinBuilderIface *this, PgfExn *err);
-    void (*add_sequence_id)(PgfLinBuilderIface *this, object seq_id, PgfExn *err);
-    void (*end_production)(PgfLinBuilderIface *this, PgfExn *err);
 } PgfLinBuilderIfaceVtbl;
 
 struct PgfLinBuilderIface {
@@ -720,10 +708,17 @@ void pgf_drop_lincat(PgfDB *db,
 PGF_API_DECL
 void pgf_create_lin(PgfDB *db,
                     PgfRevision revision, PgfConcrRevision cnc_revision,
-                    PgfText *name, size_t n_prods,
+                    PgfText *name, size_t n_rules,
                     PgfBuildLinIface *build,
                     PgfExn *err);
 
+PGF_API_DECL
+void pgf_alter_lin(PgfDB *db,
+                   PgfRevision revision, PgfConcrRevision cnc_revision,
+                   PgfText *name, size_t n_rules,
+                   PgfBuildLinIface *build,
+                   PgfExn *err);
+
 PGF_API_DECL
 void pgf_drop_lin(PgfDB *db,
                   PgfRevision revision, PgfConcrRevision cnc_revision,
@@ -916,8 +911,4 @@ pgf_align_words(PgfDB *db, PgfConcrRevision revision,
                 size_t *n_phrases /* out */,
                 PgfExn* err);
 
-PGF_API PgfText *
-pgf_graphviz_lr_automaton(PgfDB *db, PgfConcrRevision revision,
-                          PgfExn *err);
-
 #endif // PGF_H_

src/runtime/c/pgf/phrasetable.h

@@ -1,138 +1,122 @@
 #ifndef PHRASETABLE_H
 #define PHRASETABLE_H
 
-struct PgfSequence;
-struct PgfSequenceBackref;
-
-struct PGF_INTERNAL_DECL PgfPhrasetableEntry {
-    ref<PgfSequence> seq;
-
-    // Here n_backrefs tells us how many actual backrefs there are in
-    // the vector backrefs. On the other hand, backrefs->len tells us
-    // how big buffer we have allocated.
-    size_t n_backrefs;
-    vector<PgfSequenceBackref> backrefs;
-};
-
-struct PgfSequenceItor;
-typedef ref<Node<PgfPhrasetableEntry>> PgfPhrasetable;
-
-
-#if __GNUC__
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wattributes"
-#endif
-
-struct PgfPhrasetableIds {
-public:
-	PGF_INTERNAL_DECL PgfPhrasetableIds();
-	PGF_INTERNAL_DECL ~PgfPhrasetableIds() { end(); }
-
-	PGF_INTERNAL_DECL void start(ref<PgfConcr> concr);
-	PGF_INTERNAL_DECL size_t add(ref<PgfSequence> seq);
-	PGF_INTERNAL_DECL size_t get(ref<PgfSequence> seq);
-	PGF_INTERNAL_DECL void end();
-
-private:
-	size_t next_id;
-
-	struct PGF_INTERNAL_DECL SeqIdChain;
-
-	struct PGF_INTERNAL_DECL SeqIdPair {
-		SeqIdChain *chain;
-		ref<PgfSequence> seq;
-		size_t seq_id;
-	};
-
-	struct PGF_INTERNAL_DECL SeqIdChain : public SeqIdPair {
-		SeqIdChain *next;
-	};
-
-	size_t n_pairs;
-	SeqIdPair  *pairs;
-	SeqIdChain *chains;
-};
-
-#if __GNUC__
-#pragma GCC diagnostic pop
-#endif
-
-struct PgfConcrLincat;
-
-PGF_INTERNAL_DECL
-PgfPhrasetable phrasetable_internalize(PgfPhrasetable table,
-                                       ref<PgfSequence> seq,
-                                       ref<PgfConcrLincat> lincat,
-                                       object container,
-                                       size_t seq_index,
-                                       ref<PgfPhrasetableEntry> *pentry);
-
-PGF_INTERNAL_DECL
-ref<PgfSequence> phrasetable_relink(PgfPhrasetable table,
-                                    object container,
-                                    size_t seq_index,
-                                    size_t seq_id);
-
-PGF_INTERNAL_DECL
-PgfPhrasetable phrasetable_delete(PgfPhrasetable table,
-                                  object container,
-                                  size_t seq_index,
-                                  ref<PgfSequence> seq);
-
-PGF_INTERNAL_DECL
-size_t phrasetable_size(PgfPhrasetable table);
-
 struct PgfConcrLin;
+struct PgfConcrLincat;
 
 struct PGF_INTERNAL_DECL PgfTextSpot {
 	size_t pos;          // position in Unicode characters
 	const uint8_t *ptr;  // pointer into the spot location
 };
 
+struct PGF_INTERNAL_DECL PgfItem {
+    struct {
+        size_t &operator[](int i) {
+            PgfItem *item = containerof(PgfItem,vars,this);
+            return ((size_t*) (((ref<PgfSymbolCCat>*) (item+1))+item->rule->args.size()))[i];
+        }
+        size_t size() {
+            PgfItem *item = containerof(PgfItem,vars,this);
+            return (item->rule->vars != 0) ? item->rule->vars.size() : 0;
+        }
+    } vars;
+
+    struct {
+        ref<PgfSymbolCCat> &operator[](int i) {
+            PgfItem *item = containerof(PgfItem,args,this);
+            return ((ref<PgfSymbolCCat>*) (item+1))[i];
+        }
+        size_t size() {
+            PgfItem *item = containerof(PgfItem,args,this);
+            return item->rule->args.size();
+        }
+    } args;
+
+    uint16_t pre_alt;
+    uint16_t pre_dot;
+    uint16_t dot;
+    ref<PgfConcrRule> rule;
+};
+
+struct PgfPhrasetableNode;
+typedef ref<PgfPhrasetableNode> PgfPhrasetable;
+
+struct PGF_INTERNAL_DECL PgfPhrasetableNode {
+    const static size_t DELTA = 3;
+    const static size_t RATIO = 2;
+
+public:
+    PgfSymbol sym;
+
+    // Here n_items tells us how many actual items there are in
+    // the vector items. On the other hand, items.size() tells us
+    // how big buffer we have allocated.
+    size_t n_items;
+    vector<ref<PgfItem>> items;
+
+    txn_t txn_id;
+
+    size_t sz;
+    ref<PgfPhrasetableNode> left;
+    ref<PgfPhrasetableNode> right;
+
+    static
+    ref<PgfPhrasetableNode> new_node(PgfSymbol sym, size_t n_items);
+
+    static
+    ref<PgfPhrasetableNode> upd_node(ref<PgfPhrasetableNode> node, ref<PgfPhrasetableNode> left, ref<PgfPhrasetableNode> right);
+
+    static
+    ref<PgfPhrasetableNode> balanceL(ref<PgfPhrasetableNode> node);
+
+    static
+    ref<PgfPhrasetableNode> balanceR(ref<PgfPhrasetableNode> node);
+
+    static
+    size_t size(ref<PgfPhrasetableNode> node)
+    {
+        if (node == 0)
+            return 0;
+        return node->sz;
+    }
+
+    static
+    void release(ref<PgfPhrasetableNode> node);
+};
+
+PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
+                                  PgfSymbol sym,
+                                  ref<PgfItem> item);
+
+PgfPhrasetable phrasetable_insert(PgfPhrasetable table,
+                                  ref<PgfConcrLincat> lincat,
+                                  size_t value, size_t lin_idx,
+                                  ref<PgfItem> item);
+
+PGF_INTERNAL_DECL
+void phrasetable_iter(PgfPhrasetable phrasetable,ref<PgfConcrLincat> lincat,std::function<void(ref<PgfSymbolCCat> symcf,size_t,vector<ref<PgfItem>>)> &f);
+
+PGF_INTERNAL_DECL
+vector<ref<PgfItem>> phrasetable_lookup(PgfPhrasetable phrasetable, PgfSymbol sym, size_t *n_items);
+
 class PGF_INTERNAL_DECL PgfPhraseScanner {
 public:
     virtual void space(PgfTextSpot *start, PgfTextSpot *end, PgfExn* err)=0;
     virtual void start_matches(PgfTextSpot *spot, PgfExn* err)=0;
-    virtual void match(ref<PgfConcrLin> lin, size_t seq_index, PgfExn* err)=0;
+    virtual void match(ref<PgfConcrLin> lin, size_t lin_idx, PgfExn* err)=0;
     virtual void end_matches(PgfTextSpot *spot, PgfExn* err)=0;
 };
 
 PGF_INTERNAL_DECL
-void phrasetable_lookup(PgfPhrasetable table,
+void phrasetable_lookup(PgfPhrasetable phrasetable,
                         PgfText *sentence,
                         bool case_sensitive,
                         PgfPhraseScanner *scanner, PgfExn* err);
 
 PGF_INTERNAL_DECL
-void phrasetable_lookup_cohorts(PgfPhrasetable table,
+void phrasetable_lookup_cohorts(PgfPhrasetable phrasetable,
                                 PgfText *sentence,
                                 bool case_sensitive,
                                 PgfPhraseScanner *scanner, PgfExn* err);
 
-PGF_INTERNAL_DECL
-void phrasetable_iter(PgfConcr *concr,
-                      PgfPhrasetable table,
-                      PgfSequenceItor* itor,
-                      PgfMorphoCallback *callback,
-                      PgfPhrasetableIds *seq_ids, PgfExn *err);
-
-PGF_INTERNAL_DECL
-void phrasetable_release(PgfPhrasetable table);
-
-// The following are used internally in the parser
-
-enum SeqMatch { SM_FULL_MATCH, SM_PREFIX, SM_PARTIAL };
-
-PGF_INTERNAL_DECL
-int text_sequence_cmp(PgfTextSpot *spot, const uint8_t *end,
-                      ref<PgfSequence> seq, size_t *p_i,
-                      bool case_sensitive, SeqMatch sm);
-
-// The following is used internally in the grammar builder
-
-PGF_INTERNAL_DECL
-void phrasetable_add_backref(ref<PgfPhrasetableEntry> entry, txn_t txn_id,
-                             object container,
-                             size_t seq_index);
-
 #endif

src/runtime/c/pgf/printer.cxx

@@ -545,11 +545,11 @@ void PgfPrinter::symbol(PgfSymbol sym)
         auto sym_kp = ref<PgfSymbolKP>::untagged(sym);
         puts("pre {");
 
-        sequence(sym_kp->default_form);
+        symbols(sym_kp->default_form);
 
         for (size_t i = 0; i < sym_kp->alts.size(); i++) {
             puts("; ");
-            sequence(sym_kp->alts[i].form);
+            symbols(sym_kp->alts[i].form);
             puts(" /");
             for (size_t j = 0; j < sym_kp->alts[i].prefixes.size(); j++) {
                 puts(" ");
@@ -578,22 +578,100 @@ void PgfPrinter::symbol(PgfSymbol sym)
 	case PgfSymbolALLCAPIT::tag:
         puts("ALL_CAPIT");
         break;
+	case PgfSymbolACat::tag: {
+        auto symcf = ref<PgfSymbolACat>::untagged(sym);
+        efun(&symcf->name);
+        break;
+    }
+	case PgfSymbolCCat::tag: {
+        auto symcf = ref<PgfSymbolCCat>::untagged(sym);
+        efun(&symcf->lincat->name);
+        nprintf(64,"(%zu,%zu)",symcf->value,symcf->lin_idx);
+        break;
+    }
 	}
 }
 
-void PgfPrinter::sequence(ref<PgfSequence> seq)
+void PgfPrinter::symbols(vector<PgfSymbol> syms)
 {
-    for (size_t i = 0; i < seq->syms.size(); i++) {
+    for (size_t i = 0; i < syms.size(); i++) {
         if (i > 0)
             puts(" ");
 
-        symbol(seq->syms[i]);
+        symbol(syms[i]);
     }
 }
 
-void PgfPrinter::seq_id(PgfPhrasetableIds *seq_ids, ref<PgfSequence> seq)
+void PgfPrinter::item(ref<PgfItem> item)
 {
-	nprintf(5, "S%zu", seq_ids->get(seq));
+    switch (ref<PgfConcrLin>::get_tag(item->rule->container)) {
+    case PgfConcrLincat::tag: {
+        ref<PgfConcrLincat> lincat = ref<PgfConcrLincat>::untagged(item->rule->container);
+
+        if (item->rule->vars != 0) {
+            lvar_ranges(item->rule->vars, &item->vars[0]);
+            puts(" ");
+        }
+
+        puts("String(");
+        lparam(item->rule->res);
+        puts(") -> ");
+
+        efun(&lincat->name);
+        puts("[");
+        efun(&lincat->name);
+        puts("(");
+        lparam(item->rule->args[0]);
+        puts(")]; ");
+        break;
+    }
+    case PgfConcrLin::tag: {
+        ref<PgfConcrLin> lin = ref<PgfConcrLin>::untagged(item->rule->container);
+        ref<PgfDTyp> ty = lin->absfun->type;
+
+        if (item->rule->vars != 0) {
+            lvar_ranges(item->rule->vars, &item->vars[0]);
+            puts(" ");
+        }
+
+        efun(&ty->name);
+        puts("(");
+        lparam(item->rule->res);
+        puts(") -> ");
+
+        efun(&lin->name);
+        puts("[");
+        for (size_t i = 0; i < item->rule->args.size(); i++) {
+            if (i > 0)
+                puts(",");
+            if (item->args[i] == 0) {
+                efun(&ty->hypos.elem(i)->type->name);
+                puts("(");
+                lparam(item->rule->args[i]);
+                puts(")");
+            } else {
+                emeta(0);
+            }
+        }
+        puts("]; ");
+        break;
+    }
+    }
+
+    lparam(item->rule->lin_idx);
+    puts(" : ");
+
+    for (size_t i = 0; i < item->rule->syms.size(); i++) {
+        if (i > 0)
+            puts(" ");
+
+        if (item->pre_alt == 0 && item->dot == i)
+            puts(". ");
+        else if (item->pre_alt > 0 && item->pre_dot == i)
+            puts(". ");
+
+        symbol(item->rule->syms[i]);
+    }
 }
 
 void PgfPrinter::free_ref(object x)

src/runtime/c/pgf/printer.h

@@ -79,9 +79,9 @@ public:
     void lvar(size_t var);
     void lparam(ref<PgfLParam> lparam);
     void lvar_ranges(vector<PgfVariableRange> vars, size_t *values);
-    void seq_id(PgfPhrasetableIds *seq_ids, ref<PgfSequence> seq);
     void symbol(PgfSymbol sym);
-    void sequence(ref<PgfSequence> seq);
+    void symbols(vector<PgfSymbol> syms);
+    void item(ref<PgfItem> item);
 
     virtual PgfExpr eabs(PgfBindType btype, PgfText *name, PgfExpr body);
     virtual PgfExpr eapp(PgfExpr fun, PgfExpr arg);

src/runtime/c/pgf/reader.cxx

@@ -10,6 +10,7 @@ PgfReader::PgfReader(FILE *in,PgfProbsCallback *probs_callback)
     this->probs_callback = probs_callback;
     this->abstract = 0;
     this->concrete = 0;
+    this->container = 0;
 }
 
 uint8_t PgfReader::read_uint8()
@@ -161,6 +162,21 @@ ref<C> PgfReader::read_vector(inline_vector<V> C::* field, void (PgfReader::*rea
     return loc;
 }
 
+template <class V>
+vector<V> PgfReader::read_null_vector(void (PgfReader::*read_value)(ref<V> val))
+{
+    size_t len = read_len();
+    if (len == 0) {
+        return 0;
+    } else {
+        vector<V> vec = vector<V>::alloc(len);
+        for (size_t i = 0; i < len; i++) {
+            (this->*read_value)(vec.elem(i));
+        }
+        return vec;
+    }
+}
+
 template <class V>
 vector<V> PgfReader::read_vector(void (PgfReader::*read_value)(ref<V> val))
 {
@@ -572,14 +588,14 @@ PgfSymbol PgfReader::read_symbol()
         ref<PgfSymbolKP> sym_kp = inline_vector<PgfAlternative>::alloc(&PgfSymbolKP::alts,n_alts);
 
         for (size_t i = 0; i < n_alts; i++) {
-            auto form     = read_seq();
+            auto form     = read_vector(&PgfReader::read_symbol2);
             auto prefixes = read_vector(&PgfReader::read_text2);
 
             sym_kp->alts[i].form     = form;
             sym_kp->alts[i].prefixes = prefixes;
         }
 
-        auto default_form = read_seq();
+        auto default_form = read_vector(&PgfReader::read_symbol2);
         sym_kp->default_form = default_form;
 
         sym = sym_kp.tagged();
@@ -616,80 +632,50 @@ PgfSymbol PgfReader::read_symbol()
     return sym;
 }
 
-ref<PgfSequence> PgfReader::read_seq()
+ref<PgfConcrRule> PgfReader::read_rule()
 {
-	size_t n_syms = read_len();
+    size_t n_syms = read_len();
+    ref<PgfConcrRule> rule = inline_vector<PgfSymbol>::alloc(&PgfConcrRule::syms, n_syms);
 
-	ref<PgfSequence> seq = inline_vector<PgfSymbol>::alloc(&PgfSequence::syms, n_syms);
+    vector<PgfVariableRange> vars = read_null_vector(&PgfReader::read_variable_range);
+    ref<PgfLParam> res = read_lparam();
+    vector<ref<PgfLParam>> args = read_null_vector(&PgfReader::read_lparam);
+    ref<PgfLParam> lin_idx = read_lparam();
+
+    rule->vars = vars;
+    rule->res  = res;
+    rule->container = container;
+    rule->args = args;
+    rule->lin_idx = lin_idx;
 
     for (size_t i = 0; i < n_syms; i++) {
         PgfSymbol sym = read_symbol();
-        seq->syms[i] = sym;
+        rule->syms[i] = sym;
     }
 
-    return seq;
+    return rule;
-}
-
-vector<ref<PgfSequence>> PgfReader::read_seq_ids(object container)
-{
-    size_t len = read_len();
-    vector<ref<PgfSequence>> vec = vector<ref<PgfSequence>>::alloc(len);
-    for (size_t i = 0; i < len; i++) {
-        size_t seq_id = read_len();
-        ref<PgfSequence> seq = phrasetable_relink(concrete->phrasetable,
-                                                  container, i,
-                                                  seq_id);
-        if (seq == 0) {
-            throw pgf_error("Invalid sequence id");
-        }
-        vec[i] = seq;
-    }
-    return vec;
-}
-
-PgfPhrasetable PgfReader::read_phrasetable(size_t len)
-{
-    if (len == 0)
-        return 0;
-
-    PgfPhrasetableEntry value;
-
-    size_t half = len/2;
-    PgfPhrasetable left  = read_phrasetable(half);
-    value.seq = read_seq();
-    value.n_backrefs = 0;
-    value.backrefs = 0;
-    PgfPhrasetable right = read_phrasetable(len-half-1);
-
-    PgfPhrasetable table = Node<PgfPhrasetableEntry>::new_node(value);
-    table->sz    = 1+Node<PgfPhrasetableEntry>::size(left)+Node<PgfPhrasetableEntry>::size(right);
-    table->left  = left;
-    table->right = right;
-    return table;
-}
-
-PgfPhrasetable PgfReader::read_phrasetable()
-{
-    size_t len = read_len();
-    return read_phrasetable(len);
 }
 
 ref<PgfConcrLincat> PgfReader::read_lincat()
 {
     ref<PgfConcrLincat> lincat = read_name(&PgfConcrLincat::name);
 
+    container = lincat.tagged();
+
     auto fields = read_lincat_fields(lincat);
     auto n_lindefs = read_len();
-    auto args = read_vector(&PgfReader::read_parg);
+    auto rules = read_vector(&PgfReader::read_rule2);
-    auto res  = read_vector(&PgfReader::read_presult2);
+
-    auto seqs = read_seq_ids(lincat.tagged());
+    container = 0;
+
+    for (size_t i = n_lindefs; i < rules.size(); i++) {
+        table_maker->insert_rule(rules[i]);
+    }
 
     lincat->abscat = namespace_lookup(abstract->cats, &lincat->name);
     lincat->fields = fields;
     lincat->n_lindefs = n_lindefs;
-    lincat->args = args;
+    lincat->rules = rules;
-    lincat->res  = res;
-    lincat->seqs = seqs;
     return lincat;
 }
 
@@ -715,13 +701,16 @@ ref<PgfConcrLin> PgfReader::read_lin()
     if (lin->lincat == 0)
         throw pgf_error("Found a lin which uses a category without a lincat");
 
-    auto args = read_vector(&PgfReader::read_parg);
+    container  = lin.tagged();
-    auto res  = read_vector(&PgfReader::read_presult2);
-    auto seqs = read_seq_ids(lin.tagged());
 
-    lin->args = args;
+    auto rules = read_vector(&PgfReader::read_rule2);
-    lin->res  = res;
+    lin->rules = rules;
-    lin->seqs = seqs;
+
+    container  = 0;
+
+    for (size_t i = 0; i < rules.size(); i++) {
+        table_maker->insert_rule(rules[i]);
+    }
 
     return lin;
 }
@@ -740,8 +729,8 @@ ref<PgfConcr> PgfReader::read_concrete()
 	auto cflags = read_namespace<PgfFlag>(&PgfReader::read_flag);
 	concrete->cflags = cflags;
 
-	auto phrasetable = read_phrasetable();
+    PgfParseTableMaker tm(concrete);
-	concrete->phrasetable = phrasetable;
+    this->table_maker = &tm;
 
 	auto lincats = read_namespace<PgfConcrLincat>(&PgfReader::read_lincat);
 	concrete->lincats = lincats;
@@ -749,12 +738,11 @@ ref<PgfConcr> PgfReader::read_concrete()
 	auto lins = read_namespace<PgfConcrLin>(&PgfReader::read_lin);
 	concrete->lins = lins;
 
+    this->table_maker = NULL;
+
 	auto printnames = read_namespace<PgfConcrPrintname>(&PgfReader::read_printname);
 	concrete->printnames = printnames;
 
-    //PgfLRTableMaker maker(abstract, concrete);
-    //concrete->lrtable = maker.make();
-
     return concrete;
 }
 

src/runtime/c/pgf/reader.h

@@ -51,6 +51,9 @@ public:
     template <class C, class V>
     ref<C> read_vector(inline_vector<V> C::* field, void (PgfReader::*read_value)(ref<V> val));
 
+    template<class V>
+    vector<V> read_null_vector(void (PgfReader::*read_value)(ref<V> val));
+
     template<class V>
     vector<V> read_vector(void (PgfReader::*read_value)(ref<V> val));
 
@@ -70,6 +73,7 @@ public:
     void read_abstract(ref<PgfAbstr> abstract);
     void merge_abstract(ref<PgfAbstr> abstract);
 
+    ref<PgfConcrRule> read_rule();
     ref<PgfConcrLincat> read_lincat();
     vector<ref<PgfText>> read_lincat_fields(ref<PgfConcrLincat> lincat);
     ref<PgfLParam> read_lparam();
@@ -77,10 +81,6 @@ public:
     void read_parg(ref<PgfPArg> parg);
     ref<PgfPResult> read_presult();
     PgfSymbol read_symbol();
-    ref<PgfSequence> read_seq();
-    vector<ref<PgfSequence>> read_seq_ids(object container);
-    PgfPhrasetable read_phrasetable(size_t len);
-    PgfPhrasetable read_phrasetable();
     ref<PgfConcrLin> read_lin();
     ref<PgfConcrPrintname> read_printname();
 
@@ -94,6 +94,9 @@ private:
     PgfProbsCallback *probs_callback;
     ref<PgfAbstr> abstract;
     ref<PgfConcr> concrete;
+    object container;
+
+    class PgfParseTableMaker *table_maker;
 
     object read_name_internal(size_t struct_size);
     object read_text_internal(size_t struct_size);
@@ -101,6 +104,8 @@ private:
     void read_text2(ref<ref<PgfText>> r) { auto text = read_text(); *r = text; }
     void read_lparam(ref<ref<PgfLParam>> r) { auto lparam = read_lparam(); *r = lparam; }
     void read_presult2(ref<ref<PgfPResult>> r) { auto res = read_presult(); *r = res; }
+    void read_rule2(ref<ref<PgfConcrRule>> r) { auto rule = read_rule(); *r = rule; }
+    void read_symbol2(ref<PgfSymbol> r) { auto sym = read_symbol(); *r = sym; }
 
     template<class I>
     ref<I> read_symbol_idx();

src/runtime/c/pgf/writer.cxx

@@ -144,6 +144,19 @@ void PgfWriter::write_vector(vector<V> vec, void (PgfWriter::*write_value)(ref<V
     }
 }
 
+template<class V>
+void PgfWriter::write_null_vector(vector<V> vec, void (PgfWriter::*write_value)(ref<V> val))
+{
+    if (vec == 0) {
+        write_len(0);
+    } else {
+        write_len(vec.size());
+        for (size_t i = 0; i < vec.size(); i++) {
+            (this->*write_value)(vec.elem(i));
+        }
+    }
+}
+
 void PgfWriter::write_literal(PgfLiteral literal)
 {
     auto tag = ref<PgfLiteral>::get_tag(literal);
@@ -293,18 +306,19 @@ void PgfWriter::write_lparam(ref<PgfLParam> lparam)
     }
 }
 
-void PgfWriter::write_parg(ref<PgfPArg> parg)
+void PgfWriter::write_rule(ref<PgfConcrRule> rule)
 {
-    write_lparam(parg->param);
+    write_len(rule->syms.size());
-}
 
-void PgfWriter::write_presult(ref<PgfPResult> pres)
+    write_null_vector(rule->vars, &PgfWriter::write_variable_range);
-{
+    write_lparam(rule->res);
-    if (pres->vars != 0)
+    write_null_vector(rule->args, &PgfWriter::write_lparam);
-        write_vector(pres->vars, &PgfWriter::write_variable_range);
+
-    else
+    write_lparam(rule->lin_idx);
-        write_len(0);
+
-    write_lparam(ref<PgfLParam>::from_ptr(&pres->param));
+    for (PgfSymbol sym : rule->syms) {
+        write_symbol(sym);
+    }
 }
 
 void PgfWriter::write_symbol(PgfSymbol sym)
@@ -341,10 +355,10 @@ void PgfWriter::write_symbol(PgfSymbol sym)
         write_len(sym_kp->alts.size());
         for (size_t i = 0; i < sym_kp->alts.size(); i++) {
 			ref<PgfAlternative> alt = sym_kp->alts.elem(i);
-            write_vector(alt->form->syms.as_vector(), &PgfWriter::write_symbol);
+            write_vector(alt->form, &PgfWriter::write_symbol);
             write_vector(alt->prefixes, &PgfWriter::write_text);
         }
-        write_vector(sym_kp->default_form->syms.as_vector(), &PgfWriter::write_symbol);
+        write_vector(sym_kp->default_form, &PgfWriter::write_symbol);
 		break;
 	}
 	case PgfSymbolBIND::tag:
@@ -359,36 +373,12 @@ void PgfWriter::write_symbol(PgfSymbol sym)
 	}
 }
 
-void PgfWriter::write_seq(ref<PgfSequence> seq)
-{
-	seq_ids.add(seq);
-    write_vector(seq->syms.as_vector(), &PgfWriter::write_symbol);
-}
-
-void PgfWriter::write_phrasetable(PgfPhrasetable table)
-{
-    write_len(phrasetable_size(table));
-    write_phrasetable_helper(table);
-}
-
-void PgfWriter::write_phrasetable_helper(PgfPhrasetable table)
-{
-    if (table == 0)
-        return;
-
-    write_phrasetable_helper(table->left);
-    write_seq(table->value.seq);
-    write_phrasetable_helper(table->right);
-}
-
 void PgfWriter::write_lincat(ref<PgfConcrLincat> lincat)
 {
     write_name(&lincat->name);
     write_vector(lincat->fields, &PgfWriter::write_lincat_field);
     write_len(lincat->n_lindefs);
-    write_vector(lincat->args, &PgfWriter::write_parg);
+    write_vector(lincat->rules, &PgfWriter::write_rule);
-    write_vector(lincat->res, &PgfWriter::write_presult);
-    write_vector(lincat->seqs, &PgfWriter::write_seq_id);
 }
 
 void PgfWriter::write_lincat_field(ref<ref<PgfText>> field)
@@ -399,9 +389,7 @@ void PgfWriter::write_lincat_field(ref<ref<PgfText>> field)
 void PgfWriter::write_lin(ref<PgfConcrLin> lin)
 {
     write_name(&lin->name);
-    write_vector(lin->args, &PgfWriter::write_parg);
+    write_vector(lin->rules, &PgfWriter::write_rule);
-    write_vector(lin->res, &PgfWriter::write_presult);
-    write_vector(lin->seqs, &PgfWriter::write_seq_id);
 }
 
 void PgfWriter::write_printname(ref<PgfConcrPrintname> printname)
@@ -428,16 +416,11 @@ void PgfWriter::write_concrete(ref<PgfConcr> concr)
         }
     }
 
-	seq_ids.start(concr);
-
     write_name(&concr->name);
     write_namespace<PgfFlag>(concr->cflags, &PgfWriter::write_flag);
-	write_phrasetable(concr->phrasetable);
     write_namespace<PgfConcrLincat>(concr->lincats, &PgfWriter::write_lincat);
 	write_namespace<PgfConcrLin>(concr->lins, &PgfWriter::write_lin);
 	write_namespace<PgfConcrPrintname>(concr->printnames, &PgfWriter::write_printname);
-
-	seq_ids.end();
 }
 
 void PgfWriter::write_pgf(ref<PgfPGF> pgf)

src/runtime/c/pgf/writer.h

@@ -24,6 +24,8 @@ public:
 
     template<class V>
     void write_vector(vector<V> vec, void (PgfWriter::*write_value)(ref<V> val));
+    template<class V>
+    void write_null_vector(vector<V> vec, void (PgfWriter::*write_value)(ref<V> val));
 
     void write_literal(PgfLiteral literal);
     void write_expr(PgfExpr expr);
@@ -42,12 +44,7 @@ public:
     void write_lincat_field(ref<ref<PgfText>> field);
     void write_variable_range(ref<PgfVariableRange> var);
     void write_lparam(ref<PgfLParam> lparam);
-    void write_parg(ref<PgfPArg> linarg);
-    void write_presult(ref<PgfPResult> linres);
     void write_symbol(PgfSymbol sym);
-    void write_seq(ref<PgfSequence> seq);
-    void write_seq_id(ref<ref<PgfSequence>> r) { write_len(seq_ids.get(*r)); };
-    void write_phrasetable(PgfPhrasetable table);
     void write_lin(ref<PgfConcrLin> lin);
     void write_printname(ref<PgfConcrPrintname> printname);
 
@@ -58,18 +55,17 @@ public:
 private:
     template<class V>
     void write_namespace_helper(Namespace<V> nmsp, void (PgfWriter::*write_value)(ref<V>));
-    void write_phrasetable_helper(PgfPhrasetable table);
 
     void write_text(ref<ref<PgfText>> r) { write_text(&(**r)); };
     void write_lparam(ref<ref<PgfLParam>> r) { write_lparam(*r); };
+    void write_rule(ref<PgfConcrRule> rule);
     void write_symbol(ref<PgfSymbol> r) { write_symbol(*r); };
-    void write_presult(ref<ref<PgfPResult>> r) { write_presult(*r); };
+    void write_rule(ref<ref<PgfConcrRule>> r) { write_rule(*r); };
 
     FILE *out;
     PgfText **langs;
 
     ref<PgfAbstr> abstract;
-    PgfPhrasetableIds seq_ids;
 };
 
 #endif

src/runtime/haskell/PGF2.hsc

@@ -73,7 +73,7 @@ module PGF2 (-- * PGF
              graphvizAbstractTree, graphvizParseTree,
              Labels, getDepLabels,
              graphvizDependencyTree, conlls2latexDoc, getCncDepLabels,
-             graphvizWordAlignment, graphvizLRAutomaton,
+             graphvizWordAlignment,
 
              -- * Concrete syntax
              ConcName,Concr,languages,language,concreteName,languageCode,concreteFlag,
@@ -363,19 +363,14 @@ showPGF p =
           modifyIORef ref (\doc -> doc $$ text def)
 
     ppConcr name c = unsafePerformIO $ do
-      (seq_ids,doc3) <- prepareSequences c -- run first to update all seq_id
+      doc1 <- ppLincats c
-      doc1 <- ppLincats seq_ids c
+      doc2 <- ppLins c
-      doc2 <- ppLins seq_ids c
-      pgf_release_phrasetable_ids seq_ids
       return (text "concrete" <+> text name <+> char '{' $$
               nest 2 (doc1 $$
-                      doc2 $$
+                      doc2) $$
-                      (text "sequences" <+> char '{' $$ 
-                       nest 2 doc3 $$ 
-                       char '}')) $$
               char '}')
 
-    ppLincats seq_ids c = do
+    ppLincats c = do
       ref <- newIORef empty
       (allocaBytes (#size PgfItor) $ \itor ->
        bracket (wrapItorCallback (getLincats ref)) freeHaskellFunPtr $ \fptr ->
@@ -402,15 +397,15 @@ showPGF p =
                                        char ']')
           modifyIORef ref $ (\doc -> doc $$ def)
           forM_ (init [0..n_lindefs]) $ \i -> do
-            def <- bracket (pgf_print_lindef_internal seq_ids val i) free $ \c_text -> do
+            def <- bracket (pgf_print_lindef_internal val i) free $ \c_text -> do
                      fmap text (peekText c_text)
             modifyIORef ref (\doc -> doc $$ text "lindef" <+> def)
           forM_ (init [0..n_linrefs]) $ \i -> do
-            def <- bracket (pgf_print_linref_internal seq_ids val i) free $ \c_text -> do
+            def <- bracket (pgf_print_linref_internal val i) free $ \c_text -> do
                      fmap text (peekText c_text)
             modifyIORef ref $ (\doc -> doc $$ text "linref" <+> def)
 
-    ppLins seq_ids c = do
+    ppLins c = do
       ref <- newIORef empty
       (allocaBytes (#size PgfItor) $ \itor ->
        bracket (wrapItorCallback (getLins ref)) freeHaskellFunPtr $ \fptr ->
@@ -421,30 +416,13 @@ showPGF p =
       where
         getLins :: IORef Doc -> ItorCallback
         getLins ref itor key val exn = do
-          n_prods <- pgf_get_lin_get_prod_count val
+          n_prods <- pgf_get_lin_rules_count val
           forM_ (init [0..n_prods]) $ \i -> do
-            def <- bracket (pgf_print_lin_internal seq_ids val i) free $ \c_text -> do
+            def <- bracket (pgf_print_lin_internal val i) free $ \c_text -> do
                      fmap text (peekText c_text)
             modifyIORef ref (\doc -> doc $$ text "lin" <+> def)
             return ()
 
-    prepareSequences c = do
-      ref <- newIORef empty
-      seq_ids <- (allocaBytes (#size PgfSequenceItor) $ \itor ->
-                  bracket (wrapSequenceItorCallback (getSequences ref)) freeHaskellFunPtr $ \fptr ->
-                  withForeignPtr (c_revision c) $ \c_revision -> do
-                    (#poke PgfSequenceItor, fn) itor fptr
-                    withPgfExn "showPGF" (pgf_iter_sequences (a_db p) c_revision itor nullPtr))
-      doc <- readIORef ref
-      return (seq_ids, doc)
-      where
-        getSequences :: IORef Doc -> SequenceItorCallback
-        getSequences ref itor seq_id val exn = do
-          def <- bracket (pgf_print_sequence_internal seq_id val) free $ \c_text -> do
-                     fmap text (peekText c_text)
-          modifyIORef ref $ (\doc -> doc $$ def)
-          return 0
-
 -- | The abstract language name is the name of the top-level
 -- abstract module
 abstractName :: PGF -> AbsName
@@ -830,8 +808,7 @@ fullFormLexicon c = unsafePerformIO $ do
    withForeignPtr (c_revision c) $ \c_revision -> do
      (#poke PgfSequenceItor,   fn) itor1 fptr1
      (#poke PgfMorphoCallback, fn) itor2 fptr2
-     seq_ids <- withPgfExn "fullFormLexicon" (pgf_iter_sequences (c_db c) c_revision itor1 itor2)
+     withPgfExn "fullFormLexicon" (pgf_iter_sequences (c_db c) c_revision itor1 itor2))
-     pgf_release_phrasetable_ids seq_ids)
   fmap (reverse2 []) (readIORef ref)
   where
     getSequences ref _ seq_id val exn = do
@@ -1484,15 +1461,6 @@ graphvizDependencyTree
   -> String -- ^ Rendered output in the specified format
 graphvizDependencyTree format debug mlab mclab concr t = error "TODO: graphvizDependencyTree"
 
-graphvizLRAutomaton :: Concr -> String
-graphvizLRAutomaton c =
-  unsafePerformIO $
-  withForeignPtr (c_revision c) $ \c_revision ->
-  bracket (withPgfExn "graphvizLRAutomaton" (pgf_graphviz_lr_automaton (c_db c) c_revision)) free $ \c_text ->
-    if c_text == nullPtr
-      then return ""
-      else peekText c_text
-
 ---------------------- should be a separate module?
 
 -- visualization with latex output. AR Nov 2015

src/runtime/haskell/PGF2/FFI.hsc

@@ -48,7 +48,6 @@ data PgfSequenceItor
 data PgfProbsCallback
 data PgfMorphoCallback
 data PgfCohortsCallback
-data PgfPhrasetableIds
 data PgfExprEnum
 data PgfAlignmentPhrase
 
@@ -150,26 +149,22 @@ foreign import ccall "wrapper" wrapCohortsCallback :: Wrapper CohortsCallback
 
 foreign import ccall pgf_lookup_cohorts :: Ptr PgfDB -> Ptr Concr -> Ptr PgfText -> Ptr PgfCohortsCallback -> Ptr PgfExn -> IO ()
 
-foreign import ccall pgf_iter_sequences :: Ptr PgfDB -> Ptr Concr -> Ptr PgfSequenceItor -> Ptr PgfMorphoCallback -> Ptr PgfExn -> IO (Ptr PgfPhrasetableIds)
+foreign import ccall pgf_iter_sequences :: Ptr PgfDB -> Ptr Concr -> Ptr PgfSequenceItor -> Ptr PgfMorphoCallback -> Ptr PgfExn -> IO ()
 
 foreign import ccall pgf_get_lincat_counts_internal :: Ptr () -> Ptr CSize -> IO ()
 
 foreign import ccall pgf_get_lincat_field_internal :: Ptr () -> CSize -> IO (Ptr PgfText)
 
-foreign import ccall pgf_print_lindef_internal :: Ptr PgfPhrasetableIds -> Ptr () -> CSize -> IO (Ptr PgfText)
+foreign import ccall pgf_print_lindef_internal :: Ptr () -> CSize -> IO (Ptr PgfText)
 
-foreign import ccall pgf_print_linref_internal :: Ptr PgfPhrasetableIds -> Ptr () -> CSize -> IO (Ptr PgfText)
+foreign import ccall pgf_print_linref_internal :: Ptr () -> CSize -> IO (Ptr PgfText)
 
-foreign import ccall pgf_get_lin_get_prod_count :: Ptr () -> IO CSize
+foreign import ccall pgf_get_lin_rules_count :: Ptr () -> IO CSize
 
-foreign import ccall pgf_print_lin_internal :: Ptr PgfPhrasetableIds -> Ptr () -> CSize -> IO (Ptr PgfText)
+foreign import ccall pgf_print_lin_internal :: Ptr () -> CSize -> IO (Ptr PgfText)
-
-foreign import ccall pgf_print_sequence_internal :: CSize -> Ptr () -> IO (Ptr PgfText)
 
 foreign import ccall pgf_sequence_get_text_internal :: Ptr () -> IO (Ptr PgfText)
 
-foreign import ccall pgf_release_phrasetable_ids :: Ptr PgfPhrasetableIds -> IO ()
-
 type ItorCallback = Ptr PgfItor -> Ptr PgfText -> Ptr () -> Ptr PgfExn -> IO ()
 
 foreign import ccall "wrapper" wrapItorCallback :: Wrapper ItorCallback
@@ -244,7 +239,7 @@ foreign import ccall "dynamic" callLinBuilder1 :: Dynamic (Ptr PgfLinBuilderIfac
 
 foreign import ccall "dynamic" callLinBuilder2 :: Dynamic (Ptr PgfLinBuilderIface -> CSize -> CSize -> Ptr PgfExn -> IO ())
 
-foreign import ccall "dynamic" callLinBuilder3 :: Dynamic (Ptr PgfLinBuilderIface -> CSize -> CSize -> CSize -> Ptr CSize -> Ptr PgfExn -> IO ())
+foreign import ccall "dynamic" callLinBuilder3 :: Dynamic (Ptr PgfLinBuilderIface -> CSize -> CSize -> Ptr CSize -> Ptr PgfExn -> IO ())
 
 foreign import ccall "dynamic" callLinBuilder4 :: Dynamic (Ptr PgfLinBuilderIface -> CSize -> CSize -> CSize -> Ptr CSize -> Ptr PgfExn -> IO ())
 
@@ -318,8 +313,6 @@ foreign import ccall pgf_graphviz_parse_tree :: Ptr PgfDB -> Ptr Concr -> Stable
 
 foreign import ccall pgf_graphviz_word_alignment :: Ptr PgfDB -> Ptr (Ptr Concr) -> CSize -> StablePtr Expr -> Ptr PgfPrintContext -> Ptr PgfMarshaller -> Ptr PgfGraphvizOptions -> Ptr PgfExn -> IO (Ptr PgfText)
 
-foreign import ccall pgf_graphviz_lr_automaton :: Ptr PgfDB -> Ptr Concr -> Ptr PgfExn -> IO (Ptr PgfText)
-
 
 -----------------------------------------------------------------------
 -- Texts

src/runtime/haskell/PGF2/Transactions.hsc

@@ -1,3 +1,4 @@
+{-# LANGUAGE ScopedTypeVariables #-}
 module PGF2.Transactions
           ( -- transactions
             TxnID
@@ -18,15 +19,14 @@ module PGF2.Transactions
           , setAbstractFlag
 
             -- concrete syntax
-          , Token, SeqId, LIndex, LVar, LParam(..)
+          , Token, LIndex, LVar, LParam(..)
-          , PArg(..), Symbol(..), Production(..)
+          , PArg(..), Symbol(..), Rule(..)
 
           , createConcrete
           , alterConcrete
           , dropConcrete
           , mergePGF
           , setConcreteFlag
-          , SeqTable
           , createLincat
           , dropLincat
           , createLin, alterLin
@@ -251,21 +251,23 @@ data Symbol
   | SymALL_CAPIT                    -- the special ALL_CAPIT token
   deriving (Eq,Ord,Show)
 
+type Quantifiers = [(LVar,Int)]
+data Rule = Rule Quantifiers LParam [LParam] LParam [Symbol]
+                 deriving (Eq,Show)
+
 data PArg = PArg [(LIndex,LIndex)] {-# UNPACK #-} !LParam
             deriving (Eq,Show)
 
 data Production = Production [(LVar,LIndex)] [PArg] LParam [SeqId]
                  deriving (Eq,Show)
 
-type SeqTable = Seq.Seq (Either [Symbol] SeqId)
+createLincat :: Cat -> [String] -> [Rule] -> [Rule] -> Transaction Concr ()
-
+createLincat name fields lindefs linrefs = Transaction $ \c_db c_abstr c_revision c_exn ->
-createLincat :: Cat -> [String] -> [Production] -> [Production] -> SeqTable -> Transaction Concr SeqTable
-createLincat name fields lindefs linrefs seqtbl = Transaction $ \c_db c_abstr c_revision c_exn ->
   let n_fields = length fields
   in withText name $ \c_name ->
      allocaBytes (n_fields*(#size PgfText*)) $ \c_fields ->
      withTexts c_fields 0 fields $
-     withBuildLinIface (lindefs++linrefs) seqtbl $ \c_build ->
+     withBuildLinIface (lindefs++linrefs) $ \c_build ->
        pgf_create_lincat c_db c_abstr c_revision c_name
                          (fromIntegral n_fields) c_fields
                          (fromIntegral (length lindefs)) (fromIntegral (length linrefs))
@@ -282,27 +284,25 @@ dropLincat name = Transaction $ \c_db  c_abstr c_revision c_exn ->
   withText name $ \c_name ->
     pgf_drop_lincat c_db  c_abstr c_revision c_name c_exn
 
-createLin :: Fun -> [Production] -> SeqTable -> Transaction Concr SeqTable
+createLin :: Fun -> [Rule] -> Transaction Concr ()
-createLin name prods seqtbl = Transaction $ \c_db c_abstr c_revision c_exn ->
+createLin name rules = Transaction $ \c_db c_abstr c_revision c_exn ->
   withText name $ \c_name ->
-  withBuildLinIface prods seqtbl $ \c_build ->
+  withBuildLinIface rules $ \c_build ->
-    pgf_create_lin c_db c_abstr c_revision c_name (fromIntegral (length prods)) c_build c_exn
+    pgf_create_lin c_db c_abstr c_revision c_name (fromIntegral (length rules)) c_build c_exn
 
-alterLin :: Fun -> [Production] -> SeqTable -> Transaction Concr SeqTable
+alterLin :: Fun -> [Rule] -> Transaction Concr ()
-alterLin name prods seqtbl = Transaction $ \c_db c_abstr c_revision c_exn ->
+alterLin name rules = Transaction $ \c_db c_abstr c_revision c_exn ->
   withText name $ \c_name ->
-  withBuildLinIface prods seqtbl $ \c_build ->
+  withBuildLinIface rules $ \c_build ->
-    pgf_alter_lin c_db c_abstr c_revision c_name (fromIntegral (length prods)) c_build c_exn
+    pgf_alter_lin c_db c_abstr c_revision c_name (fromIntegral (length rules)) c_build c_exn
 
-withBuildLinIface prods seqtbl f = do
+withBuildLinIface rules f = do
-  ref <- newIORef seqtbl
   (allocaBytes (#size PgfBuildLinIface) $ \c_build ->
    allocaBytes (#size PgfBuildLinIfaceVtbl) $ \vtbl ->
-   bracket (wrapLinBuild (build ref)) freeHaskellFunPtr $ \c_callback -> do
+   bracket (wrapLinBuild build) freeHaskellFunPtr $ \c_callback -> do
      (#poke PgfBuildLinIface, vtbl) c_build vtbl
      (#poke PgfBuildLinIfaceVtbl, build) vtbl c_callback
      f c_build)
-  readIORef ref
   where
     forM_ []     c_exn f = return ()
     forM_ (x:xs) c_exn f = do
@@ -311,31 +311,23 @@ withBuildLinIface prods seqtbl f = do
         then f x >> forM_ xs c_exn f
         else return ()
 
-    build ref _ c_builder c_exn = do
+    build _ c_builder c_exn = do
       vtbl <- (#peek PgfLinBuilderIface, vtbl) c_builder
-      forM_ prods c_exn $ \(Production vars args res seqids) -> do
+      forM_ rules c_exn $ \(Rule vars res args lin_idx seq) -> do
-        fun <- (#peek PgfLinBuilderIfaceVtbl, start_production) vtbl
+        fun <- (#peek PgfLinBuilderIfaceVtbl, start_rule) vtbl
-        callLinBuilder0 fun c_builder c_exn
+        callLinBuilder2 fun c_builder (fromIntegral (length vars)) (fromIntegral (length seq)) c_exn
         fun <- (#peek PgfLinBuilderIfaceVtbl, add_argument) vtbl
-        forM_ args c_exn $ \(PArg hypos param) ->
+        forM_ args c_exn $ \arg ->
-          callLParam (callLinBuilder3 fun c_builder (fromIntegral (length hypos))) param c_exn
+          callLParam (callLinBuilder3 fun c_builder) arg c_exn
         fun <- (#peek PgfLinBuilderIfaceVtbl, set_result) vtbl
-        callLParam (callLinBuilder3 fun c_builder (fromIntegral (length vars))) res c_exn
+        callLParam (callLinBuilder3 fun c_builder) res c_exn
+        fun <- (#peek PgfLinBuilderIfaceVtbl, set_lin_idx) vtbl
+        callLParam (callLinBuilder3 fun c_builder) lin_idx c_exn
         fun <- (#peek PgfLinBuilderIfaceVtbl, add_variable) vtbl
         forM_ vars c_exn $ \(v,r) ->
           callLinBuilder2 fun c_builder (fromIntegral v) (fromIntegral r) c_exn
-        fun <- (#peek PgfLinBuilderIfaceVtbl, add_sequence_id) vtbl
+        forM_ seq c_exn (addSymbol c_builder vtbl c_exn)
-        seqtbl <- readIORef ref
+        fun <- (#peek PgfLinBuilderIfaceVtbl, end_rule) vtbl
-        forM_ seqids c_exn $ \seqid ->
-          case Seq.index seqtbl seqid of
-            Left syms   -> do fun    <- (#peek PgfLinBuilderIfaceVtbl, start_sequence) vtbl
-                              callLinBuilder1 fun c_builder (fromIntegral (length syms)) c_exn
-                              forM_ syms c_exn (addSymbol c_builder vtbl c_exn)
-                              fun <- (#peek PgfLinBuilderIfaceVtbl, end_sequence) vtbl
-                              seqid' <- callLinBuilder7 fun c_builder c_exn
-                              writeIORef ref $! Seq.update seqid (Right (fromIntegral seqid')) seqtbl
-            Right seqid -> do callLinBuilder1 fun c_builder (fromIntegral seqid) c_exn
-        fun <- (#peek PgfLinBuilderIfaceVtbl, end_production) vtbl
         callLinBuilder0 fun c_builder c_exn
 
     addSymbol c_builder vtbl c_exn (SymCat d r) = do