handle functions without lin rules as well as meta variables

src/runtime/c/pgf/linearizer.cxx

@@ -2,16 +2,13 @@
 #include "printer.h"
 #include "linearizer.h"
 
-PgfLinearizer::TreeNode::TreeNode(PgfLinearizer *linearizer, ref<PgfConcrLin> lin, PgfText *lit)
+PgfLinearizer::TreeNode::TreeNode(PgfLinearizer *linearizer)
 {
     this->next     = linearizer->root;
     this->next_arg = NULL;
     this->args     = linearizer->args;
 
     this->fid       = 0;
-    this->literal   = lit;
-    this->lin       = lin;
-    this->lin_index = 0;
 
     this->value     = 0;
     this->var_count = 0;
@@ -20,6 +17,135 @@ PgfLinearizer::TreeNode::TreeNode(PgfLinearizer *linearizer, ref<PgfConcrLin> li
     linearizer->root= this;
 }
 
+void PgfLinearizer::TreeNode::linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, PgfLParam *r)
+{
+    TreeNode *arg = args;
+    while (d > 0) {
+        arg = arg->next_arg;
+        if (arg == 0)
+            throw pgf_error("Found inconsistency in the PMCFG representation");
+        d--;
+    }
+    size_t lindex = eval_param(r);
+    arg->linearize(out, linearizer, lindex);
+}
+
+void PgfLinearizer::TreeNode::linearize_syms(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, ref<Vector<PgfSymbol>> syms)
+{
+    for (size_t i = 0; i < syms->len; i++) {
+        PgfSymbol sym = *vector_elem(syms, i);
+
+        switch (ref<PgfSymbol>::get_tag(sym)) {
+        case PgfSymbolCat::tag: {
+            auto sym_cat = ref<PgfSymbolCat>::untagged(sym);
+            linearize_arg(out, linearizer, sym_cat->d, &sym_cat->r);
+            break;
+        }
+        case PgfSymbolLit::tag: {
+            auto sym_lit = ref<PgfSymbolLit>::untagged(sym);
+            linearize_arg(out, linearizer, sym_lit->d, &sym_lit->r);
+            break;
+        }
+        case PgfSymbolVar::tag: {
+            auto sym_var = ref<PgfSymbolVar>::untagged(sym);
+            break;
+        }
+        case PgfSymbolKS::tag: {
+            auto sym_ks = ref<PgfSymbolKS>::untagged(sym);
+
+            linearizer->flush_pre_stack(out, &sym_ks->token);
+
+            switch (linearizer->capit) {
+            case CAPIT_NONE:
+                out->symbol_token(&sym_ks->token);
+                break;
+            case CAPIT_FIRST: {
+                PgfText *cap = (PgfText *) alloca(sizeof(PgfText)+sym_ks->token.size+6);
+
+                const uint8_t *p   = (const uint8_t *) sym_ks->token.text;
+                const uint8_t *end = p + sym_ks->token.size;
+
+                uint8_t *q = (uint8_t *) cap->text;
+
+                uint32_t ucs = pgf_utf8_decode(&p);
+                ucs = pgf_utf8_to_upper(ucs);
+                pgf_utf8_encode(ucs,&q);
+
+                memcpy(q, p, (end - p)+1);
+                q += (end - p);
+
+                cap->size = q - (uint8_t *) cap->text;
+                out->symbol_token(cap);
+
+                linearizer->capit = CAPIT_NONE;
+                break;
+            }
+            case CAPIT_ALL: {
+                PgfText *cap = (PgfText *) alloca(sizeof(PgfText)+sym_ks->token.size*6);
+
+                const uint8_t *p   = (const uint8_t *) sym_ks->token.text;
+                const uint8_t *end = p + sym_ks->token.size;
+
+                uint8_t *q = (uint8_t *) cap->text;
+
+                while (p != end) {
+                    uint32_t ucs = pgf_utf8_decode(&p);
+                    ucs = pgf_utf8_to_upper(ucs);
+                    pgf_utf8_encode(ucs,&q);
+                }
+
+                cap->size = q - (uint8_t *) cap->text;
+                *q = 0;
+
+                out->symbol_token(cap);
+
+                linearizer->capit = CAPIT_NONE;
+                break;
+            }
+            }
+            break;
+        }
+        case PgfSymbolKP::tag: {
+            auto sym_kp = ref<PgfSymbolKP>::untagged(sym);
+            PreStack *pre = new PreStack();
+            pre->next   = linearizer->pre_stack;
+            pre->node   = this;
+            pre->sym_kp = sym_kp;
+            pre->bind   = false;
+            pre->capit  = CAPIT_NONE;
+            pre->bracket_stack = NULL;
+            linearizer->pre_stack = pre;
+            break;
+        }
+        case PgfSymbolBIND::tag:
+        case PgfSymbolSOFTBIND::tag:
+            if (linearizer->pre_stack == NULL)
+                out->symbol_bind();
+            else
+                linearizer->pre_stack->bind = true;
+            break;
+        case PgfSymbolNE::tag:
+            out->symbol_ne();
+            break;
+        case PgfSymbolSOFTSPACE::tag:
+            // Nothing to do
+            break;
+        case PgfSymbolCAPIT::tag:
+            if (linearizer->pre_stack == NULL)
+                linearizer->capit = CAPIT_FIRST;
+            else
+                linearizer->pre_stack->capit = CAPIT_FIRST;
+            break;
+        case PgfSymbolALLCAPIT::tag:
+            if (linearizer->pre_stack == NULL)
+                linearizer->capit = CAPIT_ALL;
+            else
+                linearizer->pre_stack->capit = CAPIT_ALL;
+            break;
+        }
+    }
+}
+
 size_t PgfLinearizer::TreeNode::eval_param(PgfLParam *param)
 {
     size_t value = param->i0;
@@ -36,6 +162,239 @@ size_t PgfLinearizer::TreeNode::eval_param(PgfLParam *param)
     return value;
 }
 
+PgfLinearizer::TreeLinNode::TreeLinNode(PgfLinearizer *linearizer, ref<PgfConcrLin> lin)
+  : TreeNode(linearizer)
+{
+    this->lin       = lin;
+    this->lin_index = 0;
+}
+
+bool PgfLinearizer::TreeLinNode::resolve(PgfLinearizer *linearizer)
+{
+    ref<Vector<PgfHypo>> hypos = lin->absfun->type->hypos;
+    size_t n_args = lin->args->len / lin->res->len;
+
+    while (lin_index < lin->res->len) {
+        size_t offset = lin_index*n_args;
+
+        ref<PgfPResult> pres = *vector_elem(lin->res,  lin_index);
+
+        int i = 0;
+        TreeNode *arg = args;
+        while (arg != NULL) {
+            ref<PgfPArg> parg = vector_elem(lin->args, offset+i);
+            arg->check_category(linearizer, &vector_elem(hypos,i)->type->name);
+
+            if (arg->value < parg->param->i0)
+                break;
+
+            size_t value = arg->value - parg->param->i0;
+            for (size_t j = 0; j < parg->param->n_terms; j++) {
+                size_t factor    = parg->param->terms[j].factor;
+                size_t var       = parg->param->terms[j].var;
+                size_t var_value;
+
+                if (var < var_count && var_values[var] != (size_t) -1) {
+                    // The variable already has a value
+                    var_value = var_values[var];
+                } else {
+                    // The variable is not assigned yet
+                    var_value = value / factor;
+
+                    // find the range for the variable
+                    size_t range = 0;
+                    for (size_t k = 0; k < pres->vars->len; k++) {
+                        ref<PgfVariableRange> var_range = vector_elem(pres->vars, k);
+                        if (var_range->var == var) {
+                            range = var_range->range;
+                            break;
+                        }
+                    }
+                    if (range == 0)
+                        throw pgf_error("Unknown variable in resolving a linearization");
+
+                    if (var_value > range)
+                        break;
+
+                    // Assign the variable;
+                    if (var >= var_count) {
+                        var_values = (size_t*)
+                            realloc(var_values, (var+1)*sizeof(size_t));
+                        while (var_count < var) {
+                            var_values[var_count++] = (size_t) -1;
+                        }
+                        var_count++;
+                    }
+                    var_values[var] = var_value;
+                }
+
+                value -= var_value * factor;
+            }
+
+            if (value != 0)
+                break;
+
+            arg = arg->next_arg;
+            i++;
+        }
+
+        lin_index++;
+
+        if (arg == NULL) {
+            value = eval_param(&pres->param);
+            break;
+        }
+
+        // Unbind all variables
+        for (size_t j = 0; j < var_count; j++) {
+            var_values[j] = (size_t) -1;
+        }
+    }
+
+    return (lin_index <= lin->res->len);
+}
+
+void PgfLinearizer::TreeLinNode::check_category(PgfLinearizer *linearizer, PgfText *cat)
+{
+    if (textcmp(&lin->absfun->type->name, cat) != 0)
+        throw pgf_error("An attempt to linearize an expression which is not type correct");
+}
+
+void PgfLinearizer::TreeLinNode::linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex)
+{
+    PgfText *cat = &lin->absfun->type->name;
+    PgfText *field = NULL;
+    ref<PgfConcrLincat> lincat = namespace_lookup(linearizer->concr->lincats, cat);
+    if (lincat != 0) {
+        field = &(**vector_elem(lincat->fields, lindex));
+    }
+
+    if (linearizer->pre_stack == NULL)
+        out->begin_phrase(cat, fid, field, &lin->name);
+    else {
+        BracketStack *bracket = new BracketStack();
+        bracket->next  = linearizer->pre_stack->bracket_stack;
+        bracket->begin = true;
+        bracket->fid   = fid;
+        bracket->cat   = cat;
+        bracket->field = field;
+        bracket->fun   = &lin->name;
+        linearizer->pre_stack->bracket_stack = bracket;
+    }
+
+    size_t n_seqs = lin->seqs->len / lin->res->len;
+    ref<Vector<PgfSymbol>> syms = *vector_elem(lin->seqs, (lin_index-1)*n_seqs + lindex);
+    linearize_syms(out, linearizer, syms);
+
+    if (linearizer->pre_stack == NULL)
+        out->end_phrase(cat, fid, field, &lin->name);
+    else {
+        BracketStack *bracket = new BracketStack();
+        bracket->next  = linearizer->pre_stack->bracket_stack;
+        bracket->begin = false;
+        bracket->fid   = fid;
+        bracket->cat   = cat;
+        bracket->field = field;
+        bracket->fun   = &lin->name;
+        linearizer->pre_stack->bracket_stack = bracket;
+    }
+}
+
+PgfLinearizer::TreeLindefNode::TreeLindefNode(PgfLinearizer *linearizer, PgfText *literal)
+  : TreeNode(linearizer)
+{
+    this->lincat    = 0;
+    this->lin_index = 0;
+    this->literal   = literal;
+}
+
+bool PgfLinearizer::TreeLindefNode::resolve(PgfLinearizer *linearizer)
+{
+    ref<PgfPResult> pres = *vector_elem(lincat->res,  lin_index);
+    value = eval_param(&pres->param);
+    lin_index++;
+    return true;
+}
+
+void PgfLinearizer::TreeLindefNode::check_category(PgfLinearizer *linearizer, PgfText *cat)
+{
+    lincat = namespace_lookup(linearizer->concr->lincats, cat);
+    if (lincat == 0)
+        throw pgf_error("Cannot find a lincat for a category");
+}
+
+void PgfLinearizer::TreeLindefNode::linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, PgfLParam *r)
+{
+    linearizer->flush_pre_stack(out, literal);
+    out->symbol_token(literal);
+}
+
+void PgfLinearizer::TreeLindefNode::linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex)
+{
+    if (lincat != 0) {
+        PgfText *field = &(**vector_elem(lincat->fields, lindex));
+        if (linearizer->pre_stack == NULL)
+            out->begin_phrase(&lincat->name, fid, field, linearizer->wild);
+        else {
+            BracketStack *bracket = new BracketStack();
+            bracket->next  = linearizer->pre_stack->bracket_stack;
+            bracket->begin = true;
+            bracket->fid   = fid;
+            bracket->cat   = &lincat->name;
+            bracket->field = field;
+            bracket->field = linearizer->wild;
+            linearizer->pre_stack->bracket_stack = bracket;
+        }
+
+        ref<Vector<PgfSymbol>> syms = *vector_elem(lincat->seqs, (lin_index-1)*lincat->fields->len + lindex);
+        linearize_syms(out, linearizer, syms);
+
+        if (linearizer->pre_stack == NULL)
+            out->end_phrase(&lincat->name, fid, field, linearizer->wild);
+        else {
+            BracketStack *bracket = new BracketStack();
+            bracket->next  = linearizer->pre_stack->bracket_stack;
+            bracket->begin = false;
+            bracket->fid   = fid;
+            bracket->cat   = &lincat->name;
+            bracket->field = field;
+            bracket->field = linearizer->wild;
+            linearizer->pre_stack->bracket_stack = bracket;
+        }
+    } else {
+        linearize_arg(out, linearizer, 0, NULL);
+    }
+}
+
+PgfLinearizer::TreeLitNode::TreeLitNode(PgfLinearizer *linearizer, ref<PgfConcrLincat> lincat, PgfText *lit)
+  : TreeNode(linearizer)
+{
+    this->lincat  = lincat;
+    this->literal = lit;
+}
+
+void PgfLinearizer::TreeLitNode::check_category(PgfLinearizer *linearizer, PgfText *cat)
+{
+    if (textcmp(&lincat->name, cat) != 0)
+        throw pgf_error("An attempt to linearize an expression which is not type correct");
+}
+
+void PgfLinearizer::TreeLitNode::linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex)
+{    
+    PgfText *field = NULL;
+    if (lincat != 0) {
+        field = &(**vector_elem(lincat->fields, lindex));
+    }
+
+    linearizer->flush_pre_stack(out, literal);
+
+    if (lincat != 0)
+        out->begin_phrase(&lincat->name, fid, field, linearizer->wild);
+    out->symbol_token(literal);
+    if (lincat != 0)
+        out->end_phrase(&lincat->name, fid, field, linearizer->wild);
+}
+
 PgfLinearizer::PgfLinearizer(ref<PgfConcr> concr, PgfMarshaller *m) {
     this->concr = concr;
     this->m = m;
@@ -44,6 +403,11 @@ PgfLinearizer::PgfLinearizer(ref<PgfConcr> concr, PgfMarshaller *m) {
     this->args  = NULL;
     this->capit = CAPIT_NONE;
     this->pre_stack = NULL;
+    this->wild = (PgfText*) malloc(sizeof(PgfText)+2);
+    this->wild->size = 1;
+    this->wild->text[0] = '_';
+    this->wild->text[1] = 0;
+
 };
 
 PgfLinearizer::~PgfLinearizer()
@@ -67,95 +431,16 @@ PgfLinearizer::~PgfLinearizer()
         delete pre_stack;
         pre_stack = next;
     }
+
+    free(this->wild);
 }
 
 bool PgfLinearizer::resolve()
 {
     TreeNode *node = first;
     while (node != NULL) {
-        if (node->literal == NULL) {
-            size_t n_args = node->lin->args->len / node->lin->res->len;
-
-            while (node->lin_index < node->lin->res->len) {
-                size_t offset = node->lin_index*n_args;
-
-                ref<PgfPResult> pres = *vector_elem(node->lin->res,  node->lin_index);
-
-                int i = 0;
-                TreeNode *arg = node->args;
-                while (arg != NULL) {
-                    ref<PgfPArg> parg = vector_elem(node->lin->args, offset+i);
-
-                    if (arg->value < parg->param->i0)
-                        break;
-
-                    size_t value = arg->value - parg->param->i0;
-                    for (size_t j = 0; j < parg->param->n_terms; j++) {
-                        size_t factor    = parg->param->terms[j].factor;
-                        size_t var       = parg->param->terms[j].var;
-                        size_t var_value;
-
-                        if (var < node->var_count && node->var_values[var] != (size_t) -1) {
-                            // The variable already has a value
-                            var_value = node->var_values[var];
-                        } else {
-                            // The variable is not assigned yet
-                            var_value = value / factor;
-
-                            // find the range for the variable
-                            size_t range = 0;
-                            for (size_t k = 0; k < pres->vars->len; k++) {
-                                ref<PgfVariableRange> var_range = vector_elem(pres->vars, k);
-                                if (var_range->var == var) {
-                                    range = var_range->range;
-                                    break;
-                                }
-                            }
-                            if (range == 0)
-                                throw pgf_error("Unknown variable in resolving a linearization");
-
-                            if (var_value > range)
-                                break;
-
-                            // Assign the variable;
-                            if (var >= node->var_count) {
-                                node->var_values = (size_t*)
-                                    realloc(node->var_values, (var+1)*sizeof(size_t));
-                                while (node->var_count < var) {
-                                    node->var_values[node->var_count++] = (size_t) -1;
-                                }
-                                node->var_count++;
-                            }
-                            node->var_values[var] = var_value;
-                        }
-
-                        value -= var_value * factor;
-                    }
-
-                    if (value != 0)
-                        break;
-
-                    arg = arg->next_arg;
-                    i++;
-                }
-
-                node->lin_index++;
-
-                if (arg == NULL) {
-                    node->value = node->eval_param(&pres->param);
-                    break;
-                }
-
-                // Unbind all variables
-                for (size_t j = 0; j < node->var_count; j++) {
-                    node->var_values[j] = (size_t) -1;
-                }
-            }
-
-            if (node->lin_index > node->lin->res->len)
-                return false;
-        }
-
+        if (!node->resolve(this))
+            return false;
         node = node->next;
     }
 
@@ -190,14 +475,14 @@ void PgfLinearizer::flush_pre_stack(PgfLinearizationOutputIface *out, PgfText *t
                 for (size_t j = 0; j < alt->prefixes->len; j++) {
                     ref<PgfText> prefix = *vector_elem(alt->prefixes,j);
                     if (textstarts(token, &(*prefix))) {
-                        linearize(out, pre->node, alt->form);
+                        pre->node->linearize_syms(out, this, alt->form);
                         goto done;
                     }
                 }
             }
         }
 
-        linearize(out, pre->node, pre->sym_kp->default_form);
+        pre->node->linearize_syms(out, this, pre->sym_kp->default_form);
 
     done:
         if (pre->bracket_stack != NULL)
@@ -218,179 +503,9 @@ void PgfLinearizer::BracketStack::flush(PgfLinearizationOutputIface *out)
         next->flush(out);
 
     if (begin)
-        out->begin_phrase(&node->lin->absfun->type->name, node->fid, field, &node->lin->name);
+        out->begin_phrase(cat, fid, field, fun);
     else
-        out->end_phrase(&node->lin->absfun->type->name, node->fid, field, &node->lin->name);
-}
-
-void PgfLinearizer::linearize(PgfLinearizationOutputIface *out, TreeNode *node, size_t d, PgfLParam *r)
-{
-    size_t lindex = node->eval_param(r);
-    PgfText *cat = &vector_elem(node->lin->absfun->type->hypos, d)->type->name;
-    TreeNode *arg = node->args;
-    while (d > 0) {
-        arg = arg->next_arg;
-        if (arg == 0)
-            throw pgf_error("Found inconsistency in the PMCFG representation");
-        d--;
-    }
-
-    PgfText *field = NULL;
-    ref<PgfConcrLincat> lincat = namespace_lookup(concr->lincats, cat);
-    if (lincat != 0) {
-        field = &(**vector_elem(lincat->fields, lindex));
-    }
-
-    if (pre_stack == NULL)
-        out->begin_phrase(cat, arg->fid, field, &arg->lin->name);
-    else {
-        BracketStack *bracket = new BracketStack();
-        bracket->next  = pre_stack->bracket_stack;
-        bracket->begin = true;
-        bracket->node  = arg;
-        bracket->field = field;
-        pre_stack->bracket_stack = bracket;
-    }
-    linearize(out, arg, lindex);
-    if (pre_stack == NULL)
-        out->end_phrase(cat, arg->fid, field, &arg->lin->name);
-    else {
-        BracketStack *bracket = new BracketStack();
-        bracket->next  = pre_stack->bracket_stack;
-        bracket->begin = false;
-        bracket->node  = arg;
-        bracket->field = field;
-        pre_stack->bracket_stack = bracket;
-    }
-}
-
-void PgfLinearizer::linearize(PgfLinearizationOutputIface *out, TreeNode *node, ref<Vector<PgfSymbol>> syms)
-{
-    ref<Vector<PgfHypo>> hypos = node->lin->absfun->type->hypos;
-
-    for (size_t i = 0; i < syms->len; i++) {
-        PgfSymbol sym = *vector_elem(syms, i);
-
-        switch (ref<PgfSymbol>::get_tag(sym)) {
-        case PgfSymbolCat::tag: {
-            auto sym_cat = ref<PgfSymbolCat>::untagged(sym);
-            linearize(out, node, sym_cat->d, &sym_cat->r);
-            break;
-        }
-        case PgfSymbolLit::tag: {
-            auto sym_lit = ref<PgfSymbolLit>::untagged(sym);
-            linearize(out, node, sym_lit->d, &sym_lit->r);
-            break;
-        }
-        case PgfSymbolVar::tag: {
-            auto sym_var = ref<PgfSymbolVar>::untagged(sym);
-            break;
-        }
-        case PgfSymbolKS::tag: {
-            auto sym_ks = ref<PgfSymbolKS>::untagged(sym);
-
-            flush_pre_stack(out, &sym_ks->token);
-
-            switch (capit) {
-            case CAPIT_NONE:
-                out->symbol_token(&sym_ks->token);
-                break;
-            case CAPIT_FIRST: {
-                PgfText *cap = (PgfText *) alloca(sizeof(PgfText)+sym_ks->token.size+6);
-
-                const uint8_t *p   = (const uint8_t *) sym_ks->token.text;
-                const uint8_t *end = p + sym_ks->token.size;
-
-                uint8_t *q = (uint8_t *) cap->text;
-
-                uint32_t ucs = pgf_utf8_decode(&p);
-                ucs = pgf_utf8_to_upper(ucs);
-                pgf_utf8_encode(ucs,&q);
-
-                memcpy(q, p, (end - p)+1);
-                q += (end - p);
-
-                cap->size = q - (uint8_t *) cap->text;
-                out->symbol_token(cap);
-
-                capit = CAPIT_NONE;
-                break;
-            }
-            case CAPIT_ALL: {
-                PgfText *cap = (PgfText *) alloca(sizeof(PgfText)+sym_ks->token.size*6);
-
-                const uint8_t *p   = (const uint8_t *) sym_ks->token.text;
-                const uint8_t *end = p + sym_ks->token.size;
-
-                uint8_t *q = (uint8_t *) cap->text;
-
-                while (p != end) {
-                    uint32_t ucs = pgf_utf8_decode(&p);
-                    ucs = pgf_utf8_to_upper(ucs);
-                    pgf_utf8_encode(ucs,&q);
-                }
-
-                cap->size = q - (uint8_t *) cap->text;
-                *q = 0;
-
-                out->symbol_token(cap);
-
-                capit = CAPIT_NONE;
-                break;
-            }
-            }
-            break;
-        }
-        case PgfSymbolKP::tag: {
-            auto sym_kp = ref<PgfSymbolKP>::untagged(sym);
-            PreStack *pre = new PreStack();
-            pre->next   = pre_stack;
-            pre->node   = node;
-            pre->sym_kp = sym_kp;
-            pre->bind   = false;
-            pre->capit  = CAPIT_NONE;
-            pre->bracket_stack = NULL;
-            pre_stack   = pre;
-            break;
-        }
-        case PgfSymbolBIND::tag:
-        case PgfSymbolSOFTBIND::tag:
-            if (pre_stack == NULL)
-                out->symbol_bind();
-            else
-                pre_stack->bind = true;
-            break;
-        case PgfSymbolNE::tag:
-            out->symbol_ne();
-            break;
-        case PgfSymbolSOFTSPACE::tag:
-            // Nothing to do
-            break;
-        case PgfSymbolCAPIT::tag:
-            if (pre_stack == NULL)
-                capit = CAPIT_FIRST;
-            else
-                pre_stack->capit = CAPIT_FIRST;
-            break;
-        case PgfSymbolALLCAPIT::tag:
-            if (pre_stack == NULL)
-                capit = CAPIT_ALL;
-            else
-                pre_stack->capit = CAPIT_ALL;
-            break;
-        }
-    }
-}
-
-void PgfLinearizer::linearize(PgfLinearizationOutputIface *out, TreeNode *node, size_t lindex)
-{
-    if (node->literal == NULL) {
-        size_t n_seqs = node->lin->seqs->len / node->lin->res->len;
-        ref<Vector<PgfSymbol>> syms = *vector_elem(node->lin->seqs, (node->lin_index-1)*n_seqs + lindex);
-        linearize(out, node, syms);
-    } else {
-        out->symbol_token(node->literal);
-    }
+        out->end_phrase(cat, fid, field, fun);
 }
 
 PgfExpr PgfLinearizer::eabs(PgfBindType btype, PgfText *name, PgfExpr body)
@@ -416,14 +531,26 @@ PgfExpr PgfLinearizer::elit(PgfLiteral lit)
 
 PgfExpr PgfLinearizer::emeta(PgfMetaId meta)
 {
-    return 0;
+    PgfPrinter printer(NULL,0,NULL);
+    if (meta == 0)
+        printer.puts("?");
+    else
+        printer.nprintf(32,"?%d",meta);
+    return (PgfExpr) new TreeLindefNode(this, printer.get_text());
 }
 
 PgfExpr PgfLinearizer::efun(PgfText *name)
 {
     ref<PgfConcrLin> lin = namespace_lookup(concr->lins, name);
-    TreeNode *node = new TreeNode(this, lin, NULL);
-    return (PgfExpr) node;
+    if (lin != 0)
+        return (PgfExpr) new TreeLinNode(this, lin);
+    else {
+        PgfPrinter printer(NULL,0,NULL);
+        printer.puts("[");
+        printer.efun(name);
+        printer.puts("]");
+        return (PgfExpr) new TreeLindefNode(this, printer.get_text());
+    }
 }
 
 PgfExpr PgfLinearizer::evar(int index)
@@ -443,21 +570,38 @@ PgfExpr PgfLinearizer::eimplarg(PgfExpr expr)
 
 PgfLiteral PgfLinearizer::lint(size_t size, uintmax_t *v)
 {
+    PgfText *cat = (PgfText *) alloca(sizeof(PgfText)+4);
+    cat->size = 3;
+    strcpy(cat->text, "Int");
+    ref<PgfConcrLincat> lincat = namespace_lookup(concr->lincats, cat);
+
     PgfPrinter printer(NULL,0,NULL);
     printer.lint(size,v);
-    return (PgfExpr) new TreeNode(this, 0, printer.get_text());
+
+    return (PgfExpr) new TreeLitNode(this, lincat, printer.get_text());
 }
 
 PgfLiteral PgfLinearizer::lflt(double v)
-{
+{    
+    PgfText *cat = (PgfText *) alloca(sizeof(PgfText)+6);
+    cat->size = 5;
+    strcpy(cat->text, "Float");
+    ref<PgfConcrLincat> lincat = namespace_lookup(concr->lincats, cat);
+
     PgfPrinter printer(NULL,0,NULL);
     printer.lflt(v);
-    return (PgfExpr) new TreeNode(this, 0, printer.get_text());
+
+    return (PgfExpr) new TreeLitNode(this, lincat, printer.get_text());
 }
 
 PgfLiteral PgfLinearizer::lstr(PgfText *v)
 {
-    return (PgfExpr) new TreeNode(this, 0, textdup(v));
+    PgfText *cat = (PgfText *) alloca(sizeof(PgfText)+7);
+    cat->size = 6;
+    strcpy(cat->text, "String");
+    ref<PgfConcrLincat> lincat = namespace_lookup(concr->lincats, cat);
+
+    return (PgfExpr) new TreeLitNode(this, lincat, textdup(v));
 }
 
 PgfType PgfLinearizer::dtyp(size_t n_hypos, PgfTypeHypo *hypos,

src/runtime/c/pgf/linearizer.h

@@ -29,17 +29,52 @@ class PGF_INTERNAL_DECL PgfLinearizer : public PgfUnmarshaller {
         TreeNode *args;
 
         int fid;
-        PgfText *literal;      // != NULL if literal
-        ref<PgfConcrLin> lin;  // != 0    if function
-        size_t lin_index;
 
         size_t value;
         size_t var_count;
         size_t *var_values;
 
-        TreeNode(PgfLinearizer *linearizer, ref<PgfConcrLin> lin, PgfText *lit);
-        ~TreeNode() { free(literal); free(var_values); };
+        TreeNode(PgfLinearizer *linearizer);
+        virtual bool resolve(PgfLinearizer *linearizer) { return true; };
+        virtual void check_category(PgfLinearizer *linearizer, PgfText *cat)=0;
+        virtual void linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, PgfLParam *r);
+        virtual void linearize_syms(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, ref<Vector<PgfSymbol>> syms);
+        virtual void linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex)=0;
         size_t eval_param(PgfLParam *param);
+        virtual ~TreeNode() { free(var_values); };
+    };
+
+    struct TreeLinNode : public TreeNode {
+        ref<PgfConcrLin> lin;
+        size_t lin_index;
+
+        TreeLinNode(PgfLinearizer *linearizer, ref<PgfConcrLin> lin);
+        virtual bool resolve(PgfLinearizer *linearizer);
+        virtual void check_category(PgfLinearizer *linearizer, PgfText *cat);
+        virtual void linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex);
+    };
+
+    struct TreeLindefNode : public TreeNode {
+        ref<PgfConcrLincat> lincat;
+        size_t lin_index;
+        PgfText *literal;
+
+        TreeLindefNode(PgfLinearizer *linearizer, PgfText *lit);
+        virtual bool resolve(PgfLinearizer *linearizer);
+        virtual void check_category(PgfLinearizer *linearizer, PgfText *cat);
+        virtual void linearize_arg(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t d, PgfLParam *r);
+        virtual void linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex);
+        ~TreeLindefNode() { free(literal); };
+    };
+
+    struct TreeLitNode : public TreeNode {
+        ref<PgfConcrLincat> lincat;
+        PgfText *literal;
+
+        TreeLitNode(PgfLinearizer *linearizer, ref<PgfConcrLincat> lincat, PgfText *lit);
+        virtual void check_category(PgfLinearizer *linearizer, PgfText *cat);
+        virtual void linearize(PgfLinearizationOutputIface *out, PgfLinearizer *linearizer, size_t lindex);
+        ~TreeLitNode() { free(literal); };
     };
 
     TreeNode *root;
@@ -53,8 +88,10 @@ class PGF_INTERNAL_DECL PgfLinearizer : public PgfUnmarshaller {
     struct BracketStack {
         BracketStack *next;
         bool begin;
-        TreeNode *node;
+        int fid;
+        PgfText *cat;
         PgfText *field;
+        PgfText *fun;
 
         void flush(PgfLinearizationOutputIface *out);
     };
@@ -71,9 +108,7 @@ class PGF_INTERNAL_DECL PgfLinearizer : public PgfUnmarshaller {
     PreStack *pre_stack;
     void flush_pre_stack(PgfLinearizationOutputIface *out, PgfText *token);
 
-    void linearize(PgfLinearizationOutputIface *out, TreeNode *node, size_t d, PgfLParam *r);
-    void linearize(PgfLinearizationOutputIface *out, TreeNode *node, ref<Vector<PgfSymbol>> syms);
-    void linearize(PgfLinearizationOutputIface *out, TreeNode *node, size_t lindex);
+    PgfText *wild;
 
 public:
     PgfLinearizer(ref<PgfConcr> concr, PgfMarshaller *m);
@@ -81,7 +116,7 @@ public:
     bool resolve();
     void reverse_and_label();
     void linearize(PgfLinearizationOutputIface *out) {
-        linearize(out, root, 0);
+        root->linearize(out, this, 0);
         flush_pre_stack(out, NULL);
     }