exhaustive generation without HOAS and depth limit

src/runtime/c/pgf/data.h

@@ -6,6 +6,8 @@
 #include <exception>
 #include <stdexcept>
 #include <functional>
+#include <queue>
+#include <map>
 
 #include "pgf.h"
 

src/runtime/c/pgf/generator.cxx

@@ -178,11 +178,11 @@ again:  {
     if (expr == 0) {
         if (strcmp(cat->text, "Int") == 0) {
             uintmax_t value = 999;
-            PgfExpr lint = u->lint(1,&value);
+            PgfLiteral lint = u->lint(1,&value);
             expr = u->elit(lint);
             u->free_ref(lint);
         } else if (strcmp(cat->text, "Float") == 0) {
-            PgfExpr lflt = u->lflt(3.14);
+            PgfLiteral lflt = u->lflt(3.14);
             expr = u->elit(lflt);
             u->free_ref(lflt);
         } else if (strcmp(cat->text, "String") == 0) {
@@ -190,7 +190,7 @@ again:  {
             value->size = 3;
             strcpy(value->text, "Foo");
 
-            PgfExpr lstr = u->lstr(value);
+            PgfLiteral lstr = u->lstr(value);
             expr = u->elit(lstr);
             u->free_ref(lstr);
         } else {
@@ -268,3 +268,323 @@ PgfExpr PgfRandomGenerator::descend(PgfExpr expr,
 
     return expr;
 }
+
+
+
+
+
+
+PgfExhaustiveGenerator::PgfExhaustiveGenerator(ref<PgfPGF> pgf,
+                                               size_t depth,
+                                               PgfMarshaller *m, PgfUnmarshaller *u)
+{
+    this->pgf = pgf;
+    this->depth = depth;
+    this->m = m;
+    this->top_res = NULL;
+    this->top_res_index = 0;
+    
+    PgfText *text_Int = string2text("Int");
+    ref<PgfAbsCat> cat_Int =
+        namespace_lookup(pgf->abstract.cats, text_Int);
+    free(text_Int);
+    if (cat_Int != 0) {
+        uintmax_t value = 999;
+        PgfLiteral lint = u->lint(1,&value);
+        PgfExpr expr = u->elit(lint);
+        u->free_ref(lint);
+        Result *res = new Result();
+        res->exprs.push_back(std::pair<PgfExpr,prob_t>(expr,0));
+        results[ref<PgfText>::from_ptr(&cat_Int->name)] = res;
+    }
+
+    PgfText *text_Float = string2text("Float");
+    ref<PgfAbsCat> cat_Float =
+        namespace_lookup(pgf->abstract.cats, text_Float);
+    free(text_Float);
+    if (cat_Float != 0) {
+        PgfLiteral lflt = u->lflt(3.14);
+        PgfExpr expr = u->elit(lflt);
+        u->free_ref(lflt);
+        Result *res = new Result();
+        res->exprs.push_back(std::pair<PgfExpr,prob_t>(expr,0));
+        results[ref<PgfText>::from_ptr(&cat_Float->name)] = res;
+    }
+
+    PgfText *text_String = string2text("String");
+    ref<PgfAbsCat> cat_String =
+        namespace_lookup(pgf->abstract.cats, text_String);
+    free(text_String);
+    if (cat_String != 0) {
+        PgfText *value = (PgfText *) alloca(sizeof(PgfText)+4);
+        value->size = 3;
+        strcpy(value->text, "Foo");
+
+        PgfLiteral lstr = u->lstr(value);
+        PgfExpr expr = u->elit(lstr);
+        u->free_ref(lstr);
+        Result *res = new Result();
+        res->exprs.push_back(std::pair<PgfExpr,prob_t>(expr,0));
+        results[ref<PgfText>::from_ptr(&cat_String->name)] = res;
+    }
+}
+
+PgfExpr PgfExhaustiveGenerator::eabs(PgfBindType btype, PgfText *name, PgfExpr body)
+{
+    return 0;
+}
+
+PgfExpr PgfExhaustiveGenerator::eapp(PgfExpr fun, PgfExpr arg)
+{
+    return 0;
+}
+
+PgfExpr PgfExhaustiveGenerator::elit(PgfLiteral lit)
+{
+    return 0;
+}
+
+PgfExpr PgfExhaustiveGenerator::emeta(PgfMetaId meta)
+{
+    return 0;
+}
+
+PgfExpr PgfExhaustiveGenerator::efun(PgfText *name)
+{
+    return 0;
+}
+
+PgfExpr PgfExhaustiveGenerator::evar(int index)
+{
+    return 0;
+}
+
+PgfExpr PgfExhaustiveGenerator::etyped(PgfExpr expr, PgfType typ)
+{
+    return 0;
+}
+
+PgfExpr PgfExhaustiveGenerator::eimplarg(PgfExpr expr)
+{
+    return 0;
+}
+
+PgfLiteral PgfExhaustiveGenerator::lint(size_t size, uintmax_t *v)
+{
+    return 0;
+}
+
+PgfLiteral PgfExhaustiveGenerator::lflt(double v)
+{
+    return 0;
+}
+
+PgfLiteral PgfExhaustiveGenerator::lstr(PgfText *v)
+{
+    return 0;
+}
+
+void PgfExhaustiveGenerator::push_left_states(PgfProbspace space, PgfText *cat, Result *res)
+{
+    while (space != 0) {
+        int cmp = textcmp(cat,&(*space->value.cat));
+        if (cmp < 0) {
+            space = space->left;
+        } else if (cmp > 0) {
+            space = space->right;
+        } else {
+            if (space->value.is_result()) {
+                res->ref_count++;
+
+                State0 *state = new State0();
+                state->res   = res;
+                state->prob  = res->outside_prob(this) +
+                               space->value.fun->prob;
+                state->space = space;
+                queue.push(state);
+            } else {
+                push_left_states(space->right, cat, res);
+            }
+            space = space->left;
+        }
+    }
+}
+
+PgfType PgfExhaustiveGenerator::dtyp(size_t n_hypos, PgfTypeHypo *hypos,
+                                     PgfText *cat,
+                                     size_t n_exprs, PgfExpr *exprs)
+{
+    ref<PgfAbsCat> abscat =
+        namespace_lookup(pgf->abstract.cats, cat);
+    if (abscat == 0)
+        return 0;
+
+    Result *&res = results[ref<PgfText>::from_ptr(&abscat->name)];
+    if (res == NULL) {
+        res = new Result();
+    }
+    top_res = res;
+
+    push_left_states(pgf->abstract.funs_by_cat, cat, top_res);
+    return 0;
+}
+
+void PgfExhaustiveGenerator::free_ref(object x)
+{
+}
+
+void PgfExhaustiveGenerator::State::free_refs(PgfUnmarshaller *u)
+{
+}
+
+void PgfExhaustiveGenerator::State::release(State *state, PgfUnmarshaller *u)
+{
+    state->res->ref_count--;
+    if (state->res->ref_count == 0) {
+        while (!state->res->states.empty()) {
+            State1 *parent = state->res->states.back();
+            u->free_ref(parent->expr);
+            State::release(parent,u);
+            state->res->states.pop_back();
+        }
+        state->res->states.shrink_to_fit();
+    }
+
+    delete state;
+}
+
+bool PgfExhaustiveGenerator::State0::process(PgfExhaustiveGenerator *gen, PgfUnmarshaller *u)
+{
+    gen->push_left_states(space->right, &(*space->value.cat), res);
+
+    ref<PgfAbsFun> fun = space->value.fun;
+    PgfExpr expr = u->efun(&fun->name);
+
+    res->ref_count++;
+
+    State1 *state = new State1();
+    state->res    = res;
+    state->prob   = prob;
+    state->type   = fun->type;
+    state->n_args = 0;
+    state->expr   = expr;
+
+    if (state->process(gen, u)) {
+        State::release(state,u);
+    }
+
+    return true;
+}
+
+bool PgfExhaustiveGenerator::State1::process(PgfExhaustiveGenerator *gen, PgfUnmarshaller *u)
+{
+    if (n_args >= type->hypos->len) {
+        complete(gen, u);
+        return true;
+    }
+
+    PgfDTyp *arg_type = vector_elem(type->hypos, n_args)->type;
+    Result *&res = gen->results[ref<PgfText>::from_ptr(&arg_type->name)];
+    Result *tmp  = res;
+    if (res == NULL) {
+        res = new Result();
+    }
+    res->states.push_back(this);
+
+    if (tmp == NULL) {
+        gen->push_left_states(gen->pgf->abstract.funs_by_cat, &arg_type->name, res);
+    } else {
+        for (std::pair<PgfExpr,prob_t> p : res->exprs) {
+            this->combine(gen,p.first,p.second,u);
+        }
+    }
+
+    return false;
+}
+
+void PgfExhaustiveGenerator::State1::combine(PgfExhaustiveGenerator *gen, 
+                                             PgfExpr expr, prob_t prob,
+                                             PgfUnmarshaller *u)
+{
+    PgfBindType bind_type = vector_elem(type->hypos, n_args)->bind_type;
+
+    if (bind_type == PGF_BIND_TYPE_IMPLICIT) {
+        expr = u->eimplarg(expr);
+    }
+
+    PgfExpr app = u->eapp(this->expr, expr);
+
+    if (bind_type == PGF_BIND_TYPE_IMPLICIT) {
+        u->free_ref(expr);
+    }
+
+    res->ref_count++;
+
+    State1 *app_state = new State1();
+    app_state->res    = res;
+    app_state->prob   = this->prob + prob;
+    app_state->type   = type;
+    app_state->n_args = n_args+1;
+    app_state->expr   = app;
+    gen->queue.push(app_state);
+}
+
+void PgfExhaustiveGenerator::State1::complete(PgfExhaustiveGenerator *gen, PgfUnmarshaller *u)
+{
+    prob_t inside_prob = prob-res->outside_prob(gen);
+    res->exprs.push_back(std::pair<PgfExpr,prob_t>(expr,inside_prob));
+    for (State1 *state : res->states) {
+        state->combine(gen,expr,inside_prob,u);
+    }
+}
+
+void PgfExhaustiveGenerator::State1::free_refs(PgfUnmarshaller *u)
+{
+    u->free_ref(expr);
+}
+
+PgfExhaustiveGenerator::Result::Result()
+{
+    ref_count = 0;
+}
+
+PgfExpr PgfExhaustiveGenerator::fetch(PgfDB *db, PgfUnmarshaller *u, prob_t *prob)
+{
+    DB_scope scope(db, READER_SCOPE);
+
+    for (;;) {
+        if (top_res_index < top_res->exprs.size()) {
+            auto pair = top_res->exprs[top_res_index++];
+            *prob = pair.second;
+            return pair.first;
+        }
+
+        if (queue.empty())
+            return 0;
+
+        State *state = queue.top(); queue.pop();
+
+        if (state->process(this, u)) {
+            State::release(state, u);
+        }
+    }
+}
+
+void PgfExhaustiveGenerator::free_refs(PgfUnmarshaller *u)
+{
+    while (!queue.empty()) {
+        State  *state = queue.top(); queue.pop();
+        state->free_refs(u);
+        State::release(state,u);
+    }
+
+    for (auto i : results) {
+        for (auto j : i.second->exprs) {
+            free_ref(j.first);
+        }
+    }
+}
+
+PgfExhaustiveGenerator::~PgfExhaustiveGenerator()
+{
+}

src/runtime/c/pgf/generator.h

@@ -54,4 +54,98 @@ public:
     virtual void free_ref(object x);
 };
 
+class PGF_INTERNAL_DECL PgfExhaustiveGenerator : public PgfUnmarshaller, public PgfExprEnum
+{
+    struct Result;
+
+    ref<PgfPGF> pgf;
+    size_t depth;
+    PgfMarshaller *m;
+    Result *top_res;
+    size_t top_res_index;
+
+    struct State {
+        Result *res;
+        prob_t prob;
+        virtual bool process(PgfExhaustiveGenerator *gen, PgfUnmarshaller *u) = 0;
+        virtual void free_refs(PgfUnmarshaller *u);
+        static void release(State *state, PgfUnmarshaller *u);
+    };
+
+    struct State0 : State {
+        PgfProbspace space;
+        virtual bool process(PgfExhaustiveGenerator *gen, PgfUnmarshaller *u);
+    };
+
+    struct State1 : State {
+        ref<PgfDTyp> type;
+        size_t n_args;
+        PgfExpr expr;
+
+        virtual bool process(PgfExhaustiveGenerator *gen, PgfUnmarshaller *u);
+        virtual void free_refs(PgfUnmarshaller *u);
+        void combine(PgfExhaustiveGenerator *gen, 
+                     PgfExpr expr, prob_t prob,
+                     PgfUnmarshaller *u);
+        void complete(PgfExhaustiveGenerator *gen, PgfUnmarshaller *u);
+    };
+
+    struct Result {
+        size_t ref_count;
+        std::vector<State1*> states;
+        std::vector<std::pair<PgfExpr,prob_t>> exprs;
+
+        Result();
+
+        prob_t outside_prob(PgfExhaustiveGenerator *gen) {
+            if (this == gen->top_res)
+                return 0;
+            return states[0]->prob;
+        }
+    };
+
+    class CompareState : public std::less<State*> {
+    public:
+        bool operator() (const State* s1, const State* s2) const {
+            return s1->prob > s2->prob;
+        }
+    };
+
+    class CompareText : public std::less<ref<PgfText>> {
+    public:
+        bool operator() (const ref<PgfText> t1, const ref<PgfText> t2) const {
+            return textcmp(t1, t2) < 0;
+        }
+    };
+
+    std::map<ref<PgfText>, Result*, CompareText> results;
+    std::priority_queue<State*, std::vector<State*>, CompareState> queue;
+
+    void push_left_states(PgfProbspace space, PgfText *cat, Result *res);
+
+public:
+    PgfExhaustiveGenerator(ref<PgfPGF> pgf, size_t depth,
+                           PgfMarshaller *m, PgfUnmarshaller *u);
+    virtual ~PgfExhaustiveGenerator();
+
+    virtual PgfExpr eabs(PgfBindType btype, PgfText *name, PgfExpr body);
+    virtual PgfExpr eapp(PgfExpr fun, PgfExpr arg);
+    virtual PgfExpr elit(PgfLiteral lit);
+    virtual PgfExpr emeta(PgfMetaId meta);
+    virtual PgfExpr efun(PgfText *name);
+    virtual PgfExpr evar(int index);
+    virtual PgfExpr etyped(PgfExpr expr, PgfType typ);
+    virtual PgfExpr eimplarg(PgfExpr expr);
+    virtual PgfLiteral lint(size_t size, uintmax_t *v);
+    virtual PgfLiteral lflt(double v);
+    virtual PgfLiteral lstr(PgfText *v);
+    virtual PgfType dtyp(size_t n_hypos, PgfTypeHypo *hypos,
+                         PgfText *cat,
+                         size_t n_exprs, PgfExpr *exprs);
+    virtual void free_ref(object x);
+
+    virtual PgfExpr fetch(PgfDB *db, PgfUnmarshaller *u, prob_t *prob);
+    virtual void free_refs(PgfUnmarshaller *u);
+};
+
 #endif

src/runtime/c/pgf/parser.cxx

@@ -794,6 +794,10 @@ PgfExpr PgfParser::fetch(PgfDB *db, PgfUnmarshaller *u, prob_t *prob)
     return 0;
 }
 
+void PgfParser::free_refs(PgfUnmarshaller *u)
+{
+}
+
 PgfParser::~PgfParser()
 {
     free(sentence);

src/runtime/c/pgf/parser.h

@@ -13,6 +13,7 @@ public:
     void prepare();
     PgfExpr fetch(PgfDB *db, PgfUnmarshaller *u, prob_t *prob);
 
+    virtual void free_refs(PgfUnmarshaller *u);
     virtual ~PgfParser();
 
 private:

src/runtime/c/pgf/pgf.cxx

@@ -1247,6 +1247,25 @@ PgfExpr pgf_generate_random_from
     return 0;
 }
 
+PGF_API
+PgfExprEnum *pgf_generate_all(PgfDB *db, PgfRevision revision,
+                              PgfType type, size_t depth,
+                              PgfMarshaller *m, PgfUnmarshaller *u,
+                              PgfExn *err)
+{
+    PGF_API_BEGIN {
+        DB_scope scope(db, READER_SCOPE);
+
+        ref<PgfPGF> pgf = db->revision2pgf(revision);
+
+        PgfExhaustiveGenerator *gen = new PgfExhaustiveGenerator(pgf, depth, m, u);
+        m->match_type(gen, type);
+        return gen;
+    } PGF_API_END
+
+    return NULL;
+}
+
 PGF_API
 PgfRevision pgf_start_transaction(PgfDB *db, PgfExn *err)
 {
@@ -2644,8 +2663,9 @@ PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision,
 }
 
 PGF_API
-void pgf_free_expr_enum(PgfExprEnum *en)
+void pgf_free_expr_enum(PgfUnmarshaller *u, PgfExprEnum *en)
 {
+    en->free_refs(u);
     delete en;
 }
 

src/runtime/c/pgf/pgf.h

@@ -524,6 +524,29 @@ PgfExpr pgf_generate_random_from
                             PgfMarshaller *m, PgfUnmarshaller *u,
                             PgfExn *err);
 
+#ifdef __cplusplus
+struct PgfExprEnum {
+    virtual PgfExpr fetch(PgfDB *db, PgfUnmarshaller *u, prob_t *prob)=0;
+    virtual void free_refs(PgfUnmarshaller *u)=0;
+    virtual ~PgfExprEnum() {};
+};
+#else
+typedef struct PgfExprEnum PgfExprEnum;
+typedef struct PgfExprEnumVtbl PgfExprEnumVtbl;
+struct PgfExprEnumVtbl {
+    PgfExpr (*fetch)(PgfExprEnum *this, PgfDB *db, PgfUnmarshaller *u, prob_t *prob);
+};
+struct PgfExprEnum {
+    PgfExprEnumVtbl *vtbl;
+};
+#endif
+
+PGF_API_DECL
+PgfExprEnum *pgf_generate_all(PgfDB *db, PgfRevision revision,
+                              PgfType type, size_t depth,
+                              PgfMarshaller *m, PgfUnmarshaller *u,
+                              PgfExn *err);
+
 PGF_API_DECL
 PgfRevision pgf_start_transaction(PgfDB *db, PgfExn *err);
 
@@ -759,22 +782,6 @@ void pgf_bracketed_linearize_all(PgfDB *db, PgfConcrRevision revision,
                                  PgfLinearizationOutputIface *out,
                                  PgfExn* err);
 
-#ifdef __cplusplus
-struct PgfExprEnum {
-    virtual PgfExpr fetch(PgfDB *db, PgfUnmarshaller *u, prob_t *prob)=0;
-    virtual ~PgfExprEnum() {};
-};
-#else
-typedef struct PgfExprEnum PgfExprEnum;
-typedef struct PgfExprEnumVtbl PgfExprEnumVtbl;
-struct PgfExprEnumVtbl {
-    PgfExpr (*fetch)(PgfExprEnum *this, PgfDB *db, PgfUnmarshaller *u, prob_t *prob);
-};
-struct PgfExprEnum {
-    PgfExprEnumVtbl *vtbl;
-};
-#endif
-
 PGF_API_DECL
 PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision,
                        PgfType ty, PgfMarshaller *m,
@@ -782,7 +789,7 @@ PgfExprEnum *pgf_parse(PgfDB *db, PgfConcrRevision revision,
                        PgfExn * err);
 
 PGF_API_DECL
-void pgf_free_expr_enum(PgfExprEnum *en);
+void pgf_free_expr_enum(PgfUnmarshaller *u, PgfExprEnum *en);
 
 PGF_API_DECL
 PgfText *pgf_get_printname(PgfDB *db, PgfConcrRevision revision,

src/runtime/c/pgf/probspace.cxx

@@ -1,6 +1,10 @@
 #include "data.h"
 #include "math.h"
 
+bool PgfProbspaceEntry::is_result() {
+    return cat == ref<PgfText>::from_ptr(&fun->type->name);
+}
+
 static
 int entry_cmp(PgfProbspaceEntry *entry1, PgfProbspaceEntry *entry2)
 {
@@ -37,7 +41,7 @@ PgfProbspace probspace_insert(PgfProbspace space,
         space = Node<PgfProbspaceEntry>::upd_node(space,space->left,space->right);
         space->value = *entry;
         return space;
-     }
+    }
 }
 
 static
@@ -188,7 +192,7 @@ void probspace_iter(PgfProbspace space, PgfText *cat,
         if (err->type != PGF_EXN_NONE)
             return;
 
-        if (all || space->value.cat == ref<PgfText>::from_ptr(&space->value.fun->type->name)) {
+        if (all || space->value.is_result()) {
             itor->fn(itor, &space->value.fun->name, space->value.fun.as_object(), err);
             if (err->type != PGF_EXN_NONE)
                 return;
@@ -220,7 +224,7 @@ ref<PgfAbsFun> probspace_random(PgfProbspace space,
         if (fun != 0)
             return fun;
 
-        bool is_res = (space->value.cat == ref<PgfText>::from_ptr(&space->value.fun->type->name));
+        bool is_res = space->value.is_result();
         if (is_res) {
             *rand -= exp(-space->value.fun->prob);
             if (*rand <= 0)

src/runtime/c/pgf/probspace.h

@@ -47,6 +47,8 @@
 struct PGF_INTERNAL_DECL PgfProbspaceEntry {
     ref<PgfText> cat;
     ref<PgfAbsFun> fun;
+    
+    bool is_result();
 };
 
 typedef ref<Node<PgfProbspaceEntry>> PgfProbspace;

src/runtime/c/pgf/text.cxx

@@ -113,6 +113,18 @@ bool textistarts(PgfText *t, PgfText *prefix)
     }
 }
 
+PGF_INTERNAL
+PgfText* string2text(const char *s)
+{
+    size_t size = strlen(s);
+    PgfText *text = (PgfText *) malloc(sizeof(PgfText)+size+1);
+    if (text != NULL) {
+        text->size = size;
+        strcpy(text->text, s);
+    }
+    return text;
+}
+
 PGF_INTERNAL
 PgfText* textdup(PgfText *t1)
 {