better handling for nonExist in the C runtime. BIND is also supported in the linearizer but not in the parser yet

src/runtime/c/pgf/linearizer.c

@@ -454,19 +454,22 @@ pgf_lzr_concretize(PgfConcr* concr, PgfExpr expr, GuPool* pool)
 }
 
 void
-pgf_lzr_linearize_sequence(PgfConcr* concr, 
+pgf_lzr_linearize_sequence(PgfConcr* concr, PgfCncTreeApp* fapp, 
-                           PgfCncTreeApp* fapp, PgfSequence* seq, 
+                           PgfSequence* seq, uint16_t seq_idx,
                            PgfLinFuncs** fnsp)
 {
 	size_t nsyms = gu_seq_length(seq);
 	PgfSymbol* syms = gu_seq_data(seq);
-	for (size_t i = 0; i < nsyms; i++) {
+	for (size_t i = seq_idx; i < nsyms; i++) {
 		PgfSymbol sym = syms[i];
 		GuVariantInfo sym_i = gu_variant_open(sym);
 		switch (sym_i.tag) {
 		case PGF_SYMBOL_CAT:
 		case PGF_SYMBOL_VAR:
 		case PGF_SYMBOL_LIT: {
+			if (fapp == NULL)
+				return;
+
 			PgfSymbolIdx* sidx = sym_i.data;
 			gu_assert((unsigned) sidx->d < fapp->n_args);
 
@@ -484,11 +487,19 @@ pgf_lzr_linearize_sequence(PgfConcr* concr,
 		case PGF_SYMBOL_KP: {
 			// TODO: correct prefix-dependencies
 			PgfSymbolKP* kp = sym_i.data;
-			pgf_lzr_linearize_sequence(concr, fapp, kp->default_form, fnsp);
+			pgf_lzr_linearize_sequence(concr, fapp, kp->default_form, 0, fnsp);
 			break;
 		}
 		case PGF_SYMBOL_NE: {
-			// Nothing to be done here
+			if ((*fnsp)->symbol_ne) {
+				(*fnsp)->symbol_ne(fnsp);
+			}
+			break;
+		}
+		case PGF_SYMBOL_BIND: {
+			if ((*fnsp)->symbol_bind) {
+				(*fnsp)->symbol_bind(fnsp);
+			}
 			break;
 		}
 		default:
@@ -518,7 +529,7 @@ pgf_lzr_linearize(PgfConcr* concr, PgfCncTree ctree, size_t lin_idx, PgfLinFuncs
 		gu_require(lin_idx < fun->n_lins);
 
 		PgfSequence* seq = fun->lins[lin_idx];
-		pgf_lzr_linearize_sequence(concr, fapp, seq, fnsp);
+		pgf_lzr_linearize_sequence(concr, fapp, seq, 0, fnsp);
 		
 		if (fns->end_phrase) {
 			fns->end_phrase(fnsp,
@@ -572,11 +583,22 @@ typedef struct PgfSimpleLin PgfSimpleLin;
 
 struct PgfSimpleLin {
 	PgfLinFuncs* funcs;
-	int n_tokens;
+	bool bind;
 	GuOut* out;
 	GuExn* err;
 };
 
+GU_DEFINE_TYPE(PgfLinNonExist, abstract, _);
+
+static void
+pgf_file_lzn_put_space(PgfSimpleLin* flin)
+{
+	if (flin->bind)
+		flin->bind = false;
+	else
+		gu_putc(' ', flin->out, flin->err);
+}
+
 static void
 pgf_file_lzn_symbol_token(PgfLinFuncs** funcs, PgfToken tok)
 {
@@ -584,12 +606,9 @@ pgf_file_lzn_symbol_token(PgfLinFuncs** funcs, PgfToken tok)
 	if (!gu_ok(flin->err)) {
 		return;
 	}
-	if (flin->n_tokens > 0)
-		gu_putc(' ', flin->out, flin->err);
 
+	pgf_file_lzn_put_space(flin);
 	gu_string_write(tok, flin->out, flin->err);
-	
-	flin->n_tokens++;
 }
 
 static void
@@ -600,8 +619,7 @@ pgf_file_lzn_expr_literal(PgfLinFuncs** funcs, PgfLiteral lit)
 		return;
 	}
 
-	if (flin->n_tokens > 0)
+	pgf_file_lzn_put_space(flin);
-		gu_putc(' ', flin->out, flin->err);
 
 	GuVariantInfo i = gu_variant_open(lit);
     switch (i.tag) {
@@ -623,8 +641,20 @@ pgf_file_lzn_expr_literal(PgfLinFuncs** funcs, PgfLiteral lit)
 	default:
 		gu_impossible();
 	}
+}
 
-	flin->n_tokens++;
+static void
+pgf_file_lzn_symbol_ne(PgfLinFuncs** funcs)
+{
+	PgfSimpleLin* flin = gu_container(funcs, PgfSimpleLin, funcs);
+	gu_raise(flin->err, PgfLinNonExist);
+}
+
+static void
+pgf_file_lzn_symbol_bind(PgfLinFuncs** funcs)
+{
+	PgfSimpleLin* flin = gu_container(funcs, PgfSimpleLin, funcs);
+	flin->bind = true;
 }
 
 static PgfLinFuncs pgf_file_lin_funcs = {
@@ -632,6 +662,8 @@ static PgfLinFuncs pgf_file_lin_funcs = {
 	.expr_literal = pgf_file_lzn_expr_literal,
 	.begin_phrase = NULL,
 	.end_phrase   = NULL,
+	.symbol_ne    = pgf_file_lzn_symbol_ne,
+	.symbol_bind  = pgf_file_lzn_symbol_bind
 };
 
 void
@@ -640,9 +672,34 @@ pgf_lzr_linearize_simple(PgfConcr* concr, PgfCncTree ctree,
 {
 	PgfSimpleLin flin = {
 		.funcs = &pgf_file_lin_funcs,
-		.n_tokens = 0,
+		.bind = true,
 		.out = out,
 		.err = err
 	};
 	pgf_lzr_linearize(concr, ctree, lin_idx, &flin.funcs);
 }
+
+GuString
+pgf_get_tokens(PgfSequence* seq, uint16_t seq_idx, GuPool* pool)
+{
+	GuPool* tmp_pool = gu_new_pool();
+	GuExn* err = gu_new_exn(NULL, gu_kind(type), tmp_pool);
+	GuStringBuf* sbuf = gu_string_buf(tmp_pool);
+	GuOut* out = gu_string_buf_out(sbuf);
+
+	PgfSimpleLin flin = {
+		.funcs = &pgf_file_lin_funcs,
+		.bind = true,
+		.out = out,
+		.err = err
+	};
+
+	pgf_lzr_linearize_sequence(NULL, NULL, seq, seq_idx, &flin.funcs);
+
+	GuString tokens = gu_ok(err) ? gu_string_buf_freeze(sbuf, pool)
+	                             : gu_empty_string;
+
+	gu_pool_free(tmp_pool);
+
+	return tokens;
+}

src/runtime/c/pgf/linearizer.h

@@ -1,21 +1,5 @@
-/* 
+#ifndef PGF_LINEARIZER_H_
- * Copyright 2010-2011 University of Helsinki.
+#define PGF_LINEARIZER_H_
- *   
- * This file is part of libpgf.
- * 
- * Libpgf is free software: you can redistribute it and/or modify it under
- * the terms of the GNU Lesser General Public License as published by the
- * Free Software Foundation, either version 3 of the License, or (at your
- * option) any later version.
- * 
- * Libpgf is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
- * License for more details.
- * 
- * You should have received a copy of the GNU Lesser General Public
- * License along with libpgf. If not, see <http://www.gnu.org/licenses/>.
- */
 
 #include <gu/type.h>
 #include <gu/enum.h>
@@ -46,6 +30,11 @@ typedef GuEnum PgfCncTreeEnum;
 PgfCncTreeEnum*
 pgf_lzr_concretize(PgfConcr* concr, PgfExpr expr, GuPool* pool);
 
+typedef struct {
+} PgfLinNonExist;
+
+extern GU_DECLARE_TYPE(PgfLinNonExist, abstract);
+
 typedef struct PgfLinFuncs PgfLinFuncs;
 
 struct PgfLinFuncs
@@ -61,6 +50,12 @@ struct PgfLinFuncs
 
 	/// End phrase
 	void (*end_phrase)(PgfLinFuncs** self, PgfCId cat, int fid, int lindex, PgfCId fun);
+
+	/// handling nonExist
+	void (*symbol_ne)(PgfLinFuncs** self);
+
+	/// token binding
+	void (*symbol_bind)(PgfLinFuncs** self);
 };
 
 
@@ -76,3 +71,10 @@ pgf_lzr_linearize(PgfConcr* concr, PgfCncTree ctree, size_t lin_idx,
 void
 pgf_lzr_linearize_simple(PgfConcr* concr, PgfCncTree ctree,
 			 size_t lin_idx, GuOut* out, GuExn* err);
+#endif
+
+#ifdef PGF_PARSER_H_
+// Used internally in the parser
+GuString
+pgf_get_tokens(PgfSequence* seq, uint16_t seq_idx, GuPool* pool);
+#endif

src/runtime/c/pgf/parser.c

@@ -1,4 +1,5 @@
 #include <pgf/parser.h>
+#include <pgf/linearizer.h>
 #include <gu/seq.h>
 #include <gu/assert.h>
 
@@ -1734,38 +1735,6 @@ typedef struct {
 	PgfParseState* state;
 } PgfPrefixTokenState;
 
-static GuString
-pgf_get_tokens(PgfSequence* seq, uint16_t seq_idx, GuPool* pool)
-{
-	GuPool* tmp_pool = gu_new_pool();
-	GuExn* err = gu_new_exn(NULL, gu_kind(type), tmp_pool);
-	GuStringBuf* sbuf = gu_string_buf(tmp_pool);
-	GuOut* out = gu_string_buf_out(sbuf);
-
-	// collect the tokens in the production
-	size_t len = gu_seq_length(seq);
-	for (size_t i = seq_idx; i < len; i++) {
-		PgfSymbol sym = gu_seq_get(seq, PgfSymbol, i);
-
-		GuVariantInfo i = gu_variant_open(sym);
-		switch (i.tag) {
-		case PGF_SYMBOL_KS: {
-			PgfSymbolKS* symks = i.data;
-			gu_string_write(symks->token, out, err);						
-		}
-		default:
-			goto end;
-		}
-	}
-	
-end:;
-	GuString tokens = gu_string_buf_freeze(sbuf, pool);
-	
-	gu_pool_free(tmp_pool);
-	
-	return tokens;
-}
-
 static bool
 pgf_prefix_match_token(PgfTokenState* ts0, PgfToken tok, PgfItem* item)
 {

src/runtime/python/pypgf.c

@@ -1378,8 +1378,12 @@ Concr_linearize(ConcrObject* self, PyObject *args)
 	
 	pgf_linearize(self->concr, pyexpr->expr, out, err);
 	if (!gu_ok(err)) {
-		PyErr_SetString(PGFError, "The abstract tree cannot be linearized");
+		if (gu_exn_caught(err) == gu_type(PgfLinNonExist))
-		return NULL;
+			Py_RETURN_NONE;
+		else {
+			PyErr_SetString(PGFError, "The abstract tree cannot be linearized");
+			return NULL;
+		}
 	}
 
 	GuString str = gu_string_buf_freeze(sbuf, tmp_pool);