more comments

src/runtime/c/gu/exn.c

@@ -18,12 +18,24 @@ gu_exn_is_raised(GuExn* err) {
 	return err && (err->state == GU_EXN_RAISED);
 }
 
+GU_API_DECL void
+gu_exn_clear(GuExn* err) {
+	err->caught = NULL;
+	err->state = GU_EXN_OK;
+}
+
 GU_API bool
 gu_exn_caught_(GuExn* err, const char* type)
 {
 	return (err->caught && strcmp(err->caught, type) == 0);
 }
 
+GU_API_DECL void*
+gu_exn_caught_data(GuExn* err)
+{
+	return err->data.data;
+}
+
 GU_API void
 gu_exn_block(GuExn* err)
 {

src/runtime/c/gu/exn.h

@@ -71,11 +71,13 @@ gu_new_exn(GuPool* pool);
 GU_API_DECL bool
 gu_exn_is_raised(GuExn* err);
 
-static inline void
-gu_exn_clear(GuExn* err) {
-	err->caught = NULL;
-	err->state = GU_EXN_OK;
-}
+// static inline void
+// gu_exn_clear(GuExn* err) {
+// 	err->caught = NULL;
+// 	err->state = GU_EXN_OK;
+// }
+GU_API_DECL void
+gu_exn_clear(GuExn* err);
 
 #define gu_exn_caught(err, type) \
 	(err->caught && strcmp(err->caught, #type) == 0)
@@ -83,11 +85,13 @@ gu_exn_clear(GuExn* err) {
 GU_API_DECL bool
 gu_exn_caught_(GuExn* err, const char* type);
 
-static inline const void*
-gu_exn_caught_data(GuExn* err)
-{
-	return err->data.data;
-}
+// static inline const void*
+// gu_exn_caught_data(GuExn* err)
+// {
+// 	return err->data.data;
+// }
+GU_API_DECL void*
+gu_exn_caught_data(GuExn* err);
 
 /// Temporarily block a raised exception.
 GU_API_DECL void

src/runtime/c/pgf/jit.c

@@ -26,6 +26,16 @@ PGF_INTERNAL void
 pgf_jit_predicate(PgfReader* rdr, PgfAbstr* abstr,
                   PgfAbsCat* abscat)
 {
+    size_t n_funs = pgf_read_len(rdr);
+	gu_return_on_exn(rdr->err, );
+
+    for (size_t i = 0; i < n_funs; i++) {
+        gu_in_f64be(rdr->in, rdr->err);  // ignore
+        gu_return_on_exn(rdr->err, );
+
+        PgfCId name = pgf_read_cid(rdr, rdr->tmp_pool);
+        gu_return_on_exn(rdr->err, );
+    }
 }
 
 PGF_INTERNAL void

src/runtime/c/pgf/reader.c

@@ -1326,7 +1326,7 @@ pgf_read_concretes(PgfReader* rdr, PgfAbstr* abstr, bool with_content)
 PGF_INTERNAL PgfPGF*
 pgf_read_pgf(PgfReader* rdr) {
 	PgfPGF* pgf = gu_new(PgfPGF, rdr->opool);
-	
+
 	pgf->major_version = gu_in_u16be(rdr->in, rdr->err);
 	gu_return_on_exn(rdr->err, NULL);
 
@@ -1335,7 +1335,7 @@ pgf_read_pgf(PgfReader* rdr) {
 
 	pgf->gflags = pgf_read_flags(rdr);
 	gu_return_on_exn(rdr->err, NULL);
-		
+
 	pgf_read_abstract(rdr, &pgf->abstract);
 	gu_return_on_exn(rdr->err, NULL);
 

src/runtime/javascript/.gitignore

@@ -0,0 +1 @@
+.libs/

src/runtime/javascript/DEPRECATED.md

@@ -1,4 +0,0 @@
-# Deprecation notice
-
-As of June 2019, this JavaScript version of the GF runtime is considered deprecated,
-in favour of the TypeScript version in <https://github.com/GrammaticalFramework/gf-typescript>.

src/runtime/javascript/Dockerfile

@@ -0,0 +1,48 @@
+FROM emscripten/emsdk:latest
+
+RUN apt update
+RUN apt install -y autoconf automake libtool make
+
+WORKDIR /tmp/c
+COPY gu/*.c gu/*.h /tmp/c/gu/
+COPY pgf/*.c pgf/*.h /tmp/c/pgf/
+COPY pgf/lightning/i386/*.h /tmp/c/pgf/lightning/i386/
+COPY pgf/lightning/*.h /tmp/c/pgf/lightning/
+COPY \
+    Makefile.am \
+    configure.ac \
+    lib*.pc.in \
+    /tmp/c/
+RUN autoreconf -i
+RUN emconfigure ./configure
+RUN emmake make
+RUN emcc .libs/libgu.a .libs/libpgf.a -o pgf.js \
+    -sALLOW_MEMORY_GROWTH \
+    -sEXPORTED_FUNCTIONS="\
+        _pgf_read,\
+        _pgf_abstract_name,\
+        _pgf_read_expr,\
+        _pgf_print_expr,\
+        _pgf_expr_arity,\
+        _gu_new_pool,\
+        _gu_new_exn,\
+        _gu_data_in,\
+        _gu_exn_is_raised,\
+        _gu_exn_caught_,\
+        _gu_exn_caught_data,\
+        _gu_exn_clear,\
+        _gu_new_string_buf,\
+        _gu_string_buf_out,\
+        _gu_string_buf_data,\
+        _malloc,\
+        _free\
+        "\
+    -sEXPORTED_RUNTIME_METHODS="\
+        ccall,\
+        FS,\
+        getValue,\
+        AsciiToString,\
+        stringToUTF8,\
+        UTF8ToString,\
+        allocateUTF8\
+        "

src/runtime/javascript/README.md

@@ -0,0 +1,11 @@
+# JavaScript runtime using Web Assembly
+
+This folder contains very early work experimenting with a pure JavaScript runtime,
+compiled to Web Assembly (WASM) using [Emscripten](https://emscripten.org/).
+
+1. Compile the WASM files (inside Docker) using `build-wasm.sh`, placing them in `.libs/`
+2. Test in Node.js by running `node test-node.js [path to PGF]`
+3. Test in a web browser
+    1. Start a server with `npx serve -l 41296`
+    2. Browse to `http://localhost:41296/test-web.html`
+    3. Check JavaScript console

src/runtime/javascript/build-wasm.sh

@@ -0,0 +1,10 @@
+#! /usr/bin/env bash
+set -e
+
+# Build inside Docker image
+IMAGE="gf/build-c-runtime-wasm"
+docker build ../c --file Dockerfile --tag $IMAGE
+
+# Copy bulit files from container to host
+mkdir -p .libs
+docker run --rm --volume "$PWD":/tmp/host $IMAGE bash -c "cp pgf.js pgf.wasm /tmp/host/.libs/"

src/runtime/javascript/editor-grammar/Editor.gf

@@ -1,62 +0,0 @@
-abstract Editor = {
-
-cat Adjective ;
-    Noun ;
-    Verb ;
-    Determiner ;
-    Sentence ;
-
-fun Available : Adjective ;
-    Next      : Adjective ;
-    Previous  : Adjective ;
- 
-fun Bulgarian : Noun ;
-    Danish    : Noun ;
-    English   : Noun ;
-    Finnish   : Noun ;
-    French    : Noun ;
-    German    : Noun ;
-    Italian   : Noun ;
-    Norwegian : Noun ;
-    Russian   : Noun ;
-    Spanish   : Noun ;
-    Swedish   : Noun ;
-
-fun Float_N    : Noun ;
-    Integer_N  : Noun ;
-    String_N   : Noun ;
-    
-    Language   : Noun ;
-    Node       : Noun ;
-    Page       : Noun ;
-    Refinement : Noun ;
-    Tree       : Noun ;
-    Wrapper    : Noun ;
-
-fun Copy    : Verb ;
-    Cut     : Verb ;
-    Delete  : Verb ;
-    Enter   : Verb ;
-    Parse   : Verb ;
-    Paste   : Verb ;
-    Redo    : Verb ;
-    Refine  : Verb ;
-    Replace : Verb ;
-    Select  : Verb ;
-    Show    : Verb ;
-    Undo    : Verb ;
-    Wrap    : Verb ;
-
-fun DefPlDet   : Determiner ;
-    DefSgDet   : Determiner ;
-    IndefPlDet : Determiner ;
-    IndefSgDet : Determiner ;
-
-fun Command           : Verb -> Determiner -> Noun -> Sentence ;
-    CommandAdj        : Verb -> Determiner -> Adjective -> Noun -> Sentence ;
-    ErrorMessage      : Adjective -> Noun -> Sentence ;   
-    Label             : Noun -> Sentence ;
-    RandomlyCommand   : Verb -> Determiner -> Noun -> Sentence ;
-    SingleWordCommand : Verb -> Sentence ;
-
-}

src/runtime/javascript/editor-grammar/EditorEng.gf

@@ -1,63 +0,0 @@
---# -path=alltenses
-concrete EditorEng of Editor = open GrammarEng, ParadigmsEng in {
-
-lincat Adjective = A ;
-       Noun = N ;
-       Verb = V ;
-       Determiner = Det ;
-       Sentence = Utt ;
-
-lin Available = mkA "available" ;
-    Next      = mkA "next" ;
-    Previous  = mkA "previous" ;
-
-lin Bulgarian = mkN "Bulgarian" ;
-    Danish    = mkN "Danish" ;
-    English   = mkN "English" ;
-    Finnish   = mkN "Finnish" ;
-    French    = mkN "French" ;
-    German    = mkN "German" ;
-    Italian   = mkN "Italian" ;
-    Norwegian = mkN "Norwegian" ;
-    Russian   = mkN "Russian" ;
-    Spanish   = mkN "Spanish" ;
-    Swedish   = mkN "Swedish" ;
-
-lin Float_N    = mkN "float" ;
-    Integer_N  = mkN "integer" ;
-    String_N   = mkN "string" ;
-    
-    Language   = mkN "language" ;
-    Node       = mkN "node" ;
-    Page       = mkN "page" ;
-    Refinement = mkN "refinement" ;
-    Tree       = mkN "tree" ;
-    Wrapper    = mkN "wrapper" ;
-
-lin Copy    = mkV "copy" ;
-    Cut     = mkV "cut" ;
-    Delete  = mkV "delete" ;
-    Enter   = mkV "enter" ;
-    Parse   = mkV "parse" ;
-    Paste   = mkV "paste" ;
-    Redo    = mkV "redo" ;
-    Refine  = mkV "refine" ;
-    Replace = mkV "replace" ;
-    Select  = mkV "select" ;
-    Show    = mkV "show" ;
-    Undo    = mkV "undo" ;
-    Wrap    = mkV "wrap" ;
-
-lin DefPlDet   = DetQuant DefArt   NumPl ;
-    DefSgDet   = DetQuant DefArt   NumSg ;
-    IndefPlDet = DetQuant IndefArt NumPl ;
-    IndefSgDet = DetQuant IndefArt NumSg ;
-
-lin Command v d n = UttImpSg PPos (ImpVP (ComplSlash (SlashV2a (mkV2 v)) (DetCN d (UseN n)))) ;
-    CommandAdj v d a n = UttImpSg PPos (ImpVP (ComplSlash (SlashV2a (mkV2 v)) (DetCN d (AdjCN (PositA a) (UseN n))))) ;
-    ErrorMessage a n = UttNP (DetCN (DetQuant no_Quant NumPl) (AdjCN (PositA a) (UseN n))) ;
-    Label n = UttNP (MassNP (UseN n)) ;
-    RandomlyCommand v d n = UttImpSg PPos (ImpVP (AdvVP (ComplSlash (SlashV2a (mkV2 v)) (DetCN d (UseN n))) (PrepNP (mkPrep "at") (MassNP (UseN (mkN "random")))))) ;
-    SingleWordCommand v = UttImpSg PPos (ImpVP (UseV v)) ;
-
-}

src/runtime/javascript/editor.html

@@ -1,17 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-	<head>
-		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-  		<link rel="stylesheet" type="text/css" href="style.css" />
-		<script type="text/javascript" src="gflib.js"></script>
-		<script type="text/javascript" src="editorGrammar.js"></script>
-		<script type="text/javascript" src="grammar.js"></script>
-		<script type="text/javascript" src="gfjseditor.js"></script>
-  		<title>Web-based Syntax Editor</title>
-	</head>
-	<body onload="mkEditor('editor', Foods)" onkeydown="return hotKeys(event)">
-		<div id="editor">
-		</div>
-	</body>
-</html>

src/runtime/javascript/gflib-xhtml-voice.js

@@ -1,54 +0,0 @@
-/* Output */
-
-function sayText(text) {
-  document.voice_output_text = text;
-  activateForm("voice_output");
-}
-
-/* XHTML+Voice Utilities */
-
-function activateForm(formid) {
-  var form = document.getElementById(formid);
-  var e = document.createEvent("UIEvents");
-  e.initEvent("DOMActivate","true","true");
-  form.dispatchEvent(e); 
-}
-
-
-/* DOM utilities */
-
-/* Gets the head element of the document. */
-function getHeadElement() {
-  var hs = document.getElementsByTagName("head");
-  if (hs.length == 0) {
-    var head = document.createElement("head");
-    document.documentElement.insertBefore(head, document.documentElement.firstChild);
-    return head;
-  } else {
-    return hs[0];
-  }
-}
-
-/* Gets the body element of the document. */
-function getBodyElement() {
-  var bs = document.getElementsByTagName("body");
-  if (bs.length == 0) {
-    var body = document.createElement("body");
-    document.documentElement.appendChild(body);
-    return body;
-  } else {
-    return bs[0];
-  }
-}
-
-/* Removes all the children of a node */
-function removeChildren(node) {
-  while (node.hasChildNodes()) {
-    node.removeChild(node.firstChild);
-  }
-}
-
-function setText(node, text) {
-  removeChildren(node);
-  node.appendChild(document.createTextNode(text));
-}

src/runtime/javascript/jspgf.js

@@ -0,0 +1,543 @@
+/**
+ * This module is the high-level JavaScript wrapper around the WASM-compiled version.
+ */
+
+async function mkAPI() {
+
+    const sizeof_GuMapItor = 4;
+    const offsetof_GuMapItor_fn = 0;
+
+    var asm = null;
+    var wasmTable = null;
+    var freeTableIndexes = [];
+
+    function setErrNo(value) {
+        HEAP32[asm.__errno_location() >> 2] = value;
+        return value;
+    }
+
+    function abortOnCannotGrowMemory(requestedSize) {
+        abort('Cannot enlarge memory arrays to size ' + requestedSize + ' bytes (OOM). Either (1) compile with  -s INITIAL_MEMORY=X  with X higher than the current value ' + HEAP8.length + ', (2) compile with  -s ALLOW_MEMORY_GROWTH=1  which allows increasing the size at runtime, or (3) if you want malloc to return NULL (0) instead of this abort, compile with  -s ABORTING_MALLOC=0 ');
+    }
+
+    function _emscripten_resize_heap(requestedSize) {
+        var oldSize = HEAPU8.length;
+        requestedSize = requestedSize >>> 0;
+        abortOnCannotGrowMemory(requestedSize);
+    }
+
+    var tempRet0 = 0;
+    var urlData = {};
+    var fdData  = {};
+    var fdMax   = 0;
+    var asmLibraryArg = {
+        "__syscall_fcntl64":
+            function (fd, cmd, varargs) {
+                setErrNo(134);
+                return -1;
+            },
+
+        "__syscall_ioctl":
+            function (fd, op, varargs) {
+                setErrNo(134);
+                return -1;
+            },
+
+        "__syscall_open":
+            function (pathPtr, flags, varargs) {
+                const path = UTF8ToString(pathPtr);
+                const data = urlData[path];
+                if (data == null) {
+                    setErrNo(129);
+                    return -1;
+                }
+                fdMax++;
+                fdData[fdMax] = {data: data, pos: 0};
+                delete urlData[path];
+                return fdMax;
+            },
+
+        "_munmap_js":
+            function (addr, len, prot, flags, fd, offset) {
+                setErrNo(134);
+                return -1;
+            },
+
+        "abort":
+            function () {
+                console.log('native code called abort()');
+            },
+
+        "emscripten_memcpy_big":
+            function (dest, src, num) {
+                HEAPU8.copyWithin(dest, src, src + num);
+            },
+
+        "emscripten_resize_heap":
+            function _emscripten_resize_heap(requestedSize) {
+                var oldSize = HEAPU8.length;
+                requestedSize = requestedSize >>> 0;
+                abortOnCannotGrowMemory(requestedSize);
+            },
+
+      "fd_close":
+            function (fd) {
+                delete fdData[fd];
+                return 0;
+            },
+
+      "fd_read":
+            function (fd, iov, iovcnt, pnum) {
+                const info = fdData[fd];
+                if (info == null) {
+                    setErrNo(121);
+                    return -1;
+                }
+
+                let num = 0;
+                for (let i = 0; i < iovcnt; i++) {
+                  const ptr = HEAP32[(((iov)+(i*8))>>2)];
+                  const len = HEAP32[(((iov)+(i*8 + 4))>>2)];
+
+                  let cnt = 0;
+                  while (cnt < len && info.pos < info.data.length) {
+                    HEAP8[ptr+cnt] = info.data[info.pos];
+                    info.pos++
+                    cnt++;
+                  }
+
+                  num += cnt;
+                  if (cnt < len) break; // nothing more to read
+                }
+
+                HEAP32[((pnum)>>2)] = num;
+                return 0;
+            },
+
+      "fd_seek":
+            function (fd, offset_low, offset_high, whence, newOffset) {
+                setErrNo(134);
+                return -1;
+            },
+
+      "fd_write":
+            function _fd_write(fd, iov, iovcnt, pnum) {
+                setErrNo(134);
+                return -1;
+            },
+
+      "setTempRet0":
+            function (value) {
+                tempRet0 = value;
+            },
+
+      "__assert_fail":
+            function (condition, filename, line, func) {
+                abort('Assertion failed: ' + UTF8ToString(condition) + ', at: ' + [filename ? UTF8ToString(filename) : 'unknown filename', line, func ? UTF8ToString(func) : 'unknown function']);
+            }
+    };
+
+    // Wraps a JS function as a wasm function with a given signature.
+    function convertJsFunctionToWasm(func, sig) {
+
+        // If the type reflection proposal is available, use the new
+        // "WebAssembly.Function" constructor.
+        // Otherwise, construct a minimal wasm module importing the JS function and
+        // re-exporting it.
+        if (typeof WebAssembly.Function == "function") {
+            var typeNames = {
+                'i': 'i32',
+                'j': 'i64',
+                'f': 'f32',
+                'd': 'f64'
+            };
+            var type = {
+                parameters: [],
+                results: sig[0] == 'v' ? [] : [typeNames[sig[0]]]
+            };
+            for (var i = 1; i < sig.length; ++i) {
+                type.parameters.push(typeNames[sig[i]]);
+            }
+            return new WebAssembly.Function(type, func);
+        }
+
+        // The module is static, with the exception of the type section, which is
+        // generated based on the signature passed in.
+        var typeSection = [
+            0x01, // id: section,
+            0x00, // length: 0 (placeholder)
+            0x01, // count: 1
+            0x60, // form: func
+        ];
+        var sigRet = sig.slice(0, 1);
+        var sigParam = sig.slice(1);
+        var typeCodes = {
+            'i': 0x7f, // i32
+            'j': 0x7e, // i64
+            'f': 0x7d, // f32
+            'd': 0x7c, // f64
+        };
+
+        // Parameters, length + signatures
+        typeSection.push(sigParam.length);
+        for (var i = 0; i < sigParam.length; ++i) {
+            typeSection.push(typeCodes[sigParam[i]]);
+        }
+
+        // Return values, length + signatures
+        // With no multi-return in MVP, either 0 (void) or 1 (anything else)
+        if (sigRet == 'v') {
+            typeSection.push(0x00);
+        } else {
+            typeSection = typeSection.concat([0x01, typeCodes[sigRet]]);
+        }
+
+        // Write the overall length of the type section back into the section header
+        // (excepting the 2 bytes for the section id and length)
+        typeSection[1] = typeSection.length - 2;
+
+        // Rest of the module is static
+        var bytes = new Uint8Array([
+            0x00, 0x61, 0x73, 0x6d, // magic ("\0asm")
+            0x01, 0x00, 0x00, 0x00, // version: 1
+        ].concat(typeSection, [
+            0x02, 0x07, // import section
+            // (import "e" "f" (func 0 (type 0)))
+            0x01, 0x01, 0x65, 0x01, 0x66, 0x00, 0x00,
+            0x07, 0x05, // export section
+            // (export "f" (func 0 (type 0)))
+            0x01, 0x01, 0x66, 0x00, 0x00,
+        ]));
+
+        // We can compile this wasm module synchronously because it is very small.
+        // This accepts an import (at "e.f"), that it reroutes to an export (at "f")
+        var module = new WebAssembly.Module(bytes);
+        var instance = new WebAssembly.Instance(module, {
+                'e': {'f': func}
+            });
+        var wrappedFunc = instance.exports['f'];
+        return wrappedFunc;
+    }
+
+    function addFunction(func, sig) {
+        func = convertJsFunctionToWasm(func, sig);
+
+        let index;
+
+        // Reuse a free index if there is one, otherwise grow.
+        if (freeTableIndexes.length) {
+            index = freeTableIndexes.pop();
+        } else {
+            // Grow the table
+            try {
+                wasmTable.grow(1);
+            } catch (err) {
+                if (!(err instanceof RangeError)) {
+                    throw err;
+                }
+                throw 'Unable to grow wasm table. Set ALLOW_TABLE_GROWTH.';
+            }
+            index = wasmTable.length - 1;
+        }
+
+        wasmTable.set(index, func);
+        return index;
+    }
+
+    function removeFunction(index) {
+      freeTableIndexes.push(index);
+    }
+
+    const response = await fetch("pgf.wasm", { credentials: 'same-origin' });
+
+    const info = {
+        'env': asmLibraryArg,
+        'wasi_snapshot_preview1': asmLibraryArg,
+    };
+
+    // Suppress closure warning here since the upstream definition for
+    // instantiateStreaming only allows Promise<Repsponse> rather than
+    // an actual Response.
+    // TODO(https://github.com/google/closure-compiler/pull/3913): Remove if/when upstream closure is fixed.
+    /** @suppress {checkTypes} */
+    const result = await WebAssembly.instantiateStreaming(response, info);
+
+    asm = result["instance"].exports;
+    wasmTable = asm['__indirect_function_table'];
+    const buf = asm['memory'].buffer;
+    const HEAP8 = new Int8Array(buf);
+    const HEAP16 = new Int16Array(buf);
+    const HEAP32 = new Int32Array(buf);
+    const HEAPU8 = new Uint8Array(buf);
+    const HEAPU16 = new Uint16Array(buf);
+    const HEAPU32 = new Uint32Array(buf);
+    const HEAPF32 = new Float32Array(buf);
+    const HEAPF64 = new Float64Array(buf);
+
+    // Returns the number of bytes the given Javascript string takes if encoded as a UTF8 byte array, EXCLUDING the null terminator byte.
+    function lengthBytesUTF8(str) {
+        var len = 0;
+        for (var i = 0; i < str.length; ++i) {
+            // Gotcha: charCodeAt returns a 16-bit word that is a UTF-16 encoded code unit, not a Unicode code point of the character! So decode UTF16->UTF32->UTF8.
+            // See http://unicode.org/faq/utf_bom.html#utf16-3
+            var u = str.charCodeAt(i); // possibly a lead surrogate
+            if (u >= 0xD800 && u <= 0xDFFF) u = 0x10000 + ((u & 0x3FF) << 10) | (str.charCodeAt(++i) & 0x3FF);
+            if (u <= 0x7F) ++len;
+            else if (u <= 0x7FF) len += 2;
+            else if (u <= 0xFFFF) len += 3;
+            else len += 4;
+        }
+        return len;
+    }
+
+    function stringToUTF8Array(str, heap, outIdx, maxBytesToWrite) {
+        if (!(maxBytesToWrite > 0)) // Parameter maxBytesToWrite is not optional. Negative values, 0, null, undefined and false each don't write out any bytes.
+            return 0;
+
+        var startIdx = outIdx;
+        var endIdx = outIdx + maxBytesToWrite - 1; // -1 for string null terminator.
+        for (var i = 0; i < str.length; ++i) {
+            // Gotcha: charCodeAt returns a 16-bit word that is a UTF-16 encoded code unit, not a Unicode code point of the character! So decode UTF16->UTF32->UTF8.
+            // See http://unicode.org/faq/utf_bom.html#utf16-3
+            // For UTF8 byte structure, see http://en.wikipedia.org/wiki/UTF-8#Description and https://www.ietf.org/rfc/rfc2279.txt and https://tools.ietf.org/html/rfc3629
+            var u = str.charCodeAt(i); // possibly a lead surrogate
+            if (u >= 0xD800 && u <= 0xDFFF) {
+              var u1 = str.charCodeAt(++i);
+              u = 0x10000 + ((u & 0x3FF) << 10) | (u1 & 0x3FF);
+            }
+            if (u <= 0x7F) {
+              if (outIdx >= endIdx) break;
+              heap[outIdx++] = u;
+            } else if (u <= 0x7FF) {
+              if (outIdx + 1 >= endIdx) break;
+              heap[outIdx++] = 0xC0 | (u >> 6);
+              heap[outIdx++] = 0x80 | (u & 63);
+            } else if (u <= 0xFFFF) {
+              if (outIdx + 2 >= endIdx) break;
+              heap[outIdx++] = 0xE0 | (u >> 12);
+              heap[outIdx++] = 0x80 | ((u >> 6) & 63);
+              heap[outIdx++] = 0x80 | (u & 63);
+            } else {
+              if (outIdx + 3 >= endIdx) break;
+              if (u > 0x10FFFF) warnOnce('Invalid Unicode code point 0x' + u.toString(16) + ' encountered when serializing a JS string to a UTF-8 string in wasm memory! (Valid unicode code points should be in range 0-0x10FFFF).');
+              heap[outIdx++] = 0xF0 | (u >> 18);
+              heap[outIdx++] = 0x80 | ((u >> 12) & 63);
+              heap[outIdx++] = 0x80 | ((u >> 6) & 63);
+              heap[outIdx++] = 0x80 | (u & 63);
+            }
+        }
+
+        // Null-terminate the pointer to the buffer.
+        heap[outIdx] = 0;
+        return outIdx - startIdx;
+    }
+
+    function allocateUTF8(pool,str) {
+        var size = lengthBytesUTF8(str) + 1;
+        var ptr = asm.gu_malloc(pool,size);
+        if (ptr) stringToUTF8Array(str, HEAP8, ptr, size);
+        return ptr;
+    }
+
+    const UTF8Decoder = typeof TextDecoder != 'undefined' ? new TextDecoder('utf8') : undefined;
+
+    /**
+     * @param {number} idx
+     * @param {number=} maxBytesToRead
+     * @return {string}
+     */
+    function UTF8ArrayToString(heap, idx, maxBytesToRead) {
+        var endIdx = idx + maxBytesToRead;
+        var endPtr = idx;
+        // TextDecoder needs to know the byte length in advance, it doesn't stop on null terminator by itself.
+        // Also, use the length info to avoid running tiny strings through TextDecoder, since .subarray() allocates garbage.
+        // (As a tiny code save trick, compare endPtr against endIdx using a negation, so that undefined means Infinity)
+        while (heap[endPtr] && !(endPtr >= endIdx)) ++endPtr;
+
+        if (endPtr - idx > 16 && heap.subarray && UTF8Decoder) {
+            return UTF8Decoder.decode(heap.subarray(idx, endPtr));
+        } else {
+            var str = '';
+            // If building with TextDecoder, we have already computed the string length above, so test loop end condition against that
+            while (idx < endPtr) {
+                // For UTF8 byte structure, see:
+                // http://en.wikipedia.org/wiki/UTF-8#Description
+                // https://www.ietf.org/rfc/rfc2279.txt
+                // https://tools.ietf.org/html/rfc3629
+                var u0 = heap[idx++];
+                if (!(u0 & 0x80)) { str += String.fromCharCode(u0); continue; }
+                var u1 = heap[idx++] & 63;
+                if ((u0 & 0xE0) == 0xC0) { str += String.fromCharCode(((u0 & 31) << 6) | u1); continue; }
+                var u2 = heap[idx++] & 63;
+                if ((u0 & 0xF0) == 0xE0) {
+                    u0 = ((u0 & 15) << 12) | (u1 << 6) | u2;
+                } else {
+                    if ((u0 & 0xF8) != 0xF0) warnOnce('Invalid UTF-8 leading byte 0x' + u0.toString(16) + ' encountered when deserializing a UTF-8 string in wasm memory to a JS string!');
+                    u0 = ((u0 & 7) << 18) | (u1 << 12) | (u2 << 6) | (heap[idx++] & 63);
+                }
+
+                if (u0 < 0x10000) {
+                    str += String.fromCharCode(u0);
+                } else {
+                    var ch = u0 - 0x10000;
+                    str += String.fromCharCode(0xD800 | (ch >> 10), 0xDC00 | (ch & 0x3FF));
+                }
+            }
+        }
+        return str;
+    }
+
+    function UTF8ToString(ptr, maxBytesToRead) {
+      return ptr ? UTF8ArrayToString(HEAPU8, ptr, maxBytesToRead) : '';
+    }
+
+    const GuErrnoStrPtr = asm.malloc(8);
+    stringToUTF8Array("GuErrno", HEAP8, GuErrnoStrPtr, 8);
+
+    const PgfExnStrPtr = asm.malloc(8);
+    stringToUTF8Array("PgfExn", HEAP8, PgfExnStrPtr, 8);
+
+    function pgfError(err) {
+        if (asm.gu_exn_caught_(err, GuErrnoStrPtr)) {
+            errDataPtr = asm.gu_exn_caught_data(err);
+            return new Error("errno="+HEAP32[errDataPtr >> 2]);
+        } else if (asm.gu_exn_caught_(err, PgfExnStrPtr)) {
+            msgPtr = asm.gu_exn_caught_data(err);
+            return new Error(UTF8ToString(msgPtr));
+        }
+        return new Error();
+    }
+
+    const registry = new FinalizationRegistry((pool) => {
+        asm.gu_pool_free(pool);
+    });
+
+    function PGF(pgfPtr,name,pool) {
+        this.pgfPtr = pgfPtr;
+        this.abstractName = name;
+        this.pool   = pool;
+        this.languages = {};
+        registry.register(this,pool);
+    }
+
+    function Concr(pgf,concrPtr,name) {
+        this.pgf  = pgf;
+        this.name = name;
+        this.concrPtr = concrPtr;
+    }
+    Concr.prototype.linearize = function(expr) {
+        const tmp_pool = asm.gu_new_pool();
+        const err = asm.gu_new_exn(tmp_pool);
+        const sb = asm.gu_new_string_buf(tmp_pool);
+        const out = asm.gu_string_buf_out(sb);
+
+        asm.pgf_linearize(this.concrPtr, expr.exprPtr, out, err);
+        if (asm.gu_exn_is_raised(err)) {
+            const e = pgfError(err);
+            asm.gu_pool_free(tmp_pool);
+            throw e;
+        }
+
+        const strPtr = asm.gu_string_buf_data(sb);
+        const len = asm.gu_string_buf_length(sb);
+        const str = UTF8ToString(strPtr,len);
+        asm.gu_pool_free(tmp_pool);
+
+        return str
+    }
+
+    async function readPGF(pgfURL) {
+        const response = await fetch(pgfURL);
+        urlData[pgfURL] = new Int8Array(await response.arrayBuffer());
+
+        const pool = asm.gu_new_pool();
+
+        const tmp_pool = asm.gu_new_pool();
+        const err = asm.gu_new_exn(tmp_pool);
+        const strPtr = allocateUTF8(tmp_pool,pgfURL);
+        const pgfPtr = asm.pgf_read(strPtr,pool,err);
+        if (asm.gu_exn_is_raised(err)) {
+            const e = pgfError(err);
+            asm.gu_pool_free(tmp_pool);
+            throw e;
+        }
+
+        const namePtr = asm.pgf_abstract_name(pgfPtr);
+        const abstractName = UTF8ToString(namePtr);
+
+        const pgf = new PGF(pgfPtr,abstractName,pool);
+
+        const itor = asm.gu_malloc(tmp_pool,sizeof_GuMapItor);
+        const fn =
+          addFunction(
+             (itor,namePtr,concrPtrPtr,err) => {
+                 const name = UTF8ToString(namePtr);
+                 const concrPtr = HEAP32[concrPtrPtr >> 2];
+                 pgf.languages[name] = new Concr(pgf,concrPtr,name);
+             },
+             "viiii"
+             );
+        HEAP32[(itor+offsetof_GuMapItor_fn) >> 2] = fn;
+        asm.pgf_iter_languages(pgfPtr,itor,err);
+        removeFunction(fn);
+
+        asm.gu_pool_free(tmp_pool);
+        return pgf;
+    }
+
+    function Expr(exprPtr,pool) {
+        this.exprPtr = exprPtr;
+        this.pool = pool;
+        registry.register(this,pool);
+    }
+    Expr.prototype.toString = function() {
+        const tmp_pool = asm.gu_new_pool();
+
+        const sb = asm.gu_new_string_buf(tmp_pool);
+        const out = asm.gu_string_buf_out(sb);
+        const err = asm.gu_new_exn(tmp_pool);
+        asm.pgf_print_expr(this.exprPtr, 0, 0, out, err);
+        if (asm.gu_exn_is_raised(err)) {
+            const e = pgfError(err);
+            asm.gu_pool_free(tmp_pool);
+            throw e;
+        }
+
+        const strPtr = asm.gu_string_buf_data(sb);
+        const len = asm.gu_string_buf_length(sb);
+        const str = UTF8ToString(strPtr,len);
+        asm.gu_pool_free(tmp_pool);
+
+        return str;
+    };
+    Expr.prototype.arity = function(expr) {
+        return asm.pgf_expr_arity(this.expr);
+    }
+
+    function readExpr(exprStr) {
+        const tmp_pool = asm.gu_new_pool();
+
+        const strPtr = allocateUTF8(tmp_pool,exprStr);
+        const in_ = asm.gu_data_in(strPtr, exprStr.length, tmp_pool);
+        const err = asm.gu_new_exn(tmp_pool);
+        const pool = asm.gu_new_pool();
+        const expr = asm.pgf_read_expr(in_, pool, tmp_pool, err);
+        asm.gu_pool_free(tmp_pool);
+
+        if (asm.gu_exn_is_raised(err)) {
+            throw pgfError(err);
+        }
+        if (expr == 0) {
+            throw new Error('Expression cannot be parsed');
+        }
+
+        return new Expr(expr,pool);
+    }
+
+    return { readPGF, readExpr };
+}
+
+// This allows us to use both from Node and in browser
+if (typeof module != 'undefined') {
+    module.exports = mkAPI;
+}

src/runtime/javascript/style.css

@@ -1,252 +0,0 @@
-body {
-	font-family:arial,helvetica,sans-serif;
-	font-size:12px;
-	background-color: white;
-}
-
-#wrapper {
-	width:740px;
-	height:520px;
-	margin:auto 50px;
-	border:1px solid gray;
-	padding:10px;
-
-}
-
-#absFrame {
- 	width:250px;
- 	height:250px;
-	padding:10px;
-	border:1px solid gray;
-   	float:left;
-   	white-space: nowrap;
-}
-
-#conFrame {
-	width:436px;
-	height:250px;
-	margin-left:10px;
-	padding:10px;
-	border:1px solid gray;
-   	float:left;
-   	white-space: normal;
-   	overflow:auto;
-}
-
-#actFrame {
-	width:250px;
-	height:170px;
-	margin-top:10px;
-	padding:10px;
-	border:1px solid gray;
-   	float:left;
-   	overflow:auto;
-}
-
-#refFrame {
-	width:436px;
-	height:170px;
-	margin-left:10px;
-	margin-top:10px;
-	padding:10px;
-	border:1px solid gray;
-   	float:left;
-   	overflow:auto;
-}
-
-#messageFrame {
-	width:506px;
-	height:15px;
-	margin-top:10px;
-	margin-right:10px;
-	padding:10px;
-	border:1px solid gray;
-   	float:left;
-   	overflow:hidden;
-}
-
-#clipboardFrame {
-	width:180px;
-	height:15px;
-	margin-top:10px;
-	padding:10px;
-	border:1px solid gray;
-   	float:left;
-   	overflow:auto;
-}
-
-#tree {
-	left: -10px;
-	top: -10px;	
-	margin: 0px;
-	padding: 10px;
-	overflow: auto;
-}
-
-ul {
-	position: relative;
-	list-style: none;
-	margin-left: 20px;
-	padding: 0px;
-}
-
-li {
-	position: relative;
-}
-
-img.tree-menu {
-	margin-right: 5px;
-}
-
-a.tree:link, a.tree:visited, a.tree:active {
-	color: black;
-	background-color: white;
-	text-decoration: none;
-	margin-right:10px;
-}
-
-a.tree:hover {
-	color: blue;
-	background-color: white;
-	text-decoration: underline;
-	margin-right:10px;
-}
-
-a.treeSelected:link, a.treeSelected:visited, a.treeSelected:active {
-	color: white;
-	background-color: #3366CC;
-	text-decoration: none;
-	margin-right:10px;
-}
-
-a.treeSelected:hover {
-	color: white;
-	background-color: #3366CC;
-	text-decoration: underline;
-	margin-right:10px;
-}
-
-a.treeGray:link, a.treeGray:visited, a.treeGray:active {
-	color: silver;
-	background-color: white;
-	text-decoration: none;
-	margin-right:10px;
-}
-
-a.treeGray:hover {
-	color: silver;
-	background-color: white;
-	text-decoration: none;
-	margin-right:10px;
-}
-
-table.action, table.refinement, table.wrapper, table.tree, table.language {
-	margin: 0px;
-	padding: 0px;
-	border-style: none;
-	border-collapse: collapse;
-	border-spacing: 0px;
-}
-
-tr.selected {
-	color: white;
-	background-color: #3366CC;
-}
-
-tr.unavailable, tr.closed {
-	color: silver;
-	background-color: white;
-}
-
-tr.unavailable:hover {
-	color: silver;
-	background-color: #3366CC;
-}
-
-tr.action, tr.refinement, tr.wrapper, tr.tree {
-	color: black;
-	background-color: white;
-}
-
-tr.action:hover, tr.refinement:hover, tr.wrapper:hover, tr.tree:hover {
-	color: white;
-	background-color: #3366CC;
-}
-
-td.action {
-	width: 220px;
-	margin: 0px;
-	padding: 0px;
-}
-
-td.refinement, td.wrapper, td.tree {
-	width: 515px;
-	margin: 0px;
-	padding: 0px;
-}
-
-td.hotKey {
-	width: 30px;
-	margin: 0px;
-	padding: 0px;
-	text-align: right;
-}
-
-td.language {
-	color: black;
-	background-color: white;
-	margin: 1px;
-	padding: 1px;
-}
-
-td.language:hover {
-	color: blue;
-	background-color: white;
-	text-decoration: underline;
-	margin: 1px;
-	padding: 1px;
-}
-
-td.selected {
-	color: white;
-	background-color: #3366CC;
-	margin: 1px;
-	padding: 1px;
-}
-
-td.selected:hover {
-	color: white;
-	background-color: #3366CC;
-	text-decoration: underline;
-	margin: 1px;
-	padding: 1px;
-}
-	
-p {
-	margin-bottom: 40px;
-}
-
-span.normal {
-	color: black;
-	background-color: white;
-	text-decoration: none;
-    padding-left: 2px;
-    padding-right: 2px;
-}
-
-span.edit {
-	color: black;
-	background-color: white;
-	text-decoration: none;
-    border:2px inset;
-    padding-left: 2px;
-    padding-right: 2px;
-}
-
-span.selected {
-	color: white;
-	background-color: #3366CC;
-	text-decoration: none;
-    padding-left: 2px;
-    padding-right: 2px;
-}

src/runtime/javascript/test-node.js

@@ -0,0 +1,33 @@
+const Module = require('./.libs/pgf.js');
+const JSPGF = require('./jspgf.js')(Module);
+const fs = require('fs');
+const path = require('path');
+
+Module.onRuntimeInitialized = () => {
+
+    // Read PGF path from args
+    if (process.argv.length > 2) {
+        const pgfPathHost = process.argv[2];
+
+        // Copy file into filesystem
+        const pgfPathFS = '/tmp/' + path.basename(pgfPathHost);
+        const rawPgf = fs.readFileSync(pgfPathHost);
+        Module.FS.writeFile(pgfPathFS, rawPgf);
+
+        // Read PGF
+        const pgf = JSPGF.readPGF(pgfPathFS);
+
+        // Print its name
+        console.log(JSPGF.abstractName(pgf));
+    }
+
+    // Parse expression
+    const expr = JSPGF.readExpr("Pred (Another (x f))");
+
+    // Show it
+    console.log(JSPGF.showExpr(expr));
+
+    // Print its arity
+    console.log('arity', JSPGF.arity(expr));
+}
+

src/runtime/javascript/test-web.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+  </head>
+  <body>
+    <script type="text/javascript" src="./jspgf.js"></script>
+    <script type="text/javascript" src="./test-web.js"></script>
+  </body>
+</html>
+
+

src/runtime/javascript/test-web.js

@@ -0,0 +1,21 @@
+mkAPI().then((pgf) => {
+    // Parse expression
+    const expr = pgf.readExpr("Pred (This Fish) Fresh");
+
+    // Show it
+    console.log(expr.toString());
+
+    // Print its arity
+    console.log('arity', expr.arity());
+
+    pgf.readPGF("Foods.pgf").then((gr) => {
+        // Print the grammar name
+        console.log(gr.abstractName);
+        
+        // Access a language and print the concrete name
+        console.log(gr.languages["FoodsEng"].name);
+        
+        // Linearize an expression
+        console.log(gr.languages["FoodsEng"].linearize(expr));
+    });
+});

src/runtime/javascript/translator.css

@@ -1,54 +0,0 @@
-body {
-  color: black;
-  background-color: white;
-}
-
-dl {
-
-}
-
-dt {
-  margin: 0;
-  padding: 0;
-}
-
-dl dd {
-  margin: 0;
-  padding: 0;
-}
-
-dl.fromLang dt {
-  display: none;
-}
-
-dl.toLang {
-  border-width: 1px 0 0 0;
-  border-style: solid;
-  border-color: #c0c0c0;
-}
-
-dl.toLang dt {
-  color: #c0c0c0;
-  display: block;
-  float: left;
-  width: 5em;
-}
-
-
-dl.toLang dd {
-  border-width: 0 0 1px 0;
-  border-style: solid;
-  border-color: #c0c0c0;
-}
-
-
-ul {
-  margin: 0;
-  padding: 0;
-}
-
-li {
-  list-style-type: none;
-  margin: 0;
-  padding: 0;
-}

src/runtime/javascript/translator.html

@@ -1,48 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-  <head>
-    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-    <link rel="stylesheet" type="text/css" href="translator.css" />
-    <script type="text/javascript" src="gflib.js"></script>
-    <script type="text/javascript" src="grammar.js"></script>
-    <script type="text/javascript" src="translator.js"></script>
-    <script type="text/javascript">
-      /* CHANGE ME */
-     var grammar = Foods;
-
-     function updateTranslation () {
-        var input = document.getElementById('inputText').value;
-        var fromLang = document.getElementById('fromLang').value;
-        var toLang = document.getElementById('toLang').value;
-        var output = document.getElementById('output');
-        var translation = grammar.translate(input, fromLang, toLang);
-        removeChildren(output);
-        output.appendChild(formatTranslation(translation));
-      }
-
-      function populateLangs () {
-        var f = document.getElementById('fromLang');
-        var t = document.getElementById('toLang');
-        for (var c in grammar.concretes) {
-          addOption(f, c, c);
-          addOption(t, c, c);
-        }
-      }
-    </script>
-    <title>Web-based GF Translator</title>
-  </head>
-  <body onload="populateLangs(grammar, 'fromLang', 'toLang')">
-    <form id="translate">
-      <p>
-        <input type="text" name="inputText" id="inputText" value="this cheese is warm" size="50" />
-      </p>
-      <p>
-        From: <select name="fromLang" id="fromLang" onchange=""><option value="">Any language</option></select>
-        To: <select name="toLang" id="toLang"><option value="">All languages</option></select>
-        <input type="button" value="Translate" onclick="updateTranslation()" />
-      </p>
-    </form>
-    <div id="output"></div>
-  </body>
-</html>

src/runtime/javascript/translator.js

@@ -1,51 +0,0 @@
-function formatTranslation (outputs) {
-  var dl1 = document.createElement("dl");
-  dl1.className = "fromLang";
-  for (var fromLang in outputs) {
-    var ul = document.createElement("ul");
-    addDefinition(dl1, document.createTextNode(fromLang), ul);
-    for (var i in outputs[fromLang]) {
-      var dl2 = document.createElement("dl");
-      dl2.className = "toLang";
-      for (var toLang in outputs[fromLang][i]) {
-	addDefinition(dl2, document.createTextNode(toLang), document.createTextNode(outputs[fromLang][i][toLang]));
-      }
-      addItem(ul, dl2);
-    }
-  }
-
-  return dl1;
-}
-
-/* DOM utilities for specific tags */
-
-function addDefinition (dl, t, d) {
-    var dt = document.createElement("dt");
-    dt.appendChild(t);
-    dl.appendChild(dt);
-    var dd = document.createElement("dd");
-    dd.appendChild(d);
-    dl.appendChild(dd);
-}
-
-function addItem (ul, i) {
-    var li = document.createElement("li");
-    li.appendChild(i);
-    ul.appendChild(li);
-}
-
-function addOption (select, value, content) {
-  var option = document.createElement("option");
-  option.value = value;
-  option.appendChild(document.createTextNode(content));
-  select.appendChild(option);
-}
-
-/* General DOM utilities */
-
-/* Removes all the children of a node */
-function removeChildren(node) {
-  while (node.hasChildNodes()) {
-    node.removeChild(node.firstChild);
-  }
-}

src/runtime/typescript/MOVED.md

@@ -1,7 +0,0 @@
-# Project moved
-
-The GF TypeScript runtime has been moved to the repository:
-<https://github.com/GrammaticalFramework/gf-typescript>
-
-If you are looking for an updated version of the JavaScript runtime,
-you should also look there.