replace broken link to best practices

This fixes nix builds of gf-lsp

.github/workflows/build-all-versions.yml

@@ -12,28 +12,34 @@ jobs:
     name: ${{ matrix.os }} / ghc ${{ matrix.ghc }}
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         cabal: ["latest"]
         ghc:
           - "8.6.5"
           - "8.8.3"
-          - "8.10.1"
+          - "8.10.7"
+          - "9.6.7"
         exclude:
           - os: macos-latest
             ghc: 8.8.3
           - os: macos-latest
             ghc: 8.6.5
+          - os: macos-latest
+            ghc: 8.10.7
           - os: windows-latest
             ghc: 8.8.3
           - os: windows-latest
             ghc: 8.6.5
+          - os: windows-latest
+            ghc: 8.10.7
 
     steps:
     - uses: actions/checkout@v2
       if: github.event.action == 'opened' || github.event.action == 'synchronize' || github.event.ref == 'refs/heads/master'
 
-    - uses: haskell/actions/setup@v1
+    - uses: haskell-actions/setup@v2
       id: setup-haskell-cabal
       name: Setup Haskell
       with:
@@ -44,7 +50,7 @@ jobs:
       run: |
         cabal freeze
 
-    - uses: actions/cache@v1
+    - uses: actions/cache@v4
       name: Cache ~/.cabal/store
       with:
         path: ${{ steps.setup-haskell-cabal.outputs.cabal-store }}
@@ -53,43 +59,50 @@ jobs:
 
     - name: Build
       run: |
-        cabal configure --enable-tests --test-show-details=direct
+        cabal configure --enable-tests --enable-benchmarks --test-show-details=direct
-        cabal build
+        cabal build all
 
-    - name: Test
+    # - name: Test
-      run: |
+    #   run: |
-        PATH="$PWD/dist/build/gf:$PATH" cabal test gf-tests
+    #     cabal test all
 
   stack:
     name: stack / ghc ${{ matrix.ghc }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.ghc == '7.10.3' && 'ubuntu-20.04' || 'ubuntu-latest' }}
     strategy:
+      fail-fast: false
       matrix:
         stack: ["latest"]
-        ghc: ["7.10.3","8.0.2", "8.2.2", "8.4.4", "8.6.5", "8.8.4"]
+        ghc: ["8.4.4", "8.6.5", "8.8.4", "8.10.7", "9.0.2", "9.6.7"]
-        # ghc: ["8.8.3"]
 
     steps:
     - uses: actions/checkout@v2
       if: github.event.action == 'opened' || github.event.action == 'synchronize' || github.event.ref == 'refs/heads/master'
 
-    - uses: haskell/actions/setup@v1
+    - uses: haskell-actions/setup@v2
       name: Setup Haskell Stack
       with:
         ghc-version: ${{ matrix.ghc }}
         stack-version: 'latest'
         enable-stack: true
 
-    - uses: actions/cache@v1
+
+    # Fix linker errrors on ghc-7.10.3 for ubuntu (see https://github.com/commercialhaskell/stack/blob/255cd830627870cdef34b5e54d670ef07882523e/doc/faq.md#i-get-strange-ld-errors-about-recompiling-with--fpic)
+    - run: sed -i.bak 's/"C compiler link flags", "/&-no-pie /' /home/runner/.ghcup/ghc/7.10.3/lib/ghc-7.10.3/settings
+      if: matrix.ghc == '7.10.3'
+
+    - uses: actions/cache@v4
       name: Cache ~/.stack
       with:
         path: ~/.stack
-        key: ${{ runner.os }}-${{ matrix.ghc }}-stack
+        key: ${{ runner.os }}-${{ matrix.ghc }}-stack--${{ hashFiles(format('stack-ghc{0}', matrix.ghc)) }}
+        restore-keys: |
+             ${{ runner.os }}-${{ matrix.ghc }}-stack
 
     - name: Build
       run: |
-        stack build --system-ghc --stack-yaml stack-ghc${{ matrix.ghc }}.yaml --test --no-run-tests
+        stack build --test --no-run-tests --system-ghc --stack-yaml stack-ghc${{ matrix.ghc }}.yaml
 
     - name: Test
       run: |
-        stack test --system-ghc --stack-yaml stack-ghc${{ matrix.ghc }}.yaml gf:test:gf-tests
+        stack test --system-ghc --stack-yaml stack-ghc${{ matrix.ghc }}.yaml

.github/workflows/build-binary-packages.yml

@@ -13,9 +13,9 @@ jobs:
     name: Build Ubuntu package
     strategy:
       matrix:
-        os:
+        ghc: ["9.6"]
-        - ubuntu-18.04
+        cabal: ["3.10"]
-        - ubuntu-20.04
+        os: ["ubuntu-24.04"]
 
     runs-on: ${{ matrix.os }}
 
@@ -25,12 +25,13 @@ jobs:
     # Note: `haskell-platform` is listed as requirement in debian/control,
     # which is why it's installed using apt instead of the Setup Haskell action.
 
-    # - name: Setup Haskell
+    - name: Setup Haskell
-    #   uses: actions/setup-haskell@v1
+      uses: haskell-actions/setup@v2
-    #   id: setup-haskell-cabal
+      id: setup-haskell-cabal
-    #   with:
+      with:
-    #     ghc-version: ${{ matrix.ghc }}
+        ghc-version: ${{ matrix.ghc }}
-    #     cabal-version: ${{ matrix.cabal }}
+        cabal-version: ${{ matrix.cabal }}
+      if: matrix.os == 'ubuntu-24.04'
 
     - name: Install build tools
       run: |
@@ -39,14 +40,15 @@ jobs:
           make \
           dpkg-dev \
           debhelper \
-          haskell-platform \
           libghc-json-dev \
-          python-dev \
           default-jdk \
+          python-dev-is-python3 \
           libtool-bin 
+        cabal install alex happy
         
     - name: Build package
       run: |
+        export PYTHONPATH="/home/runner/work/gf-core/gf-core/debian/gf/usr/local/lib/python3.12/dist-packages/"
         make deb
 
     - name: Copy package
@@ -54,7 +56,7 @@ jobs:
         cp ../gf_*.deb dist/
 
     - name: Upload artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: gf-${{ github.event.release.tag_name }}-${{ matrix.os }}.deb
         path: dist/gf_*.deb
@@ -64,14 +66,14 @@ jobs:
       run: |
         mv dist/gf_*.deb dist/gf-${{ github.event.release.tag_name }}-${{ matrix.os }}.deb
 
-    - uses: actions/upload-release-asset@v1.0.2
+    #- uses: actions/upload-release-asset@v1.0.2
-      env:
+    #  env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    #    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
+    #  with:
-        upload_url: ${{ github.event.release.upload_url }}
+    #    upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dist/gf-${{ github.event.release.tag_name }}-${{ matrix.os }}.deb
+    #    asset_path: dist/gf-${{ github.event.release.tag_name }}-${{ matrix.os }}.deb
-        asset_name: gf-${{ github.event.release.tag_name }}-${{ matrix.os }}.deb
+    #    asset_name: gf-${{ github.event.release.tag_name }}-${{ matrix.os }}.deb
-        asset_content_type: application/octet-stream
+    #    asset_content_type: application/octet-stream
 
 # ---
 
@@ -79,16 +81,16 @@ jobs:
     name: Build macOS package
     strategy:
       matrix:
-        ghc: ["8.6.5"]
+        ghc: ["9.6"]
-        cabal: ["2.4"]
+        cabal: ["3.10"]
-        os: ["macos-10.15"]
+        os: ["macos-latest", "macos-13"]
     runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v2
 
     - name: Setup Haskell
-      uses: actions/setup-haskell@v1
+      uses: haskell-actions/setup@v2
       id: setup-haskell-cabal
       with:
         ghc-version: ${{ matrix.ghc }}
@@ -97,8 +99,10 @@ jobs:
     - name: Install build tools
       run: |
         brew install \
-          automake
+          automake \
+          libtool
         cabal v1-install alex happy
+        pip install setuptools
 
     - name: Build package
       run: |
@@ -107,9 +111,9 @@ jobs:
         make pkg
 
     - name: Upload artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
-        name: gf-${{ github.event.release.tag_name }}-macos
+        name: gf-${{ github.event.release.tag_name }}-${{ matrix.os }}
         path: dist/gf-*.pkg
         if-no-files-found: error
 
@@ -117,14 +121,14 @@ jobs:
       run: |
         mv dist/gf-*.pkg dist/gf-${{ github.event.release.tag_name }}-macos.pkg
 
-    - uses: actions/upload-release-asset@v1.0.2
+    #- uses: actions/upload-release-asset@v1.0.2
-      env:
+    #  env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    #    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
+    #  with:
-        upload_url: ${{ github.event.release.upload_url }}
+    #    upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dist/gf-${{ github.event.release.tag_name }}-macos.pkg
+    #    asset_path: dist/gf-${{ github.event.release.tag_name }}-macos.pkg
-        asset_name: gf-${{ github.event.release.tag_name }}-macos.pkg
+    #    asset_name: gf-${{ github.event.release.tag_name }}-macos.pkg
-        asset_content_type: application/octet-stream
+    #    asset_content_type: application/octet-stream
 
 # ---
 
@@ -132,9 +136,9 @@ jobs:
     name: Build Windows package
     strategy:
       matrix:
-        ghc: ["8.6.5"]
+        ghc: ["9.6.7"]
-        cabal: ["2.4"]
+        cabal: ["3.10"]
-        os: ["windows-2019"]
+        os: ["windows-2022"]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -147,6 +151,7 @@ jobs:
           base-devel
           gcc
           python-devel
+          autotools
 
     - name: Prepare dist folder
       shell: msys2 {0}
@@ -171,7 +176,8 @@ jobs:
     - name: Build Java bindings
       shell: msys2 {0}
       run: |
-        export JDKPATH=/c/hostedtoolcache/windows/Java_Adopt_jdk/8.0.292-10/x64
+        echo $JAVA_HOME_8_X64
+        export JDKPATH="$(cygpath -u "${JAVA_HOME_8_X64}")"
         export PATH="${PATH}:${JDKPATH}/bin"
         cd src/runtime/java
         make \
@@ -180,6 +186,9 @@ jobs:
         make install
         cp .libs/msys-jpgf-0.dll /c/tmp-dist/java/jpgf.dll
         cp jpgf.jar /c/tmp-dist/java
+      if: false
+    
+    # - uses: actions/setup-python@v5
 
     - name: Build Python bindings
       shell: msys2 {0}
@@ -188,12 +197,13 @@ jobs:
         EXTRA_LIB_DIRS: /mingw64/lib
       run: |
         cd src/runtime/python
+        pacman --noconfirm -S python-setuptools
         python setup.py build
         python setup.py install
-        cp /usr/lib/python3.9/site-packages/pgf* /c/tmp-dist/python
+        cp -r /usr/lib/python3.12/site-packages/pgf* /c/tmp-dist/python
 
     - name: Setup Haskell
-      uses: actions/setup-haskell@v1
+      uses: haskell-actions/setup@v2
       id: setup-haskell-cabal
       with:
         ghc-version: ${{ matrix.ghc }}
@@ -205,13 +215,13 @@ jobs:
 
     - name: Build GF
       run: |
-        cabal install --only-dependencies -fserver
+        cabal install -fserver --only-dependencies 
         cabal configure -fserver
         cabal build
-        copy dist\build\gf\gf.exe C:\tmp-dist
+        copy dist-newstyle/build/x86_64-windows/ghc-${{matrix.ghc}}/*/x/gf/build/gf/gf.exe C:/tmp-dist
 
     - name: Upload artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: gf-${{ github.event.release.tag_name }}-windows
         path: C:\tmp-dist\*
@@ -220,11 +230,11 @@ jobs:
     - name: Create archive
       run: |
         Compress-Archive C:\tmp-dist C:\gf-${{ github.event.release.tag_name }}-windows.zip
-    - uses: actions/upload-release-asset@v1.0.2
+    #- uses: actions/upload-release-asset@v1.0.2
-      env:
+    #  env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    #    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
+    #  with:
-        upload_url: ${{ github.event.release.upload_url }}
+    #    upload_url: ${{ github.event.release.upload_url }}
-        asset_path: C:\gf-${{ github.event.release.tag_name }}-windows.zip
+    #    asset_path: C:\gf-${{ github.event.release.tag_name }}-windows.zip
-        asset_name: gf-${{ github.event.release.tag_name }}-windows.zip
+    #    asset_name: gf-${{ github.event.release.tag_name }}-windows.zip
-        asset_content_type: application/zip
+    #    asset_content_type: application/zip

.github/workflows/build-python-package.yml

@@ -13,24 +13,25 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-18.04, macos-10.15]
+        os: [ubuntu-latest, macos-latest, macos-13]
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
 
-    - uses: actions/setup-python@v1
+    - uses: actions/setup-python@v5
       name: Install Python
       with:
-        python-version: '3.7'
+        python-version: '3.x'
 
     - name: Install cibuildwheel
       run: |
-        python -m pip install git+https://github.com/joerick/cibuildwheel.git@main
+        python -m pip install cibuildwheel
 
     - name: Install build tools for OSX
       if: startsWith(matrix.os, 'macos')
       run: |
         brew install automake
+        brew install libtool
 
     - name: Build wheels on Linux
       if: startsWith(matrix.os, 'macos') != true
@@ -42,30 +43,32 @@ jobs:
     - name: Build wheels on OSX
       if: startsWith(matrix.os, 'macos')
       env:
-        CIBW_BEFORE_BUILD: cd src/runtime/c && glibtoolize && autoreconf -i && ./configure && make && make install
+        CIBW_BEFORE_BUILD: cd src/runtime/c && glibtoolize && autoreconf -i && ./configure && make && sudo make install
       run: |
         python -m cibuildwheel src/runtime/python --output-dir wheelhouse
 
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v4
       with:
+        name: wheel-${{ matrix.os }}
         path: ./wheelhouse
 
   build_sdist:
     name: Build source distribution
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         name: Install Python
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - name: Build sdist
         run: cd src/runtime/python && python setup.py sdist
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
+          name: wheel-source
           path: ./src/runtime/python/dist/*.tar.gz
 
   upload_pypi:
@@ -75,24 +78,25 @@ jobs:
     if: github.ref == 'refs/heads/master' && github.event_name == 'push'
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
 
       - name: Install twine
         run: pip install twine
 
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4.1.7
         with:
-          name: artifact
+          pattern: wheel-*
+          merge-multiple: true
           path: ./dist
 
       - name: Publish
         env:
           TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.pypi_password }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
-          (cd ./src/runtime/python && curl -I --fail https://pypi.org/project/$(python setup.py --name)/$(python setup.py --version)/) || twine upload dist/*
+          twine upload --verbose --non-interactive --skip-existing dist/*

.gitignore

@@ -5,7 +5,6 @@
 *.jar
 *.gfo
 *.pgf
-*.lpgf
 debian/.debhelper
 debian/debhelper-build-stamp
 debian/gf
@@ -49,7 +48,7 @@ src/runtime/java/.libs/
 src/runtime/python/build/
 .cabal-sandbox
 cabal.sandbox.config
-.stack-work*
+.stack-work
 DATA_DIR
 
 stack*.yaml.lock
@@ -74,10 +73,9 @@ doc/icfp-2012.html
 download/*.html
 gf-book/index.html
 src/www/gf-web-api.html
-
+.devenv
-DEBUG/
+.direnv
-PROF/
+result
-*.aux
+.vscode
-*.hp
+.envrc
-*.prof
+.pre-commit-config.yaml
-*.ps

.travis.yml

@@ -1,14 +0,0 @@
-sudo: required
-
-language: c
-
-services:
-  - docker
-
-before_install:
-  - docker pull odanoburu/gf-src:3.9
-
-script:
-  - |
-    docker run --mount src="$(pwd)",target=/home/gfer,type=bind odanoburu/gf-src:3.9 /bin/bash -c "cd /home/gfer/src/runtime/c &&
-    autoreconf -i && ./configure && make && make install ; cd /home/gfer ; cabal install -fserver -fc-runtime --extra-lib-dirs='/usr/local/lib'"

CHANGELOG.md

@@ -0,0 +1,12 @@
+### New since 3.12 (WIP)
+
+### 3.12
+See <https://www.grammaticalframework.org/download/release-3.12.html>
+
+### 3.11
+
+See <https://www.grammaticalframework.org/download/release-3.11.html>
+
+### 3.10
+
+See <https://www.grammaticalframework.org/download/release-3.10.html>

Makefile

@@ -50,7 +50,7 @@ html::
 # number to the top of debian/changelog.
 # (Tested on Ubuntu 15.04. You need to install dpkg-dev & debhelper.)
 deb:
-	dpkg-buildpackage -b -uc
+	dpkg-buildpackage -b -uc -d
 
 # Make a macOS installer package
 pkg:
@@ -65,6 +65,6 @@ bintar:
 
 # Make a source tar.gz distribution using git to make sure that everything is included.
 # We put the distribution in dist/ so it is removed on `make clean`
-sdist:
+# sdist:
-	test -d dist || mkdir dist
+# 	test -d dist || mkdir dist
-	git archive --format=tar.gz --output=dist/gf-${VERSION}.tar.gz HEAD
+# 	git archive --format=tar.gz --output=dist/gf-${VERSION}.tar.gz HEAD

README.md

@@ -1,4 +1,4 @@
-![GF Logo](doc/Logos/gf1.svg)
+![GF Logo](https://www.grammaticalframework.org/doc/Logos/gf1.svg)
 
 # Grammatical Framework (GF)
 
@@ -38,8 +38,23 @@ or:
 ```
 stack install
 ```
+Note that if you are unlucky to have Cabal 3.0 or later, then it uses
+the so-called Nix style commands. Using those for GF development is
+a pain. Every time when you change something in the source code, Cabal
+will generate a new folder for GF to look for the GF libraries and
+the GF cloud. Either reinstall everything with every change in the
+compiler, or be sane and stop using cabal-install. Instead you can do:
+```
+runghc Setup.hs configure
+runghc Setup.hs build
+sudo runghc Setup.hs install
+```
+The script will install the GF dependencies globally. The only solution
+to the Nix madness that I found is radical:
 
-For more information, including links to precompiled binaries, see the [download page](http://www.grammaticalframework.org/download/index.html).
+  "No person, no problem" (Нет человека – нет проблемы).
+
+For more information, including links to precompiled binaries, see the [download page](https://www.grammaticalframework.org/download/index.html).
 
 ## About this repository
 

RELEASE.md

@@ -47,11 +47,14 @@ but the generated _artifacts_ must be manually attached to the release as _asset
 
 In order to do this you will need to be added the [GF maintainers](https://hackage.haskell.org/package/gf/maintainers/) on Hackage.
 
-1. Run `make sdist`
+1. Run `stack sdist --test-tarball` and address any issues.
 2. Upload the package, either:
-    1. **Manually**: visit <https://hackage.haskell.org/upload> and upload the file `dist/gf-X.Y.tar.gz`
+    1. **Manually**: visit <https://hackage.haskell.org/upload> and upload the file generated by the previous command.
-    2. **via Cabal (≥2.4)**: `cabal upload dist/gf-X.Y.tar.gz`
+    2. **via Stack**: `stack upload . --candidate`
-3. If the documentation-building fails on the Hackage server, do:
+3. After testing the candidate, publish it:
+    1. **Manually**: visit <https://hackage.haskell.org/package/gf-X.Y.Z/candidate/publish>
+    1. **via Stack**: `stack upload .`
+4. If the documentation-building fails on the Hackage server, do:
 ```
 cabal v2-haddock --builddir=dist/docs --haddock-for-hackage --enable-doc
 cabal upload --documentation dist/docs/*-docs.tar.gz

Setup.hs

@@ -4,23 +4,54 @@ import Distribution.Simple.LocalBuildInfo(LocalBuildInfo(..),absoluteInstallDirs
 import Distribution.Simple.Setup(BuildFlags(..),Flag(..),InstallFlags(..),CopyDest(..),CopyFlags(..),SDistFlags(..))
 import Distribution.PackageDescription(PackageDescription(..),emptyHookedBuildInfo)
 import Distribution.Simple.BuildPaths(exeExtension)
+import System.Directory
 import System.FilePath((</>),(<.>))
+import System.Process
+import Control.Monad(forM_,unless)
+import Control.Exception(bracket_)
+import Data.Char(isSpace)
 
 import WebSetup
 
--- | Notice about RGL not built anymore
-noRGLmsg :: IO ()
-noRGLmsg = putStrLn "Notice: the RGL is not built as part of GF anymore. See https://github.com/GrammaticalFramework/gf-rgl"
-
 main :: IO ()
 main = defaultMainWithHooks simpleUserHooks
-  { preBuild  = gfPreBuild
+  { preConf   = gfPreConf
+  , preBuild  = gfPreBuild
   , postBuild = gfPostBuild
   , preInst   = gfPreInst
   , postInst  = gfPostInst
   , postCopy  = gfPostCopy
   }
   where
+    gfPreConf args flags = do
+      pkgs <- fmap (map (dropWhile isSpace) . tail . lines)
+                   (readProcess "ghc-pkg" ["list"] "")
+      forM_ dependencies $ \pkg -> do
+        let name = takeWhile (/='/') (drop 36 pkg)
+        unless (name `elem` pkgs) $ do
+          let fname = name <.> ".tar.gz"
+          callProcess "wget" [pkg,"-O",fname]
+          callProcess "tar"  ["-xzf",fname]
+          removeFile fname
+          bracket_ (setCurrentDirectory name) (setCurrentDirectory ".." >> removeDirectoryRecursive name) $ do
+            exists <- doesFileExist "Setup.hs"
+            unless exists $ do
+              writeFile "Setup.hs" (unlines [
+                  "import Distribution.Simple",
+                  "main = defaultMain"
+                ])
+            let to_descr = reverse .
+                           (++) (reverse ".cabal") . 
+                           drop 1 . 
+                           dropWhile (/='-') . 
+                           reverse
+            callProcess "wget"   [to_descr pkg, "-O", to_descr name]
+            callProcess "runghc" ["Setup.hs","configure"]
+            callProcess "runghc" ["Setup.hs","build"]
+            callProcess "sudo" ["runghc","Setup.hs","install"]
+          
+      preConf simpleUserHooks args flags
+
     gfPreBuild args = gfPre args . buildDistPref
     gfPreInst  args = gfPre args . installDistPref
 
@@ -28,18 +59,13 @@ main = defaultMainWithHooks simpleUserHooks
       return emptyHookedBuildInfo
 
     gfPostBuild args flags pkg lbi = do
-      -- noRGLmsg
       let gf = default_gf lbi
       buildWeb gf flags (pkg,lbi)
 
     gfPostInst args flags pkg lbi = do
-      -- noRGLmsg
-      saveInstallPath args flags (pkg,lbi)
       installWeb (pkg,lbi)
 
     gfPostCopy args flags  pkg lbi = do
-      -- noRGLmsg
-      saveCopyPath args flags (pkg,lbi)
       copyWeb flags (pkg,lbi)
 
     -- `cabal sdist` will not make a proper dist archive, for that see `make sdist`
@@ -47,27 +73,16 @@ main = defaultMainWithHooks simpleUserHooks
     gfSDist pkg lbi hooks flags = do
       return ()
 
-saveInstallPath :: [String] -> InstallFlags -> (PackageDescription, LocalBuildInfo) -> IO ()
+dependencies = [
-saveInstallPath args flags bi = do
+  "https://hackage.haskell.org/package/utf8-string-1.0.2/utf8-string-1.0.2.tar.gz",
-  let
+  "https://hackage.haskell.org/package/json-0.10/json-0.10.tar.gz",
-    dest = NoCopyDest
+  "https://hackage.haskell.org/package/network-bsd-2.8.1.0/network-bsd-2.8.1.0.tar.gz",
-    dir = datadir (uncurry absoluteInstallDirs bi dest)
+  "https://hackage.haskell.org/package/httpd-shed-0.4.1.1/httpd-shed-0.4.1.1.tar.gz",
-  writeFile dataDirFile dir
+  "https://hackage.haskell.org/package/exceptions-0.10.5/exceptions-0.10.5.tar.gz",
-
+  "https://hackage.haskell.org/package/stringsearch-0.3.6.6/stringsearch-0.3.6.6.tar.gz",
-saveCopyPath :: [String] -> CopyFlags -> (PackageDescription, LocalBuildInfo) -> IO ()
+  "https://hackage.haskell.org/package/multipart-0.2.1/multipart-0.2.1.tar.gz",
-saveCopyPath args flags bi = do
+  "https://hackage.haskell.org/package/cgi-3001.5.0.0/cgi-3001.5.0.0.tar.gz"
-  let
+  ]
-    dest = case copyDest flags of
-      NoFlag -> NoCopyDest
-      Flag d -> d
-    dir = datadir (uncurry absoluteInstallDirs bi dest)
-  writeFile dataDirFile dir
-
--- | Name of file where installation's data directory is recording
--- This is a last-resort way in which the seprate RGL build script
--- can determine where to put the compiled RGL files
-dataDirFile :: String
-dataDirFile = "DATA_DIR"
 
 -- | Get path to locally-built gf
 default_gf :: LocalBuildInfo -> FilePath

bin/build-binary-dist.sh

@@ -32,7 +32,7 @@ set -x                             # print commands before executing them
 pushd src/runtime/c
 bash setup.sh configure --prefix="$prefix"
 bash setup.sh build
-bash setup.sh install prefix="$prefix" # hack required for GF build on macOS
+# bash setup.sh install prefix="$prefix" # hack required for GF build on macOS
 bash setup.sh install prefix="$destdir$prefix"
 popd
 
@@ -46,7 +46,7 @@ if which >/dev/null python; then
         pyver=$(ls "$destdir$prefix/lib" | sed -n 's/^python//p')
         pydest="$destdir/Library/Python/$pyver/site-packages"
         mkdir -p "$pydest"
-        ln "$destdir$prefix/lib/python$pyver/site-packages"/pgf* "$pydest"
+        ln "$destdir$prefix/lib/python$pyver/site-packages"/pgf*.so "$pydest"
     fi
     popd
 else

debian/changelog

@@ -1,3 +1,8 @@
+gf (3.12) noble; urgency=low
+
+  * GF 3.12
+
+ -- Inari Listenmaa <inari@digitalgrammars.com>  Fri, 8 Aug 2025 18:29:29 +0100
 gf (3.11) bionic focal; urgency=low
 
   * GF 3.11

debian/control

@@ -3,7 +3,7 @@ Section: devel
 Priority: optional
 Maintainer: Thomas Hallgren <hallgren@chalmers.se>
 Standards-Version: 3.9.2
-Build-Depends: debhelper (>= 5), haskell-platform (>= 2011.2.0.1), libghc-haskeline-dev, libghc-mtl-dev, libghc-json-dev, autoconf, automake, libtool-bin, python-dev, java-sdk
+Build-Depends: debhelper (>= 5), libghc-haskeline-dev, libghc-mtl-dev, libghc-json-dev, autoconf, automake, libtool-bin, python-dev-is-python3, java-sdk
 Homepage: http://www.grammaticalframework.org/
 
 Package: gf

debian/rules

@@ -16,7 +16,7 @@ override_dh_shlibdeps:
 override_dh_auto_configure:
 	cd src/runtime/c && bash setup.sh configure --prefix=/usr
 	cd src/runtime/c && bash setup.sh build
-	cabal v1-update
+	cabal update
 	cabal v1-install --only-dependencies
 	cabal v1-configure --prefix=/usr -fserver -fc-runtime --extra-lib-dirs=$(CURDIR)/src/runtime/c/.libs --extra-include-dirs=$(CURDIR)/src/runtime/c
 
@@ -24,7 +24,7 @@ SET_LDL=LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(CURDIR)/src/runtime/c/.libs
 
 override_dh_auto_build:
 	cd src/runtime/python && EXTRA_INCLUDE_DIRS=$(CURDIR)/src/runtime/c EXTRA_LIB_DIRS=$(CURDIR)/src/runtime/c/.libs python setup.py build
-	cd src/runtime/java && make CFLAGS="-I$(CURDIR)/src/runtime/c -L$(CURDIR)/src/runtime/c/.libs" INSTALL_PATH=/usr
+	# cd src/runtime/java && make CFLAGS="-I$(CURDIR)/src/runtime/c -L$(CURDIR)/src/runtime/c/.libs" INSTALL_PATH=/usr
 	echo $(SET_LDL)
 	-$(SET_LDL) cabal v1-build
 
@@ -32,13 +32,15 @@ override_dh_auto_install:
 	$(SET_LDL) cabal v1-copy --destdir=$(CURDIR)/debian/gf
 	cd src/runtime/c && bash setup.sh copy prefix=$(CURDIR)/debian/gf/usr
 	cd src/runtime/python && python setup.py install --prefix=$(CURDIR)/debian/gf/usr
-	cd src/runtime/java && make INSTALL_PATH=$(CURDIR)/debian/gf/usr install
+	# cd src/runtime/java && make INSTALL_PATH=$(CURDIR)/debian/gf/usr install
-	D="`find debian/gf -name site-packages`" && [ -n "$$D" ] && cd $$D && cd .. && mv site-packages dist-packages
+	# D="`find debian/gf -name dist-packages`" && [ -n "$$D" ] && cd $$D && cd .. && mv dist-packages dist-packages
+
+override_dh_usrlocal:
 
 override_dh_auto_clean:
 	rm -fr dist/build
 	-cd src/runtime/python && rm -fr build
-	-cd src/runtime/java && make clean
+	# -cd src/runtime/java && make clean
 	-cd src/runtime/c && make clean
 
 override_dh_auto_test:

doc/gf-developers.t2t

@@ -150,11 +150,9 @@ Open a terminal, go to the top directory (``gf-core``), and type the following c
 $ stack install
 ```
 
-It will install GF and all necessary tools and libraries to do that.
-
-
 === Alternative: use Cabal ===
-You can also install GF using Cabal, if you prefer Cabal to Stack. In that case, you may need to install some prerequisites yourself.
+
+If you prefer Cabal, then you just need to manually choose a suitable GHC to build GF. We recommend GHC 9.6.7, see other supported options in [gf.cabal https://github.com/GrammaticalFramework/gf-core/blob/master/gf.cabal#L14].
 
 The actual installation process is similar to Stack: open a terminal, go to the top directory (``gf-core``), and type the following command.
 
@@ -162,7 +160,13 @@ The actual installation process is similar to Stack: open a terminal, go to the
 $ cabal install
 ```
 
-//The old (potentially outdated) instructions for Cabal are moved to a [separate page ../doc/gf-developers-old-cabal.html]. If you run into trouble with ``cabal install``, you may want to take a look.//
+=== Nix ===
+
+As of 3.12, GF can also be installed via Nix. You can install GF from github with the following command:
+
+```
+nix profile install github:GrammaticalFramework/gf-core#gf
+```
 
 == Compiling GF with C runtime system support ==
 
@@ -197,7 +201,7 @@ Depending on what you want to do with the C runtime, you can follow one or more
 
 === Use the C runtime from another programming language ===[bindings]
 
-% **If you just want to use the C runtime from Python, Java, or Haskell, you don't need to change your GF installation.**
+% **If you just want to use the C runtime from Python or Haskell, you don't need to change your GF installation.**
 
 - **What —**
 This is the most common use case for the C runtime: compile
@@ -230,20 +234,13 @@ modes (use the ``help`` command in the shell for details).
 
 (Re)compiling your GF with these flags will also give you
 Haskell bindings to the C runtime, as a library called ``PGF2``,
-but if you want Python or Java bindings, you need to do [the previous step #bindings].
+but if you want Python bindings, you need to do [the previous step #bindings].
 
 % ``PGF2``: a module to import in Haskell programs, providing a binding to the C run-time system.
 
 - **How —**
-If you use cabal, run the following command:
 
-```
+Add (or uncomment) the following lines in the ``stack.yaml`` file:
-cabal install -fc-runtime
-```
-
-from the top directory (``gf-core``).
-
-If you use stack, uncomment the following lines in the ``stack.yaml`` file:
 
 ```
 flags:
@@ -254,6 +251,32 @@ extra-lib-dirs:
 ```
 and then run ``stack install`` from the top directory (``gf-core``).
 
+Run the newly built executable with the flag ``-cshell``, and you should see the following welcome message:
+
+```
+$ gf -cshell
+
+         *  *  *
+      *           *
+    *               *
+   *
+   *
+   *        * * * * * *
+   *        *         *
+    *       * * * *  *
+      *     *      *
+         *  *  *
+
+This is GF version 3.12.0.
+Built on ...
+Git info: ...
+
+Flags: interrupt server c-runtime
+License: see help -license.
+
+This shell uses the C run-time system. See help for available commands.
+>
+```
 
 //If you get an "``error while loading shared libraries``" when trying to run GF with C runtime, remember to declare your ``LD_LIBRARY_PATH``.//
 //Add ``export LD_LIBRARY_PATH="/usr/local/lib"`` to either your ``.bashrc`` or ``.profile``. You should now be able to start GF with C runtime.//
@@ -266,14 +289,8 @@ With this feature, ``gf -server`` mode is extended with new requests to call the
 system, e.g. ``c-parse``, ``c-linearize`` and ``c-translate``.
 
 - **How —**
-If you use cabal, run the following command:
 
-```
+Add the following lines in the ``stack.yaml`` file:
-cabal install -fc-runtime -fserver
-```
-from the top directory.
-
-If you use stack, add the following lines in the ``stack.yaml`` file:
 
 ```
 flags:

doc/gf-editor-modes.md

@@ -0,0 +1,75 @@
+# Editor modes & IDE integration for GF
+
+We collect GF modes for various editors on this page. Contributions are welcome!
+
+## Emacs
+
+[gf.el](https://github.com/GrammaticalFramework/gf-emacs-mode) by Johan
+Bockgård provides syntax highlighting and automatic indentation and
+lets you run the GF Shell in an emacs buffer.  See installation
+instructions inside.
+
+## Atom
+
+[language-gf](https://atom.io/packages/language-gf), by John J. Camilleri
+
+## Visual Studio Code
+
+* [Grammatical Framework Language Server](https://marketplace.visualstudio.com/items?itemName=anka-213.gf-vscode) by Andreas Källberg.
+This provides syntax highlighting and a client for the Grammatical Framework language server. Follow the installation instructions in the link.
+* [Grammatical Framework](https://marketplace.visualstudio.com/items?itemName=GrammaticalFramework.gf-vscode) is a simpler extension
+without any external dependencies which provides only syntax highlighting.
+
+## Eclipse
+
+[GF Eclipse Plugin](https://github.com/GrammaticalFramework/gf-eclipse-plugin/), by John J. Camilleri
+
+## Gedit
+
+By John J. Camilleri
+
+Copy the file below to
+`~/.local/share/gtksourceview-3.0/language-specs/gf.lang` (under Ubuntu).
+
+ * [gf.lang](../src/tools/gf.lang)
+
+Some helpful notes/links:
+
+ * The code is based heavily on the `haskell.lang` file which I found in
+`/usr/share/gtksourceview-2.0/language-specs/haskell.lang`.
+ * Ruslan Osmanov recommends
+[registering your file extension as its own MIME type](http://osmanov-dev-notes.blogspot.com/2011/04/how-to-add-new-highlight-mode-in-gedit.html)
+(see also [here](https://help.ubuntu.com/community/AddingMimeTypes)),
+however on my system the `.gf` extension was already registered
+as a generic font (`application/x-tex-gf`) and I didn't want to risk
+messing any of that up.
+ * This is a quick 5-minute job and might require some tweaking.
+[The GtkSourceView language definition tutorial](http://developer.gnome.org/gtksourceview/stable/lang-tutorial.html)
+is the place to start looking.
+ * Contributions are welcome!
+
+## Geany
+
+By John J. Camilleri
+
+[Custom filetype](http://www.geany.org/manual/dev/index.html#custom-filetypes)
+config files for syntax highlighting in [Geany](http://www.geany.org/).
+
+For version 1.36 and above, copy one of the files below to
+`/usr/share/geany/filedefs/filetypes.GF.conf` (under Ubuntu).
+If you're using a version older than 1.36, copy the file to `/usr/share/geany/filetypes.GF.conf`.
+You will need to manually create the file.
+
+ * [light-filetypes.GF.conf](../src/tools/light-filetypes.GF.conf)
+ * [dark-filetypes.GF.conf](../src/tools/dark-filetypes.GF.conf)
+
+You will also need to edit the `filetype_extensions.conf` file and add the
+following line somewhere:
+
+```
+GF=*.gf
+```
+
+## Vim
+
+[vim-gf](https://github.com/gdetrez/vim-gf)

doc/gf-editor-modes.t2t

@@ -1,78 +0,0 @@
-Editor modes & IDE integration for GF
-
-
-We collect GF modes for various editors on this page. Contributions are
-welcome!
-
-
-==Emacs==
-
-[gf.el https://github.com/GrammaticalFramework/gf-emacs-mode] by Johan
-Bockgård provides syntax highlighting and automatic indentation and
-lets you run the GF Shell in an emacs buffer.  See installation
-instructions inside.
-
-==Atom==
-[language-gf https://atom.io/packages/language-gf], by John J. Camilleri
-
-==Visual Studio Code==
-
-[Grammatical Framework Language Server https://marketplace.visualstudio.com/items?itemName=anka-213.gf-vscode] by Andreas Källberg.
-
-This provides syntax highlighting and a client for the Grammatical Framework language server. Follow the installation instructions in the link.
-
-==Eclipse==
-
-[GF Eclipse Plugin https://github.com/GrammaticalFramework/gf-eclipse-plugin/], by John J. Camilleri
-
-==Gedit==
-
-By John J. Camilleri
-
-Copy the file below to
-``~/.local/share/gtksourceview-3.0/language-specs/gf.lang`` (under Ubuntu).
-
-- [gf.lang ../src/tools/gf.lang]
-
-
-Some helpful notes/links:
-
-- The code is based heavily on the ``haskell.lang`` file which I found in
-  ``/usr/share/gtksourceview-2.0/language-specs/haskell.lang``.
-- Ruslan Osmanov recommends
-  [registering your file extension as its own MIME type http://osmanov-dev-notes.blogspot.com/2011/04/how-to-add-new-highlight-mode-in-gedit.html]
-  (see also [here https://help.ubuntu.com/community/AddingMimeTypes]),
-  however on my system the ``.gf`` extension was already registered
-  as a generic font (``application/x-tex-gf``) and I didn't want to risk
-  messing any of that up.
-- This is a quick 5-minute job and might require some tweaking.
-  [The GtkSourceView language definition tutorial http://developer.gnome.org/gtksourceview/stable/lang-tutorial.html]
-  is the place to start looking.
-- Contributions are welcome!
-
-
-==Geany==
-
-By John J. Camilleri
-
-[Custom filetype http://www.geany.org/manual/dev/index.html#custom-filetypes]
-config files for syntax highlighting in [Geany http://www.geany.org/].
-
-Copy one of the files below to ``/usr/share/geany/filetypes.GF.conf``
-(under Ubuntu). You will need to manually create the file.
-
-- [light-filetypes.GF.conf ../src/tools/light-filetypes.GF.conf]
-- [dark-filetypes.GF.conf ../src/tools/dark-filetypes.GF.conf]
-
-
-You will also need to edit the ``filetype_extensions.conf`` file and add the
-following line somewhere:
-
-```
-GF=*.gf
-```
-
-
-==Vim==
-
-[vim-gf https://github.com/gdetrez/vim-gf]

doc/gf-help-full.txt

@@ -303,7 +303,7 @@ but the resulting .gf file must be imported separately.
 
 #TINY
 
-Generates a list of random trees, by default one tree.
+Generates a list of random trees, by default one tree up to depth 5.
 If a tree argument is given, the command completes the Tree with values to
 all metavariables in the tree. The generation can be biased by probabilities,
 given in a file in the -probs flag.
@@ -315,13 +315,14 @@ given in a file in the -probs flag.
  | ``-cat`` | generation category
  | ``-lang`` | uses only functions that have linearizations in all these languages
  | ``-number`` | number of trees generated
- | ``-depth`` | the maximum generation depth
+ | ``-depth`` | the maximum generation depth (default: 5)
  | ``-probs`` | file with biased probabilities (format 'f 0.4' one by line)
 
 - Examples:
 
  | ``gr`` | one tree in the startcat of the current grammar
  | ``gr -cat=NP -number=16`` | 16 trees in the category NP
+ | ``gr -cat=NP -depth=2`` | one tree in the category NP, up to depth 2
  | ``gr -lang=LangHin,LangTha -cat=Cl`` | Cl, both in LangHin and LangTha
  | ``gr -probs=FILE`` | generate with bias
  | ``gr (AdjCN ? (UseN ?))`` | generate trees of form (AdjCN ? (UseN ?))
@@ -339,7 +340,7 @@ given in a file in the -probs flag.
 #TINY
 
 Generates all trees of a given category. By default,
-the depth is limited to 4, but this can be changed by a flag.
+the depth is limited to 5, but this can be changed by a flag.
 If a Tree argument is given, the command completes the Tree with values
 to all metavariables in the tree.
 
@@ -353,7 +354,7 @@ to all metavariables in the tree.
 
 - Examples:
 
- | ``gt`` | all trees in the startcat, to depth 4
+ | ``gt`` | all trees in the startcat, to depth 5
  | ``gt -cat=NP -number=16`` | 16 trees in the category NP
  | ``gt -cat=NP -depth=2`` | trees in the category NP to depth 2
  | ``gt (AdjCN ? (UseN ?))`` | trees of form (AdjCN ? (UseN ?))

doc/gf-people.md

@@ -7,7 +7,6 @@ title: "Grammatical Framework: Authors and Acknowledgements"
 The current maintainers of GF are
 
 [Krasimir Angelov](http://www.chalmers.se/cse/EN/organization/divisions/computing-science/people/angelov-krasimir),
-[Thomas Hallgren](http://www.cse.chalmers.se/~hallgren/),
 [Aarne Ranta](http://www.cse.chalmers.se/~aarne/),
 [John J. Camilleri](http://johnjcamilleri.com), and
 [Inari Listenmaa](https://inariksit.github.io/).
@@ -22,6 +21,7 @@ and
 
 The following people have contributed code to some of the versions:
 
+- [Thomas Hallgren](http://www.cse.chalmers.se/~hallgren/) (University of Gothenburg)
 - Grégoire Détrez (University of Gothenburg)
 - Ramona Enache (University of Gothenburg)
 - [Björn Bringert](http://www.cse.chalmers.se/alumni/bringert) (University of Gothenburg)

doc/tutorial/gf-tutorial.t2t

@@ -1188,7 +1188,7 @@ use ``generate_trees = gt``.
   this wine is fresh
   this wine is warm
 ```
-The default **depth** is 3; the depth can be
+The default **depth** is 5; the depth can be
 set by using the ``depth`` flag:
 ```
   > generate_trees -depth=2 | l
@@ -1265,10 +1265,16 @@ Human eye may prefer to see a visualization: ``visualize_tree = vt``:
   > parse "this delicious cheese is very Italian" | visualize_tree
 ```
 The tree is generated in postscript (``.ps``) file. The ``-view`` option is used for
-telling what command to use to view the file. Its default is ``"open"``, which works
+telling what command to use to view the file.
-on Mac OS X. On Ubuntu Linux, one can write
+
+This works on Mac OS X:
 ```
-  > parse "this delicious cheese is very Italian" | visualize_tree -view="eog"
+  > parse "this delicious cheese is very Italian" | visualize_tree -view=open
+```
+On Linux, one can use one of the following commands.
+```
+  > parse "this delicious cheese is very Italian" | visualize_tree -view=eog
+  > parse "this delicious cheese is very Italian" | visualize_tree -view=xdg-open
 ```
 
 
@@ -1733,6 +1739,13 @@ A new module can **extend** an old one:
       Pizza : Kind ;
   }
 ```
+Note that the extended grammar doesn't inherit the start
+category from the grammar it extends, so if you want to
+generate sentences with this grammar, you'll have to either
+add a startcat (e.g. ``flags startcat = Question ;``),
+or in the GF shell, specify the category to ``generate_random`` or ``geneate_trees``
+(e.g. ``gr -cat=Comment`` or ``gt -cat=Question``).
+
 Parallel to the abstract syntax, extensions can
 be built for concrete syntaxes:
 ```
@@ -4578,7 +4591,7 @@ in any multilingual grammar between any languages in the grammar.
 module Main where
 
 import PGF
-import System (getArgs)
+import System.Environment (getArgs)
 
 main :: IO ()
 main = do

download/index-3.11.md

@@ -53,26 +53,39 @@ You will probably need to update the `PATH` environment variable to include your
 
 For more information, see [Using GF on Windows](https://www.grammaticalframework.org/~inari/gf-windows.html) (latest updated for Windows 10).
 
-<!--## Installing the latest Hackage release (macOS, Linux, and WSL2 on Windows)
+## Installing from Hackage
+
+_Instructions applicable for macOS, Linux, and WSL2 on Windows._
 
 [GF is on Hackage](http://hackage.haskell.org/package/gf), so under
 normal circumstances the procedure is fairly simple:
 
-1.  Install ghcup https://www.haskell.org/ghcup/
+```
-2.  `ghcup install ghc 8.10.4`
+cabal update
-3.  `ghcup set ghc 8.10.4`
+cabal install gf-3.11
-4.  `cabal update`
+```
-5.  On Linux: install some C libraries from your Linux distribution (see note below)
-6.  `cabal install gf-3.11`
-
-You can also download the source code release from [GitHub](https://github.com/GrammaticalFramework/gf-core/releases),
-and follow the instructions below under **Installing from the latest developer source code**.
 
 ### Notes
 
+**GHC version**
+
+The GF source code is known to be compilable with GHC versions 7.10 through to 8.10.
+
+**Obtaining Haskell**
+
+There are various ways of obtaining Haskell, including:
+
+- ghcup
+    1.  Install from https://www.haskell.org/ghcup/
+    2.  `ghcup install ghc 8.10.4`
+    3.  `ghcup set ghc 8.10.4`
+- Haskell Platform https://www.haskell.org/platform/
+- Stack https://haskellstack.org/
+
+
 **Installation location**
 
-The above steps installs GF for a single user.
+The above steps install GF for a single user.
 The executables are put in `$HOME/.cabal/bin` (or on macOS in `$HOME/Library/Haskell/bin`),
 so you might want to add this directory to your path (in `.bash_profile` or similar):
 
@@ -84,32 +97,34 @@ PATH=$HOME/.cabal/bin:$PATH
 
 GF uses [`haskeline`](http://hackage.haskell.org/package/haskeline), which
 on Linux depends on some non-Haskell libraries that won't be installed
-automatically by cabal, and therefore need to be installed manually.
+automatically by Cabal, and therefore need to be installed manually.
 Here is one way to do this:
 
 - On Ubuntu: `sudo apt-get install libghc-haskeline-dev`
 - On Fedora: `sudo dnf install ghc-haskeline-devel`
 
-**GHC version**
+## Installing from source code
 
-The GF source code has been updated to compile with GHC versions 7.10 through to 8.8.
+**Obtaining**
--->
-## Installing from the latest developer source code
 
-If you haven't already, clone the repository with:
+To obtain the source code for the **release**,
+download it from [GitHub](https://github.com/GrammaticalFramework/gf-core/releases).
 
+Alternatively, to obtain the **latest version** of the source code:
+
+1. If you haven't already, clone the repository with:
 ```
 git clone https://github.com/GrammaticalFramework/gf-core.git
 ```
-
+2. If you've already cloned the repository previously, update with:
-If you've already cloned the repository previously, update with:
-
 ```
 git pull
 ```
 
-Then install with:
 
+**Installing**
+
+You can then install with:
 ```
 cabal install
 ```
@@ -124,6 +139,8 @@ stack install
 For more info on working with the GF source code, see the
 [GF Developers Guide](../doc/gf-developers.html).
 
+For macOS Sequoia, you need to downgrade the LLVM package, see instructions [here](https://github.com/GrammaticalFramework/gf-core/issues/172#issuecomment-2599365457).
+
 ## Installing the Python bindings from PyPI
 
 The Python library is available on PyPI as `pgf`, so it can be installed using:

download/index-3.12.md

@@ -0,0 +1,191 @@
+---
+title: Grammatical Framework Download and Installation
+date: 8 August 2025
+---
+
+**GF 3.12** was released on 8 August 2025.
+
+What's new? See the [release notes](release-3.12.html).
+
+#### Note: GF core and the RGL
+
+The following instructions explain how to install **GF core**, i.e. the compiler, shell and run-time systems.
+Obtaining the **Resource Grammar Library (RGL)** is done separately; see the section [at the bottom of this page](#installing-the-rgl-from-a-binary-release).
+
+---
+
+## Installing from a binary package
+
+Binary packages are available for Debian/Ubuntu, macOS, and Windows and include:
+
+- GF shell and grammar compiler
+- `gf -server` mode
+- C run-time system
+- Python bindings to the C run-time system
+
+[Binary packages on GitHub](https://github.com/GrammaticalFramework/gf-core/releases/tag/release-3.12)
+
+#### Debian/Ubuntu
+
+The package targets Ubuntu 24.04 (Noble).
+To install it, use:
+
+```
+sudo apt install ./gf-3.12-ubuntu-24.04.deb 
+```
+
+#### macOS
+
+If you are on an Intel Mac (2019 or older), use `gf-3.12-macos-intel.pkg`.<br>
+For newer ARM-based Macs (Apple Silicon M1, M2, M3), use `gf-3.12-macos-arm.pkg`.
+
+After downloading, right click on the file and click on Open.[^1]
+You will see a dialog saying that "macOS cannot verify the developer of "gf-3.12-macos-intel.pkg". Are you sure you want to open it?". 
+Press Open.
+
+[^1]: If you just double click on the file, you will get an error message "gf-3.12-macos-intel.pkg" cannot be opened because it is from an unidentified developer.
+
+#### Windows
+
+To install the package:
+
+1. unpack it anywhere and take note of the full path to the folder containing the `.exe` file. 
+2. add it to the `PATH` environment variable
+
+For more information, see [Using GF on Windows](https://www.grammaticalframework.org/~inari/gf-windows.html) (latest updated for Windows 10).
+
+## Installing from Hackage
+
+_Instructions applicable for macOS, Linux, and WSL2 on Windows._
+
+[GF is on Hackage](http://hackage.haskell.org/package/gf), so under
+normal circumstances the procedure is fairly simple:
+
+```
+cabal update
+cabal install gf-3.12
+```
+
+### Notes
+
+#### GHC version
+
+The GF source code is known to be compilable with GHC versions 7.10 through to 9.6.7.
+
+#### Obtaining Haskell
+
+There are various ways of obtaining Haskell, including:
+
+- ghcup
+    1. Install from https://www.haskell.org/ghcup/
+    2. `ghcup install ghc 9.6.7`
+    3. `ghcup set ghc 9.6.7`
+- Stack: https://haskellstack.org/
+
+
+#### Installation location
+
+The above steps install GF for a single user.
+The executables are put in `$HOME/.cabal/bin` (or on macOS in `$HOME/Library/Haskell/bin`),
+so you might want to add this directory to your path (in `.bash_profile` or similar):
+
+```
+PATH=$HOME/.cabal/bin:$PATH
+```
+
+#### Haskeline
+
+GF uses [`haskeline`](http://hackage.haskell.org/package/haskeline), which
+on Linux depends on some non-Haskell libraries that won't be installed
+automatically by Cabal, and therefore need to be installed manually.
+Here is one way to do this:
+
+- On Ubuntu: `sudo apt-get install libghc-haskeline-dev`
+- On Fedora: `sudo dnf install ghc-haskeline-devel`
+
+## Installing from source code
+
+### Obtaining
+
+To obtain the source code for the **release**,
+download it from [GitHub](https://github.com/GrammaticalFramework/gf-core/releases).
+
+Alternatively, to obtain the **latest version** of the source code:
+
+1. If you haven't already, clone the repository with:
+```
+git clone https://github.com/GrammaticalFramework/gf-core.git
+```
+2. If you've already cloned the repository previously, update with:
+```
+git pull
+```
+
+### Installing
+
+You can then install with:
+```
+cabal install
+```
+
+or, if you're a Stack user:
+
+```
+stack install
+```
+
+<!--The above notes for installing from source apply also in these cases.-->
+For more info on working with the GF source code, see the
+[GF Developers Guide](../doc/gf-developers.html).
+
+## Installing the Python bindings from PyPI
+
+The Python library is available on PyPI as `pgf`, so it can be installed using:
+
+```
+pip install pgf
+```
+
+If this doesn't work, you will need to install the C runtime manually; see the instructions [here](https://www.grammaticalframework.org/doc/gf-developers.html#toc12).
+
+---
+
+## Installing the RGL from a binary release
+
+Binary releases of the RGL are made available on [GitHub](https://github.com/GrammaticalFramework/gf-rgl/releases).
+In general the steps to follow are:
+
+1. Download a binary release and extract it somewhere on your system.
+2. Set the environment variable `GF_LIB_PATH` to point to wherever you extracted the RGL.
+
+For more information, see [Using GF on Windows](https://www.grammaticalframework.org/~inari/gf-windows.html) (latest updated for Windows 10).
+
+## Installing the RGL from source
+
+To compile the RGL, you will need to have GF already installed and in your path.
+
+1. Obtain the RGL source code, either by:
+  - cloning with `git clone https://github.com/GrammaticalFramework/gf-rgl.git`
+  - downloading a source archive [here](https://github.com/GrammaticalFramework/gf-rgl/archive/master.zip)
+2. Run `make` in the source code folder.
+
+For more options, see the [RGL README](https://github.com/GrammaticalFramework/gf-rgl/blob/master/README.md).
+
+---
+
+## Older releases
+
+- [GF 3.11](index-3.11.html) (July 2021)
+- [GF 3.10](index-3.10.html) (December 2018)
+- [GF 3.9](index-3.9.html) (August 2017)
+- [GF 3.8](index-3.8.html) (June 2016)
+- [GF 3.7.1](index-3.7.1.html) (October 2015)
+- [GF 3.7](index-3.7.html) (June 2015)
+- [GF 3.6](index-3.6.html) (June 2014)
+- [GF 3.5](index-3.5.html) (August 2013)
+- [GF 3.4](index-3.4.html) (January 2013)
+- [GF 3.3.3](index-3.3.3.html) (March 2012)
+- [GF 3.3](index-3.3.html) (October 2011)
+- [GF 3.2.9](index-3.2.9.html) source-only snapshot (September 2011)
+- [GF 3.2](index-3.2.html) (December 2010)
+- [GF 3.1.6](index-3.1.6.html) (April 2010)

download/index.html

@@ -1,8 +1,8 @@
 <html>
 <head>
-  <meta http-equiv="refresh" content="0; URL=/download/index-3.11.html" />
+  <meta http-equiv="refresh" content="0; URL=/download/index-3.12.html" />
 </head>
 <body>
-  You are being redirected to <a href="index-3.11.html">the current version</a> of this page.
+  You are being redirected to <a href="index-3.12.html">the current version</a> of this page.
 </body>
 </html>

download/release-3.12.md

@@ -0,0 +1,37 @@
+---
+title: GF 3.12 Release Notes
+date: 08 August 2025
+---
+
+## Installation
+
+See the [download page](index-3.12.html).
+
+## What's new
+This release adds support for Apple Silicon M1 Mac computers and newer versions of GHC, along with various improvements and bug fixes.
+
+Over 70 commits have been merged to gf-core since the release of GF 3.11 in July 2021.
+
+## General
+- Support for ARM, allowing to run GF on Mac computers with Apple Silicon M1
+- Support for newer versions of GHC (8.10.7, 9.0.2, 9.2.4, 9.4, 9.6.7)
+- Support compiling with Nix
+- Better error messages
+- Improvements to several GF shell commands
+- Several bug fixes and performance improvements
+- Temporarily dropped support for Java bindings
+
+## GF compiler and run-time library
+- Syntactic sugar for table update: `table {cases ; vvv => t \! vvv}.t` can now be written as `t ** { cases }`  
+- Adjust the `-view` command depending on the OS
+- Improve output of the `visualize_dependencies` (`vd`) command for large dependency trees
+- Reintroduce syntactic transfer with `pt -transfer` and fix a bug in `pt -compute`
+- Bug fix: apply `gt` to all arguments when piped
+- Fix many "Invalid character" messages by always encoding GF files in UTF-8
+- Improve performance with long extend-lists
+- Improve syntax error messages
+- Add support for BIND tokens in the Python bindings
+- Allow compilation with emscripten
+
+## Other
+- Add support for Visual Studio Code

flake.lock

@@ -0,0 +1,43 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1704290814,
+        "narHash": "sha256-LWvKHp7kGxk/GEtlrGYV68qIvPHkU9iToomNFGagixU=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "70bdadeb94ffc8806c0570eb5c2695ad29f0e421",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs",
+        "systems": "systems"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,50 @@
+{
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
+    systems.url = "github:nix-systems/default";
+  };
+
+  nixConfig = {
+    # extra-trusted-public-keys =
+    #   "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=";
+    # extra-substituters = "https://devenv.cachix.org";
+  };
+
+  outputs = { self, nixpkgs, systems, ... }@inputs:
+    let forEachSystem = nixpkgs.lib.genAttrs (import systems);
+    in {
+      packages = forEachSystem (system:
+        let
+          pkgs = nixpkgs.legacyPackages.${system};
+          haskellPackages = pkgs.haskell.packages.ghc925.override {
+            overrides = self: _super: {
+              cgi = pkgs.haskell.lib.unmarkBroken (pkgs.haskell.lib.dontCheck
+                (self.callHackage "cgi" "3001.5.0.1" { }));
+            };
+          };
+
+        in {
+          gf = pkgs.haskell.lib.overrideCabal
+            (haskellPackages.callCabal2nixWithOptions "gf" self "--flag=-server"
+              { }) (_old: {
+                # Fix utf8 encoding problems
+                patches = [
+                  # Already applied in master
+                  # (
+                  #   pkgs.fetchpatch {
+                  #     url = "https://github.com/anka-213/gf-core/commit/6f1ca05fddbcbc860898ddf10a557b513dfafc18.patch";
+                  #     sha256 = "17vn3hncxm1dwbgpfmrl6gk6wljz3r28j191lpv5zx741pmzgbnm";
+                  #   }
+                  # )
+                  ./nix/expose-all.patch
+                  ./nix/revert-new-cabal-madness.patch
+                ];
+                jailbreak = true;
+                # executableSystemDepends = [
+                #   (pkgs.ncurses.override { enableStatic = true; })
+                # ];
+                # executableHaskellDepends = [ ];
+              });
+        });
+    };
+}

gf-book/examples/chapter2/FoodIta.gf

@@ -2,7 +2,7 @@ concrete FoodIta of Food = {
   lincat
     Comment, Item, Kind, Quality = Str ;
   lin
-    Pred item quality = item ++ "è" ++ quality ;
+    Pred item quality = item ++ "è" ++ quality ;
     This kind = "questo" ++ kind ;
     That kind = "quel" ++ kind ;
     Mod quality kind = kind ++ quality ;

gf-book/examples/chapter3/ResIta.gf

@@ -32,5 +32,5 @@ resource ResIta = open Prelude in {
       in 
       adjective nero (ner+"a") (ner+"i") (ner+"e") ;
     copula : Number => Str = 
-      table {Sg => "è" ; Pl => "sono"} ;
+      table {Sg => "è" ; Pl => "sono"} ;
 }

gf-book/examples/chapter5/LexFoodsFin.gf

@@ -9,12 +9,12 @@ instance LexFoodsFin of LexFoods =
     fish_N = mkN "kala" ;
     fresh_A = mkA "tuore" ;
     warm_A = mkA
-    (mkN "lämmin" "lämpimän" "lämmintä" "lämpimänä" "lämpimään" 
+    (mkN "lämmin" "lämpimän" "lämmintä" "lämpimänä" "lämpimään"
-         "lämpiminä" "lämpimiä" "lämpimien" "lämpimissä" "lämpimiin"
+         "lämpiminä" "lämpimiä" "lämpimien" "lämpimissä" "lämpimiin"
 	 )
-    "lämpimämpi" "lämpimin" ;
+    "lämpimämpi" "lämpimin" ;
     italian_A = mkA "italialainen" ;
     expensive_A = mkA "kallis" ;
     delicious_A = mkA "herkullinen" ;
-    boring_A = mkA "tylsä" ;
+    boring_A = mkA "tylsä" ;
 }

gf-book/examples/chapter5/LexFoodsGer.gf

@@ -5,12 +5,12 @@ instance LexFoodsGer of LexFoods =
   oper
     wine_N = mkN "Wein" ;
     pizza_N = mkN "Pizza" "Pizzen" feminine ;
-    cheese_N = mkN "Käse" "Käse" masculine ;
+    cheese_N = mkN "Käse" "Käse" masculine ;
     fish_N = mkN "Fisch" ;
     fresh_A = mkA "frisch" ;
-    warm_A = mkA "warm" "wärmer" "wärmste" ;
+    warm_A = mkA "warm" "wärmer" "wärmste" ;
     italian_A = mkA "italienisch" ;
     expensive_A = mkA "teuer" ;
-    delicious_A = mkA "köstlich" ;
+    delicious_A = mkA "köstlich" ;
     boring_A = mkA "langweilig" ;
 }

gf-book/examples/chapter5/LexFoodsSwe.gf

@@ -7,10 +7,10 @@ instance LexFoodsSwe of LexFoods =
     pizza_N = mkN "pizza" ;
     cheese_N = mkN "ost" ;
     fish_N = mkN "fisk" ;
-    fresh_A = mkA "färsk" ;
+    fresh_A = mkA "färsk" ;
     warm_A = mkA "varm" ;
     italian_A = mkA "italiensk" ;
     expensive_A = mkA "dyr" ;
-    delicious_A = mkA "läcker" ;
+    delicious_A = mkA "läcker" ;
-    boring_A = mkA "tråkig" ;
+    boring_A = mkA "tråkig" ;
 }

gf-book/examples/chapter7/QueryFin.gf

@@ -6,7 +6,7 @@ concrete QueryFin of Query = {
     Odd   = pred "pariton" ;
     Prime = pred "alkuluku" ;
     Number i = i.s ;
-    Yes = "kyllä" ;
+    Yes = "kyllä" ;
     No = "ei" ;
   oper
     pred : Str -> Str -> Str = \f,x -> "onko" ++ x ++ f ;

gf-book/examples/chapter9/ResIta.gf

@@ -46,7 +46,7 @@ oper
     Avere =>
       mkVerb "avere" "ho" "hai" "ha" "abbiamo" "avete" "hanno" "avuto" Avere ;
     Essere =>
-      mkVerb "essere" "sono" "sei" "è" "siamo" "siete" "sono" "stato" Essere
+      mkVerb "essere" "sono" "sei" "è" "siamo" "siete" "sono" "stato" Essere
     } ;
 
   agrPart : Verb -> Agr -> ClitAgr -> Str = \v,a,c -> case v.aux of {

gf.cabal

@@ -1,19 +1,24 @@
 name: gf
-version: 3.11.0-git
+version: 3.12.0
 
 cabal-version: 1.22
-build-type: Custom
+build-type: Simple
 license: OtherLicense
 license-file: LICENSE
 category: Natural Language Processing, Compiler
 synopsis: Grammatical Framework
 description: GF, Grammatical Framework, is a programming language for multilingual grammar applications
+maintainer: John J. Camilleri <john@digitalgrammars.com>
 homepage: https://www.grammaticalframework.org/
 bug-reports: https://github.com/GrammaticalFramework/gf-core/issues
-tested-with: GHC==7.10.3, GHC==8.0.2, GHC==8.10.4
+tested-with: GHC==7.10.3, GHC==8.0.2, GHC==8.10.4, GHC==9.0.2, GHC==9.2.4, GHC==9.6.7
 
 data-dir: src
-extra-source-files: WebSetup.hs
+extra-source-files:
+  README.md
+  CHANGELOG.md
+  WebSetup.hs
+  doc/Logos/gf0.png
 data-files:
   www/*.html
   www/*.css
@@ -39,14 +44,6 @@ data-files:
   www/translator/*.css
   www/translator/*.js
 
-custom-setup
-  setup-depends:
-    base >= 4.9.1 && < 4.15,
-    Cabal >= 1.22.0.0,
-    directory >= 1.3.0 && < 1.4,
-    filepath >= 1.4.1 && < 1.5,
-    process >= 1.0.1.1 && < 1.7
-
 source-repository head
   type: git
   location: https://github.com/GrammaticalFramework/gf-core.git
@@ -76,23 +73,20 @@ library
   build-depends:
     -- GHC 8.0.2 to GHC 8.10.4
     array >= 0.5.1 && < 0.6,
-    base >= 4.9.1 && < 4.15,
+    base >= 4.9.1 && < 4.22,
-    bytestring >= 0.10.8 && < 0.11,
+    bytestring >= 0.10.8 && < 0.12,
     containers >= 0.5.7 && < 0.7,
     exceptions >= 0.8.3 && < 0.11,
-    ghc-prim >= 0.5.0 && < 0.7,
+    ghc-prim >= 0.5.0 && <= 0.10.0,
-    hashable >= 1.2.6 && < 1.4,
+    mtl >= 2.2.1 && <= 2.3.1,
-    mtl >= 2.2.1 && < 2.3,
     pretty >= 1.1.3 && < 1.2,
     random >= 1.1 && < 1.3,
-    text >= 1.2.2 && < 1.3,
+    utf8-string >= 1.0.1.1 && < 1.1
-    unordered-containers >= 0.2.8 && < 0.3,
-    utf8-string >= 1.0.1.1 && < 1.1,
-    -- We need transformers-compat >= 0.6.3, but that is only in newer snapshots where it is redundant.
-    transformers-compat >= 0.5.1.4 && < 0.7
 
   if impl(ghc<8.0)
     build-depends:
+      -- We need this in order for ghc-7.10 to build
+      transformers-compat >= 0.6.3 && < 0.7,
       fail >= 4.9.0 && < 4.10
 
   hs-source-dirs: src/runtime/haskell
@@ -109,16 +103,14 @@ library
 --ghc-options: -fwarn-unused-imports
 --if impl(ghc>=7.8)
 --  ghc-options: +RTS -A20M -RTS
-  ghc-prof-options: -fprof-auto
+  -- ghc-prof-options: -fprof-auto
 
   exposed-modules:
-    LPGF
     PGF
     PGF.Internal
     PGF.Haskell
 
   other-modules:
-    LPGF.Internal
     PGF.Data
     PGF.Macros
     PGF.Binary
@@ -163,10 +155,11 @@ library
     directory >= 1.3.0 && < 1.4,
     filepath >= 1.4.1 && < 1.5,
     haskeline >= 0.7.3 && < 0.9,
-    json >= 0.9.1 && < 0.11,
+    json >= 0.9.1 && <= 0.11,
     parallel >= 3.2.1.1 && < 3.3,
     process >= 1.4.3 && < 1.7,
-    time >= 1.6.0 && < 1.10
+    time >= 1.6.0 && <= 1.12.2,
+    template-haskell >= 2.13.0.0 && < 2.21
 
   hs-source-dirs: src/compiler
   exposed-modules:
@@ -211,7 +204,6 @@ library
     GF.Compile.Export
     GF.Compile.GenerateBC
     GF.Compile.GeneratePMCFG
-    GF.Compile.GrammarToLPGF
     GF.Compile.GrammarToPGF
     GF.Compile.Multi
     GF.Compile.Optimize
@@ -240,7 +232,6 @@ library
     GF.Data.ErrM
     GF.Data.Graph
     GF.Data.Graphviz
-    GF.Data.IntMapBuilder
     GF.Data.Relation
     GF.Data.Str
     GF.Data.Utilities
@@ -304,14 +295,14 @@ library
     build-depends:
       cgi >= 3001.3.0.2 && < 3001.6,
       httpd-shed >= 0.4.0 && < 0.5,
-      network>=2.3 && <2.7
+      network>=2.3 && <3.2
     if flag(network-uri)
       build-depends:
         network-uri >= 2.6.1.0 && < 2.7,
-        network>=2.6 && <2.7
+        network>=2.6 && <3.2
     else
       build-depends:
-        network >= 2.5 && <2.6
+        network >= 2.5 && <3.2
 
     cpp-options: -DSERVER_MODE
     other-modules:
@@ -356,8 +347,14 @@ library
       Win32 >= 2.3.1.1 && < 2.7
   else
     build-depends:
-      terminfo >=0.4.0 && < 0.5,
+      terminfo >=0.4.0 && < 0.5
-      unix >= 2.7.2 && < 2.8
+
+    if impl(ghc >= 9.6)
+      build-depends: unix >= 2.8 && < 2.9
+
+    else
+      build-depends: unix >= 2.7.2 && < 2.8
+
 
   if impl(ghc>=8.2)
     ghc-options: -fhide-source-paths
@@ -368,7 +365,7 @@ executable gf
   default-language: Haskell2010
   build-depends:
     gf,
-    base
+    base >= 4.9.1 && < 4.22
   ghc-options: -threaded
 --ghc-options: -fwarn-unused-imports
 
@@ -377,7 +374,7 @@ executable gf
     if impl(ghc<7.8)
       ghc-options: -with-rtsopts=-K64M
 
-  ghc-prof-options: -auto-all
+  -- ghc-prof-options: -auto-all
 
   if impl(ghc>=8.2)
     ghc-options: -fhide-source-paths
@@ -402,422 +399,10 @@ test-suite gf-tests
   main-is: run.hs
   hs-source-dirs: testsuite
   build-depends:
-    base >= 4.9.1 && < 4.15,
+    base >= 4.9.1 && < 4.22,
     Cabal >= 1.8,
     directory >= 1.3.0 && < 1.4,
     filepath >= 1.4.1 && < 1.5,
     process >= 1.4.3 && < 1.7
   build-tool-depends: gf:gf
   default-language: Haskell2010
-
-test-suite lpgf
-  type: exitcode-stdio-1.0
-  main-is: test.hs
-  hs-source-dirs:
-    src/compiler
-    src/runtime/haskell
-    testsuite/lpgf
-  other-modules:
-    Data.Binary
-    Data.Binary.Builder
-    Data.Binary.Get
-    Data.Binary.IEEE754
-    Data.Binary.Put
-    GF
-    GF.Command.Abstract
-    GF.Command.CommandInfo
-    GF.Command.Commands
-    GF.Command.CommonCommands
-    GF.Command.Help
-    GF.Command.Importing
-    GF.Command.Interpreter
-    GF.Command.Messages
-    GF.Command.Parse
-    GF.Command.SourceCommands
-    GF.Command.TreeOperations
-    GF.Compile
-    GF.Compile.CFGtoPGF
-    GF.Compile.CheckGrammar
-    GF.Compile.Compute.Concrete
-    GF.Compile.Compute.Predef
-    GF.Compile.Compute.Value
-    GF.Compile.ConcreteToHaskell
-    GF.Compile.ExampleBased
-    GF.Compile.Export
-    GF.Compile.GenerateBC
-    GF.Compile.GeneratePMCFG
-    GF.Compile.GetGrammar
-    GF.Compile.GrammarToCanonical
-    GF.Compile.GrammarToLPGF
-    GF.Compile.GrammarToPGF
-    GF.Compile.Multi
-    GF.Compile.Optimize
-    GF.Compile.PGFtoHaskell
-    GF.Compile.PGFtoJava
-    GF.Compile.PGFtoJS
-    GF.Compile.PGFtoJSON
-    GF.Compile.PGFtoProlog
-    GF.Compile.PGFtoPython
-    GF.Compile.ReadFiles
-    GF.Compile.Rename
-    GF.Compile.SubExOpt
-    GF.Compile.Tags
-    GF.Compile.ToAPI
-    GF.Compile.TypeCheck.Abstract
-    GF.Compile.TypeCheck.Concrete
-    GF.Compile.TypeCheck.ConcreteNew
-    GF.Compile.TypeCheck.Primitives
-    GF.Compile.TypeCheck.TC
-    GF.Compile.Update
-    GF.CompileInParallel
-    GF.CompileOne
-    GF.Compiler
-    GF.Data.BacktrackM
-    GF.Data.ErrM
-    GF.Data.Graph
-    GF.Data.Graphviz
-    GF.Data.IntMapBuilder
-    GF.Data.Operations
-    GF.Data.Relation
-    GF.Data.Str
-    GF.Data.Utilities
-    GF.Data.XML
-    GF.Grammar
-    GF.Grammar.Analyse
-    GF.Grammar.Binary
-    GF.Grammar.BNFC
-    GF.Grammar.Canonical
-    GF.Grammar.CanonicalJSON
-    GF.Grammar.CFG
-    GF.Grammar.EBNF
-    GF.Grammar.Grammar
-    GF.Grammar.Lexer
-    GF.Grammar.Lockfield
-    GF.Grammar.Lookup
-    GF.Grammar.Macros
-    GF.Grammar.Parser
-    GF.Grammar.PatternMatch
-    GF.Grammar.Predef
-    GF.Grammar.Printer
-    GF.Grammar.ShowTerm
-    GF.Grammar.Unify
-    GF.Grammar.Values
-    GF.Haskell
-    GF.Infra.BuildInfo
-    GF.Infra.CheckM
-    GF.Infra.Concurrency
-    GF.Infra.Dependencies
-    GF.Infra.GetOpt
-    GF.Infra.Ident
-    GF.Infra.Location
-    GF.Infra.Option
-    GF.Infra.SIO
-    GF.Infra.UseIO
-    GF.Interactive
-    GF.JavaScript.AbsJS
-    GF.JavaScript.PrintJS
-    GF.Main
-    GF.Quiz
-    GF.Speech.CFGToFA
-    GF.Speech.FiniteState
-    GF.Speech.GSL
-    GF.Speech.JSGF
-    GF.Speech.PGFToCFG
-    GF.Speech.PrRegExp
-    GF.Speech.RegExp
-    GF.Speech.SISR
-    GF.Speech.SLF
-    GF.Speech.SRG
-    GF.Speech.SRGS_ABNF
-    GF.Speech.SRGS_XML
-    GF.Speech.VoiceXML
-    GF.Support
-    GF.System.Catch
-    GF.System.Concurrency
-    GF.System.Console
-    GF.System.Directory
-    GF.System.Process
-    GF.System.Signal
-    GF.Text.Clitics
-    GF.Text.Coding
-    GF.Text.Lexing
-    GF.Text.Pretty
-    GF.Text.Transliterations
-    LPGF
-    LPGF.Internal
-    PGF
-    PGF.Binary
-    PGF.ByteCode
-    PGF.CId
-    PGF.Data
-    PGF.Expr
-    PGF.Forest
-    PGF.Generate
-    PGF.Internal
-    PGF.Linearize
-    PGF.Macros
-    PGF.Morphology
-    PGF.OldBinary
-    PGF.Optimize
-    PGF.Paraphrase
-    PGF.Parse
-    PGF.Printer
-    PGF.Probabilistic
-    PGF.Tree
-    PGF.TrieMap
-    PGF.Type
-    PGF.TypeCheck
-    PGF.Utilities
-    PGF.VisualizeTree
-    Paths_gf
-  if flag(interrupt)
-    cpp-options: -DUSE_INTERRUPT
-    other-modules: GF.System.UseSignal
-  else
-    other-modules: GF.System.NoSignal
-  build-depends:
-    ansi-terminal >= 0.6.3 && < 0.12,
-    array >= 0.5.1 && < 0.6,
-    base >=4.6 && < 5,
-    bytestring >= 0.10.8 && < 0.11,
-    containers >= 0.5.7 && < 0.7,
-    directory >= 1.3.0 && < 1.4,
-    filepath >= 1.4.1 && < 1.5,
-    ghc-prim >= 0.5.0 && < 0.7,
-    hashable >= 1.2.6 && < 1.4,
-    haskeline >= 0.7.3 && < 0.9,
-    json >= 0.9.1 && < 0.11,
-    mtl >= 2.2.1 && < 2.3,
-    parallel >= 3.2.1.1 && < 3.3,
-    pretty >= 1.1.3 && < 1.2,
-    process >= 1.4.3 && < 1.7,
-    random >= 1.1 && < 1.3,
-    text >= 1.2.2 && < 1.3,
-    time >= 1.6.0 && < 1.10,
-    transformers-compat >= 0.5.1.4 && < 0.7,
-    unordered-containers >= 0.2.8 && < 0.3,
-    utf8-string >= 1.0.1.1 && < 1.1
-
-  if impl(ghc<8.0)
-    build-depends:
-      fail >= 4.9.0 && < 4.10
-
-  if os(windows)
-    build-depends:
-      Win32 >= 2.3.1.1 && < 2.7
-  else
-    build-depends:
-      unix >= 2.7.2 && < 2.8,
-      terminfo >=0.4.0 && < 0.5
-
-  default-language: Haskell2010
-
-benchmark lpgf-bench
-  type: exitcode-stdio-1.0
-  main-is: bench.hs
-  hs-source-dirs:
-    src/compiler
-    src/runtime/haskell
-    testsuite/lpgf
-  other-modules:
-    Data.Binary
-    Data.Binary.Builder
-    Data.Binary.Get
-    Data.Binary.IEEE754
-    Data.Binary.Put
-    GF
-    GF.Command.Abstract
-    GF.Command.CommandInfo
-    GF.Command.Commands
-    GF.Command.CommonCommands
-    GF.Command.Help
-    GF.Command.Importing
-    GF.Command.Interpreter
-    GF.Command.Messages
-    GF.Command.Parse
-    GF.Command.SourceCommands
-    GF.Command.TreeOperations
-    GF.Compile
-    GF.Compile.CFGtoPGF
-    GF.Compile.CheckGrammar
-    GF.Compile.Compute.Concrete
-    GF.Compile.Compute.Predef
-    GF.Compile.Compute.Value
-    GF.Compile.ConcreteToHaskell
-    GF.Compile.ExampleBased
-    GF.Compile.Export
-    GF.Compile.GenerateBC
-    GF.Compile.GeneratePMCFG
-    GF.Compile.GetGrammar
-    GF.Compile.GrammarToCanonical
-    GF.Compile.GrammarToLPGF
-    GF.Compile.GrammarToPGF
-    GF.Compile.Multi
-    GF.Compile.Optimize
-    GF.Compile.PGFtoHaskell
-    GF.Compile.PGFtoJS
-    GF.Compile.PGFtoJSON
-    GF.Compile.PGFtoJava
-    GF.Compile.PGFtoProlog
-    GF.Compile.PGFtoPython
-    GF.Compile.ReadFiles
-    GF.Compile.Rename
-    GF.Compile.SubExOpt
-    GF.Compile.Tags
-    GF.Compile.ToAPI
-    GF.Compile.TypeCheck.Abstract
-    GF.Compile.TypeCheck.Concrete
-    GF.Compile.TypeCheck.ConcreteNew
-    GF.Compile.TypeCheck.Primitives
-    GF.Compile.TypeCheck.TC
-    GF.Compile.Update
-    GF.CompileInParallel
-    GF.CompileOne
-    GF.Compiler
-    GF.Data.BacktrackM
-    GF.Data.ErrM
-    GF.Data.Graph
-    GF.Data.Graphviz
-    GF.Data.IntMapBuilder
-    GF.Data.Operations
-    GF.Data.Relation
-    GF.Data.Str
-    GF.Data.Utilities
-    GF.Data.XML
-    GF.Grammar
-    GF.Grammar.Analyse
-    GF.Grammar.BNFC
-    GF.Grammar.Binary
-    GF.Grammar.CFG
-    GF.Grammar.Canonical
-    GF.Grammar.CanonicalJSON
-    GF.Grammar.EBNF
-    GF.Grammar.Grammar
-    GF.Grammar.Lexer
-    GF.Grammar.Lockfield
-    GF.Grammar.Lookup
-    GF.Grammar.Macros
-    GF.Grammar.Parser
-    GF.Grammar.PatternMatch
-    GF.Grammar.Predef
-    GF.Grammar.Printer
-    GF.Grammar.ShowTerm
-    GF.Grammar.Unify
-    GF.Grammar.Values
-    GF.Haskell
-    GF.Infra.BuildInfo
-    GF.Infra.CheckM
-    GF.Infra.Concurrency
-    GF.Infra.Dependencies
-    GF.Infra.GetOpt
-    GF.Infra.Ident
-    GF.Infra.Location
-    GF.Infra.Option
-    GF.Infra.SIO
-    GF.Infra.UseIO
-    GF.Interactive
-    GF.JavaScript.AbsJS
-    GF.JavaScript.PrintJS
-    GF.Main
-    GF.Quiz
-    GF.Speech.CFGToFA
-    GF.Speech.FiniteState
-    GF.Speech.GSL
-    GF.Speech.JSGF
-    GF.Speech.PGFToCFG
-    GF.Speech.PrRegExp
-    GF.Speech.RegExp
-    GF.Speech.SISR
-    GF.Speech.SLF
-    GF.Speech.SRG
-    GF.Speech.SRGS_ABNF
-    GF.Speech.SRGS_XML
-    GF.Speech.VoiceXML
-    GF.Support
-    GF.System.Catch
-    GF.System.Concurrency
-    GF.System.Console
-    GF.System.Directory
-    GF.System.Process
-    GF.System.Signal
-    GF.Text.Clitics
-    GF.Text.Coding
-    GF.Text.Lexing
-    GF.Text.Pretty
-    GF.Text.Transliterations
-    LPGF
-    LPGF.Internal
-    PGF
-    PGF.Binary
-    PGF.ByteCode
-    PGF.CId
-    PGF.Data
-    PGF.Expr
-    PGF.Expr
-    PGF.Forest
-    PGF.Generate
-    PGF.Internal
-    PGF.Linearize
-    PGF.Macros
-    PGF.Morphology
-    PGF.OldBinary
-    PGF.Optimize
-    PGF.Paraphrase
-    PGF.Parse
-    PGF.Printer
-    PGF.Probabilistic
-    PGF.Tree
-    PGF.TrieMap
-    PGF.Type
-    PGF.TypeCheck
-    PGF.Utilities
-    PGF.VisualizeTree
-    PGF2
-    PGF2.Expr
-    PGF2.Type
-    PGF2.FFI
-    Paths_gf
-  if flag(interrupt)
-    cpp-options: -DUSE_INTERRUPT
-    other-modules: GF.System.UseSignal
-  else
-    other-modules: GF.System.NoSignal
-
-  hs-source-dirs:
-    src/runtime/haskell-bind
-  other-modules:
-    PGF2
-    PGF2.FFI
-    PGF2.Expr
-    PGF2.Type
-  build-tools:     hsc2hs
-  extra-libraries: pgf gu
-  c-sources:       src/runtime/haskell-bind/utils.c
-  cc-options:      -std=c99
-
-  build-depends:
-    ansi-terminal,
-    array,
-    base>=4.6 && <5,
-    bytestring,
-    containers,
-    deepseq,
-    directory,
-    filepath,
-    ghc-prim,
-    hashable,
-    haskeline,
-    json,
-    mtl,
-    parallel>=3,
-    pretty,
-    process,
-    random,
-    terminfo,
-    text,
-    time,
-    transformers-compat,
-    unix,
-    unordered-containers,
-    utf8-string
-  default-language: Haskell2010

index.html

@@ -8,7 +8,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
   <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
-  <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.4.2/css/all.css" integrity="sha384-/rXc/GQVaYpyDdyxK+ecHPVYJSN9bmVFBvjA/9eOB+pb3F2w2N6fc5qB9Ew5yIns" crossorigin="anonymous">
+  <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.15.4/css/all.css" crossorigin="anonymous">
 
   <link rel="alternate" href="https://github.com/GrammaticalFramework/gf-core/" title="GF GitHub repository">
 </head>
@@ -55,13 +55,14 @@
       <li><a href="gf-book">The GF Book</a></li>
       <li><a href="doc/gf-refman.html">Reference Manual</a></li>
       <li><a href="doc/gf-shell-reference.html">Shell Reference</a></li>
-      <li><a href="http://www.molto-project.eu/sites/default/files/MOLTO_D2.3.pdf">Best Practices</a> <small>[PDF]</small></li>
+      <li><a href="https://www.grammaticalframework.org/doc/MOLTO_D2.3.pdf">Best Practices</a> <small>[PDF]</small></li>
       <li><a href="https://www.mitpressjournals.org/doi/pdf/10.1162/COLI_a_00378">Scaling Up (Computational Linguistics 2020)</a></li>
+      <li><a href="https://inariksit.github.io/blog/">GF blog</a></li>
     </ul>
 
     <a href="lib/doc/synopsis/index.html" class="btn btn-primary ml-3">
       <i class="fab fa-readme mr-1"></i>
-      RGL Synopsis
+      RGL API
     </a>
   </div>
 
@@ -71,8 +72,12 @@
       <li><a href="doc/gf-developers.html">Developers Guide</a></li>
       <!-- <li><a href="/~hallgren/gf-experiment/browse/">Browse Source Code</a></li> -->
       <li>PGF library API:<br>
-        <a href="http://hackage.haskell.org/package/gf/docs/PGF.html">Haskell</a> /
+        <ul>
-        <a href="doc/runtime-api.html">C&nbsp;runtime</a>
+          <li><a href="http://hackage.haskell.org/package/gf/docs/PGF.html">Haskell</a>
+          </li><li><a href="doc/runtime-api.html#python">Python</a>
+          </li><li><a href="doc/runtime-api.html">C&nbsp;runtime</a>
+          </li>
+        </ul>
       </li>
       <li><a href="http://hackage.haskell.org/package/gf/docs/GF.html">GF compiler API</a></li>
       <!-- <li><a href="src/ui/android/README">GF on Android (new)</a></li>
@@ -85,10 +90,22 @@
   <div class="col-sm-6 col-md-3 mb-4">
     <h3>Contribute</h3>
     <ul class="mb-2">
-      <li><a href="http://groups.google.com/group/gf-dev">Mailing List</a></li>
+      <li>
+        <a href="https://discord.gg/EvfUsjzmaz">
+          <i class="fab fa-discord"></i>
+          Discord
+        </a>
+      </li>
+      <li>
+        <a href="https://stackoverflow.com/questions/tagged/gf">
+          <i class="fab fa-stack-overflow"></i>
+          Stack Overflow
+        </a>
+      </li>
+      <li><a href="https://groups.google.com/group/gf-dev">Mailing List</a></li>
       <li><a href="https://github.com/GrammaticalFramework/gf-core/issues">Issue Tracker</a></li>
+      <li><a href="//school.grammaticalframework.org/">Summer School</a></li>
       <li><a href="doc/gf-people.html">Authors</a></li>
-      <li><a href="//school.grammaticalframework.org/2020/">Summer School</a></li>
     </ul>
     <a href="https://github.com/GrammaticalFramework/" class="btn btn-primary ml-3">
       <i class="fab fa-github mr-1"></i>
@@ -154,7 +171,7 @@ least one, it may help you to get a first idea of what GF is.
 <div class="row">
 
   <div class="col-md-6">
-    <h2>Applications & Availability</h2>
+    <h2>Applications & availability</h2>
     <p>
     GF can be used for building
     <a href="//cloud.grammaticalframework.org/translator/">translation systems</a>,
@@ -214,64 +231,50 @@ least one, it may help you to get a first idea of what GF is.
     </p>
 
     <p>
-    We run the IRC channel <strong><code>#gf</code></strong> on the Libera network, where you are welcome to look for help with small questions or just start a general discussion.
+    We run the <a href="https://discord.gg/EvfUsjzmaz">GF server on Discord</a>, where you are welcome to look for help with small questions or just start a general discussion.
-    You can <a href="https://web.libera.chat/?channels=#gf">open a web chat</a>
-    or <a href="https://www.grammaticalframework.org/irc/?C=M;O=D">browse the channel logs</a>.
     </p>
+
+
     <p>
-    If you have a larger question which the community may benefit from, we recommend you ask it on the <a href="http://groups.google.com/group/gf-dev">mailing list</a>.
+    For bug reports and feature requests, please create an issue in the
+    <a href="https://github.com/GrammaticalFramework/gf-core/issues">GF Core</a> or
+    <a href="https://github.com/GrammaticalFramework/gf-rgl/issues">RGL</a> repository.
+
+    For programming questions, consider asking them on <a href="https://stackoverflow.com/questions/tagged/gf">Stack Overflow with the <code>gf</code> tag</a>.
+    If you have a more general question to the community, we recommend you ask it on the <a href="http://groups.google.com/group/gf-dev">mailing list</a>.
     </p>
 
   </div>
 
   <div class="col-md-6">
     <h2>News</h2>
+    <dl class="row">
+      <dt class="col-sm-3 text-center text-nowrap">2025-08-08</dt>
+      <dd class="col-sm-9">
+        <strong>GF 3.12 released.</strong>
+        <a href="download/release-3.12.html">Release notes</a>
+      </dd>
+      <dt class="col-sm-3 text-center text-nowrap">2025-01-18</dt>
+      <dd class="col-sm-9">
+        <a href="//school.grammaticalframework.org/2025/">9th GF Summer School</a>, in Gothenburg, Sweden, 18 &ndash; 29 August 2025.
+      </dd>
+      <dt class="col-sm-3 text-center text-nowrap">2023-01-24</dt>
+      <dd class="col-sm-9">
+        <a href="//school.grammaticalframework.org/2023/">8th GF Summer School</a>, in Tampere, Finland, 14 &ndash; 25 August 2023.
+      </dd>
       <dt class="col-sm-3 text-center text-nowrap">2021-07-25</dt>
       <dd class="col-sm-9">
         <strong>GF 3.11 released.</strong>
         <a href="download/release-3.11.html">Release notes</a>
       </dd>
-    <dl class="row">
       <dt class="col-sm-3 text-center text-nowrap">2021-05-05</dt>
       <dd class="col-sm-9">
         <a href="https://cloud.grammaticalframework.org/wordnet/">GF WordNet</a> now supports languages for which there are no other WordNets. New additions: Afrikaans, German, Korean, Maltese, Polish, Somali, Swahili.
       </dd>
-      <dt class="col-sm-3 text-center text-nowrap">2021-03-01</dt>
-      <dd class="col-sm-9">
-        <a href="//school.grammaticalframework.org/2020/">Seventh GF Summer School</a>, in Singapore and online, 26 July &ndash; 6 August 2021.
-      </dd>
       <dt class="col-sm-3 text-center text-nowrap">2020-09-29</dt>
       <dd class="col-sm-9">
         <a href="https://www.mitpressjournals.org/doi/pdf/10.1162/COLI_a_00378">Abstract Syntax as Interlingua</a>: Scaling Up the Grammatical Framework from Controlled Languages to Robust Pipelines. A paper in Computational Linguistics (2020) summarizing much of the development in GF in the past ten years.
       </dd>
-      <dt class="col-sm-3 text-center text-nowrap">2018-12-03</dt>
-      <dd class="col-sm-9">
-        <a href="//school.grammaticalframework.org/2018/">Sixth GF Summer School</a> in Stellenbosch (South Africa), 3–14 December 2018
-      </dd>
-      <dt class="col-sm-3 text-center text-nowrap">2018-12-02</dt>
-      <dd class="col-sm-9">
-        <strong>GF 3.10 released.</strong>
-        <a href="download/release-3.10.html">Release notes</a>
-      </dd>
-      <dt class="col-sm-3 text-center text-nowrap">2018-07-25</dt>
-      <dd class="col-sm-9">
-        The GF repository has been split in two:
-        <a href="https://github.com/GrammaticalFramework/gf-core">gf-core</a> and
-        <a href="https://github.com/GrammaticalFramework/gf-rgl">gf-rgl</a>.
-        The original <a href="https://github.com/GrammaticalFramework/GF">GF</a> repository is now archived.
-      </dd>
-      <dt class="col-sm-3 text-center text-nowrap">2017-08-11</dt>
-      <dd class="col-sm-9">
-        <strong>GF 3.9 released.</strong>
-        <a href="download/release-3.9.html">Release notes</a>
-      </dd>
-      <dt class="col-sm-3 text-center text-nowrap">2017-06-29</dt>
-      <dd class="col-sm-9">
-        GF is moving to <a href="https://github.com/GrammaticalFramework/GF/">GitHub</a>.</dd>
-      <dt class="col-sm-3 text-center text-nowrap">2017-03-13</dt>
-      <dd class="col-sm-9">
-        <a href="//school.grammaticalframework.org/2017/">GF Summer School</a> in Riga (Latvia), 14-25 August 2017
-      </dd>
     </dl>
 
     <h2>Projects</h2>
@@ -341,7 +344,7 @@ least one, it may help you to get a first idea of what GF is.
     Libraries are at the heart of modern software engineering. In natural language
     applications, libraries are a way to cope with thousands of details involved in
     syntax, lexicon, and inflection. The
-    <a href="lib/doc/synopsis/index.html">GF resource grammar library</a> has
+    <a href="lib/doc/synopsis/index.html">GF resource grammar library</a> (RGL) has
     support for an increasing number of languages, currently including
     Afrikaans,
     Amharic (partial),

nix/expose-all.patch

@@ -0,0 +1,12 @@
+diff --git a/gf.cabal b/gf.cabal
+index 0076e7638..8d3fe4b49 100644
+--- a/gf.cabal
++++ b/gf.cabal
+@@ -168,7 +168,6 @@ Library
+     GF.Text.Lexing
+     GF.Grammar.Canonical
+
+-  other-modules:
+     GF.Main
+     GF.Compiler
+     GF.Interactive

nix/revert-new-cabal-madness.patch

@@ -0,0 +1,193 @@
+commit 45e5473fcd5707af93646d9a116867a4d4e3e9c9
+Author: Andreas Källberg <anka.213@gmail.com>
+Date:   Mon Oct 10 14:57:12 2022 +0200
+
+    Revert "workaround for the Nix madness"
+    
+    This reverts commit 1294269cd60f3db7b056135104615625baeb528c.
+    
+    There are easier workarounds, like using
+    
+      cabal v1-build
+    
+    etc. instead of just `cabal build`
+    
+    These changes also broke a whole bunch of other stuff
+
+diff --git a/README.md b/README.md
+index ba35795a4..79e6ab68f 100644
+--- a/README.md
++++ b/README.md
+@@ -38,21 +38,6 @@ or:
+ ```
+ stack install
+ ```
+-Note that if you are unlucky to have Cabal 3.0 or later, then it uses
+-the so-called Nix style commands. Using those for GF development is
+-a pain. Every time when you change something in the source code, Cabal
+-will generate a new folder for GF to look for the GF libraries and
+-the GF cloud. Either reinstall everything with every change in the
+-compiler, or be sane and stop using cabal-install. Instead you can do:
+-```
+-runghc Setup.hs configure
+-runghc Setup.hs build
+-sudo runghc Setup.hs install
+-```
+-The script will install the GF dependencies globally. The only solution
+-to the Nix madness that I found is radical:
+-
+-  "No person, no problem" (Нет человека – нет проблемы).
+ 
+ For more information, including links to precompiled binaries, see the [download page](https://www.grammaticalframework.org/download/index.html).
+ 
+diff --git a/Setup.hs b/Setup.hs
+index 58dc3e0c6..f8309cc00 100644
+--- a/Setup.hs
++++ b/Setup.hs
+@@ -4,68 +4,42 @@ import Distribution.Simple.LocalBuildInfo(LocalBuildInfo(..),absoluteInstallDirs
+ import Distribution.Simple.Setup(BuildFlags(..),Flag(..),InstallFlags(..),CopyDest(..),CopyFlags(..),SDistFlags(..))
+ import Distribution.PackageDescription(PackageDescription(..),emptyHookedBuildInfo)
+ import Distribution.Simple.BuildPaths(exeExtension)
+-import System.Directory
+ import System.FilePath((</>),(<.>))
+-import System.Process
+-import Control.Monad(forM_,unless)
+-import Control.Exception(bracket_)
+-import Data.Char(isSpace)
+ 
+ import WebSetup
+ 
++-- | Notice about RGL not built anymore
++noRGLmsg :: IO ()
++noRGLmsg = putStrLn "Notice: the RGL is not built as part of GF anymore. See https://github.com/GrammaticalFramework/gf-rgl"
++
+ main :: IO ()
+ main = defaultMainWithHooks simpleUserHooks
+-  { preConf   = gfPreConf
+-  , preBuild  = gfPreBuild
++  { preBuild  = gfPreBuild
+   , postBuild = gfPostBuild
+   , preInst   = gfPreInst
+   , postInst  = gfPostInst
+   , postCopy  = gfPostCopy
+   }
+   where
+-    gfPreConf args flags = do
+-      pkgs <- fmap (map (dropWhile isSpace) . tail . lines)
+-                   (readProcess "ghc-pkg" ["list"] "")
+-      forM_ dependencies $ \pkg -> do
+-        let name = takeWhile (/='/') (drop 36 pkg)
+-        unless (name `elem` pkgs) $ do
+-          let fname = name <.> ".tar.gz"
+-          callProcess "wget" [pkg,"-O",fname]
+-          callProcess "tar"  ["-xzf",fname]
+-          removeFile fname
+-          bracket_ (setCurrentDirectory name) (setCurrentDirectory ".." >> removeDirectoryRecursive name) $ do
+-            exists <- doesFileExist "Setup.hs"
+-            unless exists $ do
+-              writeFile "Setup.hs" (unlines [
+-                  "import Distribution.Simple",
+-                  "main = defaultMain"
+-                ])
+-            let to_descr = reverse .
+-                           (++) (reverse ".cabal") . 
+-                           drop 1 . 
+-                           dropWhile (/='-') . 
+-                           reverse
+-            callProcess "wget"   [to_descr pkg, "-O", to_descr name]
+-            callProcess "runghc" ["Setup.hs","configure"]
+-            callProcess "runghc" ["Setup.hs","build"]
+-            callProcess "sudo" ["runghc","Setup.hs","install"]
+-          
+-      preConf simpleUserHooks args flags
+-
+-    gfPreBuild args = gfPre args . buildDistPref
+-    gfPreInst  args = gfPre args . installDistPref
++    gfPreBuild args  = gfPre args . buildDistPref
++    gfPreInst args = gfPre args . installDistPref
+ 
+     gfPre args distFlag = do
+       return emptyHookedBuildInfo
+ 
+     gfPostBuild args flags pkg lbi = do
++      -- noRGLmsg
+       let gf = default_gf lbi
+       buildWeb gf flags (pkg,lbi)
+ 
+     gfPostInst args flags pkg lbi = do
++      -- noRGLmsg
++      saveInstallPath args flags (pkg,lbi)
+       installWeb (pkg,lbi)
+ 
+     gfPostCopy args flags  pkg lbi = do
++      -- noRGLmsg
++      saveCopyPath args flags (pkg,lbi)
+       copyWeb flags (pkg,lbi)
+ 
+     -- `cabal sdist` will not make a proper dist archive, for that see `make sdist`
+@@ -73,16 +47,27 @@ main = defaultMainWithHooks simpleUserHooks
+     gfSDist pkg lbi hooks flags = do
+       return ()
+ 
+-dependencies = [
+-  "https://hackage.haskell.org/package/utf8-string-1.0.2/utf8-string-1.0.2.tar.gz",
+-  "https://hackage.haskell.org/package/json-0.10/json-0.10.tar.gz",
+-  "https://hackage.haskell.org/package/network-bsd-2.8.1.0/network-bsd-2.8.1.0.tar.gz",
+-  "https://hackage.haskell.org/package/httpd-shed-0.4.1.1/httpd-shed-0.4.1.1.tar.gz",
+-  "https://hackage.haskell.org/package/exceptions-0.10.5/exceptions-0.10.5.tar.gz",
+-  "https://hackage.haskell.org/package/stringsearch-0.3.6.6/stringsearch-0.3.6.6.tar.gz",
+-  "https://hackage.haskell.org/package/multipart-0.2.1/multipart-0.2.1.tar.gz",
+-  "https://hackage.haskell.org/package/cgi-3001.5.0.0/cgi-3001.5.0.0.tar.gz"
+-  ]
++saveInstallPath :: [String] -> InstallFlags -> (PackageDescription, LocalBuildInfo) -> IO ()
++saveInstallPath args flags bi = do
++  let
++    dest = NoCopyDest
++    dir = datadir (uncurry absoluteInstallDirs bi dest)
++  writeFile dataDirFile dir
++
++saveCopyPath :: [String] -> CopyFlags -> (PackageDescription, LocalBuildInfo) -> IO ()
++saveCopyPath args flags bi = do
++  let
++    dest = case copyDest flags of
++      NoFlag -> NoCopyDest
++      Flag d -> d
++    dir = datadir (uncurry absoluteInstallDirs bi dest)
++  writeFile dataDirFile dir
++
++-- | Name of file where installation's data directory is recording
++-- This is a last-resort way in which the seprate RGL build script
++-- can determine where to put the compiled RGL files
++dataDirFile :: String
++dataDirFile = "DATA_DIR"
+ 
+ -- | Get path to locally-built gf
+ default_gf :: LocalBuildInfo -> FilePath
+diff --git a/gf.cabal b/gf.cabal
+index a055b86be..d00a5b935 100644
+--- a/gf.cabal
++++ b/gf.cabal
+@@ -2,7 +2,7 @@ name: gf
+ version: 3.11.0-git
+ 
+ cabal-version: 1.22
+-build-type: Simple
++build-type: Custom
+ license: OtherLicense
+ license-file: LICENSE
+ category: Natural Language Processing, Compiler
+@@ -44,6 +44,14 @@ data-files:
+   www/translator/*.css
+   www/translator/*.js
+ 
++custom-setup
++  setup-depends:
++    base >= 4.9.1 && < 4.16,
++    Cabal >= 1.22.0.0,
++    directory >= 1.3.0 && < 1.4,
++    filepath >= 1.4.1 && < 1.5,
++    process >= 1.0.1.1 && < 1.7
++
+ source-repository head
+   type: git
+   location: https://github.com/GrammaticalFramework/gf-core.git

src/compiler/GF/Command/Commands.hs

@@ -4,6 +4,7 @@ module GF.Command.Commands (
   options,flags,
   ) where
 import Prelude hiding (putStrLn,(<>)) -- GHC 8.4.1 clash with Text.PrettyPrint
+import System.Info(os)
 
 import PGF
 
@@ -21,6 +22,7 @@ import GF.Infra.SIO
 import GF.Command.Abstract
 import GF.Command.CommandInfo
 import GF.Command.CommonCommands
+import qualified GF.Command.CommonCommands as Common
 import GF.Text.Clitics
 import GF.Quiz
 
@@ -165,14 +167,15 @@ pgfCommands = Map.fromList [
      synopsis = "generate random trees in the current abstract syntax",
      syntax = "gr [-cat=CAT] [-number=INT]",
      examples = [
-       mkEx "gr                     -- one tree in the startcat of the current grammar",
+       mkEx $ "gr                     -- one tree in the startcat of the current grammar, up to depth " ++ Common.default_depth_str,
        mkEx   "gr -cat=NP -number=16  -- 16 trees in the category NP",
+       mkEx   "gr -cat=NP -depth=2    -- one tree in the category NP, up to depth 2",
        mkEx   "gr -lang=LangHin,LangTha -cat=Cl  -- Cl, both in LangHin and LangTha",
        mkEx   "gr -probs=FILE         -- generate with bias",
        mkEx   "gr (AdjCN ? (UseN ?))  -- generate trees of form (AdjCN ? (UseN ?))"
        ],
      explanation = unlines [
-       "Generates a list of random trees, by default one tree.",
+       "Generates a list of random trees, by default one tree up to depth " ++ Common.default_depth_str ++ ".",
        "If a tree argument is given, the command completes the Tree with values to",
        "all metavariables in the tree. The generation can be biased by probabilities,",
        "given in a file in the -probs flag."
@@ -181,13 +184,13 @@ pgfCommands = Map.fromList [
        ("cat","generation category"),
        ("lang","uses only functions that have linearizations in all these languages"),
        ("number","number of trees generated"),
-       ("depth","the maximum generation depth"),
+       ("depth","the maximum generation depth (default: " ++ Common.default_depth_str ++ ")"),
        ("probs", "file with biased probabilities (format 'f 0.4' one by line)")
        ],
      exec = getEnv $ \ opts arg (Env pgf mos) -> do
        pgf <- optProbs opts (optRestricted opts pgf)
        gen <- newStdGen
-       let dp = valIntOpts "depth" 4 opts
+       let dp = valIntOpts "depth" Common.default_depth opts
        let ts  = case mexp (toExprs arg) of
                    Just ex -> generateRandomFromDepth gen pgf ex (Just dp)
                    Nothing -> generateRandomDepth     gen pgf (optType pgf opts) (Just dp)
@@ -198,28 +201,28 @@ pgfCommands = Map.fromList [
      synopsis = "generates a list of trees, by default exhaustive",
      explanation = unlines [
        "Generates all trees of a given category. By default, ",
-       "the depth is limited to 4, but this can be changed by a flag.",
+       "the depth is limited to " ++ Common.default_depth_str ++ ", but this can be changed by a flag.",
        "If a Tree argument is given, the command completes the Tree with values",
        "to all metavariables in the tree."
        ],
      flags = [
        ("cat","the generation category"),
-       ("depth","the maximum generation depth"),
+       ("depth","the maximum generation depth (default: " ++ Common.default_depth_str ++ ")"),
        ("lang","excludes functions that have no linearization in this language"),
        ("number","the number of trees generated")
        ],
      examples = [
-       mkEx "gt                     -- all trees in the startcat, to depth 4",
+       mkEx $ "gt                     -- all trees in the startcat, to depth " ++ Common.default_depth_str,
        mkEx   "gt -cat=NP -number=16  -- 16 trees in the category NP",
        mkEx   "gt -cat=NP -depth=2    -- trees in the category NP to depth 2",
        mkEx   "gt (AdjCN ? (UseN ?))  -- trees of form (AdjCN ? (UseN ?))"
        ],
      exec = getEnv $ \ opts arg (Env pgf mos) -> do
        let pgfr = optRestricted opts pgf
-       let dp = valIntOpts "depth" 4 opts
+       let dp = valIntOpts "depth" Common.default_depth opts
-       let ts = case mexp (toExprs arg) of
+       let ts = case toExprs arg of
-                  Just ex -> generateFromDepth pgfr ex (Just dp)
+                  [] -> generateAllDepth pgfr (optType pgf opts) (Just dp)
-                  Nothing -> generateAllDepth pgfr (optType pgf opts) (Just dp)
+                  es -> concat [generateFromDepth pgfr e (Just dp) | e <- es]
        returnFromExprs $ take (optNumInf opts) ts
      }),
   ("i", emptyCommandInfo {
@@ -427,7 +430,8 @@ pgfCommands = Map.fromList [
        "are type checking and semantic computation."
        ],
      examples = [
-       mkEx "pt -compute (plus one two)                               -- compute value"
+       mkEx "pt -compute (plus one two)                               -- compute value",
+       mkEx ("p \"the 4 dogs\" | pt -transfer=digits2numeral | l  -- \"the four dogs\" ")
        ],
      exec = getEnv $ \ opts arg (Env pgf mos) ->
             returnFromExprs . takeOptNum opts . treeOps pgf opts $ toExprs arg,
@@ -758,7 +762,7 @@ pgfCommands = Map.fromList [
                   []        -> [parse_ pgf lang (optType pgf opts) (Just dp) s | lang <- optLangs pgf opts]
                   open_typs -> [parseWithRecovery pgf lang (optType pgf opts) open_typs (Just dp) s | lang <- optLangs pgf opts]
      where
-       dp = valIntOpts "depth" 4 opts
+       dp = valIntOpts "depth" Common.default_depth opts
 
    fromParse opts = foldr (joinPiped . fromParse1 opts) void
 
@@ -882,11 +886,15 @@ pgfCommands = Map.fromList [
                        Right ty   -> ty
           Nothing -> error ("Can't parse '"++str++"' as a type")
    optViewFormat opts = valStrOpts "format" "png" opts
-   optViewGraph opts = valStrOpts "view" "open" opts
+   optViewGraph opts = valStrOpts "view" open_cmd opts
    optNum opts = valIntOpts "number" 1 opts
    optNumInf opts = valIntOpts "number" 1000000000 opts ---- 10^9
    takeOptNum opts = take (optNumInf opts)
 
+   open_cmd | os == "linux"   = "xdg-open"
+            | os == "mingw32" = "start"
+            | otherwise       = "open"
+
    returnFromExprs es = return $ case es of
      [] -> pipeMessage "no trees found"
      _  -> fromExprs es

src/compiler/GF/Command/CommonCommands.hs

@@ -19,6 +19,12 @@ import Data.Char (isSpace)
 
 import qualified PGF as H(showCId,showExpr,toATree,toTrie,Trie(..))
 
+-- store default generation depth in a variable and use everywhere
+default_depth :: Int
+default_depth = 5
+default_depth_str = show default_depth
+
+
 extend old new = Map.union (Map.fromList new) old -- Map.union is left-biased
 
 commonCommands :: (Monad m,MonadSIO m) => Map.Map String (CommandInfo m)

src/compiler/GF/Command/TreeOperations.hs

@@ -5,6 +5,8 @@ module GF.Command.TreeOperations (
   ) where
 
 import PGF(Expr,PGF,CId,compute,mkApp,unApp,unapply,unMeta,exprSize,exprFunctions)
+import PGF.Data(Expr(EApp,EFun))
+import PGF.TypeCheck(inferExpr)
 import Data.List
 
 type TreeOp = [Expr] -> [Expr]
@@ -16,15 +18,17 @@ allTreeOps :: PGF -> [(String,(String,Either TreeOp (CId -> TreeOp)))]
 allTreeOps pgf = [
    ("compute",("compute by using semantic definitions (def)",
       Left  $ map (compute pgf))),
+   ("transfer",("apply this transfer function to all maximal subtrees of suitable type",
+      Right $ \f -> map (transfer pgf f))), -- HL 12/24, modified from gf-3.3
    ("largest",("sort trees from largest to smallest, in number of nodes",
       Left  $ largest)),
-   ("nub",("remove duplicate trees",
+   ("nub\t",("remove duplicate trees",
       Left  $ nub)),
    ("smallest",("sort trees from smallest to largest, in number of nodes",
       Left  $ smallest)),
    ("subtrees",("return all fully applied subtrees (stopping at abstractions), by default sorted from the largest",
       Left  $ concatMap subtrees)),
-   ("funs",("return all fun functions appearing in the tree, with duplications",
+   ("funs\t",("return all fun functions appearing in the tree, with duplications",
       Left  $ \es -> [mkApp f [] | e <- es, f <- exprFunctions e]))
   ]
 
@@ -48,3 +52,18 @@ subtrees :: Expr -> [Expr]
 subtrees t = t : case unApp t of
   Just (f,ts) -> concatMap subtrees ts
   _ -> []  -- don't go under abstractions
+
+-- Apply transfer function f:C -> D to all maximal subtrees s:C of tree e and replace
+-- these s by the values of f(s). This modifies the 'simple-minded transfer' of gf-3.3.
+-- If applied to strict subtrees s of e, better use with f:C -> C only.  HL 12/2024
+
+transfer :: PGF -> CId -> Expr -> Expr
+transfer pgf f e = case inferExpr pgf (appf e) of
+     Left _err -> case e of
+                    EApp g a -> EApp (transfer pgf f g) (transfer pgf f a)
+                    _ -> e
+     Right _ty -> case (compute pgf (appf e)) of
+                    v | v /= (appf e) -> v
+                    _ -> e          -- default case of f, or f has no computation rule
+ where
+  appf = EApp (EFun f)

src/compiler/GF/Compile.hs

@@ -1,7 +1,6 @@
-module GF.Compile (compileToPGF, compileToLPGF, link, linkl, batchCompile, srcAbsName) where
+module GF.Compile (compileToPGF, link, batchCompile, srcAbsName) where
 
 import GF.Compile.GrammarToPGF(mkCanon2pgf)
-import GF.Compile.GrammarToLPGF(mkCanon2lpgf)
 import GF.Compile.ReadFiles(ModEnv,getOptionsFromFile,getAllFiles,
                             importsOfModule)
 import GF.CompileOne(compileOne)
@@ -15,7 +14,7 @@ import GF.Infra.UseIO(IOE,FullPath,liftIO,getLibraryDirectory,putIfVerb,
                       justModuleName,extendPathEnv,putStrE,putPointE)
 import GF.Data.Operations(raise,(+++),err)
 
-import Control.Monad(foldM,when,(<=<),filterM)
+import Control.Monad(foldM,when,(<=<),filterM,liftM)
 import GF.System.Directory(doesFileExist,getModificationTime)
 import System.FilePath((</>),isRelative,dropFileName)
 import qualified Data.Map as Map(empty,insert,elems) --lookup
@@ -25,16 +24,12 @@ import GF.Text.Pretty(render,($$),(<+>),nest)
 
 import PGF.Internal(optimizePGF)
 import PGF(PGF,defaultProbabilities,setProbabilities,readProbabilitiesFromFile)
-import LPGF(LPGF)
 
 -- | Compiles a number of source files and builds a 'PGF' structure for them.
 -- This is a composition of 'link' and 'batchCompile'.
 compileToPGF :: Options -> [FilePath] -> IOE PGF
 compileToPGF opts fs = link opts . snd =<< batchCompile opts fs
 
-compileToLPGF :: Options -> [FilePath] -> IOE LPGF
-compileToLPGF opts fs = linkl opts . snd =<< batchCompile opts fs
-
 -- | Link a grammar into a 'PGF' that can be used to 'PGF.linearize' and
 -- 'PGF.parse' with the "PGF" run-time system.
 link :: Options -> (ModuleName,Grammar) -> IOE PGF
@@ -47,14 +42,6 @@ link opts (cnc,gr) =
     return $ setProbabilities probs 
            $ if flag optOptimizePGF opts then optimizePGF pgf else pgf
 
--- | Link a grammar into a 'LPGF' that can be used for linearization only.
-linkl :: Options -> (ModuleName,Grammar) -> IOE LPGF
-linkl opts (cnc,gr) =
-  putPointE Normal opts "linking ... " $ do
-    let abs = srcAbsName gr cnc
-    lpgf <- mkCanon2lpgf opts gr abs
-    return lpgf
-
 -- | Returns the name of the abstract syntax corresponding to the named concrete syntax
 srcAbsName gr cnc = err (const cnc) id $ abstractOfConcrete gr cnc
 

src/compiler/GF/Compile/Compute/Concrete.hs

@@ -172,11 +172,11 @@ value env t0 =
     ImplArg t -> (VImplArg.) # value env t
     Table p res -> liftM2 VTblType # value env p <# value env res
     RecType rs -> do lovs <- mapPairsM (value env) rs
-                     return $ \vs->VRecType $ mapSnd ($vs) lovs
+                     return $ \vs->VRecType $ mapSnd ($ vs) lovs
     t@(ExtR t1 t2) -> ((extR t.)# both id) # both (value env) (t1,t2)
     FV ts   -> ((vfv .) # sequence) # mapM (value env) ts
     R as    -> do lovs <- mapPairsM (value env.snd) as
-                  return $ \ vs->VRec $ mapSnd ($vs) lovs
+                  return $ \ vs->VRec $ mapSnd ($ vs) lovs
     T i cs  -> valueTable env i cs
     V ty ts -> do pvs <- paramValues env ty
                   ((VV ty pvs .) . sequence) # mapM (value env) ts
@@ -376,10 +376,10 @@ valueTable env i cs =
   where
     dynamic cs' ty _ = cases cs' # value env ty
 
-    cases cs' vty vs = err keep ($vs) (convertv cs' (vty vs))
+    cases cs' vty vs = err keep ($ vs) (convertv cs' (vty vs))
       where
         keep msg = --trace (msg++"\n"++render (ppTerm Unqualified 0 (T i cs))) $
-                   VT wild (vty vs) (mapSnd ($vs) cs')
+                   VT wild (vty vs) (mapSnd ($ vs) cs')
 
     wild = case i of TWild _ -> True; _ -> False
 
@@ -392,7 +392,7 @@ valueTable env i cs =
     convert' cs' ((pty,vs),pvs) =
       do sts  <- mapM (matchPattern cs') vs
          return $ \ vs -> VV pty pvs $ map (err bug id . valueMatch env)
-                                           (mapFst ($vs) sts)
+                                           (mapFst ($ vs) sts)
 
     valueCase (p,t) = do p' <- measurePatt # inlinePattMacro p
                          pvs <- linPattVars p'
@@ -430,19 +430,19 @@ apply' :: CompleteEnv -> Term -> [OpenValue] -> Err OpenValue
 apply' env t [] = value env t
 apply' env t vs =
   case t of
-    QC x                   -> return $ \ svs -> VCApp x (map ($svs) vs)
+    QC x                   -> return $ \ svs -> VCApp x (map ($ svs) vs)
 {-
     Q x@(m,f) | m==cPredef -> return $
                               let constr = --trace ("predef "++show x) .
                                            VApp x
                               in \ svs -> maybe constr id (Map.lookup f predefs)
-                                          $ map ($svs) vs
+                                          $ map ($ svs) vs
               | otherwise  -> do r <- resource env x
-                                 return $ \ svs -> vapply (gloc env) r (map ($svs) vs)
+                                 return $ \ svs -> vapply (gloc env) r (map ($ svs) vs)
 -}
     App t1 t2              -> apply' env t1 . (:vs) =<< value env t2
     _                      -> do fv <- value env t
-                                 return $ \ svs -> vapply (gloc env) (fv svs) (map ($svs) vs)
+                                 return $ \ svs -> vapply (gloc env) (fv svs) (map ($ svs) vs)
 
 vapply :: GLocation -> Value -> [Value] -> Value
 vapply loc v [] = v

src/compiler/GF/Compile/GeneratePMCFG.hs

@@ -201,11 +201,11 @@ instance Fail.MonadFail CnvMonad where
   fail = bug
 
 instance Applicative CnvMonad where
-  pure = return
+  pure a = CM (\gr c s -> c a s)
   (<*>) = ap
 
 instance Monad CnvMonad where
-    return a   = CM (\gr c s -> c a s)
+    return     = pure
     CM m >>= k = CM (\gr c s -> m gr (\a s -> unCM (k a) gr c s) s)
 
 instance MonadState ([ProtoFCat],[Symbol]) CnvMonad where

src/compiler/GF/Compile/GetGrammar.hs

@@ -42,11 +42,12 @@ getSourceModule opts file0 =
      raw <- liftIO $ keepTemp tmp
    --ePutStrLn $ "1 "++file0
      (optCoding,parsed) <- parseSource opts pModDef raw
+     let indentLines = unlines . map ("   "++) . lines
      case parsed of
        Left (Pn l c,msg) -> do file <- liftIO $ writeTemp tmp
                                cwd <- getCurrentDirectory
                                let location = makeRelative cwd file++":"++show l++":"++show c
-                               raise (location++":\n   "++msg)
+                               raise (location++":\n" ++ indentLines msg)
        Right (i,mi0) ->
          do liftIO $ removeTemp tmp
             let mi =mi0 {mflags=mflags mi0 `addOptions` opts, msrc=file0}

src/compiler/GF/Compile/GrammarToLPGF.hs

@@ -1,429 +0,0 @@
-module GF.Compile.GrammarToLPGF (mkCanon2lpgf) where
-
-import LPGF.Internal (LPGF (..))
-import qualified LPGF.Internal as L
-
-import PGF.CId
-import GF.Grammar.Grammar
-import qualified GF.Grammar.Canonical as C
-import GF.Compile.GrammarToCanonical (grammar2canonical)
-
-import GF.Data.Operations (ErrorMonad (..))
-import qualified GF.Data.IntMapBuilder as IntMapBuilder
-import GF.Infra.Ident (rawIdentS, showRawIdent)
-import GF.Infra.Option (Options)
-import GF.Infra.UseIO (IOE)
-import GF.Text.Pretty (pp, render)
-
-import Control.Applicative ((<|>))
-import Control.Monad (when, unless, forM, forM_)
-import qualified Control.Monad.State.Strict as CMS
-import Data.Either (lefts, rights)
-import Data.List (elemIndex)
-import qualified Data.List as L
-import qualified Data.Map.Strict as Map
-import Data.Maybe (fromJust, isJust)
-import Data.Text (Text)
-import qualified Data.Text as T
-import System.Environment (lookupEnv)
-import System.FilePath ((</>), (<.>))
-import Text.Printf (printf)
-
-import qualified Debug.Trace
-trace x = Debug.Trace.trace ("> " ++ show x) (return ())
-
-mkCanon2lpgf :: Options -> SourceGrammar -> ModuleName -> IOE LPGF
-mkCanon2lpgf opts gr am = do
-  debug <- isJust <$> lookupEnv "DEBUG"
-  when debug $ do
-    ppCanonical debugDir canon
-    dumpCanonical debugDir canon
-  (an,abs) <- mkAbstract ab
-  cncs     <- mapM (mkConcrete debug ab) cncs
-  let lpgf = LPGF {
-    L.absname = an,
-    L.abstract = abs,
-    L.concretes = Map.fromList cncs
-  }
-  when debug $ ppLPGF debugDir lpgf
-  return lpgf
-  where
-    canon@(C.Grammar ab cncs) = grammar2canonical opts am gr
-
-mkAbstract :: (ErrorMonad err) => C.Abstract -> err (CId, L.Abstract)
-mkAbstract (C.Abstract modId flags cats funs) = return (mdi2i modId, L.Abstract {})
-
-mkConcrete :: (ErrorMonad err) => Bool -> C.Abstract -> C.Concrete -> err (CId, L.Concrete)
-mkConcrete debug (C.Abstract _ _ _ funs) (C.Concrete modId absModId flags params0 lincats0 lindefs0) = do
-  let
-    -- Some transformations on canonical grammar
-
-    params :: [C.ParamDef]
-    params = inlineParamAliases params0
-
-    lincats :: [C.LincatDef]
-    lincats = s:i:f:lincats0
-      where
-        ss = C.RecordType [C.RecordRow (C.LabelId (rawIdentS "s")) C.StrType]
-        s = C.LincatDef (C.CatId (rawIdentS "String")) ss
-        i = C.LincatDef (C.CatId (rawIdentS "Int")) ss
-        f = C.LincatDef (C.CatId (rawIdentS "Float")) ss
-
-    lindefs :: [C.LinDef]
-    lindefs =
-      [ C.LinDef funId varIds linValue
-      | (C.LinDef funId varIds linValue) <- lindefs0
-      , let Right linType = lookupLinType funId
-      ]
-
-    -- Builds maps for lookups
-
-    paramValueMap :: Map.Map C.ParamId C.ParamDef -- constructor -> definition
-    paramValueMap = Map.fromList [ (v,d) | d@(C.ParamDef _ vs) <- params, (C.Param v _) <- vs ]
-
-    lincatMap :: Map.Map C.CatId C.LincatDef
-    lincatMap = Map.fromList [ (cid,d) | d@(C.LincatDef cid _) <- lincats ]
-
-    funMap :: Map.Map C.FunId C.FunDef
-    funMap = Map.fromList [ (fid,d) | d@(C.FunDef fid _) <- funs ]
-
-    -- | Lookup paramdef
-    lookupParamDef :: C.ParamId -> Either String C.ParamDef
-    lookupParamDef pid = m2e (printf "Cannot find param definition: %s" (show pid)) (Map.lookup pid paramValueMap)
-
-    -- | Lookup lintype for a function
-    lookupLinType :: C.FunId -> Either String C.LinType
-    lookupLinType funId = do
-      fun <- m2e (printf "Cannot find type for: %s" (show funId)) (Map.lookup funId funMap)
-      let (C.FunDef _ (C.Type _ (C.TypeApp catId _))) = fun
-      lincat <- m2e (printf "Cannot find lincat for: %s" (show catId)) (Map.lookup catId lincatMap)
-      let (C.LincatDef _ lt) = lincat
-      return lt
-
-    -- | Lookup lintype for a function's argument
-    lookupLinTypeArg :: C.FunId -> Int -> Either String C.LinType
-    lookupLinTypeArg funId argIx = do
-      fun <- m2e (printf "Cannot find type for: %s" (show funId)) (Map.lookup funId funMap)
-      let (C.FunDef _ (C.Type args _)) = fun
-      let (C.TypeBinding _ (C.Type _ (C.TypeApp catId _))) = args !! argIx
-      lincat <- m2e (printf "Cannot find lincat for: %s" (show catId)) (Map.lookup catId lincatMap)
-      let (C.LincatDef _ lt) = lincat
-      return lt
-
-    -- Code generation
-
-    -- | Main code generation function
-    mkLin :: C.LinDef -> CodeGen (CId, L.LinFun)
-    mkLin (C.LinDef funId varIds linValue) = do
-      -- when debug $ trace funId
-      (lf, _) <- val2lin linValue
-      return (fi2i funId, lf)
-      where
-        val2lin :: C.LinValue -> CodeGen (L.LinFun, Maybe C.LinType)
-        val2lin lv = case lv of
-
-          C.ConcatValue v1 v2 -> do
-            (v1',t1) <- val2lin v1
-            (v2',t2) <- val2lin v2
-            return (L.Concat v1' v2', t1 <|> t2) -- t1 else t2
-
-          C.LiteralValue ll -> case ll of
-            C.FloatConstant f -> return (L.Token $ T.pack $ show f, Just C.FloatType)
-            C.IntConstant i -> return (L.Token $ T.pack $ show i, Just C.IntType)
-            C.StrConstant s -> return (L.Token $ T.pack s, Just C.StrType)
-
-          C.ErrorValue err -> return (L.Error err, Nothing)
-
-          C.ParamConstant (C.Param pid lvs) -> do
-            let
-              collectProjections :: C.LinValue -> CodeGen [L.LinFun]
-              collectProjections (C.ParamConstant (C.Param pid lvs)) = do
-                def <- lookupParamDef pid
-                let (C.ParamDef tpid defpids) = def
-                pidIx <- eitherElemIndex pid [ p | C.Param p _ <- defpids ]
-                rest <- mapM collectProjections lvs
-                return $ L.Ix (pidIx+1) : concat rest
-              collectProjections lv = do
-                (lf,_) <- val2lin lv
-                return [lf]
-            lfs <- collectProjections lv
-            let term = L.Tuple lfs
-            def <- lookupParamDef pid
-            let (C.ParamDef tpid _) = def
-            return (term, Just $ C.ParamType (C.ParamTypeId tpid))
-
-          C.PredefValue (C.PredefId pid) -> case showRawIdent pid of
-            "BIND" -> return (L.Bind, Nothing)
-            "SOFT_BIND" -> return (L.Bind, Nothing)
-            "SOFT_SPACE" -> return (L.Space, Nothing)
-            "CAPIT" -> return (L.Capit, Nothing)
-            "ALL_CAPIT" -> return (L.AllCapit, Nothing)
-            x -> Left $ printf "Unknown predef function: %s" x
-
-          C.RecordValue rrvs -> do
-            ts <- sequence [ val2lin lv | C.RecordRow lid lv <- rrvs ]
-            return (L.Tuple (map fst ts), Just $ C.RecordType [ C.RecordRow lid lt | (C.RecordRow lid _, (_, Just lt)) <- zip rrvs ts])
-
-          C.TableValue lt trvs -> do
-            -- group the rows by "left-most" value
-            let
-              groupRow :: C.TableRowValue -> C.TableRowValue -> Bool
-              groupRow (C.TableRow p1 _) (C.TableRow p2 _) = groupPattern p1 p2
-
-              groupPattern :: C.LinPattern -> C.LinPattern -> Bool
-              groupPattern p1 p2 = case (p1,p2) of
-                (C.ParamPattern (C.Param pid1 _), C.ParamPattern (C.Param pid2 _)) -> pid1 == pid2 -- compare only constructors
-                (C.RecordPattern (C.RecordRow lid1 patt1:_), C.RecordPattern (C.RecordRow lid2 patt2:_)) -> groupPattern patt1 patt2 -- lid1 == lid2 necessarily
-                _ -> error $ printf "Mismatched patterns in grouping:\n%s\n%s" (show p1) (show p2)
-
-              grps :: [[C.TableRowValue]]
-              grps = L.groupBy groupRow trvs
-
-            -- remove one level of depth and recurse
-            let
-              handleGroup :: [C.TableRowValue] -> CodeGen (L.LinFun, Maybe C.LinType)
-              handleGroup [C.TableRow patt lv] =
-                case reducePattern patt of
-                  Just patt' -> do
-                    (lf,lt) <- handleGroup [C.TableRow patt' lv]
-                    return (L.Tuple [lf],lt)
-                  Nothing    -> val2lin lv
-              handleGroup rows = do
-                let rows' = map reduceRow rows
-                val2lin (C.TableValue lt rows') -- lt is wrong here, but is unused
-
-              reducePattern :: C.LinPattern -> Maybe C.LinPattern
-              reducePattern patt =
-                case patt of
-                  C.ParamPattern (C.Param _ []) -> Nothing
-                  C.ParamPattern (C.Param _ patts) -> Just $ C.ParamPattern (C.Param pid' patts')
-                    where
-                      C.ParamPattern (C.Param pid1 patts1) = head patts
-                      pid' = pid1
-                      patts' = patts1 ++ tail patts
-
-                  C.RecordPattern [] -> Nothing
-                  C.RecordPattern (C.RecordRow lid patt:rrs) ->
-                    case reducePattern patt of
-                      Just patt' -> Just $ C.RecordPattern (C.RecordRow lid patt':rrs)
-                      Nothing    -> if null rrs then Nothing else Just $ C.RecordPattern rrs
-
-                  _ -> error $ printf "Unhandled pattern in reducing: %s" (show patt)
-
-              reduceRow :: C.TableRowValue -> C.TableRowValue
-              reduceRow (C.TableRow patt lv) =
-                let Just patt' = reducePattern patt
-                in  C.TableRow patt' lv
-
-            -- ts :: [(L.LinFun, Maybe C.LinType)]
-            ts <- mapM handleGroup grps
-
-            -- return
-            let typ = case ts of
-                  (_, Just tst):_ -> Just $ C.TableType lt tst
-                  _ -> Nothing
-            return (L.Tuple (map fst ts), typ)
-
-          -- TODO TuplePattern, WildPattern?
-
-          C.TupleValue lvs -> do
-            ts <- mapM val2lin lvs
-            return (L.Tuple (map fst ts), Just $ C.TupleType (map (fromJust.snd) ts))
-
-          C.VariantValue [] -> return (L.Empty, Nothing) -- TODO Just C.StrType ?
-          C.VariantValue (vr:_) -> val2lin vr -- NOTE variants not supported, just pick first
-
-          C.VarValue (C.VarValueId (C.Unqual v)) -> do
-            ix <- eitherElemIndex (C.VarId v) varIds
-            lt <- lookupLinTypeArg funId ix
-            return (L.Argument (ix+1), Just lt)
-
-          C.PreValue pts df -> do
-            pts' <- forM pts $ \(pfxs, lv) -> do
-              (lv', _) <- val2lin lv
-              return (map T.pack pfxs, lv')
-            (df', lt) <- val2lin df
-            return (L.Pre pts' df', lt)
-
-          C.Projection v1 lblId -> do
-            (v1', mtyp) <- val2lin v1
-            -- find label index in argument type
-            let Just (C.RecordType rrs) = mtyp
-            let rrs' = [ lid | C.RecordRow lid _ <- rrs ]
-            -- lblIx <- eitherElemIndex lblId rrs'
-            let
-              lblIx = case eitherElemIndex lblId rrs' of
-                Right x -> x
-                Left _ -> 0 -- corresponds to Prelude.False
-            -- lookup lintype for record row
-            let C.RecordRow _ lt = rrs !! lblIx
-            return (L.Projection v1' (L.Ix (lblIx+1)), Just lt)
-
-          C.Selection v1 v2 -> do
-            (v1', t1) <- val2lin v1
-            (v2', t2) <- val2lin v2
-            let Just (C.TableType t11 t12) = t1 -- t11 == t2
-            return (L.Projection v1' v2', Just t12)
-
-          -- C.CommentedValue cmnt lv -> val2lin lv
-          C.CommentedValue cmnt lv -> case cmnt of
-            "impossible" -> return (L.Empty, Nothing)
-            -- "impossible" -> val2lin lv >>= \(_, typ) -> return (L.Empty, typ)
-            _ -> val2lin lv
-
-          v -> Left $ printf "val2lin not implemented for: %s" (show v)
-
-  -- Invoke code generation
-
-  let es = map mkLin lindefs
-  unless (null $ lefts es) (raise $ unlines (lefts es))
-
-  let maybeOptimise = if debug then id else extractStrings
-  let concr = maybeOptimise $ L.Concrete {
-    L.toks = IntMapBuilder.emptyIntMap,
-    L.lins = Map.fromList (rights es)
-  }
-  return (mdi2i modId, concr)
-
-type CodeGen a = Either String a
-
--- | Remove ParamAliasDefs by inlining their definitions
-inlineParamAliases :: [C.ParamDef] -> [C.ParamDef]
-inlineParamAliases defs = if null aliases then defs else map rp' pdefs
-  where
-    (aliases,pdefs) = L.partition isParamAliasDef defs
-
-    rp' :: C.ParamDef -> C.ParamDef
-    rp' (C.ParamDef pid pids) = C.ParamDef pid (map rp'' pids)
-    rp' (C.ParamAliasDef _ _) = error "inlineParamAliases called on ParamAliasDef" -- impossible
-
-    rp'' :: C.ParamValueDef -> C.ParamValueDef
-    rp'' (C.Param pid pids) = C.Param pid (map rp''' pids)
-
-    rp''' :: C.ParamId -> C.ParamId
-    rp''' pid = case L.find (\(C.ParamAliasDef p _) -> p == pid) aliases of
-      Just (C.ParamAliasDef _ (C.ParamType (C.ParamTypeId p))) -> p
-      _ -> pid
-
-isParamAliasDef :: C.ParamDef -> Bool
-isParamAliasDef (C.ParamAliasDef _ _) = True
-isParamAliasDef _ = False
-
-isParamType :: C.LinType -> Bool
-isParamType (C.ParamType _) = True
-isParamType _ = False
-
-isRecordType :: C.LinType -> Bool
-isRecordType (C.RecordType _) = True
-isRecordType _ = False
-
--- | Find all token strings, put them in a map and replace with token indexes
-extractStrings :: L.Concrete -> L.Concrete
-extractStrings concr = L.Concrete { L.toks = toks', L.lins = lins' }
-  where
-    imb = IntMapBuilder.fromIntMap (L.toks concr)
-    (lins',imb') = CMS.runState (go0 (L.lins concr)) imb
-    toks' = IntMapBuilder.toIntMap imb'
-
-    go0 :: Map.Map CId L.LinFun -> CMS.State (IntMapBuilder.IMB Text) (Map.Map CId L.LinFun)
-    go0 mp = do
-      xs <- mapM (\(cid,lin) -> go lin >>= \lin' -> return (cid,lin')) (Map.toList mp)
-      return $ Map.fromList xs
-
-    go :: L.LinFun -> CMS.State (IntMapBuilder.IMB Text) L.LinFun
-    go lf = case lf of
-      L.Token str -> do
-        imb <- CMS.get
-        let (ix,imb') = IntMapBuilder.insert' str imb
-        CMS.put imb'
-        return $ L.TokenIx ix
-
-      L.Pre pts df -> do
-        -- pts' <- mapM (\(pfxs,lv) -> go lv >>= \lv' -> return (pfxs,lv')) pts
-        pts' <- forM pts $ \(pfxs,lv) -> do
-          imb <- CMS.get
-          let str = T.pack $ show pfxs
-          let (ix,imb') = IntMapBuilder.insert' str imb
-          CMS.put imb'
-          lv' <- go lv
-          return (ix,lv')
-        df' <- go df
-        return $ L.PreIx pts' df'
-      L.Concat s t -> do
-        s' <- go s
-        t' <- go t
-        return $ L.Concat s' t'
-      L.Tuple ts -> do
-        ts' <- mapM go ts
-        return $ L.Tuple ts'
-      L.Projection t u -> do
-        t' <- go t
-        u' <- go u
-        return $ L.Projection t' u'
-      t -> return t
-
--- | Convert Maybe to Either value with error
-m2e :: String -> Maybe a -> Either String a
-m2e err = maybe (Left err) Right
-
--- | Wrap elemIndex into Either value
-eitherElemIndex :: (Eq a, Show a) => a -> [a] -> Either String Int
-eitherElemIndex x xs = m2e (printf "Cannot find: %s in %s" (show x) (show xs)) (elemIndex x xs)
-
-mdi2s :: C.ModId -> String
-mdi2s (C.ModId i) = showRawIdent i
-
-mdi2i :: C.ModId -> CId
-mdi2i (C.ModId i) = mkCId (showRawIdent i)
-
-fi2i :: C.FunId -> CId
-fi2i (C.FunId i) = mkCId (showRawIdent i)
-
--- Debugging
-
-debugDir :: FilePath
-debugDir = "DEBUG"
-
--- | Pretty-print canonical grammars to file
-ppCanonical :: FilePath -> C.Grammar -> IO ()
-ppCanonical path (C.Grammar ab cncs) = do
-  let (C.Abstract modId flags cats funs) = ab
-  writeFile (path </> mdi2s modId <.> "canonical.gf") (render $ pp ab)
-  forM_ cncs $ \cnc@(C.Concrete modId absModId flags params lincats lindefs) ->
-    writeFile' (path </> mdi2s modId <.> "canonical.gf") (render $ pp cnc)
-
--- | Dump canonical grammars to file
-dumpCanonical :: FilePath -> C.Grammar -> IO ()
-dumpCanonical path (C.Grammar ab cncs) = do
-  let (C.Abstract modId flags cats funs) = ab
-  let body = unlines $ map show cats ++ [""] ++ map show funs
-  writeFile' (path </> mdi2s modId <.> "canonical.dump") body
-
-  forM_ cncs $ \(C.Concrete modId absModId flags params lincats lindefs) -> do
-    let body = unlines $ concat [
-          map show params,
-          [""],
-          map show lincats,
-          [""],
-          map show lindefs
-          ]
-    writeFile' (path </> mdi2s modId <.> "canonical.dump") body
-
--- | Pretty-print LPGF to file
-ppLPGF :: FilePath -> LPGF -> IO ()
-ppLPGF path lpgf =
-  forM_ (Map.toList $ L.concretes lpgf) $ \(cid,concr) ->
-    writeFile' (path </> showCId cid <.> "lpgf.txt") (L.render $ L.pp concr)
-
--- | Dump LPGF to file
-dumpLPGF :: FilePath -> LPGF -> IO ()
-dumpLPGF path lpgf =
-  forM_ (Map.toList $ L.concretes lpgf) $ \(cid,concr) -> do
-    let body = unlines $ map show (Map.toList $ L.lins concr)
-    writeFile' (path </> showCId cid <.> "lpgf.dump") body
-
--- | Write a file and report it to console
-writeFile' :: FilePath -> String -> IO ()
-writeFile' p b = do
-  writeFile p b
-  putStrLn $ "Wrote " ++ p

src/compiler/GF/Compile/PGFtoHaskell.hs

@@ -51,7 +51,7 @@ grammar2haskell opts name gr = foldr (++++) [] $
           derivingClause
                  | dataExt = "deriving (Show,Data)"
                  | otherwise = "deriving Show"
-          extraImports | gadt = ["import Control.Monad.Identity", "import Data.Monoid"]
+          extraImports | gadt = ["import Control.Monad.Identity", "import Control.Monad", "import Data.Monoid"]
                        | dataExt = ["import Data.Data"]
                        | otherwise = []
           pgfImports | pgf2 = ["import PGF2 hiding (Tree)", "", "showCId :: CId -> String", "showCId = id"]

src/compiler/GF/Compile/TypeCheck/ConcreteNew.hs

@@ -644,7 +644,7 @@ data TcResult a
 newtype TcM a = TcM {unTcM :: MetaStore -> [Message] -> TcResult a}
 
 instance Monad TcM where
-  return x = TcM (\ms msgs -> TcOk x ms msgs)
+  return = pure
   f >>= g  = TcM (\ms msgs -> case unTcM f ms msgs of
                                 TcOk x ms msgs -> unTcM (g x) ms msgs
                                 TcFail    msgs -> TcFail msgs)
@@ -659,7 +659,7 @@ instance Fail.MonadFail TcM where
 
 
 instance Applicative TcM where
-  pure = return
+  pure x = TcM (\ms msgs -> TcOk x ms msgs)
   (<*>) = ap
 
 instance Functor TcM where

src/compiler/GF/CompileInParallel.hs

@@ -61,10 +61,10 @@ parallelBatchCompile jobs opts rootfiles0 =
 
         usesPresent (_,paths) = take 1 libs==["present"]
           where
-            libs = [p|path<-paths,
+            libs = [p | path<-paths,
                         let (d,p0) = splitAt n path
                             p = dropSlash p0,
-                      d==lib_dir,p `elem` all_modes]
+                      d==lib_dir, p `elem` all_modes]
             n = length lib_dir
 
         all_modes = ["alltenses","present"]
@@ -175,7 +175,7 @@ batchCompile1 lib_dir (opts,filepaths) =
                    " from being compiled."
        else return (maximum ts,(cnc,gr))
 
-splitEither es = ([x|Left x<-es],[y|Right y<-es])
+splitEither es = ([x | Left x<-es], [y | Right y<-es])
 
 canonical path = liftIO $ D.canonicalizePath path `catch` const (return path)
 
@@ -239,11 +239,11 @@ instance Functor m => Functor (CollectOutput m) where
    fmap f (CO m) = CO (fmap (fmap f) m)
 
 instance (Functor m,Monad m) => Applicative (CollectOutput m) where
-  pure = return
+  pure x = CO (return (return (),x))
   (<*>) = ap
 
 instance Monad m => Monad (CollectOutput m) where
-  return x = CO (return (return (),x))
+  return     = pure
   CO m >>= f = CO $ do (o1,x) <- m
                        let CO m2 = f x
                        (o2,y) <- m2

src/compiler/GF/Compiler.hs

@@ -1,11 +1,9 @@
-module GF.Compiler (mainGFC, linkGrammars, writePGF, writeLPGF, writeOutputs) where
+module GF.Compiler (mainGFC, linkGrammars, writePGF, writeOutputs) where
 
 import PGF
 import PGF.Internal(concretes,optimizePGF,unionPGF)
 import PGF.Internal(putSplitAbs,encodeFile,runPut)
-import LPGF(LPGF)
+import GF.Compile as S(batchCompile,link,srcAbsName)
-import qualified LPGF.Internal as LPGF
-import GF.Compile as S(batchCompile,link,linkl,srcAbsName)
 import GF.CompileInParallel as P(parallelBatchCompile)
 import GF.Compile.Export
 import GF.Compile.ConcreteToHaskell(concretes2haskell)
@@ -13,8 +11,7 @@ import GF.Compile.GrammarToCanonical--(concretes2canonical)
 import GF.Compile.CFGtoPGF
 import GF.Compile.GetGrammar
 import GF.Grammar.BNFC
-import GF.Grammar.CFG hiding (Grammar)
+import GF.Grammar.CFG
-import GF.Grammar.Grammar (Grammar, ModuleName)
 
 --import GF.Infra.Ident(showIdent)
 import GF.Infra.UseIO
@@ -26,11 +23,10 @@ import GF.Text.Pretty(render,render80)
 import Data.Maybe
 import qualified Data.Map as Map
 import qualified Data.Set as Set
-import Data.Time(UTCTime)
 import qualified Data.ByteString.Lazy as BSL
 import GF.Grammar.CanonicalJSON (encodeJSON)
 import System.FilePath
-import Control.Monad(when,unless,forM,void)
+import Control.Monad(when,unless,forM_)
 
 -- | Compile the given GF grammar files. The result is a number of @.gfo@ files
 -- and, depending on the options, a @.pgf@ file. (@gf -batch@, @gf -make@)
@@ -97,10 +93,6 @@ compileSourceFiles opts fs =
 -- If a @.pgf@ file by the same name already exists and it is newer than the
 -- source grammar files (as indicated by the 'UTCTime' argument), it is not
 -- recreated. Calls 'writePGF' and 'writeOutputs'.
-linkGrammars :: Options -> (UTCTime,[(ModuleName, Grammar)]) -> IOE ()
-linkGrammars opts (_,cnc_grs) | FmtLPGF `elem` flag optOutputFormats opts = do
-  lpgf <- linkl opts (head cnc_grs)
-  void $ writeLPGF opts lpgf
 linkGrammars opts (t_src,~cnc_grs@(~(cnc,gr):_)) =
     do let abs = render (srcAbsName gr cnc)
            pgfFile = outputPath opts (grammarName' opts abs<.>"pgf")
@@ -153,7 +145,7 @@ unionPGFFiles opts fs =
              pgfFile = outputPath opts (grammarName opts pgf <.> "pgf")
          if pgfFile `elem` fs
            then putStrLnE $ "Refusing to overwrite " ++ pgfFile
-           else void $ writePGF opts pgf
+           else writePGF opts pgf
          writeOutputs opts pgf
 
     readPGFVerbose f =
@@ -170,39 +162,26 @@ writeOutputs opts pgf = do
 -- | Write the result of compiling a grammar (e.g. with 'compileToPGF' or
 -- 'link') to a @.pgf@ file.
 -- A split PGF file is output if the @-split-pgf@ option is used.
-writePGF :: Options -> PGF -> IOE [FilePath]
+writePGF :: Options -> PGF -> IOE ()
 writePGF opts pgf =
     if flag optSplitPGF opts then writeSplitPGF else writeNormalPGF
   where
     writeNormalPGF =
        do let outfile = outputPath opts (grammarName opts pgf <.> "pgf")
           writing opts outfile $ encodeFile outfile pgf
-          return [outfile]
 
     writeSplitPGF =
       do let outfile = outputPath opts (grammarName opts pgf <.> "pgf")
          writing opts outfile $ BSL.writeFile outfile (runPut (putSplitAbs pgf))
                                 --encodeFile_ outfile (putSplitAbs pgf)
-         outfiles <- forM (Map.toList (concretes pgf)) $ \cnc -> do
+         forM_ (Map.toList (concretes pgf)) $ \cnc -> do
            let outfile = outputPath opts (showCId (fst cnc) <.> "pgf_c")
            writing opts outfile $ encodeFile outfile cnc
-           return outfile
 
-         return (outfile:outfiles)
 
-writeLPGF :: Options -> LPGF -> IOE FilePath
+writeOutput :: Options -> FilePath-> String -> IOE ()
-writeLPGF opts lpgf = do
+writeOutput opts file str = writing opts path $ writeUTF8File path str
-  let
+  where path = outputPath opts file
-    grammarName = fromMaybe (showCId (LPGF.absname lpgf)) (flag optName opts)
-    outfile = outputPath opts (grammarName <.> "lpgf")
-  writing opts outfile $ liftIO $ LPGF.encodeFile outfile lpgf
-  return outfile
-
-writeOutput :: Options -> FilePath-> String -> IOE FilePath
-writeOutput opts file str = do
-  let outfile = outputPath opts file
-  writing opts outfile $ writeUTF8File outfile str
-  return outfile
 
 -- * Useful helper functions
 

src/compiler/GF/Data/BacktrackM.hs

@@ -64,11 +64,11 @@ finalStates :: BacktrackM s () -> s -> [s]
 finalStates bm = map fst . runBM bm
 
 instance Applicative (BacktrackM s) where
-    pure = return
+    pure a = BM (\c s b -> c a s b)
     (<*>) = ap
 
 instance Monad (BacktrackM s) where
-    return a   = BM (\c s b -> c a s b)
+    return     = pure
     BM m >>= k = BM (\c s b -> m (\a s b -> unBM (k a) c s b) s b)
         where unBM (BM m) = m
 

src/compiler/GF/Data/ErrM.hs

@@ -34,7 +34,7 @@ fromErr :: a -> Err a -> a
 fromErr a = err (const a) id
 
 instance Monad Err where
-  return      = Ok
+  return      = pure
   Ok a  >>= f = f a
   Bad s >>= f = Bad s
 
@@ -54,7 +54,7 @@ instance Functor Err where
   fmap f (Bad s) = Bad s
 
 instance Applicative Err where
-  pure = return
+  pure = Ok
   (<*>) = ap
 
 -- | added by KJ

src/compiler/GF/Data/IntMapBuilder.hs

@@ -1,61 +0,0 @@
--- | In order to build an IntMap in one pass, we need a map data structure with
--- fast lookup in both keys and values.
--- This is achieved by keeping a separate reversed map of values to keys during building.
-module GF.Data.IntMapBuilder where
-
-import Data.IntMap (IntMap)
-import qualified Data.IntMap as IntMap
-import Data.Hashable (Hashable)
-import Data.HashMap.Strict (HashMap)
-import qualified Data.HashMap.Strict as HashMap
-import Data.Tuple (swap)
-import Prelude hiding (lookup)
-
-data IMB a = IMB {
-  intMap :: IntMap a,
-  valMap :: HashMap a Int
-}
-
--- | An empty IMB
-empty :: (Eq a, Hashable a) => IMB a
-empty = IMB {
-  intMap = IntMap.empty,
-  valMap = HashMap.empty
-}
-
--- | An empty IntMap
-emptyIntMap :: IntMap a
-emptyIntMap = IntMap.empty
-
--- | Lookup a value
-lookup :: (Eq a, Hashable a) => a -> IMB a -> Maybe Int
-lookup a IMB { valMap = vm } = HashMap.lookup a vm
-
--- | Insert without any lookup
-insert :: (Eq a, Hashable a) => a -> IMB a -> (Int, IMB a)
-insert a IMB { intMap = im, valMap = vm } =
-  let
-    ix = IntMap.size im
-    im' = IntMap.insert ix a im
-    vm' = HashMap.insert a ix vm
-    imb' = IMB { intMap = im', valMap = vm' }
-  in
-    (ix, imb')
-
--- | Insert only when lookup fails
-insert' :: (Eq a, Hashable a) => a -> IMB a -> (Int, IMB a)
-insert' a imb =
-  case lookup a imb of
-    Just ix -> (ix, imb)
-    Nothing -> insert a imb
-
--- | Build IMB from existing IntMap
-fromIntMap :: (Eq a, Hashable a) => IntMap a -> IMB a
-fromIntMap im = IMB {
-  intMap = im,
-  valMap = HashMap.fromList (map swap (IntMap.toList im))
-}
-
--- | Get IntMap from IMB
-toIntMap :: (Eq a, Hashable a) => IMB a -> IntMap a
-toIntMap = intMap

src/compiler/GF/Grammar/Canonical.hs

@@ -31,7 +31,7 @@ data TypeApp  = TypeApp CatId [Type]        deriving Show
 data TypeBinding = TypeBinding VarId Type   deriving Show
 
 --------------------------------------------------------------------------------
--- ** Concrete syntax
+-- ** Concreate syntax
 
 -- | Concrete Syntax
 data Concrete  = Concrete ModId ModId Flags [ParamDef] [LincatDef] [LinDef]
@@ -105,7 +105,7 @@ data TableRow  rhs = TableRow  LinPattern rhs
 
 newtype PredefId = PredefId Id        deriving (Eq,Ord,Show)
 newtype LabelId  = LabelId Id         deriving (Eq,Ord,Show)
-newtype VarValueId = VarValueId QualId  deriving (Eq,Ord,Show)
+data VarValueId  = VarValueId QualId  deriving (Eq,Ord,Show)
 
 -- | Name of param type or param value
 newtype ParamId = ParamId QualId  deriving (Eq,Ord,Show)
@@ -116,9 +116,9 @@ newtype ParamId = ParamId QualId  deriving (Eq,Ord,Show)
 newtype ModId = ModId Id  deriving (Eq,Ord,Show)
 
 newtype CatId = CatId Id  deriving (Eq,Ord,Show)
-newtype FunId = FunId Id  deriving (Eq,Ord,Show)
+newtype FunId = FunId Id  deriving (Eq,Show)
 
-data VarId = Anonymous | VarId Id  deriving (Eq,Show)
+data VarId = Anonymous | VarId Id  deriving Show
 
 newtype Flags = Flags [(FlagName,FlagValue)] deriving Show
 type FlagName = Id

src/compiler/GF/Grammar/Grammar.hs

@@ -78,6 +78,7 @@ import PGF.Internal (FId, FunId, SeqId, LIndex, Sequence, BindType(..))
 import Data.Array.IArray(Array)
 import Data.Array.Unboxed(UArray)
 import qualified Data.Map as Map
+import qualified Data.Set as Set
 import GF.Text.Pretty
 
 
@@ -125,10 +126,20 @@ extends :: ModuleInfo -> [ModuleName]
 extends = map fst . mextend
 
 isInherited :: MInclude -> Ident -> Bool
-isInherited c i = case c of
+isInherited c =
-  MIAll -> True
+  case c of
-  MIOnly is -> elem i is
+    MIAll -> const True
-  MIExcept is -> notElem i is
+    MIOnly is -> elemOrd is
+    MIExcept is -> not . elemOrd is
+
+-- | Faster version of `elem`, using a `Set`.
+-- Make sure you give this the first argument _outside_ of the inner loop
+--
+-- Example:
+-- > myIntersection xs ys = filter (elemOrd xs) ys
+elemOrd :: Ord a => [a] -> a -> Bool
+elemOrd list = (`Set.member` set)
+  where set = Set.fromList list
 
 inheritAll :: ModuleName -> (ModuleName,MInclude)
 inheritAll i = (i,MIAll)

src/compiler/GF/Grammar/Lexer.x

@@ -4,7 +4,7 @@
 module GF.Grammar.Lexer
          ( Token(..), Posn(..)
          , P, runP, runPartial, token, lexer, getPosn, failLoc
-         , isReservedWord
+         , isReservedWord, invMap
          ) where
 
 import Control.Applicative
@@ -134,7 +134,7 @@ data Token
  | T_Double  Double          -- double precision float literals
  | T_Ident   Ident
  | T_EOF
--- deriving Show -- debug
+  deriving (Eq, Ord, Show) -- debug
 
 res = eitherResIdent
 eitherResIdent :: (Ident -> Token) -> Ident -> Token
@@ -224,6 +224,13 @@ resWords = Map.fromList
  ]
  where b s t = (identS s, t)
 
+invMap :: Map.Map Token String
+invMap = res
+  where
+    lst = Map.toList resWords
+    flp = map (\(k,v) -> (v,showIdent k)) lst
+    res = Map.fromList flp
+
 unescapeInitTail :: String -> String
 unescapeInitTail = unesc . tail where
   unesc s = case s of
@@ -276,11 +283,11 @@ instance Functor P where
   fmap = liftA
 
 instance Applicative P where
-  pure = return
+  pure a = a `seq` (P $ \s -> POk s a)
   (<*>) = ap
 
 instance Monad P where
-  return a    = a `seq` (P $ \s -> POk s a)
+  return      = pure
   (P m) >>= k = P $ \ s -> case m s of
                              POk s a          -> unP (k a) s
                              PFailed posn err -> PFailed posn err

src/compiler/GF/Grammar/Parser.y

@@ -37,6 +37,9 @@ import PGF(mkCId)
 %name pBNFCRules ListCFRule
 %name pEBNFRules ListEBNFRule
 
+%errorhandlertype explist
+%error { happyError }
+
 -- no lexer declaration
 %monad { P } { >>= } { return }
 %lexer { lexer } { T_EOF }
@@ -430,6 +433,7 @@ Exp3
                                          RecType xs -> RecType (xs ++ [(tupleLabel (length xs+1),$3)])
                                          t          -> RecType [(tupleLabel 1,$1), (tupleLabel 2,$3)]  }
   | Exp3 '**' Exp4                   { ExtR $1 $3    }
+  | Exp3 '**'  '{' ListCase '}'      { let v = identS "$vvv" in T TRaw ($4 ++ [(PV v, S $1 (Vr v))]) }
   | Exp4                             { $1            }
 
 Exp4 :: { Term }
@@ -701,8 +705,18 @@ Posn
 
 {
 
-happyError :: P a
+happyError :: (Token, [String]) -> P a
-happyError = fail "syntax error"
+happyError (t,strs) = fail $ 
+  "Syntax error:\n     Unexpected " ++ showToken t ++ ".\n     Expected one of:\n" 
+    ++ unlines (map (("     - "++).cleanupToken) strs)
+  
+  where
+    cleanupToken "Ident" = "an identifier"
+    cleanupToken x = x
+    showToken (T_Ident i) = "identifier '" ++ showIdent i ++ "'"
+    showToken t = case Map.lookup t invMap of 
+      Nothing -> show t
+      Just s -> "token '" ++ s ++"'"
 
 mkListId,mkConsId,mkBaseId  :: Ident -> Ident
 mkListId = prefixIdent "List"

src/compiler/GF/Infra/BuildInfo.hs

@@ -1,13 +1,34 @@
 {-# LANGUAGE CPP #-}
+{-# LANGUAGE TemplateHaskell #-}
+
 module GF.Infra.BuildInfo where
 import System.Info
 import Data.Version(showVersion)
 
+import Language.Haskell.TH.Syntax
+import Control.Monad.IO.Class
+import Control.Exception
+import Data.Time hiding (buildTime)
+import System.Process
+
+-- Use Template Haskell to get compile time
+buildTime :: String
+buildTime = $(do
+                 timeZone <- liftIO getCurrentTimeZone
+                 time <- liftIO $ utcToLocalTime timeZone <$> getCurrentTime
+                 return $ LitE $ StringL $ formatTime defaultTimeLocale "%F %T" time )
+
+-- Use Template Haskell to get current Git information
+gitInfo :: String
+gitInfo = $(do
+               info <- liftIO $ try $ readProcess "git" ["log", "--format=commit %h tag %(describe:tags=true)", "-1"] "" :: Q (Either SomeException String)
+               return $ LitE $ StringL $ either (\_ -> "unavailable") id info )
+
 {-# NOINLINE buildInfo #-}
 buildInfo =
     "Built on "++os++"/"++arch
-    ++" with "++compilerName++"-"++showVersion compilerVersion
+    ++" with "++compilerName++"-"++showVersion compilerVersion ++ " at " ++ buildTime ++ "\nGit info: " ++ gitInfo
-    ++", flags:"
+    ++"\nFlags:"
 #ifdef USE_INTERRUPT
     ++" interrupt"
 #endif

src/compiler/GF/Infra/CheckM.hs

@@ -48,7 +48,7 @@ newtype Check a
 instance Functor Check where fmap = liftM
 
 instance Monad Check where
-  return x = Check $ \{-ctxt-} ws -> (ws,Success x)
+  return   = pure
   f >>= g  = Check $ \{-ctxt-} ws ->
                case unCheck f {-ctxt-} ws of
                  (ws,Success x) -> unCheck (g x) {-ctxt-} ws
@@ -58,7 +58,7 @@ instance Fail.MonadFail Check where
   fail = raise
 
 instance Applicative Check where
-  pure = return
+  pure x = Check $ \{-ctxt-} ws -> (ws,Success x)
   (<*>) = ap
 
 instance ErrorMonad Check where

src/compiler/GF/Infra/Option.hs

@@ -87,8 +87,7 @@ data Verbosity = Quiet | Normal | Verbose | Debug
 data Phase = Preproc | Convert | Compile | Link
   deriving (Show,Eq,Ord)
 
-data OutputFormat = FmtLPGF
+data OutputFormat = FmtPGFPretty
-                  | FmtPGFPretty
                   | FmtCanonicalGF
                   | FmtCanonicalJson
                   | FmtJavaScript
@@ -336,7 +335,7 @@ optDescr =
      Option ['f'] ["output-format"] (ReqArg outFmt "FMT")
         (unlines ["Output format. FMT can be one of:",
                   "Canonical GF grammar: canonical_gf, canonical_json, (and haskell with option --haskell=concrete)",
-                  "Multiple concrete: pgf (default), lpgf, json, js, pgf_pretty, prolog, python, ...", -- gar,
+                  "Multiple concrete: pgf (default), json, js, pgf_pretty, prolog, python, ...", -- gar,
                   "Single concrete only: bnf, ebnf, fa, gsl, jsgf, regexp, slf, srgs_xml, srgs_abnf, vxml, ....", -- cf, lbnf,
                   "Abstract only: haskell, ..."]), -- prolog_abs,
      Option [] ["sisr"] (ReqArg sisrFmt "FMT")
@@ -478,8 +477,7 @@ outputFormats = map fst outputFormatsExpl
 
 outputFormatsExpl :: [((String,OutputFormat),String)]
 outputFormatsExpl =
-    [(("lpgf",         FmtLPGF),"Linearisation-only PGF"),
+    [(("pgf_pretty",   FmtPGFPretty),"human-readable pgf"),
-     (("pgf_pretty",   FmtPGFPretty),"Human-readable PGF"),
      (("canonical_gf", FmtCanonicalGF),"Canonical GF source files"),
      (("canonical_json", FmtCanonicalJson),"Canonical JSON source files"),
      (("js",           FmtJavaScript),"JavaScript (whole grammar)"),

src/compiler/GF/Infra/SIO.hs

@@ -52,11 +52,11 @@ newtype SIO a = SIO {unS::PutStr->IO a}
 instance Functor SIO where fmap = liftM
 
 instance Applicative SIO where
-  pure = return
+  pure x = SIO (const (pure x))
   (<*>) = ap
 
 instance Monad SIO where
-  return x = SIO (const (return x))
+  return         = pure
   SIO m1 >>= xm2 = SIO $ \ h -> m1 h >>= \ x -> unS (xm2 x) h
 
 instance Fail.MonadFail SIO where

src/compiler/GF/Interactive.hs

@@ -32,14 +32,17 @@ import qualified Text.ParserCombinators.ReadP as RP
 import System.Directory({-getCurrentDirectory,-}getAppUserDataDirectory)
 import Control.Exception(SomeException,fromException,evaluate,try)
 import Control.Monad.State hiding (void)
+import Control.Monad (join, when, (<=<))
 import qualified GF.System.Signal as IO(runInterruptibly)
 #ifdef SERVER_MODE
 import GF.Server(server)
 #endif
 
 import GF.Command.Messages(welcome)
--- Provides an orphan instance of MonadFail for StateT in ghc versions < 8
+#if !(MIN_VERSION_base(4,9,0))
+-- Needed to make it compile on GHC < 8
 import Control.Monad.Trans.Instances ()
+#endif
 
 -- | Run the GF Shell in quiet mode (@gf -run@).
 mainRunGFI :: Options -> [FilePath] -> IO ()

src/compiler/GF/Interactive2.hs

@@ -12,7 +12,7 @@ import GF.Command.Abstract
 import GF.Command.Parse(readCommandLine,pCommand)
 import GF.Data.Operations (Err(..))
 import GF.Data.Utilities(whenM,repeatM)
-
+import Control.Monad (join, when, (<=<))
 import GF.Infra.UseIO(ioErrorText,putStrLnE)
 import GF.Infra.SIO
 import GF.Infra.Option

src/compiler/GF/Main.hs

@@ -16,6 +16,7 @@ import Data.Version
 import System.Directory
 import System.Environment (getArgs)
 import System.Exit
+import GHC.IO.Encoding
 -- import GF.System.Console (setConsoleEncoding)
 
 -- | Run the GF main program, taking arguments from the command line.
@@ -23,6 +24,7 @@ import System.Exit
 -- Run @gf --help@ for usage info.
 main :: IO ()
 main = do
+  setLocaleEncoding utf8
   -- setConsoleEncoding
   uncurry mainOpts =<< getOptions
 
@@ -46,7 +48,10 @@ getOptions = do
 mainOpts :: Options -> [FilePath] -> IO ()
 mainOpts opts files =
     case flag optMode opts of
-      ModeVersion     -> putStrLn $ "Grammatical Framework (GF) version " ++ showVersion version ++ "\n" ++ buildInfo
+      ModeVersion     -> do datadir <- getDataDir
+                            putStrLn $ "Grammatical Framework (GF) version " ++ showVersion version ++ "\n" ++ 
+                                       buildInfo ++ "\n" ++
+                                       "Shared folder: " ++ datadir
       ModeHelp        -> putStrLn helpMessage
       ModeServer port -> GFI1.mainServerGFI opts port files
       ModeCompiler    -> mainGFC opts files

src/compiler/GF/Text/Coding.hs

@@ -38,7 +38,7 @@ decodeUnicode :: TextEncoding -> ByteString -> String
 decodeUnicode enc bs = unsafePerformIO $ decodeUnicodeIO enc bs
 
 decodeUnicodeIO enc (PS fptr l len) = do
-    let bbuf = Buffer{bufRaw=fptr, bufState=ReadBuffer, bufSize=len, bufL=l, bufR=l+len}
+    let bbuf = (emptyBuffer fptr len ReadBuffer) { bufL=l, bufR=l+len }
     cbuf <- newCharBuffer 128 WriteBuffer
     case enc of
       TextEncoding {mkTextDecoder=mk} -> do decoder <- mk

src/programs/gf-main.hs

@@ -1,3 +1,5 @@
+module Main where
+
 import qualified GF
 
 main = GF.main

src/runtime/c/INSTALL

@@ -44,7 +44,7 @@ $ make install
 For Windows users
 -----------------
 
-- Install MinGW: http://www.mingw.org/. From the installer you need
+- Install MinGW: http://www.mingw-w64.org/. From the installer you need
   to select at least the following packages:
      - Mingw-developer-toolkit
      - Mingw-base

src/runtime/c/configure.ac

@@ -30,6 +30,7 @@ AM_PROG_CC_C_O
  -Wall\
  -Wextra\
  -Wno-missing-field-initializers\
+ -fpermissive\
  -Wno-unused-parameter\
  -Wno-unused-value"
 fi]
@@ -45,6 +46,8 @@ case "$target_cpu" in
 		   [Define if lightning is targeting the powerpc architecture]) ;;
 	arm*)    cpu=arm; AC_DEFINE(LIGHTNING_ARM, 1,
 		   [Define if lightning is targeting the arm architecture]) ;;
+	aarch64) cpu=aarch64; AC_DEFINE(LIGHTNING_AARCH64, 1,
+		   [Define if lightning is targeting the aarch64 architecture]) ;;
 	*)	 AC_MSG_ERROR([cpu $target_cpu not supported])	;;
 esac
 

src/runtime/c/gu/fun.h

@@ -12,17 +12,17 @@ typedef void (*GuFn2)(GuFn* clo, void* arg1, void* arg2);
 
 static inline void
 gu_apply0(GuFn* fn) {
-	(*fn)(fn);
+	((GuFn0)(*fn))(fn);
 }
 
 static inline void
 gu_apply1(GuFn* fn, void* arg1) {
-	(*fn)(fn, arg1);
+	((GuFn1)(*fn))(fn, arg1);
 }
 
 static inline void
 gu_apply2(GuFn* fn, void* arg1, void* arg2) {
-	(*fn)(fn, arg1, arg2);
+	((GuFn2)(*fn))(fn, arg1, arg2);
 }
 
 #define gu_apply(fn_, ...)			\

src/runtime/c/pgf/jit.c

@@ -4,10 +4,49 @@
 #include <pgf/data.h>
 #include <pgf/reasoner.h>
 #include <pgf/reader.h>
+
+#if !defined(__aarch64__)
 #include "lightning.h"
+#endif
 
 //#define PGF_JIT_DEBUG
 
+#if defined(EMSCRIPTEN) || defined(__aarch64__)
+
+PGF_INTERNAL PgfJitState*
+pgf_new_jit(PgfReader* rdr)
+{
+	return NULL;
+}
+
+PGF_INTERNAL PgfEvalGates*
+pgf_jit_gates(PgfReader* rdr)
+{
+    PgfEvalGates* gates = gu_new(PgfEvalGates, rdr->opool);
+    memset(gates, 0, sizeof(*gates));
+    return gates;
+}
+
+PGF_INTERNAL void
+pgf_jit_predicate(PgfReader* rdr, PgfAbstr* abstr,
+                  PgfAbsCat* abscat)
+{
+	size_t n_funs = pgf_read_len(rdr);
+	gu_return_on_exn(rdr->err, );
+
+	for (size_t i = 0; i < n_funs; i++) {
+		gu_in_f64be(rdr->in, rdr->err);  // ignore
+		gu_return_on_exn(rdr->err,);
+		pgf_read_cid(rdr, rdr->tmp_pool);
+	}
+}
+
+PGF_INTERNAL void
+pgf_jit_done(PgfReader* rdr, PgfAbstr* abstr)
+{
+}
+
+#else
 
 struct PgfJitState {
 	jit_state jit;
@@ -1329,3 +1368,5 @@ pgf_jit_done(PgfReader* rdr, PgfAbstr* abstr)
 	
 	jit_flush_code(rdr->jit_state->buf, jit_get_ip().ptr);
 }
+
+#endif

src/runtime/c/pgf/lightning/aarch64/asm.h

@@ -0,0 +1 @@
+// DUMMY

src/runtime/c/pgf/lightning/aarch64/core.h

@@ -0,0 +1 @@
+

src/runtime/c/pgf/lightning/aarch64/fp.h

@@ -0,0 +1 @@
+

src/runtime/c/pgf/lightning/aarch64/funcs.h

@@ -0,0 +1 @@
+// DUMMY

src/runtime/c/pgf/parser.c

@@ -44,6 +44,7 @@ typedef struct {
     PgfParseState *before;
     PgfParseState *after;
     PgfToken prefix;
+    bool prefix_bind;
     PgfTokenProb* tp;
     PgfExprEnum en;    // enumeration for the generated trees/tokens
 #ifdef PGF_COUNTS_DEBUG
@@ -1009,6 +1010,7 @@ pgf_new_parse_state(PgfParsing* ps, size_t start_offset,
 	                    (start_offset == end_offset);
 	state->start_offset = start_offset;
 	state->end_offset = end_offset;
+
 	state->viterbi_prob = viterbi_prob;
  	state->lexicon_idx =
 		gu_new_buf(PgfLexiconIdxEntry, ps->pool);
@@ -1381,6 +1383,15 @@ pgf_parsing_symbol(PgfParsing* ps, PgfItem* item, PgfSymbol sym)
 		break;
 	}
 	case PGF_SYMBOL_BIND: {
+        if (!ps->prefix_bind && ps->prefix != NULL && *(ps->sentence + ps->before->end_offset) == 0) {
+            PgfProductionApply* papp = gu_variant_data(item->prod);
+
+			ps->tp = gu_new(PgfTokenProb, ps->out_pool);
+			ps->tp->tok  = NULL;
+			ps->tp->cat  = item->conts->ccat->cnccat->abscat->name;
+			ps->tp->fun  = papp->fun->absfun->name;
+			ps->tp->prob = item->inside_prob + item->conts->outside_prob;
+        } else {
             if (ps->before->start_offset == ps->before->end_offset &&
                 ps->before->needs_bind) {
                 PgfParseState* state =
@@ -1394,6 +1405,7 @@ pgf_parsing_symbol(PgfParsing* ps, PgfItem* item, PgfSymbol sym)
                 }
             } else {
                 pgf_item_free(ps, item);
+            }
         }
 		break;
 	}
@@ -2337,7 +2349,8 @@ pgf_parser_completions_next(GuEnum* self, void* to, GuPool* pool)
 
 PGF_API GuEnum*
 pgf_complete(PgfConcr* concr, PgfType* type, GuString sentence, 
-             GuString prefix, GuExn *err, GuPool* pool)
+             GuString prefix, bool prefix_bind,
+             GuExn *err, GuPool* pool)
 {
 	if (concr->sequences == NULL ||
 	    concr->cnccats == NULL) {
@@ -2377,6 +2390,7 @@ pgf_complete(PgfConcr* concr, PgfType* type, GuString sentence,
 	// Now begin enumerating the completions
 	ps->en.next = pgf_parser_completions_next;
 	ps->prefix  = prefix;
+    ps->prefix_bind = prefix_bind;
 	ps->tp      = NULL;
 	return &ps->en;
 }

src/runtime/c/pgf/pgf.h

@@ -251,7 +251,8 @@ typedef struct {
 
 PGF_API_DECL GuEnum*
 pgf_complete(PgfConcr* concr, PgfType* type, GuString string, 
-             GuString prefix, GuExn* err, GuPool* pool);
+             GuString prefix, bool prefix_bind,
+             GuExn* err, GuPool* pool);
 
 typedef struct PgfLiteralCallback PgfLiteralCallback;
 

src/runtime/haskell-bind/PGF2.hsc

@@ -1026,7 +1026,10 @@ complete lang (Type ctype _) sent pfx =
           touchConcr lang
           return []
         else do
-          tok <- peekUtf8CString =<< (#peek PgfTokenProb, tok) cmpEntry
+          p_tok <- (#peek PgfTokenProb, tok) cmpEntry
+          tok <- if p_tok == nullPtr
+                   then return "&+"
+                   else peekUtf8CString p_tok
           cat <- peekUtf8CString =<< (#peek PgfTokenProb, cat) cmpEntry
           fun <- peekUtf8CString =<< (#peek PgfTokenProb, fun) cmpEntry
           prob <- (#peek PgfTokenProb, prob) cmpEntry

src/runtime/haskell-bind/Setup.hs

@@ -0,0 +1,3 @@
+import Distribution.Simple
+
+main = defaultMain

src/runtime/haskell-bind/pgf2.cabal

@@ -15,7 +15,7 @@ homepage: https://www.grammaticalframework.org/
 bug-reports: https://github.com/GrammaticalFramework/gf-core/issues
 author: Krasimir Angelov
 extra-source-files: CHANGELOG.md, README.md
-tested-with: GHC==7.10.3, GHC==8.0.2, GHC==8.10.4
+tested-with: GHC==7.10.3, GHC==8.0.2, GHC==8.10.4, GHC=9.6.6
 
 library
   exposed-modules:
@@ -26,7 +26,7 @@ library
     PGF2.Expr,
     PGF2.Type
   build-depends:
-    base >= 4.9.1 && < 4.15,
+    base >= 4.9.1 && < 4.22,
     containers >= 0.5.7 && < 0.7,
     pretty >= 1.1.3 && < 1.2
   default-language: Haskell2010

src/runtime/haskell/Data/Binary/Builder.hs

@@ -74,12 +74,18 @@ import qualified Data.ByteString.Internal as S
 #endif
 
 #if defined(__GLASGOW_HASKELL__) && !defined(__HADDOCK__)
-import GHC.Base(Int(..),uncheckedShiftRL# )
+import GHC.Base(Int(..),uncheckedShiftRL#,)
 import GHC.Word (Word32(..),Word16(..),Word64(..))
 
+#if MIN_VERSION_base(4,16,0)
+import GHC.Exts (wordToWord16#, word16ToWord#, wordToWord32#, word32ToWord#)
+#endif
 #if WORD_SIZE_IN_BITS < 64 && __GLASGOW_HASKELL__ >= 608 
 import GHC.Word (uncheckedShiftRL64#)
 #endif
+#if __GLASGOW_HASKELL__ >= 900 
+import GHC.Word (uncheckedShiftRL64#)
+#endif
 #endif
 
 ------------------------------------------------------------------------
@@ -108,7 +114,7 @@ instance Semigroup Builder where
 instance Monoid Builder where
     mempty  = empty
     {-# INLINE mempty #-}
-    mappend = append
+    mappend = (<>)
     {-# INLINE mappend #-}
 
 ------------------------------------------------------------------------
@@ -411,8 +417,14 @@ shiftr_w32 :: Word32 -> Int -> Word32
 shiftr_w64 :: Word64 -> Int -> Word64
 
 #if defined(__GLASGOW_HASKELL__) && !defined(__HADDOCK__)
+#if MIN_VERSION_base(4,16,0)
+shiftr_w16 (W16# w) (I# i) = W16# (wordToWord16# ((word16ToWord# w) `uncheckedShiftRL#`   i))
+shiftr_w32 (W32# w) (I# i) = W32# (wordToWord32# ((word32ToWord# w) `uncheckedShiftRL#`   i))
+#else
 shiftr_w16 (W16# w) (I# i) = W16# (w `uncheckedShiftRL#`   i)
 shiftr_w32 (W32# w) (I# i) = W32# (w `uncheckedShiftRL#`   i)
+#endif
+
 
 #if WORD_SIZE_IN_BITS < 64
 shiftr_w64 (W64# w) (I# i) = W64# (w `uncheckedShiftRL64#` i)
@@ -424,7 +436,11 @@ foreign import ccall unsafe "stg_uncheckedShiftRL64"
 #endif
 
 #else
+#if __GLASGOW_HASKELL__ <= 810
 shiftr_w64 (W64# w) (I# i) = W64# (w `uncheckedShiftRL#` i)
+#else
+shiftr_w64 (W64# w) (I# i) = W64# (w `uncheckedShiftRL64#` i)
+#endif
 #endif
 
 #else

src/runtime/haskell/Data/Binary/Get.hs

@@ -1,4 +1,6 @@
 {-# LANGUAGE CPP, MagicHash #-}
+-- This module makes profiling a lot slower, so don't add automatic cost centres 
+{-# OPTIONS_GHC -fno-prof-auto #-}
 -- for unboxed shifts
 
 -----------------------------------------------------------------------------
@@ -99,6 +101,12 @@ import Data.STRef
 import GHC.Base
 import GHC.Word
 --import GHC.Int
+#if MIN_VERSION_base(4,16,0)
+import GHC.Exts (wordToWord16#, word16ToWord#, wordToWord32#, word32ToWord#)
+#endif
+#if __GLASGOW_HASKELL__ >= 900 
+import GHC.Word (uncheckedShiftL64#)
+#endif
 #endif
 
 -- Control.Monad.Fail import will become redundant in GHC 8.8+
@@ -119,11 +127,11 @@ instance Functor Get where
     {-# INLINE fmap #-}
 
 instance Applicative Get where
-    pure  = return
+    pure a = Get (\s -> (a, s))
     (<*>) = ap
 
 instance Monad Get where
-    return a  = Get (\s -> (a, s))
+    return = pure
     {-# INLINE return #-}
 
     m >>= k   = Get (\s -> case unGet m s of
@@ -530,8 +538,13 @@ shiftl_w32 :: Word32 -> Int -> Word32
 shiftl_w64 :: Word64 -> Int -> Word64
 
 #if defined(__GLASGOW_HASKELL__) && !defined(__HADDOCK__)
+#if MIN_VERSION_base(4,16,0)
+shiftl_w16 (W16# w) (I# i) = W16# (wordToWord16# ((word16ToWord# w) `uncheckedShiftL#`   i))
+shiftl_w32 (W32# w) (I# i) = W32# (wordToWord32# ((word32ToWord# w) `uncheckedShiftL#`   i))
+#else
 shiftl_w16 (W16# w) (I# i) = W16# (w `uncheckedShiftL#`   i)
 shiftl_w32 (W32# w) (I# i) = W32# (w `uncheckedShiftL#`   i)
+#endif
 
 #if WORD_SIZE_IN_BITS < 64
 shiftl_w64 (W64# w) (I# i) = W64# (w `uncheckedShiftL64#` i)
@@ -543,7 +556,12 @@ foreign import ccall unsafe "stg_uncheckedShiftL64"
 #endif
 
 #else
+#if __GLASGOW_HASKELL__ <= 810
 shiftl_w64 (W64# w) (I# i) = W64# (w `uncheckedShiftL#` i)
+#else
+shiftl_w64 (W64# w) (I# i) = W64# (w `uncheckedShiftL64#` i)
+#endif
+
 #endif
 
 #else

src/runtime/haskell/Data/Binary/Put.hs

@@ -77,15 +77,20 @@ instance Functor PutM where
         {-# INLINE fmap #-}
 
 instance Applicative PutM where
-        pure    = return
+        pure a  = Put $ PairS a mempty
         m <*> k = Put $
             let PairS f w  = unPut m
                 PairS x w' = unPut k
             in PairS (f x) (w `mappend` w')
+        m *> k  = Put $
+            let PairS _ w  = unPut m
+                PairS b w' = unPut k
+            in PairS b (w `mappend` w')
+        {-# INLINE (*>) #-}
 
 -- Standard Writer monad, with aggressive inlining
 instance Monad PutM where
-    return a = Put $ PairS a mempty
+    return = pure
     {-# INLINE return #-}
 
     m >>= k  = Put $
@@ -94,10 +99,7 @@ instance Monad PutM where
         in PairS b (w `mappend` w')
     {-# INLINE (>>=) #-}
 
-    m >> k  = Put $
+    (>>) = (*>)
-        let PairS _ w  = unPut m
-            PairS b w' = unPut k
-        in PairS b (w `mappend` w')
     {-# INLINE (>>) #-}
 
 tell :: Builder -> Put

src/runtime/haskell/LPGF.hs

@@ -1,234 +0,0 @@
-{-# LANGUAGE LambdaCase #-}
-{-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE ScopedTypeVariables #-}
-
--- | Linearisation-only portable grammar format.
---
--- LPGF is an output format from the GF compiler, intended as a smaller and faster alternative to PGF.
--- This API allows LPGF files to be used in Haskell programs.
---
--- The implementation closely follows description in Section 2 of Angelov, Bringert, Ranta (2009):
--- "PGF: A Portable Run-Time Format for Type-Theoretical Grammars".
--- http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.640.6330&rep=rep1&type=pdf
-module LPGF (
-  -- * LPGF
-  LPGF,
-  showLPGF,
-  readLPGF,
-
-  -- * Identifiers
-  CId,
-  mkCId,
-  showCId,
-  readCId,
-
-  -- * Abstract syntax
-  Abstract,
-  abstractName,
-
-  -- ** Categories
-
-  -- ** Functions
-
-  -- ** Expressions
-  Expr,
-  PGF.showExpr,
-  PGF.readExpr,
-
-  -- ** Types
-
-  -- ** Type checking
-
-  -- * Concrete syntax
-  Language,
-  PGF.showLanguage,
-  PGF.readLanguage,
-  languages,
-  Concrete,
-  LPGF.concretes,
-
-  -- ** Linearization
-  linearize,
-  linearizeText,
-  linearizeConcrete,
-  linearizeConcreteText
-) where
-
-import LPGF.Internal
-import PGF (Language)
-import PGF.CId
-import PGF.Expr (Expr, Literal (..))
-import PGF.Tree (Tree (..), expr2tree, prTree)
-import qualified PGF
-
-import Data.Binary (decodeFile)
-import Data.Either (isLeft)
-import qualified Data.IntMap as IntMap
-import qualified Data.Map.Strict as Map
-import Data.Text (Text)
-import qualified Data.Text as T
-import Numeric (showFFloat)
-import Text.Printf (printf)
-
-import Prelude hiding ((!!))
-import qualified Prelude
-
--- | The abstract language name is the name of the top-level abstract module.
-abstractName :: LPGF -> CId
-abstractName = absname
-
--- | List of all languages available in the given grammar.
-languages :: LPGF -> [Language]
-languages = Map.keys . LPGF.Internal.concretes
-
--- | Map of all languages and their corresponding concrete sytaxes.
-concretes :: LPGF -> Map.Map Language Concrete
-concretes = LPGF.Internal.concretes
-
--- | Reads file in LPGF and produces 'LPGF' term.
--- The file is usually produced with:
---
--- > $ gf --make --output-format=lpgf <grammar file name>
-readLPGF :: FilePath -> IO LPGF
-readLPGF = Data.Binary.decodeFile
-
--- | Produce pretty-printed representation of an LPGF.
-showLPGF :: LPGF -> String
-showLPGF = render . pp
-
--- | Main linearize function, to 'String'
-linearize :: LPGF -> Language -> Expr -> String
-linearize lpgf lang expr = T.unpack $ linearizeText lpgf lang expr
-
--- | Main linearize function, to 'Data.Text.Text'
-linearizeText :: LPGF -> Language -> Expr -> Text
-linearizeText lpgf lang =
-  case Map.lookup lang (LPGF.Internal.concretes lpgf) of
-    Just concr -> linearizeConcreteText concr
-    Nothing -> error $ printf "Unknown language: %s" (showCId lang)
-
--- | Language-specific linearize function, to 'String'
-linearizeConcrete :: Concrete -> Expr -> String
-linearizeConcrete concr expr = T.unpack $ linearizeConcreteText concr expr
-
--- | Language-specific linearize function, to 'Data.Text.Text'
-linearizeConcreteText :: Concrete -> Expr -> Text
-linearizeConcreteText concr expr = lin2string $ lin (expr2tree expr)
-  where
-    lin :: Tree -> LinFun
-    lin = \case
-      Fun f as ->
-        case Map.lookup f (lins concr) of
-          Just t -> eval cxt t
-            where cxt = Context { cxToks = toks concr, cxArgs = map lin as }
-          _ -> Missing f
-      Lit l -> Tuple [Token (T.pack s)]
-        where
-          s = case l of
-            LStr s -> s
-            LInt i -> show i
-            LFlt f -> showFFloat (Just 6) f ""
-      x -> error $ printf "Cannot lin: %s" (prTree x)
-
--- | Evaluation context
-data Context = Context {
-  cxArgs :: [LinFun], -- ^  is a sequence of terms
-  cxToks :: IntMap.IntMap Text -- ^ token map
-}
-
--- | Operational semantics
-eval :: Context -> LinFun -> LinFun
-eval cxt t = case t of
-  Error err -> error err
-  Pre pts df -> Pre pts' df'
-    where
-      pts' = [(pfxs, eval cxt t) | (pfxs, t) <- pts]
-      df' = eval cxt df
-
-  Concat s t -> Concat v w
-    where
-      v = eval cxt s
-      w = eval cxt t
-  Tuple ts -> Tuple vs
-    where vs = map (eval cxt) ts
-  Projection t u ->
-    case (eval cxt t, eval cxt u) of
-      (Missing f, _) -> Missing f
-      (Tuple vs, Missing _) | not (null vs) -> vs !! 0 -- cannot know how deep to unpack; this gives best results with current testsuite
-      (_, Missing f) -> Missing f
-      (Tuple vs, Ix i) -> vs !! (i-1)
-      (t', tv@(Tuple _)) -> eval cxt $ foldl Projection t' (flattenTuple tv)
-      (t',u') -> error $ printf "Incompatible projection:\n- %s\n⇓ %s\n- %s\n⇓ %s" (show t) (show t') (show u) (show u')
-  Argument i -> cxArgs cxt !! (i-1)
-
-  PreIx pts df -> Pre pts' df'
-    where
-      pts' = [(pfxs, eval cxt t) | (ix, t) <- pts, let pfxs = maybe [] (read . T.unpack) $ IntMap.lookup ix (cxToks cxt)]
-      df' = eval cxt df
-  TokenIx i -> maybe Empty Token $ IntMap.lookup i (cxToks cxt)
-
-  _ -> t
-
-flattenTuple :: LinFun -> [LinFun]
-flattenTuple = \case
-  Tuple vs -> concatMap flattenTuple vs
-  lf -> [lf]
-
--- | Turn concrete syntax terms into an actual string.
--- This is done in two passes, first to flatten concats & evaluate pre's, then to
--- apply BIND and other predefs.
-lin2string :: LinFun -> Text
-lin2string lf = T.unwords $ join $ flatten [lf]
-  where
-    -- Process bind et al into final token list
-    join :: [Either LinFun Text] -> [Text]
-    join elt = case elt of
-      Right tok:Left Bind:ls ->
-        case join ls of
-          next:ls' -> tok `T.append` next : ls'
-          _ -> []
-      Right tok:ls -> tok : join ls
-      Left Space:ls -> join ls
-      Left Capit:ls ->
-        case join ls of
-          next:ls' -> T.toUpper (T.take 1 next) `T.append` T.drop 1 next : ls'
-          _ -> []
-      Left AllCapit:ls ->
-        case join ls of
-          next:ls' -> T.toUpper next : ls'
-          _ -> []
-      Left (Missing cid):ls -> join (Right (T.pack (printf "[%s]" (show cid))) : ls)
-      [] -> []
-      x -> error $ printf "Unhandled term in lin2string: %s" (show x)
-
-    -- Process concats, tuples, pre into flat list
-    flatten :: [LinFun] -> [Either LinFun Text]
-    flatten [] = []
-    flatten (l:ls) = case l of
-      Empty -> flatten ls
-      Token "" -> flatten ls
-      Token tok -> Right tok : flatten ls
-      Concat l1 l2 -> flatten (l1 : l2 : ls)
-      Tuple [l] -> flatten (l:ls)
-      Tuple (l:_) -> flatten (l:ls) -- unselected table, just choose first option (see e.g. FoodsJpn)
-      Pre pts df ->
-        let
-          f = flatten ls
-          ch = case dropWhile isLeft f of
-            Right next:_ ->
-              let matches = [ l | (pfxs, l) <- pts, any (`T.isPrefixOf` next) pfxs ]
-              in  if null matches then df else head matches
-            _ -> df
-        in flatten (ch:ls)
-      x -> Left x : flatten ls
-
--- | List indexing with more verbose error messages
-(!!) :: (Show a) => [a] -> Int -> a
-(!!) xs i
-  | i < 0 = error $ printf "!!: index %d too small for list: %s" i (show xs)
-  | i > length xs - 1 = error $ printf "!!: index %d too large for list: %s" i (show xs)
-  | otherwise = xs Prelude.!! i
-
-isIx :: LinFun -> Bool
-isIx (Ix _) = True
-isIx _ = False

src/runtime/haskell/LPGF/Internal.hs

@@ -1,227 +0,0 @@
-{-# LANGUAGE LambdaCase #-}
-
-module LPGF.Internal where
-
-import PGF.CId
-import PGF ()
-
-import Control.Monad (liftM, liftM2, forM_)
-import qualified Control.Monad.Writer as CMW
-import Data.Binary (Binary, put, get, putWord8, getWord8, encodeFile)
-import qualified Data.IntMap as IntMap
-import qualified Data.Map.Strict as Map
-import Data.Text (Text)
-import qualified Data.Text as T
-import qualified Data.Text.Encoding as TE
-
--- | Linearisation-only PGF
-data LPGF = LPGF {
-  absname   :: CId,
-  abstract  :: Abstract,
-  concretes :: Map.Map CId Concrete
-} deriving (Show)
-
--- | Abstract syntax (currently empty)
-data Abstract = Abstract {
-} deriving (Show)
-
--- | Concrete syntax
-data Concrete = Concrete {
-  toks    :: IntMap.IntMap Text, -- ^ all strings are stored exactly once here
-  -- lincats :: Map.Map CId LinType, -- ^ a linearization type for each category
-  lins    :: Map.Map CId LinFun  -- ^ a linearization function for each function
-} deriving (Show)
-
--- | Abstract function type
--- data Type = Type [CId] CId
---   deriving (Show)
-
--- -- | Linearisation type
--- data LinType =
---     StrType
---   | IxType Int
---   | ProductType [LinType]
---   deriving (Show)
-
--- | Linearisation function
-data LinFun =
-  -- Additions
-    Error String -- ^ a runtime error, should probably not be supported at all
-  | Bind -- ^ join adjacent tokens
-  | Space -- ^ space between adjacent tokens
-  | Capit -- ^ capitalise next character
-  | AllCapit -- ^ capitalise next word
-  | Pre [([Text], LinFun)] LinFun
-  | Missing CId -- ^ missing definition (inserted at runtime)
-
-  -- From original definition in paper
-  | Empty
-  | Token Text
-  | Concat LinFun LinFun
-  | Ix Int
-  | Tuple [LinFun]
-  | Projection LinFun LinFun
-  | Argument Int
-
-  -- For reducing LPGF file when stored
-  | PreIx [(Int, LinFun)] LinFun -- ^ index into `toks` map (must apply read to convert to list)
-  | TokenIx Int -- ^ index into `toks` map
-
-  deriving (Show, Read)
-
-instance Binary LPGF where
-  put lpgf = do
-    put (absname lpgf)
-    put (abstract lpgf)
-    put (concretes lpgf)
-  get = do
-    an <- get
-    abs <- get
-    concs <- get
-    return $ LPGF {
-      absname = an,
-      abstract = abs,
-      concretes = concs
-    }
-
-instance Binary Abstract where
-  put abs = return ()
-  get = return $ Abstract {}
-
-instance Binary Concrete where
-  put concr = do
-    put (toks concr)
-    put (lins concr)
-  get = do
-    ts <- get
-    ls <- get
-    return $ Concrete {
-      toks = ts,
-      lins = ls
-    }
-
-instance Binary LinFun where
-  put = \case
-    Error e          -> putWord8 0 >> put e
-    Bind             -> putWord8 1
-    Space            -> putWord8 2
-    Capit            -> putWord8 3
-    AllCapit         -> putWord8 4
-    Pre ps d         -> putWord8 5 >> put (ps,d)
-    Missing f        -> putWord8 13 >> put f
-
-    Empty            -> putWord8 6
-    Token t          -> putWord8 7 >> put t
-    Concat l1 l2     -> putWord8 8 >> put (l1,l2)
-    Ix i             -> putWord8 9 >> put i
-    Tuple ls         -> putWord8 10 >> put ls
-    Projection l1 l2 -> putWord8 11 >> put (l1,l2)
-    Argument i       -> putWord8 12 >> put i
-
-    PreIx ps d       -> putWord8 15 >> put (ps,d)
-    TokenIx i        -> putWord8 14 >> put i
-
-  get = do
-    tag <- getWord8
-    case tag of
-      0  -> liftM  Error get
-      1  -> return Bind
-      2  -> return Space
-      3  -> return Capit
-      4  -> return AllCapit
-      5  -> liftM2 Pre get get
-      13 -> liftM  Missing get
-
-      6  -> return Empty
-      7  -> liftM  Token get
-      8  -> liftM2 Concat get get
-      9  -> liftM  Ix get
-      10 -> liftM  Tuple get
-      11 -> liftM2 Projection get get
-      12 -> liftM  Argument get
-
-      15 -> liftM2 PreIx get get
-      14 -> liftM  TokenIx get
-      _  -> fail "Failed to decode LPGF binary format"
-
-instance Binary Text where
-  put = put . TE.encodeUtf8
-  get = liftM TE.decodeUtf8 get
-
-encodeFile :: FilePath -> LPGF -> IO ()
-encodeFile = Data.Binary.encodeFile
-
-------------------------------------------------------------------------------
--- Utilities
-
--- | Helper for building concat trees
-mkConcat :: [LinFun] -> LinFun
-mkConcat [] = Empty
-mkConcat [x] = x
-mkConcat xs = foldl1 Concat xs
-
--- | Helper for unfolding concat trees
-unConcat :: LinFun -> [LinFun]
-unConcat (Concat l1 l2) = concatMap unConcat [l1, l2]
-unConcat lf = [lf]
-
-------------------------------------------------------------------------------
--- Pretty-printing
-
-type Doc = CMW.Writer [String] ()
-
-render :: Doc -> String
-render = unlines . CMW.execWriter
-
-class PP a where
-  pp :: a -> Doc
-
-instance PP LPGF where
-  pp (LPGF _ _ cncs) = mapM_ pp cncs
-
-instance PP Concrete where
-  pp (Concrete toks lins) = do
-    forM_ (IntMap.toList toks) $ \(i,tok) ->
-      CMW.tell [show i ++ " " ++ T.unpack tok]
-    CMW.tell [""]
-    forM_ (Map.toList lins) $ \(cid,lin) -> do
-      CMW.tell ["# " ++ showCId cid]
-      pp lin
-      CMW.tell [""]
-
-instance PP LinFun where
-  pp = pp' 0
-    where
-      pp' n = \case
-        Pre ps d -> do
-          p "Pre"
-          CMW.tell [ replicate (2*(n+1)) ' ' ++ show p | p <- ps ]
-          pp' (n+1) d
-
-        c@(Concat l1 l2) -> do
-          let ts = unConcat c
-          if any isDeep ts
-          then do
-            p "Concat"
-            mapM_ (pp' (n+1)) ts
-          else
-            p $ "Concat " ++ show ts
-        Tuple ls | any isDeep ls -> do
-          p "Tuple"
-          mapM_ (pp' (n+1)) ls
-        Projection l1 l2 | isDeep l1 || isDeep l2 -> do
-          p "Projection"
-          pp' (n+1) l1
-          pp' (n+1) l2
-        t -> p $ show t
-        where
-          p :: String -> Doc
-          p t = CMW.tell [ replicate (2*n) ' ' ++ t ]
-
-      isDeep = not . isTerm
-      isTerm = \case
-        Pre _ _ -> False
-        Concat _ _ -> False
-        Tuple _ -> False
-        Projection _ _ -> False
-        _ -> True

src/runtime/haskell/PGF.hs

@@ -31,7 +31,7 @@ module PGF(
            languages, abstractName, languageCode,
 
            -- * Types
-           Type, Hypo,
+           Type, Hypo, BindType(..),
            showType, readType,
            mkType, mkHypo, mkDepHypo, mkImplHypo,
            unType,

src/runtime/haskell/PGF/Expr.hs

@@ -17,7 +17,8 @@ module PGF.Expr(Tree, BindType(..), Expr(..), Literal(..), Patt(..), Equation(..
                 MetaId,
 
                 -- helpers
-                pMeta,pArg,pLit,freshName,ppMeta,ppLit,ppParens
+                pMeta,pArg,pLit,freshName,ppMeta,ppLit,ppParens,
+                freshBoundVars
                ) where
 
 import PGF.CId
@@ -235,10 +236,11 @@ pLit = liftM LStr (RP.readS_to_P reads)
 
 ppExpr :: Int -> [CId] -> Expr -> PP.Doc
 ppExpr d scope (EAbs b x e) = let (bs,xs,e1) = getVars [] [] (EAbs b x e)
+                                  xs' = freshBoundVars scope xs
                               in ppParens (d > 1) (PP.char '\\' PP.<>
-                                                   PP.hsep (PP.punctuate PP.comma (reverse (List.zipWith ppBind bs xs))) PP.<+>
+                                                   PP.hsep (PP.punctuate PP.comma (reverse (List.zipWith ppBind bs xs'))) PP.<+>
                                                    PP.text "->" PP.<+>
-                                                   ppExpr 1 (xs++scope) e1)
+                                                   ppExpr 1 (xs' ++ scope) e1)
                               where
                                 getVars bs xs (EAbs b x e) = getVars (b:bs) ((freshName x xs):xs) e
                                 getVars bs xs e            = (bs,xs,e)
@@ -289,6 +291,15 @@ freshName x xs0 = loop 1 x
       | elem y xs = loop (i+1) (mkCId (show x++show i))
       | otherwise = y
 
+-- refresh new vars xs in scope if needed. AR 2024-03-01
+freshBoundVars :: [CId] -> [CId] -> [CId]
+freshBoundVars scope xs = foldr fresh [] xs
+  where
+    fresh x xs' = mkCId (freshName (showCId x) xs') : xs'
+    freshName s xs' = 
+      if elem (mkCId s) (xs' ++ scope)
+      then freshName (s ++ "'") xs'
+      else s
 
 -----------------------------------------------------
 -- Computation
@@ -397,7 +408,7 @@ match sig f eqs as0 =
         tryMatch (p          ) (VMeta i envi vs  ) env            = VSusp i envi vs (\v -> tryMatch p v env)
         tryMatch (p          ) (VGen  i vs       ) env            = VConst f as0
         tryMatch (p          ) (VSusp i envi vs k) env            = VSusp i envi vs (\v -> tryMatch p (k v) env)
-        tryMatch (p          ) v@(VConst _ _     ) env            = VConst f as0
+        tryMatch (p          ) v@(VConst _ _     ) env            = match sig f eqs as0
         tryMatch (PApp f1 ps1) (VApp f2 vs2      ) env | f1 == f2 = tryMatches eqs (ps1++ps) (vs2++as) res env
         tryMatch (PLit l1    ) (VLit l2          ) env | l1 == l2 = tryMatches eqs  ps        as  res env
         tryMatch (PImplArg p ) (VImplArg v       ) env            = tryMatch p v env

src/runtime/haskell/PGF/Linearize.hs

@@ -81,7 +81,7 @@ linTree pgf cnc e = nub (map snd (lin Nothing 0 e [] [] e []))
   where
     lp    = lproductions cnc
 
-    lin mb_cty n_fid e0 ys xs (EAbs _ x e) es = lin   mb_cty n_fid e0 ys (x:xs) e      es
+    lin mb_cty n_fid e0 ys xs (EAbs _ x e) es = lin   mb_cty n_fid e0 ys (freshBoundVars (xs ++ ys) [x] ++ xs) e      es --fresh: AR 2024
     lin mb_cty n_fid e0 ys xs (EApp e1 e2) es = lin   mb_cty n_fid e0 ys    xs  e1 (e2:es)
     lin mb_cty n_fid e0 ys xs (EImplArg e) es = lin   mb_cty n_fid e0 ys    xs  e      es
     lin mb_cty n_fid e0 ys xs (ETyped e _) es = lin   mb_cty n_fid e0 ys    xs  e      es

src/runtime/haskell/PGF/TypeCheck.hs

@@ -94,11 +94,11 @@ class Selector s where
   select        :: CId -> Scope -> Maybe Int -> TcM s (Expr,TType)
 
 instance Applicative (TcM s) where
-  pure = return
+  pure x = TcM (\abstr k h -> k x)
   (<*>) = ap
 
 instance Monad (TcM s) where
-  return x = TcM (\abstr k h -> k x)
+  return   = pure
   f >>= g  = TcM (\abstr k h -> unTcM f abstr (\x -> unTcM (g x) abstr k h) h)
 
 instance Selector s => Alternative (TcM s) where
@@ -147,9 +147,9 @@ typeGenerators scope cat = fmap normalize (liftM2 (++) x y)
          where
            Scope gamma = scope
 
-    y | cat == cidInt    = return [(1.0,ELit (LInt 999),  TTyp [] (DTyp [] cat []))]
+    y | cat == cidInt    = return [(0.1, ELit (LInt n),  TTyp [] (DTyp [] cat [])) | n <- ints]
-      | cat == cidFloat  = return [(1.0,ELit (LFlt 3.14), TTyp [] (DTyp [] cat []))]
+      | cat == cidFloat  = return [(0.1, ELit (LFlt d), TTyp [] (DTyp [] cat [])) | d <- floats]
-      | cat == cidString = return [(1.0,ELit (LStr "Foo"),TTyp [] (DTyp [] cat []))]
+      | cat == cidString = return [(0.1, ELit (LStr s),TTyp [] (DTyp [] cat [])) | s <- strs]
       | otherwise        = TcM (\abstr k h ms ->
                                     case Map.lookup cat (cats abstr) of
                                       Just (_,fns,_) -> unTcM (mapM helper fns) abstr k h ms
@@ -163,6 +163,11 @@ typeGenerators scope cat = fmap normalize (liftM2 (++) x y)
       where
         s = sum [p | (p,_,_) <- gens]
 
+ -- random elements of predefined types: many instead of one AR 2025-01-17
+    ints = [1, 2, 3, 14, 42, 123, 999, 2025, 1000000, 1234567890]
+    floats = [0.0, 1.0, 3.14, 0.999, 0.5772156649, 2.71828, 6.62607015, 19.3, 0.0001, 1.60934]
+    strs = words "A B X Y b c x y foo bar"
+
 emptyMetaStore :: MetaStore s
 emptyMetaStore = IntMap.empty
 

src/runtime/haskell/PGF/VisualizeTree.hs

@@ -651,6 +651,7 @@ app macro arg = text "\\" <> text macro <> text "{" <> arg <> text "}"
 latexDoc :: Doc -> Doc
 latexDoc body =
   vcat [text "\\documentclass{article}",
+        text "\\usepackage[a4paper,margin=0.5in,landscape]{geometry}",
         text "\\usepackage[utf8]{inputenc}",
         text "\\begin{document}",
         body,

src/runtime/haskell/pgf.cabal

@@ -9,16 +9,16 @@ synopsis: Grammatical Framework
 description: A library for interpreting the Portable Grammar Format (PGF)
 homepage: https://www.grammaticalframework.org/
 bug-reports: https://github.com/GrammaticalFramework/gf-core/issues
-tested-with: GHC==7.10.3, GHC==8.0.2, GHC==8.10.4
+tested-with: GHC==7.10.3, GHC==8.0.2, GHC==8.10.4, GHC==9.4.5
 
 library
   default-language: Haskell2010
   build-depends:
-    array >= 0.5.1 && < 0.6,
+    array >= 0.5.1 && < 0.7,
-    base >= 4.9.1 && < 4.15,
+    base >= 4.9.1 && < 5.0,
-    bytestring >= 0.10.8 && < 0.11,
+    bytestring >= 0.10.8 && < 0.12,
-    containers >= 0.5.7 && < 0.7,
+    containers >= 0.5.7 && < 0.8,
-    ghc-prim >= 0.5.0 && < 0.7,
+    ghc-prim >= 0.5.0 && < 0.10,
     mtl >= 2.2.1 && < 2.3,
     pretty >= 1.1.3 && < 1.2,
     random >= 1.1 && < 1.3,

src/runtime/python/README.org

@@ -0,0 +1,152 @@
+* INSTALL
+
+You will need the python-devel package or similar.
+
+You must have installed the PGF C runtime (see ../c/INSTALL)
+
+#+begin_src sh
+  $ python setup.py build
+  $ sudo python setup.py install
+#+end_src
+
+* Apple Silicon
+
+The following install instructions were written with the following config in mind:
+
+| OS                    | Hardware | GF                       |
+|-----------------------+----------+--------------------------+
+| MacOS Monterey 12.2.1 | Apple M1 | 3.11 from binary package |
+
+We assume that you may have installed GF as a binary package downloaded from Github.
+
+From that starting point, try all the solutions below, in sequence, until you achieve success.
+
+** Validation Goal
+
+Our goal is to be able to
+- run python 3
+- import pgf
+- type "pgf."
+- hit tab
+
+and get this:
+
+#+begin_example
+  >>> import pgf
+  >>> pgf.
+  pgf.BIND(        pgf.Concr(       pgf.Iter(        pgf.PGFError(    pgf.Type(        pgf.readExpr(    pgf.readType(
+  pgf.Bracket(     pgf.Expr(        pgf.PGF(         pgf.ParseError(  pgf.TypeError(   pgf.readPGF(
+#+end_example
+
+When that works, we can consider the Python PGF bindings to be installed successfully.
+
+** The GF binary package won't install
+
+We assume you've tried [[https://github.com/GrammaticalFramework/gf-core/releases][downloading a binary package]].
+
+If MacOS is being secure, go to System Preferences, Security & Privacy, General, and click Open Anyway.
+
+** gu/mem.h file not found
+
+Maybe you tried running something like ~pip install pgf~ or ~pip3 install pgf~.
+
+Did you get this error?
+
+#+begin_example
+  python3 setup.py build
+  running build
+  running build_ext
+  creating build/temp.macosx-12-arm64-3.9
+  clang -Wno-unused-result -Wsign-compare -Wunreachable-code -fno-common -dynamic -DNDEBUG -g -fwrapv -O3 -Wall -isysroot /Library/Developer/CommandLineTools/SDKs/MacOSX12.sdk -I/opt/homebrew/opt/python@3.9/Frameworks/Python.framework/Versions/3.9/include/python3.9 -c pypgf.c -o build/temp.macosx-12-arm64-3.9/pypgf.o -std=c99
+  pypgf.c:5:10: fatal error: 'gu/mem.h' file not found
+  #include <gu/mem.h>
+           ^~~~~~~~~~
+  1 error generated.
+  error: command '/usr/bin/clang' failed with exit code 1
+#+end_example
+
+Solution:
+
+#+begin_example
+  $ EXTRA_INCLUDE_DIRS=/usr/local/include EXTRA_LIB_DIRS=/usr/local/lib pip install pgf
+#+end_example
+
+This should tell the build where to find the include and lib files it needs to compile:
+
+#+begin_example
+  $ ls /usr/local/include/gu
+  assert.h  choice.h  enum.h    file.h    hash.h    map.h     out.h     seq.h     sysdeps.h utf8.h
+  bits.h    defs.h    exn.h     fun.h     in.h      mem.h     prime.h   string.h  ucs.h     variant.h
+
+  $ ls /usr/local/lib/libgu*
+  /usr/local/lib/libgu.0.dylib /usr/local/lib/libgu.a       /usr/local/lib/libgu.dylib   /usr/local/lib/libgu.la
+#+end_example
+
+If those files don't exist, or you get the following error, you will need to rebuild the C runtime.
+
+** symbol not found in flat namespace
+
+Did you get this error?
+
+#+begin_example
+  Python 3.9.10 (main, Jan 15 2022, 11:40:53)
+  [Clang 13.0.0 (clang-1300.0.29.3)] on darwin
+  Type "help", "copyright", "credits" or "license" for more information.
+  >>> import pgf
+  Traceback (most recent call last):
+    File "<stdin>", line 1, in <module>
+  ImportError: dlopen(/opt/homebrew/lib/python3.9/site-packages/pgf.cpython-39-darwin.so, 0x0002): symbol not found in flat namespace '_gu_alloc_variant'
+#+end_example
+
+This may be a sign that you're trying to get binaries and libraries compiled with different compilers to play nicely. We're trying to get three things to align:
+
+- the Python interpreter
+- the C runtime libraries
+- the Python pgf libraries
+
+Solution:
+
+Maybe your Python isn't the Apple-provided Python. In the above error message we see ~python3~ is provided by Homebrew. Assuming you prefer to keep things this way, we'll try to rebuild things to match your Python.
+
+Rebuilding needs a C compiler. The Apple-provided system ~clang~ is preferred. If you have multiple ~clang~ compilers installed, try disabling the others. For example, if your ~clang~ was provided by nix, run ~nix-env --uninstall clang~. Similarly for brew.
+
+Then try rebuilding the C runtime.
+
+** How to re-build the C runtime
+
+Maybe the C runtime is missing from ~/usr/local/lib~, or maybe the version you have installed is causing the "symbol not found" error.
+
+Build the C runtime by following the instructions in ~gf-core/src/runtime/c/INSTALL~.
+
+After a successful ~make install~, rebuild the Python bindings.
+
+** How to re-build the Python bindings using pip
+
+Sometimes a ~pip install pgf~ will decline to recompile, because a cached wheel exists.
+
+To return to a more pristine state,
+
+#+begin_example
+  pip uninstall pgf
+  pip cache remove pgf
+#+end_example
+
+You may need to ~sudo~ some of the above commands.
+
+Then you can repeat
+
+#+begin_example
+  $ EXTRA_INCLUDE_DIRS=/usr/local/include EXTRA_LIB_DIRS=/usr/local/lib pip install pgf
+#+end_example
+
+** How to re-build the Python bindings manually
+
+If the ~pip install pgf~ just isn't working, try building it directly in ~gf-core/src/runtime/python~:
+
+#+begin_example
+  $ python setup.py build
+  $ sudo python setup.py install
+#+end_example
+
+You may need to add the ~EXTRA~ environment prefixes as shown in previous commands.
+

src/runtime/python/pgf.egg-info/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: pgf
+Version: 1.0
+Summary: Python bindings to the Grammatical Framework's PGF runtime
+Home-page: https://www.grammaticalframework.org/
+Author: Krasimir Angelov
+Author-email: kr.angelov@gmail.com
+License: BSD
+Dynamic: author
+Dynamic: author-email
+Dynamic: description
+Dynamic: home-page
+Dynamic: license
+Dynamic: summary
+
+Grammatical Framework (GF) is a programming language for multilingual grammar applications.
+This package provides Python bindings to GF runtime, which allows you to parse and generate text using GF grammars compiled into the PGF format.

src/runtime/python/pgf.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+INSTALL
+README.org
+pypgf.c
+setup.py
+test.py
+examples/README
+examples/gf_utils.py
+examples/translation_pipeline.py
+pgf.egg-info/PKG-INFO
+pgf.egg-info/SOURCES.txt
+pgf.egg-info/dependency_links.txt
+pgf.egg-info/top_level.txt

src/runtime/python/pgf.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

src/runtime/python/pgf.egg-info/top_level.txt

@@ -0,0 +1 @@
+pgf

src/runtime/python/pypgf.c

@@ -385,7 +385,7 @@ Expr_call(ExprObject* self, PyObject* args, PyObject* kw)
 	pyexpr->pool = gu_new_pool();
 	pyexpr->expr = self->expr;
 
-	for (Py_ssize_t i = 0; i < n_args; i++) {
+	for (size_t i = 0; i < n_args; i++) {
 		PyObject* obj = PyTuple_GetItem(args, i);
 		if (obj->ob_type != &pgf_ExprType) {
 			PyErr_SetString(PyExc_TypeError, "the arguments must be expressions");
@@ -549,7 +549,7 @@ Expr_visit(ExprObject* self, PyObject *args)
 				return NULL;
 			}
 
-			for (size_t i = 0; i < app->n_args; i++) {
+			for (size_t i = 0; i < (size_t) app->n_args; i++) {
 				ExprObject* pyarg = (ExprObject*) pgf_ExprType.tp_alloc(&pgf_ExprType, 0);
 				if (pyarg == NULL) {
 					Py_DECREF(args);
@@ -856,7 +856,7 @@ Type_init(TypeObject *self, PyObject *args, PyObject *kwds)
 	self->type->cid = gu_string_copy(catname_s, self->pool);
 
 	self->type->n_exprs = n_exprs;
-	for (Py_ssize_t i = 0; i < n_exprs; i++) {
+	for (size_t i = 0; i < n_exprs; i++) {
 		PyObject* obj = PyList_GetItem(py_exprs, i);
 		if (Py_TYPE(obj) != &pgf_ExprType) {
 			PyErr_SetString(PyExc_TypeError, "the arguments in the second list must be expressions");
@@ -1155,6 +1155,82 @@ Iter_fetch_expr(IterObject* self)
 	return res;
 }
 
+typedef struct {
+	PyObject_HEAD
+} BINDObject;
+
+static PyObject *BIND_instance = NULL;
+
+static void
+BIND_dealloc(PyTypeObject *self)
+{
+    BIND_instance = NULL;
+}
+
+static PyObject *
+BIND_repr(BINDObject *self)
+{
+	return PyString_FromString("pgf.BIND");
+}
+
+static PyObject *
+BIND_str(BINDObject *self)
+{
+	return PyString_FromString("&+");
+}
+
+static PyObject *
+BIND_alloc(PyTypeObject *self, Py_ssize_t nitems)
+{
+    if (BIND_instance == NULL)
+        BIND_instance = PyType_GenericAlloc(self, nitems);
+    else
+        Py_INCREF(BIND_instance); 
+    return BIND_instance;
+}
+
+static PyTypeObject pgf_BINDType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    //0,                       /*ob_size*/
+    "pgf.BINDType",            /*tp_name*/
+    sizeof(BINDObject),        /*tp_basicsize*/
+    0,                         /*tp_itemsize*/
+    (destructor) BIND_dealloc, /*tp_dealloc*/
+    0,                         /*tp_print*/
+    0,                         /*tp_getattr*/
+    0,                         /*tp_setattr*/
+    0,                         /*tp_compare*/
+    (reprfunc) BIND_repr,      /*tp_repr*/
+    0,                         /*tp_as_number*/
+    0,                         /*tp_as_sequence*/
+    0,                         /*tp_as_mapping*/
+    0,                         /*tp_hash */
+    0,                         /*tp_call*/
+    (reprfunc) BIND_str,       /*tp_str*/
+    0,                         /*tp_getattro*/
+    0,                         /*tp_setattro*/
+    0,                         /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
+    "a marker for BIND in a bracketed string", /*tp_doc*/
+    0,		                   /*tp_traverse */
+    0,		                   /*tp_clear */
+    0,		                   /*tp_richcompare */
+    0,		                   /*tp_weaklistoffset */
+    0,		                   /*tp_iter */
+    0,		                   /*tp_iternext */
+    0,                         /*tp_methods */
+    0,                         /*tp_members */
+    0,                         /*tp_getset */
+    0,                         /*tp_base */
+    0,                         /*tp_dict */
+    0,                         /*tp_descr_get */
+    0,                         /*tp_descr_set */
+    0,                         /*tp_dictoffset */
+    0,                         /*tp_init */
+    BIND_alloc,                /*tp_alloc */
+    0,                         /*tp_new */
+};
+
 static PyObject*
 Iter_fetch_token(IterObject* self)
 {
@@ -1162,7 +1238,9 @@ Iter_fetch_token(IterObject* self)
 	if (tp == NULL)
 		return NULL;
 
-	PyObject* py_tok = PyString_FromString(tp->tok);
+	PyObject* py_tok =
+        (tp->tok != NULL) ? PyString_FromString(tp->tok)
+                          : pgf_BINDType.tp_alloc(&pgf_BINDType, 0);
 	PyObject* py_cat = PyString_FromString(tp->cat);
 	PyObject* py_fun = PyString_FromString(tp->fun);
 	PyObject* res = Py_BuildValue("(f,O,O,O)", tp->prob, py_tok, py_cat, py_fun);
@@ -1601,12 +1679,14 @@ Concr_complete(ConcrObject* self, PyObject *args, PyObject *keywds)
 {
     static char *kwlist[] = {"sentence", "cat", "prefix", "n", NULL};
 
-	const char *sentence = NULL;
+	PyObject* sentence0 = NULL;
+	const char* sentence = NULL;
 	PyObject* start = NULL;
     GuString prefix = "";
+    bool prefix_bind = false;
     int max_count = -1;
-	if (!PyArg_ParseTupleAndKeywords(args, keywds, "s|Osi", kwlist,
+    if (!PyArg_ParseTupleAndKeywords(args, keywds, "O|Osi", kwlist,
-	                                 &sentence, &start,
+                                     &sentence0, &start,
                                      &prefix, &max_count))
         return NULL;
 
@@ -1630,6 +1710,21 @@ Concr_complete(ConcrObject* self, PyObject *args, PyObject *keywds)
 
 	GuExn* parse_err = gu_new_exn(tmp_pool);
 
+    if (PyTuple_Check(sentence0) && 
+        PyTuple_GET_SIZE(sentence0) == 2 && 
+        PyTuple_GET_ITEM(sentence0,1) == pgf_BINDType.tp_alloc(&pgf_BINDType, 0))
+    {
+        sentence0 = PyTuple_GET_ITEM(sentence0,0);
+        prefix_bind = true;
+    }
+
+    if (PyUnicode_Check(sentence0)) {
+        sentence = PyUnicode_AsUTF8(sentence0);
+    } else {
+        PyErr_SetString(PyExc_TypeError, "The sentence must be either a string or a tuple of string and pgf.BIND");
+        return NULL;
+    }
+
 	PgfType* type;
 	if (start == NULL) {
 		type = pgf_start_cat(self->grammar->pgf, pyres->pool);
@@ -1642,7 +1737,7 @@ Concr_complete(ConcrObject* self, PyObject *args, PyObject *keywds)
 	}
 
 	pyres->res =
-		pgf_complete(self->concr, type, sentence, prefix, parse_err, pyres->pool);
+		pgf_complete(self->concr, type, sentence, prefix, prefix_bind, parse_err, pyres->pool);
 
 	if (!gu_ok(parse_err)) {
 		Py_DECREF(pyres);
@@ -2077,58 +2172,6 @@ static PyTypeObject pgf_BracketType = {
     0,                         /*tp_new */
 };
 
-typedef struct {
-	PyObject_HEAD
-} BINDObject;
-
-static PyObject *
-BIND_repr(BINDObject *self)
-{
-	return PyString_FromString("&+");
-}
-
-static PyTypeObject pgf_BINDType = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    //0,                       /*ob_size*/
-    "pgf.BIND",                /*tp_name*/
-    sizeof(BINDObject),        /*tp_basicsize*/
-    0,                         /*tp_itemsize*/
-    0,                         /*tp_dealloc*/
-    0,                         /*tp_print*/
-    0,                         /*tp_getattr*/
-    0,                         /*tp_setattr*/
-    0,                         /*tp_compare*/
-    0,                         /*tp_repr*/
-    0,                         /*tp_as_number*/
-    0,                         /*tp_as_sequence*/
-    0,                         /*tp_as_mapping*/
-    0,                         /*tp_hash */
-    0,                         /*tp_call*/
-    (reprfunc) BIND_repr,      /*tp_str*/
-    0,                         /*tp_getattro*/
-    0,                         /*tp_setattro*/
-    0,                         /*tp_as_buffer*/
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
-    "a marker for BIND in a bracketed string", /*tp_doc*/
-    0,		                   /*tp_traverse */
-    0,		                   /*tp_clear */
-    0,		                   /*tp_richcompare */
-    0,		                   /*tp_weaklistoffset */
-    0,		                   /*tp_iter */
-    0,		                   /*tp_iternext */
-    0,                         /*tp_methods */
-    0,                         /*tp_members */
-    0,                         /*tp_getset */
-    0,                         /*tp_base */
-    0,                         /*tp_dict */
-    0,                         /*tp_descr_get */
-    0,                         /*tp_descr_set */
-    0,                         /*tp_dictoffset */
-    0,                         /*tp_init */
-    0,                         /*tp_alloc */
-    0,                         /*tp_new */
-};
-
 typedef struct {
 	PgfLinFuncs* funcs;
 	GuBuf* stack;
@@ -2726,6 +2769,11 @@ static PyMethodDef Concr_methods[] = {
     },
     {"complete", (PyCFunction)Concr_complete, METH_VARARGS | METH_KEYWORDS,
      "Parses a partial string and returns a list with the top n possible next tokens"
+     "Named arguments:\n"
+     "- sentence (string or a (string,pgf.BIND) tuple. The later indicates that the sentence ends with a BIND token)\n"
+     "- cat (string); OPTIONAL, default: the startcat of the grammar\n"
+     "- prefix (string); OPTIONAL, the prefix of predicted tokens"
+     "- n (int), max. number of predicted tokens"
     },
     {"parseval", (PyCFunction)Concr_parseval, METH_VARARGS,
      "Computes precision, recall and exact match for the parser on a given abstract tree"
@@ -3670,7 +3718,7 @@ MOD_INIT(pgf)
     PyModule_AddObject(m, "Bracket", (PyObject *) &pgf_BracketType);
     Py_INCREF(&pgf_BracketType);
 
-    PyModule_AddObject(m, "BIND", (PyObject *) &pgf_BINDType);
+    PyModule_AddObject(m, "BIND", pgf_BINDType.tp_alloc(&pgf_BINDType, 0));
     Py_INCREF(&pgf_BINDType);
 
 	return MOD_SUCCESS_VAL(m);

src/runtime/python/setup.py

@@ -1,4 +1,4 @@
-from distutils.core import setup, Extension
+from setuptools import setup, Extension
 import os
 
 includes = os.getenv('EXTRA_INCLUDE_DIRS','').split(':')
@@ -16,7 +16,7 @@ pgf_module = Extension('pgf',
                        libraries = ['gu', 'pgf'])
 
 setup (name = 'pgf',
-       version = '1.0',
+       version = '1.1',
        description = 'Python bindings to the Grammatical Framework\'s PGF runtime',
        long_description="""\
 Grammatical Framework (GF) is a programming language for multilingual grammar applications.

src/server/CGIUtils.hs

@@ -34,8 +34,13 @@ stderrToFile :: FilePath -> IO ()
 stderrToFile file =
     do let mode = ownerReadMode<>ownerWriteMode<>groupReadMode<>otherReadMode
            (<>) = unionFileModes
+#if MIN_VERSION_unix(2,8,0)
+           flags = defaultFileFlags { append = True, creat = Just mode }
+       fileFd <- openFd file WriteOnly flags
+#else
            flags = defaultFileFlags { append = True }
        fileFd <- openFd file WriteOnly (Just mode) flags
+#endif
        dupTo fileFd stdError
        return ()
 #else

src/server/PGFService.hs

@@ -159,13 +159,13 @@ cpgfMain qsem command (t,(pgf,pc)) =
                     -> out t=<< bracketedLin # tree % to
     "c-linearizeAll"-> out t=<< linAll # tree % to
     "c-translate"   -> withQSem qsem $
-                       out t=<<join(trans # input % cat % to % start % limit%treeopts)
+                       out t=<<join(trans # input % cat % to % start % limit % treeopts)
     "c-lookupmorpho"-> out t=<< morpho # from1 % textInput
     "c-lookupcohorts"->out t=<< cohorts # from1 % getInput "filter" % textInput
     "c-flush"       -> out t=<< flush
     "c-grammar"     -> out t grammar
     "c-abstrtree"   -> outputGraphviz=<< C.graphvizAbstractTree pgf C.graphvizDefaults # tree
-    "c-parsetree"   -> outputGraphviz=<< (\cnc -> C.graphvizParseTree cnc C.graphvizDefaults) . snd # from1 %tree
+    "c-parsetree"   -> outputGraphviz=<< (\cnc -> C.graphvizParseTree cnc C.graphvizDefaults) . snd # from1 % tree
     "c-wordforword" -> out t =<< wordforword # input % cat % to
     _               -> badRequest "Unknown command" command
   where
@@ -448,7 +448,7 @@ pgfMain lcs@(alc,clc) path command tpgf@(t,pgf) =
       "linearizeTable" -> o =<< doLinearizeTabular pgf # tree % to
       "random"         -> o =<< join (doRandom pgf # cat % depth % limit % to)
       "generate"       -> o =<< doGenerate pgf # cat % depth % limit % to
-      "translate"      -> o =<< doTranslate pgf # input % cat %to%limit%treeopts
+      "translate"      -> o =<< doTranslate pgf # input % cat % to % limit % treeopts
       "translategroup" -> o =<< doTranslateGroup pgf # input % cat % to % limit
       "lookupmorpho"   -> o =<< doLookupMorpho pgf # from1 % textInput
       "grammar"        -> join $ doGrammar tpgf
@@ -571,6 +571,8 @@ limit, depth :: CGI (Maybe Int)
 limit = readInput "limit"
 depth = readInput "depth"
 
+default_depth_server = 4
+
 start :: CGI Int
 start = maybe 0 id # readInput "start"
 
@@ -781,7 +783,7 @@ doRandom pgf mcat mdepth mlimit to =
              | tree <- limit trees]
   where cat = fromMaybe (PGF.startCat pgf) mcat
         limit = take (fromMaybe 1 mlimit)
-        depth = fromMaybe 4 mdepth
+        depth = fromMaybe default_depth_server mdepth
 
 doGenerate :: PGF -> Maybe PGF.Type -> Maybe Int -> Maybe Int -> To -> JSValue
 doGenerate pgf mcat mdepth mlimit tos =
@@ -794,7 +796,7 @@ doGenerate pgf mcat mdepth mlimit tos =
     trees = PGF.generateAllDepth pgf cat (Just depth)
     cat = fromMaybe (PGF.startCat pgf) mcat
     limit = take (fromMaybe 1 mlimit)
-    depth = fromMaybe 4 mdepth
+    depth = fromMaybe default_depth_server mdepth
 
 doGrammar :: (UTCTime,PGF) -> Either IOError (UTCTime,l) -> Maybe (Accept Language) -> CGI CGIResult
 doGrammar (t1,pgf) elbls macc = out t $ showJSON $ makeObj
@@ -1092,7 +1094,7 @@ linearizeTabular pgf (tos,unlex) tree =
     [(to,lintab to (transfer to tree)) | to <- langs]
   where
     langs = if null tos then PGF.languages pgf else tos
-    lintab to t = [(p,map unlex (nub [t|(p',t)<-vs,p'==p]))|p<-ps]
+    lintab to t = [(p,map unlex (nub [t | (p',t)<-vs,p'==p])) | p<-ps]
       where
         ps = nub (map fst vs)
         vs = concat (PGF.tabularLinearizes pgf to t)

src/tools/c/GFCC/ErrM.hs

@@ -1,3 +1,5 @@
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE DeriveFunctor #-}
 -- BNF Converter: Error Monad
 -- Copyright (C) 2004  Author:  Aarne Ranta
 
@@ -6,12 +8,17 @@ module GFCC.ErrM where
 
 -- Control.Monad.Fail import will become redundant in GHC 8.8+
 import qualified Control.Monad.Fail as Fail
+import Control.Monad (ap)
 
 
 -- the Error monad: like Maybe type with error msgs
 
 data Err a = Ok a | Bad String
-  deriving (Read, Show, Eq)
+  deriving (Read, Show, Eq, Functor)
+
+instance Applicative Err where
+  pure = Ok
+  (<*>) = ap
 
 instance Monad Err where
   return      = Ok