Merge branch 'master' into majestic

.github/workflows/build-all-versions.yml

@@ -18,7 +18,7 @@ jobs:
         ghc:
           - "8.6.5"
           - "8.8.3"
-          - "8.10.1"
+          - "8.10.7"
         exclude:
           - os: macos-latest
             ghc: 8.8.3
@@ -33,7 +33,7 @@ jobs:
     - uses: actions/checkout@v2
       if: github.event.action == 'opened' || github.event.action == 'synchronize' || github.event.ref == 'refs/heads/master'
 
-    - uses: haskell/actions/setup@v1
+    - uses: haskell/actions/setup@v1.2.9
       id: setup-haskell-cabal
       name: Setup Haskell
       with:
@@ -66,25 +66,32 @@ jobs:
     strategy:
       matrix:
         stack: ["latest"]
-        ghc: ["7.10.3","8.0.2", "8.2.2", "8.4.4", "8.6.5", "8.8.4"]
+        ghc: ["7.10.3","8.0.2", "8.2.2", "8.4.4", "8.6.5", "8.8.4", "8.10.7", "9.0.2"]
         # ghc: ["8.8.3"]
 
     steps:
     - uses: actions/checkout@v2
       if: github.event.action == 'opened' || github.event.action == 'synchronize' || github.event.ref == 'refs/heads/master'
 
-    - uses: haskell/actions/setup@v1
+    - uses: haskell/actions/setup@v1.2.9
       name: Setup Haskell Stack
       with:
         ghc-version: ${{ matrix.ghc }}
         stack-version: 'latest'
         enable-stack: true
 
+
+    # Fix linker errrors on ghc-7.10.3 for ubuntu (see https://github.com/commercialhaskell/stack/blob/255cd830627870cdef34b5e54d670ef07882523e/doc/faq.md#i-get-strange-ld-errors-about-recompiling-with--fpic)
+    - run: sed -i.bak 's/"C compiler link flags", "/&-no-pie /' /home/runner/.ghcup/ghc/7.10.3/lib/ghc-7.10.3/settings
+      if: matrix.ghc == '7.10.3'
+
     - uses: actions/cache@v1
       name: Cache ~/.stack
       with:
         path: ~/.stack
-        key: ${{ runner.os }}-${{ matrix.ghc }}-stack
+        key: ${{ runner.os }}-${{ matrix.ghc }}-stack--${{ hashFiles(format('stack-ghc{0}', matrix.ghc)) }}
+        restore-keys: |
+             ${{ runner.os }}-${{ matrix.ghc }}-stack
 
     - name: Build
       run: |

gf.cabal

@@ -8,12 +8,17 @@ license-file: LICENSE
 category: Natural Language Processing, Compiler
 synopsis: Grammatical Framework
 description: GF, Grammatical Framework, is a programming language for multilingual grammar applications
+maintainer: John J. Camilleri <john@digitalgrammars.com>
 homepage: https://www.grammaticalframework.org/
 bug-reports: https://github.com/GrammaticalFramework/gf-core/issues
-tested-with: GHC==7.10.3, GHC==8.0.2, GHC==8.10.4
+tested-with: GHC==7.10.3, GHC==8.0.2, GHC==8.10.4, GHC==9.0.2
 
 data-dir: src
-extra-source-files: WebSetup.hs
+extra-source-files:
+  README.md
+  CHANGELOG.md
+  WebSetup.hs
+  doc/Logos/gf0.png
 data-files:
   www/*.html
   www/*.css
@@ -41,7 +46,7 @@ data-files:
 
 custom-setup
   setup-depends:
-    base >= 4.9.1 && < 4.15,
+    base >= 4.9.1 && < 4.16,
     Cabal >= 1.22.0.0,
     directory >= 1.3.0 && < 1.4,
     filepath >= 1.4.1 && < 1.5,
@@ -81,7 +86,6 @@ executable gf
                  filepath, directory>=1.2, time,
                  process, haskeline, parallel>=3, json
   ghc-options: -threaded
-
   other-modules:
     GF
     GF.Support
@@ -216,14 +220,14 @@ executable gf
     build-depends:
       cgi >= 3001.3.0.2 && < 3001.6,
       httpd-shed >= 0.4.0 && < 0.5,
-      network>=2.3 && <2.7
+      network>=2.3 && <3.2
     if flag(network-uri)
       build-depends:
         network-uri >= 2.6.1.0 && < 2.7,
-        network>=2.6 && <2.7
+        network>=2.6 && <3.2
     else
       build-depends:
-        network >= 2.5 && <2.6
+        network >= 2.5 && <3.2
 
     cpp-options: -DSERVER_MODE
     other-modules:
@@ -252,7 +256,7 @@ test-suite gf-tests
   main-is: run.hs
   hs-source-dirs: testsuite
   build-depends:
-    base >= 4.9.1 && < 4.15,
+    base >= 4.9.1 && < 4.16,
     Cabal >= 1.8,
     directory >= 1.3.0 && < 1.4,
     filepath >= 1.4.1 && < 1.5,

src/compiler/Data/Binary/Get.hs

@@ -1,4 +1,6 @@
 {-# LANGUAGE CPP, MagicHash #-}
+-- This module makes profiling a lot slower, so don't add automatic cost centres 
+{-# OPTIONS_GHC -fno-prof-auto #-}
 -- for unboxed shifts
 
 -----------------------------------------------------------------------------

src/compiler/GF/Main.hs

@@ -13,6 +13,7 @@ import Data.Version
 import System.Directory
 import System.Environment (getArgs)
 import System.Exit
+import GHC.IO.Encoding
 -- import GF.System.Console (setConsoleEncoding)
 
 -- | Run the GF main program, taking arguments from the command line.
@@ -20,6 +21,7 @@ import System.Exit
 -- Run @gf --help@ for usage info.
 main :: IO ()
 main = do
+  setLocaleEncoding utf8
   -- setConsoleEncoding
   uncurry mainOpts =<< getOptions
 

src/compiler/GF/Text/Coding.hs

@@ -38,7 +38,7 @@ decodeUnicode :: TextEncoding -> ByteString -> String
 decodeUnicode enc bs = unsafePerformIO $ decodeUnicodeIO enc bs
 
 decodeUnicodeIO enc (PS fptr l len) = do
-    let bbuf = Buffer{bufRaw=fptr, bufState=ReadBuffer, bufSize=len, bufL=l, bufR=l+len}
+    let bbuf = (emptyBuffer fptr len ReadBuffer) { bufL=l, bufR=l+len }
     cbuf <- newCharBuffer 128 WriteBuffer
     case enc of
       TextEncoding {mkTextDecoder=mk} -> do decoder <- mk

src/runtime/python/README.org

@@ -0,0 +1,152 @@
+* INSTALL
+
+You will need the python-devel package or similar.
+
+You must have installed the PGF C runtime (see ../c/INSTALL)
+
+#+begin_src sh
+  $ python setup.py build
+  $ sudo python setup.py install
+#+end_src
+
+* Apple Silicon
+
+The following install instructions were written with the following config in mind:
+
+| OS                    | Hardware | GF                       |
+|-----------------------+----------+--------------------------+
+| MacOS Monterey 12.2.1 | Apple M1 | 3.11 from binary package |
+
+We assume that you may have installed GF as a binary package downloaded from Github.
+
+From that starting point, try all the solutions below, in sequence, until you achieve success.
+
+** Validation Goal
+
+Our goal is to be able to
+- run python 3
+- import pgf
+- type "pgf."
+- hit tab
+
+and get this:
+
+#+begin_example
+  >>> import pgf
+  >>> pgf.
+  pgf.BIND(        pgf.Concr(       pgf.Iter(        pgf.PGFError(    pgf.Type(        pgf.readExpr(    pgf.readType(
+  pgf.Bracket(     pgf.Expr(        pgf.PGF(         pgf.ParseError(  pgf.TypeError(   pgf.readPGF(
+#+end_example
+
+When that works, we can consider the Python PGF bindings to be installed successfully.
+
+** The GF binary package won't install
+
+We assume you've tried [[https://github.com/GrammaticalFramework/gf-core/releases][downloading a binary package]].
+
+If MacOS is being secure, go to System Preferences, Security & Privacy, General, and click Open Anyway.
+
+** gu/mem.h file not found
+
+Maybe you tried running something like ~pip install pgf~ or ~pip3 install pgf~.
+
+Did you get this error?
+
+#+begin_example
+  python3 setup.py build
+  running build
+  running build_ext
+  creating build/temp.macosx-12-arm64-3.9
+  clang -Wno-unused-result -Wsign-compare -Wunreachable-code -fno-common -dynamic -DNDEBUG -g -fwrapv -O3 -Wall -isysroot /Library/Developer/CommandLineTools/SDKs/MacOSX12.sdk -I/opt/homebrew/opt/python@3.9/Frameworks/Python.framework/Versions/3.9/include/python3.9 -c pypgf.c -o build/temp.macosx-12-arm64-3.9/pypgf.o -std=c99
+  pypgf.c:5:10: fatal error: 'gu/mem.h' file not found
+  #include <gu/mem.h>
+           ^~~~~~~~~~
+  1 error generated.
+  error: command '/usr/bin/clang' failed with exit code 1
+#+end_example
+
+Solution:
+
+#+begin_example
+  $ EXTRA_INCLUDE_DIRS=/usr/local/include EXTRA_LIB_DIRS=/usr/local/lib pip install pgf
+#+end_example
+
+This should tell the build where to find the include and lib files it needs to compile:
+
+#+begin_example
+  $ ls /usr/local/include/gu
+  assert.h  choice.h  enum.h    file.h    hash.h    map.h     out.h     seq.h     sysdeps.h utf8.h
+  bits.h    defs.h    exn.h     fun.h     in.h      mem.h     prime.h   string.h  ucs.h     variant.h
+
+  $ ls /usr/local/lib/libgu*
+  /usr/local/lib/libgu.0.dylib /usr/local/lib/libgu.a       /usr/local/lib/libgu.dylib   /usr/local/lib/libgu.la
+#+end_example
+
+If those files don't exist, or you get the following error, you will need to rebuild the C runtime.
+
+** symbol not found in flat namespace
+
+Did you get this error?
+
+#+begin_example
+  Python 3.9.10 (main, Jan 15 2022, 11:40:53)
+  [Clang 13.0.0 (clang-1300.0.29.3)] on darwin
+  Type "help", "copyright", "credits" or "license" for more information.
+  >>> import pgf
+  Traceback (most recent call last):
+    File "<stdin>", line 1, in <module>
+  ImportError: dlopen(/opt/homebrew/lib/python3.9/site-packages/pgf.cpython-39-darwin.so, 0x0002): symbol not found in flat namespace '_gu_alloc_variant'
+#+end_example
+
+This may be a sign that you're trying to get binaries and libraries compiled with different compilers to play nicely. We're trying to get three things to align:
+
+- the Python interpreter
+- the C runtime libraries
+- the Python pgf libraries
+
+Solution:
+
+Maybe your Python isn't the Apple-provided Python. In the above error message we see ~python3~ is provided by Homebrew. Assuming you prefer to keep things this way, we'll try to rebuild things to match your Python.
+
+Rebuilding needs a C compiler. The Apple-provided system ~clang~ is preferred. If you have multiple ~clang~ compilers installed, try disabling the others. For example, if your ~clang~ was provided by nix, run ~nix-env --uninstall clang~. Similarly for brew.
+
+Then try rebuilding the C runtime.
+
+** How to re-build the C runtime
+
+Maybe the C runtime is missing from ~/usr/local/lib~, or maybe the version you have installed is causing the "symbol not found" error.
+
+Build the C runtime by following the instructions in ~gf-core/src/runtime/c/INSTALL~.
+
+After a successful ~make install~, rebuild the Python bindings.
+
+** How to re-build the Python bindings using pip
+
+Sometimes a ~pip install pgf~ will decline to recompile, because a cached wheel exists.
+
+To return to a more pristine state,
+
+#+begin_example
+  pip uninstall pgf
+  pip cache remove pgf
+#+end_example
+
+You may need to ~sudo~ some of the above commands.
+
+Then you can repeat
+
+#+begin_example
+  $ EXTRA_INCLUDE_DIRS=/usr/local/include EXTRA_LIB_DIRS=/usr/local/lib pip install pgf
+#+end_example
+
+** How to re-build the Python bindings manually
+
+If the ~pip install pgf~ just isn't working, try building it directly in ~gf-core/src/runtime/python~:
+
+#+begin_example
+  $ python setup.py build
+  $ sudo python setup.py install
+#+end_example
+
+You may need to add the ~EXTRA~ environment prefixes as shown in previous commands.
+

src/www/index.html

@@ -22,7 +22,7 @@
       (bilingual document editor)
   <!--<li><a href="wc.html">Wide Coverage Translation Demo</a>-->
   <li><a href="gfmorpho/">Word inflection with smart paradigms</a>
-  <li><a href="https://cloud.grammaticalframework.org/wordnet">GF WordNet</a> (an online browser and editor for the WordNet lexicon)</li>
+  <li><a href="wordnet/">GF WordNet</a> (an online browser and editor for the WordNet lexicon)</li>
 </ul>
 
 <h2>Documentation</h2>

stack-ghc8.10.4.yaml

@@ -1,14 +0,0 @@
-resolver: lts-18.0 # ghc 8.10.4
-
-extra-deps:
-- network-2.6.3.6
-- httpd-shed-0.4.0.3
-- cgi-3001.5.0.0@sha256:3d1193a328d5f627a021a0ef3927c1ae41dd341e32dba612fed52d0e3a6df056,2990
-- json-0.10@sha256:d9fc6b07ce92b8894825a17d2cf14799856767eb30c8bf55962baa579207d799,3210
-- multipart-0.2.0@sha256:b8770e3ff6089be4dd089a8250894b31287cca671f3d258190a505f9351fa8a9,1084
-
-# flags:
-#   gf:
-#     c-runtime: true
-# extra-lib-dirs:
-#   - /usr/local/lib

stack-ghc8.10.7.yaml

@@ -0,0 +1,12 @@
+resolver: lts-18.27 # ghc 8.10.7
+
+extra-deps:
+- network-2.6.3.6
+- httpd-shed-0.4.0.3
+
+# flags:
+#   gf:
+#     server: true
+#     c-runtime: true
+# extra-lib-dirs:
+#   - /usr/local/lib

stack-ghc9.0.2.yaml

@@ -0,0 +1,14 @@
+resolver: lts-19.6
+
+extra-deps:
+# - network-2.6.3.6
+# - httpd-shed-0.4.0.3
+# - cgi-3001.5.0.0@sha256:3d1193a328d5f627a021a0ef3927c1ae41dd341e32dba612fed52d0e3a6df056,2990
+# - json-0.10@sha256:d9fc6b07ce92b8894825a17d2cf14799856767eb30c8bf55962baa579207d799,3210
+# - multipart-0.2.0@sha256:b8770e3ff6089be4dd089a8250894b31287cca671f3d258190a505f9351fa8a9,1084
+
+# flags:
+#   gf:
+#     c-runtime: true
+# extra-lib-dirs:
+#   - /usr/local/lib

stack.yaml

@@ -1,18 +1,15 @@
-# This default stack file is a copy of stack-ghc8.10.4.yaml
+# This default stack file is a copy of stack-ghc8.10.7.yaml
 # But committing a symlink can be problematic on Windows, so it's a real copy.
 # See: https://github.com/GrammaticalFramework/gf-core/pull/106
-
+resolver: lts-18.27 # ghc 8.10.7
-resolver: lts-18.0 # ghc 8.10.4
 
 extra-deps:
 - network-2.6.3.6
 - httpd-shed-0.4.0.3
-- cgi-3001.5.0.0@sha256:3d1193a328d5f627a021a0ef3927c1ae41dd341e32dba612fed52d0e3a6df056,2990
-- json-0.10@sha256:d9fc6b07ce92b8894825a17d2cf14799856767eb30c8bf55962baa579207d799,3210
-- multipart-0.2.0@sha256:b8770e3ff6089be4dd089a8250894b31287cca671f3d258190a505f9351fa8a9,1084
 
 # flags:
 #   gf:
+#     server: true
 #     c-runtime: true
 # extra-lib-dirs:
 #   - /usr/local/lib