From 56f530abab732e0b6c357c4bcfaa0ce7a39b0380 Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Mon, 30 Jul 2018 11:52:55 +0200 Subject: [PATCH 01/19] Clean up Make.sh and add verbosity flag --- Make.sh | 26 +++++++++++++++----------- README.md | 6 +++++- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Make.sh b/Make.sh index 2f8124103..c5eaa5129 100755 --- a/Make.sh +++ b/Make.sh @@ -8,6 +8,7 @@ modules_api="Try Symbolic" # Defaults (may be overridden by options) gf="gf" dest="" +verbose="false" # Check command line options for arg in "$@"; do @@ -16,6 +17,8 @@ for arg in "$@"; do gf="${arg#*=}"; shift ;; --dest=*) dest="${arg#*=}"; shift ;; + --verbose|-v) + verbose="true"; shift ;; *) echo "Unknown option: ${arg}" ; exit 1 ;; esac done @@ -25,7 +28,7 @@ if [ -z "$dest" ]; then dest="$GF_LIB_PATH" fi if [ -z "$dest" ] && [ -f "../gf-core/DATA_DIR" ]; then - dest=`cat ../gf-core/DATA_DIR` + dest=$(cat ../gf-core/DATA_DIR) if [ -n "$dest" ]; then dest="${dest}/lib"; fi fi if [ -z "$dest" ]; then @@ -47,37 +50,38 @@ mkdir -p "${dist}/present" mkdir -p "${dist}/alltenses" # Build: prelude -echo "Building prelude" +echo "Building [prelude]" ${gfc} --gfo-dir="${dist}"/prelude "${src}"/prelude/*.gf # Gather all language modules for building for mod in $modules_langs; do - for file in "${src}"/*/${mod}???.gf; do - [[ ! -e $file ]] && continue + for file in "${src}"/*/"${mod}"???.gf; do modules="${modules} ${file}" done done for mod in $modules_api; do - for file in "${src}"/api/${mod}???.gf; do - [[ ! -e $file ]] && continue + for file in "${src}"/api/"${mod}"???.gf; do modules="${modules} ${file}" done done +# Redirect stderr if not verbose +if [ $verbose = false ]; then + exec 2> /dev/null +fi + # Build: present -echo "Building present" -# ${gfc} -no-pmcfg --gfo-dir="${dist}"/present -preproc=mkPresent "${modules}" +echo "Building [present]" for module in $modules; do ${gfc} --no-pmcfg --gfo-dir="${dist}"/present -preproc=mkPresent "${module}" done # Build: alltenses -echo "Building alltenses" -# ${gfc} -no-pmcfg --gfo-dir="${dist}"/alltenses "${modules}" +echo "Building [alltenses]" for module in $modules; do ${gfc} --no-pmcfg --gfo-dir="${dist}"/alltenses "${module}" done # Copy echo "Copying to ${dest}" -cp -R ${dist}/* ${dest} +cp -R "${dist}"/* "${dest}" diff --git a/README.md b/README.md index 36d14df2e..36b1cae19 100644 --- a/README.md +++ b/README.md @@ -78,11 +78,15 @@ clean ## Shell script: `Make.sh` -This method is provided as an alternative for those who don't have Haskell installed. Simply run the script to build the entire RGL and install in the default location: +This method is provided as an alternative for those who don't have Haskell installed. +Simply run the script to build the entire RGL and install in the default location: You can pass the following flags: - `--dest=...` to manually specify the install location - `--gf=...` to specify the path to the `gf` executable, if not available on the system path +- `--verbose` or `-v` to show all GF warnings and errors + +This build method tries to build all languages found in the `src` directory, even those which are not considered complete. ## Windows batch file: `Make.bat` From 19020167f7d91988ba18393e592610dd6653b572 Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Mon, 30 Jul 2018 15:58:36 +0200 Subject: [PATCH 02/19] First version of Make.bat, untested but basically complete --- Make.bat | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++--- Make.sh | 18 ++++++---- README.md | 7 +++- 3 files changed, 114 insertions(+), 12 deletions(-) diff --git a/Make.bat b/Make.bat index 63c6a410f..d4b9418f9 100644 --- a/Make.bat +++ b/Make.bat @@ -1,8 +1,101 @@ +@echo off +Setlocal EnableDelayedExpansion + +REM --- REM Non-Haskell RGL build script for Windows machines -@ECHO OFF +REM --- -REM Prelude +REM Modules to compile for each language +set langs=Afr Amh Ara Eus Bul Cat Chi Dan Dut Eng Est Fin Fre Grc Gre Heb Hin Ger Ice Ina Ita Jpn Lat Lav Mlt Mon Nep Nor Nno Pes Pol Por Pnb Ron Rus Snd Spa Swe Tha Tur Urd +set modules_langs=All Symbol Compatibility +set modules_api=Try Symbolic -REM Present +REM Defaults (may be overridden by options) +set gf=gf-default +set dest= +set verbose=false -REM All tenses +REM Check command line options +set arg_gf_next=false +set arg_dest_next=false +for %%i in (%*) do ( + if !arg_gf_next!==true ( + set gf=%%i + set arg_gf_next=false + ) + if !arg_dest_next!==true ( + set dest=%%i + set arg_dest_next=false + ) + if %%i==-v set verbose=true + if %%i==--verbose set verbose=true + if %%i==--gf set arg_gf_next=true + if %%i==--dest set arg_dest_next=true +) + +REM Try to determine install location +if "%dest%"=="" ( + set dest=%GF_LIB_PATH% +) +if "%dest%"=="" ( + REM TODO Look in ../gf-core/DATA=DIR +) +if "%dest%"=="" ( + echo Unable to determine where to install the RGL. Please do one of the following: + echo - Pass the --dest=... flag to this script + echo - Set the GF_LIB_PATH environment variable + REM echo - Compile & install GF from the gf-core repository (must be in same directory as gf-rgl) + exit /b +) + +REM A few more definitions before we get started +set src=src +set dist=dist +set gfc=gf --batch --gf-lib-path=%src% --quiet + +REM Redirect stderr if not verbose +if %verbose%==false ( + set gfc=2>NUL !gfc! +) + +REM Make directories if not present +mkdir %dist%\prelude +mkdir %dist%\present +mkdir %dist%\alltenses + +REM Build: prelude +echo Building [prelude] +for /r %src%\prelude %%m in (*.gf) do ( + %gfc% --gfo-dir=%dist%\prelude %%m +) + +REM Gather all language modules for building +set modules= +for %%l in (%langs%) do ( + for %%m in (%modules_langs%) do ( + for /r %src% %%m in (*%%m%%l.gf) do ( + set modules=!modules! %%m + ) + ) + for %%m in (%modules_api%) do ( + for /r %src%\api %%m in (*%%m%%l.gf) do ( + set modules=!modules! %%m + ) + ) +) + +REM Build: present +echo Building [present] +for %%m in (%modules%) do ( + %gfc% --no-pmcfg --gfo-dir=%dist%\present --preproc=mkPresent %%m +) + +REM Build: alltenses +echo Building [alltenses] +for %%m in (%modules%) do ( + %gfc% --no-pmcfg --gfo-dir=%dist%\alltenses %%m +) + +REM Copy +echo Copying to %dest% +copy %dist% %dest% diff --git a/Make.sh b/Make.sh index c5eaa5129..b5b6512f9 100755 --- a/Make.sh +++ b/Make.sh @@ -1,7 +1,11 @@ #!/bin/sh +# --- # Non-Haskell RGL build script for Unix-based machines +# --- +# Modules to compile for each language +# langs="Afr Amh Ara Eus Bul Cat Chi Dan Dut Eng Est Fin Fre Grc Gre Heb Hin Ger Ice Ina Ita Jpn Lat Lav Mlt Mon Nep Nor Nno Pes Pol Por Pnb Ron Rus Snd Spa Swe Tha Tur Urd" modules_langs="All Symbol Compatibility" modules_api="Try Symbolic" @@ -42,7 +46,12 @@ fi # A few more definitions before we get started src="src" dist="dist" -gfc="${gf} --batch --gf-lib-path=${src} --quiet " +gfc="${gf} --batch --gf-lib-path=${src} --quiet" + +# Redirect stderr if not verbose +if [ $verbose = false ]; then + exec 2> /dev/null +fi # Make directories if not present mkdir -p "${dist}/prelude" @@ -65,15 +74,10 @@ for mod in $modules_api; do done done -# Redirect stderr if not verbose -if [ $verbose = false ]; then - exec 2> /dev/null -fi - # Build: present echo "Building [present]" for module in $modules; do - ${gfc} --no-pmcfg --gfo-dir="${dist}"/present -preproc=mkPresent "${module}" + ${gfc} --no-pmcfg --gfo-dir="${dist}"/present --preproc=mkPresent "${module}" done # Build: alltenses diff --git a/README.md b/README.md index 36b1cae19..b87f08eb3 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,14 @@ This build method tries to build all languages found in the `src` directory, eve ## Windows batch file: `Make.bat` +**This script is still untested.** + This method is provided as an alternative for Windows users who don't have Haskell installed. -**COMING SOON** +It is supposed to be a port of Make.sh and works in largely the same way. +In particular, it accepts the same flags (in the same format) as described above. + +One difference is that the list of languages to be compiled is specified manually in the script in the `langs` variable. ## About this repository From 0df5b29602c7803fa8756b7afa67b4b5f4e10bf3 Mon Sep 17 00:00:00 2001 From: Kristian Kankainen Date: Thu, 2 Aug 2018 10:11:46 +0300 Subject: [PATCH 03/19] corrected flag name (was 'to', now 'dest') --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b87f08eb3..08b51fc5e 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ clean - You can _remove_ languages from the default list with `langs=-...` - `LANG` is a 3-letter language code, e.g. `Eng`, `Swe` etc. - The path to GF installed on your system can be specified via the `gf` flag (default is that the `gf` executable is in the global system path). -- The `to` flag can be used to manually specify where the compiled RGL modules should be copied/installed. This is the same place as `GF_LIB_PATH`. +- The `dest` flag can be used to manually specify where the compiled RGL modules should be copied/installed. This is the same place as `GF_LIB_PATH`. ## Shell script: `Make.sh` From cc8bb0248902eeb05415b9dcd00c8f091028d359 Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Sat, 4 Aug 2018 16:27:20 +0200 Subject: [PATCH 04/19] Remove `pgf` build command --- Make.hs | 20 ++++++++++---------- README.md | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Make.hs b/Make.hs index fba736c30..2fb989a51 100644 --- a/Make.hs +++ b/Make.hs @@ -191,16 +191,16 @@ rglCommands = , RGLCommand "api" False $ gfcp [t,sc] , RGLCommand "compat" False $ gfcp [c] - , RGLCommand "pgf" False $ \modes args bi -> - parallel_ [ - do let dir = getRGLBuildDir bi mode - createDirectoryIfMissing True dir - sequence_ [run_gfc bi ["-s","-make","-name=Lang"++la, - dir ++ "/Lang" ++ la ++ ".gfo"] - | l <- optl langsPGF args, let la = langCode l] - run_gfc bi (["-s","-make","-name=Lang"]++ - ["Lang" ++ langCode l ++ ".pgf"|l <- optl langsPGF args]) - | mode <- modes] + -- , RGLCommand "pgf" False $ \modes args bi -> + -- parallel_ [ + -- do let dir = getRGLBuildDir bi mode + -- createDirectoryIfMissing True dir + -- sequence_ [run_gfc bi ["-s","-make","-name=Lang"++la, + -- dir ++ "/Lang" ++ la ++ ".gfo"] + -- | l <- optl langsPGF args, let la = langCode l] + -- run_gfc bi (["-s","-make","-name=Lang"]++ + -- ["Lang" ++ langCode l ++ ".pgf"|l <- optl langsPGF args]) + -- | mode <- modes] , RGLCommand "parse" False $ \modes args bi -> gfc bi modes (summary parse) (map parse (optl langsParse args)) diff --git a/README.md b/README.md index 08b51fc5e..46977ddfc 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ clean `lang`, `api`, `compat`, -`pgf`, + `parse` (default is `all`) - `MODE` is one of: From 7f5fae2031a7d20754679c0a12f9d19799b647f5 Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Sat, 4 Aug 2018 18:10:37 +0200 Subject: [PATCH 05/19] Add support for building/copying individual modules in Make.hs Particularly useful for ExtraEng, large monolingual dicts, etc. --- Make.hs | 189 ++++++++++++++++++++++++++++++++---------------------- README.md | 21 +++--- 2 files changed, 125 insertions(+), 85 deletions(-) diff --git a/Make.hs b/Make.hs index 2fb989a51..f87a4ffe6 100644 --- a/Make.hs +++ b/Make.hs @@ -1,12 +1,12 @@ -import Data.List (find,intersect,isPrefixOf) +import Data.List (find,intersect,isPrefixOf,isSuffixOf) import Data.Maybe (fromJust,isJust,catMaybes) import System.IO (hPutStrLn,stderr) import System.IO.Error (catchIOError) import System.Exit (ExitCode(..),die) import System.Environment (getArgs,lookupEnv) import System.Process (rawSystem,readProcess) -import System.FilePath ((),(<.>)) -import System.Directory (createDirectoryIfMissing,copyFile,getDirectoryContents,removeDirectoryRecursive) +import System.FilePath ((),(<.>),takeFileName,addExtension,dropExtension) +import System.Directory (createDirectoryIfMissing,copyFile,getDirectoryContents,removeDirectoryRecursive,doesFileExist) import Control.Monad (unless,when) main :: IO () @@ -14,7 +14,7 @@ main = do aargs <- getArgs case aargs of [] -> putStrLn $ "Must specify command, one of: " ++ unwords commands - a:_ | a `notElem` commands -> putStrLn $ "Unknown command: " ++ a ++ " (valid commands: " ++ unwords commands ++ ")" + a:_ | a `notElem` commands -> putStrLn $ "Unknown command: " ++ a ++ ". Valid commands: " ++ unwords commands "build":args -> buildRGL args "copy":args -> copyRGL args "install":args -> buildRGL args >> copyRGL args @@ -25,8 +25,7 @@ main = do -- | Build grammars into dist buildRGL :: [String] -> IO () buildRGL args = do - checkArgs args - let cmds = getRGLCommands args + let cmds = getCommands args let modes = getOptMode args info <- mkInfo mapM_ (\cmd -> cmdAction cmd modes args info) cmds @@ -37,8 +36,15 @@ copyRGL args = do let modes = getOptMode args info <- mkInfo gf_lib_dir <- maybe (die errLocation) return (infoInstallDir info) - copyAll "prelude" (infoBuildDir info "prelude") (gf_lib_dir "prelude") - sequence_ [copyAll (show mode) (getRGLBuildDir info mode) (gf_lib_dir getRGLBuildSubDir mode)|mode<-modes] + let files = getOptModules args + if not (null files) + then do + -- Copy single files + sequence_ [copyOne (flip addExtension "gfo" . dropExtension . takeFileName $ file) (getRGLBuildDir info mode) (gf_lib_dir getRGLBuildSubDir mode)|file<-files, mode<-modes] + else do + -- Copy everything + copyAll "prelude" (infoBuildDir info "prelude") (gf_lib_dir "prelude") + sequence_ [copyAll (show mode) (getRGLBuildDir info mode) (gf_lib_dir getRGLBuildSubDir mode)|mode<-modes] -- | Error message when install location cannot be determined errLocation :: String @@ -49,10 +55,17 @@ errLocation = unlines $ , " - Compile & install GF from the gf-core repository (must be in same directory as gf-rgl)" ] +-- | Copy single file between directories +copyOne :: String -> FilePath -> FilePath -> IO () +copyOne file from to = do + putStrLn $ "Copying [" ++ file ++ "] " ++ to + createDirectoryIfMissing True to + copyFile (from file) (to file) + -- | Copy all files between directories copyAll :: String -> FilePath -> FilePath -> IO () copyAll msg from to = do - putStrLn $ "Installing [" ++ msg ++ "] " ++ to + putStrLn $ "Copying [" ++ msg ++ "] " ++ to createDirectoryIfMissing True to mapM_ (\file -> when (file /= "." && file /= "..") $ copyFile (from file) (to file)) =<< getDirectoryContents from @@ -62,54 +75,6 @@ clean = do info <- mkInfo removeDirectoryRecursive (infoBuildDir info) --- | Flag for specifying languages --- '=' can optionally be followed by '+' or '-' to alter the default languages -lang_flag :: String -lang_flag = "--langs=" - --- | Flag for specifying gf location -gf_flag :: String -gf_flag = "--gf=" - --- | Flag for specifying RGL install location -destination_flag :: String -destination_flag = "--dest=" - --- | Check arguments are valid -checkArgs :: [String] -> IO () -checkArgs args = do - let args' = flip filter args (\arg -> not - ( arg `elem` (map cmdName rglCommands) - || arg `elem` all_modes - || lang_flag `isPrefixOf` arg - || gf_flag `isPrefixOf` arg - || destination_flag `isPrefixOf` arg - )) - unless (null args') $ die $ "Unrecognised flags: " ++ unwords args' - return () - --- | List of languages overriding the definitions below -getOptLangs :: [LangInfo] -> [String] -> [LangInfo] -getOptLangs defaultLangs args = - case [ls | arg <- args, - let (f,ls) = splitAt (length lang_flag) arg, - f==lang_flag] of - ('+':ls):_ -> foldr addLang defaultLangs (seps ls) - ('-':ls):_ -> foldr removeLang defaultLangs (seps ls) - ls:_ -> findLangs langs (seps ls) - _ -> defaultLangs - where - seps = words . map (\c -> if c==',' then ' ' else c) - findLangs langs ls = [lang | lang <- langs, langCode lang `elem` ls] - removeLang l ls = [lang | lang <- ls, langCode lang /= l] - addLang l ls = if null (findLangs ls [l]) - then findLangs langs [l]++ls - else ls - --- | Get flag value from list of args -getFlag :: String -> [String] -> Maybe String -getFlag flag args = fmap (drop (length flag)) $ find (isPrefixOf flag) args - ------------------------------------------------------------------------------- -- Paths and directories @@ -122,7 +87,7 @@ data Info = Info { infoBuildDir :: FilePath -- ^ where to put built RGL modules (fixed) , infoInstallDir :: Maybe FilePath -- ^ install directory (found dynamically) , infoGFPath :: FilePath -- ^ path to GF - } + } deriving (Show) -- | Build info object from command line args mkInfo :: IO Info @@ -178,19 +143,33 @@ data RGLCommand = RGLCommand -- | Possible build commands rglCommands :: [RGLCommand] rglCommands = - [ RGLCommand "prelude" True $ \mode args bi -> do - putStrLn $ "Compiling [prelude]" - let prelude_src_dir = sourceDir "prelude" - prelude_dst_dir = infoBuildDir bi "prelude" - createDirectoryIfMissing True prelude_dst_dir - files <- getDirectoryContents prelude_src_dir - run_gfc bi (["-s", "--gfo-dir="++prelude_dst_dir] ++ [prelude_src_dir file | file <- files, file /= "." && file /= ".."]) + [ RGLCommand "prelude" True $ \modes args bi -> do + putStrLn $ "Building [prelude]" + let prelude_src_dir = sourceDir "prelude" + prelude_dst_dir = infoBuildDir bi "prelude" + createDirectoryIfMissing True prelude_dst_dir + files <- getDirectoryContents prelude_src_dir + run_gfc bi (["-s", "--gfo-dir="++prelude_dst_dir] ++ [prelude_src_dir file | file <- files, file /= "." && file /= ".."]) , RGLCommand "all" True $ gfcp [l,s,c,t,sc] , RGLCommand "lang" False $ gfcp [l,s] , RGLCommand "api" False $ gfcp [t,sc] , RGLCommand "compat" False $ gfcp [c] + , RGLCommand "modules" False $ \modes args bi -> do + let modules = getOptModules args + flip mapM_ modules $ \m -> do + -- TODO search for modules so that folder name is not needed + -- TODO determine dependants, e.g. include ExtraEngAbs when specifying ExtraEng + let mfull = sourceDir m + ex <- doesFileExist mfull + if not ex + then die $ "Cannot find module: " ++ m + else flip mapM_ modes $ \mode -> do + let dst = getRGLBuildDir bi mode + putStrLn $ "Building [" ++ m ++ "] " ++ dst + run_gfc bi ["-s", "--gfo-dir="++dst, mfull] + -- , RGLCommand "pgf" False $ \modes args bi -> -- parallel_ [ -- do let dir = getRGLBuildDir bi mode @@ -232,6 +211,36 @@ rglCommands = Present -> intersect langsPresent _ -> id +-- | Get commands from args +-- Fails on command error +getCommands :: [String] -> [RGLCommand] +getCommands args = + let + cmdModules = head $ filter (\cmd -> cmdName cmd == "modules") rglCommands + cmds0 = + [ cmd + | arg <- args + , cmd <- rglCommands + , cmdName cmd == arg + , cmdName cmd `notElem` all_modes + ] ++ (if not (null (getOptModules args)) then [cmdModules] else []) + in if null cmds0 + then [cmd | cmd <- rglCommands, cmdIsDef cmd] + else cmds0 + +-- -- | Check arguments are valid +-- checkArgs :: [String] -> IO () +-- checkArgs args = do +-- let args' = flip filter args (\arg -> not +-- ( arg `elem` (map cmdName rglCommands) +-- || arg `elem` all_modes +-- || lang_flag `isPrefixOf` arg +-- || gf_flag `isPrefixOf` arg +-- || destination_flag `isPrefixOf` arg +-- )) +-- unless (null args') $ die $ "Unrecognised flags: " ++ unwords args' +-- return () + -- | Get mode from args (may be missing) getOptMode :: [String] -> [Mode] getOptMode args = @@ -244,15 +253,45 @@ getOptMode args = [AllTenses|have "alltenses"] have mode = mode `elem` args --- | Get RGL command from args -getRGLCommands :: [String] -> [RGLCommand] -getRGLCommands args = - let cmds0 = [cmd | arg <- args, - cmd <- rglCommands, - cmdName cmd == arg] - in if null cmds0 - then [cmd | cmd <- rglCommands, cmdIsDef cmd] - else cmds0 +-- | List of languages overriding the definitions below +getOptLangs :: [LangInfo] -> [String] -> [LangInfo] +getOptLangs defaultLangs args = + case [ls | arg <- args, + let (f,ls) = splitAt (length lang_flag) arg, + f==lang_flag] of + ('+':ls):_ -> foldr addLang defaultLangs (seps ls) + ('-':ls):_ -> foldr removeLang defaultLangs (seps ls) + ls:_ -> findLangs langs (seps ls) + _ -> defaultLangs + where + seps = words . map (\c -> if c==',' then ' ' else c) + findLangs langs ls = [lang | lang <- langs, langCode lang `elem` ls] + removeLang l ls = [lang | lang <- ls, langCode lang /= l] + addLang l ls = if null (findLangs ls [l]) + then findLangs langs [l]++ls + else ls + +-- | Get module names from arguments +-- TODO check if they exist +getOptModules :: [String] -> [FilePath] +getOptModules = filter (isSuffixOf ".gf") + +-- | Flag for specifying languages +-- '=' can optionally be followed by '+' or '-' to alter the default languages +lang_flag :: String +lang_flag = "--langs=" + +-- | Flag for specifying gf location +gf_flag :: String +gf_flag = "--gf=" + +-- | Flag for specifying RGL install location +destination_flag :: String +destination_flag = "--dest=" + +-- | Get flag value from list of args +getFlag :: String -> [String] -> Maybe String +getFlag flag args = fmap (drop (length flag)) $ find (isPrefixOf flag) args ------------------------------------------------------------------------------- -- Languages of the RGL @@ -398,7 +437,7 @@ gfcn bi mode summary files = do AllTenses -> "" Present -> "-preproc=mkPresent" createDirectoryIfMissing True dir - putStrLn $ "Compiling [" ++ show mode ++ "] " ++ summary + putStrLn $ "Building [" ++ show mode ++ "] " ++ summary run_gfc bi (["-s", "-no-pmcfg", preproc, "--gfo-dir="++dir] ++ files) -- | Runs the gf executable in compile mode with the given arguments diff --git a/README.md b/README.md index 46977ddfc..b910ed283 100644 --- a/README.md +++ b/README.md @@ -50,25 +50,26 @@ runghc Make.hs ... Where `...` is one of: ``` -build [CMD] [MODE] [--langs=[+|-]LANG,LANG,...] [--gf=...] -copy [--dest=...] -install [CMD] [MODE] [--langs=[+|-]LANG,LANG,...] [--gf=...] [--dest=...] +build [CMDS] [MODE] [--langs=[+|-]LANG,LANG,...] [--gf=...] +copy [FILES] [MODE] [--dest=...] +install [CMDS] [MODE] [--langs=[+|-]LANG,LANG,...] [--gf=...] [--dest=...] clean ``` -- `CMD` is one of: +- `CMDS` is one or more of: `prelude`, -`all`, +`all` (default), `lang`, `api`, `compat`, -`parse` -(default is `all`) +`parse`, +or an explicit module name (e.g. `english/ExtraEng.gf`). - `MODE` is one of: `present`, `alltenses` -(default is both) +(default is both). +- `FILES` is a space separated list of module names (e.g. `english/ExtraEng.gf german/DictGer.gf`) - You can _override_ the default language list with `--langs=...` - You can _add_ languages to the default list with `--langs=+...` - You can _remove_ languages from the default list with `langs=-...` @@ -79,7 +80,7 @@ clean ## Shell script: `Make.sh` This method is provided as an alternative for those who don't have Haskell installed. -Simply run the script to build the entire RGL and install in the default location: +Simply run the script to build the entire RGL and install in the default location. You can pass the following flags: - `--dest=...` to manually specify the install location @@ -104,7 +105,7 @@ One difference is that the list of languages to be compiled is specified manuall On 2018-07-25, the monolithic [GF repository](https://github.com/GrammaticalFramework/GF) was split in two: -1. [gf-core](https://github.com/GrammaticalFramework/gf-core) — the GF compiler, shell and runtimes +1. [gf-core](https://github.com/GrammaticalFramework/gf-core) — the GF compiler, shell and runtimes 2. [gf-rgl](https://github.com/GrammaticalFramework/gf-rgl) — the resource grammar library The former repository is now archived and no longer updated. From 860011226498800282f2137ab36d603e40ff98dd Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Sat, 4 Aug 2018 18:40:40 +0200 Subject: [PATCH 06/19] Search for module by name and do not require language folder --- Make.hs | 31 ++++++++++++++++++------------- README.md | 4 ++-- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Make.hs b/Make.hs index f87a4ffe6..71c0c8428 100644 --- a/Make.hs +++ b/Make.hs @@ -4,10 +4,10 @@ import System.IO (hPutStrLn,stderr) import System.IO.Error (catchIOError) import System.Exit (ExitCode(..),die) import System.Environment (getArgs,lookupEnv) -import System.Process (rawSystem,readProcess) -import System.FilePath ((),(<.>),takeFileName,addExtension,dropExtension) -import System.Directory (createDirectoryIfMissing,copyFile,getDirectoryContents,removeDirectoryRecursive,doesFileExist) -import Control.Monad (unless,when) +import System.Process (rawSystem) +import System.FilePath ((),takeFileName,addExtension,dropExtension) +import System.Directory (createDirectoryIfMissing,copyFile,getDirectoryContents,removeDirectoryRecursive,findFile) +import Control.Monad (when) main :: IO () main = do @@ -159,16 +159,14 @@ rglCommands = , RGLCommand "modules" False $ \modes args bi -> do let modules = getOptModules args flip mapM_ modules $ \m -> do - -- TODO search for modules so that folder name is not needed -- TODO determine dependants, e.g. include ExtraEngAbs when specifying ExtraEng - let mfull = sourceDir m - ex <- doesFileExist mfull - if not ex - then die $ "Cannot find module: " ++ m - else flip mapM_ modes $ \mode -> do - let dst = getRGLBuildDir bi mode - putStrLn $ "Building [" ++ m ++ "] " ++ dst - run_gfc bi ["-s", "--gfo-dir="++dst, mfull] + mex <- findModule m + case mex of + Nothing -> die $ "Cannot find module: " ++ m + Just mfull -> flip mapM_ modes $ \mode -> do + let dst = getRGLBuildDir bi mode + putStrLn $ "Building [" ++ m ++ "] " ++ dst + run_gfc bi ["-s", "--gfo-dir="++dst, mfull] -- , RGLCommand "pgf" False $ \modes args bi -> -- parallel_ [ @@ -211,6 +209,13 @@ rglCommands = Present -> intersect langsPresent _ -> id +-- | Search all language dirs for module name +findModule :: String -> IO (Maybe FilePath) +findModule file = do + let langdirs = map langDir langs + let searchdirs = map (() sourceDir) langdirs + findFile searchdirs file + -- | Get commands from args -- Fails on command error getCommands :: [String] -> [RGLCommand] diff --git a/README.md b/README.md index b910ed283..5d45def2c 100644 --- a/README.md +++ b/README.md @@ -64,12 +64,12 @@ clean `compat`, `parse`, -or an explicit module name (e.g. `english/ExtraEng.gf`). +or an explicit module name (e.g. `ExtraEng.gf`). - `MODE` is one of: `present`, `alltenses` (default is both). -- `FILES` is a space separated list of module names (e.g. `english/ExtraEng.gf german/DictGer.gf`) +- `FILES` is a space separated list of module names (e.g. `ExtraEng.gf DictGer.gf`) - You can _override_ the default language list with `--langs=...` - You can _add_ languages to the default list with `--langs=+...` - You can _remove_ languages from the default list with `langs=-...` From 59672ef8ac08fdf6c6345b5b46a74cec0d57a25a Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Sat, 4 Aug 2018 18:44:16 +0200 Subject: [PATCH 07/19] Re-add argument checking in Make.hs --- Make.hs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Make.hs b/Make.hs index 71c0c8428..5a255c51b 100644 --- a/Make.hs +++ b/Make.hs @@ -1,4 +1,4 @@ -import Data.List (find,intersect,isPrefixOf,isSuffixOf) +import Data.List (find,intersect,isPrefixOf,isSuffixOf,(\\)) import Data.Maybe (fromJust,isJust,catMaybes) import System.IO (hPutStrLn,stderr) import System.IO.Error (catchIOError) @@ -7,7 +7,7 @@ import System.Environment (getArgs,lookupEnv) import System.Process (rawSystem) import System.FilePath ((),takeFileName,addExtension,dropExtension) import System.Directory (createDirectoryIfMissing,copyFile,getDirectoryContents,removeDirectoryRecursive,findFile) -import Control.Monad (when) +import Control.Monad (when,unless) main :: IO () main = do @@ -25,6 +25,7 @@ main = do -- | Build grammars into dist buildRGL :: [String] -> IO () buildRGL args = do + checkArgs args let cmds = getCommands args let modes = getOptMode args info <- mkInfo @@ -216,8 +217,21 @@ findModule file = do let searchdirs = map (() sourceDir) langdirs findFile searchdirs file +-- | Check arguments are valid, failing on error +checkArgs :: [String] -> IO () +checkArgs args = do + let args'' = args \\ (getOptModules args) + let args' = flip filter args'' (\arg -> not + ( arg `elem` (map cmdName rglCommands) + || arg `elem` all_modes + || lang_flag `isPrefixOf` arg + || gf_flag `isPrefixOf` arg + || destination_flag `isPrefixOf` arg + )) + unless (null args') $ die $ "Unrecognised argument: " ++ unwords args' + return () + -- | Get commands from args --- Fails on command error getCommands :: [String] -> [RGLCommand] getCommands args = let @@ -233,19 +247,6 @@ getCommands args = then [cmd | cmd <- rglCommands, cmdIsDef cmd] else cmds0 --- -- | Check arguments are valid --- checkArgs :: [String] -> IO () --- checkArgs args = do --- let args' = flip filter args (\arg -> not --- ( arg `elem` (map cmdName rglCommands) --- || arg `elem` all_modes --- || lang_flag `isPrefixOf` arg --- || gf_flag `isPrefixOf` arg --- || destination_flag `isPrefixOf` arg --- )) --- unless (null args') $ die $ "Unrecognised flags: " ++ unwords args' --- return () - -- | Get mode from args (may be missing) getOptMode :: [String] -> [Mode] getOptMode args = @@ -277,7 +278,6 @@ getOptLangs defaultLangs args = else ls -- | Get module names from arguments --- TODO check if they exist getOptModules :: [String] -> [FilePath] getOptModules = filter (isSuffixOf ".gf") From e44ab11609c2ff867b1a6b54d1254a9883b557f6 Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Sun, 5 Aug 2018 17:22:46 +0200 Subject: [PATCH 08/19] `copy` copies everything from dist, not just specified modules This was problematic when copying only a module and not its dependents. --- Make.hs | 18 +++++++++--------- README.md | 5 ++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Make.hs b/Make.hs index 5a255c51b..947fda281 100644 --- a/Make.hs +++ b/Make.hs @@ -31,21 +31,21 @@ buildRGL args = do info <- mkInfo mapM_ (\cmd -> cmdAction cmd modes args info) cmds --- | Copy from dist to install location +-- | Copy everything from dist to install location copyRGL :: [String] -> IO () copyRGL args = do let modes = getOptMode args info <- mkInfo gf_lib_dir <- maybe (die errLocation) return (infoInstallDir info) - let files = getOptModules args - if not (null files) - then do - -- Copy single files - sequence_ [copyOne (flip addExtension "gfo" . dropExtension . takeFileName $ file) (getRGLBuildDir info mode) (gf_lib_dir getRGLBuildSubDir mode)|file<-files, mode<-modes] - else do + -- let files = getOptModules args + -- if not (null files) + -- then do + -- -- Copy single files + -- sequence_ [copyOne (flip addExtension "gfo" . dropExtension . takeFileName $ file) (getRGLBuildDir info mode) (gf_lib_dir getRGLBuildSubDir mode)|file<-files, mode<-modes] + -- else do -- Copy everything - copyAll "prelude" (infoBuildDir info "prelude") (gf_lib_dir "prelude") - sequence_ [copyAll (show mode) (getRGLBuildDir info mode) (gf_lib_dir getRGLBuildSubDir mode)|mode<-modes] + copyAll "prelude" (infoBuildDir info "prelude") (gf_lib_dir "prelude") + sequence_ [copyAll (show mode) (getRGLBuildDir info mode) (gf_lib_dir getRGLBuildSubDir mode)|mode<-modes] -- | Error message when install location cannot be determined errLocation :: String diff --git a/README.md b/README.md index 5d45def2c..dd307ab11 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ runghc Make.hs ... Where `...` is one of: ``` build [CMDS] [MODE] [--langs=[+|-]LANG,LANG,...] [--gf=...] -copy [FILES] [MODE] [--dest=...] +copy [MODE] [--dest=...] install [CMDS] [MODE] [--langs=[+|-]LANG,LANG,...] [--gf=...] [--dest=...] clean ``` @@ -64,12 +64,11 @@ clean `compat`, `parse`, -or an explicit module name (e.g. `ExtraEng.gf`). +or an explicit module name (e.g. `ExtraEng.gf`. You don't need to specify to language subdirectory, but there is a restriction that the module must exist in a **direct** subdirectory of `src`). - `MODE` is one of: `present`, `alltenses` (default is both). -- `FILES` is a space separated list of module names (e.g. `ExtraEng.gf DictGer.gf`) - You can _override_ the default language list with `--langs=...` - You can _add_ languages to the default list with `--langs=+...` - You can _remove_ languages from the default list with `langs=-...` From 70297f6ac27ce663a51ea4659cc7c892eced23c1 Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Mon, 6 Aug 2018 10:29:42 +0200 Subject: [PATCH 09/19] Add --verbose/-v switch to Make.hs --- Make.hs | 23 +++++++++++++++++------ README.md | 4 ++-- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/Make.hs b/Make.hs index 947fda281..4b010287e 100644 --- a/Make.hs +++ b/Make.hs @@ -88,6 +88,7 @@ data Info = Info { infoBuildDir :: FilePath -- ^ where to put built RGL modules (fixed) , infoInstallDir :: Maybe FilePath -- ^ install directory (found dynamically) , infoGFPath :: FilePath -- ^ path to GF + , infoVerbose :: Bool } deriving (Show) -- | Build info object from command line args @@ -104,11 +105,13 @@ mkInfo = do case catMaybes [mflag,menvar,mbuilt] of [] -> Nothing p:_ -> Just p + let verbose = verbose_switch `elem` args || verbose_switch_short `elem` args return $ Info { infoBuildDir = "dist" , infoInstallDir = inst_dir , infoGFPath = maybe default_gf id (getFlag gf_flag args) + , infoVerbose = verbose } where default_gf = "gf" @@ -150,7 +153,7 @@ rglCommands = prelude_dst_dir = infoBuildDir bi "prelude" createDirectoryIfMissing True prelude_dst_dir files <- getDirectoryContents prelude_src_dir - run_gfc bi (["-s", "--gfo-dir="++prelude_dst_dir] ++ [prelude_src_dir file | file <- files, file /= "." && file /= ".."]) + run_gfc bi ([if infoVerbose bi then "--verbose" else "--quiet", "--gfo-dir="++prelude_dst_dir] ++ [prelude_src_dir file | file <- files, file /= "." && file /= ".."]) , RGLCommand "all" True $ gfcp [l,s,c,t,sc] , RGLCommand "lang" False $ gfcp [l,s] @@ -167,7 +170,7 @@ rglCommands = Just mfull -> flip mapM_ modes $ \mode -> do let dst = getRGLBuildDir bi mode putStrLn $ "Building [" ++ m ++ "] " ++ dst - run_gfc bi ["-s", "--gfo-dir="++dst, mfull] + run_gfc bi [if infoVerbose bi then "--verbose" else "--quiet", "--gfo-dir="++dst, mfull] -- , RGLCommand "pgf" False $ \modes args bi -> -- parallel_ [ @@ -227,6 +230,7 @@ checkArgs args = do || lang_flag `isPrefixOf` arg || gf_flag `isPrefixOf` arg || destination_flag `isPrefixOf` arg + || arg `elem` [verbose_switch, verbose_switch_short] )) unless (null args') $ die $ "Unrecognised argument: " ++ unwords args' return () @@ -294,6 +298,13 @@ gf_flag = "--gf=" destination_flag :: String destination_flag = "--dest=" +-- | Switch for making verbose +verbose_switch :: String +verbose_switch = "--verbose" + +verbose_switch_short :: String +verbose_switch_short = "-v" + -- | Get flag value from list of args getFlag :: String -> [String] -> Maybe String getFlag flag args = fmap (drop (length flag)) $ find (isPrefixOf flag) args @@ -422,7 +433,7 @@ parse l = sourceDir "parse" ("Parse" ++ langCode l ++ ".gf") -- | Get unlexer flags for languages unlexer :: String -> [LangInfo] -> String unlexer abstr ls = - "-unlexer=\\\"" ++ unwords + "--unlexer=\\\"" ++ unwords [ abstr ++ langCode lang ++ "=" ++ fromJust unl | lang <- ls , let unl = langUnlexer lang @@ -440,16 +451,16 @@ gfcn bi mode summary files = do let dir = getRGLBuildDir bi mode preproc = case mode of AllTenses -> "" - Present -> "-preproc=mkPresent" + Present -> "--preproc=mkPresent" createDirectoryIfMissing True dir putStrLn $ "Building [" ++ show mode ++ "] " ++ summary - run_gfc bi (["-s", "-no-pmcfg", preproc, "--gfo-dir="++dir] ++ files) + run_gfc bi ([if infoVerbose bi then "--verbose" else "--quiet", "--no-pmcfg", preproc, "--gfo-dir="++dir] ++ files) -- | Runs the gf executable in compile mode with the given arguments run_gfc :: Info -> [String] -> IO () run_gfc bi args = do let - args' = ["-batch","-gf-lib-path="++sourceDir] ++ filter (not . null) args + args' = ["--batch","--gf-lib-path="++sourceDir] ++ filter (not . null) args gf = infoGFPath bi execute gf args' diff --git a/README.md b/README.md index dd307ab11..815b86fd6 100644 --- a/README.md +++ b/README.md @@ -50,9 +50,9 @@ runghc Make.hs ... Where `...` is one of: ``` -build [CMDS] [MODE] [--langs=[+|-]LANG,LANG,...] [--gf=...] +build [CMDS] [MODE] [--langs=[+|-]LANG,LANG,...] [--gf=...] [--verbose|-v] copy [MODE] [--dest=...] -install [CMDS] [MODE] [--langs=[+|-]LANG,LANG,...] [--gf=...] [--dest=...] +install [CMDS] [MODE] [--langs=[+|-]LANG,LANG,...] [--gf=...] [--dest=...] [--verbose|-v] clean ``` From 557ae6da4444a6373c34f6c288ea40849555c548 Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Mon, 6 Aug 2018 16:53:23 +0200 Subject: [PATCH 10/19] Factor out language config to CSV file Compiles, not tested --- Make.hs | 189 ++++++++++++++++++++------------------------------ README.md | 18 +++++ languages.csv | 46 ++++++++++++ 3 files changed, 141 insertions(+), 112 deletions(-) create mode 100644 languages.csv diff --git a/Make.hs b/Make.hs index 4b010287e..f2ea0d3e5 100644 --- a/Make.hs +++ b/Make.hs @@ -1,4 +1,4 @@ -import Data.List (find,intersect,isPrefixOf,isSuffixOf,(\\)) +import Data.List (find,intersect,isPrefixOf,isSuffixOf,(\\),unfoldr) import Data.Maybe (fromJust,isJust,catMaybes) import System.IO (hPutStrLn,stderr) import System.IO.Error (catchIOError) @@ -163,7 +163,6 @@ rglCommands = , RGLCommand "modules" False $ \modes args bi -> do let modules = getOptModules args flip mapM_ modules $ \m -> do - -- TODO determine dependants, e.g. include ExtraEngAbs when specifying ExtraEng mex <- findModule m case mex of Nothing -> die $ "Cannot find module: " ++ m @@ -172,50 +171,41 @@ rglCommands = putStrLn $ "Building [" ++ m ++ "] " ++ dst run_gfc bi [if infoVerbose bi then "--verbose" else "--quiet", "--gfo-dir="++dst, mfull] - -- , RGLCommand "pgf" False $ \modes args bi -> - -- parallel_ [ - -- do let dir = getRGLBuildDir bi mode - -- createDirectoryIfMissing True dir - -- sequence_ [run_gfc bi ["-s","-make","-name=Lang"++la, - -- dir ++ "/Lang" ++ la ++ ".gfo"] - -- | l <- optl langsPGF args, let la = langCode l] - -- run_gfc bi (["-s","-make","-name=Lang"]++ - -- ["Lang" ++ langCode l ++ ".pgf"|l <- optl langsPGF args]) - -- | mode <- modes] - - , RGLCommand "parse" False $ \modes args bi -> - gfc bi modes (summary parse) (map parse (optl langsParse args)) + , RGLCommand "parse" False $ \modes args bi -> do + langs <- loadLangs + let mylangs = (optml AllTenses langsParse args) langs + gfc bi modes (summary parse) (map parse mylangs) ] where - gfcp :: [Mode -> [String] -> (LangInfo -> FilePath,[LangInfo])] -> [Mode] -> [String] -> Info -> IO () + gfcp :: [Mode -> [String] -> (LangInfo -> FilePath,[LangInfo] -> [LangInfo])] -> [Mode] -> [String] -> Info -> IO () gfcp cs modes args bi = parallel_ [gfcp' bi mode args cs | mode <- modes] - gfcp' :: Info -> Mode -> [String] -> [Mode -> [String] -> (LangInfo -> FilePath,[LangInfo])] -> IO () - gfcp' bi mode args cs = gfcn bi mode (unwords ss) (concat fss) - where (ss,fss) = unzip [(summary f,map f as)|c<-cs,let (f,as)=c mode args] + gfcp' :: Info -> Mode -> [String] -> [Mode -> [String] -> (LangInfo -> FilePath,[LangInfo] -> [LangInfo])] -> IO () + gfcp' bi mode args cs = do + langs <- loadLangs + let (ss,fss) = unzip [ (summary f,map f (as langs)) | c<-cs, let (f,as) = c mode args] + gfcn bi mode (unwords ss) (concat fss) summary :: (LangInfo -> FilePath) -> FilePath - summary f = f (LangInfo "*" "*" Nothing Nothing) + summary f = f (LangInfo "*" "*" Nothing Nothing False False False False) l mode args = (lang,optml mode langsLang args) s mode args = (symbol,optml mode langsAPI args) - c mode args = (compat,optl langsCompat args) + c mode args = (compat,optml AllTenses langsCompat args) t mode args = (try,optml mode langsAPI args) sc mode args = (symbolic,optml mode langsSymbolic args) - optl :: [LangInfo] -> [String] -> [LangInfo] - optl = optml AllTenses - - optml :: Mode -> [LangInfo] -> [String] -> [LangInfo] - optml mode ls args = getOptLangs (shrink ls) args + optml :: Mode -> ([LangInfo] -> [LangInfo]) -> [String] -> ([LangInfo] -> [LangInfo]) + optml mode ls args = \langs -> getOptLangs langs (shrink langs) args where - shrink = case mode of - Present -> intersect langsPresent - _ -> id + shrink langs = case mode of + Present -> langsPresent langs + _ -> langs -- | Search all language dirs for module name findModule :: String -> IO (Maybe FilePath) findModule file = do + langs <- loadLangs let langdirs = map langDir langs let searchdirs = map (() sourceDir) langdirs findFile searchdirs file @@ -263,9 +253,9 @@ getOptMode args = [AllTenses|have "alltenses"] have mode = mode `elem` args --- | List of languages overriding the definitions below -getOptLangs :: [LangInfo] -> [String] -> [LangInfo] -getOptLangs defaultLangs args = +-- | List of languages overriding the default definitions +getOptLangs :: [LangInfo] -> [LangInfo] -> [String] -> [LangInfo] +getOptLangs langs defaultLangs args = case [ls | arg <- args, let (f,ls) = splitAt (length lang_flag) arg, f==lang_flag] of @@ -318,94 +308,69 @@ data LangInfo = LangInfo , langDir :: String -- ^ directory name , langFunctor :: Maybe String -- ^ functor (not used) , langUnlexer :: Maybe String -- ^ decoding for postprocessing linearizations - } deriving (Eq) + , langPresent :: Bool + , langAPI :: Bool + , langSymbolic :: Bool + , langCompatibility :: Bool + } deriving (Show,Eq) --- | List of all languages known to RGL -langs :: [LangInfo] -langs = - [ LangInfo "Afr" "afrikaans" Nothing Nothing - , LangInfo "Amh" "amharic" Nothing Nothing - , LangInfo "Ara" "arabic" Nothing Nothing - , LangInfo "Eus" "basque" Nothing Nothing - , LangInfo "Bul" "bulgarian" Nothing Nothing - , LangInfo "Cat" "catalan" (Just "Romance") Nothing - , LangInfo "Chi" "chinese" Nothing Nothing - , LangInfo "Dan" "danish" (Just "Scand") Nothing - , LangInfo "Dut" "dutch" Nothing Nothing - , LangInfo "Eng" "english" Nothing Nothing - , LangInfo "Est" "estonian" Nothing Nothing - , LangInfo "Fin" "finnish" Nothing Nothing - , LangInfo "Fre" "french" (Just "Romance") Nothing - , LangInfo "Grc" "ancient_greek" Nothing Nothing - , LangInfo "Gre" "greek" Nothing Nothing - , LangInfo "Heb" "hebrew" Nothing Nothing - , LangInfo "Hin" "hindi" (Just "Hindustani") (Just "to_devanagari") - , LangInfo "Ger" "german" Nothing Nothing - , LangInfo "Ice" "icelandic" Nothing Nothing - , LangInfo "Ina" "interlingua" Nothing Nothing - , LangInfo "Ita" "italian" (Just "Romance") Nothing - , LangInfo "Jpn" "japanese" Nothing Nothing - , LangInfo "Lat" "latin" Nothing Nothing - , LangInfo "Lav" "latvian" Nothing Nothing - , LangInfo "Mlt" "maltese" Nothing Nothing - , LangInfo "Mon" "mongolian" Nothing Nothing - , LangInfo "Nep" "nepali" Nothing Nothing - , LangInfo "Nor" "norwegian" (Just "Scand") Nothing - , LangInfo "Nno" "nynorsk" Nothing Nothing - , LangInfo "Pes" "persian" Nothing Nothing - , LangInfo "Pol" "polish" Nothing Nothing - , LangInfo "Por" "portuguese" (Just "Romance") Nothing - , LangInfo "Pnb" "punjabi" Nothing Nothing - , LangInfo "Ron" "romanian" Nothing Nothing - , LangInfo "Rus" "russian" Nothing Nothing - , LangInfo "Snd" "sindhi" Nothing Nothing - , LangInfo "Spa" "spanish" (Just "Romance") Nothing - , LangInfo "Swe" "swedish" (Just "Scand") Nothing - , LangInfo "Tha" "thai" Nothing (Just "to_thai") - , LangInfo "Tur" "turkish" Nothing Nothing - , LangInfo "Urd" "urdu" (Just "Hindustani") Nothing - ] +-- | Load language information from config file +loadLangs :: IO [LangInfo] +loadLangs = do + lns <- readFile conffile >>= return . lines + mapM mkLangInfo (tail lns) + where + conffile = "languages.csv" + maybeBit bits n = if length bits >= (n+1) && length (bits !! n) > 0 then Just (bits !! n) else Nothing + boolBit bits n def = if length bits >= (n+1) && length (bits !! n) > 0 then (bits !! n == if def then "n" else "y") else def + mkLangInfo s = + let bits = separateBy ',' s in + if length bits < 2 + then die $ "Invalid entry in " ++ conffile ++ ": " ++ s + else return $ LangInfo + { langCode = bits !! 0 + , langDir = bits !! 1 + , langFunctor = maybeBit bits 2 + , langUnlexer = maybeBit bits 3 + , langPresent = boolBit bits 4 False + , langAPI = boolBit bits 5 True + , langSymbolic = boolBit bits 6 True + , langCompatibility = boolBit bits 7 False + } --- | Languagues for which to compile Lang -langsLang :: [LangInfo] -langsLang = langs +-- | Separate a string on a character +-- Source: https://stackoverflow.com/a/4978733/98600 +separateBy :: Eq a => a -> [a] -> [[a]] +separateBy chr = unfoldr sep where + sep [] = Nothing + sep l = Just . fmap (drop 1) . break (== chr) $ l --- | Languages that have notpresent marked -langsPresent :: [LangInfo] -langsPresent = langsLang `except` ["Afr","Chi","Eus","Gre","Heb","Ice","Jpn","Mlt","Mon","Nep","Pes","Snd","Tha","Thb","Est"] - --- | Languages for which to compile Try -langsAPI :: [LangInfo] -langsAPI = langsLang `except` langsIncomplete - --- | Languages which compile but which are incomplete -langsIncomplete :: [String] -langsIncomplete = ["Amh","Ara","Grc","Heb","Ina","Lat","Tur"] - --- | Languages for which to compile Symbolic -langsSymbolic :: [LangInfo] -langsSymbolic = langsAPI `except` ["Afr","Ice","Mon","Nep"] - --- | Languages for which to compile parsing grammars -langsParse :: [LangInfo] -langsParse = langs `only` ["Eng"] - --- | Languages for which langs.pgf is built -langsPGF :: [LangInfo] -langsPGF = langsLang `except` ["Ara","Hin","Ron","Tha"] - --- | Languages for which Compatibility exists (to be extended) -langsCompat :: [LangInfo] -langsCompat = langsLang `only` ["Cat","Eng","Fin","Fre","Ita","Lav","Spa","Swe"] - --- | Exclude langs from list by code -except :: [LangInfo] -> [String] -> [LangInfo] -except ls es = filter (flip notElem es . langCode) ls +-- -- | Exclude langs from list by code +-- exceptLangs :: [LangInfo] -> [String] -> [LangInfo] +-- exceptLangs ls es = filter (flip notElem es . langCode) ls -- | Only specified langs by code only :: [LangInfo] -> [String] -> [LangInfo] only ls es = filter (flip elem es . langCode) ls +langsLang :: [LangInfo] -> [LangInfo] +langsLang = id + +langsPresent :: [LangInfo] -> [LangInfo] +langsPresent = filter langPresent + +langsAPI :: [LangInfo] -> [LangInfo] +langsAPI = filter langAPI + +langsSymbolic :: [LangInfo] -> [LangInfo] +langsSymbolic = filter langSymbolic + +langsCompat :: [LangInfo] -> [LangInfo] +langsCompat = filter langCompatibility + +langsParse :: [LangInfo] -> [LangInfo] +langsParse = flip only ["Eng"] + ------------------------------------------------------------------------------- -- Getting module paths/names diff --git a/README.md b/README.md index 815b86fd6..22728a3d7 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,24 @@ It will look for, in this order: - the file `../gf-core/DATA_DIR` (relative to this directory). This only works if you have the `gf-core` and `gf-rgl` repositories in the same top-level directory **and** you have already compiled GF from source. (This is considered a bit hacky and will probably disappear in the future). +## Language config + +A list of all languages and their properties is maintained centrally in `languages.csv`. +This file should be kept up-to-date and all build methods should read this config file. +If you see something wrong, please report/fix it. + +Description of columns: +- Code +- Directory +- Functor: (not used) +- Unlexer +- Present: languages that have notpresent marked +- API: languages for which to compile Try +- Symbolic: languages for which to compile Symbolic +- Compatibility: languages for which Compatibility exists + +Columns can be a string, just `y`'s (where nothing means `n`) or just (`n`'s where nothing means `y`). + ## Haskell script: `Make.hs` This build method gives you most options. diff --git a/languages.csv b/languages.csv new file mode 100644 index 000000000..84d9c2c68 --- /dev/null +++ b/languages.csv @@ -0,0 +1,46 @@ +Code,Directory,Functor,Unlexer,Present,API,Symbolic,Compatibility +Afr,afrikaans,,,y,,n, +Amh,amharic,,,,n,n, +Ara,arabic,,,,n,n, +Eus,basque,,,,,, +Bul,bulgarian,,,y,,,y +Cat,catalan,Romance,,y,,,y +Chi,chinese,,,y,,, +Dan,danish,Scand,,y,,, +Dut,dutch,,,y,,, +Eng,english,,,y,,,y +Est,estonian,,,y,,, +Fin,finnish,,,y,,,y +Fre,french,Romance,,y,,,y +Grc,ancient_greek,,,y,n,n, +Gre,greek,,,y,,, +Heb,hebrew,,,,n,n, +Hin,hindi,Hindustani,to_devanagari,y,,, +Hun,hungarian,,,y,,, +Ger,german,,,,,, +Ice,icelandic,,,y,,n, +Ina,interlingua,,,y,n,n, +Ita,italian,Romance,,y,,,y +Jpn,japanese,,,y,,, +Lat,latin,,,y,n,n, +Lav,latvian,,,,,,y +Mlt,maltese,,,,,, +Mon,mongolian,,,y,,n, +Nep,nepali,,,y,,n, +Nor,norwegian,Scand,,y,,, +Nno,nynorsk,,,y,,, +Pes,persian,,,y,,, +Pol,polish,,,,,, +Por,portuguese,Romance,,y,,,y +Pnb,punjabi,,,y,,, +Ron,romanian,,,y,,, +Rus,russian,,,y,,, +Snd,sindhi,,,y,,, +Spa,spanish,Romance,,y,,,y +Swa,swahili,,,,,,y +Swe,swedish,Scand,,y,,,y +Tel,telugu,,,y,,, +Tha,thai,,to_thai,y,,, +Tur,turkish,,,,n,n, +Urd,urdu,Hindustani,,y,,, +Vot,votic,,,,,, From bd636d805a844468a3da1c0a85e350cf31d52959 Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Mon, 6 Aug 2018 21:08:46 +0200 Subject: [PATCH 11/19] Simplify specifying languages in Make.hs --- Make.hs | 50 ++++++++++++-------------------------------------- 1 file changed, 12 insertions(+), 38 deletions(-) diff --git a/Make.hs b/Make.hs index f2ea0d3e5..da9b61183 100644 --- a/Make.hs +++ b/Make.hs @@ -129,7 +129,7 @@ getRGLBuildSubDir mode = -- Build modes data Mode = AllTenses | Present - deriving (Show) + deriving (Show,Eq) all_modes :: [String] all_modes = ["alltenses","present"] @@ -173,6 +173,7 @@ rglCommands = , RGLCommand "parse" False $ \modes args bi -> do langs <- loadLangs + let langsParse = \l -> langCode l `elem` ["Eng"] let mylangs = (optml AllTenses langsParse args) langs gfc bi modes (summary parse) (map parse mylangs) ] @@ -189,18 +190,17 @@ rglCommands = summary :: (LangInfo -> FilePath) -> FilePath summary f = f (LangInfo "*" "*" Nothing Nothing False False False False) - l mode args = (lang,optml mode langsLang args) - s mode args = (symbol,optml mode langsAPI args) - c mode args = (compat,optml AllTenses langsCompat args) - t mode args = (try,optml mode langsAPI args) - sc mode args = (symbolic,optml mode langsSymbolic args) + l mode args = (lang,optml mode (const True) args) + s mode args = (symbol,optml mode langAPI args) + c mode args = (compat,optml AllTenses langCompatibility args) + t mode args = (try,optml mode langAPI args) + sc mode args = (symbolic,optml mode langSymbolic args) - optml :: Mode -> ([LangInfo] -> [LangInfo]) -> [String] -> ([LangInfo] -> [LangInfo]) - optml mode ls args = \langs -> getOptLangs langs (shrink langs) args - where - shrink langs = case mode of - Present -> langsPresent langs - _ -> langs + optml :: Mode -> (LangInfo -> Bool) -> [String] -> ([LangInfo] -> [LangInfo]) + optml mode ls args = + \langs -> + let defLangs = filter (if mode == Present then langPresent else const True) (filter ls langs) + in getOptLangs langs defLangs args -- | Search all language dirs for module name findModule :: String -> IO (Maybe FilePath) @@ -345,32 +345,6 @@ separateBy chr = unfoldr sep where sep [] = Nothing sep l = Just . fmap (drop 1) . break (== chr) $ l --- -- | Exclude langs from list by code --- exceptLangs :: [LangInfo] -> [String] -> [LangInfo] --- exceptLangs ls es = filter (flip notElem es . langCode) ls - --- | Only specified langs by code -only :: [LangInfo] -> [String] -> [LangInfo] -only ls es = filter (flip elem es . langCode) ls - -langsLang :: [LangInfo] -> [LangInfo] -langsLang = id - -langsPresent :: [LangInfo] -> [LangInfo] -langsPresent = filter langPresent - -langsAPI :: [LangInfo] -> [LangInfo] -langsAPI = filter langAPI - -langsSymbolic :: [LangInfo] -> [LangInfo] -langsSymbolic = filter langSymbolic - -langsCompat :: [LangInfo] -> [LangInfo] -langsCompat = filter langCompatibility - -langsParse :: [LangInfo] -> [LangInfo] -langsParse = flip only ["Eng"] - ------------------------------------------------------------------------------- -- Getting module paths/names From bad78999dc4181d16bea9622c9527a90ea37ca2a Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Mon, 6 Aug 2018 21:28:29 +0200 Subject: [PATCH 12/19] Make summary more explicit as it was a bit misleading --- Make.hs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Make.hs b/Make.hs index da9b61183..cf8aeaa95 100644 --- a/Make.hs +++ b/Make.hs @@ -172,10 +172,10 @@ rglCommands = run_gfc bi [if infoVerbose bi then "--verbose" else "--quiet", "--gfo-dir="++dst, mfull] , RGLCommand "parse" False $ \modes args bi -> do - langs <- loadLangs + langsAll <- loadLangs let langsParse = \l -> langCode l `elem` ["Eng"] - let mylangs = (optml AllTenses langsParse args) langs - gfc bi modes (summary parse) (map parse mylangs) + let langs = (optml AllTenses langsParse args) langsAll + gfc bi modes (summary parse langs) (map parse langs) ] where gfcp :: [Mode -> [String] -> (LangInfo -> FilePath,[LangInfo] -> [LangInfo])] -> [Mode] -> [String] -> Info -> IO () @@ -183,12 +183,13 @@ rglCommands = gfcp' :: Info -> Mode -> [String] -> [Mode -> [String] -> (LangInfo -> FilePath,[LangInfo] -> [LangInfo])] -> IO () gfcp' bi mode args cs = do - langs <- loadLangs - let (ss,fss) = unzip [ (summary f,map f (as langs)) | c<-cs, let (f,as) = c mode args] + langsAll <- loadLangs + let (ss,fss) = unzip [ (summary f langs,map f langs) | c<-cs, let (f,as) = c mode args, let langs = as langsAll] gfcn bi mode (unwords ss) (concat fss) - summary :: (LangInfo -> FilePath) -> FilePath - summary f = f (LangInfo "*" "*" Nothing Nothing False False False False) + summary :: (LangInfo -> FilePath) -> [LangInfo] -> String + summary f langs = unwords (map (drop (length sourceDir + 1) . f) langs) + -- summary f _ = f (LangInfo "*" "*" Nothing Nothing False False False False) l mode args = (lang,optml mode (const True) args) s mode args = (symbol,optml mode langAPI args) From cfb6bf03015d94805ed2c05c8e27832fcea4d334 Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Mon, 6 Aug 2018 21:55:58 +0200 Subject: [PATCH 13/19] Update language config with "All" column; fix those marked for notpresent Also fix bug in boolBit --- Make.hs | 12 ++++--- README.md | 1 + languages.csv | 91 +++++++++++++++++++++++++-------------------------- 3 files changed, 53 insertions(+), 51 deletions(-) diff --git a/Make.hs b/Make.hs index cf8aeaa95..dd1d06f7c 100644 --- a/Make.hs +++ b/Make.hs @@ -191,7 +191,7 @@ rglCommands = summary f langs = unwords (map (drop (length sourceDir + 1) . f) langs) -- summary f _ = f (LangInfo "*" "*" Nothing Nothing False False False False) - l mode args = (lang,optml mode (const True) args) + l mode args = (lang,optml mode langAll args) s mode args = (symbol,optml mode langAPI args) c mode args = (compat,optml AllTenses langCompatibility args) t mode args = (try,optml mode langAPI args) @@ -310,6 +310,7 @@ data LangInfo = LangInfo , langFunctor :: Maybe String -- ^ functor (not used) , langUnlexer :: Maybe String -- ^ decoding for postprocessing linearizations , langPresent :: Bool + , langAll :: Bool , langAPI :: Bool , langSymbolic :: Bool , langCompatibility :: Bool @@ -323,7 +324,7 @@ loadLangs = do where conffile = "languages.csv" maybeBit bits n = if length bits >= (n+1) && length (bits !! n) > 0 then Just (bits !! n) else Nothing - boolBit bits n def = if length bits >= (n+1) && length (bits !! n) > 0 then (bits !! n == if def then "n" else "y") else def + boolBit bits n def = if length bits >= (n+1) && length (bits !! n) > 0 then (if def then bits !! n /= "n" else bits !! n == "y") else def mkLangInfo s = let bits = separateBy ',' s in if length bits < 2 @@ -334,9 +335,10 @@ loadLangs = do , langFunctor = maybeBit bits 2 , langUnlexer = maybeBit bits 3 , langPresent = boolBit bits 4 False - , langAPI = boolBit bits 5 True - , langSymbolic = boolBit bits 6 True - , langCompatibility = boolBit bits 7 False + , langAll = boolBit bits 5 True + , langAPI = boolBit bits 6 True + , langSymbolic = boolBit bits 7 True + , langCompatibility = boolBit bits 8 False } -- | Separate a string on a character diff --git a/README.md b/README.md index 22728a3d7..a16590d14 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Description of columns: - Functor: (not used) - Unlexer - Present: languages that have notpresent marked +- All: languages for which to compile All - API: languages for which to compile Try - Symbolic: languages for which to compile Symbolic - Compatibility: languages for which Compatibility exists diff --git a/languages.csv b/languages.csv index 84d9c2c68..7ee44f866 100644 --- a/languages.csv +++ b/languages.csv @@ -1,46 +1,45 @@ -Code,Directory,Functor,Unlexer,Present,API,Symbolic,Compatibility -Afr,afrikaans,,,y,,n, -Amh,amharic,,,,n,n, -Ara,arabic,,,,n,n, -Eus,basque,,,,,, -Bul,bulgarian,,,y,,,y -Cat,catalan,Romance,,y,,,y -Chi,chinese,,,y,,, -Dan,danish,Scand,,y,,, -Dut,dutch,,,y,,, -Eng,english,,,y,,,y -Est,estonian,,,y,,, -Fin,finnish,,,y,,,y -Fre,french,Romance,,y,,,y -Grc,ancient_greek,,,y,n,n, -Gre,greek,,,y,,, -Heb,hebrew,,,,n,n, -Hin,hindi,Hindustani,to_devanagari,y,,, -Hun,hungarian,,,y,,, -Ger,german,,,,,, -Ice,icelandic,,,y,,n, -Ina,interlingua,,,y,n,n, -Ita,italian,Romance,,y,,,y -Jpn,japanese,,,y,,, -Lat,latin,,,y,n,n, -Lav,latvian,,,,,,y -Mlt,maltese,,,,,, -Mon,mongolian,,,y,,n, -Nep,nepali,,,y,,n, -Nor,norwegian,Scand,,y,,, -Nno,nynorsk,,,y,,, -Pes,persian,,,y,,, -Pol,polish,,,,,, -Por,portuguese,Romance,,y,,,y -Pnb,punjabi,,,y,,, -Ron,romanian,,,y,,, -Rus,russian,,,y,,, -Snd,sindhi,,,y,,, -Spa,spanish,Romance,,y,,,y -Swa,swahili,,,,,,y -Swe,swedish,Scand,,y,,,y -Tel,telugu,,,y,,, -Tha,thai,,to_thai,y,,, -Tur,turkish,,,,n,n, -Urd,urdu,Hindustani,,y,,, -Vot,votic,,,,,, +Code,Directory,Functor,Unlexer,Present,All,API,Symbolic,Compatibility +Afr,afrikaans,,,,,,n, +Amh,amharic,,,,,n,n, +Ara,arabic,,,,,n,n, +Eus,basque,,,,,,, +Bul,bulgarian,,,y,,,, +Cat,catalan,Romance,,y,,,,y +Chi,chinese,,,,,,, +Dan,danish,Scand,,y,,,, +Dut,dutch,,,y,,,, +Eng,english,,,y,,,,y +Est,estonian,,,,,,, +Fin,finnish,,,y,,,,y +Fre,french,Romance,,y,,,,y +Grc,ancient_greek,,,y,,n,n, +Gre,greek,,,,,,, +Heb,hebrew,,,,,n,n, +Hin,hindi,Hindustani,to_devanagari,y,,,, +Hun,hungarian,,,y,n,n,n, +Ger,german,,,,,,, +Ice,icelandic,,,,,,n, +Ina,interlingua,,,y,,n,n, +Ita,italian,Romance,,y,,,,y +Jpn,japanese,,,,,,, +Lat,latin,,,y,,n,n, +Lav,latvian,,,,,,,y +Mlt,maltese,,,,,,, +Mon,mongolian,,,,,,n, +Nep,nepali,,,,,,n, +Nor,norwegian,Scand,,y,,,, +Nno,nynorsk,,,y,,,, +Pes,persian,,,,,,, +Pol,polish,,,,,,, +Por,portuguese,Romance,,y,,,,y +Pnb,punjabi,,,y,,,, +Ron,romanian,,,y,,,, +Rus,russian,,,y,,,, +Snd,sindhi,,,,,,, +Spa,spanish,Romance,,y,,,,y +Swa,swahili,,,,n,n,n,y +Swe,swedish,Scand,,y,,,,y +Tel,telugu,,,y,n,n,n, +Tha,thai,,to_thai,,,,, +Tur,turkish,,,,,n,n, +Urd,urdu,Hindustani,,,,,, \ No newline at end of file From d99420849936ee8dbbeaa97e236301d3d5b5f191 Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Tue, 7 Aug 2018 08:41:45 +0200 Subject: [PATCH 14/19] Cleanup Make.hs, remove 'parse' build command. --- Make.hs | 87 +++++++++++++++++++++++++------------------------------ README.md | 4 +-- 2 files changed, 40 insertions(+), 51 deletions(-) diff --git a/Make.hs b/Make.hs index dd1d06f7c..011c0dcdc 100644 --- a/Make.hs +++ b/Make.hs @@ -1,11 +1,11 @@ -import Data.List (find,intersect,isPrefixOf,isSuffixOf,(\\),unfoldr) -import Data.Maybe (fromJust,isJust,catMaybes) +import Data.List (find,isPrefixOf,isSuffixOf,(\\),unfoldr) +import Data.Maybe (catMaybes) import System.IO (hPutStrLn,stderr) import System.IO.Error (catchIOError) import System.Exit (ExitCode(..),die) import System.Environment (getArgs,lookupEnv) import System.Process (rawSystem) -import System.FilePath ((),takeFileName,addExtension,dropExtension) +import System.FilePath (()) -- ,takeFileName,addExtension,dropExtension) import System.Directory (createDirectoryIfMissing,copyFile,getDirectoryContents,removeDirectoryRecursive,findFile) import Control.Monad (when,unless) @@ -44,7 +44,7 @@ copyRGL args = do -- sequence_ [copyOne (flip addExtension "gfo" . dropExtension . takeFileName $ file) (getRGLBuildDir info mode) (gf_lib_dir getRGLBuildSubDir mode)|file<-files, mode<-modes] -- else do -- Copy everything - copyAll "prelude" (infoBuildDir info "prelude") (gf_lib_dir "prelude") + copyAll "Prelude" (infoBuildDir info "prelude") (gf_lib_dir "prelude") sequence_ [copyAll (show mode) (getRGLBuildDir info mode) (gf_lib_dir getRGLBuildSubDir mode)|mode<-modes] -- | Error message when install location cannot be determined @@ -83,6 +83,11 @@ clean = do sourceDir :: FilePath sourceDir = "src" +-- | Drop source directory prefix +-- TODO use functions from System.FilePath +dropSourceDir :: FilePath -> FilePath +dropSourceDir = drop (length sourceDir + 1) + -- | Information needed in build data Info = Info { infoBuildDir :: FilePath -- ^ where to put built RGL modules (fixed) @@ -95,7 +100,6 @@ data Info = Info mkInfo :: IO Info mkInfo = do args <- getArgs - -- Look for install location in a few different places let mflag = getFlag destination_flag args mbuilt <- catchIOError (readFile "../gf-core/DATA_DIR" >>= \d -> return (Just (d "lib"))) (\e -> return Nothing) @@ -106,7 +110,6 @@ mkInfo = do [] -> Nothing p:_ -> Just p let verbose = verbose_switch `elem` args || verbose_switch_short `elem` args - return $ Info { infoBuildDir = "dist" , infoInstallDir = inst_dir @@ -148,18 +151,20 @@ data RGLCommand = RGLCommand rglCommands :: [RGLCommand] rglCommands = [ RGLCommand "prelude" True $ \modes args bi -> do - putStrLn $ "Building [prelude]" let prelude_src_dir = sourceDir "prelude" prelude_dst_dir = infoBuildDir bi "prelude" createDirectoryIfMissing True prelude_dst_dir files <- getDirectoryContents prelude_src_dir - run_gfc bi ([if infoVerbose bi then "--verbose" else "--quiet", "--gfo-dir="++prelude_dst_dir] ++ [prelude_src_dir file | file <- files, file /= "." && file /= ".."]) + let modules = [prelude_src_dir file | file <- files, file /= "." && file /= ".."] + putStrLn $ "Building [Prelude] " ++ unwords (map dropSourceDir modules) + run_gfc bi ([if infoVerbose bi then "--verbose" else "--quiet", "--gfo-dir="++prelude_dst_dir] ++ modules) , RGLCommand "all" True $ gfcp [l,s,c,t,sc] , RGLCommand "lang" False $ gfcp [l,s] , RGLCommand "api" False $ gfcp [t,sc] , RGLCommand "compat" False $ gfcp [c] + -- Special command, invoked when command ends in .gf , RGLCommand "modules" False $ \modes args bi -> do let modules = getOptModules args flip mapM_ modules $ \m -> do @@ -168,14 +173,9 @@ rglCommands = Nothing -> die $ "Cannot find module: " ++ m Just mfull -> flip mapM_ modes $ \mode -> do let dst = getRGLBuildDir bi mode - putStrLn $ "Building [" ++ m ++ "] " ++ dst + putStrLn $ "Building [" ++ show mode ++ "] " ++ dropSourceDir mfull run_gfc bi [if infoVerbose bi then "--verbose" else "--quiet", "--gfo-dir="++dst, mfull] - , RGLCommand "parse" False $ \modes args bi -> do - langsAll <- loadLangs - let langsParse = \l -> langCode l `elem` ["Eng"] - let langs = (optml AllTenses langsParse args) langsAll - gfc bi modes (summary parse langs) (map parse langs) ] where gfcp :: [Mode -> [String] -> (LangInfo -> FilePath,[LangInfo] -> [LangInfo])] -> [Mode] -> [String] -> Info -> IO () @@ -188,7 +188,7 @@ rglCommands = gfcn bi mode (unwords ss) (concat fss) summary :: (LangInfo -> FilePath) -> [LangInfo] -> String - summary f langs = unwords (map (drop (length sourceDir + 1) . f) langs) + summary f langs = unwords (map (dropSourceDir . f) langs) -- summary f _ = f (LangInfo "*" "*" Nothing Nothing False False False False) l mode args = (lang,optml mode langAll args) @@ -203,6 +203,9 @@ rglCommands = let defLangs = filter (if mode == Present then langPresent else const True) (filter ls langs) in getOptLangs langs defLangs args +------------------------------------------------------------------------------- +-- Getting module paths/names + -- | Search all language dirs for module name findModule :: String -> IO (Maybe FilePath) findModule file = do @@ -211,6 +214,27 @@ findModule file = do let searchdirs = map (() sourceDir) langdirs findFile searchdirs file +lang :: LangInfo -> FilePath +lang l = sourceDir langDir l ("All" ++ langCode l ++ ".gf") + +compat :: LangInfo -> FilePath +compat l = sourceDir langDir l ("Compatibility" ++ langCode l ++ ".gf") + +symbol :: LangInfo -> FilePath +symbol l = sourceDir langDir l ("Symbol" ++ langCode l ++ ".gf") + +try :: LangInfo -> FilePath +try l = sourceDir "api" ("Try" ++ langCode l ++ ".gf") + +syntax :: LangInfo -> FilePath +syntax l = sourceDir "api" ("Syntax" ++ langCode l ++ ".gf") + +symbolic :: LangInfo -> FilePath +symbolic l = sourceDir "api" ("Symbolic" ++ langCode l ++ ".gf") + +------------------------------------------------------------------------------- +-- Argument helpers + -- | Check arguments are valid, failing on error checkArgs :: [String] -> IO () checkArgs args = do @@ -348,39 +372,6 @@ separateBy chr = unfoldr sep where sep [] = Nothing sep l = Just . fmap (drop 1) . break (== chr) $ l -------------------------------------------------------------------------------- --- Getting module paths/names - -lang :: LangInfo -> FilePath -lang l = sourceDir langDir l ("All" ++ langCode l ++ ".gf") - -compat :: LangInfo -> FilePath -compat l = sourceDir langDir l ("Compatibility" ++ langCode l ++ ".gf") - -symbol :: LangInfo -> FilePath -symbol l = sourceDir langDir l ("Symbol" ++ langCode l ++ ".gf") - -try :: LangInfo -> FilePath -try l = sourceDir "api" ("Try" ++ langCode l ++ ".gf") - -syntax :: LangInfo -> FilePath -syntax l = sourceDir "api" ("Syntax" ++ langCode l ++ ".gf") - -symbolic :: LangInfo -> FilePath -symbolic l = sourceDir "api" ("Symbolic" ++ langCode l ++ ".gf") - -parse :: LangInfo -> FilePath -parse l = sourceDir "parse" ("Parse" ++ langCode l ++ ".gf") - --- | Get unlexer flags for languages -unlexer :: String -> [LangInfo] -> String -unlexer abstr ls = - "--unlexer=\\\"" ++ unwords - [ abstr ++ langCode lang ++ "=" ++ fromJust unl - | lang <- ls - , let unl = langUnlexer lang - , isJust unl] ++ "\\\"" - ------------------------------------------------------------------------------- -- Executing GF diff --git a/README.md b/README.md index a16590d14..062969898 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ There is also `make clean` available. For more fine-grained control over the build process, you can run the build script directly: ``` -runghc Make.hs ... +runghc Make ... ``` Where `...` is one of: @@ -81,8 +81,6 @@ clean `lang`, `api`, `compat`, - -`parse`, or an explicit module name (e.g. `ExtraEng.gf`. You don't need to specify to language subdirectory, but there is a restriction that the module must exist in a **direct** subdirectory of `src`). - `MODE` is one of: `present`, From 1208cdf9bc69ed511960da71ec166cadd1fdfbaa Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Tue, 7 Aug 2018 09:44:03 +0200 Subject: [PATCH 15/19] Remove english/CompatibilityCat.gf There's no reason why that should be there --- src/english/CompatibilityCat.gf | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 src/english/CompatibilityCat.gf diff --git a/src/english/CompatibilityCat.gf b/src/english/CompatibilityCat.gf deleted file mode 100644 index e12287c45..000000000 --- a/src/english/CompatibilityCat.gf +++ /dev/null @@ -1,11 +0,0 @@ ---# -path=.:../romance:../abstract:../common - -concrete CompatibilityCat of Compatibility = CatCat ** open Prelude, CommonRomance in { - --- from Noun 19/4/2008 - -lin - NumInt n = {s = \\_ => n.s ; isNum = True ; n = Pl} ; - OrdInt n = {s = \\_ => n.s ++ "."} ; --- - -} From b76d8512e40f956d08edb7ab9bbc73e5f4575e83 Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Tue, 7 Aug 2018 10:06:12 +0200 Subject: [PATCH 16/19] Updade Make.sh to read languages.csv, more specific about what to compile --- .gitignore | 1 + Make.bat | 1 + Make.hs | 8 ++++---- Make.sh | 36 ++++++++++++++++++++++++------------ README.md | 30 ++++++++++++++---------------- languages.csv | 4 ++-- 6 files changed, 46 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 94e6cb952..1e3357253 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist/ *.hi *.o *.pgf +*.tmp diff --git a/Make.bat b/Make.bat index d4b9418f9..16ba98fa2 100644 --- a/Make.bat +++ b/Make.bat @@ -6,6 +6,7 @@ REM Non-Haskell RGL build script for Windows machines REM --- REM Modules to compile for each language +REM TODO read from languages.csv set langs=Afr Amh Ara Eus Bul Cat Chi Dan Dut Eng Est Fin Fre Grc Gre Heb Hin Ger Ice Ina Ita Jpn Lat Lav Mlt Mon Nep Nor Nno Pes Pol Por Pnb Ron Rus Snd Spa Swe Tha Tur Urd set modules_langs=All Symbol Compatibility set modules_api=Try Symbolic diff --git a/Make.hs b/Make.hs index 011c0dcdc..686067d16 100644 --- a/Make.hs +++ b/Make.hs @@ -192,9 +192,9 @@ rglCommands = -- summary f _ = f (LangInfo "*" "*" Nothing Nothing False False False False) l mode args = (lang,optml mode langAll args) - s mode args = (symbol,optml mode langAPI args) + s mode args = (symbol,optml mode langTry args) c mode args = (compat,optml AllTenses langCompatibility args) - t mode args = (try,optml mode langAPI args) + t mode args = (try,optml mode langTry args) sc mode args = (symbolic,optml mode langSymbolic args) optml :: Mode -> (LangInfo -> Bool) -> [String] -> ([LangInfo] -> [LangInfo]) @@ -335,7 +335,7 @@ data LangInfo = LangInfo , langUnlexer :: Maybe String -- ^ decoding for postprocessing linearizations , langPresent :: Bool , langAll :: Bool - , langAPI :: Bool + , langTry :: Bool , langSymbolic :: Bool , langCompatibility :: Bool } deriving (Show,Eq) @@ -360,7 +360,7 @@ loadLangs = do , langUnlexer = maybeBit bits 3 , langPresent = boolBit bits 4 False , langAll = boolBit bits 5 True - , langAPI = boolBit bits 6 True + , langTry = boolBit bits 6 True , langSymbolic = boolBit bits 7 True , langCompatibility = boolBit bits 8 False } diff --git a/Make.sh b/Make.sh index b5b6512f9..4aa66b7e2 100755 --- a/Make.sh +++ b/Make.sh @@ -4,8 +4,14 @@ # Non-Haskell RGL build script for Unix-based machines # --- +# Get languages from config +langs=$(tail -n +2 languages.csv | awk -F ',' '{ if ($6 != "n") { print $1 } }') +langs_present=$(tail -n +2 languages.csv | awk -F ',' '{ if ($5 == "y") { print $1 } }') +langs_try=$(tail -n +2 languages.csv | awk -F ',' '{ if ($7 != "n") { print $1 } }') +langs_symbolic=$(tail -n +2 languages.csv | awk -F ',' '{ if ($8 != "n") { print $1 } }') +langs_compat=$(tail -n +2 languages.csv | awk -F ',' '{ if ($9 == "y") { print $1 } }') + # Modules to compile for each language -# langs="Afr Amh Ara Eus Bul Cat Chi Dan Dut Eng Est Fin Fre Grc Gre Heb Hin Ger Ice Ina Ita Jpn Lat Lav Mlt Mon Nep Nor Nno Pes Pol Por Pnb Ron Rus Snd Spa Swe Tha Tur Urd" modules_langs="All Symbol Compatibility" modules_api="Try Symbolic" @@ -46,7 +52,7 @@ fi # A few more definitions before we get started src="src" dist="dist" -gfc="${gf} --batch --gf-lib-path=${src} --quiet" +gfc="${gf} --batch --quiet" # Redirect stderr if not verbose if [ $verbose = false ]; then @@ -60,29 +66,35 @@ mkdir -p "${dist}/alltenses" # Build: prelude echo "Building [prelude]" +if [ $verbose = true ]; then echo "${src}"/prelude/*.gf; fi ${gfc} --gfo-dir="${dist}"/prelude "${src}"/prelude/*.gf # Gather all language modules for building -for mod in $modules_langs; do - for file in "${src}"/*/"${mod}"???.gf; do - modules="${modules} ${file}" - done -done -for mod in $modules_api; do - for file in "${src}"/api/"${mod}"???.gf; do - modules="${modules} ${file}" +modules_present= +modules_alltenses= +for lang in $langs; do + for mod in $modules_langs $modules_api; do + if [ $mod == "Compatibility" ] && [[ "$langs_compat" != *"$lang"* ]]; then continue; fi + if [ $mod == "Try" ] && [[ "$langs_try" != *"$lang"* ]]; then continue; fi + if [ $mod == "Symbolic" ] && [[ "$langs_symbolic" != *"$lang"* ]]; then continue; fi + for file in "${src}"/*/"${mod}${lang}".gf; do + if [[ "$langs_present" = *"$lang"* ]]; then modules_present="${modules_present} ${file}"; fi + modules_alltenses="${modules_alltenses} ${file}" + done done done # Build: present echo "Building [present]" -for module in $modules; do +if [ $verbose = true ]; then echo $modules_present; fi +for module in $modules_present; do ${gfc} --no-pmcfg --gfo-dir="${dist}"/present --preproc=mkPresent "${module}" done # Build: alltenses echo "Building [alltenses]" -for module in $modules; do +if [ $verbose = true ]; then echo $modules_alltenses; fi +for module in $modules_alltenses; do ${gfc} --no-pmcfg --gfo-dir="${dist}"/alltenses "${module}" done diff --git a/README.md b/README.md index 062969898..db12cad0c 100644 --- a/README.md +++ b/README.md @@ -27,18 +27,18 @@ It will look for, in this order: A list of all languages and their properties is maintained centrally in `languages.csv`. This file should be kept up-to-date and all build methods should read this config file. -If you see something wrong, please report/fix it. +**If you see something wrong, please report/fix it.** Description of columns: -- Code -- Directory -- Functor: (not used) -- Unlexer -- Present: languages that have notpresent marked -- All: languages for which to compile All -- API: languages for which to compile Try -- Symbolic: languages for which to compile Symbolic -- Compatibility: languages for which Compatibility exists +- Code, e,g, `Eng` +- Directory, e.g. `english` +- Functor (not used) +- Unlexer (not used) +- Present: languages that have `--# notpresent` marked +- All: languages for which to compile `All` +- Try: languages for which to compile `Try` +- Symbolic: languages for which to compile `Symbolic` +- Compatibility: languages for which to complile `Compatibility` Columns can be a string, just `y`'s (where nothing means `n`) or just (`n`'s where nothing means `y`). @@ -86,12 +86,12 @@ or an explicit module name (e.g. `ExtraEng.gf`. You don't need to specify to lan `present`, `alltenses` (default is both). +- `LANG` is a 3-letter language code, e.g. `Eng`, `Swe` etc. - You can _override_ the default language list with `--langs=...` - You can _add_ languages to the default list with `--langs=+...` - You can _remove_ languages from the default list with `langs=-...` -- `LANG` is a 3-letter language code, e.g. `Eng`, `Swe` etc. -- The path to GF installed on your system can be specified via the `gf` flag (default is that the `gf` executable is in the global system path). -- The `dest` flag can be used to manually specify where the compiled RGL modules should be copied/installed. This is the same place as `GF_LIB_PATH`. +- The path to GF installed on your system can be specified via the `--gf` flag (default is that the `gf` executable is in the global system path). +- The `--dest` flag can be used to manually specify where the compiled RGL modules should be copied/installed. This is the same place as `GF_LIB_PATH`. ## Shell script: `Make.sh` @@ -103,8 +103,6 @@ You can pass the following flags: - `--gf=...` to specify the path to the `gf` executable, if not available on the system path - `--verbose` or `-v` to show all GF warnings and errors -This build method tries to build all languages found in the `src` directory, even those which are not considered complete. - ## Windows batch file: `Make.bat` **This script is still untested.** @@ -114,7 +112,7 @@ This method is provided as an alternative for Windows users who don't have Haske It is supposed to be a port of Make.sh and works in largely the same way. In particular, it accepts the same flags (in the same format) as described above. -One difference is that the list of languages to be compiled is specified manually in the script in the `langs` variable. +However it currently tries to build all modules for all languages and doesn't consider the details of which modules should be compiled for each language (specified in `languages.csv`) ## About this repository diff --git a/languages.csv b/languages.csv index 7ee44f866..633fabd15 100644 --- a/languages.csv +++ b/languages.csv @@ -1,4 +1,4 @@ -Code,Directory,Functor,Unlexer,Present,All,API,Symbolic,Compatibility +Code,Directory,Functor,Unlexer,Present,All,Try,Symbolic,Compatibility Afr,afrikaans,,,,,,n, Amh,amharic,,,,,n,n, Ara,arabic,,,,,n,n, @@ -42,4 +42,4 @@ Swe,swedish,Scand,,y,,,,y Tel,telugu,,,y,n,n,n, Tha,thai,,to_thai,,,,, Tur,turkish,,,,,n,n, -Urd,urdu,Hindustani,,,,,, \ No newline at end of file +Urd,urdu,Hindustani,,,,,, From 32647437ad334b008a32432f7883533fe69df997 Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Tue, 7 Aug 2018 10:11:23 +0200 Subject: [PATCH 17/19] Remove unneccessary --gf-lib-path, which could give misleading errors --- Make.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Make.hs b/Make.hs index 686067d16..05872407f 100644 --- a/Make.hs +++ b/Make.hs @@ -393,7 +393,7 @@ gfcn bi mode summary files = do run_gfc :: Info -> [String] -> IO () run_gfc bi args = do let - args' = ["--batch","--gf-lib-path="++sourceDir] ++ filter (not . null) args + args' = ["--batch"] ++ filter (not . null) args gf = infoGFPath bi execute gf args' From 0130a0012d210cbdfd1c0c893ae69f8c490bbe0e Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Tue, 7 Aug 2018 11:01:02 +0200 Subject: [PATCH 18/19] Better output when no files specified in Make.hs --- Make.hs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Make.hs b/Make.hs index 05872407f..1118fb397 100644 --- a/Make.hs +++ b/Make.hs @@ -198,10 +198,10 @@ rglCommands = sc mode args = (symbolic,optml mode langSymbolic args) optml :: Mode -> (LangInfo -> Bool) -> [String] -> ([LangInfo] -> [LangInfo]) - optml mode ls args = - \langs -> - let defLangs = filter (if mode == Present then langPresent else const True) (filter ls langs) - in getOptLangs langs defLangs args + optml mode pred args = + \langsAll -> + let langsDefault = filter (if mode == Present then langPresent else const True) (filter pred langsAll) + in getOptLangs langsAll langsDefault args ------------------------------------------------------------------------------- -- Getting module paths/names @@ -327,6 +327,10 @@ getFlag flag args = fmap (drop (length flag)) $ find (isPrefixOf flag) args ------------------------------------------------------------------------------- -- Languages of the RGL +-- | Path to language config file +configFile :: FilePath +configFile = "languages.csv" + -- | Information about a language data LangInfo = LangInfo { langCode :: String -- ^ 3-letter ISO 639-2/B code @@ -343,16 +347,15 @@ data LangInfo = LangInfo -- | Load language information from config file loadLangs :: IO [LangInfo] loadLangs = do - lns <- readFile conffile >>= return . lines + lns <- readFile configFile >>= return . lines mapM mkLangInfo (tail lns) where - conffile = "languages.csv" maybeBit bits n = if length bits >= (n+1) && length (bits !! n) > 0 then Just (bits !! n) else Nothing boolBit bits n def = if length bits >= (n+1) && length (bits !! n) > 0 then (if def then bits !! n /= "n" else bits !! n == "y") else def mkLangInfo s = let bits = separateBy ',' s in if length bits < 2 - then die $ "Invalid entry in " ++ conffile ++ ": " ++ s + then die $ "Invalid entry in " ++ configFile ++ ": " ++ s else return $ LangInfo { langCode = bits !! 0 , langDir = bits !! 1 @@ -380,6 +383,7 @@ gfc bi modes summary files = parallel_ [gfcn bi mode summary files | mode<-modes] gfcn :: Info -> Mode -> String -> [String] -> IO () +gfcn _ _ _ [] = die $ "No files specified.\nMake sure the language is in " ++ configFile ++ " and that it supports the modes/modules specified." gfcn bi mode summary files = do let dir = getRGLBuildDir bi mode preproc = case mode of @@ -393,7 +397,7 @@ gfcn bi mode summary files = do run_gfc :: Info -> [String] -> IO () run_gfc bi args = do let - args' = ["--batch"] ++ filter (not . null) args + args' = ["--batch","--gf-lib-path="] ++ filter (not . null) args gf = infoGFPath bi execute gf args' From faf66b4a8839d6b93b086da93e890b79dbdfd15f Mon Sep 17 00:00:00 2001 From: "John J. Camilleri" Date: Tue, 7 Aug 2018 11:21:58 +0200 Subject: [PATCH 19/19] Update Clone.hs to add copy api files and add language to config --- src/Clone.hs | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/Clone.hs b/src/Clone.hs index 06c57bbe0..d86c563e6 100644 --- a/src/Clone.hs +++ b/src/Clone.hs @@ -6,19 +6,24 @@ import Data.Char import Data.List import System.Process import System.Directory +import System.FilePath import System.Environment (getArgs) import System.Exit - +import Text.Printf -- To clone a project from one language to another: --- Clone fromdir todir fromlang tolang (--comment) +-- Clone fromdir todir fromlang tolang -- -- 1. for each Module in 'fromdir', copy Module(fromlang) to todir/Module(tolang) ; create todir if it doesn't exist -- 2. in each Module(tolang), replace substrings fromlang by tolang, if proper suffixes of identifiers --- 3. If the option --comment is present, comment out every line in the body +-- 3. repeat the above for api/Module(fromlang) to api/Module(tolang) +-- 4. add the language to config file if not present +-- - If the option --comment-body is present, comment out every line in the body +-- - If the option --comment-body is present, comment out every line in the body -- -- Example: runghc Clone swedish danish Swe Dan +main :: IO () main = do args <- getArgs if length args < 4 @@ -31,15 +36,32 @@ main = do createDirectoryIfMissing True todir mapM_ (clone options fromdir todir fromlang tolang) modules + mapM_ (\md -> clone options "api" "api" fromlang tolang (md,"")) apiModules + conf <- readFile configFile + if not (any (isPrefixOf tolang) (lines conf)) + then do + appendFile configFile (printf "%s,%s\n" tolang todir) + printf "Language '%s' has been added to %s\n" tolang configFile + else return () + +configFile :: FilePath +configFile = ".." "languages.csv" + +apiModules :: [String] +apiModules = ["Try","Symbolic","Syntax","Constructors","Combinators"] + +clone :: [String] -> String -> String -> String -> String -> (String, String) -> IO () clone options fromdir todir from to (absname,absfx) = do s <- readFile (fromdir ++ "/" ++ absname ++ from ++ absfx ++ ".gf") writeAndReportFile (todir ++ "/" ++ absname ++ to ++ absfx ++ ".gf") (commentIf options (replaceLang from to s)) +getAbstractName :: String -> String -> (String, String) getAbstractName from file | isSuffixOf (from ++ "Abs.gf") file = (take (length file - (length from + 6)) file, "Abs") -- (NewDict, Abs) | isSuffixOf (from ++ ".gf") file = (take (length file - (length from + 3)) file, "") -- (NewDict, []) | otherwise = error ("Need suffix " ++ (from ++ ".gf") ++ " or " ++ (from ++ "Abs.gf") ++ " therefore cannot clone file name " ++ file) +replaceLang :: String -> String -> String -> String replaceLang s1 s2 = repl where repl s = case s of c:cs -> case splitAt lgs s of @@ -49,12 +71,14 @@ replaceLang s1 s2 = repl where _ -> s lgs = length s1 +commentIf :: [String] -> String -> String commentIf options = let commentbody = if (elem "--comment-body" options) then commentBody else id dropcomments = if (elem "--drop-comments" options) then dropComments else id in unlines . commentbody . dropcomments . lines +commentBody :: [String] -> [String] commentBody ss = header ++ map comment body ++ ["}"] where (header,body) = break (isJment . words) ss isJment ws = case ws of @@ -65,12 +89,14 @@ commentBody ss = header ++ map comment body ++ ["}"] where _ | all isSpace l -> l -- empty line _ -> "--" ++ l +dropComments :: [String] -> [String] dropComments = filter (not . isComment) where isComment line = case dropWhile isSpace line of '-':'-':'#':_ -> False '-':'-':_ -> True _ -> False +writeAndReportFile :: FilePath -> String -> IO () writeAndReportFile file s = do writeFile file s putStrLn $ "wrote " ++ file