diff --git a/src/GF/Compile/GetGrammar.hs b/src/GF/Compile/GetGrammar.hs index 41c2c063d..62826746c 100644 --- a/src/GF/Compile/GetGrammar.hs +++ b/src/GF/Compile/GetGrammar.hs @@ -44,6 +44,7 @@ import GF.Infra.ReadFiles ---- import Data.Char (toUpper) import Data.List (nub) +import qualified Data.ByteString.Char8 as BS import Control.Monad (foldM) import System (system) @@ -61,14 +62,14 @@ getSourceModule opts file0 = do let string = case getOptVal opts uniCoding of Just "utf8" -> decodeUTF8 string0 _ -> string0 - let tokens = myLexer string + let tokens = myLexer (BS.pack string) mo1 <- ioeErr $ err2err $ pModDef tokens ioeErr $ transModDef mo1 getSourceGrammar :: Options -> FilePath -> IOE SourceGrammar getSourceGrammar opts file = do string <- readFileIOE file - let tokens = myLexer string + let tokens = myLexer (BS.pack string) gr1 <- ioeErr $ err2err $ pGrammar tokens ioeErr $ transGrammar gr1 @@ -117,7 +118,7 @@ ioeEErr = ioeErr . err2err -- change them by turning the final letter to upper case. --- There is a risk of clash. oldLexer :: String -> [L.Token] -oldLexer = map change . L.tokens where +oldLexer = map change . L.tokens . BS.pack where change t = case t of (L.PT p (L.TS s)) | elem s newReservedWords -> (L.PT p (L.TV (init s ++ [toUpper (last s)]))) diff --git a/src/GF/Compile/PGrammar.hs b/src/GF/Compile/PGrammar.hs index 1c6f0b411..e4e60d623 100644 --- a/src/GF/Compile/PGrammar.hs +++ b/src/GF/Compile/PGrammar.hs @@ -29,10 +29,11 @@ import GF.Grammar.Macros import GF.Grammar.MMacros import GF.Data.Operations +import qualified Data.ByteString.Char8 as BS pTerm :: String -> Err Term pTerm s = do - e <- err2err $ pExp $ myLexer s + e <- err2err $ pExp $ myLexer (BS.pack s) transExp e pTrm :: String -> Term