GitHub/gf-core
1
0
Fork 1
mirror of https://github.com/GrammaticalFramework/gf-core.git synced 2026-04-23 11:42:49 -06:00
Code Issues Packages Projects Releases Wiki Activity

GF.Compile.ReadFiles: a simpler & faster way to convert from Latin1 to UTF-8

Browse Source
This commit is contained in:
hallgren
2014-08-19 14:27:15 +00:00
parent 60b5990792
commit 61760e4205
1 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/compiler/GF/Compile/ReadFiles.hs
View File

@@ -232,12 +232,15 @@ toUTF8 opts0 raw =
coding = getEncoding $ opts0 `addOptions` opts coding = getEncoding $ opts0 `addOptions` opts
utf8 <- if coding=="UTF-8" utf8 <- if coding=="UTF-8"
then return raw then return raw
else lift $ do --ePutStrLn $ "toUTF8 from "++coding else if coding=="CP1252" -- Latin1
enc <- mkTextEncoding coding then return . UTF8.fromString $ BS.unpack raw -- faster
-- decodeUnicodeIO uses a lot of stack space, else lift $
-- so we need to split the file into smaller pieces do --ePutStrLn $ "toUTF8 from "++coding
ls <- mapM (decodeUnicodeIO enc) (BS.lines raw) enc <- mkTextEncoding coding
return $ UTF8.fromString (unlines ls) -- decodeUnicodeIO uses a lot of stack space,
-- so we need to split the file into smaller pieces
ls <- mapM (decodeUnicodeIO enc) (BS.lines raw)
return $ UTF8.fromString (unlines ls)
return (given,utf8) return (given,utf8)
--lift io = ioe (fmap Ok io `catch` (return . Bad . show)) --lift io = ioe (fmap Ok io `catch` (return . Bad . show))