GitHub/gf-core
1
0
Fork 1
mirror of https://github.com/GrammaticalFramework/gf-core.git synced 2026-04-23 19:42:50 -06:00
Code Issues Packages Projects Releases Wiki Activity

started Italian Swadesh

Browse Source
This commit is contained in:
aarne
2005-04-04 16:29:18 +00:00
parent 41108818c9
commit f7ecc10d13
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/GF/UseGrammar/Tokenize.hs
View File

@@ -5,9 +5,9 @@
-- Stability : (stable) -- Stability : (stable)
-- Portability : (portable) -- Portability : (portable)
-- --
-- > CVS $Date: 2005/04/04 15:50:27 $ -- > CVS $Date: 2005/04/04 17:29:18 $
-- > CVS $Author: aarne $ -- > CVS $Author: aarne $
-- > CVS $Revision: 1.11 $ -- > CVS $Revision: 1.12 $
-- --
-- lexers = tokenizers, to prepare input for GF grammars. AR 4\/1\/2002. -- lexers = tokenizers, to prepare input for GF grammars. AR 4\/1\/2002.
-- an entry for each is included in 'Custom.customTokenizer' -- an entry for each is included in 'Custom.customTokenizer'
@@ -40,8 +40,8 @@ tokWords = map tS . words
tokLits :: String -> [CFTok] tokLits :: String -> [CFTok]
tokLits = map mkCFTok . mergeStr . words where tokLits = map mkCFTok . mergeStr . words where
mergeStr ss = case ss of mergeStr ss = case ss of
w@(c:_):rest | elem c "\'\"" -> getStr [w] rest w@(c:cs):rest | elem c "\'\"" && c /= last w -> getStr [w] rest
w :rest -> w : mergeStr rest w :rest -> w : mergeStr rest
[] -> [] [] -> []
getStr v ss = case ss of getStr v ss = case ss of
w@(_:_):rest | elem (last w) "\'\"" -> (unwords (reverse (w:v))) : mergeStr rest w@(_:_):rest | elem (last w) "\'\"" -> (unwords (reverse (w:v))) : mergeStr rest