From f7ecc10d13ae3b35b36e27b4f51c1a68cf45fb11 Mon Sep 17 00:00:00 2001
From: aarne <unknown>
Date: Mon, 4 Apr 2005 16:29:18 +0000
Subject: [PATCH] started Italian Swadesh

---
 src/GF/UseGrammar/Tokenize.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs
index 067404468..da4d419b3 100644
--- a/src/GF/UseGrammar/Tokenize.hs
+++ b/src/GF/UseGrammar/Tokenize.hs
@@ -5,9 +5,9 @@
 -- Stability   : (stable)
 -- Portability : (portable)
 --
--- > CVS $Date: 2005/04/04 15:50:27 $ 
+-- > CVS $Date: 2005/04/04 17:29:18 $ 
 -- > CVS $Author: aarne $
--- > CVS $Revision: 1.11 $
+-- > CVS $Revision: 1.12 $
 --
 -- lexers = tokenizers, to prepare input for GF grammars. AR 4\/1\/2002.
 -- an entry for each is included in 'Custom.customTokenizer'
@@ -40,8 +40,8 @@ tokWords = map tS . words
 tokLits :: String -> [CFTok]
 tokLits = map mkCFTok . mergeStr . words where
   mergeStr ss = case ss of
-    w@(c:_):rest | elem c "\'\"" -> getStr [w] rest
-    w      :rest                 -> w : mergeStr rest
+    w@(c:cs):rest | elem c "\'\"" && c /= last w -> getStr [w] rest
+    w      :rest                  -> w : mergeStr rest
     [] -> []
   getStr v ss = case ss of
     w@(_:_):rest | elem (last w) "\'\"" -> (unwords (reverse (w:v))) : mergeStr rest