some var parser bug fixes

src/GF/UseGrammar/Custom.hs

@@ -320,6 +320,7 @@ customTokenizer =
   ,(strCI "vars",      const $ tokVars)
   ,(strCI "chars",     const $ map (tS . singleton))
   ,(strCI "code",      const $ lexHaskell)
+  ,(strCI "codevars",  const $ (mkTokVars lexHaskell))
   ,(strCI "text",      const $ lexText)
   ,(strCI "unglue",    \gr -> map tS . decomposeWords (stateMorpho gr))
   ,(strCI "codelit",   lexHaskellLiteral . stateIsWord)

src/GF/UseGrammar/Parsing.hs

@@ -20,6 +20,7 @@ import Option
 import Custom
 import ShellState
 
+import PPrCF (prCFTree)
 import qualified ParseGFC as N
 
 import Operations
@@ -69,7 +70,10 @@ trees2trms opts sg cn as ts0 info = do
     _ | null ts0 -> checkWarn "No success in cf parsing" >> return []
     _ | raw      -> do
       ts1 <- return (map cf2trm0 ts0) ----- should not need annot
-      mapM (checkErr . (annotate gr) . trExp) ts1 ---- complicated; often fails
+      checks [
+         mapM (checkErr . (annotate gr) . trExp) ts1 ---- complicated, often fails
+        ,checkWarn (unlines ("Raw CF trees:":(map prCFTree ts0))) >> return []
+        ]
     _ -> do
       let num = optIntOrN opts flagRawtrees 99999
       let (ts01,rest) = splitAt num ts0

src/GF/UseGrammar/Tokenize.hs

@@ -35,6 +35,11 @@ mkCFTokVar s = case s of
   '$':xs@(_:_) -> if last s == '$' then tV (init xs) else tS s
   _            -> tS s
 
+mkTokVars :: (String -> [CFTok]) -> String -> [CFTok]
+mkTokVars tok = map tv . tok where
+  tv (TS s) = mkCFTokVar s
+  tv t = t
+
 mkLit :: String -> CFTok
 mkLit s = if (all isDigit s) then (tI s) else (tL s)