codevars

grammars/logic/Arithm.gf

@@ -44,7 +44,7 @@ def
               (LtNat one n) 
               (Univ Nat (\x -> Impl (Conj (LtNat one x) (Div n x)) (EqNat x n))) ;
 
---- data Elem = zero | succ ;
+---  data Elem = zero | succ ;
 
 fun ex1 : Text ;
 def ex1 = 

src/GF/UseGrammar/Custom.hs

@@ -320,7 +320,7 @@ customTokenizer =
   ,(strCI "vars",      const $ tokVars)
   ,(strCI "chars",     const $ map (tS . singleton))
   ,(strCI "code",      const $ lexHaskell)
-  ,(strCI "codevars",  const $ (mkTokVars lexHaskell))
+  ,(strCI "codevars",  lexHaskellVar . stateIsWord)
   ,(strCI "text",      const $ lexText)
   ,(strCI "unglue",    \gr -> map tS . decomposeWords (stateMorpho gr))
   ,(strCI "codelit",   lexHaskellLiteral . stateIsWord)

src/GF/UseGrammar/Tokenize.hs

@@ -134,9 +134,18 @@ unknown2string isKnown = map mkOne where
   mkOne t@(TC s) = if isKnown s then t else mkTL s
   mkOne t        = t
 
+unknown2var :: (String -> Bool) -> [CFTok] -> [CFTok]
+unknown2var isKnown = map mkOne where
+  mkOne t@(TS "??") = if isKnown "??" then t else tM "??"
+  mkOne t@(TS s) = if isKnown s then t else tV s
+  mkOne t@(TC s) = if isKnown s then t else tV s
+  mkOne t        = t
+
 lexTextLiteral    isKnown = unknown2string (eitherUpper isKnown) . lexText
 lexHaskellLiteral isKnown = unknown2string isKnown . lexHaskell
 
+lexHaskellVar     isKnown = unknown2var isKnown . lexHaskell
+
 eitherUpper isKnown w@(c:cs) = isKnown (toLower c : cs) || isKnown (toUpper c : cs)
 eitherUpper isKnown w = isKnown w
 

src/HelpFile

@@ -418,7 +418,7 @@ q, quit: q
     -lexer=vars          like words, but "x","x_...","$...$" as vars, "?..." as meta
     -lexer=chars         each character is a token
     -lexer=code          use Haskell's lex
-    -lexer=codevars      like code, but "x", "x_..." as vars, "??" as meta
+    -lexer=codevars      like code, but treat unknown words as variables, ?? as meta 
     -lexer=text          with conventions on punctuation and capital letters
     -lexer=codelit       like code, but treat unknown words as string literals
     -lexer=textlit       like text, but treat unknown words as string literals

src/HelpFile.hs

@@ -431,7 +431,7 @@ txtHelpFile =
   "\n    -lexer=vars          like words, but \"x\",\"x_...\",\"$...$\" as vars, \"?...\" as meta" ++
   "\n    -lexer=chars         each character is a token" ++
   "\n    -lexer=code          use Haskell's lex" ++
-  "\n    -lexer=codevars      like code, but \"x\", \"x_...\" as vars, \"??\" as meta" ++
+  "\n    -lexer=codevars      like code, but treat unknown words as variables, ?? as meta " ++
   "\n    -lexer=text          with conventions on punctuation and capital letters" ++
   "\n    -lexer=codelit       like code, but treat unknown words as string literals" ++
   "\n    -lexer=textlit       like text, but treat unknown words as string literals" ++