From 42e8522757114469a9044729ebcd2b80699a77f3 Mon Sep 17 00:00:00 2001 From: aarne Date: Wed, 25 Aug 2004 15:17:13 +0000 Subject: [PATCH] codevars --- grammars/logic/Arithm.gf | 2 +- src/GF/UseGrammar/Custom.hs | 2 +- src/GF/UseGrammar/Tokenize.hs | 9 +++++++++ src/HelpFile | 2 +- src/HelpFile.hs | 2 +- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/grammars/logic/Arithm.gf b/grammars/logic/Arithm.gf index 7674e9551..0cf07e0c1 100644 --- a/grammars/logic/Arithm.gf +++ b/grammars/logic/Arithm.gf @@ -44,7 +44,7 @@ def (LtNat one n) (Univ Nat (\x -> Impl (Conj (LtNat one x) (Div n x)) (EqNat x n))) ; ---- data Elem = zero | succ ; +--- data Elem = zero | succ ; fun ex1 : Text ; def ex1 = diff --git a/src/GF/UseGrammar/Custom.hs b/src/GF/UseGrammar/Custom.hs index e7989de30..9df79e983 100644 --- a/src/GF/UseGrammar/Custom.hs +++ b/src/GF/UseGrammar/Custom.hs @@ -320,7 +320,7 @@ customTokenizer = ,(strCI "vars", const $ tokVars) ,(strCI "chars", const $ map (tS . singleton)) ,(strCI "code", const $ lexHaskell) - ,(strCI "codevars", const $ (mkTokVars lexHaskell)) + ,(strCI "codevars", lexHaskellVar . stateIsWord) ,(strCI "text", const $ lexText) ,(strCI "unglue", \gr -> map tS . decomposeWords (stateMorpho gr)) ,(strCI "codelit", lexHaskellLiteral . stateIsWord) diff --git a/src/GF/UseGrammar/Tokenize.hs b/src/GF/UseGrammar/Tokenize.hs index 77c6222ac..7ae3463a0 100644 --- a/src/GF/UseGrammar/Tokenize.hs +++ b/src/GF/UseGrammar/Tokenize.hs @@ -134,9 +134,18 @@ unknown2string isKnown = map mkOne where mkOne t@(TC s) = if isKnown s then t else mkTL s mkOne t = t +unknown2var :: (String -> Bool) -> [CFTok] -> [CFTok] +unknown2var isKnown = map mkOne where + mkOne t@(TS "??") = if isKnown "??" then t else tM "??" + mkOne t@(TS s) = if isKnown s then t else tV s + mkOne t@(TC s) = if isKnown s then t else tV s + mkOne t = t + lexTextLiteral isKnown = unknown2string (eitherUpper isKnown) . lexText lexHaskellLiteral isKnown = unknown2string isKnown . lexHaskell +lexHaskellVar isKnown = unknown2var isKnown . lexHaskell + eitherUpper isKnown w@(c:cs) = isKnown (toLower c : cs) || isKnown (toUpper c : cs) eitherUpper isKnown w = isKnown w diff --git a/src/HelpFile b/src/HelpFile index 3e2b04335..85b39c8d9 100644 --- a/src/HelpFile +++ b/src/HelpFile @@ -418,7 +418,7 @@ q, quit: q -lexer=vars like words, but "x","x_...","$...$" as vars, "?..." as meta -lexer=chars each character is a token -lexer=code use Haskell's lex - -lexer=codevars like code, but "x", "x_..." as vars, "??" as meta + -lexer=codevars like code, but treat unknown words as variables, ?? as meta -lexer=text with conventions on punctuation and capital letters -lexer=codelit like code, but treat unknown words as string literals -lexer=textlit like text, but treat unknown words as string literals diff --git a/src/HelpFile.hs b/src/HelpFile.hs index 66be956fa..3cc6b2ada 100644 --- a/src/HelpFile.hs +++ b/src/HelpFile.hs @@ -431,7 +431,7 @@ txtHelpFile = "\n -lexer=vars like words, but \"x\",\"x_...\",\"$...$\" as vars, \"?...\" as meta" ++ "\n -lexer=chars each character is a token" ++ "\n -lexer=code use Haskell's lex" ++ - "\n -lexer=codevars like code, but \"x\", \"x_...\" as vars, \"??\" as meta" ++ + "\n -lexer=codevars like code, but treat unknown words as variables, ?? as meta " ++ "\n -lexer=text with conventions on punctuation and capital letters" ++ "\n -lexer=codelit like code, but treat unknown words as string literals" ++ "\n -lexer=textlit like text, but treat unknown words as string literals" ++