GitHub/gf-core
1
0
Fork 1
mirror of https://github.com/GrammaticalFramework/gf-core.git synced 2026-04-27 13:32:51 -06:00
Code Issues Packages Projects Releases Wiki Activity

codevars

Browse Source
This commit is contained in:
aarne
2004-08-25 15:17:13 +00:00
parent 173a773334
commit 42e8522757
5 changed files with 13 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/GF/UseGrammar/Custom.hs
View File

@@ -320,7 +320,7 @@ customTokenizer =
,(strCI "vars", const $ tokVars)
,(strCI "chars", const $ map (tS . singleton))
,(strCI "code", const $ lexHaskell)
,(strCI "codevars", const $ (mkTokVars lexHaskell))
,(strCI "codevars", lexHaskellVar . stateIsWord)
,(strCI "text", const $ lexText)
,(strCI "unglue", \gr -> map tS . decomposeWords (stateMorpho gr))
,(strCI "codelit", lexHaskellLiteral . stateIsWord)

9
src/GF/UseGrammar/Tokenize.hs
View File

@@ -134,9 +134,18 @@ unknown2string isKnown = map mkOne where
mkOne t@(TC s) = if isKnown s then t else mkTL s
mkOne t = t
unknown2var :: (String -> Bool) -> [CFTok] -> [CFTok]
unknown2var isKnown = map mkOne where
mkOne t@(TS "??") = if isKnown "??" then t else tM "??"
mkOne t@(TS s) = if isKnown s then t else tV s
mkOne t@(TC s) = if isKnown s then t else tV s
mkOne t = t
lexTextLiteral isKnown = unknown2string (eitherUpper isKnown) . lexText
lexHaskellLiteral isKnown = unknown2string isKnown . lexHaskell
lexHaskellVar isKnown = unknown2var isKnown . lexHaskell
eitherUpper isKnown w@(c:cs) = isKnown (toLower c : cs) || isKnown (toUpper c : cs)
eitherUpper isKnown w = isKnown w