msyds/gf-core
1
0
Fork 0
forked from GitHub/gf-core
Code Pull Requests Activity

codevars

Browse Source
This commit is contained in:
aarne
2004-08-25 15:17:13 +00:00
parent 173a773334
commit 42e8522757
5 changed files with 13 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/GF/UseGrammar/Tokenize.hs
View File

@@ -134,9 +134,18 @@ unknown2string isKnown = map mkOne where
mkOne t@(TC s) = if isKnown s then t else mkTL s
mkOne t = t
unknown2var :: (String -> Bool) -> [CFTok] -> [CFTok]
unknown2var isKnown = map mkOne where
mkOne t@(TS "??") = if isKnown "??" then t else tM "??"
mkOne t@(TS s) = if isKnown s then t else tV s
mkOne t@(TC s) = if isKnown s then t else tV s
mkOne t = t
lexTextLiteral isKnown = unknown2string (eitherUpper isKnown) . lexText
lexHaskellLiteral isKnown = unknown2string isKnown . lexHaskell
lexHaskellVar isKnown = unknown2var isKnown . lexHaskell
eitherUpper isKnown w@(c:cs) = isKnown (toLower c : cs) || isKnown (toUpper c : cs)
eitherUpper isKnown w = isKnown w