GitHub/gf-core
1
0
Fork 1
mirror of https://github.com/GrammaticalFramework/gf-core.git synced 2026-05-10 03:32:51 -06:00
Code Issues Packages Projects Releases Wiki Activity

Fixed several things, e.g. tokenizer.

Browse Source
This commit is contained in:
aarne
2003-11-03 16:27:55 +00:00
parent 2728e6e7ce
commit 94326929b1
18 changed files with 133 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/GF/UseGrammar/Tokenize.hs
View File

@@ -21,7 +21,11 @@ tokVars :: String -> [CFTok]
tokVars = map mkCFTokVar . words
mkCFTok :: String -> CFTok
mkCFTok s = tS s ---- if (isLiteral s) then (mkLit s) else (tS s)
mkCFTok s = case s of
'"' :cs@(_:_) -> tL $ init cs
'\'':cs@(_:_) -> tL $ init cs --- 's Gravenhage
_:_ | all isDigit s -> tI s
_ -> tS s
mkCFTokVar :: String -> CFTok
mkCFTokVar s = case s of