GitHub/gf-core
1
0
Fork 1
mirror of https://github.com/GrammaticalFramework/gf-core.git synced 2026-05-22 09:32:53 -06:00
Code Issues Packages Projects Releases Wiki Activity

lexer documentation in help and tutorial updated

Browse Source
This commit is contained in:
aarne
2008-11-10 15:53:38 +00:00
parent 7e82e4a710
commit d9ff5aa48c
2 changed files with 12 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
doc/gf-tutorial.txt
View File

@@ -4277,24 +4277,12 @@ In linearization, we use a corresponding **unlexer**:
===Most common lexers and unlexers=== ===Most common lexers and unlexers===
|| lexer | description || || lexer | unlexer | description ||
| ``words`` | (default) tokens are separated by spaces or newlines | ``chars`` | ``unchars`` | each character is a token
| ``literals`` | like words, but integer and string literals recognized | ``lexcode`` | ``unlexcode`` | program code conventions (uses Haskell's lex)
| ``chars`` | each character is a token | ``lexmixed`` | ``unlexmixed`` | like text, but between $ signs like code
| ``code`` | program code conventions (uses Haskell's lex) | ``lextext`` | ``unlextext`` | with conventions on punctuation and capitals
| ``text`` | with conventions on punctuation and capital letters | ``words`` | ``unwords`` | (default) tokens separated by space characters
| ``codelit`` | like code, but recognize literals (unknown words as strings)
| ``textlit`` | like text, but recognize literals (unknown words as strings)
|| unlexer | description ||
| ``unwords`` | (default) space-separated token list
| ``text`` | format as text: punctuation, capitals, paragraph <p>
| ``code`` | format as code (spacing, indentation)
| ``textlit`` | like text, but remove string literal quotes
| ``codelit`` | like code, but remove string literal quotes
| ``concat`` | remove all spaces
%TODO: update the names
%TODO: also on alphabet encodings - although somewhere else %TODO: also on alphabet encodings - although somewhere else

13
src/GF/Command/Commands.hs
View File

@@ -360,11 +360,10 @@ allCommands cod pgf = Map.fromList [
"To see transliteration tables, use command ut." "To see transliteration tables, use command ut."
], ],
examples = [ examples = [
"l (EAdd 3 4) | ps -code -- linearize code-like output", "l (EAdd 3 4) | ps -code -- linearize code-like output",
"ps -lexer=code | p -cat=Exp -- parse code-like input", "ps -lexer=code | p -cat=Exp -- parse code-like input",
"gr -cat=QCl | l | ps -bind -to_utf8 -- linearization output from LangFin", "gr -cat=QCl | l | ps -bind -- linearization output from LangFin",
"ps -from_utf8 \"jag ?r h?r\" | p -- parser in LangSwe in UTF8 terminal", "ps -to_devanagari \"A-p\" -- show Devanagari in UTF8 terminal"
"ps -to_devanagari -to_utf8 \"A-p\" -- show Devanagari in UTF8 terminal"
], ],
exec = \opts -> return . fromString . stringOps (map prOpt opts) . toString, exec = \opts -> return . fromString . stringOps (map prOpt opts) . toString,
options = stringOpOptions options = stringOpOptions
@@ -626,7 +625,7 @@ stringOpOptions = [
("from_devanagari","from unicode to GF Devanagari transliteration"), ("from_devanagari","from unicode to GF Devanagari transliteration"),
("from_thai","from unicode to GF Telugu transliteration"), ("from_thai","from unicode to GF Telugu transliteration"),
("from_thai","from unicode to GF Thai transliteration"), ("from_thai","from unicode to GF Thai transliteration"),
("from_utf8","decode from utf8"), ("from_utf8","decode from utf8 (default)"),
("lextext","text-like lexer"), ("lextext","text-like lexer"),
("lexcode","code-like lexer"), ("lexcode","code-like lexer"),
("lexmixed","mixture of text and code (code between $...$)"), ("lexmixed","mixture of text and code (code between $...$)"),
@@ -636,7 +635,7 @@ stringOpOptions = [
("to_html","wrap in a html file with linebreaks"), ("to_html","wrap in a html file with linebreaks"),
("to_telugu","from GF Telugu transliteration to unicode"), ("to_telugu","from GF Telugu transliteration to unicode"),
("to_thai","from GF Thai transliteration to unicode"), ("to_thai","from GF Thai transliteration to unicode"),
("to_utf8","encode to utf8"), ("to_utf8","encode to utf8 (default)"),
("unlextext","text-like unlexer"), ("unlextext","text-like unlexer"),
("unlexcode","code-like unlexer"), ("unlexcode","code-like unlexer"),
("unlexmixed","mixture of text and code (code between $...$)"), ("unlexmixed","mixture of text and code (code between $...$)"),