Adding a basic lexicon-based tokenizer and the asociated command in gf shell

2026-04-22 11:19:32 -06:00 · 2011-02-10 15:00:06 +00:00
parent d7ae73f1c7
commit 45ecae4b77
3 changed files with 37 additions and 0 deletions
--- a/gf.cabal
+++ b/gf.cabal
@@ -22,6 +22,7 @@ flag interrupt
 library
  build-depends: base >= 4.2 && <5,
                 array,
+                 fst,
                 containers,
                 bytestring,
                 random,
@@ -42,6 +43,7 @@ library
    PGF.Expr
    PGF.Type
    PGF.Tree
+    PGF.Tokenizer
    PGF.Paraphrase
    PGF.TypeCheck
    PGF.Binary
@@ -72,6 +74,7 @@ executable gf
                 containers,
                 bytestring,
                 filepath,
+                 fst,
                 directory,
                 random,
                 old-time,