Adding a basic lexicon-based tokenizer and the asociated command in gf shell

src/runtime/haskell/PGF.hs

@@ -109,6 +109,9 @@ module PGF(
            -- ** Morphological Analysis
            Lemma, Analysis, Morpho,
            lookupMorpho, buildMorpho, fullFormLexicon,
+           
+           -- ** Tokenizing
+           mkTokenizer,
 
            -- ** Visualizations
            graphvizAbstractTree,
@@ -141,6 +144,7 @@ import PGF.Expr (Tree)
 import PGF.Morphology
 import PGF.Data
 import PGF.Binary
+import PGF.Tokenizer
 import qualified PGF.Forest as Forest
 import qualified PGF.Parse as Parse