introduce the type synonym Token=String in the PGF API

src/compiler/GF/Speech/CFG.hs

@@ -27,7 +27,6 @@ import qualified Data.Set as Set
 --
 
 type Cat = String
-type Token = String
 
 data Symbol c t = NonTerminal c | Terminal t
   deriving (Eq, Ord, Show)

src/compiler/GF/Speech/SRGS_XML.hs

@@ -13,7 +13,7 @@ import GF.Speech.CFG
 import GF.Speech.RegExp
 import GF.Speech.SISR as SISR
 import GF.Speech.SRG
-import PGF (PGF, CId)
+import PGF (PGF, CId, Token)
 
 import Control.Monad
 import Data.Char (toUpper,toLower)

src/runtime/haskell/PGF.hs

@@ -54,7 +54,7 @@ module PGF(
            groupResults, -- lins of trees by language, removing duplicates
            showPrintName,
            
-           BracketedString(..), FId, LIndex,
+           BracketedString(..), FId, LIndex, Token,
            Forest.showBracketedString,
 
            -- ** Parsing

src/runtime/haskell/PGF/Data.hs

@@ -44,19 +44,20 @@ data Concr = Concr {
   totalCats    :: {-# UNPACK #-} !FId
   }
 
+type Token  = String
 type FId    = Int
 type LIndex = Int
 type DotPos = Int
 data Symbol
   = SymCat {-# UNPACK #-} !Int {-# UNPACK #-} !LIndex
   | SymLit {-# UNPACK #-} !Int {-# UNPACK #-} !LIndex
-  | SymKS [String]
-  | SymKP [String] [Alternative]
+  | SymKS [Token]
+  | SymKP [Token] [Alternative]
   deriving (Eq,Ord,Show)
 data Production
   = PApply  {-# UNPACK #-} !FunId [FId]
   | PCoerce {-# UNPACK #-} !FId
-  | PConst  CId Expr [String]
+  | PConst  CId Expr [Token]
   deriving (Eq,Ord,Show)
 data CncCat = CncCat {-# UNPACK #-} !FId {-# UNPACK #-} !FId {-# UNPACK #-} !(Array LIndex String)
 data CncFun = CncFun CId {-# UNPACK #-} !(UArray LIndex SeqId) deriving (Eq,Ord,Show)
@@ -65,7 +66,7 @@ type FunId = Int
 type SeqId = Int
 
 data Alternative =
-   Alt [String] [String]
+   Alt [Token] [String]
   deriving (Eq,Ord,Show)
 
 

src/runtime/haskell/PGF/Macros.hs

@@ -145,7 +145,7 @@ _V = mkCId "__gfV"
 -- as usual but we also want to retain the ''brackets'' that
 -- mark the beginning and the end of each constituent.
 data BracketedString
-  = Leaf String                                                                -- ^ this is the leaf i.e. a single token
+  = Leaf Token                                                                -- ^ this is the leaf i.e. a single token
   | Bracket CId {-# UNPACK #-} !FId {-# UNPACK #-} !LIndex [Expr] [BracketedString]   
                                                                                -- ^ this is a bracket. The 'CId' is the category of
                                                                                -- the phrase. The 'FId' is an unique identifier for
@@ -160,8 +160,8 @@ data BracketedString
                                                                                -- that represents the same constituent.
 
 data BracketedTokn
-  = LeafKS [String]
-  | LeafKP [String] [Alternative]
+  = LeafKS [Token]
+  | LeafKP [Token] [Alternative]
   | Bracket_ CId {-# UNPACK #-} !FId {-# UNPACK #-} !LIndex [Expr] [BracketedTokn]    -- Invariant: the list is not empty
   deriving Eq
 

src/runtime/haskell/PGF/Parse.hs

@@ -89,7 +89,7 @@ initState pgf lang (DTyp _ start _) =
 -- is consumed and the current position is shifted by one.
 -- If the new token cannot be accepted then an error state 
 -- is returned.
-nextState :: ParseState -> String -> Either ErrorState ParseState
+nextState :: ParseState -> Token -> Either ErrorState ParseState
 nextState (PState pgf cnc chart items) t =
   let (mb_agenda,map_items) = TMap.decompose items
       agenda = maybe [] Set.toList mb_agenda
@@ -147,7 +147,7 @@ feedLiteral (PState pgf cnc chart items) (ELit lit) =
 -- then the 'getCompletions' function can be used to calculate the possible
 -- next words and the consequent states. This is used for word completions in
 -- the GF interpreter.
-getCompletions :: ParseState -> String -> Map.Map String ParseState
+getCompletions :: ParseState -> String -> Map.Map Token ParseState
 getCompletions (PState pgf cnc chart items) w =
   let (mb_agenda,map_items) = TMap.decompose items
       agenda = maybe [] Set.toList mb_agenda
@@ -164,7 +164,7 @@ getCompletions (PState pgf cnc chart items) w =
       | isPrefixOf w tok    = Map.insertWith (TMap.unionWith Set.union) tok (TMap.singleton toks (Set.singleton item)) acc
     add _          item acc = acc
 
-recoveryStates :: [Type] -> ErrorState -> (ParseState, Map.Map String ParseState)
+recoveryStates :: [Type] -> ErrorState -> (ParseState, Map.Map Token ParseState)
 recoveryStates open_types (EState pgf cnc chart) =
   let open_fcats = concatMap type2fcats open_types
       agenda = foldl (complete open_fcats) [] (actives chart)