diff --git a/src/compiler/GF/Speech/CFG.hs b/src/compiler/GF/Speech/CFG.hs index 9ec8416c5..f396faace 100644 --- a/src/compiler/GF/Speech/CFG.hs +++ b/src/compiler/GF/Speech/CFG.hs @@ -27,7 +27,6 @@ import qualified Data.Set as Set -- type Cat = String -type Token = String data Symbol c t = NonTerminal c | Terminal t deriving (Eq, Ord, Show) diff --git a/src/compiler/GF/Speech/SRGS_XML.hs b/src/compiler/GF/Speech/SRGS_XML.hs index 1f94de66d..5f3579406 100644 --- a/src/compiler/GF/Speech/SRGS_XML.hs +++ b/src/compiler/GF/Speech/SRGS_XML.hs @@ -13,7 +13,7 @@ import GF.Speech.CFG import GF.Speech.RegExp import GF.Speech.SISR as SISR import GF.Speech.SRG -import PGF (PGF, CId) +import PGF (PGF, CId, Token) import Control.Monad import Data.Char (toUpper,toLower) diff --git a/src/runtime/haskell/PGF.hs b/src/runtime/haskell/PGF.hs index 2f9f21e1e..a4d9f4aa1 100644 --- a/src/runtime/haskell/PGF.hs +++ b/src/runtime/haskell/PGF.hs @@ -54,7 +54,7 @@ module PGF( groupResults, -- lins of trees by language, removing duplicates showPrintName, - BracketedString(..), FId, LIndex, + BracketedString(..), FId, LIndex, Token, Forest.showBracketedString, -- ** Parsing diff --git a/src/runtime/haskell/PGF/Data.hs b/src/runtime/haskell/PGF/Data.hs index 8b2fb41f8..fd357a678 100644 --- a/src/runtime/haskell/PGF/Data.hs +++ b/src/runtime/haskell/PGF/Data.hs @@ -44,19 +44,20 @@ data Concr = Concr { totalCats :: {-# UNPACK #-} !FId } +type Token = String type FId = Int type LIndex = Int type DotPos = Int data Symbol = SymCat {-# UNPACK #-} !Int {-# UNPACK #-} !LIndex | SymLit {-# UNPACK #-} !Int {-# UNPACK #-} !LIndex - | SymKS [String] - | SymKP [String] [Alternative] + | SymKS [Token] + | SymKP [Token] [Alternative] deriving (Eq,Ord,Show) data Production = PApply {-# UNPACK #-} !FunId [FId] | PCoerce {-# UNPACK #-} !FId - | PConst CId Expr [String] + | PConst CId Expr [Token] deriving (Eq,Ord,Show) data CncCat = CncCat {-# UNPACK #-} !FId {-# UNPACK #-} !FId {-# UNPACK #-} !(Array LIndex String) data CncFun = CncFun CId {-# UNPACK #-} !(UArray LIndex SeqId) deriving (Eq,Ord,Show) @@ -65,7 +66,7 @@ type FunId = Int type SeqId = Int data Alternative = - Alt [String] [String] + Alt [Token] [String] deriving (Eq,Ord,Show) diff --git a/src/runtime/haskell/PGF/Macros.hs b/src/runtime/haskell/PGF/Macros.hs index f6116ba60..f0d9b92a8 100644 --- a/src/runtime/haskell/PGF/Macros.hs +++ b/src/runtime/haskell/PGF/Macros.hs @@ -145,7 +145,7 @@ _V = mkCId "__gfV" -- as usual but we also want to retain the ''brackets'' that -- mark the beginning and the end of each constituent. data BracketedString - = Leaf String -- ^ this is the leaf i.e. a single token + = Leaf Token -- ^ this is the leaf i.e. a single token | Bracket CId {-# UNPACK #-} !FId {-# UNPACK #-} !LIndex [Expr] [BracketedString] -- ^ this is a bracket. The 'CId' is the category of -- the phrase. The 'FId' is an unique identifier for @@ -160,8 +160,8 @@ data BracketedString -- that represents the same constituent. data BracketedTokn - = LeafKS [String] - | LeafKP [String] [Alternative] + = LeafKS [Token] + | LeafKP [Token] [Alternative] | Bracket_ CId {-# UNPACK #-} !FId {-# UNPACK #-} !LIndex [Expr] [BracketedTokn] -- Invariant: the list is not empty deriving Eq diff --git a/src/runtime/haskell/PGF/Parse.hs b/src/runtime/haskell/PGF/Parse.hs index 0f3a885e5..073e2516a 100644 --- a/src/runtime/haskell/PGF/Parse.hs +++ b/src/runtime/haskell/PGF/Parse.hs @@ -89,7 +89,7 @@ initState pgf lang (DTyp _ start _) = -- is consumed and the current position is shifted by one. -- If the new token cannot be accepted then an error state -- is returned. -nextState :: ParseState -> String -> Either ErrorState ParseState +nextState :: ParseState -> Token -> Either ErrorState ParseState nextState (PState pgf cnc chart items) t = let (mb_agenda,map_items) = TMap.decompose items agenda = maybe [] Set.toList mb_agenda @@ -147,7 +147,7 @@ feedLiteral (PState pgf cnc chart items) (ELit lit) = -- then the 'getCompletions' function can be used to calculate the possible -- next words and the consequent states. This is used for word completions in -- the GF interpreter. -getCompletions :: ParseState -> String -> Map.Map String ParseState +getCompletions :: ParseState -> String -> Map.Map Token ParseState getCompletions (PState pgf cnc chart items) w = let (mb_agenda,map_items) = TMap.decompose items agenda = maybe [] Set.toList mb_agenda @@ -164,7 +164,7 @@ getCompletions (PState pgf cnc chart items) w = | isPrefixOf w tok = Map.insertWith (TMap.unionWith Set.union) tok (TMap.singleton toks (Set.singleton item)) acc add _ item acc = acc -recoveryStates :: [Type] -> ErrorState -> (ParseState, Map.Map String ParseState) +recoveryStates :: [Type] -> ErrorState -> (ParseState, Map.Map Token ParseState) recoveryStates open_types (EState pgf cnc chart) = let open_fcats = concatMap type2fcats open_types agenda = foldl (complete open_fcats) [] (actives chart)