probabilistic model for the new syntax

src/experimental/transfer/TrainFreq.hs

@@ -0,0 +1,31 @@
+import Data.List
+import qualified Data.Map
+
+main = do
+
+  ws <- readFile "new-trees.txt" >>= return . words
+
+  let freqs ws = Data.Map.fromListWith (+) [(w,1) | w <- ws]
+
+  let freqmap = freqs ws
+
+  fs <- readFile "all-ndfuns.txt" >>= return . lines
+
+  let catf ws = case ws of f:ty -> (f,last(init ty))
+
+  let catmap = Data.Map.fromList [catf (words f) | f <- fs]
+
+  let allist = [(f,(c,Data.Map.lookup f freqmap)) | (f,c) <- Data.Map.assocs catmap]
+
+  let catlist = Data.List.sortBy (\(f,(c,_)) (_,(k,_)) -> compare c k) allist
+
+  let gcatlist = Data.List.groupBy (\(f,(c,_)) (_,(k,_)) -> c==k) catlist
+
+  let fcatfreqs fcs = let cat = fst (snd (head fcs)) in let tot = sum [maybe 0 id mn | (f,(c,mn)) <- fcs] in [(f,maybe 0 id mn, cat, tot) | (f,(c,mn)) <- fcs]
+
+  let fcatfreqlist = map fcatfreqs gcatlist
+
+  let relprobs cat = [(f, (fromIntegral i+1 :: Double) / (fromIntegral tot :: Double)) | (f,i,c,t) <- cat, let tot = t + length cat]
+
+--  writeFile "allFunFreqs.txt" $ unlines $ [unwords [f,show i,c,show t] | (f,i,c,t) <- concat fcatfreqlist]
+  writeFile "NDPredTrans.probs" $ unlines $ [unwords [f,show n] | (f,n) <- concatMap relprobs fcatfreqlist]