uniform encoding: gfo and pgf in UTF8, internal in unicode

2008-06-26 16:35:45 +00:00
parent 712717e461
commit 7f9a0cdd34
41 changed files with 159 additions and 73 deletions
--- a/lib/resource-1.4/Make.hs
+++ b/lib/resource-1.4/Make.hs
@@ -19,21 +19,21 @@ import System
 
 langsCoding = [
  (("arabic",   "Ara"),""),
-  (("bulgarian","Bul"),"from_cp1251,to_utf8"),
-  (("catalan",  "Cat"),"to_utf8"),
-  (("danish",   "Dan"),"to_utf8"),
+  (("bulgarian","Bul"),""),
+  (("catalan",  "Cat"),""),
+  (("danish",   "Dan"),""),
  (("english",  "Eng"),""),
-  (("finnish",  "Fin"),"to_utf8"),
-  (("french",   "Fre"),"to_utf8"),
-  (("hindi",    "Hin"),"to_devanagari,to_utf8"),
-  (("german",   "Ger"),"to_utf8"),
+  (("finnish",  "Fin"),""),
+  (("french",   "Fre"),""),
+  (("hindi",    "Hin"),"to_devanagari"),
+  (("german",   "Ger"),""),
  (("interlingua","Ina"),""),
-  (("italian",  "Ita"),"to_utf8"),
-  (("norwegian","Nor"),"to_utf8"),
+  (("italian",  "Ita"),""),
+  (("norwegian","Nor"),""),
  (("russian",  "Rus"),""),
-  (("spanish",  "Spa"),"to_utf8"),
-  (("swedish",  "Swe"),"to_utf8"), 
-  (("thai",     "Tha"),"to_thai,to_utf8")
+  (("spanish",  "Spa"),""),
+  (("swedish",  "Swe"),""), 
+  (("thai",     "Tha"),"to_thai")
  ]

 langs = map fst langsCoding
--- a/lib/resource-1.4/bulgarian/AdjectiveBul.gf
+++ b/lib/resource-1.4/bulgarian/AdjectiveBul.gf
@@ -1,4 +1,6 @@
 concrete AdjectiveBul of Adjective = CatBul ** open ResBul, Prelude in {
+  flags coding=cp1251 ;
+
  lin
    PositA  a = {
      s = \\aform => a.s ! aform ;
--- a/lib/resource-1.4/bulgarian/AdverbBul.gf
+++ b/lib/resource-1.4/bulgarian/AdverbBul.gf
@@ -1,4 +1,6 @@
 concrete AdverbBul of Adverb = CatBul ** open ResBul, Prelude in {
+  flags coding=cp1251 ;
+
  lin
    PositAdvAdj a = {s = a.adv} ;
    ComparAdvAdj cadv a np = {
--- a/lib/resource-1.4/bulgarian/Bulgarian.gf
+++ b/lib/resource-1.4/bulgarian/Bulgarian.gf
@@ -3,4 +3,6 @@
 concrete Bulgarian of BulgarianAbs = 
  LangBul,
  ExtraBul
-  ** {} ;
+  ** {
+  flags coding=cp1251 ;
+} ;
--- a/lib/resource-1.4/bulgarian/BulgarianAbs.gf
+++ b/lib/resource-1.4/bulgarian/BulgarianAbs.gf
@@ -3,4 +3,6 @@
 abstract BulgarianAbs = 
  Lang,
  ExtraBulAbs
-  ** {} ;
+  ** {
+  flags coding=cp1251 ;
+} ;
--- a/lib/resource-1.4/bulgarian/CatBul.gf
+++ b/lib/resource-1.4/bulgarian/CatBul.gf
@@ -1,4 +1,6 @@
-concrete CatBul of Cat = open ResBul, Prelude, (R = ParamX) in {
+concrete CatBul of Cat = open ResBul, Prelude, (R = ParamX) in {
+  flags coding=cp1251 ;
+

  flags optimize=all_subs ;

--- a/lib/resource-1.4/bulgarian/ConjunctionBul.gf
+++ b/lib/resource-1.4/bulgarian/ConjunctionBul.gf
@@ -1,5 +1,7 @@
 concrete ConjunctionBul of Conjunction = 
  CatBul ** open ResBul, Coordination, Prelude in {
+  flags coding=cp1251 ;
+

  flags optimize=all_subs ;

--- a/lib/resource-1.4/bulgarian/ExtraBul.gf
+++ b/lib/resource-1.4/bulgarian/ExtraBul.gf
@@ -1,5 +1,7 @@
 concrete ExtraBul of ExtraBulAbs = CatBul ** 
  open ResBul, Coordination, Prelude in {
+  flags coding=cp1251 ;
+

  lin
    PossIndefPron p = {
--- a/lib/resource-1.4/bulgarian/ExtraBulAbs.gf
+++ b/lib/resource-1.4/bulgarian/ExtraBulAbs.gf
@@ -1,4 +1,6 @@
 abstract ExtraBulAbs = Extra ** {
+  flags coding=cp1251 ;
+

 fun
 -- Feminine variants of pronouns (those in $Structural$ are
--- a/lib/resource-1.4/bulgarian/GrammarBul.gf
+++ b/lib/resource-1.4/bulgarian/GrammarBul.gf
@@ -15,6 +15,8 @@ concrete GrammarBul of Grammar =
  StructuralBul,
  IdiomBul
  ** {
+  flags coding=cp1251 ;
+

 flags startcat = Phr ; unlexer = text ; lexer = text ;

--- a/lib/resource-1.4/bulgarian/IdiomBul.gf
+++ b/lib/resource-1.4/bulgarian/IdiomBul.gf
@@ -1,4 +1,6 @@
 concrete IdiomBul of Idiom = CatBul ** open Prelude, ParadigmsBul, ResBul in {
+  flags coding=cp1251 ;
+
  flags optimize=all_subs ;

  lin
--- a/lib/resource-1.4/bulgarian/LangBul.gf
+++ b/lib/resource-1.4/bulgarian/LangBul.gf
@@ -4,6 +4,8 @@ concrete LangBul of Lang =
  GrammarBul,
  LexiconBul
  ** {
+  flags coding=cp1251 ;
+

 flags startcat = Phr ; unlexer = text ; lexer = text ; erasing = on ; coding = cp1251 ;

--- a/lib/resource-1.4/bulgarian/LexiconBul.gf
+++ b/lib/resource-1.4/bulgarian/LexiconBul.gf
@@ -2,6 +2,8 @@

 concrete LexiconBul of Lexicon = CatBul ** 
  open ParadigmsBul, ResBul, Prelude in {
+  flags coding=cp1251 ;
+

 flags 
  optimize=values ;
--- a/lib/resource-1.4/bulgarian/MorphoBul.gf
+++ b/lib/resource-1.4/bulgarian/MorphoBul.gf
@@ -12,7 +12,9 @@ resource MorphoBul = ResBul ** open
  Predef,
  Prelude,
  CatBul
-  in {
+  in {
+  flags coding=cp1251 ;
+

  flags optimize=all ;

--- a/lib/resource-1.4/bulgarian/MorphoFunsBul.gf
+++ b/lib/resource-1.4/bulgarian/MorphoFunsBul.gf
@@ -4,7 +4,9 @@ resource MorphoFunsBul = open
  Prelude, 
  CatBul,
  MorphoBul
-  in {
+  in {
+  flags coding=cp1251 ;
+

 oper
 --2 Adverbs
--- a/lib/resource-1.4/bulgarian/NounBul.gf
+++ b/lib/resource-1.4/bulgarian/NounBul.gf
@@ -1,4 +1,6 @@
-concrete NounBul of Noun = CatBul ** open ResBul, Prelude in {
+concrete NounBul of Noun = CatBul ** open ResBul, Prelude in {
+  flags coding=cp1251 ;
+

  flags optimize=all_subs ;

--- a/lib/resource-1.4/bulgarian/NumeralBul.gf
+++ b/lib/resource-1.4/bulgarian/NumeralBul.gf
@@ -1,4 +1,6 @@
 concrete NumeralBul of Numeral = CatBul ** open Prelude, ResBul in {
+  flags coding=cp1251 ;
+

 lincat 
  Digit      = {s : DForm => CardOrd => Str} ;
--- a/lib/resource-1.4/bulgarian/ParadigmsBul.gf
+++ b/lib/resource-1.4/bulgarian/ParadigmsBul.gf
@@ -3,7 +3,9 @@ resource ParadigmsBul = MorphoFunsBul ** open
  Prelude,
  MorphoBul,
  CatBul
-  in {
+  in {
+  flags coding=cp1251 ;
+
 oper
  mkN001 : Str -> N ;
  mkN001 base = let v0 = base
--- a/lib/resource-1.4/bulgarian/PhraseBul.gf
+++ b/lib/resource-1.4/bulgarian/PhraseBul.gf
@@ -1,4 +1,6 @@
 concrete PhraseBul of Phrase = CatBul ** open Prelude, ResBul in {
+  flags coding=cp1251 ;
+

  lin
    PhrUtt pconj utt voc = {s = pconj.s ++ utt.s ++ voc.s} ;
--- a/lib/resource-1.4/bulgarian/QuestionBul.gf
+++ b/lib/resource-1.4/bulgarian/QuestionBul.gf
@@ -1,4 +1,6 @@
 concrete QuestionBul of Question = CatBul ** open ResBul, Prelude in {
+  flags coding=cp1251 ;
+

  flags optimize=all_subs ;

--- a/lib/resource-1.4/bulgarian/RelativeBul.gf
+++ b/lib/resource-1.4/bulgarian/RelativeBul.gf
@@ -1,4 +1,6 @@
 concrete RelativeBul of Relative = CatBul ** open ResBul in {
+  flags coding=cp1251 ;
+

  flags optimize=all_subs ;

--- a/lib/resource-1.4/bulgarian/ResBul.gf
+++ b/lib/resource-1.4/bulgarian/ResBul.gf
@@ -8,6 +8,8 @@
 -- patterns needed for $Lex$.

 resource ResBul = ParamX ** open Prelude in {
+  flags coding=cp1251 ;
+

  flags optimize=all ;

--- a/lib/resource-1.4/bulgarian/SentenceBul.gf
+++ b/lib/resource-1.4/bulgarian/SentenceBul.gf
@@ -1,4 +1,6 @@
 concrete SentenceBul of Sentence = CatBul ** open Prelude, ResBul in {
+  flags coding=cp1251 ;
+

  flags optimize=all_subs ;

--- a/lib/resource-1.4/bulgarian/StructuralBul.gf
+++ b/lib/resource-1.4/bulgarian/StructuralBul.gf
@@ -1,5 +1,7 @@
 concrete StructuralBul of Structural = CatBul ** 
  open MorphoBul, ParadigmsBul, Prelude in {
+  flags coding=cp1251 ;
+

  flags optimize=all ;

--- a/lib/resource-1.4/bulgarian/TextBul.gf
+++ b/lib/resource-1.4/bulgarian/TextBul.gf
@@ -1,4 +1,6 @@
 concrete TextBul of Text = CatBul ** {
+  flags coding=cp1251 ;
+

 -- This will work for almost all languages except Spanish.

--- a/lib/resource-1.4/bulgarian/VerbBul.gf
+++ b/lib/resource-1.4/bulgarian/VerbBul.gf
@@ -1,4 +1,6 @@
 concrete VerbBul of Verb = CatBul ** open Prelude, ResBul, ParadigmsBul in {
+  flags coding=cp1251 ;
+

  flags optimize=all_subs ;

--- a/lib/resource-1.4/demo/DemoRus.gf
+++ b/lib/resource-1.4/demo/DemoRus.gf
@@ -18,6 +18,6 @@ concrete DemoRus of Demo =
  LexiconRus
  ** {

-flags startcat = Phr ; unlexer = text ; lexer = text ;
+flags startcat = Phr ; unlexer = text ; lexer = text ; coding = utf8 ;

 } ;
--- a/lib/resource-1.4/mathematical/SymbolGer.gf
+++ b/lib/resource-1.4/mathematical/SymbolGer.gf
@@ -4,7 +4,7 @@ lin
  SymbPN i = {s = \\c => i.s ; g = Neutr} ; --- c
  IntPN i  = {s = \\c => i.s ; g = Neutr} ; --- c
  FloatPN i  = {s = \\c => i.s ; g = Neutr} ; --- c
-  NumPN i  = {s = \\c => i.s ; g = Neutr} ; --- c
+  NumPN i  = {s = i.s ! Neutr ; g = Neutr} ; --- c

  CNIntNP cn i = {
    s = \\c => cn.s ! Weak ! Sg ! Nom ++ i.s ;
@@ -17,14 +17,14 @@ lin
    isPron = False
    } ;
  CNNumNP cn i = {
-    s = \\c => artDef ! (GSg cn.g) ! c ++ cn.s ! Weak ! Sg ! Nom ++ i.s ;
+    s = \\c => artDef ! (GSg cn.g) ! c ++ cn.s ! Weak ! Sg ! Nom ++ i.s ! Neutr ! c ;
    a = agrP3 Sg ;
    isPron = False
    } ;

  SymbS sy = {s = \\_ => sy.s} ;

-  SymbNum n = {s = n.s ; n = Pl ; isNum = True} ;
+  SymbNum n = {s = \\_,_ => n.s ; n = Pl ; isNum = True} ;
  SymbOrd n = {s = \\_   => n.s ++ "."} ;


--- a/lib/resource-1.4/russian/ExtraRus.gf
+++ b/lib/resource-1.4/russian/ExtraRus.gf
@@ -1,3 +1,3 @@
 concrete ExtraRus of ExtraRusAbs = CatRus ** {
-
+flags coding=utf8 ;
 } 
--- a/lib/resource-1.4/russian/GrammarRus.gf
+++ b/lib/resource-1.4/russian/GrammarRus.gf
@@ -14,4 +14,4 @@ concrete GrammarRus of Grammar =
  TextX,
  StructuralRus,
  IdiomRus
-   ** { flags  startcat = Phr ; unlexer = text ; lexer = text ;} ;
+   ** { flags  startcat = Phr ; unlexer = text ; lexer = text ; coding=utf8 ;} ;
--- a/lib/resource-1.4/russian/IrregRus.gf
+++ b/lib/resource-1.4/russian/IrregRus.gf
@@ -3,7 +3,7 @@
 --
 concrete IrregRus of IrregRusAbs = CatRus ** open ParadigmsRus in {
 --
--flags optimize=values ;
+flags optimize=values ; coding=utf8 ;
 --
 --  lin
 --  awake_V = irregV "awake" "awoke" "awoken" ;
--- a/lib/resource-1.4/russian/NounRus.gf
+++ b/lib/resource-1.4/russian/NounRus.gf
@@ -2,7 +2,7 @@

 concrete NounRus of Noun = CatRus ** open ResRus, Prelude, MorphoRus in {

-  flags optimize=all_subs ;
+  flags optimize=all_subs ; coding=utf8 ;

  lin
    DetCN kazhduj okhotnik = {
--- a/lib/resource-1.4/russian/PhraseRus.gf
+++ b/lib/resource-1.4/russian/PhraseRus.gf
@@ -2,6 +2,7 @@

 concrete PhraseRus of Phrase = CatRus ** open Prelude, ResRus in {

+  flags coding=utf8 ;
  lin
    PhrUtt pconj utt voc = {s = pconj.s ++ utt.s ++ voc.s} ;

--- a/lib/resource-1.4/russian/QuestionRus.gf
+++ b/lib/resource-1.4/russian/QuestionRus.gf
@@ -2,7 +2,7 @@

 concrete QuestionRus of Question = CatRus ** open ResRus, Prelude in {

-  flags optimize=all_subs ;
+  flags optimize=all_subs ; coding=utf8 ;

  lin

--- a/lib/resource-1.4/russian/Russian.gf
+++ b/lib/resource-1.4/russian/Russian.gf
@@ -3,4 +3,4 @@
 concrete Russian of RussianAbs = 
  LangRus,
  ExtraRus
-  ** {} ;
+  ** {flags coding=utf8 ;} ;