support parsing with literals

src/compiler/api/GF/Compile/CFGtoPGF.hs

@@ -95,7 +95,7 @@ cf2concr opts abstr cfg =
 
     mkSequence rule = snd $ mapAccumL convertSymbol 0 (ruleRhs rule)
       where
-        convertSymbol d (NonTerminal (c,_)) = (d+1,if c `elem` ["Int","Float","String"] then SymLit d 0 else SymCat d 0)
+        convertSymbol d (NonTerminal (c,_)) = (d+1,SymCat d 0)
         convertSymbol d (Terminal t)        = (d,  SymKS t)
 
     mkCncCat fid (cat,n)

src/compiler/api/GF/Compile/GrammarToPGF.hs

@@ -386,14 +386,6 @@ compareCaseInsensitive (x:xs) (y:ys) =
                       EQ -> r1 `compare` r2
                       x  -> x
                _ -> LT
-        SymLit d1 r1
-          -> case s2 of
-               SymCat {} -> GT
-               SymLit d2 r2
-                 -> case compare d1 d2 of
-                      EQ -> r1 `compare` r2
-                      x  -> x
-               _ -> LT
         SymVar d1 r1
           -> if tagToEnum# (getTag s2 ># 2#)
                then LT

src/compiler/api/GF/Compile/pgf.schema.json

@@ -203,7 +203,6 @@
           "type": "string",
           "enum": [
             "SymCat",
-            "SymLit",
             "SymVar",
             "SymKS",
             "SymKP",

src/compiler/api/GF/Grammar/Binary.hs

@@ -307,7 +307,6 @@ instance Binary Literal where
 
 instance Binary Symbol where
   put (SymCat d r)       = putWord8 0 >> put (d,r)
-  put (SymLit d r)       = putWord8 1 >> put (d,r)
   put (SymVar n l)       = putWord8 2 >> put (n,l)
   put (SymKS ts)         = putWord8 3 >> put ts
   put (SymKP d vs)       = putWord8 4 >> put (d,vs)
@@ -320,7 +319,6 @@ instance Binary Symbol where
   get = do tag <- getWord8
            case tag of
              0 -> liftM2 SymCat get get
-             1 -> liftM2 SymLit get get
              2 -> liftM2 SymVar get get
              3 -> liftM  SymKS  get
              4 -> liftM2 (\d vs -> SymKP d vs) get get

src/compiler/api/GF/Grammar/Printer.hs

@@ -380,7 +380,6 @@ ppLit (LInt n) = pp n
 ppLit (LFlt d) = pp d
 
 ppSymbol (SymCat d r)= pp '<' <> pp d <> pp ',' <> ppLParam r <> pp '>'
-ppSymbol (SymLit d r)= pp '{' <> pp d <> pp ',' <> ppLParam r <> pp '}'
 ppSymbol (SymVar d r)   = pp '<' <> pp d <> pp ',' <> pp '$' <> pp r <> pp '>'
 ppSymbol (SymKS t)      = doubleQuotes (pp t)
 ppSymbol SymNE          = pp "nonExist"

src/runtime/c/pgf/data.cxx

@@ -78,11 +78,6 @@ static void symbols_release(vector<PgfSymbol> syms)
             PgfDB::free(sym_cat, sym_cat->r.n_terms*sizeof(sym_cat->r.terms[0]));
             break;
         }
-        case PgfSymbolLit::tag: {
-            auto sym_lit = ref<PgfSymbolLit>::untagged(sym);
-            PgfDB::free(sym_lit, sym_lit->r.n_terms*sizeof(sym_lit->r.terms[0]));
-            break;
-        }
         case PgfSymbolVar::tag:
             PgfDB::free(ref<PgfSymbolVar>::untagged(sym));
             break;

src/runtime/c/pgf/data.h

@@ -159,12 +159,6 @@ struct PGF_INTERNAL_DECL PgfSymbolCat {
     PgfLParam r;
 };
 
-struct PGF_INTERNAL_DECL PgfSymbolLit {
-    static const uint8_t tag = 1;
-    size_t d;
-    PgfLParam r;
-};
-
 struct PGF_INTERNAL_DECL PgfSymbolVar {
     static const uint8_t tag = 2;
     size_t d, r;

src/runtime/c/pgf/expr.cxx

@@ -305,10 +305,10 @@ PgfType PgfInternalMarshaller::match_type(PgfUnmarshaller *u, PgfType ty)
                     tp->exprs.size(), tp->exprs.get_data());
 }
 
-PgfExprParser::PgfExprParser(PgfText *input, PgfUnmarshaller *unmarshaller)
+PgfExprParser::PgfExprParser(PgfText *input, size_t byte_pos, PgfUnmarshaller *unmarshaller)
 {
     inp = input;
-    pos = (const char*) &inp->text;
+    pos = (const char*) &inp->text[byte_pos];
 	ch  = ' ';
     u   = unmarshaller;
     token_pos = NULL;
@@ -354,11 +354,6 @@ void PgfExprParser::putc(uint32_t ucs)
     *(p++) = 0;
 }
 
-bool PgfExprParser::eof()
-{
-    return (token_tag == PGF_TOKEN_EOF);
-}
-
 PGF_INTERNAL bool
 pgf_is_ident_first(uint32_t ucs)
 {

src/runtime/c/pgf/expr.h

@@ -170,7 +170,7 @@ class PGF_INTERNAL_DECL PgfExprParser {
     void putc(uint32_t ch);
 
 public:
-    PgfExprParser(PgfText* input, PgfUnmarshaller *unmarshaller);
+    PgfExprParser(PgfText* input, size_t byte_pos, PgfUnmarshaller *unmarshaller);
     ~PgfExprParser();
 
     bool str_char();
@@ -189,8 +189,15 @@ public:
     PgfType parse_type();
     PgfTypeHypo *parse_context(size_t *p_n_hypos);
 
-    bool eof();
+    bool is_eof()   { return (token_tag == PGF_TOKEN_EOF); }
+    bool is_int()   { return (token_tag == PGF_TOKEN_INT); }
+    bool is_flt()   { return (token_tag == PGF_TOKEN_FLT); }
+    bool is_str()   { return (token_tag == PGF_TOKEN_STR); }
+    bool is_ident() { return (token_tag == PGF_TOKEN_IDENT); }
 
+    void ident2str() { token_tag = PGF_TOKEN_STR; }
+
+    const PgfText *get_token_value() { return token_value; }
     const char *get_token_pos() { return token_pos; }
 };
 

src/runtime/c/pgf/linearizer.cxx

@@ -100,13 +100,6 @@ bool PgfLinearizer::TreeNode::linearize_item(PgfLinearizationOutputIface *out, P
                 return false;
             break;
         }
-        case PgfSymbolLit::tag: {
-            auto sym_lit = ref<PgfSymbolLit>::untagged(sym);
-            size_t r = item->eval(ref<PgfLParam>::from_ptr(&sym_lit->r));
-            if (!linearize_arg(out, linearizer, sym_lit->d, r))
-                return false;
-            break;
-        }
         case PgfSymbolVar::tag: {
             auto sym_var = ref<PgfSymbolVar>::untagged(sym);
             linearize_var(out, linearizer, sym_var->d, sym_var->r);

src/runtime/c/pgf/parser.cxx

@@ -16,17 +16,22 @@ PgfAbstractParser::PgfAbstractParser(ref<PgfConcr> concr)
 
 PgfAbstractParser::ItemProbComparator PgfAbstractParser::item_prob_comp;
 
-void PgfAbstractParser::get_info(CCat *ccat, ref<PgfConcrRule> *prule, size_t **pvalues)
+bool PgfAbstractParser::get_info(CCat *ccat, ref<PgfConcrRule> *prule, size_t **pvalues)
 {
     if (ccat->fid <= concr->last_fid) {
+        if (ccat->epsilon->items.size() == 0)
+            return false;
         ref<PgfItem> pitem = ccat->epsilon->items[0];
         *prule = pitem->rule;
         *pvalues = &pitem->vars[0];
     } else {
+        if (ccat->prods.size() == 0)
+            return false;
         Production *prod = ccat->prods[0];
         *prule = prod->rule;
         *pvalues = &prod->vars[0];
     }
+    return true;
 }
 
 PgfAbstractParser::CCat *PgfAbstractParser::get_epsilon_ccat(PgfText *name, PgfMetaId fid)
@@ -57,7 +62,8 @@ PgfAbstractParser::CCat::~CCat()
         delete prod;
     }
     for (ExprState *estate : pending) {
-        delete estate;
+        if (estate != NULL)
+            delete estate;
     }
 }
 
@@ -493,18 +499,19 @@ void PgfAbstractParser::combine(State *state, Item *item, CCat *ccat)
 
     ref<PgfConcrRule> rule;
     size_t *values;
-    get_info(ccat, &rule,&values);
-    values = CLONE_VALUES(rule, values);
+    if (get_info(ccat, &rule,&values)) {
+        values = CLONE_VALUES(rule, values);
 
-    if (!instantiate(item->rule, &item->vars[0], item->rule->args[sym_cat->d],
-                     rule,       values,         rule->res)) {
-        delete item;
-        return;
-    }
-    if (!instantiate(item->rule, &item->vars[0], ref<PgfLParam>::from_ptr(&sym_cat->r),
-                     rule,       values,         rule->lin_idx)) {
-        delete item;
-        return;
+        if (!instantiate(item->rule, &item->vars[0], item->rule->args[sym_cat->d],
+                         rule,       values,         rule->res)) {
+            delete item;
+            return;
+        }
+        if (!instantiate(item->rule, &item->vars[0], ref<PgfLParam>::from_ptr(&sym_cat->r),
+                         rule,       values,         rule->lin_idx)) {
+            delete item;
+            return;
+        }
     }
 
     item->dot++;
@@ -877,7 +884,7 @@ PgfParser::~PgfParser()
                         continue;
 
                     for (ExprState *estate : it3.second->pending) {
-                        if (estate->expr != 0)
+                        if (estate != NULL && estate->expr != 0)
                             u->free_ref(estate->expr);
                     }
                     for (ExprProb &ep : it3.second->exprs) {
@@ -944,6 +951,64 @@ void PgfParser::bu_predict(PgfPhrasetable<PgfSymbolKS> phrasetable,
      }
 }
 
+void PgfParser::bu_literal(State *state, const char *name, PgfExprParser *eparser, prob_t viterbi_prob)
+{
+    PgfText* tname = string2text(name);
+    ref<PgfConcrLincat> lincat =
+        namespace_lookup(concr->lincats, tname);
+    free(tname);
+
+    if (lincat == 0)
+        return;
+
+    Cont *&cont = state->conts1[lincat];
+    if (cont == NULL) {
+        cont = new Cont;
+        cont->ccat = NULL;
+        cont->lincat = lincat;
+        cont->state = state;
+    }
+
+    PgfTextSpot current = cont->state->end;
+    current.byte_pos += eparser->get_token_value()->size;
+    current.pos      += eparser->get_token_value()->size;
+
+    State *next_state = new_state(current, viterbi_prob);
+
+    interval_t zero = {0,0};
+    CCat *&ccat = next_state->completed[cont][zero][zero];
+    if (ccat == NULL) {
+        ccat = new CCat;
+        ccat->fid = (++last_fid);
+        ccat->epsilon = 0;
+        ccat->cont  = cont;
+        ccat->state = next_state;
+        ccat->lin_idx = zero;
+        ccat->value = zero;
+        ccat->covered = false;
+        ccat->viterbi_prob = 0;
+
+#ifdef DEBUG_PARSER
+        {
+            PgfPrinter printer(NULL,0,NULL);
+            printer.nprintf(64,"literal [%zd-%zd; ",state->end.pos,next_state->start.pos);
+            printer.efun(&ccat->cont->lincat->name);
+            printer.puts("(0); 0; ");
+            printer.emeta(ccat->fid);
+            printer.puts("]");
+            PgfText *text = printer.get_text();
+            fprintf(stderr, "%s\n", text->text);
+            free(text);
+        }
+#endif
+
+        ccat->pending.push_back(NULL);
+        ccat->exprs.emplace_back(eparser->parse_term(), 0, 0);
+    }
+
+    bu_predict(next_state, viterbi_prob, ccat);
+}
+
 void PgfParser::bu_predict(PgfPhrasetable<PgfSymbolBIND> phrasetable,
                            State *state, prob_t outside_prob)
 {
@@ -1492,6 +1557,15 @@ void PgfParser::suspend(Cont *cont,Item *item,bool do_predict,ref<PgfSymbolCat>
                 if (cont->state->needs_bind) {
                     bu_predict(concr->phrasetable4, cont->state, viterbi_prob);
                 } else {
+                    PgfExprParser eparser(sentence, cont->state->end.byte_pos, u);
+                    if (eparser.is_int()) {
+                        bu_literal(cont->state, "Int", &eparser, viterbi_prob);
+                    } else if (eparser.is_flt()) {
+                        bu_literal(cont->state, "Float", &eparser, viterbi_prob);
+                    } else if (eparser.is_ident()) {
+                        eparser.ident2str();
+                        bu_literal(cont->state, "String", &eparser, viterbi_prob);
+                    }
                     bu_predict(concr->phrasetable1, cont->state, viterbi_prob, 1, sentence->size);
                 }
             }

src/runtime/c/pgf/parser.h

@@ -257,7 +257,7 @@ protected:
     static
     interval_t interval(ref<PgfConcrRule> rule, size_t *values, ref<PgfLParam> lparam);
 
-    void get_info(CCat *ccat, ref<PgfConcrRule> *rule, size_t **pvalues);
+    bool get_info(CCat *ccat, ref<PgfConcrRule> *rule, size_t **pvalues);
     CCat *get_epsilon_ccat(PgfText *name, PgfMetaId fid);
 
     static
@@ -292,6 +292,7 @@ class PGF_INTERNAL_DECL PgfParser : private PgfAbstractParser, public PgfExprEnu
     virtual void bu_predict(State *state, prob_t outside_prob, CCat *ccat);
 
     void bu_predict(PgfPhrasetable<PgfSymbolBIND> phrasetable, State *state, prob_t outside_prob);
+    void bu_literal(State *state, const char *name, PgfExprParser *eparser, prob_t viterbi_prob);
     void bu_predict(PgfPhrasetable<PgfSymbolKS> phrasetable, State *state, prob_t outside_prob, ptrdiff_t min, ptrdiff_t max);
     void make_chunks(State *state, std::vector<CCat*> &chunks, prob_t prob);
     PgfExpr process_expr(ExprState *estate, prob_t *prob);

src/runtime/c/pgf/pgf.cxx

@@ -774,9 +774,9 @@ PgfText *pgf_print_ident(PgfText *name)
 PGF_API
 PgfExpr pgf_read_expr(PgfText *input, PgfUnmarshaller *u)
 {
-    PgfExprParser parser(input, u);
+    PgfExprParser parser(input, 0, u);
     PgfExpr res = parser.parse_expr();
-    if (!parser.eof()) {
+    if (!parser.is_eof()) {
         if (res != 0)
             u->free_ref(res);
         return 0;
@@ -787,7 +787,7 @@ PgfExpr pgf_read_expr(PgfText *input, PgfUnmarshaller *u)
 PGF_API
 PgfExpr pgf_read_expr_ex(PgfText *input, const char **end_pos, PgfUnmarshaller *u)
 {
-    PgfExprParser parser(input, u);
+    PgfExprParser parser(input, 0, u);
     PgfExpr expr = parser.parse_expr();
     *end_pos = parser.get_token_pos();
     return expr;
@@ -838,9 +838,9 @@ PgfText *pgf_print_context(size_t n_hypos, PgfTypeHypo *hypos,
 PGF_API
 PgfType pgf_read_type(PgfText *input, PgfUnmarshaller *u)
 {
-    PgfExprParser parser(input, u);
+    PgfExprParser parser(input, 0, u);
     PgfType res = parser.parse_type();
-    if (!parser.eof()) {
+    if (!parser.is_eof()) {
         u->free_ref(res);
         return 0;
     }
@@ -850,9 +850,9 @@ PgfType pgf_read_type(PgfText *input, PgfUnmarshaller *u)
 PGF_API
 PgfTypeHypo *pgf_read_context(PgfText *input, PgfUnmarshaller *u, size_t *n_hypos)
 {
-    PgfExprParser parser(input, u);
+    PgfExprParser parser(input, 0, u);
     PgfTypeHypo *res = parser.parse_context(n_hypos);
-    if (!parser.eof()) {
+    if (!parser.is_eof()) {
         for (size_t i = 0; i < *n_hypos; i++) {
             free(res[i].cid);
             u->free_ref(res[i].type);
@@ -2050,33 +2050,6 @@ public:
         } PGF_API_END
     }
 
-    void add_symlit(size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err)
-    {
-        if (err->type != PGF_EXN_NONE)
-            return;
-
-        PGF_API_BEGIN {
-            if (syms == 0 || sym_index == (size_t) -1 || sym_index >= syms.size())
-                throw pgf_error(builder_error_msg);
-
-            if (d > n_args)
-                throw pgf_error(builder_error_msg);
-
-            ref<PgfSymbolLit> symlit = PgfDB::malloc<PgfSymbolLit>(n_terms*2*sizeof(size_t));
-            symlit->d  = d;
-            symlit->r.i0 = i0;
-            symlit->r.n_terms = n_terms;
-
-            for (size_t i = 0; i < n_terms; i++) {
-                symlit->r.terms[i].factor = terms[2*i];
-                symlit->r.terms[i].var    = terms[2*i+1];
-            }
-
-            syms[sym_index] = symlit.tagged();
-            sym_index++;
-        } PGF_API_END
-    }
-
     void add_symvar(size_t d, size_t r, PgfExn *err)
     {
         if (err->type != PGF_EXN_NONE)

src/runtime/c/pgf/pgf.h

@@ -638,7 +638,6 @@ struct PgfLinBuilderIface {
     virtual void set_lin_idx(size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0;
     virtual void add_variable(size_t range, PgfExn *err)=0;
     virtual void add_symcat(size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0;
-    virtual void add_symlit(size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err)=0;
     virtual void add_symvar(size_t d, size_t r, PgfExn *err)=0;
     virtual void add_symks(PgfText *token, PgfExn *err)=0;
     virtual void start_symkp(size_t n_syms, size_t n_alts, PgfExn *err)=0;
@@ -667,7 +666,6 @@ typedef struct {
     void (*set_lin_idx)(PgfLinBuilderIface *this, size_t i0, size_t n_terms, size_t *terms, PgfExn *err);
     void (*add_variable)(PgfLinBuilderIface *this, size_t range, PgfExn *err);
     void (*add_symcat)(PgfLinBuilderIface *this, size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err);
-    void (*add_symlit)(PgfLinBuilderIface *this, size_t d, size_t i0, size_t n_terms, size_t *terms, PgfExn *err);
     void (*add_symvar)(PgfLinBuilderIface *this, size_t d, size_t r, PgfExn *err);
     void (*add_symks)(PgfLinBuilderIface *this, PgfText *token, PgfExn *err);
     void (*start_symkp)(PgfLinBuilderIface *this, size_t n_syms, size_t n_alts, PgfExn *err);

src/runtime/c/pgf/printer.cxx

@@ -524,13 +524,6 @@ void PgfPrinter::symbol(PgfSymbol sym)
         puts(">");
 		break;
 	}
-	case PgfSymbolLit::tag: {
-        auto sym_lit = ref<PgfSymbolLit>::untagged(sym);
-        nprintf(32, "{%ld,",sym_lit->d);
-        lparam(ref<PgfLParam>::from_ptr(&sym_lit->r));
-        puts("}");
-		break;
-	}
 	case PgfSymbolVar::tag: {
         auto sym_var = ref<PgfSymbolVar>::untagged(sym);
         nprintf(64, "<%ld,$%ld>",sym_var->d, sym_var->r);

src/runtime/c/pgf/reader.cxx

@@ -537,11 +537,6 @@ PgfSymbol PgfReader::read_symbol()
         ref<PgfSymbolCat> sym_cat = read_symbol_idx<PgfSymbolCat>();
         sym = sym_cat.tagged();
 		break;
-    }
-	case PgfSymbolLit::tag: {
-        ref<PgfSymbolLit> sym_lit = read_symbol_idx<PgfSymbolLit>();
-        sym = sym_lit.tagged();
-		break;
     }
 	case PgfSymbolVar::tag: {
         ref<PgfSymbolVar> sym_var = PgfDB::malloc<PgfSymbolVar>();

src/runtime/c/pgf/writer.cxx

@@ -332,12 +332,6 @@ void PgfWriter::write_symbol(PgfSymbol sym)
         write_lparam(ref<PgfLParam>::from_ptr(&sym_cat->r));
 		break;
 	}
-	case PgfSymbolLit::tag: {
-        auto sym_lit = ref<PgfSymbolLit>::untagged(sym);
-        write_int(sym_lit->d);
-        write_lparam(ref<PgfLParam>::from_ptr(&sym_lit->r));
-		break;
-	}
 	case PgfSymbolVar::tag: {
         auto sym_var = ref<PgfSymbolVar>::untagged(sym);
         write_int(sym_var->d);

src/runtime/haskell/PGF2/Transactions.hsc

@@ -246,7 +246,6 @@ data LParam  = LParam {-# UNPACK #-} !LIndex [(LIndex,LVar)]
 
 data Symbol
   = SymCat {-# UNPACK #-} !Int {-# UNPACK #-} !LParam
-  | SymLit {-# UNPACK #-} !Int {-# UNPACK #-} !LParam
   | SymVar {-# UNPACK #-} !Int {-# UNPACK #-} !Int
   | SymKS Token
   | SymKP [Symbol] [([Symbol],[String])]
@@ -337,9 +336,6 @@ withBuildLinIface rules f = do
     addSymbol c_builder vtbl c_exn (SymCat d r) = do
       fun <- (#peek PgfLinBuilderIfaceVtbl, add_symcat) vtbl
       callLParam (callLinBuilder4 fun c_builder (fromIntegral d)) r c_exn
-    addSymbol c_builder vtbl c_exn (SymLit d r) = do
-      fun <- (#peek PgfLinBuilderIfaceVtbl, add_symlit) vtbl
-      callLParam (callLinBuilder4 fun c_builder (fromIntegral d)) r c_exn
     addSymbol c_builder vtbl c_exn (SymVar d r) = do
       fun <- (#peek PgfLinBuilderIfaceVtbl, add_symvar) vtbl
       callLinBuilder2 fun c_builder (fromIntegral d) (fromIntegral r) c_exn