This patch removes Gregoire's parse_tokens function in the python binding and adds another implementation which builds on the existing API for lexers in the C runtime. Now it is possible to write incremental Lexers in Python

src/runtime/c/utils/pgf-parse.c

@@ -123,7 +123,7 @@ int main(int argc, char* argv[]) {
     }
 
     GuReader *rdr = gu_string_reader(gu_str_string(line, ppool), ppool);
-    PgfLexer *lexer = pgf_new_lexer(rdr, ppool);
+    PgfLexer *lexer = pgf_new_simple_lexer(rdr, ppool);
     GuEnum* result = pgf_parse(concr, cat, lexer, ppool);
 
     PgfExprProb* ep = NULL;