GitHub/gf-core
1
0
Fork 1
mirror of https://github.com/GrammaticalFramework/gf-core.git synced 2026-05-22 09:32:53 -06:00
Code Issues Packages Projects Releases Wiki Activity

bugfix in the lexer from the C runtime. the input sentence doesn't have to terminate with whitespace

Browse Source
This commit is contained in:
kr.angelov
2012-12-13 16:45:44 +00:00
parent 0f0b7158c9
commit f7a5eb0df1
2 changed files with 20 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
src/runtime/c/pgf/lexer.c
View File

@@ -21,6 +21,16 @@ pgf_new_lexer(GuReader *rdr, GuPool *pool)
return lexer; return lexer;
} }
static void
pgf_lexer_read_ucs(PgfLexer *lexer, GuExn* err)
{
lexer->ucs = gu_read_ucs(lexer->rdr, err);
if (gu_exn_is_raised(err)) {
gu_exn_clear(err);
lexer->ucs = ' ';
}
}
PgfToken PgfToken
pgf_lexer_read_token(PgfLexer *lexer, GuExn* err) pgf_lexer_read_token(PgfLexer *lexer, GuExn* err)
{ {
@@ -44,9 +54,7 @@ pgf_lexer_read_token(PgfLexer *lexer, GuExn* err)
if (gu_exn_is_raised(err)) if (gu_exn_is_raised(err))
goto stop; goto stop;
counter++; counter++;
lexer->ucs = gu_read_ucs(lexer->rdr, err); pgf_lexer_read_ucs(lexer, err);
if (gu_exn_is_raised(err))
goto stop;
if (lexer->ucs == '.' && counter < 4) { if (lexer->ucs == '.' && counter < 4) {
// perhaps an abreviation // perhaps an abreviation
@@ -54,9 +62,7 @@ pgf_lexer_read_token(PgfLexer *lexer, GuExn* err)
if (gu_exn_is_raised(err)) if (gu_exn_is_raised(err))
goto stop; goto stop;
counter = 0; counter = 0;
lexer->ucs = gu_read_ucs(lexer->rdr, err); pgf_lexer_read_ucs(lexer, err);
if (gu_exn_is_raised(err))
goto stop;
} }
} while (iswalnum(lexer->ucs) || } while (iswalnum(lexer->ucs) ||
lexer->ucs == '\'' || lexer->ucs == '\'' ||
@@ -64,12 +70,10 @@ pgf_lexer_read_token(PgfLexer *lexer, GuExn* err)
} else if (iswdigit(lexer->ucs) || lexer->ucs == '-') { } else if (iswdigit(lexer->ucs) || lexer->ucs == '-') {
if (lexer->ucs == '-') { if (lexer->ucs == '-') {
gu_ucs_write(lexer->ucs, wtr, err); gu_ucs_write(lexer->ucs, wtr, err);
if (gu_exn_is_raised(err))
goto stop;
lexer->ucs = gu_read_ucs(lexer->rdr, err);
if (gu_exn_is_raised(err)) if (gu_exn_is_raised(err))
goto stop; goto stop;
pgf_lexer_read_ucs(lexer, err);
if (!iswdigit(lexer->ucs)) if (!iswdigit(lexer->ucs))
goto stop; goto stop;
} }
@@ -78,9 +82,8 @@ pgf_lexer_read_token(PgfLexer *lexer, GuExn* err)
gu_ucs_write(lexer->ucs, wtr, err); gu_ucs_write(lexer->ucs, wtr, err);
if (gu_exn_is_raised(err)) if (gu_exn_is_raised(err))
goto stop; goto stop;
lexer->ucs = gu_read_ucs(lexer->rdr, err);
if (gu_exn_is_raised(err)) pgf_lexer_read_ucs(lexer, err);
goto stop;
} while (iswdigit(lexer->ucs)); } while (iswdigit(lexer->ucs));
if (lexer->ucs == '.') { if (lexer->ucs == '.') {
@@ -88,26 +91,19 @@ pgf_lexer_read_token(PgfLexer *lexer, GuExn* err)
if (gu_exn_is_raised(err)) if (gu_exn_is_raised(err))
goto stop; goto stop;
lexer->ucs = gu_read_ucs(lexer->rdr, err); pgf_lexer_read_ucs(lexer, err);
if (gu_exn_is_raised(err))
goto stop;
while (iswdigit(lexer->ucs)) { while (iswdigit(lexer->ucs)) {
gu_ucs_write(lexer->ucs, wtr, err); gu_ucs_write(lexer->ucs, wtr, err);
if (gu_exn_is_raised(err)) if (gu_exn_is_raised(err))
goto stop; goto stop;
lexer->ucs = gu_read_ucs(lexer->rdr, err); pgf_lexer_read_ucs(lexer, err);
if (gu_exn_is_raised(err))
goto stop;
} }
} }
} else { } else {
gu_ucs_write(lexer->ucs, wtr, err); gu_ucs_write(lexer->ucs, wtr, err);
if (gu_exn_is_raised(err)) if (gu_exn_is_raised(err))
goto stop; goto stop;
lexer->ucs = gu_read_ucs(lexer->rdr, err); pgf_lexer_read_ucs(lexer, err);
if (gu_exn_is_raised(err))
goto stop;
} }
stop: stop:

1
src/runtime/c/utils/pgf-translate.c
View File

@@ -6,7 +6,6 @@
#include <gu/file.h> #include <gu/file.h>
#include <pgf/pgf.h> #include <pgf/pgf.h>
#include <pgf/parser.h> #include <pgf/parser.h>
#include <pgf/lexer.h>
#include <pgf/literals.h> #include <pgf/literals.h>
#include <pgf/linearize.h> #include <pgf/linearize.h>
#include <pgf/edsl.h> #include <pgf/edsl.h>