GitHub/gf-core
1
0
Fork 1
mirror of https://github.com/GrammaticalFramework/gf-core.git synced 2026-04-09 04:59:31 -06:00
Code Issues Packages Projects Releases Wiki Activity

bugfix in the lexer from the C runtime. the input sentence doesn't have to terminate with whitespace

Browse Source
This commit is contained in:
kr.angelov
2012-12-13 16:45:44 +00:00
parent 0f0b7158c9
commit f7a5eb0df1
2 changed files with 20 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/runtime/c/pgf/lexer.c
View File

@@ -21,6 +21,16 @@ pgf_new_lexer(GuReader *rdr, GuPool *pool)
return lexer;
}
static void
pgf_lexer_read_ucs(PgfLexer *lexer, GuExn* err)
{
lexer->ucs = gu_read_ucs(lexer->rdr, err);
if (gu_exn_is_raised(err)) {
gu_exn_clear(err);
lexer->ucs = ' ';
}
}
PgfToken
pgf_lexer_read_token(PgfLexer *lexer, GuExn* err)
{
@@ -44,19 +54,15 @@ pgf_lexer_read_token(PgfLexer *lexer, GuExn* err)
if (gu_exn_is_raised(err))
goto stop;
counter++;
lexer->ucs = gu_read_ucs(lexer->rdr, err);
if (gu_exn_is_raised(err))
goto stop;
pgf_lexer_read_ucs(lexer, err);
if (lexer->ucs == '.' && counter < 4) {
// perhaps an abreviation
gu_ucs_write(lexer->ucs, wtr, err);
if (gu_exn_is_raised(err))
goto stop;
counter = 0;
lexer->ucs = gu_read_ucs(lexer->rdr, err);
if (gu_exn_is_raised(err))
goto stop;
pgf_lexer_read_ucs(lexer, err);
}
} while (iswalnum(lexer->ucs) ||
lexer->ucs == '\'' ||
@@ -66,10 +72,8 @@ pgf_lexer_read_token(PgfLexer *lexer, GuExn* err)
gu_ucs_write(lexer->ucs, wtr, err);
if (gu_exn_is_raised(err))
goto stop;
lexer->ucs = gu_read_ucs(lexer->rdr, err);
if (gu_exn_is_raised(err))
goto stop;
pgf_lexer_read_ucs(lexer, err);
if (!iswdigit(lexer->ucs))
goto stop;
}
@@ -78,9 +82,8 @@ pgf_lexer_read_token(PgfLexer *lexer, GuExn* err)
gu_ucs_write(lexer->ucs, wtr, err);
if (gu_exn_is_raised(err))
goto stop;
lexer->ucs = gu_read_ucs(lexer->rdr, err);
if (gu_exn_is_raised(err))
goto stop;
pgf_lexer_read_ucs(lexer, err);
} while (iswdigit(lexer->ucs));
if (lexer->ucs == '.') {
@@ -88,26 +91,19 @@ pgf_lexer_read_token(PgfLexer *lexer, GuExn* err)
if (gu_exn_is_raised(err))
goto stop;
lexer->ucs = gu_read_ucs(lexer->rdr, err);
if (gu_exn_is_raised(err))
goto stop;
pgf_lexer_read_ucs(lexer, err);
while (iswdigit(lexer->ucs)) {
gu_ucs_write(lexer->ucs, wtr, err);
if (gu_exn_is_raised(err))
goto stop;
lexer->ucs = gu_read_ucs(lexer->rdr, err);
if (gu_exn_is_raised(err))
goto stop;
pgf_lexer_read_ucs(lexer, err);
}
}
} else {
gu_ucs_write(lexer->ucs, wtr, err);
if (gu_exn_is_raised(err))
goto stop;
lexer->ucs = gu_read_ucs(lexer->rdr, err);
if (gu_exn_is_raised(err))
goto stop;
pgf_lexer_read_ucs(lexer, err);
}
stop:

1
src/runtime/c/utils/pgf-translate.c
View File

@@ -6,7 +6,6 @@
#include <gu/file.h>
#include <pgf/pgf.h>
#include <pgf/parser.h>
#include <pgf/lexer.h>
#include <pgf/literals.h>
#include <pgf/linearize.h>
#include <pgf/edsl.h>