libpgf: added simple lexer

src/runtime/c/pgf/lexer.c

@@ -0,0 +1,103 @@
+#include <gu/list.h>
+#include <pgf/lexer.h>
+#include <pgf/data.h>
+#include <wctype.h>
+
+struct PgfLexer {
+	GuReader* rdr;
+	GuUCS ucs;
+};
+
+PgfLexer*
+pgf_new_lexer(GuReader *rdr, GuPool *pool)
+{
+	PgfLexer* lexer = gu_new(PgfLexer, pool);
+	lexer->rdr = rdr;
+	lexer->ucs = ' ';
+	return lexer;
+}
+
+PgfToken
+pgf_lexer_next_token(PgfLexer *lexer, GuExn* err, GuPool *pool)
+{
+	GuPool* tmp_pool = gu_new_pool();
+
+	PgfToken tok;
+
+	GuStringBuf* buf = gu_string_buf(tmp_pool);
+	GuWriter* wtr = gu_string_buf_writer(buf);
+
+	while (iswspace(lexer->ucs)) {
+		lexer->ucs = gu_read_ucs(lexer->rdr, err);
+		if (gu_exn_is_raised(err))
+			goto stop;
+	}
+
+	if (iswalpha(lexer->ucs) ||
+	    lexer->ucs == '\''   ||
+	    lexer->ucs == '_') {
+		do {
+			gu_ucs_write(lexer->ucs, wtr, err);
+			if (gu_exn_is_raised(err))
+				goto stop;
+			lexer->ucs = gu_read_ucs(lexer->rdr, err);
+			if (gu_exn_is_raised(err))
+				goto stop;
+		} while (iswalnum(lexer->ucs) ||
+		         lexer->ucs == '\''   ||
+		         lexer->ucs == '_');
+	} else if (iswdigit(lexer->ucs) || lexer->ucs == '-') {
+		if (lexer->ucs == '-') {
+			gu_ucs_write(lexer->ucs, wtr, err);
+			if (gu_exn_is_raised(err))
+				goto stop;
+			lexer->ucs = gu_read_ucs(lexer->rdr, err);
+			if (gu_exn_is_raised(err))
+				goto stop;
+
+			if (!iswdigit(lexer->ucs))
+				goto stop;
+		}
+
+		do {
+			gu_ucs_write(lexer->ucs, wtr, err);
+			if (gu_exn_is_raised(err))
+				goto stop;
+			lexer->ucs = gu_read_ucs(lexer->rdr, err);
+			if (gu_exn_is_raised(err))
+				goto stop;
+		} while (iswdigit(lexer->ucs));
+		
+		if (lexer->ucs == '.') {
+			gu_ucs_write(lexer->ucs, wtr, err);
+			if (gu_exn_is_raised(err))
+				goto stop;
+
+			lexer->ucs = gu_read_ucs(lexer->rdr, err);
+			if (gu_exn_is_raised(err))
+				goto stop;
+
+			while (iswdigit(lexer->ucs)) {
+				gu_ucs_write(lexer->ucs, wtr, err);
+				if (gu_exn_is_raised(err))
+					goto stop;
+				lexer->ucs = gu_read_ucs(lexer->rdr, err);
+				if (gu_exn_is_raised(err))
+					goto stop;
+			}
+		}
+	} else {
+		gu_ucs_write(lexer->ucs, wtr, err);
+		if (gu_exn_is_raised(err))
+			goto stop;
+		lexer->ucs = gu_read_ucs(lexer->rdr, err);
+		if (gu_exn_is_raised(err))
+			goto stop;
+	}
+
+stop:
+	tok = gu_string_buf_freeze(buf, pool);
+
+	gu_pool_free(tmp_pool);
+	return tok;
+}

src/runtime/c/pgf/lexer.h

@@ -0,0 +1,15 @@
+#ifndef PGF_LEXER_H_
+#define PGF_LEXER_H_
+
+#include <gu/read.h>
+#include <pgf/data.h>
+
+typedef struct PgfLexer PgfLexer;
+
+PgfLexer*
+pgf_new_lexer(GuReader *rdr, GuPool *pool);
+
+PgfToken
+pgf_lexer_next_token(PgfLexer *lexer, GuExn* err, GuPool *pool);
+
+#endif // PGF_LEXER_H_

src/runtime/c/pgf/parser.c

@@ -953,8 +953,8 @@ typedef struct {
     GuPool *pool;
 } PgfParseTokenCallback;
 
-static
-void pgf_match_token(PgfLexCallback* self, PgfToken tok, PgfItem* item)
+static void 
+pgf_match_token(PgfLexCallback* self, PgfToken tok, PgfItem* item)
 {
     PgfParseTokenCallback *clo = (PgfParseTokenCallback *) self;