fix(parser): fix blatant bug with lexer handling

main
Jef Roosens 2024-03-15 22:24:19 +01:00
parent 06a9d1e37f
commit 33824534bc
Signed by: Jef Roosens
GPG Key ID: B75D4F293C7052DB
4 changed files with 52 additions and 34 deletions

View File

@ -1,7 +1,7 @@
#ifndef MRK_PARSER_INTERNAL #ifndef MRK_PARSER_INTERNAL
#define MRK_PARSER_INTERNAL #define MRK_PARSER_INTERNAL
#include "mrk/lexer_internal.h" #include "mrk/lexer.h"
#include "mrk/parser.h" #include "mrk/parser.h"
#define MRK_PARSER_LOOKAHEAD_BUF_SIZE 4 #define MRK_PARSER_LOOKAHEAD_BUF_SIZE 4
@ -12,10 +12,7 @@
struct mrk_parser { struct mrk_parser {
mrk_lexer *lexer; mrk_lexer *lexer;
struct { mrk_token next_token;
mrk_token buf[MRK_PARSER_LOOKAHEAD_BUF_SIZE];
size_t index;
} lookahead;
struct { struct {
mrk_token token; mrk_token token;
const char *msg; const char *msg;

View File

@ -1,4 +1,5 @@
#include "mrk/parser_internal.h" #include "mrk/parser_internal.h"
#include <stdio.h>
mrk_err mrk_parser_init(mrk_parser **out) { mrk_err mrk_parser_init(mrk_parser **out) {
MRK_CALLOC(out, 1, sizeof(mrk_parser)); MRK_CALLOC(out, 1, sizeof(mrk_parser));
@ -9,11 +10,8 @@ mrk_err mrk_parser_init(mrk_parser **out) {
void mrk_parser_open(mrk_parser *parser, mrk_lexer *lexer) { void mrk_parser_open(mrk_parser *parser, mrk_lexer *lexer) {
parser->lexer = lexer; parser->lexer = lexer;
// Prefill lookahead buffer // Prefill lookahead character
for (size_t i = 0; mrk_parser_advance(parser);
i < MRK_PARSER_LOOKAHEAD_BUF_SIZE && !mrk_lexer_done(lexer); i++) {
mrk_lexer_next(&parser->lookahead.buf[i], lexer);
}
} }
mrk_err mrk_parser_parse(mrk_ast_node **out, mrk_parser *parser) { mrk_err mrk_parser_parse(mrk_ast_node **out, mrk_parser *parser) {
@ -28,6 +26,10 @@ mrk_err mrk_parser_parse(mrk_ast_node **out, mrk_parser *parser) {
case mrk_token_type_header_start: case mrk_token_type_header_start:
MRK_RES(mrk_parser_parse_header(child, parser)); MRK_RES(mrk_parser_parse_header(child, parser));
break; break;
case mrk_token_type_text:
case mrk_token_type_left_bracket:
MRK_RES(mrk_parser_parse_paragraph(child, parser));
break;
} }
} }

View File

@ -2,35 +2,21 @@
bool mrk_parser_done(mrk_parser *parser) { bool mrk_parser_done(mrk_parser *parser) {
return mrk_lexer_done(parser->lexer) && return mrk_lexer_done(parser->lexer) &&
parser->lookahead.buf[parser->lookahead.index].type == parser->next_token.type == mrk_token_type_none;
mrk_token_type_none;
} }
mrk_token mrk_parser_peek(mrk_parser *parser) { mrk_token mrk_parser_peek(mrk_parser *parser) { return parser->next_token; }
mrk_token out;
if (mrk_parser_done(parser)) {
out.type = mrk_token_type_none;
} else {
out = parser->lookahead.buf[parser->lookahead.index];
}
return out;
}
void mrk_parser_advance(mrk_parser *parser) { void mrk_parser_advance(mrk_parser *parser) {
if (mrk_parser_done(parser)) { switch (mrk_lexer_next(&parser->next_token, parser->lexer)) {
return; case mrk_lexer_err_ok:
// TODO handle this better
case mrk_lexer_err_unexpected_char:
break;
case mrk_lexer_err_done:
parser->next_token.type = mrk_token_type_none;
break;
} }
// Append new element of the lexer to the lookahead buffer
if (!mrk_lexer_done(parser->lexer)) {
mrk_lexer_next(&parser->lookahead.buf[parser->lookahead.index],
parser->lexer);
}
parser->lookahead.index =
(parser->lookahead.index + 1) % MRK_PARSER_LOOKAHEAD_BUF_SIZE;
} }
mrk_err mrk_parser_eat(mrk_token *out, mrk_parser *parser, mrk_err mrk_parser_eat(mrk_token *out, mrk_parser *parser,

View File

@ -0,0 +1,33 @@
#include "test.h"
#include "mrk/lexer.h"
#include "mrk/parser_internal.h"
#define LEXER_INIT() \
mrk_lexer *lxr; \
TEST_CHECK(mrk_lexer_init(&lxr) == mrk_err_ok)
#define PARSER_INIT() \
mrk_parser *parser; \
TEST_CHECK(mrk_parser_init(&parser) == mrk_err_ok)
#define PARSER_OPEN(buf) \
mrk_lexer *lxr; \
TEST_CHECK(mrk_lexer_init(&lxr) == mrk_err_ok); \
mrk_parser *parser; \
TEST_CHECK(mrk_parser_init(&parser) == mrk_err_ok); \
mrk_lexer_open(lxr, buf, 0); \
mrk_parser_open(parser, lxr)
void test_parse_simple_header_paragraph() {
const char *buf = "# this is a header\n\nthis is a paragraph with a [link](https://example.com)";
PARSER_OPEN(buf);
mrk_ast_node *root;
TEST_CHECK(mrk_parser_parse(&root, parser) == mrk_err_ok);
}
TEST_LIST = {
{ "parser simple header and paragraph", test_parse_simple_header_paragraph },
{ NULL, NULL }
};