From: cassowarii Date: Fri, 26 Jun 2026 16:10:20 +0000 (-0700) Subject: basic Pratt expression parser X-Git-Url: https://git.cassowary.me/gitweb.cgi?a=commitdiff_plain;h=8c0cbdd9d86ea5f51bb2557d37588fd07fe70f0e;p=sarabande.git basic Pratt expression parser --- diff --git a/src/data/symbol.c b/src/data/symbol.c index ab29e8c..666a954 100644 --- a/src/data/symbol.c +++ b/src/data/symbol.c @@ -53,6 +53,10 @@ void sbSymbol_sys_init() { sbSymbol_from_bytes("", 0); } +void sbSymbol_sys_deinit() { + // nothing +} + char *sbSymbol_name(hSymbol sym) { return (char*)sym; } diff --git a/src/data/symbol.h b/src/data/symbol.h index 091b66e..c341687 100644 --- a/src/data/symbol.h +++ b/src/data/symbol.h @@ -3,6 +3,7 @@ #include "data/value.h" void sbSymbol_sys_init(); +void sbSymbol_sys_deinit(); hSymbol sbSymbol_from_string(hString str); diff --git a/src/main.c b/src/main.c index e1c0d78..867aff4 100644 --- a/src/main.c +++ b/src/main.c @@ -1,82 +1,25 @@ #include "common.h" -#include "parse/filereader.h" -#include "parse/lexer.h" +#include "parse/parser.h" #include "data/string.h" #include "data/hashtable.h" #include "data/symbol.h" int main(int argc, char **argv) { if (argc == 2) { - hFileReader fr = sbFileReader_open(argv[1]); - - if (!sbFileReader_ok(fr)) { - fprintf(stderr, "Error getting input stream.\n"); - return -1; - } - - sbLexer lx; - sbLexer_initialize(&lx, fr); - sbString_sys_init(); sbHash_sys_init(); sbSymbol_sys_init(); - sbLexToken t; - char scratch[8]; - - printf("ok here i go\n"); - hString hello_str = OBJSL("hello! "); - hString long_str = OBJSL(""); - hString test_str = OBJSL("hello! hello! hello! hello! hello! "); - for (int i = 0; i < 300; i++) { - sbString_release(long_str); - long_str = sbString_concat(long_str, hello_str); - if (sbString_eq(test_str, long_str)) { - printf("BREAK!\n"); - break; - } - } - printf("%s\n", sbString_get_value(long_str, scratch, NULL)); - - do { - t = sbLexer_next(&lx); - - if (t.type == T_EOF) { - printf(""); - } else if (t.type == T_SPACE) { - printf(""); - } else if (t.type == T_NEWLINE) { - printf("\n"); - } else if (t.type == T_IDENTIFIER) { - printf("%s", t.cstr); - } else if (t.type == T_INTEGER) { - printf("", t.i); - } else if (t.type == T_STRING) { - printf("", sbString_get_value(t.hstr, scratch, NULL)); - } else if (t.type == T_SYMBOL) { - printf("", t.cstr); - } else if (t.type >= T_rAND && t.type <= T_rWHILE) { - printf("<%s>", t.cstr); - } else if (t.type == T_SEMICOLON) { - printf(";\n"); - } else { - switch (t.type) { - case T_COLONBRACE: - printf(":{"); break; - case T_DOUBLEEQUALS: - printf("=="); break; - default: - printf("%c", t.type); - } - } - } while (t.type != T_EOF); - printf("\n"); + sbParser pr; + sbParser_initialize(&pr); - sbLexer_deinitialize(&lx); + sbParser_parse_file(&pr, argv[1]); - sbFileReader_close(fr); + sbParser_deinitialize(&pr); + sbSymbol_sys_deinit(); + sbHash_sys_deinit(); sbString_sys_deinit(); } else { fprintf(stderr, "please provide a file as input\n"); diff --git a/src/parse/ast.h b/src/parse/ast.h new file mode 100644 index 0000000..c552c4e --- /dev/null +++ b/src/parse/ast.h @@ -0,0 +1,71 @@ +#include "common.h" + +#include "data/value.h" +#include "parse/token.h" + +typedef enum sbAstType { + AST_NULL, + AST_VAL_NIL, + AST_VAL_INT, + AST_VAL_STRING, + AST_VAL_FLOAT, + AST_VAL_SYMBOL, + AST_VAL_BOOLEAN, + AST_NODE_NAME, + AST_NODE_SEQ, + AST_NODE_OP, + AST_NODE_DEF, + AST_NODE_LET, + AST_NODE_IF, + AST_NODE_THENELSE, + AST_NODE_WHILE, + AST_NODE_LIST, + AST_NODE_HASH, + AST_NODE_HASHENTRY, + AST_NODE_NEXT, + AST_NODE_GROUPING, + AST_NODE_MULTIVAL, + AST_NODE_FUNCCALL, + AST_NODE_METHODCALL, + AST_NODE_SEND, + AST_NODE_ELLIPSIS, +} sbAstType; + +typedef enum sbAstOp { + AST_OP_NULL, + AST_OP_PLUS = '+', + AST_OP_MINUS = '-', + AST_OP_TIMES = '*', + AST_OP_DIV = '/', + AST_OP_MOD = '%', + AST_OP_EQ = '=', + AST_OP_LT = '<', + AST_OP_GT = '>', + AST_OP_LE = 128, + AST_OP_GE, + AST_OP_NE, + AST_OP_FLDIV, + AST_OP_DIVBY, + AST_OP_PN, +} sbAstOp; + +typedef struct sbAstNode { + sbAstType type; + union { + hString str; + hSymbol symb; + hInteger i; + double fl; + struct { + const struct sbAstNode *this; + const struct sbAstNode *next; + } seq; + struct { + sbTokenType type; + const struct sbAstNode *left; + const struct sbAstNode *right; + } op; + }; +} sbAstNode; + +typedef const sbAstNode *sbAst; diff --git a/src/parse/parser.c b/src/parse/parser.c new file mode 100644 index 0000000..11fe90b --- /dev/null +++ b/src/parse/parser.c @@ -0,0 +1,208 @@ +#include "parser.h" + +#include "parse/filereader.h" +#include "data/symbol.h" + +static sbAst do_parse(hParser pr); + +void sbParser_initialize(hParser pr) { + sbArena_initialize(&pr->node_arena, 32768); + sbTokenQueue_initialize(&pr->input_queue, 8); +} + +void sbParser_deinitialize(hParser pr) { + sbTokenQueue_deinitialize(&pr->input_queue); + sbArena_deinitialize(&pr->node_arena); +} + +sbAst sbParser_parse_file(hParser pr, const char *filename) { + hFileReader fr = sbFileReader_open(filename); + + if (!sbFileReader_ok(fr)) { + fprintf(stderr, "Error getting input stream.\n"); + return NULL; + } + + sbLexer_initialize(&pr->lexer, fr); + + sbAst result = do_parse(pr); + + sbFileReader_close(fr); + + return result; +} + +/* --- */ + +typedef struct binop { + sbTokenType type; + u8 left_precedence; + u8 right_precedence; +} binop; + +static const sbAstNode SENTINEL_VALUE = {0}; +static sbAst NO_NODE = &SENTINEL_VALUE; + +static sbLexToken peek_ahead(hParser pr, usize count) { + while (sbTokenQueue_size(&pr->input_queue) < count + 1) { + sbTokenQueue_enqueue(&pr->input_queue, sbLexer_next(&pr->lexer)); + } + + return sbTokenQueue_at(&pr->input_queue, count); +} + +static sbLexToken next_token(hParser pr) { + if (sbTokenQueue_size(&pr->input_queue) > 0) { + sbTokenQueue_shift(&pr->input_queue); + } + + return sbTokenQueue_at(&pr->input_queue, 0); +} + +static flag accept(hParser pr, sbTokenType type, sbLexToken *tok_out) { + sbLexToken up_next = peek_ahead(pr, 0); + if (up_next.type == type) { + next_token(pr); + if (tok_out) *tok_out = up_next; + return TRUE; + } else { + return FALSE; + } +} + +static flag expect(hParser pr, sbTokenType type, sbLexToken *tok_out) { + sbLexToken up_next = peek_ahead(pr, 0); + if (up_next.type == type) { + next_token(pr); + if (tok_out) *tok_out = up_next; + return TRUE; + } else { + fprintf(stderr, "syntax error ! unexpected type %d\n", up_next.type); // TODO add line information, etc. + return FALSE; + } +} + +static sbAst new_node(hParser pr, sbAstNode *n) { + sbAstNode *new_node = sbArena_alloc(&pr->node_arena, sizeof(sbAstNode)); + *new_node = *n; + return new_node; +} + +static sbAst unop_node(hParser pr, sbTokenType operation, sbAstNode *child) { + sbAstNode n = (sbAstNode) { + .type = AST_NODE_OP, + .op.type = operation, + .op.left = child, + }; + return new_node(pr, &n); +} + +static sbAst binop_node(hParser pr, sbTokenType operation, sbAst left, sbAst right) { + sbAstNode n = (sbAstNode) { + .type = AST_NODE_OP, + .op.type = operation, + .op.left = left, + .op.right = right, + }; + return new_node(pr, &n); +} + +static binop binops[] = { + { T_rOR, 3, 4 }, + { T_rAND, 6, 7 }, + { T_DOUBLEEQUALS, 10, 11 }, + { T_GREATER, 10, 11 }, + { T_LESS, 10, 11 }, + { T_GREATEREQUALS, 10, 11 }, + { T_LESSEQUALS, 10, 11 }, + { T_NOTEQUALS, 10, 11 }, + { T_PLUS, 15, 16 }, + { T_MINUS, 15, 16 }, + { T_ASTERISK, 30, 31 }, + { T_SLASH, 30, 31 }, + { T_PERCENT, 30, 31 }, + { T_DOUBLESLASH, 30, 31 }, + { T_DOUBLEPERCENT, 30, 31 }, + { T_DOUBLEASTERISK, 51, 50 }, + { T_TWODOT, 60, 61 }, + { T_PAAMAYIM_NEKUDOTAYIM, 70, 71 }, +}; + +const int NUM_BINOPS = sizeof(binops) / sizeof(binops[0]); + +/* https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html */ +static sbAst parse_expr(hParser pr, u8 min_precedence) { + sbLexToken t = peek_ahead(pr, 0); + sbAst lhs = NO_NODE; + if (t.type == T_IDENTIFIER) { + expect(pr, T_IDENTIFIER, NULL); + sbAstNode n = { .type = AST_NODE_NAME, .symb = t.symb }; + lhs = new_node(pr, &n); + } else if (t.type == T_LPAREN) { + next_token(pr); + lhs = parse_expr(pr, 0); + if (!expect(pr, T_RPAREN, NULL)) { + fprintf(stderr, "expected closing parenthesis!\n"); + return NO_NODE; + } + } else { + fprintf(stderr, "expected identifier!\n"); + return NO_NODE; + } + + while (1) { + sbLexToken op = peek_ahead(pr, 0); + if (op.type == T_EOF || op.type == T_SEMICOLON) { + break; + } else { + binop *infix = NULL; + for (int i = 0; i < NUM_BINOPS; i++) { + if (binops[i].type == op.type) { + infix = &binops[i]; + break; + } + } + + if (!infix) { + /* something that isn't an infix operator: means end of expr */ + break; + } + + if (infix->left_precedence < min_precedence) { + /* end of this expr because something with lower precedence */ + break; + } + + next_token(pr); /* consume operator token */ + + sbAst rhs = parse_expr(pr, infix->right_precedence); + + lhs = binop_node(pr, infix->type, lhs, rhs); + } + } + + return lhs; +} + +static void print_ast_node(sbAst n) { + if (n->type == AST_NODE_NAME) { + printf("%s", sbSymbol_name(n->symb)); + } else { + printf("("); + if (n->type == AST_NODE_OP) { + printf("%c ", n->op.type); + print_ast_node(n->op.left); + printf(" "); + print_ast_node(n->op.right); + } else { + printf("?<%d>", n->type); + } + printf(")"); + } +} + +static sbAst do_parse(hParser pr) { + sbAst program = parse_expr(pr, 0); + print_ast_node(program); + return program; +} diff --git a/src/parse/parser.h b/src/parse/parser.h new file mode 100644 index 0000000..5cbf389 --- /dev/null +++ b/src/parse/parser.h @@ -0,0 +1,18 @@ +#include "common.h" + +#include "parse/ast.h" +#include "parse/lexer.h" + +typedef struct sbParser { + sbLexer lexer; + sbTokenQueue input_queue; + sbArena node_arena; +} sbParser; + +typedef sbParser *hParser; + +const sbAstNode *sbParser_parse_file(hParser pr, const char *filename); + +void sbParser_initialize(hParser pr); + +void sbParser_deinitialize(hParser pr); diff --git a/src/parse/scanner.c b/src/parse/scanner.c index c297bed..fb6e50f 100644 --- a/src/parse/scanner.c +++ b/src/parse/scanner.c @@ -69,7 +69,7 @@ sbLexToken sbScanner_next(hScanner sc) { } void sbScanner_deinitialize(hScanner sc) { - sbArena_destroy(&sc->arena); + sbArena_deinitialize(&sc->arena); sbBuffer_deinitialize(&sc->dynamic_buffer); } diff --git a/src/parse/scanner.h b/src/parse/scanner.h index e3aabd3..e2a8bd7 100644 --- a/src/parse/scanner.h +++ b/src/parse/scanner.h @@ -1,3 +1,6 @@ +#ifndef __SARABANDE_SCANNER_H__ +#define __SARABANDE_SCANNER_H__ + #include "common.h" #include "filereader.h" @@ -20,3 +23,5 @@ sbLexToken sbScanner_next(hScanner sc); sbLexToken sbScanner_peek(hScanner sc); void sbScanner_deinitialize(hScanner sc); + +#endif