From eb8513bd963f3d5b649bb95fd2f7358dc663b008 Mon Sep 17 00:00:00 2001 From: cassowarii Date: Sat, 27 Jun 2026 17:23:11 -0700 Subject: [PATCH] i think this is all the basic syntax i had in mind (except @declarations, but those are just syntax sugar, we'll come back to those!) --- src/main.c | 10 ++- src/mem/buffer.c | 1 + src/parse/ast.h | 9 ++- src/parse/filereader.c | 10 +-- src/parse/lexer.c | 10 ++- src/parse/parser.c | 209 +++++++++++++++++++++++++++++++++++-------------- src/parse/parser.h | 1 + src/parse/scanner.c | 5 +- src/parse/scanner.h | 1 - src/parse/token.h | 2 + 10 files changed, 186 insertions(+), 72 deletions(-) diff --git a/src/main.c b/src/main.c index 867aff4..684b332 100644 --- a/src/main.c +++ b/src/main.c @@ -14,7 +14,15 @@ int main(int argc, char **argv) { sbParser pr; sbParser_initialize(&pr); - sbParser_parse_file(&pr, argv[1]); + sbAst parse_result = sbParser_parse_file(&pr, argv[1]); + + if (parse_result == NULL) { + fprintf(stderr, "fatal error: Could not open script '%s'\n", argv[1]); + } else if (parse_result->type == AST_ERROR) { + fprintf(stderr, "fatal error: Could not run '%s' due to syntax errors.\n", argv[1]); + } else { + printf("wow! the syntax is good!\n"); + } sbParser_deinitialize(&pr); diff --git a/src/mem/buffer.c b/src/mem/buffer.c index b844c76..61898cf 100644 --- a/src/mem/buffer.c +++ b/src/mem/buffer.c @@ -35,6 +35,7 @@ void *sbBuffer_expand(hBuffer buf, usize expand_size) { char *new_data = realloc(buf->data, new_capacity); if (new_data) { + buf->capacity = new_capacity; buf->data = new_data; } else { fprintf(stderr, "failed to expand buffer!"); diff --git a/src/parse/ast.h b/src/parse/ast.h index 33238ca..1cef746 100644 --- a/src/parse/ast.h +++ b/src/parse/ast.h @@ -12,7 +12,10 @@ typedef enum sbAstType { AST_VAL_FLOAT, AST_VAL_SYMBOL, AST_VAL_BOOLEAN, + AST_VAL_LIST, + AST_VAL_HASH, AST_VAL_FUNC, + AST_VAL_IMFUNC, AST_VAL_OBJ, AST_NODE_NAME, AST_NODE_SEQ, @@ -20,6 +23,8 @@ typedef enum sbAstType { AST_NODE_DEF, AST_NODE_LET, AST_NODE_ASSIGN, + AST_NODE_INCR, + AST_NODE_DECR, AST_NODE_IF, AST_NODE_THENELSE, AST_NODE_WHILE, @@ -27,7 +32,6 @@ typedef enum sbAstType { AST_NODE_CASE, AST_NODE_MATCH, AST_NODE_LIST, - AST_NODE_HASH, AST_NODE_HASHENTRY, AST_NODE_NEXT, AST_NODE_MULTIVAL, @@ -48,6 +52,8 @@ typedef enum sbAstOp { AST_OP_EQ = '=', AST_OP_LT = '<', AST_OP_GT = '>', + AST_OP_REF = '&', + AST_OP_DEREF = '*', AST_OP_PIPE = '|', AST_OP_LE = 128, AST_OP_GE, @@ -64,6 +70,7 @@ typedef enum sbAstOp { AST_OP_AND, AST_OP_NOT, AST_OP_IN, + AST_OP_SEND, AST_OP_SPLAT, } sbAstOp; diff --git a/src/parse/filereader.c b/src/parse/filereader.c index fce211c..38a7196 100644 --- a/src/parse/filereader.c +++ b/src/parse/filereader.c @@ -4,10 +4,10 @@ #define N_READBACK_LINES 3 -/* this seems pointless right now as a wrapper for FILE* but later, we can make - * this work to read from stdin in interactive mode as well, which will be easier - * if we just have these few functions separate from the actual lexer code. - * we may also want to have multiple files open for imports and stuff... */ +/* reads in files a line at a time and provides the file data one character + * at a time. also keeps track of the current line and column number (counting + * tabs as 4 characters), and can print the previous few lines for error- + * reporting purposes. (highlighting syntax errors on a certain line) */ struct sbFileReader { flag valid; @@ -122,7 +122,7 @@ void sbFileReader_close(hFileReader r) { static void read_in_new_line(hFileReader r) { if (!r->valid) return; - /* recycle line buffers */ + /* rotate line buffers */ sbBuffer oldest_line_buffer = r->readback_lines[0]; for (int i = 0; i < N_READBACK_LINES - 1; i++) { r->readback_lines[i] = r->readback_lines[i + 1]; diff --git a/src/parse/lexer.c b/src/parse/lexer.c index c19eaa7..4a86fbc 100644 --- a/src/parse/lexer.c +++ b/src/parse/lexer.c @@ -166,7 +166,10 @@ static flag can_only_start_expression(sbTokenType type, flag brace_terminated_st * surrounded by spaces or neither space */ static flag maybe_can_start_expression(sbTokenType type) { return type == T_PLUS - || type == T_MINUS; + || type == T_MINUS + || type == T_ASTERISK + || type == T_AMPERSAND + || type == T_ELLIPSIS; } /* can come after something like a ) and still be the beginning of a function @@ -204,6 +207,7 @@ static flag begins_brace_terminated_state(sbTokenType type) { || type == T_rDO || type == T_rREPEAT || type == T_rCASE + || type == T_rMATCH || type == T_rWHEN || type == T_FATARROW /* fat arrow => a, b { ... } introduces block header also */ || type == T_SQUIGARROW; /* squiggle arrow ~> a, b { ... } introduces block header also */ @@ -213,8 +217,7 @@ static flag begins_brace_terminated_state(sbTokenType type) { static flag can_end_expression(sbTokenType type) { return type == T_IDENTIFIER || type == T_RPAREN - || type == T_RBRACKET - || type == T_RBRACE; + || type == T_RBRACKET; } /* when in brace-terminated state, one of these must precede a { character @@ -225,6 +228,7 @@ static flag block_header_can_end_after(sbTokenType type) { || type == T_FATARROW || type == T_SQUIGARROW || type == T_rCASE + || type == T_ELLIPSIS || type == T_rDO; } diff --git a/src/parse/parser.c b/src/parse/parser.c index 667f8ea..acb9afc 100644 --- a/src/parse/parser.c +++ b/src/parse/parser.c @@ -21,14 +21,18 @@ sbAst sbParser_parse_file(hParser pr, const char *filename) { hFileReader fr = sbFileReader_open(filename); if (!sbFileReader_ok(fr)) { - fprintf(stderr, "Error getting input stream.\n"); return NULL; } + pr->error_state = FALSE; + pr->any_error = FALSE; + sbLexer_initialize(&pr->lexer, fr); sbAst result = do_parse(pr); + sbLexer_deinitialize(&pr->lexer); + sbFileReader_close(fr); return result; @@ -61,6 +65,7 @@ static sbAst ERROR_NODE = &ERROR_SENTINEL_VALUE; static tokenspelling token_spellings[] = { { T_NEWLINE, "newline" }, + { T_SPACE, "space" }, { T_EOF, "end-of-file" }, { T_IDENTIFIER, "identifier" }, { T_SYMBOL, "symbol" }, @@ -107,8 +112,9 @@ static tokenspelling token_spellings[] = { { T_GREATER, "'>'" }, { T_COLON, "':'" }, { T_SEMICOLON, "';'" }, + { T_AMPERSAND, "'&'" }, + { T_AT, "'@'" }, { T_BACKSLASH, "'\\'" }, - { T_SPACE, "' '" }, { T_ARROW, "'->'" }, { T_FATARROW, "'=>'" }, { T_SQUIGARROW, "'~>'" }, @@ -135,11 +141,12 @@ static tokenspelling token_spellings[] = { { T_ELLIPSIS, "'...'" }, }; -const char *const TOKEN_SPELLING_UNKNOWN = ""; +static const char *const TOKEN_SPELLING_UNKNOWN = ""; static binop binops[] = { { T_PIPE, 6, 7, AST_OP_PIPE }, - { T_rOR, 10, 11, AST_OP_OR }, + { T_BACKSQUIGARROW, 8, 9, AST_OP_SEND }, + { T_rOR, 15, 16, AST_OP_OR }, { T_rAND, 20, 21, AST_OP_AND }, { T_rIN, 25, 26, AST_OP_IN }, { T_DOUBLEEQUALS, 30, 31, AST_OP_EQ }, @@ -158,6 +165,7 @@ static binop binops[] = { { T_DOUBLEASTERISK, 71, 70, AST_OP_POW }, { T_TWODOT, 80, 81, AST_OP_RANGE }, { T_DOT, 90, 91, AST_OP_NULL }, + { T_ARROW, 90, 91, AST_OP_NULL }, { T_LPAREN, 90, 91, AST_OP_NULL }, { T_LBRACKET, 90, 91, AST_OP_INDEX }, { T_PAAMAYIM_NEKUDOTAYIM, 90, 91, AST_OP_SCOPE }, @@ -167,11 +175,13 @@ static unop unops[] = { { T_rNOT, 25, AST_OP_NOT }, { T_PLUS, 85, AST_OP_UNPLUS }, { T_MINUS, 85, AST_OP_UNMINUS }, + { T_AMPERSAND, 85, AST_OP_REF }, + { T_ASTERISK, 85, AST_OP_DEREF }, }; -const usize NUM_BINOPS = sizeof(binops) / sizeof(binops[0]); -const usize NUM_UNOPS = sizeof(unops) / sizeof(unops[0]); -const usize NUM_TOKEN_SPELLINGS = sizeof(token_spellings) / sizeof(token_spellings[0]); +static const usize NUM_BINOPS = sizeof(binops) / sizeof(binops[0]); +static const usize NUM_UNOPS = sizeof(unops) / sizeof(unops[0]); +static const usize NUM_TOKEN_SPELLINGS = sizeof(token_spellings) / sizeof(token_spellings[0]); static sbLexToken peek_ahead(hParser pr, usize count) { while (sbTokenQueue_size(&pr->input_queue) < count + 1) { @@ -187,22 +197,14 @@ static sbLexToken next_token(hParser pr) { sbTokenQueue_shift(&pr->input_queue); } - pr->error_state = 0; + pr->error_state = FALSE; sbLexToken t = peek_ahead(pr, 0); return t; } -flag is_literal_token(sbTokenType type) { - return type == T_IDENTIFIER - || type == T_SYMBOL - || type == T_INTEGER - || type == T_FLOAT - || type == T_STRING; -} - -const char *token_spelling(sbTokenType type) { +static const char *token_spelling(sbTokenType type) { for (usize i = 0; i < NUM_TOKEN_SPELLINGS; i++) { if (token_spellings[i].type == type) { return token_spellings[i].name; @@ -247,7 +249,7 @@ static sbAst syntax_error(hParser pr) { if (unexpected_token.type == T_ERROR) { fprintf(stderr, "syntax error: invalid character"); } else if (unexpected_token.type == T_WRONGBRACKET) { - fprintf(stderr, "syntax error: mismatched bracket"); + fprintf(stderr, "syntax error: mismatched or missing bracket"); } else if (unexpected_token.type == T_BADNUMBER) { fprintf(stderr, "syntax error: invalid number"); } else if (unexpected_token.type == T_SEMICOLON && unexpected_token.invisible) { @@ -271,6 +273,7 @@ static sbAst syntax_error(hParser pr) { /* set error flag until we move to the next token so that syntax error * isn't printed multiple times */ pr->error_state = TRUE; + pr->any_error = TRUE; return ERROR_NODE; } @@ -385,50 +388,102 @@ static sbAst parse_comma_exprs(hParser pr, sbAst after) { *put_here = seq_node(pr, AST_NODE_MULTIVAL, expr, NO_NODE); put_here = &(*put_here)->seq.right; + + /* these | and <~ operators can break a comma list */ + if (expect(pr, T_PIPE)) break; + if (expect(pr, T_BACKSQUIGARROW)) break; } while (expect(pr, T_COMMA)); return result; } -static sbAst parse_name(hParser pr) { +static sbAst parse_literal(hParser pr) { sbLexToken t = peek_ahead(pr, 0); - if (t.type == T_IDENTIFIER) { - next_token(pr); - return name_node(pr, t); - } - return NO_NODE; -} - -static sbAst parse_block(hParser pr); - -/* https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html */ -static sbAst parse_expr(hParser pr, u8 min_precedence) { - sbLexToken t = peek_ahead(pr, 0); - sbAst lhs = NO_NODE; if (t.type == T_rNIL) { next_token(pr); sbAstNode n = { .type = AST_VAL_NIL }; - lhs = new_node(pr, &n); + return new_node(pr, &n); } else if (t.type == T_rTRUE || t.type == T_rFALSE) { next_token(pr); sbAstNode n = { .type = AST_VAL_BOOLEAN, .i = (t.type == T_rTRUE) }; - lhs = new_node(pr, &n); + return new_node(pr, &n); } else if (t.type == T_INTEGER) { next_token(pr); sbAstNode n = { .type = AST_VAL_INT, .i = t.i }; - lhs = new_node(pr, &n); + return new_node(pr, &n); } else if (t.type == T_FLOAT) { next_token(pr); sbAstNode n = { .type = AST_VAL_FLOAT, .fl = t.fl }; - lhs = new_node(pr, &n); + return new_node(pr, &n); } else if (t.type == T_SYMBOL) { next_token(pr); sbAstNode n = { .type = AST_VAL_SYMBOL, .symb = t.symb }; - lhs = new_node(pr, &n); + return new_node(pr, &n); } else if (t.type == T_STRING) { next_token(pr); sbAstNode n = { .type = AST_VAL_STRING, .str = t.hstr }; - lhs = new_node(pr, &n); + return new_node(pr, &n); + } + + return NO_NODE; +} + +static sbAst parse_name(hParser pr) { + sbLexToken t = peek_ahead(pr, 0); + if (t.type == T_IDENTIFIER) { + next_token(pr); + return name_node(pr, t); + } + return NO_NODE; +} + +static sbAst parse_hash_key(hParser pr) { + sbLexToken t = peek_ahead(pr, 0); + if (t.type == T_IDENTIFIER) { + return parse_name(pr); + } + + sbAst literal = parse_literal(pr); + if (literal != NO_NODE) { + return literal; + } else if (expect(pr, '[')) { + sbAst key = parse_expr(pr, 0); + if (!expect(pr, ']')) return syntax_error(pr); + return key; + } else { + return NO_NODE; + } +} + +static sbAst parse_inside_hash(hParser pr) { + sbAst result = NO_NODE; + sbAst *put_here = &result; + + do { + sbAst key = parse_hash_key(pr); + if (key == NO_NODE) break; + if (!expect(pr, ':')) return syntax_error(pr); + sbAst value = parse_expr(pr, 0); + + *put_here = seq_node(pr, AST_NODE_NEXT, seq_node(pr, AST_NODE_HASHENTRY, key, value), NO_NODE); + put_here = &(*put_here)->seq.right; + } while (expect(pr, ',')); + + return result; +} + +static sbAst parse_stmt(hParser pr); +static sbAst parse_stmtseq(hParser pr); +static sbAst parse_block(hParser pr); + +/* https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html */ +static sbAst parse_expr(hParser pr, u8 min_precedence) { + sbLexToken t = peek_ahead(pr, 0); + sbAst lhs = NO_NODE; + + sbAst literal = parse_literal(pr); + if (literal != NO_NODE) { + lhs = literal; } else if (t.type == T_IDENTIFIER) { lhs = parse_name(pr); } else if (t.type == T_FATARROW) { @@ -437,14 +492,29 @@ static sbAst parse_expr(hParser pr, u8 min_precedence) { sbAst params = parse_comma_exprs(pr, NULL); if (!expect(pr, T_RPAREN)) return syntax_error(pr); sbAst body = parse_block(pr); - return seq_node(pr, AST_VAL_FUNC, params, body); + lhs = seq_node(pr, AST_VAL_FUNC, params, body); } else if (t.type == T_SQUIGARROW) { next_token(pr); if (!expect(pr, T_LPAREN)) return syntax_error(pr); sbAst params = parse_comma_exprs(pr, NULL); if (!expect(pr, T_RPAREN)) return syntax_error(pr); sbAst body = parse_block(pr); - return seq_node(pr, AST_VAL_OBJ, params, body); + lhs = seq_node(pr, AST_VAL_OBJ, params, body); + } else if (t.type == T_COLONBRACE) { + next_token(pr); + sbAst body = parse_stmtseq(pr); + if (!expect(pr, T_RBRACE)) return syntax_error(pr); + lhs = wrap_node(pr, AST_VAL_IMFUNC, body); + } else if (t.type == T_LBRACKET) { + next_token(pr); + sbAst content = parse_comma_exprs(pr, NULL); + if (!expect(pr, T_RBRACKET)) return syntax_error(pr); + lhs = wrap_node(pr, AST_VAL_LIST, content); + } else if (t.type == T_LBRACE) { + next_token(pr); + sbAst content = parse_inside_hash(pr); + if (!expect(pr, T_RBRACE)) return syntax_error(pr); + lhs = wrap_node(pr, AST_VAL_HASH, content); } else if (t.type == T_LPAREN) { next_token(pr); lhs = parse_expr(pr, 0); @@ -504,13 +574,20 @@ static sbAst parse_expr(hParser pr, u8 min_precedence) { /* indexing */ rhs = parse_expr(pr, 0); if (!expect(pr, T_RBRACKET)) return syntax_error(pr); - } else if (op.type == T_DOT) { + } else if (op.type == T_DOT || op.type == T_ARROW) { sbAst method_name = parse_name(pr); if (!expect(pr, T_LPAREN)) return syntax_error(pr); sbAst params = parse_comma_exprs(pr, NULL); if (!expect(pr, T_RPAREN)) return syntax_error(pr); ast_type = AST_NODE_METHODCALL; + if (op.type == T_ARROW) { + /* (whatever)->x is rewritten as (*whatever).x */ + lhs = unop_node(pr, AST_OP_DEREF, lhs); + } rhs = seq_node(pr, AST_NODE_NEXT, method_name, params); + } else if (op.type == T_BACKSQUIGARROW) { + /* a <~ b, c, d can have multiple comma things on the right side */ + rhs = parse_comma_exprs(pr, NULL); } else { rhs = parse_expr(pr, infix->right_precedence); } @@ -525,9 +602,6 @@ static sbAst parse_expr(hParser pr, u8 min_precedence) { return lhs; } -static sbAst parse_stmt(hParser pr); -static sbAst parse_stmtseq(hParser pr); - static sbAst parse_block(hParser pr) { if (!expect(pr, '{')) return syntax_error(pr); @@ -620,7 +694,8 @@ static sbAst parse_stmt(hParser pr) { sbAst condition = parse_expr(pr, 0); return seq_node(pr, AST_NODE_REPEAT, body, condition); } else { - return syntax_error(pr); + /* repeat { ... } # infinite loop */ + return seq_node(pr, AST_NODE_REPEAT, body, NO_NODE); } } else if (t.type == T_rCASE) { /* case x, y { ... } */ @@ -669,25 +744,32 @@ static sbAst parse_stmt(hParser pr) { return_node = with_trailing_conditional(pr, return_node); return return_node; } else { - /* (statement that is just an expression) */ + /* (statement that is just an expression maybe followed by other stuff) */ sbAst expr = parse_expr(pr, 0); if (expr == NO_NODE) return NO_NODE; - if (peek_ahead(pr, 0).type == ',') { - /* (implicit return) a, b, c */ - next_token(pr); - expr = parse_comma_exprs(pr, expr); - } + if (expect(pr, T_DOUBLEPLUS)) { + expr = wrap_node(pr, AST_NODE_INCR, expr); + } else if (expect(pr, T_DOUBLEMINUS)) { + expr = wrap_node(pr, AST_NODE_DECR, expr); + } else { + if (expect(pr, ',')) { + /* (implicit return) a, b, c */ + expr = parse_comma_exprs(pr, expr); + } - if (peek_ahead(pr, 0).type == '=') { - /* a, b, c = 1, 2, 3 */ - next_token(pr); - sbAst assigned_values = parse_comma_exprs(pr, 0); - expr = seq_node(pr, AST_NODE_ASSIGN, expr, assigned_values); + if (peek_ahead(pr, 0).type == '=') { + /* a, b, c = 1, 2, 3 */ + next_token(pr); + sbAst assigned_values = parse_comma_exprs(pr, 0); + expr = seq_node(pr, AST_NODE_ASSIGN, expr, assigned_values); + } } if (expr != NO_NODE) { - /* a, b, c = 1, 2, 3 (or other expr) if condition */ + /* a, b, c = 1, 2, 3 if condition */ + /* x++ unless condition */ + /* f() if condition */ expr = with_trailing_conditional(pr, expr); } @@ -695,7 +777,18 @@ static sbAst parse_stmt(hParser pr) { } } -static sbAst do_parse(hParser pr) { +static sbAst parse_program(hParser pr) { sbAst program = parse_stmtseq(pr); + if (!expect(pr, T_EOF)) return syntax_error(pr); + return program; } + +static sbAst do_parse(hParser pr) { + sbAst program = parse_program(pr); + if (pr->any_error) { + return ERROR_NODE; + } else { + return program; + } +} diff --git a/src/parse/parser.h b/src/parse/parser.h index cbfe3d2..7b1b087 100644 --- a/src/parse/parser.h +++ b/src/parse/parser.h @@ -8,6 +8,7 @@ typedef struct sbParser { sbTokenQueue input_queue; sbArena node_arena; flag error_state; + flag any_error; } sbParser; typedef sbParser *hParser; diff --git a/src/parse/scanner.c b/src/parse/scanner.c index fa45aab..344a8be 100644 --- a/src/parse/scanner.c +++ b/src/parse/scanner.c @@ -50,7 +50,6 @@ static void check_if_start(hScanner sc); static sbLexToken compute_next_token(hScanner sc); void sbScanner_initialize(hScanner sc, hFileReader fr) { - sbArena_initialize(&sc->arena, MEM_BLOCK_SIZE); sc->file_reader = fr; sc->next_token = (sbLexToken) {0}; @@ -70,7 +69,6 @@ sbLexToken sbScanner_next(hScanner sc) { } void sbScanner_deinitialize(hScanner sc) { - sbArena_deinitialize(&sc->arena); sbBuffer_deinitialize(&sc->dynamic_buffer); } @@ -237,7 +235,8 @@ static sbLexToken compute_next_token(hScanner sc) { || ch == '[' || ch == ']' || ch == '{' || ch == '}' || ch == ';' || ch == '|' - || ch == ',' || ch == '\\') { + || ch == '&' || ch == ',' + || ch == '@' || ch == '\\') { /* unambiguously single-character tokens */ new_token.type = ch; NEXT; diff --git a/src/parse/scanner.h b/src/parse/scanner.h index e2a8bd7..ec1b07e 100644 --- a/src/parse/scanner.h +++ b/src/parse/scanner.h @@ -10,7 +10,6 @@ typedef struct sbScanner { sbLexToken next_token; hFileReader file_reader; - sbArena arena; sbBuffer dynamic_buffer; } sbScanner; diff --git a/src/parse/token.h b/src/parse/token.h index e001be4..beedd84 100644 --- a/src/parse/token.h +++ b/src/parse/token.h @@ -30,6 +30,8 @@ typedef enum sbTokenType { T_COLON = ':', T_SEMICOLON = ';', T_BACKSLASH = '\\', + T_AMPERSAND = '&', + T_AT = '@', T_NEWLINE = '\n', T_SPACE = ' ', T_IDENTIFIER = 128, // abc -- 1.8.3.1