more parser stuff + much more friendly syntax error reporting

diff --git a/src/parse/ast.h b/src/parse/ast.h
index 2ed4424..33238ca 100644 (file)
--- a/src/parse/ast.h
+++ b/src/parse/ast.h
@@ -5,6 +5,7 @@
 
 typedef enum sbAstType {
   AST_NULL,
+  AST_ERROR,
   AST_VAL_NIL,
   AST_VAL_INT,
   AST_VAL_STRING,

diff --git a/src/parse/filereader.c b/src/parse/filereader.c
index b30f260..fce211c 100644 (file)
--- a/src/parse/filereader.c
+++ b/src/parse/filereader.c
@@ -1,5 +1,9 @@
 #include "filereader.h"
 
+#include "mem/mem.h"
+
+#define N_READBACK_LINES 3
+
 /* this seems pointless right now as a wrapper for FILE* but later, we can make
  * this work to read from stdin in interactive mode as well, which will be easier
  * if we just have these few functions separate from the actual lexer code.
@@ -7,20 +11,34 @@
 
 struct sbFileReader {
     flag valid;
+    i32 line;
+    i32 col;
+    i32 char_index;
+    sbBuffer current_line;
+    sbBuffer readback_lines[N_READBACK_LINES];
     FILE *file;
 };
 
+static void read_in_new_line(hFileReader r);
+
 hFileReader sbFileReader_open(const char *filename) {
-    hFileReader r = malloc(sizeof(struct sbFileReader));
-    r->file = fopen(filename, "r");
-    if (!r->file) {
-        perror("Error opening file for reading");
-        r->valid = 0;
-    } else {
-        r->valid = 1;
+  hFileReader r = calloc(1, sizeof(struct sbFileReader));
+  r->file = fopen(filename, "r");
+  r->line = 1;
+  r->char_index = 0;
+  r->col = 1;
+  if (!r->file) {
+    perror("Error opening file for reading");
+  } else {
+    r->valid = 1;
+    for (usize i = 0; i < N_READBACK_LINES; i++) {
+      sbBuffer_initialize(&r->readback_lines[i], 128);
     }
+    sbBuffer_initialize(&r->current_line, 128);
+    read_in_new_line(r);
+  }
 
-    return r;
+  return r;
 }
 
 flag sbFileReader_ok(hFileReader r) {
@@ -28,27 +46,96 @@ flag sbFileReader_ok(hFileReader r) {
 }
 
 int sbFileReader_peek(hFileReader r) {
-    if (r->valid) {
-        char c = fgetc(r->file);
-        ungetc(c, r->file);
-        return c;
+  if (r->valid) {
+    int c = ((int*)r->current_line.data)[r->char_index];
+    return c;
+  } else {
+    return EOF;
+  }
+}
+
+int sbFileReader_next(hFileReader r) {
+  if (r->valid) {
+    r->char_index ++;
+    int c = sbFileReader_peek(r);
+    if (c == '\t') {
+      r->col += 4;
+    } else if (c == 0) {
+      read_in_new_line(r);
+      r->line ++;
+      r->char_index = 0;
+      r->col = 1;
+      c = sbFileReader_peek(r);
     } else {
-        return EOF;
+      r->col ++;
     }
+    return c;
+  } else {
+    return EOF;
+  }
 }
 
-int sbFileReader_next(hFileReader r) {
-    if (r->valid) {
-        fgetc(r->file);
-        return sbFileReader_peek(r);
+int sbFileReader_get_line(hFileReader r) {
+  return r->line;
+}
+
+int sbFileReader_get_col(hFileReader r) {
+  return r->col;
+}
+
+void sbFileReader_fprint_line_at(FILE *out, hFileReader r, int line_num) {
+  isize readback_amount = r->line - line_num;
+  sbBuffer *buffer;
+  if (readback_amount == 0) {
+    buffer = &r->current_line;
+  } else if (readback_amount > 0 && readback_amount <= N_READBACK_LINES) {
+    buffer = &r->readback_lines[N_READBACK_LINES - readback_amount];
+  } else {
+    fprintf(out, "<line not available>\n");
+    return;
+  }
+  int *to_read = (int*)buffer->data;
+  for (int i = 0; i < buffer->size / sizeof(int); i++) {
+    if (to_read[i] == 0 || to_read[i] == '\n') break;
+    if (to_read[i] == '\t') {
+      fputs("    ", out);
     } else {
-        return EOF;
+      fputc(to_read[i], out);
     }
+  }
+  fputc('\n', out);
 }
 
 void sbFileReader_close(hFileReader r) {
     if (r->valid) {
         fclose(r->file);
     }
+    sbBuffer_deinitialize(&r->current_line);
+    for (usize i = 0; i < N_READBACK_LINES; i++) {
+      sbBuffer_deinitialize(&r->readback_lines[i]);
+    }
     free(r);
 }
+
+/* --- */
+
+static void read_in_new_line(hFileReader r) {
+    if (!r->valid) return;
+
+    /* recycle line buffers */
+    sbBuffer oldest_line_buffer = r->readback_lines[0];
+    for (int i = 0; i < N_READBACK_LINES - 1; i++) {
+      r->readback_lines[i] = r->readback_lines[i + 1];
+    }
+    r->readback_lines[N_READBACK_LINES - 1] = r->current_line;
+    r->current_line = oldest_line_buffer;
+    sbBuffer_reset(&r->current_line);
+
+    int c;
+    do {
+      c = fgetc(r->file);
+      sbBuffer_append(&r->current_line, &c, sizeof(int));
+    } while (c != '\n' && c != EOF);
+    int zero = 0;
+    sbBuffer_append(&r->current_line, &zero, sizeof(int));
+}

diff --git a/src/parse/filereader.h b/src/parse/filereader.h
index 06ba22b..fa90516 100644 (file)
--- a/src/parse/filereader.h
+++ b/src/parse/filereader.h
@@ -10,4 +10,10 @@ int sbFileReader_next(hFileReader r);
 
 int sbFileReader_peek(hFileReader r);
 
+int sbFileReader_get_line(hFileReader r);
+
+int sbFileReader_get_col(hFileReader r);
+
+void sbFileReader_fprint_line_at(FILE *out, hFileReader r, int line_num);
+
 void sbFileReader_close(hFileReader r);

diff --git a/src/parse/lexer.c b/src/parse/lexer.c
index d6101c0..c19eaa7 100644 (file)
--- a/src/parse/lexer.c
+++ b/src/parse/lexer.c
@@ -75,6 +75,14 @@ static void enqueue_output_token(hLexer lx, sbLexToken token) {
         unstack_visible_bracket(lx, token);
     }
 
+    if (token.line == 0) {
+      /* fake tokens that are inserted get their location set as just after the
+       * last token seen */
+      token.line = lx->last_token_seen.line;
+      token.start_col = lx->last_token_seen.end_col;
+      token.end_col = lx->last_token_seen.end_col + 1;
+    }
+
     sbTokenQueue_enqueue(&lx->output_queue, token);
 
     if (is_closing_bracket(token.type) && !token.invisible) {
@@ -203,7 +211,7 @@ static flag begins_brace_terminated_state(sbTokenType type) {
 
 /* things we can potentially insert a ( after because they may be a call */
 static flag can_end_expression(sbTokenType type) {
-    return is_literal(type)
+    return type == T_IDENTIFIER
         || type == T_RPAREN
         || type == T_RBRACKET
         || type == T_RBRACE;
@@ -213,6 +221,7 @@ static flag can_end_expression(sbTokenType type) {
  * in order to exit brace-terminated state */
 static flag block_header_can_end_after(sbTokenType type) {
     return can_end_expression(type)
+        || is_literal(type)
         || type == T_FATARROW
         || type == T_SQUIGARROW
         || type == T_rCASE
@@ -353,14 +362,7 @@ static void unstack_visible_bracket(hLexer lx, sbLexToken closing_token) {
         }
     }
 
-    if (lx->brackets_stack.size != 0) {
-        fprintf(stderr, "syntax error: mismatched bracket ('%c' for '%c')\n",
-                lx->brackets_stack.data[lx->brackets_stack.size - 1], closing_token.type);
-    } else {
-        fprintf(stderr, "syntax error: mismatched bracket\n");
-    }
-
-    enqueue_output_token(lx, (sbLexToken) { .type = T_ERROR });
+    enqueue_output_token(lx, (sbLexToken) { .type = T_WRONGBRACKET });
 }
 
 static void compute_next_token(hLexer lx) {
@@ -397,8 +399,6 @@ static void compute_next_token(hLexer lx) {
         unstack_all_invisible_parentheses(lx);
     }
 
-    //printf("\nstack: %s\n", lx->brackets_stack.data);
-
     /* --- HERE IS WHERE THE TOKEN ACTUALLY GETS OUTPUT TO THE STREAM --- */
     /* Everything before this gets put into the stream ahead of this token. */
     /* Everything after this comes after the token. */
@@ -485,6 +485,12 @@ static void compute_next_token(hLexer lx) {
         }
     }
 
+    if (token.line == 0) {
+        token.line = lx->last_token_seen.line;
+        token.start_col = lx->last_token_seen.start_col;
+        token.end_col = lx->last_token_seen.end_col;
+    }
+
     if (token.type != T_SPACE && token.type != T_NEWLINE) {
         lx->last_token_seen = token;
     }
@@ -501,7 +507,13 @@ static void compute_next_token(hLexer lx) {
             sbLexToken after_newline = input_peek_ahead(lx, 0);
 
             if (!cancel_semicolon_before(after_newline.type)) {
-                sbLexToken invisible_semicolon = { .type = T_SEMICOLON, .invisible = 1 };
+                sbLexToken invisible_semicolon = {
+                  .type = T_SEMICOLON,
+                  .invisible = 1,
+                  .line = lx->last_token_seen.line,
+                  .start_col = lx->last_token_seen.start_col,
+                  .end_col = lx->last_token_seen.end_col,
+                };
                 enqueue_output_token(lx, invisible_semicolon);
                 lx->last_token_seen = invisible_semicolon;
             }

diff --git a/src/parse/parser.c b/src/parse/parser.c
index 83c09ee..667f8ea 100644 (file)
--- a/src/parse/parser.c
+++ b/src/parse/parser.c
@@ -6,6 +6,7 @@
 static sbAst do_parse(hParser pr);
 
 void sbParser_initialize(hParser pr) {
+  *pr = (sbParser) {0};
   sbArena_initialize(&pr->node_arena, 32768);
   sbTokenQueue_initialize(&pr->input_queue, 8);
 }
@@ -13,6 +14,7 @@ void sbParser_initialize(hParser pr) {
 void sbParser_deinitialize(hParser pr) {
   sbTokenQueue_deinitialize(&pr->input_queue);
   sbArena_deinitialize(&pr->node_arena);
+  *pr = (sbParser) {0};
 }
 
 sbAst sbParser_parse_file(hParser pr, const char *filename) {
@@ -47,13 +49,129 @@ typedef struct unop {
   sbAstOp ast_op;
 } unop;
 
-typedef struct opspelling {
-  sbAstOp ast_op;
+typedef struct tokenspelling {
+  sbTokenType type;
   const char *name;
-} opspelling;
+} tokenspelling;
+
+static sbAstNode NONE_SENTINEL_VALUE = {0};
+static sbAst NO_NODE = &NONE_SENTINEL_VALUE;
+static sbAstNode ERROR_SENTINEL_VALUE = { .type = AST_ERROR };
+static sbAst ERROR_NODE = &ERROR_SENTINEL_VALUE;
+
+static tokenspelling token_spellings[] = {
+  { T_NEWLINE, "newline" },
+  { T_EOF, "end-of-file" },
+  { T_IDENTIFIER, "identifier" },
+  { T_SYMBOL, "symbol" },
+  { T_INTEGER, "integer" },
+  { T_FLOAT, "floating-point literal" },
+  { T_STRING, "string literal" },
+  { T_rAND, "'and'" },
+  { T_rAS, "'as'" },
+  { T_rCASE, "'case'" },
+  { T_rDEF, "'def'" },
+  { T_rDO, "'do'" },
+  { T_rELSE, "'else'" },
+  { T_rFALSE, "'false'" },
+  { T_rIF, "'if'" },
+  { T_rIN, "'in'" },
+  { T_rLET, "'let'" },
+  { T_rMATCH, "'match'" },
+  { T_rNIL, "'nil'" },
+  { T_rNOT, "'not'" },
+  { T_rOR, "'or'" },
+  { T_rREPEAT, "'repeat'" },
+  { T_rRETURN, "'return'" },
+  { T_rTRUE, "'true'" },
+  { T_rUNLESS, "'unless'" },
+  { T_rUNTIL, "'until'" },
+  { T_rWHEN, "'when'" },
+  { T_rWHILE, "'while'" },
+  { T_LPAREN, "'('" },
+  { T_RPAREN, "')'" },
+  { T_LBRACKET, "'['" },
+  { T_RBRACKET, "']'" },
+  { T_LBRACE, "'{'" },
+  { T_RBRACE, "'}'" },
+  { T_ASTERISK, "'*'" },
+  { T_SLASH, "'/'" },
+  { T_PLUS, "'+'" },
+  { T_MINUS, "'-'" },
+  { T_PERCENT, "'%'" },
+  { T_PIPE, "'|'" },
+  { T_DOT, "'.'" },
+  { T_COMMA, "','" },
+  { T_EQUALS, "'='" },
+  { T_LESS, "'<'" },
+  { T_GREATER, "'>'" },
+  { T_COLON, "':'" },
+  { T_SEMICOLON, "';'" },
+  { T_BACKSLASH, "'\\'" },
+  { T_SPACE, "' '" },
+  { T_ARROW, "'->'" },
+  { T_FATARROW, "'=>'" },
+  { T_SQUIGARROW, "'~>'" },
+  { T_BACKSQUIGARROW, "'<~'" },
+  { T_COLONBRACE, "':{'" },
+  { T_PAAMAYIM_NEKUDOTAYIM, "'::'" },
+  { T_DOUBLEEQUALS, "'=='" },
+  { T_DOUBLEMINUS, "'--'" },
+  { T_DOUBLEPLUS, "'++'" },
+  { T_DOUBLEASTERISK, "'**'" },
+  { T_DOUBLESLASH, "'//'" },
+  { T_DOUBLEPERCENT, "'%%'" },
+  { T_DOUBLEGREATER, "'>>'" },
+  { T_DOUBLELESS, "'<<'" },
+  { T_MINUSEQUALS, "'-='" },
+  { T_PLUSEQUALS, "'+='" },
+  { T_ASTERISKEQUALS, "'*='" },
+  { T_SLASHEQUALS, "'/='" },
+  { T_PERCENTEQUALS, "'%='" },
+  { T_LESSEQUALS, "'<='" },
+  { T_GREATEREQUALS, "'>='" },
+  { T_NOTEQUALS, "'!='" },
+  { T_TWODOT, "'..'" },
+  { T_ELLIPSIS, "'...'" },
+};
 
-static sbAstNode SENTINEL_VALUE = {0};
-static sbAst NO_NODE = &SENTINEL_VALUE;
+const char *const TOKEN_SPELLING_UNKNOWN = "<unknown-token>";
+
+static binop binops[] = {
+  { T_PIPE, 6, 7, AST_OP_PIPE },
+  { T_rOR, 10, 11, AST_OP_OR },
+  { T_rAND, 20, 21, AST_OP_AND },
+  { T_rIN, 25, 26, AST_OP_IN },
+  { T_DOUBLEEQUALS, 30, 31, AST_OP_EQ },
+  { T_GREATER, 30, 31, AST_OP_GT },
+  { T_LESS, 30, 31, AST_OP_LT },
+  { T_GREATEREQUALS, 30, 31, AST_OP_GE },
+  { T_LESSEQUALS, 30, 31, AST_OP_LE },
+  { T_NOTEQUALS, 30, 31, AST_OP_NE },
+  { T_DOUBLEPERCENT, 30, 31, AST_OP_DIVBY },
+  { T_PLUS, 45, 46, AST_OP_ADD },
+  { T_MINUS, 45, 46, AST_OP_SUB },
+  { T_ASTERISK, 60, 61, AST_OP_MUL },
+  { T_SLASH, 60, 61, AST_OP_DIV },
+  { T_PERCENT, 60, 61, AST_OP_MOD },
+  { T_DOUBLESLASH, 60, 61, AST_OP_FLDIV },
+  { T_DOUBLEASTERISK, 71, 70, AST_OP_POW },
+  { T_TWODOT, 80, 81, AST_OP_RANGE },
+  { T_DOT, 90, 91, AST_OP_NULL },
+  { T_LPAREN, 90, 91, AST_OP_NULL },
+  { T_LBRACKET, 90, 91, AST_OP_INDEX },
+  { T_PAAMAYIM_NEKUDOTAYIM, 90, 91, AST_OP_SCOPE },
+};
+
+static unop unops[] = {
+  { T_rNOT, 25, AST_OP_NOT },
+  { T_PLUS, 85, AST_OP_UNPLUS },
+  { T_MINUS, 85, AST_OP_UNMINUS },
+};
+
+const usize NUM_BINOPS = sizeof(binops) / sizeof(binops[0]);
+const usize NUM_UNOPS = sizeof(unops) / sizeof(unops[0]);
+const usize NUM_TOKEN_SPELLINGS = sizeof(token_spellings) / sizeof(token_spellings[0]);
 
 static sbLexToken peek_ahead(hParser pr, usize count) {
   while (sbTokenQueue_size(&pr->input_queue) < count + 1) {
@@ -63,19 +181,103 @@ static sbLexToken peek_ahead(hParser pr, usize count) {
   return sbTokenQueue_at(&pr->input_queue, count);
 }
 
+static void fprint_token(FILE *out, sbLexToken t);
 static sbLexToken next_token(hParser pr) {
   if (sbTokenQueue_size(&pr->input_queue) > 0) {
     sbTokenQueue_shift(&pr->input_queue);
   }
 
-  return sbTokenQueue_at(&pr->input_queue, 0);
+  pr->error_state = 0;
+
+  sbLexToken t = peek_ahead(pr, 0);
+
+  return t;
+}
+
+flag is_literal_token(sbTokenType type) {
+  return type == T_IDENTIFIER
+      || type == T_SYMBOL
+      || type == T_INTEGER
+      || type == T_FLOAT
+      || type == T_STRING;
+}
+
+const char *token_spelling(sbTokenType type) {
+  for (usize i = 0; i < NUM_TOKEN_SPELLINGS; i++) {
+    if (token_spellings[i].type == type) {
+      return token_spellings[i].name;
+    }
+  }
+  return TOKEN_SPELLING_UNKNOWN;
+}
+
+static void fprint_token(FILE *out, sbLexToken t) {
+  fprintf(out, "%s", token_spelling(t.type));
+
+  if (t.type == T_IDENTIFIER) {
+    fprintf(out, " '%s'", sbSymbol_name(t.symb));
+  } else if (t.type == T_SYMBOL) {
+    fprintf(out, " ':%s'", sbSymbol_name(t.symb));
+  } else if (t.type == T_INTEGER) {
+    fprintf(out, " '%d'", t.i);
+  } else if (t.type == T_FLOAT) {
+    fprintf(out, " '%g'", t.fl);
+  }
+}
+
+static void fprint_repeat_char(FILE *out, int ch, usize length) {
+  for (int i = 0; i < length; i++) {
+    fputc(ch, out);
+  }
 }
 
-static flag expect(hParser pr, sbTokenType type, sbLexToken *tok_out) {
+static void fprint_left_error_margin(FILE *out, usize line_num) {
+  /* indentation */
+  if (line_num) {
+    fprintf(out, "%6zu | ", line_num);
+  } else {
+    fprintf(out, "       | ");
+  }
+}
+
+static sbAst syntax_error(hParser pr) {
+  if (pr->error_state) return ERROR_NODE;
+
+  sbLexToken unexpected_token = peek_ahead(pr, 0);
+  if (unexpected_token.type == T_ERROR) {
+    fprintf(stderr, "syntax error: invalid character");
+  } else if (unexpected_token.type == T_WRONGBRACKET) {
+    fprintf(stderr, "syntax error: mismatched bracket");
+  } else if (unexpected_token.type == T_BADNUMBER) {
+    fprintf(stderr, "syntax error: invalid number");
+  } else if (unexpected_token.type == T_SEMICOLON && unexpected_token.invisible) {
+    fprintf(stderr, "syntax error: unexpected end-of-line");
+  } else if (unexpected_token.invisible) {
+    fprintf(stderr, "syntax error");
+  } else {
+    fprintf(stderr, "syntax error: unexpected ");
+    fprint_token(stderr, unexpected_token);
+  }
+  fprintf(stderr, " at line %d\n", unexpected_token.line);
+  fprint_left_error_margin(stderr, 0);
+  fprintf(stderr, "\n");
+  fprint_left_error_margin(stderr, unexpected_token.line);
+  sbFileReader_fprint_line_at(stderr, pr->lexer.scanner.file_reader, unexpected_token.line);
+  fprint_left_error_margin(stderr, 0);
+  fprint_repeat_char(stderr, ' ', unexpected_token.start_col - 1);
+  fprint_repeat_char(stderr, '^', unexpected_token.end_col - unexpected_token.start_col + 1);
+  fprintf(stderr, "\n\n");
+
+  /* set error flag until we move to the next token so that syntax error
+   * isn't printed multiple times */
+  pr->error_state = TRUE;
+  return ERROR_NODE;
+}
+
+static flag expect(hParser pr, sbTokenType type) {
   sbLexToken up_next = peek_ahead(pr, 0);
   if (up_next.type == type) {
     next_token(pr);
-    if (tok_out) *tok_out = up_next;
     return TRUE;
   } else {
     return FALSE;
@@ -143,7 +345,7 @@ static sbAst atomic_node(hParser pr, sbAstType type) {
 
 static sbAst name_node(hParser pr, sbLexToken token) {
   if (token.type != T_IDENTIFIER) {
-    PANIC("can't create name node with token of type %d\n", token.type);
+    PANIC("can't create name node with token of type %d", token.type);
   }
 
   sbAstNode n = (sbAstNode) {
@@ -153,68 +355,6 @@ static sbAst name_node(hParser pr, sbLexToken token) {
   return new_node(pr, &n);
 }
 
-static binop binops[] = {
-  { T_PIPE, 6, 7, AST_OP_PIPE },
-  { T_rOR, 10, 11, AST_OP_OR },
-  { T_rAND, 20, 21, AST_OP_AND },
-  { T_rIN, 25, 26, AST_OP_IN },
-  { T_DOUBLEEQUALS, 30, 31, AST_OP_EQ },
-  { T_GREATER, 30, 31, AST_OP_GT },
-  { T_LESS, 30, 31, AST_OP_LT },
-  { T_GREATEREQUALS, 30, 31, AST_OP_GE },
-  { T_LESSEQUALS, 30, 31, AST_OP_LE },
-  { T_NOTEQUALS, 30, 31, AST_OP_NE },
-  { T_DOUBLEPERCENT, 30, 31, AST_OP_DIVBY },
-  { T_PLUS, 45, 46, AST_OP_ADD },
-  { T_MINUS, 45, 46, AST_OP_SUB },
-  { T_ASTERISK, 60, 61, AST_OP_MUL },
-  { T_SLASH, 60, 61, AST_OP_DIV },
-  { T_PERCENT, 60, 61, AST_OP_MOD },
-  { T_DOUBLESLASH, 60, 61, AST_OP_FLDIV },
-  { T_DOUBLEASTERISK, 71, 70, AST_OP_POW },
-  { T_TWODOT, 80, 81, AST_OP_RANGE },
-  { T_DOT, 90, 91, AST_OP_NULL },
-  { T_LPAREN, 90, 91, AST_OP_NULL },
-  { T_LBRACKET, 90, 91, AST_OP_INDEX },
-  { T_PAAMAYIM_NEKUDOTAYIM, 100, 101, AST_OP_SCOPE },
-};
-
-static unop unops[] = {
-  { T_rNOT, 25, AST_OP_NOT },
-  { T_PLUS, 85, AST_OP_UNPLUS },
-  { T_MINUS, 85, AST_OP_UNMINUS },
-};
-
-static opspelling op_spellings[] = {
-  { AST_OP_OR, "or" },
-  { AST_OP_AND, "and" },
-  { AST_OP_NOT, "not" },
-  { AST_OP_IN, "in" },
-  { AST_OP_EQ, "==" },
-  { AST_OP_NE, "!=" },
-  { AST_OP_GT, ">" },
-  { AST_OP_LT, "<" },
-  { AST_OP_GE, ">=" },
-  { AST_OP_LE, "<=" },
-  { AST_OP_DIVBY, "%%" },
-  { AST_OP_ADD, "+" },
-  { AST_OP_UNPLUS, "+@" },
-  { AST_OP_SUB, "-" },
-  { AST_OP_UNMINUS, "-@" },
-  { AST_OP_MUL, "*" },
-  { AST_OP_DIV, "/" },
-  { AST_OP_MOD, "%" },
-  { AST_OP_FLDIV, "//" },
-  { AST_OP_POW, "**" },
-  { AST_OP_RANGE, ".." },
-  { AST_OP_INDEX, "[]" },
-  { AST_OP_SCOPE, "::" },
-};
-
-const int NUM_BINOPS = sizeof(binops) / sizeof(binops[0]);
-const int NUM_UNOPS = sizeof(unops) / sizeof(unops[0]);
-const int NUM_SPELLINGS = sizeof(op_spellings) / sizeof(op_spellings[0]);
-
 static sbAst parse_expr(hParser pr, u8 min_precedence);
 static sbAst parse_comma_exprs(hParser pr, sbAst after) {
   sbAst result = NO_NODE;
@@ -227,7 +367,7 @@ static sbAst parse_comma_exprs(hParser pr, sbAst after) {
   }
 
   do {
-    if (expect(pr, T_ELLIPSIS, NULL)) {
+    if (expect(pr, T_ELLIPSIS)) {
       sbAst splatted_expr = parse_expr(pr, 0);
       if (splatted_expr != NO_NODE) {
         /* "...something" */
@@ -245,7 +385,7 @@ static sbAst parse_comma_exprs(hParser pr, sbAst after) {
 
     *put_here = seq_node(pr, AST_NODE_MULTIVAL, expr, NO_NODE);
     put_here = &(*put_here)->seq.right;
-  } while (expect(pr, T_COMMA, NULL));
+  } while (expect(pr, T_COMMA));
 
   return result;
 }
@@ -253,7 +393,8 @@ static sbAst parse_comma_exprs(hParser pr, sbAst after) {
 static sbAst parse_name(hParser pr) {
   sbLexToken t = peek_ahead(pr, 0);
   if (t.type == T_IDENTIFIER) {
-    return name_node(pr, next_token(pr));
+    next_token(pr);
+    return name_node(pr, t);
   }
   return NO_NODE;
 }
@@ -265,62 +406,53 @@ static sbAst parse_expr(hParser pr, u8 min_precedence) {
   sbLexToken t = peek_ahead(pr, 0);
   sbAst lhs = NO_NODE;
   if (t.type == T_rNIL) {
+    next_token(pr);
     sbAstNode n = { .type = AST_VAL_NIL };
     lhs = new_node(pr, &n);
   } else if (t.type == T_rTRUE || t.type == T_rFALSE) {
+    next_token(pr);
     sbAstNode n = { .type = AST_VAL_BOOLEAN, .i = (t.type == T_rTRUE) };
     lhs = new_node(pr, &n);
   } else if (t.type == T_INTEGER) {
+    next_token(pr);
     sbAstNode n = { .type = AST_VAL_INT, .i = t.i };
     lhs = new_node(pr, &n);
   } else if (t.type == T_FLOAT) {
+    next_token(pr);
     sbAstNode n = { .type = AST_VAL_FLOAT, .fl = t.fl };
     lhs = new_node(pr, &n);
   } else if (t.type == T_SYMBOL) {
+    next_token(pr);
     sbAstNode n = { .type = AST_VAL_SYMBOL, .symb = t.symb };
     lhs = new_node(pr, &n);
   } else if (t.type == T_STRING) {
+    next_token(pr);
     sbAstNode n = { .type = AST_VAL_STRING, .str = t.hstr };
     lhs = new_node(pr, &n);
   } else if (t.type == T_IDENTIFIER) {
     lhs = parse_name(pr);
   } else if (t.type == T_FATARROW) {
     next_token(pr);
-    if (!expect(pr, T_LPAREN, NULL)) {
-      fprintf(stderr, "expected '(' after '=>' token\n");
-      return NO_NODE;
-    }
+    if (!expect(pr, T_LPAREN)) return syntax_error(pr);
     sbAst params = parse_comma_exprs(pr, NULL);
-    if (!expect(pr, T_RPAREN, NULL)) {
-      fprintf(stderr, "expected ')' after function parameters\n");
-      return NO_NODE;
-    }
+    if (!expect(pr, T_RPAREN)) return syntax_error(pr);
     sbAst body = parse_block(pr);
     return seq_node(pr, AST_VAL_FUNC, params, body);
   } else if (t.type == T_SQUIGARROW) {
     next_token(pr);
-    if (!expect(pr, T_LPAREN, NULL)) {
-      fprintf(stderr, "expected '(' after '~>' token\n");
-      return NO_NODE;
-    }
+    if (!expect(pr, T_LPAREN)) return syntax_error(pr);
     sbAst params = parse_comma_exprs(pr, NULL);
-    if (!expect(pr, T_RPAREN, NULL)) {
-      fprintf(stderr, "expected ')' after function parameters\n");
-      return NO_NODE;
-    }
+    if (!expect(pr, T_RPAREN)) return syntax_error(pr);
     sbAst body = parse_block(pr);
     return seq_node(pr, AST_VAL_OBJ, params, body);
   } else if (t.type == T_LPAREN) {
     next_token(pr);
     lhs = parse_expr(pr, 0);
-    if (!expect(pr, T_RPAREN, NULL)) {
-      fprintf(stderr, "expected closing parenthesis!\n");
-      return NO_NODE;
-    }
+    if (!expect(pr, T_RPAREN)) return syntax_error(pr);
   } else {
     sbLexToken op = peek_ahead(pr, 0);
     unop *prefix = NULL;
-    for (int i = 0; i < NUM_UNOPS; i++) {
+    for (usize i = 0; i < NUM_UNOPS; i++) {
       if (unops[i].type == op.type) {
         prefix = &unops[i];
         break;
@@ -343,7 +475,7 @@ static sbAst parse_expr(hParser pr, u8 min_precedence) {
     sbLexToken op = peek_ahead(pr, 0);
     sbAstType ast_type = AST_NODE_OP;
     binop *infix = NULL;
-    for (int i = 0; i < NUM_BINOPS; i++) {
+    for (usize i = 0; i < NUM_BINOPS; i++) {
       if (binops[i].type == op.type) {
         infix = &binops[i];
         break;
@@ -367,35 +499,18 @@ static sbAst parse_expr(hParser pr, u8 min_precedence) {
       /* function call */
       rhs = parse_comma_exprs(pr, NULL);
       ast_type = AST_NODE_FUNCCALL;
-      if (!expect(pr, T_RPAREN, NULL)) {
-        fprintf(stderr, "expected ')' after function parameter list\n");
-        return NO_NODE;
-      }
+      if (!expect(pr, T_RPAREN)) return syntax_error(pr);
     } else if (op.type == T_LBRACKET) {
       /* indexing */
       rhs = parse_expr(pr, 0);
-      if (!expect(pr, T_RBRACKET, NULL)) {
-        fprintf(stderr, "expected ']'\n");
-        return NO_NODE;
-      }
+      if (!expect(pr, T_RBRACKET)) return syntax_error(pr);
     } else if (op.type == T_DOT) {
-      sbLexToken method_name = {0};
-      if (!expect(pr, T_IDENTIFIER, &method_name)) {
-        fprintf(stderr, "expected identifier after '.' token\n");
-        return NO_NODE;
-      }
-      sbAst name = name_node(pr, method_name);
-      if (!expect(pr, T_LPAREN, NULL)) {
-        fprintf(stderr, "expected opening-parenthesis after '.%s'\n", sbSymbol_name(method_name.symb));
-        return NO_NODE;
-      }
+      sbAst method_name = parse_name(pr);
+      if (!expect(pr, T_LPAREN)) return syntax_error(pr);
       sbAst params = parse_comma_exprs(pr, NULL);
-      if (!expect(pr, T_RPAREN, NULL)) {
-        fprintf(stderr, "expected closing-parenthesis after '.%s(...'\n", sbSymbol_name(method_name.symb));
-        return NO_NODE;
-      }
+      if (!expect(pr, T_RPAREN)) return syntax_error(pr);
       ast_type = AST_NODE_METHODCALL;
-      rhs = seq_node(pr, AST_NODE_NEXT, name, params);
+      rhs = seq_node(pr, AST_NODE_NEXT, method_name, params);
     } else {
       rhs = parse_expr(pr, infix->right_precedence);
     }
@@ -414,17 +529,11 @@ static sbAst parse_stmt(hParser pr);
 static sbAst parse_stmtseq(hParser pr);
 
 static sbAst parse_block(hParser pr) {
-  if (!expect(pr, '{', NULL)) {
-    fprintf(stderr, "syntax error! expecting '{'\n");
-    return NO_NODE;
-  }
+  if (!expect(pr, '{')) return syntax_error(pr);
 
   sbAst result = parse_stmtseq(pr);
 
-  if (!expect(pr, '}', NULL)) {
-    fprintf(stderr, "syntax error! expecting '}'\n");
-    return NO_NODE;
-  }
+  if (!expect(pr, '}')) return syntax_error(pr);
 
   return result;
 }
@@ -438,12 +547,12 @@ static sbAst parse_stmtseq(hParser pr) {
     if (stmt == NO_NODE) {
       /* if failed to parse a statement, try eating an
        * additional semicolon first */
-      if (expect(pr, ';', NULL)) continue;
+      if (expect(pr, ';')) continue;
       break;
     }
     *put_here = seq_node(pr, AST_NODE_SEQ, stmt, NO_NODE);
     put_here = &(*put_here)->seq.right;
-  } while (expect(pr, ';', NULL));
+  } while (expect(pr, ';'));
 
   return result;
 }
@@ -451,10 +560,10 @@ static sbAst parse_stmtseq(hParser pr) {
 static sbAst with_trailing_conditional(hParser pr, sbAst stmt) {
   sbAst result = stmt;
 
-  if (expect(pr, T_rIF, NULL)) {
+  if (expect(pr, T_rIF)) {
     sbAst condition = parse_expr(pr, 0);
     result = tri_node(pr, AST_NODE_IF, condition, stmt, NO_NODE);
-  } else if (expect(pr, T_rUNLESS, NULL)) {
+  } else if (expect(pr, T_rUNLESS)) {
     sbAst condition = parse_expr(pr, 0);
     result = tri_node(pr, AST_NODE_IF, unop_node(pr, AST_OP_NOT, condition), stmt, NO_NODE);
   }
@@ -465,11 +574,12 @@ static sbAst with_trailing_conditional(hParser pr, sbAst stmt) {
 static sbAst parse_stmt(hParser pr) {
   sbLexToken t = peek_ahead(pr, 0);
   if (t.type == T_rIF) {
+    /* if condition1 { ... } else if condition2 { ... } else { ... } */
     next_token(pr);
     sbAst condition = parse_expr(pr, 0);
     sbAst then_body = parse_block(pr);
     sbAst else_body = NO_NODE;
-    if (expect(pr, T_rELSE, NULL)) {
+    if (expect(pr, T_rELSE)) {
       sbLexToken t = peek_ahead(pr, 0);
       if (t.type == T_rIF) {
         /* else if ...as above... */
@@ -480,17 +590,20 @@ static sbAst parse_stmt(hParser pr) {
     }
     return tri_node(pr, AST_NODE_IF, condition, then_body, else_body);
   } else if (t.type == T_rUNLESS) {
+    /* unless condition { ... } */
     next_token(pr);
     sbAst condition = parse_expr(pr, 0);
     /* won't permit unless..else. it is too confusing. unless can only have 1 block */
     sbAst unless_body = parse_block(pr);
     return tri_node(pr, AST_NODE_IF, unop_node(pr, AST_OP_NOT, condition), unless_body, NO_NODE);
   } else if (t.type == T_rWHILE) {
+    /* while condition { ... } */
     next_token(pr);
     sbAst condition = parse_expr(pr, 0);
     sbAst body = parse_block(pr);
     return seq_node(pr, AST_NODE_WHILE, condition, body);
   } else if (t.type == T_rUNTIL) {
+    /* until condition { ... } */
     next_token(pr);
     sbAst condition = parse_expr(pr, 0);
     sbAst body = parse_block(pr);
@@ -498,31 +611,32 @@ static sbAst parse_stmt(hParser pr) {
   } else if (t.type == T_rREPEAT) {
     next_token(pr);
     sbAst body = parse_block(pr);
-    if (expect(pr, T_rWHILE, NULL)) {
-      /* repeat..while */
+    if (expect(pr, T_rWHILE)) {
+      /* repeat { ... } while */
       sbAst condition = parse_expr(pr, 0);
       return seq_node(pr, AST_NODE_REPEAT, body, condition);
-    } else if (expect(pr, T_rUNTIL, NULL)) {
-      /* repeat..until */
+    } else if (expect(pr, T_rUNTIL)) {
+      /* repeat { ... } until */
       sbAst condition = parse_expr(pr, 0);
       return seq_node(pr, AST_NODE_REPEAT, body, condition);
     } else {
-      fprintf(stderr, "syntax error! 'while' or 'until' required after 'repeat { ... }'\n");
-      return NO_NODE;
+      return syntax_error(pr);
     }
   } else if (t.type == T_rCASE) {
+    /* case x, y { ... } */
     next_token(pr);
     sbAst values = parse_comma_exprs(pr, NULL);
     sbAst body = parse_block(pr);
     return seq_node(pr, AST_NODE_CASE, values, body);
   } else if (t.type == T_rMATCH) {
+    /* match :x if j { ... } */
     next_token(pr);
     sbAst pattern = parse_comma_exprs(pr, NULL);
     sbAst guard_clause = NO_NODE;
-    if (expect(pr, T_rIF, NULL)) {
+    if (expect(pr, T_rIF)) {
       next_token(pr);
       guard_clause = parse_expr(pr, 0);
-    } else if (expect(pr, T_rUNLESS, NULL)) {
+    } else if (expect(pr, T_rUNLESS)) {
       next_token(pr);
       guard_clause = parse_expr(pr, 0);
       guard_clause = unop_node(pr, AST_OP_NOT, guard_clause);
@@ -530,111 +644,58 @@ static sbAst parse_stmt(hParser pr) {
     sbAst body = parse_block(pr);
     return tri_node(pr, AST_NODE_MATCH, pattern, guard_clause, body);
   } else if (t.type == T_rLET) {
+    /* let a, b = ... */
     next_token(pr);
     sbAst bindings = parse_comma_exprs(pr, NULL);
-    if (!expect(pr, T_EQUALS, NULL)) {
-      fprintf(stderr, "expected '=' after let ...\n");
-      return NO_NODE;
-    }
+    if (!expect(pr, T_EQUALS)) return syntax_error(pr);
     sbAst values = parse_comma_exprs(pr, NULL);
     return seq_node(pr, AST_NODE_LET, bindings, values);
   } else if (t.type == T_rDEF) {
+    /* def a b, c { ... } */
     next_token(pr);
     sbAst name = parse_name(pr);
-    if (!expect(pr, T_LPAREN, NULL)) {
-      fprintf(stderr, "expected '(' after 'def' and name\n");
-      return NO_NODE;
-    }
+    if (!expect(pr, T_LPAREN)) return syntax_error(pr);
     sbAst params = parse_comma_exprs(pr, NULL);
-    if (!expect(pr, T_RPAREN, NULL)) {
-      fprintf(stderr, "expected ')' after function parameters\n");
-      return NO_NODE;
-    }
+    if (!expect(pr, T_RPAREN)) return syntax_error(pr);
     sbAst body = parse_block(pr);
     sbAst func_node = seq_node(pr, AST_VAL_FUNC, params, body);
     return seq_node(pr, AST_NODE_DEF, name, func_node);
   } else if (t.type == T_rRETURN) {
+    /* return ... */
     next_token(pr);
     sbAst returned_val = parse_comma_exprs(pr, 0);
     sbAst return_node = wrap_node(pr, AST_NODE_RETURN, returned_val);
+    /* return ... unless condition */
     return_node = with_trailing_conditional(pr, return_node);
     return return_node;
   } else {
+    /* (statement that is just an expression) */
     sbAst expr = parse_expr(pr, 0);
+    if (expr == NO_NODE) return NO_NODE;
+
     if (peek_ahead(pr, 0).type == ',') {
+      /* (implicit return) a, b, c */
       next_token(pr);
       expr = parse_comma_exprs(pr, expr);
     }
+
     if (peek_ahead(pr, 0).type == '=') {
+      /* a, b, c = 1, 2, 3 */
       next_token(pr);
       sbAst assigned_values = parse_comma_exprs(pr, 0);
       expr = seq_node(pr, AST_NODE_ASSIGN, expr, assigned_values);
     }
+
     if (expr != NO_NODE) {
+      /* a, b, c = 1, 2, 3 (or other expr) if condition */
       expr = with_trailing_conditional(pr, expr);
-      return expr;
-    }
-  }
-
-  return NO_NODE;
-}
-
-static void print_ast_node(sbAst n, int indent) {
-  if (n == NULL) return;
-
-  printf("\n");
-  for (int i = 0; i < indent; i++) {
-    printf("  ");
-  }
-
-  if (n->type == AST_NODE_NAME) {
-    printf("%s", sbSymbol_name(n->symb));
-  } else if (n->type == AST_NODE_MULTIVAL) {
-    print_ast_node(n->seq.left, indent + 1);
-    printf(",");
-    print_ast_node(n->seq.right, indent + 1);
-  } else {
-    printf("(");
-    if (n->type == AST_NODE_OP) {
-      flag found = 0;
-      for (int i = 0; i < NUM_SPELLINGS; i++) {
-        if (op_spellings[i].ast_op == n->op.type) {
-          printf("%s", op_spellings[i].name);
-          found = 1;
-          break;
-        }
-      }
-      if (!found) printf("??");
-      print_ast_node(n->op.left, indent + 1);
-      if (n->op.right) {
-        print_ast_node(n->op.right, indent + 1);
-      }
-    } else if (n->type == AST_NODE_FUNCCALL) {
-      print_ast_node(n->seq.left, indent + 1);
-      printf("(");
-      print_ast_node(n->seq.right, indent + 1);
-      printf(")");
-    } else if (n->type == AST_NODE_METHODCALL) {
-      print_ast_node(n->seq.left, indent + 1);
-      printf(".");
-      print_ast_node(n->seq.right->seq.left, indent + 1);
-      printf("(");
-      print_ast_node(n->seq.right->seq.right, indent + 1);
-      printf(")");
-    } else {
-      printf("?<%d>", n->type);
     }
 
-    printf("\n");
-    for (int i = 0; i < indent; i++) {
-      printf("  ");
-    }
-    printf(")");
+    return expr;
   }
 }
 
 static sbAst do_parse(hParser pr) {
-  sbAst program = parse_expr(pr, 0);
-  print_ast_node(program, 0);
+  sbAst program = parse_stmtseq(pr);
   return program;
 }

diff --git a/src/parse/parser.h b/src/parse/parser.h
index 3847ef9..cbfe3d2 100644 (file)
--- a/src/parse/parser.h
+++ b/src/parse/parser.h
@@ -7,6 +7,7 @@ typedef struct sbParser {
   sbLexer lexer;
   sbTokenQueue input_queue;
   sbArena node_arena;
+  flag error_state;
 } sbParser;
 
 typedef sbParser *hParser;

diff --git a/src/parse/scanner.c b/src/parse/scanner.c
index f444e10..fa45aab 100644 (file)
--- a/src/parse/scanner.c
+++ b/src/parse/scanner.c
@@ -204,6 +204,8 @@ static usize read_symbol(hScanner sc) {
 static sbLexToken compute_next_token(hScanner sc) {
     int ch_int = PEEK;
     unsigned char ch = (unsigned char)ch_int;
+    u32 token_line = sbFileReader_get_line(sc->file_reader);
+    u32 token_start_col = sbFileReader_get_col(sc->file_reader);
 
     sbBuffer_reset(&sc->dynamic_buffer);
 
@@ -245,8 +247,7 @@ static sbLexToken compute_next_token(hScanner sc) {
             new_token.type = T_NOTEQUALS;
             NEXT;
         } else {
-            /* ! on its own is not an operator i guess. not sure how to handle this elegantly */
-            fprintf(stderr, "don't know how to handle character: '!'\n");
+            /* ! on its own is not an operator */
             new_token.type = T_ERROR;
         }
     } else if (ch == '~') {
@@ -256,7 +257,6 @@ static sbLexToken compute_next_token(hScanner sc) {
             NEXT;
         } else {
             /* ~ on its own is not an operator */
-            fprintf(stderr, "don't know how to handle character: '~'\n");
             new_token.type = T_ERROR;
         }
     } else if (ch == '>') {
@@ -379,7 +379,6 @@ static sbLexToken compute_next_token(hScanner sc) {
 
             token_size = read_symbol(sc);
             new_token.symb = sbSymbol_from_bytes(sc->dynamic_buffer.data, sc->dynamic_buffer.size - 1);
-            printf("%s -> %p\n", (char*)new_token.symb, (void*)new_token.symb);
             new_token.size = token_size;
         } else {
             /* : */
@@ -448,13 +447,13 @@ static sbLexToken compute_next_token(hScanner sc) {
 
         token_size = read_identifier(sc);
         new_token.symb = sbSymbol_from_bytes(sc->dynamic_buffer.data, sc->dynamic_buffer.size - 1);
-        printf("%s -> %p\n", (char*)new_token.symb, (void*)new_token.symb);
         new_token.size = token_size;
     } else if (is_digit(ch)) {
         new_token.type = T_INTEGER;
 
         i64 intval = 0;
         int base = 10;
+        int num_done = FALSE;
 
         if (ch == '0') {
             /* skip a leading zero, and see if it has some base indicator */
@@ -468,37 +467,36 @@ static sbLexToken compute_next_token(hScanner sc) {
             } else if (ch == 'x') {
                 ch = NEXT;
                 base = 16;
+            } else if (is_space(ch) || ch == '\n') {
+                /* it was literally just the number 0 (probably) */
+                num_done = TRUE;
+            } else {
+                new_token.type = T_BADNUMBER;
+                num_done = TRUE;
             }
         }
 
-        do {
-            /* TODO: promote to bigint, don't allow overflow */
-            intval *= base;
-            intval += base_digit_value(ch);
+        if (!num_done) {
             do {
-                /* skip over underscores in numeric literals */
-                ch = NEXT;
-            } while (ch == '_');
-        } while (is_base_digit(ch, base));
-
-        /* if we are now looking at still a character that's a letter or digit, throw error */
-        if (is_digit(ch) || is_alpha(ch)) {
-            if (base == 10) {
-                fprintf(stderr, "unexpected character in numeric literal: '%c'\n", ch);
+                /* TODO: promote to bigint, don't allow overflow */
+                intval *= base;
+                intval += base_digit_value(ch);
+                do {
+                    /* skip over underscores in numeric literals */
+                    ch = NEXT;
+                } while (ch == '_');
+            } while (is_base_digit(ch, base));
+
+            /* if we are now looking at still a character that's a letter or digit, throw error */
+            if (is_digit(ch) || is_alpha(ch)) {
+                new_token.type = T_BADNUMBER;
+                NEXT;
             } else {
-                fprintf(stderr, "unexpected character in base %d numeric literal: '%c'\n", base, ch);
+                new_token.i = intval;
             }
-
-            new_token.type = T_ERROR;
-            NEXT;
-        } else {
-            new_token.i = intval;
         }
     } else {
         new_token.type = T_ERROR;
-
-        fprintf(stderr, "don't know how to process character: '%c'\n", ch);
-
         NEXT;
     }
 
@@ -512,6 +510,11 @@ static sbLexToken compute_next_token(hScanner sc) {
         }
     }
 
+    u32 token_end_col = sbFileReader_get_col(sc->file_reader) - 1;
+
+    new_token.line = token_line;
+    new_token.start_col = token_start_col;
+    new_token.end_col = token_end_col;
 
     return new_token;
 }

diff --git a/src/parse/token.h b/src/parse/token.h
index 7fa61ed..e001be4 100644 (file)
--- a/src/parse/token.h
+++ b/src/parse/token.h
@@ -8,6 +8,8 @@
 typedef enum sbTokenType {
     T_NULL                  = 0,
     T_ERROR                 = 1,
+    T_WRONGBRACKET          = 2,
+    T_BADNUMBER             = 3,
     T_LPAREN                = '(', // op::call
     T_RPAREN                = ')',
     T_LBRACKET              = '[', // op::index
@@ -90,6 +92,9 @@ typedef struct sbLexToken {
     sbTokenType type;
     usize size;
     flag invisible;
+    i32 line;
+    i32 start_col;
+    i32 end_col;
     union {
         char *cstr;
         hString hstr;