From: cassowarii <2374677+cassowarii@users.noreply.github.com>
Date: Fri, 19 Jun 2026 18:27:12 +0000 (-0700)
Subject: way more lexing capability
X-Git-Url: https://git.cassowary.me/gitweb.cgi?a=commitdiff_plain;h=e1fc8890011234e8d55abcc27045e11b9a9552ff;p=sarabande.git

way more lexing capability
---

diff --git a/src/main.c b/src/main.c
index b91319e..d85ddc4 100644
--- a/src/main.c
+++ b/src/main.c
@@ -30,7 +30,12 @@ int main(int argc, char **argv) {
             } else if (t.type == T_INTEGER) {
                 printf("(INT %d)", t.i);
             } else {
-                printf("  %c  ", t.type);
+                switch (t.type) {
+                    case T_DOUBLEEQUALS:
+                        printf("  ==  "); break;
+                    default:
+                        printf("  %c  ", t.type);
+                }
             }
         } while (t.type != T_EOF);
 
diff --git a/src/mem/arena.c b/src/mem/arena.c
index 176fc12..00f61d2 100644
--- a/src/mem/arena.c
+++ b/src/mem/arena.c
@@ -37,28 +37,31 @@ hArena sbArena_create(usize initial_size) {
 void *sbArena_alloc(hArena arena, usize size) {
     while (size % ALIGN != 0) size++;
 
-    if (size > arena->current->capacity - arena->current->used) {
-        /* not enough space in current block. need to move to next block or allocate a new one */
-        if (arena->current != arena->last) {
-            /* move to next block */
-            if (arena->current->next) {
-                arena->current = arena->current->next;
-            } else {
-                PANIC("lost track of memory block in arena somehow; should not happen");
-            }
+    if (arena->current->used > arena->current->capacity) {
+        PANIC("somehow arena values are out of sync");
+    }
+
+    while (arena->current != arena->last && size > arena->current->capacity - arena->current->used) {
+        /* move to next block until we find one that fits */
+        if (arena->current->next) {
+            arena->current = arena->current->next;
         } else {
-            /* need to allocate a new block */
-            usize new_capacity = arena->current->capacity;
-            if (size > new_capacity) new_capacity = size;
-            struct block *block = calloc(sizeof(struct block) + new_capacity, 1);
-            block->capacity = new_capacity;
-            block->next = NULL;
-
-            arena->current->next = block;
-            arena->current = arena->last = block;
+            PANIC("lost track of memory block in arena somehow; should not happen");
         }
     }
 
+    if (size > arena->current->capacity - arena->current->used) {
+        /* if still doesn't fit, we must be at the end, need to allocate a new block */
+        usize new_capacity = arena->current->capacity;
+        if (size > new_capacity) new_capacity = size;
+        struct block *block = calloc(sizeof(struct block) + new_capacity, 1);
+        block->capacity = new_capacity;
+        block->next = NULL;
+
+        arena->current->next = block;
+        arena->current = arena->last = block;
+    }
+
     void *allocated_ptr = &arena->current->data[arena->current->used];
     arena->current->used += size;
     memset(allocated_ptr, 0, size);
diff --git a/src/mem/string.c b/src/mem/string.c
index b9f01fe..5eec54a 100644
--- a/src/mem/string.c
+++ b/src/mem/string.c
@@ -1,6 +1,6 @@
 #include "string.h"
 
-usize mystrncpy(char *dst, char *src, usize limit) {
+usize sbstrncpy(char *dst, char *src, usize limit) {
     usize count = 0;
     char ch = 0;
 
diff --git a/src/mem/string.h b/src/mem/string.h
index c73b83a..f98b0ee 100644
--- a/src/mem/string.h
+++ b/src/mem/string.h
@@ -1,3 +1,3 @@
 #include "common.h"
 
-usize mystrncpy(char *dst, char *src, usize limit);
+usize sbstrncpy(char *dst, char *src, usize limit);
diff --git a/src/parse/scanner.c b/src/parse/scanner.c
index 5220444..6713e71 100644
--- a/src/parse/scanner.c
+++ b/src/parse/scanner.c
@@ -13,6 +13,7 @@
  * which have different semantics. */
 
 #define MEM_BLOCK_SIZE 65536
+#define TOKEN_BUFFER_SIZE 256
 
 typedef struct sbScanner {
     hFileReader file_reader;
@@ -52,114 +53,283 @@ flag is_space(char c) {
     return c == ' ' || c == '\t' || c == '\r';
 }
 
-flag accept_char(char expected, hFileReader fr) {
-    char actual = sbFileReader_peek(fr);
-    if (expected == actual) {
-        sbFileReader_next(fr);
-        return 1;
+flag is_start_identifier(char c) {
+    return is_alpha(c);
+}
+
+flag is_mid_identifier(char c) {
+    return is_start_identifier(c) || is_digit(c);
+}
+
+flag is_end_identifier(char c) {
+    return is_mid_identifier(c) || c == '?';
+}
+
+flag is_base_digit(char c, int base) {
+    if (base == 10) {
+        return is_digit(c);
+    } else if (base == 16) {
+        return is_digit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F');
+    } else if (base == 8) {
+        return '0' <= c && c <= '7';
+    } else if (base == 2) {
+        return '0' <= c && c <= '1';
     } else {
         return 0;
     }
 }
 
-char accept_digit(hFileReader fr) {
-    char actual_char = sbFileReader_peek(fr);
-    if (is_digit(actual_char)) {
-        sbFileReader_next(fr);
-        return actual_char;
+int base_digit_value(char c) {
+    if (is_digit(c)) {
+        return c - '0';
+    } else if ('a' <= c && c <= 'z') {
+        return c - 'a' + 10;
+    } else if ('A' <= c && c <= 'Z') {
+        return c - 'A' + 10;
     } else {
         return 0;
     }
 }
 
-char accept_alpha(hFileReader fr) {
-    char actual_char = sbFileReader_peek(fr);
-    if (is_alpha(actual_char)) {
-        sbFileReader_next(fr);
-        return actual_char;
-    } else {
-        return 0;
+#define NEXT sbFileReader_next(sc->file_reader)
+#define PEEK sbFileReader_peek(sc->file_reader)
+
+usize read_identifier(hScanner sc, char *token_buffer, usize max_size) {
+    usize token_size = 0;
+    char ch = 0;
+
+    do {
+        token_buffer[token_size] = NEXT;
+        token_size ++;
+        ch = PEEK;
+    } while (is_mid_identifier(ch) && token_size < max_size - 1);
+
+    if (is_end_identifier(ch) && token_size < max_size - 1) {
+        token_buffer[token_size] = NEXT;
+        token_size ++;
     }
+
+    token_buffer[token_size] = '\0';
+
+    return token_size;
 }
 
 sbLexToken compute_next_token(hScanner sc) {
-    char ch = sbFileReader_peek(sc->file_reader);
+    int next = PEEK;
+    char ch = (char)next;
 
-    char token_buffer[256];
-    int token_size = 0;
+    char token_buffer[TOKEN_BUFFER_SIZE];
+    usize token_size;
 
     sbLexToken new_token = {0};
 
-    if (ch == EOF) {
+    if (next == EOF) {
         new_token.type = T_EOF;
     } else if (ch == '\n') {
         new_token.type = T_NEWLINE;
 
         /* skip all spaces after the newline */
         do {
-            sbFileReader_next(sc->file_reader);
-        } while (is_space(sbFileReader_peek(sc->file_reader)));
+            NEXT;
+        } while (is_space(PEEK));
+    } else if (ch == '#') {
+        /* Comment. Ignore everything until the end of the line. */
+        do {
+            NEXT;
+            ch = PEEK;
+        } while (ch != '\n');
+        NEXT; /* eat the newline */
+        new_token.type = T_NEWLINE;
+    } else if (ch == '(' || ch == ')'
+            || ch == '[' || ch == ']'
+            || ch == '{' || ch == '}'
+            || ch == ';' || ch == '|'
+            || ch == '.' || ch == ','
+            || ch == '\\') {
+        /* unambiguously single-character tokens */
+        new_token.type = ch;
+        NEXT;
+    } else if (ch == '%') {
+        NEXT;
+        ch = PEEK;
+        if (ch == '=') {
+            new_token.type = T_PERCENTEQUALS;
+        } else {
+            new_token.type = T_PERCENT;
+        }
+    } else if (ch == '/') {
+        NEXT;
+        ch = PEEK;
+        if (ch == '/') {
+            new_token.type = T_DOUBLESLASH;
+            NEXT;
+        } else if (ch == '=') {
+            new_token.type = T_SLASHEQUALS;
+            NEXT;
+        } else {
+            new_token.type = T_SLASH;
+        }
+    } else if (ch == '*') {
+        NEXT;
+        ch = PEEK;
+        if (ch == '*') {
+            new_token.type = T_DOUBLEASTERISK;
+            NEXT;
+        } else if (ch == '=') {
+            new_token.type = T_ASTERISKEQUALS;
+            NEXT;
+        } else {
+            new_token.type = T_ASTERISK;
+        }
+    } else if (ch == '=') {
+        NEXT;
+        ch = PEEK;
+        if (ch == '=') {
+            new_token.type = T_DOUBLEEQUALS;
+            NEXT;
+        } else if (ch == '>') {
+            new_token.type = T_FATARROW;
+            NEXT;
+        } else {
+            new_token.type = T_EQUALS;
+        }
+    } else if (ch == '+') {
+        NEXT;
+        ch = PEEK;
+        if (ch == '+') {
+            new_token.type = T_DOUBLEPLUS;
+            NEXT;
+        } else if (ch == '=') {
+            new_token.type = T_PLUSEQUALS;
+            NEXT;
+        } else {
+            new_token.type = T_PLUS;
+        }
+    } else if (ch == '-') {
+        NEXT;
+        ch = PEEK;
+        if (ch == '-') {
+            new_token.type = T_DOUBLEMINUS;
+            NEXT;
+        } else if (ch == '>') {
+            new_token.type = T_ARROW;
+            NEXT;
+        } else if (ch == '=') {
+            new_token.type = T_MINUSEQUALS;
+            NEXT;
+        } else {
+            new_token.type = T_MINUS;
+        }
+    } else if (ch == ':') {
+        NEXT;
+        ch = PEEK;
+        if (ch == ':') {
+            /* :: */
+            new_token.type = T_PAAMAYIM_NEKUDOTAYIM;
+        } else if (ch == '{') {
+            /* :{ */
+            new_token.type = T_COLONBRACE;
+        } else if (is_start_identifier(ch)) {
+            /* :abc... */
+            new_token.type = T_SYMBOL;
+
+            token_size = read_identifier(sc, token_buffer, TOKEN_BUFFER_SIZE);
+
+            char *storage = sbArena_alloc(sc->arena, token_size + 1);
+
+            sbstrncpy(storage, token_buffer, token_size + 1);
+
+            new_token.str = storage;
+            new_token.size = token_size;
+        } else {
+            /* : */
+            new_token.type = T_COLON;
+        }
     } else if (is_space(ch)) {
         new_token.type = T_SPACE;
 
         /* skip all subsequent spaces */
         do {
-            sbFileReader_next(sc->file_reader);
-        } while (is_space(sbFileReader_peek(sc->file_reader)));
+            NEXT;
+        } while (is_space(PEEK));
+
+        /* if we run into a comment, skip to the end of the line */
+        if (ch == '#') {
+            do {
+                NEXT;
+            } while (PEEK != '\n');
+        }
 
         /* if we run into a newline, forget about the spaces */
-        if (sbFileReader_peek(sc->file_reader) == '\n') {
+        if (PEEK == '\n') {
             new_token.type = T_NEWLINE;
 
             /* and skip all spaces after the newline */
             do {
-                sbFileReader_next(sc->file_reader);
-            } while (is_space(sbFileReader_peek(sc->file_reader)));
+                NEXT;
+            } while (is_space(PEEK));
         }
-    } else if (is_alpha(ch)) {
+    } else if (is_start_identifier(ch)) {
         new_token.type = T_IDENTIFIER;
 
-        do {
-            token_buffer[token_size] = sbFileReader_next(sc->file_reader);
-            token_size ++;
-            ch = sbFileReader_peek(sc->file_reader);
-        } while (is_alpha(ch) || is_digit(ch));
-
-        /* TODO: I am sleepy. Please check that this has not overflowed the buffer. */
-
-        token_buffer[token_size] = '\0';
+        token_size = read_identifier(sc, token_buffer, TOKEN_BUFFER_SIZE);
 
         char *storage = sbArena_alloc(sc->arena, token_size + 1);
 
-        mystrncpy(storage, token_buffer, token_size + 1);
+        sbstrncpy(storage, token_buffer, token_size + 1);
 
         new_token.str = storage;
+        new_token.size = token_size;
     } else if (is_digit(ch)) {
         new_token.type = T_INTEGER;
-        /* TODO: yes, yes. this isn't correct. listen. i am tired. */
 
-        i64 intval = ch - '0';
+        i64 intval = 0;
+        int base = 10;
+
+        if (ch == '0') {
+            NEXT;
+            ch = PEEK;
+            if (ch == 'b') {
+                NEXT;
+                ch = PEEK;
+                base = 2;
+            } else if (ch == 'o') {
+                NEXT;
+                ch = PEEK;
+                base = 8;
+            } else if (ch == 'x') {
+                NEXT;
+                ch = PEEK;
+                base = 16;
+            }
+        }
 
-        while (is_digit(sbFileReader_peek(sc->file_reader))) {
-            ch = sbFileReader_next(sc->file_reader);
-            intval *= 10;
-            intval += ch - '0';
+        do {
+            /* TODO: promote to bigint, don't allow overflow */
+            NEXT;
+            intval *= base;
+            intval += base_digit_value(ch);
+            ch = PEEK;
+        } while (is_base_digit(ch, base));
+
+        /* if we are now looking at still a character that's a letter or digit, throw error */
+        if (is_digit(ch) || is_alpha(ch)) {
+            if (base == 10) {
+                fprintf(stderr, "unexpected character in numeric literal: '%c'\n", ch);
+            } else {
+                fprintf(stderr, "unexpected character in base %d numeric literal: '%c'\n", base, ch);
+            }
+            new_token.type = T_ERROR;
+            NEXT;
+        } else {
+            new_token.i = intval;
         }
     } else {
         new_token.type = T_ERROR;
 
-        switch (ch) {
-            case '(':
-            case ')':
-            case '.':
-                /* TODO ok blah blah this isn't right */
-                new_token.type = ch;
-                break;
-            default:
-                fprintf(stderr, "don't know how to process character: '%c'\n", ch);
-        }
+        fprintf(stderr, "don't know how to process character: '%c'\n", ch);
 
-        sbFileReader_next(sc->file_reader);
+        NEXT;
     }
 
     return new_token;
diff --git a/src/parse/scanner.h b/src/parse/scanner.h
index 608f5f9..7af527c 100644
--- a/src/parse/scanner.h
+++ b/src/parse/scanner.h
@@ -8,37 +8,51 @@ typedef enum sbTokenType {
     T_LPAREN = '(',
     T_RPAREN = ')',
     T_LBRACKET = '[',
-    T_RBRACKET = '[',
+    T_RBRACKET = ']',
     T_LBRACE = '{',
     T_RBRACE = '}',
     T_ASTERISK = '*',
-    T_SLASH = '*',
+    T_SLASH = '/',
     T_PLUS = '+',
     T_MINUS = '-',
     T_PERCENT = '%',
     T_PIPE = '|',
     T_DOT = '.',
     T_COMMA = ',',
+    T_EQUALS = '=',
     T_COLON = ':',
     T_SEMICOLON = ';',
+    T_BACKSLASH = '\\',
     T_NEWLINE = '\n',
     T_SPACE = ' ',
     T_NULL = 0,
     T_ERROR = 1,
+    T_KEYWORD,
+    T_IDENTIFIER,
+    T_SYMBOL,
     T_INTEGER,
     T_FLOAT,
     T_STRING,
-    T_KEYWORD,
-    T_IDENTIFIER,
     T_ARROW,
-    T_FUNCARROW,
+    T_FATARROW,
     T_COLONBRACE,
+    T_DOUBLEEQUALS,
+    T_DOUBLEMINUS,
+    T_DOUBLEPLUS,
+    T_DOUBLEASTERISK,
+    T_DOUBLESLASH,
+    T_MINUSEQUALS,
+    T_PLUSEQUALS,
+    T_ASTERISKEQUALS,
+    T_SLASHEQUALS,
+    T_PERCENTEQUALS,
     T_PAAMAYIM_NEKUDOTAYIM,
     T_EOF = 255,
 } sbTokenType;
 
 typedef struct sbLexToken {
     sbTokenType type;
+    usize size;
     union {
         char *str;
         float fl;