From: cassowarii <2374677+cassowarii@users.noreply.github.com>
Date: Fri, 19 Jun 2026 06:03:19 +0000 (-0700)
Subject: world's worst lexer
X-Git-Url: https://git.cassowary.me/gitweb.cgi?a=commitdiff_plain;h=f5c96a17eac78f0eaff004dfcaf6e63c71dfc546;p=sarabande.git

world's worst lexer
---

f5c96a17eac78f0eaff004dfcaf6e63c71dfc546
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..579f8a0
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,22 @@
+CC=gcc
+CFLAGS=-Wall -g -DDEBUG -fsanitize=undefined
+#CFLAGS=-Wall -O3 -flto -DGLEW_STATIC -DDEBUG -fsanitize=undefined
+LIBS=
+
+SRC := src
+OBJ := obj
+BUILDDIR := build
+
+SOURCES := $(shell find $(SRC) -type f -name '*.c' -not -path '$(SRC)/$(RES)/*')
+OBJECTS := $(patsubst $(SRC)/%.c, $(OBJ)/%.o, $(SOURCES))
+
+build/a.out: $(OBJECTS)
+	@mkdir -p $(dir $@)
+	$(CC) $(CFLAGS) -I$(SRC) $(OBJECTS) $(LIBS) -o $@
+
+$(OBJ)/%.o: $(SRC)/%.c
+	@mkdir -p $(dir $@)
+	$(CC) $(CFLAGS) $(LIBS) -I$(SRC) -c $< -o $@
+
+clean:
+	rm -rf $(OBJ) $(BUILDDIR)
diff --git a/src/.main.c.swp b/src/.main.c.swp
new file mode 100644
index 0000000..db6743f
Binary files /dev/null and b/src/.main.c.swp differ
diff --git a/src/common.h b/src/common.h
new file mode 100644
index 0000000..498ac27
--- /dev/null
+++ b/src/common.h
@@ -0,0 +1,23 @@
+#ifndef __SARABANDE_COMMON_H__
+#define __SARABANDE_COMMON_H__
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <stdint.h>
+
+typedef uint64_t u64;
+typedef int64_t i64;
+typedef uint32_t u32;
+typedef int32_t i32;
+typedef uint16_t u16;
+typedef int16_t i16;
+typedef uint8_t u8;
+typedef int8_t i8;
+typedef uint8_t flag;
+typedef size_t usize;
+typedef ptrdiff_t isize;
+
+#include "global.h"
+
+#endif
diff --git a/src/global.h b/src/global.h
new file mode 100644
index 0000000..4abd545
--- /dev/null
+++ b/src/global.h
@@ -0,0 +1,5 @@
+#ifdef DEBUG
+#define PANIC(msg) do { fprintf(stderr, "panic: " __FILE__ ":%d: " msg "\n", __LINE__); abort(); } while (0)
+#else
+#define PANIC(x) 0
+#endif
diff --git a/src/main.c b/src/main.c
new file mode 100644
index 0000000..edcc69f
--- /dev/null
+++ b/src/main.c
@@ -0,0 +1,38 @@
+#include "common.h"
+
+#include "parse/filereader.h"
+#include "parse/scanner.h"
+
+int main(int argc, char **argv) {
+    if (argc == 2) {
+        hFileReader fr = sbFileReader_open(argv[1]);
+
+        hScanner sc = sbScanner_create(fr);
+
+        sbLexToken t;
+
+        do {
+            t = sbScanner_next(sc);
+
+            if (t.type == T_EOF) {
+                printf("(EOF)");
+            } else if (t.type == T_SPACE) {
+                printf("( )");
+            } else if (t.type == T_NEWLINE) {
+                printf("(\\n)");
+            } else if (t.type == T_IDENTIFIER) {
+                printf("(ID %s)", t.str);
+            } else if (t.type == T_INTEGER) {
+                printf("(INT %d)", t.i);
+            } else {
+                printf("  %c  ", t.type);
+            }
+        } while (t.type != T_EOF);
+
+        sbScanner_destroy(sc);
+
+        sbFileReader_close(fr);
+    } else {
+        fprintf(stderr, "please provide a file as input\n");
+    }
+}
diff --git a/src/mem/.arena.c.swp b/src/mem/.arena.c.swp
new file mode 100644
index 0000000..dda4630
Binary files /dev/null and b/src/mem/.arena.c.swp differ
diff --git a/src/mem/.arena.h.swp b/src/mem/.arena.h.swp
new file mode 100644
index 0000000..3996008
Binary files /dev/null and b/src/mem/.arena.h.swp differ
diff --git a/src/mem/.string.c.swp b/src/mem/.string.c.swp
new file mode 100644
index 0000000..4fea13e
Binary files /dev/null and b/src/mem/.string.c.swp differ
diff --git a/src/mem/arena.c b/src/mem/arena.c
new file mode 100644
index 0000000..0d56062
--- /dev/null
+++ b/src/mem/arena.c
@@ -0,0 +1,88 @@
+#include "arena.h"
+
+#include <string.h>
+
+#define ALIGN 8
+
+struct block {
+    struct block *next;
+    usize used;
+    usize capacity;
+    char data[];
+};
+
+struct sbArena {
+    struct block *first;
+    struct block *current;
+    struct block *last;
+};
+
+hArena sbArena_create(usize initial_size) {
+    hArena arena = malloc(sizeof(struct sbArena));
+
+    while (initial_size % ALIGN != 0) initial_size++;
+
+    struct block *block = malloc(initial_size);
+    block->used = 0;
+    block->next = NULL;
+    block->capacity = initial_size;
+
+    arena->first = block;
+    arena->current = block;
+    arena->last = block;
+
+    return arena;
+}
+
+void *sbArena_alloc(hArena arena, usize size) {
+    while (size % ALIGN != 0) size++;
+
+    if (size > arena->current->capacity - arena->current->used) {
+        /* not enough space in current block. need to move to next block or allocate a new one */
+        if (arena->current != arena->last) {
+            /* move to next block */
+            if (arena->current->next) {
+                arena->current = arena->current->next;
+            } else {
+                PANIC("lost track of memory block in arena somehow; should not happen");
+            }
+        } else {
+            /* need to allocate a new block */
+            usize new_capacity = arena->current->capacity;
+            if (size > new_capacity) new_capacity = size;
+            struct block *block = malloc(sizeof(struct block) + new_capacity);
+            block->capacity = new_capacity;
+            block->next = NULL;
+
+            arena->current->next = block;
+            arena->current = arena->last = block;
+        }
+    }
+
+    void *allocated_ptr = &arena->current->data[arena->current->used];
+    arena->current->used += size;
+    memset(allocated_ptr, 0, size);
+
+    return allocated_ptr;
+}
+
+void sbArena_reset(hArena arena) {
+    struct block *blk = arena->first;
+    do {
+        blk->used = 0;
+    } while (blk != arena->current);
+
+    arena->current = arena->first;
+}
+
+void sbArena_destroy(hArena arena) {
+    struct block *blk = arena->first;
+    struct block *blk_next = arena->first->next;
+    do {
+        blk_next = blk->next;
+        free(blk);
+        blk = blk_next;
+    } while (blk);
+
+    free(arena);
+}
diff --git a/src/mem/arena.h b/src/mem/arena.h
new file mode 100644
index 0000000..6eb5836
--- /dev/null
+++ b/src/mem/arena.h
@@ -0,0 +1,11 @@
+#include "common.h"
+
+typedef struct sbArena *hArena;
+
+hArena sbArena_create(usize initial_size);
+
+void *sbArena_alloc(hArena arena, usize size);
+
+void sbArena_reset(hArena arena);
+
+void sbArena_destroy(hArena arena);
diff --git a/src/mem/mem.h b/src/mem/mem.h
new file mode 100644
index 0000000..fdd935a
--- /dev/null
+++ b/src/mem/mem.h
@@ -0,0 +1,4 @@
+#include "common.h"
+
+#include "arena.h"
+#include "string.h"
diff --git a/src/mem/string.c b/src/mem/string.c
new file mode 100644
index 0000000..b9f01fe
--- /dev/null
+++ b/src/mem/string.c
@@ -0,0 +1,14 @@
+#include "string.h"
+
+usize mystrncpy(char *dst, char *src, usize limit) {
+    usize count = 0;
+    char ch = 0;
+
+    do {
+        ch = dst[count] = src[count];
+        count ++;
+    } while (ch && count < limit);
+    dst[--count] = 0;
+
+    return count;
+}
diff --git a/src/mem/string.h b/src/mem/string.h
new file mode 100644
index 0000000..c73b83a
--- /dev/null
+++ b/src/mem/string.h
@@ -0,0 +1,3 @@
+#include "common.h"
+
+usize mystrncpy(char *dst, char *src, usize limit);
diff --git a/src/parse/.filereader.h.swp b/src/parse/.filereader.h.swp
new file mode 100644
index 0000000..fc6b1d5
Binary files /dev/null and b/src/parse/.filereader.h.swp differ
diff --git a/src/parse/.scanner.c.swp b/src/parse/.scanner.c.swp
new file mode 100644
index 0000000..5487a19
Binary files /dev/null and b/src/parse/.scanner.c.swp differ
diff --git a/src/parse/.scanner.h.swp b/src/parse/.scanner.h.swp
new file mode 100644
index 0000000..dab6bd2
Binary files /dev/null and b/src/parse/.scanner.h.swp differ
diff --git a/src/parse/filereader.c b/src/parse/filereader.c
new file mode 100644
index 0000000..4f18799
--- /dev/null
+++ b/src/parse/filereader.c
@@ -0,0 +1,38 @@
+#include "filereader.h"
+
+/* this seems pointless right now as a wrapper for FILE* but later, we can make
+ * this work to read from stdin in interactive mode as well, which will be easier
+ * if we just have these few functions separate from the actual lexer code.
+ * we may also want to have multiple files open for imports and stuff... */
+
+struct sbFileReader {
+    FILE *file;
+};
+
+hFileReader sbFileReader_open(const char *filename) {
+    hFileReader fr = malloc(sizeof(struct sbFileReader));
+    fr->file = fopen(filename, "r");
+    if (!fr->file) {
+        perror("Error opening file for reading:");
+    } else {
+        return fr;
+    }
+
+    return fr;
+}
+
+char sbFileReader_next(hFileReader r) {
+    char c = fgetc(r->file);
+    return c;
+}
+
+char sbFileReader_peek(hFileReader r) {
+    char c = fgetc(r->file);
+    ungetc(c, r->file);
+    return c;
+}
+
+void sbFileReader_close(hFileReader r) {
+    fclose(r->file);
+    free(r);
+}
diff --git a/src/parse/filereader.h b/src/parse/filereader.h
new file mode 100644
index 0000000..145ac6d
--- /dev/null
+++ b/src/parse/filereader.h
@@ -0,0 +1,11 @@
+#include "common.h"
+
+typedef struct sbFileReader *hFileReader;
+
+hFileReader sbFileReader_open(const char *filename);
+
+char sbFileReader_next(hFileReader r);
+
+char sbFileReader_peek(hFileReader r);
+
+void sbFileReader_close(hFileReader r);
diff --git a/src/parse/scanner.c b/src/parse/scanner.c
new file mode 100644
index 0000000..a84651a
--- /dev/null
+++ b/src/parse/scanner.c
@@ -0,0 +1,175 @@
+#include "scanner.h"
+
+#include "filereader.h"
+#include "mem/mem.h"
+
+/* This is like, the pre lexing stage. I mean, this does a lot of the tokenizing,
+ * but it includes stuff like spaces and newlines, too. Think of this as 'normalizing'
+ * the input. The lexer in the second stage will use some additional context and state
+ * on top of the stream of tokens output from this module in order to insert some
+ * additional "ghost" tokens into the stream (invisible parentheses, semicolon insertion),
+ * and will delete stuff like spaces and newlines. But we need to preserve spaces and such
+ * after this stage so we can differentiate between things like 'a.b (1)' and 'a.b(1)',
+ * which have different semantics. */
+
+typedef struct sbScanner {
+    hFileReader file_reader;
+    hArena arena;
+    sbLexToken next_token;
+} sbScanner;
+
+hScanner sbScanner_create(hFileReader fr) {
+    hScanner sc = malloc(sizeof(sbScanner));
+    sc->arena = sbArena_create(65536);
+    sc->file_reader = fr;
+    sc->next_token = (sbLexToken) {0};
+
+    return sc;
+}
+
+sbLexToken sbScanner_peek(hScanner sc) {
+    return sc->next_token;
+}
+
+void sbScanner_destroy(hScanner sc) {
+    sbArena_destroy(sc->arena);
+    free(sc);
+}
+
+/* yeah, yeah, unicode! get off my achin' back! i'll do it later! (maybe) */
+
+flag is_digit(char c) {
+    return ('0' <= c && c <= '9');
+}
+
+flag is_alpha(char c) {
+    return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '_';
+}
+
+flag is_space(char c) {
+    return c == ' ' || c == '\t' || c == '\r';
+}
+
+flag accept_char(char expected, hFileReader fr) {
+    char actual = sbFileReader_peek(fr);
+    if (expected == actual) {
+        sbFileReader_next(fr);
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+char accept_digit(hFileReader fr) {
+    char actual_char = sbFileReader_peek(fr);
+    if (is_digit(actual_char)) {
+        sbFileReader_next(fr);
+        return actual_char;
+    } else {
+        return 0;
+    }
+}
+
+char accept_alpha(hFileReader fr) {
+    char actual_char = sbFileReader_peek(fr);
+    if (is_alpha(actual_char)) {
+        sbFileReader_next(fr);
+        return actual_char;
+    } else {
+        return 0;
+    }
+}
+
+sbLexToken compute_next_token(hScanner sc) {
+    char ch = sbFileReader_peek(sc->file_reader);
+
+    char token_buffer[256];
+    int token_size = 0;
+
+    sbLexToken new_token = {0};
+
+    if (ch == EOF) {
+        new_token.type = T_EOF;
+    } else if (ch == '\n') {
+        new_token.type = T_NEWLINE;
+
+        /* skip all spaces after the newline */
+        do {
+            sbFileReader_next(sc->file_reader);
+        } while (is_space(sbFileReader_peek(sc->file_reader)));
+    } else if (is_space(ch)) {
+        new_token.type = T_SPACE;
+
+        /* skip all subsequent spaces */
+        do {
+            sbFileReader_next(sc->file_reader);
+        } while (is_space(sbFileReader_peek(sc->file_reader)));
+
+        /* if we run into a newline, forget about the spaces */
+        if (sbFileReader_peek(sc->file_reader) == '\n') {
+            new_token.type = T_NEWLINE;
+
+            /* and skip all spaces after the newline */
+            do {
+                sbFileReader_next(sc->file_reader);
+            } while (is_space(sbFileReader_peek(sc->file_reader)));
+        }
+    } else if (is_alpha(ch)) {
+        new_token.type = T_IDENTIFIER;
+
+        do {
+            token_buffer[token_size] = sbFileReader_next(sc->file_reader);
+            token_size ++;
+            ch = sbFileReader_peek(sc->file_reader);
+        } while (is_alpha(ch) || is_digit(ch));
+
+        /* TODO: I am sleepy. Please check that this has not overflowed the buffer. */
+
+        token_buffer[token_size] = '\0';
+
+        char *storage = sbArena_alloc(sc->arena, token_size + 1);
+
+        mystrncpy(storage, token_buffer, token_size + 1);
+
+        new_token.str = storage;
+    } else if (is_digit(ch)) {
+        new_token.type = T_INTEGER;
+        /* TODO: yes, yes. this isn't correct. listen. i am tired. */
+
+        i64 intval = ch - '0';
+
+        while (is_digit(sbFileReader_peek(sc->file_reader))) {
+            ch = sbFileReader_next(sc->file_reader);
+            intval *= 10;
+            intval += ch - '0';
+        }
+    } else {
+        new_token.type = T_ERROR;
+
+        switch (ch) {
+            case '(':
+            case ')':
+            case '.':
+                /* TODO ok blah blah this isn't right */
+                new_token.type = ch;
+                break;
+            default:
+                fprintf(stderr, "don't know how to process character: '%c'\n", ch);
+        }
+
+        sbFileReader_next(sc->file_reader);
+    }
+
+    return new_token;
+}
+
+sbLexToken sbScanner_next(hScanner sc) {
+    if (sc->next_token.type == T_NULL) {
+        /* if we just started, queue up the first token */
+        sc->next_token = compute_next_token(sc);
+    }
+
+    sbLexToken to_return = sc->next_token;
+    sc->next_token = compute_next_token(sc);
+    return to_return;
+}
diff --git a/src/parse/scanner.h b/src/parse/scanner.h
new file mode 100644
index 0000000..608f5f9
--- /dev/null
+++ b/src/parse/scanner.h
@@ -0,0 +1,55 @@
+#include "common.h"
+
+#include "filereader.h"
+
+typedef struct sbScanner *hScanner;
+
+typedef enum sbTokenType {
+    T_LPAREN = '(',
+    T_RPAREN = ')',
+    T_LBRACKET = '[',
+    T_RBRACKET = '[',
+    T_LBRACE = '{',
+    T_RBRACE = '}',
+    T_ASTERISK = '*',
+    T_SLASH = '*',
+    T_PLUS = '+',
+    T_MINUS = '-',
+    T_PERCENT = '%',
+    T_PIPE = '|',
+    T_DOT = '.',
+    T_COMMA = ',',
+    T_COLON = ':',
+    T_SEMICOLON = ';',
+    T_NEWLINE = '\n',
+    T_SPACE = ' ',
+    T_NULL = 0,
+    T_ERROR = 1,
+    T_INTEGER,
+    T_FLOAT,
+    T_STRING,
+    T_KEYWORD,
+    T_IDENTIFIER,
+    T_ARROW,
+    T_FUNCARROW,
+    T_COLONBRACE,
+    T_PAAMAYIM_NEKUDOTAYIM,
+    T_EOF = 255,
+} sbTokenType;
+
+typedef struct sbLexToken {
+    sbTokenType type;
+    union {
+        char *str;
+        float fl;
+        int i;
+    };
+} sbLexToken;
+
+hScanner sbScanner_create(hFileReader fr);
+
+sbLexToken sbScanner_next(hScanner sc);
+
+sbLexToken sbScanner_peek(hScanner sc);
+
+void sbScanner_destroy(hScanner sc);