dynamically sized token queue

diff --git a/src/mem/buffer.c b/src/mem/buffer.c
index f3580a4..c474aea 100644 (file)
--- a/src/mem/buffer.c
+++ b/src/mem/buffer.c
@@ -48,6 +48,22 @@ void *sbBuffer_expand(hBuffer buf, usize expand_size) {
     return result;
 }
 
+void *sbBuffer_shrink(hBuffer buf, usize shrink_size) {
+    if (shrink_size > buf->size) {
+        fprintf(stderr, "internal warning: can't shrink buffer past zero");
+        shrink_size = buf->size;
+    }
+
+    buf->size -= shrink_size;
+
+    /* ok to return this even though it's "outside" the buffer,
+     * because we never shrink the actual allocation of the buffer.
+     * however, bear in mind not to keep this pointer around very
+     * long, because if we append to the buffer it could change
+     * again. */
+    return &buf->data[buf->size];
+}
+
 void sbBuffer_append(hBuffer buf, const char *data, usize data_length) {
     if (data_length == 0) return;
 

diff --git a/src/mem/buffer.h b/src/mem/buffer.h
index 3767a75..f04347c 100644 (file)
--- a/src/mem/buffer.h
+++ b/src/mem/buffer.h
@@ -19,6 +19,8 @@ void sbBuffer_initialize(hBuffer buf, usize initial_size);
 
 void *sbBuffer_expand(hBuffer buf, usize expand_size);
 
+void *sbBuffer_shrink(hBuffer buf, usize shrink_size);
+
 void sbBuffer_append(hBuffer buf, const char *data, usize data_length);
 
 void sbBuffer_reset(hBuffer buf);

diff --git a/src/parse/lexer.c b/src/parse/lexer.c
index af6eb2b..5e2cb9f 100644 (file)
--- a/src/parse/lexer.c
+++ b/src/parse/lexer.c
@@ -34,25 +34,21 @@ static sbLexToken advance_output_queue(hLexer lx);
 void sbLexer_initialize(hLexer lx, hFileReader fr) {
     sbBuffer_initialize(&lx->brackets_stack, 64);
     sbScanner_initialize(&lx->scanner, fr);
-    memset(lx->input_queue, 0, LEXER_QUEUE_LENGTH * sizeof(sbLexToken));
-    memset(lx->output_queue, 0, LEXER_QUEUE_LENGTH * sizeof(sbLexToken));
-    lx->input_queue_length = 0;
-    lx->output_queue_length = 0;
+    sbTokenQueue_initialize(&lx->input_queue, 8);
+    sbTokenQueue_initialize(&lx->output_queue, 8);
     lx->last_token_seen = (sbLexToken) {0};
-    lx->n_brackets = 0;
-    lx->brace_terminated_state = 0;
 }
 
 sbLexToken sbLexer_peek(hLexer lx) {
-    if (lx->output_queue_length == 0) {
+    if (sbTokenQueue_size(&lx->output_queue) == 0) {
         compute_next_token(lx);
     }
 
-    return lx->output_queue[0];
+    return sbTokenQueue_at(&lx->output_queue, 0);
 }
 
 sbLexToken sbLexer_next(hLexer lx) {
-    if (lx->output_queue_length == 0) {
+    if (sbTokenQueue_size(&lx->output_queue) == 0) {
         compute_next_token(lx);
     }
 
@@ -62,8 +58,8 @@ sbLexToken sbLexer_next(hLexer lx) {
 void sbLexer_deinitialize(hLexer lx) {
     sbScanner_deinitialize(&lx->scanner);
     sbBuffer_deinitialize(&lx->brackets_stack);
-    lx->input_queue_length = 0;
-    lx->output_queue_length = 0;
+    sbTokenQueue_deinitialize(&lx->input_queue);
+    sbTokenQueue_deinitialize(&lx->output_queue);
 }
 
 struct ReservedWord {
@@ -78,13 +74,16 @@ static struct ReservedWord reserved_words[] = {
     { "def", T_rDEF },
     { "do", T_rDO },
     { "else", T_rELSE },
+    { "false", T_rFALSE },
     { "if", T_rIF },
     { "let", T_rLET },
     { "in", T_rIN },
+    { "match", T_rMATCH },
     { "not", T_rNOT },
     { "or", T_rOR },
     { "repeat", T_rREPEAT },
     { "return", T_rRETURN },
+    { "true", T_rTRUE},
     { "unless", T_rUNLESS },
     { "until", T_rUNTIL },
     { "when", T_rWHEN },
@@ -93,44 +92,6 @@ static struct ReservedWord reserved_words[] = {
 
 #define N_RESERVED_WORDS ((sizeof(reserved_words))/(sizeof(reserved_words[0])))
 
-static sbLexToken advance_token_queue(sbLexToken *q, int *length, char version) {
-    if (*length == 0) {
-        if (version == 'o') {
-            PANIC("can't advance empty lexer output queue!");
-        } else {
-            PANIC("can't advance empty lexer input queue!");
-        }
-    }
-
-    sbLexToken result = q[0];
-
-    (*length)--;
-    for (int i = 0; i < *length; i++) {
-        if (i < LEXER_QUEUE_LENGTH - 1) {
-            q[i] = q[i + 1];
-        } else {
-            q[i] = (sbLexToken) {0};
-        }
-    }
-
-    q[*length] = (sbLexToken) {0};
-
-    return result;
-}
-
-static void enqueue_token(sbLexToken *q, int *length, sbLexToken token, char version) {
-    if (*length == LEXER_QUEUE_LENGTH) {
-        if (version == 'o') {
-            PANIC("output token queue is full!");
-        } else {
-            PANIC("input token queue is full!");
-        }
-    }
-
-    q[*length] = token;
-    (*length)++;
-}
-
 static flag is_stackable(sbTokenType type);
 static void stack_token(hLexer lx, sbLexToken token);
 static flag is_closing_bracket(sbTokenType type);
@@ -144,7 +105,7 @@ static void enqueue_output_token(hLexer lx, sbLexToken token) {
         unstack_visible_bracket(lx, token);
     }
 
-    enqueue_token(lx->output_queue, &lx->output_queue_length, token, 'o');
+    sbTokenQueue_enqueue(&lx->output_queue, token);
 
     if (is_closing_bracket(token.type) && !token.invisible) {
         unstack_sticky_invisible_parentheses(lx);
@@ -169,31 +130,33 @@ static void enqueue_input_token(hLexer lx, sbLexToken token) {
         }
     }
 
-    enqueue_token(lx->input_queue, &lx->input_queue_length, token, 'i');
+    sbTokenQueue_enqueue(&lx->input_queue, token);
 }
 
 static sbLexToken advance_output_queue(hLexer lx) {
-    while (lx->output_queue_length == 0) {
+    while (sbTokenQueue_size(&lx->output_queue) == 0) {
         compute_next_token(lx);
     }
-    return advance_token_queue(lx->output_queue, &lx->output_queue_length, 'o');
+
+    return sbTokenQueue_shift(&lx->output_queue);
 }
 
 static sbLexToken advance_input_queue(hLexer lx) {
-    if (lx->input_queue_length == 0) {
+    if (sbTokenQueue_size(&lx->input_queue) == 0) {
         enqueue_input_token(lx, sbScanner_next(&lx->scanner));
     }
-    return advance_token_queue(lx->input_queue, &lx->input_queue_length, 'i');
+
+    return sbTokenQueue_shift(&lx->input_queue);
 }
 
 static sbLexToken input_peek_ahead(hLexer lx, int count) {
     /* 0 = next token; 1 = token after that; etc.
      * so if count is 0, length must be at least 1 */
-    while (lx->input_queue_length <= count) {
+    while (sbTokenQueue_size(&lx->input_queue) <= count) {
         enqueue_input_token(lx, sbScanner_next(&lx->scanner));
     }
 
-    return lx->input_queue[count];
+    return sbTokenQueue_at(&lx->input_queue, count);
 }
 
 static flag is_literal(sbTokenType type) {
@@ -201,7 +164,9 @@ static flag is_literal(sbTokenType type) {
         || type == T_STRING
         || type == T_INTEGER
         || type == T_FLOAT
-        || type == T_SYMBOL;
+        || type == T_SYMBOL
+        || type == T_rTRUE
+        || type == T_rFALSE;
 }
 
 static flag insert_semicolon_after(sbTokenType type) {
@@ -312,10 +277,9 @@ static char brackets_stack_top(hLexer lx) {
 static char brackets_stack_pop(hLexer lx) {
     if (lx->brackets_stack.size == 0) return 0;
 
-    char result = brackets_stack_top(lx);
-    lx->brackets_stack.data[lx->brackets_stack.size - 1] = 0;
-    lx->brackets_stack.size --;
-    return result;
+    char *result = sbBuffer_shrink(&lx->brackets_stack, 1);
+
+    return *result;
 }
 
 static void brackets_stack_push(hLexer lx, char c) {

diff --git a/src/parse/lexer.h b/src/parse/lexer.h
index 8eaac99..98bd427 100644 (file)
--- a/src/parse/lexer.h
+++ b/src/parse/lexer.h
@@ -4,19 +4,12 @@
 #include "token.h"
 #include "scanner.h"
 
-#define LEXER_QUEUE_LENGTH 32
-
-/* TODO: input_queue and output_queue should be sbBuffer */
 typedef struct sbLexer {
     sbScanner scanner;
-    sbLexToken input_queue[LEXER_QUEUE_LENGTH];
-    sbLexToken output_queue[LEXER_QUEUE_LENGTH];
-    int input_queue_length;
-    int output_queue_length;
-    sbLexToken last_token_seen;
-    flag brace_terminated_state;
+    sbTokenQueue input_queue;
+    sbTokenQueue output_queue;
     sbBuffer brackets_stack;
-    int n_brackets;
+    sbLexToken last_token_seen;
 } sbLexer;
 
 typedef struct sbLexer *hLexer;

diff --git a/src/parse/scanner.c b/src/parse/scanner.c
index 608c714..6229cf6 100644 (file)
--- a/src/parse/scanner.c
+++ b/src/parse/scanner.c
@@ -9,7 +9,7 @@
  * after this stage so we can differentiate between things like 'a.b (1)' and 'a.b(1)',
  * which have different semantics. */
 
-#define MEM_BLOCK_SIZE 8
+#define MEM_BLOCK_SIZE 65536
 #define TOKEN_BUFFER_SIZE 64
 
 static void check_if_start(hScanner sc);
@@ -206,11 +206,10 @@ static sbLexToken compute_next_token(hScanner sc) {
             || ch == '[' || ch == ']'
             || ch == '{' || ch == '}'
             || ch == ';' || ch == '|'
-            || ch == '.' || ch == ','
-            || ch == '\\') {
+            || ch == ',' || ch == '\\') {
         /* unambiguously single-character tokens */
-        new_token.type = ch;
         NEXT;
+        new_token.type = ch;
     } else if (ch == '!') {
         NEXT;
         ch = PEEK;
@@ -245,6 +244,20 @@ static sbLexToken compute_next_token(hScanner sc) {
         } else {
             new_token.type = T_GREATER;
         }
+    } else if (ch == '.') {
+        new_token.type = T_DOT;
+        NEXT;
+        ch = PEEK;
+        if (ch == '.') {
+            new_token.type = T_TWODOT;
+            NEXT;
+            ch = PEEK;
+
+            if (ch == '.') {
+                new_token.type = T_ELLIPSIS;
+                NEXT;
+            }
+        }
     } else if (ch == '<') {
         NEXT;
         ch = PEEK;

diff --git a/src/parse/token.h b/src/parse/token.h
index 0158ffd..8ea30b4 100644 (file)
--- a/src/parse/token.h
+++ b/src/parse/token.h
@@ -1,26 +1,28 @@
 #ifndef __SB_TOKEN_H__
 #define __SB_TOKEN_H__
 
+#include "common.h"
+
 typedef enum sbTokenType {
     T_NULL                  = 0,
     T_ERROR                 = 1,
-    T_LPAREN                = '(',
+    T_LPAREN                = '(', // op::call
     T_RPAREN                = ')',
-    T_LBRACKET              = '[',
+    T_LBRACKET              = '[', // op::index
     T_RBRACKET              = ']',
     T_LBRACE                = '{',
     T_RBRACE                = '}',
-    T_ASTERISK              = '*',
-    T_SLASH                 = '/',
-    T_PLUS                  = '+',
-    T_MINUS                 = '-',
-    T_PERCENT               = '%',
+    T_ASTERISK              = '*', // op::mul
+    T_SLASH                 = '/', // op::div
+    T_PLUS                  = '+', // op::add
+    T_MINUS                 = '-', // op::sub
+    T_PERCENT               = '%', // op::mod
     T_PIPE                  = '|',
     T_DOT                   = '.',
     T_COMMA                 = ',',
     T_EQUALS                = '=',
-    T_LESS                  = '<',
-    T_GREATER               = '>',
+    T_LESS                  = '<', // op::lt
+    T_GREATER               = '>', // op::gt
     T_COLON                 = ':',
     T_SEMICOLON             = ';',
     T_BACKSLASH             = '\\',
@@ -36,22 +38,25 @@ typedef enum sbTokenType {
     T_SQUIGARROW,           // ~>
     T_BACKSQUIGARROW,       // <~
     T_COLONBRACE,           // :{
-    T_DOUBLEEQUALS,         // ==
-    T_DOUBLEMINUS,          // --
-    T_DOUBLEPLUS,           // ++
-    T_DOUBLEASTERISK,       // **
-    T_DOUBLESLASH,          // //
-    T_DOUBLEGREATER,        // >>
-    T_DOUBLELESS,           // <<
+    T_PAAMAYIM_NEKUDOTAYIM, // :: op::scope
+    T_DOUBLEEQUALS,         // == op::eq
+    T_DOUBLEMINUS,          // -- op::decr
+    T_DOUBLEPLUS,           // ++ op::incr
+    T_DOUBLEASTERISK,       // ** op::pow
+    T_DOUBLESLASH,          // // op::floordiv
+    T_DOUBLEPERCENT,        // %% op::divisible
+    T_DOUBLEGREATER,        // >> op::rshift
+    T_DOUBLELESS,           // << op::lshift
     T_MINUSEQUALS,          // -=
     T_PLUSEQUALS,           // +=
     T_ASTERISKEQUALS,       // *=
     T_SLASHEQUALS,          // /=
     T_PERCENTEQUALS,        // %=
-    T_LESSEQUALS,           // <=
-    T_GREATEREQUALS,        // >=
-    T_NOTEQUALS,            // !=
-    T_PAAMAYIM_NEKUDOTAYIM, // ::
+    T_LESSEQUALS,           // <= op::le
+    T_GREATEREQUALS,        // >= op::ge
+    T_NOTEQUALS,            // != op::ne
+    T_TWODOT,               // ..
+    T_ELLIPSIS,             // ...
 
     /* reserved words */
     T_rAND,                 // and
@@ -60,13 +65,16 @@ typedef enum sbTokenType {
     T_rDEF,                 // def
     T_rDO,                  // do
     T_rELSE,                // else
+    T_rFALSE,               // false
     T_rIF,                  // if
     T_rIN,                  // in
     T_rLET,                 // let
+    T_rMATCH,               // match
     T_rNOT,                 // not
     T_rOR,                  // or
     T_rREPEAT,              // repeat
     T_rRETURN,              // return
+    T_rTRUE,                // true
     T_rUNLESS,              // unless
     T_rUNTIL,               // until
     T_rWHEN,                // when
@@ -86,4 +94,22 @@ typedef struct sbLexToken {
     };
 } sbLexToken;
 
+typedef struct sbTokenQueue {
+    sbBuffer buffer;
+} sbTokenQueue;
+
+typedef sbTokenQueue *hTokenQueue;
+
+void sbTokenQueue_initialize(hTokenQueue q, i16 initial_size);
+
+void sbTokenQueue_enqueue(hTokenQueue q, sbLexToken t);
+
+sbLexToken sbTokenQueue_shift(hTokenQueue q);
+
+i32 sbTokenQueue_size(hTokenQueue q);
+
+sbLexToken sbTokenQueue_at(hTokenQueue q, i32 index);
+
+void sbTokenQueue_deinitialize(hTokenQueue q);
+
 #endif