#define STRINGS_PER_BLOCK 256
/* highest bit set on handle value means it is a tinystr */
-#define FLAG_TINY (1L << 63)
+#define FLAG_TINY (1ULL << 63)
/* if string length < REQ_ALLOC_LENGTH, we can just put the
* string in the handle itself. this should be 64 / 8 = 8,
/* TODO: Move these globals into some kind of "execution context" struct
* that we can pass around to be reentrant */
/* also TODO a better (slightly more annoying to code) way to structure
- * this would be SoA style: instead of storing free_count and allocated
+ * this would be SoA style: instead of storing used_count and allocated
* inside the block and entry respectively, store them in a separate
* array next to the pointers that we can scan quickly. less important
- * for allocated since it's already in the strblk anyway, but for free_count
+ * for allocated since it's already in the strblk anyway, but for used_count
* this could be important because the counts will otherwise be scattered
* to the four winds */
sbBuffer g_string_blocks;
usize g_free_block_index;
-typedef u64 hString_mutable;
-
typedef struct strentry {
usize length;
- hString_mutable handle;
+ hString handle;
flag is_external;
flag allocated;
union {
typedef struct strblk {
usize id;
- usize free_count;
+ usize used_count;
usize next_free_index;
strentry entries[STRINGS_PER_BLOCK];
} strblk;
static strentry *new_entry(usize length);
static void place_str_at_offset(strentry *e, usize offset, const char *str, usize length);
static flag is_tinystr(hString handle);
+static strblk *new_strblk();
void sbString_sys_init() {
g_free_block_index = 0;
sbBuffer_initialize(&g_string_blocks, sizeof(strblk*) * 8);
+
+ /* we need to have at least one strblk to start */
+ new_strblk();
}
void sbString_sys_deinit() {
- sbBuffer_deinitialize(&g_string_blocks);
+ /* TODO free these i guess, or don't deinitialize this buffer, right now this
+ * would just drop the pointers to no benefit. look, i'm going to write a
+ * garbage collector, ok? give me a second */
+ //sbBuffer_deinitialize(&g_string_blocks);
}
hString sbString_new(const char *value, usize length) {
return e->handle;
} else {
/* tinystr */
- hString_mutable new_handle = 0;
+ hString new_handle = 0;
for (int i = 0; i < length; i++) {
new_handle <<= 8;
new_handle |= value[i];
if (length_out) *length_out = tinylength;
if (buffer_i_might_use) {
usize index = 0;
- hString_mutable handle_to_eat = handle;
+ hString handle_to_eat = handle;
while (index < tinylength) {
buffer_i_might_use[index] = (handle_to_eat & 0xFF); /* get lowest byte and move down */
handle_to_eat >>= 8;
static strblk *get_strblk(usize index) {
usize nblocks = g_string_blocks.size / sizeof(strblk*);
- if (index >= nblocks) PANIC("attempt to get a string from a block (block #%zu) that does not exist!", index);
+ if (index >= nblocks) PANIC("attempt to get a string from a block (#%zu) that does not exist!", index);
return ((strblk**)g_string_blocks.data)[index];
}
static strentry *get_entry(strblk *blk, usize index) {
- if (index >= STRINGS_PER_BLOCK) PANIC("attempt to get a string (string #%zu) that does not exist in block at %zu!", index, blk->id);
+ if (index >= STRINGS_PER_BLOCK) PANIC("attempt to get a string (#%zu) that does not exist in block (#%zu)!", index, blk->id);
return &blk->entries[index];
}
}
static strentry *alloc_entry(strblk *blk) {
- if (blk->free_count == 0) PANIC("attempt to allocate from a full strblk (%zu)!", blk->id);
+ if (blk->used_count == STRINGS_PER_BLOCK) PANIC("attempt to allocate from a full strblk (#%zu)!", blk->id);
- blk->free_count --;
+ blk->used_count ++;
usize free_index = blk->next_free_index;
- if (blk->free_count > 0) {
+ if (blk->used_count < STRINGS_PER_BLOCK) {
usize next_free_index = blk->next_free_index;
do {
next_free_index ++;
} while (blk->entries[free_index].allocated && next_free_index != blk->next_free_index);
if (next_free_index == blk->next_free_index) {
- PANIC("free count lied! for strblk #%zu", blk->id);
+ PANIC("free count lied! for strblk (#%zu)", blk->id);
}
blk->next_free_index = next_free_index;
static strblk *find_free_block() {
strblk *block_with_free = get_strblk(g_free_block_index);
- if (block_with_free->free_count == 0) {
+ if (block_with_free->used_count == STRINGS_PER_BLOCK) {
usize nblocks = g_string_blocks.size / sizeof(strblk*);
usize start_index = g_free_block_index;
g_free_block_index ++;
g_free_block_index %= nblocks;
block_with_free = get_strblk(g_free_block_index);
- } while (g_free_block_index != start_index && block_with_free->free_count == 0);
+ } while (g_free_block_index != start_index && block_with_free->used_count == STRINGS_PER_BLOCK);
if (g_free_block_index == start_index) {
/* Where did that bring you? Back to me. */
if (e->is_external) {
sbBuffer_set_size(&e->somewhere_else_value, length + 1);
/* if we shortened string, need to replace NUL at end */
+ e->length = length;
e->somewhere_else_value.data[length] = '\0';
} else {
if (length > INLINE_BUFFER_SIZE - 1) {
if (e->length > 0) {
sbBuffer_append(&new_buf, e->inline_value, e->length + 1);
}
+ e->length = length;
e->is_external = TRUE;
e->somewhere_else_value = new_buf;
e->somewhere_else_value.data[length] = '\0';
} else {
/* we can remain inline, but need to set NUL in the right place */
+ e->length = length;
e->inline_value[length] = '\0';
}
}
#include "parse/filereader.h"
#include "parse/lexer.h"
+#include "data/string.h"
int main(int argc, char **argv) {
if (argc == 2) {
sbLexer lx;
sbLexer_initialize(&lx, fr);
+ sbString_sys_init();
+
sbLexToken t;
+ char scratch[8];
do {
t = sbLexer_next(&lx);
} else if (t.type == T_NEWLINE) {
printf("\n");
} else if (t.type == T_IDENTIFIER) {
- printf("%s", t.str);
+ printf("%s", t.cstr);
} else if (t.type == T_INTEGER) {
printf("<INT %d>", t.i);
} else if (t.type == T_STRING) {
- printf("<STRING '%s'>", t.str);
+ printf("<STRING '%s'>", sbString_get_value(t.hstr, scratch, NULL));
} else if (t.type == T_SYMBOL) {
- printf("<SYMBOL :%s>", t.str);
+ printf("<SYMBOL :%s>", t.cstr);
} else if (t.type >= T_rAND && t.type <= T_rWHILE) {
- printf("<%s>", t.str);
+ printf("<%s>", t.cstr);
} else if (t.type == T_SEMICOLON) {
printf(";\n");
} else {
sbLexer_deinitialize(&lx);
sbFileReader_close(fr);
+
+ sbString_sys_deinit();
} else {
fprintf(stderr, "please provide a file as input\n");
}
sbTokenQueue_deinitialize(&lx->output_queue);
}
-struct ReservedWord {
- const char *name;
- sbTokenType token_type;
-};
-
-static struct ReservedWord reserved_words[] = {
- { "and", T_rAND },
- { "as", T_rAS },
- { "case", T_rCASE },
- { "def", T_rDEF },
- { "do", T_rDO },
- { "else", T_rELSE },
- { "false", T_rFALSE },
- { "if", T_rIF },
- { "let", T_rLET },
- { "in", T_rIN },
- { "match", T_rMATCH },
- { "not", T_rNOT },
- { "or", T_rOR },
- { "repeat", T_rREPEAT },
- { "return", T_rRETURN },
- { "true", T_rTRUE},
- { "unless", T_rUNLESS },
- { "until", T_rUNTIL },
- { "when", T_rWHEN },
- { "while", T_rWHILE },
-};
-
-#define N_RESERVED_WORDS ((sizeof(reserved_words))/(sizeof(reserved_words[0])))
-
static flag is_stackable(sbTokenType type);
static void stack_token(hLexer lx, sbLexToken token);
static flag is_closing_bracket(sbTokenType type);
}
static void enqueue_input_token(hLexer lx, sbLexToken token) {
- /* if we receive an identifier, check if it is a reserved word or not.
- * (we need to do this as soon as tokens enter the input queue, because we
- * may be peeking ahead to see whether something is an identifier or not.) */
- if (token.type == T_IDENTIFIER) {
- for (int i = 0; i < N_RESERVED_WORDS; i++) {
- if (!sbstrncmp(reserved_words[i].name, token.str, token.size + 1)) {
- token.type = reserved_words[i].token_type;
- break;
- }
- }
- }
-
sbTokenQueue_enqueue(&lx->input_queue, token);
}
#define MEM_BLOCK_SIZE 65536
#define TOKEN_BUFFER_SIZE 64
+struct ReservedWord {
+ const char *name;
+ sbTokenType token_type;
+};
+
+static struct ReservedWord reserved_words[] = {
+ { "and", T_rAND },
+ { "as", T_rAS },
+ { "case", T_rCASE },
+ { "def", T_rDEF },
+ { "do", T_rDO },
+ { "else", T_rELSE },
+ { "false", T_rFALSE },
+ { "if", T_rIF },
+ { "let", T_rLET },
+ { "in", T_rIN },
+ { "match", T_rMATCH },
+ { "not", T_rNOT },
+ { "or", T_rOR },
+ { "repeat", T_rREPEAT },
+ { "return", T_rRETURN },
+ { "true", T_rTRUE},
+ { "unless", T_rUNLESS },
+ { "until", T_rUNTIL },
+ { "when", T_rWHEN },
+ { "while", T_rWHILE },
+};
+
+#define N_RESERVED_WORDS ((sizeof(reserved_words))/(sizeof(reserved_words[0])))
+
static void check_if_start(hScanner sc);
static sbLexToken compute_next_token(hScanner sc);
token_size = read_symbol(sc);
char *storage = save_buffer(sc);
- new_token.str = storage;
+ new_token.cstr = storage;
new_token.size = token_size;
} else {
/* : */
read_char_into_buffer(sc, '\0');
finalize_char_buffer(sc);
- char *storage = save_buffer(sc);
+ hString hstr = sbString_new(sc->dynamic_buffer.data, sc->dynamic_buffer.size);
- new_token.str = storage;
+ new_token.hstr = hstr;
new_token.size = sc->dynamic_buffer.size;
} else if (is_start_identifier(ch)) {
new_token.type = T_IDENTIFIER;
token_size = read_identifier(sc);
char *storage = save_buffer(sc);
- new_token.str = storage;
+ new_token.cstr = storage;
new_token.size = token_size;
} else if (is_digit(ch)) {
new_token.type = T_INTEGER;
NEXT;
}
+ /* if we receive an identifier, check if it is a reserved word or not. */
+ if (new_token.type == T_IDENTIFIER) {
+ for (int i = 0; i < N_RESERVED_WORDS; i++) {
+ if (!sbstrncmp(reserved_words[i].name, new_token.cstr, new_token.size + 1)) {
+ new_token.type = reserved_words[i].token_type;
+ break;
+ }
+ }
+ }
+
+
return new_token;
}
git://git.cassowary.me / sarabande.git / commitdiff