Missing commit

build.sh

@@ -2,4 +2,4 @@
 CFLAGS=${CFLAGS:-"-fsanitize=address -ggdb"}
 CC=${CC:-gcc}
 set -x
-${CC} -DDEBUG -pipe  -Wall -Wextra -Wno-unused-parameter -Wno-unused-variable -std=gnu2x $CFLAGS src/*c
+${CC} -DDEBUG -pipe  -Wall -Wextra -Wno-unused-parameter -Wno-unused-variable -std=gnu2x $CFLAGS src/*c src/base/*c

src/base/arena.c

@@ -1,5 +1,5 @@
 #include "arena.h"
-#include "base_core.h"
+#include "core.h"
 #include "platform.h"
 
 
@@ -12,15 +12,15 @@ static MemoryBlock *memoryblock_allocate(U32 size, U32 commit_size, U32 page_siz
     Assert(block != NULL);
     memory_commit(block, commit_size);
 
+    ASAN_POISON_MEMORY_REGION(block, size);
+    ASAN_UNPOISON_MEMORY_REGION(block, MemoryBlock_Header_Size);
+
     block->base          = (U8 *)block;
     block->size          = size;
     block->first_free    = MemoryBlock_Header_Size;
     block->arena_offset  = 0;
     block->prev          = NULL;
     block->committed_end = commit_size;
-
-    ASAN_POISON_MEMORY_REGION(block->base, block->size);
-    ASAN_UNPOISON_MEMORY_REGION(block, MemoryBlock_Header_Size);
     return block;
 }
 
@@ -38,7 +38,7 @@ void arena_init(Arena *arena, const ArenaOptions *opts)
     MemoryBlock *block  = memoryblock_allocate(aligned_size, aligned_commit_size, page_size);
 
     arena->flags            = opts->flags;
-    arena->checkpoint_level = 0;
+    arena->temp_level = 0;
     arena->current          = block;
     arena->block_size       = aligned_size;
     arena->commit_size      = aligned_commit_size;
@@ -48,7 +48,7 @@ void arena_init(Arena *arena, const ArenaOptions *opts)
 void arena_free(Arena *arena)
 {
     MemoryBlock *block = arena->current;
-    Assert(arena->checkpoint_level == 0);
+    Assert(arena->temp_level == 0);
     while (block != NULL) {
         MemoryBlock *prev = block->prev;
         memory_free(block->base, block->size);
@@ -115,7 +115,7 @@ void arena_reset_to(Arena *arena, U64 total_offset)
         MemoryBlock *prev = block->prev;
 
         if (block->arena_offset >= total_offset) {
-            ASAN_UNPOISON_MEMORY_REGION(block->base, block->size);
+            ASAN_UNPOISON_MEMORY_REGION(block->base, block->committed_end);
             memory_free(block->base, block->size);
             arena->current = prev;
         } else {
@@ -123,8 +123,14 @@ void arena_reset_to(Arena *arena, U64 total_offset)
             new_pos = Max(MemoryBlock_Header_Size, new_pos);
             // Disallow arbitrary pops to positions that previously were not commited.
             Assert(new_pos <= block->committed_end);
+            Assert(new_pos <= block->size);
             block->first_free = new_pos;
             ASAN_POISON_MEMORY_REGION(&block->base[block->first_free], memoryblock_available(block));
+
+            if ((arena->flags & ArenaFlag_NoDecommit) == 0) {
+                arena_trim(arena);
+            }
+
             break;
         }
 
@@ -132,25 +138,39 @@ void arena_reset_to(Arena *arena, U64 total_offset)
     }
 }
 
-ArenaCheckpoint arena_temp_begin(Arena *arena)
+void arena_trim(Arena *arena)
 {
-    ArenaCheckpoint checkpoint = {};
+    MemoryBlock *block = arena->current;
+    U32 aligned_pos = RoundUpToMultiple(block->first_free, arena->commit_size);
+    if (aligned_pos < block->committed_end) {
+        void *trim_start = &block->base[aligned_pos];
+        // Note(nos): Should we unpoision ?
+        //ASAN_UNPOISON_MEMORY_REGION(trim_start, block->committed_end);
+        memory_decommit(trim_start, block->committed_end - aligned_pos);
+        block->committed_end = aligned_pos;
+    }
+}
+
+ATemp arena_temp_begin(Arena *arena)
+{
+    ATemp checkpoint = {};
     checkpoint.arena           = arena;
     checkpoint.total_offset    = arena_used(arena);
-    arena->checkpoint_level++;
+    arena->temp_level++;
 
     return checkpoint;
 }
 
-void arena_temp_end(ArenaCheckpoint *checkpoint)
+void arena_temp_end(ATemp *checkpoint)
 {
     Arena *arena = checkpoint->arena;
-    Assert(arena->checkpoint_level > 0);
+    Assert(arena->temp_level > 0);
     arena_reset_to(arena, checkpoint->total_offset);
-    arena->checkpoint_level--;
+    arena->temp_level--;
 }
 
 void arena_reset(Arena *arena)
 {
     arena_reset_to(arena, 0);
 }
+

src/base/arena.h

@@ -1,13 +1,17 @@
 #pragma once
-#include "base_core.h"
+#include "core.h"
 
 
 typedef U32 ArenaFlags;
 typedef struct ArenaOptions ArenaOptions;
+enum {
+    ArenaFlag_NoDecommit = 1, // operations which shrinks the arena does not decommit memory.
+                              // (arena_trim() can still decommit when called explicitly)
+};
 struct ArenaOptions {
     U32 size;
     U32 commit_size;
-    ArenaFlags flags; // No flags defined yet
+    ArenaFlags flags;
 };
 
 typedef struct MemoryBlock MemoryBlock;
@@ -27,16 +31,16 @@ struct Arena {
     U32 commit_size;    // requested commit size
     U32 page_size;      // cached page size
     ArenaFlags flags;
-    S32 checkpoint_level;
+    S32 temp_level;
 };
 
-typedef struct ArenaCheckpoint ArenaCheckpoint;
-struct ArenaCheckpoint {
+typedef struct ATemp ATemp;
+struct ATemp {
     Arena *arena;
     U64 total_offset;
 };
-#ifndef ArenaParams_Default
-#define ArenaParams_Default { .size = MB(32), .commit_size = KB(64) }
+#ifndef ArenaOptions_Default
+#define ArenaOptions_Default ((ArenaOptions){ .size = MB(32), .commit_size = KB(64) })
 #endif
 
 StaticAssert(sizeof(MemoryBlock) <= 64);
@@ -51,8 +55,10 @@ void *arena_push(Arena *arena, U32 size, U32 align);
 // Invalid for arrays > U32
 #define arena_push_array_of(arena, type, len) arena_push(arena, (U32) (sizeof(type) * (len)), alignof(type))
 
-ArenaCheckpoint arena_temp_begin(Arena *arena);
-void arena_temp_end(ArenaCheckpoint *checkpoint);
+void arena_trim(Arena *arena);
+
+ATemp arena_temp_begin(Arena *arena);
+void arena_temp_end(ATemp *checkpoint);
 void arena_reset_to(Arena *arena, U64 total_offset);
 void arena_reset(Arena *arena);
 
@@ -66,3 +72,30 @@ static INLINE U64 arena_used(const Arena *arena)
     const MemoryBlock *block = arena->current;
     return block->arena_offset + block->first_free;
 }
+
+
+static INLINE void *arena_allocator_func(Allocator *allocator, AllocOperation operation , void *memory, U32 size, U32 align)
+{
+    switch (operation) {
+        case Alloc_Op_Aligned:
+            return arena_push(allocator->data, size, align);
+
+        case Alloc_Op_Free: // can't free from  an marena
+            return NULL;
+
+        case Alloc_Op_Free_All:
+            arena_free(allocator->data);
+            return NULL;
+    }
+
+    Assert(false);
+}
+
+static INLINE Allocator arena_allocator(Arena *arena)
+{
+    Allocator allocator = {};
+    allocator.op_func_ptr = arena_allocator_func;
+    allocator.data = arena;
+
+    return allocator;
+}

src/base/core.h

@@ -0,0 +1,580 @@
+#pragma once
+
+#include <stdalign.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#ifdef __has_feature
+    #if __has_feature(address_sanitizer)
+    #define USE_ASAN 1
+    #endif
+#endif
+#if defined(__SANITIZE_ADDRESS__)
+    #define USE_ASAN 1
+#endif
+
+// Types
+typedef int8_t      S8;
+typedef int16_t     S16;
+typedef int32_t     S32;
+typedef int64_t     S64;
+typedef uint8_t     U8;
+typedef uint16_t    U16;
+typedef uint32_t    U32;
+typedef uint64_t    U64;
+typedef float       F32;
+typedef double      F64;
+#if defined(__FLT128_MANT_DIG__)
+    typedef _Float128 F128;
+#elif defined(__SIZEOF_FLOAT128__)
+    typedef __float128 F128;
+#endif
+#ifdef __SIZEOF_INT128__
+typedef __int128          S128;
+typedef unsigned __int128 U128;
+#endif
+
+#define  U8_Max     ((U8) 0xff)
+#define U16_Max     ((U16)0xffff)
+#define U32_Max     ((U32)0xffffffff)
+#define U64_Max     ((U64)0xffffffffffffffffULL)
+
+// Duh ..
+#define  U8_Min     ((U8)0x0)
+#define U16_Min     ((U16)0x0)
+#define U32_Min     ((U32)0x0)
+#define U64_Min     ((U64)0x0)
+
+#define  S8_Max     ((S8) 0x7f)
+#define S16_Max     ((S16)0x7fff)
+#define S32_Max     ((S32)0x7fffffff)
+#define S64_Max     ((S64)0x7fffffffffffffffLL)
+
+#define  S8_Min     ((S8) 0x80)
+#define S16_Min     ((S16)0x8000)
+#define S32_Min     ((S32)0x80000000)
+#define S64_Min     ((S64)0x8000000000000000LL)
+// asserts
+
+#define Trap() __builtin_trap()
+#define Abort() __builtin_abort()
+
+
+#define Assert(x)       do { if (!(x)) { Trap(); }} while(0)
+#define NotImplemented  Assert(!"Not Implemented")
+#define ShouldNotHappen  Assert(!"This should not happen")
+
+#define StaticAssert(expr) typedef int STATIC_ASSERT_FAILED[(expr) ? 1 : -1]
+
+// misc utilities
+#define Max(a, b)             \
+  ({                          \
+    __typeof__(a) _amx = (a); \
+    __typeof__(b) _bmx = (b); \
+    _amx > _bmx ? _amx : _bmx;\
+  })
+
+#define Min(a, b)              \
+  ({                           \
+    __typeof__(a) _amn = (a);  \
+    __typeof__(b) _bmn = (b);  \
+    _amn < _bmn ? _amn : _bmn; \
+  })
+
+#define ArraySize(a) (sizeof(a) / sizeof((a)[0]))
+
+#define Swap(a, b)            \
+  do {                        \
+    __typeof__(a) tmp_ = (a); \
+    (a) = (b);                \
+    (b) = tmp_;               \
+  } while (0)
+
+#define ContainerOf(ptr, type, member)                                  \
+  ({                                                                    \
+    typeof(((type *)0)->member) *__mptr = (ptr);                        \
+    (type *)((void *)(unsigned char *)__mptr - offsetof(type, member)); \
+  })
+
+#define ConstContainerOf(ptr, type, member)                           \
+  ({                                                                  \
+    const typeof(((const type *)0)->member) *__mptr = (ptr);          \
+    (const type *)((const void *)(const unsigned char *)__mptr - offsetof(const type, member));                     \
+  })
+
+// `align` must be a power of two.
+//
+// These keep the result in the *same integer type as the first argument*
+#define AlignUp(val, align)                                                \
+  ({                                                                       \
+    __typeof__(val) AlignUp_v_ = (val);                                    \
+    __typeof__(val) AlignUp_a_ = (__typeof__(val))(align);                 \
+    (__typeof__(val))((AlignUp_v_ + (AlignUp_a_ - 1)) &                    \
+                      ~(__typeof__(val))(AlignUp_a_ - 1));                 \
+  })
+
+#define AlignDown(val, align)                                              \
+  ({                                                                       \
+    __typeof__(val) AlignDown_v_ = (val);                                  \
+    __typeof__(val) AlignDown_a_ = (__typeof__(val))(align);               \
+    (__typeof__(val))(AlignDown_v_ & ~(__typeof__(val))(AlignDown_a_ - 1));\
+  })
+
+#define IsPow2(x)                                                          \
+  ({                                                                       \
+    __typeof__(x) IsPow2_x_ = (x);                                         \
+    IsPow2_x_ && !(IsPow2_x_ & (__typeof__(x))(IsPow2_x_ - 1));            \
+  })
+
+#define IsPow2OrZero(x)                                                    \
+  ({                                                                       \
+    __typeof__(x) IsPow2OrZero_x_ = (x);                                   \
+    !(IsPow2OrZero_x_ & (__typeof__(x))(IsPow2OrZero_x_ - 1));             \
+  })
+
+#define IsAlignedTo(x, align)                                              \
+  ({                                                                       \
+    __typeof__(x) IsAlignedTo_x_ = (x);                                    \
+    __typeof__(x) IsAlignedTo_a_ = (__typeof__(x))(align);                 \
+    IsPow2(IsAlignedTo_a_) &&                                              \
+        !(IsAlignedTo_x_ & (__typeof__(x))(IsAlignedTo_a_ - 1));           \
+  })
+
+#define GetBit(val, idx) (((val) >> (idx)) & 1)
+// Get n_bits starting at idx going left
+#define GetBits(val, n_bits, idx) \
+  (((n_bits) == 0 ? 0 : ((value) >> (idx)) & ((((__typeof__(val))(1) << (n_bits)) - 1)))
+
+#define STRINGIFY(x) STRINGIFY_HLP(x)
+#define STRINGIFY_HLP(x) #x
+
+#define SRC_LOCATION __FILE__ ":" STRINGIFY(__LINE__)
+
+#ifdef DEBUG
+#define TRACEF(fmt, ...)                                         \
+  do {                                                           \
+    fprintf(stdout, "%s (%s)\t" fmt, SRC_LOCATION, __FUNCTION__, \
+            ##__VA_ARGS__);                                      \
+    fflush(stdout);                                              \
+  } while (0)
+#else
+#define TRACEF(fmt, ...) \
+  do {                   \
+  } while (0)
+#endif
+
+/**
+ * Ceiling of X / y , or  round up x / y
+ * (e.g 100/9 == 12, 3/2 == 2)
+ * x + y must not overflow.
+ *
+ * @param x numerator, must be positive
+ * @param y denominator
+ * @return x/y rounded up
+ */
+#define CeilDiv(x, y) (((x) + ((y) - 1)) / (y))
+
+/**
+ * Round x up to nearest multiple of y:
+ * (((x) + (y) - 1) - (((x) + (y) - 1) % (y)))
+ * e.g. RoundUp(30,48) == 48
+ * e.g. RoundUp(49,48) == 96
+ *
+ * @param x numerator, must be positive
+ * @param y denominator
+ * @return x rounded up to nearest multiple of y
+ */
+#define RoundUpToMultiple(x, y)       \
+({                                    \
+    __typeof__(x) _x   = (x);         \
+    __typeof__(y) _y   = (y);         \
+    Assert(_y         != 0);          \
+    __typeof__(x) _tmp = _x + _y - 1; \
+    _tmp - _tmp % _y;                 \
+})
+
+
+/**
+ * Round x down to nearest multiple of y
+ * e.g. RoundDown(30,48) == 0
+ * e.g. RoundDown(49,48) == 48
+ *
+ * @param x numerator, must be positive
+ * @param y denominator
+ * @return x rounded down nearest multiple of y
+ */
+#define RoundDownToMultiple(x, y) ((x) / (y) * (y))
+#define Clamp(val, min, max) (((val) < (min)) ? (min) : ((val) > (max)) ? (max) : (val))
+
+#ifdef __GNUC__
+#define INLINE inline __attribute__((always_inline, flatten))
+#else
+#define INLINE inline
+#endif
+
+#ifdef __cplusplus
+#define C_LINKAGE_BEGIN extern "C" {
+#define C_LINKAGE_END }
+#define C_LINKAGE extern "C"
+#else
+#define C_LINKAGE_BEGIN
+#define C_LINKAGE_END
+#define C_LINKAGE
+#endif
+
+/** Indicate the variable is unused.
+ */
+#define UNUSED(x) x __attribute__((unused))
+
+// sizes
+#define Bytes(n) ((U64)(n))
+#define KB(n)    (((U64)(n)) << 10)
+#define MB(n)    (((U64)(n)) << 20)
+#define GB(n)    (((U64)(n)) << 30)
+#define TB(n)    (((U64)(n)) << 40)
+
+#define Thousand(n) (((U64)(n) * 1000))
+#define Million(n)  (((U64)(n) * 1000000))
+#define Billion(n)  (((U64)(n) * 1000000000))
+
+// memory
+#define MemSet(ptr, val, len)       memset(ptr, val, len)
+#ifdef USE_ASAN
+#define MemoryZero(ptr, len)        memset(ptr, 0, len)
+// explicity bzero is usually not being intercepted
+#else
+#define MemoryZero(ptr, len)        explicit_bzero(ptr, len)
+#endif
+#define MemMove(dst, src, len)      memmove(dst, src, len)
+#define MemCopy(dst, src, len)       memcpy(dst, src, len)
+#define MemAppend(dst, src, len)    mempcpy(dst, src, len)
+#define MemCompare(first, second, len) memcmp(first, second, len)
+
+static INLINE U8 *MemFind(U8 *mem, U64 mem_len, U8 *needle, U64 needle_len)
+{
+  if (needle_len > mem_len) {
+      return NULL;
+  }
+  for (U64 i = 0 ; i + needle_len <= mem_len; i++) {
+      if (MemCompare(&mem[i], needle, needle_len) == 0) {
+        return &mem[i];
+      }
+  }
+  return NULL;
+}
+// atomics
+#define Atomic(type) _Atomic(type)
+#define AtomicSub(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+#define AtomicAdd(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+#define AtomicDec(ptr) AtomicSub((ptr), 1)
+#define AtomicInc(ptr) AtomicAdd((ptr), 1)
+#define AtomicGet(ptr) __c11_atomic_load((ptr), __ATOMIC_SEQ_CST)
+#define AtomicCAS(ptr, oldval, newval) \
+    __atomic_compare_exchange((ptr), (oldval), (newval),0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
+
+#define MemoryBarrier() __atomic_thread_fence(UC_ORDER_CST)
+#define CompilerBarrier() __asm__ __volatile__("": : :"memory")
+
+// Saturating operations
+#define SaturatingAdd(a, b, type, max_val)\
+    ({type res ;\
+    if (__builtin_add_overflow(a, b, &res)) {\
+        res = max_val;}\
+    res;})
+
+
+#define SaturatingSub(a, b, type)\
+    ({type res ;\
+    if (__builtin_sub_overflow(a, b, &res)) {\
+        res = 0;}\
+    res;})
+
+#define SaturatingMult(a, b, type, max_val)\
+    ({type res ;\
+    if (__builtin_mul_overflow(a, b, &res)) {\
+        res = max_val;}\
+    res;})
+
+static INLINE U8 SaturatingAddU8(U8 a, U8 b)
+{
+    return SaturatingAdd(a, b, U8, U8_Max);
+}
+
+static INLINE U8 SaturatingSubU8(U8 a, U8 b)
+{
+    return SaturatingSub(a, b, U8);
+}
+
+static INLINE U8 SaturatingMult8(U8 a, U8 b)
+{
+    return SaturatingMult(a, b, U8, U8_Max);
+}
+
+static INLINE U16 SaturatingAdd16(U16 a, U16 b)
+{
+    return SaturatingAdd(a, b, U16, U16_Max);
+}
+
+static INLINE U16 SaturatingSubU16(U16 a, U16 b)
+{
+    return SaturatingSub(a, b, U16);
+}
+
+static INLINE U16 SaturatingMultU16(U16 a, U16 b)
+{
+    return SaturatingMult(a, b, U16, U16_Max);
+}
+
+static INLINE U32 SaturatingAdd32(U32 a, U32 b)
+{
+    return SaturatingAdd(a, b, U32, U32_Max);
+}
+
+static INLINE U32 SaturatingSub32(U32 a, U32 b)
+{
+    return SaturatingSub(a, b, U32);
+}
+
+static INLINE U32 SaturatingMult32(U32 a, U32 b)
+{
+    return SaturatingMult(a, b, U32, U32_Max);
+}
+
+static INLINE U64 SaturatingAdd64(U64 a, U64 b)
+{
+    return SaturatingAdd(a, b, U64, U64_Max);
+}
+
+static INLINE U64 SaturatingSubU64(U64 a, U64 b)
+{
+    return SaturatingSub(a, b, U64);
+}
+
+static INLINE U64 SaturatingMult64(U64 a, U64 b)
+{
+    return SaturatingMult(a, b, U64, U64_Max);
+}
+
+// Allocators
+typedef enum AllocOperation AllocOperation;
+enum AllocOperation {
+    Alloc_Op_Aligned = 1,
+    Alloc_Op_Free,
+    Alloc_Op_Free_All,
+};
+
+typedef struct Allocator Allocator;
+typedef void *(*AllocOpFunc)(Allocator *allocator, AllocOperation operation, void *memory, U32 size, U32 align);
+struct Allocator {
+    AllocOpFunc op_func_ptr;
+    void *data;
+};
+
+static INLINE void *allocator_alloc_aligned(Allocator *allocator, U32 size, U32 align)
+{
+    return allocator->op_func_ptr(allocator, Alloc_Op_Aligned, NULL, size, align);
+}
+
+static INLINE void allocator_free(Allocator *allocator, void *memory)
+{
+    allocator->op_func_ptr(allocator, Alloc_Op_Free, memory, 0, 0);
+}
+
+static INLINE void  allocator_free_all(Allocator *allocator)
+{
+    allocator->op_func_ptr(allocator, Alloc_Op_Free_All, NULL, 0, 0);
+}
+
+
+#define allocator_push_type(allocator, type_)\
+    (allocator)->op_func_ptr((allocator), Alloc_Op_Aligned, NULL, sizeof(type_), _Alignof(type_))
+
+#define allocator_push_array(allocator, type_, nr_elements)\
+    (allocator)->op_func_ptr((allocator), Alloc_Op_Aligned, NULL, sizeof(type_) * (nr_elements), _Alignof(type_) )
+
+#define allocator_push(allocator, size, align)\
+    (allocator)->op_func_ptr((allocator), Alloc_Op_Aligned, NULL, size, align)
+
+// address-sanitizer helpers (-fsanitize=address)
+#ifdef USE_ASAN
+#include <sanitizer/asan_interface.h>
+#define ASAN_POISON_MEMORY_REGION(addr, size) \
+  __asan_poison_memory_region((addr), (size))
+#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+  __asan_unpoison_memory_region((addr), (size))
+#else
+#define ASAN_POISON_MEMORY_REGION(addr, size) \
+  ((void)(addr), (void)(size))
+#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+  ((void)(addr), (void)(size))
+#endif
+
+
+// linked lists
+
+/*
+  SList - singly linked lists
+    "list" struct must have a "*first" and "**last" member to entrys
+    "first"/"last" structs must have a "*next" member
+*/
+
+/**
+ * Initialize a list
+ *
+ * @param list struct pointer with a *first and a **last member to an entry struct
+ */
+#define SListInit(list)           \
+do {                               \
+    (list)->first = NULL;          \
+    (list)->last = &(list)->first; \
+} while (0)
+
+/**
+ * Adds entry to the head of a list
+ *
+ * @param list struct pointer with a *first and a **last member to an entry struct
+ * @param entry struct pointer with a *next member to an entry struct
+ */
+#define SListAddFirst(list, entry)    \
+do {                                   \
+    (entry)->next = (list)->first;     \
+    if ((entry)->next == NULL) {       \
+        (list)->last = &(entry)->next; \
+    }                                  \
+    (list)->first = (entry);           \
+} while (0)
+
+/**
+ * Adds entry to the tail of a list
+ *
+ * @param list struct pointer with a *first and a **last member to an entry struct
+ * @param entry struct pointer with a *next member to an entry struct
+ */
+#define SListAddLast(list, entry) \
+do {                                \
+    (entry)->next = NULL;           \
+    *(list)->last = (entry);        \
+    (list)->last = &(entry)->next;  \
+} while (0)
+
+/**
+ * Remove head from list, list MUST not be empty
+ *
+ * @param entry struct pointer with a *next member to an entry struct
+ */
+#define SListRemoveFirst(list)          \
+do {                                     \
+    (list)->first = (list)->first->next; \
+    if ((list)->first == NULL) {         \
+        (list)->last = &(list)->first;   \
+    }                                    \
+} while (0)
+
+#define SListIsEmpty(list)    ((list)->first == NULL)
+#define SListResetNode(node)  ((node)->next = NULL)
+
+/*
+  DList - doubly linked lists
+    "list" struct must have a "*first" and "*last" member to entrys
+    "first"/"last" structs must have a "*next" and a "*prev" member
+*/
+
+/**
+ * Initialize a list
+ *
+ * @param list struct pointer with a *first and a *last member to an entry struct
+ */
+#define DListInit(list)           \
+do {                              \
+    (list)->first = NULL;         \
+    (list)->last = NULL;          \
+} while (0)
+
+/**
+ * Adds entry to the head of a list
+ *
+ * @param list struct pointer with a *first and a *last member to an entry struct
+ * @param entry struct pointer with a *next and *prev member to an entry struct
+ */
+#define DListAddFirst(list, entry)             \
+do {                                          \
+  (entry)->next = (list)->first;              \
+  (entry)->prev = NULL;                       \
+    if ((list)->first != NULL) {              \
+        (list)->first->prev = (entry);        \
+    } else {                                  \
+        (list)->last = (entry);               \
+    }                                         \
+    (list)->first = (entry);                  \
+} while (0)
+/**
+ * Adds entry to the tail of a list
+ *
+ * @param list struct pointer with a *first and a *last member to an entry struct
+ * @param entry struct pointer with a *next and *prev member to an entry struct
+ */
+#define DListAddLast(list, entry)  \
+do {                                \
+  (entry)->prev = (list)->last;     \
+  (entry)->next = NULL;             \
+  if ((list)->last != NULL) {       \
+    (list)->last->next = (entry);   \
+  } else {                          \
+    (list)->first = (entry);        \
+  }                                 \
+  (list)->last = (entry);           \
+} while (0)
+
+/**
+ * Remove first entry from list, list MUST not be empty
+ *
+ * @param entry struct pointer with a *next and *prev member to an entry struct
+ */
+#define DListRemoveFirst(list)           \
+do {                                     \
+    (list)->first = (list)->first->next; \
+    if ((list)->first != NULL) {         \
+        (list)->first->prev = NULL;      \
+    } else {                             \
+        (list)->last = NULL;             \
+    }                                    \
+} while (0)
+
+/**
+ * Remove last entry from list, list MUST not be empty
+ *
+ * @param entry struct pointer with a *next and *prev member to an entry struct
+ */
+#define DListRemoveLast(list)             \
+do {                                      \
+    (list)->last = (list)->last->prev;    \
+    if ((list)->last != NULL) {           \
+        (list)->last->next = NULL;        \
+    } else {                              \
+        (list)->first = NULL;             \
+    }                                     \
+} while (0)
+
+/**
+ * Remove last entry from list, entry must be in the list
+ *
+ * @param entry struct pointer with a *next and *prev member to an entry struct
+ */
+#define DListRemove(list, entry)                \
+do {                                            \
+    if ((entry)->next != NULL) {                \
+      (entry)->next->prev = (entry)->prev;      \
+    } else {                                    \
+      (list)->last = (entry)->prev;             \
+    }                                           \
+    if ((entry)->prev != NULL) {                \
+      (entry)->prev->next = (entry)->next;      \
+    } else {                                    \
+      (list)->first = (entry)->next;            \
+    }                                           \
+} while (0)
+
+#define DListIsEmpty(list)    ((list)->first == NULL)
+#define DListResetNode(node) ((node)->next = (node)->prev = NULL)

src/base/marena.h

@@ -0,0 +1,121 @@
+#pragma once
+#include "core.h"
+
+// Mini Arena for arena allocations from a supplied buffer
+
+typedef struct MArena MArena;
+struct MArena {
+    U8 *base;         // start of buffer
+    U32 first_free;
+    U32 size;
+    S32 temp_level;  // level of marena_temp_begin nesting
+};
+
+typedef struct MTemp MTemp;
+struct MTemp {
+    U32 offset;
+    MArena *arena;
+};
+
+#define MARENA_STATIC_INITIALIZER(data) MARENA_INITIALIZER((data), sizeof(data))
+#define MARENA_INITIALIZER(data, size_) {.base = (data), .first_free = 0, .size = (U32)size_, .temp_level = 0}
+#define MARENA_DEFINE(var_name, size, alignment_)  \
+    struct { \
+        _Alignas(alignment_) U8 memory[size];\
+    } var_name##_marena_memory; \
+    MArena var_name = MARENA_STATIC_INITIALIZER(var_name##_marena_memory.memory)
+
+
+
+static INLINE void marena_init(MArena *arena, void *memory, U32 size)
+{
+    Assert(memory != NULL);
+    arena->base       = (U8 *)memory;
+    arena->first_free = 0;
+    arena->size       = size;
+}
+
+static INLINE MTemp marena_temp_begin(MArena *arena)
+{
+    MTemp checkpoint = {};
+
+    checkpoint.arena  = arena;
+    checkpoint.offset = arena->first_free;
+    arena->temp_level++;
+
+    return checkpoint;
+}
+
+static INLINE void marena_temp_end(MTemp *state)
+{
+    MArena *arena = state->arena;
+    Assert(arena->temp_level > 0);
+    arena->first_free = state->offset;
+    arena->temp_level--;
+}
+
+static INLINE void *marena_push(MArena *arena, U32 size)
+{
+    if (size > arena->size - arena->first_free) {
+        return NULL;
+    }
+    void *base = &arena->base[arena->first_free];
+    arena->first_free += size;
+    return base;
+}
+
+[[gnu::alloc_align(3)]]
+static INLINE void *marena_push_aligned(MArena *arena, U32 size, U32 align)
+{
+    Assert(IsPow2(align));
+
+    U32 aligned = AlignUp(arena->first_free, align);
+    if (aligned < arena->first_free) { // aligned wrapped around
+        return NULL;
+    }
+
+    if (aligned > arena->size || size > arena->size - aligned) {
+        return NULL;
+    }
+
+    void *base = &arena->base[aligned];
+    arena->first_free = aligned + size;
+
+    return base;
+}
+
+#define marena_push_type(arena, type_)\
+        marena_push_aligned((arena), sizeof(type_), _Alignof(type_))
+#define marena_push_array(arena, type_, nr_elements_)\
+        marena_push_aligned((arena), (nr_elements_) * (U32)sizeof(type_), _Alignof(type_))
+
+
+static INLINE void marena_reset(MArena *arena)
+{
+    arena->first_free = 0;
+}
+
+static INLINE void *marena_allocator_func(Allocator *allocator, AllocOperation operation , void *memory, U32 size, U32 align)
+{
+    switch (operation) {
+        case Alloc_Op_Aligned:
+            return marena_push_aligned(allocator->data, size, align);
+
+        case Alloc_Op_Free: // can't free/Free_all from  an marena
+            return NULL;
+
+        case Alloc_Op_Free_All:
+            return NULL;
+    }
+
+    Assert(false);
+}
+
+static INLINE Allocator marena_allocator(MArena *arena)
+{
+    Allocator allocator = {};
+    allocator.op_func_ptr = marena_allocator_func;
+    allocator.data = arena;
+
+    return allocator;
+}

src/base/platform.c

@@ -1,4 +1,4 @@
-#include "base_core.h"
+#include "core.h"
 #include "platform.h"
 #include <stdlib.h>
 #include <stdio.h>
@@ -23,6 +23,34 @@ void platform_free(void *mem)
     free(mem);
 }
 
+
+static void *platform_allocator_func(Allocator *allocator, AllocOperation operation , void *memory, U32 size, U32 align)
+{
+    switch (operation) {
+        case Alloc_Op_Aligned:
+            return aligned_alloc(size, align);
+
+        case Alloc_Op_Free: // can't free/Free_all from  an marena
+            free(memory);
+            return NULL;
+
+        case Alloc_Op_Free_All:
+            NotImplemented;
+            return NULL;
+    }
+
+    Assert(false);
+}
+
+Allocator platform_allocator(void)
+{
+    Allocator allocator = {};
+    allocator.op_func_ptr = platform_allocator_func;
+    allocator.data = NULL;
+
+    return allocator;
+}
+
 void *memory_allocate_and_commit(size_t size)
 {
     Assert(IsPageAligned(size));

src/base/platform.h

@@ -1,12 +1,16 @@
 #pragma once
 #include <stdlib.h>
+#include "core.h"
 
 void *platform_malloc(size_t size);
+void platform_free(void *mem);
+Allocator platform_allocator(void);
+
 U32 platform_page_size(void);
 
+// Memory functions expect mem and size to be page aligned
 void *memory_allocate_and_commit(size_t size);
 void *memory_reserve(size_t size);
 void memory_commit(void *mem, size_t size);
 void memory_decommit(void *mem, size_t size);
-
 void memory_free(void *mem, size_t size);

src/base/pool.c

@@ -0,0 +1,57 @@
+#include "pool.h"
+
+APool *apool_create_ex(Allocator *allocator, U32 n_elements, U32 element_size, U32 element_align)
+{
+    Assert(IsPow2(element_align));
+    U32 align = element_align;
+    if (element_align < alignof(APoolElement)) {
+        element_align = alignof(APoolElement);
+    }
+    U32 size = AlignUp(element_size, element_align);
+
+    APool *pool = allocator_push_type(allocator, APool);
+    Assert(pool != NULL);
+    pool->align = align;
+    pool->element_size = size;
+    pool->allocator = allocator;
+    pool->n_elements = n_elements;
+    pool->free_list = NULL;
+    pool->start = NULL;
+
+    if (size != APOOL_AUTOGROW) {
+        Assert(n_elements <= U32_Max / size );
+        U32 total_size = n_elements * size;
+        U8 * start = allocator_push(allocator, total_size, align);
+        for (U32 i = 0; i < total_size; i += size) {
+            APoolElement *element = (APoolElement *)&start[i];
+            element->next = pool->free_list;
+            pool->free_list = element;
+        }
+        pool->start = start;
+    }
+
+    return pool;
+}
+
+void *apool_alloc(APool *pool)
+{
+    APoolElement *element = pool->free_list;
+    if (element != NULL) {
+        pool->free_list = element->next;
+    } else if(pool->n_elements == APOOL_AUTOGROW) {
+        element = allocator_push(pool->allocator, pool->element_size, pool->align);
+    }
+
+    return element;
+}
+
+void apool_free(APool *pool, void *ptr)
+{
+    if (ptr != NULL) {
+        Assert((U8 *)ptr >= pool->start);
+
+        APoolElement *element = (APoolElement *)ptr;
+        element->next = pool->free_list;
+        pool->free_list = element;
+    }
+}

src/base/pool.h

@@ -0,0 +1,26 @@
+#pragma once
+#include "core.h"
+
+typedef union APoolElement APoolElement;
+union APoolElement {
+    APoolElement *next;       // offset from start to next free block (when not allocated)
+};
+
+typedef struct APool APool;
+struct APool {
+    APoolElement *free_list;  // free list
+    U8 *start;                // start of user buffer
+    Allocator *allocator;
+    U32 element_size;         // aligned size of each block
+    U32 n_elements;           // total number of blocks
+    U32 align;                // alignment
+};
+enum {
+    APOOL_AUTOGROW = 0
+};
+
+#define apool_create(arena, n_elements, type) \
+        apool_create_ex(arena, n_elements, sizeof(type), alignof(type));
+APool *apool_create_ex(Allocator *allocator, U32 n_elements, U32 element_size, uint32_t element_align);
+void *apool_alloc(APool *pool);
+void apool_free(APool *pool, void *element);

src/base/string.c

@@ -0,0 +1,583 @@
+#include "string.h"
+
+Str str_empty(void)
+{
+    static const U8 empty_;
+    static const Str str = {(U8 *)&empty_, 0};
+    return str;
+}
+
+Str str_slice(Str str, U32 start, U32 len)
+{
+    Assert(start <= str.len);
+    Assert(len <= str.len - start);
+
+    Str new_str = str_create(&str.str[start], len);
+    return new_str;
+}
+
+Str str_concat(Allocator *alloc, Str first, Str second)
+{
+    Str str;
+    Assert(first.len <= U32_Max - second.len);
+    str.len = first.len + second.len;
+    str.str = allocator_push(alloc, str.len, 1);;
+
+    MemCopy(str.str, first.str, first.len);
+    MemCopy(&str.str[first.len], second.str, second.len);
+
+    return str;
+}
+
+Str str_push(Allocator *alloc, Str str)
+{
+    U8 *mem = (U8 *)allocator_push(alloc, str.len, 1);
+    MemCopy(mem, str.str, str.len);
+
+    return str_create(mem, str.len);
+}
+
+Str str_ascii_trim_left(Str str)
+{
+    U8 *start = str.str;
+    U8 *end = str.str + str.len;
+    while (start != end) {
+        if (!char_is_ascii_whitespace(*start)) {
+            break;
+        }
+        start++;
+    }
+
+    return str_create(start, (U32)(end - start));
+}
+
+Str str_ascii_trim_right(Str str)
+{
+    U8 *start = str.str;
+    U8 *end = str.str + str.len;
+    while (start != end) {
+        if (!char_is_ascii_whitespace(*(end - 1))) {
+            break;
+        }
+        end--;
+    }
+
+    return str_create(start, (U32)(end - start));
+}
+
+Str str_ascii_trim(Str str)
+{
+    Str new_str = str_ascii_trim_left(str);
+    new_str = str_ascii_trim_right(new_str);
+    return new_str;
+}
+
+S64 str_index(Str haystack, Str needle)
+{
+    if (needle.len == 0) {
+        return 0;
+    }
+
+    if (needle.len == 1) {
+        return str_find_byte(haystack, needle.str[0]);
+    }
+    if (needle.len > haystack.len) {
+        return -1;
+    }
+
+    U8 *found = MemFind(haystack.str, haystack.len, needle.str, needle.len);
+    if (found != NULL) {
+        return (S64)(found - haystack.str);
+    }
+
+    return -1;
+}
+
+
+
+static INLINE S16 char_hex_to_binary(U8 c)
+{
+    if (c >='0' && c <='9')
+        return c - '0';
+    else if (c >= 'A' && c <= 'F')
+        return c - 'A' + 10;
+    else if (c >= 'a' && c <= 'f')
+        return c - 'a' + 10;
+
+    return -1;
+}
+
+static const char hex_chars[] = "0123456789ABCDEF";
+
+Str str_hex_encode(Str binary_data, Str hex_result)
+{
+    Assert(hex_result.len  >= binary_data.len * 2);
+    U32 hex_len = binary_data.len * 2;
+    for (U32 i = 0, t = 0; i < binary_data.len; ++i,t+=2) {
+        U8 high_nibble;
+        U8 low_nibble;
+
+        high_nibble = (binary_data.str[i] & 0xF0) >> 4 ;
+        low_nibble =  binary_data.str[i] & 0x0F ;
+
+        hex_result.str[t]   = hex_chars[high_nibble];
+        hex_result.str[t+1] = hex_chars[low_nibble];
+    }
+
+    hex_result.len = hex_len;
+    return hex_result;
+}
+
+Str str_hex_decode(Str hex_str, Str result)
+{
+    Assert(result.len >= hex_str.len/2);
+    U32 dest_pos = 0;
+    U32 source_pos = 0;
+
+    while (source_pos + 1 < hex_str.len) {
+
+        if (char_is_ascii_whitespace(hex_str.str[source_pos])) {
+            source_pos++;
+            continue;
+        }
+
+        S16 high_nibble;
+        S16 low_nibble;
+
+        high_nibble = char_hex_to_binary(hex_str.str[source_pos]);
+        low_nibble = char_hex_to_binary(hex_str.str[source_pos + 1]);
+        if (high_nibble == -1 || low_nibble == -1) {
+            result.len = 0;
+            return result;
+        }
+
+        result.str[dest_pos] = (high_nibble << 4 ) | low_nibble;
+        source_pos += 2;
+        dest_pos++;
+    }
+
+    result.len = dest_pos;
+    return result;
+}
+// split @str on any byte in @separators, place split strings in @vec
+void str_split(Str str, Str separators, StrVec *vec)
+{
+    if (vec->max_len == 0 || str.len == 0) {
+        return;
+    }
+
+    U32 start_pos = 0;
+    for (U32 i = 0; i < str.len; i++) {
+        U8 byte = str.str[i];
+        if (str_find_byte(separators, byte) != -1) {
+            if (vec->len >= vec->max_len) {
+                break;
+            }
+
+            Str sub_str = str_slice(str, start_pos, i - start_pos);
+            vec->strings[vec->len] = sub_str;
+            vec->len++;
+            start_pos = i + 1;
+        }
+    }
+    if (start_pos <= str.len && vec->len < vec->max_len) {
+        Str sub_str = str_slice(str, start_pos, str.len - start_pos);
+        vec->strings[vec->len] = sub_str;
+        vec->len++;
+    }
+}
+
+Str str_join(Allocator *allocator, StrVec strings, Str separator)
+{
+    U32 joined_str_len = 0;
+
+    for (U32 i = 0; i < strings.len; i++) {
+        U32 part_len = strings.strings[i].len;
+        Assert(joined_str_len <= U32_Max - part_len); // overflow
+        joined_str_len += part_len;
+    }
+    if (strings.len > 1) {
+        U32 separator_count = strings.len - 1;
+        Assert(separator.len <= U32_Max / separator_count); // overflow
+        Assert(joined_str_len <= U32_Max - separator.len * separator_count); // overflow
+        joined_str_len += separator.len * separator_count;
+    }
+
+    U8 *joined_str = allocator_push(allocator, joined_str_len, 1);
+    if (strings.len > 0) {
+        U32 pos = 0;
+        Str *str = &strings.strings[0];
+
+        MemCopy(&joined_str[pos], str->str, str->len);
+        pos += str->len;
+
+        for (U32 i = 1; i < strings.len; i++) {
+            str = &strings.strings[i];
+            MemCopy(&joined_str[pos], separator.str, separator.len);
+            pos += separator.len;
+            MemCopy(&joined_str[pos], str->str, str->len);
+            pos += str->len;
+        }
+    }
+
+    return str_create(joined_str, joined_str_len);
+}
+
+
+
+/*
+ * Each entry in utf8_lead[] is an 8-bit descriptor:
+ *
+ *     7 6 5 4 3 2 1 0
+ *    +---------+-----+
+ *    |  range  | len |
+ *    +---------+-----+
+ *
+ * len:
+ *     Number of bytes in the UTF-8 sequence.
+ *
+ *     0 = byte cannot start a valid UTF-8 sequence
+ *     1 = ASCII
+ *     2 = two-byte UTF-8 sequence
+ *     3 = three-byte UTF-8 sequence
+ *     4 = four-byte UTF-8 sequence
+ *
+ * range:
+ *     Selects the legal range for byte 1, the first continuation byte.
+ *
+ * This range is enough to encode all of UTF-8's special boundary rules:
+ *
+ *     E0 A0..BF    prevents overlong 3-byte encodings
+ *     ED 80..9F    excludes UTF-16 surrogate code points
+ *     F0 90..BF    prevents overlong 4-byte encodings
+ *     F4 80..8F    limits Unicode to U+10FFFF
+ *
+ * All continuation bytes after byte 1 simply use the normal 80..BF range.
+ */
+
+#define UTF8_LENGTH_BITS 3u
+#define UTF8_LENGTH_MASK ((1u << UTF8_LENGTH_BITS) - 1u)
+
+#define UTF8_DESC(length, range) \
+    ((U8)((length) | ((U8)(range) << UTF8_LENGTH_BITS)))
+
+#define UTF8_DESC_LENGTH(desc) \
+    ((U8)((desc) & UTF8_LENGTH_MASK))
+
+#define UTF8_DESC_RANGE(desc) \
+    ((U8)((desc) >> UTF8_LENGTH_BITS))
+
+
+/*
+ * Classes of allowed values for the first continuation byte.
+ */
+enum Utf8RangeClass {
+    /*
+     * No continuation byte exists.
+     *
+     * Used only by ASCII descriptors.
+     */
+    UTF8_RANGE_NONE = 0,
+
+    /*
+     * Normal continuation-byte range.
+     */
+    UTF8_RANGE_80_BF,
+
+    /*
+     * E0 xx xx:
+     *
+     * E0 80..9F would encode values below U+0800 and therefore be
+     * non-canonical overlong UTF-8.
+     */
+    UTF8_RANGE_A0_BF,
+
+    /*
+     * ED xx xx:
+     *
+     * ED A0..BF would encode U+D800..U+DFFF, which are UTF-16 surrogate
+     * code points and are not Unicode scalar values.
+     */
+    UTF8_RANGE_80_9F,
+
+    /*
+     * F0 xx xx xx:
+     *
+     * F0 80..8F would encode values below U+10000 and therefore be
+     * non-canonical overlong UTF-8.
+     */
+    UTF8_RANGE_90_BF,
+
+    /*
+     * F4 xx xx xx:
+     *
+     * F4 90..BF would encode values above Unicode's maximum scalar value,
+     * U+10FFFF.
+     */
+    UTF8_RANGE_80_8F,
+};
+
+
+typedef struct {
+    U8 lo;
+    U8 hi;
+} Utf8Range;
+
+/*
+ * Allowed range of the first continuation byte for each range class.
+ *
+ * The normal continuation-byte interval is 0x80..0xBF.
+ */
+static const Utf8Range utf8_ranges[] = {
+    [UTF8_RANGE_NONE]  = { 0x00, 0x00 },
+    [UTF8_RANGE_80_BF] = { 0x80, 0xBF },
+    [UTF8_RANGE_A0_BF] = { 0xA0, 0xBF },
+    [UTF8_RANGE_80_9F] = { 0x80, 0x9F },
+    [UTF8_RANGE_90_BF] = { 0x90, 0xBF },
+    [UTF8_RANGE_80_8F] = { 0x80, 0x8F },
+};
+
+
+
+/*
+ * Each entry in utf8_lead[] is an 8-bit descriptor:
+ *
+ *     7 6 5 4 3 2 1 0
+ *    +---------+-----+
+ *    |  range  | len |
+ *    +---------+-----+
+ *
+ * len:
+ *     Number of bytes in the UTF-8 sequence.
+ *
+ *     0 = byte cannot start a valid UTF-8 sequence
+ *     1 = ASCII
+ *     2 = two-byte UTF-8 sequence
+ *     3 = three-byte UTF-8 sequence
+ *     4 = four-byte UTF-8 sequence
+ *
+ * range:
+ *     Selects the legal range for byte 1, the first continuation byte.
+ *
+ * This range is enough to encode all of UTF-8's special boundary rules:
+ *
+ *     E0 A0..BF    prevents overlong 3-byte encodings
+ *     ED 80..9F    excludes UTF-16 surrogate code points
+ *     F0 90..BF    prevents overlong 4-byte encodings
+ *     F4 80..8F    limits Unicode to U+10FFFF
+ *
+ * All continuation bytes after byte 1 simply use the normal 80..BF range.
+ */
+/*
+ * Classification of every possible first byte.
+ *
+ * The zero-initialized entries are invalid UTF-8 leading bytes:
+ *
+ *     80..BF  continuation bytes cannot start a sequence
+ *     C0..C1  would produce overlong two-byte encodings
+ *     F5..FF  would encode values above U+10FFFF or are otherwise invalid
+ *
+ * Valid leading-byte ranges:
+ *
+ *     00..7F  ASCII
+ *     C2..DF  2 bytes
+ *     E0..EF  3 bytes
+ *     F0..F4  4 bytes
+ *
+ * E0, ED, F0, and F4 receive special range classes because their second
+ * byte is constrained beyond the normal 80..BF continuation-byte range.
+ *
+ */
+static const U8 utf8_lead[256] = {
+    [0x00 ... 0x7F] = UTF8_DESC(1, UTF8_RANGE_NONE),
+
+    [0xC2 ... 0xDF] = UTF8_DESC(2, UTF8_RANGE_80_BF),
+
+    [0xE0]          = UTF8_DESC(3, UTF8_RANGE_A0_BF),
+
+    [0xE1 ... 0xEC] = UTF8_DESC(3, UTF8_RANGE_80_BF),
+
+    [0xED]          = UTF8_DESC(3, UTF8_RANGE_80_9F),
+
+    [0xEE ... 0xEF] = UTF8_DESC(3, UTF8_RANGE_80_BF),
+
+    [0xF0]          = UTF8_DESC(4, UTF8_RANGE_90_BF),
+
+    [0xF1 ... 0xF3] = UTF8_DESC(4, UTF8_RANGE_80_BF),
+
+    [0xF4]          = UTF8_DESC(4, UTF8_RANGE_80_8F),
+};
+
+/*
+ * Decode one Unicode scalar value from UTF-8.
+ *
+ * Valid input:
+ *
+ *     returns the decoded Rune
+ *     valid = true
+ *     width = number of UTF-8 bytes consumed
+ *
+ * Invalid input:
+ *
+ *     returns U+FFFD
+ *     valid = false
+ *     width = 1
+ *
+ * Consuming one byte on malformed input guarantees forward progress and
+ * allows decoding to resume immediately after the offending byte.
+ *
+ * End of input:
+ *
+ *     returns RUNE_EOF
+ *     valid = false
+ *     width = 0
+ */
+RuneDecode _str_utf8_decode_full(Str str, U32 offset)
+{
+    if (str.len == 0 || str.len <= offset) {
+        return (RuneDecode) {
+            .rune  = RUNE_EOF,
+            .width = 0,
+            .valid = false,
+        };
+    }
+    U8 b0 = str.str[offset];
+    U8 desc = utf8_lead[b0];
+    U8 width = UTF8_DESC_LENGTH(desc);
+    /*
+     * width == 0 covers:
+     *
+     *     80..BF
+     *     C0..C1
+     *     F5..FF
+     */
+    if (width == 0)
+        goto invalid;
+
+    /*
+     * We must have the complete candidate sequence before accessing any
+     * continuation bytes below.
+     */
+    if (str.len - offset < width)
+        goto invalid;
+
+    /*
+     * Validate the first continuation byte.
+     *
+     * Its descriptor-selected range handles both the normal continuation
+     * range and the E0 / ED / F0 / F4 boundary cases.
+     */
+    Utf8Range range = utf8_ranges[UTF8_DESC_RANGE(desc)];
+    U8 b1 = str.str[offset + 1];
+
+    if (b1 < range.lo || b1 > range.hi)
+        goto invalid;
+
+    /*
+     * Decode paths are intentionally unrolled.
+     *
+     * Besides avoiding a loop in this hot function, each case combines
+     * continuation-byte validation with scalar construction. The compiler
+     * therefore sees fixed-width operations for every UTF-8 length.
+     */
+    switch (width) {
+    case 2: {
+        /*
+         * 110xxxxx 10xxxxxx
+         *
+         * C2..DF in the lead table already guarantees:
+         *
+         *     - exactly a legal two-byte leader
+         *     - no C0/C1 overlong encoding
+         *
+         * b1 has already been checked against 80..BF.
+         */
+        Rune rune =
+            ((Rune)(b0 & 0x1F) << 6) |
+             (Rune)(b1 & 0x3F);
+
+        return (RuneDecode) {
+            .rune  = rune,
+            .width = 2,
+            .valid = true,
+        };
+    }
+
+    case 3: {
+        /*
+         * 1110xxxx 10xxxxxx 10xxxxxx
+         *
+         * b1 has already been validated using the descriptor-specific
+         * range. Only b2 still needs a continuation-byte check.
+         */
+        U8 b2 = str.str[offset + 2];
+
+        if ((b2 & 0xC0) != 0x80)
+            goto invalid;
+
+        Rune rune =
+            ((Rune)(b0 & 0x0F) << 12) |
+            ((Rune)(b1 & 0x3F) <<  6) |
+             (Rune)(b2 & 0x3F);
+
+        return (RuneDecode) {
+            .rune  = rune,
+            .width = 3,
+            .valid = true,
+        };
+    }
+
+    case 4: {
+        /*
+         * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+         *
+         * b1 has already been validated using the descriptor-specific
+         * range. b2 and b3 use the ordinary continuation-byte form.
+         */
+        U8 b2 = str.str[offset + 2];
+        U8 b3 = str.str[offset + 3];
+
+        /*
+         * Validate both remaining continuation bytes with one expression.
+         *
+         * For a valid continuation byte:
+         *
+         *     byte & 0xC0 == 0x80
+         *
+         * XOR with 0x80 therefore gives zero in the high two bits.
+         * ORing the two transformed bytes lets us test both at once.
+         */
+        if ((((b2 ^ 0x80) | (b3 ^ 0x80)) & 0xC0) != 0)
+            goto invalid;
+
+        Rune rune =
+            ((Rune)(b0 & 0x07) << 18) |
+            ((Rune)(b1 & 0x3F) << 12) |
+            ((Rune)(b2 & 0x3F) <<  6) |
+             (Rune)(b3 & 0x3F);
+
+        return (RuneDecode) {
+            .rune  = rune,
+            .width = 4,
+            .valid = true,
+        };
+    }
+
+    default:
+        /*
+         * The descriptor table can only produce widths 0..4.
+         * Width 1 was handled by the ASCII fast path.
+         */
+        goto invalid;
+    }
+
+
+invalid:
+    return (RuneDecode) {
+        .rune  = RUNE_ERROR,
+        .width = 1,
+        .valid = false,
+    };
+}
+
+

src/base/string.h

@@ -0,0 +1,295 @@
+#pragma once
+#include "core.h"
+
+typedef struct Str Str;
+struct Str {
+    U8 *str;
+    U32 len;
+};
+
+Str str_empty(void);
+Str str_slice(Str str, U32 start, U32 len);
+// returns -1 if not found, or index start of first needle
+S64 str_index(Str haystack, Str needle);
+
+Str str_concat(Allocator *alloc, Str first, Str second);
+Str str_push(Allocator *alloc, Str str);
+Str str_ascii_trim(Str str);
+Str str_ascii_trim_left(Str str);
+Str str_ascii_trim_right(Str str);
+
+
+#define  str_from_lit(str) str_create(((U8 *)str), sizeof(str) -1)
+#define  S(str) str_from_lit(str)
+#define cstr_len(str) (U32)strlen(str)
+#define str_dup(alloc, str) str_push((alloc), (str))
+
+static INLINE bool str_eq(Str first, Str second)
+{
+    if (first.len != second.len) {
+        return false;
+    }
+    return MemCompare(first.str, second.str, first.len) == 0;
+}
+
+static INLINE bool str_is_empty(Str str)
+{
+    return str.len == 0;
+}
+
+static INLINE Str str_create(U8 *start, U32 len)
+{
+    Str str = {start, len};
+    return str;
+}
+
+static INLINE Str str_from_cstr(const char *cstr)
+{
+    Str str = str_create((U8 *)cstr, cstr_len(cstr));
+    return str;
+}
+
+static INLINE Str str_push_cstr(Allocator *alloc, const char *cstr)
+{
+    Str str = str_from_cstr(cstr);
+    return str_push(alloc, str);
+}
+
+static INLINE S64 str_find_byte(Str str, U8 byte)
+{
+    for (U32 i = 0; i < str.len; i++) {
+        if (str.str[i] == byte) {
+            return i;
+        }
+    }
+
+    return -1;
+}
+
+static INLINE Str str_replace_byte(Str str, U8 orig_byte, U8 new_byte)
+{
+    for (U32 i = 0; i < str.len; i++) {
+        if (str.str[i] == orig_byte) {
+            str.str[i] = new_byte;
+        }
+    }
+
+    return str;
+}
+
+static INLINE bool str_starts_with(Str str, Str prefix)
+{
+    if (prefix.len > str.len) {
+        return false;
+    }
+
+    return MemCompare(str.str, prefix.str, prefix.len) == 0;
+}
+
+static INLINE bool str_ends_with(Str str, Str suffix)
+{
+    if (suffix.len > str.len) {
+        return false;
+    }
+    U32 start = str.len - suffix.len;
+    return MemCompare(&str.str[start], suffix.str, suffix.len) == 0;
+}
+
+static INLINE Str str_skip(Str str, U32 len)
+{
+    U32 skip_len = Min(str.len, len);
+    Str new_str = str_slice(str, skip_len, str.len - skip_len);
+
+    return new_str;
+}
+
+// hex_result must be at least source.len * 2. Returns hex_result.
+Str str_hex_encode(Str binary_data, Str hex_result);
+// result must be at least source.len / 2. Returns result.
+Str str_hex_decode(Str hex_str, Str result);
+
+static INLINE Str str_hex_encode_alloc(Allocator *allocator, Str binary_data)
+{
+    Assert(binary_data.len <= U32_Max / 2);
+    U8 hex_len = binary_data.len * 2;
+    U8 *result = allocator_push_array(allocator, U8, hex_len);
+    Str destination = str_create(result, hex_len);
+
+    return str_hex_encode(binary_data, destination);
+}
+
+static INLINE Str str_hex_decode_alloc(Allocator *allocator, Str hex_str)
+{
+    U8 binary_len = hex_str.len / 2;
+    U8 *result = allocator_push_array(allocator, U8, binary_len);
+    Str destination = str_create(result, binary_len);
+
+    return str_hex_decode(hex_str, destination);
+}
+// String Vector
+typedef struct StrVec StrVec;
+struct StrVec {
+    Str *strings;
+    U32 len;
+    U32 max_len;
+};
+#define STRVEC_STATIC_INITIALIZER(strings_) STRVEC_INITIALIZER((strings_), ArraySize(strings_), ArraySize(strings_))
+#define STRVEC_STATIC_INITIALIZER_EMPTY(strings_) STRVEC_INITIALIZER((strings_), 0, ArraySize(strings_))
+#define STRVEC_INITIALIZER(strings_, len_, max_len_) {.strings = (strings_), .len = (len_), .max_len = (max_len_)}
+
+#define STRVEC_DEFINE(var_name, len_)  \
+    struct { \
+        Str strings[len_];\
+    } var_name##_strvec_array; \
+    StrVec var_name = STRVEC_STATIC_INITIALIZER_EMPTY(var_name##_strvec_array.strings)
+
+void str_split(Str str, Str separators, StrVec *vec);
+Str str_join(Allocator *allocator, StrVec strings, Str separator);
+
+
+// Characters
+
+static INLINE bool char_is_ascii_whitespace(U8 byte)
+{
+    return byte == ' ' || byte == '\t' || byte == '\v' || byte == '\f' || byte == '\r' || byte == '\n';
+}
+
+static INLINE bool char_is_digit(U8 byte)
+{
+    return byte >= '0' && byte <= '9';
+}
+
+
+// UTF-8/Unicode
+
+typedef S32 Rune;
+
+
+/*
+ * Unicode replacement character.
+ *
+ * Returned for malformed UTF-8
+ */
+#define RUNE_ERROR ((Rune)0xFFFD)
+
+#define RUNE_EOF ((Rune)-1)
+#define RUNE_INVALID (-1)
+
+// Maximum valid Unicode code point.
+#define RUNE_MAX  0x10FFFF
+#define RUNE_SURROGATE_MIN 0xD800
+#define RUNE_SURROGATE_MAX 0xDFFF
+#define RUNE_1_MAX ((1 <<  7)- 1)
+#define RUNE_2_MAX ((1 << 11)- 1)
+#define RUNE_3_MAX ((1 << 16)- 1)
+
+typedef struct {
+    Rune rune;
+
+    /*
+     * Number of input bytes consumed.
+     *
+     *   valid UTF-8 : 1..4
+     *   invalid     : 1
+     *   end of input: 0
+     */
+    U8 width;
+
+    /*
+     * True only when rune was decoded from valid UTF-8.
+     *
+     *  U+FFFD itself may legitimately occur in  valid
+     *  input, so rune == RUNE_ERROR does not imply an error.
+     */
+    bool valid;
+} RuneDecode;
+
+// Helper:
+RuneDecode _str_utf8_decode_full(Str str, U32 offset);
+/*
+ * Decode one Unicode scalar value from UTF-8.
+ *
+ * Valid input:
+ *
+ *     returns the decoded Rune
+ *     valid = true
+ *     width = number of UTF-8 bytes consumed
+ *
+ * Invalid input:
+ *
+ *     returns U+FFFD
+ *     valid = false
+ *     width = 1
+ *
+ * Consuming one byte on malformed input guarantees forward progress and
+ * allows decoding to resume immediately after the offending byte.
+ *
+ * End of input:
+ *
+ *     returns RUNE_EOF
+ *     valid = false
+ *     width = 0
+ */
+static INLINE RuneDecode str_utf8_decode(Str str, U32 offset)
+{
+    if (str.len == 0 || str.len <= offset) {
+        return (RuneDecode) {
+            .rune  = RUNE_EOF,
+            .width = 0,
+            .valid = false,
+        };
+    }
+
+    U8 b0 = str.str[offset];
+
+    /*
+     * Testing b0 directly also avoids consulting utf8_ranges[] for the
+     * overwhelmingly common single-byte/ascii case.
+     */
+    if (b0 <= RUNE_1_MAX) {
+        return (RuneDecode) {
+            .rune  = (Rune)b0,
+            .width = 1,
+            .valid = true,
+        };
+    }
+
+    return _str_utf8_decode_full(str, offset);
+}
+
+// Length of bytes to encode the Rune in utf-8.
+// returns -1 if the Rune is not a valid value to encode in UTF-8.
+static INLINE S32 rune_utf8_len(Rune rune) {
+
+	if (rune < 0) {
+		return RUNE_INVALID;
+    } else if (rune <= RUNE_1_MAX) {
+		return 1;
+	} else if (rune <= RUNE_2_MAX) {
+		return 2;
+	} else if (rune >= RUNE_SURROGATE_MIN && rune <= RUNE_SURROGATE_MAX) {
+		return RUNE_INVALID;
+	} else if (rune <= RUNE_3_MAX) {
+		return 3;
+	} else if (rune <= RUNE_MAX) {
+		return 4;
+	}
+
+	return RUNE_INVALID;
+}
+
+// Number of unicode Runes in str. 1 invalid UTF-8 byte is counted as 1 Rune.
+static INLINE U32 str_rune_count(Str str)
+{
+    U32 offset = 0;
+    U32 runes = 0;
+
+    RuneDecode d = str_utf8_decode(str, offset);
+    while (d.rune != RUNE_EOF) {
+        offset += d.width;
+        runes++;
+        d = str_utf8_decode(str, offset);
+    }
+
+    return runes;
+}

src/base_core.h

@@ -1,319 +0,0 @@
-#pragma once
-
-#include <stdalign.h>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <string.h>
-
-#ifdef __has_feature
-    #if __has_feature(address_sanitizer)
-    #define USE_ASAN 1
-    #endif
-#endif
-#if defined(__SANITIZE_ADDRESS__)
-    #define USE_ASAN 1
-#endif
-
-// Types
-typedef int8_t      S8;
-typedef int16_t     S16;
-typedef int32_t     S32;
-typedef int64_t     S64;
-typedef uint8_t     U8;
-typedef uint16_t    U16;
-typedef uint32_t    U32;
-typedef uint64_t    U64;
-typedef float       F32;
-typedef double      F64;
-#if defined(__FLT128_MANT_DIG__)
-    typedef _Float128 F128;
-#elif defined(__SIZEOF_FLOAT128__)
-    typedef __float128 F128;
-#endif
-#ifdef __SIZEOF_INT128__
-typedef __int128          S128;
-typedef unsigned __int128 U128;
-#endif
-
-#define  U8_Max     ((U8) 0xff)
-#define U16_Max     ((U16)0xffff)
-#define U32_Max     ((U32)0xffffffff)
-#define U64_Max     ((U64)0xffffffffffffffffULL)
-
-// Duh ..
-#define  U8_Min     ((U8)0x0)
-#define U16_Min     ((U16)0x0)
-#define U32_Min     ((U32)0x0)
-#define U64_Min     ((U64)0x0)
-
-#define  S8_Max     ((S8) 0x7f)
-#define S16_Max     ((S16)0x7fff)
-#define S32_Max     ((S32)0x7fffffff)
-#define S64_Max     ((S64)0x7fffffffffffffffLL)
-
-#define  S8_Min     ((S8) 0x80)
-#define S16_Min     ((S16)0x8000)
-#define S32_Min     ((S32)0x80000000)
-#define S64_Min     ((S64)0x8000000000000000LL)
-// asserts
-
-#define Trap() __builtin_trap()
-#define Abort() __builtin_abort()
-
-
-#define Assert(x)       do { if (!(x)) { Trap(); }} while(0)
-#define NotImplemented  Assert(!"Not Implemented")
-#define ShouldNotHappen  Assert(!"This should not happen")
-
-#define StaticAssert(expr) typedef int STATIC_ASSERT_FAILED[(expr) ? 1 : -1]
-
-// misc utilities
-#define Max(a, b)             \
-  ({                          \
-    __typeof__(a) _amx = (a); \
-    __typeof__(b) _bmx = (b); \
-    _amx > _bmx ? _amx : _bmx;\
-  })
-
-#define Min(a, b)              \
-  ({                           \
-    __typeof__(a) _amn = (a);  \
-    __typeof__(b) _bmn = (b);  \
-    _amn < _bmn ? _amn : _bmn; \
-  })
-
-#define ArraySize(a) (sizeof(a) / sizeof((a)[0]))
-
-#define Swap(a, b)            \
-  do {                        \
-    __typeof__(a) tmp_ = (a); \
-    (a) = (b);                \
-    (b) = tmp_;               \
-  } while (0)
-
-#define ContainerOf(ptr, type, member)                                  \
-  ({                                                                    \
-    typeof(((type *)0)->member) *__mptr = (ptr);                        \
-    (type *)((void *)(unsigned char *)__mptr - offsetof(type, member)); \
-  })
-
-#define ConstContainerOf(ptr, type, member)                           \
-  ({                                                                  \
-    const typeof(((const type *)0)->member) *__mptr = (ptr);          \
-    (const type *)((const void *)(const unsigned char *)__mptr - offsetof(const type, member));                     \
-  })
-
-// align needs to be pow2
-#define AlignUp(val, align)     (((val) + (align) - 1UL) & ~((align) - 1UL))
-#define AlignDown(val, align)   ((val) & ~((align) - 1UL))
-#define IsPow2(x)                   ((x) && !((x) & ((x) - 1UL)))
-#define IsPow2OrZero(x)             (!((x) & ((x) - 1UL)))
-#define IsAlignedTo(x, align)           (IsPow2(align) && !((x) & ((align) - 1UL)))
-
-#define GetBit(val, idx) (((val) >> (idx)) & 1)
-// Get n_bits starting at idx going left
-#define GetBits(val, n_bits, idx) \
-  (((n_bits) == 0 ? 0 : ((value) >> (idx)) & ((((__typeof__(val))(1) << (n_bits)) - 1)))
-
-#define STRINGIFY(x) STRINGIFY_HLP(x)
-#define STRINGIFY_HLP(x) #x
-
-#define SRC_LOCATION __FILE__ ":" STRINGIFY(__LINE__)
-
-#ifdef DEBUG
-#define TRACEF(fmt, ...)                                         \
-  do {                                                           \
-    fprintf(stdout, "%s (%s)\t" fmt, SRC_LOCATION, __FUNCTION__, \
-            ##__VA_ARGS__);                                      \
-    fflush(stdout);                                              \
-  } while (0)
-#else
-#define TRACEF(fmt, ...) \
-  do {                   \
-  } while (0)
-#endif
-
-/**
- * Ceiling of X / y , or  round up x / y
- * (e.g 100/9 == 12, 3/2 == 2)
- * x + y must not overflow.
- *
- * @param x numerator, must be positive
- * @param y denominator
- * @return x/y rounded up
- */
-#define CeilDiv(x, y) (((x) + ((y) - 1)) / (y))
-
-/**
- * Round x up to nearest multiple of y
- * e.g. RoundUp(30,48) == 48
- * e.g. RoundUp(49,48) == 96
- *
- * @param x numerator, must be positive
- * @param y denominator
- * @return x rounded up to nearest multiple of y
- */
-#define RoundUpToMultiple(x, y) (CeilDiv((x), (y)) * (y))
-
-/**
- * Round x down to nearest multiple of y
- * e.g. RoundDown(30,48) == 0
- * e.g. RoundDown(49,48) == 48
- *
- * @param x numerator, must be positive
- * @param y denominator
- * @return x rounded down nearest multiple of y
- */
-#define RoundDownToMultiple(x, y) ((x) / (y) * (y))
-#define Clamp(val, min, max) (((val) < (min)) ? (min) : ((val) > (max)) ? (max) : (val))
-
-#ifdef __GNUC__
-#define INLINE inline __attribute__((always_inline, flatten))
-#else
-#define INLINE inline
-#endif
-
-#ifdef __cplusplus
-#define C_LINKAGE_BEGIN extern "C" {
-#define C_LINKAGE_END }
-#define C_LINKAGE extern "C"
-#else
-#define C_LINKAGE_BEGIN
-#define C_LINKAGE_END
-#define C_LINKAGE
-#endif
-
-/** Indicate the variable is unused.
- */
-#define UNUSED(x) x __attribute__((unused))
-
-// sizes
-#define Bytes(n) ((U64)(n))
-#define KB(n)    (((U64)(n)) << 10)
-#define MB(n)    (((U64)(n)) << 20)
-#define GB(n)    (((U64)(n)) << 30)
-#define TB(n)    (((U64)(n)) << 40)
-
-#define Thousand(n) (((U64)(n) * 1000))
-#define Million(n)  (((U64)(n) * 1000000))
-#define Billion(n)  (((U64)(n) * 1000000000))
-
-// memory
-#define MemSet(ptr, val, len)       memset(ptr, val, len)
-#ifdef USE_ASAN
-#define MemoryZero(ptr, len)        memset(ptr, 0, len)
-// explicity bzero is usually not being intercepted
-#else
-#define MemoryZero(ptr, len)        explicit_bzero(ptr, len)
-#endif
-#define MemMove(dst, src, len)      memmove(dst, src, len)
-#define MemCopy(dst, src, len)       memcpy(dst, src, len)
-#define MemAppend(dst, src, len)    mempcpy(dst, src, len)
-
-// atomics
-#define Atomic(type) _Atomic(type)
-#define AtomicSub(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define AtomicAdd(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define AtomicDec(ptr) AtomicSub((ptr), 1)
-#define AtomicInc(ptr) AtomicAdd((ptr), 1)
-#define AtomicGet(ptr) __c11_atomic_load((ptr), __ATOMIC_SEQ_CST)
-#define AtomicCAS(ptr, oldval, newval) \
-    __atomic_compare_exchange((ptr), (oldval), (newval),0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
-
-#define MemoryBarrier() __atomic_thread_fence(UC_ORDER_CST)
-#define CompilerBarrier() __asm__ __volatile__("": : :"memory")
-
-// Saturating operations
-#define SaturatingAdd(a, b, type, max_val)\
-    ({type res ;\
-    if (__builtin_add_overflow(a, b, &res)) {\
-        res = max_val;}\
-    res;})
-
-
-#define SaturatingSub(a, b, type)\
-    ({type res ;\
-    if (__builtin_sub_overflow(a, b, &res)) {\
-        res = 0;}\
-    res;})
-
-#define SaturatingMult(a, b, type, max_val)\
-    ({type res ;\
-    if (__builtin_mul_overflow(a, b, &res)) {\
-        res = max_val;}\
-    res;})
-
-static INLINE U8 SaturatingAddU8(U8 a, U8 b)
-{
-    return SaturatingAdd(a, b, U8, U8_Max);
-}
-
-static INLINE U8 SaturatingSubU8(U8 a, U8 b)
-{
-    return SaturatingSub(a, b, U8);
-}
-
-static INLINE U8 SaturatingMult8(U8 a, U8 b)
-{
-    return SaturatingMult(a, b, U8, U8_Max);
-}
-
-static INLINE U16 SaturatingAdd16(U16 a, U16 b)
-{
-    return SaturatingAdd(a, b, U16, U16_Max);
-}
-
-static INLINE U16 SaturatingSubU16(U16 a, U16 b)
-{
-    return SaturatingSub(a, b, U16);
-}
-
-static INLINE U16 SaturatingMultU16(U16 a, U16 b)
-{
-    return SaturatingMult(a, b, U16, U16_Max);
-}
-
-static INLINE U32 SaturatingAdd32(U32 a, U32 b)
-{
-    return SaturatingAdd(a, b, U32, U32_Max);
-}
-
-static INLINE U32 SaturatingSub32(U32 a, U32 b)
-{
-    return SaturatingSub(a, b, U32);
-}
-
-static INLINE U32 SaturatingMult32(U32 a, U32 b)
-{
-    return SaturatingMult(a, b, U32, U32_Max);
-}
-
-static INLINE U64 SaturatingAdd64(U64 a, U64 b)
-{
-    return SaturatingAdd(a, b, U64, U64_Max);
-}
-
-static INLINE U64 SaturatingSubU64(U64 a, U64 b)
-{
-    return SaturatingSub(a, b, U64);
-}
-
-static INLINE U64 SaturatingMult64(U64 a, U64 b)
-{
-    return SaturatingMult(a, b, U64, U64_Max);
-}
-
-// address-sanitizer helpers (-fsanitize=address)
-#ifdef USE_ASAN
-#include <sanitizer/asan_interface.h>
-#define ASAN_POISON_MEMORY_REGION(addr, size) \
-  __asan_poison_memory_region((addr), (size))
-#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
-  __asan_unpoison_memory_region((addr), (size))
-#else
-#define ASAN_POISON_MEMORY_REGION(addr, size) \
-  ((void)(addr), (void)(size))
-#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
-  ((void)(addr), (void)(size))
-#endif

src/main.c

@@ -1,6 +1,9 @@
-#include "base_core.h"
-#include "arena.h"
-#include "platform.h"
+#include "base/core.h"
+#include "base/arena.h"
+#include "base/marena.h"
+#include "base/platform.h"
+#include "base/pool.h"
+#include "base/string.h"
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -67,7 +70,7 @@ static void arena_test_2(void)
     printf("Allocated ptr %p\n", ptr);
 
     arena_dump(&arena);
-    ArenaCheckpoint checkpoint = arena_temp_begin(&arena);
+    ATemp checkpoint = arena_temp_begin(&arena);
     ptr = arena_push(&arena, MB(4)+3, 8);
     MemoryZero(ptr, MB(2));
 
@@ -76,7 +79,7 @@ static void arena_test_2(void)
     ptr = arena_push(&arena, commit_size , 8);
     MemoryZero(ptr, commit_size  );
 
-    ArenaCheckpoint checkpoint2 = arena_temp_begin(&arena);
+    ATemp checkpoint2 = arena_temp_begin(&arena);
     ptr = arena_push(&arena, 64, 8);
     MemoryZero(ptr, 64);
 
@@ -91,12 +94,12 @@ static void arena_test_2(void)
     ((U8*)ptr)[0] = 0x73;
     ptr = arena_push(&arena, 0, KB(4));
     //((U8*)ptr)[0] = 0x73;
-    arena_reset_to(&arena, 1089);
+  //  arena_reset_to(&arena, 1089);
     arena_dump(&arena);
     arena_reset_to(&arena, 0);
-    for (U64 i = 0; i < KB(1); i++) {
-        ptr = arena_push(&arena, MB(1)  + 3, 8);
-        MemoryZero(ptr, MB(1));
+    for (U64 i = 0; i < MB(4); i++) {
+        ptr = arena_push(&arena,  3, 1);
+        MemoryZero(ptr, 3);
     }
     arena_free(&arena);
 }
@@ -162,11 +165,534 @@ static void arena_test_4(void)
     arena_free(&arena);
 }
 
-int main(int argc, char *argv[])
+static void arena_test_allocator_interface(void)
 {
-    // arena_test_1();
+    TRACEF("\n");
+
+    U32 page_size = platform_page_size();
+    ArenaOptions arena_params = {};
+    U32 arena_size  = MB(1) + MemoryBlock_Header_Size;
+    U32 commit_size = KB(64);
+    arena_params.size = arena_size;
+    arena_params.commit_size = commit_size;
+    Arena arena;
+    arena_init(&arena, &arena_params);
+    Allocator allocator = arena_allocator(&arena);
+
+    // block_size is the page-aligned arena_size. Request nearly the whole block
+    // but with page alignment, so the alignment gap (a full page) forces a grow.
+    U32 block_size = arena.block_size;
+    U32 push_size  = block_size - MemoryBlock_Header_Size;
+
+    void *ptr = allocator_push(&allocator, push_size, page_size);
+    Assert(IsAlignedTo((uintptr_t)ptr, page_size));
+    MemoryZero(ptr, push_size);
+    printf("test_4: allocated %u bytes page-aligned at %p\n", push_size, ptr);
+    arena_dump(&arena);
+
+    // A second page-aligned push to make sure the cursor math still holds.
+    ptr = allocator_push(&allocator, page_size, page_size);
+    Assert(IsAlignedTo((uintptr_t)ptr, page_size));
+    MemoryZero(ptr, page_size);
+    printf("test_4: allocated %u bytes page-aligned at %p\n", page_size, ptr);
+
+    allocator_free_all(&allocator);
+    arena_free(&arena);
+
+}
+
+static void arena_test_decommit(void)
+{
+    TRACEF("\n");
+
+    U32 page_size = platform_page_size();
+    ArenaOptions arena_params = {};
+    U32 arena_size  = MB(1) + MemoryBlock_Header_Size;
+    U32 commit_size = KB(64);
+    arena_params.size = arena_size;
+    arena_params.commit_size = commit_size;
+    Arena arena;
+    arena_init(&arena, &arena_params);
+    void *ptr = arena_push(&arena, commit_size * 10, 8);
+    MemoryZero(ptr, commit_size * 10);
+    arena_reset(&arena);
+
+    void *pre_ptr = arena_push(&arena, 5, 4);
+
+    ATemp tmp = arena_temp_begin(&arena);
+    ptr = arena_push(&arena, commit_size * 2 + 13, 8);
+    MemoryZero(ptr, commit_size * 2 + 13);
+    arena_temp_end(&tmp);
+    MemoryZero(pre_ptr, 5);
+
+    arena_free(&arena);
+}
+
+void miniarena_test1(void)
+{
+    TRACEF("\n");
+
+    #define  arena_size  4096
+    MARENA_DEFINE(arena, arena_size, 8);
+    int i;
+    for (i = 0; ; i++) {
+        void *ptr = marena_push(&arena, 1);
+        if (ptr == NULL) {
+            break;
+        }
+    }
+
+    printf("miniarena_test1,size %u pushed %d bytes first_free %u\n", arena.size, i, arena.first_free);
+    marena_reset(&arena);
+    MTemp checkpoint = marena_temp_begin(&arena);
+
+    void *ptr = marena_push(&arena, arena_size);
+    Assert(ptr != NULL);
+    Assert(arena.first_free == arena_size);
+    MemoryZero(ptr, arena_size);
+    marena_temp_end(&checkpoint);
+    Assert(arena.first_free == 0);
+    U32 *val = marena_push_type(&arena, U32);
+    *val = 0xa0b0c0d0;
+    printf("marena val 0x%x\n", *val);
+
+    printf("miniarena_test1,size %u bytes first_free %u\n", arena.size, arena.first_free);
+    Assert(arena.first_free == sizeof(U32));
+}
+
+typedef struct PoolTest PoolTest;
+struct PoolTest{
+    U32 n;
+    PoolTest *next;
+    char name[12];
+};
+
+void apool_test(void)
+{
+    TRACEF("\n");
+
+    Arena arena;
+    arena_init(&arena, &ArenaOptions_Default);
+    Allocator allocator = arena_allocator(&arena);
+    APool *pool = apool_create_ex(&allocator, 16, sizeof(PoolTest), alignof(PoolTest));
+    PoolTest *list = NULL;
+    for (U32 i = 0; i < U32_Max; i++) {
+        PoolTest *pt = apool_alloc(pool);
+        if (pt == NULL){
+            break;
+        }
+        pt->n = i;
+        pt->next = list;
+        list = pt;
+    }
+    for (PoolTest *p = list; p; p = p->next){
+        printf("Pooltest %u\n",p->n);
+    }
+
+    PoolTest *next;
+    U32 count = 0;
+    for (PoolTest *p = list; count < 7 && p; p = next){
+        count++;
+        next = p->next;
+        printf("Pooltest freeing %u\n",p->n);
+        apool_free(pool, p);
+    }
+    list = NULL;
+    for (U32 i = 0; i < U32_Max; i++) {
+        PoolTest *pt = apool_alloc(pool);
+        if (pt == NULL){
+            break;
+        }
+        pt->n = i;
+        pt->next = list;
+        list = pt;
+    }
+    for (PoolTest *p = list; p; p = p->next){
+        printf("Pooltest %u\n",p->n);
+    }
+    allocator_free_all(&allocator);
+    arena_free(&arena);
+}
+
+void apool_test_miniarena(void)
+{
+    TRACEF("\n");
+
+    MArena arena;
+    U8 data[4096];
+    marena_init(&arena, data, sizeof data);
+    Allocator allocator = marena_allocator(&arena);
+    APool *pool = apool_create_ex(&allocator, 16, sizeof(PoolTest), alignof(PoolTest));
+    PoolTest *list = NULL;
+    for (U32 i = 0; i < U32_Max; i++) {
+        PoolTest *pt = apool_alloc(pool);
+        if (pt == NULL){
+            break;
+        }
+        pt->n = i;
+        pt->next = list;
+        list = pt;
+    }
+    for (PoolTest *p = list; p; p = p->next){
+        printf("Pooltest %u\n",p->n);
+    }
+
+    PoolTest *next;
+    U32 count = 0;
+    for (PoolTest *p = list; count < 7 && p; p = next){
+        count++;
+        next = p->next;
+        printf("Pooltest freeing %u\n",p->n);
+        apool_free(pool, p);
+    }
+    list = NULL;
+    for (U32 i = 0; i < U32_Max; i++) {
+        PoolTest *pt = apool_alloc(pool);
+        if (pt == NULL){
+            break;
+        }
+        pt->n = i;
+        pt->next = list;
+        list = pt;
+    }
+    for (PoolTest *p = list; p; p = p->next){
+        printf("Pooltest %u\n",p->n);
+    }
+    allocator_free_all(&allocator);
+}
+
+
+void apool_test_autogrow(void)
+{
+    TRACEF("\n");
+
+    Arena arena;
+    arena_init(&arena, &ArenaOptions_Default);
+    Allocator allocator = arena_allocator(&arena);
+
+    APool *pool = apool_create(&allocator, APOOL_AUTOGROW, PoolTest);
+    PoolTest *list = NULL;
+    for (U32 i = 0; i < 4; i++) {
+        PoolTest *pt = apool_alloc(pool);
+        if (pt == NULL){
+            break;
+        }
+        pt->n = i;
+        pt->next = list;
+        list = pt;
+    }
+    for (PoolTest *p = list; p; p = p->next){
+        printf("Pooltest %u\n",p->n);
+    }
+
+    PoolTest *next;
+    U32 count = 0;
+    for (PoolTest *p = list; count < 3 && p; p = next){
+        count++;
+        next = p->next;
+        printf("Pooltest freeing %u\n",p->n);
+        apool_free(pool, p);
+    }
+    list = NULL;
+    for (U32 i = 0; i < 5; i++) {
+        PoolTest *pt = apool_alloc(pool);
+        if (pt == NULL){
+            break;
+        }
+        pt->n = i;
+        pt->next = list;
+        list = pt;
+    }
+    for (PoolTest *p = list; p; p = p->next){
+        printf("Pooltest %u\n",p->n);
+    }
+
+    allocator_free_all(&allocator);
+    arena_free(&arena);
+
+}
+
+void string_test_1(void)
+{
+    TRACEF("\n");
+
+    Str str1 = S("Hello");
+    printf("str1: %.*s len %d\n", str1.len, str1.str, str1.len);
+    U64 str1_index_of_l = str_find_byte(str1, 'l');
+    U64 str1_index_of_g = str_find_byte(str1, 'g');
+    printf("str1: index of l %ld, index of g: %ld\n", str1_index_of_l, str1_index_of_g);
+
+    MARENA_DEFINE(arena, 512, 1);
+    Allocator alloc = marena_allocator(&arena);
+    Str str2 = str_dup(&alloc, S(" World"));
+    Str str3 = str_concat(&alloc, str1, str2);
+    printf("str3: %.*s len %d\n", str3.len, str3.str, str3.len);
+}
+
+void string_split_test_1(void)
+{
+    TRACEF("\n");
+
+    Str str1 = S("|a:bb|ccc|");
+    Str sep = S("|:");
+    STRVEC_DEFINE(strings, 16);
+
+    str_split(str1, sep, &strings);
+    for (U32 i = 0; i < strings.len; i++) {
+        Str *str = &strings.strings[i];
+        printf("split %u: '%.*s'\n", i, str->len, str->str);
+    }
+}
+void string_test_misc(void)
+{
+    TRACEF("\n");
+
+    Str str1 = S("Hello World");
+    bool prefix1 = str_starts_with(str1, S("He"));
+    bool prefix2 = str_starts_with(str1, S(""));
+    bool prefix3 = str_starts_with(str1, S("false"));
+    bool prefix4 = str_starts_with(str1, S("Hello World"));
+
+    bool suffix1 = str_ends_with(str1, S("orld"));
+    bool suffix2 = str_ends_with(str1, S(""));
+    bool suffix3 = str_ends_with(str1, S("false"));
+    bool suffix4 = str_ends_with(str1, S("Hello World"));
+    printf("matches: %d %d %d %d %d %d %d %d\n", prefix1, prefix2, prefix3, prefix4, suffix1, suffix2, suffix3, suffix4);
+
+    Str str2 = S(" Hello World\t \n");
+    Str str2_trim = str_ascii_trim(str2);
+    printf("str2_trim: '%.*s'\n", str2_trim.len, str2_trim.str);
+    printf("str2_trim == str1: %d\n", str_eq(str1, str2_trim));
+
+    Str str3 = str_empty();
+    Str str3_trim = str_ascii_trim(str3);
+    printf("str3_trim: '%.*s'\n", str3_trim.len, str3_trim.str);
+
+    Str str4 = S("   \t\t    ");
+    Str str4_trim = str_ascii_trim(str4);
+    printf("str4_trim: '%.*s'\n", str4_trim.len, str4_trim.str);
+
+    S64 found = str_index(str1, S("World"));
+    printf("1. found %ld\n", found);
+    found = str_index(str_empty(), S("World"));
+    printf("2. found %ld\n", found);
+    found = str_index(str_empty(), S(""));
+    printf("3. found %ld\n", found);
+    found = str_index(str1, S(" "));
+    printf("4. found %ld\n", found);
+
+}
+void string_test_join(void)
+{
+    TRACEF("\n");
+    Arena arena;
+    arena_init(&arena, &ArenaOptions_Default);
+    Allocator allocator = arena_allocator(&arena);
+    Str strs[3] = {S("1"), S("two"), S("3-3")};
+    StrVec vec = STRVEC_STATIC_INITIALIZER(strs);
+
+    Str str = str_join(&allocator, vec, S("/"));
+    printf("str_join 1 '%.*s'\n", str.len, str.str);
+    str = str_join(&allocator, vec, S(""));
+    printf("str_join 2 '%.*s'\n", str.len, str.str);
+    vec.len = 0;
+    str = str_join(&allocator, vec, S("/"));
+    printf("str_join 2 '%.*s'\n", str.len, str.str);
+
+}
+
+void print_runes(Str str)
+{
+    U32 offset = 0;
+    U32 len = str_rune_count(str);
+    printf("rune len %d\n", len);
+    while (offset < str.len) {
+        RuneDecode d = str_utf8_decode(str, offset);
+        printf("Rune: %d  valid: %d\n", d.rune, d.valid);
+        offset += d.width;
+    }
+}
+
+void string_test_utf8(void)
+{
+    TRACEF("\n");
+
+    Str str1 = S("Test 1ø2 ☺");
+
+    TRACEF("str1:\n");
+    print_runes(str1);
+
+    Str str2 = str_empty();
+    TRACEF("str2:\n");
+    print_runes(str2);
+
+    Str str3 = S("1Å 2\xc6" "3\xc3");
+    TRACEF("str3:\n");
+    print_runes(str3);
+
+    const char rune_error[] = {0xEF, 0xBF, 0xBD, 0x00} ; // RUNE_ERROR encoded in utf-8
+    Str str4 = S(rune_error);
+    TRACEF("str4:\n");
+    print_runes(str4);  // this should be a valid code point
+
+}
+
+void string_test_hex(void)
+{
+    TRACEF("\n");
+
+    MArena arena;
+    U8 data[4096];
+    marena_init(&arena, data, sizeof data);
+    Allocator allocator = marena_allocator(&arena);
+
+    Str str1 = S("Hello World ☺");
+
+    Str str1_encoded = str_hex_encode_alloc(&allocator, str1);
+    printf("str1_encoded: '%.*s'\n", str1_encoded.len, str1_encoded.str);
+
+    Str str1_decoded = str_hex_decode_alloc(&allocator, str1_encoded);
+    printf("str1_decoded: '%.*s'\n", str1_decoded.len, str1_decoded.str);
+
+    Str str2 = S("");
+
+    Str str2_encoded = str_hex_encode_alloc(&allocator, str2);
+    printf("str2_encoded: '%.*s'\n", str2_encoded.len, str2_encoded.str);
+
+    Str str2_decoded = str_hex_decode_alloc(&allocator, str2_encoded);
+    printf("str2_decoded: '%.*s'\n", str2_decoded.len, str2_decoded.str);
+}
+
+typedef struct StrSLNode StrSLNode;
+struct StrSLNode {
+    Str str;
+    StrSLNode *next;
+};
+
+typedef struct StrSList StrSList;
+struct StrSList {
+    StrSLNode *first;
+    StrSLNode **last;
+};
+
+void string_test_slist(void)
+{
+    TRACEF("\n");
+
+    Str str1 = S("one");
+    Str str2 = S("two");
+    Str str3 = S("three");
+    StrSList strlist = {};
+    StrSLNode n1 = {str1, NULL};
+    StrSLNode n2 = {str2, NULL};
+    StrSLNode n3 = {str3, NULL};
+
+    SListInit(&strlist);
+    SListAddFirst(&strlist, &n1);
+    SListAddFirst(&strlist, &n2);
+    SListAddFirst(&strlist, &n3);
+    TRACEF("List test first\n");
+
+    for (StrSLNode *n = strlist.first; n != NULL; n = n->next) {
+        printf("Node: %.*s\n", n->str.len, n->str.str);
+    }
+    TRACEF("List test first done\n");
+    SListInit(&strlist);
+    SListAddLast(&strlist, &n1);
+    SListAddLast(&strlist, &n2);
+    SListAddLast(&strlist, &n3);
+    SListRemoveFirst(&strlist);
+    TRACEF("List test last\n");
+
+    for (StrSLNode *n = strlist.first; n != NULL; n = n->next) {
+        printf("Node: %.*s\n", n->str.len, n->str.str);
+    }
+    TRACEF("List test last done\n");
+
+}
+typedef struct StrDLNode StrDLNode;
+struct StrDLNode {
+    Str str;
+    StrDLNode *next;
+    StrDLNode *prev;
+};
+
+typedef struct StrDList StrDList;
+struct StrDList {
+    StrDLNode *first;
+    StrDLNode *last;
+};
+
+void string_test_dlist(void)
+{
+    TRACEF("\n");
+
+    Str str1 = S("one");
+    Str str2 = S("two");
+    Str str3 = S("three");
+    Str str4 = S("four");
+    StrDList strlist = {};
+    StrDLNode n1 = {str1, NULL, NULL};
+    StrDLNode n2 = {str2, NULL, NULL};
+    StrDLNode n3 = {str3, NULL, NULL};
+    StrDLNode n4 = {str4, NULL, NULL};
+
+    DListInit(&strlist);
+    DListAddFirst(&strlist, &n1);
+    DListAddFirst(&strlist, &n2);
+    DListAddFirst(&strlist, &n3);
+    TRACEF("List test first\n");
+
+    for (StrDLNode *n = strlist.first; n != NULL; n = n->next) {
+        printf("Node: %.*s\n", n->str.len, n->str.str);
+    }
+    TRACEF("List test first done\n");
+
+    DListInit(&strlist);
+    DListAddLast(&strlist, &n2);
+    DListAddLast(&strlist, &n3);
+    DListAddLast(&strlist, &n4);
+    DListAddFirst(&strlist, &n1);
+
+    TRACEF("List test full\n");
+
+    for (StrDLNode *n = strlist.first; n != NULL; n = n->next) {
+        printf("Node: %.*s\n", n->str.len, n->str.str);
+    }
+    TRACEF("List test full done\n");
+
+    DListRemove(&strlist, &n1);
+    DListRemoveFirst(&strlist);
+    DListRemoveLast(&strlist);
+    TRACEF("List test tail removed\n");
+
+    for (StrDLNode *n = strlist.first; n != NULL; n = n->next) {
+        printf("Node: %.*s\n", n->str.len, n->str.str);
+    }
+    TRACEF("List test tail removed done\n");
+
+}
+
+int main(int argc, char *argv[])
+{
+    arena_test_1();
     arena_test_2();
-    // arena_test_3();
+    arena_test_3();
     arena_test_4();
+    arena_test_decommit();
+    arena_test_allocator_interface();
+    miniarena_test1();
+    apool_test();
+    apool_test_autogrow();
+    apool_test_miniarena();
+    string_test_1();
+    string_split_test_1();
+    string_test_misc();
+    string_test_join();
+    string_test_utf8();
+    string_test_hex();
+    string_test_slist();
+    string_test_dlist();
+
 }
 

src/mini_arena.h

@@ -1,96 +0,0 @@
-#pragma once
-#include "base_core.h"
-
-typedef struct MiniArena MiniArena;
-struct MiniArena {
-    U8 *start;  // start of user buffer
-    U8 *curr;   // next alloc point
-    U8 *end;    // one past end of user buffer
-    int checkpoint_level;  // level of miniarenatmp_begin stacks
-};
-
-typedef struct MiniArenaCheckpoint MiniArenaCheckpoint;
-struct MiniArenaCheckpoint {
-    U8 *alloc_point;
-    MiniArena *arena;
-};
-
-#define LARENA_STATIC_INITIALIZER(data) LARENA_INITIALIZER((data), sizeof(data))
-#define LARENA_INITIALIZER(data, sz) {.start = (data), .curr = (data), .end = (data) + sz, .checkpoint_level = 0}
-#define LARENA_DEFINE(var_name, size_, alignment_)  \
-    struct {                                        \
-        _Alignas(alignment_) U8 memory[size_]; \
-    } var_name##_arena_memory;                      \
-    MiniArena var_name = LARENA_STATIC_INITIALIZER(var_name##_arena_memory.memory)
-#define LARENA_ALLOC_TYPE(allocator, type_) miniarenaalloc_aligned((allocator), sizeof(type_), _Alignof(type_))
-#define LARENA_ALLOC_ARRAY(allocator, type_, nr_elements_) miniarenaalloc_aligned((allocator), (nr_elements_) * sizeof(type_), _Alignof(type_))
-static INLINE size_t miniarena_size(const MiniArena *arena)
-{
-    return (size_t)(arena->end - arena->start);
-}
-
-static INLINE size_t miniarena_used(const MiniArena *arena)
-{
-    return (size_t)(arena->curr - arena->start);
-}
-static INLINE size_t miniarena_available(const MiniArena *arena)
-{
-    return miniarena_size(arena) - miniarena_used(arena);
-}
-
-static INLINE void miniarena_init(MiniArena *arena, void *memory, size_t sz)
-{
-    arena->start = arena->curr = (U8 *)memory;
-    arena->end = arena->start + sz;
-}
-
-static INLINE MiniArenaCheckpoint miniarenatmp_begin(MiniArena *arena)
-{
-    MiniArenaCheckpoint checkpoint = {};
-
-    checkpoint.arena = arena;
-    checkpoint.alloc_point = arena->curr;
-    arena->checkpoint_level++;
-
-    return checkpoint;
-}
-
-static INLINE void miniarenatmp_end(MiniArenaCheckpoint *state)
-{
-    MiniArena *arena = state->arena;
-    Assert(arena->checkpoint_level > 0);
-    arena->curr = state->alloc_point;
-    arena->checkpoint_level--;
-}
-static INLINE void *miniarena_push(MiniArena *arena, size_t sz)
-{
-    if (sz > miniarena_available(arena)) {
-        return NULL;
-    }
-    void *start = arena->curr;
-    arena->curr += sz;
-    return start;
-}
-
-[[gnu::alloc_align(3)]]
-static INLINE void *miniaren_push_aligned(MiniArena *arena, size_t sz, unsigned int align)
-{
-    Assert(IsPow2(align));
-
-    uintptr_t curr = (uintptr_t)arena->curr;
-    uintptr_t aligned = AlignUp(curr, align);
-    U8 *start = (U8 *)aligned;
-
-    if (sz > (size_t)(arena->end - start)) {
-        return NULL;
-    }
-
-    arena->curr = start + sz;
-
-    return start;
-}
-
-static INLINE void miniarena_reset(MiniArena *arena)
-{
-    arena->curr = arena->start;
-}