Add lazy commit option

build.sh

@@ -2,4 +2,4 @@
 CFLAGS=${CFLAGS:-"-fsanitize=address -ggdb"}
 CC=${CC:-gcc}
 set -x
-${CC} -DDEBUG -pipe  -Wall -Wextra -Wno-unused-parameter -Wno-unused-variable -std=gnu2x $CFLAGS src/*c
+${CC} -DDEBUG -pipe  -Wall -Wextra -Wno-unused-parameter -Wno-unused-variable -std=gnu2x $CFLAGS src/*c src/base/*c

src/base/arena.c

@@ -1,5 +1,5 @@
 #include "arena.h"
-#include "base_core.h"
+#include "core.h"
 #include "platform.h"
 
 
@@ -10,7 +10,7 @@ static MemoryBlock *memoryblock_allocate(U32 size, U32 commit_size, U32 page_siz
 
     MemoryBlock *block = memory_reserve(size);
     Assert(block != NULL);
-    memory_commit(block, commit_size);
+    memory_commit(block, commit_size, false);
 
     ASAN_POISON_MEMORY_REGION(block, size);
     ASAN_UNPOISON_MEMORY_REGION(block, MemoryBlock_Header_Size);
@@ -61,7 +61,6 @@ void *arena_push(Arena *arena, U32 size, U32 align)
 {
     Assert(IsPow2(align));
 
-
     MemoryBlock *block = arena->current;
     U32 start_pos = AlignUp(block->first_free, align);  // might align past block->size, handled in below check
     U32 available = block->size - start_pos;
@@ -97,7 +96,7 @@ void *arena_push(Arena *arena, U32 size, U32 align)
         U32 aligned_size      = RoundUpToMultiple(aligned_end_pos - block->committed_end, arena->commit_size);
         aligned_size = Min(aligned_size, block->size - block->committed_end);
 
-        memory_commit(&block->base[block->committed_end], aligned_size);
+        memory_commit(&block->base[block->committed_end], aligned_size, false);
         block->committed_end  += aligned_size;
         Assert(block->committed_end <= block->size);
     }
@@ -144,8 +143,6 @@ void arena_trim(Arena *arena)
     U32 aligned_pos = RoundUpToMultiple(block->first_free, arena->commit_size);
     if (aligned_pos < block->committed_end) {
         void *trim_start = &block->base[aligned_pos];
-        // Note(nos): Should we unpoision ?
-        //ASAN_UNPOISON_MEMORY_REGION(trim_start, block->committed_end);
         memory_decommit(trim_start, block->committed_end - aligned_pos);
         block->committed_end = aligned_pos;
     }

src/base/arena.h

@@ -1,16 +1,17 @@
 #pragma once
-#include "base_core.h"
+#include "core.h"
 
 
 typedef U32 ArenaFlags;
 typedef struct ArenaOptions ArenaOptions;
 enum {
-    ArenaFlag_NoDecommit, // arena_ temp_end/reset/reset_to doesn't decommit memory
+    ArenaFlag_NoDecommit = (1 << 0), // operations which shrinks the arena does not decommit memory.
+                              // (arena_trim() can still decommit when called explicitly)
 };
 struct ArenaOptions {
     U32 size;
     U32 commit_size;
-    ArenaFlags flags; // No flags defined yet
+    ArenaFlags flags;
 };
 
 typedef struct MemoryBlock MemoryBlock;
@@ -79,7 +80,8 @@ static INLINE void *arena_allocator_func(Allocator *allocator, AllocOperation op
         case Alloc_Op_Aligned:
             return arena_push(allocator->data, size, align);
 
-        case Alloc_Op_Free: // can't free from  an marena
+        case Alloc_Op_Free: // can't free from  an arena
+            NotImplemented;
             return NULL;
 
         case Alloc_Op_Free_All:
@@ -87,7 +89,7 @@ static INLINE void *arena_allocator_func(Allocator *allocator, AllocOperation op
             return NULL;
     }
 
-    Assert(false);
+    ShouldNotHappen;
 }
 
 static INLINE Allocator arena_allocator(Arena *arena)

src/base/core.h

@@ -24,6 +24,8 @@ typedef uint8_t     U8;
 typedef uint16_t    U16;
 typedef uint32_t    U32;
 typedef uint64_t    U64;
+typedef uintptr_t   UMM;
+typedef intptr_t    IMM;
 typedef float       F32;
 typedef double      F64;
 #if defined(__FLT128_MANT_DIG__)
@@ -410,3 +412,171 @@ static INLINE void  allocator_free_all(Allocator *allocator)
 #define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
   ((void)(addr), (void)(size))
 #endif
+
+
+// linked lists
+
+/*
+  SList - singly linked lists
+    "list" struct must have a "*first" and "**last" member to entrys
+    "first"/"last" structs must have a "*next" member
+*/
+
+/**
+ * Initialize a list, can also just be zero initialized
+ *
+ * @param list struct pointer with a *first and a **last member to an entry struct
+ */
+#define SListInit(list)           \
+do {                               \
+    (list)->first = NULL;          \
+    (list)->last = &(list)->first; \
+} while (0)
+
+/**
+ * Adds entry to the head of a list
+ *
+ * @param list struct pointer with a *first and a **last member to an entry struct
+ * @param entry struct pointer with a *next member to an entry struct
+ */
+#define SListAddFirst(list, entry)    \
+do {                                   \
+    (entry)->next = (list)->first;     \
+    if ((entry)->next == NULL) {       \
+        (list)->last = &(entry)->next; \
+    }                                  \
+    (list)->first = (entry);           \
+} while (0)
+
+/**
+ * Adds entry to the tail of a list
+ *
+ * @param list struct pointer with a *first and a **last member to an entry struct
+ * @param entry struct pointer with a *next member to an entry struct
+ */
+#define SListAddLast(list, entry) \
+do {                                \
+    (entry)->next = NULL;           \
+    *(list)->last = (entry);        \
+    (list)->last = &(entry)->next;  \
+} while (0)
+
+/**
+ * Remove head from list, list MUST not be empty
+ *
+ * @param entry struct pointer with a *next member to an entry struct
+ */
+#define SListRemoveFirst(list)          \
+do {                                     \
+    (list)->first = (list)->first->next; \
+    if ((list)->first == NULL) {         \
+        (list)->last = &(list)->first;   \
+    }                                    \
+} while (0)
+
+#define SListIsEmpty(list)    ((list)->first == NULL)
+#define SListResetNode(node)  ((node)->next = NULL)
+
+/*
+  DList - doubly linked lists
+    "list" struct must have a "*first" and "*last" member to entrys
+    "first"/"last" structs must have a "*next" and a "*prev" member
+*/
+
+/**
+ * Initialize a list, can also just be zero initialized
+ *
+ * @param list struct pointer with a *first and a *last member to an entry struct
+ */
+#define DListInit(list)           \
+do {                              \
+    (list)->first = NULL;         \
+    (list)->last = NULL;          \
+} while (0)
+
+/**
+ * Adds entry to the head of a list
+ *
+ * @param list struct pointer with a *first and a *last member to an entry struct
+ * @param entry struct pointer with a *next and *prev member to an entry struct
+ */
+#define DListAddFirst(list, entry)             \
+do {                                          \
+  (entry)->next = (list)->first;              \
+  (entry)->prev = NULL;                       \
+    if ((list)->first != NULL) {              \
+        (list)->first->prev = (entry);        \
+    } else {                                  \
+        (list)->last = (entry);               \
+    }                                         \
+    (list)->first = (entry);                  \
+} while (0)
+/**
+ * Adds entry to the tail of a list
+ *
+ * @param list struct pointer with a *first and a *last member to an entry struct
+ * @param entry struct pointer with a *next and *prev member to an entry struct
+ */
+#define DListAddLast(list, entry)  \
+do {                                \
+  (entry)->prev = (list)->last;     \
+  (entry)->next = NULL;             \
+  if ((list)->last != NULL) {       \
+    (list)->last->next = (entry);   \
+  } else {                          \
+    (list)->first = (entry);        \
+  }                                 \
+  (list)->last = (entry);           \
+} while (0)
+
+/**
+ * Remove first entry from list, list MUST not be empty
+ *
+ * @param entry struct pointer with a *next and *prev member to an entry struct
+ */
+#define DListRemoveFirst(list)           \
+do {                                     \
+    (list)->first = (list)->first->next; \
+    if ((list)->first != NULL) {         \
+        (list)->first->prev = NULL;      \
+    } else {                             \
+        (list)->last = NULL;             \
+    }                                    \
+} while (0)
+
+/**
+ * Remove last entry from list, list MUST not be empty
+ *
+ * @param entry struct pointer with a *next and *prev member to an entry struct
+ */
+#define DListRemoveLast(list)             \
+do {                                      \
+    (list)->last = (list)->last->prev;    \
+    if ((list)->last != NULL) {           \
+        (list)->last->next = NULL;        \
+    } else {                              \
+        (list)->first = NULL;             \
+    }                                     \
+} while (0)
+
+/**
+ * Remove last entry from list, entry must be in the list
+ *
+ * @param entry struct pointer with a *next and *prev member to an entry struct
+ */
+#define DListRemove(list, entry)                \
+do {                                            \
+    if ((entry)->next != NULL) {                \
+      (entry)->next->prev = (entry)->prev;      \
+    } else {                                    \
+      (list)->last = (entry)->prev;             \
+    }                                           \
+    if ((entry)->prev != NULL) {                \
+      (entry)->prev->next = (entry)->next;      \
+    } else {                                    \
+      (list)->first = (entry)->next;            \
+    }                                           \
+} while (0)
+
+#define DListIsEmpty(list)    ((list)->first == NULL)
+#define DListResetNode(node) ((node)->next = (node)->prev = NULL)

src/base/lalloc.c

@@ -0,0 +1,176 @@
+#include "lalloc.h"
+
+#define BLOCK_ALIGNMENT sizeof(LBlockHeader)
+#define BLOCK_SPLIT_MIN_SIZE BLOCK_ALIGNMENT
+#define NULL_BLOCK (0xffffffff)
+#define BLOCK_FREE ((U32)1)
+
+static inline void lblock_set_free(LBlockHeader *block)
+{
+    block->size_and_flags |= BLOCK_FREE;
+}
+
+static inline void lblock_set_used(LBlockHeader *block)
+{
+    block->size_and_flags &= ~BLOCK_FREE;
+}
+
+static inline void lblock_set_size(LBlockHeader *block, U32 size)
+{
+    U32 flags = block->size_and_flags & BLOCK_FREE;
+    Assert((size & BLOCK_FREE) == 0);
+    block->size_and_flags = size | flags;
+}
+
+U32 lblock_size(const LBlockHeader *block)
+{
+    return block->size_and_flags & ~BLOCK_FREE;
+}
+
+int lblock_is_free(const LBlockHeader *block)
+{
+    return (block->size_and_flags & BLOCK_FREE) != 0;
+}
+
+LBlockHeader *lblock_from_offset(const LAlloc *allocator, U32 offset)
+{
+    if (offset == NULL_BLOCK) {
+        return NULL;
+    }
+
+    return (LBlockHeader *)((UMM)allocator->base + offset);
+}
+
+static inline U32 lbh_offset(LAlloc *allocator, LBlockHeader *block)
+{
+    if (block == NULL) {
+        return NULL_BLOCK;
+    }
+
+    return (U32)((UMM)block - (uintptr_t)allocator->base);
+}
+
+static void lmaybe_split_block(LAlloc *allocator, LBlockHeader *block, U32 size)
+{
+    // this function assumes size is already aligned
+
+    U32 bsize = lblock_size(block);
+    if (size > U32_Max - (sizeof(LBlockHeader) + BLOCK_SPLIT_MIN_SIZE)) {
+        return; // size is so large that size + header + min would overflow U32 anyway
+    }
+    if (bsize < size + sizeof(LBlockHeader) + BLOCK_SPLIT_MIN_SIZE) {
+        return; // no gain from splitting this block
+    }
+
+    U8 *base = (U8 *)block;
+    LBlockHeader *new_block = (LBlockHeader *)(base + size + sizeof(LBlockHeader) );
+
+    U32 new_size = bsize - size - sizeof(LBlockHeader);
+
+    lblock_set_size(block, size);
+    lblock_set_used(block);
+    lblock_set_size(new_block, new_size);
+    lblock_set_free(new_block);
+    new_block->next = block->next;
+    new_block->prev = lbh_offset(allocator, block);
+
+    LBlockHeader *next = lblock_from_offset(allocator, new_block->next);
+    if (next) {
+        next->prev = lbh_offset(allocator, new_block);
+    }
+
+    block->next = lbh_offset(allocator, new_block);
+}
+
+static void lmerge_with_next(LAlloc *allocator, LBlockHeader *block)
+{
+    LBlockHeader *next = lblock_from_offset(allocator, block->next);
+    if (next == NULL || !lblock_is_free(next)) {
+        return;
+    }
+
+    // block and next are both existing blocks within the same backing store, whose
+    // total size is asserted in lalloc_init to be < 0xfffffff0, so this can't overflow
+    // unless allocator state is already corrupted.
+    Assert(lblock_size(block) <= U32_Max - sizeof(LBlockHeader) - lblock_size(next));
+    U32 new_size = lblock_size(block) + sizeof(LBlockHeader) + lblock_size(next);
+
+    lblock_set_size(block, new_size);
+    block->next = next->next;
+    LBlockHeader *nextnext = lblock_from_offset(allocator, next->next);
+    if (nextnext != NULL) {
+         nextnext->prev = lbh_offset(allocator, block);
+    }
+    // note - caller have to adjust next_alloc on return
+}
+
+void lalloc_init(LAlloc *allocator, void *backing_store, U32 size)
+{
+    // backing store must be aligned, otherwise  all alignment guarantees are lies
+
+    Assert(IsAlignedTo((UMM)backing_store, BLOCK_ALIGNMENT));
+    Assert(size >= sizeof(LBlockHeader));
+    Assert(size < 0xfffffff0);
+
+    allocator->base = backing_store;
+    allocator->size = size;
+
+    LBlockHeader *first = backing_store;
+    // We need to keep size aligned to at least 2 as we reuse the LSB for the FREE flag
+    U32 usable = AlignDown(size, 2) - sizeof(LBlockHeader);
+    lblock_set_free(first);
+    lblock_set_size(first, usable);
+    first->next = NULL_BLOCK;
+    first->prev = NULL_BLOCK;
+
+    allocator->next_alloc = first;
+}
+
+void *lalloc(LAlloc *allocator, U32 size)
+{
+    if (size > U32_Max - (BLOCK_ALIGNMENT - 1)) { // -1 so AlignUp below doesn't overflow
+        return NULL;
+    }
+    size = AlignUp(size, BLOCK_ALIGNMENT);
+
+    LBlockHeader *start = allocator->next_alloc;
+    LBlockHeader *curr = start;
+
+    do {
+        if (lblock_is_free(curr) && lblock_size(curr) >= size) {
+            lmaybe_split_block(allocator, curr, size);
+            lblock_set_used(curr);
+            LBlockHeader *next = lblock_from_offset(allocator, curr->next);
+            allocator->next_alloc = next ? next : (LBlockHeader *)allocator->base;
+            return (U8 *)curr + sizeof(LBlockHeader);
+        }
+        LBlockHeader *next = lblock_from_offset(allocator, curr->next);
+        curr = next ? next : (LBlockHeader *)allocator->base;
+    } while (curr != start);
+
+    return NULL;
+}
+
+void lfree(LAlloc *allocator, void *ptr)
+{
+    if (ptr == NULL) {
+        return;
+    }
+
+    LBlockHeader *block = (LBlockHeader *)((U8 *)ptr - sizeof(LBlockHeader));
+
+    Assert((U8 *)block >= allocator->base && (U8 *)ptr < allocator->base + allocator->size);
+    // double free detection
+    Assert(!lblock_is_free(block));
+
+    lblock_set_free(block);
+    lmerge_with_next(allocator, block);
+
+    LBlockHeader *prev = lblock_from_offset(allocator, block->prev);
+    if (prev != NULL && lblock_is_free(prev)) {
+        lmerge_with_next(allocator, prev);
+        allocator->next_alloc = prev;
+    } else {
+        allocator->next_alloc = block;
+    }
+}

src/base/lalloc.h

@@ -0,0 +1,41 @@
+#pragma once
+#include "core.h"
+
+// Internal header added in front of each allocation
+typedef struct LBlockHeader LBlockHeader;
+struct LBlockHeader {
+    U32 next;            // offset from base
+    U32 prev;            // offset from base
+    U32 size_and_flags;  // size | free_bit
+    // Padding to ensure we always return 16 byte aligned pointers.
+    uint8_t pad[4];
+};
+_Static_assert(sizeof(LBlockHeader) == 16, "sizeof(BlockHeader) is not 16");
+
+/**
+ * General purpose allocator from a pre-defined backing store
+ * - Allocated pointers are aligned to 16 bytes.
+ * - Backing buffer must be aligned to 16 bytes
+ * - 4Gb - 16 is max for the allocator backing buffer.
+ * - Allocation of > 0 bytes returns a size aligned up to 16 bytes
+ * - Allocation of 0 bytes returns a valid pointer with 0 usable space
+ * The allocator maintains a doubly linked list of blocks,
+ * when blocks are free'd, adjacent blocks are merged to keep.
+ * This is also known as a buddy allocator
+ * fragmentation to a minimum
+ */
+typedef struct LAlloc LAlloc;
+struct LAlloc {
+    uint8_t *base;              // backing store
+    U32 size;                   // total size
+    LBlockHeader *next_alloc;   // start of free block search
+};
+
+void lalloc_init(LAlloc *allocator, void *backing_store, U32 size);
+
+void *lalloc(LAlloc *allocator, U32 size);
+void lfree(LAlloc *allocator, void *ptr);
+
+LBlockHeader *lblock_from_offset(const LAlloc *allocator, U32 offset);
+U32 lblock_size(const LBlockHeader *block);
+int lblock_is_free(const LBlockHeader *block);

src/base/marena.h

@@ -1,5 +1,5 @@
 #pragma once
-#include "base_core.h"
+#include "core.h"
 
 // Mini Arena for arena allocations from a supplied buffer
 
@@ -102,13 +102,15 @@ static INLINE void *marena_allocator_func(Allocator *allocator, AllocOperation o
             return marena_push_aligned(allocator->data, size, align);
 
         case Alloc_Op_Free: // can't free/Free_all from  an marena
+            NotImplemented;
             return NULL;
 
         case Alloc_Op_Free_All:
+            marena_reset(allocator->data);
             return NULL;
     }
 
-    Assert(false);
+    ShouldNotHappen;
 }
 
 static INLINE Allocator marena_allocator(MArena *arena)

src/base/platform.c

@@ -1,4 +1,4 @@
-#include "base_core.h"
+#include "core.h"
 #include "platform.h"
 #include <stdlib.h>
 #include <stdio.h>
@@ -30,7 +30,7 @@ static void *platform_allocator_func(Allocator *allocator, AllocOperation operat
         case Alloc_Op_Aligned:
             return aligned_alloc(size, align);
 
-        case Alloc_Op_Free: // can't free/Free_all from  an marena
+        case Alloc_Op_Free:
             free(memory);
             return NULL;
 
@@ -73,7 +73,7 @@ void *memory_allocate_and_commit(size_t size)
 void memory_free(void *mem, size_t size)
 {
     TRACEF("        memory_free     size=%7zu mem=%p\n", size, mem);
-    Assert(IsPageAligned((uintptr_t)mem));
+    Assert(IsPageAligned((UMM)mem));
     Assert(IsPageAligned(size));
     int rc = munmap(mem, size);
 
@@ -91,23 +91,27 @@ void *memory_reserve(size_t size)
     return mem;
 }
 
-void memory_commit(void *mem, size_t size)
+void memory_commit(void *mem, size_t size, bool lazy)
 {
-    Assert(IsPageAligned((uintptr_t)mem));
+    Assert(IsPageAligned((UMM)mem));
     Assert(IsPageAligned(size));
     TRACEF("memory_commit   size=%7zu mem=%p\n", size, mem);
 
     int rc;
     rc = mprotect(mem, size, PROT_READ | PROT_WRITE);
     Assert(rc == 0);
+    if (lazy) {
+        rc = madvise(mem, size, MADV_POPULATE_WRITE);
+    } else {
         rc = madvise(mem, size, MADV_WILLNEED);
+    }
     Assert(rc == 0);
 }
 
 void memory_decommit(void *mem, size_t size)
 {
     TRACEF("memory_decommit size=%7zu mem=%p\n", size, mem);
-    Assert(IsPageAligned((uintptr_t)mem));
+    Assert(IsPageAligned((UMM)mem));
     Assert(IsPageAligned(size));
     int rc;
     rc = mprotect(mem, size, PROT_NONE);

src/base/platform.h

@@ -1,6 +1,6 @@
 #pragma once
 #include <stdlib.h>
-#include "base_core.h"
+#include "core.h"
 
 void *platform_malloc(size_t size);
 void platform_free(void *mem);
@@ -11,6 +11,7 @@ U32 platform_page_size(void);
 // Memory functions expect mem and size to be page aligned
 void *memory_allocate_and_commit(size_t size);
 void *memory_reserve(size_t size);
-void memory_commit(void *mem, size_t size);
+
+void memory_commit(void *mem, size_t size, bool lazy);
 void memory_decommit(void *mem, size_t size);
 void memory_free(void *mem, size_t size);

src/base/pool.c

@@ -0,0 +1,63 @@
+#include "pool.h"
+
+APool *apool_create_ex(Allocator *allocator, const APoolCreateOptions *options)
+{
+    Assert(IsPow2(options->element_align));
+    Assert(options->n_reserved > 0 || options->flags & APoolFlag_AutoGrow);
+    U32 element_align = options->element_align;
+    if (element_align < alignof(APoolElement)) {
+        element_align = alignof(APoolElement);
+    }
+    U32 element_size = AlignUp(options->element_size, options->element_align);
+    APool *pool = allocator_push_type(allocator, APool);
+    Assert(pool != NULL);
+    pool->element_align= element_align;
+    pool->element_size = element_size;
+    pool->allocator    = allocator;
+    pool->n_elements   = 0;
+    pool->n_free       = 0;
+    pool->free_list    = NULL;
+    pool->flags        = options->flags;
+
+    if (options->n_reserved != 0) {
+        Assert(options->n_reserved <= U32_Max / element_size );
+        U32 total_size = options->n_reserved * element_size;
+        U8 *start = allocator_push(allocator, total_size, element_align);
+        for (U32 i = 0; i < total_size; i += element_size) {
+            APoolElement *element = (APoolElement *)&start[i];
+            element->next = pool->free_list;
+            pool->free_list = element;
+            pool->n_elements++;
+        }
+        pool->n_free = pool->n_elements;
+    }
+
+    return pool;
+}
+
+void *apool_alloc(APool *pool)
+{
+    APoolElement *element = pool->free_list;
+    if (element != NULL) {
+        pool->free_list = element->next;
+        pool->n_free--;
+    } else if((pool->flags & APoolFlag_AutoGrow) != 0) {
+        element = allocator_push(pool->allocator, pool->element_size, pool->element_align);
+        if (element != NULL) {
+            pool->n_elements++;
+        }
+    }
+
+    return element;
+}
+
+void apool_free(APool *pool, void *ptr)
+{
+    if (ptr != NULL) {
+        APoolElement *element = (APoolElement *)ptr;
+        element->next = pool->free_list;
+        pool->free_list = element;
+        pool->n_free++;
+
+    }
+}

src/base/pool.h

@@ -0,0 +1,43 @@
+#pragma once
+#include "core.h"
+
+enum {
+    APoolFlag_AutoGrow = (1 << 0)
+};
+typedef U32 APoolFlags;
+typedef union APoolElement APoolElement;
+union APoolElement {
+    APoolElement *next;       // offset from start to next free block (when not allocated)
+};
+
+typedef struct APool APool;
+struct APool {
+    APoolElement *free_list;  // free list
+    Allocator *allocator;
+    APoolFlags flags;
+    U32 element_size;         // aligned size of each element
+    U32 element_align;        // alignment
+    U32 n_elements;           // total elements in the pool
+    U32 n_free;               // number of free elements
+};
+typedef struct APoolCreateOptions APoolCreateOptions;
+struct APoolCreateOptions {
+    U32 element_size;         // aligned size of each element
+    U32 element_align;        // alignment of elements
+    U32 n_reserved;           // number of elements reserved
+    APoolFlags flags;
+
+};
+#define apool_create(arena, n_reserved, type) \
+        apool_create_ex(arena, &(APoolCreateOptions){sizeof(type), alignof(type), n_reserved, 0})
+APool *apool_create_ex(Allocator *allocator, const APoolCreateOptions *options);
+void *apool_alloc(APool *pool);
+INLINE void *apool_allocz(APool *pool)
+{
+    void *ptr = apool_alloc(pool);
+    if (ptr != NULL) {
+        MemoryZero(ptr, pool->element_size);
+    }
+    return ptr;
+}
+void apool_free(APool *pool, void *element);

src/base/string.c

@@ -93,6 +93,72 @@ S64 str_index(Str haystack, Str needle)
     return -1;
 }
 
+
+
+static INLINE S16 char_hex_to_binary(U8 c)
+{
+    if (c >='0' && c <='9')
+        return c - '0';
+    else if (c >= 'A' && c <= 'F')
+        return c - 'A' + 10;
+    else if (c >= 'a' && c <= 'f')
+        return c - 'a' + 10;
+
+    return -1;
+}
+
+static const char hex_chars[] = "0123456789ABCDEF";
+
+Str str_hex_encode(Str binary_data, Str hex_result)
+{
+    Assert(hex_result.len  >= binary_data.len * 2);
+    U32 hex_len = binary_data.len * 2;
+    for (U32 i = 0, t = 0; i < binary_data.len; ++i,t+=2) {
+        U8 high_nibble;
+        U8 low_nibble;
+
+        high_nibble = (binary_data.str[i] & 0xF0) >> 4 ;
+        low_nibble =  binary_data.str[i] & 0x0F ;
+
+        hex_result.str[t]   = hex_chars[high_nibble];
+        hex_result.str[t+1] = hex_chars[low_nibble];
+    }
+
+    hex_result.len = hex_len;
+    return hex_result;
+}
+
+Str str_hex_decode(Str hex_str, Str result)
+{
+    Assert(result.len >= hex_str.len/2);
+    U32 dest_pos = 0;
+    U32 source_pos = 0;
+
+    while (source_pos + 1 < hex_str.len) {
+
+        if (char_is_ascii_whitespace(hex_str.str[source_pos])) {
+            source_pos++;
+            continue;
+        }
+
+        S16 high_nibble;
+        S16 low_nibble;
+
+        high_nibble = char_hex_to_binary(hex_str.str[source_pos]);
+        low_nibble = char_hex_to_binary(hex_str.str[source_pos + 1]);
+        if (high_nibble == -1 || low_nibble == -1) {
+            result.len = 0;
+            return result;
+        }
+
+        result.str[dest_pos] = (high_nibble << 4 ) | low_nibble;
+        source_pos += 2;
+        dest_pos++;
+    }
+
+    result.len = dest_pos;
+    return result;
+}
 // split @str on any byte in @separators, place split strings in @vec
 void str_split(Str str, Str separators, StrVec *vec)
 {
@@ -158,6 +224,105 @@ Str str_join(Allocator *allocator, StrVec strings, Str separator)
 }
 
 
+
+/*
+ * Each entry in utf8_lead[] is an 8-bit descriptor:
+ *
+ *     7 6 5 4 3 2 1 0
+ *    +---------+-----+
+ *    |  range  | len |
+ *    +---------+-----+
+ *
+ * len:
+ *     Number of bytes in the UTF-8 sequence.
+ *
+ *     0 = byte cannot start a valid UTF-8 sequence
+ *     1 = ASCII
+ *     2 = two-byte UTF-8 sequence
+ *     3 = three-byte UTF-8 sequence
+ *     4 = four-byte UTF-8 sequence
+ *
+ * range:
+ *     Selects the legal range for byte 1, the first continuation byte.
+ *
+ * This range is enough to encode all of UTF-8's special boundary rules:
+ *
+ *     E0 A0..BF    prevents overlong 3-byte encodings
+ *     ED 80..9F    excludes UTF-16 surrogate code points
+ *     F0 90..BF    prevents overlong 4-byte encodings
+ *     F4 80..8F    limits Unicode to U+10FFFF
+ *
+ * All continuation bytes after byte 1 simply use the normal 80..BF range.
+ */
+
+#define UTF8_LENGTH_BITS 3u
+#define UTF8_LENGTH_MASK ((1u << UTF8_LENGTH_BITS) - 1u)
+
+#define UTF8_DESC(length, range) \
+    ((U8)((length) | ((U8)(range) << UTF8_LENGTH_BITS)))
+
+#define UTF8_DESC_LENGTH(desc) \
+    ((U8)((desc) & UTF8_LENGTH_MASK))
+
+#define UTF8_DESC_RANGE(desc) \
+    ((U8)((desc) >> UTF8_LENGTH_BITS))
+
+
+/*
+ * Classes of allowed values for the first continuation byte.
+ */
+enum Utf8RangeClass {
+    /*
+     * No continuation byte exists.
+     *
+     * Used only by ASCII descriptors.
+     */
+    UTF8_RANGE_NONE = 0,
+
+    /*
+     * Normal continuation-byte range.
+     */
+    UTF8_RANGE_80_BF,
+
+    /*
+     * E0 xx xx:
+     *
+     * E0 80..9F would encode values below U+0800 and therefore be
+     * non-canonical overlong UTF-8.
+     */
+    UTF8_RANGE_A0_BF,
+
+    /*
+     * ED xx xx:
+     *
+     * ED A0..BF would encode U+D800..U+DFFF, which are UTF-16 surrogate
+     * code points and are not Unicode scalar values.
+     */
+    UTF8_RANGE_80_9F,
+
+    /*
+     * F0 xx xx xx:
+     *
+     * F0 80..8F would encode values below U+10000 and therefore be
+     * non-canonical overlong UTF-8.
+     */
+    UTF8_RANGE_90_BF,
+
+    /*
+     * F4 xx xx xx:
+     *
+     * F4 90..BF would encode values above Unicode's maximum scalar value,
+     * U+10FFFF.
+     */
+    UTF8_RANGE_80_8F,
+};
+
+
+typedef struct {
+    U8 lo;
+    U8 hi;
+} Utf8Range;
+
 /*
  * Allowed range of the first continuation byte for each range class.
  *
@@ -173,6 +338,36 @@ static const Utf8Range utf8_ranges[] = {
 };
 
 
+
+/*
+ * Each entry in utf8_lead[] is an 8-bit descriptor:
+ *
+ *     7 6 5 4 3 2 1 0
+ *    +---------+-----+
+ *    |  range  | len |
+ *    +---------+-----+
+ *
+ * len:
+ *     Number of bytes in the UTF-8 sequence.
+ *
+ *     0 = byte cannot start a valid UTF-8 sequence
+ *     1 = ASCII
+ *     2 = two-byte UTF-8 sequence
+ *     3 = three-byte UTF-8 sequence
+ *     4 = four-byte UTF-8 sequence
+ *
+ * range:
+ *     Selects the legal range for byte 1, the first continuation byte.
+ *
+ * This range is enough to encode all of UTF-8's special boundary rules:
+ *
+ *     E0 A0..BF    prevents overlong 3-byte encodings
+ *     ED 80..9F    excludes UTF-16 surrogate code points
+ *     F0 90..BF    prevents overlong 4-byte encodings
+ *     F4 80..8F    limits Unicode to U+10FFFF
+ *
+ * All continuation bytes after byte 1 simply use the normal 80..BF range.
+ */
 /*
  * Classification of every possible first byte.
  *
@@ -239,9 +434,16 @@ static const U8 utf8_lead[256] = {
  */
 RuneDecode _str_utf8_decode_full(Str str, U32 offset)
 {
-    const uint8_t b0 = str.str[offset];
+    if (str.len == 0 || str.len <= offset) {
-    const uint8_t desc = utf8_lead[b0];
+        return (RuneDecode) {
-    const uint8_t width = UTF8_DESC_LENGTH(desc);
+            .rune  = RUNE_EOF,
+            .width = 0,
+            .valid = false,
+        };
+    }
+    U8 b0 = str.str[offset];
+    U8 desc = utf8_lead[b0];
+    U8 width = UTF8_DESC_LENGTH(desc);
     /*
      * width == 0 covers:
      *
@@ -265,8 +467,8 @@ RuneDecode _str_utf8_decode_full(Str str, U32 offset)
      * Its descriptor-selected range handles both the normal continuation
      * range and the E0 / ED / F0 / F4 boundary cases.
      */
-    const Utf8Range range = utf8_ranges[UTF8_DESC_RANGE(desc)];
+    Utf8Range range = utf8_ranges[UTF8_DESC_RANGE(desc)];
-    const uint8_t b1 = str.str[offset + 1];
+    U8 b1 = str.str[offset + 1];
 
     if (b1 < range.lo || b1 > range.hi)
         goto invalid;
@@ -290,7 +492,7 @@ RuneDecode _str_utf8_decode_full(Str str, U32 offset)
          *
          * b1 has already been checked against 80..BF.
          */
-        const Rune rune =
+        Rune rune =
             ((Rune)(b0 & 0x1F) << 6) |
              (Rune)(b1 & 0x3F);
 
@@ -308,12 +510,12 @@ RuneDecode _str_utf8_decode_full(Str str, U32 offset)
          * b1 has already been validated using the descriptor-specific
          * range. Only b2 still needs a continuation-byte check.
          */
-        const uint8_t b2 = str.str[offset + 2];
+        U8 b2 = str.str[offset + 2];
 
         if ((b2 & 0xC0) != 0x80)
             goto invalid;
 
-        const Rune rune =
+        Rune rune =
             ((Rune)(b0 & 0x0F) << 12) |
             ((Rune)(b1 & 0x3F) <<  6) |
              (Rune)(b2 & 0x3F);
@@ -332,8 +534,8 @@ RuneDecode _str_utf8_decode_full(Str str, U32 offset)
          * b1 has already been validated using the descriptor-specific
          * range. b2 and b3 use the ordinary continuation-byte form.
          */
-        const uint8_t b2 = str.str[offset + 2];
+        U8 b2 = str.str[offset + 2];
-        const uint8_t b3 = str.str[offset + 3];
+        U8 b3 = str.str[offset + 3];
 
         /*
          * Validate both remaining continuation bytes with one expression.
@@ -348,7 +550,7 @@ RuneDecode _str_utf8_decode_full(Str str, U32 offset)
         if ((((b2 ^ 0x80) | (b3 ^ 0x80)) & 0xC0) != 0)
             goto invalid;
 
-        const Rune rune =
+        Rune rune =
             ((Rune)(b0 & 0x07) << 18) |
             ((Rune)(b1 & 0x3F) << 12) |
             ((Rune)(b2 & 0x3F) <<  6) |

src/base/string.h

@@ -1,5 +1,5 @@
 #pragma once
-#include "base_core.h"
+#include "core.h"
 
 typedef struct Str Str;
 struct Str {
@@ -9,6 +9,7 @@ struct Str {
 
 Str str_empty(void);
 Str str_slice(Str str, U32 start, U32 len);
+// returns -1 if not found, or index start of first needle
 S64 str_index(Str haystack, Str needle);
 
 Str str_concat(Allocator *alloc, Str first, Str second);
@@ -65,6 +66,17 @@ static INLINE S64 str_find_byte(Str str, U8 byte)
     return -1;
 }
 
+static INLINE Str str_replace_byte(Str str, U8 orig_byte, U8 new_byte)
+{
+    for (U32 i = 0; i < str.len; i++) {
+        if (str.str[i] == orig_byte) {
+            str.str[i] = new_byte;
+        }
+    }
+
+    return str;
+}
+
 static INLINE bool str_starts_with(Str str, Str prefix)
 {
     if (prefix.len > str.len) {
@@ -91,7 +103,29 @@ static INLINE Str str_skip(Str str, U32 len)
     return new_str;
 }
 
+// hex_result must be at least source.len * 2. Returns hex_result.
+Str str_hex_encode(Str binary_data, Str hex_result);
+// result must be at least source.len / 2. Returns result.
+Str str_hex_decode(Str hex_str, Str result);
 
+static INLINE Str str_hex_encode_alloc(Allocator *allocator, Str binary_data)
+{
+    Assert(binary_data.len <= U32_Max / 2);
+    U8 hex_len = binary_data.len * 2;
+    U8 *result = allocator_push_array(allocator, U8, hex_len);
+    Str destination = str_create(result, hex_len);
+
+    return str_hex_encode(binary_data, destination);
+}
+
+static INLINE Str str_hex_decode_alloc(Allocator *allocator, Str hex_str)
+{
+    U8 binary_len = hex_str.len / 2;
+    U8 *result = allocator_push_array(allocator, U8, binary_len);
+    Str destination = str_create(result, binary_len);
+
+    return str_hex_decode(hex_str, destination);
+}
 // String Vector
 typedef struct StrVec StrVec;
 struct StrVec {
@@ -113,11 +147,11 @@ void str_split(Str str, Str separators, StrVec *vec);
 Str str_join(Allocator *allocator, StrVec strings, Str separator);
 
 
+// Characters
 
 static INLINE bool char_is_ascii_whitespace(U8 byte)
 {
     return byte == ' ' || byte == '\t' || byte == '\v' || byte == '\f' || byte == '\r' || byte == '\n';
-
 }
 
 static INLINE bool char_is_digit(U8 byte)
@@ -126,7 +160,6 @@ static INLINE bool char_is_digit(U8 byte)
 }
 
 
-
 // UTF-8/Unicode
 
 typedef S32 Rune;
@@ -165,111 +198,12 @@ typedef struct {
     /*
      * True only when rune was decoded from valid UTF-8.
      *
-     * This is required because U+FFFD itself may legitimately occur in
+     *  U+FFFD itself may legitimately occur in  valid
-     * valid input, so rune == RUNE_ERROR does not imply an error.
+     *  input, so rune == RUNE_ERROR does not imply an error.
      */
     bool valid;
 } RuneDecode;
 
-
-/*
- * Each entry in utf8_lead[] is an 8-bit descriptor:
- *
- *     7 6 5 4 3 2 1 0
- *    +---------+-----+
- *    |  range  | len |
- *    +---------+-----+
- *
- * len:
- *     Number of bytes in the UTF-8 sequence.
- *
- *     0 = byte cannot start a valid UTF-8 sequence
- *     1 = ASCII
- *     2 = two-byte UTF-8 sequence
- *     3 = three-byte UTF-8 sequence
- *     4 = four-byte UTF-8 sequence
- *
- * range:
- *     Selects the legal range for byte 1, the first continuation byte.
- *
- * This range is enough to encode all of UTF-8's special boundary rules:
- *
- *     E0 A0..BF    prevents overlong 3-byte encodings
- *     ED 80..9F    excludes UTF-16 surrogate code points
- *     F0 90..BF    prevents overlong 4-byte encodings
- *     F4 80..8F    limits Unicode to U+10FFFF
- *
- * All continuation bytes after byte 1 simply use the normal 80..BF range.
- */
-
-#define UTF8_LENGTH_BITS 3u
-#define UTF8_LENGTH_MASK ((1u << UTF8_LENGTH_BITS) - 1u)
-
-#define UTF8_DESC(length, range) \
-    ((U8)((length) | ((U8)(range) << UTF8_LENGTH_BITS)))
-
-#define UTF8_DESC_LENGTH(desc) \
-    ((U8)((desc) & UTF8_LENGTH_MASK))
-
-#define UTF8_DESC_RANGE(desc) \
-    ((U8)((desc) >> UTF8_LENGTH_BITS))
-
-
-/*
- * Classes of allowed values for the first continuation byte.
- */
-enum Utf8RangeClass {
-    /*
-     * No continuation byte exists.
-     *
-     * Used only by ASCII descriptors.
-     */
-    UTF8_RANGE_NONE = 0,
-
-    /*
-     * Normal continuation-byte range.
-     */
-    UTF8_RANGE_80_BF,
-
-    /*
-     * E0 xx xx:
-     *
-     * E0 80..9F would encode values below U+0800 and therefore be
-     * non-canonical overlong UTF-8.
-     */
-    UTF8_RANGE_A0_BF,
-
-    /*
-     * ED xx xx:
-     *
-     * ED A0..BF would encode U+D800..U+DFFF, which are UTF-16 surrogate
-     * code points and are not Unicode scalar values.
-     */
-    UTF8_RANGE_80_9F,
-
-    /*
-     * F0 xx xx xx:
-     *
-     * F0 80..8F would encode values below U+10000 and therefore be
-     * non-canonical overlong UTF-8.
-     */
-    UTF8_RANGE_90_BF,
-
-    /*
-     * F4 xx xx xx:
-     *
-     * F4 90..BF would encode values above Unicode's maximum scalar value,
-     * U+10FFFF.
-     */
-    UTF8_RANGE_80_8F,
-};
-
-
-typedef struct {
-    U8 lo;
-    U8 hi;
-} Utf8Range;
-
 // Helper:
 RuneDecode _str_utf8_decode_full(Str str, U32 offset);
 /*
@@ -306,16 +240,13 @@ static INLINE RuneDecode str_utf8_decode(Str str, U32 offset)
         };
     }
 
-    const uint8_t b0 = str.str[offset];
+    U8 b0 = str.str[offset];
-
 
     /*
-     * Handle ascii, helps inlining for ascii cases
-     *
      * Testing b0 directly also avoids consulting utf8_ranges[] for the
-     * overwhelmingly common single-byte case.
+     * overwhelmingly common single-byte/ascii case.
      */
-    if (b0 < 0x80) {
+    if (b0 <= RUNE_1_MAX) {
         return (RuneDecode) {
             .rune  = (Rune)b0,
             .width = 1,
@@ -336,7 +267,7 @@ static INLINE S32 rune_utf8_len(Rune rune) {
 		return 1;
 	} else if (rune <= RUNE_2_MAX) {
 		return 2;
-	} else if (rune > RUNE_SURROGATE_MIN && rune <= RUNE_SURROGATE_MAX) {
+	} else if (rune >= RUNE_SURROGATE_MIN && rune <= RUNE_SURROGATE_MAX) {
 		return RUNE_INVALID;
 	} else if (rune <= RUNE_3_MAX) {
 		return 3;
@@ -347,8 +278,7 @@ static INLINE S32 rune_utf8_len(Rune rune) {
 	return RUNE_INVALID;
 }
 
-// Number of unicode Runes in str. Invalid UTF-8 is counted as 1 Rune.
+// Number of unicode Runes in str. 1 invalid UTF-8 byte is counted as 1 Rune.
-// (1 invalud UTF-8 byte will be decoded to 1 Rune of RUNE_ERROR
 static INLINE U32 str_rune_count(Str str)
 {
     U32 offset = 0;

src/main.c

@@ -1,16 +1,21 @@
-#include "base_core.h"
+#include "base/core.h"
-#include "arena.h"
+#include "base/arena.h"
-#include "mini_arena.h"
+#include "base/marena.h"
-#include "platform.h"
+#include "base/platform.h"
-#include "pool.h"
+#include "base/pool.h"
-#include "string.h"
+#include "base/string.h"
+#include "base/lalloc.h"
 #include <stdio.h>
 #include <stdlib.h>
 
+#define ARENA_DEBUG 1
 
 void arena_dump(const Arena *arena)
 {
+    if (!ARENA_DEBUG) {
         return;
+    }
+
     MemoryBlock *block = arena->current;
     puts("@@@@@@@@@@");
     while (block) {
@@ -18,7 +23,7 @@ void arena_dump(const Arena *arena)
         printf("base:        %p\n", block->base);
         printf("size:        %u\n", block->size);
         printf("first_free:         %u\n", block->first_free);
-        printf("total_used:    %lu\n",block->arena_offset);
+        printf("arena_offset:    %lu\n",block->arena_offset);
         printf("prev: %p\n", block->prev);
         puts("-----Arena  end-");
         block = block->prev;
@@ -26,7 +31,7 @@ void arena_dump(const Arena *arena)
 }
 static void arena_test_1(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
     ArenaOptions arena_params = {};
     U32 arena_size = MB(1) + MemoryBlock_Header_Size;
     U32 commit_size = KB(8);
@@ -53,7 +58,7 @@ static void arena_test_1(void)
 
 static void arena_test_2(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
 
     ArenaOptions arena_params = {};
     U32 arena_size = MB(3) + MemoryBlock_Header_Size;
@@ -106,7 +111,7 @@ static void arena_test_2(void)
 
 static void arena_test_3(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
 
     ArenaOptions arena_params = {};
     U32 arena_size = KB(97) + MemoryBlock_Header_Size;
@@ -134,7 +139,7 @@ static void arena_test_3(void)
 // gap was not accounted for. After the fix this should allocate a larger block.
 static void arena_test_4(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
 
     U32 page_size = platform_page_size();
     ArenaOptions arena_params = {};
@@ -151,23 +156,45 @@ static void arena_test_4(void)
     U32 push_size  = block_size - MemoryBlock_Header_Size;
 
     void *ptr = arena_push(&arena, push_size, page_size);
-    Assert(IsAlignedTo((uintptr_t)ptr, page_size));
+    Assert(IsAlignedTo((UMM)ptr, page_size));
     MemoryZero(ptr, push_size);
     printf("test_4: allocated %u bytes page-aligned at %p\n", push_size, ptr);
     arena_dump(&arena);
 
     // A second page-aligned push to make sure the cursor math still holds.
     ptr = arena_push(&arena, page_size, page_size);
-    Assert(IsAlignedTo((uintptr_t)ptr, page_size));
+    Assert(IsAlignedTo((UMM)ptr, page_size));
     MemoryZero(ptr, page_size);
     printf("test_4: allocated %u bytes page-aligned at %p\n", page_size, ptr);
 
     arena_free(&arena);
 }
 
+static void arena_test_bigger_push(void)
+{
+    TRACEF("\n\n");
+    ArenaOptions arena_params = {};
+    U32 arena_size = KB(256) + MemoryBlock_Header_Size;
+    U32 commit_size = KB(8);
+    arena_params.size = arena_size;
+    arena_params.commit_size = commit_size;
+    Arena arena;
+    arena_init(&arena, &arena_params);
+    arena_dump(&arena);
+    void *ptr = arena_push(&arena, KB(256), 1);
+    MemoryZero(ptr, 512);
+    printf("Allocated ptr %p\n", ptr);
+    arena_dump(&arena);
+    ptr = arena_push(&arena, MB(4), 4);
+    MemoryZero(ptr, MB(4));
+    printf("Allocated ptr %p\n", ptr);
+    arena_dump(&arena);
+    arena_free(&arena);
+}
+
 static void arena_test_allocator_interface(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
 
     U32 page_size = platform_page_size();
     ArenaOptions arena_params = {};
@@ -185,14 +212,14 @@ static void arena_test_allocator_interface(void)
     U32 push_size  = block_size - MemoryBlock_Header_Size;
 
     void *ptr = allocator_push(&allocator, push_size, page_size);
-    Assert(IsAlignedTo((uintptr_t)ptr, page_size));
+    Assert(IsAlignedTo((UMM)ptr, page_size));
     MemoryZero(ptr, push_size);
     printf("test_4: allocated %u bytes page-aligned at %p\n", push_size, ptr);
     arena_dump(&arena);
 
     // A second page-aligned push to make sure the cursor math still holds.
     ptr = allocator_push(&allocator, page_size, page_size);
-    Assert(IsAlignedTo((uintptr_t)ptr, page_size));
+    Assert(IsAlignedTo((UMM)ptr, page_size));
     MemoryZero(ptr, page_size);
     printf("test_4: allocated %u bytes page-aligned at %p\n", page_size, ptr);
 
@@ -203,7 +230,7 @@ static void arena_test_allocator_interface(void)
 
 static void arena_test_decommit(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
 
     U32 page_size = platform_page_size();
     ArenaOptions arena_params = {};
@@ -230,7 +257,7 @@ static void arena_test_decommit(void)
 
 void miniarena_test1(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
 
     #define  arena_size  4096
     MARENA_DEFINE(arena, arena_size, 8);
@@ -269,12 +296,18 @@ struct PoolTest{
 
 void apool_test(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
 
     Arena arena;
     arena_init(&arena, &ArenaOptions_Default);
     Allocator allocator = arena_allocator(&arena);
-    APool *pool = apool_create_ex(&allocator, 16, sizeof(PoolTest), alignof(PoolTest));
+    APoolCreateOptions opts = {
+        .n_reserved= 16,
+        .element_size  = sizeof(PoolTest),
+        .element_align = alignof(PoolTest),
+        .flags = 0
+    };
+    APool *pool = apool_create_ex(&allocator, &opts);
     PoolTest *list = NULL;
     for (U32 i = 0; i < U32_Max; i++) {
         PoolTest *pt = apool_alloc(pool);
@@ -297,6 +330,7 @@ void apool_test(void)
         printf("Pooltest freeing %u\n",p->n);
         apool_free(pool, p);
     }
+    printf("Pooltest total: %u free %u\n", pool->n_elements, pool->n_free);
     list = NULL;
     for (U32 i = 0; i < U32_Max; i++) {
         PoolTest *pt = apool_alloc(pool);
@@ -316,13 +350,13 @@ void apool_test(void)
 
 void apool_test_miniarena(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
 
     MArena arena;
     U8 data[4096];
     marena_init(&arena, data, sizeof data);
     Allocator allocator = marena_allocator(&arena);
-    APool *pool = apool_create_ex(&allocator, 16, sizeof(PoolTest), alignof(PoolTest));
+    APool *pool = apool_create(&allocator, 16, PoolTest);
     PoolTest *list = NULL;
     for (U32 i = 0; i < U32_Max; i++) {
         PoolTest *pt = apool_alloc(pool);
@@ -358,19 +392,26 @@ void apool_test_miniarena(void)
     for (PoolTest *p = list; p; p = p->next){
         printf("Pooltest %u\n",p->n);
     }
+    printf("Pooltest total: %u free %u\n", pool->n_elements, pool->n_free);
+
     allocator_free_all(&allocator);
 }
 
 
 void apool_test_autogrow(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
 
     Arena arena;
     arena_init(&arena, &ArenaOptions_Default);
     Allocator allocator = arena_allocator(&arena);
-
+    APoolCreateOptions opts = {
-    APool *pool = apool_create(&allocator, APOOL_AUTOGROW, PoolTest);
+        .n_reserved = 0,
+        .element_size  = sizeof(PoolTest),
+        .element_align = alignof(PoolTest),
+        .flags = APoolFlag_AutoGrow
+    };
+    APool *pool = apool_create_ex(&allocator, &opts);
     PoolTest *list = NULL;
     for (U32 i = 0; i < 4; i++) {
         PoolTest *pt = apool_alloc(pool);
@@ -384,6 +425,7 @@ void apool_test_autogrow(void)
     for (PoolTest *p = list; p; p = p->next){
         printf("Pooltest %u\n",p->n);
     }
+    printf("Pooltest total: %u free %u\n", pool->n_elements, pool->n_free);
 
     PoolTest *next;
     U32 count = 0;
@@ -406,6 +448,7 @@ void apool_test_autogrow(void)
     for (PoolTest *p = list; p; p = p->next){
         printf("Pooltest %u\n",p->n);
     }
+    printf("Pooltest total: %u free %u\n", pool->n_elements, pool->n_free);
 
     allocator_free_all(&allocator);
     arena_free(&arena);
@@ -414,7 +457,7 @@ void apool_test_autogrow(void)
 
 void string_test_1(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
 
     Str str1 = S("Hello");
     printf("str1: %.*s len %d\n", str1.len, str1.str, str1.len);
@@ -431,7 +474,7 @@ void string_test_1(void)
 
 void string_split_test_1(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
 
     Str str1 = S("|a:bb|ccc|");
     Str sep = S("|:");
@@ -445,7 +488,7 @@ void string_split_test_1(void)
 }
 void string_test_misc(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
 
     Str str1 = S("Hello World");
     bool prefix1 = str_starts_with(str1, S("He"));
@@ -484,7 +527,7 @@ void string_test_misc(void)
 }
 void string_test_join(void)
 {
-    TRACEF("\n");
+    TRACEF("\n\n");
     Arena arena;
     arena_init(&arena, &ArenaOptions_Default);
     Allocator allocator = arena_allocator(&arena);
@@ -515,6 +558,8 @@ void print_runes(Str str)
 
 void string_test_utf8(void)
 {
+    TRACEF("\n\n");
+
     Str str1 = S("Test 1ø2 ☺");
 
     TRACEF("str1:\n");
@@ -527,8 +572,193 @@ void string_test_utf8(void)
     Str str3 = S("1Å 2\xc6" "3\xc3");
     TRACEF("str3:\n");
     print_runes(str3);
+
+    const char rune_error[] = {0xEF, 0xBF, 0xBD, 0x00} ; // RUNE_ERROR encoded in utf-8
+    Str str4 = S(rune_error);
+    TRACEF("str4:\n");
+    print_runes(str4);  // this should be a valid code point
+
 }
 
+void string_test_hex(void)
+{
+    TRACEF("\n\n");
+
+    MArena arena;
+    U8 data[4096];
+    marena_init(&arena, data, sizeof data);
+    Allocator allocator = marena_allocator(&arena);
+
+    Str str1 = S("Hello World ☺");
+
+    Str str1_encoded = str_hex_encode_alloc(&allocator, str1);
+    printf("str1_encoded: '%.*s'\n", str1_encoded.len, str1_encoded.str);
+
+    Str str1_decoded = str_hex_decode_alloc(&allocator, str1_encoded);
+    printf("str1_decoded: '%.*s'\n", str1_decoded.len, str1_decoded.str);
+
+    Str str2 = S("");
+
+    Str str2_encoded = str_hex_encode_alloc(&allocator, str2);
+    printf("str2_encoded: '%.*s'\n", str2_encoded.len, str2_encoded.str);
+
+    Str str2_decoded = str_hex_decode_alloc(&allocator, str2_encoded);
+    printf("str2_decoded: '%.*s'\n", str2_decoded.len, str2_decoded.str);
+}
+
+typedef struct StrSLNode StrSLNode;
+struct StrSLNode {
+    Str str;
+    StrSLNode *next;
+};
+
+typedef struct StrSList StrSList;
+struct StrSList {
+    StrSLNode *first;
+    StrSLNode **last;
+};
+
+void string_test_slist(void)
+{
+    TRACEF("\n\n");
+
+    Str str1 = S("one");
+    Str str2 = S("two");
+    Str str3 = S("three");
+    StrSList strlist = {};
+    StrSLNode n1 = {str1, NULL};
+    StrSLNode n2 = {str2, NULL};
+    StrSLNode n3 = {str3, NULL};
+
+    SListInit(&strlist);
+    SListAddFirst(&strlist, &n1);
+    SListAddFirst(&strlist, &n2);
+    SListAddFirst(&strlist, &n3);
+    TRACEF("List test first\n");
+
+    for (StrSLNode *n = strlist.first; n != NULL; n = n->next) {
+        printf("Node: %.*s\n", n->str.len, n->str.str);
+    }
+    TRACEF("List test first done\n");
+    SListInit(&strlist);
+    SListAddLast(&strlist, &n1);
+    SListAddLast(&strlist, &n2);
+    SListAddLast(&strlist, &n3);
+    SListRemoveFirst(&strlist);
+    TRACEF("List test last\n");
+
+    for (StrSLNode *n = strlist.first; n != NULL; n = n->next) {
+        printf("Node: %.*s\n", n->str.len, n->str.str);
+    }
+    TRACEF("List test last done\n");
+
+}
+typedef struct StrDLNode StrDLNode;
+struct StrDLNode {
+    Str str;
+    StrDLNode *next;
+    StrDLNode *prev;
+};
+
+typedef struct StrDList StrDList;
+struct StrDList {
+    StrDLNode *first;
+    StrDLNode *last;
+};
+
+void string_test_dlist(void)
+{
+    TRACEF("\n\n");
+
+    Str str1 = S("one");
+    Str str2 = S("two");
+    Str str3 = S("three");
+    Str str4 = S("four");
+    StrDList strlist = {};
+    StrDLNode n1 = {str1, NULL, NULL};
+    StrDLNode n2 = {str2, NULL, NULL};
+    StrDLNode n3 = {str3, NULL, NULL};
+    StrDLNode n4 = {str4, NULL, NULL};
+
+    DListInit(&strlist);
+    DListAddFirst(&strlist, &n1);
+    DListAddFirst(&strlist, &n2);
+    DListAddFirst(&strlist, &n3);
+    TRACEF("List test first\n");
+
+    for (StrDLNode *n = strlist.first; n != NULL; n = n->next) {
+        printf("Node: %.*s\n", n->str.len, n->str.str);
+    }
+    TRACEF("List test first done\n");
+
+    DListInit(&strlist);
+    DListAddLast(&strlist, &n2);
+    DListAddLast(&strlist, &n3);
+    DListAddLast(&strlist, &n4);
+    DListAddFirst(&strlist, &n1);
+
+    TRACEF("List test full\n");
+
+    for (StrDLNode *n = strlist.first; n != NULL; n = n->next) {
+        printf("Node: %.*s\n", n->str.len, n->str.str);
+    }
+    TRACEF("List test full done\n");
+
+    DListRemove(&strlist, &n1);
+    DListRemoveFirst(&strlist);
+    DListRemoveLast(&strlist);
+    TRACEF("List test tail removed\n");
+
+    for (StrDLNode *n = strlist.first; n != NULL; n = n->next) {
+        printf("Node: %.*s\n", n->str.len, n->str.str);
+    }
+    TRACEF("List test tail removed done\n");
+
+}
+
+void debug_lalloc(const LAlloc *allocator)
+{
+    puts("--debug start--");
+    for (LBlockHeader *b = (LBlockHeader *)allocator->base; b; b = lblock_from_offset(allocator, b->next)) {
+        printf("Block size %u free %d prev %u next %u \n", lblock_size(b), lblock_is_free(b), b->prev, b->next);
+    }
+    puts("--debug end--");
+}
+
+static void lalloc_test_basic(void)
+{
+    printf("bh size %zu\n", sizeof(LBlockHeader));
+#define SZ 64
+    LAlloc a;
+    char *ptrs[SZ] = {};
+    alignas(16) char buff[1023];
+    lalloc_init(&a, buff, sizeof(buff));
+    debug_lalloc(&a);
+    ptrs[0] = lalloc(&a, 1023 - 16 * 2 - 144);
+    ptrs[1] = lalloc(&a, 80);
+    ptrs[2] = lalloc(&a, 32);
+    debug_lalloc(&a);
+    lfree(&a, ptrs[0]);
+    ptrs[3] = lalloc(&a, 12);
+    ptrs[4] = lalloc(&a, 12);
+    debug_lalloc(&a);
+
+    lfree(&a, ptrs[1]);
+    lfree(&a, ptrs[2]);
+    lfree(&a, ptrs[3]);
+    lfree(&a, ptrs[4]);
+
+    for (int i = 0; i < SZ; i++) {
+        ptrs[i] = lalloc(&a, rand() % 256 + 1);
+    }
+
+    for (int i = 0; i < SZ; i++) {
+        // if (rand() % 2 == 0) {
+        lfree(&a, ptrs[i]);
+        // }
+    }
+    debug_lalloc(&a);
+}
 int main(int argc, char *argv[])
 {
     arena_test_1();
@@ -537,6 +767,7 @@ int main(int argc, char *argv[])
     arena_test_4();
     arena_test_decommit();
     arena_test_allocator_interface();
+    arena_test_bigger_push();
     miniarena_test1();
     apool_test();
     apool_test_autogrow();
@@ -546,6 +777,10 @@ int main(int argc, char *argv[])
     string_test_misc();
     string_test_join();
     string_test_utf8();
+    string_test_hex();
+    string_test_slist();
+    string_test_dlist();
+    lalloc_test_basic();
 
 }
 

src/pool.c

@@ -1,57 +0,0 @@
-#include "pool.h"
-
-APool *apool_create_ex(Allocator *allocator, U32 n_elements, U32 element_size, U32 element_align)
-{
-    Assert(IsPow2(element_align));
-    U32 align = element_align;
-    if (element_align < alignof(APoolElement)) {
-        element_align = alignof(APoolElement);
-    }
-    U32 size = AlignUp(element_size, element_align);
-
-    APool *pool = allocator_push_type(allocator, APool);
-    Assert(pool != NULL);
-    pool->align = align;
-    pool->element_size = size;
-    pool->allocator = allocator;
-    pool->n_elements = n_elements;
-    pool->free_list = NULL;
-    pool->start = NULL;
-
-    if (size != APOOL_AUTOGROW) {
-        Assert(n_elements <= U32_Max / size );
-        U32 total_size = n_elements * size;
-        U8 * start = allocator_push(allocator, total_size, align);
-        for (U32 i = 0; i < total_size; i += size) {
-            APoolElement *element = (APoolElement *)&start[i];
-            element->next = pool->free_list;
-            pool->free_list = element;
-        }
-        pool->start = start;
-    }
-
-    return pool;
-}
-
-void *apool_alloc(APool *pool)
-{
-    APoolElement *element = pool->free_list;
-    if (element != NULL) {
-        pool->free_list = element->next;
-    } else if(pool->n_elements == APOOL_AUTOGROW) {
-        element = allocator_push(pool->allocator, pool->element_size, pool->align);
-    }
-
-    return element;
-}
-
-void apool_free(APool *pool, void *ptr)
-{
-    if (ptr != NULL) {
-        Assert((U8 *)ptr >= pool->start);
-
-        APoolElement *element = (APoolElement *)ptr;
-        element->next = pool->free_list;
-        pool->free_list = element;
-    }
-}

src/pool.h

@@ -1,26 +0,0 @@
-#pragma once
-#include "base_core.h"
-
-typedef union APoolElement APoolElement;
-union APoolElement {
-    APoolElement *next;       // offset from start to next free block (when not allocated)
-};
-
-typedef struct APool APool;
-struct APool {
-    APoolElement *free_list;  // free list
-    U8 *start;                // start of user buffer
-    Allocator *allocator;
-    U32 element_size;         // aligned size of each block
-    U32 n_elements;           // total number of blocks
-    U32 align;                // alignment
-};
-enum {
-    APOOL_AUTOGROW = 0
-};
-
-#define apool_create(arena, n_elements, type) \
-        apool_create_ex(arena, n_elements, sizeof(type), alignof(type));
-APool *apool_create_ex(Allocator *allocator, U32 n_elements, U32 element_size, uint32_t element_align);
-void *apool_alloc(APool *pool);
-void apool_free(APool *pool, void *element);