Add lazy commit option

src/base/arena.c

@@ -10,7 +10,7 @@ static MemoryBlock *memoryblock_allocate(U32 size, U32 commit_size, U32 page_siz
 
     MemoryBlock *block = memory_reserve(size);
     Assert(block != NULL);
-    memory_commit(block, commit_size);
+    memory_commit(block, commit_size, false);
 
     ASAN_POISON_MEMORY_REGION(block, size);
     ASAN_UNPOISON_MEMORY_REGION(block, MemoryBlock_Header_Size);
@@ -38,7 +38,7 @@ void arena_init(Arena *arena, const ArenaOptions *opts)
     MemoryBlock *block  = memoryblock_allocate(aligned_size, aligned_commit_size, page_size);
 
     arena->flags            = opts->flags;
-    arena->temp_level = 0;
+    arena->temp_level       = 0;
     arena->current          = block;
     arena->block_size       = aligned_size;
     arena->commit_size      = aligned_commit_size;
@@ -61,7 +61,6 @@ void *arena_push(Arena *arena, U32 size, U32 align)
 {
     Assert(IsPow2(align));
 
-
     MemoryBlock *block = arena->current;
     U32 start_pos = AlignUp(block->first_free, align);  // might align past block->size, handled in below check
     U32 available = block->size - start_pos;
@@ -97,7 +96,7 @@ void *arena_push(Arena *arena, U32 size, U32 align)
         U32 aligned_size      = RoundUpToMultiple(aligned_end_pos - block->committed_end, arena->commit_size);
         aligned_size = Min(aligned_size, block->size - block->committed_end);
 
-        memory_commit(&block->base[block->committed_end], aligned_size);
+        memory_commit(&block->base[block->committed_end], aligned_size, false);
         block->committed_end  += aligned_size;
         Assert(block->committed_end <= block->size);
     }
@@ -144,8 +143,6 @@ void arena_trim(Arena *arena)
     U32 aligned_pos = RoundUpToMultiple(block->first_free, arena->commit_size);
     if (aligned_pos < block->committed_end) {
         void *trim_start = &block->base[aligned_pos];
-        // Note(nos): Should we unpoision ?
-        //ASAN_UNPOISON_MEMORY_REGION(trim_start, block->committed_end);
         memory_decommit(trim_start, block->committed_end - aligned_pos);
         block->committed_end = aligned_pos;
     }

src/base/arena.h

@@ -5,7 +5,7 @@
 typedef U32 ArenaFlags;
 typedef struct ArenaOptions ArenaOptions;
 enum {
-    ArenaFlag_NoDecommit = 1, // operations which shrinks the arena does not decommit memory.
+    ArenaFlag_NoDecommit = (1 << 0), // operations which shrinks the arena does not decommit memory.
                               // (arena_trim() can still decommit when called explicitly)
 };
 struct ArenaOptions {
@@ -89,7 +89,7 @@ static INLINE void *arena_allocator_func(Allocator *allocator, AllocOperation op
             return NULL;
     }
 
-    Assert(false);
+    ShouldNotHappen;
 }
 
 static INLINE Allocator arena_allocator(Arena *arena)

src/base/core.h

@@ -24,6 +24,8 @@ typedef uint8_t     U8;
 typedef uint16_t    U16;
 typedef uint32_t    U32;
 typedef uint64_t    U64;
+typedef uintptr_t   UMM;
+typedef intptr_t    IMM;
 typedef float       F32;
 typedef double      F64;
 #if defined(__FLT128_MANT_DIG__)
@@ -421,7 +423,7 @@ static INLINE void  allocator_free_all(Allocator *allocator)
 */
 
 /**
- * Initialize a list
+ * Initialize a list, can also just be zero initialized
  *
  * @param list struct pointer with a *first and a **last member to an entry struct
  */
@@ -482,7 +484,7 @@ do {                                     \
 */
 
 /**
- * Initialize a list
+ * Initialize a list, can also just be zero initialized
  *
  * @param list struct pointer with a *first and a *last member to an entry struct
  */

src/base/lalloc.c

@@ -0,0 +1,176 @@
+#include "lalloc.h"
+
+#define BLOCK_ALIGNMENT sizeof(LBlockHeader)
+#define BLOCK_SPLIT_MIN_SIZE BLOCK_ALIGNMENT
+#define NULL_BLOCK (0xffffffff)
+#define BLOCK_FREE ((U32)1)
+
+static inline void lblock_set_free(LBlockHeader *block)
+{
+    block->size_and_flags |= BLOCK_FREE;
+}
+
+static inline void lblock_set_used(LBlockHeader *block)
+{
+    block->size_and_flags &= ~BLOCK_FREE;
+}
+
+static inline void lblock_set_size(LBlockHeader *block, U32 size)
+{
+    U32 flags = block->size_and_flags & BLOCK_FREE;
+    Assert((size & BLOCK_FREE) == 0);
+    block->size_and_flags = size | flags;
+}
+
+U32 lblock_size(const LBlockHeader *block)
+{
+    return block->size_and_flags & ~BLOCK_FREE;
+}
+
+int lblock_is_free(const LBlockHeader *block)
+{
+    return (block->size_and_flags & BLOCK_FREE) != 0;
+}
+
+LBlockHeader *lblock_from_offset(const LAlloc *allocator, U32 offset)
+{
+    if (offset == NULL_BLOCK) {
+        return NULL;
+    }
+
+    return (LBlockHeader *)((UMM)allocator->base + offset);
+}
+
+static inline U32 lbh_offset(LAlloc *allocator, LBlockHeader *block)
+{
+    if (block == NULL) {
+        return NULL_BLOCK;
+    }
+
+    return (U32)((UMM)block - (uintptr_t)allocator->base);
+}
+
+static void lmaybe_split_block(LAlloc *allocator, LBlockHeader *block, U32 size)
+{
+    // this function assumes size is already aligned
+
+    U32 bsize = lblock_size(block);
+    if (size > U32_Max - (sizeof(LBlockHeader) + BLOCK_SPLIT_MIN_SIZE)) {
+        return; // size is so large that size + header + min would overflow U32 anyway
+    }
+    if (bsize < size + sizeof(LBlockHeader) + BLOCK_SPLIT_MIN_SIZE) {
+        return; // no gain from splitting this block
+    }
+
+    U8 *base = (U8 *)block;
+    LBlockHeader *new_block = (LBlockHeader *)(base + size + sizeof(LBlockHeader) );
+
+    U32 new_size = bsize - size - sizeof(LBlockHeader);
+
+    lblock_set_size(block, size);
+    lblock_set_used(block);
+    lblock_set_size(new_block, new_size);
+    lblock_set_free(new_block);
+    new_block->next = block->next;
+    new_block->prev = lbh_offset(allocator, block);
+
+    LBlockHeader *next = lblock_from_offset(allocator, new_block->next);
+    if (next) {
+        next->prev = lbh_offset(allocator, new_block);
+    }
+
+    block->next = lbh_offset(allocator, new_block);
+}
+
+static void lmerge_with_next(LAlloc *allocator, LBlockHeader *block)
+{
+    LBlockHeader *next = lblock_from_offset(allocator, block->next);
+    if (next == NULL || !lblock_is_free(next)) {
+        return;
+    }
+
+    // block and next are both existing blocks within the same backing store, whose
+    // total size is asserted in lalloc_init to be < 0xfffffff0, so this can't overflow
+    // unless allocator state is already corrupted.
+    Assert(lblock_size(block) <= U32_Max - sizeof(LBlockHeader) - lblock_size(next));
+    U32 new_size = lblock_size(block) + sizeof(LBlockHeader) + lblock_size(next);
+
+    lblock_set_size(block, new_size);
+    block->next = next->next;
+    LBlockHeader *nextnext = lblock_from_offset(allocator, next->next);
+    if (nextnext != NULL) {
+         nextnext->prev = lbh_offset(allocator, block);
+    }
+    // note - caller have to adjust next_alloc on return
+}
+
+void lalloc_init(LAlloc *allocator, void *backing_store, U32 size)
+{
+    // backing store must be aligned, otherwise  all alignment guarantees are lies
+
+    Assert(IsAlignedTo((UMM)backing_store, BLOCK_ALIGNMENT));
+    Assert(size >= sizeof(LBlockHeader));
+    Assert(size < 0xfffffff0);
+
+    allocator->base = backing_store;
+    allocator->size = size;
+
+    LBlockHeader *first = backing_store;
+    // We need to keep size aligned to at least 2 as we reuse the LSB for the FREE flag
+    U32 usable = AlignDown(size, 2) - sizeof(LBlockHeader);
+    lblock_set_free(first);
+    lblock_set_size(first, usable);
+    first->next = NULL_BLOCK;
+    first->prev = NULL_BLOCK;
+
+    allocator->next_alloc = first;
+}
+
+void *lalloc(LAlloc *allocator, U32 size)
+{
+    if (size > U32_Max - (BLOCK_ALIGNMENT - 1)) { // -1 so AlignUp below doesn't overflow
+        return NULL;
+    }
+    size = AlignUp(size, BLOCK_ALIGNMENT);
+
+    LBlockHeader *start = allocator->next_alloc;
+    LBlockHeader *curr = start;
+
+    do {
+        if (lblock_is_free(curr) && lblock_size(curr) >= size) {
+            lmaybe_split_block(allocator, curr, size);
+            lblock_set_used(curr);
+            LBlockHeader *next = lblock_from_offset(allocator, curr->next);
+            allocator->next_alloc = next ? next : (LBlockHeader *)allocator->base;
+            return (U8 *)curr + sizeof(LBlockHeader);
+        }
+        LBlockHeader *next = lblock_from_offset(allocator, curr->next);
+        curr = next ? next : (LBlockHeader *)allocator->base;
+    } while (curr != start);
+
+    return NULL;
+}
+
+void lfree(LAlloc *allocator, void *ptr)
+{
+    if (ptr == NULL) {
+        return;
+    }
+
+    LBlockHeader *block = (LBlockHeader *)((U8 *)ptr - sizeof(LBlockHeader));
+
+    Assert((U8 *)block >= allocator->base && (U8 *)ptr < allocator->base + allocator->size);
+    // double free detection
+    Assert(!lblock_is_free(block));
+
+    lblock_set_free(block);
+    lmerge_with_next(allocator, block);
+
+    LBlockHeader *prev = lblock_from_offset(allocator, block->prev);
+    if (prev != NULL && lblock_is_free(prev)) {
+        lmerge_with_next(allocator, prev);
+        allocator->next_alloc = prev;
+    } else {
+        allocator->next_alloc = block;
+    }
+}

src/base/lalloc.h

@@ -0,0 +1,41 @@
+#pragma once
+#include "core.h"
+
+// Internal header added in front of each allocation
+typedef struct LBlockHeader LBlockHeader;
+struct LBlockHeader {
+    U32 next;            // offset from base
+    U32 prev;            // offset from base
+    U32 size_and_flags;  // size | free_bit
+    // Padding to ensure we always return 16 byte aligned pointers.
+    uint8_t pad[4];
+};
+_Static_assert(sizeof(LBlockHeader) == 16, "sizeof(BlockHeader) is not 16");
+
+/**
+ * General purpose allocator from a pre-defined backing store
+ * - Allocated pointers are aligned to 16 bytes.
+ * - Backing buffer must be aligned to 16 bytes
+ * - 4Gb - 16 is max for the allocator backing buffer.
+ * - Allocation of > 0 bytes returns a size aligned up to 16 bytes
+ * - Allocation of 0 bytes returns a valid pointer with 0 usable space
+ * The allocator maintains a doubly linked list of blocks,
+ * when blocks are free'd, adjacent blocks are merged to keep.
+ * This is also known as a buddy allocator
+ * fragmentation to a minimum
+ */
+typedef struct LAlloc LAlloc;
+struct LAlloc {
+    uint8_t *base;              // backing store
+    U32 size;                   // total size
+    LBlockHeader *next_alloc;   // start of free block search
+};
+
+void lalloc_init(LAlloc *allocator, void *backing_store, U32 size);
+
+void *lalloc(LAlloc *allocator, U32 size);
+void lfree(LAlloc *allocator, void *ptr);
+
+LBlockHeader *lblock_from_offset(const LAlloc *allocator, U32 offset);
+U32 lblock_size(const LBlockHeader *block);
+int lblock_is_free(const LBlockHeader *block);

src/base/marena.h

@@ -110,7 +110,7 @@ static INLINE void *marena_allocator_func(Allocator *allocator, AllocOperation o
             return NULL;
     }
 
-    Assert(false);
+    ShouldNotHappen;
 }
 
 static INLINE Allocator marena_allocator(MArena *arena)

src/base/platform.c

@@ -73,7 +73,7 @@ void *memory_allocate_and_commit(size_t size)
 void memory_free(void *mem, size_t size)
 {
     TRACEF("        memory_free     size=%7zu mem=%p\n", size, mem);
-    Assert(IsPageAligned((uintptr_t)mem));
+    Assert(IsPageAligned((UMM)mem));
     Assert(IsPageAligned(size));
     int rc = munmap(mem, size);
 
@@ -91,23 +91,27 @@ void *memory_reserve(size_t size)
     return mem;
 }
 
-void memory_commit(void *mem, size_t size)
+void memory_commit(void *mem, size_t size, bool lazy)
 {
-    Assert(IsPageAligned((uintptr_t)mem));
+    Assert(IsPageAligned((UMM)mem));
     Assert(IsPageAligned(size));
     TRACEF("memory_commit   size=%7zu mem=%p\n", size, mem);
 
     int rc;
     rc = mprotect(mem, size, PROT_READ | PROT_WRITE);
     Assert(rc == 0);
-    rc = madvise(mem, size, MADV_WILLNEED);
+    if (lazy) {
+        rc = madvise(mem, size, MADV_POPULATE_WRITE);
+    } else {
+        rc = madvise(mem, size, MADV_WILLNEED);
+    }
     Assert(rc == 0);
 }
 
 void memory_decommit(void *mem, size_t size)
 {
     TRACEF("memory_decommit size=%7zu mem=%p\n", size, mem);
-    Assert(IsPageAligned((uintptr_t)mem));
+    Assert(IsPageAligned((UMM)mem));
     Assert(IsPageAligned(size));
     int rc;
     rc = mprotect(mem, size, PROT_NONE);

src/base/platform.h

@@ -11,6 +11,7 @@ U32 platform_page_size(void);
 // Memory functions expect mem and size to be page aligned
 void *memory_allocate_and_commit(size_t size);
 void *memory_reserve(size_t size);
-void memory_commit(void *mem, size_t size);
+
+void memory_commit(void *mem, size_t size, bool lazy);
 void memory_decommit(void *mem, size_t size);
 void memory_free(void *mem, size_t size);

src/base/pool.c

@@ -3,12 +3,12 @@
 APool *apool_create_ex(Allocator *allocator, const APoolCreateOptions *options)
 {
     Assert(IsPow2(options->element_align));
+    Assert(options->n_reserved > 0 || options->flags & APoolFlag_AutoGrow);
     U32 element_align = options->element_align;
     if (element_align < alignof(APoolElement)) {
         element_align = alignof(APoolElement);
     }
     U32 element_size = AlignUp(options->element_size, options->element_align);
-
     APool *pool = allocator_push_type(allocator, APool);
     Assert(pool != NULL);
     pool->element_align= element_align;

src/base/pool.h

@@ -32,4 +32,12 @@ struct APoolCreateOptions {
         apool_create_ex(arena, &(APoolCreateOptions){sizeof(type), alignof(type), n_reserved, 0})
 APool *apool_create_ex(Allocator *allocator, const APoolCreateOptions *options);
 void *apool_alloc(APool *pool);
+INLINE void *apool_allocz(APool *pool)
+{
+    void *ptr = apool_alloc(pool);
+    if (ptr != NULL) {
+        MemoryZero(ptr, pool->element_size);
+    }
+    return ptr;
+}
 void apool_free(APool *pool, void *element);

src/main.c

@@ -4,13 +4,18 @@
 #include "base/platform.h"
 #include "base/pool.h"
 #include "base/string.h"
+#include "base/lalloc.h"
 #include <stdio.h>
 #include <stdlib.h>
 
+#define ARENA_DEBUG 1
 
 void arena_dump(const Arena *arena)
 {
-    return;
+    if (!ARENA_DEBUG) {
+        return;
+    }
+
     MemoryBlock *block = arena->current;
     puts("@@@@@@@@@@");
     while (block) {
@@ -18,7 +23,7 @@ void arena_dump(const Arena *arena)
         printf("base:        %p\n", block->base);
         printf("size:        %u\n", block->size);
         printf("first_free:         %u\n", block->first_free);
-        printf("total_used:    %lu\n",block->arena_offset);
+        printf("arena_offset:    %lu\n",block->arena_offset);
         printf("prev: %p\n", block->prev);
         puts("-----Arena  end-");
         block = block->prev;
@@ -151,20 +156,42 @@ static void arena_test_4(void)
     U32 push_size  = block_size - MemoryBlock_Header_Size;
 
     void *ptr = arena_push(&arena, push_size, page_size);
-    Assert(IsAlignedTo((uintptr_t)ptr, page_size));
+    Assert(IsAlignedTo((UMM)ptr, page_size));
     MemoryZero(ptr, push_size);
     printf("test_4: allocated %u bytes page-aligned at %p\n", push_size, ptr);
     arena_dump(&arena);
 
     // A second page-aligned push to make sure the cursor math still holds.
     ptr = arena_push(&arena, page_size, page_size);
-    Assert(IsAlignedTo((uintptr_t)ptr, page_size));
+    Assert(IsAlignedTo((UMM)ptr, page_size));
     MemoryZero(ptr, page_size);
     printf("test_4: allocated %u bytes page-aligned at %p\n", page_size, ptr);
 
     arena_free(&arena);
 }
 
+static void arena_test_bigger_push(void)
+{
+    TRACEF("\n\n");
+    ArenaOptions arena_params = {};
+    U32 arena_size = KB(256) + MemoryBlock_Header_Size;
+    U32 commit_size = KB(8);
+    arena_params.size = arena_size;
+    arena_params.commit_size = commit_size;
+    Arena arena;
+    arena_init(&arena, &arena_params);
+    arena_dump(&arena);
+    void *ptr = arena_push(&arena, KB(256), 1);
+    MemoryZero(ptr, 512);
+    printf("Allocated ptr %p\n", ptr);
+    arena_dump(&arena);
+    ptr = arena_push(&arena, MB(4), 4);
+    MemoryZero(ptr, MB(4));
+    printf("Allocated ptr %p\n", ptr);
+    arena_dump(&arena);
+    arena_free(&arena);
+}
+
 static void arena_test_allocator_interface(void)
 {
     TRACEF("\n\n");
@@ -185,14 +212,14 @@ static void arena_test_allocator_interface(void)
     U32 push_size  = block_size - MemoryBlock_Header_Size;
 
     void *ptr = allocator_push(&allocator, push_size, page_size);
-    Assert(IsAlignedTo((uintptr_t)ptr, page_size));
+    Assert(IsAlignedTo((UMM)ptr, page_size));
     MemoryZero(ptr, push_size);
     printf("test_4: allocated %u bytes page-aligned at %p\n", push_size, ptr);
     arena_dump(&arena);
 
     // A second page-aligned push to make sure the cursor math still holds.
     ptr = allocator_push(&allocator, page_size, page_size);
-    Assert(IsAlignedTo((uintptr_t)ptr, page_size));
+    Assert(IsAlignedTo((UMM)ptr, page_size));
     MemoryZero(ptr, page_size);
     printf("test_4: allocated %u bytes page-aligned at %p\n", page_size, ptr);
 
@@ -689,6 +716,49 @@ void string_test_dlist(void)
 
 }
 
+void debug_lalloc(const LAlloc *allocator)
+{
+    puts("--debug start--");
+    for (LBlockHeader *b = (LBlockHeader *)allocator->base; b; b = lblock_from_offset(allocator, b->next)) {
+        printf("Block size %u free %d prev %u next %u \n", lblock_size(b), lblock_is_free(b), b->prev, b->next);
+    }
+    puts("--debug end--");
+}
+
+static void lalloc_test_basic(void)
+{
+    printf("bh size %zu\n", sizeof(LBlockHeader));
+#define SZ 64
+    LAlloc a;
+    char *ptrs[SZ] = {};
+    alignas(16) char buff[1023];
+    lalloc_init(&a, buff, sizeof(buff));
+    debug_lalloc(&a);
+    ptrs[0] = lalloc(&a, 1023 - 16 * 2 - 144);
+    ptrs[1] = lalloc(&a, 80);
+    ptrs[2] = lalloc(&a, 32);
+    debug_lalloc(&a);
+    lfree(&a, ptrs[0]);
+    ptrs[3] = lalloc(&a, 12);
+    ptrs[4] = lalloc(&a, 12);
+    debug_lalloc(&a);
+
+    lfree(&a, ptrs[1]);
+    lfree(&a, ptrs[2]);
+    lfree(&a, ptrs[3]);
+    lfree(&a, ptrs[4]);
+
+    for (int i = 0; i < SZ; i++) {
+        ptrs[i] = lalloc(&a, rand() % 256 + 1);
+    }
+
+    for (int i = 0; i < SZ; i++) {
+        // if (rand() % 2 == 0) {
+        lfree(&a, ptrs[i]);
+        // }
+    }
+    debug_lalloc(&a);
+}
 int main(int argc, char *argv[])
 {
     arena_test_1();
@@ -697,6 +767,7 @@ int main(int argc, char *argv[])
     arena_test_4();
     arena_test_decommit();
     arena_test_allocator_interface();
+    arena_test_bigger_push();
     miniarena_test1();
     apool_test();
     apool_test_autogrow();
@@ -709,6 +780,7 @@ int main(int argc, char *argv[])
     string_test_hex();
     string_test_slist();
     string_test_dlist();
+    lalloc_test_basic();
 
 }