v/thirdparty/walloc/walloc.c

// walloc.c: a small malloc implementation for use in WebAssembly targets
// Copyright (c) 2020 Igalia, S.L.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

typedef __SIZE_TYPE__ size_t;
typedef __UINTPTR_TYPE__ uintptr_t;
typedef __UINT8_TYPE__ uint8_t;

#define NULL ((void *)0)

#define STATIC_ASSERT_EQ(a, b) _Static_assert((a) == (b), "eq")

#ifndef NDEBUG
#define ASSERT(x)             \
    do                        \
    {                         \
        if (!(x))             \
            __builtin_trap(); \
    } while (0)
#else
#define ASSERT(x) \
    do            \
    {             \
    } while (0)
#endif
#define ASSERT_EQ(a, b) ASSERT((a) == (b))

static inline size_t max(size_t a, size_t b)
{
    return a < b ? b : a;
}
static inline uintptr_t align(uintptr_t val, uintptr_t alignment)
{
    return (val + alignment - 1) & ~(alignment - 1);
}
#define ASSERT_ALIGNED(x, y) ASSERT((x) == align((x), y))

#define CHUNK_SIZE 256
#define CHUNK_SIZE_LOG_2 8
#define CHUNK_MASK (CHUNK_SIZE - 1)
STATIC_ASSERT_EQ(CHUNK_SIZE, 1 << CHUNK_SIZE_LOG_2);

#define PAGE_SIZE 65536
#define PAGE_SIZE_LOG_2 16
#define PAGE_MASK (PAGE_SIZE - 1)
STATIC_ASSERT_EQ(PAGE_SIZE, 1 << PAGE_SIZE_LOG_2);

#define CHUNKS_PER_PAGE 256
STATIC_ASSERT_EQ(PAGE_SIZE, CHUNK_SIZE *CHUNKS_PER_PAGE);

#define GRANULE_SIZE 8
#define GRANULE_SIZE_LOG_2 3
#define LARGE_OBJECT_THRESHOLD 256
#define LARGE_OBJECT_GRANULE_THRESHOLD 32

STATIC_ASSERT_EQ(GRANULE_SIZE, 1 << GRANULE_SIZE_LOG_2);
STATIC_ASSERT_EQ(LARGE_OBJECT_THRESHOLD,
                 LARGE_OBJECT_GRANULE_THRESHOLD *GRANULE_SIZE);

struct chunk
{
    char data[CHUNK_SIZE];
};

// There are small object pages for allocations of these sizes.
#define FOR_EACH_SMALL_OBJECT_GRANULES(M) \
    M(1)                                  \
    M(2) M(3) M(4) M(5) M(6) M(8) M(10) M(16) M(32)

enum chunk_kind
{
#define DEFINE_SMALL_OBJECT_CHUNK_KIND(i) GRANULES_##i,
    FOR_EACH_SMALL_OBJECT_GRANULES(DEFINE_SMALL_OBJECT_CHUNK_KIND)
#undef DEFINE_SMALL_OBJECT_CHUNK_KIND

        SMALL_OBJECT_CHUNK_KINDS,
    FREE_LARGE_OBJECT = 254,
    LARGE_OBJECT = 255
};

static const uint8_t small_object_granule_sizes[] =
    {
#define SMALL_OBJECT_GRANULE_SIZE(i) i,
        FOR_EACH_SMALL_OBJECT_GRANULES(SMALL_OBJECT_GRANULE_SIZE)
#undef SMALL_OBJECT_GRANULE_SIZE
};

static enum chunk_kind granules_to_chunk_kind(unsigned granules)
{
#define TEST_GRANULE_SIZE(i) \
    if (granules <= i)       \
        return GRANULES_##i;
    FOR_EACH_SMALL_OBJECT_GRANULES(TEST_GRANULE_SIZE);
#undef TEST_GRANULE_SIZE
    return LARGE_OBJECT;
}

static unsigned chunk_kind_to_granules(enum chunk_kind kind)
{
    switch (kind)
    {
#define CHUNK_KIND_GRANULE_SIZE(i) \
    case GRANULES_##i:             \
        return i;
        FOR_EACH_SMALL_OBJECT_GRANULES(CHUNK_KIND_GRANULE_SIZE);
#undef CHUNK_KIND_GRANULE_SIZE
    default:
        return -1;
    }
}

// Given a pointer P returned by malloc(), we get a header pointer via
// P&~PAGE_MASK, and a chunk index via (P&PAGE_MASK)/CHUNKS_PER_PAGE.  If
// chunk_kinds[chunk_idx] is [FREE_]LARGE_OBJECT, then the pointer is a large
// object, otherwise the kind indicates the size in granules of the objects in
// the chunk.
struct page_header
{
    uint8_t chunk_kinds[CHUNKS_PER_PAGE];
};

struct page
{
    union
    {
        struct page_header header;
        struct chunk chunks[CHUNKS_PER_PAGE];
    };
};

#define PAGE_HEADER_SIZE (sizeof(struct page_header))
#define FIRST_ALLOCATABLE_CHUNK 1
STATIC_ASSERT_EQ(PAGE_HEADER_SIZE, FIRST_ALLOCATABLE_CHUNK *CHUNK_SIZE);

static struct page *get_page(void *ptr)
{
    return (struct page *)(char *)(((uintptr_t)ptr) & ~PAGE_MASK);
}
static unsigned get_chunk_index(void *ptr)
{
    return (((uintptr_t)ptr) & PAGE_MASK) / CHUNK_SIZE;
}

struct freelist
{
    struct freelist *next;
};

struct large_object
{
    struct large_object *next;
    size_t size;
};

#define LARGE_OBJECT_HEADER_SIZE (sizeof(struct large_object))

static inline void *get_large_object_payload(struct large_object *obj)
{
    return ((char *)obj) + LARGE_OBJECT_HEADER_SIZE;
}
static inline struct large_object *get_large_object(void *ptr)
{
    return (struct large_object *)(((char *)ptr) - LARGE_OBJECT_HEADER_SIZE);
}

static struct freelist *small_object_freelists[SMALL_OBJECT_CHUNK_KINDS];
static struct large_object *large_objects;

extern void __heap_base;
static size_t walloc_heap_size;

static struct page *
allocate_pages(size_t payload_size, size_t *n_allocated)
{
    size_t needed = payload_size + PAGE_HEADER_SIZE;
    size_t heap_size = __builtin_wasm_memory_size(0) * PAGE_SIZE;
    uintptr_t base = heap_size;
    uintptr_t preallocated = 0, grow = 0;

    if (!walloc_heap_size)
    {
        // We are allocating the initial pages, if any.  We skip the first 64 kB,
        // then take any additional space up to the memory size.
        uintptr_t heap_base = align((uintptr_t)&__heap_base, PAGE_SIZE);
        preallocated = heap_size - heap_base; // Preallocated pages.
        walloc_heap_size = preallocated;
        base -= preallocated;
    }

    if (preallocated < needed)
    {
        // Always grow the walloc heap at least by 50%.
        grow = align(max(walloc_heap_size / 2, needed - preallocated),
                     PAGE_SIZE);
        ASSERT(grow);
        if (__builtin_wasm_memory_grow(0, grow >> PAGE_SIZE_LOG_2) == -1)
        {
            return NULL;
        }
        walloc_heap_size += grow;
    }

    struct page *ret = (struct page *)base;
    size_t size = grow + preallocated;
    ASSERT(size);
    ASSERT_ALIGNED(size, PAGE_SIZE);
    *n_allocated = size / PAGE_SIZE;
    return ret;
}

static char *
allocate_chunk(struct page *page, unsigned idx, enum chunk_kind kind)
{
    page->header.chunk_kinds[idx] = kind;
    return page->chunks[idx].data;
}

// It's possible for splitting to produce a large object of size 248 (256 minus
// the header size) -- i.e. spanning a single chunk.  In that case, push the
// chunk back on the GRANULES_32 small object freelist.
static void maybe_repurpose_single_chunk_large_objects_head(void)
{
    if (large_objects->size < CHUNK_SIZE)
    {
        unsigned idx = get_chunk_index(large_objects);
        char *ptr = allocate_chunk(get_page(large_objects), idx, GRANULES_32);
        large_objects = large_objects->next;
        struct freelist *head = (struct freelist *)ptr;
        head->next = small_object_freelists[GRANULES_32];
        small_object_freelists[GRANULES_32] = head;
    }
}

// If there have been any large-object frees since the last large object
// allocation, go through the freelist and merge any adjacent objects.
static int pending_large_object_compact = 0;
static struct large_object **
maybe_merge_free_large_object(struct large_object **prev)
{
    struct large_object *obj = *prev;
    while (1)
    {
        char *end = get_large_object_payload(obj) + obj->size;
        ASSERT_ALIGNED((uintptr_t)end, CHUNK_SIZE);
        unsigned chunk = get_chunk_index(end);
        if (chunk < FIRST_ALLOCATABLE_CHUNK)
        {
            // Merging can't create a large object that newly spans the header chunk.
            // This check also catches the end-of-heap case.
            return prev;
        }
        struct page *page = get_page(end);
        if (page->header.chunk_kinds[chunk] != FREE_LARGE_OBJECT)
        {
            return prev;
        }
        struct large_object *next = (struct large_object *)end;

        struct large_object **prev_prev = &large_objects, *walk = large_objects;
        while (1)
        {
            ASSERT(walk);
            if (walk == next)
            {
                obj->size += LARGE_OBJECT_HEADER_SIZE + walk->size;
                *prev_prev = walk->next;
                if (prev == &walk->next)
                {
                    prev = prev_prev;
                }
                break;
            }
            prev_prev = &walk->next;
            walk = walk->next;
        }
    }
}
static void
maybe_compact_free_large_objects(void)
{
    if (pending_large_object_compact)
    {
        pending_large_object_compact = 0;
        struct large_object **prev = &large_objects;
        while (*prev)
        {
            prev = &(*maybe_merge_free_large_object(prev))->next;
        }
    }
}

// Allocate a large object with enough space for SIZE payload bytes.  Returns a
// large object with a header, aligned on a chunk boundary, whose payload size
// may be larger than SIZE, and whose total size (header included) is
// chunk-aligned.  Either a suitable allocation is found in the large object
// freelist, or we ask the OS for some more pages and treat those pages as a
// large object.  If the allocation fits in that large object and there's more
// than an aligned chunk's worth of data free at the end, the large object is
// split.
//
// The return value's corresponding chunk in the page as starting a large
// object.
static struct large_object *
allocate_large_object(size_t size)
{
    maybe_compact_free_large_objects();
    struct large_object *best = NULL, **best_prev = &large_objects;
    size_t best_size = -1;
    for (struct large_object **prev = &large_objects, *walk = large_objects;
         walk;
         prev = &walk->next, walk = walk->next)
    {
        if (walk->size >= size && walk->size < best_size)
        {
            best_size = walk->size;
            best = walk;
            best_prev = prev;
            if (best_size + LARGE_OBJECT_HEADER_SIZE == align(size + LARGE_OBJECT_HEADER_SIZE, CHUNK_SIZE))
                // Not going to do any better than this; just return it.
                break;
        }
    }

    if (!best)
    {
        // The large object freelist doesn't have an object big enough for this
        // allocation.  Allocate one or more pages from the OS, and treat that new
        // sequence of pages as a fresh large object.  It will be split if
        // necessary.
        size_t size_with_header = size + sizeof(struct large_object);
        size_t n_allocated = 0;
        struct page *page = allocate_pages(size_with_header, &n_allocated);
        if (!page)
        {
            return NULL;
        }
        char *ptr = allocate_chunk(page, FIRST_ALLOCATABLE_CHUNK, LARGE_OBJECT);
        best = (struct large_object *)ptr;
        size_t page_header = ptr - ((char *)page);
        best->next = large_objects;
        best->size = best_size =
            n_allocated * PAGE_SIZE - page_header - LARGE_OBJECT_HEADER_SIZE;
        ASSERT(best_size >= size_with_header);
    }

    allocate_chunk(get_page(best), get_chunk_index(best), LARGE_OBJECT);

    struct large_object *next = best->next;
    *best_prev = next;

    size_t tail_size = (best_size - size) & ~CHUNK_MASK;
    if (tail_size)
    {
        // The best-fitting object has 1 or more aligned chunks free after the
        // requested allocation; split the tail off into a fresh aligned object.
        struct page *start_page = get_page(best);
        char *start = get_large_object_payload(best);
        char *end = start + best_size;

        if (start_page == get_page(end - tail_size - 1))
        {
            // The allocation does not span a page boundary; yay.
            ASSERT_ALIGNED((uintptr_t)end, CHUNK_SIZE);
        }
        else if (size < PAGE_SIZE - LARGE_OBJECT_HEADER_SIZE - CHUNK_SIZE)
        {
            // If the allocation itself smaller than a page, split off the head, then
            // fall through to maybe split the tail.
            ASSERT_ALIGNED((uintptr_t)end, PAGE_SIZE);
            size_t first_page_size = PAGE_SIZE - (((uintptr_t)start) & PAGE_MASK);
            struct large_object *head = best;
            allocate_chunk(start_page, get_chunk_index(start), FREE_LARGE_OBJECT);
            head->size = first_page_size;
            head->next = large_objects;
            large_objects = head;

            maybe_repurpose_single_chunk_large_objects_head();

            struct page *next_page = start_page + 1;
            char *ptr = allocate_chunk(next_page, FIRST_ALLOCATABLE_CHUNK, LARGE_OBJECT);
            best = (struct large_object *)ptr;
            best->size = best_size = best_size - first_page_size - CHUNK_SIZE - LARGE_OBJECT_HEADER_SIZE;
            ASSERT(best_size >= size);
            start = get_large_object_payload(best);
            tail_size = (best_size - size) & ~CHUNK_MASK;
        }
        else
        {
            // A large object that spans more than one page will consume all of its
            // tail pages.  Therefore if the split traverses a page boundary, round up
            // to page size.
            ASSERT_ALIGNED((uintptr_t)end, PAGE_SIZE);
            size_t first_page_size = PAGE_SIZE - (((uintptr_t)start) & PAGE_MASK);
            size_t tail_pages_size = align(size - first_page_size, PAGE_SIZE);
            size = first_page_size + tail_pages_size;
            tail_size = best_size - size;
        }
        best->size -= tail_size;

        unsigned tail_idx = get_chunk_index(end - tail_size);
        while (tail_idx < FIRST_ALLOCATABLE_CHUNK && tail_size)
        {
            // We would be splitting in a page header; don't do that.
            tail_size -= CHUNK_SIZE;
            tail_idx++;
        }

        if (tail_size)
        {
            struct page *page = get_page(end - tail_size);
            char *tail_ptr = allocate_chunk(page, tail_idx, FREE_LARGE_OBJECT);
            struct large_object *tail = (struct large_object *)tail_ptr;
            tail->next = large_objects;
            tail->size = tail_size - LARGE_OBJECT_HEADER_SIZE;
            ASSERT_ALIGNED((uintptr_t)(get_large_object_payload(tail) + tail->size), CHUNK_SIZE);
            large_objects = tail;

            maybe_repurpose_single_chunk_large_objects_head();
        }
    }

    ASSERT_ALIGNED((uintptr_t)(get_large_object_payload(best) + best->size), CHUNK_SIZE);
    return best;
}

static struct freelist *
obtain_small_objects(enum chunk_kind kind)
{
    struct freelist **whole_chunk_freelist = &small_object_freelists[GRANULES_32];
    void *chunk;
    if (*whole_chunk_freelist)
    {
        chunk = *whole_chunk_freelist;
        *whole_chunk_freelist = (*whole_chunk_freelist)->next;
    }
    else
    {
        chunk = allocate_large_object(0);
        if (!chunk)
        {
            return NULL;
        }
    }
    char *ptr = allocate_chunk(get_page(chunk), get_chunk_index(chunk), kind);
    char *end = ptr + CHUNK_SIZE;
    struct freelist *next = NULL;
    size_t size = chunk_kind_to_granules(kind) * GRANULE_SIZE;
    for (size_t i = size; i <= CHUNK_SIZE; i += size)
    {
        struct freelist *head = (struct freelist *)(end - i);
        head->next = next;
        next = head;
    }
    return next;
}

static inline size_t size_to_granules(size_t size)
{
    return (size + GRANULE_SIZE - 1) >> GRANULE_SIZE_LOG_2;
}
static struct freelist **get_small_object_freelist(enum chunk_kind kind)
{
    ASSERT(kind < SMALL_OBJECT_CHUNK_KINDS);
    return &small_object_freelists[kind];
}

static void *
allocate_small(enum chunk_kind kind)
{
    struct freelist **loc = get_small_object_freelist(kind);
    if (!*loc)
    {
        struct freelist *freelist = obtain_small_objects(kind);
        if (!freelist)
        {
            return NULL;
        }
        *loc = freelist;
    }
    struct freelist *ret = *loc;
    *loc = ret->next;
    return (void *)ret;
}

static void *
allocate_large(size_t size)
{
    struct large_object *obj = allocate_large_object(size);
    return obj ? get_large_object_payload(obj) : NULL;
}

void *
malloc(size_t size)
{
    size_t granules = size_to_granules(size);
    enum chunk_kind kind = granules_to_chunk_kind(granules);
    return (kind == LARGE_OBJECT) ? allocate_large(size) : allocate_small(kind);
}

void free(void *ptr)
{
    if (!ptr)
        return;
    struct page *page = get_page(ptr);
    unsigned chunk = get_chunk_index(ptr);
    uint8_t kind = page->header.chunk_kinds[chunk];
    if (kind == LARGE_OBJECT)
    {
        struct large_object *obj = get_large_object(ptr);
        obj->next = large_objects;
        large_objects = obj;
        allocate_chunk(page, chunk, FREE_LARGE_OBJECT);
        pending_large_object_compact = 1;
    }
    else
    {
        size_t granules = kind;
        struct freelist **loc = get_small_object_freelist(granules);
        struct freelist *obj = ptr;
        obj->next = *loc;
        *loc = obj;
    }
}