Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize garbage collection by using realloc #939

Merged
merged 1 commit into from
Apr 10, 2024
Merged

Optimize garbage collection by using realloc #939

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 180 additions & 8 deletions src/libAtomVM/memory.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
* SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

Expand All @@ -43,6 +44,9 @@
static void memory_scan_and_copy(HeapFragment *old_fragment, term *mem_start, const term *mem_end, term **new_heap_pos, term *mso_list, bool move);
static term memory_shallow_copy_term(HeapFragment *old_fragment, term t, term **new_heap, bool move);
static enum MemoryGCResult memory_gc(Context *ctx, size_t new_size, size_t num_roots, term *roots);
static enum MemoryGCResult memory_shrink(Context *ctx, size_t new_size, size_t num_roots, term *roots);
static void memory_scan_and_rewrite(size_t count, term *terms, const term *old_start, const term *old_end, intptr_t delta, bool is_heap);
static inline term *memory_rewrite_pointer(term *ptr, const term *old_start, const term *old_end, intptr_t delta);

enum MemoryGCResult memory_init_heap(Heap *heap, size_t size)
{
Expand All @@ -64,6 +68,20 @@ void memory_init_heap_root_fragment(Heap *heap, HeapFragment *root, size_t size)
heap->heap_end = heap->heap_start + size;
}

static inline enum MemoryGCResult memory_realloc_heap_root(Heap *heap, size_t size)
{
uintptr_t used_size = heap->heap_ptr - heap->heap_start;
HeapFragment *new_root = (HeapFragment *) realloc(heap->root, sizeof(HeapFragment) + size * sizeof(term));
if (IS_NULL_PTR(new_root)) {
return MEMORY_GC_ERROR_FAILED_ALLOCATION;
}
heap->root = new_root;
heap->heap_start = new_root->storage;
heap->heap_ptr = heap->heap_start + used_size;
heap->heap_end = heap->heap_start + size;
return MEMORY_GC_OK;
}

static inline enum MemoryGCResult memory_heap_alloc_new_fragment(Heap *heap, size_t size)
{
HeapFragment *root_fragment = heap->root;
Expand Down Expand Up @@ -171,6 +189,9 @@ enum MemoryGCResult memory_ensure_free_with_roots(Context *c, size_t size, size_
UNREACHABLE();
}
target_size = MAX(c->has_min_heap_size ? c->min_heap_size : 0, target_size);
if (UNLIKELY(c->has_max_heap_size && (target_size > c->max_heap_size))) {
return MEMORY_GC_DENIED_ALLOCATION;
}
if (target_size != memory_size) {
if (UNLIKELY(memory_gc(c, target_size, num_roots, roots) != MEMORY_GC_OK)) {
// TODO: handle this more gracefully
Expand Down Expand Up @@ -205,7 +226,7 @@ enum MemoryGCResult memory_ensure_free_with_roots(Context *c, size_t size, size_
if (should_gc) {
new_target_size = MAX(c->has_min_heap_size ? c->min_heap_size : 0, new_target_size);
if (new_target_size != new_memory_size) {
if (UNLIKELY(memory_gc(c, new_target_size, num_roots, roots) != MEMORY_GC_OK)) {
if (UNLIKELY(memory_shrink(c, new_target_size, num_roots, roots) != MEMORY_GC_OK)) {
TRACE("Unable to allocate memory for GC shrink. new_memory_size=%zu new_free_space=%zu size=%u\n", new_memory_size, new_free_space, size);
return MEMORY_GC_ERROR_FAILED_ALLOCATION;
}
Expand All @@ -227,13 +248,6 @@ static inline void push_to_stack(term **stack, term value)
static enum MemoryGCResult memory_gc(Context *ctx, size_t new_size, size_t num_roots, term *roots)
{
TRACE("Going to perform gc on process %i\n", ctx->process_id);
size_t min_heap_size = ctx->has_min_heap_size ? ctx->min_heap_size : 0;
new_size = MAX(new_size, min_heap_size);

if (UNLIKELY(ctx->has_max_heap_size && (new_size > ctx->max_heap_size))) {
return MEMORY_GC_DENIED_ALLOCATION;
}

term old_mso_list = ctx->heap.root->mso_list;
term *old_stack_ptr = context_stack_base(ctx);
term *old_heap_end = ctx->heap.heap_end;
Expand Down Expand Up @@ -293,6 +307,73 @@ static enum MemoryGCResult memory_gc(Context *ctx, size_t new_size, size_t num_r
return MEMORY_GC_OK;
}

static enum MemoryGCResult memory_shrink(Context *ctx, size_t new_size, size_t num_roots, term *roots)
{
TRACE("Going to perform shrink on process %i\n", ctx->process_id);

// First, move stack up.
term *old_stack_ptr = context_stack_base(ctx);
size_t stack_size = old_stack_ptr - ctx->e;
term *new_ctx_e = ctx->heap.heap_start + new_size - stack_size;
memmove(new_ctx_e, ctx->e, stack_size * sizeof(term));
ctx->e = new_ctx_e;

term *old_heap_root = ctx->heap.root->storage;
term *old_end = ctx->heap.heap_ptr;

if (UNLIKELY(memory_realloc_heap_root(&ctx->heap, new_size) != MEMORY_GC_OK)) {
return MEMORY_GC_ERROR_FAILED_ALLOCATION;
}

term *new_heap_root = ctx->heap.root->storage;
intptr_t delta = new_heap_root - old_heap_root;
ctx->e += delta;

if (delta == 0) {
return MEMORY_GC_OK;
}

// Rewrite all boxed pointers...
// ...in stack
memory_scan_and_rewrite(ctx->heap.heap_end - ctx->e, ctx->e, old_heap_root, old_end, delta, false);
// ...in heap
memory_scan_and_rewrite(old_end - old_heap_root, new_heap_root, old_heap_root, old_end, delta, true);
// ...in roots
memory_scan_and_rewrite(num_roots, roots, old_heap_root, old_end, delta, true);
// ...in process dictionary
struct ListHead *item;
LIST_FOR_EACH (item, &ctx->dictionary) {
struct DictEntry *entry = GET_LIST_ENTRY(item, struct DictEntry, head);
memory_scan_and_rewrite(2, &entry->key, old_heap_root, old_end, delta, true);
}
// ...exit_reason
memory_scan_and_rewrite(1, &ctx->exit_reason, old_heap_root, old_end, delta, true);
// ...and MSO list.
term *mso_ptr = &ctx->heap.root->mso_list;
while (!term_is_nil(*mso_ptr)) {
term *list_ptr = term_get_list_ptr(*mso_ptr);
term *new_list_ptr = memory_rewrite_pointer(list_ptr, old_heap_root, old_end, delta);
if (list_ptr != new_list_ptr) {
*mso_ptr = ((term) new_list_ptr) | 0x1;
}
// Process head.
term head = new_list_ptr[1];
if (UNLIKELY(!term_is_boxed(head))) {
fprintf(stderr, "Expected a boxed term, got %" TERM_X_FMT "\n", head);
AVM_ABORT();
}
term *boxed_ptr = term_to_term_ptr(head);
term *new_boxed_ptr = memory_rewrite_pointer(boxed_ptr, old_heap_root, old_end, delta);
if (boxed_ptr != new_boxed_ptr) {
new_list_ptr[1] = ((term) new_boxed_ptr) | TERM_BOXED_VALUE_TAG;
}
// Loop with tail.
mso_ptr = new_list_ptr;
}

return MEMORY_GC_OK;
}

static inline int memory_is_moved_marker(term *t)
{
// 0x2B is an unused tag
Expand Down Expand Up @@ -573,6 +654,97 @@ static void memory_scan_and_copy(HeapFragment *old_fragment, term *mem_start, co
*new_heap_pos = new_heap;
}

static inline term *memory_rewrite_pointer(term *ptr, const term *old_start, const term *old_end, intptr_t delta)
{
if (ptr >= old_start && ptr < old_end) {
return ptr + delta;
}
return ptr;
}

static void memory_scan_and_rewrite(size_t count, term *terms, const term *old_start, const term *old_end, intptr_t delta, bool is_heap)
{
term *ptr = terms;
term *end = terms + count;
while (ptr < end) {
term t = *ptr++;
if (is_heap && (t & 0x3) == 0x0) {
switch (t & TERM_BOXED_TAG_MASK) {
case TERM_BOXED_TUPLE:
// Skip header and process elements
break;

case TERM_BOXED_BIN_MATCH_STATE: {
// there is a boxed binary that needs to be rewritten and then it's integers.
term binary_or_state = *ptr;
if (UNLIKELY(!term_is_boxed(binary_or_state))) {
fprintf(stderr, "Expected a boxed term, got %" TERM_X_FMT "\n", binary_or_state);
AVM_ABORT();
}
term *boxed_ptr = term_to_term_ptr(binary_or_state);
term *new_boxed_ptr = memory_rewrite_pointer(boxed_ptr, old_start, old_end, delta);
if (boxed_ptr != new_boxed_ptr) {
*ptr = ((term) new_boxed_ptr) | TERM_BOXED_VALUE_TAG;
}
ptr += term_get_size_from_boxed_header(t);
break;
}

case TERM_BOXED_POSITIVE_INTEGER:
ptr += term_get_size_from_boxed_header(t);
break;

case TERM_BOXED_REF:
ptr += term_get_size_from_boxed_header(t);
break;

case TERM_BOXED_FUN:
// Skip header and module and process next terms
ptr++;
break;

case TERM_BOXED_FLOAT:
ptr += term_get_size_from_boxed_header(t);
break;

case TERM_BOXED_REFC_BINARY:
ptr += term_get_size_from_boxed_header(t);
break;

case TERM_BOXED_SUB_BINARY:
// Skip to binary
ptr += 2;
break;

case TERM_BOXED_HEAP_BINARY:
ptr += term_get_size_from_boxed_header(t);
break;

case TERM_BOXED_MAP:
// Skip header and process next term
break;

default:
fprintf(stderr, "- Found unknown boxed type: %" TERM_X_FMT "\n", t & TERM_BOXED_TAG_MASK);
AVM_ABORT();
}
} else if (term_is_nonempty_list(t)) {
term *list_ptr = term_get_list_ptr(t);
term *new_list_ptr = memory_rewrite_pointer(list_ptr, old_start, old_end, delta);
if (list_ptr != new_list_ptr) {
ptr[-1] = ((term) new_list_ptr) | 0x1;
}

} else if (term_is_boxed(t)) {
term *boxed_ptr = term_to_term_ptr(t);
term *new_boxed_ptr = memory_rewrite_pointer(boxed_ptr, old_start, old_end, delta);
if (boxed_ptr != new_boxed_ptr) {
ptr[-1] = ((term) new_boxed_ptr) | TERM_BOXED_VALUE_TAG;
}
}
}
}

HOT_FUNC static inline bool memory_heap_fragment_contains_pointer(HeapFragment *old_fragment, term *ptr)
{
do {
Expand Down
Loading