Skip to content

Commit

Permalink
ARAL: fast path to quickly allocate elements on a new page (netdata#1…
Browse files Browse the repository at this point in the history 
…9376)

* fast path to quickly allocate elements on a new page

* partitions on aral incoming lock
  • Loading branch information
@ktsaou
ktsaou authored Jan 12, 2025
1 parent 5f59772 commit 0232abd
Showing 1 changed file with 75 additions and 69 deletions.
144 changes: 75 additions & 69 deletions src/libnetdata/aral/aral.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,23 @@
// use anonymous private mmap pages
#define ARAL_MMAP_PAGES_ABOVE (32ULL * 1024)

#define ARAL_PAGE_INCOMING_PARTITIONS 4 // up to 32 (32-bits bitmap)

typedef struct aral_free {
size_t size;
struct aral_free *next;
} ARAL_FREE;

typedef struct aral_page {
const char *filename;
uint8_t *data;

bool marked;
bool started_marked;
bool mapped;
uint32_t size; // the allocation size of the page
const char *filename;
uint8_t *data;

uint32_t max_elements; // the number of elements that can fit on this page
uint64_t elements_segmented; // fast path for acquiring new elements in this page

struct {
uint32_t used_elements; // the number of used elements on this page
Expand All @@ -61,7 +64,10 @@ typedef struct aral_page {
struct {
SPINLOCK spinlock;
ARAL_FREE *list;
} incoming;
char pad[48];
} incoming[ARAL_PAGE_INCOMING_PARTITIONS];

uint32_t incoming_partition_bitmap; // atomic

} ARAL_PAGE;

Expand Down Expand Up @@ -227,14 +233,14 @@ static inline void aral_page_available_unlock(ARAL *ar, ARAL_PAGE *page) {
spinlock_unlock(&page->available.spinlock);
}

static inline void aral_page_incoming_lock(ARAL *ar, ARAL_PAGE *page) {
static inline void aral_page_incoming_lock(ARAL *ar, ARAL_PAGE *page, size_t partition) {
if(likely(!(ar->config.options & ARAL_LOCKLESS)))
spinlock_lock(&page->incoming.spinlock);
spinlock_lock(&page->incoming[partition].spinlock);
}

static inline void aral_page_incoming_unlock(ARAL *ar, ARAL_PAGE *page) {
static inline void aral_page_incoming_unlock(ARAL *ar, ARAL_PAGE *page, size_t partition) {
if(likely(!(ar->config.options & ARAL_LOCKLESS)))
spinlock_unlock(&page->incoming.spinlock);
spinlock_unlock(&page->incoming[partition].spinlock);
}

static inline bool aral_adders_trylock(ARAL *ar, bool marked) {
Expand Down Expand Up @@ -566,7 +572,10 @@ static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar, size_t size TRACE_
#endif

spinlock_init(&page->available.spinlock);
spinlock_init(&page->incoming.spinlock);

for(size_t p = 0; p < ARAL_PAGE_INCOMING_PARTITIONS ;p++)
spinlock_init(&page->incoming[p].spinlock);

page->size = size;
page->max_elements = aral_elements_in_page_size(ar, page->size);
page->aral_lock.free_elements = page->max_elements;
Expand All @@ -582,14 +591,8 @@ static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar, size_t size TRACE_
__atomic_add_fetch(&ar->stats->structures.allocations, 1, __ATOMIC_RELAXED);
__atomic_add_fetch(&ar->stats->structures.allocated_bytes, structures_size, __ATOMIC_RELAXED);

// link the free space to its page
ARAL_FREE *fr = (ARAL_FREE *)page->data;

fr->size = page->max_elements * ar->config.element_size;
fr->next = NULL;
page->available.list = fr;

aral_free_validate_internal_check(ar, fr);
// Initialize elements_segmented last with RELEASE
__atomic_store_n(&page->elements_segmented, 0, __ATOMIC_RELEASE);

return page;
}
Expand Down Expand Up @@ -693,9 +696,9 @@ static inline ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, bool mar

DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(*head_ptr_free, page, aral_lock.prev, aral_lock.next);

//#ifdef NETDATA_ARAL_INTERNAL_CHECKS
// added = true;
//#endif
//#ifdef NETDATA_ARAL_INTERNAL_CHECKS
// added = true;
//#endif

aral_adders_lock(ar, marked);
ar->ops[idx].adders.allocating_elements -= aral_elements_in_page_size(ar, page_allocation_size);
Expand All @@ -721,12 +724,12 @@ static inline ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, bool mar

internal_fatal(marked && !page->marked, "ARAL: requested a marked page, but the page found is not marked");

//#ifdef NETDATA_ARAL_INTERNAL_CHECKS
// if(added) {
// f2 = check_free_space___aral_lock_needed(ar, page, marked);
// internal_fatal(f2.failed, "hey!");
// }
//#endif
//#ifdef NETDATA_ARAL_INTERNAL_CHECKS
// if(added) {
// f2 = check_free_space___aral_lock_needed(ar, page, marked);
// internal_fatal(f2.failed, "hey!");
// }
//#endif

internal_fatal(!page || !page->aral_lock.free_elements,
"ARAL: '%s' selected page does not have a free slot in it",
Expand Down Expand Up @@ -769,49 +772,50 @@ static inline ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, bool mar
}

static void *aral_get_free_slot___no_lock_required(ARAL *ar, ARAL_PAGE *page, bool marked) {
aral_page_available_lock(ar, page);
// Try fast path first
uint64_t slot = __atomic_fetch_add(&page->elements_segmented, 1, __ATOMIC_ACQUIRE);
if (slot < page->max_elements) {
// Fast path - we got a valid slot
uint8_t *data = page->data + (slot * ar->config.element_size);

if(!page->available.list) {
aral_page_incoming_lock(ar, page);
page->available.list = page->incoming.list;
page->incoming.list = NULL;
aral_page_incoming_unlock(ar, page);
}

ARAL_FREE *found_fr;
found_fr = page->available.list;

internal_fatal(!found_fr,
"ARAL: '%s' incoming free list, cannot be NULL.", ar->config.name);
// Set the page pointer after the element
aral_set_page_pointer_after_element___do_NOT_have_aral_lock(ar, page, data, marked);

internal_fatal(found_fr->size < ar->config.element_size,
"ARAL: '%s' free element size %zu, cannot be smaller than %zu",
ar->config.name, page->available.list->size, ar->config.element_size);

// check if the remaining size (after we use this slot) is not enough for another element
if(unlikely(found_fr->size - ar->config.element_size < ar->config.element_size)) {
// we can use the entire free space entry

page->available.list = found_fr->next;
aral_element_given(ar, page);
return data;
}
else {
// we can split the free space entry

uint8_t *data = (uint8_t *)found_fr;
ARAL_FREE *fr = (ARAL_FREE *)&data[ar->config.element_size];

fr->size = found_fr->size - ar->config.element_size;

// link the free slot first in the page
fr->next = found_fr->next;
page->available.list = fr;
// Fall back to existing mechanism for reused memory
aral_page_available_lock(ar, page);

aral_free_validate_internal_check(ar, fr);
if(!page->available.list) {
uint32_t bitmap = __atomic_load_n(&page->incoming_partition_bitmap, __ATOMIC_RELAXED);
if(!bitmap)
fatal("ARAL: bitmap of incoming free elements cannot be empty at this point");

size_t partition = __builtin_ffs((int)bitmap) - 1;
// for(partition = 0; partition < ARAL_PAGE_INCOMING_PARTITIONS ; partition++) {
// if (bitmap & (1U << partition))
// break;
// }

if(partition >= ARAL_PAGE_INCOMING_PARTITIONS)
fatal("ARAL: partition %zu must be smaller than %d", partition, ARAL_PAGE_INCOMING_PARTITIONS);

aral_page_incoming_lock(ar, page, partition);
page->available.list = page->incoming[partition].list;
page->incoming[partition].list = NULL;
__atomic_fetch_and(&page->incoming_partition_bitmap, ~(1U << partition), __ATOMIC_RELAXED);
aral_page_incoming_unlock(ar, page, partition);
}

ARAL_FREE *found_fr = page->available.list;
internal_fatal(!found_fr, "ARAL: '%s' incoming free list, cannot be NULL.", ar->config.name);
page->available.list = found_fr->next;

aral_page_available_unlock(ar, page);

// put the page pointer after the element
// Set the page pointer after the element
aral_set_page_pointer_after_element___do_NOT_have_aral_lock(ar, page, found_fr, marked);

aral_element_given(ar, page);
Expand All @@ -823,10 +827,12 @@ static inline void aral_add_free_slot___no_lock_required(ARAL *ar, ARAL_PAGE *pa
ARAL_FREE *fr = (ARAL_FREE *)ptr;
fr->size = ar->config.element_size;

aral_page_incoming_lock(ar, page);
fr->next = page->incoming.list;
page->incoming.list = fr;
aral_page_incoming_unlock(ar, page);
size_t partition = gettid_cached() % ARAL_PAGE_INCOMING_PARTITIONS;
aral_page_incoming_lock(ar, page, partition);
fr->next = page->incoming[partition].list;
page->incoming[partition].list = fr;
__atomic_fetch_or(&page->incoming_partition_bitmap, 1U << partition, __ATOMIC_RELAXED);
aral_page_incoming_unlock(ar, page, partition);
}

void *aral_callocz_internal(ARAL *ar, bool marked TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
Expand Down Expand Up @@ -935,7 +941,7 @@ void aral_freez_internal(ARAL *ar, void *ptr TRACE_ALLOCATIONS_FUNCTION_DEFINITI
(size_t)page->max_elements,
(size_t)page->aral_lock.used_elements, (size_t)page->aral_lock.free_elements,
(size_t)page->aral_lock.used_elements + (size_t)page->aral_lock.free_elements
);
);

ARAL_PAGE **head_ptr = page->aral_lock.free_elements ? aral_pages_head_free(ar, page->marked) : aral_pages_head_full(ar, page->marked);
internal_fatal(!is_page_in_list(*head_ptr, page), "Page is not in this list");
Expand Down Expand Up @@ -1230,10 +1236,10 @@ void aral_by_size_release(ARAL *ar) {
fatal("ARAL BY SIZE: double release detected");

aral_by_size_globals.array[size].refcount--;
// if(!aral_by_size_globals.array[size].refcount) {
// aral_destroy(aral_by_size_globals.array[size].ar);
// aral_by_size_globals.array[size].ar = NULL;
// }
// if(!aral_by_size_globals.array[size].refcount) {
// aral_destroy(aral_by_size_globals.array[size].ar);
// aral_by_size_globals.array[size].ar = NULL;
// }

spinlock_unlock(&aral_by_size_globals.spinlock);
}
Expand Down

0 comments on commit 0232abd

Please sign in to comment.