public/azerothcore-wotlk
1
0
Fork 0
mirror of https://github.com/liyunfan1223/azerothcore-wotlk.git synced 2026-08-08 07:48:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(Deps/Jemalloc): update Jemalloc to 5.2.1 (#2413)

Browse Source
This commit is contained in:
Viste
2019-11-14 23:17:38 +03:00
committed by Kargatum
parent fae7ae95a3
commit 685538b01b
103 changed files with 10904 additions and 3886 deletions
Download Patch File Download Diff File Expand all files Collapse all files

897
deps/jemalloc/src/arena.c vendored
View File

File diff suppressed because it is too large Load Diff

201
deps/jemalloc/src/background_thread.c vendored
View File

@@ -4,6 +4,8 @@
#include "jemalloc/internal/assert.h"
JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
/******************************************************************************/
/* Data. */
@@ -11,38 +13,37 @@
#define BACKGROUND_THREAD_DEFAULT false
/* Read-only after initialization. */
bool opt_background_thread = BACKGROUND_THREAD_DEFAULT;
size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT + 1;
/* Used for thread creation, termination and stats. */
malloc_mutex_t background_thread_lock;
/* Indicates global state. Atomic because decay reads this w/o locking. */
atomic_b_t background_thread_enabled_state;
size_t n_background_threads;
size_t max_background_threads;
/* Thread info per-index. */
background_thread_info_t *background_thread_info;
/* False if no necessary runtime support. */
bool can_enable_background_thread;
/******************************************************************************/
#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
#include <dlfcn.h>
static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *,
void *(*)(void *), void *__restrict);
static pthread_once_t once_control = PTHREAD_ONCE_INIT;
static void
pthread_create_wrapper_once(void) {
pthread_create_wrapper_init(void) {
#ifdef JEMALLOC_LAZY_LOCK
isthreaded = true;
if (!isthreaded) {
isthreaded = true;
}
#endif
}
int
pthread_create_wrapper(pthread_t *__restrict thread, const pthread_attr_t *attr,
void *(*start_routine)(void *), void *__restrict arg) {
pthread_once(&once_control, pthread_create_wrapper_once);
pthread_create_wrapper_init();
return pthread_create_fptr(thread, attr, start_routine, arg);
}
@@ -78,7 +79,7 @@ background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) {
}
static inline bool
set_current_thread_affinity(UNUSED int cpu) {
set_current_thread_affinity(int cpu) {
#if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
@@ -286,7 +287,7 @@ background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info, unsigne
uint64_t min_interval = BACKGROUND_THREAD_INDEFINITE_SLEEP;
unsigned narenas = narenas_total_get();
for (unsigned i = ind; i < narenas; i += ncpus) {
for (unsigned i = ind; i < narenas; i += max_background_threads) {
arena_t *arena = arena_get(tsdn, i, false);
if (!arena) {
continue;
@@ -379,35 +380,32 @@ background_thread_create_signals_masked(pthread_t *thread,
return create_err;
}
static void
static bool
check_background_thread_creation(tsd_t *tsd, unsigned *n_created,
bool *created_threads) {
bool ret = false;
if (likely(*n_created == n_background_threads)) {
return;
return ret;
}
malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_info[0].mtx);
label_restart:
malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
for (unsigned i = 1; i < ncpus; i++) {
tsdn_t *tsdn = tsd_tsdn(tsd);
malloc_mutex_unlock(tsdn, &background_thread_info[0].mtx);
for (unsigned i = 1; i < max_background_threads; i++) {
if (created_threads[i]) {
continue;
}
background_thread_info_t *info = &background_thread_info[i];
malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
assert(info->state != background_thread_paused);
malloc_mutex_lock(tsdn, &info->mtx);
/*
* In case of the background_thread_paused state because of
* arena reset, delay the creation.
*/
bool create = (info->state == background_thread_started);
malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
malloc_mutex_unlock(tsdn, &info->mtx);
if (!create) {
continue;
}
/*
* To avoid deadlock with prefork handlers (which waits for the
* mutex held here), unlock before calling pthread_create().
*/
malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
pre_reentrancy(tsd, NULL);
int err = background_thread_create_signals_masked(&info->thread,
NULL, background_thread_entry, (void *)(uintptr_t)i);
@@ -423,19 +421,21 @@ label_restart:
abort();
}
}
/* Restart since we unlocked. */
goto label_restart;
/* Return to restart the loop since we unlocked. */
ret = true;
break;
}
malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_info[0].mtx);
malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
malloc_mutex_lock(tsdn, &background_thread_info[0].mtx);
return ret;
}
static void
background_thread0_work(tsd_t *tsd) {
/* Thread0 is also responsible for launching / terminating threads. */
VARIABLE_ARRAY(bool, created_threads, ncpus);
VARIABLE_ARRAY(bool, created_threads, max_background_threads);
unsigned i;
for (i = 1; i < ncpus; i++) {
for (i = 1; i < max_background_threads; i++) {
created_threads[i] = false;
}
/* Start working, and create more threads when asked. */
@@ -445,8 +445,10 @@ background_thread0_work(tsd_t *tsd) {
&background_thread_info[0])) {
continue;
}
check_background_thread_creation(tsd, &n_created,
(bool *)&created_threads);
if (check_background_thread_creation(tsd, &n_created,
(bool *)&created_threads)) {
continue;
}
background_work_sleep_once(tsd_tsdn(tsd),
&background_thread_info[0], 0);
}
@@ -456,15 +458,20 @@ background_thread0_work(tsd_t *tsd) {
* the global background_thread mutex (and is waiting) for us.
*/
assert(!background_thread_enabled());
for (i = 1; i < ncpus; i++) {
for (i = 1; i < max_background_threads; i++) {
background_thread_info_t *info = &background_thread_info[i];
assert(info->state != background_thread_paused);
if (created_threads[i]) {
background_threads_disable_single(tsd, info);
} else {
malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
/* Clear in case the thread wasn't created. */
info->state = background_thread_stopped;
if (info->state != background_thread_stopped) {
/* The thread was not created. */
assert(info->state ==
background_thread_started);
n_background_threads--;
info->state = background_thread_stopped;
}
malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
}
}
@@ -498,9 +505,11 @@ background_work(tsd_t *tsd, unsigned ind) {
static void *
background_thread_entry(void *ind_arg) {
unsigned thread_ind = (unsigned)(uintptr_t)ind_arg;
assert(thread_ind < ncpus);
assert(thread_ind < max_background_threads);
#ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP
pthread_setname_np(pthread_self(), "jemalloc_bg_thd");
#elif defined(__FreeBSD__)
pthread_set_name_np(pthread_self(), "jemalloc_bg_thd");
#endif
if (opt_percpu_arena != percpu_arena_disabled) {
set_current_thread_affinity((int)thread_ind);
@@ -525,14 +534,13 @@ background_thread_init(tsd_t *tsd, background_thread_info_t *info) {
n_background_threads++;
}
/* Create a new background thread if needed. */
bool
background_thread_create(tsd_t *tsd, unsigned arena_ind) {
static bool
background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) {
assert(have_background_thread);
malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
/* We create at most NCPUs threads. */
size_t thread_ind = arena_ind % ncpus;
size_t thread_ind = arena_ind % max_background_threads;
background_thread_info_t *info = &background_thread_info[thread_ind];
bool need_new_thread;
@@ -580,37 +588,53 @@ background_thread_create(tsd_t *tsd, unsigned arena_ind) {
return false;
}
/* Create a new background thread if needed. */
bool
background_thread_create(tsd_t *tsd, unsigned arena_ind) {
assert(have_background_thread);
bool ret;
malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
ret = background_thread_create_locked(tsd, arena_ind);
malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
return ret;
}
bool
background_threads_enable(tsd_t *tsd) {
assert(n_background_threads == 0);
assert(background_thread_enabled());
malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
VARIABLE_ARRAY(bool, marked, ncpus);
VARIABLE_ARRAY(bool, marked, max_background_threads);
unsigned i, nmarked;
for (i = 0; i < ncpus; i++) {
for (i = 0; i < max_background_threads; i++) {
marked[i] = false;
}
nmarked = 0;
/* Thread 0 is required and created at the end. */
marked[0] = true;
/* Mark the threads we need to create for thread 0. */
unsigned n = narenas_total_get();
for (i = 1; i < n; i++) {
if (marked[i % ncpus] ||
if (marked[i % max_background_threads] ||
arena_get(tsd_tsdn(tsd), i, false) == NULL) {
continue;
}
background_thread_info_t *info = &background_thread_info[i];
background_thread_info_t *info = &background_thread_info[
i % max_background_threads];
malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
assert(info->state == background_thread_stopped);
background_thread_init(tsd, info);
malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
marked[i % ncpus] = true;
if (++nmarked == ncpus) {
marked[i % max_background_threads] = true;
if (++nmarked == max_background_threads) {
break;
}
}
return background_thread_create(tsd, 0);
return background_thread_create_locked(tsd, 0);
}
bool
@@ -720,14 +744,14 @@ background_thread_prefork0(tsdn_t *tsdn) {
void
background_thread_prefork1(tsdn_t *tsdn) {
for (unsigned i = 0; i < ncpus; i++) {
for (unsigned i = 0; i < max_background_threads; i++) {
malloc_mutex_prefork(tsdn, &background_thread_info[i].mtx);
}
}
void
background_thread_postfork_parent(tsdn_t *tsdn) {
for (unsigned i = 0; i < ncpus; i++) {
for (unsigned i = 0; i < max_background_threads; i++) {
malloc_mutex_postfork_parent(tsdn,
&background_thread_info[i].mtx);
}
@@ -736,7 +760,7 @@ background_thread_postfork_parent(tsdn_t *tsdn) {
void
background_thread_postfork_child(tsdn_t *tsdn) {
for (unsigned i = 0; i < ncpus; i++) {
for (unsigned i = 0; i < max_background_threads; i++) {
malloc_mutex_postfork_child(tsdn,
&background_thread_info[i].mtx);
}
@@ -749,7 +773,7 @@ background_thread_postfork_child(tsdn_t *tsdn) {
malloc_mutex_lock(tsdn, &background_thread_lock);
n_background_threads = 0;
background_thread_enabled_set(tsdn, false);
for (unsigned i = 0; i < ncpus; i++) {
for (unsigned i = 0; i < max_background_threads; i++) {
background_thread_info_t *info = &background_thread_info[i];
malloc_mutex_lock(tsdn, &info->mtx);
info->state = background_thread_stopped;
@@ -773,9 +797,15 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) {
stats->num_threads = n_background_threads;
uint64_t num_runs = 0;
nstime_init(&stats->run_interval, 0);
for (unsigned i = 0; i < ncpus; i++) {
for (unsigned i = 0; i < max_background_threads; i++) {
background_thread_info_t *info = &background_thread_info[i];
malloc_mutex_lock(tsdn, &info->mtx);
if (malloc_mutex_trylock(tsdn, &info->mtx)) {
/*
* Each background thread run may take a long time;
* avoid waiting on the stats if the thread is active.
*/
continue;
}
if (info->state != background_thread_stopped) {
num_runs += info->tot_n_runs;
nstime_add(&stats->run_interval, &info->tot_sleep_time);
@@ -795,6 +825,39 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) {
#undef BILLION
#undef BACKGROUND_THREAD_MIN_INTERVAL_NS
#ifdef JEMALLOC_HAVE_DLSYM
#include <dlfcn.h>
#endif
static bool
pthread_create_fptr_init(void) {
if (pthread_create_fptr != NULL) {
return false;
}
/*
* Try the next symbol first, because 1) when use lazy_lock we have a
* wrapper for pthread_create; and 2) application may define its own
* wrapper as well (and can call malloc within the wrapper).
*/
#ifdef JEMALLOC_HAVE_DLSYM
pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create");
#else
pthread_create_fptr = NULL;
#endif
if (pthread_create_fptr == NULL) {
if (config_lazy_lock) {
malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, "
"\"pthread_create\")\n");
abort();
} else {
/* Fall back to the default symbol. */
pthread_create_fptr = pthread_create;
}
}
return false;
}
/*
* When lazy lock is enabled, we need to make sure setting isthreaded before
* taking any background_thread locks. This is called early in ctl (instead of
@@ -805,7 +868,8 @@ void
background_thread_ctl_init(tsdn_t *tsdn) {
malloc_mutex_assert_not_owner(tsdn, &background_thread_lock);
#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
pthread_once(&once_control, pthread_create_wrapper_once);
pthread_create_fptr_init();
pthread_create_wrapper_init();
#endif
}
@@ -818,18 +882,10 @@ background_thread_boot0(void) {
"supports pthread only\n");
return true;
}
#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create");
if (pthread_create_fptr == NULL) {
can_enable_background_thread = false;
if (config_lazy_lock || opt_background_thread) {
malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, "
"\"pthread_create\")\n");
abort();
}
} else {
can_enable_background_thread = true;
if ((config_lazy_lock || opt_background_thread) &&
pthread_create_fptr_init()) {
return true;
}
#endif
return false;
@@ -841,6 +897,11 @@ background_thread_boot1(tsdn_t *tsdn) {
assert(have_background_thread);
assert(narenas_total_get() > 0);
if (opt_max_background_threads > MAX_BACKGROUND_THREAD_LIMIT) {
opt_max_background_threads = DEFAULT_NUM_BACKGROUND_THREAD;
}
max_background_threads = opt_max_background_threads;
background_thread_enabled_set(tsdn, opt_background_thread);
if (malloc_mutex_init(&background_thread_lock,
"background_thread_global",
@@ -848,17 +909,15 @@ background_thread_boot1(tsdn_t *tsdn) {
malloc_mutex_rank_exclusive)) {
return true;
}
if (opt_background_thread) {
background_thread_ctl_init(tsdn);
}
background_thread_info = (background_thread_info_t *)base_alloc(tsdn,
b0get(), ncpus * sizeof(background_thread_info_t), CACHELINE);
b0get(), opt_max_background_threads *
sizeof(background_thread_info_t), CACHELINE);
if (background_thread_info == NULL) {
return true;
}
for (unsigned i = 0; i < ncpus; i++) {
for (unsigned i = 0; i < max_background_threads; i++) {
background_thread_info_t *info = &background_thread_info[i];
/* Thread mutex is rank_inclusive because of thread0. */
if (malloc_mutex_init(&info->mtx, "background_thread",

174
deps/jemalloc/src/base.c vendored
View File

@@ -10,25 +10,40 @@
/******************************************************************************/
/* Data. */
static base_t *b0;
static base_t *b0;
metadata_thp_mode_t opt_metadata_thp = METADATA_THP_DEFAULT;
const char *metadata_thp_mode_names[] = {
"disabled",
"auto",
"always"
};
/******************************************************************************/
static inline bool
metadata_thp_madvise(void) {
return (metadata_thp_enabled() &&
(init_system_thp_mode == thp_mode_default));
}
static void *
base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size) {
void *addr;
bool zero = true;
bool commit = true;
/* Use huge page sizes and alignment regardless of opt_metadata_thp. */
assert(size == HUGEPAGE_CEILING(size));
size_t alignment = HUGEPAGE;
if (extent_hooks == &extent_hooks_default) {
addr = extent_alloc_mmap(NULL, size, PAGE, &zero, &commit);
addr = extent_alloc_mmap(NULL, size, alignment, &zero, &commit);
} else {
/* No arena context as we are creating new arenas. */
tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
pre_reentrancy(tsd, NULL);
addr = extent_hooks->alloc(extent_hooks, NULL, size, PAGE,
addr = extent_hooks->alloc(extent_hooks, NULL, size, alignment,
&zero, &commit, ind);
post_reentrancy(tsd);
}
@@ -51,16 +66,16 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr,
*/
if (extent_hooks == &extent_hooks_default) {
if (!extent_dalloc_mmap(addr, size)) {
return;
goto label_done;
}
if (!pages_decommit(addr, size)) {
return;
goto label_done;
}
if (!pages_purge_forced(addr, size)) {
return;
goto label_done;
}
if (!pages_purge_lazy(addr, size)) {
return;
goto label_done;
}
/* Nothing worked. This should never happen. */
not_reached();
@@ -70,27 +85,33 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr,
if (extent_hooks->dalloc != NULL &&
!extent_hooks->dalloc(extent_hooks, addr, size, true,
ind)) {
goto label_done;
goto label_post_reentrancy;
}
if (extent_hooks->decommit != NULL &&
!extent_hooks->decommit(extent_hooks, addr, size, 0, size,
ind)) {
goto label_done;
goto label_post_reentrancy;
}
if (extent_hooks->purge_forced != NULL &&
!extent_hooks->purge_forced(extent_hooks, addr, size, 0,
size, ind)) {
goto label_done;
goto label_post_reentrancy;
}
if (extent_hooks->purge_lazy != NULL &&
!extent_hooks->purge_lazy(extent_hooks, addr, size, 0, size,
ind)) {
goto label_done;
goto label_post_reentrancy;
}
/* Nothing worked. That's the application's problem. */
label_done:
label_post_reentrancy:
post_reentrancy(tsd);
return;
}
label_done:
if (metadata_thp_madvise()) {
/* Set NOHUGEPAGE after unmap to avoid kernel defrag. */
assert(((uintptr_t)addr & HUGEPAGE_MASK) == 0 &&
(size & HUGEPAGE_MASK) == 0);
pages_nohuge(addr, size);
}
}
@@ -105,6 +126,56 @@ base_extent_init(size_t *extent_sn_next, extent_t *extent, void *addr,
extent_binit(extent, addr, size, sn);
}
static size_t
base_get_num_blocks(base_t *base, bool with_new_block) {
base_block_t *b = base->blocks;
assert(b != NULL);
size_t n_blocks = with_new_block ? 2 : 1;
while (b->next != NULL) {
n_blocks++;
b = b->next;
}
return n_blocks;
}
static void
base_auto_thp_switch(tsdn_t *tsdn, base_t *base) {
assert(opt_metadata_thp == metadata_thp_auto);
malloc_mutex_assert_owner(tsdn, &base->mtx);
if (base->auto_thp_switched) {
return;
}
/* Called when adding a new block. */
bool should_switch;
if (base_ind_get(base) != 0) {
should_switch = (base_get_num_blocks(base, true) ==
BASE_AUTO_THP_THRESHOLD);
} else {
should_switch = (base_get_num_blocks(base, true) ==
BASE_AUTO_THP_THRESHOLD_A0);
}
if (!should_switch) {
return;
}
base->auto_thp_switched = true;
assert(!config_stats || base->n_thp == 0);
/* Make the initial blocks THP lazily. */
base_block_t *block = base->blocks;
while (block != NULL) {
assert((block->size & HUGEPAGE_MASK) == 0);
pages_huge(block, block->size);
if (config_stats) {
base->n_thp += HUGEPAGE_CEILING(block->size -
extent_bsize_get(&block->extent)) >> LG_HUGEPAGE;
}
block = block->next;
assert(block == NULL || (base_ind_get(base) == 0));
}
}
static void *
base_extent_bump_alloc_helper(extent_t *extent, size_t *gap_size, size_t size,
size_t alignment) {
@@ -124,8 +195,8 @@ base_extent_bump_alloc_helper(extent_t *extent, size_t *gap_size, size_t size,
}
static void
base_extent_bump_alloc_post(tsdn_t *tsdn, base_t *base, extent_t *extent,
size_t gap_size, void *addr, size_t size) {
base_extent_bump_alloc_post(base_t *base, extent_t *extent, size_t gap_size,
void *addr, size_t size) {
if (extent_bsize_get(extent) > 0) {
/*
* Compute the index for the largest size class that does not
@@ -140,23 +211,31 @@ base_extent_bump_alloc_post(tsdn_t *tsdn, base_t *base, extent_t *extent,
base->allocated += size;
/*
* Add one PAGE to base_resident for every page boundary that is
* crossed by the new allocation.
* crossed by the new allocation. Adjust n_thp similarly when
* metadata_thp is enabled.
*/
base->resident += PAGE_CEILING((uintptr_t)addr + size) -
PAGE_CEILING((uintptr_t)addr - gap_size);
assert(base->allocated <= base->resident);
assert(base->resident <= base->mapped);
if (metadata_thp_madvise() && (opt_metadata_thp ==
metadata_thp_always || base->auto_thp_switched)) {
base->n_thp += (HUGEPAGE_CEILING((uintptr_t)addr + size)
- HUGEPAGE_CEILING((uintptr_t)addr - gap_size)) >>
LG_HUGEPAGE;
assert(base->mapped >= base->n_thp << LG_HUGEPAGE);
}
}
}
static void *
base_extent_bump_alloc(tsdn_t *tsdn, base_t *base, extent_t *extent,
size_t size, size_t alignment) {
base_extent_bump_alloc(base_t *base, extent_t *extent, size_t size,
size_t alignment) {
void *ret;
size_t gap_size;
ret = base_extent_bump_alloc_helper(extent, &gap_size, size, alignment);
base_extent_bump_alloc_post(tsdn, base, extent, gap_size, ret, size);
base_extent_bump_alloc_post(base, extent, gap_size, ret, size);
return ret;
}
@@ -166,8 +245,8 @@ base_extent_bump_alloc(tsdn_t *tsdn, base_t *base, extent_t *extent,
* On success a pointer to the initialized base_block_t header is returned.
*/
static base_block_t *
base_block_alloc(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind,
pszind_t *pind_last, size_t *extent_sn_next, size_t size,
base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks,
unsigned ind, pszind_t *pind_last, size_t *extent_sn_next, size_t size,
size_t alignment) {
alignment = ALIGNMENT_CEILING(alignment, QUANTUM);
size_t usize = ALIGNMENT_CEILING(size, alignment);
@@ -183,8 +262,8 @@ base_block_alloc(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind,
*/
size_t min_block_size = HUGEPAGE_CEILING(sz_psz2u(header_size + gap_size
+ usize));
pszind_t pind_next = (*pind_last + 1 < NPSIZES) ? *pind_last + 1 :
*pind_last;
pszind_t pind_next = (*pind_last + 1 < sz_psz2ind(SC_LARGE_MAXCLASS)) ?
*pind_last + 1 : *pind_last;
size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next));
size_t block_size = (min_block_size > next_block_size) ? min_block_size
: next_block_size;
@@ -193,6 +272,25 @@ base_block_alloc(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind,
if (block == NULL) {
return NULL;
}
if (metadata_thp_madvise()) {
void *addr = (void *)block;
assert(((uintptr_t)addr & HUGEPAGE_MASK) == 0 &&
(block_size & HUGEPAGE_MASK) == 0);
if (opt_metadata_thp == metadata_thp_always) {
pages_huge(addr, block_size);
} else if (opt_metadata_thp == metadata_thp_auto &&
base != NULL) {
/* base != NULL indicates this is not a new base. */
malloc_mutex_lock(tsdn, &base->mtx);
base_auto_thp_switch(tsdn, base);
if (base->auto_thp_switched) {
pages_huge(addr, block_size);
}
malloc_mutex_unlock(tsdn, &base->mtx);
}
}
*pind_last = sz_psz2ind(block_size);
block->size = block_size;
block->next = NULL;
@@ -216,7 +314,7 @@ base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
* called.
*/
malloc_mutex_unlock(tsdn, &base->mtx);
base_block_t *block = base_block_alloc(tsdn, extent_hooks,
base_block_t *block = base_block_alloc(tsdn, base, extent_hooks,
base_ind_get(base), &base->pind_last, &base->extent_sn_next, size,
alignment);
malloc_mutex_lock(tsdn, &base->mtx);
@@ -229,8 +327,16 @@ base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
base->allocated += sizeof(base_block_t);
base->resident += PAGE_CEILING(sizeof(base_block_t));
base->mapped += block->size;
if (metadata_thp_madvise() &&
!(opt_metadata_thp == metadata_thp_auto
&& !base->auto_thp_switched)) {
assert(base->n_thp > 0);
base->n_thp += HUGEPAGE_CEILING(sizeof(base_block_t)) >>
LG_HUGEPAGE;
}
assert(base->allocated <= base->resident);
assert(base->resident <= base->mapped);
assert(base->n_thp << LG_HUGEPAGE <= base->mapped);
}
return &block->extent;
}
@@ -244,7 +350,7 @@ base_t *
base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
pszind_t pind_last = 0;
size_t extent_sn_next = 0;
base_block_t *block = base_block_alloc(tsdn, extent_hooks, ind,
base_block_t *block = base_block_alloc(tsdn, NULL, extent_hooks, ind,
&pind_last, &extent_sn_next, sizeof(base_t), QUANTUM);
if (block == NULL) {
return NULL;
@@ -265,17 +371,22 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
base->pind_last = pind_last;
base->extent_sn_next = extent_sn_next;
base->blocks = block;
for (szind_t i = 0; i < NSIZES; i++) {
base->auto_thp_switched = false;
for (szind_t i = 0; i < SC_NSIZES; i++) {
extent_heap_new(&base->avail[i]);
}
if (config_stats) {
base->allocated = sizeof(base_block_t);
base->resident = PAGE_CEILING(sizeof(base_block_t));
base->mapped = block->size;
base->n_thp = (opt_metadata_thp == metadata_thp_always) &&
metadata_thp_madvise() ? HUGEPAGE_CEILING(sizeof(base_block_t))
>> LG_HUGEPAGE : 0;
assert(base->allocated <= base->resident);
assert(base->resident <= base->mapped);
assert(base->n_thp << LG_HUGEPAGE <= base->mapped);
}
base_extent_bump_alloc_post(tsdn, base, &block->extent, gap_size, base,
base_extent_bump_alloc_post(base, &block->extent, gap_size, base,
base_size);
return base;
@@ -315,7 +426,7 @@ base_alloc_impl(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment,
extent_t *extent = NULL;
malloc_mutex_lock(tsdn, &base->mtx);
for (szind_t i = sz_size2index(asize); i < NSIZES; i++) {
for (szind_t i = sz_size2index(asize); i < SC_NSIZES; i++) {
extent = extent_heap_remove_first(&base->avail[i]);
if (extent != NULL) {
/* Use existing space. */
@@ -332,7 +443,7 @@ base_alloc_impl(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment,
goto label_return;
}
ret = base_extent_bump_alloc(tsdn, base, extent, usize, alignment);
ret = base_extent_bump_alloc(base, extent, usize, alignment);
if (esn != NULL) {
*esn = extent_sn_get(extent);
}
@@ -368,7 +479,7 @@ base_alloc_extent(tsdn_t *tsdn, base_t *base) {
void
base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, size_t *resident,
size_t *mapped) {
size_t *mapped, size_t *n_thp) {
cassert(config_stats);
malloc_mutex_lock(tsdn, &base->mtx);
@@ -377,6 +488,7 @@ base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, size_t *resident,
*allocated = base->allocated;
*resident = base->resident;
*mapped = base->mapped;
*n_thp = base->n_thp;
malloc_mutex_unlock(tsdn, &base->mtx);
}

95
deps/jemalloc/src/bin.c vendored Normal file
View File

@@ -0,0 +1,95 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/bin.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/witness.h"
bin_info_t bin_infos[SC_NBINS];
static void
bin_infos_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
bin_info_t bin_infos[SC_NBINS]) {
for (unsigned i = 0; i < SC_NBINS; i++) {
bin_info_t *bin_info = &bin_infos[i];
sc_t *sc = &sc_data->sc[i];
bin_info->reg_size = ((size_t)1U << sc->lg_base)
+ ((size_t)sc->ndelta << sc->lg_delta);
bin_info->slab_size = (sc->pgs << LG_PAGE);
bin_info->nregs =
(uint32_t)(bin_info->slab_size / bin_info->reg_size);
bin_info->n_shards = bin_shard_sizes[i];
bitmap_info_t bitmap_info = BITMAP_INFO_INITIALIZER(
bin_info->nregs);
bin_info->bitmap_info = bitmap_info;
}
}
bool
bin_update_shard_size(unsigned bin_shard_sizes[SC_NBINS], size_t start_size,
size_t end_size, size_t nshards) {
if (nshards > BIN_SHARDS_MAX || nshards == 0) {
return true;
}
if (start_size > SC_SMALL_MAXCLASS) {
return false;
}
if (end_size > SC_SMALL_MAXCLASS) {
end_size = SC_SMALL_MAXCLASS;
}
/* Compute the index since this may happen before sz init. */
szind_t ind1 = sz_size2index_compute(start_size);
szind_t ind2 = sz_size2index_compute(end_size);
for (unsigned i = ind1; i <= ind2; i++) {
bin_shard_sizes[i] = (unsigned)nshards;
}
return false;
}
void
bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]) {
/* Load the default number of shards. */
for (unsigned i = 0; i < SC_NBINS; i++) {
bin_shard_sizes[i] = N_BIN_SHARDS_DEFAULT;
}
}
void
bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) {
assert(sc_data->initialized);
bin_infos_init(sc_data, bin_shard_sizes, bin_infos);
}
bool
bin_init(bin_t *bin) {
if (malloc_mutex_init(&bin->lock, "bin", WITNESS_RANK_BIN,
malloc_mutex_rank_exclusive)) {
return true;
}
bin->slabcur = NULL;
extent_heap_new(&bin->slabs_nonfull);
extent_list_init(&bin->slabs_full);
if (config_stats) {
memset(&bin->stats, 0, sizeof(bin_stats_t));
}
return false;
}
void
bin_prefork(tsdn_t *tsdn, bin_t *bin) {
malloc_mutex_prefork(tsdn, &bin->lock);
}
void
bin_postfork_parent(tsdn_t *tsdn, bin_t *bin) {
malloc_mutex_postfork_parent(tsdn, &bin->lock);
}
void
bin_postfork_child(tsdn_t *tsdn, bin_t *bin) {
malloc_mutex_postfork_child(tsdn, &bin->lock);
}

7
deps/jemalloc/src/ckh.c vendored
View File

@@ -275,7 +275,8 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh) {
lg_curcells++;
usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) {
if (unlikely(usize == 0
|| usize > SC_LARGE_MAXCLASS)) {
ret = true;
goto label_return;
}
@@ -320,7 +321,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) {
lg_prevbuckets = ckh->lg_curbuckets;
lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1;
usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) {
if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
return;
}
tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, NULL,
@@ -396,7 +397,7 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
ckh->keycomp = keycomp;
usize = sz_sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE);
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) {
if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
ret = true;
goto label_return;
}

1041
deps/jemalloc/src/ctl.c vendored
View File

File diff suppressed because it is too large Load Diff

55
deps/jemalloc/src/div.c vendored Normal file
View File

@@ -0,0 +1,55 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/div.h"
#include "jemalloc/internal/assert.h"
/*
* Suppose we have n = q * d, all integers. We know n and d, and want q = n / d.
*
* For any k, we have (here, all division is exact; not C-style rounding):
* floor(ceil(2^k / d) * n / 2^k) = floor((2^k + r) / d * n / 2^k), where
* r = (-2^k) mod d.
*
* Expanding this out:
* ... = floor(2^k / d * n / 2^k + r / d * n / 2^k)
* = floor(n / d + (r / d) * (n / 2^k)).
*
* The fractional part of n / d is 0 (because of the assumption that d divides n
* exactly), so we have:
* ... = n / d + floor((r / d) * (n / 2^k))
*
* So that our initial expression is equal to the quantity we seek, so long as
* (r / d) * (n / 2^k) < 1.
*
* r is a remainder mod d, so r < d and r / d < 1 always. We can make
* n / 2 ^ k < 1 by setting k = 32. This gets us a value of magic that works.
*/
void
div_init(div_info_t *div_info, size_t d) {
/* Nonsensical. */
assert(d != 0);
/*
* This would make the value of magic too high to fit into a uint32_t
* (we would want magic = 2^32 exactly). This would mess with code gen
* on 32-bit machines.
*/
assert(d != 1);
uint64_t two_to_k = ((uint64_t)1 << 32);
uint32_t magic = (uint32_t)(two_to_k / d);
/*
* We want magic = ceil(2^k / d), but C gives us floor. We have to
* increment it unless the result was exact (i.e. unless d is a power of
* two).
*/
if (two_to_k % d != 0) {
magic++;
}
div_info->magic = magic;
#ifdef JEMALLOC_DEBUG
div_info->d = d;
#endif
}

888
deps/jemalloc/src/extent.c vendored
View File

File diff suppressed because it is too large Load Diff

12
deps/jemalloc/src/extent_dss.c vendored
View File

@@ -113,7 +113,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
cassert(have_dss);
assert(size > 0);
assert(alignment > 0);
assert(alignment == ALIGNMENT_CEILING(alignment, PAGE));
/*
* sbrk() uses a signed increment argument, so take care not to
@@ -154,9 +154,10 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
(uintptr_t)gap_addr_page;
if (gap_size_page != 0) {
extent_init(gap, arena, gap_addr_page,
gap_size_page, false, NSIZES,
gap_size_page, false, SC_NSIZES,
arena_extent_sn_next(arena),
extent_state_active, false, true);
extent_state_active, false, true, true,
EXTENT_NOT_HEAD);
}
/*
* Compute the address just past the end of the desired
@@ -198,8 +199,9 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
extent_t extent;
extent_init(&extent, arena, ret, size,
size, false, NSIZES,
extent_state_active, false, true);
size, false, SC_NSIZES,
extent_state_active, false, true,
true, EXTENT_NOT_HEAD);
if (extent_purge_forced_wrapper(tsdn,
arena, &extent_hooks, &extent, 0,
size)) {

4
deps/jemalloc/src/extent_mmap.c vendored
View File

@@ -21,8 +21,8 @@ bool opt_retain =
void *
extent_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero,
bool *commit) {
void *ret = pages_map(new_addr, size, ALIGNMENT_CEILING(alignment,
PAGE), commit);
assert(alignment == ALIGNMENT_CEILING(alignment, PAGE));
void *ret = pages_map(new_addr, size, alignment, commit);
if (ret == NULL) {
return NULL;
}

195
deps/jemalloc/src/hook.c vendored Normal file
View File

@@ -0,0 +1,195 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/hook.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/seq.h"
typedef struct hooks_internal_s hooks_internal_t;
struct hooks_internal_s {
hooks_t hooks;
bool in_use;
};
seq_define(hooks_internal_t, hooks)
static atomic_u_t nhooks = ATOMIC_INIT(0);
static seq_hooks_t hooks[HOOK_MAX];
static malloc_mutex_t hooks_mu;
bool
hook_boot() {
return malloc_mutex_init(&hooks_mu, "hooks", WITNESS_RANK_HOOK,
malloc_mutex_rank_exclusive);
}
static void *
hook_install_locked(hooks_t *to_install) {
hooks_internal_t hooks_internal;
for (int i = 0; i < HOOK_MAX; i++) {
bool success = seq_try_load_hooks(&hooks_internal, &hooks[i]);
/* We hold mu; no concurrent access. */
assert(success);
if (!hooks_internal.in_use) {
hooks_internal.hooks = *to_install;
hooks_internal.in_use = true;
seq_store_hooks(&hooks[i], &hooks_internal);
atomic_store_u(&nhooks,
atomic_load_u(&nhooks, ATOMIC_RELAXED) + 1,
ATOMIC_RELAXED);
return &hooks[i];
}
}
return NULL;
}
void *
hook_install(tsdn_t *tsdn, hooks_t *to_install) {
malloc_mutex_lock(tsdn, &hooks_mu);
void *ret = hook_install_locked(to_install);
if (ret != NULL) {
tsd_global_slow_inc(tsdn);
}
malloc_mutex_unlock(tsdn, &hooks_mu);
return ret;
}
static void
hook_remove_locked(seq_hooks_t *to_remove) {
hooks_internal_t hooks_internal;
bool success = seq_try_load_hooks(&hooks_internal, to_remove);
/* We hold mu; no concurrent access. */
assert(success);
/* Should only remove hooks that were added. */
assert(hooks_internal.in_use);
hooks_internal.in_use = false;
seq_store_hooks(to_remove, &hooks_internal);
atomic_store_u(&nhooks, atomic_load_u(&nhooks, ATOMIC_RELAXED) - 1,
ATOMIC_RELAXED);
}
void
hook_remove(tsdn_t *tsdn, void *opaque) {
if (config_debug) {
char *hooks_begin = (char *)&hooks[0];
char *hooks_end = (char *)&hooks[HOOK_MAX];
char *hook = (char *)opaque;
assert(hooks_begin <= hook && hook < hooks_end
&& (hook - hooks_begin) % sizeof(seq_hooks_t) == 0);
}
malloc_mutex_lock(tsdn, &hooks_mu);
hook_remove_locked((seq_hooks_t *)opaque);
tsd_global_slow_dec(tsdn);
malloc_mutex_unlock(tsdn, &hooks_mu);
}
#define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr) \
for (int for_each_hook_counter = 0; \
for_each_hook_counter < HOOK_MAX; \
for_each_hook_counter++) { \
bool for_each_hook_success = seq_try_load_hooks( \
(hooks_internal_ptr), &hooks[for_each_hook_counter]); \
if (!for_each_hook_success) { \
continue; \
} \
if (!(hooks_internal_ptr)->in_use) { \
continue; \
}
#define FOR_EACH_HOOK_END \
}
static bool *
hook_reentrantp() {
/*
* We prevent user reentrancy within hooks. This is basically just a
* thread-local bool that triggers an early-exit.
*
* We don't fold in_hook into reentrancy. There are two reasons for
* this:
* - Right now, we turn on reentrancy during things like extent hook
* execution. Allocating during extent hooks is not officially
* supported, but we don't want to break it for the time being. These
* sorts of allocations should probably still be hooked, though.
* - If a hook allocates, we may want it to be relatively fast (after
* all, it executes on every allocator operation). Turning on
* reentrancy is a fairly heavyweight mode (disabling tcache,
* redirecting to arena 0, etc.). It's possible we may one day want
* to turn on reentrant mode here, if it proves too difficult to keep
* this working. But that's fairly easy for us to see; OTOH, people
* not using hooks because they're too slow is easy for us to miss.
*
* The tricky part is
* that this code might get invoked even if we don't have access to tsd.
* This function mimics getting a pointer to thread-local data, except
* that it might secretly return a pointer to some global data if we
* know that the caller will take the early-exit path.
* If we return a bool that indicates that we are reentrant, then the
* caller will go down the early exit path, leaving the global
* untouched.
*/
static bool in_hook_global = true;
tsdn_t *tsdn = tsdn_fetch();
tcache_t *tcache = tsdn_tcachep_get(tsdn);
if (tcache != NULL) {
return &tcache->in_hook;
}
return &in_hook_global;
}
#define HOOK_PROLOGUE \
if (likely(atomic_load_u(&nhooks, ATOMIC_RELAXED) == 0)) { \
return; \
} \
bool *in_hook = hook_reentrantp(); \
if (*in_hook) { \
return; \
} \
*in_hook = true;
#define HOOK_EPILOGUE \
*in_hook = false;
void
hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
uintptr_t args_raw[3]) {
HOOK_PROLOGUE
hooks_internal_t hook;
FOR_EACH_HOOK_BEGIN(&hook)
hook_alloc h = hook.hooks.alloc_hook;
if (h != NULL) {
h(hook.hooks.extra, type, result, result_raw, args_raw);
}
FOR_EACH_HOOK_END
HOOK_EPILOGUE
}
void
hook_invoke_dalloc(hook_dalloc_t type, void *address, uintptr_t args_raw[3]) {
HOOK_PROLOGUE
hooks_internal_t hook;
FOR_EACH_HOOK_BEGIN(&hook)
hook_dalloc h = hook.hooks.dalloc_hook;
if (h != NULL) {
h(hook.hooks.extra, type, address, args_raw);
}
FOR_EACH_HOOK_END
HOOK_EPILOGUE
}
void
hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]) {
HOOK_PROLOGUE
hooks_internal_t hook;
FOR_EACH_HOOK_BEGIN(&hook)
hook_expand h = hook.hooks.expand_hook;
if (h != NULL) {
h(hook.hooks.extra, type, address, old_usize, new_usize,
result_raw, args_raw);
}
FOR_EACH_HOOK_END
HOOK_EPILOGUE
}

1224
deps/jemalloc/src/jemalloc.c vendored
View File

File diff suppressed because it is too large Load Diff

27
deps/jemalloc/src/jemalloc_cpp.cpp vendored
View File

@@ -39,12 +39,10 @@ void operator delete(void *ptr, std::size_t size) noexcept;
void operator delete[](void *ptr, std::size_t size) noexcept;
#endif
template <bool IsNoExcept>
void *
newImpl(std::size_t size) noexcept(IsNoExcept) {
void *ptr = je_malloc(size);
if (likely(ptr != nullptr))
return ptr;
JEMALLOC_NOINLINE
static void *
handleOOM(std::size_t size, bool nothrow) {
void *ptr = nullptr;
while (ptr == nullptr) {
std::new_handler handler;
@@ -68,11 +66,22 @@ newImpl(std::size_t size) noexcept(IsNoExcept) {
ptr = je_malloc(size);
}
if (ptr == nullptr && !IsNoExcept)
if (ptr == nullptr && !nothrow)
std::__throw_bad_alloc();
return ptr;
}
template <bool IsNoExcept>
JEMALLOC_ALWAYS_INLINE
void *
newImpl(std::size_t size) noexcept(IsNoExcept) {
void *ptr = je_malloc(size);
if (likely(ptr != nullptr))
return ptr;
return handleOOM(size, IsNoExcept);
}
void *
operator new(std::size_t size) {
return newImpl<false>(size);
@@ -119,14 +128,14 @@ operator delete(void *ptr, std::size_t size) noexcept {
if (unlikely(ptr == nullptr)) {
return;
}
je_sdallocx(ptr, size, /*flags=*/0);
je_sdallocx_noflags(ptr, size);
}
void operator delete[](void *ptr, std::size_t size) noexcept {
if (unlikely(ptr == nullptr)) {
return;
}
je_sdallocx(ptr, size, /*flags=*/0);
je_sdallocx_noflags(ptr, size);
}
#endif // __cpp_sized_deallocation

52
deps/jemalloc/src/large.c vendored
View File

@@ -28,7 +28,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
assert(!tsdn_null(tsdn) || arena != NULL);
ausize = sz_sa2u(usize, alignment);
if (unlikely(ausize == 0 || ausize > LARGE_MAXCLASS)) {
if (unlikely(ausize == 0 || ausize > SC_LARGE_MAXCLASS)) {
return NULL;
}
@@ -42,7 +42,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
*/
is_zeroed = zero;
if (likely(!tsdn_null(tsdn))) {
arena = arena_choose(tsdn_tsd(tsdn), arena);
arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize);
}
if (unlikely(arena == NULL) || (extent = arena_extent_alloc_large(tsdn,
arena, usize, alignment, &is_zeroed)) == NULL) {
@@ -109,7 +109,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) {
if (diff != 0) {
extent_t *trail = extent_split_wrapper(tsdn, arena,
&extent_hooks, extent, usize + sz_large_pad,
sz_size2index(usize), false, diff, NSIZES, false);
sz_size2index(usize), false, diff, SC_NSIZES, false);
if (trail == NULL) {
return true;
}
@@ -154,17 +154,17 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
bool new_mapping;
if ((trail = extents_alloc(tsdn, arena, &extent_hooks,
&arena->extents_dirty, extent_past_get(extent), trailsize, 0,
CACHELINE, false, NSIZES, &is_zeroed_trail, &commit)) != NULL
CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL
|| (trail = extents_alloc(tsdn, arena, &extent_hooks,
&arena->extents_muzzy, extent_past_get(extent), trailsize, 0,
CACHELINE, false, NSIZES, &is_zeroed_trail, &commit)) != NULL) {
CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) {
if (config_stats) {
new_mapping = false;
}
} else {
if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks,
extent_past_get(extent), trailsize, 0, CACHELINE, false,
NSIZES, &is_zeroed_trail, &commit)) == NULL) {
SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) {
return true;
}
if (config_stats) {
@@ -221,9 +221,10 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
size_t oldusize = extent_usize_get(extent);
/* The following should have been caught by callers. */
assert(usize_min > 0 && usize_max <= LARGE_MAXCLASS);
assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS);
/* Both allocation sizes must be large to avoid a move. */
assert(oldusize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS);
assert(oldusize >= SC_LARGE_MINCLASS
&& usize_max >= SC_LARGE_MINCLASS);
if (usize_max > oldusize) {
/* Attempt to expand the allocation in-place. */
@@ -270,17 +271,23 @@ large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
}
void *
large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
size_t alignment, bool zero, tcache_t *tcache) {
size_t oldusize = extent_usize_get(extent);
large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
size_t alignment, bool zero, tcache_t *tcache,
hook_ralloc_args_t *hook_args) {
extent_t *extent = iealloc(tsdn, ptr);
size_t oldusize = extent_usize_get(extent);
/* The following should have been caught by callers. */
assert(usize > 0 && usize <= LARGE_MAXCLASS);
assert(usize > 0 && usize <= SC_LARGE_MAXCLASS);
/* Both allocation sizes must be large to avoid a move. */
assert(oldusize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS);
assert(oldusize >= SC_LARGE_MINCLASS
&& usize >= SC_LARGE_MINCLASS);
/* Try to avoid moving the allocation. */
if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero)) {
hook_invoke_expand(hook_args->is_realloc
? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize,
usize, (uintptr_t)ptr, hook_args->args);
return extent_addr_get(extent);
}
@@ -295,6 +302,12 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
return NULL;
}
hook_invoke_alloc(hook_args->is_realloc
? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret,
hook_args->args);
hook_invoke_dalloc(hook_args->is_realloc
? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args);
size_t copysize = (usize < oldusize) ? usize : oldusize;
memcpy(ret, extent_addr_get(extent), copysize);
isdalloct(tsdn, extent_addr_get(extent), oldusize, tcache, NULL, true);
@@ -318,8 +331,9 @@ large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
large_dalloc_maybe_junk(extent_addr_get(extent),
extent_usize_get(extent));
} else {
malloc_mutex_assert_owner(tsdn, &arena->large_mtx);
/* Only hold the large_mtx if necessary. */
if (!arena_is_auto(arena)) {
malloc_mutex_assert_owner(tsdn, &arena->large_mtx);
extent_list_remove(&arena->large, extent);
}
}
@@ -369,3 +383,13 @@ void
large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent) {
large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U);
}
nstime_t
large_prof_alloc_time_get(const extent_t *extent) {
return extent_prof_alloc_time_get(extent);
}
void
large_prof_alloc_time_set(extent_t *extent, nstime_t t) {
extent_prof_alloc_time_set(extent, t);
}

78
deps/jemalloc/src/log.c vendored Normal file
View File

@@ -0,0 +1,78 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/log.h"
char log_var_names[JEMALLOC_LOG_VAR_BUFSIZE];
atomic_b_t log_init_done = ATOMIC_INIT(false);
/*
* Returns true if we were able to pick out a segment. Fills in r_segment_end
* with a pointer to the first character after the end of the string.
*/
static const char *
log_var_extract_segment(const char* segment_begin) {
const char *end;
for (end = segment_begin; *end != '\0' && *end != '|'; end++) {
}
return end;
}
static bool
log_var_matches_segment(const char *segment_begin, const char *segment_end,
const char *log_var_begin, const char *log_var_end) {
assert(segment_begin <= segment_end);
assert(log_var_begin < log_var_end);
ptrdiff_t segment_len = segment_end - segment_begin;
ptrdiff_t log_var_len = log_var_end - log_var_begin;
/* The special '.' segment matches everything. */
if (segment_len == 1 && *segment_begin == '.') {
return true;
}
if (segment_len == log_var_len) {
return strncmp(segment_begin, log_var_begin, segment_len) == 0;
} else if (segment_len < log_var_len) {
return strncmp(segment_begin, log_var_begin, segment_len) == 0
&& log_var_begin[segment_len] == '.';
} else {
return false;
}
}
unsigned
log_var_update_state(log_var_t *log_var) {
const char *log_var_begin = log_var->name;
const char *log_var_end = log_var->name + strlen(log_var->name);
/* Pointer to one before the beginning of the current segment. */
const char *segment_begin = log_var_names;
/*
* If log_init done is false, we haven't parsed the malloc conf yet. To
* avoid log-spew, we default to not displaying anything.
*/
if (!atomic_load_b(&log_init_done, ATOMIC_ACQUIRE)) {
return LOG_INITIALIZED_NOT_ENABLED;
}
while (true) {
const char *segment_end = log_var_extract_segment(
segment_begin);
assert(segment_end < log_var_names + JEMALLOC_LOG_VAR_BUFSIZE);
if (log_var_matches_segment(segment_begin, segment_end,
log_var_begin, log_var_end)) {
atomic_store_u(&log_var->state, LOG_ENABLED,
ATOMIC_RELAXED);
return LOG_ENABLED;
}
if (*segment_end == '\0') {
/* Hit the end of the segment string with no match. */
atomic_store_u(&log_var->state,
LOG_INITIALIZED_NOT_ENABLED, ATOMIC_RELAXED);
return LOG_INITIALIZED_NOT_ENABLED;
}
/* Otherwise, skip the delimiter and continue. */
segment_begin = segment_end + 1;
}
}

20
deps/jemalloc/src/malloc_io.c vendored
View File

@@ -70,20 +70,7 @@ static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
/* malloc_message() setup. */
static void
wrtmessage(void *cbopaque, const char *s) {
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_write)
/*
* Use syscall(2) rather than write(2) when possible in order to avoid
* the possibility of memory allocation within libc. This is necessary
* on FreeBSD; most operating systems do not have this problem though.
*
* syscall() returns long or int, depending on platform, so capture the
* unused result in the widest plausible type to avoid compiler
* warnings.
*/
UNUSED long result = syscall(SYS_write, STDERR_FILENO, s, strlen(s));
#else
UNUSED ssize_t result = write(STDERR_FILENO, s, strlen(s));
#endif
malloc_write_fd(STDERR_FILENO, s, strlen(s));
}
JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s);
@@ -111,7 +98,7 @@ buferror(int err, char *buf, size_t buflen) {
FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0,
(LPSTR)buf, (DWORD)buflen, NULL);
return 0;
#elif defined(__GLIBC__) && defined(_GNU_SOURCE)
#elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE)
char *b = strerror_r(err, buf, buflen);
if (b != buf) {
strncpy(buf, b, buflen);
@@ -375,7 +362,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
} \
} while (0)
#define GET_ARG_NUMERIC(val, len) do { \
switch (len) { \
switch ((unsigned char)len) { \
case '?': \
val = va_arg(ap, int); \
break; \
@@ -645,7 +632,6 @@ malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
*/
write_cb = (je_malloc_message != NULL) ? je_malloc_message :
wrtmessage;
cbopaque = NULL;
}
malloc_vsnprintf(buf, sizeof(buf), format, ap);

14
deps/jemalloc/src/mutex.c vendored
View File

@@ -4,6 +4,7 @@
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/spin.h"
#ifndef _CRT_SPINCOUNT
#define _CRT_SPINCOUNT 4000
@@ -45,7 +46,7 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
void
malloc_mutex_lock_slow(malloc_mutex_t *mutex) {
mutex_prof_data_t *data = &mutex->prof_data;
UNUSED nstime_t before = NSTIME_ZERO_INITIALIZER;
nstime_t before = NSTIME_ZERO_INITIALIZER;
if (ncpus == 1) {
goto label_spin_done;
@@ -53,8 +54,9 @@ malloc_mutex_lock_slow(malloc_mutex_t *mutex) {
int cnt = 0, max_cnt = MALLOC_MUTEX_MAX_SPIN;
do {
CPU_SPINWAIT;
if (!malloc_mutex_trylock_final(mutex)) {
spin_cpu_spinwait();
if (!atomic_load_b(&mutex->locked, ATOMIC_RELAXED)
&& !malloc_mutex_trylock_final(mutex)) {
data->n_spin_acquired++;
return;
}
@@ -143,9 +145,7 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
}
# endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
mutex->lock = OS_UNFAIR_LOCK_INIT;
#elif (defined(JEMALLOC_OSSPIN))
mutex->lock = 0;
mutex->lock = OS_UNFAIR_LOCK_INIT;
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
if (postpone_init) {
mutex->postponed_next = postponed_mutexes;
@@ -173,7 +173,7 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
mutex->lock_order = lock_order;
if (lock_order == malloc_mutex_address_ordered) {
witness_init(&mutex->witness, name, rank,
mutex_addr_comp, &mutex);
mutex_addr_comp, mutex);
} else {
witness_init(&mutex->witness, name, rank, NULL, NULL);
}

278
deps/jemalloc/src/pages.c vendored
View File

@@ -10,6 +10,9 @@
#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
#include <sys/sysctl.h>
#ifdef __FreeBSD__
#include <vm/vm_param.h>
#endif
#endif
/******************************************************************************/
@@ -25,6 +28,18 @@ static int mmap_flags;
#endif
static bool os_overcommits;
const char *thp_mode_names[] = {
"default",
"always",
"never",
"not supported"
};
thp_mode_t opt_thp = THP_MODE_DEFAULT;
thp_mode_t init_system_thp_mode;
/* Runtime support for lazy purge. Irrelevant when !pages_can_purge_lazy. */
static bool pages_can_purge_lazy_runtime = true;
/******************************************************************************/
/*
* Function prototypes for static functions that are referenced prior to
@@ -165,6 +180,35 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
assert(alignment >= PAGE);
assert(ALIGNMENT_ADDR2BASE(addr, alignment) == addr);
#if defined(__FreeBSD__) && defined(MAP_EXCL)
/*
* FreeBSD has mechanisms both to mmap at specific address without
* touching existing mappings, and to mmap with specific alignment.
*/
{
if (os_overcommits) {
*commit = true;
}
int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
int flags = mmap_flags;
if (addr != NULL) {
flags |= MAP_FIXED | MAP_EXCL;
} else {
unsigned alignment_bits = ffs_zu(alignment);
assert(alignment_bits > 1);
flags |= MAP_ALIGNED(alignment_bits - 1);
}
void *ret = mmap(addr, size, prot, flags, -1, 0);
if (ret == MAP_FAILED) {
ret = NULL;
}
return ret;
}
#endif
/*
* Ideally, there would be a way to specify alignment to mmap() (like
* NetBSD has), but in the absence of such a feature, we have to work
@@ -246,19 +290,31 @@ pages_decommit(void *addr, size_t size) {
bool
pages_purge_lazy(void *addr, size_t size) {
assert(PAGE_ADDR2BASE(addr) == addr);
assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
assert(PAGE_CEILING(size) == size);
if (!pages_can_purge_lazy) {
return true;
}
if (!pages_can_purge_lazy_runtime) {
/*
* Built with lazy purge enabled, but detected it was not
* supported on the current system.
*/
return true;
}
#ifdef _WIN32
VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
return false;
#elif defined(JEMALLOC_PURGE_MADVISE_FREE) && \
!defined(PAGES_CAN_PURGE_LAZY)
return (madvise(addr, size, MADV_FREE) != 0);
#elif defined(JEMALLOC_PURGE_MADVISE_FREE)
return (madvise(addr, size,
# ifdef MADV_FREE
MADV_FREE
# else
JEMALLOC_MADV_FREE
# endif
) != 0);
#elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \
!defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)
return (madvise(addr, size, MADV_DONTNEED) != 0);
@@ -287,12 +343,13 @@ pages_purge_forced(void *addr, size_t size) {
#endif
}
bool
pages_huge(void *addr, size_t size) {
assert(HUGEPAGE_ADDR2BASE(addr) == addr);
assert(HUGEPAGE_CEILING(size) == size);
#ifdef JEMALLOC_THP
static bool
pages_huge_impl(void *addr, size_t size, bool aligned) {
if (aligned) {
assert(HUGEPAGE_ADDR2BASE(addr) == addr);
assert(HUGEPAGE_CEILING(size) == size);
}
#ifdef JEMALLOC_HAVE_MADVISE_HUGE
return (madvise(addr, size, MADV_HUGEPAGE) != 0);
#else
return true;
@@ -300,23 +357,74 @@ pages_huge(void *addr, size_t size) {
}
bool
pages_nohuge(void *addr, size_t size) {
assert(HUGEPAGE_ADDR2BASE(addr) == addr);
assert(HUGEPAGE_CEILING(size) == size);
pages_huge(void *addr, size_t size) {
return pages_huge_impl(addr, size, true);
}
#ifdef JEMALLOC_THP
static bool
pages_huge_unaligned(void *addr, size_t size) {
return pages_huge_impl(addr, size, false);
}
static bool
pages_nohuge_impl(void *addr, size_t size, bool aligned) {
if (aligned) {
assert(HUGEPAGE_ADDR2BASE(addr) == addr);
assert(HUGEPAGE_CEILING(size) == size);
}
#ifdef JEMALLOC_HAVE_MADVISE_HUGE
return (madvise(addr, size, MADV_NOHUGEPAGE) != 0);
#else
return false;
#endif
}
bool
pages_nohuge(void *addr, size_t size) {
return pages_nohuge_impl(addr, size, true);
}
static bool
pages_nohuge_unaligned(void *addr, size_t size) {
return pages_nohuge_impl(addr, size, false);
}
bool
pages_dontdump(void *addr, size_t size) {
assert(PAGE_ADDR2BASE(addr) == addr);
assert(PAGE_CEILING(size) == size);
#ifdef JEMALLOC_MADVISE_DONTDUMP
return madvise(addr, size, MADV_DONTDUMP) != 0;
#else
return false;
#endif
}
bool
pages_dodump(void *addr, size_t size) {
assert(PAGE_ADDR2BASE(addr) == addr);
assert(PAGE_CEILING(size) == size);
#ifdef JEMALLOC_MADVISE_DONTDUMP
return madvise(addr, size, MADV_DODUMP) != 0;
#else
return false;
#endif
}
static size_t
os_page_detect(void) {
#ifdef _WIN32
SYSTEM_INFO si;
GetSystemInfo(&si);
return si.dwPageSize;
#elif defined(__FreeBSD__)
/*
* This returns the value obtained from
* the auxv vector, avoiding a syscall.
*/
return getpagesize();
#else
long result = sysconf(_SC_PAGESIZE);
if (result == -1) {
@@ -333,9 +441,19 @@ os_overcommits_sysctl(void) {
size_t sz;
sz = sizeof(vm_overcommit);
#if defined(__FreeBSD__) && defined(VM_OVERCOMMIT)
int mib[2];
mib[0] = CTL_VM;
mib[1] = VM_OVERCOMMIT;
if (sysctl(mib, 2, &vm_overcommit, &sz, NULL, 0) != 0) {
return false; /* Error. */
}
#else
if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0) {
return false; /* Error. */
}
#endif
return ((vm_overcommit & 0x3) == 0);
}
@@ -351,27 +469,44 @@ static bool
os_overcommits_proc(void) {
int fd;
char buf[1];
ssize_t nread;
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY |
O_CLOEXEC);
#if defined(O_CLOEXEC)
fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY |
O_CLOEXEC);
#else
fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY);
if (fd != -1) {
fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
}
#endif
#elif defined(JEMALLOC_USE_SYSCALL) && defined(SYS_openat)
fd = (int)syscall(SYS_openat,
AT_FDCWD, "/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC);
#if defined(O_CLOEXEC)
fd = (int)syscall(SYS_openat,
AT_FDCWD, "/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC);
#else
fd = (int)syscall(SYS_openat,
AT_FDCWD, "/proc/sys/vm/overcommit_memory", O_RDONLY);
if (fd != -1) {
fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
}
#endif
#else
fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC);
#if defined(O_CLOEXEC)
fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC);
#else
fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
if (fd != -1) {
fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
}
#endif
#endif
if (fd == -1) {
return false; /* Error. */
}
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read)
nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf));
#else
nread = read(fd, &buf, sizeof(buf));
#endif
ssize_t nread = malloc_read_fd(fd, &buf, sizeof(buf));
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
syscall(SYS_close, fd);
#else
@@ -391,6 +526,75 @@ os_overcommits_proc(void) {
}
#endif
void
pages_set_thp_state (void *ptr, size_t size) {
if (opt_thp == thp_mode_default || opt_thp == init_system_thp_mode) {
return;
}
assert(opt_thp != thp_mode_not_supported &&
init_system_thp_mode != thp_mode_not_supported);
if (opt_thp == thp_mode_always
&& init_system_thp_mode != thp_mode_never) {
assert(init_system_thp_mode == thp_mode_default);
pages_huge_unaligned(ptr, size);
} else if (opt_thp == thp_mode_never) {
assert(init_system_thp_mode == thp_mode_default ||
init_system_thp_mode == thp_mode_always);
pages_nohuge_unaligned(ptr, size);
}
}
static void
init_thp_state(void) {
if (!have_madvise_huge) {
if (metadata_thp_enabled() && opt_abort) {
malloc_write("<jemalloc>: no MADV_HUGEPAGE support\n");
abort();
}
goto label_error;
}
static const char sys_state_madvise[] = "always [madvise] never\n";
static const char sys_state_always[] = "[always] madvise never\n";
static const char sys_state_never[] = "always madvise [never]\n";
char buf[sizeof(sys_state_madvise)];
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
int fd = (int)syscall(SYS_open,
"/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
#else
int fd = open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
#endif
if (fd == -1) {
goto label_error;
}
ssize_t nread = malloc_read_fd(fd, &buf, sizeof(buf));
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
syscall(SYS_close, fd);
#else
close(fd);
#endif
if (nread < 0) {
goto label_error;
}
if (strncmp(buf, sys_state_madvise, (size_t)nread) == 0) {
init_system_thp_mode = thp_mode_default;
} else if (strncmp(buf, sys_state_always, (size_t)nread) == 0) {
init_system_thp_mode = thp_mode_always;
} else if (strncmp(buf, sys_state_never, (size_t)nread) == 0) {
init_system_thp_mode = thp_mode_never;
} else {
goto label_error;
}
return;
label_error:
opt_thp = init_system_thp_mode = thp_mode_not_supported;
}
bool
pages_boot(void) {
os_page = os_page_detect();
@@ -419,5 +623,27 @@ pages_boot(void) {
os_overcommits = false;
#endif
init_thp_state();
#ifdef __FreeBSD__
/*
* FreeBSD doesn't need the check; madvise(2) is known to work.
*/
#else
/* Detect lazy purge runtime support. */
if (pages_can_purge_lazy) {
bool committed = false;
void *madv_free_page = os_pages_map(NULL, PAGE, PAGE, &committed);
if (madv_free_page == NULL) {
return true;
}
assert(pages_can_purge_lazy_runtime);
if (pages_purge_lazy(madv_free_page, PAGE)) {
pages_can_purge_lazy_runtime = false;
}
os_pages_unmap(madv_free_page, PAGE);
}
#endif
return false;
}

742
deps/jemalloc/src/prof.c vendored
View File

@@ -7,6 +7,7 @@
#include "jemalloc/internal/hash.h"
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/emitter.h"
/******************************************************************************/
@@ -23,7 +24,7 @@
*/
#undef _Unwind_Backtrace
#include <unwind.h>
#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook)
#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook)
#endif
/******************************************************************************/
@@ -38,6 +39,7 @@ bool opt_prof_gdump = false;
bool opt_prof_final = false;
bool opt_prof_leak = false;
bool opt_prof_accum = false;
bool opt_prof_log = false;
char opt_prof_prefix[
/* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF
@@ -70,6 +72,100 @@ uint64_t prof_interval = 0;
size_t lg_prof_sample;
typedef enum prof_logging_state_e prof_logging_state_t;
enum prof_logging_state_e {
prof_logging_state_stopped,
prof_logging_state_started,
prof_logging_state_dumping
};
/*
* - stopped: log_start never called, or previous log_stop has completed.
* - started: log_start called, log_stop not called yet. Allocations are logged.
* - dumping: log_stop called but not finished; samples are not logged anymore.
*/
prof_logging_state_t prof_logging_state = prof_logging_state_stopped;
#ifdef JEMALLOC_JET
static bool prof_log_dummy = false;
#endif
/* Incremented for every log file that is output. */
static uint64_t log_seq = 0;
static char log_filename[
/* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF
PATH_MAX +
#endif
1];
/* Timestamp for most recent call to log_start(). */
static nstime_t log_start_timestamp = NSTIME_ZERO_INITIALIZER;
/* Increment these when adding to the log_bt and log_thr linked lists. */
static size_t log_bt_index = 0;
static size_t log_thr_index = 0;
/* Linked list node definitions. These are only used in prof.c. */
typedef struct prof_bt_node_s prof_bt_node_t;
struct prof_bt_node_s {
prof_bt_node_t *next;
size_t index;
prof_bt_t bt;
/* Variable size backtrace vector pointed to by bt. */
void *vec[1];
};
typedef struct prof_thr_node_s prof_thr_node_t;
struct prof_thr_node_s {
prof_thr_node_t *next;
size_t index;
uint64_t thr_uid;
/* Variable size based on thr_name_sz. */
char name[1];
};
typedef struct prof_alloc_node_s prof_alloc_node_t;
/* This is output when logging sampled allocations. */
struct prof_alloc_node_s {
prof_alloc_node_t *next;
/* Indices into an array of thread data. */
size_t alloc_thr_ind;
size_t free_thr_ind;
/* Indices into an array of backtraces. */
size_t alloc_bt_ind;
size_t free_bt_ind;
uint64_t alloc_time_ns;
uint64_t free_time_ns;
size_t usize;
};
/*
* Created on the first call to prof_log_start and deleted on prof_log_stop.
* These are the backtraces and threads that have already been logged by an
* allocation.
*/
static bool log_tables_initialized = false;
static ckh_t log_bt_node_set;
static ckh_t log_thr_node_set;
/* Store linked lists for logged data. */
static prof_bt_node_t *log_bt_first = NULL;
static prof_bt_node_t *log_bt_last = NULL;
static prof_thr_node_t *log_thr_first = NULL;
static prof_thr_node_t *log_thr_last = NULL;
static prof_alloc_node_t *log_alloc_first = NULL;
static prof_alloc_node_t *log_alloc_last = NULL;
/* Protects the prof_logging_state and any log_{...} variable. */
static malloc_mutex_t log_mtx;
/*
* Table of mutexes that are shared among gctx's. These are leaf locks, so
* there is no problem with using them for more than one gctx at the same time.
@@ -145,6 +241,12 @@ static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
bool even_if_attached);
static char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name);
/* Hashtable functions for log_bt_node_set and log_thr_node_set. */
static void prof_thr_node_hash(const void *key, size_t r_hash[2]);
static bool prof_thr_node_keycomp(const void *k1, const void *k2);
static void prof_bt_node_hash(const void *key, size_t r_hash[2]);
static bool prof_bt_node_keycomp(const void *k1, const void *k2);
/******************************************************************************/
/* Red-black trees. */
@@ -242,6 +344,12 @@ prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
prof_tctx_t *tctx) {
prof_tctx_set(tsdn, ptr, usize, NULL, tctx);
/* Get the current time and set this in the extent_t. We'll read this
* when free() is called. */
nstime_t t = NSTIME_ZERO_INITIALIZER;
nstime_update(&t);
prof_alloc_time_set(tsdn, ptr, NULL, t);
malloc_mutex_lock(tsdn, tctx->tdata->lock);
tctx->cnts.curobjs++;
tctx->cnts.curbytes += usize;
@@ -253,14 +361,174 @@ prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
malloc_mutex_unlock(tsdn, tctx->tdata->lock);
}
static size_t
prof_log_bt_index(tsd_t *tsd, prof_bt_t *bt) {
assert(prof_logging_state == prof_logging_state_started);
malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx);
prof_bt_node_t dummy_node;
dummy_node.bt = *bt;
prof_bt_node_t *node;
/* See if this backtrace is already cached in the table. */
if (ckh_search(&log_bt_node_set, (void *)(&dummy_node),
(void **)(&node), NULL)) {
size_t sz = offsetof(prof_bt_node_t, vec) +
(bt->len * sizeof(void *));
prof_bt_node_t *new_node = (prof_bt_node_t *)
iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL,
true, arena_get(TSDN_NULL, 0, true), true);
if (log_bt_first == NULL) {
log_bt_first = new_node;
log_bt_last = new_node;
} else {
log_bt_last->next = new_node;
log_bt_last = new_node;
}
new_node->next = NULL;
new_node->index = log_bt_index;
/*
* Copy the backtrace: bt is inside a tdata or gctx, which
* might die before prof_log_stop is called.
*/
new_node->bt.len = bt->len;
memcpy(new_node->vec, bt->vec, bt->len * sizeof(void *));
new_node->bt.vec = new_node->vec;
log_bt_index++;
ckh_insert(tsd, &log_bt_node_set, (void *)new_node, NULL);
return new_node->index;
} else {
return node->index;
}
}
static size_t
prof_log_thr_index(tsd_t *tsd, uint64_t thr_uid, const char *name) {
assert(prof_logging_state == prof_logging_state_started);
malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx);
prof_thr_node_t dummy_node;
dummy_node.thr_uid = thr_uid;
prof_thr_node_t *node;
/* See if this thread is already cached in the table. */
if (ckh_search(&log_thr_node_set, (void *)(&dummy_node),
(void **)(&node), NULL)) {
size_t sz = offsetof(prof_thr_node_t, name) + strlen(name) + 1;
prof_thr_node_t *new_node = (prof_thr_node_t *)
iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL,
true, arena_get(TSDN_NULL, 0, true), true);
if (log_thr_first == NULL) {
log_thr_first = new_node;
log_thr_last = new_node;
} else {
log_thr_last->next = new_node;
log_thr_last = new_node;
}
new_node->next = NULL;
new_node->index = log_thr_index;
new_node->thr_uid = thr_uid;
strcpy(new_node->name, name);
log_thr_index++;
ckh_insert(tsd, &log_thr_node_set, (void *)new_node, NULL);
return new_node->index;
} else {
return node->index;
}
}
static void
prof_try_log(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx) {
malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
prof_tdata_t *cons_tdata = prof_tdata_get(tsd, false);
if (cons_tdata == NULL) {
/*
* We decide not to log these allocations. cons_tdata will be
* NULL only when the current thread is in a weird state (e.g.
* it's being destroyed).
*/
return;
}
malloc_mutex_lock(tsd_tsdn(tsd), &log_mtx);
if (prof_logging_state != prof_logging_state_started) {
goto label_done;
}
if (!log_tables_initialized) {
bool err1 = ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS,
prof_bt_node_hash, prof_bt_node_keycomp);
bool err2 = ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS,
prof_thr_node_hash, prof_thr_node_keycomp);
if (err1 || err2) {
goto label_done;
}
log_tables_initialized = true;
}
nstime_t alloc_time = prof_alloc_time_get(tsd_tsdn(tsd), ptr,
(alloc_ctx_t *)NULL);
nstime_t free_time = NSTIME_ZERO_INITIALIZER;
nstime_update(&free_time);
size_t sz = sizeof(prof_alloc_node_t);
prof_alloc_node_t *new_node = (prof_alloc_node_t *)
iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true,
arena_get(TSDN_NULL, 0, true), true);
const char *prod_thr_name = (tctx->tdata->thread_name == NULL)?
"" : tctx->tdata->thread_name;
const char *cons_thr_name = prof_thread_name_get(tsd);
prof_bt_t bt;
/* Initialize the backtrace, using the buffer in tdata to store it. */
bt_init(&bt, cons_tdata->vec);
prof_backtrace(&bt);
prof_bt_t *cons_bt = &bt;
/* We haven't destroyed tctx yet, so gctx should be good to read. */
prof_bt_t *prod_bt = &tctx->gctx->bt;
new_node->next = NULL;
new_node->alloc_thr_ind = prof_log_thr_index(tsd, tctx->tdata->thr_uid,
prod_thr_name);
new_node->free_thr_ind = prof_log_thr_index(tsd, cons_tdata->thr_uid,
cons_thr_name);
new_node->alloc_bt_ind = prof_log_bt_index(tsd, prod_bt);
new_node->free_bt_ind = prof_log_bt_index(tsd, cons_bt);
new_node->alloc_time_ns = nstime_ns(&alloc_time);
new_node->free_time_ns = nstime_ns(&free_time);
new_node->usize = usize;
if (log_alloc_first == NULL) {
log_alloc_first = new_node;
log_alloc_last = new_node;
} else {
log_alloc_last->next = new_node;
log_alloc_last = new_node;
}
label_done:
malloc_mutex_unlock(tsd_tsdn(tsd), &log_mtx);
}
void
prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) {
prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize,
prof_tctx_t *tctx) {
malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
assert(tctx->cnts.curobjs > 0);
assert(tctx->cnts.curbytes >= usize);
tctx->cnts.curobjs--;
tctx->cnts.curbytes -= usize;
prof_try_log(tsd, ptr, usize, tctx);
if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) {
prof_tctx_destroy(tsd, tctx);
} else {
@@ -871,15 +1139,12 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) {
void
prof_sample_threshold_update(prof_tdata_t *tdata) {
#ifdef JEMALLOC_PROF
uint64_t r;
double u;
if (!config_prof) {
return;
}
if (lg_prof_sample == 0) {
tdata->bytes_until_sample = 0;
tsd_bytes_until_sample_set(tsd_fetch(), 0);
return;
}
@@ -901,11 +1166,16 @@ prof_sample_threshold_update(prof_tdata_t *tdata) {
* pp 500
* (http://luc.devroye.org/rnbookindex.html)
*/
r = prng_lg_range_u64(&tdata->prng_state, 53);
u = (double)r * (1.0/9007199254740992.0L);
tdata->bytes_until_sample = (uint64_t)(log(u) /
uint64_t r = prng_lg_range_u64(&tdata->prng_state, 53);
double u = (double)r * (1.0/9007199254740992.0L);
uint64_t bytes_until_sample = (uint64_t)(log(u) /
log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
+ (uint64_t)1U;
if (bytes_until_sample > SSIZE_MAX) {
bytes_until_sample = SSIZE_MAX;
}
tsd_bytes_until_sample_set(tsd_fetch(), bytes_until_sample);
#endif
}
@@ -978,7 +1248,7 @@ prof_dump_flush(bool propagate_err) {
cassert(config_prof);
err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
err = malloc_write_fd(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
if (err == -1) {
if (!propagate_err) {
malloc_write("<jemalloc>: write() failed during heap "
@@ -1022,7 +1292,7 @@ prof_dump_write(bool propagate_err, const char *s) {
}
}
if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) {
if (prof_dump_buf_end + slen - i <= PROF_DUMP_BUFSIZE) {
/* Finish writing. */
n = slen - i;
} else {
@@ -1033,6 +1303,7 @@ prof_dump_write(bool propagate_err, const char *s) {
prof_dump_buf_end += n;
i += n;
}
assert(i == slen);
return false;
}
@@ -1409,7 +1680,15 @@ prof_open_maps(const char *format, ...) {
va_start(ap, format);
malloc_vsnprintf(filename, sizeof(filename), format, ap);
va_end(ap);
#if defined(O_CLOEXEC)
mfd = open(filename, O_RDONLY | O_CLOEXEC);
#else
mfd = open(filename, O_RDONLY);
if (mfd != -1) {
fcntl(mfd, F_SETFD, fcntl(mfd, F_GETFD) | FD_CLOEXEC);
}
#endif
return mfd;
}
@@ -1463,8 +1742,9 @@ prof_dump_maps(bool propagate_err) {
goto label_return;
}
}
nread = read(mfd, &prof_dump_buf[prof_dump_buf_end],
PROF_DUMP_BUFSIZE - prof_dump_buf_end);
nread = malloc_read_fd(mfd,
&prof_dump_buf[prof_dump_buf_end], PROF_DUMP_BUFSIZE
- prof_dump_buf_end);
} while (nread > 0);
} else {
ret = true;
@@ -1772,7 +2052,7 @@ prof_idump(tsdn_t *tsdn) {
cassert(config_prof);
if (!prof_booted || tsdn_null(tsdn)) {
if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) {
return;
}
tsd = tsdn_tsd(tsdn);
@@ -1829,7 +2109,7 @@ prof_gdump(tsdn_t *tsdn) {
cassert(config_prof);
if (!prof_booted || tsdn_null(tsdn)) {
if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) {
return;
}
tsd = tsdn_tsd(tsdn);
@@ -1878,6 +2158,33 @@ prof_bt_keycomp(const void *k1, const void *k2) {
return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
}
static void
prof_bt_node_hash(const void *key, size_t r_hash[2]) {
const prof_bt_node_t *bt_node = (prof_bt_node_t *)key;
prof_bt_hash((void *)(&bt_node->bt), r_hash);
}
static bool
prof_bt_node_keycomp(const void *k1, const void *k2) {
const prof_bt_node_t *bt_node1 = (prof_bt_node_t *)k1;
const prof_bt_node_t *bt_node2 = (prof_bt_node_t *)k2;
return prof_bt_keycomp((void *)(&bt_node1->bt),
(void *)(&bt_node2->bt));
}
static void
prof_thr_node_hash(const void *key, size_t r_hash[2]) {
const prof_thr_node_t *thr_node = (prof_thr_node_t *)key;
hash(&thr_node->thr_uid, sizeof(uint64_t), 0x94122f35U, r_hash);
}
static bool
prof_thr_node_keycomp(const void *k1, const void *k2) {
const prof_thr_node_t *thr_node1 = (prof_thr_node_t *)k1;
const prof_thr_node_t *thr_node2 = (prof_thr_node_t *)k2;
return thr_node1->thr_uid == thr_node2->thr_uid;
}
static uint64_t
prof_thr_uid_alloc(tsdn_t *tsdn) {
uint64_t thr_uid;
@@ -2110,6 +2417,368 @@ prof_active_set(tsdn_t *tsdn, bool active) {
return prof_active_old;
}
#ifdef JEMALLOC_JET
size_t
prof_log_bt_count(void) {
size_t cnt = 0;
prof_bt_node_t *node = log_bt_first;
while (node != NULL) {
cnt++;
node = node->next;
}
return cnt;
}
size_t
prof_log_alloc_count(void) {
size_t cnt = 0;
prof_alloc_node_t *node = log_alloc_first;
while (node != NULL) {
cnt++;
node = node->next;
}
return cnt;
}
size_t
prof_log_thr_count(void) {
size_t cnt = 0;
prof_thr_node_t *node = log_thr_first;
while (node != NULL) {
cnt++;
node = node->next;
}
return cnt;
}
bool
prof_log_is_logging(void) {
return prof_logging_state == prof_logging_state_started;
}
bool
prof_log_rep_check(void) {
if (prof_logging_state == prof_logging_state_stopped
&& log_tables_initialized) {
return true;
}
if (log_bt_last != NULL && log_bt_last->next != NULL) {
return true;
}
if (log_thr_last != NULL && log_thr_last->next != NULL) {
return true;
}
if (log_alloc_last != NULL && log_alloc_last->next != NULL) {
return true;
}
size_t bt_count = prof_log_bt_count();
size_t thr_count = prof_log_thr_count();
size_t alloc_count = prof_log_alloc_count();
if (prof_logging_state == prof_logging_state_stopped) {
if (bt_count != 0 || thr_count != 0 || alloc_count || 0) {
return true;
}
}
prof_alloc_node_t *node = log_alloc_first;
while (node != NULL) {
if (node->alloc_bt_ind >= bt_count) {
return true;
}
if (node->free_bt_ind >= bt_count) {
return true;
}
if (node->alloc_thr_ind >= thr_count) {
return true;
}
if (node->free_thr_ind >= thr_count) {
return true;
}
if (node->alloc_time_ns > node->free_time_ns) {
return true;
}
node = node->next;
}
return false;
}
void
prof_log_dummy_set(bool new_value) {
prof_log_dummy = new_value;
}
#endif
bool
prof_log_start(tsdn_t *tsdn, const char *filename) {
if (!opt_prof || !prof_booted) {
return true;
}
bool ret = false;
size_t buf_size = PATH_MAX + 1;
malloc_mutex_lock(tsdn, &log_mtx);
if (prof_logging_state != prof_logging_state_stopped) {
ret = true;
} else if (filename == NULL) {
/* Make default name. */
malloc_snprintf(log_filename, buf_size, "%s.%d.%"FMTu64".json",
opt_prof_prefix, prof_getpid(), log_seq);
log_seq++;
prof_logging_state = prof_logging_state_started;
} else if (strlen(filename) >= buf_size) {
ret = true;
} else {
strcpy(log_filename, filename);
prof_logging_state = prof_logging_state_started;
}
if (!ret) {
nstime_update(&log_start_timestamp);
}
malloc_mutex_unlock(tsdn, &log_mtx);
return ret;
}
/* Used as an atexit function to stop logging on exit. */
static void
prof_log_stop_final(void) {
tsd_t *tsd = tsd_fetch();
prof_log_stop(tsd_tsdn(tsd));
}
struct prof_emitter_cb_arg_s {
int fd;
ssize_t ret;
};
static void
prof_emitter_write_cb(void *opaque, const char *to_write) {
struct prof_emitter_cb_arg_s *arg =
(struct prof_emitter_cb_arg_s *)opaque;
size_t bytes = strlen(to_write);
#ifdef JEMALLOC_JET
if (prof_log_dummy) {
return;
}
#endif
arg->ret = write(arg->fd, (void *)to_write, bytes);
}
/*
* prof_log_emit_{...} goes through the appropriate linked list, emitting each
* node to the json and deallocating it.
*/
static void
prof_log_emit_threads(tsd_t *tsd, emitter_t *emitter) {
emitter_json_array_kv_begin(emitter, "threads");
prof_thr_node_t *thr_node = log_thr_first;
prof_thr_node_t *thr_old_node;
while (thr_node != NULL) {
emitter_json_object_begin(emitter);
emitter_json_kv(emitter, "thr_uid", emitter_type_uint64,
&thr_node->thr_uid);
char *thr_name = thr_node->name;
emitter_json_kv(emitter, "thr_name", emitter_type_string,
&thr_name);
emitter_json_object_end(emitter);
thr_old_node = thr_node;
thr_node = thr_node->next;
idalloc(tsd, thr_old_node);
}
emitter_json_array_end(emitter);
}
static void
prof_log_emit_traces(tsd_t *tsd, emitter_t *emitter) {
emitter_json_array_kv_begin(emitter, "stack_traces");
prof_bt_node_t *bt_node = log_bt_first;
prof_bt_node_t *bt_old_node;
/*
* Calculate how many hex digits we need: twice number of bytes, two for
* "0x", and then one more for terminating '\0'.
*/
char buf[2 * sizeof(intptr_t) + 3];
size_t buf_sz = sizeof(buf);
while (bt_node != NULL) {
emitter_json_array_begin(emitter);
size_t i;
for (i = 0; i < bt_node->bt.len; i++) {
malloc_snprintf(buf, buf_sz, "%p", bt_node->bt.vec[i]);
char *trace_str = buf;
emitter_json_value(emitter, emitter_type_string,
&trace_str);
}
emitter_json_array_end(emitter);
bt_old_node = bt_node;
bt_node = bt_node->next;
idalloc(tsd, bt_old_node);
}
emitter_json_array_end(emitter);
}
static void
prof_log_emit_allocs(tsd_t *tsd, emitter_t *emitter) {
emitter_json_array_kv_begin(emitter, "allocations");
prof_alloc_node_t *alloc_node = log_alloc_first;
prof_alloc_node_t *alloc_old_node;
while (alloc_node != NULL) {
emitter_json_object_begin(emitter);
emitter_json_kv(emitter, "alloc_thread", emitter_type_size,
&alloc_node->alloc_thr_ind);
emitter_json_kv(emitter, "free_thread", emitter_type_size,
&alloc_node->free_thr_ind);
emitter_json_kv(emitter, "alloc_trace", emitter_type_size,
&alloc_node->alloc_bt_ind);
emitter_json_kv(emitter, "free_trace", emitter_type_size,
&alloc_node->free_bt_ind);
emitter_json_kv(emitter, "alloc_timestamp",
emitter_type_uint64, &alloc_node->alloc_time_ns);
emitter_json_kv(emitter, "free_timestamp", emitter_type_uint64,
&alloc_node->free_time_ns);
emitter_json_kv(emitter, "usize", emitter_type_uint64,
&alloc_node->usize);
emitter_json_object_end(emitter);
alloc_old_node = alloc_node;
alloc_node = alloc_node->next;
idalloc(tsd, alloc_old_node);
}
emitter_json_array_end(emitter);
}
static void
prof_log_emit_metadata(emitter_t *emitter) {
emitter_json_object_kv_begin(emitter, "info");
nstime_t now = NSTIME_ZERO_INITIALIZER;
nstime_update(&now);
uint64_t ns = nstime_ns(&now) - nstime_ns(&log_start_timestamp);
emitter_json_kv(emitter, "duration", emitter_type_uint64, &ns);
char *vers = JEMALLOC_VERSION;
emitter_json_kv(emitter, "version",
emitter_type_string, &vers);
emitter_json_kv(emitter, "lg_sample_rate",
emitter_type_int, &lg_prof_sample);
int pid = prof_getpid();
emitter_json_kv(emitter, "pid", emitter_type_int, &pid);
emitter_json_object_end(emitter);
}
bool
prof_log_stop(tsdn_t *tsdn) {
if (!opt_prof || !prof_booted) {
return true;
}
tsd_t *tsd = tsdn_tsd(tsdn);
malloc_mutex_lock(tsdn, &log_mtx);
if (prof_logging_state != prof_logging_state_started) {
malloc_mutex_unlock(tsdn, &log_mtx);
return true;
}
/*
* Set the state to dumping. We'll set it to stopped when we're done.
* Since other threads won't be able to start/stop/log when the state is
* dumping, we don't have to hold the lock during the whole method.
*/
prof_logging_state = prof_logging_state_dumping;
malloc_mutex_unlock(tsdn, &log_mtx);
emitter_t emitter;
/* Create a file. */
int fd;
#ifdef JEMALLOC_JET
if (prof_log_dummy) {
fd = 0;
} else {
fd = creat(log_filename, 0644);
}
#else
fd = creat(log_filename, 0644);
#endif
if (fd == -1) {
malloc_printf("<jemalloc>: creat() for log file \"%s\" "
" failed with %d\n", log_filename, errno);
if (opt_abort) {
abort();
}
return true;
}
/* Emit to json. */
struct prof_emitter_cb_arg_s arg;
arg.fd = fd;
emitter_init(&emitter, emitter_output_json, &prof_emitter_write_cb,
(void *)(&arg));
emitter_begin(&emitter);
prof_log_emit_metadata(&emitter);
prof_log_emit_threads(tsd, &emitter);
prof_log_emit_traces(tsd, &emitter);
prof_log_emit_allocs(tsd, &emitter);
emitter_end(&emitter);
/* Reset global state. */
if (log_tables_initialized) {
ckh_delete(tsd, &log_bt_node_set);
ckh_delete(tsd, &log_thr_node_set);
}
log_tables_initialized = false;
log_bt_index = 0;
log_thr_index = 0;
log_bt_first = NULL;
log_bt_last = NULL;
log_thr_first = NULL;
log_thr_last = NULL;
log_alloc_first = NULL;
log_alloc_last = NULL;
malloc_mutex_lock(tsdn, &log_mtx);
prof_logging_state = prof_logging_state_stopped;
malloc_mutex_unlock(tsdn, &log_mtx);
#ifdef JEMALLOC_JET
if (prof_log_dummy) {
return false;
}
#endif
return close(fd);
}
const char *
prof_thread_name_get(tsd_t *tsd) {
prof_tdata_t *tdata;
@@ -2346,6 +3015,35 @@ prof_boot2(tsd_t *tsd) {
}
}
if (opt_prof_log) {
prof_log_start(tsd_tsdn(tsd), NULL);
}
if (atexit(prof_log_stop_final) != 0) {
malloc_write("<jemalloc>: Error in atexit() "
"for logging\n");
if (opt_abort) {
abort();
}
}
if (malloc_mutex_init(&log_mtx, "prof_log",
WITNESS_RANK_PROF_LOG, malloc_mutex_rank_exclusive)) {
return true;
}
if (ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS,
prof_bt_node_hash, prof_bt_node_keycomp)) {
return true;
}
if (ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS,
prof_thr_node_hash, prof_thr_node_keycomp)) {
return true;
}
log_tables_initialized = true;
gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd),
b0get(), PROF_NCTX_LOCKS * sizeof(malloc_mutex_t),
CACHELINE);
@@ -2373,16 +3071,14 @@ prof_boot2(tsd_t *tsd) {
return true;
}
}
}
#ifdef JEMALLOC_PROF_LIBGCC
/*
* Cause the backtracing machinery to allocate its internal state
* before enabling profiling.
*/
_Unwind_Backtrace(prof_unwind_init_callback, NULL);
/*
* Cause the backtracing machinery to allocate its internal
* state before enabling profiling.
*/
_Unwind_Backtrace(prof_unwind_init_callback, NULL);
#endif
}
prof_booted = true;
return false;

4
deps/jemalloc/src/rtree.c vendored
View File

@@ -39,7 +39,7 @@ rtree_node_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *node) {
/* Nodes are never deleted during normal operation. */
not_reached();
}
UNUSED rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc =
rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc =
rtree_node_dalloc_impl;
static rtree_leaf_elm_t *
@@ -54,7 +54,7 @@ rtree_leaf_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *leaf) {
/* Leaves are never deleted during normal operation. */
not_reached();
}
UNUSED rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc =
rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc =
rtree_leaf_dalloc_impl;
#ifdef JEMALLOC_JET

24
deps/jemalloc/src/safety_check.c vendored Normal file
View File

@@ -0,0 +1,24 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
static void (*safety_check_abort)(const char *message);
void safety_check_set_abort(void (*abort_fn)(const char *)) {
safety_check_abort = abort_fn;
}
void safety_check_fail(const char *format, ...) {
char buf[MALLOC_PRINTF_BUFSIZE];
va_list ap;
va_start(ap, format);
malloc_vsnprintf(buf, MALLOC_PRINTF_BUFSIZE, format, ap);
va_end(ap);
if (safety_check_abort == NULL) {
malloc_write(buf);
abort();
} else {
safety_check_abort(buf);
}
}

313
deps/jemalloc/src/sc.c vendored Normal file
View File

@@ -0,0 +1,313 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/pages.h"
#include "jemalloc/internal/sc.h"
/*
* This module computes the size classes used to satisfy allocations. The logic
* here was ported more or less line-by-line from a shell script, and because of
* that is not the most idiomatic C. Eventually we should fix this, but for now
* at least the damage is compartmentalized to this file.
*/
sc_data_t sc_data_global;
static size_t
reg_size_compute(int lg_base, int lg_delta, int ndelta) {
return (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta);
}
/* Returns the number of pages in the slab. */
static int
slab_size(int lg_page, int lg_base, int lg_delta, int ndelta) {
size_t page = (ZU(1) << lg_page);
size_t reg_size = reg_size_compute(lg_base, lg_delta, ndelta);
size_t try_slab_size = page;
size_t try_nregs = try_slab_size / reg_size;
size_t perfect_slab_size = 0;
bool perfect = false;
/*
* This loop continues until we find the least common multiple of the
* page size and size class size. Size classes are all of the form
* base + ndelta * delta == (ndelta + base/ndelta) * delta, which is
* (ndelta + ngroup) * delta. The way we choose slabbing strategies
* means that delta is at most the page size and ndelta < ngroup. So
* the loop executes for at most 2 * ngroup - 1 iterations, which is
* also the bound on the number of pages in a slab chosen by default.
* With the current default settings, this is at most 7.
*/
while (!perfect) {
perfect_slab_size = try_slab_size;
size_t perfect_nregs = try_nregs;
try_slab_size += page;
try_nregs = try_slab_size / reg_size;
if (perfect_slab_size == perfect_nregs * reg_size) {
perfect = true;
}
}
return (int)(perfect_slab_size / page);
}
static void
size_class(
/* Output. */
sc_t *sc,
/* Configuration decisions. */
int lg_max_lookup, int lg_page, int lg_ngroup,
/* Inputs specific to the size class. */
int index, int lg_base, int lg_delta, int ndelta) {
sc->index = index;
sc->lg_base = lg_base;
sc->lg_delta = lg_delta;
sc->ndelta = ndelta;
sc->psz = (reg_size_compute(lg_base, lg_delta, ndelta)
% (ZU(1) << lg_page) == 0);
size_t size = (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta);
if (index == 0) {
assert(!sc->psz);
}
if (size < (ZU(1) << (lg_page + lg_ngroup))) {
sc->bin = true;
sc->pgs = slab_size(lg_page, lg_base, lg_delta, ndelta);
} else {
sc->bin = false;
sc->pgs = 0;
}
if (size <= (ZU(1) << lg_max_lookup)) {
sc->lg_delta_lookup = lg_delta;
} else {
sc->lg_delta_lookup = 0;
}
}
static void
size_classes(
/* Output. */
sc_data_t *sc_data,
/* Determined by the system. */
size_t lg_ptr_size, int lg_quantum,
/* Configuration decisions. */
int lg_tiny_min, int lg_max_lookup, int lg_page, int lg_ngroup) {
int ptr_bits = (1 << lg_ptr_size) * 8;
int ngroup = (1 << lg_ngroup);
int ntiny = 0;
int nlbins = 0;
int lg_tiny_maxclass = (unsigned)-1;
int nbins = 0;
int npsizes = 0;
int index = 0;
int ndelta = 0;
int lg_base = lg_tiny_min;
int lg_delta = lg_base;
/* Outputs that we update as we go. */
size_t lookup_maxclass = 0;
size_t small_maxclass = 0;
int lg_large_minclass = 0;
size_t large_maxclass = 0;
/* Tiny size classes. */
while (lg_base < lg_quantum) {
sc_t *sc = &sc_data->sc[index];
size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index,
lg_base, lg_delta, ndelta);
if (sc->lg_delta_lookup != 0) {
nlbins = index + 1;
}
if (sc->psz) {
npsizes++;
}
if (sc->bin) {
nbins++;
}
ntiny++;
/* Final written value is correct. */
lg_tiny_maxclass = lg_base;
index++;
lg_delta = lg_base;
lg_base++;
}
/* First non-tiny (pseudo) group. */
if (ntiny != 0) {
sc_t *sc = &sc_data->sc[index];
/*
* See the note in sc.h; the first non-tiny size class has an
* unusual encoding.
*/
lg_base--;
ndelta = 1;
size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index,
lg_base, lg_delta, ndelta);
index++;
lg_base++;
lg_delta++;
if (sc->psz) {
npsizes++;
}
if (sc->bin) {
nbins++;
}
}
while (ndelta < ngroup) {
sc_t *sc = &sc_data->sc[index];
size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index,
lg_base, lg_delta, ndelta);
index++;
ndelta++;
if (sc->psz) {
npsizes++;
}
if (sc->bin) {
nbins++;
}
}
/* All remaining groups. */
lg_base = lg_base + lg_ngroup;
while (lg_base < ptr_bits - 1) {
ndelta = 1;
int ndelta_limit;
if (lg_base == ptr_bits - 2) {
ndelta_limit = ngroup - 1;
} else {
ndelta_limit = ngroup;
}
while (ndelta <= ndelta_limit) {
sc_t *sc = &sc_data->sc[index];
size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index,
lg_base, lg_delta, ndelta);
if (sc->lg_delta_lookup != 0) {
nlbins = index + 1;
/* Final written value is correct. */
lookup_maxclass = (ZU(1) << lg_base)
+ (ZU(ndelta) << lg_delta);
}
if (sc->psz) {
npsizes++;
}
if (sc->bin) {
nbins++;
/* Final written value is correct. */
small_maxclass = (ZU(1) << lg_base)
+ (ZU(ndelta) << lg_delta);
if (lg_ngroup > 0) {
lg_large_minclass = lg_base + 1;
} else {
lg_large_minclass = lg_base + 2;
}
}
large_maxclass = (ZU(1) << lg_base)
+ (ZU(ndelta) << lg_delta);
index++;
ndelta++;
}
lg_base++;
lg_delta++;
}
/* Additional outputs. */
int nsizes = index;
unsigned lg_ceil_nsizes = lg_ceil(nsizes);
/* Fill in the output data. */
sc_data->ntiny = ntiny;
sc_data->nlbins = nlbins;
sc_data->nbins = nbins;
sc_data->nsizes = nsizes;
sc_data->lg_ceil_nsizes = lg_ceil_nsizes;
sc_data->npsizes = npsizes;
sc_data->lg_tiny_maxclass = lg_tiny_maxclass;
sc_data->lookup_maxclass = lookup_maxclass;
sc_data->small_maxclass = small_maxclass;
sc_data->lg_large_minclass = lg_large_minclass;
sc_data->large_minclass = (ZU(1) << lg_large_minclass);
sc_data->large_maxclass = large_maxclass;
/*
* We compute these values in two ways:
* - Incrementally, as above.
* - In macros, in sc.h.
* The computation is easier when done incrementally, but putting it in
* a constant makes it available to the fast paths without having to
* touch the extra global cacheline. We assert, however, that the two
* computations are equivalent.
*/
assert(sc_data->npsizes == SC_NPSIZES);
assert(sc_data->lg_tiny_maxclass == SC_LG_TINY_MAXCLASS);
assert(sc_data->small_maxclass == SC_SMALL_MAXCLASS);
assert(sc_data->large_minclass == SC_LARGE_MINCLASS);
assert(sc_data->lg_large_minclass == SC_LG_LARGE_MINCLASS);
assert(sc_data->large_maxclass == SC_LARGE_MAXCLASS);
/*
* In the allocation fastpath, we want to assume that we can
* unconditionally subtract the requested allocation size from
* a ssize_t, and detect passing through 0 correctly. This
* results in optimal generated code. For this to work, the
* maximum allocation size must be less than SSIZE_MAX.
*/
assert(SC_LARGE_MAXCLASS < SSIZE_MAX);
}
void
sc_data_init(sc_data_t *sc_data) {
assert(!sc_data->initialized);
int lg_max_lookup = 12;
size_classes(sc_data, LG_SIZEOF_PTR, LG_QUANTUM, SC_LG_TINY_MIN,
lg_max_lookup, LG_PAGE, 2);
sc_data->initialized = true;
}
static void
sc_data_update_sc_slab_size(sc_t *sc, size_t reg_size, size_t pgs_guess) {
size_t min_pgs = reg_size / PAGE;
if (reg_size % PAGE != 0) {
min_pgs++;
}
/*
* BITMAP_MAXBITS is actually determined by putting the smallest
* possible size-class on one page, so this can never be 0.
*/
size_t max_pgs = BITMAP_MAXBITS * reg_size / PAGE;
assert(min_pgs <= max_pgs);
assert(min_pgs > 0);
assert(max_pgs >= 1);
if (pgs_guess < min_pgs) {
sc->pgs = (int)min_pgs;
} else if (pgs_guess > max_pgs) {
sc->pgs = (int)max_pgs;
} else {
sc->pgs = (int)pgs_guess;
}
}
void
sc_data_update_slab_size(sc_data_t *data, size_t begin, size_t end, int pgs) {
assert(data->initialized);
for (int i = 0; i < data->nsizes; i++) {
sc_t *sc = &data->sc[i];
if (!sc->bin) {
break;
}
size_t reg_size = reg_size_compute(sc->lg_base, sc->lg_delta,
sc->ndelta);
if (begin <= reg_size && reg_size <= end) {
sc_data_update_sc_slab_size(sc, reg_size, pgs);
}
}
}
void
sc_boot(sc_data_t *data) {
sc_data_init(data);
}

1968
deps/jemalloc/src/stats.c vendored
View File

File diff suppressed because it is too large Load Diff

154
deps/jemalloc/src/sz.c vendored
View File

@@ -2,105 +2,63 @@
#include "jemalloc/internal/sz.h"
JEMALLOC_ALIGNED(CACHELINE)
const size_t sz_pind2sz_tab[NPSIZES+1] = {
#define PSZ_yes(lg_grp, ndelta, lg_delta) \
(((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))),
#define PSZ_no(lg_grp, ndelta, lg_delta)
#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \
PSZ_##psz(lg_grp, ndelta, lg_delta)
SIZE_CLASSES
#undef PSZ_yes
#undef PSZ_no
#undef SC
(LARGE_MAXCLASS + PAGE)
};
size_t sz_pind2sz_tab[SC_NPSIZES+1];
static void
sz_boot_pind2sz_tab(const sc_data_t *sc_data) {
int pind = 0;
for (unsigned i = 0; i < SC_NSIZES; i++) {
const sc_t *sc = &sc_data->sc[i];
if (sc->psz) {
sz_pind2sz_tab[pind] = (ZU(1) << sc->lg_base)
+ (ZU(sc->ndelta) << sc->lg_delta);
pind++;
}
}
for (int i = pind; i <= (int)SC_NPSIZES; i++) {
sz_pind2sz_tab[pind] = sc_data->large_maxclass + PAGE;
}
}
JEMALLOC_ALIGNED(CACHELINE)
const size_t sz_index2size_tab[NSIZES] = {
#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \
((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)),
SIZE_CLASSES
#undef SC
};
size_t sz_index2size_tab[SC_NSIZES];
static void
sz_boot_index2size_tab(const sc_data_t *sc_data) {
for (unsigned i = 0; i < SC_NSIZES; i++) {
const sc_t *sc = &sc_data->sc[i];
sz_index2size_tab[i] = (ZU(1) << sc->lg_base)
+ (ZU(sc->ndelta) << (sc->lg_delta));
}
}
/*
* To keep this table small, we divide sizes by the tiny min size, which gives
* the smallest interval for which the result can change.
*/
JEMALLOC_ALIGNED(CACHELINE)
const uint8_t sz_size2index_tab[] = {
#if LG_TINY_MIN == 0
#warning "Dangerous LG_TINY_MIN"
#define S2B_0(i) i,
#elif LG_TINY_MIN == 1
#warning "Dangerous LG_TINY_MIN"
#define S2B_1(i) i,
#elif LG_TINY_MIN == 2
#warning "Dangerous LG_TINY_MIN"
#define S2B_2(i) i,
#elif LG_TINY_MIN == 3
#define S2B_3(i) i,
#elif LG_TINY_MIN == 4
#define S2B_4(i) i,
#elif LG_TINY_MIN == 5
#define S2B_5(i) i,
#elif LG_TINY_MIN == 6
#define S2B_6(i) i,
#elif LG_TINY_MIN == 7
#define S2B_7(i) i,
#elif LG_TINY_MIN == 8
#define S2B_8(i) i,
#elif LG_TINY_MIN == 9
#define S2B_9(i) i,
#elif LG_TINY_MIN == 10
#define S2B_10(i) i,
#elif LG_TINY_MIN == 11
#define S2B_11(i) i,
#else
#error "Unsupported LG_TINY_MIN"
#endif
#if LG_TINY_MIN < 1
#define S2B_1(i) S2B_0(i) S2B_0(i)
#endif
#if LG_TINY_MIN < 2
#define S2B_2(i) S2B_1(i) S2B_1(i)
#endif
#if LG_TINY_MIN < 3
#define S2B_3(i) S2B_2(i) S2B_2(i)
#endif
#if LG_TINY_MIN < 4
#define S2B_4(i) S2B_3(i) S2B_3(i)
#endif
#if LG_TINY_MIN < 5
#define S2B_5(i) S2B_4(i) S2B_4(i)
#endif
#if LG_TINY_MIN < 6
#define S2B_6(i) S2B_5(i) S2B_5(i)
#endif
#if LG_TINY_MIN < 7
#define S2B_7(i) S2B_6(i) S2B_6(i)
#endif
#if LG_TINY_MIN < 8
#define S2B_8(i) S2B_7(i) S2B_7(i)
#endif
#if LG_TINY_MIN < 9
#define S2B_9(i) S2B_8(i) S2B_8(i)
#endif
#if LG_TINY_MIN < 10
#define S2B_10(i) S2B_9(i) S2B_9(i)
#endif
#if LG_TINY_MIN < 11
#define S2B_11(i) S2B_10(i) S2B_10(i)
#endif
#define S2B_no(i)
#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \
S2B_##lg_delta_lookup(index)
SIZE_CLASSES
#undef S2B_3
#undef S2B_4
#undef S2B_5
#undef S2B_6
#undef S2B_7
#undef S2B_8
#undef S2B_9
#undef S2B_10
#undef S2B_11
#undef S2B_no
#undef SC
};
uint8_t sz_size2index_tab[(SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1];
static void
sz_boot_size2index_tab(const sc_data_t *sc_data) {
size_t dst_max = (SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1;
size_t dst_ind = 0;
for (unsigned sc_ind = 0; sc_ind < SC_NSIZES && dst_ind < dst_max;
sc_ind++) {
const sc_t *sc = &sc_data->sc[sc_ind];
size_t sz = (ZU(1) << sc->lg_base)
+ (ZU(sc->ndelta) << sc->lg_delta);
size_t max_ind = ((sz + (ZU(1) << SC_LG_TINY_MIN) - 1)
>> SC_LG_TINY_MIN);
for (; dst_ind <= max_ind && dst_ind < dst_max; dst_ind++) {
sz_size2index_tab[dst_ind] = sc_ind;
}
}
}
void
sz_boot(const sc_data_t *sc_data) {
sz_boot_pind2sz_tab(sc_data);
sz_boot_index2size_tab(sc_data);
sz_boot_size2index_tab(sc_data);
}

228
deps/jemalloc/src/tcache.c vendored
View File

@@ -4,7 +4,8 @@
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/safety_check.h"
#include "jemalloc/internal/sc.h"
/******************************************************************************/
/* Data. */
@@ -12,7 +13,7 @@
bool opt_tcache = true;
ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
tcache_bin_info_t *tcache_bin_info;
cache_bin_info_t *tcache_bin_info;
static unsigned stack_nelms; /* Total stack elms per tcache. */
unsigned nhbins;
@@ -40,8 +41,8 @@ void
tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
szind_t binind = tcache->next_gc_bin;
tcache_bin_t *tbin;
if (binind < NBINS) {
cache_bin_t *tbin;
if (binind < SC_NBINS) {
tbin = tcache_small_bin_get(tcache, binind);
} else {
tbin = tcache_large_bin_get(tcache, binind);
@@ -50,7 +51,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
/*
* Flush (ceiling) 3/4 of the objects below the low water mark.
*/
if (binind < NBINS) {
if (binind < SC_NBINS) {
tcache_bin_flush_small(tsd, tcache, tbin, binind,
tbin->ncached - tbin->low_water + (tbin->low_water
>> 2));
@@ -58,7 +59,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
* Reduce fill count by 2X. Limit lg_fill_div such that
* the fill count is always at least 1.
*/
tcache_bin_info_t *tbin_info = &tcache_bin_info[binind];
cache_bin_info_t *tbin_info = &tcache_bin_info[binind];
if ((tbin_info->ncached_max >>
(tcache->lg_fill_div[binind] + 1)) >= 1) {
tcache->lg_fill_div[binind]++;
@@ -72,7 +73,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
* Increase fill count by 2X for small bins. Make sure
* lg_fill_div stays greater than 0.
*/
if (binind < NBINS && tcache->lg_fill_div[binind] > 1) {
if (binind < SC_NBINS && tcache->lg_fill_div[binind] > 1) {
tcache->lg_fill_div[binind]--;
}
}
@@ -86,7 +87,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
void *
tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
tcache_bin_t *tbin, szind_t binind, bool *tcache_success) {
cache_bin_t *tbin, szind_t binind, bool *tcache_success) {
void *ret;
assert(tcache->arena != NULL);
@@ -95,33 +96,72 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
if (config_prof) {
tcache->prof_accumbytes = 0;
}
ret = tcache_alloc_easy(tbin, tcache_success);
ret = cache_bin_alloc_easy(tbin, tcache_success);
return ret;
}
/* Enabled with --enable-extra-size-check. */
static void
tbin_extents_lookup_size_check(tsdn_t *tsdn, cache_bin_t *tbin, szind_t binind,
size_t nflush, extent_t **extents){
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
/*
* Verify that the items in the tcache all have the correct size; this
* is useful for catching sized deallocation bugs, also to fail early
* instead of corrupting metadata. Since this can be turned on for opt
* builds, avoid the branch in the loop.
*/
szind_t szind;
size_t sz_sum = binind * nflush;
for (unsigned i = 0 ; i < nflush; i++) {
rtree_extent_szind_read(tsdn, &extents_rtree,
rtree_ctx, (uintptr_t)*(tbin->avail - 1 - i), true,
&extents[i], &szind);
sz_sum -= szind;
}
if (sz_sum != 0) {
safety_check_fail("<jemalloc>: size mismatch in thread cache "
"detected, likely caused by sized deallocation bugs by "
"application. Abort.\n");
abort();
}
}
void
tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
szind_t binind, unsigned rem) {
bool merged_stats = false;
assert(binind < NBINS);
assert(rem <= tbin->ncached);
assert(binind < SC_NBINS);
assert((cache_bin_sz_t)rem <= tbin->ncached);
arena_t *arena = tcache->arena;
assert(arena != NULL);
unsigned nflush = tbin->ncached - rem;
VARIABLE_ARRAY(extent_t *, item_extent, nflush);
/* Look up extent once per item. */
for (unsigned i = 0 ; i < nflush; i++) {
item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i));
}
/* Look up extent once per item. */
if (config_opt_safety_checks) {
tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind,
nflush, item_extent);
} else {
for (unsigned i = 0 ; i < nflush; i++) {
item_extent[i] = iealloc(tsd_tsdn(tsd),
*(tbin->avail - 1 - i));
}
}
while (nflush > 0) {
/* Lock the arena bin associated with the first object. */
extent_t *extent = item_extent[0];
arena_t *bin_arena = extent_arena_get(extent);
arena_bin_t *bin = &bin_arena->bins[binind];
unsigned bin_arena_ind = extent_arena_ind_get(extent);
arena_t *bin_arena = arena_get(tsd_tsdn(tsd), bin_arena_ind,
false);
unsigned binshard = extent_binshard_get(extent);
assert(binshard < bin_infos[binind].n_shards);
bin_t *bin = &bin_arena->bins[binind].bin_shards[binshard];
if (config_prof && bin_arena == arena) {
if (arena_prof_accum(tsd_tsdn(tsd), arena,
@@ -132,8 +172,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
}
malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
if (config_stats && bin_arena == arena) {
assert(!merged_stats);
if (config_stats && bin_arena == arena && !merged_stats) {
merged_stats = true;
bin->stats.nflushes++;
bin->stats.nrequests += tbin->tstats.nrequests;
@@ -145,9 +184,10 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
extent = item_extent[i];
assert(ptr != NULL && extent != NULL);
if (extent_arena_get(extent) == bin_arena) {
if (extent_arena_ind_get(extent) == bin_arena_ind
&& extent_binshard_get(extent) == binshard) {
arena_dalloc_bin_junked_locked(tsd_tsdn(tsd),
bin_arena, extent, ptr);
bin_arena, bin, binind, extent, ptr);
} else {
/*
* This object was allocated via a different
@@ -169,8 +209,9 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
* The flush loop didn't happen to flush to this thread's
* arena, so the stats didn't get merged. Manually do so now.
*/
arena_bin_t *bin = &arena->bins[binind];
malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
unsigned binshard;
bin_t *bin = arena_bin_choose_lock(tsd_tsdn(tsd), arena, binind,
&binshard);
bin->stats.nflushes++;
bin->stats.nrequests += tbin->tstats.nrequests;
tbin->tstats.nrequests = 0;
@@ -180,63 +221,76 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem *
sizeof(void *));
tbin->ncached = rem;
if ((low_water_t)tbin->ncached < tbin->low_water) {
if (tbin->ncached < tbin->low_water) {
tbin->low_water = tbin->ncached;
}
}
void
tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind,
unsigned rem, tcache_t *tcache) {
bool merged_stats = false;
assert(binind < nhbins);
assert(rem <= tbin->ncached);
assert((cache_bin_sz_t)rem <= tbin->ncached);
arena_t *arena = tcache->arena;
assert(arena != NULL);
arena_t *tcache_arena = tcache->arena;
assert(tcache_arena != NULL);
unsigned nflush = tbin->ncached - rem;
VARIABLE_ARRAY(extent_t *, item_extent, nflush);
#ifndef JEMALLOC_EXTRA_SIZE_CHECK
/* Look up extent once per item. */
for (unsigned i = 0 ; i < nflush; i++) {
item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i));
}
#else
tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, nflush,
item_extent);
#endif
while (nflush > 0) {
/* Lock the arena associated with the first object. */
extent_t *extent = item_extent[0];
arena_t *locked_arena = extent_arena_get(extent);
UNUSED bool idump;
unsigned locked_arena_ind = extent_arena_ind_get(extent);
arena_t *locked_arena = arena_get(tsd_tsdn(tsd),
locked_arena_ind, false);
bool idump;
if (config_prof) {
idump = false;
}
malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx);
bool lock_large = !arena_is_auto(locked_arena);
if (lock_large) {
malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx);
}
for (unsigned i = 0; i < nflush; i++) {
void *ptr = *(tbin->avail - 1 - i);
assert(ptr != NULL);
extent = item_extent[i];
if (extent_arena_get(extent) == locked_arena) {
if (extent_arena_ind_get(extent) == locked_arena_ind) {
large_dalloc_prep_junked_locked(tsd_tsdn(tsd),
extent);
}
}
if ((config_prof || config_stats) && locked_arena == arena) {
if ((config_prof || config_stats) &&
(locked_arena == tcache_arena)) {
if (config_prof) {
idump = arena_prof_accum(tsd_tsdn(tsd), arena,
tcache->prof_accumbytes);
idump = arena_prof_accum(tsd_tsdn(tsd),
tcache_arena, tcache->prof_accumbytes);
tcache->prof_accumbytes = 0;
}
if (config_stats) {
merged_stats = true;
arena_stats_large_nrequests_add(tsd_tsdn(tsd),
&arena->stats, binind,
arena_stats_large_flush_nrequests_add(
tsd_tsdn(tsd), &tcache_arena->stats, binind,
tbin->tstats.nrequests);
tbin->tstats.nrequests = 0;
}
}
malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx);
if (lock_large) {
malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx);
}
unsigned ndeferred = 0;
for (unsigned i = 0; i < nflush; i++) {
@@ -244,7 +298,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
extent = item_extent[i];
assert(ptr != NULL && extent != NULL);
if (extent_arena_get(extent) == locked_arena) {
if (extent_arena_ind_get(extent) == locked_arena_ind) {
large_dalloc_finish(tsd_tsdn(tsd), extent);
} else {
/*
@@ -270,15 +324,15 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
* The flush loop didn't happen to flush to this thread's
* arena, so the stats didn't get merged. Manually do so now.
*/
arena_stats_large_nrequests_add(tsd_tsdn(tsd), &arena->stats,
binind, tbin->tstats.nrequests);
arena_stats_large_flush_nrequests_add(tsd_tsdn(tsd),
&tcache_arena->stats, binind, tbin->tstats.nrequests);
tbin->tstats.nrequests = 0;
}
memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem *
sizeof(void *));
tbin->ncached = rem;
if ((low_water_t)tbin->ncached < tbin->low_water) {
if (tbin->ncached < tbin->low_water) {
tbin->low_water = tbin->ncached;
}
}
@@ -291,8 +345,15 @@ tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
if (config_stats) {
/* Link into list of extant tcaches. */
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
ql_elm_new(tcache, link);
ql_tail_insert(&arena->tcache_ql, tcache, link);
cache_bin_array_descriptor_init(
&tcache->cache_bin_array_descriptor, tcache->bins_small,
tcache->bins_large);
ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
&tcache->cache_bin_array_descriptor, link);
malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
}
}
@@ -316,6 +377,8 @@ tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) {
assert(in_ql);
}
ql_remove(&arena->tcache_ql, tcache, link);
ql_remove(&arena->cache_bin_array_descriptor_ql,
&tcache->cache_bin_array_descriptor, link);
tcache_stats_merge(tsdn, tcache, arena);
malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
}
@@ -354,10 +417,10 @@ tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) {
size_t stack_offset = 0;
assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
memset(tcache->tbins_small, 0, sizeof(tcache_bin_t) * NBINS);
memset(tcache->tbins_large, 0, sizeof(tcache_bin_t) * (nhbins - NBINS));
memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS);
memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS));
unsigned i = 0;
for (; i < NBINS; i++) {
for (; i < SC_NBINS; i++) {
tcache->lg_fill_div[i] = 1;
stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
/*
@@ -449,16 +512,16 @@ static void
tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
assert(tcache->arena != NULL);
for (unsigned i = 0; i < NBINS; i++) {
tcache_bin_t *tbin = tcache_small_bin_get(tcache, i);
for (unsigned i = 0; i < SC_NBINS; i++) {
cache_bin_t *tbin = tcache_small_bin_get(tcache, i);
tcache_bin_flush_small(tsd, tcache, tbin, i, 0);
if (config_stats) {
assert(tbin->tstats.nrequests == 0);
}
}
for (unsigned i = NBINS; i < nhbins; i++) {
tcache_bin_t *tbin = tcache_large_bin_get(tcache, i);
for (unsigned i = SC_NBINS; i < nhbins; i++) {
cache_bin_t *tbin = tcache_large_bin_get(tcache, i);
tcache_bin_flush_large(tsd, tbin, i, 0, tcache);
if (config_stats) {
@@ -482,6 +545,7 @@ tcache_flush(tsd_t *tsd) {
static void
tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
tcache_flush_cache(tsd, tcache);
arena_t *arena = tcache->arena;
tcache_arena_dissociate(tsd_tsdn(tsd), tcache);
if (tsd_tcache) {
@@ -494,6 +558,23 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
/* Release both the tcache struct and avail array. */
idalloctm(tsd_tsdn(tsd), tcache, NULL, NULL, true, true);
}
/*
* The deallocation and tcache flush above may not trigger decay since
* we are on the tcache shutdown path (potentially with non-nominal
* tsd). Manually trigger decay to avoid pathological cases. Also
* include arena 0 because the tcache array is allocated from it.
*/
arena_decay(tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false),
false, false);
if (arena_nthreads_get(arena, false) == 0 &&
!background_thread_enabled()) {
/* Force purging when no threads assigned to the arena anymore. */
arena_decay(tsd_tsdn(tsd), arena, false, true);
} else {
arena_decay(tsd_tsdn(tsd), arena, false, false);
}
}
/* For auto tcache (embedded in TSD) only. */
@@ -523,18 +604,18 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
cassert(config_stats);
/* Merge and reset tcache stats. */
for (i = 0; i < NBINS; i++) {
arena_bin_t *bin = &arena->bins[i];
tcache_bin_t *tbin = tcache_small_bin_get(tcache, i);
malloc_mutex_lock(tsdn, &bin->lock);
for (i = 0; i < SC_NBINS; i++) {
cache_bin_t *tbin = tcache_small_bin_get(tcache, i);
unsigned binshard;
bin_t *bin = arena_bin_choose_lock(tsdn, arena, i, &binshard);
bin->stats.nrequests += tbin->tstats.nrequests;
malloc_mutex_unlock(tsdn, &bin->lock);
tbin->tstats.nrequests = 0;
}
for (; i < nhbins; i++) {
tcache_bin_t *tbin = tcache_large_bin_get(tcache, i);
arena_stats_large_nrequests_add(tsdn, &arena->stats, i,
cache_bin_t *tbin = tcache_large_bin_get(tcache, i);
arena_stats_large_flush_nrequests_add(tsdn, &arena->stats, i,
tbin->tstats.nrequests);
tbin->tstats.nrequests = 0;
}
@@ -605,23 +686,32 @@ label_return:
}
static tcache_t *
tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm) {
tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm, bool allow_reinit) {
malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx);
if (elm->tcache == NULL) {
return NULL;
}
tcache_t *tcache = elm->tcache;
elm->tcache = NULL;
if (allow_reinit) {
elm->tcache = TCACHES_ELM_NEED_REINIT;
} else {
elm->tcache = NULL;
}
if (tcache == TCACHES_ELM_NEED_REINIT) {
return NULL;
}
return tcache;
}
void
tcaches_flush(tsd_t *tsd, unsigned ind) {
malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind]);
tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind], true);
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
if (tcache != NULL) {
/* Destroy the tcache; recreate in tcaches_get() if needed. */
tcache_destroy(tsd, tcache, false);
}
}
@@ -630,7 +720,7 @@ void
tcaches_destroy(tsd_t *tsd, unsigned ind) {
malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
tcaches_t *elm = &tcaches[ind];
tcache_t *tcache = tcaches_elm_remove(tsd, elm);
tcache_t *tcache = tcaches_elm_remove(tsd, elm, false);
elm->next = tcaches_avail;
tcaches_avail = elm;
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
@@ -643,8 +733,8 @@ bool
tcache_boot(tsdn_t *tsdn) {
/* If necessary, clamp opt_lg_tcache_max. */
if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) <
SMALL_MAXCLASS) {
tcache_maxclass = SMALL_MAXCLASS;
SC_SMALL_MAXCLASS) {
tcache_maxclass = SC_SMALL_MAXCLASS;
} else {
tcache_maxclass = (ZU(1) << opt_lg_tcache_max);
}
@@ -657,21 +747,21 @@ tcache_boot(tsdn_t *tsdn) {
nhbins = sz_size2index(tcache_maxclass) + 1;
/* Initialize tcache_bin_info. */
tcache_bin_info = (tcache_bin_info_t *)base_alloc(tsdn, b0get(), nhbins
* sizeof(tcache_bin_info_t), CACHELINE);
tcache_bin_info = (cache_bin_info_t *)base_alloc(tsdn, b0get(), nhbins
* sizeof(cache_bin_info_t), CACHELINE);
if (tcache_bin_info == NULL) {
return true;
}
stack_nelms = 0;
unsigned i;
for (i = 0; i < NBINS; i++) {
if ((arena_bin_info[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) {
for (i = 0; i < SC_NBINS; i++) {
if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) {
tcache_bin_info[i].ncached_max =
TCACHE_NSLOTS_SMALL_MIN;
} else if ((arena_bin_info[i].nregs << 1) <=
} else if ((bin_infos[i].nregs << 1) <=
TCACHE_NSLOTS_SMALL_MAX) {
tcache_bin_info[i].ncached_max =
(arena_bin_info[i].nregs << 1);
(bin_infos[i].nregs << 1);
} else {
tcache_bin_info[i].ncached_max =
TCACHE_NSLOTS_SMALL_MAX;

12
deps/jemalloc/src/test_hooks.c vendored Normal file
View File

@@ -0,0 +1,12 @@
#include "jemalloc/internal/jemalloc_preamble.h"
/*
* The hooks are a little bit screwy -- they're not genuinely exported in the
* sense that we want them available to end-users, but we do want them visible
* from outside the generated library, so that we can use them in test code.
*/
JEMALLOC_EXPORT
void (*test_hooks_arena_new_hook)() = NULL;
JEMALLOC_EXPORT
void (*test_hooks_libc_hook)() = NULL;

253
deps/jemalloc/src/tsd.c vendored
View File

@@ -12,12 +12,16 @@
static unsigned ncleanups;
static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX];
/* TSD_INITIALIZER triggers "-Wmissing-field-initializer" */
JEMALLOC_DIAGNOSTIC_PUSH
JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
__thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER;
__thread bool JEMALLOC_TLS_MODEL tsd_initialized = false;
JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER;
JEMALLOC_TSD_TYPE_ATTR(bool) JEMALLOC_TLS_MODEL tsd_initialized = false;
bool tsd_booted = false;
#elif (defined(JEMALLOC_TLS))
__thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER;
JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER;
pthread_key_t tsd_tsd;
bool tsd_booted = false;
#elif (defined(_WIN32))
@@ -41,6 +45,7 @@ tsd_init_head_t tsd_init_head = {
ql_head_initializer(blocks),
MALLOC_MUTEX_INITIALIZER
};
tsd_wrapper_t tsd_boot_wrapper = {
false,
TSD_INITIALIZER
@@ -48,17 +53,164 @@ tsd_wrapper_t tsd_boot_wrapper = {
bool tsd_booted = false;
#endif
JEMALLOC_DIAGNOSTIC_POP
/******************************************************************************/
/* A list of all the tsds in the nominal state. */
typedef ql_head(tsd_t) tsd_list_t;
static tsd_list_t tsd_nominal_tsds = ql_head_initializer(tsd_nominal_tsds);
static malloc_mutex_t tsd_nominal_tsds_lock;
/* How many slow-path-enabling features are turned on. */
static atomic_u32_t tsd_global_slow_count = ATOMIC_INIT(0);
static bool
tsd_in_nominal_list(tsd_t *tsd) {
tsd_t *tsd_list;
bool found = false;
/*
* We don't know that tsd is nominal; it might not be safe to get data
* out of it here.
*/
malloc_mutex_lock(TSDN_NULL, &tsd_nominal_tsds_lock);
ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) {
if (tsd == tsd_list) {
found = true;
break;
}
}
malloc_mutex_unlock(TSDN_NULL, &tsd_nominal_tsds_lock);
return found;
}
static void
tsd_add_nominal(tsd_t *tsd) {
assert(!tsd_in_nominal_list(tsd));
assert(tsd_state_get(tsd) <= tsd_state_nominal_max);
ql_elm_new(tsd, TSD_MANGLE(tcache).tsd_link);
malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link);
malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
}
static void
tsd_remove_nominal(tsd_t *tsd) {
assert(tsd_in_nominal_list(tsd));
assert(tsd_state_get(tsd) <= tsd_state_nominal_max);
malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link);
malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
}
static void
tsd_force_recompute(tsdn_t *tsdn) {
/*
* The stores to tsd->state here need to synchronize with the exchange
* in tsd_slow_update.
*/
atomic_fence(ATOMIC_RELEASE);
malloc_mutex_lock(tsdn, &tsd_nominal_tsds_lock);
tsd_t *remote_tsd;
ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) {
assert(tsd_atomic_load(&remote_tsd->state, ATOMIC_RELAXED)
<= tsd_state_nominal_max);
tsd_atomic_store(&remote_tsd->state, tsd_state_nominal_recompute,
ATOMIC_RELAXED);
}
malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock);
}
void
tsd_global_slow_inc(tsdn_t *tsdn) {
atomic_fetch_add_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED);
/*
* We unconditionally force a recompute, even if the global slow count
* was already positive. If we didn't, then it would be possible for us
* to return to the user, have the user synchronize externally with some
* other thread, and then have that other thread not have picked up the
* update yet (since the original incrementing thread might still be
* making its way through the tsd list).
*/
tsd_force_recompute(tsdn);
}
void tsd_global_slow_dec(tsdn_t *tsdn) {
atomic_fetch_sub_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED);
/* See the note in ..._inc(). */
tsd_force_recompute(tsdn);
}
static bool
tsd_local_slow(tsd_t *tsd) {
return !tsd_tcache_enabled_get(tsd)
|| tsd_reentrancy_level_get(tsd) > 0;
}
bool
tsd_global_slow() {
return atomic_load_u32(&tsd_global_slow_count, ATOMIC_RELAXED) > 0;
}
/******************************************************************************/
static uint8_t
tsd_state_compute(tsd_t *tsd) {
if (!tsd_nominal(tsd)) {
return tsd_state_get(tsd);
}
/* We're in *a* nominal state; but which one? */
if (malloc_slow || tsd_local_slow(tsd) || tsd_global_slow()) {
return tsd_state_nominal_slow;
} else {
return tsd_state_nominal;
}
}
void
tsd_slow_update(tsd_t *tsd) {
if (tsd_nominal(tsd)) {
if (malloc_slow || !tsd_tcache_enabled_get(tsd) ||
tsd_reentrancy_level_get(tsd) > 0) {
tsd->state = tsd_state_nominal_slow;
uint8_t old_state;
do {
uint8_t new_state = tsd_state_compute(tsd);
old_state = tsd_atomic_exchange(&tsd->state, new_state,
ATOMIC_ACQUIRE);
} while (old_state == tsd_state_nominal_recompute);
}
void
tsd_state_set(tsd_t *tsd, uint8_t new_state) {
/* Only the tsd module can change the state *to* recompute. */
assert(new_state != tsd_state_nominal_recompute);
uint8_t old_state = tsd_atomic_load(&tsd->state, ATOMIC_RELAXED);
if (old_state > tsd_state_nominal_max) {
/*
* Not currently in the nominal list, but it might need to be
* inserted there.
*/
assert(!tsd_in_nominal_list(tsd));
tsd_atomic_store(&tsd->state, new_state, ATOMIC_RELAXED);
if (new_state <= tsd_state_nominal_max) {
tsd_add_nominal(tsd);
}
} else {
/*
* We're currently nominal. If the new state is non-nominal,
* great; we take ourselves off the list and just enter the new
* state.
*/
assert(tsd_in_nominal_list(tsd));
if (new_state > tsd_state_nominal_max) {
tsd_remove_nominal(tsd);
tsd_atomic_store(&tsd->state, new_state,
ATOMIC_RELAXED);
} else {
tsd->state = tsd_state_nominal;
/*
* This is the tricky case. We're transitioning from
* one nominal state to another. The caller can't know
* about any races that are occuring at the same time,
* so we always have to recompute no matter what.
*/
tsd_slow_update(tsd);
}
}
}
@@ -71,12 +223,23 @@ tsd_data_init(tsd_t *tsd) {
*/
rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd));
/*
* A nondeterministic seed based on the address of tsd reduces
* the likelihood of lockstep non-uniform cache index
* utilization among identical concurrent processes, but at the
* cost of test repeatability. For debug builds, instead use a
* deterministic seed.
*/
*tsd_offset_statep_get(tsd) = config_debug ? 0 :
(uint64_t)(uintptr_t)tsd;
return tsd_tcache_enabled_data_init(tsd);
}
static void
assert_tsd_data_cleanup_done(tsd_t *tsd) {
assert(!tsd_nominal(tsd));
assert(!tsd_in_nominal_list(tsd));
assert(*tsd_arenap_get_unsafe(tsd) == NULL);
assert(*tsd_iarenap_get_unsafe(tsd) == NULL);
assert(*tsd_arenas_tdata_bypassp_get_unsafe(tsd) == true);
@@ -87,8 +250,8 @@ assert_tsd_data_cleanup_done(tsd_t *tsd) {
static bool
tsd_data_init_nocleanup(tsd_t *tsd) {
assert(tsd->state == tsd_state_reincarnated ||
tsd->state == tsd_state_minimal_initialized);
assert(tsd_state_get(tsd) == tsd_state_reincarnated ||
tsd_state_get(tsd) == tsd_state_minimal_initialized);
/*
* During reincarnation, there is no guarantee that the cleanup function
* will be called (deallocation may happen after all tsd destructors).
@@ -107,27 +270,33 @@ tsd_t *
tsd_fetch_slow(tsd_t *tsd, bool minimal) {
assert(!tsd_fast(tsd));
if (tsd->state == tsd_state_nominal_slow) {
/* On slow path but no work needed. */
assert(malloc_slow || !tsd_tcache_enabled_get(tsd) ||
tsd_reentrancy_level_get(tsd) > 0 ||
*tsd_arenas_tdata_bypassp_get(tsd));
} else if (tsd->state == tsd_state_uninitialized) {
if (tsd_state_get(tsd) == tsd_state_nominal_slow) {
/*
* On slow path but no work needed. Note that we can't
* necessarily *assert* that we're slow, because we might be
* slow because of an asynchronous modification to global state,
* which might be asynchronously modified *back*.
*/
} else if (tsd_state_get(tsd) == tsd_state_nominal_recompute) {
tsd_slow_update(tsd);
} else if (tsd_state_get(tsd) == tsd_state_uninitialized) {
if (!minimal) {
tsd->state = tsd_state_nominal;
tsd_slow_update(tsd);
/* Trigger cleanup handler registration. */
tsd_set(tsd);
tsd_data_init(tsd);
if (tsd_booted) {
tsd_state_set(tsd, tsd_state_nominal);
tsd_slow_update(tsd);
/* Trigger cleanup handler registration. */
tsd_set(tsd);
tsd_data_init(tsd);
}
} else {
tsd->state = tsd_state_minimal_initialized;
tsd_state_set(tsd, tsd_state_minimal_initialized);
tsd_set(tsd);
tsd_data_init_nocleanup(tsd);
}
} else if (tsd->state == tsd_state_minimal_initialized) {
} else if (tsd_state_get(tsd) == tsd_state_minimal_initialized) {
if (!minimal) {
/* Switch to fully initialized. */
tsd->state = tsd_state_nominal;
tsd_state_set(tsd, tsd_state_nominal);
assert(*tsd_reentrancy_levelp_get(tsd) >= 1);
(*tsd_reentrancy_levelp_get(tsd))--;
tsd_slow_update(tsd);
@@ -135,12 +304,12 @@ tsd_fetch_slow(tsd_t *tsd, bool minimal) {
} else {
assert_tsd_data_cleanup_done(tsd);
}
} else if (tsd->state == tsd_state_purgatory) {
tsd->state = tsd_state_reincarnated;
} else if (tsd_state_get(tsd) == tsd_state_purgatory) {
tsd_state_set(tsd, tsd_state_reincarnated);
tsd_set(tsd);
tsd_data_init_nocleanup(tsd);
} else {
assert(tsd->state == tsd_state_reincarnated);
assert(tsd_state_get(tsd) == tsd_state_reincarnated);
}
return tsd;
@@ -204,7 +373,7 @@ void
tsd_cleanup(void *arg) {
tsd_t *tsd = (tsd_t *)arg;
switch (tsd->state) {
switch (tsd_state_get(tsd)) {
case tsd_state_uninitialized:
/* Do nothing. */
break;
@@ -222,7 +391,7 @@ tsd_cleanup(void *arg) {
case tsd_state_nominal:
case tsd_state_nominal_slow:
tsd_do_data_cleanup(tsd);
tsd->state = tsd_state_purgatory;
tsd_state_set(tsd, tsd_state_purgatory);
tsd_set(tsd);
break;
case tsd_state_purgatory:
@@ -250,6 +419,10 @@ malloc_tsd_boot0(void) {
tsd_t *tsd;
ncleanups = 0;
if (malloc_mutex_init(&tsd_nominal_tsds_lock, "tsd_nominal_tsds_lock",
WITNESS_RANK_OMIT, malloc_mutex_rank_exclusive)) {
return NULL;
}
if (tsd_boot0()) {
return NULL;
}
@@ -300,7 +473,7 @@ _tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) {
# pragma comment(linker, "/INCLUDE:_tls_callback")
# else
# pragma comment(linker, "/INCLUDE:_tls_used")
# pragma comment(linker, "/INCLUDE:tls_callback")
# pragma comment(linker, "/INCLUDE:" STRINGIFY(tls_callback) )
# endif
# pragma section(".CRT$XLY",long,read)
#endif
@@ -339,3 +512,23 @@ tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) {
malloc_mutex_unlock(TSDN_NULL, &head->lock);
}
#endif
void
tsd_prefork(tsd_t *tsd) {
malloc_mutex_prefork(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
}
void
tsd_postfork_parent(tsd_t *tsd) {
malloc_mutex_postfork_parent(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
}
void
tsd_postfork_child(tsd_t *tsd) {
malloc_mutex_postfork_child(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
ql_new(&tsd_nominal_tsds);
if (tsd_state_get(tsd) <= tsd_state_nominal_max) {
tsd_add_nominal(tsd);
}
}

30
deps/jemalloc/src/zone.c vendored
View File

@@ -89,6 +89,7 @@ JEMALLOC_ATTR(weak_import);
static malloc_zone_t *default_zone, *purgeable_zone;
static malloc_zone_t jemalloc_zone;
static struct malloc_introspection_t jemalloc_zone_introspect;
static pid_t zone_force_lock_pid = -1;
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
@@ -270,6 +271,12 @@ zone_log(malloc_zone_t *zone, void *address) {
static void
zone_force_lock(malloc_zone_t *zone) {
if (isthreaded) {
/*
* See the note in zone_force_unlock, below, to see why we need
* this.
*/
assert(zone_force_lock_pid == -1);
zone_force_lock_pid = getpid();
jemalloc_prefork();
}
}
@@ -277,14 +284,25 @@ zone_force_lock(malloc_zone_t *zone) {
static void
zone_force_unlock(malloc_zone_t *zone) {
/*
* Call jemalloc_postfork_child() rather than
* jemalloc_postfork_parent(), because this function is executed by both
* parent and child. The parent can tolerate having state
* reinitialized, but the child cannot unlock mutexes that were locked
* by the parent.
* zone_force_lock and zone_force_unlock are the entry points to the
* forking machinery on OS X. The tricky thing is, the child is not
* allowed to unlock mutexes locked in the parent, even if owned by the
* forking thread (and the mutex type we use in OS X will fail an assert
* if we try). In the child, we can get away with reinitializing all
* the mutexes, which has the effect of unlocking them. In the parent,
* doing this would mean we wouldn't wake any waiters blocked on the
* mutexes we unlock. So, we record the pid of the current thread in
* zone_force_lock, and use that to detect if we're in the parent or
* child here, to decide which unlock logic we need.
*/
if (isthreaded) {
jemalloc_postfork_child();
assert(zone_force_lock_pid != -1);
if (getpid() == zone_force_lock_pid) {
jemalloc_postfork_parent();
} else {
jemalloc_postfork_child();
}
zone_force_lock_pid = -1;
}
}