diff --git a/include/maestro/i_memlock.h b/include/maestro/i_memlock.h index 5d5a15438c847c1716014b64602ad6289b141278..5d4f16a0a923177dbe6b7b77ea6cbe2b606f9bb5 100644 --- a/include/maestro/i_memlock.h +++ b/include/maestro/i_memlock.h @@ -70,9 +70,13 @@ mstro_status mstro_memunlock(void* addr, size_t len); -/**@brief Initialize the memlock subsystem */ +/**@brief Initialize the memlock subsystem + * + * Check resource limits whether at least min_required bytes can be locked (RLIMIT_MEMLOCK), and + * if not, return MSTRO_NOMEM. + */ mstro_status -mstro_memlock_init(void); +mstro_memlock_init(size_t min_required); /**@} (end of group MSTRO_I_MEMLOCK) */ /**@} (end of group MSTRO_Internal) */ diff --git a/maestro/core.c b/maestro/core.c index 79134a06af068a29fd5905a422c5deff345d260d..7bac92b88684f8a9beab40470520045af9f53472 100644 --- a/maestro/core.c +++ b/maestro/core.c @@ -141,6 +141,10 @@ BAILOUT: return MSTRO_OK; } + +/** minimum mlock() limit */ +#define MSTRO_MIN_MEMLOCK (4*sizeof(g_component_descriptor)) + mstro_status mstro_core_init(const char *workflow_name, const char *component_name, @@ -148,7 +152,7 @@ mstro_core_init(const char *workflow_name, { mstro_status status = MSTRO_UNIMPL; - status = mstro_memlock_init(); + status = mstro_memlock_init(MSTRO_MIN_MEMLOCK); if(status!=MSTRO_OK) { return status; } diff --git a/maestro/memlock.c b/maestro/memlock.c index e6552c3d10a73ac7def943df73043aa61fe1c61f..b89bd6fee700bac1c60d503ee72955c3ad6f393c 100644 --- a/maestro/memlock.c +++ b/maestro/memlock.c @@ -88,7 +88,7 @@ static pthread_mutex_t g_locked_pages_mtx = PTHREAD_MUTEX_INITIALIZER; mstro_status -mstro_memlock_init(void) +mstro_memlock_init(size_t min_required) { g_pagesize = sysconf(_SC_PAGESIZE); assert(popcount(g_pagesize)==1); @@ -98,6 +98,18 @@ mstro_memlock_init(void) DEBUG("Page size is %ld, %zu bits, mask %" PRIxPTR "\n", g_pagesize, g_page_bits, g_pagify_mask); + struct rlimit l; + int s = getrlimit(RLIMIT_MEMLOCK, &l); + DEBUG("RLIMIT_MEMLOCK is %zu (soft), %zu (hard)\n", + l.rlim_cur, l.rlim_max); + if(l.rlim_cur < min_required) { + ERR("RLIMIT_MEMLOCK too small; please set ulimit -l %d or higher\n", + min_required/1024); + /* We tried to do setrlimit here, but get a segv on linux (Cray XC) + * after it, so ask the user to do it */ + return MSTRO_NOMEM; + } + g_locked_pages = kh_init(page_set); if(g_locked_pages==NULL) diff --git a/maestro/ofi.c b/maestro/ofi.c index be9651a411089b69293245fa7fbea4d9bee9a329..a69e119daf4f7ee7c3b485ca02618d983cd6e2e0 100644 --- a/maestro/ofi.c +++ b/maestro/ofi.c @@ -1638,26 +1638,7 @@ mstro_ofi__order_fi_list(struct fi_info **fi) return MSTRO_OK; } -/** minimum mlock() limit */ -#define MSTRO_MIN_MEMLOCK (4*sizeof(g_component_descriptor)) -static inline -mstro_status -mstro__init_memlock() -{ - struct rlimit l; - int s = getrlimit(RLIMIT_MEMLOCK,&l); - DEBUG("RLIMIT_MEMLOCK is %zu (soft), %zu (hard)\n", - l.rlim_cur, l.rlim_max); - if(l.rlim_cur <MSTRO_MIN_MEMLOCK) { - ERR("RLIMIT_MEMLOCK too small; please set ulimit -l %d or higher\n", - MSTRO_MIN_MEMLOCK/1024); - /* We tried to do setrlimit here, but get a segv on linux (Cray XC) - * after it, so ask the user to do it */ - return MSTRO_NOMEM; - } - return MSTRO_OK; -} /** Populate @ref g_endpoints with an enabled endpoint for each useful * OFI endpoint discovered */ @@ -1671,11 +1652,6 @@ mstro_ofi_init(void) struct fi_info *fi = NULL, *hints = NULL; - /* ensure we can use some RDMA */ - retstat = mstro__init_memlock(); - if(retstat!=MSTRO_OK) - return retstat; - /* prepare for DRC */ /* (only needed on GNI/Cray, but dummy emulation is provided) */ if(g_drc_info==NULL) { diff --git a/tests/check_memlock.c b/tests/check_memlock.c index b155ec568e442aea3a1782ec2ea9d8b760588242..85ec86300a4e33807f64c2cd4dfbbf109962c566 100644 --- a/tests/check_memlock.c +++ b/tests/check_memlock.c @@ -46,7 +46,7 @@ CHEAT_TEST(lock_unlock, { - cheat_assert(MSTRO_OK==mstro_memlock_init()); + cheat_assert(MSTRO_OK==mstro_memlock_init(2*8000)); void *x =malloc(8000); cheat_assert(x!=NULL);