From 48e19a786223ad1f1beb1bf391160c84435b4614 Mon Sep 17 00:00:00 2001
From: Utz-Uwe Haus <uhaus@cray.com>
Date: Fri, 20 Nov 2020 19:05:00 +0100
Subject: [PATCH] Check RLIMIT_MEMLOCK when initializing memlock subsystem
---
include/maestro/i_memlock.h | 8 ++++++--
maestro/core.c | 6 +++++-
maestro/memlock.c | 14 +++++++++++++-
maestro/ofi.c | 24 ------------------------
tests/check_memlock.c | 2 +-
5 files changed, 25 insertions(+), 29 deletions(-)
diff --git a/include/maestro/i_memlock.h b/include/maestro/i_memlock.h
index 5d5a1543..5d4f16a0 100644
--- a/include/maestro/i_memlock.h
+++ b/include/maestro/i_memlock.h
@@ -70,9 +70,13 @@ mstro_status
mstro_memunlock(void* addr, size_t len);
-/**@brief Initialize the memlock subsystem */
+/**@brief Initialize the memlock subsystem
+ *
+ * Check resource limits whether at least min_required bytes can be locked (RLIMIT_MEMLOCK), and
+ * if not, return MSTRO_NOMEM.
+ */
mstro_status
-mstro_memlock_init(void);
+mstro_memlock_init(size_t min_required);
/**@} (end of group MSTRO_I_MEMLOCK) */
/**@} (end of group MSTRO_Internal) */
diff --git a/maestro/core.c b/maestro/core.c
index 79134a06..7bac92b8 100644
--- a/maestro/core.c
+++ b/maestro/core.c
@@ -141,6 +141,10 @@ BAILOUT:
return MSTRO_OK;
}
+
+/** minimum mlock() limit */
+#define MSTRO_MIN_MEMLOCK (4*sizeof(g_component_descriptor))
+
mstro_status
mstro_core_init(const char *workflow_name,
const char *component_name,
@@ -148,7 +152,7 @@ mstro_core_init(const char *workflow_name,
{
mstro_status status = MSTRO_UNIMPL;
- status = mstro_memlock_init();
+ status = mstro_memlock_init(MSTRO_MIN_MEMLOCK);
if(status!=MSTRO_OK) {
return status;
}
diff --git a/maestro/memlock.c b/maestro/memlock.c
index e6552c3d..b89bd6fe 100644
--- a/maestro/memlock.c
+++ b/maestro/memlock.c
@@ -88,7 +88,7 @@ static pthread_mutex_t g_locked_pages_mtx = PTHREAD_MUTEX_INITIALIZER;
mstro_status
-mstro_memlock_init(void)
+mstro_memlock_init(size_t min_required)
{
g_pagesize = sysconf(_SC_PAGESIZE);
assert(popcount(g_pagesize)==1);
@@ -98,6 +98,18 @@ mstro_memlock_init(void)
DEBUG("Page size is %ld, %zu bits, mask %" PRIxPTR "\n",
g_pagesize, g_page_bits, g_pagify_mask);
+ struct rlimit l;
+ int s = getrlimit(RLIMIT_MEMLOCK, &l);
+ DEBUG("RLIMIT_MEMLOCK is %zu (soft), %zu (hard)\n",
+ l.rlim_cur, l.rlim_max);
+ if(l.rlim_cur < min_required) {
+ ERR("RLIMIT_MEMLOCK too small; please set ulimit -l %d or higher\n",
+ min_required/1024);
+ /* We tried to do setrlimit here, but get a segv on linux (Cray XC)
+ * after it, so ask the user to do it */
+ return MSTRO_NOMEM;
+ }
+
g_locked_pages = kh_init(page_set);
if(g_locked_pages==NULL)
diff --git a/maestro/ofi.c b/maestro/ofi.c
index be9651a4..a69e119d 100644
--- a/maestro/ofi.c
+++ b/maestro/ofi.c
@@ -1638,26 +1638,7 @@ mstro_ofi__order_fi_list(struct fi_info **fi)
return MSTRO_OK;
}
-/** minimum mlock() limit */
-#define MSTRO_MIN_MEMLOCK (4*sizeof(g_component_descriptor))
-static inline
-mstro_status
-mstro__init_memlock()
-{
- struct rlimit l;
- int s = getrlimit(RLIMIT_MEMLOCK,&l);
- DEBUG("RLIMIT_MEMLOCK is %zu (soft), %zu (hard)\n",
- l.rlim_cur, l.rlim_max);
- if(l.rlim_cur <MSTRO_MIN_MEMLOCK) {
- ERR("RLIMIT_MEMLOCK too small; please set ulimit -l %d or higher\n",
- MSTRO_MIN_MEMLOCK/1024);
- /* We tried to do setrlimit here, but get a segv on linux (Cray XC)
- * after it, so ask the user to do it */
- return MSTRO_NOMEM;
- }
- return MSTRO_OK;
-}
/** Populate @ref g_endpoints with an enabled endpoint for each useful
* OFI endpoint discovered */
@@ -1671,11 +1652,6 @@ mstro_ofi_init(void)
struct fi_info *fi = NULL,
*hints = NULL;
- /* ensure we can use some RDMA */
- retstat = mstro__init_memlock();
- if(retstat!=MSTRO_OK)
- return retstat;
-
/* prepare for DRC */
/* (only needed on GNI/Cray, but dummy emulation is provided) */
if(g_drc_info==NULL) {
diff --git a/tests/check_memlock.c b/tests/check_memlock.c
index b155ec56..85ec8630 100644
--- a/tests/check_memlock.c
+++ b/tests/check_memlock.c
@@ -46,7 +46,7 @@
CHEAT_TEST(lock_unlock,
{
- cheat_assert(MSTRO_OK==mstro_memlock_init());
+ cheat_assert(MSTRO_OK==mstro_memlock_init(2*8000));
void *x =malloc(8000);
cheat_assert(x!=NULL);
--
GitLab