diff --git a/maestro/ofi.c b/maestro/ofi.c index c76a084e2bfd9f1d5aa004487fb436fdc2c61ea9..72a1572df8ded128f7f66375d169c9eb9b56f951 100644 --- a/maestro/ofi.c +++ b/maestro/ofi.c @@ -44,7 +44,7 @@ #include "i_pool_client.h" #include "i_pool_manager.h" - +#include "maestro/i_memlock.h" #include "../transport/transport.h" #include <rdma/fabric.h> @@ -1331,19 +1331,12 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst, /* register component block as RDMA-readable memory chunk */ /* to be safe: mlock the descriptor. GNI needs it, others will like it */ - assert(sizeof(g_component_descriptor)%sysconf(_SC_PAGESIZE)==0); - stat = mlock(&g_component_descriptor, sizeof(g_component_descriptor)); - if(stat!=0) { + retstat = mstro_memlock(&g_component_descriptor, sizeof(g_component_descriptor)); + if(retstat!=MSTRO_OK) { ERR("Failed to mlock component descriptor (%d bytes at %p, pagesize %d): %d (%s)\n", sizeof(g_component_descriptor), &g_component_descriptor,sysconf(_SC_PAGESIZE), - errno, strerror(errno)); -#if defined(RLIMIT_MEMLOCK) - struct rlimit l; - getrlimit(RLIMIT_MEMLOCK,&l); - DEBUG("RLIMIT_MEMLOCK is %zu (soft), %zu (hard)\n", - l.rlim_cur, l.rlim_max); -#endif - retstat=MSTRO_FAIL; goto BAILOUT_FAIL; + retstat, mstro_status_description(retstat)); + goto BAILOUT_FAIL; } struct fid_mr *mr; @@ -1747,34 +1740,22 @@ mstro_ofi_init(void) /* to be safe: mlock the descriptor. GNI needs it, others will like it */ assert(sizeof(g_component_descriptor)%sysconf(_SC_PAGESIZE)==0); - stat = mlock(&g_component_descriptor, sizeof(g_component_descriptor)); - if(stat!=0) { + retstat = mstro_memlock(&g_component_descriptor, sizeof(g_component_descriptor)); + if(retstat!=MSTRO_OK) { ERR("Failed to mlock our component descriptor (%d bytes at %p, pagesize %ul): %d (%s)\n", sizeof(g_component_descriptor), &g_component_descriptor, sysconf(_SC_PAGESIZE), - errno, strerror(errno)); -#if defined(RLIMIT_MEMLOCK) - struct rlimit l; - getrlimit(RLIMIT_MEMLOCK,&l); - DEBUG("RLIMIT_MEMLOCK is %zu (soft), %zu (hard)\n", - l.rlim_cur, l.rlim_max); -#endif - retstat=MSTRO_FAIL; goto BAILOUT_FAIL; + retstat, mstro_status_description(retstat)); + goto BAILOUT_FAIL; } /* also do it for the PM descriptor block we'll RDMA read into */ assert(sizeof(g_pm_component_descriptor)%sysconf(_SC_PAGESIZE)==0); - stat = mlock(&g_pm_component_descriptor, sizeof(g_pm_component_descriptor)); - if(stat!=0) { + retstat = mstro_memlock(&g_pm_component_descriptor, sizeof(g_pm_component_descriptor)); + if(retstat!=MSTRO_OK) { ERR("Failed to mlock PM component descriptor (%d bytes at %p, pagesize %ul): %d (%s)\n", sizeof(g_pm_component_descriptor), &g_pm_component_descriptor, sysconf(_SC_PAGESIZE), - errno, strerror(errno)); -#if defined(RLIMIT_MEMLOCK) - struct rlimit l; - getrlimit(RLIMIT_MEMLOCK,&l); - DEBUG("RLIMIT_MEMLOCK is %zu (soft), %zu (hard)\n", - l.rlim_cur, l.rlim_max); -#endif - retstat=MSTRO_FAIL; goto BAILOUT_FAIL; + retstat, mstro_status_description(retstat)); + goto BAILOUT_FAIL; } /* create one entry in our global EP list per entry */ @@ -1859,6 +1840,8 @@ mstro_ofi_finalize(void) i, s, fi_strerror(-s)); } } + /* every EP has one lock on the component descriptor */ + mstro_memunlock(&g_component_descriptor, sizeof(g_component_descriptor)); #define CLOSE_FID(member, descr) do { \ if(e->member) { \ @@ -1876,15 +1859,16 @@ mstro_ofi_finalize(void) CLOSE_FID(domain,"DOMAIN"); // CLOSE_FID(fi,"FABRIC"); #undef CLOSE_FID - + } } DEBUG("OFI endpoints closed\n"); - /* It's safe to call this even if we've not yet locked the page; we ignore the EINVAL that we'd get in this case */ - munlock(&g_component_descriptor, sizeof(g_component_descriptor)); - munlock(&g_pm_component_descriptor, sizeof(g_pm_component_descriptor)); - + /* It's safe to call this even if we've not yet locked the page; we + * ignore the error that we'd get in this case */ + mstro_memunlock(&g_component_descriptor, sizeof(g_component_descriptor)); + mstro_memunlock(&g_pm_component_descriptor, sizeof(g_pm_component_descriptor)); + return MSTRO_OK; }