From 26430fe18f67ed497f40f1fa614950667fb2e719 Mon Sep 17 00:00:00 2001
From: Ali Mohammed <ali.mohammed@hpe.com>
Date: Tue, 17 Jan 2023 16:23:29 +0100
Subject: [PATCH] separate num cq threads from op threads
---
examples/run_core_bench.sh | 2 +-
include/maestro/env.h | 6 ++++++
maestro/ofi.c | 13 ++++++++++++-
3 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/examples/run_core_bench.sh b/examples/run_core_bench.sh
index aab6da42..32c23070 100755
--- a/examples/run_core_bench.sh
+++ b/examples/run_core_bench.sh
@@ -39,7 +39,7 @@ export MPICH_MAX_THREAD_SAFETY=multiple
# number of completion queue handler threads
export MSTRO_OFI_CQ_NUM_THREADS=1
-
+export MSTRO_OPERATIONS_NUM_THREADS=1
# core_bench supports multiple consumer modes
# MSTRO_CONSUMER_SINK_ALL >> One consumer sinks in all data from all producers
# MSTRO_CONSUMER_ONE2ONE >> #consumers == #procducers. each consumer is assigned CDOs from one producer
diff --git a/include/maestro/env.h b/include/maestro/env.h
index 6d1b6d33..c7243b6d 100644
--- a/include/maestro/env.h
+++ b/include/maestro/env.h
@@ -274,6 +274,12 @@
**/
#define MSTRO_ENV_OFI_CQ_NUM_THREADS "MSTRO_OFI_CQ_NUM_THREADS"
+/**
+ ** @brief Number of threads servicing PM or PC operations
+ **
+ ** The default is 1.
+ **/
+#define MSTRO_ENV_OPERATIONS_NUM_THREADS "MSTRO_OPERATIONS_NUM_THREADS"
/**
* @berief Pin PM/PC thread to a certain core
diff --git a/maestro/ofi.c b/maestro/ofi.c
index 242ea91a..e43de4e3 100644
--- a/maestro/ofi.c
+++ b/maestro/ofi.c
@@ -1939,6 +1939,8 @@ mstro_ofi_init(void)
#define DEFAULT_OFI_CQ_NUM_THREADS 1
char *e = getenv(MSTRO_ENV_OFI_CQ_NUM_THREADS);
int num_cq_threads;
+ int num_op_threads = 1;
+
if(e!=NULL) {
num_cq_threads = atoi(e);
if(num_cq_threads<1) {
@@ -1948,6 +1950,15 @@ mstro_ofi_init(void)
num_cq_threads = DEFAULT_OFI_CQ_NUM_THREADS;
}
+ char *tmp_char_p = getenv(MSTRO_ENV_OPERATIONS_NUM_THREADS);
+ if(tmp_char_p !=NULL) {
+ num_op_threads = atoi(tmp_char_p);
+ if(num_op_threads<1) {
+ ERR("Illegal value for number of Operations threads, using %d\n", 1);
+ num_op_threads = 1;
+ }
+ }
+
retstat = erl_thread_team_create("OFI CQ entry handler",
"CQ",
num_cq_threads,
@@ -1963,7 +1974,7 @@ mstro_ofi_init(void)
/**Start operation handler thread team*/
retstat = erl_thread_team_create("Pool operation handler",
"OP",
- num_cq_threads, //FIXME create an env var
+ num_op_threads,
mstro_pool_op_engine,
NULL,
&g_pool_operations_team);
--
GitLab