Use Lmod in Python mode (!4) · Merge requests · hps-public / xenv

Merged Benedikt Steinbusch requested to merge bs-lmod-python into master Dec 21, 2021

This prevents issues with escaped characters in Lmod's default output format meant for evaluation by a Bourne shell. See #4 (closed).

These are the changes in the resulting environment when loading GCC and OpenMPI:

--- old.env     2021-12-21 18:52:36.104545189 +0100
+++ new.env     2021-12-21 20:06:29.102588436 +0100
@@ -94,9 +94,9 @@
 LMOD_PKG=/p/software/juwels/lmod/8.4.1
 LMOD_PREPEND_BLOCK=normal
 LMOD_RC=/p/software/juwelstest/configs/lmodrc.lua
-__LMOD_REF_COUNT__LMFILES_=/p/software/juwelstest/productionstages/Stages/2020.lua:1\
-__LMOD_REF_COUNT_LOADEDMODULES=Stages/2020:1\
-__LMOD_REF_COUNT_MODULEPATH=/p/software/juwels/stages/2020/modules/all/MPI_settings/OpenMPI/4.1:1\
+__LMOD_REF_COUNT__LMFILES_=/p/software/juwelstest/productionstages/Stages/2020.lua:1;/p/software/juwels/stages/2020/UI/Compilers/GCCcore/.10.3.0.lua:1;/p/software/juwels/stages/2020/modules/all/Compiler/GCCcore/10.3.0/zlib/.1.2.11.lua:1;/p/software/juwels/stages/2020/modules/all/Compiler/GCCcore/10.3.0/binutils/.2.36.1.lua:1;/p/software/juwels/stages/2020/UI/Tools/StdEnv/2020.lua:1;/p/software/juwels/stages/2020/UI/Compilers/GCC/10.3.0.lua:1;/p/software/juwels/stages/2020/UI/Tools/numactl/2.0.14.lua:1;/p/software/juwels/stages/2020/modules/all/Compiler/GCCcore/10.3.0/XZ/.5.2.5.lua:1;/p/software/juwels/stages/2020/modules/all/Compiler/GCCcore/10.3.0/libxml2/.2.9.10.lua:1;/p/software/juwels/stages/2020/modules/all/Compiler/GCCcore/10.3.0/libpciaccess/.0.16.lua:1;/p/software/juwels/stages/2020/UI/Tools/nvidia-driver/.default.lua:1;/p/software/juwels/stages/2020/UI/Tools/CUDA/11.3.lua:1;/p/software/juwels/stages/2020/modules/all/Compiler/GCCcore/10.3.0/hwloc/2.4.1.lua:1;/p/software/juwels/stages/2020/UI/Tools/UCX/1.10.1.lua:1;/p/software/juwels/stages/2020/modules/all/Compiler/GCCcore/10.3.0/libevent/.2.1.12.lua:1;/p/software/juwels/stages/2020/modules/all/MPI_settings/OpenMPI/4.1/mpi-settings/CUDA.lua:1;/p/software/juwels/stages/2020/modules/all/Compiler/mpi/GCC/10.3.0/OpenMPI/4.1.1.lua:1
+__LMOD_REF_COUNT_LOADEDMODULES=Stages/2020:1;GCCcore/.10.3.0:1;zlib/.1.2.11:1;binutils/.2.36.1:1;StdEnv/2020:1;GCC/10.3.0:1;numactl/2.0.14:1;XZ/.5.2.5:1;libxml2/.2.9.10:1;libpciaccess/.0.16:1;nvidia-driver/.default:1;CUDA/11.3:1;hwloc/2.4.1:1;UCX/1.10.1:1;libevent/.2.1.12:1;mpi-settings/CUDA:1;OpenMPI/4.1.1:1
+__LMOD_REF_COUNT_MODULEPATH=/p/software/juwels/stages/2020/modules/all/MPI_settings/OpenMPI/4.1:1;/p/software/juwels/stages/2020/modules/all/MPI/GCC/10.3.0/OpenMPI/4.1:1;/p/software/juwels/stages/2020/modules/all/Compiler/mpi/GCC/10.3.0:1;/p/software/juwels/stages/2020/modules/all/Compiler/GCC/10.3.0:1;/p/software/juwels/stages/2020/modules/all/Compiler/GCCcore/10.3.0:1;/p/software/juwels/stages/2020/UI/Defaults:1;/p/software/juwels/stages/2020/UI/Tools:1;/p/software/juwels/stages/2020/UI/Compilers:1;/p/software/juwelstest/supercomputer_modules:1;/p/software/juwelstest/productionstages:1
 LMOD_ROOT=/p/software/juwels/lmod
 __LMOD_SET_FPATH=1
 LMOD_SETTARG_CMD=:
@@ -131,10 +131,10 @@
 OMPI_MCA_bml_r2_show_unreach_errors=0
 OMPI_MCA_btl_openib_allow_ib=1
 OMPI_MCA_btl_openib_warn_default_gid_prefix=0
-OMPI_MCA_btl=\^uct,openib
+OMPI_MCA_btl=^uct,openib
 OMPI_MCA_coll_hcoll_enable=1
 OMPI_MCA_coll_hcoll_np=0
-OMPI_MCA_coll=\^ml
+OMPI_MCA_coll=^ml
 OMPI_MCA_io=romio321
 OMPI_MCA_mca_base_component_show_load_errors=1
 OMPI_MCA_mpi_param_check=1
@@ -143,7 +143,7 @@
 OMPI_MCA_opal_abort_print_stack=1
 OMPI_MCA_opal_event_include=epoll
 OMPI_MCA_opal_set_max_sys_limits=1
-OMPI_MCA_osc=\^rdma
+OMPI_MCA_osc=^rdma
 OMPI_MCA_pml=ucx
 OTHERSTAGES=/p/software/juwelstest/otherstages
 PATH=/p/software/juwels/stages/2020/software/OpenMPI/4.1.1-GCC-10.3.0/bin:/p/software/juwels/stages/2020/software/libevent/2.1.12-GCCcore-10.3.0/bin:/p/software/juwels/stages/2020/software/UCX/1.10.1/bin:/p/software/juwels/stages/2020/software/hwloc/2.4.1-GCCcore-10.3.0/sbin:/p/software/juwels/stages/2020/software/hwloc/2.4.1-GCCcore-10.3.0/bin:/p/software/juwels/stages/2020/software/CUDA/11.3/nvvm/bin:/p/software/juwels/stages/2020/software/CUDA/11.3/bin:/p/software/juwels/stages/2020/software/nvidia-driver/default:/p/software/juwels/stages/2020/software/nvidia-driver/default/bin:/p/software/juwels/stages/2020/software/libxml2/2.9.10-GCCcore-10.3.0/bin:/p/software/juwels/stages/2020/software/XZ/5.2.5-GCCcore-10.3.0/bin:/p/software/juwels/stages/2020/software/numactl/2.0.14/bin:/p/software/juwels/stages/2020/software/binutils/2.36.1-GCCcore-10.3.0/bin:/p/software/juwels/stages/2020/software/GCCcore/10.3.0/bin:/home/bsteinb/.local/bin:/home/bsteinb/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/jsc/bin:/usr/local/jsc/bin:/opt/parastation/bin:/p/software/juwelstest/bin

The OMPI_MCA settings are now set correctly with the ^ unescaped. Additionally, the __LMOD_REF_COUNT variables are not truncated at the first semicolon.

The MR is somewhat bloated by the changes that were necessary to make the linters shut up.

Previous contributors here are @vonst.vieth1 and @thust1. Who is in the mood for reviewing this?