testing on jsc

training_data_distribution/distribution.py

@@ -14,7 +14,7 @@ import random
 import itertools
 import functools
 
-from hpc4neuro.errors import DataDistributionError
+from errors import DataDistributionError
 
 
 @enum.unique
@@ -136,7 +136,7 @@ class DataDistributor:
                     used is from the 'random' package in the Python
                     standard library.
 
-        :raises hpc4neuro.errors DataDistributionError: can be
+        :raises errors DataDistributionError: can be
                     raised if 'shutdown on error' is not requested at
                     the time of object creation.
 

training_data_distribution/jureca_job.sh

@@ -18,6 +18,8 @@ module load GCC
 module load OpenMPI
 module load TensorFlow
 module load Horovod
+export HOROVOD_MPI_THREADS_DISABLE=0
+module load mpi4py
 
 # Make all GPUs visible per node
 export CUDA_VISIBLE_DEVICES=0,1,2,3

training_data_distribution/jusuf_job.sh

@@ -18,6 +18,8 @@ module load GCC
 module load OpenMPI
 module load TensorFlow
 module load Horovod
+export HOROVOD_MPI_THREADS_DISABLE=0
+module load mpi4py
 
 # Make all GPUs visible per node
 export CUDA_VISIBLE_DEVICES=0

training_data_distribution/juwels_booster_job.sh

@@ -18,6 +18,8 @@ module load GCC
 module load OpenMPI
 module load TensorFlow
 module load Horovod
+export HOROVOD_MPI_THREADS_DISABLE=0
+module load mpi4py
 
 # Make all GPUs visible per node
 export CUDA_VISIBLE_DEVICES=0,1,2,3

training_data_distribution/juwels_job.sh

@@ -18,6 +18,8 @@ module load GCC
 module load OpenMPI
 module load TensorFlow
 module load Horovod
+export HOROVOD_MPI_THREADS_DISABLE=0
+module load mpi4py
 
 # Make all GPUs visible per node
 export CUDA_VISIBLE_DEVICES=0,1,2,3