diff --git a/README.md b/README.md
index a36ef6218ebf89ecdcbb102c57796fcf9f717e4d..dc59596c5436f8576836a23bc79a8f5dbf679fb5 100644
--- a/README.md
+++ b/README.md
@@ -15,12 +15,17 @@ visit [this](https://gitlab.version.fz-juelich.de/MLDL_FZJ/MLDL_FZJ_Wiki/wikis/E
### Announcements
-* Tensorflow and Keras examples (with and without Horovod) are now fully functional on JUWELS as well.
-* Python 2 support has been removed from the tutorial for all frameworks except Caffe.
-* Even though PyTorch is available as as system-wide module on the JSC supercomputers, all PyTorch
+* **November 18, 2019:** The `horovod_data_distributed` directory has been added that contains code
+samples to illustrate proper data-distributed training with Horovod, i.e., a distribution mechanism
+where the training data is distributed instead of epochs. Further information is available in the
+directory-local `README.md`.
+* **September 02, 2019:** Even though PyTorch is available as as system-wide module on the JSC supercomputers, all PyTorch
examples have been removed from this tutorial. This is due to the fact that the tutorial
developers are not currently working with PyTorch, and are therefore not in a position to provide
support for PyTorch related issues.
+* **August 23, 2019:**
+ * Tensorflow and Keras examples (with and without Horovod) are now fully functional on JUWELS as well.
+ * Python 2 support has been removed from the tutorial for all frameworks except Caffe.
# Table of contents
<!-- TOC -->
diff --git a/horovod/keras/.run_mnist_data_dist b/horovod/keras/.run_mnist_data_dist
deleted file mode 100755
index b9b19fa5617b0573fd480b1c77b3e04eb122d755..0000000000000000000000000000000000000000
--- a/horovod/keras/.run_mnist_data_dist
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-PYTHONHASHSEED=0 mpirun -np 1 python -u mnist_data_distributed.py
diff --git a/horovod_data_distributed/README.md b/horovod_data_distributed/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..c0934fc91d3305c4cecf1d144d9f3c204c848a4b
--- /dev/null
+++ b/horovod_data_distributed/README.md
@@ -0,0 +1,24 @@
+# Introduction
+
+Please see the main docstring in each program for details.
+
+# Notes
+
+The `mnist_data_distributed.py` program requires the [`slns.distribution`](
+https://gitlab.version.fz-juelich.de/hpc4ns/slns_utils#1-slnsdistribution)
+module for distribution of training data filenames across multiple ranks.
+Please follow the steps below to install the required package.
+
+1. Change to the source directory for this sample, i.e., to `dl_on_supercomputers/horovod_data_distributed`
+2. Load the system-wide Python module.
+ * On JURECA and JUWELS: `module load Python/3.6.8`
+ * On JURON: `module load Python/3.6.1`
+3. Create a Python virtual environment: `python -m venv venv_dl_slns`
+4. Activate the virtual environment: `source activate venv_dl_slns/bin/activate`
+5. Install the `slns` package: `python -m pip install git+https://gitlab.version.fz-juelich.de/hpc4ns/slns_utils.git`
+6. Open the job submission script you intend to use, and make sure the path to the virtual environment is correct
+
+Once all the above steps are completed, the job can be submitted.
+
+**Note:** A maximum of eight ranks can be used to run `mnist_data_distributed.py`, as there
+only eight training files.
\ No newline at end of file
diff --git a/horovod/keras/mnist_data_distributed.py b/horovod_data_distributed/mnist_data_distributed.py
similarity index 87%
rename from horovod/keras/mnist_data_distributed.py
rename to horovod_data_distributed/mnist_data_distributed.py
index c1632cd724eff179f6a7afb2d953e1c0ab2d62e5..b6add7e7e9210f1b734e02a8475b71ab1e3e0181 100644
--- a/horovod/keras/mnist_data_distributed.py
+++ b/horovod_data_distributed/mnist_data_distributed.py
@@ -2,14 +2,14 @@
# This code is licensed under MIT license (see the LICENSE file for details).
"""
- This program program distributes the partitioned MNIST data across
- multiple MPI ranks for truly data distributed training of a shallow ANN
- for handwritten digit classification.
+ This program distributes the partitioned MNIST data across multiple ranks
+ for truly data distributed training of a shallow ANN for handwritten digit
+ classification.
- The Horovod framework is used for seamless distributed training. Instead
- of distributing epochs, this program distributes data amongst the ranks,
- so that each rank contributes training based on its local subset of the
- training data.
+ The Horovod framework is used for seamless distributed training. However,
+ instead of distributing epochs, this program distributes data amongst the
+ ranks, so that each rank contributes training based on its local subset of
+ the training data.
"""
@@ -93,14 +93,15 @@ def initialize_hvd_and_mpi():
are no conflicts between Horovod and mpi4py communicator
initialization.
- :exception: hpcns.errors.MpiInitError is raised in the case
+ :exception: slns.errors.MpiInitError is raised in the case
of initialization failure.
"""
# Initialize Horovod.
hvd.init()
- # Pin the GPU to be used to process local rank (one GPU per process)
+ # Bind the local rank to a specific GPU, so that each rank uses
+ # a different GPU
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
tf_config.gpu_options.visible_device_list = str(hvd.local_rank())
@@ -139,14 +140,19 @@ def main():
# Flag to indicate whether this is the MPI root
is_root = hvd.rank() == 0
+ # Decorate the get_filenames function so that instead of returning
+ # a list of all filenames, it returns a list of the subset of
+ # filenames that are to be processed by the local rank.
dist_decorator = DataDistributor(
mpi_comm=mpi4py.MPI.COMM_WORLD, shutdown_on_error=True
)
get_rank_local_filenames = dist_decorator(get_filenames)
+ # Data directory paths
data_sub_dir = 'mnist/partitioned'
data_dir = DataValidator.validated_data_dir(data_sub_dir)
+ # Prepare training data
train_filenames = get_rank_local_filenames(
f'{os.path.join(data_dir, data_sub_dir)}/train')
x_train, y_train = load_dataset(
@@ -156,6 +162,7 @@ def main():
x_train = x_train / 255.0
if is_root:
+ # Prepare test data
test_filenames = get_filenames(
f'{os.path.join(data_dir, data_sub_dir)}/test')
x_test, y_test = load_dataset(
@@ -174,7 +181,7 @@ def main():
# Optimizer
optimizer = tf.keras.optimizers.Adam()
- # Horovod: add Horovod Distributed Optimizer.
+ # Decorate the optimizer with the Horovod Distributed Optimizer
optimizer = hvd.DistributedOptimizer(optimizer)
# Compile the model
@@ -189,9 +196,6 @@ def main():
# Training callbacks
callbacks = [
- # Horovod: broadcast initial variable states from rank 0 to all other processes.
- # This is necessary to ensure consistent initialization of all workers when
- # training is started with random weights or restored from a checkpoint.
hvd.callbacks.BroadcastGlobalVariablesCallback(0)
]
diff --git a/horovod_data_distributed/submit_job_juwels.sh b/horovod_data_distributed/submit_job_juwels.sh
new file mode 100755
index 0000000000000000000000000000000000000000..24f50d842239dcaecd8c4942d1a27f189919a352
--- /dev/null
+++ b/horovod_data_distributed/submit_job_juwels.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Slurm job configuration
+#SBATCH --nodes=2
+#SBATCH --ntasks=8
+#SBATCH --ntasks-per-node=4
+#SBATCH --output=output_%j.out
+#SBATCH --error=error_%j.er
+#SBATCH --time=00:10:00
+#SBATCH --job-name=HVD_DATA_DIST
+#SBATCH --gres=gpu:4 --partition=develgpus
+#SBATCH --mail-type=ALL
+
+# Load the required modules
+module load GCC/8.3.0
+module load MVAPICH2/2.3.1-GDR
+module load TensorFlow/1.13.1-GPU-Python-3.6.8
+module load Horovod/0.16.2-GPU-Python-3.6.8
+
+# Source the virtual environment
+source activate venv_dl_slns/bin/activate
+
+# Run the program
+srun python -u mnist_data_distributed.py