From a2c93bf31c4ad7662a518f389f8ef781bdc92856 Mon Sep 17 00:00:00 2001 From: Fahad Khalid <f.khalid@fz-juelich.de> Date: Mon, 18 Nov 2019 14:29:00 +0100 Subject: [PATCH] Added licensing information. --- LICENSE | 21 ++++ NOTICE | 116 ++++++++++++++++++ hpcns/errors.py | 6 + hpcns/tutorials/distribution/decorator_0.py | 10 +- hpcns/tutorials/distribution/decorator_1.py | 16 ++- hpcns/tutorials/distribution/decorator_3.py | 10 +- hpcns/tutorials/distribution/decorator_4.py | 16 ++- hpcns/tutorials/dl/mnist_data_distributed.py | 37 +++--- hpcns/tutorials/dl/mnist_data_sequential.py | 25 ++-- hpcns/tutorials/dl/mnist_epoch_distributed.py | 39 +++--- hpcns/tutorials/dl/mnist_sequential.py | 31 +++-- hpcns/tutorials/mpi/broadcast.py | 8 +- hpcns/tutorials/mpi/info_only.py | 8 +- hpcns/tutorials/mpi/spmd.py | 14 ++- hpcns/utils/data.py | 6 + hpcns/utils/distribution.py | 6 + 16 files changed, 290 insertions(+), 79 deletions(-) create mode 100644 LICENSE create mode 100644 NOTICE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8fbce06 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Forschungszentrum Juelich GmbH + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..872ddc2 --- /dev/null +++ b/NOTICE @@ -0,0 +1,116 @@ +This project includes derived work from the following: + +The SLNS library of Python utilities +Copyright (c) 2019 Forschungszentrum Juelich GmbH + +Licensed under The MIT License. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Getting started with Deep Learning on Supercomputers +Copyright (c) 2019 Forschungszentrum Juelich GmbH + +Licensed under The MIT License. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Horovod +Copyright 2018 Uber Technologies, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Tensorflow +Copyright 2016 The TensorFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Keras +All contributions by François Chollet: +Copyright (c) 2015 - 2019, François Chollet. +All rights reserved. + +All contributions by Google: +Copyright (c) 2015 - 2019, Google, Inc. +All rights reserved. + +All contributions by Microsoft: +Copyright (c) 2017 - 2019, Microsoft, Inc. +All rights reserved. + +All other contributions: +Copyright (c) 2015 - 2019, the respective contributors. +All rights reserved. + +Licensed under The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/hpcns/errors.py b/hpcns/errors.py index e22c546..12c513a 100644 --- a/hpcns/errors.py +++ b/hpcns/errors.py @@ -1,3 +1,9 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH +# This code is licensed under MIT license (see the LICENSE file for details) +# This code is derived from "The SLNS library of Python utilities", +# which is licensed under the MIT License (see the NOTICE file for details). + + """ A collection of custom error classes for the slns module. """ diff --git a/hpcns/tutorials/distribution/decorator_0.py b/hpcns/tutorials/distribution/decorator_0.py index c31bed6..7d89df4 100644 --- a/hpcns/tutorials/distribution/decorator_0.py +++ b/hpcns/tutorials/distribution/decorator_0.py @@ -1,6 +1,12 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from "The SLNS library of Python utilities", +# which is licensed under the MIT License (see the NOTICE file for details). + """ -Illustration of a sequential program that defines and uses -a function to read a list of filenames from the disk. + Illustration of a sequential program that defines and uses + a function to read a list of filenames from the disk. + """ import os diff --git a/hpcns/tutorials/distribution/decorator_1.py b/hpcns/tutorials/distribution/decorator_1.py index e768d83..f5161ef 100644 --- a/hpcns/tutorials/distribution/decorator_1.py +++ b/hpcns/tutorials/distribution/decorator_1.py @@ -1,10 +1,16 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from "The SLNS library of Python utilities", +# which is licensed under the MIT License (see the NOTICE file for details). + """ -This program demonstrates how a function that reads and returns -a list of filenames can be easily decorated to automatically -distribute the list across multiple MPI ranks. + This program demonstrates how a function that reads and returns + a list of filenames can be easily decorated to automatically + distribute the list across multiple MPI ranks. + + The hpcns.utils.distribution.FilenamesDistributor is used as + the decorator. -The hpcns.utils.distribution.FilenamesDistributor is used as -the decorator. """ import os diff --git a/hpcns/tutorials/distribution/decorator_3.py b/hpcns/tutorials/distribution/decorator_3.py index 4b8cce6..c147847 100644 --- a/hpcns/tutorials/distribution/decorator_3.py +++ b/hpcns/tutorials/distribution/decorator_3.py @@ -1,6 +1,12 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from "The SLNS library of Python utilities", +# which is licensed under the MIT License (see the NOTICE file for details). + """ -Illustration of a sequential program that uses a library -function to read a list of filenames from the disk. + Illustration of a sequential program that uses a library + function to read a list of filenames from the disk. + """ import os diff --git a/hpcns/tutorials/distribution/decorator_4.py b/hpcns/tutorials/distribution/decorator_4.py index 1b598e7..a91ae44 100644 --- a/hpcns/tutorials/distribution/decorator_4.py +++ b/hpcns/tutorials/distribution/decorator_4.py @@ -1,10 +1,16 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from "The SLNS library of Python utilities", +# which is licensed under the MIT License (see the NOTICE file for details). + """ -This program demonstrates how a library function can be decorated -using the hpcns.utils.distribution.FilenamesDistributor. + This program demonstrates how a library function can be decorated + using the hpcns.utils.distribution.FilenamesDistributor. + + The syntax for decoration presented in this program can be useful not + only for library functions, but also in situations where a reference + to the MPI object is not available at the time of function definition. -The syntax for decoration presented in this program can be useful not -only for library functions, but also in situations where a reference -to the MPI object is not available at the time of function definition. """ import os diff --git a/hpcns/tutorials/dl/mnist_data_distributed.py b/hpcns/tutorials/dl/mnist_data_distributed.py index 44d0f52..5eabc49 100644 --- a/hpcns/tutorials/dl/mnist_data_distributed.py +++ b/hpcns/tutorials/dl/mnist_data_distributed.py @@ -1,25 +1,30 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH +# This code is licensed under MIT license (see the LICENSE file for details) +# This code is derived from "Getting started with Deep Learning on Supercomputers", +# which is licensed under the MIT License (see the NOTICE file for details). + """ -This program is an adaptation of the following code sample: -https://github.com/horovod/horovod/blob/master/examples/keras_mnist.py. -The modified program distributes the partitioned MNIST data across -multiple MPI ranks for truly data distributed training of a shallow ANN -for handwritten digit classification. + This program is an adaptation of the following code sample: + https://github.com/horovod/horovod/blob/master/examples/keras_mnist.py. + The modified program distributes the partitioned MNIST data across + multiple MPI ranks for truly data distributed training of a shallow ANN + for handwritten digit classification. -The Horovod framework is used for seamless distributed training. Instead -of distributing epochs, this program distributes data amongst the ranks, -so that each rank contributes training based on its local subset of the -training data. + The Horovod framework is used for seamless distributed training. Instead + of distributing epochs, this program distributes data amongst the ranks, + so that each rank contributes training based on its local subset of the + training data. -To run this sample from the repository root, use the following -command on your workstation/laptop equipped with a GPU: + To run this sample from the repository root, use the following + command on your workstation/laptop equipped with a GPU: -mpirun -np 1 python -u -m hpcns.tutorials.dl.mnist_data_distributed + mpirun -np 1 python -u -m hpcns.tutorials.dl.mnist_data_distributed -If you have more than one GPU on your system, you can increase the -number of ranks accordingly. + If you have more than one GPU on your system, you can increase the + number of ranks accordingly. -To run this sample on JUWELS, please use the provided 'submit_parallel_job_juwels.sh' -script. Please make sure the srun command in the script points to this file. + To run this sample on JUWELS, please use the provided 'submit_parallel_job_juwels.sh' + script. Please make sure the srun command in the script points to this file. """ diff --git a/hpcns/tutorials/dl/mnist_data_sequential.py b/hpcns/tutorials/dl/mnist_data_sequential.py index e501725..7c08d97 100644 --- a/hpcns/tutorials/dl/mnist_data_sequential.py +++ b/hpcns/tutorials/dl/mnist_data_sequential.py @@ -1,18 +1,23 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH. +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from Tensorflow tutorials, which is licensed under the Apache License, +# Version 2.0 (see the NOTICE file for details). + """ -This program is an adaptation of the code sample available at -https://www.tensorflow.org/tutorials/. The modified program uses -the MNIST data partitioned into multiple files to train a -shallow ANN for handwritten digit classification. + This program is an adaptation of the code sample available at + https://www.tensorflow.org/tutorials/. The modified program uses + the MNIST data partitioned into multiple files to train a + shallow ANN for handwritten digit classification. -Custom functions are defined to read the partitioned datasets. + Custom functions are defined to read the partitioned datasets. -To run this sample from the repository root, use the following -command on your workstation/laptop equipped with a GPU: + To run this sample from the repository root, use the following + command on your workstation/laptop equipped with a GPU: -python -u -m hpcns.tutorials.dl.mnist_data_sequential + python -u -m hpcns.tutorials.dl.mnist_data_sequential -To run this sample on JUWELS, please use the provided 'submit_sequential_job_juwels.sh' -script. Please make sure the srun command in the script points to this file. + To run this sample on JUWELS, please use the provided 'submit_sequential_job_juwels.sh' + script. Please make sure the srun command in the script points to this file. """ diff --git a/hpcns/tutorials/dl/mnist_epoch_distributed.py b/hpcns/tutorials/dl/mnist_epoch_distributed.py index 034777f..84e3849 100644 --- a/hpcns/tutorials/dl/mnist_epoch_distributed.py +++ b/hpcns/tutorials/dl/mnist_epoch_distributed.py @@ -1,27 +1,32 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH. +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from Horovod, which is licensed under the Apache License, +# Version 2.0 (see the NOTICE file for details). + """ -This program is an adaptation of the following code sample: -https://github.com/horovod/horovod/blob/master/examples/keras_mnist.py. -The program creates and trains a shallow ANN for handwritten digit -classification using the MNIST dataset. + This program is an adaptation of the following code sample: + https://github.com/horovod/horovod/blob/master/examples/keras_mnist.py. + The program creates and trains a shallow ANN for handwritten digit + classification using the MNIST dataset. -The Horovod framework is used for seamless distributed training. In this -example epochs are distributed across the MPI ranks, not data. + The Horovod framework is used for seamless distributed training. In this + example epochs are distributed across the MPI ranks, not data. -The code has been adapted to load the MNIST dataset from the -'data' directory in the repository root, rather than -downloading it from the Internet. The correspondingly updated -lines code are preceded by a comment beginning with [HPCNS]. + The code has been adapted to load the MNIST dataset from the + 'data' directory in the repository root, rather than + downloading it from the Internet. The correspondingly updated + lines code are preceded by a comment beginning with [HPCNS]. -To run this sample from the repository root, use the following -command on your workstation/laptop equipped with a GPU: + To run this sample from the repository root, use the following + command on your workstation/laptop equipped with a GPU: -mpirun -np 1 python -u -m hpcns.tutorials.dl.mnist_epoch_distributed + mpirun -np 1 python -u -m hpcns.tutorials.dl.mnist_epoch_distributed -If you have more than one GPU on your system, you can increase the -number of ranks accordingly. + If you have more than one GPU on your system, you can increase the + number of ranks accordingly. -To run this sample on JUWELS, please use the provided 'submit_parallel_job_juwels.sh' -script. Please make sure the srun command in the script points to this file. + To run this sample on JUWELS, please use the provided 'submit_parallel_job_juwels.sh' + script. Please make sure the srun command in the script points to this file. """ diff --git a/hpcns/tutorials/dl/mnist_sequential.py b/hpcns/tutorials/dl/mnist_sequential.py index 5ef36b9..cbe6a0b 100644 --- a/hpcns/tutorials/dl/mnist_sequential.py +++ b/hpcns/tutorials/dl/mnist_sequential.py @@ -1,21 +1,26 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH. +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from Tensorflow tutorials, which is licensed under the Apache License, +# Version 2.0 (see the NOTICE file for details). + """ -This program is an adaptation of the code sample available at -https://www.tensorflow.org/tutorials/. The program creates -and trains a shallow ANN for handwritten digit classification -using the MNIST dataset. + This program is an adaptation of the code sample available at + https://www.tensorflow.org/tutorials/. The program creates + and trains a shallow ANN for handwritten digit classification + using the MNIST dataset. -The code has been adapted to load the MNIST dataset from the -'data' directory in the repository root, rather than -downloading it from the Internet. The correspondingly updated -lines of code are preceded by a comment beginning with [HPCNS]. + The code has been adapted to load the MNIST dataset from the + 'data' directory in the repository root, rather than + downloading it from the Internet. The correspondingly updated + lines of code are preceded by a comment beginning with [HPCNS]. -To run this sample from the repository root, use the following -command on your workstation/laptop equipped with a GPU: + To run this sample from the repository root, use the following + command on your workstation/laptop equipped with a GPU: -python -u -m hpcns.tutorials.dl.mnist_sequential + python -u -m hpcns.tutorials.dl.mnist_sequential -To run this sample on JUWELS, please use the provided 'submit_sequential_job_juwels.sh' -script. Please make sure the srun command in the script points to this file. + To run this sample on JUWELS, please use the provided 'submit_sequential_job_juwels.sh' + script. Please make sure the srun command in the script points to this file. """ diff --git a/hpcns/tutorials/mpi/broadcast.py b/hpcns/tutorials/mpi/broadcast.py index e4ab549..51ee147 100644 --- a/hpcns/tutorials/mpi/broadcast.py +++ b/hpcns/tutorials/mpi/broadcast.py @@ -1,6 +1,10 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH +# This code is licensed under MIT license (see the LICENSE file for details) + """ -This program demonstrates how to use mpi4py to broadcast data -from the root to all other ranks + This program demonstrates how to use mpi4py to broadcast data + from the root to all other ranks. + """ # Import the package and initialize MPI diff --git a/hpcns/tutorials/mpi/info_only.py b/hpcns/tutorials/mpi/info_only.py index 234ffd2..473f057 100644 --- a/hpcns/tutorials/mpi/info_only.py +++ b/hpcns/tutorials/mpi/info_only.py @@ -1,6 +1,10 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH +# This code is licensed under MIT license (see the LICENSE file for details) + """ -This program demonstrates how to use mpi4py to retrieve basic -information from the default MPI communicator. + This program demonstrates how to use mpi4py to retrieve basic + information from the default MPI communicator. + """ # Import the package and initialize MPI diff --git a/hpcns/tutorials/mpi/spmd.py b/hpcns/tutorials/mpi/spmd.py index cf4a864..40a525b 100644 --- a/hpcns/tutorials/mpi/spmd.py +++ b/hpcns/tutorials/mpi/spmd.py @@ -1,10 +1,14 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH +# This code is licensed under MIT license (see the LICENSE file for details) + """ -This program demonstrates the Single Program Multiple Data (SPMD) -concept. The same program is executed by two ranks, but each -rank operates on a different chunk of data. + This program demonstrates the Single Program Multiple Data (SPMD) + concept. The same program is executed by two ranks, but each + rank operates on a different chunk of data. + + Exercise: What happens when you run this program with more than + two ranks? Can you explain the behavior? -Exercise: What happens when you run this program with more than - two ranks? Can you explain the behavior? """ # Import the package and initialize MPI diff --git a/hpcns/utils/data.py b/hpcns/utils/data.py index 904a37c..58bb6c1 100644 --- a/hpcns/utils/data.py +++ b/hpcns/utils/data.py @@ -1,3 +1,9 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH +# This code is licensed under MIT license (see the LICENSE file for details) +# This code is derived from Getting started with Deep Learning on Supercomputers, +# which is licensed under the MIT License (see the NOTICE file for details). + + """ A collections of utilities for data manipulation. diff --git a/hpcns/utils/distribution.py b/hpcns/utils/distribution.py index f92891f..fa4456d 100644 --- a/hpcns/utils/distribution.py +++ b/hpcns/utils/distribution.py @@ -1,3 +1,9 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH +# This code is licensed under MIT license (see the LICENSE file for details) +# This code is derived from "The SLNS library of Python utilities", +# which is licensed under the MIT License (see the NOTICE file for details). + + """ A collection of utilities for seamless distribution of data across multiple MPI ranks. -- GitLab