diff --git a/.gitignore b/.gitignore index 7f5ba6e51700856088e0b4379f4163ef9baf1bb6..340d044d241882508ed86b968f390a76bc7a8881 100644 --- a/.gitignore +++ b/.gitignore @@ -117,4 +117,6 @@ mnist_convnet_model/ # Error and output files from the supercomputers *.er -*.out \ No newline at end of file +*.out + +horovod/keras/mnist_data_distributed.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..78d9698282c7162167dfe5930388a7d8eae47117 --- /dev/null +++ b/LICENSE @@ -0,0 +1,26 @@ +All contents of this work, except for the contents of the "datasets/mnist" +sub-directory are licensed under The MIT License (see license details below). +Contents of the "datasets/mnist" sub-directory are licensed under the Creative +Commons Attribution-ShareAlike 3.0 Unported License (see "datasets/mnist/LICENSE"). + +MIT License + +Copyright (c) 2019 Forschungszentrum Juelich GmbH + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..22a9d695eaaf27a76217db1873fd8544854f4451 --- /dev/null +++ b/NOTICE @@ -0,0 +1,71 @@ +This project includes derived work from the following: + + +Horovod +Copyright 2018 Uber Technologies, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Tensorflow +Copyright 2016 The TensorFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Keras +All contributions by François Chollet: +Copyright (c) 2015 - 2019, François Chollet. +All rights reserved. + +All contributions by Google: +Copyright (c) 2015 - 2019, Google, Inc. +All rights reserved. + +All contributions by Microsoft: +Copyright (c) 2017 - 2019, Microsoft, Inc. +All rights reserved. + +All other contributions: +Copyright (c) 2015 - 2019, the respective contributors. +All rights reserved. + +Licensed under The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/datasets/mnist/LICENSE b/datasets/mnist/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..65f46d5ba1251569045f48757b724f95dfa8a488 --- /dev/null +++ b/datasets/mnist/LICENSE @@ -0,0 +1,3 @@ +The mnist directory is licensed under the Creative Commons Attribution-ShareAlike 3.0 Unported License. +To view a copy of this license, visit http://creativecommons.org/licenses/by-sa/3.0/ or send +a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. diff --git a/datasets/mnist/NOTICE b/datasets/mnist/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..ff1a1c31ca00ff94110cbcfc389c8491412c9768 --- /dev/null +++ b/datasets/mnist/NOTICE @@ -0,0 +1,12 @@ +The contents of the mnist directory are derived from the MNIST dataset: + +Yann LeCun (Courant Institute, NYU) and Corinna Cortes (Google Labs, New York) +hold the copyright of MNIST dataset (http://yann.lecun.com/exdb/mnist), which is +a derivative work from original NIST datasets. MNIST dataset is made available +under the terms of the Creative Commons Attribution-Share Alike 3.0 license. The +license details are available via the following URL: + +http://creativecommons.org/licenses/by-sa/3.0/ + +Individual images and labels have not been changed in this work. The only changes +made are to the dataset format. diff --git a/horovod/keras/mnist.py b/horovod/keras/mnist.py index e31aa8a009e2fee923023dc18d5e5979c6d70203..0c46a771047d0adfa6d61017176d6ef2c6de0d67 100644 --- a/horovod/keras/mnist.py +++ b/horovod/keras/mnist.py @@ -1,3 +1,8 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH. +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from https://github.com/horovod/horovod/blob/master/examples/keras_mnist.py, +# which is licensed under the Apache License, Version 2.0 (see the NOTICE file for details). + from __future__ import print_function import os import sys diff --git a/horovod/keras/mnist_advanced.py b/horovod/keras/mnist_advanced.py index bf52fddb7e312d10e4a4bd7a0da7f9ab87368ff3..ba60b6d64d61feac9e19e0af213b5134087f887b 100644 --- a/horovod/keras/mnist_advanced.py +++ b/horovod/keras/mnist_advanced.py @@ -1,3 +1,9 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH. +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from https://github.com/horovod/horovod/blob/master/examples/keras_mnist_advanced.py, +# which is licensed under the Apache License, Version 2.0 (see the NOTICE file for details). + + from __future__ import print_function import os import sys diff --git a/horovod/tensorflow/mnist.py b/horovod/tensorflow/mnist.py index 8099f1c22a3927c9b38adb7375a60f752b28acf2..3c780accef6f40d6bb3c95196f4feb69aafb96fe 100644 --- a/horovod/tensorflow/mnist.py +++ b/horovod/tensorflow/mnist.py @@ -1,17 +1,7 @@ -# Copyright 2017 Uber Technologies, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== +# Copyright (c) 2019 Forschungszentrum Juelich GmbH. +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from https://github.com/horovod/horovod/blob/master/examples/tensorflow_mnist.py, +# which is licensed under the Apache License, Version 2.0 (see the NOTICE file for details). import os import sys diff --git a/horovod/tensorflow/mnist_estimator.py b/horovod/tensorflow/mnist_estimator.py index 861de50549b470685462a688643dbf3cd8e86288..792c0577f5e6324eddca6e54d23d6669a21ab3c4 100644 --- a/horovod/tensorflow/mnist_estimator.py +++ b/horovod/tensorflow/mnist_estimator.py @@ -1,17 +1,8 @@ -# Copyright 2018 Uber Technologies, Inc. All Rights Reserved. -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright (c) 2019 Forschungszentrum Juelich GmbH. +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from https://github.com/horovod/horovod/blob/master/examples/tensorflow_mnist_estimator.py, +# which is licensed under the Apache License, Version 2.0 (see the NOTICE file for details). + """Convolutional Neural Network Estimator for MNIST, built with tf.layers.""" from __future__ import absolute_import diff --git a/keras/mnist.py b/keras/mnist.py index c1831694e02ccc3d1546fb8955f5798474c870e6..9fc93f2a56f0aa38318a114e151b7e2a6c2ea15c 100644 --- a/keras/mnist.py +++ b/keras/mnist.py @@ -1,3 +1,9 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH. +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from https://github.com/keras-team/keras/blob/master/examples/mnist_cnn.py, +# which is also licensed under The MIT License (see the NOTICE file for details). + + """Trains a simple convnet on the MNIST dataset. Gets to 99.25% test accuracy after 12 epochs diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..79144dccd44dd967fb51438abbcd9589c6d81937 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,24 @@ +absl-py==0.8.0 +astor==0.8.0 +cffi==1.12.3 +cloudpickle==1.2.1 +gast==0.3.1 +grpcio==1.23.0 +h5py==2.10.0 +Markdown==3.1.1 +mock==3.0.5 +mpi4py==3.0.2 +numpy==1.17.2 +protobuf==3.9.1 +psutil==5.6.3 +pycparser==2.19 +six==1.12.0 +Werkzeug==0.15.6 +Keras-Applications==1.0.8 +Keras-Preprocessing==1.1.0 +tensorboard==1.13.1 +tensorflow-estimator==1.13.0 +tensorflow-gpu==1.13.1 +termcolor==1.1.0 +keras==2.3.1 +horovod==0.16.2 \ No newline at end of file diff --git a/tensorflow/mnist.py b/tensorflow/mnist.py index 7ba4bdc5fb1b25bc0744308a26ad22856f729c26..30477e153e9c59d2a151b90686049b39885155e4 100644 --- a/tensorflow/mnist.py +++ b/tensorflow/mnist.py @@ -1,17 +1,7 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== +# Copyright (c) 2019 Forschungszentrum Juelich GmbH. +# This code is licensed under MIT license (see the LICENSE file for details). +# This code is derived from https://github.com/tensorflow/models/blob/master/tutorials/image/mnist/convolutional.py, +# which is licensed under the Apache License, Version 2.0 (see the NOTICE file for details). """Simple, end-to-end, LeNet-5-like convolutional MNIST model example. diff --git a/utils/data_utils.py b/utils/data_utils.py index 048885633c636c08a14ce04e3519f7ead932a7e1..bab6e035ea82e22108998da561519a70eea94eac 100644 --- a/utils/data_utils.py +++ b/utils/data_utils.py @@ -1,3 +1,6 @@ +# Copyright (c) 2019 Forschungszentrum Juelich GmbH. +# This code is licensed under MIT license (see the LICENSE file for details). + """ A collections of utilities for data manipulation.