From 9e3f20910369eea6e281ea5819f09862f83ddbb6 Mon Sep 17 00:00:00 2001 From: Fahad Khalid <f.khalid@fz-juelich.de> Date: Tue, 3 Dec 2019 08:59:36 +0100 Subject: [PATCH 1/3] Package renamed to hpc4neuro. --- README.md | 30 ++-- doc/conf.py | 4 +- doc/html/.buildinfo | 2 +- doc/html/.doctrees/environment.pickle | Bin 11574 -> 11642 bytes doc/html/.doctrees/index.doctree | Bin 16331 -> 16448 bytes doc/html/_sources/index.rst.txt | 12 ++ doc/html/genindex.html | 132 ++++++++++++++++++ doc/html/index.html | 28 ++-- doc/html/objects.inv | 5 +- doc/html/search.html | 111 +++++++++++++++ doc/html/searchindex.js | 1 + doc/index.rst | 8 +- doc/text/index.txt | 10 +- {hpc4ns => hpc4neuro}/__init__.py | 0 {hpc4ns => hpc4neuro}/distribution.py | 4 +- {hpc4ns => hpc4neuro}/errors.py | 0 {hpc4ns => hpc4neuro}/examples/__init__.py | 0 .../examples/distribution/__init__.py | 0 .../dynamic_filenames_decorator.py | 6 +- .../distribution/sequential_filenames.py | 2 +- .../static_filenames_decorator.py | 6 +- pyproject.toml | 6 +- tests/test_distribution.py | 14 +- tests/{test_hpc4ns.py => test_hpc4neuro.py} | 2 +- 24 files changed, 321 insertions(+), 62 deletions(-) create mode 100644 doc/html/_sources/index.rst.txt create mode 100644 doc/html/genindex.html create mode 100644 doc/html/search.html create mode 100644 doc/html/searchindex.js rename {hpc4ns => hpc4neuro}/__init__.py (100%) rename {hpc4ns => hpc4neuro}/distribution.py (98%) rename {hpc4ns => hpc4neuro}/errors.py (100%) rename {hpc4ns => hpc4neuro}/examples/__init__.py (100%) rename {hpc4ns => hpc4neuro}/examples/distribution/__init__.py (100%) rename {hpc4ns => hpc4neuro}/examples/distribution/dynamic_filenames_decorator.py (83%) rename {hpc4ns => hpc4neuro}/examples/distribution/sequential_filenames.py (91%) rename {hpc4ns => hpc4neuro}/examples/distribution/static_filenames_decorator.py (84%) rename tests/{test_hpc4ns.py => test_hpc4neuro.py} (62%) diff --git a/README.md b/README.md index 9247ef6..01e2016 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# The HPC4NS library of Python utilities +# The hpc4neuro library of Python utilities This project brings together a collection of utilities that have been factored out from different projects. In certain cases we need a specific functionality @@ -8,18 +8,18 @@ to this collection in the hope that it may be useful to others. ## Setup and Requirements -The `hpc4ns` package requires `Python 3.6` or above. To install, please +The `hpc4neuro` package requires `Python 3.6` or above. To install, please use the following command: ``` -python -m pip install git+https://gitlab.version.fz-juelich.de/hpc4ns/hpc4ns_utils.git +python -m pip install git+https://gitlab.version.fz-juelich.de/hpc4neuro/hpc4neuro_utils.git ``` ## Available modules The following modules are available at this time: -### 1. `hpc4ns.distribution` +### 1. `hpc4neuro.distribution` **Note:** This module requires `mpi4py`. To install `mpi4py`, please follow installation instructions available @@ -41,13 +41,13 @@ networks in a data-parallel fashion using Horovod. Even though Horovod hides the intricate details of distributed training, proper distribution of training/validation data is only possible via MPI programming. -The `hpc4ns.distribution` module provides a high-level interface for data distribution +The `hpc4neuro.distribution` module provides a high-level interface for data distribution with MPI, without the explicit need to write MPI code on the user's part. The following examples show what the module does, and how it can be useful. #### Examples -**Note:** All examples are available in the `hpc4ns.examples.distribution` package. +**Note:** All examples are available in the `hpc4neuro.examples.distribution` package. Consider the following code that defines a simple function which returns a list of files read from a given directory. @@ -58,28 +58,28 @@ import os def get_filenames(path): return os.listdir(path) -# List of the filenames in the 'hpc4ns' directory -filenames = get_filenames('./hpc4ns') +# List of the filenames in the 'hpc4neuro' directory +filenames = get_filenames('./hpc4neuro') ``` ##### Distributed case 1: Using the static decorator syntax Now consider a scenario in which we need to run this code on multiple processors across multiple nodes in a cluster, and distribute the returned filenames across all the processes. -The following example shows how the `hpc4ns.distribution` module can help with that. +The following example shows how the `hpc4neuro.distribution` module can help with that. ``` import os from mpi4py import MPI -from hpc4ns.distribution import DataDistributor +from hpc4neuro.distribution import DataDistributor @DataDistributor(MPI.COMM_WORLD) def get_filenames(path): return os.listdir(path) # List of rank-local file names -filenames = get_filenames('./hpc4ns') +filenames = get_filenames('./hpc4neuro') ``` `DataDistributor` decorates the `get_filenames` function such that calling @@ -98,7 +98,7 @@ of `DataDistributor` in such cases. import os from mpi4py import MPI -from hpc4ns.distribution import DataDistributor +from hpc4neuro.distribution import DataDistributor # Initialize the decorator dist_decorator = DataDistributor(MPI.COMM_WORLD) @@ -107,7 +107,7 @@ dist_decorator = DataDistributor(MPI.COMM_WORLD) get_rank_local_filenames = dist_decorator(os.listdir) # Use the decorated function to get the rank-local list of filenames -filenames = get_rank_local_filenames('./hpc4ns') +filenames = get_rank_local_filenames('./hpc4neuro') ``` #### Support for graceful application shutdown @@ -127,7 +127,7 @@ static and dynamic decoration syntax: #### API documentation -API documentation for `hpc4ns.distribution` is available [here](doc/text/index.txt). +API documentation for `hpc4neuro.distribution` is available [here](doc/text/index.txt). ## Notes for contributors @@ -139,6 +139,8 @@ API documentation for `hpc4ns.distribution` is available [here](doc/text/index.t 4. If you use [`poetry`](https://github.com/sdispater/poetry), run `poetry install` to install all the required dependencies + + ### Test setup [`pytest`](https://docs.pytest.org/en/latest/) is required for running and working with test code diff --git a/doc/conf.py b/doc/conf.py index 5ebdf96..5c165ba 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -17,7 +17,7 @@ sys.path.insert(0, os.path.abspath('..')) # -- Project information ----------------------------------------------------- -project = 'HPC4NS Python Utilities' +project = 'hpc4neuro Python Utilities' copyright = '2019, Forschungszentrum Juelich GmbH' author = 'Forschungszentrum Juelich GmbH' @@ -50,4 +50,4 @@ html_theme = 'alabaster' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] \ No newline at end of file +html_static_path = ['html/_static'] \ No newline at end of file diff --git a/doc/html/.buildinfo b/doc/html/.buildinfo index b5d32ce..6beb825 100644 --- a/doc/html/.buildinfo +++ b/doc/html/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: d3256f5f0c4fd21a832ef83fc1ba514a +config: 8b6399c0ddf0c03d19200c9c922f1102 tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/doc/html/.doctrees/environment.pickle b/doc/html/.doctrees/environment.pickle index 99464a1661f7736f2754fc4086187d8b5bec1506..16192fb6d04b438a18a5f76ad02335dbd3afeb9f 100644 GIT binary patch delta 2000 zcmdlM^(%^{fn{p`MwU!QE~$)yWRtwq(xUv$C5#npToMTWXZDXwT$*tHWF2nx$wJ)1 zNJ5)+xFuLMW#CE*DoZl*^GZuHb23XZQ;YRVQY)tPAlZ;EpunVxumMf^lpblgNt2fe z6(GAsR`@w15`Qwc=sqq<gz5{TZ@J{*+|5VC1W;7p7T4rLGMif>1<9<<WfHqtm@+&! zJIHD=^X4!yFt|HjuWS`bedaZ}T*1l;B+@@cqen(hPakfDeoAI>Nl|7}X-Q^&o?b!a zlnKQ>9GQ8^Ii)G7DO0khF)=V?&7Ax~K~)*doX*6+(8H3FnOrhu@|4yoK~s9zN-|4w zQj4e5PRYudY@p~Mu?nmtB((yhq{jem(_|ZE1OD|ONj-$vK}7?`wUa+7Mj5VVVqhpu zD(>OR$<IqiG85#0wM+~Q#XT$~`N<G_Kw_&V*D6WrgUrkD%-X=jzz{nn3v4xKPG)gQ zd}c{%?v%+>G#jVXPSMDMDVuyu$<zhLLDo~0nwDCWnwOl4sRyELGsxkno~Z{Jl(hro zn4KWvF4zZ&S-Y4R7(kZpf#}=^p$<T(LlEi+m;!}#NorAEV$PI~9=4*?w9=x?DOsR+ z>fuVvOU}qI%1g{mos!{Mnsp3h!)*`&w(ABH14Dgj5-FBJ&A0|P;}Y1tS!b9S7|?<V z?6|C}ATP6*<R_ODrD6*gD{z>T;9ItW#G=%^5>P@}qSDX9!OXw_b=0@X_NpQEV4r;k zyNV2toksImDhUw*c199I)&;m3XDM+8C>`^c<R{1HmFC8$=9Ls>!ZJvSniFHr<n3zd zjJcEL)s@6^nHU(n8A^LNk~2#ZL0J%-TyrN!sN0Dbfx@3HxH2~>KL?R4cTQfb-fITa z$CL%i5gJ+M%%C*a!=9U;QVPwDFeNOxi6t3eW6dWQXh>*+l2>PO7EGF{pb{bpHbJ9D z4Usw5X~;{0RQIqZ=Oh*vPidRdgD5yAuhWp7q{;0MR=}2<T9T2U0#cz2*RO|Cw73=( z<rjG*=B4DM7D3ZE!dN4?rpY0i+J<OmmS#nQ9S3(ab5U^#lB2V7CvVVHU^Jh6QB#(& zYVvzcUB>FkvRdW>^~?+mSq&hf5kxdiPSH|Jsbywh(8xlS>f!MP2b_XuUP@}kl#(fp zQ_xgobulwAv`=Z9lGHw>hpjj@u_!rXO4bBs28ImJEDR$jzta-4MztbqBEo=DQ1;Hx z%SlX1&4I*U707)cBEz$7N>;~Ydu=frkl|1p+NPAkJYU+wlL>Mpa<T&FR}GCGw&KjZ z^qkZwJ%*DfYRg-JDnpb=(43N$40f?!$rO#O1<VW#Jy!AY$%#2R@$nj(3L1!5Sf?Q? ztpJHs%_&)FXqxmU%jn2BU^6}wO{Gg}MP^An$bzE8yzF9BONw-~8FMDj({X29HTjuN F001sklDPl? delta 1869 zcmewrwJnOJfn}=pMwU!Qc5#mYXA{5R&E<?0Z0w>?&QJD_Ozdh9&SVR2^~uuQ!Z7}3 z3vLNkWyy?!WRtw&g36MN{JheV%$&@U%+zANlGKVRJuo9H1r(U%MX@TJ(jx&ieV<SP zJ5<eP3*qOCFgHxr6y3)z237P?^ewv#guVHNm;gc%lY}O_1jL5PrV=SI&6^iX>}KKb z;Ze{~FDS~-N=+_N*POCBK~{^IH<yWl!QJtCsdL7sU3rtID_9AEMEa*_^hoOI=|hcr zrXbHYor!@VYu03LMO9@ma|ROwLk~+zW^&1t$x~XV1WoB-E6FU$NiCjIJ0&Z3a<HO< z=xVT%kkks0k{(@%gD0<6G~n3)lF^0=yizn^TsK)-DJp6W69YqOQgIJgPJUiGl4&5T z*D*0L6!)-{<R?RH0EvMu;7%&d$w@7V&&ezX*`wJwrFM!&7RZJS&#aA13=FYTvQ|$% zuOz_%6Px^A$<z_Xfhqtwh_fg)Ewv~$FFAF}<SAIq*#c6A<dF!FDOo!~ZrKGQ?twj! zn6;aUfdS--y%5v)L#Trg>M(>l3Z{BEGV@AOi}Dh4rgZeM6{V(?7G;9M?}M^P{c(_y zcR&Q#@S7wXUYZ3q;yT=j%U~n2&N4ADVD;ZMkpI|A@{>!7Qd50#g#rn_V=G83O3f<) z`RIa5KMyA}0|V4i-zR6QhSY;S_66)JQbG-C#u+q^rIHZvU?Y+kvM$2SI7f*yK*^TB zBtJPmuQWG4HLs*76P6Pes5vp_PJXPG&X_mZTU|*!kBNc7o1wIaBRR7qu_QA;51d}} zCby{Di5G*ypDnmDHz_{{5&F9(-&F54gXv?+0%ZV=EDL5(8tY-t%}*(XW;>V?mfXaW z46v~lljmqisDZL{XK@xxnyH`?A_+D@qemH<&F^Z+i-DB(uqNju78g%xo6@5Lmy%V| zi;quE%*lz5pOOVu#g>~|l98VRQl<b^)}tvU0hL9V0V_p%G_?)UG?r#Xf!zRi8go%` z36k5g@+QB~RA982%&8^ISUp)?OP8@`a;TQMKm#)aLslb*XaW(<ljmru)z>jIFlc0< zO7-ygg2P9_GcP5zVoJ%B#wlnjvbvcW7}}?_O-X8>(!*Aqnpl*aF(qpvGXq10XBJi? zxr4!q6ao^{Q;|&MM!09vWFKuA(`t~@KtzUT+mx(MW(I~H?t)6)-29YGuw%f&Ad8^3 zwM{9_%9}h_TT&PlOtDimG<w*IGxO4OQm6FjO+KkDZ&Iw6l383*l$lftN;sNRvQog# z^edU7k+l$HzBw$8G&B`7ppkM{Lw2&g4wqm$np*A2Q93eK7)EBHDRWG%$SjEmMSM|W dUUo5(!9AK%lP~B9GUiTxq~p%Gda{;o0002yQmg;~ diff --git a/doc/html/.doctrees/index.doctree b/doc/html/.doctrees/index.doctree index 12abcf84852bce0f304ede6f5f1f397f603d6efa..b696c717a500b882528add43a6fc200bc7d887d0 100644 GIT binary patch delta 703 zcmX?If1rVpwSi@-sl!Cp3NF2jf@G7t)Y78-i7jSa25{cSgCUHJhLgpZ<mEt;dMTO3 zB}JJ@r6rm9d74vt)Zhv?`!d-uGFDGk<TYjishpgzstjTHGRaS7WKqx(hpR;~MQ=(5 zd+d}9(H?~SW>=PpOk84c^_%aqF)`yZdviEf1EZK#e0*|ZPELHhhNglBT;pV4CRq`M z+|-hc{1nY8J^Ba<TOp0f4m=MSTd+9TR!DR6BHkEAWEZgV-)F+6c(bk06fUkdguxSK zRVM$D>EqHx7SciGs&0NM>&C{YH`zc%25Yb?BQ*QzOHHm**~3_Y#i1a%wW@-ogzO_# zeiVQGP^)CbY4#HhZ$_8N2AblKi1gE!l2#}#Ni4}sMue#mvQxDuFV(!y*pAhyI-5DQ hZ=u-zN{5>Xo8rxW`YSn*Qsm@+<_1XI&88O9*a0sm@@oJ9 delta 588 zcmX@mz<9czwSi^o0sD!p73|s`0nR3V!4uof*mWVyjmJY68TBU1Fv&9(PwrO{V=SI* z&m+6Jj7gr6v2=2Ssxf2n<a%CZi11`V79|ytjDlp7ykfnS%;J)w%%swi%=|pPDH-gs zQ!+$*ppu(oStc^E3qw?H{=vq?jA_{BGOi{@Ve|O-<iwnu_;?LX1r3OLkOL-b^Kh~2 zK=_;Oc{msu>(CsdvH2WtEF;_wQ~vu*Sk%lG%Hd*fhL|&1P<9i$7M!iZ4|6n<*k*M( zcQ!`t$@wZW5~yC(oYJEJ)q77>X7W*$U5rI2PP(V6O;p%Sj%AUV?63A2?sOmZN=6K` zWG6dndPq8^R%DjMgTkjMF)zCqDU9x^N=@FcDafu5cazlQSZ%?{tXlUO8`0b)wK-P% l4I-$5b-9@^)l81n7TWw$zm|ht8){^Mg#nDcd6mU9b^s_gz!Lxf diff --git a/doc/html/_sources/index.rst.txt b/doc/html/_sources/index.rst.txt new file mode 100644 index 0000000..5865604 --- /dev/null +++ b/doc/html/_sources/index.rst.txt @@ -0,0 +1,12 @@ +hpc4neuro Python Utilities's API documentation +============================================== + +.. autoclass:: hpc4neuro.distribution.DataDistributor + :members: __call__ + +.. autoclass:: hpc4neuro.distribution.ErrorHandler + :members: + +.. toctree:: + :maxdepth: 2 + :caption: Contents: diff --git a/doc/html/genindex.html b/doc/html/genindex.html new file mode 100644 index 0000000..511e35b --- /dev/null +++ b/doc/html/genindex.html @@ -0,0 +1,132 @@ + + +<!DOCTYPE html> + +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <meta charset="utf-8" /> + <title>Index — hpc4neuro Python Utilities documentation</title> + <link rel="stylesheet" href="_static/alabaster.css" type="text/css" /> + <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> + <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script> + <script type="text/javascript" src="_static/jquery.js"></script> + <script type="text/javascript" src="_static/underscore.js"></script> + <script type="text/javascript" src="_static/doctools.js"></script> + <script type="text/javascript" src="_static/language_data.js"></script> + <link rel="index" title="Index" href="#" /> + <link rel="search" title="Search" href="search.html" /> + + <link rel="stylesheet" href="_static/custom.css" type="text/css" /> + + + <meta name="viewport" content="width=device-width, initial-scale=0.9, maximum-scale=0.9" /> + + </head><body> + + + <div class="document"> + <div class="documentwrapper"> + <div class="bodywrapper"> + + + <div class="body" role="main"> + + +<h1 id="index">Index</h1> + +<div class="genindex-jumpbox"> + <a href="#_"><strong>_</strong></a> + | <a href="#D"><strong>D</strong></a> + | <a href="#E"><strong>E</strong></a> + +</div> +<h2 id="_">_</h2> +<table style="width: 100%" class="indextable genindextable"><tr> + <td style="width: 33%; vertical-align: top;"><ul> + <li><a href="index.html#hpc4neuro.distribution.DataDistributor.__call__">__call__() (hpc4neuro.distribution.DataDistributor method)</a> +</li> + </ul></td> +</tr></table> + +<h2 id="D">D</h2> +<table style="width: 100%" class="indextable genindextable"><tr> + <td style="width: 33%; vertical-align: top;"><ul> + <li><a href="index.html#hpc4neuro.distribution.DataDistributor">DataDistributor (class in hpc4neuro.distribution)</a> +</li> + </ul></td> +</tr></table> + +<h2 id="E">E</h2> +<table style="width: 100%" class="indextable genindextable"><tr> + <td style="width: 33%; vertical-align: top;"><ul> + <li><a href="index.html#hpc4neuro.distribution.ErrorHandler">ErrorHandler (class in hpc4neuro.distribution)</a> +</li> + </ul></td> + <td style="width: 33%; vertical-align: top;"><ul> + <li><a href="index.html#hpc4neuro.distribution.ErrorHandler.exit_all_ranks">exit_all_ranks() (hpc4neuro.distribution.ErrorHandler static method)</a> +</li> + </ul></td> +</tr></table> + + + + </div> + + </div> + </div> + <div class="sphinxsidebar" role="navigation" aria-label="main navigation"> + <div class="sphinxsidebarwrapper"> +<h1 class="logo"><a href="index.html">hpc4neuro Python Utilities</a></h1> + + + + + + + + +<h3>Navigation</h3> + +<div class="relations"> +<h3>Related Topics</h3> +<ul> + <li><a href="index.html">Documentation overview</a><ul> + </ul></li> +</ul> +</div> +<div id="searchbox" style="display: none" role="search"> + <h3 id="searchlabel">Quick search</h3> + <div class="searchformwrapper"> + <form class="search" action="search.html" method="get"> + <input type="text" name="q" aria-labelledby="searchlabel" /> + <input type="submit" value="Go" /> + </form> + </div> +</div> +<script type="text/javascript">$('#searchbox').show(0);</script> + + + + + + + + + </div> + </div> + <div class="clearer"></div> + </div> + <div class="footer"> + ©2019, Forschungszentrum Juelich GmbH. + + | + Powered by <a href="http://sphinx-doc.org/">Sphinx 2.2.2</a> + & <a href="https://github.com/bitprophet/alabaster">Alabaster 0.7.12</a> + + </div> + + + + + </body> +</html> \ No newline at end of file diff --git a/doc/html/index.html b/doc/html/index.html index 4460612..f016381 100644 --- a/doc/html/index.html +++ b/doc/html/index.html @@ -4,7 +4,7 @@ <html xmlns="http://www.w3.org/1999/xhtml"> <head> <meta charset="utf-8" /> - <title>HPC4NS Python Utilities’s API documentation — HPC4NS Python Utilities documentation</title> + <title>hpc4neuro Python Utilities’s API documentation — hpc4neuro Python Utilities documentation</title> <link rel="stylesheet" href="_static/alabaster.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script> @@ -30,11 +30,11 @@ <div class="body" role="main"> - <div class="section" id="hpc4ns-python-utilities-s-api-documentation"> -<h1>HPC4NS Python Utilities’s API documentation<a class="headerlink" href="#hpc4ns-python-utilities-s-api-documentation" title="Permalink to this headline">¶</a></h1> + <div class="section" id="hpc4neuro-python-utilities-s-api-documentation"> +<h1>hpc4neuro Python Utilities’s API documentation<a class="headerlink" href="#hpc4neuro-python-utilities-s-api-documentation" title="Permalink to this headline">¶</a></h1> <dl class="class"> -<dt id="hpc4ns.distribution.DataDistributor"> -<em class="property">class </em><code class="sig-prename descclassname">hpc4ns.distribution.</code><code class="sig-name descname">DataDistributor</code><span class="sig-paren">(</span><em class="sig-param">mpi_comm</em>, <em class="sig-param">shutdown_on_error=False</em><span class="sig-paren">)</span><a class="headerlink" href="#hpc4ns.distribution.DataDistributor" title="Permalink to this definition">¶</a></dt> +<dt id="hpc4neuro.distribution.DataDistributor"> +<em class="property">class </em><code class="sig-prename descclassname">hpc4neuro.distribution.</code><code class="sig-name descname">DataDistributor</code><span class="sig-paren">(</span><em class="sig-param">mpi_comm</em>, <em class="sig-param">shutdown_on_error=False</em><span class="sig-paren">)</span><a class="headerlink" href="#hpc4neuro.distribution.DataDistributor" title="Permalink to this definition">¶</a></dt> <dd><p>A callable that serves as a decorator for any function that returns an iterable of data items, and needs its functionality to be extended such that the iterable returned is not the entire collection, but rather only @@ -42,8 +42,8 @@ a part of the collection that is to be processed by the local MPI rank.</p> <p>Therefore, this decorator converts a function into one that can be used for distributed processing with MPI.</p> <dl class="method"> -<dt id="hpc4ns.distribution.DataDistributor.__call__"> -<code class="sig-name descname">__call__</code><span class="sig-paren">(</span><em class="sig-param">data_loader</em>, <em class="sig-param">shuffle=False</em><span class="sig-paren">)</span><a class="headerlink" href="#hpc4ns.distribution.DataDistributor.__call__" title="Permalink to this definition">¶</a></dt> +<dt id="hpc4neuro.distribution.DataDistributor.__call__"> +<code class="sig-name descname">__call__</code><span class="sig-paren">(</span><em class="sig-param">data_loader</em>, <em class="sig-param">shuffle=False</em><span class="sig-paren">)</span><a class="headerlink" href="#hpc4neuro.distribution.DataDistributor.__call__" title="Permalink to this definition">¶</a></dt> <dd><p>Decorates the given function so that instead of returning all the data_items, the given function returns only those data_items that are to be processed by the local MPI rank. All the MPI communication @@ -65,7 +65,7 @@ distribution amongst MPI ranks. RNG used is from the </ul> </dd> <dt class="field-even">Raises</dt> -<dd class="field-even"><p><strong>hpc4ns.errors import DataDistributionError</strong> – can be raised if +<dd class="field-even"><p><strong>hpc4neuro.errors import DataDistributionError</strong> – can be raised if ‘shutdown on error’ is not requested at the time of object creation.</p> </dd> @@ -79,13 +79,13 @@ of data items to be processed by the local MPI rank.</p> </dd></dl> <dl class="class"> -<dt id="hpc4ns.distribution.ErrorHandler"> -<em class="property">class </em><code class="sig-prename descclassname">hpc4ns.distribution.</code><code class="sig-name descname">ErrorHandler</code><a class="headerlink" href="#hpc4ns.distribution.ErrorHandler" title="Permalink to this definition">¶</a></dt> +<dt id="hpc4neuro.distribution.ErrorHandler"> +<em class="property">class </em><code class="sig-prename descclassname">hpc4neuro.distribution.</code><code class="sig-name descname">ErrorHandler</code><a class="headerlink" href="#hpc4neuro.distribution.ErrorHandler" title="Permalink to this definition">¶</a></dt> <dd><p>Set of functions that can gracefully handle exceptions/errors in MPI programs running with one or more ranks.</p> <dl class="method"> -<dt id="hpc4ns.distribution.ErrorHandler.exit_all_ranks"> -<em class="property">static </em><code class="sig-name descname">exit_all_ranks</code><span class="sig-paren">(</span><em class="sig-param">error</em>, <em class="sig-param">mpi_comm</em><span class="sig-paren">)</span><a class="headerlink" href="#hpc4ns.distribution.ErrorHandler.exit_all_ranks" title="Permalink to this definition">¶</a></dt> +<dt id="hpc4neuro.distribution.ErrorHandler.exit_all_ranks"> +<em class="property">static </em><code class="sig-name descname">exit_all_ranks</code><span class="sig-paren">(</span><em class="sig-param">error</em>, <em class="sig-param">mpi_comm</em><span class="sig-paren">)</span><a class="headerlink" href="#hpc4neuro.distribution.ErrorHandler.exit_all_ranks" title="Permalink to this definition">¶</a></dt> <dd><p>Ensures a clean program exit by synchronizing all MPI ranks available via the given communicator. Also, writes the message string from the given error object to stderr.</p> @@ -115,7 +115,7 @@ error matters.</p></li> </div> <div class="sphinxsidebar" role="navigation" aria-label="main navigation"> <div class="sphinxsidebarwrapper"> -<h1 class="logo"><a href="#">HPC4NS Python Utilities</a></h1> +<h1 class="logo"><a href="#">hpc4neuro Python Utilities</a></h1> @@ -159,7 +159,7 @@ error matters.</p></li> ©2019, Forschungszentrum Juelich GmbH. | - Powered by <a href="http://sphinx-doc.org/">Sphinx 2.2.1</a> + Powered by <a href="http://sphinx-doc.org/">Sphinx 2.2.2</a> & <a href="https://github.com/bitprophet/alabaster">Alabaster 0.7.12</a> | diff --git a/doc/html/objects.inv b/doc/html/objects.inv index 9b4b374..a654f4c 100644 --- a/doc/html/objects.inv +++ b/doc/html/objects.inv @@ -1,5 +1,6 @@ # Sphinx inventory version 2 -# Project: HPC4NS Python Utilities +# Project: hpc4neuro Python Utilities # Version: # The remainder of this file is compressed using zlib. -xڕ�Kj�0��>�@�� �Uv!)$�C�ZL�!KFC��5r���Q�J������CV�gI�Ћ�A�Qp= 1A7,c�0,��������n��Af�KLK��[�SJ1m0X��N��*:y�eA��Zq�pU �]0�����^9ؖ���l4�#6�j���f��x�^<���s�e����o)��U�7��PO�1tm�tkτɸ��Q���o�H�(�& \ No newline at end of file +xڝ��j1���������B=�9��`Bg�%�������Ij�nU�"� �|~�3�@}���,�z�1� +�&!&膥a������r�����=U��6ȬuijI\�OW�S�� �e��mZ�ɋ. �{���H�@�d<C=�I��`S��j��Ӎ� ��`/��x�_�^� �C}KA��Vm�f���c�6ڞ� &���6�w��#U�D�. \ No newline at end of file diff --git a/doc/html/search.html b/doc/html/search.html new file mode 100644 index 0000000..7673841 --- /dev/null +++ b/doc/html/search.html @@ -0,0 +1,111 @@ + +<!DOCTYPE html> + +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <meta charset="utf-8" /> + <title>Search — hpc4neuro Python Utilities documentation</title> + <link rel="stylesheet" href="_static/alabaster.css" type="text/css" /> + <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> + + <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script> + <script type="text/javascript" src="_static/jquery.js"></script> + <script type="text/javascript" src="_static/underscore.js"></script> + <script type="text/javascript" src="_static/doctools.js"></script> + <script type="text/javascript" src="_static/language_data.js"></script> + <script type="text/javascript" src="_static/searchtools.js"></script> + <link rel="index" title="Index" href="genindex.html" /> + <link rel="search" title="Search" href="#" /> + <script type="text/javascript" src="searchindex.js" defer></script> + + + <link rel="stylesheet" href="_static/custom.css" type="text/css" /> + + + <meta name="viewport" content="width=device-width, initial-scale=0.9, maximum-scale=0.9" /> + + + </head><body> + + + <div class="document"> + <div class="documentwrapper"> + <div class="bodywrapper"> + + + <div class="body" role="main"> + + <h1 id="search-documentation">Search</h1> + <div id="fallback" class="admonition warning"> + <script type="text/javascript">$('#fallback').hide();</script> + <p> + Please activate JavaScript to enable the search + functionality. + </p> + </div> + <p> + From here you can search these documents. Enter your search + words into the box below and click "search". Note that the search + function will automatically search for all of the words. Pages + containing fewer words won't appear in the result list. + </p> + <form action="" method="get"> + <input type="text" name="q" aria-labelledby="search-documentation" value="" /> + <input type="submit" value="search" /> + <span id="search-progress" style="padding-left: 10px"></span> + </form> + + <div id="search-results"> + + </div> + + </div> + + </div> + </div> + <div class="sphinxsidebar" role="navigation" aria-label="main navigation"> + <div class="sphinxsidebarwrapper"> +<h1 class="logo"><a href="index.html">hpc4neuro Python Utilities</a></h1> + + + + + + + + +<h3>Navigation</h3> + +<div class="relations"> +<h3>Related Topics</h3> +<ul> + <li><a href="index.html">Documentation overview</a><ul> + </ul></li> +</ul> +</div> + + + + + + + + + </div> + </div> + <div class="clearer"></div> + </div> + <div class="footer"> + ©2019, Forschungszentrum Juelich GmbH. + + | + Powered by <a href="http://sphinx-doc.org/">Sphinx 2.2.2</a> + & <a href="https://github.com/bitprophet/alabaster">Alabaster 0.7.12</a> + + </div> + + + + + </body> +</html> \ No newline at end of file diff --git a/doc/html/searchindex.js b/doc/html/searchindex.js new file mode 100644 index 0000000..815b626 --- /dev/null +++ b/doc/html/searchindex.js @@ -0,0 +1 @@ +Search.setIndex({docnames:["index"],envversion:{"sphinx.domains.c":1,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":1,"sphinx.domains.javascript":1,"sphinx.domains.math":2,"sphinx.domains.python":1,"sphinx.domains.rst":1,"sphinx.domains.std":1,sphinx:56},filenames:["index.rst"],objects:{"hpc4neuro.distribution":{DataDistributor:[0,0,1,""],ErrorHandler:[0,0,1,""]},"hpc4neuro.distribution.DataDistributor":{__call__:[0,1,1,""]},"hpc4neuro.distribution.ErrorHandler":{exit_all_ranks:[0,1,1,""]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"]},objtypes:{"0":"py:class","1":"py:method"},terms:{"class":0,"function":0,"import":0,"return":0,"static":0,"true":0,The:0,__call__:0,all:0,also:0,amongst:0,ani:0,avail:0,befor:0,call:0,callabl:0,caller:0,can:0,caus:0,clean:0,collect:0,comm:0,commun:0,convert:0,creation:0,data:0,data_item:0,data_load:0,datadistributionerror:0,datadistributor:0,decor:0,deriv:0,detail:0,distribut:0,doe:0,either:0,ensur:0,entir:0,error:0,errorhandl:0,except:0,exit:0,exit_all_rank:0,extend:0,fals:0,from:0,gener:0,given:0,gracefulli:0,handl:0,hpc4n:[],implement:0,instead:0,intern:0,item:0,iter:0,its:0,len:0,less:0,librari:0,list:0,local:0,matter:0,messag:0,more:0,mpi4pi:0,mpi:0,mpi_comm:0,need:0,number:0,object:0,one:0,onli:0,packag:0,paramet:0,part:0,possibl:0,process:0,program:0,rais:0,random:0,rank:0,rather:0,repres:0,request:0,rng:0,run:0,serv:0,set:0,should:0,shuffl:0,shutdown:0,shutdown_on_error:0,size:0,standard:0,stderr:0,string:0,synchron:0,than:0,therefor:0,thi:0,those:0,time:0,type:0,used:0,via:0,when:0,which:0,write:0},titles:["hpc4neuro Python Utilities\u2019s API documentation"],titleterms:{api:0,document:0,hpc4neuro:0,python:0,util:0}}) \ No newline at end of file diff --git a/doc/index.rst b/doc/index.rst index 8c3cc6d..5865604 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,10 +1,10 @@ -HPC4NS Python Utilities's API documentation -=========================================== +hpc4neuro Python Utilities's API documentation +============================================== -.. autoclass:: hpc4ns.distribution.DataDistributor +.. autoclass:: hpc4neuro.distribution.DataDistributor :members: __call__ -.. autoclass:: hpc4ns.distribution.ErrorHandler +.. autoclass:: hpc4neuro.distribution.ErrorHandler :members: .. toctree:: diff --git a/doc/text/index.txt b/doc/text/index.txt index 1d44a63..2c9f822 100644 --- a/doc/text/index.txt +++ b/doc/text/index.txt @@ -1,7 +1,7 @@ -HPC4NS Python Utilities's API documentation -******************************************* +hpc4neuro Python Utilities's API documentation +********************************************** -class hpc4ns.distribution.DataDistributor(mpi_comm, shutdown_on_error=False) +class hpc4neuro.distribution.DataDistributor(mpi_comm, shutdown_on_error=False) A callable that serves as a decorator for any function that returns an iterable of data items, and needs its functionality to be @@ -33,7 +33,7 @@ class hpc4ns.distribution.DataDistributor(mpi_comm, shutdown_on_error=False) from the 'random' package in the Python standard library. Raises: - **hpc4ns.errors import DataDistributionError** -- can be + **hpc4neuro.errors import DataDistributionError** -- can be raised if 'shutdown on error' is not requested at the time of object creation. @@ -41,7 +41,7 @@ class hpc4ns.distribution.DataDistributor(mpi_comm, shutdown_on_error=False) Decorated function. The function when called returns a list of data items to be processed by the local MPI rank. -class hpc4ns.distribution.ErrorHandler +class hpc4neuro.distribution.ErrorHandler Set of functions that can gracefully handle exceptions/errors in MPI programs running with one or more ranks. diff --git a/hpc4ns/__init__.py b/hpc4neuro/__init__.py similarity index 100% rename from hpc4ns/__init__.py rename to hpc4neuro/__init__.py diff --git a/hpc4ns/distribution.py b/hpc4neuro/distribution.py similarity index 98% rename from hpc4ns/distribution.py rename to hpc4neuro/distribution.py index 1365b31..a7751e6 100644 --- a/hpc4ns/distribution.py +++ b/hpc4neuro/distribution.py @@ -12,7 +12,7 @@ import random import itertools import functools -from hpc4ns.errors import DataDistributionError +from hpc4neuro.errors import DataDistributionError @enum.unique @@ -130,7 +130,7 @@ class DataDistributor: distribution amongst MPI ranks. RNG used is from the 'random' package in the Python standard library. - :raises hpc4ns.errors import DataDistributionError: can be raised if + :raises hpc4neuro.errors import DataDistributionError: can be raised if 'shutdown on error' is not requested at the time of object creation. diff --git a/hpc4ns/errors.py b/hpc4neuro/errors.py similarity index 100% rename from hpc4ns/errors.py rename to hpc4neuro/errors.py diff --git a/hpc4ns/examples/__init__.py b/hpc4neuro/examples/__init__.py similarity index 100% rename from hpc4ns/examples/__init__.py rename to hpc4neuro/examples/__init__.py diff --git a/hpc4ns/examples/distribution/__init__.py b/hpc4neuro/examples/distribution/__init__.py similarity index 100% rename from hpc4ns/examples/distribution/__init__.py rename to hpc4neuro/examples/distribution/__init__.py diff --git a/hpc4ns/examples/distribution/dynamic_filenames_decorator.py b/hpc4neuro/examples/distribution/dynamic_filenames_decorator.py similarity index 83% rename from hpc4ns/examples/distribution/dynamic_filenames_decorator.py rename to hpc4neuro/examples/distribution/dynamic_filenames_decorator.py index a55a6dc..113ba8f 100644 --- a/hpc4ns/examples/distribution/dynamic_filenames_decorator.py +++ b/hpc4neuro/examples/distribution/dynamic_filenames_decorator.py @@ -3,7 +3,7 @@ """ This program demonstrates how a library function can be decorated - using the hpc4ns.distribution.DataDistributor. + using the hpc4neuro.distribution.DataDistributor. The syntax for decoration presented in this program can be useful not only for library functions, but also in situations where a reference @@ -12,14 +12,14 @@ To run this sample from the repository root, use the following command on your workstation/laptop: - mpirun -np 3 python -u -m hpc4ns.examples.distribution.dynamic_filenames_decorator + mpirun -np 3 python -u -m hpc4neuro.examples.distribution.dynamic_filenames_decorator """ import os from mpi4py import MPI -from hpc4ns.distribution import DataDistributor +from hpc4neuro.distribution import DataDistributor # Initialize the decorator, and get a reference to the object. diff --git a/hpc4ns/examples/distribution/sequential_filenames.py b/hpc4neuro/examples/distribution/sequential_filenames.py similarity index 91% rename from hpc4ns/examples/distribution/sequential_filenames.py rename to hpc4neuro/examples/distribution/sequential_filenames.py index f106091..15ed85a 100644 --- a/hpc4ns/examples/distribution/sequential_filenames.py +++ b/hpc4neuro/examples/distribution/sequential_filenames.py @@ -8,7 +8,7 @@ To run this sample from the repository root, use the following command on your workstation/laptop: - python -u -m hpc4ns.examples.distribution.sequential_filenames + python -u -m hpc4neuro.examples.distribution.sequential_filenames """ diff --git a/hpc4ns/examples/distribution/static_filenames_decorator.py b/hpc4neuro/examples/distribution/static_filenames_decorator.py similarity index 84% rename from hpc4ns/examples/distribution/static_filenames_decorator.py rename to hpc4neuro/examples/distribution/static_filenames_decorator.py index 4e08c44..ec1887a 100644 --- a/hpc4ns/examples/distribution/static_filenames_decorator.py +++ b/hpc4neuro/examples/distribution/static_filenames_decorator.py @@ -6,20 +6,20 @@ a list of filenames can be easily decorated to automatically distribute the list across multiple MPI ranks. - The hpc4ns.distribution.DataDistributor is used as + The hpc4neuro.distribution.DataDistributor is used as the decorator. To run this sample from the repository root, use the following command on your workstation/laptop: - mpirun -np 3 python -u -m hpc4ns.examples.distribution.static_filenames_decorator + mpirun -np 3 python -u -m hpc4neuro.examples.distribution.static_filenames_decorator """ import os from mpi4py import MPI -from hpc4ns.distribution import DataDistributor +from hpc4neuro.distribution import DataDistributor # Decorate the function so that instead of returning the list diff --git a/pyproject.toml b/pyproject.toml index a10be6e..e89b88e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "hpc4ns" +name = "hpc4neuro" version = "0.1.0" description = """\ A library of Python utilities developed at the Simulation \ @@ -9,8 +9,8 @@ description = """\ authors = ["Forschungszentrum Juelich GmbH"] license = "MIT" readme = "README.md" -homepage = "https://gitlab.version.fz-juelich.de/hpc4ns/hpc4ns_utils" -repository = "https://gitlab.version.fz-juelich.de/hpc4ns/hpc4ns_utils" +homepage = "https://gitlab.version.fz-juelich.de/hpc4neuro/hpc4ns_utils" +repository = "https://gitlab.version.fz-juelich.de/hpc4neuro/hpc4ns_utils" [tool.poetry.dependencies] python = "^3.6" diff --git a/tests/test_distribution.py b/tests/test_distribution.py index 73f20d6..1f347ff 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -2,7 +2,7 @@ # This code is licensed under MIT license (see the LICENSE file for details) """ - Test suite to test the hpc4ns.distribution module. + Test suite to test the hpc4neuro.distribution module. """ @@ -15,8 +15,8 @@ import pytest import numpy as np from mpi4py import MPI -from hpc4ns.distribution import DataDistributor -from hpc4ns.errors import DataDistributionError +from hpc4neuro.distribution import DataDistributor +from hpc4neuro.errors import DataDistributionError # MPI communicator info required by multiple test classes @@ -173,7 +173,7 @@ class TestExceptionHandling: def test_error_from_data_loader(self): """ An error thrown by the decorated data loader function should - be raised as hpc4ns.errors.DataDistributionError when + be raised as hpc4neuro.errors.DataDistributionError when automatic shutdown is not requested. """ @@ -189,7 +189,7 @@ class TestExceptionHandling: def test_error_for_non_iterable(self, tmpdir): """ When the function to be decorated returns an object that is - not iterable, the hpc4ns.errors.DataDistributionError should + not iterable, the hpc4neuro.errors.DataDistributionError should be raised (when automatic shutdown is not requested). :param tmpdir: Temporary directory created by Pytest fixture. @@ -210,7 +210,7 @@ class TestExceptionHandling: def test_error_for_non_sized(self, tmpdir): """ When the function to be decorated returns an object that is - not sized, the hpc4ns.errors.DataDistributionError should + not sized, the hpc4neuro.errors.DataDistributionError should be raised (when automatic shutdown is not requested). :param tmpdir: Temporary directory created by Pytest fixture. @@ -232,7 +232,7 @@ class TestExceptionHandling: """ If the number of items to be distributed returned by the decorated function is less the number of MPI ranks, the - hpc4ns.errors.DataDistributionError should be raised (when + hpc4neuro.errors.DataDistributionError should be raised (when automatic shutdown is not requested). :param tmpdir: Temporary directory created by Pytest fixture. diff --git a/tests/test_hpc4ns.py b/tests/test_hpc4neuro.py similarity index 62% rename from tests/test_hpc4ns.py rename to tests/test_hpc4neuro.py index 928ae80..15ac315 100644 --- a/tests/test_hpc4ns.py +++ b/tests/test_hpc4neuro.py @@ -1,4 +1,4 @@ -from hpc4ns import __version__ +from hpc4neuro import __version__ def test_version(): -- GitLab From 96ac23fc45fabca1ee905f7d4df29f5f5c0b8b23 Mon Sep 17 00:00:00 2001 From: Fahad Khalid <f.khalid@fz-juelich.de> Date: Tue, 3 Dec 2019 09:40:32 +0100 Subject: [PATCH 2/3] Added instructions for document generation. Made minor name corrections. --- README.md | 3 +++ hpc4neuro/errors.py | 6 +++--- pyproject.toml | 4 ++-- tests/test_distribution.py | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 01e2016..1a050a7 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,10 @@ API documentation for `hpc4neuro.distribution` is available [here](doc/text/inde 4. If you use [`poetry`](https://github.com/sdispater/poetry), run `poetry install` to install all the required dependencies +To generate API documentation using `sphinx`, issue the following commands from the repository root: +* `sphinx-build -b html doc doc/html` +* `sphinx-build -b text doc doc/text` ### Test setup diff --git a/hpc4neuro/errors.py b/hpc4neuro/errors.py index d7b82db..3b1d93a 100644 --- a/hpc4neuro/errors.py +++ b/hpc4neuro/errors.py @@ -7,13 +7,13 @@ """ -class BaseHpc4nsError(Exception): +class BaseHpc4neuroError(Exception): """ Base class for all error classes in the module. """ -class MpiInitError(BaseHpc4nsError): +class MpiInitError(BaseHpc4neuroError): """ Raised if MPI initialization fails. """ -class DataDistributionError(BaseHpc4nsError): +class DataDistributionError(BaseHpc4neuroError): """ Raised if data distribution amongst MPI ranks fails. """ diff --git a/pyproject.toml b/pyproject.toml index e89b88e..0086ee8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,8 +9,8 @@ description = """\ authors = ["Forschungszentrum Juelich GmbH"] license = "MIT" readme = "README.md" -homepage = "https://gitlab.version.fz-juelich.de/hpc4neuro/hpc4ns_utils" -repository = "https://gitlab.version.fz-juelich.de/hpc4neuro/hpc4ns_utils" +homepage = "https://gitlab.version.fz-juelich.de/hpc4ns/hpc4neuro" +repository = "https://gitlab.version.fz-juelich.de/hpc4ns/hpc4neuro" [tool.poetry.dependencies] python = "^3.6" diff --git a/tests/test_distribution.py b/tests/test_distribution.py index 1f347ff..c05790f 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -139,7 +139,7 @@ def distributed_filenames(request): """ # Create a temporary data directory - with tempfile.TemporaryDirectory(prefix='hpc4ns_test_') as data_dir: + with tempfile.TemporaryDirectory(prefix='hpc4neuro_test_') as data_dir: # Generate files in the created directory generate_files(data_dir, num_files=num_ranks) -- GitLab From 676a7d2f2d0fd3f0b883e7f0d6418adab4e85c84 Mon Sep 17 00:00:00 2001 From: Fahad Khalid <f.khalid@fz-juelich.de> Date: Tue, 3 Dec 2019 09:48:48 +0100 Subject: [PATCH 3/3] Minor update to the readme. --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1a050a7..40c4385 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The `hpc4neuro` package requires `Python 3.6` or above. To install, please use the following command: ``` -python -m pip install git+https://gitlab.version.fz-juelich.de/hpc4neuro/hpc4neuro_utils.git +python -m pip install git+https://gitlab.version.fz-juelich.de/hpc4neuro/hpc4neuro.git ``` ## Available modules @@ -139,7 +139,8 @@ API documentation for `hpc4neuro.distribution` is available [here](doc/text/inde 4. If you use [`poetry`](https://github.com/sdispater/poetry), run `poetry install` to install all the required dependencies -To generate API documentation using `sphinx`, issue the following commands from the repository root: +To generate API documentation using [`sphinx`](http://www.sphinx-doc.org/en/master/), issue the following commands +from the repository root: * `sphinx-build -b html doc doc/html` * `sphinx-build -b text doc doc/text` -- GitLab