update for pyunicore 1.0.0

dc572ccb · Jens Henrik Goebbert · 63561e23 · dc572ccb
Commit dc572ccb authored 1 year ago by Jens Henrik Goebbert
--- a/day2_hpcenv/7_cloud-hpc_challenges/1-hdf-cloud_mount-hpc-storage.ipynb
+++ b/day2_hpcenv/7_cloud-hpc_challenges/1-hdf-cloud_mount-hpc-storage.ipynb
@@ -131,8 +131,20 @@
   "source": [
    "def get_mount_command(access_token):\n",
    "    _auth = \"https://uftp.fz-juelich.de:9112/UFTP_Auth/rest/auth/\"\n",
+    "    try:\n",
    "        _tr = unicore_client.Transport(access_token)\n",
    "        _info = _tr.get(url=_auth)\n",
+    "    except AttributeError:\n",
+    "        # pyunicore 1.0.0 has a different API\n",
+    "        import pyunicore.credentials as unicore_credentials\n",
+    "        credential = unicore_credentials.OIDCToken(access_token, None)\n",
+    "        transport_kwargs = {\n",
+    "            \"credential\": credential,\n",
+    "            \"verify\": False,\n",
+    "            \"timeout\": 30\n",
+    "        }\n",
+    "        _tr = unicore_client.Transport(**transport_kwargs)\n",
+    "        _info = _tr.get(url=_auth)\n",
    "    _uid = _info[\"JUDAC\"][\"uid\"]\n",
    "\n",
    "    remote_base_dir = \"/p/home/jusers/%s\" % _uid\n",
@@ -224,6 +236,14 @@
    "# unmount\n",
    "!fusermount -u $HOME/uftp-mount-rw"
   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c0c2fb3a-549c-4993-86d8-b2fc2c7dbdc5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
  }
 ],
 "metadata": {
@@ -242,7 +262,7 @@
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
-   "version": "3.10.4"
+   "version": "3.11.3"
  }
 },
 "nbformat": 4,

 %% Cell type:markdown id:exclusive-nursery tags:
 ![jsc-logo.jpg](attachment:67258d94-84e6-4a0c-ae8f-c74332ec082e.jpg)
 Author: [Jens Henrik Göbbert](mailto:j.goebbert@fz-juelich.de)
 ------------------------------------
 %% Cell type:markdown id:worldwide-presence tags:
 # Supercomputing with JupyterLab - Tips & Tricks
 This is the first time you are using JupyterLab on an HPC system? Let us show you some tips & tricks.
 This notebook might be special to the environment at Juelich Supercomputing Centre, Forschungszentrum Juelich, Germany.# JupyterLab Tour
 -------------------------
 %% Cell type:markdown id:retired-crest tags:
 #### Exercise 1:
 **ONLY FOR HDF-CLOUD**
 **!!DANGEROUS!! YOU ARE MOUNTING WRITEABLE**
 Mount the $HOMEs from GPFS on the HDF Cloud
 %% Cell type:code id:abd7420b-36b8-4f8b-91fc-cc10f6d45013 tags:
 ``` python
 import json
 import os
 from contextlib import closing
 import pyunicore.client as unicore_client
 import requests
 ```
 %% Cell type:markdown id:2da2cb6f-d6c6-4bbe-a7bb-8dd488615181 tags:
 During the login process in Jupyter-JSC, you have authenticated yourself against the identity management system [*Unity-IdM*](https://www.unity-idm.eu) .
 This IdM issues temporary tokens (comparable to user+password) on request, which can be use for authentication to other services that are also connected to the same IdM.
 Here we use the JupyterHub-API (+your JUPYTERHUB_API_TOKEN) to get a new temporary token for the next steps.
 %% Cell type:code id:76076cad-cc9c-4590-b51b-d0a51432fb80 tags:
 ``` python
 def get_access_token():
    api_url = os.getenv("JUPYTERHUB_API_URL")
    user_api_url = f"{api_url}/user_oauth"
    headers = {"Authorization": "token {}".format(os.getenv("JUPYTERHUB_API_TOKEN"))}
    with closing(requests.get(user_api_url, headers=headers)) as r:
        if r.status_code == 200:
            resp = json.loads(r.content.decode("utf-8"))
        else:
            raise Exception(
                "Could not receive access token: {} {}".format(
                    r.status_code, r.content.decode("utf-8")
                )
            )
    # No HPC accounts -> no access token in this script
    if (
        "auth_state" in resp.keys()
        and "access_token" in resp["auth_state"].keys()
    ):
        return resp["auth_state"]["access_token"]
    else:
        return None
 ```
 %% Cell type:markdown id:540563d1-59ca-4d28-86f7-171c98310a68 tags:
 Having that `access_token` we can now ask unicore for information about the HPC account of the user the token belongs to.
 We will receive the `uid` and the base of all its home directories as `remote_base_dir`.
 That information is required to ask the service [*UFTP*](https://apps.fz-juelich.de/jsc/hps/judac/uftp.html) all details needed for the mound command.
 %% Cell type:markdown id:aa5c5156-9afd-4b7b-a78f-7bb40aef2222 tags:
 **ATTENTION:** Set `ro` or `rw` for mounting readonly or writable
 %% Cell type:code id:05daf3b3-4582-4f51-b51a-25791b25da3e tags:
 ``` python
 def get_mount_command(access_token):
    _auth = "https://uftp.fz-juelich.de:9112/UFTP_Auth/rest/auth/"
-    _tr = unicore_client.Transport(access_token)
+    try:
-    _info = _tr.get(url=_auth)
+        _tr = unicore_client.Transport(access_token)
+        _info = _tr.get(url=_auth)
+    except AttributeError:
+        # pyunicore 1.0.0 has a different API
+        import pyunicore.credentials as unicore_credentials
+        credential = unicore_credentials.OIDCToken(access_token, None)
+        transport_kwargs = {
+            "credential": credential,
+            "verify": False,
+            "timeout": 30
+        }
+        _tr = unicore_client.Transport(**transport_kwargs)
+        _info = _tr.get(url=_auth)
    _uid = _info["JUDAC"]["uid"]
    remote_base_dir = "/p/home/jusers/%s" % _uid
    # authenticate
    _req = {
        "persistent": "true",
        "serverPath": "%s/___UFTP___MULTI___FILE___SESSION___MODE___" % remote_base_dir,
    }
    _reply = _tr.post(url=_auth + "JUDAC", json=_req).json()
    uftp_pwd = _reply["secret"]
    uftp_host = _reply["serverHost"]
    uftp_port = _reply["serverPort"]
    return f"curlftpfs -s -o ro,uid=1000,gid=100,ftp_method=singlecwd,enable_epsv,user=anonymous:{uftp_pwd} {uftp_host}:{uftp_port}"
 ```
 %% Cell type:markdown id:e7f6af90-bd33-4a55-9659-973bcaa3da67 tags:
 ------------------------------------
 Let's bring all pieces together:
 %% Cell type:code id:2cdd8bf4-aab5-4cd8-bfda-cd99ca82e086 tags:
 ``` python
 access_token = get_access_token()
 if not access_token:
    print("Error: No access token")
 ```
 %% Cell type:code id:0368679f-e3dd-4499-8b2e-ed429ee5ed53 tags:
 ``` python
 mount_cmd = get_mount_command(access_token)
 # e.g. curlftpfs -s -o ro,uid=1094,gid=100,ftp_method=singlecwd,enable_epsv,user=anonymous:fwngibbflwngfllaycpc judacsrv.fz-juelich.de:64333
 print(mount_cmd.partition("anonymous:")[0], "<...>") # do not print the token
 ```
 %% Cell type:code id:45c2ff24-a735-459a-8b53-ceeb74752635 tags:
 ``` python
 %%bash -s "$mount_cmd"
 mkdir -p $HOME/uftp-mount-rw
 $1 $HOME/uftp-mount-rw
 ```
 %% Cell type:code id:30a5b1e5-4f11-465a-a27d-e0b2c5973ec0 tags:
 ``` python
 # ...
 !ls -l $HOME/uftp-mount-rw
 ```
 %% Cell type:code id:5a4ff2ac-8731-4ce2-ab79-e89b6d3887a6 tags:
 ``` python
 # unmount
 !fusermount -u $HOME/uftp-mount-rw
 ```
+%% Cell type:code id:c0c2fb3a-549c-4993-86d8-b2fc2c7dbdc5 tags:
+``` python
+```