From 84b8b1225163a9b44a960c62c666be60193e9b0e Mon Sep 17 00:00:00 2001
From: Jens Henrik Goebbert <j.goebbert@fz-juelich.de>
Date: Wed, 24 Apr 2024 06:46:32 +0000
Subject: [PATCH] update mount
---
.../1-hdf-cloud_mount-hpc-storage.ipynb | 77 ++++++++++++++++---
1 file changed, 66 insertions(+), 11 deletions(-)
diff --git a/day2_hpcenv/7_cloud-hpc_challenges/1-hdf-cloud_mount-hpc-storage.ipynb b/day2_hpcenv/7_cloud-hpc_challenges/1-hdf-cloud_mount-hpc-storage.ipynb
index 8560ec5..6625c16 100644
--- a/day2_hpcenv/7_cloud-hpc_challenges/1-hdf-cloud_mount-hpc-storage.ipynb
+++ b/day2_hpcenv/7_cloud-hpc_challenges/1-hdf-cloud_mount-hpc-storage.ipynb
@@ -104,6 +104,38 @@
" return None"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "76076cad-cc9c-4590-b51b-d0a51432fb80",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "def get_access_token():\n",
+ " api_url = os.getenv(\"JUPYTERHUB_API_URL\")\n",
+ " user_api_url = f\"{api_url}/user_oauth\"\n",
+ " headers = {\"Authorization\": \"token {}\".format(os.getenv(\"JUPYTERHUB_API_TOKEN\"))}\n",
+ " with closing(requests.get(user_api_url, headers=headers)) as r:\n",
+ " if r.status_code == 200:\n",
+ " resp = json.loads(r.content.decode(\"utf-8\"))\n",
+ " else:\n",
+ " raise Exception(\n",
+ " \"Could not receive access token: {} {}\".format(\n",
+ " r.status_code, r.content.decode(\"utf-8\")\n",
+ " )\n",
+ " )\n",
+ " # No HPC accounts -> no access token in this script\n",
+ " if (\n",
+ " \"auth_state\" in resp.keys()\n",
+ " and \"access_token\" in resp[\"auth_state\"].keys()\n",
+ " ):\n",
+ " return resp[\"auth_state\"][\"access_token\"]\n",
+ " else:\n",
+ " return None"
+ ]
+ },
{
"cell_type": "markdown",
"id": "540563d1-59ca-4d28-86f7-171c98310a68",
@@ -127,7 +159,9 @@
"cell_type": "code",
"execution_count": 3,
"id": "78c3f65d-1039-482f-900b-c9b0173b490d",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
"outputs": [],
"source": [
"def get_mount_command(access_token):\n",
@@ -158,24 +192,45 @@
"Let's bring all pieces together:"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "2cdd8bf4-aab5-4cd8-bfda-cd99ca82e086",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "access_token = get_access_token()\n",
+ "if not access_token:\n",
+ " print(\"Error: No access token\")"
+ ]
+ },
{
"cell_type": "code",
"execution_count": 7,
- "id": "b0fa1183-2983-4702-94b1-b4567fea4df3",
- "metadata": {},
+ "id": "0368679f-e3dd-4499-8b2e-ed429ee5ed53",
+ "metadata": {
+ "tags": []
+ },
"outputs": [
{
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "curlftpfs -s -o rw,uid=1094,gid=100,ftp_method=singlecwd,enable_epsv,user= <...>\n"
+ "ename": "HTTPError",
+ "evalue": "400 Server Error: Bad Request for url: https://uftp.fz-juelich.de:9112/UFTP_Auth/rest/auth//JUDAC",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mHTTPError\u001b[0m Traceback (most recent call last)",
+ "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m mount_cmd \u001b[38;5;241m=\u001b[39m \u001b[43mget_mount_command\u001b[49m\u001b[43m(\u001b[49m\u001b[43maccess_token\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m# e.g. curlftpfs -s -o ro,uid=1094,gid=100,ftp_method=singlecwd,enable_epsv,user=anonymous:fwngibbflwngfllaycpc judacsrv.fz-juelich.de:64333\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28mprint\u001b[39m(mount_cmd\u001b[38;5;241m.\u001b[39mpartition(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124manonymous:\u001b[39m\u001b[38;5;124m\"\u001b[39m)[\u001b[38;5;241m0\u001b[39m], \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m<...>\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;66;03m# do not print the token\u001b[39;00m\n",
+ "Cell \u001b[0;32mIn[3], line 13\u001b[0m, in \u001b[0;36mget_mount_command\u001b[0;34m(access_token)\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;66;03m# authenticate\u001b[39;00m\n\u001b[1;32m 9\u001b[0m _req \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 10\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpersistent\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrue\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 11\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mserverPath\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m/___UFTP___MULTI___FILE___SESSION___MODE___\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m remote_base_dir,\n\u001b[1;32m 12\u001b[0m }\n\u001b[0;32m---> 13\u001b[0m _reply \u001b[38;5;241m=\u001b[39m \u001b[43m_tr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_auth\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m/JUDAC\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_req\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mjson()\n\u001b[1;32m 14\u001b[0m uftp_pwd \u001b[38;5;241m=\u001b[39m _reply[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msecret\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 15\u001b[0m uftp_host \u001b[38;5;241m=\u001b[39m _reply[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mserverHost\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",
+ "File \u001b[0;32m/p/software/jsccloud/stages/2023/software/JupyterLab/2023.3.6-GCCcore-11.3.0/lib/python3.10/site-packages/pyunicore/client.py:189\u001b[0m, in \u001b[0;36mTransport.post\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpost\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 188\u001b[0m \u001b[38;5;124;03m\"\"\"do a POST and return the response\"\"\"\u001b[39;00m\n\u001b[0;32m--> 189\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpost\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+ "File \u001b[0;32m/p/software/jsccloud/stages/2023/software/JupyterLab/2023.3.6-GCCcore-11.3.0/lib/python3.10/site-packages/pyunicore/client.py:165\u001b[0m, in \u001b[0;36mTransport.run_method\u001b[0;34m(self, method, **args)\u001b[0m\n\u001b[1;32m 158\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrepeat_required(res, _headers):\n\u001b[1;32m 159\u001b[0m res \u001b[38;5;241m=\u001b[39m method(\n\u001b[1;32m 160\u001b[0m headers\u001b[38;5;241m=\u001b[39m_headers,\n\u001b[1;32m 161\u001b[0m verify\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mverify,\n\u001b[1;32m 162\u001b[0m timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtimeout,\n\u001b[1;32m 163\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39margs,\n\u001b[1;32m 164\u001b[0m )\n\u001b[0;32m--> 165\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_error\u001b[49m\u001b[43m(\u001b[49m\u001b[43mres\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 166\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_security_sessions:\n\u001b[1;32m 167\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlast_session_id \u001b[38;5;241m=\u001b[39m res\u001b[38;5;241m.\u001b[39mheaders\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mX-UNICORE-SecuritySession\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n",
+ "File \u001b[0;32m/p/software/jsccloud/stages/2023/software/JupyterLab/2023.3.6-GCCcore-11.3.0/lib/python3.10/site-packages/pyunicore/client.py:143\u001b[0m, in \u001b[0;36mTransport.check_error\u001b[0;34m(self, res)\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[1;32m 142\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mres\u001b[38;5;241m.\u001b[39mstatus_code\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m Server Error: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreason\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m for url: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mres\u001b[38;5;241m.\u001b[39murl\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 143\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mHTTPError(msg, response\u001b[38;5;241m=\u001b[39mres)\n\u001b[1;32m 144\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 145\u001b[0m res\u001b[38;5;241m.\u001b[39mraise_for_status()\n",
+ "\u001b[0;31mHTTPError\u001b[0m: 400 Server Error: Bad Request for url: https://uftp.fz-juelich.de:9112/UFTP_Auth/rest/auth//JUDAC"
]
}
],
"source": [
- "access_token = get_access_token()\n",
- "if not access_token:\n",
- " print(\"Error: No access token\")\n",
"mount_cmd = get_mount_command(access_token)\n",
"# e.g. curlftpfs -s -o ro,uid=1094,gid=100,ftp_method=singlecwd,enable_epsv,user=anonymous:fwngibbflwngfllaycpc judacsrv.fz-juelich.de:64333\n",
"print(mount_cmd.partition(\"anonymous:\")[0], \"<...>\") # do not print the token"
@@ -248,7 +303,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.5"
+ "version": "3.10.4"
}
},
"nbformat": 4,
--
GitLab