From 4107b26f286d132dfb83623c35d08d9308b224b4 Mon Sep 17 00:00:00 2001
From: jrybicki-jsc <j.rybicki@fz-juelich.de>
Date: Thu, 7 Apr 2022 14:58:48 +0200
Subject: [PATCH] bash based download
---
dags/just_reg.py | 5 +++--
dags/uploadflow.py | 2 +-
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/dags/just_reg.py b/dags/just_reg.py
index 827eef9..0f4af0b 100644
--- a/dags/just_reg.py
+++ b/dags/just_reg.py
@@ -85,12 +85,13 @@ def datacat_registration_example():
- step1 = BashOperator(bash_command='ls', task_id='nothing')
+ step1 = BashOperator(bash_command='curl -X GET -k https://bscgrid20.bsc.es/image_creation/images/download/wordcount_skylake.sif -o /work/ww', task_id='nothing')
step2 = register(
object_url='https://b2share-testing.fz-juelich.de/records/7a12fda26b2a4d248f96d012d54769b7')
step3 = get_template()
- step1 >> step2 >> step3
+ stepr = BashOperator(bash_command='rm /work/ww', task_id='nothing')
+ step1 >> step2 >> step3 >> stepr
dag = datacat_registration_example()
diff --git a/dags/uploadflow.py b/dags/uploadflow.py
index 4ceacb9..17c845b 100644
--- a/dags/uploadflow.py
+++ b/dags/uploadflow.py
@@ -152,7 +152,7 @@ def upload_example():
reg = register(object_url=uid)
- setup_task >> files >> uid >> en
+ setup_task >> files >> uid >> reg >> en
dag = upload_example()
--
GitLab