From d6889f39ceb0d7164ee9f473e0f638100206844b Mon Sep 17 00:00:00 2001
From: jrybicki-jsc <j.rybicki@fz-juelich.de>
Date: Fri, 6 May 2022 10:54:40 +0200
Subject: [PATCH] housekeeping + url return
---
dags/image_transfer_alt.py | 2 +-
dags/{just_reg.py => justreg.py} | 6 ++----
dags/uploadflow.py | 6 +++---
3 files changed, 6 insertions(+), 8 deletions(-)
rename dags/{just_reg.py => justreg.py} (91%)
diff --git a/dags/image_transfer_alt.py b/dags/image_transfer_alt.py
index f15ea5a..49772e5 100644
--- a/dags/image_transfer_alt.py
+++ b/dags/image_transfer_alt.py
@@ -6,7 +6,7 @@ from airflow.decorators import dag, task
from airflow.utils.dates import days_ago
from airflow.operators.python import PythonOperator
from airflow.models import Variable
-from just_reg import get_parameter
+from justreg import get_parameter
from decors import setup, get_connection, remove
default_args = {
diff --git a/dags/just_reg.py b/dags/justreg.py
similarity index 91%
rename from dags/just_reg.py
rename to dags/justreg.py
index 8018a65..160829f 100644
--- a/dags/just_reg.py
+++ b/dags/justreg.py
@@ -1,8 +1,6 @@
from airflow.decorators import dag, task
-from airflow.models.connection import Connection
from airflow.operators.bash import BashOperator
-from airflow.providers.http.hooks.http import HttpHook
from airflow.utils.dates import days_ago
from datacat_integration.hooks import DataCatalogHook
from datacat_integration.connection import DataCatalogEntry
@@ -65,7 +63,7 @@ def datacat_registration_example():
try:
r = hook.create_entry(datacat_type='dataset', entry=entry)
print("Hook registration returned: ", r)
- return r
+ return f"{hook.base_url}/dataset/{r}"
except ConnectionError as e:
print('Registration failed', e)
return -1
@@ -85,7 +83,7 @@ def datacat_registration_example():
- step1 = BashOperator(bash_command='curl -X GET -k https://bscgrid20.bsc.es/image_creation/images/download/wordcount_skylake.sif -o /work/ww', task_id='nothing')
+ step1 = BashOperator(task_id='jj', bash_command='ls') #BashOperator(bash_command='curl -X GET -k https://bscgrid20.bsc.es/image_creation/images/download/wordcount_skylake.sif -o /work/ww', task_id='nothing')
step2 = register(
object_url='https://b2share-testing.fz-juelich.de/records/7a12fda26b2a4d248f96d012d54769b7')
diff --git a/dags/uploadflow.py b/dags/uploadflow.py
index 40a442e..72c4d7e 100644
--- a/dags/uploadflow.py
+++ b/dags/uploadflow.py
@@ -12,7 +12,7 @@ from airflow.utils.dates import days_ago
from b2shareoperator import (add_file, create_draft_record, get_community,
submit_draft)
from decors import get_connection, remove, setup
-from just_reg import get_parameter
+from justreg import get_parameter
from datacat_integration.hooks import DataCatalogHook
from datacat_integration.connection import DataCatalogEntry
@@ -120,7 +120,7 @@ def upload_example():
return 0
hook = DataCatalogHook()
- print("Connected to datacat via hook", hook.list_type('dataset'))
+ print("Connected to datacat via hook")
entry = DataCatalogEntry(name=f"DLS results {kwargs['run_id']}",
url=object_url,
@@ -131,7 +131,7 @@ def upload_example():
try:
r = hook.create_entry(datacat_type='dataset', entry=entry)
print("Hook registration returned: ", r)
- return r
+ return f"{hook.base_url}/dataset/{r}"
except ConnectionError as e:
print('Registration failed', e)
return -1
--
GitLab