diff --git a/dags/image_transfer_alt.py b/dags/image_transfer_alt.py index f15ea5a48f6b0580d14e5d38b7a5441cfb10255a..49772e5ca50bd794fc4f5d2cc09e096646e935c4 100644 --- a/dags/image_transfer_alt.py +++ b/dags/image_transfer_alt.py @@ -6,7 +6,7 @@ from airflow.decorators import dag, task from airflow.utils.dates import days_ago from airflow.operators.python import PythonOperator from airflow.models import Variable -from just_reg import get_parameter +from justreg import get_parameter from decors import setup, get_connection, remove default_args = { diff --git a/dags/just_reg.py b/dags/justreg.py similarity index 91% rename from dags/just_reg.py rename to dags/justreg.py index 8018a652805b214c7b286a4903b0e55037a98571..160829f3a989e4efd3c4ed7fc45d348d22381f6e 100644 --- a/dags/just_reg.py +++ b/dags/justreg.py @@ -1,8 +1,6 @@ from airflow.decorators import dag, task -from airflow.models.connection import Connection from airflow.operators.bash import BashOperator -from airflow.providers.http.hooks.http import HttpHook from airflow.utils.dates import days_ago from datacat_integration.hooks import DataCatalogHook from datacat_integration.connection import DataCatalogEntry @@ -65,7 +63,7 @@ def datacat_registration_example(): try: r = hook.create_entry(datacat_type='dataset', entry=entry) print("Hook registration returned: ", r) - return r + return f"{hook.base_url}/dataset/{r}" except ConnectionError as e: print('Registration failed', e) return -1 @@ -85,7 +83,7 @@ def datacat_registration_example(): - step1 = BashOperator(bash_command='curl -X GET -k https://bscgrid20.bsc.es/image_creation/images/download/wordcount_skylake.sif -o /work/ww', task_id='nothing') + step1 = BashOperator(task_id='jj', bash_command='ls') #BashOperator(bash_command='curl -X GET -k https://bscgrid20.bsc.es/image_creation/images/download/wordcount_skylake.sif -o /work/ww', task_id='nothing') step2 = register( object_url='https://b2share-testing.fz-juelich.de/records/7a12fda26b2a4d248f96d012d54769b7') diff --git a/dags/uploadflow.py b/dags/uploadflow.py index 40a442ed9e22a6aa489878b1585d886545ac062c..72c4d7e0d1f0f1737c27bd94d8543c4cf94c5c5c 100644 --- a/dags/uploadflow.py +++ b/dags/uploadflow.py @@ -12,7 +12,7 @@ from airflow.utils.dates import days_ago from b2shareoperator import (add_file, create_draft_record, get_community, submit_draft) from decors import get_connection, remove, setup -from just_reg import get_parameter +from justreg import get_parameter from datacat_integration.hooks import DataCatalogHook from datacat_integration.connection import DataCatalogEntry @@ -120,7 +120,7 @@ def upload_example(): return 0 hook = DataCatalogHook() - print("Connected to datacat via hook", hook.list_type('dataset')) + print("Connected to datacat via hook") entry = DataCatalogEntry(name=f"DLS results {kwargs['run_id']}", url=object_url, @@ -131,7 +131,7 @@ def upload_example(): try: r = hook.create_entry(datacat_type='dataset', entry=entry) print("Hook registration returned: ", r) - return r + return f"{hook.base_url}/dataset/{r}" except ConnectionError as e: print('Registration failed', e) return -1