diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 050bb77730bb6e85316ec7a7859231b49a46086f..08ca84cd9fda568a86513d3cec112df13c09e6bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,5 +20,6 @@ test: - pwd - cp dags/* /opt/airflow/dags/ - airflow dags list + - airflow connections list - airflow dags test firsto 2021-08-18 diff --git a/dags/b2shareoperator.py b/dags/b2shareoperator.py index a7225195466b36569dda226685f549203b6b0336..afe6e2fd2c94610f7c3713f19bc88cd177f472e7 100644 --- a/dags/b2shareoperator.py +++ b/dags/b2shareoperator.py @@ -1,11 +1,13 @@ from airflow.models.baseoperator import BaseOperator from airflow.models.connection import Connection import requests +from urllib.parse import urljoin def get_objects(server): lst = requests.get(urljoin(server, 'api/records')).json() return lst['hits']['hits'] +server='https://b2share-testing.fz-juelich.de/' class B2ShareOperator(BaseOperator): @@ -16,15 +18,15 @@ class B2ShareOperator(BaseOperator): **kwargs) -> None: super().__init__(**kwargs) self.name = name - self.connection = Connection.get_connection_from_secrets(conn_id) + #self.connection = Connection.get_connection_from_secrets(conn_id) def execute(self, context): message = "Hello {}".format(self.name) print(message) - print(self.connection.host) + #print(self.connection.host) - print(f"Retrieving info from {self.connection.host}") - lst = get_objects(server=self.connection.schema+self.connection.host) + #print(f"Retrieving info from {self.connection.host}") + lst = get_objects(server=server) print(f"GOT: {lst}") return message