From 9e6d43b5d9ee7de9d8cc7d0a40713d5bfa62acc5 Mon Sep 17 00:00:00 2001
From: Christian Boettcher <c.boettcher@fz-juelich.de>
Date: Tue, 14 Dec 2021 12:55:21 +0100
Subject: [PATCH] delay some imports, use dict.get() to prevent keyErrors
---
src/datacat_integration/connection.py | 14 +++++++-------
src/datacat_integration/secrets.py | 19 +++++++++----------
2 files changed, 16 insertions(+), 17 deletions(-)
diff --git a/src/datacat_integration/connection.py b/src/datacat_integration/connection.py
index fd78ab1..0b50e3c 100644
--- a/src/datacat_integration/connection.py
+++ b/src/datacat_integration/connection.py
@@ -4,17 +4,17 @@ import json
from urllib.parse import urljoin
-from airflow.models.connection import Connection
-
import requests
-def get_connection_from_entry(data: Dict[str, Any], datacat_type: str, oid: str) -> Connection:
+def get_connection_from_entry(data: Dict[str, Any], datacat_type: str, oid: str):
"""returns an airflow connection from the data provided in the datacat entry."""
- conn_type = data['metadata']['conn_type']
- host = data['metadata']['host']
- port = data['metadata']['port']
- schema = data['metadata']['schema']
+ # delay import to prevent circular dependencies during config startup
+ from airflow.models.connection import Connection
+ conn_type = data['metadata'].get('conn_type')
+ host = data['metadata'].get('host')
+ port = data['metadata'].get('port')
+ schema = data['metadata'].get('schema')
conn_id = f"{datacat_type}/{oid}-connection"
# set all remaining metadata as extra
extra = {}
diff --git a/src/datacat_integration/secrets.py b/src/datacat_integration/secrets.py
index a39c793..2939a41 100644
--- a/src/datacat_integration/secrets.py
+++ b/src/datacat_integration/secrets.py
@@ -1,7 +1,6 @@
from typing import Any, Dict, Set
from urllib.parse import urljoin
from airflow.secrets import BaseSecretsBackend
-from airflow.models.connection import Connection
import requests
import logging
@@ -11,12 +10,14 @@ connection_backend_type = "airflow_connections"
log = logging.getLogger(__name__)
+class Empty:
+ pass
-def get_connection_with_secrets_from_entry(data: Dict[str, Any], secrets: Dict[str, str] , datacat_type: str, oid: str) -> Connection:
+def get_connection_with_secrets_from_entry(data: Dict[str, Any], secrets: Dict[str, str] , datacat_type: str, oid: str):
"""returns an aiflow connection from the data provided in the datacat entry and the secrets."""
conn = get_connection_from_entry(data, datacat_type, oid)
- conn.password = secrets['password']
- conn.login = secrets['login']
+ conn.password = secrets.get('password')
+ conn.login = secrets.get('login')
# add all remaining secrets to extra
extra = conn.extra
for key in secrets.keys():
@@ -76,16 +77,14 @@ class DatacatSecretsBackend(BaseSecretsBackend):
def __init__(self, **kwargs):
log.debug("Init of Datacat Secrets Backend")
- self.url = kwargs["url"]
- self.user = kwargs["user"]
- self.password = kwargs["password"]
+ self.url = kwargs.get("url", "http://localhost")
+ self.user = kwargs.get("user", "")
+ self.password = kwargs.get("password", "")
def get_connection(self, conn_id: str):
"""returns a Connection created from the <conenction_type>/<conn_id> entry in the datacatalog"""
- # only for testing: check that a specific oid has been requested
+
log.debug(f"Get connection: {conn_id}")
- if conn_id != "860355e9-975f-4253-9421-1815e20c879b":
- return None
secrets_conn = DataCatConnectionWithSecrets(self.url, self.user, self.password)
data = secrets_conn.get_entry(connection_backend_type, conn_id)
--
GitLab