delay some imports, use dict.get() to prevent keyErrors

src/datacat_integration/connection.py

@@ -4,17 +4,17 @@ import json
 
 from urllib.parse import urljoin
 
-from airflow.models.connection import Connection
-
 import requests
 
 
-def get_connection_from_entry(data: Dict[str, Any], datacat_type: str, oid: str) -> Connection:
+def get_connection_from_entry(data: Dict[str, Any], datacat_type: str, oid: str):
     """returns an airflow connection from the data provided in the datacat entry."""
-    conn_type = data['metadata']['conn_type']
-    host = data['metadata']['host']
-    port = data['metadata']['port']
-    schema = data['metadata']['schema']
+    # delay import to prevent circular dependencies during config startup
+    from airflow.models.connection import Connection
+    conn_type = data['metadata'].get('conn_type')
+    host = data['metadata'].get('host')
+    port = data['metadata'].get('port')
+    schema = data['metadata'].get('schema')
     conn_id = f"{datacat_type}/{oid}-connection"
     # set all remaining metadata as extra
     extra = {}

src/datacat_integration/secrets.py

 from typing import Any, Dict, Set
 from urllib.parse import urljoin
 from airflow.secrets import BaseSecretsBackend
-from airflow.models.connection import Connection
 import requests
 import logging
 
@@ -11,12 +10,14 @@ connection_backend_type = "airflow_connections"
 
 log = logging.getLogger(__name__)
 
+class Empty:
+    pass
 
-def get_connection_with_secrets_from_entry(data: Dict[str, Any], secrets: Dict[str, str] , datacat_type: str, oid: str) -> Connection:
+def get_connection_with_secrets_from_entry(data: Dict[str, Any], secrets: Dict[str, str] , datacat_type: str, oid: str):
     """returns an aiflow connection from the data provided in the datacat entry and the secrets."""
     conn = get_connection_from_entry(data, datacat_type, oid)
-    conn.password = secrets['password']
-    conn.login = secrets['login']
+    conn.password = secrets.get('password')
+    conn.login = secrets.get('login')
     # add all remaining secrets to extra
     extra = conn.extra
     for key in secrets.keys():
@@ -76,16 +77,14 @@ class DatacatSecretsBackend(BaseSecretsBackend):
 
     def __init__(self, **kwargs):
         log.debug("Init of Datacat Secrets Backend")
-        self.url = kwargs["url"]
-        self.user = kwargs["user"]
-        self.password = kwargs["password"]
+        self.url = kwargs.get("url", "http://localhost")
+        self.user = kwargs.get("user", "")
+        self.password = kwargs.get("password", "")
 
     def get_connection(self, conn_id: str):
         """returns a Connection created from the <conenction_type>/<conn_id> entry in the datacatalog"""
-        # only for testing: check that a specific oid has been requested
+        
         log.debug(f"Get connection: {conn_id}")
-        if conn_id != "860355e9-975f-4253-9421-1815e20c879b":
-            return None
 
         secrets_conn = DataCatConnectionWithSecrets(self.url, self.user, self.password)
         data = secrets_conn.get_entry(connection_backend_type, conn_id)