Skip to content
Snippets Groups Projects
Commit 27e81d6d authored by Jedrzej Rybicki's avatar Jedrzej Rybicki
Browse files

Update secrets.py

parent 1d3a632a
No related branches found
No related tags found
No related merge requests found
Pipeline #86266 failed
from typing import Any, Dict, Set
from urllib.parse import urljoin
from airflow.secrets import BaseSecretsBackend
from airflow.models.connection import Connection
#from airflow.models.connection import Connection
import requests
import logging
from datacat_integration.connection import DataCatConnection
from datacat_integration.hooks import get_connection_from_entry
#from datacat_integration.hooks import get_connection_from_entry
connection_backend_type = "airflow_connections"
log = logging.getLogger(__name__)
def get_connection_from_entry(data: Dict[str, Any], datacat_type: str, oid: str):
"""returns an aiflow connection from the data provided in the datacat entry."""
conn_type = data['metadata']['conn_type']
host = data['metadata']['host']
port = data['metadata']['port']
schema = data['metadata']['schema']
conn_id = f"{datacat_type}/{oid}-connection"
# set all remaining metadata as extra
extra = {}
for key in data['metadata'].keys():
if key in ['conn_type', 'host', 'port', 'schema']:
continue
extra[key] = data['metadata'][key]
return dict()
#Connection(
# conn_id=conn_id,
# conn_type=conn_type,
# host=host,
# port=port,
# description="Automatically generated Connection for the given entry.",
# schema=schema,
# extra=extra
#)
def get_connection_with_secrets_from_entry(data: Dict[str, Any], secrets: Dict[str, str] , datacat_type: str, oid: str) -> Connection:
def get_connection_with_secrets_from_entry(data: Dict[str, Any], secrets: Dict[str, str] , datacat_type: str, oid: str):
"""returns an aiflow connection from the data provided in the datacat entry and the secrets."""
conn = get_connection_from_entry(data, datacat_type, oid)
conn.password = secrets['password']
......@@ -28,7 +54,6 @@ def get_connection_with_secrets_from_entry(data: Dict[str, Any], secrets: Dict[s
conn.extra = extra
return conn
class DataCatConnectionWithSecrets(DataCatConnection):
def __init__(self, catalog_url: str = "", username: str = "", password: str = ""):
super().__init__(catalog_url=catalog_url, username=username, password=password)
......@@ -92,5 +117,9 @@ class DatacatSecretsBackend(BaseSecretsBackend):
secrets_conn = DataCatConnectionWithSecrets(self.url, self.user, self.password)
data = secrets_conn.get_object(connection_backend_type, conn_id)
secrets = secrets_conn.get_all_secret_key_value(connection_backend_type, conn_id)
data = secrets_conn.get_object(connection_backend_type, conn_id)
secrets = secrets_conn.get_all_secret_key_value(connection_backend_type, conn_id)
conn = get_connection_from_entry(data, secrets, connection_backend_type, conn_id)
return conn
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment