diff --git a/src/datacat_integration/secrets.py b/src/datacat_integration/secrets.py
index a5ceec38764251f86163cf99291c0a8ce1c3264e..efaafe0e7d9a364bf1e436a891de85947ec4fd95 100644
--- a/src/datacat_integration/secrets.py
+++ b/src/datacat_integration/secrets.py
@@ -82,6 +82,22 @@ class DataCatConnectionWithSecrets(DataCatConnection):
else:
raise ConnectionError(response.text)
+ def set_secret_value(self, datacat_type: str, oid: str, key: str, val: str):
+ headers = {
+ 'accept' : 'application/json',
+ 'Content-Type' : 'application/json',
+ 'Authorization' : 'Bearer {}'.format(self._auth_token)
+ }
+ data = {
+ "key" : key,
+ "secret" : val
+ }
+ url = urljoin(self.url, f"{datacat_type}/{oid}/secrets")
+ response = requests.post(url, headers=headers, json=data)
+ if not response.ok:
+ raise ConnectionError(response.text)
+
+
class DatacatSecretsBackend(BaseSecretsBackend):
"""A Backend for directly managing connections in apache airflow"""
diff --git a/tests/test_connection.py b/tests/test_connection.py
index ce3e8c6d9c53a33c74ccda18c72670fd6ea093f5..2064619ccb516977c505678d9a67c5603b6a54b5 100644
--- a/tests/test_connection.py
+++ b/tests/test_connection.py
@@ -1,6 +1,7 @@
from unittest import TestCase
import os, random, json
from dotenv import load_dotenv
+import requests
from datacat_integration.connection import DataCatalogEntry, DataCatConnection, get_connection_from_entry
from .delete_created_entries import delete_entries
@@ -51,6 +52,9 @@ class EntryTest(TestCase):
self.assertEqual(json_from_entry.replace(" ", ""), self.json_string.replace(" ", ""))
class ConnectionTest(TestCase):
+ connA = ""
+ connB = ""
+ toDelete = []
def setUp(self) -> None:
load_dotenv("tests/testing-authentication.env", verbose=False) # does nothing if file is not present
@@ -61,9 +65,31 @@ class ConnectionTest(TestCase):
self.assertIsNotNone(self.url)
self.assertIsNotNone(self.user)
self.assertIsNotNone(self.password)
+
+ # create entries, add to toDelete
+ conn = DataCatConnection(self.url, self.user, self.password)
+ self.connA = conn.create_entry("storage_target", DataCatalogEntry("Yet Another Test Object", "1234", {"1" : "2", "3" : "4"}))
+ self.connB = conn.create_entry("storage_target", DataCatalogEntry("DLS-Testing-Connection-Object 1", "1234", {"1" : "2", "3" : "4"}))
+ self.toDelete.append(self.connA)
+ self.toDelete.append(self.connB)
+
+ for i in range(10):
+ conn.create_entry("storage_target", DataCatalogEntry("foo", str(i+1), {"1" : "2", "3" : "4"}))
def tearDown(self) -> None:
delete_entries()
+ conn = DataCatConnection(self.url, self.user, self.password)
+ token = conn._auth_token
+
+ headers = {
+ 'accept' : 'application/json',
+ 'Authorization' : f'Bearer {token}'
+ }
+
+ for entry in self.toDelete:
+ requests.delete(f"{self.url}/storage_target/{entry}", headers=headers)
+ self.toDelete = []
+
def test_create_token(self):
connection = DataCatConnection(self.url, self.user, self.password) # creates token on __init__
@@ -81,7 +107,7 @@ class ConnectionTest(TestCase):
# may fail in future when the testing instance is moved or redeployed - test may need to be updated in that case, since the tested entry may no longer exist
def test_get_entry(self):
connection = DataCatConnection(self.url, self.user, self.password)
- entry_json = connection.get_entry("storage_target", "57fac614-a664-49b6-9f47-1bde7a46486c")
+ entry_json = connection.get_entry("storage_target", self.connA)
entry = DataCatalogEntry.from_json(entry_json)
self.assertEqual(entry.name, "Yet Another Test Object")
self.assertEqual(entry.url, "1234")
@@ -101,7 +127,7 @@ class ConnectionTest(TestCase):
def test_list_entries(self):
connection = DataCatConnection(self.url, self.user, self.password)
entries = connection.list_type("storage_target")
- self.assertIn("297e21dc-27f2-4f09-98df-8c0e8b78a5a4", [e[1] for e in entries])
+ self.assertIn(self.connB, [e[1] for e in entries])
self.assertIn("DLS-Testing-Connection-Object 1", [e[0] for e in entries])
self.assertGreaterEqual(len(entries), 5)
diff --git a/tests/test_secrets.py b/tests/test_secrets.py
index a275b6977a68cb6220dc425a04e79c1a81cff927..54443ae963636a36bcd17a1326641d562ede7094 100644
--- a/tests/test_secrets.py
+++ b/tests/test_secrets.py
@@ -1,12 +1,19 @@
from unittest import TestCase
import os, json
import uuid
+
+import requests
from dotenv import load_dotenv
+from datacat_integration.connection import DataCatalogEntry
from datacat_integration.secrets import DataCatConnectionWithSecrets, get_connection_with_secrets_from_entry, DatacatSecretsBackend, validate_uuid
class TestSecretsBackend(TestCase):
+
+ conns = {}
+ toDelete = []
+
def setUp(self):
load_dotenv("tests/testing-authentication.env", verbose=False) # does nothing if file is not present
self.backend = DatacatSecretsBackend()
@@ -17,10 +24,51 @@ class TestSecretsBackend(TestCase):
self.assertIsNotNone(self.backend.url)
self.assertIsNotNone(self.backend.user)
self.assertIsNotNone(self.backend.password)
+
+ # create connections needed for tests
+ conn = DataCatConnectionWithSecrets(self.backend.url, self.backend.user, self.backend.password)
+ conns = ["A", "B", "C"]
+ data = {}
+ secrets = {}
+ type = {}
+
+ type["A"] = "airflow_connections"
+ data["A"] = {"conn_type" : "http", "port" : "443", "some_public_extra" : "12345", "some_other_extra_to_be_overwritten_by_secret" : "this should not be shown with secrets"}
+ secrets["A"] = {"some_extra" : "secret_12345", "some_other_extra_to_be_overwritten_by_secret" : "secret_67890"}
+
+ type["B"] = "storage_target"
+ data["B"] = {"1" : "2"}
+ secrets["B"] = {}
+
+ type["C"] = "dataset"
+ data["C"] = {"conn_type" : "http", "port" : "443", "some_public_extra" : "12345"}
+ secrets["C"] = {}
+
+ for i in conns:
+ self.conns[i] = conn.create_entry(type[i], DataCatalogEntry("foo", "bar", data[i]))
+ self.toDelete.append(self.conns[i])
+ for key in secrets[i]:
+ conn.set_secret_value(type[i], self.conns[i], key, secrets[i][key])
+
+
+ def tearDown(self):
+
+ conn = DataCatConnectionWithSecrets(self.backend.url, self.backend.user, self.backend.password)
+ token = conn._auth_token
+
+ headers = {
+ 'accept' : 'application/json',
+ 'Authorization' : f'Bearer {token}'
+ }
+
+ for entry in self.toDelete:
+ requests.delete(f"{self.backend.url}/storage_target/{entry}", headers=headers)
+ self.toDelete = []
+
def test_get_connection_from_oid(self):
- conn = self.backend.get_connection("1b02cde7-f079-4053-93a7-9ac299221a01")
- self.assertEqual(conn.conn_id, "airflow_connections/1b02cde7-f079-4053-93a7-9ac299221a01-connection")
+ conn = self.backend.get_connection(self.conns["A"])
+ self.assertEqual(conn.conn_id, "airflow_connections/" + self.conns["A"] + "-connection")
self.assertEqual(conn.conn_type, "http")
self.assertEqual(conn.port, "443")
self.assertEqual(conn.extra_dejson['some_extra'], "secret_12345")
@@ -38,15 +86,15 @@ class TestSecretsBackend(TestCase):
self.assertIsNone(self.backend.get_connection("invalid_uuid"))
def test_type_and_oid(self):
- conn = self.backend.get_connection("storage_target/57fac614-a664-49b6-9f47-1bde7a46486c")
- self.assertEqual(conn.conn_id, "storage_target/57fac614-a664-49b6-9f47-1bde7a46486c-connection")
+ conn = self.backend.get_connection("storage_target/" + self.conns["B"])
+ self.assertEqual(conn.conn_id, "storage_target/" + self.conns["B"] + "-connection")
self.assertEqual(conn.extra_dejson['1'], '2')
def test_no_oid(self):
self.assertIsNone(self.backend.get_connection(""))
def test_with_connection_oid_property(self):
- conn = self.backend.get_connection("dataset/7f228d93-0350-419d-874e-926ac8b7e03d")
+ conn = self.backend.get_connection("dataset/" + self.conns["C"])
self.assertEqual(conn.conn_type, "http")
self.assertEqual(conn.port, '443')
self.assertEqual(conn.extra_dejson['some_public_extra'], "12345")
@@ -76,6 +124,30 @@ class TestSecretsConnection(TestCase):
self.key_value_map = { "login": "foo", "password": "bar", "some_extra": "secret_12345", "some_other_extra": "secret_67890",
"some_other_extra_to_be_overwritten_by_secret": "secret_67890" }
+
+ # create connections needed for tests
+ conn = DataCatConnectionWithSecrets(self.url, self.user, self.password)
+
+ data = {"some_public_extra" : "12345", "some_other_extra_to_be_overwritten_by_secret" : "this should not be shown with secrets"}
+ secrets = { "login": "foo", "password": "bar", "some_extra": "secret_12345", "some_other_extra": "secret_67890",
+ "some_other_extra_to_be_overwritten_by_secret": "secret_67890" }
+
+ self.oid = conn.create_entry(self.conn_type, DataCatalogEntry("foo", "bar", data))
+
+ for key in secrets:
+ conn.set_secret_value(self.conn_type, self.oid, key, secrets[key])
+
+ def tearDown(self):
+ conn = DataCatConnectionWithSecrets(self.url, self.user, self.password)
+ token = conn._auth_token
+
+ headers = {
+ 'accept' : 'application/json',
+ 'Authorization' : f'Bearer {token}'
+ }
+
+ requests.delete(f"{self.url}/storage_target/{self.oid}", headers=headers)
+
def test_get_keys(self):
keys = json.loads(self.connection.get_secrets_keys(self.conn_type, self.oid))
self.assertListEqual(keys, self.keys)