update tests to work with new deployments

src/datacat_integration/secrets.py

@@ -82,6 +82,22 @@ class DataCatConnectionWithSecrets(DataCatConnection):
         else:
             raise ConnectionError(response.text)
 
+    def set_secret_value(self, datacat_type: str, oid: str, key: str, val: str):
+        headers = {
+            'accept' : 'application/json',
+            'Content-Type' : 'application/json',
+            'Authorization' : 'Bearer {}'.format(self._auth_token)
+        }
+        data = {
+            "key" : key,
+            "secret" : val
+        }
+        url = urljoin(self.url, f"{datacat_type}/{oid}/secrets")
+        response = requests.post(url, headers=headers, json=data)
+        if not response.ok:
+            raise ConnectionError(response.text)
+
+
 class DatacatSecretsBackend(BaseSecretsBackend):
     """A Backend for directly managing connections in apache airflow"""
 

tests/test_connection.py

 from unittest import TestCase
 import os, random, json
 from dotenv import load_dotenv
+import requests
 
 from datacat_integration.connection import DataCatalogEntry, DataCatConnection, get_connection_from_entry
 from .delete_created_entries import delete_entries
@@ -51,6 +52,9 @@ class EntryTest(TestCase):
         self.assertEqual(json_from_entry.replace(" ", ""), self.json_string.replace(" ", ""))
 
 class ConnectionTest(TestCase):
+    connA = ""
+    connB = ""
+    toDelete = []
 
     def setUp(self) -> None:
         load_dotenv("tests/testing-authentication.env", verbose=False) #  does nothing if file is not present
@@ -62,8 +66,30 @@ class ConnectionTest(TestCase):
         self.assertIsNotNone(self.user)
         self.assertIsNotNone(self.password)
 
+        # create entries, add to toDelete
+        conn = DataCatConnection(self.url, self.user, self.password)
+        self.connA = conn.create_entry("storage_target", DataCatalogEntry("Yet Another Test Object", "1234", {"1" : "2", "3" : "4"}))
+        self.connB = conn.create_entry("storage_target", DataCatalogEntry("DLS-Testing-Connection-Object 1", "1234", {"1" : "2", "3" : "4"}))
+        self.toDelete.append(self.connA)
+        self.toDelete.append(self.connB)
+
+        for i in range(10):
+            conn.create_entry("storage_target", DataCatalogEntry("foo", str(i+1), {"1" : "2", "3" : "4"}))
+    
     def tearDown(self) -> None:
         delete_entries()
+        conn = DataCatConnection(self.url, self.user, self.password)
+        token = conn._auth_token
+
+        headers = {
+            'accept' : 'application/json',
+            'Authorization' : f'Bearer {token}'
+        }
+
+        for entry in self.toDelete:
+            requests.delete(f"{self.url}/storage_target/{entry}", headers=headers)
+        self.toDelete = []
+        
 
     def test_create_token(self):
         connection = DataCatConnection(self.url, self.user, self.password) # creates token on __init__
@@ -81,7 +107,7 @@ class ConnectionTest(TestCase):
     # may fail in future when the testing instance is moved or redeployed - test may need to be updated in that case, since the tested entry may no longer exist
     def test_get_entry(self):
         connection = DataCatConnection(self.url, self.user, self.password)
-        entry_json = connection.get_entry("storage_target", "57fac614-a664-49b6-9f47-1bde7a46486c")
+        entry_json = connection.get_entry("storage_target", self.connA)
         entry = DataCatalogEntry.from_json(entry_json)
         self.assertEqual(entry.name, "Yet Another Test Object")
         self.assertEqual(entry.url, "1234")
@@ -101,7 +127,7 @@ class ConnectionTest(TestCase):
     def test_list_entries(self):
         connection = DataCatConnection(self.url, self.user, self.password)
         entries = connection.list_type("storage_target")
-        self.assertIn("297e21dc-27f2-4f09-98df-8c0e8b78a5a4", [e[1] for e in entries])
+        self.assertIn(self.connB, [e[1] for e in entries])
         self.assertIn("DLS-Testing-Connection-Object 1", [e[0] for e in entries])
         self.assertGreaterEqual(len(entries), 5)
 

tests/test_secrets.py

 from unittest import TestCase
 import os, json
 import uuid
+
+import requests
 from dotenv import load_dotenv
+from datacat_integration.connection import DataCatalogEntry
 
 from datacat_integration.secrets import DataCatConnectionWithSecrets, get_connection_with_secrets_from_entry, DatacatSecretsBackend, validate_uuid
 
 
 class TestSecretsBackend(TestCase):
+
+    conns = {}
+    toDelete = []
+
     def setUp(self):
         load_dotenv("tests/testing-authentication.env", verbose=False) #  does nothing if file is not present
         self.backend = DatacatSecretsBackend()
@@ -18,9 +25,50 @@ class TestSecretsBackend(TestCase):
         self.assertIsNotNone(self.backend.user)
         self.assertIsNotNone(self.backend.password)
 
+        # create connections needed for tests
+        conn = DataCatConnectionWithSecrets(self.backend.url, self.backend.user, self.backend.password)
+        conns = ["A", "B", "C"]
+        data = {}
+        secrets = {}
+        type = {}
+
+        type["A"] = "airflow_connections"
+        data["A"] = {"conn_type" : "http", "port" : "443", "some_public_extra" : "12345", "some_other_extra_to_be_overwritten_by_secret" : "this should not be shown with secrets"}
+        secrets["A"] = {"some_extra" : "secret_12345", "some_other_extra_to_be_overwritten_by_secret" : "secret_67890"}
+        
+        type["B"] = "storage_target"
+        data["B"] = {"1" : "2"}
+        secrets["B"] = {}
+        
+        type["C"] = "dataset"
+        data["C"] = {"conn_type" : "http", "port" : "443", "some_public_extra" : "12345"}
+        secrets["C"] = {}
+
+        for i in conns:
+            self.conns[i] = conn.create_entry(type[i], DataCatalogEntry("foo", "bar", data[i]))
+            self.toDelete.append(self.conns[i])
+            for key in secrets[i]:
+                conn.set_secret_value(type[i], self.conns[i], key, secrets[i][key])
+            
+
+    def tearDown(self):
+        
+        conn = DataCatConnectionWithSecrets(self.backend.url, self.backend.user, self.backend.password)
+        token = conn._auth_token
+
+        headers = {
+            'accept' : 'application/json',
+            'Authorization' : f'Bearer {token}'
+        }
+
+        for entry in self.toDelete:
+            requests.delete(f"{self.backend.url}/storage_target/{entry}", headers=headers)
+        self.toDelete = []
+        
+    
     def test_get_connection_from_oid(self):
-        conn = self.backend.get_connection("1b02cde7-f079-4053-93a7-9ac299221a01")
-        self.assertEqual(conn.conn_id, "airflow_connections/1b02cde7-f079-4053-93a7-9ac299221a01-connection")
+        conn = self.backend.get_connection(self.conns["A"])
+        self.assertEqual(conn.conn_id, "airflow_connections/" + self.conns["A"] + "-connection")
         self.assertEqual(conn.conn_type, "http")
         self.assertEqual(conn.port, "443")
         self.assertEqual(conn.extra_dejson['some_extra'], "secret_12345")
@@ -38,15 +86,15 @@ class TestSecretsBackend(TestCase):
         self.assertIsNone(self.backend.get_connection("invalid_uuid"))
 
     def test_type_and_oid(self):
-        conn = self.backend.get_connection("storage_target/57fac614-a664-49b6-9f47-1bde7a46486c")
-        self.assertEqual(conn.conn_id, "storage_target/57fac614-a664-49b6-9f47-1bde7a46486c-connection")
+        conn = self.backend.get_connection("storage_target/" + self.conns["B"])
+        self.assertEqual(conn.conn_id, "storage_target/" + self.conns["B"] + "-connection")
         self.assertEqual(conn.extra_dejson['1'], '2')
 
     def test_no_oid(self):
         self.assertIsNone(self.backend.get_connection(""))
 
     def test_with_connection_oid_property(self):
-        conn = self.backend.get_connection("dataset/7f228d93-0350-419d-874e-926ac8b7e03d")
+        conn = self.backend.get_connection("dataset/" + self.conns["C"])
         self.assertEqual(conn.conn_type, "http")
         self.assertEqual(conn.port, '443')
         self.assertEqual(conn.extra_dejson['some_public_extra'], "12345")
@@ -76,6 +124,30 @@ class TestSecretsConnection(TestCase):
         self.key_value_map = { "login": "foo", "password": "bar", "some_extra": "secret_12345", "some_other_extra": "secret_67890", 
         "some_other_extra_to_be_overwritten_by_secret": "secret_67890" }
 
+        
+        # create connections needed for tests
+        conn = DataCatConnectionWithSecrets(self.url, self.user, self.password)
+
+        data = {"some_public_extra" : "12345", "some_other_extra_to_be_overwritten_by_secret" : "this should not be shown with secrets"}
+        secrets = { "login": "foo", "password": "bar", "some_extra": "secret_12345", "some_other_extra": "secret_67890", 
+        "some_other_extra_to_be_overwritten_by_secret": "secret_67890" }
+
+        self.oid = conn.create_entry(self.conn_type, DataCatalogEntry("foo", "bar", data))
+        
+        for key in secrets:
+            conn.set_secret_value(self.conn_type, self.oid, key, secrets[key])
+
+    def tearDown(self):
+        conn = DataCatConnectionWithSecrets(self.url, self.user, self.password)
+        token = conn._auth_token
+
+        headers = {
+            'accept' : 'application/json',
+            'Authorization' : f'Bearer {token}'
+        }
+        
+        requests.delete(f"{self.url}/storage_target/{self.oid}", headers=headers)
+
     def test_get_keys(self):
         keys = json.loads(self.connection.get_secrets_keys(self.conn_type, self.oid))
         self.assertListEqual(keys, self.keys)