Skip to content
Snippets Groups Projects
Commit 3b640fde authored by Christian Boettcher's avatar Christian Boettcher
Browse files

rename object to entry where applicable

parent aec8d50a
No related branches found
No related tags found
No related merge requests found
...@@ -18,7 +18,7 @@ class DataCatalogEntry: ...@@ -18,7 +18,7 @@ class DataCatalogEntry:
self.metadata = metadata self.metadata = metadata
def json(self): def json(self):
"""returns a json-compatible representation of the object.""" """returns a json-compatible representation of the entry."""
return json.dumps( return json.dumps(
{ {
"name" : self.name, "name" : self.name,
...@@ -28,7 +28,7 @@ class DataCatalogEntry: ...@@ -28,7 +28,7 @@ class DataCatalogEntry:
) )
def from_json(data: json): def from_json(data: json):
"""returns a DataCatalogEntry object from the given json string""" """returns a DataCatalogEntry entry from the given json string"""
dict_data = json.loads(data) dict_data = json.loads(data)
return DataCatalogEntry(dict_data['name'], dict_data['url'], dict_data['metadata']) return DataCatalogEntry(dict_data['name'], dict_data['url'], dict_data['metadata'])
...@@ -76,22 +76,22 @@ class DataCatConnection: ...@@ -76,22 +76,22 @@ class DataCatConnection:
return self.refresh_token() return self.refresh_token()
def get_object(self, datacat_type: str, oid: uuid.UUID): # GET /<type>/<oid> def get_entry(self, datacat_type: str, oid: uuid.UUID): # GET /<type>/<oid>
"""Returns a json of the given object from the server.""" """Returns a json of the given entry from the server."""
headers = { headers = {
'accept' : 'application/json' 'accept' : 'application/json'
} }
url = urljoin(self.url, "{}/{}".format(datacat_type, oid)) url = urljoin(self.url, "{}/{}".format(datacat_type, oid))
return json.dumps(requests.get(url, headers=headers).json()) return json.dumps(requests.get(url, headers=headers).json())
def create_object(self, datacat_type: str, object: DataCatalogEntry): # POST /<type> def create_entry(self, datacat_type: str, entry: DataCatalogEntry): # POST /<type>
"""Creates a new object in the datacatalog. Returns the oid of successful.""" """Creates a new entry in the datacatalog. Returns the oid of successful."""
headers = { headers = {
'accept' : 'application/json', 'accept' : 'application/json',
'Content-Type' : 'application/json', 'Content-Type' : 'application/json',
'Authorization' : 'Bearer {}'.format(self._auth_token) 'Authorization' : 'Bearer {}'.format(self._auth_token)
} }
response = requests.post(urljoin(self.url, datacat_type), headers=headers, data=object.json()) response = requests.post(urljoin(self.url, datacat_type), headers=headers, data=entry.json())
if response.ok: if response.ok:
return response.json()[0] return response.json()[0]
else: else:
......
...@@ -13,7 +13,7 @@ from datacat_integration.auth import BearerAuth ...@@ -13,7 +13,7 @@ from datacat_integration.auth import BearerAuth
def get_connection_from_entry(data: Dict[str, Any], datacat_type: str, oid: str) -> Connection: def get_connection_from_entry(data: Dict[str, Any], datacat_type: str, oid: str) -> Connection:
"""returns an aiflow connection from the data provided in the datacat entry.""" """returns an airflow connection from the data provided in the datacat entry."""
conn_type = data['metadata']['conn_type'] conn_type = data['metadata']['conn_type']
host = data['metadata']['host'] host = data['metadata']['host']
port = data['metadata']['port'] port = data['metadata']['port']
...@@ -51,19 +51,18 @@ class DataCatalogHook(HttpHook): ...@@ -51,19 +51,18 @@ class DataCatalogHook(HttpHook):
conn.auth = BearerAuth(self.connection.get_token()) conn.auth = BearerAuth(self.connection.get_token())
return conn return conn
def get_object(self, datacat_type: str, oid: str): def get_entry(self, datacat_type: str, oid: str):
return self.connection.get_object(datacat_type, oid) return self.connection.get_entry(datacat_type, oid)
def create_object(self, datacat_type: str, object: DataCatalogEntry): def create_entry(self, datacat_type: str, entry: DataCatalogEntry):
return self.connection.create_object(datacat_type, object) return self.connection.create_entry(datacat_type, entry)
def list_type(self, datacat_type: str): def list_type(self, datacat_type: str):
return self.connection.list_type(datacat_type) return self.connection.list_type(datacat_type)
def create_get_object_connection(self, datacat_type: str, oid: str): def create_get_entry_connection(self, datacat_type: str, oid: str):
object = self.get_object(datacat_type=datacat_type, oid=oid) entry = self.get_entry(datacat_type, oid)
# TODO decide some factors, such as connection type conn = get_connection_from_entry(entry, datacat_type, oid)
conn = get_connection_from_entry(object, datacat_type, oid)
session = settings.Session() session = settings.Session()
......
...@@ -15,7 +15,7 @@ class GetDatacatalogEntryOperator(BaseOperator): ...@@ -15,7 +15,7 @@ class GetDatacatalogEntryOperator(BaseOperator):
def execute(self, context): def execute(self, context):
cat_conn = DataCatalogHook('datacatalog') cat_conn = DataCatalogHook('datacatalog')
return cat_conn.get_object(self.datacat_type, self.oid) return cat_conn.get_entry(self.datacat_type, self.oid)
class GetDatacatalogEntryConnectionOperator(BaseOperator): class GetDatacatalogEntryConnectionOperator(BaseOperator):
"""This task returns the data for an entry in the datacatalog.""" """This task returns the data for an entry in the datacatalog."""
...@@ -31,4 +31,4 @@ class GetDatacatalogEntryConnectionOperator(BaseOperator): ...@@ -31,4 +31,4 @@ class GetDatacatalogEntryConnectionOperator(BaseOperator):
def execute(self, context): def execute(self, context):
hook = DataCatalogHook("datacatalog") hook = DataCatalogHook("datacatalog")
hook.create_get_object_connection(self.datacat_type, self.oid) hook.create_get_entry_connection(self.datacat_type, self.oid)
\ No newline at end of file \ No newline at end of file
...@@ -77,20 +77,19 @@ class DatacatSecretsBackend(BaseSecretsBackend): ...@@ -77,20 +77,19 @@ class DatacatSecretsBackend(BaseSecretsBackend):
def __init__(self, **kwargs): def __init__(self, **kwargs):
log.debug("Init of Datacat Secrets Backend") log.debug("Init of Datacat Secrets Backend")
super().__init__(**kwargs)
self.url = kwargs["url"] self.url = kwargs["url"]
self.user = kwargs["user"] self.user = kwargs["user"]
self.password = kwargs["password"] self.password = kwargs["password"]
def get_connection(self, conn_id: str): def get_connection(self, conn_id: str):
"""returns a Connection object created from the <conenction_type>/<conn_id> object in the datacatalog""" """returns a Connection created from the <conenction_type>/<conn_id> entry in the datacatalog"""
# only for testing: check that a specific oid has been requested # only for testing: check that a specific oid has been requested
log.debug(f"Get connection: {conn_id}") log.debug(f"Get connection: {conn_id}")
if conn_id != "860355e9-975f-4253-9421-1815e20c879b": if conn_id != "860355e9-975f-4253-9421-1815e20c879b":
return None return None
secrets_conn = DataCatConnectionWithSecrets(self.url, self.user, self.password) secrets_conn = DataCatConnectionWithSecrets(self.url, self.user, self.password)
data = secrets_conn.get_object(connection_backend_type, conn_id) data = secrets_conn.get_entry(connection_backend_type, conn_id)
secrets = secrets_conn.get_all_secret_key_value(connection_backend_type, conn_id) secrets = secrets_conn.get_all_secret_key_value(connection_backend_type, conn_id)
conn = get_connection_from_entry(data, secrets, connection_backend_type, conn_id) conn = get_connection_from_entry(data, secrets, connection_backend_type, conn_id)
return conn return conn
\ No newline at end of file
...@@ -45,10 +45,10 @@ class ConnectionTest(TestCase): ...@@ -45,10 +45,10 @@ class ConnectionTest(TestCase):
self.assertEqual(token, token2) self.assertEqual(token, token2)
# may fail in future when the testing instance is moved or redeployed - test may need to be updated in that case, since the tested object may no longer exist # may fail in future when the testing instance is moved or redeployed - test may need to be updated in that case, since the tested entry may no longer exist
def test_get_entry(self): def test_get_entry(self):
connection = DataCatConnection(self.url, self.user, self.password) connection = DataCatConnection(self.url, self.user, self.password)
entry_json = connection.get_object("storage_target", "7aa3877e-2860-4c65-8d48-3e080ceedca2") entry_json = connection.get_entry("storage_target", "7aa3877e-2860-4c65-8d48-3e080ceedca2")
entry = DataCatalogEntry.from_json(entry_json) entry = DataCatalogEntry.from_json(entry_json)
self.assertEqual(entry.name, "Yet Another Test Object") self.assertEqual(entry.name, "Yet Another Test Object")
self.assertEqual(entry.url, "1234") self.assertEqual(entry.url, "1234")
...@@ -58,13 +58,13 @@ class ConnectionTest(TestCase): ...@@ -58,13 +58,13 @@ class ConnectionTest(TestCase):
def test_create_entry(self): def test_create_entry(self):
connection = DataCatConnection(self.url, self.user, self.password) connection = DataCatConnection(self.url, self.user, self.password)
entry = DataCatalogEntry("foo", "bar", {"1" : "2", "source" : "automated tests", "random" : str(random.getrandbits(32))}) entry = DataCatalogEntry("foo", "bar", {"1" : "2", "source" : "automated tests", "random" : str(random.getrandbits(32))})
oid = connection.create_object("storage_target", entry) oid = connection.create_entry("storage_target", entry)
entry_from_server = DataCatalogEntry.from_json(connection.get_object("storage_target", oid)) entry_from_server = DataCatalogEntry.from_json(connection.get_entry("storage_target", oid))
self.assertDictEqual(entry.metadata, entry_from_server.metadata) self.assertDictEqual(entry.metadata, entry_from_server.metadata)
self.assertEqual(entry.name, entry_from_server.name) self.assertEqual(entry.name, entry_from_server.name)
self.assertEqual(entry.url, entry_from_server.url) self.assertEqual(entry.url, entry_from_server.url)
# may fail in future when the testing instance is moved or redeployed - test may need to be updated in that case, since the tested objects may no longer exist # may fail in future when the testing instance is moved or redeployed - test may need to be updated in that case, since the tested entries may no longer exist
def test_list_entries(self): def test_list_entries(self):
connection = DataCatConnection(self.url, self.user, self.password) connection = DataCatConnection(self.url, self.user, self.password)
entries = connection.list_type("storage_target") entries = connection.list_type("storage_target")
...@@ -79,4 +79,4 @@ class ConnectionTest(TestCase): ...@@ -79,4 +79,4 @@ class ConnectionTest(TestCase):
self.assertRaises(ConnectionError, connection.refresh_token) self.assertRaises(ConnectionError, connection.refresh_token)
connection._auth_token = "foobar" connection._auth_token = "foobar"
self.assertRaises(ConnectionError, connection.get_token) self.assertRaises(ConnectionError, connection.get_token)
self.assertRaises(ConnectionError, connection.create_object, "storage_target", DataCatalogEntry("foo", "bar", {})) self.assertRaises(ConnectionError, connection.create_entry, "storage_target", DataCatalogEntry("foo", "bar", {}))
\ No newline at end of file \ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment