From 5a9255640cf3c5ffaa2b1a6b494b3881beb7ef66 Mon Sep 17 00:00:00 2001
From: Christian Boettcher <c.boettcher@fz-juelich.de>
Date: Mon, 13 Dec 2021 08:37:45 +0100
Subject: [PATCH] use custom image to get dependencies
---
.gitignore | 1 +
config/airflow.cfg | 4 ++--
dockers/eflows-airflow.docker | 7 +++++++
requirements.txt | 2 ++
4 files changed, 12 insertions(+), 2 deletions(-)
create mode 100644 dockers/eflows-airflow.docker
diff --git a/.gitignore b/.gitignore
index 6bab012..ed59847 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
dockers/.env
.vscode/*
*.pyc
+logs/
# contains data for local tests
.coverage
diff --git a/config/airflow.cfg b/config/airflow.cfg
index b304e72..ad90566 100644
--- a/config/airflow.cfg
+++ b/config/airflow.cfg
@@ -347,13 +347,13 @@ statsd_datadog_tags =
[secrets]
# Full class name of secrets backend to enable (will precede env vars and metastore in search path)
# Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend
-backend =
+backend = datacat_integration.secrets.DatacatSecretsBackend
# The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class.
# See documentation for the secrets backend you are using. JSON is expected.
# Example for AWS Systems Manager ParameterStore:
# ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}``
-backend_kwargs =
+backend_kwargs = {"url" : "https://zam10036.zam.kfa-juelich.de", "user" : "dls-testing", "password" : "dls-testing-pass"}
[cli]
# In what way should the cli access the API. The LocalClient will use the
diff --git a/dockers/eflows-airflow.docker b/dockers/eflows-airflow.docker
new file mode 100644
index 0000000..817a86e
--- /dev/null
+++ b/dockers/eflows-airflow.docker
@@ -0,0 +1,7 @@
+FROM apache/airflow:2.2.2
+
+RUN pip --version
+
+ADD requirements.txt /requirements.txt
+
+RUN pip install -r /requirements.txt
diff --git a/requirements.txt b/requirements.txt
index aeecd2f..0b4ee44 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,3 +4,5 @@ plyvel
apache-airflow-providers-ssh
apache-airflow-providers-http
apache-airflow-providers-sftp
+--index-url https://gitlab.jsc.fz-juelich.de/api/v4/projects/4405/packages/pypi/simple
+airflow-datacat-integration>=0.0.4
--
GitLab