diff --git a/.gitignore b/.gitignore index 6bab01222a52bba92a48373a7755931e5779879a..ed59847247be44bbe5ecb45d88545eec884eaa24 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ dockers/.env .vscode/* *.pyc +logs/ # contains data for local tests .coverage diff --git a/config/airflow.cfg b/config/airflow.cfg index b304e72a91e130ec0318b88dbad24cd096d8d6b2..ad90566620303d47e3c2391f55e4af5ddb8b7aed 100644 --- a/config/airflow.cfg +++ b/config/airflow.cfg @@ -347,13 +347,13 @@ statsd_datadog_tags = [secrets] # Full class name of secrets backend to enable (will precede env vars and metastore in search path) # Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend -backend = +backend = datacat_integration.secrets.DatacatSecretsBackend # The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class. # See documentation for the secrets backend you are using. JSON is expected. # Example for AWS Systems Manager ParameterStore: # ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}`` -backend_kwargs = +backend_kwargs = {"url" : "https://zam10036.zam.kfa-juelich.de", "user" : "dls-testing", "password" : "dls-testing-pass"} [cli] # In what way should the cli access the API. The LocalClient will use the diff --git a/dockers/eflows-airflow.docker b/dockers/eflows-airflow.docker new file mode 100644 index 0000000000000000000000000000000000000000..817a86e21235125f67dc58f757cb0fc5466eb9bb --- /dev/null +++ b/dockers/eflows-airflow.docker @@ -0,0 +1,7 @@ +FROM apache/airflow:2.2.2 + +RUN pip --version + +ADD requirements.txt /requirements.txt + +RUN pip install -r /requirements.txt diff --git a/requirements.txt b/requirements.txt index aeecd2f8a68c7beca6f3259e5c3642f6349f8984..0b4ee447adaaaef3371d8d4177499e2810f9ab6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ plyvel apache-airflow-providers-ssh apache-airflow-providers-http apache-airflow-providers-sftp +--index-url https://gitlab.jsc.fz-juelich.de/api/v4/projects/4405/packages/pypi/simple +airflow-datacat-integration>=0.0.4