From 0a59c66336aa3ff6593b07191fbfee3ede5147a1 Mon Sep 17 00:00:00 2001
From: jrybicki-jsc <j.rybicki@fz-juelich.de>
Date: Mon, 11 Apr 2022 15:31:45 +0200
Subject: [PATCH] branch for new version of airflow (2.2.5), bmbf stuff moved
to image
---
dockers/docker-compose.yaml | 62 ++---------------------------------
dockers/eflows-airflow.docker | 10 +++---
2 files changed, 8 insertions(+), 64 deletions(-)
diff --git a/dockers/docker-compose.yaml b/dockers/docker-compose.yaml
index a9de206..45e7ccb 100644
--- a/dockers/docker-compose.yaml
+++ b/dockers/docker-compose.yaml
@@ -64,8 +64,6 @@ x-airflow-common:
- ./config/airflow.cfg:/opt/airflow/airflow.cfg
- /persistent_data/logs:/opt/airflow/logs
- ./plugins:/opt/airflow/plugins
- - ./templates/main.html:/home/airflow/.local/lib/python3.7/site-packages/airflow/www/templates/airflow/main.html
- - ./templates/img/BMBF_gefoerdert_2017_en.jpg:/home/airflow/.local/lib/python3.7/site-packages/airflow/www/static/BMBF_gefoerdert_2017_en.jpg
user: "${AIRFLOW_UID:-50000}:0"
depends_on:
&airflow-common-depends-on
@@ -116,7 +114,7 @@ services:
- postgres-db-volume:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "airflow"]
- interval: 5s
+ interval: 60s
retries: 5
restart: always
@@ -126,7 +124,7 @@ services:
- 6379
healthcheck:
test: ["CMD", "redis-cli", "ping"]
- interval: 5s
+ interval: 60s
timeout: 30s
retries: 50
restart: always
@@ -220,62 +218,6 @@ services:
command:
- -c
- |
- function ver() {
- printf "%04d%04d%04d%04d" $${1//./ }
- }
- airflow_version=$$(gosu airflow airflow version)
- airflow_version_comparable=$$(ver $${airflow_version})
- min_airflow_version=2.2.0
- min_airflow_version_comparable=$$(ver $${min_airflow_version})
- if (( airflow_version_comparable < min_airflow_version_comparable )); then
- echo
- echo -e "\033[1;31mERROR!!!: Too old Airflow version $${airflow_version}!\e[0m"
- echo "The minimum Airflow version supported: $${min_airflow_version}. Only use this or higher!"
- echo
- exit 1
- fi
- if [[ -z "${AIRFLOW_UID}" ]]; then
- echo
- echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m"
- echo "If you are on Linux, you SHOULD follow the instructions below to set "
- echo "AIRFLOW_UID environment variable, otherwise files will be owned by root."
- echo "For other operating systems you can get rid of the warning with manually created .env file:"
- echo " See: https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#setting-the-right-airflow-user"
- echo
- fi
- one_meg=1048576
- mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg))
- cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat)
- disk_available=$$(df / | tail -1 | awk '{print $$4}')
- warning_resources="false"
- if (( mem_available < 4000 )) ; then
- echo
- echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m"
- echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))"
- echo
- warning_resources="true"
- fi
- if (( cpus_available < 2 )); then
- echo
- echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m"
- echo "At least 2 CPUs recommended. You have $${cpus_available}"
- echo
- warning_resources="true"
- fi
- if (( disk_available < one_meg * 10 )); then
- echo
- echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m"
- echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))"
- echo
- warning_resources="true"
- fi
- if [[ $${warning_resources} == "true" ]]; then
- echo
- echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m"
- echo "Please follow the instructions to increase amount of resources available:"
- echo " https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#before-you-begin"
- echo
- fi
mkdir -p /sources/logs /sources/dags /sources/plugins
chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins}
exec /entrypoint airflow version
diff --git a/dockers/eflows-airflow.docker b/dockers/eflows-airflow.docker
index 7aae525..df0e273 100644
--- a/dockers/eflows-airflow.docker
+++ b/dockers/eflows-airflow.docker
@@ -1,9 +1,11 @@
-FROM apache/airflow:2.2.3
+FROM apache/airflow:2.2.5
USER root
-RUN apt update && apt install git -y
+RUN apt update && apt install git -y && apt clean && rm -rf /var/lib/apt/lists/*
+COPY ./templates/main.html /home/airflow/.local/lib/python3.7/site-packages/airflow/www/templates/airflow/main.html
+COPY ./templates/img/BMBF_gefoerdert_2017_en.jpg /home/airflow/.local/lib/python3.7/site-packages/airflow/www/static/BMBF_gefoerdert_2017_en.jpg
USER airflow
-RUN pip --version
+RUN pip --version && python -m pip install --upgrade pip
ADD requirements.txt /requirements.txt
-RUN pip install -r /requirements.txt
+RUN pip install -r /requirements.txt
\ No newline at end of file
--
GitLab