Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
D
Data Logistics Service
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
eFlows4HPC WP2
Data Logistics Service
Commits
0a59c663
Commit
0a59c663
authored
3 years ago
by
Jedrzej Rybicki
Browse files
Options
Downloads
Patches
Plain Diff
branch for new version of airflow (2.2.5), bmbf stuff moved to image
parent
3e2a6631
No related branches found
No related tags found
1 merge request
!5
branch for new version of airflow (2.2.5), bmbf stuff moved to image
Pipeline
#97600
passed
3 years ago
Stage: test
Stage: build
Changes
2
Pipelines
1
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
dockers/docker-compose.yaml
+2
-60
2 additions, 60 deletions
dockers/docker-compose.yaml
dockers/eflows-airflow.docker
+6
-4
6 additions, 4 deletions
dockers/eflows-airflow.docker
with
8 additions
and
64 deletions
dockers/docker-compose.yaml
+
2
−
60
View file @
0a59c663
...
...
@@ -64,8 +64,6 @@ x-airflow-common:
-
./config/airflow.cfg:/opt/airflow/airflow.cfg
-
/persistent_data/logs:/opt/airflow/logs
-
./plugins:/opt/airflow/plugins
-
./templates/main.html:/home/airflow/.local/lib/python3.7/site-packages/airflow/www/templates/airflow/main.html
-
./templates/img/BMBF_gefoerdert_2017_en.jpg:/home/airflow/.local/lib/python3.7/site-packages/airflow/www/static/BMBF_gefoerdert_2017_en.jpg
user
:
"
${AIRFLOW_UID:-50000}:0"
depends_on
:
&airflow-common-depends-on
...
...
@@ -116,7 +114,7 @@ services:
-
postgres-db-volume:/var/lib/postgresql/data
healthcheck
:
test
:
[
"
CMD"
,
"
pg_isready"
,
"
-U"
,
"
airflow"
]
interval
:
5
s
interval
:
60
s
retries
:
5
restart
:
always
...
...
@@ -126,7 +124,7 @@ services:
-
6379
healthcheck
:
test
:
[
"
CMD"
,
"
redis-cli"
,
"
ping"
]
interval
:
5
s
interval
:
60
s
timeout
:
30s
retries
:
50
restart
:
always
...
...
@@ -220,62 +218,6 @@ services:
command
:
-
-c
-
|
function ver() {
printf "%04d%04d%04d%04d" $${1//./ }
}
airflow_version=$$(gosu airflow airflow version)
airflow_version_comparable=$$(ver $${airflow_version})
min_airflow_version=2.2.0
min_airflow_version_comparable=$$(ver $${min_airflow_version})
if (( airflow_version_comparable < min_airflow_version_comparable )); then
echo
echo -e "\033[1;31mERROR!!!: Too old Airflow version $${airflow_version}!\e[0m"
echo "The minimum Airflow version supported: $${min_airflow_version}. Only use this or higher!"
echo
exit 1
fi
if [[ -z "${AIRFLOW_UID}" ]]; then
echo
echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m"
echo "If you are on Linux, you SHOULD follow the instructions below to set "
echo "AIRFLOW_UID environment variable, otherwise files will be owned by root."
echo "For other operating systems you can get rid of the warning with manually created .env file:"
echo " See: https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#setting-the-right-airflow-user"
echo
fi
one_meg=1048576
mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg))
cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat)
disk_available=$$(df / | tail -1 | awk '{print $$4}')
warning_resources="false"
if (( mem_available < 4000 )) ; then
echo
echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m"
echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))"
echo
warning_resources="true"
fi
if (( cpus_available < 2 )); then
echo
echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m"
echo "At least 2 CPUs recommended. You have $${cpus_available}"
echo
warning_resources="true"
fi
if (( disk_available < one_meg * 10 )); then
echo
echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m"
echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))"
echo
warning_resources="true"
fi
if [[ $${warning_resources} == "true" ]]; then
echo
echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m"
echo "Please follow the instructions to increase amount of resources available:"
echo " https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#before-you-begin"
echo
fi
mkdir -p /sources/logs /sources/dags /sources/plugins
chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins}
exec /entrypoint airflow version
...
...
This diff is collapsed.
Click to expand it.
dockers/eflows-airflow.docker
+
6
−
4
View file @
0a59c663
FROM
apache/airflow:2.2.
3
FROM
apache/airflow:2.2.
5
USER
root
RUN
apt update
&&
apt
install
git
-y
RUN
apt update
&&
apt
install
git
-y
&&
apt clean
&&
rm
-rf
/var/lib/apt/lists/
*
COPY
./templates/main.html /home/airflow/.local/lib/python3.7/site-packages/airflow/www/templates/airflow/main.html
COPY
./templates/img/BMBF_gefoerdert_2017_en.jpg /home/airflow/.local/lib/python3.7/site-packages/airflow/www/static/BMBF_gefoerdert_2017_en.jpg
USER
airflow
RUN
pip
--version
RUN
pip
--version
&&
python
-m
pip
install
--upgrade
pip
ADD
requirements.txt /requirements.txt
RUN
pip
install
-r
/requirements.txt
\ No newline at end of file
This diff is collapsed.
Click to expand it.
Jedrzej Rybicki
@rybicki1
mentioned in issue
#53 (closed)
·
3 years ago
mentioned in issue
#53 (closed)
mentioned in issue #53
Toggle commit list
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment