Skip to content
Snippets Groups Projects
Commit 759eac14 authored by Christian Boettcher's avatar Christian Boettcher
Browse files

Merge branch 'main' of gitlab.jsc.fz-juelich.de:eflows4hpc-wp2/data-logistics-service

parents d3d1b112 ef92d374
Branches main
No related tags found
No related merge requests found
......@@ -118,6 +118,16 @@ light-deploy-testing:
- ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP 'sed -i "s_eflows-airflow:latest_eflows-airflow:testing_g" /home/airflow/data-logistics-service/dockers/docker-compose.yaml'
- ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service $TESTING_DOMAIN $AIRFLOW__SECRETS__BACKEND $AIRFLOW__SECRETS__BACKEND_KWARGS $AIRFLOW_FERNET_KEY $DAG_GIT_URL $TESTING_OAUTH_ID $SSO_CLIENT_SECRET $METADATA_URL"
force-light-deploy-testing: # for deploying images generated on other branches to testing - can only be done by logged in and authorized users
stage: deploy
when: manual
<<: *ssh_setup
environment: Testing
script:
- ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "cd /home/airflow/data-logistics-service && git stash && git stash clear && git checkout main && git checkout -f $CI_COMMIT_TAG && git pull --all"
- ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP 'sed -i "s_eflows-airflow:latest_eflows-airflow:testing_g" /home/airflow/data-logistics-service/dockers/docker-compose.yaml'
- ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service $TESTING_DOMAIN $AIRFLOW__SECRETS__BACKEND $AIRFLOW__SECRETS__BACKEND_KWARGS $AIRFLOW_FERNET_KEY $DAG_GIT_URL $TESTING_OAUTH_ID $SSO_CLIENT_SECRET $METADATA_URL"
full-deploy-production:
......
This diff is collapsed.
......@@ -24,9 +24,11 @@
# The following variables are supported:
#
# AIRFLOW_IMAGE_NAME - Docker image name used to run Airflow.
# Default: apache/airflow:2.2.1
# Default: apache/airflow:2.5.1
# AIRFLOW_UID - User ID in Airflow containers
# Default: 50000
# AIRFLOW_PROJ_DIR - Base path to which all the files will be volumed.
# Default: .
# Those configurations are useful mostly in case of standalone testing/running Airflow in test/try-out mode
#
# _AIRFLOW_WWW_USER_USERNAME - Username for the administrator account (if requested).
......@@ -49,24 +51,26 @@ x-airflow-common:
environment:
&airflow-common-env
AIRFLOW__CORE__EXECUTOR: CeleryExecutor
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow
AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY}
AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth'
AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session'
AIRFLOW__SECRETS__BACKEND_KWARGS: ${AIRFLOW__SECRETS__BACKEND_KWARGS}
AIRFLOW__SECRETS__BACKEND: ${AIRFLOW__SECRETS__BACKEND}
OAUTH_CLIENT_ID: ${OAUTH_CLIENT_ID}
OAUTH_METADATA_URL: ${OAUTH_METADATA_URL}
OAUTH_CLIENT_SECRET: ${OAUTH_CLIENT_SECRET}
# _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
_PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
volumes:
- ./dags:/opt/airflow/dags
- ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags
- ./config/airflow.cfg:/opt/airflow/airflow.cfg
- /persistent_data/logs:/opt/airflow/logs
- ./plugins:/opt/airflow/plugins
- ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs
- ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins
user: "${AIRFLOW_UID:-50000}:0"
depends_on:
&airflow-common-depends-on
......@@ -76,7 +80,6 @@ x-airflow-common:
condition: service_healthy
services:
reverse-proxy:
image: "jwilder/nginx-proxy:alpine"
container_name: "reverse-proxy"
......@@ -117,7 +120,7 @@ services:
- postgres-db-volume:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "airflow"]
interval: 60s
interval: 10s
retries: 5
restart: always
......@@ -127,23 +130,21 @@ services:
- 6379
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 60s
interval: 10s
timeout: 30s
retries: 50
restart: always
airflow-webserver:
<<: *airflow-common
command: webserver
ports:
- 7001:8080
environment:
<<: *airflow-common-env
VIRTUAL_HOST: datalogistics.eflows4hpc.eu
LETSENCRYPT_HOST: datalogistics.eflows4hpc.eu
VIRTUAL_PORT: 8080
command: webserver
ports:
- 8080:8080
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
interval: 60s
......@@ -176,7 +177,7 @@ services:
test:
- "CMD-SHELL"
- 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
interval: 30s
interval: 60s
timeout: 10s
retries: 5
environment:
......@@ -186,9 +187,10 @@ services:
DUMB_INIT_SETSID: "0"
restart: always
volumes:
- ./dags:/opt/airflow/dags
- ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags
- ./config/airflow.cfg:/opt/airflow/airflow.cfg
- /persistent_data/logs:/opt/airflow/logs
- ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs
- ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins
- ./tmp/:/work/
depends_on:
<<: *airflow-common-depends-on
......@@ -200,14 +202,9 @@ services:
command: triggerer
healthcheck:
test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
interval: 60s
interval: 10s
timeout: 10s
retries: 5
environment:
<<: *airflow-common-env
# Required to handle warm shutdown of the celery workers properly
# See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
DUMB_INIT_SETSID: "0"
restart: always
depends_on:
<<: *airflow-common-depends-on
......@@ -228,12 +225,13 @@ services:
environment:
<<: *airflow-common-env
_AIRFLOW_DB_UPGRADE: 'true'
#_AIRFLOW_WWW_USER_CREATE: 'true'
_AIRFLOW_WWW_USER_CREATE: 'true'
_AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
_AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
_PIP_ADDITIONAL_REQUIREMENTS: ''
user: "0:0"
volumes:
- .:/sources
- ${AIRFLOW_PROJ_DIR:-.}:/sources
airflow-cli:
<<: *airflow-common
......@@ -248,24 +246,6 @@ services:
- -c
- airflow
airflow-setup:
<<: *airflow-common
environment:
<<: *airflow-common-env
CONNECTION_CHECK_MAX_COUNT: "0"
entrypoint: /bin/bash
command:
- -c
- |
exec /entrypoint airflow variables import /opt/airflow/variables.json
echo "Variables added"
volumes:
- ./dockers/variables.json:/opt/airflow/variables.json
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
volumes:
postgres-db-volume:
external: true
......@@ -276,4 +256,3 @@ volumes:
html:
vhost:
dhparam:
FROM apache/airflow:2.2.5
FROM apache/airflow:slim-2.5.3-python3.8
USER root
RUN apt update && apt install git -y && apt clean && rm -rf /var/lib/apt/lists/*
COPY ./templates/main.html /home/airflow/.local/lib/python3.7/site-packages/airflow/www/templates/airflow/main.html
COPY ./templates/navbar_right.html /home/airflow/.local/lib/python3.7/site-packages/airflow/www/templates/appbuilder/navbar_right.html
COPY ./templates/img/BMBF_gefoerdert_2017_en.jpg /home/airflow/.local/lib/python3.7/site-packages/airflow/www/static/BMBF_gefoerdert_2017_en.jpg
COPY ./templates/main.html /home/airflow/.local/lib/python3.8/site-packages/airflow/www/templates/airflow/main.html
COPY ./templates/navbar_right.html /home/airflow/.local/lib/python3.8/site-packages/airflow/www/templates/appbuilder/navbar_right.html
COPY ./templates/img/BMBF_gefoerdert_2017_en.jpg /home/airflow/.local/lib/python3.8/site-packages/airflow/www/static/BMBF_gefoerdert_2017_en.jpg
USER airflow
ENV SQLALCHEMY_SILENCE_UBER_WARNING=1
RUN pip --version && python -m pip install --upgrade pip
ADD requirements.txt /requirements.txt
RUN pip install -r /requirements.txt
\ No newline at end of file
RUN pip install --no-cache-dir -r /requirements.txt
{#
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
......@@ -29,27 +28,14 @@
{% endif%}
{% endblock %}
{% block head_css %}
{% block head_meta %}
{{ super() }}
{% if not appbuilder.app_theme %}
{# airflowDefaultTheme.css file contains the styles from local bootstrap-theme.css #}
<link rel="stylesheet" type="text/css" href="{{ url_for_asset('airflowDefaultTheme.css') }}">
{% if scheduler_job is defined and (scheduler_job and scheduler_job.is_alive()) %}
<meta name="is_scheduler_running" content="True">
{% endif %}
<link rel="stylesheet" type="text/css" href="{{ url_for_asset('materialIcons.css') }}">
<link rel="stylesheet" type="text/css" href="{{ url_for_asset('main.css') }}">
<link rel="stylesheet" type="text/css" href="{{ url_for_asset('loadingDots.css') }}">
<link rel="stylesheet" type="text/css" href="{{ url_for_asset('bootstrap-datetimepicker.min.css') }}">
<style type="text/css">
{% for state, state_color in state_color_mapping.items() %}
span.{{state}} {
background-color: {{state_color}};
}
{% endfor %}
</style>
<link rel="icon" type="image/png" href="{{ url_for('static', filename='pin_32.png') }}">
{% endblock %}
{% block messages %}
{% include 'appbuilder/flash.html' %}
{% if scheduler_job is defined and (not scheduler_job or not scheduler_job.is_alive()) %}
......@@ -137,20 +123,30 @@
{% endif %}
{% endblock %}
{% block head_css %}
{{ super() }}
{% if not appbuilder.app_theme %}
{# airflowDefaultTheme.css file contains the styles from local bootstrap-theme.css #}
<link rel="stylesheet" type="text/css" href="{{ url_for_asset('airflowDefaultTheme.css') }}">
{% endif %}
<link rel="stylesheet" type="text/css" href="{{ url_for_asset('materialIcons.css') }}">
<link rel="stylesheet" type="text/css" href="{{ url_for_asset('main.css') }}">
<link rel="stylesheet" type="text/css" href="{{ url_for_asset('loadingDots.css') }}">
<link rel="stylesheet" type="text/css" href="{{ url_for_asset('bootstrap-datetimepicker.min.css') }}">
<style type="text/css">
{% for state, state_color in state_color_mapping.items() %}
span.{{state}} {
background-color: {{state_color}};
}
{% endfor %}
</style>
<link rel="icon" type="image/png" href="{{ url_for('static', filename='pin_32.png') }}">
{% endblock %}
{% block tail_js %}
{{ super() }}
<script>
// below variables are used in main.js
// keep as var, changing to const or let breaks other code
var Airflow = {
serverTimezone: '{{ server_timezone }}',
defaultUITimezone: '{{ default_ui_timezone }}',
};
var hostName = '{{ hostname }}';
var csrfToken = '{{ csrf_token() }}';
$('time[title]').tooltip();
</script>
<!-- CUSTOM FOOTER SCRIPT -->
<!-- Calculate the size of the dynamic footer to make sure that it doesn't cover the content of the page. Helps also on mobile devices. -->
<script>
......@@ -170,9 +166,18 @@
footerAlign();
});
</script>
<!--[if IE ]>
<script src="{{ url_for_asset('ie.js') }}"></script>
<![endif]-->
<script>
// below variables are used in main.js
// keep as var, changing to const or let breaks other code
var Airflow = {
serverTimezone: '{{ server_timezone }}',
defaultUITimezone: '{{ default_ui_timezone }}',
};
var hostName = '{{ hostname }}';
var csrfToken = '{{ csrf_token() }}';
$('time[title]').tooltip();
</script>
<script src="{{ url_for_asset('moment.js') }}"></script>
<script src="{{ url_for_asset('main.js') }}"></script>
<script src="{{ url_for_asset('bootstrap-datetimepicker.min.js') }}"></script>
......@@ -182,3 +187,4 @@
{% include "analytics/" + analytics_tool + ".html" %}
{% endif %}
{% endblock %}
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment