diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 207132f5c80c1bb63b73b7147bc0016bb38b5f84..a8c42b68912e328b58ce780c981f7e30e54a1e04 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -102,7 +102,7 @@ full-deploy-production: - ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP "sudo mkdir -p /persistent_data && sudo mount /dev/vdb1 /persistent_data" - until ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP ls /finished_cloudinit >/dev/null 2>&1; do sleep 30; done # wait until cloudinit script is complete - ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP "sudo service docker restart" # to use the configured docker data path - - ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service $PRODUCTION_DOMAIN $AIRFLOW__SECRETS__BACKEND $AIRFLOW__SECRETS__BACKEND_KWARGS" + - ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service $PRODUCTION_DOMAIN $AIRFLOW__SECRETS__BACKEND $AIRFLOW__SECRETS__BACKEND_KWARGS $AIRFLOW_FERNET_KEY" - echo "Done" # NOTE Light deployment did not perform well when the template/main.html file was changed (in case of the official airflow image being updated) diff --git a/dockers/docker-compose.yaml b/dockers/docker-compose.yaml index 8d0fc5824c85a17cad083cb5edb754e49b0ac64a..a9de206505b67aa9aa27108fff50029cb311c4bc 100644 --- a/dockers/docker-compose.yaml +++ b/dockers/docker-compose.yaml @@ -52,7 +52,7 @@ x-airflow-common: AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 - AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY:-} + AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY} AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' AIRFLOW__CORE__LOAD_EXAMPLES: 'false' AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth' diff --git a/scripts/deployment.sh b/scripts/deployment.sh index 0af566c7a2e9570c0903d8cda9028495eb88fbe2..98e250409f3c6c041038b9dff69abbd8ee73383d 100755 --- a/scripts/deployment.sh +++ b/scripts/deployment.sh @@ -14,10 +14,15 @@ if [ -z ${2+x} ]; then echo "No user input for starting repository location. Def if [ -z ${3+x} ]; then export SERVER_DOMAIN=dls.fz-juelich.de; else export SERVER_DOMAIN=$3; fi if [ -z ${3+x} ]; then unset AIRFLOW__SECRETS__BACKEND; else export AIRFLOW__SECRETS__BACKEND=$4; fi if [ -z ${3+x} ]; then unset AIRFLOW__SECRETS__BACKEND_KWARGS; else export AIRFLOW__SECRETS__BACKEND_KWARGS=$5; fi +if [ -z ${6+x} ]; then unset AIRFLOW__CORE__FERNET_KEY; else export AIRFLOW__CORE__FERNET_KEY=$6; fi + + echo "DEBUG values: OLD_DIR=$OLD_DIR, ENTRYPOINT_DIR=$ENTRYPOINT and GIT_REPO=$GIT_REPO" echo "DEBUG using secrets backend: $AIRFLOW__SECRETS__BACKEND" echo "DEBUG backend args length: ${#AIRFLOW__SECRETS__BACKEND_KWARGS}" +echo "DEBUG fernet key: ${AIRFLOW__CORE__FERNET_KEY}" + cd $ENTRYPOINT mkdir -p eflows-airflow