From a27795a8397b7b481c53ccb9f9fed40c2fee9f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20B=C3=B6ttcher?= <c.boettcher@fz-juelich.de> Date: Wed, 22 Feb 2023 11:11:57 +0100 Subject: [PATCH] update conditions + docs for pipeline config --- .gitlab-ci.yml | 11 +++++------ README.md | 9 +++++++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c1d40f..9bc75d9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,7 +73,7 @@ full-deploy-testing: stage: deploy environment: Testing rules: - - if: ($CI_COMMIT_BRANCH == "main" && $MANUAL_FULL_DEPLOY_TESTING == "true") + - if: ($CI_COMMIT_BRANCH == "main" && $MANUAL_FULL_DEPLOY == "true") <<: *ssh_setup script: - echo "Starting the full testing deployment of airflows." @@ -97,7 +97,7 @@ full-deploy-testing: light-deploy-testing: stage: deploy rules: - - if: ($CI_COMMIT_BRANCH == "main" && $MANUAL_FULL_DEPLOY_TESTING !~ /true/ ) + - if: ($CI_COMMIT_BRANCH == "main" && $MANUAL_FULL_DEPLOY !~ /true/ ) <<: *ssh_setup environment: Testing @@ -111,7 +111,7 @@ full-deploy-production: stage: deploy environment: Production rules: - - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY_PRODUCTION == "true") + - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY == "true") <<: *ssh_setup script: - echo "Starting the full production deployment of airflows." @@ -141,7 +141,7 @@ full-deploy-production: light-deploy-production: stage: deploy rules: - - if: ($CI_COMMIT_TAG =~ /stable/ && ($MANUAL_FULL_DEPLOY_PRODUCTION == "" || $MANUAL_FULL_DEPLOY_PRODUCTION == "false")) + - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY !~ /true/) <<: *ssh_setup environment: Production script: @@ -152,8 +152,7 @@ test-production-webserver: cache: {} stage: test-deployment rules: - - if: ($CI_COMMIT_TAG =~ /stable/ && ($MANUAL_FULL_DEPLOY_PRODUCTION == "" || $MANUAL_FULL_DEPLOY_PRODUCTION == "false")) - - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY_PRODUCTION == "true") + - if: ($CI_COMMIT_TAG =~ /stable/ script: - apt update && apt -y install curl - echo "This is a simple check if the deployment was successful and dags get executed" diff --git a/README.md b/README.md index afb609b..c5df8c7 100644 --- a/README.md +++ b/README.md @@ -50,3 +50,12 @@ Connections can also be added through env variables, like ``` AIRFLOW_CONN_MY_PROD_DATABASE=my-conn-type://login:password@host:port/schema?param1=val1¶m2=val2 ``` + +## CI/CD +The gitlab repository is set up to automatically build the customized airflow image and deploy to the production and testing environment. The pipeline and jobs for this are defined in the [.gitlab-ci.yml](.gitlab-ci.yml) file. +In general, pushes to the main branch update the testing deployment, and tags containing "stable" update the production deployment. + +Since the airflow image is pretty large, the docker image is only built when starting the job manually, to keep the docker registry at a reasonable size. + +To avoid unneeded downtime, the VMs hosting the deployments are usuallly not re-created, and instead only the updated airflow image, as well as updated airflow config is uploaded to the VM. After this, the docker containers are restarted. +If a "full-deployment" is required (i.e. the VMs shuld be newly created), the pipeline has to be started with a variable ```MANUAL_FULL_DEPLOY=true```. This can be done while starting the pipeline via the web interface. -- GitLab