From a27795a8397b7b481c53ccb9f9fed40c2fee9f1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20B=C3=B6ttcher?= <c.boettcher@fz-juelich.de>
Date: Wed, 22 Feb 2023 11:11:57 +0100
Subject: [PATCH] update conditions + docs for pipeline config

---
 .gitlab-ci.yml | 11 +++++------
 README.md      |  9 +++++++++
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1c1d40f..9bc75d9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -73,7 +73,7 @@ full-deploy-testing:
   stage: deploy
   environment: Testing
   rules:
-    - if: ($CI_COMMIT_BRANCH == "main" && $MANUAL_FULL_DEPLOY_TESTING == "true")
+    - if: ($CI_COMMIT_BRANCH == "main" && $MANUAL_FULL_DEPLOY == "true")
   <<: *ssh_setup
   script:
     - echo "Starting the full testing deployment of airflows."
@@ -97,7 +97,7 @@ full-deploy-testing:
 light-deploy-testing:
   stage: deploy
   rules:
-    - if: ($CI_COMMIT_BRANCH == "main" && $MANUAL_FULL_DEPLOY_TESTING !~ /true/ )
+    - if: ($CI_COMMIT_BRANCH == "main" && $MANUAL_FULL_DEPLOY !~ /true/ )
 
   <<: *ssh_setup
   environment: Testing
@@ -111,7 +111,7 @@ full-deploy-production:
   stage: deploy
   environment: Production
   rules:
-     - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY_PRODUCTION == "true")
+     - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY == "true")
   <<: *ssh_setup
   script:
     - echo "Starting the full production deployment of airflows."
@@ -141,7 +141,7 @@ full-deploy-production:
 light-deploy-production:
   stage: deploy
   rules:
-    - if: ($CI_COMMIT_TAG =~ /stable/ && ($MANUAL_FULL_DEPLOY_PRODUCTION == "" || $MANUAL_FULL_DEPLOY_PRODUCTION == "false"))
+    - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY !~ /true/)
   <<: *ssh_setup
   environment: Production
   script:
@@ -152,8 +152,7 @@ test-production-webserver:
   cache: {}
   stage: test-deployment 
   rules:
-    - if: ($CI_COMMIT_TAG =~ /stable/ && ($MANUAL_FULL_DEPLOY_PRODUCTION == "" || $MANUAL_FULL_DEPLOY_PRODUCTION == "false"))
-    - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY_PRODUCTION == "true")
+    - if: ($CI_COMMIT_TAG =~ /stable/
   script:
     - apt update && apt -y install curl
     - echo "This is a simple check if the deployment was successful and dags get executed"
diff --git a/README.md b/README.md
index afb609b..c5df8c7 100644
--- a/README.md
+++ b/README.md
@@ -50,3 +50,12 @@ Connections can also be added through env variables, like
 ```
 AIRFLOW_CONN_MY_PROD_DATABASE=my-conn-type://login:password@host:port/schema?param1=val1&param2=val2
 ```
+
+## CI/CD
+The gitlab repository is set up to automatically build the customized airflow image and deploy to the production and testing environment. The pipeline and jobs for this are defined in the [.gitlab-ci.yml](.gitlab-ci.yml) file.
+In general, pushes to the main branch update the testing deployment, and tags containing "stable" update the production deployment.
+
+Since the airflow image is pretty large, the docker image is only built when starting the job manually, to keep the docker registry at a reasonable size.
+
+To avoid unneeded downtime, the VMs hosting the deployments are usuallly not re-created, and instead only the updated airflow image, as well as updated airflow config is uploaded to the VM. After this, the docker containers are restarted.
+If a "full-deployment" is required (i.e. the VMs shuld be newly created), the pipeline has to be started with a variable ```MANUAL_FULL_DEPLOY=true```. This can be done while starting the pipeline via the web interface.
-- 
GitLab