diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c1d40f09931335f76f3be5474e4ac286be886af..9bc75d9b2022df576b32d24344dd3938a3e54e7d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,7 +73,7 @@ full-deploy-testing: stage: deploy environment: Testing rules: - - if: ($CI_COMMIT_BRANCH == "main" && $MANUAL_FULL_DEPLOY_TESTING == "true") + - if: ($CI_COMMIT_BRANCH == "main" && $MANUAL_FULL_DEPLOY == "true") <<: *ssh_setup script: - echo "Starting the full testing deployment of airflows." @@ -97,7 +97,7 @@ full-deploy-testing: light-deploy-testing: stage: deploy rules: - - if: ($CI_COMMIT_BRANCH == "main" && $MANUAL_FULL_DEPLOY_TESTING !~ /true/ ) + - if: ($CI_COMMIT_BRANCH == "main" && $MANUAL_FULL_DEPLOY !~ /true/ ) <<: *ssh_setup environment: Testing @@ -111,7 +111,7 @@ full-deploy-production: stage: deploy environment: Production rules: - - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY_PRODUCTION == "true") + - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY == "true") <<: *ssh_setup script: - echo "Starting the full production deployment of airflows." @@ -141,7 +141,7 @@ full-deploy-production: light-deploy-production: stage: deploy rules: - - if: ($CI_COMMIT_TAG =~ /stable/ && ($MANUAL_FULL_DEPLOY_PRODUCTION == "" || $MANUAL_FULL_DEPLOY_PRODUCTION == "false")) + - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY !~ /true/) <<: *ssh_setup environment: Production script: @@ -152,8 +152,7 @@ test-production-webserver: cache: {} stage: test-deployment rules: - - if: ($CI_COMMIT_TAG =~ /stable/ && ($MANUAL_FULL_DEPLOY_PRODUCTION == "" || $MANUAL_FULL_DEPLOY_PRODUCTION == "false")) - - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY_PRODUCTION == "true") + - if: ($CI_COMMIT_TAG =~ /stable/ script: - apt update && apt -y install curl - echo "This is a simple check if the deployment was successful and dags get executed" diff --git a/README.md b/README.md index afb609b3938f6120a00746fe25544b573016f1f1..c5df8c732507b3e6f38fa4ca4f0dbb0808f474c2 100644 --- a/README.md +++ b/README.md @@ -50,3 +50,12 @@ Connections can also be added through env variables, like ``` AIRFLOW_CONN_MY_PROD_DATABASE=my-conn-type://login:password@host:port/schema?param1=val1¶m2=val2 ``` + +## CI/CD +The gitlab repository is set up to automatically build the customized airflow image and deploy to the production and testing environment. The pipeline and jobs for this are defined in the [.gitlab-ci.yml](.gitlab-ci.yml) file. +In general, pushes to the main branch update the testing deployment, and tags containing "stable" update the production deployment. + +Since the airflow image is pretty large, the docker image is only built when starting the job manually, to keep the docker registry at a reasonable size. + +To avoid unneeded downtime, the VMs hosting the deployments are usuallly not re-created, and instead only the updated airflow image, as well as updated airflow config is uploaded to the VM. After this, the docker containers are restarted. +If a "full-deployment" is required (i.e. the VMs shuld be newly created), the pipeline has to be started with a variable ```MANUAL_FULL_DEPLOY=true```. This can be done while starting the pipeline via the web interface.