From 118c035a98a2c88bd5d495b272bcf988c11b0051 Mon Sep 17 00:00:00 2001
From: Christian Boettcher <c.boettcher@fz-juelich.de>
Date: Mon, 24 Jan 2022 10:48:30 +0100
Subject: [PATCH] Deploy datacat integration
---
.gitlab-ci.yml | 6 +++++-
scripts/cloudinit.yml | 2 +-
scripts/deployment.sh | 12 +++++++-----
3 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f16263d..162913e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,6 +11,7 @@ variables:
OLD_TEST_NAME: old-airflow-test
TESTING_NAME: airflow-testing
TESTING_URL: http://134.94.199.220:7001/home
+ TESTING_DOMAIN: zam10220.zam.kfa-juelich.de
AIRFLOW_TESTUSER: "airflow"
AIRFLOW__SECRETS__BACKEND_KWARGS: '{"url" : "https://zam10036.zam.kfa-juelich.de", "user" : "${DATACAT_TESTING_USERNAME}", "password" : "${DATACAT_TESTING_PASSWORD}"}'
AIRFLOW__SECRETS__BACKEND: datacat_integration.secrets.DatacatSecretsBackend
@@ -89,6 +90,9 @@ deploy-testing:
- INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor m4 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data scripts/cloudinit.yml --security-group ssh --security-group airflows --security-group www --security-group https $TESTING_NAME`
- while [ "`openstack server show $INSTANCE_ID -c addresses -f value`" = "{}" ]; do sleep 5; done # wait until an address is available to attach the floating ip
- openstack server add floating ip $INSTANCE_ID $TESTING_IP
+ - sleep 10 # ensure that next command reaches the new server, prevents host key problems
+ - until ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP ls /finished_cloudinit >/dev/null 2>&1; do sleep 30; done # wait until cloudinit script is complete
+ - ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service $TESTING_DOMAIN $AIRFLOW__SECRETS__BACKEND $AIRFLOW__SECRETS__BACKEND_KWARGS"
- echo "Done"
# NOTE Light deployment did not perform well when the template/main.html file was changed (in case of the official airflow image being updated)
@@ -106,7 +110,7 @@ light-deploy-testing:
environment: Testing
script:
- ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "cd /home/airflow/data-logistics-service && sudo git stash && sudo git pull --all && sudo git checkout -f $CI_COMMIT_TAG && sudo git stash clear"
- - ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service"
+ - ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service $TESTING_DOMAIN $AIRFLOW__SECRETS__BACKEND $AIRFLOW__SECRETS__BACKEND_KWARGS"
test-testingdeployment_webserver:
cache: {}
diff --git a/scripts/cloudinit.yml b/scripts/cloudinit.yml
index 9b66180..8fdb026 100644
--- a/scripts/cloudinit.yml
+++ b/scripts/cloudinit.yml
@@ -53,6 +53,6 @@ runcmd:
- echo "Current user is $(whoami)"
- sudo -u airflow git clone https://gitlab.jsc.fz-juelich.de/eflows4hpc-wp2/data-logistics-service.git ./data-logistics-service
- cd ./data-logistics-service
- - sudo -u airflow /bin/bash ./scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service
+ - touch /finished_cloudinit
final_message: "The system is finally up, after $UPTIME seconds"
\ No newline at end of file
diff --git a/scripts/deployment.sh b/scripts/deployment.sh
index 810e1ad..8c6c190 100755
--- a/scripts/deployment.sh
+++ b/scripts/deployment.sh
@@ -2,21 +2,23 @@
# @author Maria Petrova & Christian Böttcher
## USAGE:
#
-# deployment.sh <user_home_directory> <git_directory> [API_URL] [SERVER_DOMAIN]
+# deployment.sh <user_home_directory> <git_directory> [SERVER_DOMAIN] [AIRFLOW__SECRETS__BACKEND] [AIRFLOW__SECRETS__BACKEND_KWARGS]
OLD_DIR=`pwd`
GIT_REPO=$HOME/data-logistics-service
-echo "DEBUG_1 $0 $1 $2 $3 $4"
+echo "DEBUG_1 $0 $1 $2 $3 $4 $5"
#if null (var + trim empty strings)
if [ -z ${1+x} ]; then ENTRYPOINT=`pwd`; else ENTRYPOINT=$1; fi
if [ -z ${2+x} ]; then echo "No user input for starting repository location. Default value: $GIT_REPO"; else GIT_REPO=$2; fi
-# if [ -z ${2+x} ]; then API_URL=https://datacatalog.fz-juelich.de/; else API_URL=$2; fi
-# if [ -z ${3+x} ]; then SERVER_DOMAIN=datacatalog.fz-juelich.de; else SERVER_DOMAIN=$3; fi
+if [ -z ${3+x} ]; then SERVER_DOMAIN=dls.fz-juelich.de; else SERVER_DOMAIN=$3; fi
+if [ -z ${3+x} ]; then unset AIRFLOW__SECRETS__BACKEND; else AIRFLOW__SECRETS__BACKEND=$4; fi
+if [ -z ${3+x} ]; then unset AIRFLOW__SECRETS__BACKEND_KWARGS; else AIRFLOW__SECRETS__BACKEND_KWARGS=$5; fi
-echo "DEBUG_2 $0 $1 $2 $3 $4"
+echo "DEBUG_2 $0 $1 $2 $3 $4 $5"
echo "DEBUG values: OLD_DIR=$OLD_DIR, ENTRYPOINT_DIR=$ENTRYPOINT and GIT_REPO=$GIT_REPO"
+echo "DEBUG using secrets backend: $4"
cd $ENTRYPOINT
mkdir -p eflows-airflow
--
GitLab