Deploy datacat integration

.gitlab-ci.yml

@@ -11,6 +11,7 @@ variables:
   OLD_TEST_NAME: old-airflow-test
   TESTING_NAME: airflow-testing
   TESTING_URL: http://134.94.199.220:7001/home
+  TESTING_DOMAIN: zam10220.zam.kfa-juelich.de
   AIRFLOW_TESTUSER: "airflow"
   AIRFLOW__SECRETS__BACKEND_KWARGS: '{"url" : "https://zam10036.zam.kfa-juelich.de", "user" : "${DATACAT_TESTING_USERNAME}", "password" : "${DATACAT_TESTING_PASSWORD}"}'
   AIRFLOW__SECRETS__BACKEND: datacat_integration.secrets.DatacatSecretsBackend
@@ -89,6 +90,9 @@ deploy-testing:
     - INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor m4 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data scripts/cloudinit.yml --security-group ssh --security-group airflows --security-group www --security-group https $TESTING_NAME`
     - while [ "`openstack server show $INSTANCE_ID -c addresses -f value`" = "{}" ]; do sleep 5; done # wait until an address is available to attach the floating ip
     - openstack server add floating ip $INSTANCE_ID $TESTING_IP
+    - sleep 10 # ensure that next command reaches the new server, prevents host key problems
+    - until ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP ls /finished_cloudinit >/dev/null 2>&1; do sleep 30; done # wait until cloudinit script is complete
+    - ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service $TESTING_DOMAIN $AIRFLOW__SECRETS__BACKEND $AIRFLOW__SECRETS__BACKEND_KWARGS"
     - echo "Done"
 
 # NOTE Light deployment did not perform well when the template/main.html file was changed (in case of the official airflow image being updated)
@@ -106,7 +110,7 @@ light-deploy-testing:
   environment: Testing
   script:
     - ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "cd /home/airflow/data-logistics-service && sudo git stash && sudo git pull --all && sudo git checkout -f $CI_COMMIT_TAG && sudo git stash clear"
-    - ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service"
+    - ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service $TESTING_DOMAIN $AIRFLOW__SECRETS__BACKEND $AIRFLOW__SECRETS__BACKEND_KWARGS"
 
 test-testingdeployment_webserver:
   cache: {}

scripts/cloudinit.yml

@@ -53,6 +53,6 @@ runcmd:
   - echo "Current user is $(whoami)"
   - sudo -u airflow git clone https://gitlab.jsc.fz-juelich.de/eflows4hpc-wp2/data-logistics-service.git ./data-logistics-service
   - cd ./data-logistics-service
-  - sudo -u airflow /bin/bash ./scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service
+  - touch /finished_cloudinit
 
 final_message: "The system is finally up, after $UPTIME seconds"
\ No newline at end of file

scripts/deployment.sh

@@ -2,21 +2,23 @@
 # @author Maria Petrova & Christian Böttcher
 ## USAGE:
 #
-# deployment.sh <user_home_directory> <git_directory> [API_URL] [SERVER_DOMAIN]
+# deployment.sh <user_home_directory> <git_directory> [SERVER_DOMAIN] [AIRFLOW__SECRETS__BACKEND] [AIRFLOW__SECRETS__BACKEND_KWARGS]
 
 OLD_DIR=`pwd`
 GIT_REPO=$HOME/data-logistics-service
 
-echo "DEBUG_1 $0 $1 $2 $3 $4"
+echo "DEBUG_1 $0 $1 $2 $3 $4 $5"
 
 #if null (var + trim empty strings)
 if [ -z ${1+x} ]; then ENTRYPOINT=`pwd`; else ENTRYPOINT=$1; fi
 if [ -z ${2+x} ]; then echo "No user input for starting repository location. Default value: $GIT_REPO"; else GIT_REPO=$2; fi
-# if [ -z ${2+x} ]; then API_URL=https://datacatalog.fz-juelich.de/; else API_URL=$2; fi
-# if [ -z ${3+x} ]; then SERVER_DOMAIN=datacatalog.fz-juelich.de; else SERVER_DOMAIN=$3; fi
+if [ -z ${3+x} ]; then SERVER_DOMAIN=dls.fz-juelich.de; else SERVER_DOMAIN=$3; fi
+if [ -z ${3+x} ]; then unset AIRFLOW__SECRETS__BACKEND; else AIRFLOW__SECRETS__BACKEND=$4; fi
+if [ -z ${3+x} ]; then unset AIRFLOW__SECRETS__BACKEND_KWARGS; else AIRFLOW__SECRETS__BACKEND_KWARGS=$5; fi
 
-echo "DEBUG_2 $0 $1 $2 $3 $4"
+echo "DEBUG_2 $0 $1 $2 $3 $4 $5"
 echo "DEBUG values: OLD_DIR=$OLD_DIR, ENTRYPOINT_DIR=$ENTRYPOINT and GIT_REPO=$GIT_REPO"
+echo "DEBUG using secrets backend: $4"
 
 cd $ENTRYPOINT
 mkdir -p eflows-airflow