diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f16263d4ceab39bd88bc41a824d44c54d5cc2028..162913efdc80a2018c51c147ee587da85c5aaf96 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,7 @@ variables: OLD_TEST_NAME: old-airflow-test TESTING_NAME: airflow-testing TESTING_URL: http://134.94.199.220:7001/home + TESTING_DOMAIN: zam10220.zam.kfa-juelich.de AIRFLOW_TESTUSER: "airflow" AIRFLOW__SECRETS__BACKEND_KWARGS: '{"url" : "https://zam10036.zam.kfa-juelich.de", "user" : "${DATACAT_TESTING_USERNAME}", "password" : "${DATACAT_TESTING_PASSWORD}"}' AIRFLOW__SECRETS__BACKEND: datacat_integration.secrets.DatacatSecretsBackend @@ -89,6 +90,9 @@ deploy-testing: - INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor m4 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data scripts/cloudinit.yml --security-group ssh --security-group airflows --security-group www --security-group https $TESTING_NAME` - while [ "`openstack server show $INSTANCE_ID -c addresses -f value`" = "{}" ]; do sleep 5; done # wait until an address is available to attach the floating ip - openstack server add floating ip $INSTANCE_ID $TESTING_IP + - sleep 10 # ensure that next command reaches the new server, prevents host key problems + - until ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP ls /finished_cloudinit >/dev/null 2>&1; do sleep 30; done # wait until cloudinit script is complete + - ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service $TESTING_DOMAIN $AIRFLOW__SECRETS__BACKEND $AIRFLOW__SECRETS__BACKEND_KWARGS" - echo "Done" # NOTE Light deployment did not perform well when the template/main.html file was changed (in case of the official airflow image being updated) @@ -106,7 +110,7 @@ light-deploy-testing: environment: Testing script: - ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "cd /home/airflow/data-logistics-service && sudo git stash && sudo git pull --all && sudo git checkout -f $CI_COMMIT_TAG && sudo git stash clear" - - ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service" + - ssh -oStrictHostKeyChecking=accept-new airflow@$TESTING_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service $TESTING_DOMAIN $AIRFLOW__SECRETS__BACKEND $AIRFLOW__SECRETS__BACKEND_KWARGS" test-testingdeployment_webserver: cache: {} diff --git a/scripts/cloudinit.yml b/scripts/cloudinit.yml index 9b66180e9d88868f7138388ef2683e77f357ed77..8fdb0263996ed17fd810e2cd8b37c6adb61e61d6 100644 --- a/scripts/cloudinit.yml +++ b/scripts/cloudinit.yml @@ -53,6 +53,6 @@ runcmd: - echo "Current user is $(whoami)" - sudo -u airflow git clone https://gitlab.jsc.fz-juelich.de/eflows4hpc-wp2/data-logistics-service.git ./data-logistics-service - cd ./data-logistics-service - - sudo -u airflow /bin/bash ./scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service + - touch /finished_cloudinit final_message: "The system is finally up, after $UPTIME seconds" \ No newline at end of file diff --git a/scripts/deployment.sh b/scripts/deployment.sh index 810e1adc739b08bced4930ff09213089a70f5b7e..8c6c1904025ebeb6733682091dd3d1791c05a8a9 100755 --- a/scripts/deployment.sh +++ b/scripts/deployment.sh @@ -2,21 +2,23 @@ # @author Maria Petrova & Christian Böttcher ## USAGE: # -# deployment.sh <user_home_directory> <git_directory> [API_URL] [SERVER_DOMAIN] +# deployment.sh <user_home_directory> <git_directory> [SERVER_DOMAIN] [AIRFLOW__SECRETS__BACKEND] [AIRFLOW__SECRETS__BACKEND_KWARGS] OLD_DIR=`pwd` GIT_REPO=$HOME/data-logistics-service -echo "DEBUG_1 $0 $1 $2 $3 $4" +echo "DEBUG_1 $0 $1 $2 $3 $4 $5" #if null (var + trim empty strings) if [ -z ${1+x} ]; then ENTRYPOINT=`pwd`; else ENTRYPOINT=$1; fi if [ -z ${2+x} ]; then echo "No user input for starting repository location. Default value: $GIT_REPO"; else GIT_REPO=$2; fi -# if [ -z ${2+x} ]; then API_URL=https://datacatalog.fz-juelich.de/; else API_URL=$2; fi -# if [ -z ${3+x} ]; then SERVER_DOMAIN=datacatalog.fz-juelich.de; else SERVER_DOMAIN=$3; fi +if [ -z ${3+x} ]; then SERVER_DOMAIN=dls.fz-juelich.de; else SERVER_DOMAIN=$3; fi +if [ -z ${3+x} ]; then unset AIRFLOW__SECRETS__BACKEND; else AIRFLOW__SECRETS__BACKEND=$4; fi +if [ -z ${3+x} ]; then unset AIRFLOW__SECRETS__BACKEND_KWARGS; else AIRFLOW__SECRETS__BACKEND_KWARGS=$5; fi -echo "DEBUG_2 $0 $1 $2 $3 $4" +echo "DEBUG_2 $0 $1 $2 $3 $4 $5" echo "DEBUG values: OLD_DIR=$OLD_DIR, ENTRYPOINT_DIR=$ENTRYPOINT and GIT_REPO=$GIT_REPO" +echo "DEBUG using secrets backend: $4" cd $ENTRYPOINT mkdir -p eflows-airflow