Adjust the deployment scrips

.gitlab-ci.yml

@@ -42,7 +42,7 @@ deploy-test:
   script:
     - echo "Starting the full testing deployment of airflows example."
     - pip install python-openstackclient
-    - INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor s2 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data cloudinit.yml --security-group ssh --security-group airflows --security-group www --security-group https airflow-testing`
+    - INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor s2 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data scripts/cloudinit.yml --security-group ssh --security-group airflows --security-group www --security-group https airflow-testing`
     - while [ "`openstack server show $INSTANCE_ID -c addresses -f value`" = "{}" ]; do sleep 5; done # wait until an address is available to attach the floating ip
     - openstack server add floating ip $INSTANCE_ID $FLOATING_IP
     - echo "Done"

cloudinit.yml → scripts/cloudinit.yml

@@ -47,12 +47,13 @@ users:
   #     - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDQMbfKUO3NoZspgWpzFY+SwY5Tx251oBT/F22pmnqKq3A0U1EcRooYVc11HzDmLrDTkoLSWAYPuv7I8weKqUPMlypXygu7I1kw1JoAZ4veV/TO8kBIb8+fUjD4VnD0EuU9/MD4rc0IazlInUu/5H2oDj4cj3XGoOFHAPRvo1YXF2eEbXgHcos5o52idZfvZPeWmk4wLqWUI+4q1C5o+c9xGxdWkA0Z6cErw5jSfaqIMu9GnsaPE8dDZ89vtNu8kRK97/Ax0qmJ8eLBfv3qm2HnqACRUv1MRLS/s9KsdB18DV6dTn8VuErJsn9rlpx/2oEMVS5lkUSLTJHf7oNVKDtILQ/rQ2tF/f3LakmiViA4ZsWxFspP0T/sXPhjuCgEqGWG8HrJwFj8DByMpoJUsGe1czAiMdoY5Tr7UeIgK7BGaGjoVUFaVrCKlDpDNhYsHopSTTNajVxsb0LkTRIRphGlQTHlD3nDYdHIrgZiLqA1XLtTTXtWNzQ4uE59tAkIzdTK7RSBduHunqx++IEO6Huj49Vvk1vcO33iqFTTZro1vhZ2kEGxAkxNMti+/eT2rvyfkhsXaUH1/7LXvRrR+pFKcXBpaWWeEt8cOiVrMWAPDi9VRh5QPZbJ1tyTq7XzxeaQuJhL22o2BO13ZSRzr1S+UNFcmfk3esruZoxDIiQ+Bw== apiserver@gitlab
 
 runcmd:
-  - usermod -aG docker ubuntu
+  - usermod -aG docker $USER
   - 'git clone https://gitlab.jsc.fz-juelich.de/eflows4hpc-wp2/data-logistics-service.git'
   - mkdir airflow-testing
   - cd airflow-testing
-  - mkdir ./logs ./plugins
-  - cp -r ../data-logistics-service/dags/ .
+  - mkdir -p ./dags ./logs ./plugins
+  - cp -r ../data-logistics-service/dags/* ./dags
   - echo -e "AIRFLOW_UID=$(id -u)\nAIRFLOW_GID=0" > .env
   - sudo docker-compose -f ../data-logistics-service/dockers/docker-compose.yaml --project-directory . up airflow-init
   - sudo docker-compose -f ../data-logistics-service/dockers/docker-compose.yaml --project-directory . up -d
+  # - /bin/bash ../data-logistics-service/scripts/deployment.sh .

scripts/deployment.sh

+#!/bin/bash
+# From Christian B.
+## USAGE:
+#
+# deployment.sh <git_directory> [API_URL] [SERVER_DOMAIN]
+
+OLD_DIR=`pwd`
+
+echo "DEBUG_1 $0 $1 $2 $3"
+
+if [ -z ${1+x} ]; then NEW_DIR=`pwd`; else NEW_DIR=$1; fi
+# if [ -z ${2+x} ]; then API_URL=https://datacatalog.fz-juelich.de/; else API_URL=$2; fi
+# if [ -z ${3+x} ]; then SERVER_DOMAIN=datacatalog.fz-juelich.de; else SERVER_DOMAIN=$3; fi
+
+echo "DEBUG_2 $0 $1 $2 $3"
+
+cd $NEW_DIR
+
+# pip install -r requirements.txt
+
+# sed -i "s_datacatalog.fz-juelich.de_${SERVER_DOMAIN}_g" docker-compose.yml
+
+# it is at this point assumed that ip and volume are correctly assigned, and that dns is working properly
+
+docker-compose pull #  pull changed images (e.g. new latest, or specific tag)
+TIME=`date +%Y-%m-%d-%H-%M`
+mv /app/mnt/docker.log "/app/mnt/docker.log.${TIME}"
+docker-compose -f ../data-logistics-service/dockers/docker-compose.yaml --project-directory . up airflow-init
+docker-compose -f ../data-logistics-service/dockers/docker-compose.yaml --project-directory . up -d
+# docker-compose up -d # should only restart changed images, which will also update nginx and reverse-proxy image if needed
+
+# nohup docker-compose logs -f >/app/mnt/docker.log & # or similar to capture docker log TODO (seems to cause gitlab CI to hang)
+
+cd $OLD_DIR
\ No newline at end of file