diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9ac87066880cd64ddedcdcfd6913d2302ed1adae..b354d41c584919ace85938523dcb1e2d4deb960e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,7 +42,7 @@ deploy-test: script: - echo "Starting the full testing deployment of airflows example." - pip install python-openstackclient - - INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor s2 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data cloudinit.yml --security-group ssh --security-group airflows --security-group www --security-group https airflow-testing` + - INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor s2 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data scripts/cloudinit.yml --security-group ssh --security-group airflows --security-group www --security-group https airflow-testing` - while [ "`openstack server show $INSTANCE_ID -c addresses -f value`" = "{}" ]; do sleep 5; done # wait until an address is available to attach the floating ip - openstack server add floating ip $INSTANCE_ID $FLOATING_IP - echo "Done" diff --git a/cloudinit.yml b/scripts/cloudinit.yml similarity index 95% rename from cloudinit.yml rename to scripts/cloudinit.yml index 5c6f2e2f2188802c0a02d3394124a19f3b3c9bcb..4da295651cbfd5739953c8a2745305c029fb38e2 100644 --- a/cloudinit.yml +++ b/scripts/cloudinit.yml @@ -47,12 +47,13 @@ users: # - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDQMbfKUO3NoZspgWpzFY+SwY5Tx251oBT/F22pmnqKq3A0U1EcRooYVc11HzDmLrDTkoLSWAYPuv7I8weKqUPMlypXygu7I1kw1JoAZ4veV/TO8kBIb8+fUjD4VnD0EuU9/MD4rc0IazlInUu/5H2oDj4cj3XGoOFHAPRvo1YXF2eEbXgHcos5o52idZfvZPeWmk4wLqWUI+4q1C5o+c9xGxdWkA0Z6cErw5jSfaqIMu9GnsaPE8dDZ89vtNu8kRK97/Ax0qmJ8eLBfv3qm2HnqACRUv1MRLS/s9KsdB18DV6dTn8VuErJsn9rlpx/2oEMVS5lkUSLTJHf7oNVKDtILQ/rQ2tF/f3LakmiViA4ZsWxFspP0T/sXPhjuCgEqGWG8HrJwFj8DByMpoJUsGe1czAiMdoY5Tr7UeIgK7BGaGjoVUFaVrCKlDpDNhYsHopSTTNajVxsb0LkTRIRphGlQTHlD3nDYdHIrgZiLqA1XLtTTXtWNzQ4uE59tAkIzdTK7RSBduHunqx++IEO6Huj49Vvk1vcO33iqFTTZro1vhZ2kEGxAkxNMti+/eT2rvyfkhsXaUH1/7LXvRrR+pFKcXBpaWWeEt8cOiVrMWAPDi9VRh5QPZbJ1tyTq7XzxeaQuJhL22o2BO13ZSRzr1S+UNFcmfk3esruZoxDIiQ+Bw== apiserver@gitlab runcmd: - - usermod -aG docker ubuntu + - usermod -aG docker $USER - 'git clone https://gitlab.jsc.fz-juelich.de/eflows4hpc-wp2/data-logistics-service.git' - mkdir airflow-testing - cd airflow-testing - - mkdir ./logs ./plugins - - cp -r ../data-logistics-service/dags/ . + - mkdir -p ./dags ./logs ./plugins + - cp -r ../data-logistics-service/dags/* ./dags - echo -e "AIRFLOW_UID=$(id -u)\nAIRFLOW_GID=0" > .env - sudo docker-compose -f ../data-logistics-service/dockers/docker-compose.yaml --project-directory . up airflow-init - sudo docker-compose -f ../data-logistics-service/dockers/docker-compose.yaml --project-directory . up -d + # - /bin/bash ../data-logistics-service/scripts/deployment.sh . diff --git a/scripts/deployment.sh b/scripts/deployment.sh new file mode 100755 index 0000000000000000000000000000000000000000..2601fa574c337b8b408f157f3a6ff4d7213a229d --- /dev/null +++ b/scripts/deployment.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# From Christian B. +## USAGE: +# +# deployment.sh <git_directory> [API_URL] [SERVER_DOMAIN] + +OLD_DIR=`pwd` + +echo "DEBUG_1 $0 $1 $2 $3" + +if [ -z ${1+x} ]; then NEW_DIR=`pwd`; else NEW_DIR=$1; fi +# if [ -z ${2+x} ]; then API_URL=https://datacatalog.fz-juelich.de/; else API_URL=$2; fi +# if [ -z ${3+x} ]; then SERVER_DOMAIN=datacatalog.fz-juelich.de; else SERVER_DOMAIN=$3; fi + +echo "DEBUG_2 $0 $1 $2 $3" + +cd $NEW_DIR + +# pip install -r requirements.txt + +# sed -i "s_datacatalog.fz-juelich.de_${SERVER_DOMAIN}_g" docker-compose.yml + +# it is at this point assumed that ip and volume are correctly assigned, and that dns is working properly + +docker-compose pull # pull changed images (e.g. new latest, or specific tag) +TIME=`date +%Y-%m-%d-%H-%M` +mv /app/mnt/docker.log "/app/mnt/docker.log.${TIME}" +docker-compose -f ../data-logistics-service/dockers/docker-compose.yaml --project-directory . up airflow-init +docker-compose -f ../data-logistics-service/dockers/docker-compose.yaml --project-directory . up -d +# docker-compose up -d # should only restart changed images, which will also update nginx and reverse-proxy image if needed + +# nohup docker-compose logs -f >/app/mnt/docker.log & # or similar to capture docker log TODO (seems to cause gitlab CI to hang) + +cd $OLD_DIR \ No newline at end of file