Merge branch 'main' of https://gitlab.jsc.fz-juelich.de/eflows4hpc-wp2/data-logistics-service

.gitlab-ci.yml

@@ -26,3 +26,23 @@ test:
    - airflow dags test testdag 2021-08-18
    - nosetests
 
+deploy-test:
+  stage: deploy
+  environment: Testing
+  only: 
+   - mptest
+  when: manual
+  variables:
+    OS_AUTH_TYPE: v3applicationcredential
+    OS_AUTH_URL: https://hdf-cloud.fz-juelich.de:5000
+    OS_IDENTITY_API_VERSION: 3
+    OS_REGION_NAME: "HDFCloud"
+    OS_INTERFACE: public
+    FLOATING_IP: 134.94.199.220
+  script:
+    - echo "Starting the full testing deployment of airflows example."
+    - pip install python-openstackclient
+    - INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor m2 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data scripts/cloudinit.yml --security-group ssh --security-group airflows --security-group www --security-group https airflow-testing`
+    - while [ "`openstack server show $INSTANCE_ID -c addresses -f value`" = "{}" ]; do sleep 5; done # wait until an address is available to attach the floating ip
+    - openstack server add floating ip $INSTANCE_ID $FLOATING_IP
+    - echo "Done"

dockers/docker-compose.yaml

@@ -47,6 +47,8 @@ x-airflow-common:
   image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.1.3}
   environment:
     &airflow-common-env
+    AIRFLOW_HOME: /opt/airflow
+    AIRFLOW__CORE_dags_folder: /opt/airflow/dags
     AIRFLOW__CORE__EXECUTOR: CeleryExecutor
     AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
     AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow
@@ -60,6 +62,7 @@ x-airflow-common:
     - ./dags:/opt/airflow/dags
     - ./logs:/opt/airflow/logs
     - ./plugins:/opt/airflow/plugins
+    - ./config/airflow.cfg:/opt/airflow/airflow.cfg
   user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-50000}"
   depends_on:
     redis:

plugins/eFlows_menu_link.py

+from airflow.plugins_manager import AirflowPlugin
+
+appbuilder_eFlows = {
+    "name": "About eFlows4HPC",
+    "href": "https://eflows4hpc.eu/",
+}
+
+class AirflowEFlowsPlugin(AirflowPlugin):
+    name = "eFlowsLink"
+    operators = []
+    flask_blueprints = []
+    hooks = []
+    admin_views = []
+    appbuilder_menu_items = [appbuilder_eFlows]

scripts/cloudinit.yml

+#cloud-config
+
+# This is a cloud config that install most basic packages, and clones and prepares the git repo for the datacatalog
+# This should prepare everything that is possible, so that (after assigning the ip address and generating the static files) only docker-compose needs to be run
+
+# upgrade packages
+package_update: true
+package_upgrade: true
+
+# install relevant packages
+packages:
+  - python3
+  - python3-pip
+  - docker.io
+  - docker-compose
+
+# Add users to the system. Users are added after groups are added.
+
+users:
+  - name: cboettcher
+    gecos: Christian Böttcher
+    groups: sudo, docker
+    shell: /bin/bash
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    lock_passwd: true
+    ssh_authorized_keys:
+      - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDrgXm/3kbHrgPuHrru2LCLxKBPNnZkwTSbaadkYm6N+EzE7GwVPcXorPReC+2SHT2e8YnczcjHMcazmf7VWmHAQVV3fGrZiQtk+xTjXt3tC+Rm2zuqB4vvJcR5DXXomMMRJwG3sk/PcozvFfKFv6P7bbHxKOR090o4krM3mE2Vo43EnsBaPUS8cWI2EkhcR4gAJHhreFHbIS+nrFaJydfmzfwHNE1WjjtfIBA0U8ld2tk8eelMUjvkWrYXK+qqdaUKL0n/wVMo8D/Kl1lNGKym8LE6ZiojjEX0Aq0ajSHyyEWGscJunv/tJkrrOX2C4jd9pGEP6d0YyAunimsT1glv cboet@Desktop-CB
+      - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCRsldcJ7kiksXTn2hivYfZ+Y9gziBWaMPpfVPNVlPi5XizbMurXAPQ3gUbBTDRp+Plf5LiXAfFNBdPTACb5ymFhIUKj/3sJhxc92uvJktLyjObAZ74ImBzDhVwGzs/cKhWc2otFgyMwrfPuIxdarCiLTjmG+dZ0a+IZbWta241kc3qBPjuqKK/LSZOK/Jx9Dl4rURs780GdcoA7Q2r6I6Bq8m0Cpfl2Otwi5Vr4d6hxWrl8D100ssLctn4FlL4SzVHPyZJVNeFJYQv1boJwldHBST8tJ0r0KC1V5CboB+Rdh1b/Qy1y6l/y9fPX+axFSGIIxSb6egRSwcE89f3kCC1 cboettcher@zam024
+
+
+  - name: maria
+    gecos: Maria Petrova-El Sayed
+    groups: sudo, docker
+    shell: /bin/bash
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    lock_passwd: true
+    ssh_authorized_keys:
+      - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDUNFmYnaZ1raXQm04/mfdoBfn4i6xYknic2nhGDOrkhp5r6kv4F1m7wgtuL/pddRKuEoQpiXjRWciEMljFmxvVc7+9VitsAn5zBsnzY9+Sq9+si5aKe93RK8JGLX/WsfZGnPMdKPkK2GO9LFJN4TyL9hTpFdFQfxtO82NIa3WikG4RI+WQuKeQ4qr8FHNymr+gHTw/+YaM9331xnM5YqkmOC27CvVtiQx96MNMAyMQ8RJcHy1GL8donTBL+knVZdIwGt4SUy9dIF8iwTXGFkLe8V7/DIEB7RW9gvk2sG3YPo2eq56HsQKAB3yre+5QFhmH/uqUnTKVFgZLqlDUC0duFOwALCRmlEgtOeZqOzRBa6a0RveTIfccMb48ac4FpeeJdo4KId1QO1JaEZ8fYKgRVw3xRuOjDMpxCFuxELpSvx/hd1jgrK9lRizH9DXNf5/5Go2O16hj8LPufBbhX2EiChjWJEJkoRWBhQ3UHmstbqRiuNU/MsHq0FPSHMHV6BU= maria@jsc-strela
+
+#TODO do a proper ssh key if needed, this has been excluded so far so that the testing of the use case goes faster
+  # - name: airflows
+  #   gecos: Common user for running the apiserver
+  #   groups: sudo
+  #   sudo: ALL=(ALL) NOPASSWD:ALL
+  #   lock_passwd: true
+  #   ssh_authorized_keys:
+  #     - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDQMbfKUO3NoZspgWpzFY+SwY5Tx251oBT/F22pmnqKq3A0U1EcRooYVc11HzDmLrDTkoLSWAYPuv7I8weKqUPMlypXygu7I1kw1JoAZ4veV/TO8kBIb8+fUjD4VnD0EuU9/MD4rc0IazlInUu/5H2oDj4cj3XGoOFHAPRvo1YXF2eEbXgHcos5o52idZfvZPeWmk4wLqWUI+4q1C5o+c9xGxdWkA0Z6cErw5jSfaqIMu9GnsaPE8dDZ89vtNu8kRK97/Ax0qmJ8eLBfv3qm2HnqACRUv1MRLS/s9KsdB18DV6dTn8VuErJsn9rlpx/2oEMVS5lkUSLTJHf7oNVKDtILQ/rQ2tF/f3LakmiViA4ZsWxFspP0T/sXPhjuCgEqGWG8HrJwFj8DByMpoJUsGe1czAiMdoY5Tr7UeIgK7BGaGjoVUFaVrCKlDpDNhYsHopSTTNajVxsb0LkTRIRphGlQTHlD3nDYdHIrgZiLqA1XLtTTXtWNzQ4uE59tAkIzdTK7RSBduHunqx++IEO6Huj49Vvk1vcO33iqFTTZro1vhZ2kEGxAkxNMti+/eT2rvyfkhsXaUH1/7LXvRrR+pFKcXBpaWWeEt8cOiVrMWAPDi9VRh5QPZbJ1tyTq7XzxeaQuJhL22o2BO13ZSRzr1S+UNFcmfk3esruZoxDIiQ+Bw== apiserver@gitlab
+
+runcmd:
+  - 'git clone https://gitlab.jsc.fz-juelich.de/eflows4hpc-wp2/data-logistics-service.git /home/maria/data-logistics-service'
+  - cd /home/maria
+  - mkdir airflow-testing
+  - cd airflow-testing
+  - mkdir -p ./dags ./logs ./plugins
+  - cp ../data-logistics-service/dags/* ./dags
+  - echo "AIRFLOW_UID=0\nAIRFLOW_GID=0" > .env #for root
+  - docker-compose -f ../data-logistics-service/dockers/docker-compose.yaml --project-directory . up airflow-init
+  - docker-compose -f ../data-logistics-service/dockers/docker-compose.yaml --project-directory . up
+  # - /bin/bash ../data-logistics-service/scripts/deployment.sh .

scripts/deployment.sh

+#!/bin/bash
+# From Christian B.
+## USAGE:
+#
+# deployment.sh <git_directory> [API_URL] [SERVER_DOMAIN]
+
+OLD_DIR=`pwd`
+
+echo "DEBUG_1 $0 $1 $2 $3"
+
+if [ -z ${1+x} ]; then NEW_DIR=`pwd`; else NEW_DIR=$1; fi
+# if [ -z ${2+x} ]; then API_URL=https://datacatalog.fz-juelich.de/; else API_URL=$2; fi
+# if [ -z ${3+x} ]; then SERVER_DOMAIN=datacatalog.fz-juelich.de; else SERVER_DOMAIN=$3; fi
+
+echo "DEBUG_2 $0 $1 $2 $3"
+
+cd $NEW_DIR
+
+# pip install -r requirements.txt
+
+# sed -i "s_datacatalog.fz-juelich.de_${SERVER_DOMAIN}_g" docker-compose.yml
+
+# it is at this point assumed that ip and volume are correctly assigned, and that dns is working properly
+
+docker-compose pull #  pull changed images (e.g. new latest, or specific tag)
+TIME=`date +%Y-%m-%d-%H-%M`
+mv /app/mnt/docker.log "/app/mnt/docker.log.${TIME}"
+docker-compose -f ../data-logistics-service/dockers/docker-compose.yaml --project-directory . up airflow-init
+docker-compose -f ../data-logistics-service/dockers/docker-compose.yaml --project-directory . up -d
+# docker-compose up -d # should only restart changed images, which will also update nginx and reverse-proxy image if needed
+
+# nohup docker-compose logs -f >/app/mnt/docker.log & # or similar to capture docker log TODO (seems to cause gitlab CI to hang)
+
+cd $OLD_DIR
\ No newline at end of file