diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 37127cb90f1d3d5882058d42ff41738e399a9cca..2d56f6e2c35a69881a6830d57322d25139cfb208 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,33 @@ default: image: python:3-slim +# before script copied from gitlab docs +before_script: + ## + ## Install ssh-agent if not already installed, it is required by Docker. + ## (change apt-get to yum if you use an RPM-based image) + ## + - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )' + + ## + ## Run ssh-agent (inside the build environment) + ## + - eval $(ssh-agent -s) + + ## + ## Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store + ## We're using tr to fix line endings which makes ed25519 keys work + ## without extra base64 encoding. + ## https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556 + ## + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - + + ## + ## Create the SSH directory and give it the right permissions + ## + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + stages: - test - build @@ -24,3 +51,23 @@ test: - airflow connections list - airflow dags test firsto 2021-08-18 +deploy-test: + stage: deploy + environment: Testing + only: + - mptest && web + variables: + OS_AUTH_TYPE: v3applicationcredential + OS_AUTH_URL: https://hdf-cloud.fz-juelich.de:5000 + OS_IDENTITY_API_VERSION: 3 + OS_REGION_NAME: "HDFCloud" + OS_INTERFACE: public + FLOATING_IP: 134.94.199.39 + script: + - echo "Starting the full testing deployment of airflows example." + # - sed -i 's_datacatalog.fz_zam10036.zam.kfa_g' deploy_scripts/cloudinit.yml + - pip install python-openstackclient + - INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor s2 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data cloudinit.yml --security-group ssh --security-group airflows --security-group www --security-group https testing-deployment` + - while [ "`openstack server show $INSTANCE_ID -c addresses -f value`" = "{}" ]; do sleep 5; done # wait until an address is available to attach the floating ip + - openstack server add floating ip $INSTANCE_ID $FLOATING_IP + - echo "Done" diff --git a/cloudinit.yml b/cloudinit.yml new file mode 100644 index 0000000000000000000000000000000000000000..4790bae62c850565425b66a6de37e9a5ad05e420 --- /dev/null +++ b/cloudinit.yml @@ -0,0 +1,57 @@ +#cloud-config + +# This is a cloud config that install most basic packages, and clones and prepares the git repo for the datacatalog +# This should prepare everything that is possible, so that (after assigning the ip address and generating the static files) only docker-compose needs to be run + +# upgrade packages +package_update: true +package_upgrade: true + +# install relevant packages +packages: + - python3 + - python3-pip + - docker.io + # - docker-compose + +# Add users to the system. Users are added after groups are added. + +users: + - name: cboettcher + gecos: Christian Böttcher + groups: sudo + shell: /bin/bash + sudo: ALL=(ALL) NOPASSWD:ALL + lock_passwd: true + ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDrgXm/3kbHrgPuHrru2LCLxKBPNnZkwTSbaadkYm6N+EzE7GwVPcXorPReC+2SHT2e8YnczcjHMcazmf7VWmHAQVV3fGrZiQtk+xTjXt3tC+Rm2zuqB4vvJcR5DXXomMMRJwG3sk/PcozvFfKFv6P7bbHxKOR090o4krM3mE2Vo43EnsBaPUS8cWI2EkhcR4gAJHhreFHbIS+nrFaJydfmzfwHNE1WjjtfIBA0U8ld2tk8eelMUjvkWrYXK+qqdaUKL0n/wVMo8D/Kl1lNGKym8LE6ZiojjEX0Aq0ajSHyyEWGscJunv/tJkrrOX2C4jd9pGEP6d0YyAunimsT1glv cboet@Desktop-CB + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCRsldcJ7kiksXTn2hivYfZ+Y9gziBWaMPpfVPNVlPi5XizbMurXAPQ3gUbBTDRp+Plf5LiXAfFNBdPTACb5ymFhIUKj/3sJhxc92uvJktLyjObAZ74ImBzDhVwGzs/cKhWc2otFgyMwrfPuIxdarCiLTjmG+dZ0a+IZbWta241kc3qBPjuqKK/LSZOK/Jx9Dl4rURs780GdcoA7Q2r6I6Bq8m0Cpfl2Otwi5Vr4d6hxWrl8D100ssLctn4FlL4SzVHPyZJVNeFJYQv1boJwldHBST8tJ0r0KC1V5CboB+Rdh1b/Qy1y6l/y9fPX+axFSGIIxSb6egRSwcE89f3kCC1 cboettcher@zam024 + + + - name: maria + gecos: Maria Petrova-El Sayed + groups: sudo + shell: /bin/bash + sudo: ALL=(ALL) NOPASSWD:ALL + lock_passwd: true + ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDUNFmYnaZ1raXQm04/mfdoBfn4i6xYknic2nhGDOrkhp5r6kv4F1m7wgtuL/pddRKuEoQpiXjRWciEMljFmxvVc7+9VitsAn5zBsnzY9+Sq9+si5aKe93RK8JGLX/WsfZGnPMdKPkK2GO9LFJN4TyL9hTpFdFQfxtO82NIa3WikG4RI+WQuKeQ4qr8FHNymr+gHTw/+YaM9331xnM5YqkmOC27CvVtiQx96MNMAyMQ8RJcHy1GL8donTBL+knVZdIwGt4SUy9dIF8iwTXGFkLe8V7/DIEB7RW9gvk2sG3YPo2eq56HsQKAB3yre+5QFhmH/uqUnTKVFgZLqlDUC0duFOwALCRmlEgtOeZqOzRBa6a0RveTIfccMb48ac4FpeeJdo4KId1QO1JaEZ8fYKgRVw3xRuOjDMpxCFuxELpSvx/hd1jgrK9lRizH9DXNf5/5Go2O16hj8LPufBbhX2EiChjWJEJkoRWBhQ3UHmstbqRiuNU/MsHq0FPSHMHV6BU= maria@jsc-strela + +#TODO do a proper ssh key if needed, this has been excluded so far so that the testing of the use case goes faster + # - name: airflows + # gecos: Common user for running the apiserver + # groups: sudo + # sudo: ALL=(ALL) NOPASSWD:ALL + # lock_passwd: true + # ssh_authorized_keys: + # - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDQMbfKUO3NoZspgWpzFY+SwY5Tx251oBT/F22pmnqKq3A0U1EcRooYVc11HzDmLrDTkoLSWAYPuv7I8weKqUPMlypXygu7I1kw1JoAZ4veV/TO8kBIb8+fUjD4VnD0EuU9/MD4rc0IazlInUu/5H2oDj4cj3XGoOFHAPRvo1YXF2eEbXgHcos5o52idZfvZPeWmk4wLqWUI+4q1C5o+c9xGxdWkA0Z6cErw5jSfaqIMu9GnsaPE8dDZ89vtNu8kRK97/Ax0qmJ8eLBfv3qm2HnqACRUv1MRLS/s9KsdB18DV6dTn8VuErJsn9rlpx/2oEMVS5lkUSLTJHf7oNVKDtILQ/rQ2tF/f3LakmiViA4ZsWxFspP0T/sXPhjuCgEqGWG8HrJwFj8DByMpoJUsGe1czAiMdoY5Tr7UeIgK7BGaGjoVUFaVrCKlDpDNhYsHopSTTNajVxsb0LkTRIRphGlQTHlD3nDYdHIrgZiLqA1XLtTTXtWNzQ4uE59tAkIzdTK7RSBduHunqx++IEO6Huj49Vvk1vcO33iqFTTZro1vhZ2kEGxAkxNMti+/eT2rvyfkhsXaUH1/7LXvRrR+pFKcXBpaWWeEt8cOiVrMWAPDi9VRh5QPZbJ1tyTq7XzxeaQuJhL22o2BO13ZSRzr1S+UNFcmfk3esruZoxDIiQ+Bw== apiserver@gitlab + +runcmd: + - mkdir -p /app/mnt/data + - 'git clone https://github.com/btphan95/greenr-airflow /home/airflow/greenr-airflow' + - cd /home/airflow/greenr-airflow + # general startup takes long enough that no delay should bee needed - 5 to 6 minutes until these commands are executed + - bash <(curl -s https://get.docker.com/) + - bash scripts/nginx-airflow.sh + - sudo docker build -t greenr-airflow:latest . + - sudo docker run -d -p 8080:8080 -p 8008:8008 greenr-airflow \ No newline at end of file