diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4db13bf92bd2d2ab69563caa207a62eee455b737..9fd074e02b00444a87f188f101dfe5d5cb836ed6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,8 +33,6 @@ test: deploy-test: stage: deploy environment: Testing - only: - - mptest when: manual variables: OS_AUTH_TYPE: v3applicationcredential @@ -46,7 +44,7 @@ deploy-test: script: - echo "Starting the full testing deployment of airflows example." - pip install python-openstackclient - - INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor m2 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data scripts/cloudinit.yml --security-group ssh --security-group airflows --security-group www --security-group https airflow-testing` + - INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor m4 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data scripts/cloudinit.yml --security-group ssh --security-group airflows --security-group www --security-group https airflow-testing` - while [ "`openstack server show $INSTANCE_ID -c addresses -f value`" = "{}" ]; do sleep 5; done # wait until an address is available to attach the floating ip - openstack server add floating ip $INSTANCE_ID $FLOATING_IP - echo "Done" diff --git a/plugins/eFlows_menu_link.py b/plugins/eFlows_menu_link.py index da5b6603b96752fcd2f24773242b5fce75d891f0..937ad23e784fb873550d41e54b6c0e39fedea4c9 100644 --- a/plugins/eFlows_menu_link.py +++ b/plugins/eFlows_menu_link.py @@ -14,9 +14,9 @@ class AirflowEFlowsPlugin(AirflowPlugin): appbuilder_menu_items = [appbuilder_eFlows] class AirflowDataCatPlugin(AirflowPlugin): - name = "Data Catalogue" + name = "Data Catalog" operators = [] flask_blueprints = [] hooks = [] admin_views = [] - appbuilder_menu_items = [{"name": "Data Catalogue", "href": "https://datacatalog.fz-juelich.de/index.html"}] + appbuilder_menu_items = [{"name": "Data Catalog", "href": "https://datacatalog.fz-juelich.de/index.html"}] diff --git a/scripts/cloudinit.yml b/scripts/cloudinit.yml index 3a5e9dae1d3154a0729925aa48c00cbf47177193..8d7796d938dbba67164e1581c7b91270cabe39a8 100644 --- a/scripts/cloudinit.yml +++ b/scripts/cloudinit.yml @@ -12,7 +12,8 @@ packages: - python3 - python3-pip - docker.io - - docker-compose + # apt-get of ubuntu 20.04 LTS does not install the newest version of docker-compose. + # - docker-compose # Add users to the system. Users are added after groups are added. @@ -47,17 +48,26 @@ users: # - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDQMbfKUO3NoZspgWpzFY+SwY5Tx251oBT/F22pmnqKq3A0U1EcRooYVc11HzDmLrDTkoLSWAYPuv7I8weKqUPMlypXygu7I1kw1JoAZ4veV/TO8kBIb8+fUjD4VnD0EuU9/MD4rc0IazlInUu/5H2oDj4cj3XGoOFHAPRvo1YXF2eEbXgHcos5o52idZfvZPeWmk4wLqWUI+4q1C5o+c9xGxdWkA0Z6cErw5jSfaqIMu9GnsaPE8dDZ89vtNu8kRK97/Ax0qmJ8eLBfv3qm2HnqACRUv1MRLS/s9KsdB18DV6dTn8VuErJsn9rlpx/2oEMVS5lkUSLTJHf7oNVKDtILQ/rQ2tF/f3LakmiViA4ZsWxFspP0T/sXPhjuCgEqGWG8HrJwFj8DByMpoJUsGe1czAiMdoY5Tr7UeIgK7BGaGjoVUFaVrCKlDpDNhYsHopSTTNajVxsb0LkTRIRphGlQTHlD3nDYdHIrgZiLqA1XLtTTXtWNzQ4uE59tAkIzdTK7RSBduHunqx++IEO6Huj49Vvk1vcO33iqFTTZro1vhZ2kEGxAkxNMti+/eT2rvyfkhsXaUH1/7LXvRrR+pFKcXBpaWWeEt8cOiVrMWAPDi9VRh5QPZbJ1tyTq7XzxeaQuJhL22o2BO13ZSRzr1S+UNFcmfk3esruZoxDIiQ+Bw== apiserver@gitlab runcmd: + - sudo pip3 install docker-compose - 'git clone https://gitlab.jsc.fz-juelich.de/eflows4hpc-wp2/data-logistics-service.git /home/maria/data-logistics-service' - cd /home/maria - mkdir airflow - cd airflow - mkdir -p ./dags ./logs ./plugins ./config ./templates - - cp ../data-logistics-service/dags/* ./dags - - cp -r ../data-logistics-service/plugins/* ./plugins - - cp ../data-logistics-service/config/* ./config - - cp ../data-logistics-service/templates/* ./templates - - echo -e "AIRFLOW_UID=$(id -u)" > ../data-logistics-service/dockers/.env #for root + - cd ../data-logistics-service + - cp dags/* ../airflow/dags + - cp -r plugins/* ../airflow/plugins + - cp config/* ../airflow/config + - cp templates/* ../airflow/templates + # - echo -e "AIRFLOW_UID=$(id -u)" > ../data-logistics-service/dockers/.env #for root - export AIRFLOW_UID=$(id -u) - - docker-compose -f ~/data-logistics-service/dockers/docker-compose.yaml --project-directory ~/airflow --verbose up airflow-init - - docker-compose -f ~/data-logistics-service/dockers/docker-compose.yaml --project-directory ~/airflow . up - # - /bin/bash ../data-logistics-service/scripts/deployment.sh . + - echo "Collecting requirements" + - reqs=`cat requirements.txt | tr '\n' ' '` + - echo "Collected -> $reqs" + - sudo sh -c "echo \"_PIP_ADDITIONAL_REQUIREMENTS=$reqs\" >> /home/maria/data-logistics-service/dockers/.env" + - echo "Bringing up the docker containers" + - docker-compose -f ./dockers/docker-compose.yaml --project-directory ../airflow --verbose up airflow-init + - docker-compose -f ./dockers/docker-compose.yaml --project-directory ../airflow up -d + # - /bin/bash ./scripts/deployment.sh . + +final_message: "The system is finally up, after $UPTIME seconds" \ No newline at end of file