Merge branch 'main' of https://gitlab.jsc.fz-juelich.de/eflows4hpc-wp2/data-logistics-service

.gitlab-ci.yml

@@ -101,7 +101,7 @@ full-deploy-production:
     - sleep 20 # apparently it may take some time until the volume is available to the OS
     - ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP "sudo mkdir -p /persistent_data && sudo mount /dev/vdb1 /persistent_data"
     - until ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP ls /finished_cloudinit >/dev/null 2>&1; do sleep 30; done # wait until cloudinit script is complete
-    - ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP "sudo servcie docker restart" # to use the configured docker data path
+    - ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP "sudo service docker restart" # to use the configured docker data path
     - ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service $PRODUCTION_DOMAIN $AIRFLOW__SECRETS__BACKEND $AIRFLOW__SECRETS__BACKEND_KWARGS"
     - echo "Done"
 

dockers/docker-compose.yaml

@@ -62,7 +62,7 @@ x-airflow-common:
   volumes:
     - ./dags:/opt/airflow/dags
     - ./config/airflow.cfg:/opt/airflow/airflow.cfg
-    - logs:/opt/airflow/logs
+    - /persistent_data/logs:/opt/airflow/logs
     - ./plugins:/opt/airflow/plugins
     - ./templates/main.html:/home/airflow/.local/lib/python3.7/site-packages/airflow/www/templates/airflow/main.html
     - ./templates/img/BMBF_gefoerdert_2017_en.jpg:/home/airflow/.local/lib/python3.7/site-packages/airflow/www/static/BMBF_gefoerdert_2017_en.jpg
@@ -187,7 +187,7 @@ services:
     volumes:
       - ./dags:/opt/airflow/dags
       - ./config/airflow.cfg:/opt/airflow/airflow.cfg
-      - logs:/opt/airflow/logs
+      - /persistent_data/logs:/opt/airflow/logs
       - ./tmp/:/work/
     depends_on:
       <<: *airflow-common-depends-on
@@ -322,7 +322,6 @@ services:
         condition: service_completed_successfully
 
 volumes:
-  logs:
   postgres-db-volume:
   certs:
   html:

scripts/deployment.sh

@@ -33,7 +33,7 @@ mkdir -p ./dags ./logs ./plugins ./config ./templates
 cd $GIT_REPO
 cp -r dags/* $AIRFLOW_DIR/dags
 cp -r plugins/* $AIRFLOW_DIR/plugins
-cp config/* $AIRFLOW_DIR/config
+cp config/* $AIRFLOW_DIR/config/
 cp -r templates/* $AIRFLOW_DIR/templates
 # Setup environment variables and install requirements
 echo -e "AIRFLOW_UID=$(id -u)" > $GIT_REPO/dockers/.env
@@ -45,7 +45,7 @@ echo "Collected requirements: $reqs"
 echo "_PIP_ADDITIONAL_REQUIREMENTS=\"$reqs\"" >> $GIT_REPO/dockers/.env
 pip install -r $GIT_REPO/requirements.txt
 
-sed -i "s_datalogistics.eflows4hpc.eu/_${SERVER_DOMAIN}_g" docker-compose.yml
+sed -i "s_datalogistics.eflows4hpc.eu/_${SERVER_DOMAIN}_g" $GIT_REPO/dockers/docker-compose.yaml
 
 # it is at this point assumed that ip and volume are correctly assigned, and that dns is working properly
 echo "-----------Bringing up the docker containers-----------"