diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d81a34983dd34733f27adc7bc9ec07433f72dc9..cacaaac4d8b77ca7c342fbafbc345f1d76de758f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -101,7 +101,7 @@ full-deploy-production: - sleep 20 # apparently it may take some time until the volume is available to the OS - ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP "sudo mkdir -p /persistent_data && sudo mount /dev/vdb1 /persistent_data" - until ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP ls /finished_cloudinit >/dev/null 2>&1; do sleep 30; done # wait until cloudinit script is complete - - ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP "sudo servcie docker restart" # to use the configured docker data path + - ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP "sudo service docker restart" # to use the configured docker data path - ssh -oStrictHostKeyChecking=accept-new airflow@$PRODUCTION_IP "sudo /home/airflow/data-logistics-service/scripts/deployment.sh /home/airflow /home/airflow/data-logistics-service $PRODUCTION_DOMAIN $AIRFLOW__SECRETS__BACKEND $AIRFLOW__SECRETS__BACKEND_KWARGS" - echo "Done" diff --git a/dockers/docker-compose.yaml b/dockers/docker-compose.yaml index 805ea8e6a66245406f623eaea9a9e0cfc9c6e4b2..bc4b8933b68b209586389c61ba613ac1a2c661b0 100644 --- a/dockers/docker-compose.yaml +++ b/dockers/docker-compose.yaml @@ -62,7 +62,7 @@ x-airflow-common: volumes: - ./dags:/opt/airflow/dags - ./config/airflow.cfg:/opt/airflow/airflow.cfg - - logs:/opt/airflow/logs + - /persistent_data/logs:/opt/airflow/logs - ./plugins:/opt/airflow/plugins - ./templates/main.html:/home/airflow/.local/lib/python3.7/site-packages/airflow/www/templates/airflow/main.html - ./templates/img/BMBF_gefoerdert_2017_en.jpg:/home/airflow/.local/lib/python3.7/site-packages/airflow/www/static/BMBF_gefoerdert_2017_en.jpg @@ -187,7 +187,7 @@ services: volumes: - ./dags:/opt/airflow/dags - ./config/airflow.cfg:/opt/airflow/airflow.cfg - - logs:/opt/airflow/logs + - /persistent_data/logs:/opt/airflow/logs - ./tmp/:/work/ depends_on: <<: *airflow-common-depends-on @@ -322,7 +322,6 @@ services: condition: service_completed_successfully volumes: - logs: postgres-db-volume: certs: html: diff --git a/scripts/deployment.sh b/scripts/deployment.sh index a6143e4e20ef494dcae3795f75a44286a7c8991e..b040a2ed490021ca3d170ab9bb33b7a80532ab6d 100755 --- a/scripts/deployment.sh +++ b/scripts/deployment.sh @@ -33,7 +33,7 @@ mkdir -p ./dags ./logs ./plugins ./config ./templates cd $GIT_REPO cp -r dags/* $AIRFLOW_DIR/dags cp -r plugins/* $AIRFLOW_DIR/plugins -cp config/* $AIRFLOW_DIR/config +cp config/* $AIRFLOW_DIR/config/ cp -r templates/* $AIRFLOW_DIR/templates # Setup environment variables and install requirements echo -e "AIRFLOW_UID=$(id -u)" > $GIT_REPO/dockers/.env @@ -45,7 +45,7 @@ echo "Collected requirements: $reqs" echo "_PIP_ADDITIONAL_REQUIREMENTS=\"$reqs\"" >> $GIT_REPO/dockers/.env pip install -r $GIT_REPO/requirements.txt -sed -i "s_datalogistics.eflows4hpc.eu/_${SERVER_DOMAIN}_g" docker-compose.yml +sed -i "s_datalogistics.eflows4hpc.eu/_${SERVER_DOMAIN}_g" $GIT_REPO/dockers/docker-compose.yaml # it is at this point assumed that ip and volume are correctly assigned, and that dns is working properly echo "-----------Bringing up the docker containers-----------"