Skip to content
Snippets Groups Projects
Commit 16e89efd authored by Maria Petrova-El Sayed's avatar Maria Petrova-El Sayed
Browse files

Correct errors in the deployment script

parent c28f8bf1
No related branches found
No related tags found
No related merge requests found
Pipeline #85208 passed
...@@ -59,8 +59,8 @@ cleanup-successful-full-deployment: ...@@ -59,8 +59,8 @@ cleanup-successful-full-deployment:
# check if there is an old prod or test instance, and delete it if present # check if there is an old prod or test instance, and delete it if present
stage: cleanup stage: cleanup
when: on_success when: on_success
only: # only:
- web # - web
script: script:
- echo "This is the cleanup for the full-redeployment of the testing or production servers" - echo "This is the cleanup for the full-redeployment of the testing or production servers"
- echo "if this job is reached, all earlier jobs were successful, and any lingering old instances need to be removed" - echo "if this job is reached, all earlier jobs were successful, and any lingering old instances need to be removed"
......
...@@ -64,7 +64,7 @@ x-airflow-common: ...@@ -64,7 +64,7 @@ x-airflow-common:
- ./plugins:/opt/airflow/plugins - ./plugins:/opt/airflow/plugins
- ./templates/main.html:/home/airflow/.local/lib/python3.7/site-packages/airflow/www/templates/airflow/main.html - ./templates/main.html:/home/airflow/.local/lib/python3.7/site-packages/airflow/www/templates/airflow/main.html
user: "${AIRFLOW_UID:-50000}:0" user: "${AIRFLOW_UID:-50000}:0"
command: pip3 install -r requirements.txt # command: pip3 install -r requirements.txt
depends_on: depends_on:
&airflow-common-depends-on &airflow-common-depends-on
redis: redis:
......
...@@ -51,6 +51,9 @@ runcmd: ...@@ -51,6 +51,9 @@ runcmd:
- sudo pip3 install docker-compose - sudo pip3 install docker-compose
- cd /home/maria - cd /home/maria
- 'git clone https://gitlab.jsc.fz-juelich.de/eflows4hpc-wp2/data-logistics-service.git ./data-logistics-service' - 'git clone https://gitlab.jsc.fz-juelich.de/eflows4hpc-wp2/data-logistics-service.git ./data-logistics-service'
- cd ./data-logistics-service
- sudo git checkout mptest
- cd ..
# - mkdir airflow # - mkdir airflow
# - cd airflow # - cd airflow
# - mkdir -p ./dags ./logs ./plugins ./config ./templates # - mkdir -p ./dags ./logs ./plugins ./config ./templates
...@@ -69,6 +72,6 @@ runcmd: ...@@ -69,6 +72,6 @@ runcmd:
# - echo "Bringing up the docker containers" # - echo "Bringing up the docker containers"
# - docker-compose -f ./dockers/docker-compose.yaml --project-directory ../airflow --verbose up airflow-init # - docker-compose -f ./dockers/docker-compose.yaml --project-directory ../airflow --verbose up airflow-init
# - docker-compose -f ./dockers/docker-compose.yaml --project-directory ../airflow up -d # - docker-compose -f ./dockers/docker-compose.yaml --project-directory ../airflow up -d
- /bin/bash ../data-logistics-service/scripts/deployment.sh /home/maria /home/maria/data-logistics-service - /bin/bash data-logistics-service/scripts/deployment.sh /home/maria /home/maria/data-logistics-service
final_message: "The system is finally up, after $UPTIME seconds" final_message: "The system is finally up, after $UPTIME seconds"
\ No newline at end of file
#!/bin/bash #!/bin/bash
# From Christian B. # @author Maria Petrova & Christian Böttcher
## USAGE: ## USAGE:
# #
# deployment.sh <git_directory> [API_URL] [SERVER_DOMAIN] # deployment.sh <project_directory> <git_directory> [API_URL] [SERVER_DOMAIN]
OLD_DIR=`pwd` OLD_DIR=`pwd`
GIT_REPO=$OLD_DIR/data-logistics-service GIT_REPO=$OLD_DIR/data-logistics-service
echo "DEBUG_1 $0 $1 $2 $3" echo "DEBUG_1 $0 $1 $2 $3 $4"
if [ -z ${1+x} ]; then NEW_DIR=`pwd`; else NEW_DIR=$1; fi if [ -z ${1+x} ]; then NEW_DIR=`pwd`; else NEW_DIR=$1; fi
if [ -z ${2+x} ]; then GIT_REPO else GIT_REPO=$2; fi if [ -z ${2+x} ]; then GIT_REPO else GIT_REPO=$2; fi
# if [ -z ${2+x} ]; then API_URL=https://datacatalog.fz-juelich.de/; else API_URL=$2; fi # if [ -z ${2+x} ]; then API_URL=https://datacatalog.fz-juelich.de/; else API_URL=$2; fi
# if [ -z ${3+x} ]; then SERVER_DOMAIN=datacatalog.fz-juelich.de; else SERVER_DOMAIN=$3; fi # if [ -z ${3+x} ]; then SERVER_DOMAIN=datacatalog.fz-juelich.de; else SERVER_DOMAIN=$3; fi
echo "DEBUG_2 $0 $1 $2 $3" echo "DEBUG_2 $0 $1 $2 $3 $4"
echo "DEBUG values: OLD_DIR=$OLD_DIR, NEW_DIR=$NEW_DIR and GIT_REPO=$GIT_REPO"
cd $NEW_DIR cd $NEW_DIR
`mkdir airflow` `mkdir -p airflow`
cd airflow cd airflow
AIRFLOW_DIR =`pwd` AIRFLOW_DIR=`pwd`
echo "Project dir is: $AIRFLOW_DIR"
echo "User in use is: $(whoami)"
# Make the necessary folders for the airflow artefacts and copy the corresponging content
`mkdir -p ./dags ./logs ./plugins ./config ./templates` `mkdir -p ./dags ./logs ./plugins ./config ./templates`
cd $GIT_REPO cd $GIT_REPO
`cp dags/* $AIRFLOW_DIR/dags` `cp dags/* $AIRFLOW_DIR/dags`
`cp -r plugins/* $AIRFLOW_DIR/plugins` `cp -r plugins/* $AIRFLOW_DIR/plugins`
`cp config/* $AIRFLOW_DIR/config` `cp config/* $AIRFLOW_DIR/config`
`cp templates/* $AIRFLOW_DIR/templates` `cp templates/* $AIRFLOW_DIR/templates`
# Setup environment variables and install requirements
echo -e "AIRFLOW_UID=$(id -u)" > $GIT_REPO/dockers/.env echo -e "AIRFLOW_UID=$(id -u)" > $GIT_REPO/dockers/.env
export AIRFLOW_UID=$(id -u) export AIRFLOW_UID=$(id -u)
echo "Collecting requirements" echo "Collecting requirements"
reqs=`cat requirements.txt | tr '\n' ' '` reqs=`cat requirements.txt | tr '\n' ' '`
echo "Collected - $reqs" echo "Collected requirements: $reqs"
sudo sh -c "echo \"_PIP_ADDITIONAL_REQUIREMENTS=$reqs\" >> $GIT_REPO/dockers/.env" sudo sh -c "echo \"_PIP_ADDITIONAL_REQUIREMENTS=$reqs\" >> $GIT_REPO/dockers/.env"
pip install -r $GIT_REPO/requirements.txt pip install -r $GIT_REPO/requirements.txt
echo "Bringing up the docker containers"
# sed -i "s_datacatalog.fz-juelich.de_${SERVER_DOMAIN}_g" docker-compose.yml # sed -i "s_datacatalog.fz-juelich.de_${SERVER_DOMAIN}_g" docker-compose.yml
# it is at this point assumed that ip and volume are correctly assigned, and that dns is working properly # it is at this point assumed that ip and volume are correctly assigned, and that dns is working properly
echo "-----------Bringing up the docker containers-----------"
docker-compose -f $GIT_REPO/dockers/docker-compose.yaml pull # pull changed images (e.g. new latest, or specific tag)
docker-compose pull # pull changed images (e.g. new latest, or specific tag) docker-compose -f $GIT_REPO/dockers/docker-compose.yaml --project-directory $AIRFLOW_DIR --verbose up airflow-init
docker-compose -f $GIT_REPO/dockers/docker-compose.yaml --project-directory $AIRFLOW_DIR up -d
docker-compose -f $GIT_REPO/docker-compose.yaml --project-directory $AIRFLOW_DIR --verbose up airflow-init
docker-compose -f $GIT_REPO/docker-compose.yaml --project-directory $AIRFLOW_DIR up -d
# docker-compose up -d # should only restart changed images, which will also update nginx and reverse-proxy image if needed # docker-compose up -d # should only restart changed images, which will also update nginx and reverse-proxy image if needed
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment