diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..59b7beb8b53b6d8e0d9e698e00201e8458c7a482 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,185 @@ +stages: + - test + - deploy + +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + variables: + TARGET_BRANCH: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS' + when: never + - if: '$CI_COMMIT_BRANCH' + variables: + TARGET_BRANCH: '2023' + +check_style: + stage: test + tags: [ public-docker ] + image: $CI_REGISTRY/easybuild-docker/easybuild-docker/rocky8-eb-test:latest + script: + - git config --global --add safe.directory $CI_PROJECT_DIR + - ml purge && ml EasyBuild # to load latest EasyBuild dev version + - ml --version && which -a eb && eb --version && eb --show-system-info + - export SYSTEMNAME="juwelsbooster" # Make this the default for packages like NVHPC + - export EASYBUILD_INCLUDE_EASYBLOCKS="$CI_PROJECT_DIR/Custom_EasyBlocks/*.py,$CI_PROJECT_DIR/Custom_EasyBlocks/generic/*.py" + - export EASYBUILD_INCLUDE_MODULE_NAMING_SCHEMES="$CI_PROJECT_DIR/Custom_MNS/*.py" + - export EASYBUILD_INCLUDE_TOOLCHAINS="$CI_PROJECT_DIR/Custom_Toolchains/*.py" + - export EASYBUILD_ROBOT="$CI_PROJECT_DIR/Golden_Repo" + - export EASYBUILD_HOOKS="$CI_PROJECT_DIR/Custom_Hooks/eb_hooks.py" + # Let's make this a system where all the packages can be installed, so all of them can be checked without aborting + - export LMOD_SYSTEM_NAME="juwels" + - OVERLAYS=$(ls Overlays) + - for i in $OVERLAYS; do + - export EASYBUILD_ROBOT="$EASYBUILD_ROBOT:$CI_PROJECT_DIR/Overlays/$i" + - done + - export EASYBUILD_MODULE_NAMING_SCHEME="FlexibleCustomHierarchicalMNS" + - export EASYBUILD_MODULECLASSES="astro,base,bio,cae,chem,compiler,data,debugger,devel," + - export EASYBUILD_MODULECLASSES=$EASYBUILD_MODULECLASSES"geo,ide,lang,lib,math,mpi,numlib,perf,phys,quantum,sidecompiler," + - export EASYBUILD_MODULECLASSES=$EASYBUILD_MODULECLASSES"system,toolchain,tools,vis" + - export EASYBUILD_CUDA_COMPUTE_CAPABILITIES="8.0" + - faulty="" + - export TIP_COMMIT=$CI_COMMIT_SHA + - if [ "x$CI_COMMIT_BRANCH" = "x$TARGET_BRANCH" ]; then + - export BASE_COMMIT=$CI_COMMIT_BEFORE_SHA + - else + - export BASE_COMMIT=$(git merge-base "$TIP_COMMIT" "origin/$TARGET_BRANCH") + - fi + - printf "Checking changes between base %s (%s) and tip %s (%s)\n" "$BASE_COMMIT" "$(git describe --all --exclude pipeline/* $BASE_COMMIT 2> /dev/null)" "$TIP_COMMIT" "$(git describe --all --exclude pipeline/* $TIP_COMMIT 2> /dev/null)" + - eb_list=$(git diff-tree --no-commit-id -r "$BASE_COMMIT" "$TIP_COMMIT" | awk '$5 ~ /^M|^A/ && $6 ~ /eb$/ {print($6)}') + - echo "List of files to be checked $eb_list" + - for i in $eb_list; do + - echo "Checking $(basename $i) ..." + - if [[ $(basename $i) == *AMBER* ]]; then + - sg amber -c "eb --check-style --check-contrib $i" || faulty="$faulty $i" + - sg amber -c "eb --dry-run $i" || faulty="$faulty $i" + - elif [[ $(basename $i) == *CPMD* ]]; then + - sg cpmd -c "eb --check-style --check-contrib $i" || faulty="$faulty $i" + - sg cpmd -c "eb --dry-run $i" || faulty="$faulty $i" + - elif [[ $(basename $i) == *NAMD* ]]; then + - sg namd -c "eb --check-style --check-contrib $i" || faulty="$faulty $i" + - sg namd -c "eb --dry-run $i" || faulty="$faulty $i" + - elif [[ $(basename $i) == *Turbomole* ]]; then + - sg turbomol -c "eb --check-style --check-contrib $i" || faulty="$faulty $i" + - sg turbomol -c "eb --dry-run $i" || faulty="$faulty $i" + - else + - eb --check-style --check-contrib $i || faulty="$faulty $i" + - eb --dry-run $i || faulty="$faulty $i" + - fi + - done + - if [ -n "$faulty" ]; then echo "Faulty EasyConfigs... $faulty"; exit 1; fi + +full_check_style: + when: manual + stage: test + tags: [ public-docker ] + image: $CI_REGISTRY/easybuild-docker/easybuild-docker/rocky8-eb-test:latest + script: + - ml purge && ml EasyBuild # to load latest EasyBuild dev version + - export SYSTEMNAME="juwelsbooster" # Make this the default for packages like NVHPC + - export EASYBUILD_INCLUDE_EASYBLOCKS="$CI_PROJECT_DIR/Custom_EasyBlocks/*.py,$CI_PROJECT_DIR/Custom_EasyBlocks/generic/*.py" + - export EASYBUILD_INCLUDE_MODULE_NAMING_SCHEMES="$CI_PROJECT_DIR/Custom_MNS/*.py" + - export EASYBUILD_INCLUDE_TOOLCHAINS="$CI_PROJECT_DIR/Custom_Toolchains/*.py" + - export EASYBUILD_ROBOT="$CI_PROJECT_DIR/Golden_Repo" + - export EASYBUILD_HOOKS="$CI_PROJECT_DIR/Custom_Hooks/eb_hooks.py" + # Let's make this a system where all the packages can be installed, so all of them can be checked without aborting + - export LMOD_SYSTEM_NAME="juwels" + - OVERLAYS=$(ls Overlays) + - for i in $OVERLAYS; do + - export EASYBUILD_ROBOT="$EASYBUILD_ROBOT:$CI_PROJECT_DIR/Overlays/$i" + - done + - export EASYBUILD_MODULE_NAMING_SCHEME="FlexibleCustomHierarchicalMNS" + - export EASYBUILD_MODULECLASSES="astro,base,bio,cae,chem,compiler,data,debugger,devel," + - export EASYBUILD_MODULECLASSES=$EASYBUILD_MODULECLASSES"geo,ide,lang,lib,math,mpi,numlib,perf,phys,quantum,sidecompiler," + - export EASYBUILD_MODULECLASSES=$EASYBUILD_MODULECLASSES"system,toolchain,tools,vis" + - export EASYBUILD_CUDA_COMPUTE_CAPABILITIES="8.0" + - for i in $(find Golden_Repo -name "*.eb"); do + - echo "Checking $(basename $i) ..." + - if [[ $(basename $i) == *AMBER* ]]; then + - sg amber -c "eb --check-style --check-contrib $i" || faulty="$faulty $i" + - sg amber -c "eb --dry-run $i" || faulty="$faulty $i" + - elif [[ $(basename $i) == *CPMD* ]]; then + - sg cpmd -c "eb --check-style --check-contrib $i" || faulty="$faulty $i" + - sg cpmd -c "eb --dry-run $i" || faulty="$faulty $i" + - elif [[ $(basename $i) == *NAMD* ]]; then + - sg namd -c "eb --check-style --check-contrib $i" || faulty="$faulty $i" + - sg namd -c "eb --dry-run $i" || faulty="$faulty $i" + - elif [[ $(basename $i) == *Turbomole* ]]; then + - sg turbomol -c "eb --check-style --check-contrib $i" || faulty="$faulty $i" + - sg turbomol -c "eb --dry-run $i" || faulty="$faulty $i" + - else + - eb --check-style --check-contrib $i || faulty="$faulty $i" + - eb --dry-run $i || faulty="$faulty $i" + - fi + - done + - if [ -n "$faulty" ]; then echo "Faulty EasyConfigs... $faulty"; exit 1; fi + +sync_repo_on_HPC: + stage: deploy + tags: + - jacamar + - shell + script: + - cd /p/fastdata/zam/swmanage/EasyBuild/$TARGET_BRANCH + - hostname + - echo 'Pulling branch '$TARGET_BRANCH + - git pull https://$GITLAB_USER:$GITLAB_TOKEN@gitlab.jsc.fz-juelich.de/software-team/easybuild.git $TARGET_BRANCH + rules: + - if: '$CI_PROJECT_NAMESPACE == "software-team" && $CI_COMMIT_BRANCH == $TARGET_BRANCH' + +update_acls: + stage: deploy + parallel: + matrix: + - SYSTEM: + - juwels + - juwels_booster + - jurecadc + - jusuf + - hdfml + tags: + - jacamar + - ${SYSTEM} + - shell + needs: ['sync_repo_on_HPC'] + script: + - hostname + - ml Stages/$TARGET_BRANCH + - pushd /p/fastdata/zam/swmanage/EasyBuild/$STAGE/bin + - ./setacls -c ../acls.yml --force -m + rules: + - if: '$CI_PROJECT_NAMESPACE == "software-team" && $CI_COMMIT_BRANCH == $TARGET_BRANCH' + changes: + - 'acls.yml' + +push_public_mirror: + stage: deploy + tags: + - jacamar + - juwels_booster + - shell + needs: ['sync_repo_on_HPC'] + script: + - hostname + - ml Stages/$TARGET_BRANCH + - cd /p/project/cswmanage/easybuilders-JSC/$TARGET_BRANCH + - git pull + - ./filter.sh + - git add -A + - git commit -a -m "Sync" + - git push https://$GITHUB_USER:$GITHUB_TOKEN@github.com/easybuilders/JSC.git + rules: + - if: '$CI_PROJECT_NAMESPACE == "software-team" && $CI_COMMIT_BRANCH == $TARGET_BRANCH' + when: manual + +pull_public_mirror: + stage: deploy + tags: + - jacamar + - shell + script: + - hostname + - cd /p/usersoftware/swmanage/user_installations/$TARGET_BRANCH/ + - git pull + rules: + - if: '$CI_PROJECT_NAMESPACE == "software-team" && $CI_COMMIT_BRANCH == $TARGET_BRANCH'