From c60c4f5a7c346ac58f394b15f80dfeed215cf7b9 Mon Sep 17 00:00:00 2001
From: Utz-Uwe Haus <uhaus@cray.com>
Date: Mon, 20 Sep 2021 17:56:58 +0200
Subject: [PATCH] Streamline JSC and gitlab.com CI

Enable test branch CI and disable MVP branch
Make docker image registry variable in CI scripts
Use CI_REGISTRY and do all docker-based CI steps on JSC too
---
 .gitlab-ci.yml | 193 +++++++++++++++++++++++--------------------------
 1 file changed, 89 insertions(+), 104 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ec656d15..c5c21214 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -10,27 +10,41 @@
 # CI-time variables, used for configure etc:
 variables:
   WITH_MERO: "no"
-  GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID
+  #GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID
   LIBSEGFAULT_PATH: "/usr/lib64/libSegFault.so"
   TIMEOUT: 20m
   BUILDENVNAME: "debian"
   ASAN_OPTIONS: "detect_leaks=0"
-
+  REGISTRY_BASE: "${CI_REGISTRY}/cerl/maestro/maestro-core"
 
 # stages in our CI flow
 stages:
  - update-buildenvs
  - build-and-test
- 
+
+# template to special-case things for JSC gitlab
+".build-specialization":
+  rules:
+    # disable CI except on some branches
+    - if: '$CI_COMMIT_BRANCH != "main"  &&
+           $CI_COMMIT_BRANCH != "devel" &&
+           $CI_COMMIT_BRANCH != "77-update-ci"'
+      when: never
+    # set JSC-specific vars:
+    - if: '$GITLAB_INSTANCE == "JSC"'
+      variables:
+        REGISTRY_BASE: "${CI_REGISTRY}/maestro/maestro-core"
+      when: always
+    - when: always
+    
+
 # build template, common to all environments/gitlab sites
-.build: 
-  only: 
-    refs:
-      - main
-      - devel
-      - mvp
-      - jsc-ci
+".build":
   stage: build-and-test
+  extends: ".build-specialization"
+  tags:
+    - docker
+  image: ${REGISTRY_BASE}/buildenv/${BUILDENVNAME}
   script:
     - export # print env
     - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
@@ -71,7 +85,7 @@ pages:
   except: 
     variables:
       - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
-  image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/debian
+  image: ${REGISTRY_BASE}/buildenv/debian
   script:
     - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
     - autoreconf -ivf
@@ -89,7 +103,7 @@ coverage:
   except: 
     variables:
       - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
-  image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/debian
+  image: ${REGISTRY_BASE}/buildenv/debian
   script:
     - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
     - autoreconf -ivf
@@ -123,47 +137,32 @@ coverage:
 # build rule for debian docker image
 build:debian:
   extends: .build
-  except: 
-    variables:
-      - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
-  image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/debian
-  variables: 
-      LIBSEGFAULT_PATH: "/lib/x86_64-linux-gnu/libSegFault.so"
+  variables:
+    BUILDENVNAME: "debian"
 
 build:ubuntu:
   extends: .build
-  except: 
-    variables:
-      - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
-  image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/ubuntu
-  variables: 
-      LIBSEGFAULT_PATH: "/lib/x86_64-linux-gnu/libSegFault.so"
+  variables:
+    BUILDENVNAME: "ubuntu"
 
 build:centos:
   extends: .build
-  except: 
-    variables:
-      - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
-  image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/centos
-  variables: 
-      LIBSEGFAULT_PATH: "/lib64/libSegFault.so"
+  variables:
+    BUILDENVNAME: "centos"
 
 build:opensuse:
   extends: .build
-  except: 
-    variables:
-      - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
-  image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/opensuse
-  variables: 
-      LIBSEGFAULT_PATH: "/lib64/libSegFault.so"
+  variables:
+    BUILDENVNAME: "opensuse"
 
 ### shell-runner builds, for SAGE prototype:
 # template:
 .build:sage:
   extends: .build
-  except:
-    variables:
-      - $GITLAB_INSTANCE != "JSC" # this variable is set on JSC in the project config Web UI
+  rules:
+    - if: '$GITLAB_INSTANCE == "JSC"'
+      when: always
+    - when: never
   tags: 
     - Group
     - Maestro
@@ -181,95 +180,81 @@ build:sage:mio:
 # without MIO
 build:sage:nomio:
   extends: .build:sage
-  except:
-    variables:
-      - $GITLAB_INSTANCE != "JSC" # this variable is set on JSC in the project config Web UI
   variables:
     WITH_MERO: "no"
   
 
 
 ### build environments
-.docker-buildenv-template:
- stage: update-buildenvs
- except: 
-   variables:
-     - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
- only:
-  refs:
-   - main
-   - devel
- variables:
-   CI_DEBUG_TRACE: "true"
- # if we're running on docker (gitlab.com)
- # Use docker-in-docker ('dind'), set extra docker-in-docker vars and do login
- # If you want to execute this recipe on a non-docker runner you'll need to
- # comment out all the lines up to the script.
- tags:
-  - docker
- image: docker:latest
- services:
-   - docker:dind
- variables:
-   DOCKER_HOST:   tcp://docker:2375
-   DOCKER_DRIVER: overlay2
- before_script:
-   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY 
- # 
- allow_failure: true
- retry:
+".docker-buildenv-template":
+  stage: update-buildenvs
+  extends: .build-specialization
+  # rules:
+  #   - changes:
+  #       -  build-envs/Dockerfile.debian
+  #     when: always
+  #   - when: never
+  variables:
+    CI_DEBUG_TRACE: "true"
+  # if we're running on docker (gitlab.com)
+  # Use docker-in-docker ('dind'), set extra docker-in-docker vars and do login
+  # If you want to execute this recipe on a non-docker runner you'll need to
+  # comment out all the lines up to the script.
+  tags:
+    - docker
+  image: docker:latest
+  services:
+    - docker:dind
+  # variables:
+  #   DOCKER_HOST:   tcp://docker:2375
+  #   DOCKER_DRIVER: overlay2
+  before_script:
+    - docker info
+    - docker logout
+    - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} 
+  # 
+  allow_failure: true
+  retry:
     max: 1
     when:
       - stuck_or_timeout_failure
- script:
-  # build the debian-based image
-  # if we're running in gitlab.com CI there is a magic token and login occurred
-  # above; otherwise we need assume the user has done the docker login with a
-  # personal token with 'api' and 'read-registry' features
-  - docker build -f build-envs/Dockerfile.$BUILDENVNAME
-      --tag registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:$CI_COMMIT_SHA  
-      --tag registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:latest
+  script:
+    # build the debian-based image
+    # if we're running in gitlab.com CI there is a magic token and login occurred
+    # above; otherwise we need assume the user has done the docker login with a
+    # personal token with 'api' and 'read-registry' features
+    - docker build -f build-envs/Dockerfile.$BUILDENVNAME
+      --tag ${REGISTRY_BASE}/buildenv/${BUILDENVNAME}:${CI_COMMIT_SHA}  
+      --tag ${REGISTRY_BASE}/buildenv/${BUILDENVNAME}:latest
       --pull 
       --build-arg http_proxy=$http_proxy
       --build-arg https_proxy=$https_proxy
       --build-arg no_proxy=$no_proxy
       --build-arg VCS_REF=$CI_COMMIT_SHA
       --build-arg VCS_URL=$CI_PROJECT_URL . 
-  - docker push registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:$CI_COMMIT_SHA 
-  - docker push registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:latest
+    - docker push ${REGISTRY_BASE}/buildenv/${BUILDENVNAME}:${CI_COMMIT_SHA} 
+    - docker push ${REGISTRY_BASE}/buildenv/${BUILDENVNAME}:latest
 
 # build debian image
 buildenv:debian:
- extends: .docker-buildenv-template
- only:
-  changes:
-   - build-envs/Dockerfile.debian
- variables:
-   BUILDENVNAME: "debian"
+  extends: ".docker-buildenv-template"
+  variables:
+    BUILDENVNAME: "debian"
 
 # ubuntu image
 buildenv:ubuntu:
- extends: .docker-buildenv-template
- only:
-  changes:
-   - build-envs/Dockerfile.ubuntu
- variables:
-   BUILDENVNAME: "ubuntu"
+  extends: .docker-buildenv-template
+  variables:
+    BUILDENVNAME: "ubuntu"
 
 # centos image
 buildenv:centos:
- extends: .docker-buildenv-template
- only:
-  changes:
-   - build-envs/Dockerfile.centos
- variables:
-   BUILDENVNAME: "centos"
+  extends: .docker-buildenv-template
+  variables:
+    BUILDENVNAME: "centos"
 
 # opensuse image
 buildenv:opensuse:
- extends: .docker-buildenv-template
- only:
-  changes:
-   - build-envs/Dockerfile.opensuse
- variables:
-   BUILDENVNAME: "opensuse"
+  extends: .docker-buildenv-template
+  variables:
+    BUILDENVNAME: "opensuse"
-- 
GitLab