Utz-Uwe Haus · b98804b5 · af8ee62b · 69c8a2f6 · d9349803 · 3c80dff6
--- a/.gitlab-ci.yml

+ 149

− 203

View file @ 3c80dff6

Open in Web IDE
+++ b/.gitlab-ci.yml

+ 149

− 203

View file @ 3c80dff6

Open in Web IDE
-### gitlab CI config for maestro-core
+# You can override the included template(s) by including variable overrides
-#
+# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
-# Normally this runs in a docker environment, but if we don't have one (shell
+# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings
-# executor) it should try to run without
+# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings
-#
+# Note that environment variables can be set in several places
-# The image is one that we build ourselves. Bootstrapping may require pushing
+# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
-# the first one by hand; the weird naming is to accomodate local builds with
-# gitlab-runner where the CI_* vars are not set.
-# CI-time variables, used for configure etc:
 variables:
-  WITH_MERO: "no"
+  WITH_MERO: 'no'
-  GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID
+  GIT_CLONE_PATH: "$CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID"
  LIBSEGFAULT_PATH: "/usr/lib64/libSegFault.so"
  TIMEOUT: 20m
-  BUILDENVNAME: "debian"
+  BUILDENVNAME: debian
-  ASAN_OPTIONS: "detect_leaks=0"
+  ASAN_OPTIONS: detect_leaks=0
-# stages in our CI flow
 stages:
- - update-buildenvs
+- update-buildenvs
- - build-and-test
+- build-and-test
+- test
-# build template, common to all environments/gitlab sites
+".build":
-.build: 
+  only:
-  only: 
    refs:
-      - master
+    - master
-      - devel
+    - devel
-      - mvp
+    - mvp
-      - jsc-ci
+    - jsc-ci
  stage: build-and-test
  script:
-    - export # print env
+  - export
-    - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
+  - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
-    - autoreconf -ivf
+  - autoreconf -ivf
-    - ./configure --with-mero=$WITH_MERO --enable-asan
+  - "./configure --with-mero=$WITH_MERO --enable-asan"
-    - find . -type d | xargs chmod o+rwx
+  - find . -type d | xargs chmod o+rwx
-    # TMPDIR is not set in docker images
+  - ulimit -c unlimited
-    # HOMEDIR is sometimes set to /nonexistent
+  - ulimit -l 128
-    - ulimit -c unlimited
+  - ulimit -n 1024
-    - ulimit -l 128
+  - ulimit
-    - ulimit -n 1024
+  - timeout -s KILL $TIMEOUT make check
-    - ulimit
+  - timeout -s KILL $TIMEOUT make distcheck DISTCHECK_CONFIGURE_FLAGS="--with-mero=$WITH_MERO"
-    # - cat /proc/sys/kernel/core_pattern
-    # says that core dumps go to /usr/lib/systemd/systemd-coredump
-    # but we can't easily get them inside the container
-    # So: at least try to get backtraces:
-    #- export LD_PRELOAD=$LIBSEGFAULT_PATH
-    #- export SEGFAULT_SIGNALS="bus segv"
-    #- export SEGFAULT_USE_ALTSTACK=1
-    # but now we are building with ASAN, so that should not be necessary anymore
-    - timeout -s KILL $TIMEOUT make check
-    - timeout -s KILL $TIMEOUT make distcheck DISTCHECK_CONFIGURE_FLAGS="--with-mero=$WITH_MERO"
  artifacts:
    when: always
    paths:
-      - config.status
+    - config.status
-      - config.log
+    - config.log
-      - tests/*.log
+    - tests/*.log
-      - maestro-*.tar.gz
+    - maestro-*.tar.gz
-      - maestro-*/_build/sub/tests/*.log
+    - maestro-*/_build/sub/tests/*.log
-      - docs/html
+    - docs/html
-      # distcheck results, if that failed (otherwise they'll have been deleted by now)
+    - maestro-*/_build/sub/tests/*.log
-      - maestro-*/_build/sub/tests/*.log
+    - deps/libfabric/config.{status,log}
-      - deps/libfabric/config.{status,log}
 pages:
  stage: build-and-test
-  except: 
+  except:
    variables:
-      - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
+    - $GITLAB_INSTANCE == "JSC"
  image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/debian
  script:
-    - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
+  - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
-    - autoreconf -ivf
+  - autoreconf -ivf
-    - ./configure 
+  - "./configure"
-    - make docs
+  - make docs
-    - mv docs/html public
+  - mv docs/html public
  artifacts:
    paths:
    - public
  only:
  - devel
 coverage:
  stage: build-and-test
-  except: 
+  except:
    variables:
-      - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
+    - $GITLAB_INSTANCE == "JSC"
  image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/debian
  script:
-    - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
+  - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
-    - autoreconf -ivf
+  - autoreconf -ivf
-    - ./configure --enable-code-coverage
+  - "./configure --enable-code-coverage"
-    - ulimit -c unlimited
+  - ulimit -c unlimited
-    - ulimit -l 128
+  - ulimit -l 128
-    - ulimit -n 1024
+  - ulimit -n 1024
-    - ulimit
+  - ulimit
-    - timeout -s KILL $TIMEOUT make coverage
+  - timeout -s KILL $TIMEOUT make coverage
-  coverage: '/Total:\|(\d+\.?\d+\%)/'
+  coverage: "/Total:\\|(\\d+\\.?\\d+\\%)/"
  artifacts:
    paths:
    - maestro_coverage
 @@ -112,164 +91,131 @@ coverage:
 @@ -112,164 +91,131 @@ coverage:
    - maestro-*.tar.gz
    - maestro-*/_build/sub/tests/*.log
    - docs/html
-    # distcheck results, if that failed (otherwise they'll have been deleted by now)
    - maestro-*/_build/sub/tests/*.log
    - deps/libfabric/config.{status,log}
  only:
  - devel
-### docker-based builds, for gitlab.com:
-# build rule for debian docker image
 build:debian:
-  extends: .build
+  extends: ".build"
-  except: 
+  except:
    variables:
-      - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
+    - $GITLAB_INSTANCE == "JSC"
  image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/debian
-  variables: 
+  variables:
-      LIBSEGFAULT_PATH: "/lib/x86_64-linux-gnu/libSegFault.so"
+    LIBSEGFAULT_PATH: "/lib/x86_64-linux-gnu/libSegFault.so"
 build:ubuntu:
-  extends: .build
+  extends: ".build"
-  except: 
+  except:
    variables:
-      - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
+    - $GITLAB_INSTANCE == "JSC"
  image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/ubuntu
-  variables: 
+  variables:
-      LIBSEGFAULT_PATH: "/lib/x86_64-linux-gnu/libSegFault.so"
+    LIBSEGFAULT_PATH: "/lib/x86_64-linux-gnu/libSegFault.so"
 build:centos:
-  extends: .build
+  extends: ".build"
-  except: 
+  except:
    variables:
-      - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
+    - $GITLAB_INSTANCE == "JSC"
  image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/centos
-  variables: 
+  variables:
-      LIBSEGFAULT_PATH: "/lib64/libSegFault.so"
+    LIBSEGFAULT_PATH: "/lib64/libSegFault.so"
 build:opensuse:
-  extends: .build
+  extends: ".build"
-  except: 
+  except:
    variables:
-      - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
+    - $GITLAB_INSTANCE == "JSC"
  image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/opensuse
-  variables: 
+  variables:
-      LIBSEGFAULT_PATH: "/lib64/libSegFault.so"
+    LIBSEGFAULT_PATH: "/lib64/libSegFault.so"
+".build:sage":
-### shell-runner builds, for SAGE prototype:
+  extends: ".build"
-# template:
-.build:sage:
-  extends: .build
  except:
    variables:
-      - $GITLAB_INSTANCE != "JSC" # this variable is set on JSC in the project config Web UI
+    - $GITLAB_INSTANCE != "JSC"
-  tags: 
+  tags:
-    - Group
+  - Group
-    - Maestro
+  - Maestro
-    - gitlab 
+  - gitlab
  variables:
    TIMEOUT: 30m
    LIBSEGFAULT_PATH: "/usr/lib64/libSegFault.so"
-# with MIO
 build:sage:mio:
-  extends: .build:sage
+  extends: ".build:sage"
  variables:
-    WITH_MERO: "yes"
+    WITH_MERO: 'yes'
-# without MIO
 build:sage:nomio:
-  extends: .build:sage
+  extends: ".build:sage"
  except:
    variables:
-      - $GITLAB_INSTANCE != "JSC" # this variable is set on JSC in the project config Web UI
+    - $GITLAB_INSTANCE != "JSC"
  variables:
-    WITH_MERO: "no"
+    WITH_MERO: 'no'
+".docker-buildenv-template":
+  stage: update-buildenvs
+  except:
-### build environments
+    variables:
-.docker-buildenv-template:
+    - $GITLAB_INSTANCE == "JSC"
- stage: update-buildenvs
+  only:
- except: 
+    refs:
-   variables:
+    - master
-     - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI
+    - devel
- only:
+  variables:
-  refs:
+    DOCKER_HOST: tcp://docker:2375
-   - master
+    DOCKER_DRIVER: overlay2
-   - devel
+  tags:
- variables:
-   CI_DEBUG_TRACE: "true"
- # if we're running on docker (gitlab.com)
- # Use docker-in-docker ('dind'), set extra docker-in-docker vars and do login
- # If you want to execute this recipe on a non-docker runner you'll need to
- # comment out all the lines up to the script.
- tags:
  - docker
- image: docker:latest
+  image: docker:latest
- services:
+  services:
-   - docker:dind
+  - docker:dind
- variables:
+  before_script:
-   DOCKER_HOST:   tcp://docker:2375
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-   DOCKER_DRIVER: overlay2
+  allow_failure: true
- before_script:
+  retry:
-   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY 
- # 
- allow_failure: true
- retry:
    max: 1
    when:
-      - stuck_or_timeout_failure
+    - stuck_or_timeout_failure
- script:
+  script:
-  # build the debian-based image
+  - docker build -f build-envs/Dockerfile.$BUILDENVNAME --tag registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:$CI_COMMIT_SHA
-  # if we're running in gitlab.com CI there is a magic token and login occurred
+    --tag registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:latest
-  # above; otherwise we need assume the user has done the docker login with a
+    --pull --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy
-  # personal token with 'api' and 'read-registry' features
+    --build-arg no_proxy=$no_proxy --build-arg VCS_REF=$CI_COMMIT_SHA --build-arg
-  - docker build -f build-envs/Dockerfile.$BUILDENVNAME
+    VCS_URL=$CI_PROJECT_URL .
-      --tag registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:$CI_COMMIT_SHA  
+  - docker push registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:$CI_COMMIT_SHA
-      --tag registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:latest
-      --pull 
-      --build-arg http_proxy=$http_proxy
-      --build-arg https_proxy=$https_proxy
-      --build-arg no_proxy=$no_proxy
-      --build-arg VCS_REF=$CI_COMMIT_SHA
-      --build-arg VCS_URL=$CI_PROJECT_URL . 
-  - docker push registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:$CI_COMMIT_SHA 
  - docker push registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:latest
-# build debian image
 buildenv:debian:
- extends: .docker-buildenv-template
+  extends: ".docker-buildenv-template"
- only:
+  only:
-  changes:
+    changes:
-   - build-envs/Dockerfile.debian
+    - build-envs/Dockerfile.debian
- variables:
+  variables:
-   BUILDENVNAME: "debian"
+    BUILDENVNAME: debian
-# ubuntu image
 buildenv:ubuntu:
- extends: .docker-buildenv-template
+  extends: ".docker-buildenv-template"
- only:
+  only:
-  changes:
+    changes:
-   - build-envs/Dockerfile.ubuntu
+    - build-envs/Dockerfile.ubuntu
- variables:
+  variables:
-   BUILDENVNAME: "ubuntu"
+    BUILDENVNAME: ubuntu
-# centos image
 buildenv:centos:
- extends: .docker-buildenv-template
+  extends: ".docker-buildenv-template"
- only:
+  only:
-  changes:
+    changes:
-   - build-envs/Dockerfile.centos
+    - build-envs/Dockerfile.centos
- variables:
+  variables:
-   BUILDENVNAME: "centos"
+    BUILDENVNAME: centos
-# opensuse image
 buildenv:opensuse:
- extends: .docker-buildenv-template
+  extends: ".docker-buildenv-template"
- only:
+  only:
-  changes:
+    changes:
-   - build-envs/Dockerfile.opensuse
+    - build-envs/Dockerfile.opensuse
- variables:
+  variables:
-   BUILDENVNAME: "opensuse"
+    BUILDENVNAME: opensuse
+sast-sage:
+  tags:
+  - public-docker
+  stage: test
+  script:
+  - /analyzers run
+include:
+- template: Security/SAST.gitlab-ci.yml