diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69cce0139580a35f8ec4ecde686f79eb61327b9f..047ae79526c6b16aaaf04a27b2210fea9e170378 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,105 +1,84 @@ -### gitlab CI config for maestro-core -# -# Normally this runs in a docker environment, but if we don't have one (shell -# executor) it should try to run without -# -# The image is one that we build ourselves. Bootstrapping may require pushing -# the first one by hand; the weird naming is to accomodate local builds with -# gitlab-runner where the CI_* vars are not set. - -# CI-time variables, used for configure etc: +# You can override the included template(s) by including variable overrides +# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings +# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings +# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings +# Note that environment variables can be set in several places +# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence variables: - WITH_MERO: "no" - GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID + WITH_MERO: 'no' + GIT_CLONE_PATH: "$CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID" LIBSEGFAULT_PATH: "/usr/lib64/libSegFault.so" TIMEOUT: 20m - BUILDENVNAME: "debian" - ASAN_OPTIONS: "detect_leaks=0" - - -# stages in our CI flow + BUILDENVNAME: debian + ASAN_OPTIONS: detect_leaks=0 stages: - - update-buildenvs - - build-and-test - -# build template, common to all environments/gitlab sites -.build: - only: +- update-buildenvs +- build-and-test +- test +".build": + only: refs: - - master - - devel - - mvp - - jsc-ci + - master + - devel + - mvp + - jsc-ci stage: build-and-test script: - - export # print env - - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA" - - autoreconf -ivf - - ./configure --with-mero=$WITH_MERO --enable-asan - - find . -type d | xargs chmod o+rwx - # TMPDIR is not set in docker images - # HOMEDIR is sometimes set to /nonexistent - - ulimit -c unlimited - - ulimit -l 128 - - ulimit -n 1024 - - ulimit - # - cat /proc/sys/kernel/core_pattern - # says that core dumps go to /usr/lib/systemd/systemd-coredump - # but we can't easily get them inside the container - # So: at least try to get backtraces: - #- export LD_PRELOAD=$LIBSEGFAULT_PATH - #- export SEGFAULT_SIGNALS="bus segv" - #- export SEGFAULT_USE_ALTSTACK=1 - # but now we are building with ASAN, so that should not be necessary anymore - - timeout -s KILL $TIMEOUT make check - - timeout -s KILL $TIMEOUT make distcheck DISTCHECK_CONFIGURE_FLAGS="--with-mero=$WITH_MERO" + - export + - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA" + - autoreconf -ivf + - "./configure --with-mero=$WITH_MERO --enable-asan" + - find . -type d | xargs chmod o+rwx + - ulimit -c unlimited + - ulimit -l 128 + - ulimit -n 1024 + - ulimit + - timeout -s KILL $TIMEOUT make check + - timeout -s KILL $TIMEOUT make distcheck DISTCHECK_CONFIGURE_FLAGS="--with-mero=$WITH_MERO" artifacts: when: always paths: - - config.status - - config.log - - tests/*.log - - maestro-*.tar.gz - - maestro-*/_build/sub/tests/*.log - - docs/html - # distcheck results, if that failed (otherwise they'll have been deleted by now) - - maestro-*/_build/sub/tests/*.log - - deps/libfabric/config.{status,log} - + - config.status + - config.log + - tests/*.log + - maestro-*.tar.gz + - maestro-*/_build/sub/tests/*.log + - docs/html + - maestro-*/_build/sub/tests/*.log + - deps/libfabric/config.{status,log} pages: stage: build-and-test - except: + except: variables: - - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI + - $GITLAB_INSTANCE == "JSC" image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/debian script: - - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA" - - autoreconf -ivf - - ./configure - - make docs - - mv docs/html public + - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA" + - autoreconf -ivf + - "./configure" + - make docs + - mv docs/html public artifacts: paths: - public only: - devel - coverage: stage: build-and-test - except: + except: variables: - - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI + - $GITLAB_INSTANCE == "JSC" image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/debian script: - - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA" - - autoreconf -ivf - - ./configure --enable-code-coverage - - ulimit -c unlimited - - ulimit -l 128 - - ulimit -n 1024 - - ulimit - - timeout -s KILL $TIMEOUT make coverage - coverage: '/Total:\|(\d+\.?\d+\%)/' + - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA" + - autoreconf -ivf + - "./configure --enable-code-coverage" + - ulimit -c unlimited + - ulimit -l 128 + - ulimit -n 1024 + - ulimit + - timeout -s KILL $TIMEOUT make coverage + coverage: "/Total:\\|(\\d+\\.?\\d+\\%)/" artifacts: paths: - maestro_coverage @@ -112,164 +91,131 @@ coverage: - maestro-*.tar.gz - maestro-*/_build/sub/tests/*.log - docs/html - # distcheck results, if that failed (otherwise they'll have been deleted by now) - maestro-*/_build/sub/tests/*.log - deps/libfabric/config.{status,log} - only: - devel - -### docker-based builds, for gitlab.com: -# build rule for debian docker image build:debian: - extends: .build - except: + extends: ".build" + except: variables: - - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI + - $GITLAB_INSTANCE == "JSC" image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/debian - variables: - LIBSEGFAULT_PATH: "/lib/x86_64-linux-gnu/libSegFault.so" - + variables: + LIBSEGFAULT_PATH: "/lib/x86_64-linux-gnu/libSegFault.so" build:ubuntu: - extends: .build - except: + extends: ".build" + except: variables: - - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI + - $GITLAB_INSTANCE == "JSC" image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/ubuntu - variables: - LIBSEGFAULT_PATH: "/lib/x86_64-linux-gnu/libSegFault.so" - + variables: + LIBSEGFAULT_PATH: "/lib/x86_64-linux-gnu/libSegFault.so" build:centos: - extends: .build - except: + extends: ".build" + except: variables: - - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI + - $GITLAB_INSTANCE == "JSC" image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/centos - variables: - LIBSEGFAULT_PATH: "/lib64/libSegFault.so" - + variables: + LIBSEGFAULT_PATH: "/lib64/libSegFault.so" build:opensuse: - extends: .build - except: + extends: ".build" + except: variables: - - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI + - $GITLAB_INSTANCE == "JSC" image: registry.gitlab.com/cerl/maestro/maestro-core/buildenv/opensuse - variables: - LIBSEGFAULT_PATH: "/lib64/libSegFault.so" - -### shell-runner builds, for SAGE prototype: -# template: -.build:sage: - extends: .build + variables: + LIBSEGFAULT_PATH: "/lib64/libSegFault.so" +".build:sage": + extends: ".build" except: variables: - - $GITLAB_INSTANCE != "JSC" # this variable is set on JSC in the project config Web UI - tags: - - Group - - Maestro - - gitlab + - $GITLAB_INSTANCE != "JSC" + tags: + - Group + - Maestro + - gitlab variables: TIMEOUT: 30m LIBSEGFAULT_PATH: "/usr/lib64/libSegFault.so" - -# with MIO build:sage:mio: - extends: .build:sage + extends: ".build:sage" variables: - WITH_MERO: "yes" - -# without MIO + WITH_MERO: 'yes' build:sage:nomio: - extends: .build:sage + extends: ".build:sage" except: variables: - - $GITLAB_INSTANCE != "JSC" # this variable is set on JSC in the project config Web UI + - $GITLAB_INSTANCE != "JSC" variables: - WITH_MERO: "no" - - - -### build environments -.docker-buildenv-template: - stage: update-buildenvs - except: - variables: - - $GITLAB_INSTANCE == "JSC" # this variable is set on JSC in the project config Web UI - only: - refs: - - master - - devel - variables: - CI_DEBUG_TRACE: "true" - # if we're running on docker (gitlab.com) - # Use docker-in-docker ('dind'), set extra docker-in-docker vars and do login - # If you want to execute this recipe on a non-docker runner you'll need to - # comment out all the lines up to the script. - tags: + WITH_MERO: 'no' +".docker-buildenv-template": + stage: update-buildenvs + except: + variables: + - $GITLAB_INSTANCE == "JSC" + only: + refs: + - master + - devel + variables: + DOCKER_HOST: tcp://docker:2375 + DOCKER_DRIVER: overlay2 + tags: - docker - image: docker:latest - services: - - docker:dind - variables: - DOCKER_HOST: tcp://docker:2375 - DOCKER_DRIVER: overlay2 - before_script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - # - allow_failure: true - retry: + image: docker:latest + services: + - docker:dind + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + allow_failure: true + retry: max: 1 when: - - stuck_or_timeout_failure - script: - # build the debian-based image - # if we're running in gitlab.com CI there is a magic token and login occurred - # above; otherwise we need assume the user has done the docker login with a - # personal token with 'api' and 'read-registry' features - - docker build -f build-envs/Dockerfile.$BUILDENVNAME - --tag registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:$CI_COMMIT_SHA - --tag registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:latest - --pull - --build-arg http_proxy=$http_proxy - --build-arg https_proxy=$https_proxy - --build-arg no_proxy=$no_proxy - --build-arg VCS_REF=$CI_COMMIT_SHA - --build-arg VCS_URL=$CI_PROJECT_URL . - - docker push registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:$CI_COMMIT_SHA + - stuck_or_timeout_failure + script: + - docker build -f build-envs/Dockerfile.$BUILDENVNAME --tag registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:$CI_COMMIT_SHA + --tag registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:latest + --pull --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy + --build-arg no_proxy=$no_proxy --build-arg VCS_REF=$CI_COMMIT_SHA --build-arg + VCS_URL=$CI_PROJECT_URL . + - docker push registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:$CI_COMMIT_SHA - docker push registry.gitlab.com/cerl/maestro/maestro-core/buildenv/$BUILDENVNAME:latest - -# build debian image buildenv:debian: - extends: .docker-buildenv-template - only: - changes: - - build-envs/Dockerfile.debian - variables: - BUILDENVNAME: "debian" - -# ubuntu image + extends: ".docker-buildenv-template" + only: + changes: + - build-envs/Dockerfile.debian + variables: + BUILDENVNAME: debian buildenv:ubuntu: - extends: .docker-buildenv-template - only: - changes: - - build-envs/Dockerfile.ubuntu - variables: - BUILDENVNAME: "ubuntu" - -# centos image + extends: ".docker-buildenv-template" + only: + changes: + - build-envs/Dockerfile.ubuntu + variables: + BUILDENVNAME: ubuntu buildenv:centos: - extends: .docker-buildenv-template - only: - changes: - - build-envs/Dockerfile.centos - variables: - BUILDENVNAME: "centos" - -# opensuse image + extends: ".docker-buildenv-template" + only: + changes: + - build-envs/Dockerfile.centos + variables: + BUILDENVNAME: centos buildenv:opensuse: - extends: .docker-buildenv-template - only: - changes: - - build-envs/Dockerfile.opensuse - variables: - BUILDENVNAME: "opensuse" + extends: ".docker-buildenv-template" + only: + changes: + - build-envs/Dockerfile.opensuse + variables: + BUILDENVNAME: opensuse + +sast-sage: + tags: + - public-docker + stage: test + script: + - /analyzers run +include: +- template: Security/SAST.gitlab-ci.yml