From 7a8ccb8df69ced2d9185040748d420f6c0a9b10a Mon Sep 17 00:00:00 2001 From: Dennis Klein Date: Thu, 27 Nov 2025 20:48:55 +0100 Subject: [PATCH] ci: migrate from Jenkins to GitHub Actions --- .github/actions/setup-deps/action.yml | 35 +++++++ .github/workflows/buildcache.yml | 42 +++++++++ .github/workflows/ci.yml | 126 ++++++++++++++++++++++++++ FairMQTest.cmake | 93 ------------------- Jenkinsfile | 118 ------------------------ test/ci/Containerfile.fedora | 19 ---- test/ci/Containerfile.ubuntu | 36 -------- test/ci/slurm-submit.sh | 42 --------- test/ci/spack.yaml | 17 ++++ 9 files changed, 220 insertions(+), 308 deletions(-) create mode 100644 .github/actions/setup-deps/action.yml create mode 100644 .github/workflows/buildcache.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 FairMQTest.cmake delete mode 100644 Jenkinsfile delete mode 100644 test/ci/Containerfile.fedora delete mode 100644 test/ci/Containerfile.ubuntu delete mode 100755 test/ci/slurm-submit.sh create mode 100644 test/ci/spack.yaml diff --git a/.github/actions/setup-deps/action.yml b/.github/actions/setup-deps/action.yml new file mode 100644 index 00000000..02162ee1 --- /dev/null +++ b/.github/actions/setup-deps/action.yml @@ -0,0 +1,35 @@ +name: Setup Dependencies +description: Setup spack and install dependencies + +inputs: + gcc: + description: 'GCC version to use' + required: true + fresh: + description: 'Use fresh concretization' + default: 'false' + +runs: + using: composite + steps: + - name: Setup spack + uses: spack/setup-spack@v2 + with: + ref: v1.1.0 + color: true + buildcache: true + + - name: Install GCC + shell: spack-bash {0} + run: | + spack install gcc@${{ inputs.gcc }} + spack compiler find $(spack location -i gcc@${{ inputs.gcc }}) + + - name: Install dependencies + shell: spack-bash {0} + run: | + spack env create fairmq test/ci/spack.yaml + spack -e fairmq add gcc@${{ inputs.gcc }} + spack -e fairmq config add "packages:all:require:'%gcc@${{ inputs.gcc }}'" + spack -e fairmq install --fail-fast ${{ inputs.fresh == 'true' && '--fresh' || '' }} + spack env activate --sh fairmq | grep '^export ' | sed 's/^export //;s/;$//' >> $GITHUB_ENV diff --git a/.github/workflows/buildcache.yml b/.github/workflows/buildcache.yml new file mode 100644 index 00000000..0c2b249d --- /dev/null +++ b/.github/workflows/buildcache.yml @@ -0,0 +1,42 @@ +name: Spack Buildcache + +on: + workflow_dispatch: + schedule: + - cron: '0 3 * * 0' # Weekly on Sunday at 3am UTC + push: + paths: + - 'test/ci/spack.yaml' + - '.github/workflows/buildcache.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + if: github.repository == 'FairRootGroup/FairMQ' + name: gcc-${{ matrix.gcc }} + runs-on: ubuntu-latest + permissions: + packages: write + strategy: + fail-fast: false + matrix: + gcc: ['12', '13', '14', '15'] + + env: + GITHUB_USER: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/setup-deps + with: + gcc: ${{ matrix.gcc }} + fresh: true + + - name: Push to buildcache + if: ${{ !cancelled() }} + run: spack buildcache push --unsigned --update-index local-buildcache diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e1d564e0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,126 @@ +name: CI + +on: + push: + branches: [master, dev] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + if: github.repository == 'FairRootGroup/FairMQ' + name: gcc-${{ matrix.gcc }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + gcc: ['12', '13', '14', '15'] + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + fetch-tags: true + fetch-depth: 0 + + - uses: ./.github/actions/setup-deps + with: + gcc: ${{ matrix.gcc }} + + - name: Configure and Build + uses: threeal/cmake-action@v2 + with: + generator: Ninja + options: | + CMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_INSTALL_PREFIX=${{ github.workspace }}/install + BUILD_TESTING=ON + + - name: Test + uses: threeal/ctest-action@v1 + with: + test-dir: build + + - name: Install + run: cmake --install build + + sanitizers: + if: github.repository == 'FairRootGroup/FairMQ' + name: ${{ matrix.sanitizer.name }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + sanitizer: + - name: asan+lsan+ubsan + options: | + ENABLE_SANITIZER_ADDRESS=ON + ENABLE_SANITIZER_LEAK=ON + ENABLE_SANITIZER_UNDEFINED_BEHAVIOUR=ON + cxx-flags: -O1 -fno-omit-frame-pointer + - name: tsan + options: ENABLE_SANITIZER_THREAD=ON + cxx-compiler: clang++ + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + fetch-tags: true + fetch-depth: 0 + + - uses: ./.github/actions/setup-deps + with: + gcc: '14' + + - name: Configure and Build + uses: threeal/cmake-action@v2 + with: + generator: Ninja + cxx-compiler: ${{ matrix.sanitizer.cxx-compiler }} + cxx-flags: ${{ matrix.sanitizer.cxx-flags }} + options: | + CMAKE_BUILD_TYPE=Debug + BUILD_TESTING=ON + ${{ matrix.sanitizer.options }} + + - name: Test + uses: threeal/ctest-action@v1 + with: + test-dir: build + + static-analysis: + if: github.repository == 'FairRootGroup/FairMQ' + name: static-analysis + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + fetch-tags: true + fetch-depth: 0 + + - uses: ./.github/actions/setup-deps + with: + gcc: '14' + + - name: Configure and Build + uses: threeal/cmake-action@v2 + with: + generator: Ninja + options: | + CMAKE_BUILD_TYPE=Debug + BUILD_TESTING=ON + RUN_STATIC_ANALYSIS=ON + + - name: Check for warnings + run: | + if grep -q "warning:" build.log; then + echo "::warning::Static analysis found warnings" + grep "warning:" build.log + exit 1 + fi diff --git a/FairMQTest.cmake b/FairMQTest.cmake deleted file mode 100644 index 9dc097c3..00000000 --- a/FairMQTest.cmake +++ /dev/null @@ -1,93 +0,0 @@ -################################################################################ -# Copyright (C) 2021-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH # -# # -# This software is distributed under the terms of the # -# GNU Lesser General Public Licence (LGPL) version 3, # -# copied verbatim in the file "LICENSE" # -################################################################################ - -cmake_host_system_information(RESULT fqdn QUERY FQDN) - -set(CTEST_SOURCE_DIRECTORY .) -set(CTEST_BINARY_DIRECTORY build) -set(CTEST_CMAKE_GENERATOR "Ninja") -set(CTEST_USE_LAUNCHERS ON) -set(CTEST_CONFIGURATION_TYPE "RelWithDebInfo") -set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 102400) - -if(NOT NCPUS) - if(ENV{SLURM_CPUS_PER_TASK}) - set(NCPUS $ENV{SLURM_CPUS_PER_TASK}) - else() - include(ProcessorCount) - ProcessorCount(NCPUS) - if(NCPUS EQUAL 0) - set(NCPUS 1) - endif() - endif() -endif() - -if ("$ENV{CTEST_SITE}" STREQUAL "") - set(CTEST_SITE "${fqdn}") -else() - set(CTEST_SITE $ENV{CTEST_SITE}) -endif() - -if ("$ENV{LABEL}" STREQUAL "") - set(CTEST_BUILD_NAME "build") -else() - set(CTEST_BUILD_NAME $ENV{LABEL}) -endif() - -ctest_start(Continuous) - -list(APPEND options "-DDISABLE_COLOR=ON" "-DBUILD_EXAMPLES=ON" "-DBUILD_TESTING=ON") -if(RUN_STATIC_ANALYSIS) - list(APPEND options "-DRUN_STATIC_ANALYSIS=ON") -endif() -if(CMAKE_BUILD_TYPE) - set(CTEST_CONFIGURATION_TYPE ${CMAKE_BUILD_TYPE}) -endif() -if(ENABLE_SANITIZER_ADDRESS) - list(APPEND options "-DENABLE_SANITIZER_ADDRESS=ON") -endif() -if(ENABLE_SANITIZER_LEAK) - list(APPEND options "-DENABLE_SANITIZER_LEAK=ON") -endif() -if(ENABLE_SANITIZER_UNDEFINED_BEHAVIOR) - list(APPEND options "-DENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON") -endif() -if(ENABLE_SANITIZER_MEMORY) - list(APPEND options "-DENABLE_SANITIZER_MEMORY=ON") -endif() -if(ENABLE_SANITIZER_THREAD) - list(APPEND options "-DENABLE_SANITIZER_THREAD=ON") -endif() -if(CMAKE_CXX_COMPILER) - list(APPEND options "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}") -endif() -if(CMAKE_CXX_FLAGS) - list(APPEND options "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}") -endif() -list(REMOVE_DUPLICATES options) -list(JOIN options ";" optionsstr) -ctest_configure(OPTIONS "${optionsstr}") - -ctest_submit() - -ctest_build(FLAGS "-j${NCPUS}") - -ctest_submit() - -if(NOT RUN_STATIC_ANALYSIS) - ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}" - PARALLEL_LEVEL ${NCPUS} - SCHEDULE_RANDOM ON - RETURN_VALUE _ctest_test_ret_val) - - ctest_submit() -endif() - -if(_ctest_test_ret_val) - Message(FATAL_ERROR "Some tests failed.") -endif() diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 74d47372..00000000 --- a/Jenkinsfile +++ /dev/null @@ -1,118 +0,0 @@ -#!groovy - -def jobMatrix(String type, List specs) { - def nodes = [:] - for (spec in specs) { - def job = "" - def selector = "slurm" - def os = "" - def ver = "" - - if (type == 'build') { - job = "${spec.os}-${spec.ver}-${spec.arch}-${spec.compiler}" - if (spec.os =~ /^macos/) { - selector = "${spec.os}-${spec.ver}-${spec.arch}" - } - os = spec.os - ver = spec.ver - } else { // == 'check' - job = "${spec.name}" - os = 'fedora' - ver = '36' - } - - def label = "${job}" - def extra = spec.extra - - nodes[label] = { - node(selector) { - githubNotify(context: "${label}", description: 'Building ...', status: 'PENDING') - try { - deleteDir() - checkout scm - - def jobscript = 'job.sh' - def ctestcmd = "ctest ${extra} -S FairMQTest.cmake -V --output-on-failure" - sh "echo \"set -e\" >> ${jobscript}" - sh "echo \"export LABEL=\\\"\${JOB_BASE_NAME} ${label}\\\"\" >> ${jobscript}" - if (selector =~ /^macos/) { - sh """\ - echo \"${ctestcmd}\" >> ${jobscript} - """ - sh "cat ${jobscript}" - sh "bash ${jobscript}" - } else { // selector == "slurm" - def imageurl = "oras://ghcr.io/fairrootgroup/fairmq-dev/${os}-${ver}-sif:latest" - def execopts = "--ipc --uts --pid -B/shared" - def containercmd = "singularity exec ${execopts} ${imageurl} bash -l -c \\\"${ctestcmd} ${extra}\\\"" - sh """\ - echo \"echo \\\"*** Job started at .......: \\\$(date -R)\\\"\" >> ${jobscript} - echo \"echo \\\"*** Job ID ...............: \\\${SLURM_JOB_ID}\\\"\" >> ${jobscript} - echo \"echo \\\"*** Compute node .........: \\\$(hostname -f)\\\"\" >> ${jobscript} - echo \"unset http_proxy\" >> ${jobscript} - echo \"unset HTTP_PROXY\" >> ${jobscript} - echo \"${containercmd}\" >> ${jobscript} - """ - sh "cat ${jobscript}" - sh "test/ci/slurm-submit.sh \"FairMQ \${JOB_BASE_NAME} ${label}\" ${jobscript}" - - if (job == "static-analyzers") { - recordIssues(enabledForFailure: true, - tools: [gcc(pattern: 'build/Testing/Temporary/*.log')], - filters: [excludeFile('extern/*'), excludeFile('usr/*')], - skipBlames: true, - skipPublishingChecks: true) - } - } - - deleteDir() - githubNotify(context: "${label}", description: 'Success', status: 'SUCCESS') - } catch (e) { - deleteDir() - githubNotify(context: "${label}", description: 'Error', status: 'ERROR') - throw e - } - } - } - } - return nodes -} - -pipeline{ - agent none - stages { - stage("CI") { - steps{ - script { - def builds = jobMatrix('build', [ - [os: 'ubuntu', ver: '20.04', arch: 'x86_64', compiler: 'gcc-9'], - [os: 'ubuntu', ver: '22.04', arch: 'x86_64', compiler: 'gcc-11'], - [os: 'ubuntu', ver: '24.04', arch: 'x86_64', compiler: 'gcc-13'], - [os: 'fedora', ver: '33', arch: 'x86_64', compiler: 'gcc-10'], - [os: 'fedora', ver: '34', arch: 'x86_64', compiler: 'gcc-11'], - [os: 'fedora', ver: '35', arch: 'x86_64', compiler: 'gcc-11'], - [os: 'fedora', ver: '36', arch: 'x86_64', compiler: 'gcc-12'], - [os: 'fedora', ver: '37', arch: 'x86_64', compiler: 'gcc-12'], - [os: 'fedora', ver: '38', arch: 'x86_64', compiler: 'gcc-13'], - [os: 'fedora', ver: '39', arch: 'x86_64', compiler: 'gcc-13'], - [os: 'fedora', ver: '40', arch: 'x86_64', compiler: 'gcc-14'], - [os: 'macos', ver: '14', arch: 'x86_64', compiler: 'apple-clang-16'], - [os: 'macos', ver: '15', arch: 'x86_64', compiler: 'apple-clang-16'], - [os: 'macos', ver: '15', arch: 'arm64', compiler: 'apple-clang-16'], - ]) - - def all_debug = "-DCMAKE_BUILD_TYPE=Debug" - - def checks = jobMatrix('check', [ - [name: 'static-analyzers', extra: "${all_debug} -DRUN_STATIC_ANALYSIS=ON"], - [name: '{address,leak,ub}-sanitizers', - extra: "${all_debug} -DENABLE_SANITIZER_ADDRESS=ON -DENABLE_SANITIZER_LEAK=ON -DENABLE_SANITIZER_UNDEFINED_BEHAVIOUR=ON -DCMAKE_CXX_FLAGS='-O1 -fno-omit-frame-pointer'"], - [name: 'thread-sanitizer', extra: "${all_debug} -DENABLE_SANITIZER_THREAD=ON -DCMAKE_CXX_COMPILER=clang++"], - ]) - - parallel(builds + checks) - } - } - } - } -} diff --git a/test/ci/Containerfile.fedora b/test/ci/Containerfile.fedora deleted file mode 100644 index 6e9f6017..00000000 --- a/test/ci/Containerfile.fedora +++ /dev/null @@ -1,19 +0,0 @@ -ARG VERSION=latest -FROM fedora:${VERSION} -ARG VERSION=latest -LABEL org.opencontainers.image.source "https://github.com/FairRootGroup/FairMQ" -LABEL org.opencontainers.image.description "FairMQ development environment" -RUN dnf -y update -# https://git.gsi.de/SDE/packages/builder -RUN dnf -y install https://alfa-ci.gsi.de/packages/rpm/fedora-$VERSION-x86_64/fairsoft-release-dev.rpm -RUN dnf -y install clang cli11-devel ninja-build 'dnf-command(builddep)' libasan liblsan libtsan libubsan clang-tools-extra -RUN dnf -y builddep fairmq -RUN dnf -y clean all - -# buildah build --build-arg "VERSION=36" -t "ghcr.io/fairrootgroup/fairmq-dev/fedora-36:latest" -f test/ci/Containerfile.fedora . -# echo $GH_PAT | buildah login -u dennisklein --password-stdin ghcr.io -# buildah push ghcr.io/fairrootgroup/fairmq-dev/fedora-36:latest - -# apptainer pull docker://ghcr.io/fairrootgroup/fairmq-dev/fedora-36:latest -# echo $GH_PAT | apptainer remote login -u dennisklein --password-stdin oras://ghcr.io -# apptainer push ./fedora-36_latest.sif oras://ghcr.io/fairrootgroup/fairmq-dev/fedora-36-sif:latest diff --git a/test/ci/Containerfile.ubuntu b/test/ci/Containerfile.ubuntu deleted file mode 100644 index 7742927c..00000000 --- a/test/ci/Containerfile.ubuntu +++ /dev/null @@ -1,36 +0,0 @@ -ARG VERSION=latest -FROM ubuntu:${VERSION} -ARG VERSION=latest -LABEL org.opencontainers.image.source "https://github.com/FairRootGroup/FairMQ" -LABEL org.opencontainers.image.description "FairMQ development environment" - -RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections -RUN apt-get update -RUN apt-get -y upgrade -RUN apt-get -y install ca-certificates patch cmake git libboost-dev libboost-log-dev libboost-system-dev libboost-regex-dev libboost-filesystem-dev libboost-container-dev libboost-thread-dev libboost-date-time-dev libboost-program-options-dev g++ libfmt-dev ninja-build wget libczmq-dev libxml2-utils libfabric-dev libfabric-bin pkg-config -RUN apt-get -y clean - -RUN cd /tmp - -RUN git clone -b v1.19.2 --recurse-submodules https://github.com/FairRootGroup/asio -RUN cmake -GNinja -S asio -B asio_build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -RUN cmake --build asio_build --target install -RUN rm -rf asio asio_build - -RUN git clone -b v1.0.0 https://github.com/FairRootGroup/FairCMakeModules -RUN cmake -GNinja -S FairCMakeModules -B FairCMakeModules_build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -RUN cmake --build FairCMakeModules_build --target install -RUN rm -rf FairCMakeModules FairCMakeModules_build - -RUN git clone -b v1.11.0 https://github.com/FairRootGroup/FairLogger -RUN cmake -GNinja -S FairLogger -B FairLogger_build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DUSE_EXTERNAL_FMT=ON -RUN cmake --build FairLogger_build --target install -RUN rm -rf FairLogger FairLogger_build - -# buildah build --build-arg "VERSION=22.04" -t "ghcr.io/fairrootgroup/fairmq-dev/ubuntu-22.04:latest" -f test/ci/Containerfile.ubuntu . -# echo $GH_PAT | buildah login -u dennisklein --password-stdin ghcr.io -# buildah push ghcr.io/fairrootgroup/fairmq-dev/ubuntu-22.04:latest - -# apptainer pull docker://ghcr.io/fairrootgroup/fairmq-dev/ubuntu-22.04:latest -# echo $GH_PAT | apptainer remote login -u dennisklein --password-stdin oras://ghcr.io -# apptainer push ./ubuntu-22.04_latest.sif oras://ghcr.io/fairrootgroup/fairmq-dev/ubuntu-22.04-sif:latest diff --git a/test/ci/slurm-submit.sh b/test/ci/slurm-submit.sh deleted file mode 100755 index 4635e7fc..00000000 --- a/test/ci/slurm-submit.sh +++ /dev/null @@ -1,42 +0,0 @@ -#! /bin/bash - -label="$1" -jobsh="$2" - -if [ -z "$ALFACI_SLURM_CPUS" ] -then - ALFACI_SLURM_CPUS=20 -fi -CPUS_PER_JOB=$(($ALFACI_SLURM_CPUS / 2)) -if [ -z "$ALFACI_SLURM_EXTRA_OPTS" ] -then - ALFACI_SLURM_EXTRA_OPTS="--hint=compute_bound" -fi -if [ -z "$ALFACI_SLURM_TIMEOUT" ] -then - ALFACI_SLURM_TIMEOUT=30 -fi -if [ -z "$ALFACI_SLURM_QUEUE" ] -then - ALFACI_SLURM_QUEUE=main -fi - -echo "*** Slurm request options :" -echo "*** Working directory ..: $PWD" -echo "*** Queue ..............: $ALFACI_SLURM_QUEUE" -echo "*** CPUs ...............: $ALFACI_SLURM_CPUS" -echo "*** Wall Time ..........: $ALFACI_SLURM_TIMEOUT min" -echo "*** Job Name ...........: ${label}" -echo "*** Extra Options ......: ${ALFACI_SLURM_EXTRA_OPTS}" -echo "*** Submitting job at ....: $(date -R)" -( - set -x - srun -p $ALFACI_SLURM_QUEUE -c $CPUS_PER_JOB -n 1 \ - -t $ALFACI_SLURM_TIMEOUT \ - --job-name="${label}" \ - ${ALFACI_SLURM_EXTRA_OPTS} \ - bash "${jobsh}" -) -retval=$? -echo "*** Exit Code ............: $retval" -exit "$retval" diff --git a/test/ci/spack.yaml b/test/ci/spack.yaml new file mode 100644 index 00000000..0b1c23ef --- /dev/null +++ b/test/ci/spack.yaml @@ -0,0 +1,17 @@ +spack: + specs: + - "fairlogger@2.2.0 ^fmt@:11" + - "boost@1.66: +container +program_options +filesystem +date_time +regex" + - "libzmq@4.1.4:" + - ninja + - "cmake@3.15:" + + config: + install_tree: + root: /opt/spack + padded_length: 128 + + mirrors: + local-buildcache: + url: oci://ghcr.io/fairrootgroup/fairmq-spack-buildcache + signed: false