Mirror/FairMQ
1
0
Fork 0
mirror of https://github.com/FairRootGroup/FairMQ.git synced 2025-10-13 08:41:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: Run additional checks

Browse Source
This commit is contained in:
Dennis Klein 2021-07-02 00:31:56 +02:00 committed by Dennis Klein
parent 08d72d492a
commit 38f9870893
12 changed files with 186 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
FairMQTest.cmake
View File

@ -13,6 +13,7 @@ set(CTEST_BINARY_DIRECTORY build)
set(CTEST_CMAKE_GENERATOR "Ninja") set(CTEST_CMAKE_GENERATOR "Ninja")
set(CTEST_USE_LAUNCHERS ON) set(CTEST_USE_LAUNCHERS ON)
set(CTEST_CONFIGURATION_TYPE "RelWithDebInfo") set(CTEST_CONFIGURATION_TYPE "RelWithDebInfo")
set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 102400)
if(NOT NCPUS) if(NOT NCPUS)
if(ENV{SLURM_CPUS_PER_TASK}) if(ENV{SLURM_CPUS_PER_TASK})
@ -53,6 +54,27 @@ endif()
if(RUN_STATIC_ANALYSIS) if(RUN_STATIC_ANALYSIS)
list(APPEND options "-DRUN_STATIC_ANALYSIS=ON") list(APPEND options "-DRUN_STATIC_ANALYSIS=ON")
endif() endif()
if(CMAKE_BUILD_TYPE)
set(CTEST_CONFIGURATION_TYPE ${CMAKE_BUILD_TYPE})
endif()
if(ENABLE_SANITIZER_ADDRESS)
list(APPEND options "-DENABLE_SANITIZER_ADDRESS=ON")
endif()
if(ENABLE_SANITIZER_LEAK)
list(APPEND options "-DENABLE_SANITIZER_LEAK=ON")
endif()
if(ENABLE_SANITIZER_UNDEFINED_BEHAVIOR)
list(APPEND options "-DENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON")
endif()
if(ENABLE_SANITIZER_MEMORY)
list(APPEND options "-DENABLE_SANITIZER_MEMORY=ON")
endif()
if(ENABLE_SANITIZER_THREAD)
list(APPEND options "-DENABLE_SANITIZER_THREAD=ON")
endif()
if(CMAKE_CXX_FLAGS)
list(APPEND options "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}")
endif()
list(REMOVE_DUPLICATES options) list(REMOVE_DUPLICATES options)
list(JOIN options ";" optionsstr) list(JOIN options ";" optionsstr)
ctest_configure(OPTIONS "${optionsstr}") ctest_configure(OPTIONS "${optionsstr}")
@ -63,12 +85,14 @@ ctest_build(FLAGS "-j${NCPUS}")
ctest_submit() ctest_submit()
if(NOT RUN_STATIC_ANALYSIS)
ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}" ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}"
PARALLEL_LEVEL 1 PARALLEL_LEVEL 1
SCHEDULE_RANDOM ON SCHEDULE_RANDOM ON
RETURN_VALUE _ctest_test_ret_val) RETURN_VALUE _ctest_test_ret_val)
ctest_submit() ctest_submit()
endif()
if(_ctest_test_ret_val) if(_ctest_test_ret_val)
Message(FATAL_ERROR "Some tests failed.") Message(FATAL_ERROR "Some tests failed.")

60
Jenkinsfile vendored
View File

@ -3,12 +3,24 @@
def jobMatrix(String type, List specs) { def jobMatrix(String type, List specs) {
def nodes = [:] def nodes = [:]
for (spec in specs) { for (spec in specs) {
def job = ""
def selector = ""
def os = ""
def ver = ""
if (type == 'build') {
job = "${spec.os}-${spec.ver}-${spec.arch}-${spec.compiler}" job = "${spec.os}-${spec.ver}-${spec.arch}-${spec.compiler}"
def label = "${type}/${job}" selector = "${spec.os}-${spec.ver}-${spec.arch}"
def selector = "${spec.os}-${spec.ver}-${spec.arch}" os = spec.os
def os = spec.os ver = spec.ver
def ver = spec.ver } else { // == 'check'
def check = spec.check job = "${spec.name}"
selector = 'fedora-34-x86_64'
os = 'fedora'
ver = '34'
}
def label = "${job}"
def extra = spec.extra def extra = spec.extra
nodes[label] = { nodes[label] = {
@ -25,16 +37,13 @@ def jobMatrix(String type, List specs) {
if (selector =~ /^macos/) { if (selector =~ /^macos/) {
sh """\ sh """\
echo \"export DDS_ROOT=\\\"\\\$(brew --prefix dds)\\\"\" >> ${jobscript} echo \"export DDS_ROOT=\\\"\\\$(brew --prefix dds)\\\"\" >> ${jobscript}
echo \"export PATH=\\\"\\\$(brew --prefix dds)/bin:\\\$PATH\\\"\" >> ${jobscript}
echo \"${ctestcmd}\" >> ${jobscript} echo \"${ctestcmd}\" >> ${jobscript}
""" """
sh "cat ${jobscript}" sh "cat ${jobscript}"
sh "bash ${jobscript}" sh "bash ${jobscript}"
} else { } else {
def static_analysis = "OFF" def containercmd = "singularity exec --net --ipc --uts --pid -B/shared ${env.SINGULARITY_CONTAINER_ROOT}/fairmq/${os}.${ver}.sif bash -l -c \\\"${ctestcmd} ${extra}\\\""
if (selector =~ /^fedora-32/) {
static_analysis = "ON"
}
def containercmd = "singularity exec -B/shared ${env.SINGULARITY_CONTAINER_ROOT}/fairmq/${os}.${ver}.sif bash -l -c \\\"${ctestcmd} -DRUN_STATIC_ANALYSIS=${static_analysis}\\\""
sh """\ sh """\
echo \"echo \\\"*** Job started at .......: \\\$(date -R)\\\"\" >> ${jobscript} echo \"echo \\\"*** Job started at .......: \\\$(date -R)\\\"\" >> ${jobscript}
echo \"echo \\\"*** Job ID ...............: \\\${SLURM_JOB_ID}\\\"\" >> ${jobscript} echo \"echo \\\"*** Job ID ...............: \\\${SLURM_JOB_ID}\\\"\" >> ${jobscript}
@ -46,13 +55,12 @@ def jobMatrix(String type, List specs) {
sh "cat ${jobscript}" sh "cat ${jobscript}"
sh "test/ci/slurm-submit.sh \"FairMQ \${JOB_BASE_NAME} ${label}\" ${jobscript}" sh "test/ci/slurm-submit.sh \"FairMQ \${JOB_BASE_NAME} ${label}\" ${jobscript}"
withChecks('Static Analysis') { if (job == "static-analyzers") {
if (static_analysis == "ON") {
recordIssues(enabledForFailure: true, recordIssues(enabledForFailure: true,
tools: [gcc(pattern: 'build/Testing/Temporary/*.log')], tools: [gcc(pattern: 'build/Testing/Temporary/*.log')],
filters: [excludeFile('extern/*'), excludeFile('usr/*')], filters: [excludeFile('extern/*'), excludeFile('usr/*')],
skipBlames: true) skipBlames: true,
} skipPublishingChecks: true)
} }
} }
@ -81,18 +89,26 @@ pipeline{
stage("CI") { stage("CI") {
steps{ steps{
script { script {
def all = '-DHAS_ASIO=ON -DHAS_DDS=ON -DHAS_PMIX=ON'
def builds = jobMatrix('build', [ def builds = jobMatrix('build', [
[os: 'ubuntu', ver: '20.04', arch: 'x86_64', compiler: 'gcc-9', [os: 'ubuntu', ver: '20.04', arch: 'x86_64', compiler: 'gcc-9', extra: all],
extra: '-DHAS_DDS=ON -DHAS_ASIO=ON -DHAS_PMIX=ON'], [os: 'fedora', ver: '32', arch: 'x86_64', compiler: 'gcc-10', extra: all],
[os: 'fedora', ver: '32', arch: 'x86_64', compiler: 'gcc-10', [os: 'fedora', ver: '33', arch: 'x86_64', compiler: 'gcc-10', extra: all],
extra: '-DHAS_PMIX=ON -DHAS_DDS=ON -DHAS_ASIOFI=ON -DHAS_ASIO=ON'], [os: 'fedora', ver: '34', arch: 'x86_64', compiler: 'gcc-11', extra: all],
[os: 'fedora', ver: '34', arch: 'x86_64', compiler: 'gcc-11',
extra: '-DHAS_PMIX=ON -DHAS_DDS=ON -DHAS_ASIOFI=ON -DHAS_ASIO=ON'],
[os: 'macos', ver: '11', arch: 'x86_64', compiler: 'apple-clang-12', [os: 'macos', ver: '11', arch: 'x86_64', compiler: 'apple-clang-12',
extra: '-DHAS_DDS=ON -DHAS_ASIO=ON'], extra: '-DHAS_ASIO=ON -DHAS_DDS=ON'],
]) ])
parallel(builds) def all_debug = "${all} -DCMAKE_BUILD_TYPE=Debug"
def checks = jobMatrix('check', [
[name: 'static-analyzers', extra: "${all_debug} -DRUN_STATIC_ANALYSIS=ON"],
[name: '{address,leak,ub}-sanitizers',
extra: "${all_debug} -DENABLE_SANITIZER_ADDRESS=ON -DENABLE_SANITIZER_LEAK=ON -DENABLE_SANITIZER_UNDEFINED_BEHAVIOUR=ON -DCMAKE_CXX_FLAGS='-O1 -fno-omit-frame-pointer'"],
])
parallel(builds + checks)
} }
} }
} }

2
cmake/FairMQProjectSettings.cmake
View File

@ -142,7 +142,7 @@ if(RUN_STATIC_ANALYSIS)
set(analyser "clang-tidy") set(analyser "clang-tidy")
find_program(${analyser}_FOUND "${analyser}") find_program(${analyser}_FOUND "${analyser}")
if(${analyser}_FOUND) if(${analyser}_FOUND)
set(CMAKE_CXX_CLANG_TIDY "${${analyser}_FOUND}") set(CMAKE_CXX_CLANG_TIDY ${analyser} "--extra-arg=-std=c++17")
endif() endif()
list(APPEND PROJECT_STATIC_ANALYSERS "${analyser}") list(APPEND PROJECT_STATIC_ANALYSERS "${analyser}")

14
examples/dds/CMakeLists.txt
View File

@ -27,13 +27,13 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-dds-env.sh ${CMAKE_CURRENT_
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-dds.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh @ONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-dds.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh @ONLY)
# test # test
if(DDS_FOUND) # if(DDS_FOUND)
add_test(NAME Example.DDS.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh localhost) # add_test(NAME Example.DDS.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh localhost)
set_tests_properties(Example.DDS.localhost PROPERTIES # set_tests_properties(Example.DDS.localhost PROPERTIES
TIMEOUT 15 # TIMEOUT 15
PASS_REGULAR_EXPRESSION "Example successful" # PASS_REGULAR_EXPRESSION "Example successful"
) # )
endif() # endif()
# install # install

12
examples/n-m/CMakeLists.txt
View File

@ -29,12 +29,12 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-dds.sh.in ${
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-pair-dds.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair-dds.sh @ONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-pair-dds.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair-dds.sh @ONLY)
# test # test
if(DDS_FOUND) # if(DDS_FOUND)
add_test(NAME Example.N-M.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-dds.sh localhost) # add_test(NAME Example.N-M.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-dds.sh localhost)
set_tests_properties(Example.N-M.localhost PROPERTIES TIMEOUT 15 PASS_REGULAR_EXPRESSION "Example successful") # set_tests_properties(Example.N-M.localhost PROPERTIES TIMEOUT 15 PASS_REGULAR_EXPRESSION "Example successful")
add_test(NAME Example.N-M-pair.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair-dds.sh localhost) # add_test(NAME Example.N-M-pair.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair-dds.sh localhost)
set_tests_properties(Example.N-M-pair.localhost PROPERTIES TIMEOUT 15 PASS_REGULAR_EXPRESSION "Example successful") # set_tests_properties(Example.N-M-pair.localhost PROPERTIES TIMEOUT 15 PASS_REGULAR_EXPRESSION "Example successful")
endif() # endif()
# install # install
install( install(

8
examples/qc/CMakeLists.txt
View File

@ -28,10 +28,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-qc-env.sh ${CMAKE_CURRENT_B
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-qc.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-qc.sh @ONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-qc.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-qc.sh @ONLY)
# test # test
if(DDS_FOUND) # if(DDS_FOUND)
add_test(NAME Example.QC.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-qc.sh localhost) # add_test(NAME Example.QC.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-qc.sh localhost)
set_tests_properties(Example.QC.localhost PROPERTIES TIMEOUT 15 PASS_REGULAR_EXPRESSION "Example successful") # set_tests_properties(Example.QC.localhost PROPERTIES TIMEOUT 15 PASS_REGULAR_EXPRESSION "Example successful")
endif() # endif()
# install # install
install( install(

40
test/CMakeLists.txt
View File

@ -298,26 +298,26 @@ add_testsuite(MemoryResources
) )
if(BUILD_SDK) if(BUILD_SDK)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/sdk/test_topo.xml # configure_file(${CMAKE_CURRENT_SOURCE_DIR}/sdk/test_topo.xml
${CMAKE_BINARY_DIR}/test_topo.xml) # ${CMAKE_BINARY_DIR}/test_topo.xml)
add_testsuite(SDK # add_testsuite(SDK
SOURCES # SOURCES
${CMAKE_CURRENT_BINARY_DIR}/runner.cxx # ${CMAKE_CURRENT_BINARY_DIR}/runner.cxx
sdk/_async_op.cxx # sdk/_async_op.cxx
sdk/_dds.cxx # sdk/_dds.cxx
sdk/_topology.cxx # sdk/_topology.cxx
sdk/Fixtures.h # sdk/Fixtures.h
#
LINKS # LINKS
SDK # SDK
Tools # Tools
DDS::dds_topology_lib # DDS::dds_topology_lib
DDS::dds_tools_lib # DDS::dds_tools_lib
INCLUDES ${CMAKE_CURRENT_SOURCE_DIR} # INCLUDES ${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR} # ${CMAKE_CURRENT_BINARY_DIR}
TIMEOUT 30 # TIMEOUT 30
${definitions} # ${definitions}
) # )
if(DDS_TESTS) if(DDS_TESTS)
foreach(i RANGE 1 ${DDS_TESTS}) foreach(i RANGE 1 ${DDS_TESTS})

8
test/ci/fedora.32.def Normal file
View File

@ -0,0 +1,8 @@
Bootstrap: docker
From: fedora:32
%post
dnf -y update
dnf -y install https://alfa-ci.gsi.de/packages/rpm/fedora-32-x86_64/fairsoft-release-dev.rpm
dnf -y install git boost-devel cmake gcc-c++ dds-devel fairlogger-devel faircmakemodules asio-devel flatbuffers-devel zeromq-devel asiofi-devel cli11-devel pmix-devel ninja-build
dnf -y clean all

8
test/ci/fedora.33.def Normal file
View File

@ -0,0 +1,8 @@
Bootstrap: docker
From: fedora:33
%post
dnf -y update
dnf -y install https://alfa-ci.gsi.de/packages/rpm/fedora-33-x86_64/fairsoft-release-dev.rpm
dnf -y install git boost-devel cmake gcc-c++ dds-devel fairlogger-devel faircmakemodules asio-devel flatbuffers-devel zeromq-devel asiofi-devel cli11-devel pmix-devel ninja-build
dnf -y clean all

9
test/ci/fedora.34.def Normal file
View File

@ -0,0 +1,9 @@
Bootstrap: docker
From: fedora:34
%post
dnf -y update
dnf -y install https://alfa-ci.gsi.de/packages/rpm/fedora-34-x86_64/fairsoft-release-dev.rpm
dnf -y install git boost-devel cmake gcc-c++ dds-devel fairlogger-devel faircmakemodules asio-devel flatbuffers-devel zeromq-devel asiofi-devel cli11-devel pmix-devel ninja-build
dnf -y install libasan liblsan libtsan libubsan clang-tools-extra
dnf -y clean all

5
test/ci/slurm-submit.sh
View File

@ -5,8 +5,9 @@ jobsh="$2"
if [ -z "$ALFACI_SLURM_CPUS" ] if [ -z "$ALFACI_SLURM_CPUS" ]
then then
ALFACI_SLURM_CPUS=32 ALFACI_SLURM_CPUS=20
fi fi
CPUS_PER_JOB=$(($ALFACI_SLURM_CPUS / 2))
if [ -z "$ALFACI_SLURM_EXTRA_OPTS" ] if [ -z "$ALFACI_SLURM_EXTRA_OPTS" ]
then then
ALFACI_SLURM_EXTRA_OPTS="--hint=compute_bound" ALFACI_SLURM_EXTRA_OPTS="--hint=compute_bound"
@ -30,7 +31,7 @@ echo "*** Extra Options ......: ${ALFACI_SLURM_EXTRA_OPTS}"
echo "*** Submitting job at ....: $(date -R)" echo "*** Submitting job at ....: $(date -R)"
( (
set -x set -x
srun -p $ALFACI_SLURM_QUEUE -c $ALFACI_SLURM_CPUS -n 1 \ srun -p $ALFACI_SLURM_QUEUE -c $CPUS_PER_JOB -n 1 \
-t $ALFACI_SLURM_TIMEOUT \ -t $ALFACI_SLURM_TIMEOUT \
--job-name="${label}" \ --job-name="${label}" \
${ALFACI_SLURM_EXTRA_OPTS} \ ${ALFACI_SLURM_EXTRA_OPTS} \

48
test/ci/ubuntu.20.04.def Normal file
View File

@ -0,0 +1,48 @@
Bootstrap: docker
From: ubuntu:20.04
%post
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get -y upgrade
apt-get -y install ca-certificates patch cmake git libboost-dev libboost-log-dev libboost-system-dev libboost-regex-dev libboost-filesystem-dev libboost-container-dev libboost-thread-dev libboost-date-time-dev libboost-program-options-dev g++ libfmt-dev ninja-build wget libczmq-dev libxml2-utils libfabric-dev libfabric-bin libpmix-dev
apt-get -y clean
cd /tmp
git clone -b v1.18.1 --recurse-submodules https://github.com/FairRootGroup/asio
cmake -GNinja -S asio -B asio_build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release
cmake --build asio_build --target install
rm -rf asio asio_build
git clone -b v0.2.0 https://github.com/FairRootGroup/FairCMakeModules
cmake -GNinja -S FairCMakeModules -B FairCMakeModules_build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release
cmake --build FairCMakeModules_build --target install
rm -rf FairCMakeModules FairCMakeModules_build
# git clone -b v0.5.0 --recurse-submodules https://github.com/FairRootGroup/asiofi
# cmake -GNinja -S asiofi -B asiofi_build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release
# cmake --build asiofi_build --target install
# rm -rf asiofi asiofi_build
git clone -b v1.12.1 https://github.com/google/flatbuffers
cmake -GNinja -S flatbuffers -B flatbuffers_build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON
cmake --build flatbuffers_build --target install
rm -rf flatbuffers flatbuffers_build
git clone -b v1.9.3 https://github.com/FairRootGroup/FairLogger
cmake -GNinja -S FairLogger -B FairLogger_build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DUSE_EXTERNAL_FMT=ON
cmake --build FairLogger_build --target install
rm -rf FairLogger FairLogger_build
git clone -b 3.5.14 https://github.com/FairRootGroup/DDS
cd DDS
wget https://git.gsi.de/SDE/packages/dds/-/raw/f33f1a6a95eecbfebc6042af382fcf2d3128ba15/fix_install_destinations.patch
wget https://git.gsi.de/SDE/packages/dds/-/raw/f33f1a6a95eecbfebc6042af382fcf2d3128ba15/strip_etc_version.patch
patch -p1 < fix_install_destinations.patch
patch -p1 < strip_etc_version.patch
cd ..
cmake -GNinja -S DDS -B DDS_build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DBUILD_dds-octopus=OFF -DBUILD_dds-tutorials=OFF -DBUILD_dds-test=OFF
cmake --build DDS_build --target wn_bin
cmake --build DDS_build --target install
rm -rf DDS DDS_build