mirror of
https://github.com/FairRootGroup/FairMQ.git
synced 2025-10-13 00:31:14 +00:00
CI: Adapt to new alfaci build hosts
This commit is contained in:
Dennis Klein
Dennis Klein
parent
62438bd99e
commit
72f319e276
115
FairMQTest.cmake
115
FairMQTest.cmake
|
|
@ -1,87 +1,62 @@
|
|||
################################################################################
|
||||
# Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH #
|
||||
# Copyright (C) 2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH #
|
||||
# #
|
||||
# This software is distributed under the terms of the #
|
||||
# GNU Lesser General Public Licence (LGPL) version 3, #
|
||||
# copied verbatim in the file "LICENSE" #
|
||||
################################################################################
|
||||
Set(CTEST_SOURCE_DIRECTORY $ENV{SOURCEDIR})
|
||||
Set(CTEST_BINARY_DIRECTORY $ENV{BUILDDIR})
|
||||
Set(CTEST_SITE $ENV{SITE})
|
||||
Set(CTEST_BUILD_NAME $ENV{LABEL})
|
||||
Set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
|
||||
Set(CTEST_PROJECT_NAME "FairMQ")
|
||||
|
||||
Find_Program(CTEST_GIT_COMMAND NAMES git)
|
||||
Set(CTEST_UPDATE_COMMAND "${CTEST_GIT_COMMAND}")
|
||||
cmake_host_system_information(RESULT fqdn QUERY FQDN)
|
||||
|
||||
Set(BUILD_COMMAND "make")
|
||||
Set(CTEST_BUILD_COMMAND "${BUILD_COMMAND} -j$ENV{number_of_processors}")
|
||||
set(CTEST_SOURCE_DIRECTORY .)
|
||||
set(CTEST_BINARY_DIRECTORY build)
|
||||
set(CTEST_CMAKE_GENERATOR "Ninja")
|
||||
set(CTEST_USE_LAUNCHERS ON)
|
||||
set(CTEST_CONFIGURATION_TYPE "RelWithDebInfo")
|
||||
|
||||
String(TOUPPER $ENV{ctest_model} _Model)
|
||||
Set(configure_options "-DCMAKE_BUILD_TYPE=$ENV{ctest_model}")
|
||||
|
||||
Set(CTEST_USE_LAUNCHERS 1)
|
||||
Set(configure_options "${configure_options};-DCTEST_USE_LAUNCHERS=${CTEST_USE_LAUNCHERS}")
|
||||
|
||||
Set(configure_options "${configure_options};-DDISABLE_COLOR=ON")
|
||||
Set(configure_options "${configure_options};-DCMAKE_PREFIX_PATH=$ENV{SIMPATH}")
|
||||
# Set(configure_options "${configure_options};-DBUILD_OFI_TRANSPORT=ON")
|
||||
Set(configure_options "${configure_options};-DBUILD_DDS_PLUGIN=ON")
|
||||
Set(configure_options "${configure_options};-DBUILD_SDK=ON")
|
||||
Set(configure_options "${configure_options};-DBUILD_SDK_COMMANDS=ON")
|
||||
|
||||
Set(EXTRA_FLAGS $ENV{EXTRA_FLAGS})
|
||||
If(EXTRA_FLAGS)
|
||||
Set(configure_options "${configure_options};${EXTRA_FLAGS}")
|
||||
EndIf()
|
||||
|
||||
If($ENV{ctest_model} MATCHES Profile)
|
||||
Find_Program(GCOV_COMMAND gcov)
|
||||
If(GCOV_COMMAND)
|
||||
Message("Found GCOV: ${GCOV_COMMAND}")
|
||||
Set(CTEST_COVERAGE_COMMAND ${GCOV_COMMAND})
|
||||
set(CTEST_COVERAGE_EXTRA_FLAGS "-p")
|
||||
EndIf(GCOV_COMMAND)
|
||||
EndIf()
|
||||
|
||||
If($ENV{ctest_model} MATCHES Nightly OR $ENV{ctest_model} MATCHES Profile)
|
||||
Ctest_Empty_Binary_Directory(${CTEST_BINARY_DIRECTORY})
|
||||
EndIf()
|
||||
|
||||
Ctest_Start($ENV{ctest_model})
|
||||
|
||||
Ctest_Configure(BUILD "${CTEST_BINARY_DIRECTORY}"
|
||||
OPTIONS "${configure_options}"
|
||||
)
|
||||
|
||||
Ctest_Build(BUILD "${CTEST_BINARY_DIRECTORY}")
|
||||
|
||||
unset(exclude_tests)
|
||||
if($ENV{EXCLUDE_UNSTABLE_DDS_TESTS})
|
||||
set(exclude_tests EXCLUDE ".*\\.localhost$")
|
||||
if(NOT NCPUS)
|
||||
if(ENV{SLURM_CPUS_PER_TASK})
|
||||
set(NCPUS $ENV{SLURM_CPUS_PER_TASK})
|
||||
else()
|
||||
include(ProcessorCount)
|
||||
ProcessorCount(NCPUS)
|
||||
if(NCPUS EQUAL 0)
|
||||
set(NCPUS 1)
|
||||
endif()
|
||||
Ctest_Test(BUILD "${CTEST_BINARY_DIRECTORY}"
|
||||
# PARALLEL_LEVEL $ENV{number_of_processors}
|
||||
${exclude_tests}
|
||||
PARALLEL_LEVEL $ENV{number_of_processors}
|
||||
RETURN_VALUE _ctest_test_ret_val
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if ("$ENV{CTEST_SITE}" STREQUAL "")
|
||||
set(CTEST_SITE "${fqdn}")
|
||||
else()
|
||||
set(CTEST_SITE $ENV{CTEST_SITE})
|
||||
endif()
|
||||
|
||||
if ("$ENV{LABEL}" STREQUAL "")
|
||||
set(CTEST_BUILD_NAME "build")
|
||||
else()
|
||||
set(CTEST_BUILD_NAME $ENV{LABEL})
|
||||
endif()
|
||||
|
||||
ctest_start(Continuous)
|
||||
|
||||
list(APPEND options
|
||||
"-DDISABLE_COLOR=ON"
|
||||
"-DBUILD_SDK_COMMANDS=ON"
|
||||
"-DBUILD_SDK=ON"
|
||||
"-DBUILD_DDS_PLUGIN=ON"
|
||||
)
|
||||
list(JOIN options ";" optionsstr)
|
||||
ctest_configure(OPTIONS "${optionsstr}")
|
||||
|
||||
If(GCOV_COMMAND)
|
||||
Ctest_Coverage(BUILD "${CTEST_BINARY_DIRECTORY}" LABELS coverage)
|
||||
EndIf()
|
||||
ctest_build(FLAGS "-j${NCPUS}")
|
||||
|
||||
If("$ENV{do_codecov_upload}")
|
||||
Execute_Process(COMMAND curl https://codecov.io/bash -o codecov_uploader.sh
|
||||
WORKING_DIRECTORY ${CTEST_BINARY_DIRECTORY}
|
||||
TIMEOUT 60)
|
||||
Execute_Process(COMMAND bash ./codecov_uploader.sh -X gcov
|
||||
WORKING_DIRECTORY ${CTEST_BINARY_DIRECTORY}
|
||||
TIMEOUT 60)
|
||||
EndIf()
|
||||
ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}"
|
||||
PARALLEL_LEVEL 1
|
||||
SCHEDULE_RANDOM ON
|
||||
RETURN_VALUE _ctest_test_ret_val)
|
||||
|
||||
Ctest_Submit()
|
||||
ctest_submit()
|
||||
|
||||
if(_ctest_test_ret_val)
|
||||
Message(FATAL_ERROR "Some tests failed.")
|
||||
|
|
|
|||
96
Jenkinsfile
vendored
96
Jenkinsfile
vendored
|
|
@ -1,67 +1,50 @@
|
|||
#!groovy
|
||||
|
||||
def specToLabel(Map spec) {
|
||||
return "${spec.os}-${spec.arch}-${spec.compiler}-FairSoft_${spec.fairsoft}"
|
||||
}
|
||||
|
||||
def jobMatrix(String prefix, List specs, Closure callback) {
|
||||
def jobMatrix(String prefix, String type, List specs) {
|
||||
def nodes = [:]
|
||||
for (spec in specs) {
|
||||
def label = specToLabel(spec)
|
||||
def node_tag = label
|
||||
if (spec.os =~ /macOS/) {
|
||||
node_tag = spec.os
|
||||
}
|
||||
def fairsoft = spec.fairsoft
|
||||
job = "${spec.os}-${spec.ver}-${spec.arch}-${spec.compiler}"
|
||||
def label = "${type}/${job}"
|
||||
def selector = "${spec.os}-${spec.ver}-${spec.arch}"
|
||||
def os = spec.os
|
||||
def compiler = spec.compiler
|
||||
nodes["${prefix}/${label}"] = {
|
||||
node(node_tag) {
|
||||
def ver = spec.ver
|
||||
def check = spec.check
|
||||
|
||||
nodes[label] = {
|
||||
node(selector) {
|
||||
githubNotify(context: "${prefix}/${label}", description: 'Building ...', status: 'PENDING')
|
||||
try {
|
||||
deleteDir()
|
||||
checkout scm
|
||||
|
||||
def jobscript = 'job.sh'
|
||||
def ctestcmd = "ctest -S FairMQTest.cmake -V --output-on-failure"
|
||||
sh "echo \"set -e\" >> ${jobscript}"
|
||||
sh "echo \"export LABEL=\\\"\${JOB_BASE_NAME} ${label}\\\"\" >> ${jobscript}"
|
||||
if (selector =~ /^macos/) {
|
||||
sh """\
|
||||
echo "export SIMPATH=\${SIMPATH_PREFIX}${fairsoft}" >> Dart.cfg
|
||||
echo "export FAIRSOFT_VERSION=${fairsoft}" >> Dart.cfg
|
||||
echo \"export DDS_ROOT=\\\"\\\$(brew --prefix dds)\\\"\" >> ${jobscript}
|
||||
echo \"${ctestcmd}\" >> ${jobscript}
|
||||
"""
|
||||
if (os =~ /Debian/ && compiler =~ /gcc9/) {
|
||||
sh '''\
|
||||
echo "source /etc/profile.d/modules.sh" >> Dart.cfg
|
||||
echo "module use /cvmfs/it.gsi.de/modulefiles" >> Dart.cfg
|
||||
echo "module load compiler/gcc/9.1.0" >> Dart.cfg
|
||||
'''
|
||||
}
|
||||
if (os =~ /[Mm]acOS/) {
|
||||
sh "echo \"export EXTRA_FLAGS='-DCMAKE_CXX_COMPILER=clang++'\" >> Dart.cfg"
|
||||
sh "cat ${jobscript}"
|
||||
sh "bash ${jobscript}"
|
||||
} else {
|
||||
sh "echo \"export EXTRA_FLAGS='-DCMAKE_CXX_COMPILER=g++'\" >> Dart.cfg"
|
||||
def containercmd = "singularity exec -B/shared ${env.SINGULARITY_CONTAINER_ROOT}/fairmq/${os}.${ver}.sif bash -l -c \\\"${ctestcmd}\\\""
|
||||
sh """\
|
||||
echo \"echo \\\"*** Job started at .......: \\\$(date -R)\\\"\" >> ${jobscript}
|
||||
echo \"echo \\\"*** Job ID ...............: \\\${SLURM_JOB_ID}\\\"\" >> ${jobscript}
|
||||
echo \"echo \\\"*** Compute node .........: \\\$(hostname -f)\\\"\" >> ${jobscript}
|
||||
echo \"unset http_proxy\" >> ${jobscript}
|
||||
echo \"unset HTTP_PROXY\" >> ${jobscript}
|
||||
echo \"${containercmd}\" >> ${jobscript}
|
||||
"""
|
||||
sh "cat ${jobscript}"
|
||||
sh "test/ci/slurm-submit.sh \"FairMQ \${JOB_BASE_NAME} ${label}\" ${jobscript}"
|
||||
}
|
||||
|
||||
sh '''\
|
||||
echo "export BUILDDIR=$PWD/build" >> Dart.cfg
|
||||
echo "export SOURCEDIR=$PWD" >> Dart.cfg
|
||||
echo "export PATH=\\\$SIMPATH/bin:\\\$PATH" >> Dart.cfg
|
||||
echo "export GIT_BRANCH=$JOB_BASE_NAME" >> Dart.cfg
|
||||
echo "echo \\\$PATH" >> Dart.cfg
|
||||
'''
|
||||
|
||||
if (os =~ /macOS10.14/) {
|
||||
sh "echo \"export EXCLUDE_UNSTABLE_DDS_TESTS=1\" >> Dart.cfg"
|
||||
}
|
||||
|
||||
sh 'cat Dart.cfg'
|
||||
|
||||
callback.call(spec, label)
|
||||
|
||||
deleteDir()
|
||||
githubNotify(context: "${prefix}/${label}", description: 'Success', status: 'SUCCESS')
|
||||
} catch (e) {
|
||||
def tarball = "${prefix}_${label}_dds_logs.tar.gz"
|
||||
sh "tar czvf ${tarball} -C \${WORKSPACE}/build/test/ .DDS/"
|
||||
archiveArtifacts tarball
|
||||
|
||||
deleteDir()
|
||||
githubNotify(context: "${prefix}/${label}", description: 'Error', status: 'ERROR')
|
||||
throw e
|
||||
|
|
@ -78,23 +61,12 @@ pipeline{
|
|||
stage("Run CI Matrix") {
|
||||
steps{
|
||||
script {
|
||||
def build_jobs = jobMatrix('build', [
|
||||
[os: 'Debian8', arch: 'x86_64', compiler: 'gcc9.1.0', fairsoft: 'fairmq_dev'],
|
||||
[os: 'macOS10.14', arch: 'x86_64', compiler: 'AppleClang11.0', fairsoft: 'fairmq_dev'],
|
||||
[os: 'macOS10.15', arch: 'x86_64', compiler: 'AppleClang12.0', fairsoft: 'fairmq_dev'],
|
||||
]) { spec, label ->
|
||||
sh './Dart.sh alfa_ci Dart.cfg'
|
||||
}
|
||||
def builds = jobMatrix('alfa-ci', 'build', [
|
||||
[os: 'fedora', ver: '32', arch: 'x86_64', compiler: 'gcc-10'],
|
||||
[os: 'macos', ver: '11', arch: 'x86_64', compiler: 'apple-clang-12'],
|
||||
])
|
||||
|
||||
def profile_jobs = jobMatrix('codecov', [
|
||||
[os: 'Debian8', arch: 'x86_64', compiler: 'gcc9.1.0', fairsoft: 'fairmq_dev'],
|
||||
]) { spec, label ->
|
||||
withCredentials([string(credentialsId: 'fairmq_codecov_token', variable: 'CODECOV_TOKEN')]) {
|
||||
sh './Dart.sh codecov Dart.cfg'
|
||||
}
|
||||
}
|
||||
|
||||
parallel(build_jobs + profile_jobs)
|
||||
parallel(builds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,81 +0,0 @@
|
|||
#!groovy
|
||||
|
||||
def specToLabel(Map spec) {
|
||||
return "${spec.os}-${spec.arch}-${spec.compiler}-FairSoft_${spec.fairsoft}"
|
||||
}
|
||||
|
||||
def buildMatrix(List specs, Closure callback) {
|
||||
def nodes = [:]
|
||||
for (spec in specs) {
|
||||
def label = specToLabel(spec)
|
||||
def fairsoft = spec.fairsoft
|
||||
def os = spec.os
|
||||
def compiler = spec.compiler
|
||||
nodes[label] = {
|
||||
node(label) {
|
||||
try {
|
||||
deleteDir()
|
||||
checkout scm
|
||||
|
||||
sh """\
|
||||
echo "export SIMPATH=\${SIMPATH_PREFIX}${fairsoft}" >> Dart.cfg
|
||||
echo "export FAIRSOFT_VERSION=${fairsoft}" >> Dart.cfg
|
||||
"""
|
||||
if (os =~ /Debian/ && compiler =~ /gcc9/) {
|
||||
sh '''\
|
||||
echo "source /etc/profile.d/modules.sh" >> Dart.cfg
|
||||
echo "module use /cvmfs/it.gsi.de/modulefiles" >> Dart.cfg
|
||||
echo "module load compiler/gcc/9.1.0" >> Dart.cfg
|
||||
'''
|
||||
}
|
||||
if (os =~ /MacOS/) {
|
||||
sh "echo \"export EXTRA_FLAGS='-DCMAKE_CXX_COMPILER=clang++'\" >> Dart.cfg"
|
||||
} else {
|
||||
sh "echo \"export EXTRA_FLAGS='-DCMAKE_CXX_COMPILER=g++'\" >> Dart.cfg"
|
||||
}
|
||||
sh '''\
|
||||
echo "export BUILDDIR=$PWD/build" >> Dart.cfg
|
||||
echo "export SOURCEDIR=$PWD" >> Dart.cfg
|
||||
echo "export PATH=\\\$SIMPATH/bin:\\\$PATH" >> Dart.cfg
|
||||
echo "export GIT_BRANCH=dev" >> Dart.cfg
|
||||
echo "echo \\\$PATH" >> Dart.cfg
|
||||
'''
|
||||
sh 'cat Dart.cfg'
|
||||
|
||||
callback.call(spec, label)
|
||||
|
||||
deleteDir()
|
||||
} catch (e) {
|
||||
def tarball = "${label}_dds_logs.tar.gz"
|
||||
sh "tar czvf ${tarball} -C \${WORKSPACE}/build/test/ .DDS/"
|
||||
archiveArtifacts tarball
|
||||
|
||||
deleteDir()
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
pipeline{
|
||||
agent none
|
||||
triggers { cron('H 2 * * *') }
|
||||
stages {
|
||||
stage("Run Nightly Build/Test Matrix") {
|
||||
steps{
|
||||
script {
|
||||
parallel(buildMatrix([
|
||||
[os: 'Debian8', arch: 'x86_64', compiler: 'gcc9.1.0', fairsoft: 'fairmq_dev'],
|
||||
[os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
|
||||
[os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
|
||||
]) { spec, label ->
|
||||
sh './Dart.sh Nightly Dart.cfg'
|
||||
sh './Dart.sh Profile Dart.cfg'
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
41
test/ci/slurm-submit.sh
Executable file
41
test/ci/slurm-submit.sh
Executable file
|
|
@ -0,0 +1,41 @@
|
|||
#! /bin/bash
|
||||
|
||||
label="$1"
|
||||
jobsh="$2"
|
||||
|
||||
if [ -z "$ALFACI_SLURM_CPUS" ]
|
||||
then
|
||||
ALFACI_SLURM_CPUS=32
|
||||
fi
|
||||
if [ -z "$ALFACI_SLURM_EXTRA_OPTS" ]
|
||||
then
|
||||
ALFACI_SLURM_EXTRA_OPTS="--hint=compute_bound"
|
||||
fi
|
||||
if [ -z "$ALFACI_SLURM_TIMEOUT" ]
|
||||
then
|
||||
ALFACI_SLURM_TIMEOUT=30
|
||||
fi
|
||||
if [ -z "$ALFACI_SLURM_QUEUE" ]
|
||||
then
|
||||
ALFACI_SLURM_QUEUE=main
|
||||
fi
|
||||
|
||||
echo "*** Slurm request options :"
|
||||
echo "*** Working directory ..: $PWD"
|
||||
echo "*** Queue ..............: $ALFACI_SLURM_QUEUE"
|
||||
echo "*** CPUs ...............: $ALFACI_SLURM_CPUS"
|
||||
echo "*** Wall Time ..........: $ALFACI_SLURM_TIMEOUT min"
|
||||
echo "*** Job Name ...........: ${label}"
|
||||
echo "*** Extra Options ......: ${ALFACI_SLURM_EXTRA_OPTS}"
|
||||
echo "*** Submitting job at ....: $(date -R)"
|
||||
(
|
||||
set -x
|
||||
srun -p $ALFACI_SLURM_QUEUE -c $ALFACI_SLURM_CPUS -n 1 \
|
||||
-t $ALFACI_SLURM_TIMEOUT \
|
||||
--job-name="${label}" \
|
||||
${ALFACI_SLURM_EXTRA_OPTS} \
|
||||
bash "${jobsh}"
|
||||
)
|
||||
retval=$?
|
||||
echo "*** Exit Code ............: $retval"
|
||||
exit "$retval"
|
||||
Loading…
Reference in New Issue
Block a user