Update docs

Resolves #244

.clang-tidy

@@ -1,3 +1,3 @@
 ---
-Checks:          '*,-google-*,-fuchsia-*,-cert-*,-llvm-header-guard,-readability-named-parameter,-misc-non-private-member-variables-in-classes,-*-magic-numbers,-llvm-include-order,-hicpp-no-array-decay,-performance-unnecessary-value-param,-cppcoreguidelines-pro-bounds-array-to-pointer-decay'
+Checks:          '*,-google-*,-fuchsia-*,-cert-*,-llvm-header-guard,-readability-named-parameter,-misc-non-private-member-variables-in-classes,-*-magic-numbers,-llvm-include-order,-hicpp-no-array-decay,-performance-unnecessary-value-param,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-modernize-use-trailing-return-type,-readability-redundant-member-init'
 HeaderFilterRegex: '/(fairmq/)'

.codecov.yml

@@ -1,2 +1,3 @@
 comment:
   layout: "diff, files"
+  behavior: once

CMakeLists.txt

@@ -6,8 +6,8 @@
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
-cmake_minimum_required(VERSION 3.11 FATAL_ERROR)
-cmake_policy(VERSION 3.11...3.15)
+cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
+cmake_policy(VERSION 3.12...3.15)
 
 # Project ######################################################################
 set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
@@ -18,7 +18,7 @@ get_git_version()
 project(FairMQ VERSION ${PROJECT_VERSION} LANGUAGES CXX)
 message(STATUS "${BWhite}${PROJECT_NAME}${CR} ${PROJECT_GIT_VERSION} from ${PROJECT_DATE}")
 
-if(BUILD_OFI_TRANSPORT OR BUILD_SDK)
+if(BUILD_OFI_TRANSPORT OR BUILD_SDK OR BUILD_PMIX_PLUGIN)
   set(PROJECT_MIN_CXX_STANDARD 14)
 else()
   set(PROJECT_MIN_CXX_STANDARD 11)
@@ -39,14 +39,16 @@ fairmq_build_option(BUILD_NANOMSG_TRANSPORT "Build nanomsg transport."
                                              DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
 fairmq_build_option(BUILD_OFI_TRANSPORT     "Build experimental OFI transport."
                                              DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
+fairmq_build_option(BUILD_SDK_COMMANDS      "Build the FairMQ SDK commands."
+                                             DEFAULT OFF)
 fairmq_build_option(BUILD_DDS_PLUGIN        "Build DDS plugin."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
+                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
 fairmq_build_option(BUILD_PMIX_PLUGIN       "Build PMIx plugin."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
+                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
 fairmq_build_option(BUILD_EXAMPLES          "Build FairMQ examples."
                                              DEFAULT ON  REQUIRES "BUILD_FAIRMQ")
 fairmq_build_option(BUILD_SDK               "Build the FairMQ controller SDK."
-                                             DEFAULT OFF REQUIRES "BUILD_DDS_PLUGIN")
+                                             DEFAULT OFF REQUIRES "BUILD_DDS_PLUGIN;BUILD_SDK_COMMANDS")
 fairmq_build_option(BUILD_DOCS              "Build FairMQ documentation."
                                              DEFAULT OFF)
 fairmq_build_option(FAST_BUILD              "Fast production build. Not recommended for development."
@@ -83,17 +85,15 @@ if(BUILD_NANOMSG_TRANSPORT)
   )
 endif()
 
-if(BUILD_SDK)
-  set(required_dds_version 2.5.46)
-else()
-  set(required_dds_version 2.4)
+if(BUILD_SDK_COMMANDS)
+  find_package2(PRIVATE Flatbuffers REQUIRED)
 endif()
 
 if(BUILD_DDS_PLUGIN OR BUILD_SDK)
   find_package2(PRIVATE DDS REQUIRED
-    VERSION ${required_dds_version}
+    VERSION 3.0
   )
-  set(DDS_Boost_COMPONENTS system log log_setup)
+  set(DDS_Boost_COMPONENTS system log log_setup regex filesystem thread)
   set(DDS_Boost_VERSION 1.67)
 endif()
 
@@ -155,7 +155,7 @@ endif()
 
 if(BUILD_FAIRMQ)
   find_package2(PRIVATE ZeroMQ REQUIRED
-    VERSION 4.1.5
+    VERSION 4.1.4
   )
 endif()
 
@@ -230,6 +230,9 @@ endif()
 if(BUILD_SDK)
   list(APPEND PROJECT_PACKAGE_COMPONENTS sdk)
 endif()
+if(BUILD_SDK_COMMANDS)
+  list(APPEND PROJECT_PACKAGE_COMPONENTS sdk_commands)
+endif()
 ################################################################################
 
 
@@ -295,7 +298,7 @@ if(PROJECT_PACKAGE_DEPENDENCIES)
   message(STATUS "  ")
   message(STATUS "  ${Cyan}DEPENDENCY FOUND     VERSION                   PREFIX${CR}")
   foreach(dep IN LISTS PROJECT_PACKAGE_DEPENDENCIES)
-    if(${dep}_VERSION)
+    if(${dep}_VERSION AND NOT ${dep}_VERSION STREQUAL "..")
       set(version_str "${BGreen}${${dep}_VERSION}${CR}")
     else()
       set(version_str "${BYellow}unknown${CR}")
@@ -343,6 +346,13 @@ if(PROJECT_PACKAGE_DEPENDENCIES)
     elseif(${dep} STREQUAL fmt)
       get_target_property(fmt_include fmt::fmt INTERFACE_INCLUDE_DIRECTORIES)
       get_filename_component(prefix ${fmt_include}/.. ABSOLUTE)
+    elseif(${dep} STREQUAL Flatbuffers)
+      if(TARGET flatbuffers::flatbuffers)
+        get_target_property(flatbuffers_include flatbuffers::flatbuffers INTERFACE_INCLUDE_DIRECTORIES)
+      else()
+        get_target_property(flatbuffers_include flatbuffers::flatbuffers_shared INTERFACE_INCLUDE_DIRECTORIES)
+      endif()
+      get_filename_component(prefix ${flatbuffers_include}/.. ABSOLUTE)
     else()
       get_filename_component(prefix ${${dep}_INCLUDE_DIR}/.. ABSOLUTE)
     endif()
@@ -387,9 +397,9 @@ else()
 endif()
 message(STATUS "  ${BWhite}dds_plugin${CR}         ${dds_summary}")
 if(BUILD_PMIX_PLUGIN)
-  set(pmix_summary "${BGreen}YES${CR}    (disable with ${BMagenta}-DBUILD_PMIX_PLUGIN=OFF${CR})")
+  set(pmix_summary "${BGreen}YES${CR}    EXPERIMENTAL (requires C++14) (disable with ${BMagenta}-DBUILD_PMIX_PLUGIN=OFF${CR})")
 else()
-  set(pmix_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_PMIX_PLUGIN=ON${CR})")
+  set(pmix_summary "${BRed} NO${CR}    EXPERIMENTAL (requires C++14) (default, enable with ${BMagenta}-DBUILD_PMIX_PLUGIN=ON${CR})")
 endif()
 message(STATUS "  ${BWhite}pmix_plugin${CR}        ${pmix_summary}")
 if(BUILD_EXAMPLES)
@@ -410,6 +420,12 @@ else()
   set(sdk_summary "${BRed} NO${CR}    EXPERIMENTAL (required C++14) (default, enable with ${BMagenta}-DBUILD_SDK=ON${CR})")
 endif()
 message(STATUS "  ${BWhite}sdk${CR}                ${sdk_summary}")
+if(BUILD_SDK_COMMANDS)
+  set(sdk_commands_summary "${BGreen}YES${CR}    (disable with ${BMagenta}-DBUILD_SDK_COMMANDS=OFF${CR})")
+else()
+  set(sdk_commands_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_SDK_COMMANDS=ON${CR})")
+endif()
+message(STATUS "  ${BWhite}sdk_commands${CR}       ${sdk_commands_summary}")
 message(STATUS "  ")
 if(RUN_STATIC_ANALYSIS)
   list(LENGTH PROJECT_STATIC_ANALYSERS size)

CONTRIBUTORS

@@ -4,6 +4,7 @@ Eulisse, Giulio
 Karabowicz, Radoslaw
 Kretz, Matthias <kretz@kde.org>
 Krzewicki, Mikolaj 
+Lebedev, Andrey
 Mrnjavac, Teo <teo.m@cern.ch>
 Neskovic, Gvozden
 Richter, Matthias

Dart.sh

@@ -65,7 +65,7 @@ if [ "$1" == "alfa_ci" ]; then
   export ctest_model=Experimental
 elif [ "$1" == "codecov" ]; then
   export ctest_model=Profile
-  export do_codecov_upload=1
+  export do_codecov_upload=0
 else
   export ctest_model=$1
 fi

FairMQTest.cmake

@@ -29,13 +29,14 @@ Set(configure_options "${configure_options};-DCMAKE_PREFIX_PATH=$ENV{SIMPATH}")
 Set(configure_options "${configure_options};-DBUILD_NANOMSG_TRANSPORT=ON")
 # Set(configure_options "${configure_options};-DBUILD_OFI_TRANSPORT=ON")
 Set(configure_options "${configure_options};-DBUILD_DDS_PLUGIN=ON")
-Set(configure_options "${configure_options};-DBUILD_SDK=OFF")
+Set(configure_options "${configure_options};-DBUILD_SDK=ON")
+Set(configure_options "${configure_options};-DBUILD_SDK_COMMANDS=ON")
 Set(configure_options "${configure_options};-DFAST_BUILD=ON")
 Set(configure_options "${configure_options};-DCOTIRE_MAXIMUM_NUMBER_OF_UNITY_INCLUDES=-j$ENV{number_of_processors}")
 
 Set(EXTRA_FLAGS $ENV{EXTRA_FLAGS})
 If(EXTRA_FLAGS)
-  Set(configure_options "${configure_options};${EXTRA_FLAGS}") 
+  Set(configure_options "${configure_options};${EXTRA_FLAGS}")
 EndIf()
 
 If($ENV{ctest_model} MATCHES Profile)
@@ -43,6 +44,7 @@ If($ENV{ctest_model} MATCHES Profile)
   If(GCOV_COMMAND)
     Message("Found GCOV: ${GCOV_COMMAND}")
     Set(CTEST_COVERAGE_COMMAND ${GCOV_COMMAND})
+    set(CTEST_COVERAGE_EXTRA_FLAGS "-p")
   EndIf(GCOV_COMMAND)
 EndIf()
 
@@ -58,7 +60,7 @@ Ctest_Configure(BUILD "${CTEST_BINARY_DIRECTORY}"
 
 Ctest_Build(BUILD "${CTEST_BINARY_DIRECTORY}")
 
-Ctest_Test(BUILD "${CTEST_BINARY_DIRECTORY}" 
+Ctest_Test(BUILD "${CTEST_BINARY_DIRECTORY}"
            # PARALLEL_LEVEL $ENV{number_of_processors}
            PARALLEL_LEVEL $ENV{number_of_processors}
            RETURN_VALUE _ctest_test_ret_val
@@ -78,7 +80,7 @@ If("$ENV{do_codecov_upload}")
 EndIf()
 
 Ctest_Submit()
- 
+
 if (_ctest_test_ret_val)
   Message(FATAL_ERROR "Some tests failed.")
 endif()

Jenkinsfile

@@ -22,11 +22,11 @@ def jobMatrix(String prefix, List specs, Closure callback) {
             echo "export SIMPATH=\${SIMPATH_PREFIX}${fairsoft}" >> Dart.cfg
             echo "export FAIRSOFT_VERSION=${fairsoft}" >> Dart.cfg
           """
-          if (os =~ /Debian/ && compiler =~ /gcc8/) {
+          if (os =~ /Debian/ && compiler =~ /gcc9/) {
             sh '''\
               echo "source /etc/profile.d/modules.sh" >> Dart.cfg
               echo "module use /cvmfs/it.gsi.de/modulefiles" >> Dart.cfg
-              echo "module load compiler/gcc/8" >> Dart.cfg
+              echo "module load compiler/gcc/9.1.0" >> Dart.cfg
             '''
           }
           if (os =~ /MacOS/) {
@@ -49,6 +49,10 @@ def jobMatrix(String prefix, List specs, Closure callback) {
           deleteDir()
           githubNotify(context: "${prefix}/${label}", description: 'Success', status: 'SUCCESS')
         } catch (e) {
+          def tarball = "${prefix}_${label}_dds_logs.tar.gz"
+          sh "tar czvf ${tarball} -C \${WORKSPACE}/build/test/ .DDS/"
+          archiveArtifacts tarball
+
           deleteDir()
           githubNotify(context: "${prefix}/${label}", description: 'Error', status: 'ERROR')
           throw e
@@ -65,16 +69,16 @@ pipeline{
     stage("Run CI Matrix") {
       steps{
         script {
-          def build_jobs = jobMatrix('alfa-ci/build', [
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
+          def build_jobs = jobMatrix('build', [
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc9.1.0',        fairsoft: 'fairmq_dev'],
             [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
             [os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             sh './Dart.sh alfa_ci Dart.cfg'
           }
 
-          def profile_jobs = jobMatrix('alfa-ci/codecov', [
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
+          def profile_jobs = jobMatrix('codecov', [
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc9.1.0',        fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             withCredentials([string(credentialsId: 'fairmq_codecov_token', variable: 'CODECOV_TOKEN')]) {
               sh './Dart.sh codecov Dart.cfg'

Jenkinsfile.nightly

@@ -21,11 +21,11 @@ def buildMatrix(List specs, Closure callback) {
             echo "export SIMPATH=\${SIMPATH_PREFIX}${fairsoft}" >> Dart.cfg
             echo "export FAIRSOFT_VERSION=${fairsoft}" >> Dart.cfg
           """
-          if (os =~ /Debian/ && compiler =~ /gcc8/) {
+          if (os =~ /Debian/ && compiler =~ /gcc9/) {
             sh '''\
               echo "source /etc/profile.d/modules.sh" >> Dart.cfg
               echo "module use /cvmfs/it.gsi.de/modulefiles" >> Dart.cfg
-              echo "module load compiler/gcc/8" >> Dart.cfg
+              echo "module load compiler/gcc/9.1.0" >> Dart.cfg
             '''
           }
           if (os =~ /MacOS/) {
@@ -46,6 +46,10 @@ def buildMatrix(List specs, Closure callback) {
 
           deleteDir()
         } catch (e) {
+          def tarball = "${label}_dds_logs.tar.gz"
+          sh "tar czvf ${tarball} -C \${WORKSPACE}/build/test/ .DDS/"
+          archiveArtifacts tarball
+
           deleteDir()
           throw e
         }
@@ -63,7 +67,7 @@ pipeline{
       steps{
         script {
           parallel(buildMatrix([
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc9.1.0',        fairsoft: 'fairmq_dev'],
             [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
             [os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
           ]) { spec, label ->

README.md

@@ -181,4 +181,5 @@ After the `find_package(FairMQ)` call the following CMake variables are defined:
    3. [Provided Plugins](docs/Plugins.md#73-provided-plugins)
        1. [DDS](docs/Plugins.md#731-dds)
        2. [PMIx](docs/Plugins.md#732-pmix)
+8. [Controller SDK](docs/SDK.md)
 

cmake/FairMQLib.cmake

@@ -143,15 +143,17 @@ macro(set_fairmq_defaults)
   set(PROJECT_EXPORT_SET ${PROJECT_NAME}Targets)
 
   # Configure build types
-  set(CMAKE_CONFIGURATION_TYPES "Debug" "Release" "RelWithDebInfo" "Nightly" "Profile" "Experimental" "AdressSan" "ThreadSan")
-  set(CMAKE_CXX_FLAGS_DEBUG          "-g -Wshadow -Wall -Wextra")
+  set(CMAKE_CONFIGURATION_TYPES "Debug" "Release" "RelWithDebInfo" "Nightly" "Profile" "Experimental" "AddressSan" "ThreadSan")
+  set(_warnings "-Wshadow -Wall -Wextra -Wpedantic")
+  set(CMAKE_CXX_FLAGS_DEBUG          "-g ${_warnings}")
   set(CMAKE_CXX_FLAGS_RELEASE        "-O2 -DNDEBUG")
-  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -Wshadow -Wall -Wextra -DNDEBUG")
-  set(CMAKE_CXX_FLAGS_NIGHTLY        "-O2 -g -Wshadow -Wall -Wextra")
-  set(CMAKE_CXX_FLAGS_PROFILE        "-g3 -Wshadow -Wall -Wextra -fno-inline -ftest-coverage -fprofile-arcs")
-  set(CMAKE_CXX_FLAGS_EXPERIMENTAL   "-O2 -g -Wshadow -Wall -Wextra -DNDEBUG")
-  set(CMAKE_CXX_FLAGS_ADRESSSAN      "-O2 -g -Wshadow -Wall -Wextra -fsanitize=address -fno-omit-frame-pointer")
-  set(CMAKE_CXX_FLAGS_THREADSAN      "-O2 -g -Wshadow -Wall -Wextra -fsanitize=thread")
+  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g ${_warnings} -DNDEBUG")
+  set(CMAKE_CXX_FLAGS_NIGHTLY        "-O2 -g ${_warnings}")
+  set(CMAKE_CXX_FLAGS_PROFILE        "-g3 ${_warnings} -fno-inline -ftest-coverage -fprofile-arcs")
+  set(CMAKE_CXX_FLAGS_EXPERIMENTAL   "-O2 -g ${_warnings} -DNDEBUG")
+  set(CMAKE_CXX_FLAGS_ADDRESSSAN     "-O2 -g ${_warnings} -fsanitize=address -fno-omit-frame-pointer")
+  set(CMAKE_CXX_FLAGS_THREADSAN      "-O2 -g ${_warnings} -fsanitize=thread")
+  unset(_warnings)
 
   if(CMAKE_GENERATOR STREQUAL "Ninja" AND
      ((CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9) OR
@@ -194,6 +196,13 @@ macro(set_fairmq_defaults)
     endif()
     list(APPEND PROJECT_STATIC_ANALYSERS "${analyser}")
   endif()
+
+  if(CMAKE_GENERATOR STREQUAL Ninja AND ENABLE_CCACHE)
+    find_program(CCACHE ccache)
+    if(CCACHE)
+      set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE})
+    endif()
+  endif()
 endmacro()
 
 function(join VALUES GLUE OUTPUT)
@@ -445,6 +454,23 @@ macro(exec cmd)
   endif()
 endmacro()
 
+macro(exec_source cmd)
+  join("${ARGN}" " " args)
+  file(APPEND ${${package}_BUILD_LOGFILE} ">>> ${cmd} ${args}\n")
+
+  execute_process(COMMAND ${cmd} ${ARGN}
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE log
+    ERROR_VARIABLE log
+    RESULT_VARIABLE res
+  )
+  file(APPEND ${${package}_BUILD_LOGFILE} ${log})
+
+  if(res)
+    message(FATAL_ERROR "${res} \nSee also \"${${package}_BUILD_LOGFILE}\"")
+  endif()
+endmacro()
+
 function(build_bundled package bundle)
   message(STATUS "Building bundled ${package}")
 
@@ -455,7 +481,7 @@ function(build_bundled package bundle)
   file(REMOVE ${${package}_BUILD_LOGFILE})
 
   if(Git_FOUND)
-    exec(${GIT_EXECUTABLE} submodule update --init --recursive -- ${${package}_SOURCE_DIR})
+    exec_source(${GIT_EXECUTABLE} submodule update --init -- ${${package}_SOURCE_DIR})
   endif()
 
   if(${package} STREQUAL GTest)
@@ -482,7 +508,7 @@ function(build_bundled package bundle)
 endfunction()
 
 macro(fairmq_build_option option description)
-  cmake_parse_arguments(ARGS "" "DEFAULT;REQUIRES" "" ${ARGN})
+  cmake_parse_arguments(ARGS "" "DEFAULT" "REQUIRES" ${ARGN})
 
   if(ARGS_DEFAULT)
     set(__default__ ON)
@@ -493,19 +519,21 @@ macro(fairmq_build_option option description)
   if(ARGS_REQUIRES)
     include(CMakeDependentOption)
     set(__requires__ ${ARGS_REQUIRES})
-    string(REGEX REPLACE " +" ";" __requires_condition__ "${__requires__}")
-    if(${__requires_condition__})
-    else()
-      if(${option})
-        message(WARNING "Cannot enable build option ${option}, depending options are not set: ${__requires_condition__}.")
+    foreach(d ${__requires__})
+      string(REGEX REPLACE " +" ";" __requires_condition__ "${d}")
+      if(${__requires_condition__})
+      else()
+        if(${option})
+          message(FATAL_ERROR "Cannot enable build option ${option}, depending option is not set: ${__requires_condition__}")
+        endif()
       endif()
-    endif()
+    endforeach()
   else()
     set(__requires__)
   endif()
 
   if(__requires__)
-    cmake_dependent_option(${option} ${description} ${__default__} ${__requires__} OFF)
+    cmake_dependent_option(${option} ${description} ${__default__} "${__requires__}" OFF)
   else()
     option(${option} ${description} ${__default__})
   endif()

docs/SDK.md

@@ -0,0 +1,34 @@
+← [Back](../README.md)
+
+# 8. Controller SDK
+
+The FairMQ Controller Software Development Kit (`-DBUILD_SDK=ON`) contains a (as of today still experimental) set of C++ APIs that provide essential functionality to the implementer of a global controller.
+
+The FairMQ core library only provides two local controllers - `static` (a fixed sequence of state transitions) and `interactive` (a read-eval-print-loop which reads keyboard commands from standard input). A local controller only knows how steer a single [FairMQ device](Device.md) - in fact, it runs in a thread within the device process.
+
+A global controller has knowledge about the full topology of connected FairMQ devices. Its responsibility is to facilitate the lifecycle of a distributed FairMQ-based application (*executing a topology*), such as
+
+* allocating/releasing compute resources from a resource management system,
+* launching/setting up the run-time environment and the FairMQ devices,
+* driving the device state machines in lock-step across the full topology,
+* pushing the device configuration,
+* monitoring (some aspects of the application's) operation,
+* and handling/reporting (some) error cases.
+
+The low-level hook to integrate FairMQ devices with such a global contoller is the [plugin mechanism](Plugins.md) in the FairMQ core library. The FairMQ Controller SDK provides C++ APIs that communicate to the endpoints exposed by such a FairMQ plugin.
+
+At the moment, the Controller SDK only supports [DDS](https://dds.gsi.de) as resource manager and run-time environment. A second implementation based on [PMIx](https://pmix.org/) (targeting its implementation in [Slurm](https://slurm.schedmd.com/documentation.html) and [OpenRTE](https://www-lb.open-mpi.org/papers/euro-pvmmpi-2005-orte/)) is in development.
+
+The following section give a short overview on the APIs provided.
+
+## RMS and run-time environment
+
+The classes [`fair::mq::sdk::DDSEnvironment`](../fairmq/sdk/DDSEnvironment.h), [`fair::mq::sdk::DDSSession`](../fairmq/sdk/DDSSession.h), and [`fair::mq::sdk::DDSTopology`](../fairmq/sdk/DDSTopology.h) are thin wrappers of most of the synchronous APIs exposed by DDS ([`dds::tools_api`](http://dds.gsi.de/doc/api-docs/DDS/html/namespacedds_1_1tools__api.html) and [`dds::topology_api`](http://dds.gsi.de/doc/api-docs/DDS/html/namespacedds_1_1topology__api.html)). E.g. they allow to [start a DDS session](https://github.com/FairRootGroup/FairMQ/blob/077eb0ef691940d764cfd1852bf3981dc812ddbd/main.cpp#L26-L28), [allocate resources](https://github.com/FairRootGroup/FairMQ/blob/077eb0ef691940d764cfd1852bf3981dc812ddbd/main.cpp#L34) and [launch a topology](https://github.com/FairRootGroup/FairMQ/blob/077eb0ef691940d764cfd1852bf3981dc812ddbd/main.cpp#L39) from a C++ program.
+
+## Driving the global state machine
+
+The class [`fair::mq::sdk::Topology`](../fairmq/sdk/Topology.h) adds a FairMQ-specific view on an existing DDS session that is executing a topology of FairMQ devices. One can e.g. [initiate a state transition on all devices in the topology simultaneously](https://github.com/FairRootGroup/FairMQ/blob/077eb0ef691940d764cfd1852bf3981dc812ddbd/main.cpp#L48-L49). This topology transition completes once a topology-wide barrier is passed (all devices completed the transition). This effectively exposes the device state machine as a topology state machine. The implementation is based on remote procedure calls over the [DDS intercom service](http://dds.gsi.de/doc/api-docs/DDS/html/namespacedds_1_1intercom__api.html) between the controller and the DDS plugin shipped with FairMQ (`-DBUILD_DDS_PLUGIN=ON`).
+
+For future versions of the SDK new APIs are planned to inspect and modify the device configurations and also operate only on subsets of a given topology.
+
+← [Back](../README.md)

examples/1-1/Sampler.cxx

@@ -6,9 +6,6 @@
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#include <thread> // this_thread::sleep_for
-#include <chrono>
-
 #include "Sampler.h"
 
 using namespace std;

examples/CMakeLists.txt

@@ -16,6 +16,8 @@ add_subdirectory(multiple-channels)
 if(BUILD_NANOMSG_TRANSPORT)
   add_subdirectory(multiple-transports)
 endif()
+add_subdirectory(n-m)
+add_subdirectory(qc)
 add_subdirectory(readout)
 add_subdirectory(region)
 add_subdirectory(req-rep)

examples/README.md

@@ -6,41 +6,50 @@ Set of FairMQ examples. More examples that combine FairMQ with FairRoot can be f
 
 A simple topology of two devices - **Sampler** and **Sink**. **Sampler** sends data to **Sink** with the **PUSH-PULL** pattern.
 
-
 ## 1-n-1
 
 A simple topology of three device types - **Sampler**, **Processor** and **Sink**. **Sampler** sends data to one or more **Processor**s, who modify the data and send it to one **Sink**. Transport with the **PUSH-PULL** pattern. The example also shows the configuration via JSON files, as oposed to `--channel-config` that is used by other examples.
 
+## Built-in devices
 
-## DDS
-
-This example demonstrates usage of the Dynamic Deployment System ([DDS](http://dds.gsi.de/)) to dynamically deploy and configure a topology of devices. The topology is similar to those of Example 2, but now it can be easily distributed on different computing nodes without the need for manual reconfiguration of the devices.
-
+Usage of generic devies provided with FairMQ.
 
 ## Copy & Push
 
 A topology consisting of one **Sampler** and two **Sink**s. The **Sampler** uses the `Copy` method to send the same data to both sinks with the **PUSH-PULL** pattern. In countrary to the **PUB-SUB** pattern, this ensures that all receivers are connected and no data is lost, but requires additional channels to be configured.
 
+## DDS
 
-## Request & Reply
+This example demonstrates usage of the Dynamic Deployment System ([DDS](http://dds.gsi.de/)) to dynamically deploy and configure a topology of devices. The topology is similar to those of Example 2, but now it can be easily distributed on different computing nodes without the need for manual reconfiguration of the devices.
 
-This topology contains two devices that communicate with each other via the **REQ-REP** pettern. Bidirectional communication via a single socket.
+## Multipart
 
+This example shows how to send a multipart message from one device to the other. (two parts message parts - header and body).
 
 ## Multiple Channels
 
 This example demonstrates how to work with multiple channels and multiplex between them.
 
-
-## Sending Multipart messages
-
-This example shows how to send a multipart message from one device to the other. (two parts message parts - header and body).
-
-
-## Multiple Transports example
+## Multiple Transports
 
 This examples shows how to combine different channel transports (zeromq/nanomsg/shmem) inside of one device and/or topology.
 
-## Region example
+## n-m
+
+A topology consisting of three layers of devices: synchronizer -> n * senders -> m * receivers.
+
+## QC
+
+A topology consisting of 4 devices - Sampler, QCDispatcher, QCTask and Sink. The data flows from Sampler through QCDispatcher to Sink. On demand - by setting the corresponding configuration property - the QCDispatcher device will duplicate the data to the QCTask device. The property is set by the topology controller, in this example this is the `fairmq-dds-command-ui` utility.
+
+## Readout
+
+Two example topologies of setups to be distributed to two kinds of nodes - detector readout node and processing node. Detector readout node contains readout process, data builder and data sender (and optionally an additional processor), while processing node contains data receiver devices. communication within readout nodes is done via unmanaged region through shared memory transport.
+
+## Region
 
 This example demonstrates the use of a more advanced feature - UnmanagedRegion, that can be used to create a buffer through one of FairMQ transports. The contents of this buffer are managed by the user, who can also create messages out of sub-buffers of the created buffer. Such feature can be interesting in environments that have special requirements by the hardware that writes the data, to keep the transfer efficient (e.g. shared memory).
+
+## Request & Reply
+
+This topology contains two devices that communicate with each other via the **REQ-REP** pettern. Bidirectional communication via a single socket.

examples/dds/CMakeLists.txt

@@ -7,12 +7,12 @@
 ################################################################################
 
 add_library(ExampleDDSLib STATIC
-    "Sampler.cxx"
-    "Sampler.h"
-    "Processor.cxx"
-    "Processor.h"
-    "Sink.cxx"
-    "Sink.h"
+  "Sampler.cxx"
+  "Sampler.h"
+  "Processor.cxx"
+  "Processor.h"
+  "Sink.cxx"
+  "Sink.h"
 )
 
 target_link_libraries(ExampleDDSLib PUBLIC FairMQ)
@@ -28,12 +28,7 @@ target_link_libraries(fairmq-ex-dds-sink PRIVATE ExampleDDSLib)
 
 add_custom_target(ExampleDDS DEPENDS fairmq-ex-dds-sampler fairmq-ex-dds-processor fairmq-ex-dds-sink)
 
-if(DDS_VERSION VERSION_LESS 2.5.25)
-    set(WAIT_COMMAND "dds-info --wait-for-idle-agents")
-else()
-    set(WAIT_COMMAND "dds-info --idle-count --wait")
-endif()
-
+list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
 set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
 set(DATA_DIR ${CMAKE_CURRENT_BINARY_DIR})
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-dds-topology.xml ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology.xml @ONLY)
@@ -44,24 +39,24 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-dds.sh.in ${CMAKE_CUR
 
 # test
 if(DDS_FOUND)
-    add_test(NAME Example.DDS.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh localhost)
-    set_tests_properties(Example.DDS.localhost PROPERTIES
+  add_test(NAME Example.DDS.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh localhost)
+  set_tests_properties(Example.DDS.localhost PROPERTIES
     TIMEOUT 15
     RUN_SERIAL true
     PASS_REGULAR_EXPRESSION "Example successful"
-    )
+  )
 endif()
 
 # install
 
 install(
-    TARGETS
-    fairmq-ex-dds-sampler
-    fairmq-ex-dds-processor
-    fairmq-ex-dds-sink
+  TARGETS
+  fairmq-ex-dds-sampler
+  fairmq-ex-dds-processor
+  fairmq-ex-dds-sink
 
-    LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-    RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
+  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
 )
 
 # configure run script with different executable paths for build and for install directories
@@ -73,30 +68,30 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-dds-env.sh ${CMAKE_CURRENT_
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-dds.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh_install @ONLY)
 
 install(
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology.xml_install
-    DESTINATION ${PROJECT_INSTALL_DATADIR}
-    RENAME ex-dds-topology.xml
+  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology.xml_install
+  DESTINATION ${PROJECT_INSTALL_DATADIR}
+  RENAME ex-dds-topology.xml
 )
 
 install(
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology-infinite.xml_install
-    DESTINATION ${PROJECT_INSTALL_DATADIR}
-    RENAME ex-dds-topology-infinite.xml
+  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology-infinite.xml_install
+  DESTINATION ${PROJECT_INSTALL_DATADIR}
+  RENAME ex-dds-topology-infinite.xml
 )
 
 install(
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-hosts.cfg
-    DESTINATION ${PROJECT_INSTALL_DATADIR}
+  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-hosts.cfg
+  DESTINATION ${PROJECT_INSTALL_DATADIR}
 )
 
 install(
-    PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-dds-env.sh_install
-    DESTINATION ${PROJECT_INSTALL_BINDIR}
-    RENAME fairmq-ex-dds-env.sh
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-dds-env.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-ex-dds-env.sh
 )
 
 install(
-    PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh_install
-    DESTINATION ${PROJECT_INSTALL_BINDIR}
-    RENAME fairmq-start-ex-dds.sh
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-start-ex-dds.sh
 )

examples/dds/ex-dds-topology-infinite.xml

@@ -1,7 +1,7 @@
 <topology name="ExampleDDS">
 
-    <property name="data1" />
-    <property name="data2" />
+    <property name="fmqchan_data1" />
+    <property name="fmqchan_data2" />
 
     <declrequirement name="SamplerWorker" type="wnname" value="sampler"/>
     <declrequirement name="ProcessorWorker" type="wnname" value="processor"/>
@@ -14,7 +14,7 @@
             <name>SamplerWorker</name>
         </requirements>
         <properties>
-            <name access="write">data1</name>
+            <name access="write">fmqchan_data1</name>
         </properties>
     </decltask>
 
@@ -25,8 +25,8 @@
             <name>ProcessorWorker</name>
         </requirements>
         <properties>
-            <name access="read">data1</name>
-            <name access="read">data2</name>
+            <name access="read">fmqchan_data1</name>
+            <name access="read">fmqchan_data2</name>
         </properties>
     </decltask>
 
@@ -37,7 +37,7 @@
             <name>SinkWorker</name>
         </requirements>
         <properties>
-            <name access="write">data2</name>
+            <name access="write">fmqchan_data2</name>
         </properties>
     </decltask>
 

examples/dds/ex-dds-topology.xml

@@ -1,7 +1,7 @@
 <topology name="ExampleDDS">
 
-    <property name="data1" />
-    <property name="data2" />
+    <property name="fmqchan_data1" />
+    <property name="fmqchan_data2" />
 
     <declrequirement name="SamplerWorker" type="wnname" value="sampler"/>
     <declrequirement name="ProcessorWorker" type="wnname" value="processor"/>
@@ -14,7 +14,7 @@
             <name>SamplerWorker</name>
         </requirements>
         <properties>
-            <name access="write">data1</name>
+            <name access="write">fmqchan_data1</name>
         </properties>
     </decltask>
 
@@ -25,8 +25,8 @@
             <name>ProcessorWorker</name>
         </requirements>
         <properties>
-            <name access="read">data1</name>
-            <name access="read">data2</name>
+            <name access="read">fmqchan_data1</name>
+            <name access="read">fmqchan_data2</name>
         </properties>
     </decltask>
 
@@ -37,7 +37,7 @@
             <name>SinkWorker</name>
         </requirements>
         <properties>
-            <name access="write">data2</name>
+            <name access="write">fmqchan_data2</name>
         </properties>
     </decltask>
 

examples/dds/fairmq-ex-dds-env.sh

@@ -9,3 +9,8 @@
 ################################################################################
 
 export PATH=@BIN_DIR@:$PATH
+
+OS=$(uname -s 2>&1)
+if [ "$OS" == "Darwin" ]; then
+   export DYLD_LIBRARY_PATH=@LIB_DIR@:$DYLD_LIBRARY_PATH
+fi

examples/dds/fairmq-start-ex-dds.sh.in

@@ -30,52 +30,49 @@ echo "SESSION ID: ${DDS_SESSION_ID}"
 
 trap "cleanup ${DDS_SESSION_ID}" EXIT
 
-requiredNofAgents=12
+requiredNofSlots=12
 if [[ "$plugin" == "ssh" ]]; then
   dds-submit -r ${plugin} -c @DATA_DIR@/ex-dds-hosts.cfg
 else
-  dds-submit -r ${plugin} -n ${requiredNofAgents}
+  dds-submit -r ${plugin} --slots ${requiredNofSlots}
 fi
-echo "...waiting for ${requiredNofAgents} idle agents..."
-@WAIT_COMMAND@ ${requiredNofAgents}
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
 
-topologyFile=@DATA_DIR@/ex-dds-topology.xml
-echo "TOPOLOGY FILE: ${topologyFile}"
-# TODO Uncomment once DDS 2.6 is released
-# echo "TOPOLOGY NAME: $(dds-topology --disable-validation --topology-name ${topologyFile})"
+export FAIRMQ_DDS_TOPO_FILE=@DATA_DIR@/ex-dds-topology.xml
+echo "TOPOLOGY FILE: ${FAIRMQ_DDS_TOPO_FILE}"
+echo "TOPOLOGY NAME: $(dds-topology --disable-validation --topology-name ${FAIRMQ_DDS_TOPO_FILE})"
 
-# TODO Uncomment once DDS 2.6 is released
-# dds-info --active-topology
-dds-topology --activate ${topologyFile}
-# dds-info --active-topology
-# dds-info --wait-for-executing-agents ${requiredNofAgents}
-sleep 1
+dds-info --active-topology
+dds-topology --activate ${FAIRMQ_DDS_TOPO_FILE}
+dds-info --active-topology
+echo "...waiting for ${requiredNofSlots} executing slots..."
+dds-info --executing-count --wait ${requiredNofSlots}
 
 echo "------------------------"
 echo "...waiting for Topology to finish..."
 # TODO Retrieve number of devices from DDS topology API instead of having the user pass it explicitely
-fairmq-dds-command-ui -w "IDLE" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c i -w "INITIALIZING DEVICE" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c k -w "INITIALIZED" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c b -w "BOUND" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c x -w "DEVICE READY" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c j -w "READY" -n ${requiredNofAgents}
+fairmq-dds-command-ui -w "IDLE"
+fairmq-dds-command-ui -c i
+fairmq-dds-command-ui -c k
+fairmq-dds-command-ui -c b
+fairmq-dds-command-ui -c x
+fairmq-dds-command-ui -c j
 fairmq-dds-command-ui -c r
-sampler_and_sink="main/(Sampler|Sink)"
-fairmq-dds-command-ui -p $sampler_and_sink -w "RUNNING->READY" -n 2
+sampler_and_sink="main/(Sampler|Sink).*"
+fairmq-dds-command-ui -w "RUNNING->READY" -p $sampler_and_sink
 echo "...$sampler_and_sink are READY, sending shutdown..."
-fairmq-dds-command-ui -c s -w "RUNNING->READY" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c t -w "DEVICE READY" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c d -w "IDLE" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c q -w "EXITING" -n ${requiredNofAgents}
-echo "...waiting for ${requiredNofAgents} idle agents..."
-@WAIT_COMMAND@ ${requiredNofAgents}
+fairmq-dds-command-ui -c s
+fairmq-dds-command-ui -c t
+fairmq-dds-command-ui -c d
+fairmq-dds-command-ui -c q
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
 echo "------------------------"
 
-# TODO Uncomment once DDS 2.6 is released
-# dds-info --active-topology
+dds-info --active-topology
 dds-topology --stop
-# dds-info --active-topology
+dds-info --active-topology
 
 dds-agent-cmd getlog -a
 logDir="${wrkDir}/logs"

examples/multipart/CMakeLists.txt

@@ -32,19 +32,19 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multipart.sh.in ${CMA
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multipart.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh)
 
 add_test(NAME Example.Multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh zeromq)
-set_tests_properties(Example.Multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 5 parts")
+set_tests_properties(Example.Multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
 
 if(BUILD_NANOMSG_TRANSPORT)
   add_test(NAME Example.Multipart.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh nanomsg)
-  set_tests_properties(Example.Multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 5 parts")
+  set_tests_properties(Example.Multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
 endif()
 
 add_test(NAME Example.Multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh shmem)
-set_tests_properties(Example.Multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 5 parts")
+set_tests_properties(Example.Multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
 
 if(BUILD_OFI_TRANSPORT)
   add_test(NAME Example.Multipart.ofi COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh ofi)
-  set_tests_properties(Example.Multipart.ofi PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 5 parts")
+  set_tests_properties(Example.Multipart.ofi PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
 endif()
 
 # install

examples/multipart/Sampler.cxx

@@ -40,8 +40,7 @@ bool Sampler::ConditionalRun()
     header.stopFlag = 0;
 
     // Set stopFlag to 1 for last message.
-    if (fMaxIterations > 0 && fNumIterations == fMaxIterations - 1)
-    {
+    if (fMaxIterations > 0 && fNumIterations == fMaxIterations - 1) {
         header.stopFlag = 1;
     }
     LOG(info) << "Sending header with stopFlag: " << header.stopFlag;
@@ -60,13 +59,20 @@ bool Sampler::ConditionalRun()
     assert(auxData.Size() == 0);
     assert(parts.Size() == 5);
 
+    parts.AddPart(NewMessage());
+
+    assert(parts.Size() == 6);
+
+    parts.AddPart(NewMessage(100));
+
+    assert(parts.Size() == 7);
+
     LOG(info) << "Sending body of size: " << parts.At(1)->GetSize();
 
     Send(parts, "data");
 
     // Go out of the sending loop if the stopFlag was sent.
-    if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations)
-    {
+    if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
         LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
         return false;
     }
@@ -77,8 +83,4 @@ bool Sampler::ConditionalRun()
     return true;
 }
 
-Sampler::~Sampler()
-{
-}
-
 } // namespace example_multipart

examples/multipart/Sampler.h

@@ -24,7 +24,7 @@ class Sampler : public FairMQDevice
 {
   public:
     Sampler();
-    virtual ~Sampler();
+    virtual ~Sampler() {}
 
   protected:
     virtual void InitTask();

examples/multipart/Sink.cxx

@@ -20,23 +20,28 @@ using namespace std;
 namespace example_multipart
 {
 
-Sink::Sink()
-{
-    OnData("data", &Sink::HandleData);
-}
-
 bool Sink::HandleData(FairMQParts& parts, int /*index*/)
 {
+    LOG(info) << "Received message with " << parts.Size() << " parts";
+
     Header header;
     header.stopFlag = (static_cast<Header*>(parts.At(0)->GetData()))->stopFlag;
 
-    LOG(info) << "Received message with " << parts.Size() << " parts";
+    LOG(info) << "Received part 1 (header) with stopFlag: " << header.stopFlag;
+    LOG(info) << "Received part 2 of size: " << parts.At(1)->GetSize();
+    assert(parts.At(1)->GetSize() == 1000);
+    LOG(info) << "Received part 3 of size: " << parts.At(2)->GetSize();
+    assert(parts.At(2)->GetSize() == 500);
+    LOG(info) << "Received part 4 of size: " << parts.At(3)->GetSize();
+    assert(parts.At(3)->GetSize() == 600);
+    LOG(info) << "Received part 5 of size: " << parts.At(4)->GetSize();
+    assert(parts.At(4)->GetSize() == 700);
+    LOG(info) << "Received part 6 of size: " << parts.At(5)->GetSize();
+    assert(parts.At(5)->GetSize() == 0);
+    LOG(info) << "Received part 7 of size: " << parts.At(6)->GetSize();
+    assert(parts.At(6)->GetSize() == 100);
 
-    LOG(info) << "Received header with stopFlag: " << header.stopFlag;
-    LOG(info) << "Received body of size: " << parts.At(1)->GetSize();
-
-    if (header.stopFlag == 1)
-    {
+    if (header.stopFlag == 1) {
         LOG(info) << "stopFlag is 1, going IDLE";
         return false;
     }
@@ -44,8 +49,4 @@ bool Sink::HandleData(FairMQParts& parts, int /*index*/)
     return true;
 }
 
-Sink::~Sink()
-{
-}
-
 }

examples/multipart/Sink.h

@@ -23,8 +23,8 @@ namespace example_multipart
 class Sink : public FairMQDevice
 {
   public:
-    Sink();
-    virtual ~Sink();
+    Sink() { OnData("data", &Sink::HandleData); }
+    virtual ~Sink() {}
 
   protected:
     bool HandleData(FairMQParts&, int);

examples/multipart/fairmq-start-ex-multipart.sh.in

@@ -2,12 +2,24 @@
 
 export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
 
+transport="zeromq"
+
+if [[ $1 =~ ^[a-z]+$ ]]; then
+    transport=$1
+fi
+
+SESSION="$(@CMAKE_BINARY_DIR@/fairmq/fairmq-uuid-gen -h)"
+
 SAMPLER="fairmq-ex-multipart-sampler"
 SAMPLER+=" --id sampler1"
+SAMPLER+=" --transport $transport"
+SAMPLER+=" --session $SESSION"
 SAMPLER+=" --channel-config name=data,type=push,method=connect,rateLogging=0,address=tcp://127.0.0.1:5555"
 xterm -geometry 80x23+0+0 -hold -e @EX_BIN_DIR@/$SAMPLER &
 
 SINK="fairmq-ex-multipart-sink"
 SINK+=" --id sink1"
+SINK+=" --transport $transport"
+SINK+=" --session $SESSION"
 SINK+=" --channel-config name=data,type=pull,method=bind,rateLogging=0,address=tcp://127.0.0.1:5555"
 xterm -geometry 80x23+500+0 -hold -e @EX_BIN_DIR@/$SINK &

examples/multipart/test-ex-multipart.sh.in

@@ -20,7 +20,7 @@ SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --session $SESSION"
 SAMPLER+=" --max-iterations 1"
 SAMPLER+=" --control static --color false"
-SAMPLER+=" --channel-config name=data,type=pair,method=connect,rateLogging=0,address=tcp://127.0.0.1:5555"
+SAMPLER+=" --channel-config name=data,type=pair,method=connect,rateLogging=0,address=tcp://127.0.0.1:5555,linger=1000"
 @CMAKE_CURRENT_BINARY_DIR@/$SAMPLER &
 SAMPLER_PID=$!
 

examples/n-m/CMakeLists.txt

@@ -0,0 +1,99 @@
+################################################################################
+#    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+add_executable(fairmq-ex-n-m-synchronizer runSynchronizer.cxx)
+target_link_libraries(fairmq-ex-n-m-synchronizer PRIVATE FairMQ)
+
+add_executable(fairmq-ex-n-m-sender runSender.cxx)
+target_link_libraries(fairmq-ex-n-m-sender PRIVATE FairMQ)
+
+add_executable(fairmq-ex-n-m-receiver runReceiver.cxx)
+target_link_libraries(fairmq-ex-n-m-receiver PRIVATE FairMQ)
+
+add_custom_target(ExampleNM DEPENDS fairmq-ex-n-m-synchronizer fairmq-ex-n-m-sender fairmq-ex-n-m-receiver)
+
+list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
+set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
+set(DATA_DIR ${CMAKE_CURRENT_BINARY_DIR})
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-n-m-topology.xml                ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-topology.xml @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-n-m-pair-topology.xml           ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-pair-topology.xml @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-n-m-env.sh               ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-n-m-env.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m.sh.in          ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-pair.sh.in     ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-dds.sh.in      ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-dds.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-pair-dds.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair-dds.sh @ONLY)
+
+# test
+if(DDS_FOUND)
+  add_test(NAME Example.N-M.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-dds.sh localhost)
+  set_tests_properties(Example.N-M.localhost PROPERTIES TIMEOUT 15 RUN_SERIAL true PASS_REGULAR_EXPRESSION "Example successful")
+  add_test(NAME Example.N-M-pair.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair-dds.sh localhost)
+  set_tests_properties(Example.N-M-pair.localhost PROPERTIES TIMEOUT 15 RUN_SERIAL true PASS_REGULAR_EXPRESSION "Example successful")
+endif()
+
+# install
+install(
+  TARGETS
+  fairmq-ex-n-m-synchronizer
+  fairmq-ex-n-m-sender
+  fairmq-ex-n-m-receiver
+
+  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
+)
+
+# configure run script with different executable paths for build and for install directories
+set(BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
+set(DATA_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_DATADIR})
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-n-m-topology.xml                ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-topology.xml_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-n-m-pair-topology.xml           ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-pair-topology.xml_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-n-m-env.sh               ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-n-m-env.sh_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m.sh.in          ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m.sh_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-pair.sh.in     ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair.sh_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-dds.sh.in      ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-dds.sh_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-pair-dds.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair-dds.sh_install @ONLY)
+
+install(
+  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-topology.xml_install
+  DESTINATION ${PROJECT_INSTALL_DATADIR}
+  RENAME ex-n-m-topology.xml
+)
+install(
+  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-pair-topology.xml_install
+  DESTINATION ${PROJECT_INSTALL_DATADIR}
+  RENAME ex-n-m-pair-topology.xml
+)
+
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-n-m-env.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-ex-n-m-env.sh
+)
+
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-start-ex-n-m.sh
+)
+
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-start-ex-n-m-pair.sh
+)
+
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-dds.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-start-ex-n-m-dds.sh
+)
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair-dds.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-start-ex-n-m-pair-dds.sh
+)

examples/n-m/Header.h

@@ -0,0 +1,24 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+#ifndef FAIR_MQ_EXAMPLE_N_M_HEADER_H
+#define FAIR_MQ_EXAMPLE_N_M_HEADER_H
+
+#include <cstdint>
+
+namespace example_n_m
+{
+
+struct Header
+{
+    std::uint16_t id;
+    int senderIndex;
+};
+
+} // namespace example_n_m
+
+#endif /* FAIR_MQ_EXAMPLE_N_M_HEADER_H */

examples/n-m/README.md

@@ -0,0 +1,4 @@
+N-M
+==========================
+
+A topology consisting of three layers of devices: synchronizer -> sender(s) -> receiver(s). Senders distribute data to receivers based on the data id contained in the message from the synchronizer (same id goes to the same receiver from every sender). The senders send the data in a non-blocking fashion - if queue is full or receiver is down, data is discarded. Two configurations are provided - one using push/pull channels between senders/receivers, another using pair channels. In push/pull case there is only one receiving channel on the receiver device. In pair case there are as many receiver (sub-)channels as there are senders.

examples/n-m/ex-n-m-pair-topology.xml

@@ -0,0 +1,44 @@
+<topology name="myTopology">
+
+    <var name="numSenders" value="3" />
+    <var name="numReceivers" value="4" />
+
+    <property name="fmqchan_sync" />
+    <property name="fmqchan_data" />
+
+    <decltask name="Synchronizer">
+        <exe reachable="true">fairmq-ex-n-m-synchronizer --id sync --rate 100 --color false -P dds --channel-config name=sync,type=pub,method=bind</exe>
+        <env reachable="false">fairmq-ex-n-m-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_sync</id>
+        </properties>
+    </decltask>
+
+    <decltask name="Sender">
+        <exe reachable="true">fairmq-ex-n-m-sender --id sender%taskIndex% --timeframe-size 100000 --num-receivers ${numReceivers} --color false -P dds --channel-config name=sync,type=sub,method=connect name=data,type=pair,method=connect,numSockets=${numReceivers} --dds-i data:%taskIndex%</exe>
+        <env reachable="false">fairmq-ex-n-m-env.sh</env>
+        <properties>
+            <name access="read">fmqchan_sync</id>
+            <name access="read">fmqchan_data</id>
+        </properties>
+    </decltask>
+
+    <decltask name="Receiver">
+        <exe reachable="true">fairmq-ex-n-m-receiver --id receiver%taskIndex% --num-senders ${numSenders} --color false -P dds --max-timeframes 10 --channel-config name=data,type=pair,method=bind,numSockets=${numSenders}</exe>
+        <env reachable="false">fairmq-ex-n-m-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_data</id>
+        </properties>
+    </decltask>
+
+    <main name="main">
+        <task>Synchronizer</task>
+        <group name="Senders" n="${numSenders}">
+            <task>Sender</task>
+        </group>
+        <group name="Receivers" n="${numReceivers}">
+            <task>Receiver</task>
+        </group>
+    </main>
+
+</topology>

examples/n-m/ex-n-m-topology.xml

@@ -0,0 +1,44 @@
+<topology name="myTopology">
+
+    <var name="numSenders" value="3" />
+    <var name="numReceivers" value="4" />
+
+    <property name="fmqchan_sync" />
+    <property name="fmqchan_data" />
+
+    <decltask name="Synchronizer">
+        <exe reachable="true">fairmq-ex-n-m-synchronizer --id sync --rate 100 --color false -P dds --channel-config name=sync,type=pub,method=bind</exe>
+        <env reachable="false">fairmq-ex-n-m-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_sync</id>
+        </properties>
+    </decltask>
+
+    <decltask name="Sender">
+        <exe reachable="true">fairmq-ex-n-m-sender --id sender%taskIndex% --timeframe-size 100000 --num-receivers ${numReceivers} --color false -P dds --channel-config name=sync,type=sub,method=connect name=data,type=push,method=connect,numSockets=${numReceivers}</exe>
+        <env reachable="false">fairmq-ex-n-m-env.sh</env>
+        <properties>
+            <name access="read">fmqchan_sync</id>
+            <name access="read">fmqchan_data</id>
+        </properties>
+    </decltask>
+
+    <decltask name="Receiver">
+        <exe reachable="true">fairmq-ex-n-m-receiver --id receiver%taskIndex% --num-senders ${numSenders} --color false -P dds --max-timeframes 10 --channel-config name=data,type=pull,method=bind</exe>
+        <env reachable="false">fairmq-ex-n-m-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_data</id>
+        </properties>
+    </decltask>
+
+    <main name="main">
+        <task>Synchronizer</task>
+        <group name="Senders" n="${numSenders}">
+            <task>Sender</task>
+        </group>
+        <group name="Receivers" n="${numReceivers}">
+            <task>Receiver</task>
+        </group>
+    </main>
+
+</topology>

examples/n-m/fairmq-ex-n-m-env.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+export PATH=@BIN_DIR@:$PATH
+
+OS=$(uname -s 2>&1)
+if [ "$OS" == "Darwin" ]; then
+   export DYLD_LIBRARY_PATH=@LIB_DIR@:$DYLD_LIBRARY_PATH
+fi

examples/n-m/fairmq-start-ex-n-m-dds.sh.in

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+set -e
+
+cleanup() {
+  dds-session stop $1
+  echo "CLEANUP PERFORMED"
+}
+
+source @DDS_INSTALL_PREFIX@/DDS_env.sh
+export PATH=@BIN_DIR@:$PATH
+
+exec 5>&1
+output=$(dds-session start | tee >(cat - >&5))
+export DDS_SESSION_ID=$(echo ${output} | grep "DDS session ID: " | cut -d' ' -f4)
+echo "SESSION ID: ${DDS_SESSION_ID}"
+
+trap "cleanup ${DDS_SESSION_ID}" EXIT
+
+requiredNofSlots=8
+dds-submit -r localhost --slots ${requiredNofSlots}
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
+
+export FAIRMQ_DDS_TOPO_FILE=@DATA_DIR@/ex-n-m-topology.xml
+echo "TOPOLOGY FILE: ${FAIRMQ_DDS_TOPO_FILE}"
+echo "TOPOLOGY NAME: $(dds-topology --disable-validation --topology-name ${FAIRMQ_DDS_TOPO_FILE})"
+
+dds-info --active-topology
+dds-topology --activate ${FAIRMQ_DDS_TOPO_FILE}
+dds-info --active-topology
+echo "...waiting for ${requiredNofSlots} executing slots..."
+dds-info --executing-count --wait ${requiredNofSlots}
+
+echo "------------------------"
+echo "...waiting for Topology to finish..."
+# TODO Retrieve number of devices from DDS topology API instead of having the user pass it explicitely
+fairmq-dds-command-ui -w "IDLE"
+fairmq-dds-command-ui -c i
+fairmq-dds-command-ui -c k
+fairmq-dds-command-ui -c b
+fairmq-dds-command-ui -c x
+fairmq-dds-command-ui -c j
+fairmq-dds-command-ui -c r
+receivers="main/Receivers.*"
+fairmq-dds-command-ui -w "RUNNING->READY" -p $receivers
+echo "All receivers transitioned from RUNNING to READY"
+fairmq-dds-command-ui -c s
+fairmq-dds-command-ui -c t
+fairmq-dds-command-ui -c d
+fairmq-dds-command-ui -c q
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
+echo "------------------------"
+
+dds-info --active-topology
+dds-topology --stop
+dds-info --active-topology
+
+dds-agent-cmd getlog -a
+logDir="${wrkDir}/logs"
+for file in $(find "${logDir}" -name "*.tar.gz"); do tar -xf ${file} -C "${logDir}" ; done
+echo "AGENT LOG FILES IN: ${logDir}"
+
+# This string is used by ctest to detect success
+echo "Example successful :)"
+
+# Cleanup function is called by EXIT trap

examples/n-m/fairmq-start-ex-n-m-pair-dds.sh.in

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+set -e
+
+cleanup() {
+  dds-session stop $1
+  echo "CLEANUP PERFORMED"
+}
+
+source @DDS_INSTALL_PREFIX@/DDS_env.sh
+export PATH=@BIN_DIR@:$PATH
+
+exec 5>&1
+output=$(dds-session start | tee >(cat - >&5))
+export DDS_SESSION_ID=$(echo ${output} | grep "DDS session ID: " | cut -d' ' -f4)
+echo "SESSION ID: ${DDS_SESSION_ID}"
+
+trap "cleanup ${DDS_SESSION_ID}" EXIT
+
+requiredNofSlots=8
+dds-submit -r localhost --slots ${requiredNofSlots}
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
+
+export FAIRMQ_DDS_TOPO_FILE=@DATA_DIR@/ex-n-m-pair-topology.xml
+echo "TOPOLOGY FILE: ${FAIRMQ_DDS_TOPO_FILE}"
+echo "TOPOLOGY NAME: $(dds-topology --disable-validation --topology-name ${FAIRMQ_DDS_TOPO_FILE})"
+
+dds-info --active-topology
+dds-topology --activate ${FAIRMQ_DDS_TOPO_FILE}
+dds-info --active-topology
+echo "...waiting for ${requiredNofSlots} executing slots..."
+dds-info --executing-count --wait ${requiredNofSlots}
+
+echo "------------------------"
+echo "...waiting for Topology to finish..."
+# TODO Retrieve number of devices from DDS topology API instead of having the user pass it explicitely
+fairmq-dds-command-ui -w "IDLE"
+fairmq-dds-command-ui -c i
+fairmq-dds-command-ui -c k
+fairmq-dds-command-ui -c b
+fairmq-dds-command-ui -c x
+fairmq-dds-command-ui -c j
+fairmq-dds-command-ui -c r
+receivers="main/Receivers.*"
+fairmq-dds-command-ui -w "RUNNING->READY" -p $receivers
+echo "All receivers transitioned from RUNNING to READY"
+fairmq-dds-command-ui -c s
+fairmq-dds-command-ui -c t
+fairmq-dds-command-ui -c d
+fairmq-dds-command-ui -c q
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
+echo "------------------------"
+
+dds-info --active-topology
+dds-topology --stop
+dds-info --active-topology
+
+dds-agent-cmd getlog -a
+logDir="${wrkDir}/logs"
+for file in $(find "${logDir}" -name "*.tar.gz"); do tar -xf ${file} -C "${logDir}" ; done
+echo "AGENT LOG FILES IN: ${logDir}"
+
+# This string is used by ctest to detect success
+echo "Example successful :)"
+
+# Cleanup function is called by EXIT trap

examples/n-m/fairmq-start-ex-n-m-pair.sh.in

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+export PATH=@BIN_DIR@:$PATH
+
+SYNC="fairmq-ex-n-m-synchronizer"
+SYNC+=" --id Sync"
+SYNC+=" --channel-config name=sync,type=pub,method=bind,address=tcp://localhost:8010"
+SYNC+=" --rate 100"
+xterm -geometry 80x25+0+0 -hold -e $SYNC &
+
+SENDER0="fairmq-ex-n-m-sender"
+SENDER0+=" --id Sender1"
+SENDER0+=" --channel-config name=sync,type=sub,method=connect,address=tcp://localhost:8010"
+SENDER0+="                  name=data,type=pair,method=connect,address=tcp://localhost:8021,address=tcp://localhost:8022,address=tcp://localhost:8023,address=tcp://localhost:8024"
+SENDER0+=" --sender-index 0"
+SENDER0+=" --subtimeframe-size 1000000"
+SENDER0+=" --num-receivers 4"
+xterm -geometry 80x25+500+0 -hold -e $SENDER0 &
+
+SENDER1="fairmq-ex-n-m-sender"
+SENDER1+=" --id Sender2"
+SENDER1+=" --channel-config name=sync,type=sub,method=connect,address=tcp://localhost:8010"
+SENDER1+="                  name=data,type=pair,method=connect,address=tcp://localhost:8031,address=tcp://localhost:8032,address=tcp://localhost:8033,address=tcp://localhost:8034"
+SENDER1+=" --sender-index 1"
+SENDER1+=" --subtimeframe-size 1000000"
+SENDER1+=" --num-receivers 4"
+xterm -geometry 80x25+500+350 -hold -e $SENDER1 &
+
+SENDER2="fairmq-ex-n-m-sender"
+SENDER2+=" --id Sender3"
+SENDER2+=" --channel-config name=sync,type=sub,method=connect,address=tcp://localhost:8010"
+SENDER2+="                  name=data,type=pair,method=connect,address=tcp://localhost:8041,address=tcp://localhost:8042,address=tcp://localhost:8043,address=tcp://localhost:8044"
+SENDER2+=" --sender-index 2"
+SENDER2+=" --subtimeframe-size 1000000"
+SENDER2+=" --num-receivers 4"
+xterm -geometry 80x25+500+700 -hold -e $SENDER2 &
+
+RECEIVER0="fairmq-ex-n-m-receiver"
+RECEIVER0+=" --id Receiver1"
+RECEIVER0+=" --channel-config name=data,type=pair,method=bind,address=tcp://localhost:8021,address=tcp://localhost:8031,address=tcp://localhost:8041"
+RECEIVER0+=" --num-senders 3"
+xterm -geometry 80x25+1000+0 -hold -e $RECEIVER0 &
+
+RECEIVER1="fairmq-ex-n-m-receiver"
+RECEIVER1+=" --id Receiver2"
+RECEIVER1+=" --channel-config name=data,type=pair,method=bind,address=tcp://localhost:8022,address=tcp://localhost:8032,address=tcp://localhost:8042"
+RECEIVER1+=" --num-senders 3"
+xterm -geometry 80x25+1000+350 -hold -e $RECEIVER1 &
+
+RECEIVER2="fairmq-ex-n-m-receiver"
+RECEIVER2+=" --id Receiver3"
+RECEIVER2+=" --channel-config name=data,type=pair,method=bind,address=tcp://localhost:8023,address=tcp://localhost:8033,address=tcp://localhost:8043"
+RECEIVER2+=" --num-senders 3"
+xterm -geometry 80x25+1000+700 -hold -e $RECEIVER2 &
+
+RECEIVER3="fairmq-ex-n-m-receiver"
+RECEIVER3+=" --id Receiver4"
+RECEIVER3+=" --channel-config name=data,type=pair,method=bind,address=tcp://localhost:8024,address=tcp://localhost:8034,address=tcp://localhost:8044"
+RECEIVER3+=" --num-senders 3"
+xterm -geometry 80x25+1000+1050 -hold -e $RECEIVER3 &

examples/n-m/fairmq-start-ex-n-m.sh.in

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+export PATH=@BIN_DIR@:$PATH
+
+SYNC="fairmq-ex-n-m-synchronizer"
+SYNC+=" --id Sync"
+SYNC+=" --channel-config name=sync,type=pub,method=bind,address=tcp://localhost:8010"
+SYNC+=" --rate 100"
+xterm -geometry 80x25+0+0 -hold -e $SYNC &
+
+SENDER0="fairmq-ex-n-m-sender"
+SENDER0+=" --id Sender1"
+SENDER0+=" --channel-config name=sync,type=sub,method=connect,address=tcp://localhost:8010"
+SENDER0+="                  name=data,type=push,method=connect,address=tcp://localhost:8021,address=tcp://localhost:8022,address=tcp://localhost:8023,address=tcp://localhost:8024"
+SENDER0+=" --sender-index 0"
+SENDER0+=" --subtimeframe-size 1000000"
+SENDER0+=" --num-receivers 4"
+xterm -geometry 80x25+500+0 -hold -e $SENDER0 &
+
+SENDER1="fairmq-ex-n-m-sender"
+SENDER1+=" --id Sender2"
+SENDER1+=" --channel-config name=sync,type=sub,method=connect,address=tcp://localhost:8010"
+SENDER1+="                  name=data,type=push,method=connect,address=tcp://localhost:8021,address=tcp://localhost:8022,address=tcp://localhost:8023,address=tcp://localhost:8024"
+SENDER1+=" --sender-index 1"
+SENDER1+=" --subtimeframe-size 1000000"
+SENDER1+=" --num-receivers 4"
+xterm -geometry 80x25+500+350 -hold -e $SENDER1 &
+
+SENDER2="fairmq-ex-n-m-sender"
+SENDER2+=" --id Sender3"
+SENDER2+=" --channel-config name=sync,type=sub,method=connect,address=tcp://localhost:8010"
+SENDER2+="                  name=data,type=push,method=connect,address=tcp://localhost:8021,address=tcp://localhost:8022,address=tcp://localhost:8023,address=tcp://localhost:8024"
+SENDER2+=" --sender-index 2"
+SENDER2+=" --subtimeframe-size 1000000"
+SENDER2+=" --num-receivers 4"
+xterm -geometry 80x25+500+700 -hold -e $SENDER2 &
+
+RECEIVER0="fairmq-ex-n-m-receiver"
+RECEIVER0+=" --id Receiver1"
+RECEIVER0+=" --channel-config name=data,type=pull,method=bind,address=tcp://localhost:8021"
+RECEIVER0+=" --num-senders 3"
+xterm -geometry 80x25+1000+0 -hold -e $RECEIVER0 &
+
+RECEIVER1="fairmq-ex-n-m-receiver"
+RECEIVER1+=" --id Receiver2"
+RECEIVER1+=" --channel-config name=data,type=pull,method=bind,address=tcp://localhost:8022"
+RECEIVER1+=" --num-senders 3"
+xterm -geometry 80x25+1000+350 -hold -e $RECEIVER1 &
+
+RECEIVER2="fairmq-ex-n-m-receiver"
+RECEIVER2+=" --id Receiver3"
+RECEIVER2+=" --channel-config name=data,type=pull,method=bind,address=tcp://localhost:8023"
+RECEIVER2+=" --num-senders 3"
+xterm -geometry 80x25+1000+700 -hold -e $RECEIVER2 &
+
+RECEIVER3="fairmq-ex-n-m-receiver"
+RECEIVER3+=" --id Receiver4"
+RECEIVER3+=" --channel-config name=data,type=pull,method=bind,address=tcp://localhost:8024"
+RECEIVER3+=" --num-senders 3"
+xterm -geometry 80x25+1000+1050 -hold -e $RECEIVER3 &

examples/n-m/runReceiver.cxx

@@ -0,0 +1,119 @@
+/********************************************************************************
+ *    Copyright (C) 2020 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "Header.h"
+
+#include <FairMQDevice.h>
+#include <runFairMQDevice.h>
+
+#include <string>
+#include <iomanip>
+#include <chrono>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+using namespace std;
+using namespace example_n_m;
+namespace bpo = boost::program_options;
+
+struct TFBuffer
+{
+    FairMQParts parts;
+    chrono::steady_clock::time_point start;
+    chrono::steady_clock::time_point end;
+};
+
+class Receiver : public FairMQDevice
+{
+  public:
+    Receiver()
+        : fBuffer()
+        , fDiscardedSet()
+        , fNumSenders(0)
+        , fBufferTimeoutInMs(5000)
+        , fMaxTimeframes(0)
+        , fTimeframeCounter(0)
+    {
+        OnData("data", &Receiver::HandleData);
+    }
+
+    ~Receiver() = default;
+
+    void InitTask() override
+    {
+        fNumSenders = GetConfig()->GetValue<int>("num-senders");
+        fBufferTimeoutInMs = GetConfig()->GetValue<int>("buffer-timeout");
+        fMaxTimeframes = GetConfig()->GetValue<int>("max-timeframes");
+    }
+
+  protected:
+    bool HandleData(FairMQParts& parts, int /* index */)
+    {
+        Header& h = *(static_cast<Header*>(parts.At(0)->GetData()));
+        // LOG(info) << "Received sub-time frame #" << h.id << " from Sender" << h.senderIndex;
+
+        if (fDiscardedSet.find(h.id) == fDiscardedSet.end()) {
+            if (fBuffer.find(h.id) == fBuffer.end()) {
+                // if this is the first part with this ID, save the receive time.
+                fBuffer[h.id].start = chrono::steady_clock::now();
+            }
+            // if the received ID has not previously been discarded, store the data part in the buffer
+            fBuffer[h.id].parts.AddPart(move(parts.At(1)));
+        } else {
+            // if received ID has been previously discarded.
+            LOG(debug) << "Received part from an already discarded timeframe with id " << h.id;
+        }
+
+        if (fBuffer[h.id].parts.Size() == fNumSenders) {
+            LOG(info) << "Successfully completed timeframe #" << h.id;
+            fBuffer.erase(h.id);
+
+            if (fMaxTimeframes > 0 && ++fTimeframeCounter >= fMaxTimeframes) {
+                LOG(info) << "Reached configured maximum number of timeframes (" << fMaxTimeframes << "). Exiting RUNNING state.";
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    void DiscardIncompleteTimeframes()
+    {
+        auto it = fBuffer.begin();
+        while (it != fBuffer.end()) {
+            if (chrono::duration_cast<chrono::milliseconds>(chrono::steady_clock::now() - (it->second).start).count() > fBufferTimeoutInMs) {
+                LOG(debug) << "Timeframe #" << it->first << " incomplete after " << fBufferTimeoutInMs << " milliseconds, discarding";
+                fDiscardedSet.insert(it->first);
+                fBuffer.erase(it++);
+                LOG(debug) << "Number of discarded timeframes: " << fDiscardedSet.size();
+            } else {
+                // LOG(info) << "Timeframe #" << it->first << " within timeout, buffering...";
+                ++it;
+            }
+        }
+    }
+
+    unordered_map<uint16_t, TFBuffer> fBuffer;
+    unordered_set<uint16_t> fDiscardedSet;
+
+    int fNumSenders;
+    int fBufferTimeoutInMs;
+    int fMaxTimeframes;
+    int fTimeframeCounter;
+};
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()
+        ("buffer-timeout", bpo::value<int>()->default_value(1000), "Buffer timeout in milliseconds")
+        ("num-senders", bpo::value<int>()->required(), "Number of senders")
+        ("max-timeframes", bpo::value<int>()->default_value(0), "Maximum number of timeframes to receive (0 - unlimited)");
+}
+
+FairMQDevice* getDevice(const FairMQProgOptions& /* config */) { return new Receiver(); }

examples/n-m/runSender.cxx

@@ -0,0 +1,79 @@
+/********************************************************************************
+ *    Copyright (C) 2020 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "Header.h"
+
+#include <FairMQDevice.h>
+#include <runFairMQDevice.h>
+
+#include <string>
+
+using namespace std;
+using namespace example_n_m;
+namespace bpo = boost::program_options;
+
+class Sender : public FairMQDevice
+{
+  public:
+    Sender()
+        : fNumReceivers(0)
+        , fIndex(0)
+        , fSubtimeframeSize(10000)
+    {}
+
+    ~Sender() = default;
+
+  protected:
+    void InitTask() override
+    {
+        fIndex = GetConfig()->GetProperty<int>("sender-index");
+        fSubtimeframeSize = GetConfig()->GetProperty<int>("subtimeframe-size");
+        fNumReceivers = GetConfig()->GetProperty<int>("num-receivers");
+    }
+
+    void Run() override
+    {
+        FairMQChannel& dataInChannel = fChannels.at("sync").at(0);
+
+        while (!NewStatePending()) {
+            Header h;
+            FairMQMessagePtr id(NewMessage());
+            if (dataInChannel.Receive(id) > 0) {
+                h.id = *(static_cast<uint16_t*>(id->GetData()));
+                h.senderIndex = fIndex;
+            } else {
+                continue;
+            }
+
+            FairMQParts parts;
+            parts.AddPart(NewSimpleMessage(h));
+            parts.AddPart(NewMessage(fSubtimeframeSize));
+
+            uint64_t currentDataId = h.id;
+            int direction = currentDataId % fNumReceivers;
+
+            if (Send(parts, "data", direction, 0) < 0) {
+                LOG(debug) << "Failed to queue Subtimeframe #" << currentDataId << " to Receiver[" << direction << "]";
+            }
+        }
+    }
+
+  private:
+    int fNumReceivers;
+    unsigned int fIndex;
+    int fSubtimeframeSize;
+};
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()
+        ("sender-index", bpo::value<int>()->default_value(0), "Sender Index")
+        ("subtimeframe-size", bpo::value<int>()->default_value(1000), "Subtimeframe size in bytes")
+        ("num-receivers", bpo::value<int>()->required(), "Number of EPNs");
+}
+FairMQDevice* getDevice(const FairMQProgOptions& /* config */) { return new Sender(); }

examples/n-m/runSynchronizer.cxx

@@ -0,0 +1,46 @@
+/********************************************************************************
+ *    Copyright (C) 2020 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include <FairMQDevice.h>
+#include <runFairMQDevice.h>
+
+#include <string>
+#include <cstdint>
+
+using namespace std;
+namespace bpo = boost::program_options;
+
+class Synchronizer : public FairMQDevice
+{
+  public:
+    Synchronizer()
+        : fTimeframeId(0)
+    {}
+    ~Synchronizer() = default;
+
+  protected:
+    bool ConditionalRun() override
+    {
+        FairMQMessagePtr msg(NewSimpleMessage(fTimeframeId));
+
+        if (Send(msg, "sync") > 0) {
+            if (++fTimeframeId == UINT16_MAX - 1) {
+                fTimeframeId = 0;
+            }
+        } else {
+            return false;
+        }
+
+        return true;
+    }
+
+    uint16_t fTimeframeId;
+};
+
+void addCustomOptions(bpo::options_description& /* options */) {}
+FairMQDevice* getDevice(const FairMQProgOptions& /* config */) { return new Synchronizer(); }

examples/qc/CMakeLists.txt

@@ -0,0 +1,71 @@
+################################################################################
+#    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+add_executable(fairmq-ex-qc-sampler runSampler.cxx)
+target_link_libraries(fairmq-ex-qc-sampler PRIVATE FairMQ)
+
+add_executable(fairmq-ex-qc-dispatcher runQCDispatcher.cxx)
+target_link_libraries(fairmq-ex-qc-dispatcher PRIVATE FairMQ)
+
+add_executable(fairmq-ex-qc-task runQCTask.cxx)
+target_link_libraries(fairmq-ex-qc-task PRIVATE FairMQ)
+
+add_executable(fairmq-ex-qc-sink runSink.cxx)
+target_link_libraries(fairmq-ex-qc-sink PRIVATE FairMQ)
+
+add_custom_target(ExampleQC DEPENDS fairmq-ex-qc-sampler fairmq-ex-qc-dispatcher fairmq-ex-qc-task fairmq-ex-qc-sink)
+
+list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
+set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
+set(DATA_DIR ${CMAKE_CURRENT_BINARY_DIR})
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-qc-topology.xml ${CMAKE_CURRENT_BINARY_DIR}/ex-qc-topology.xml @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-qc-env.sh ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-qc-env.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-qc.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-qc.sh @ONLY)
+
+# test
+if(DDS_FOUND)
+  add_test(NAME Example.QC.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-qc.sh localhost)
+  set_tests_properties(Example.QC.localhost PROPERTIES TIMEOUT 15 RUN_SERIAL true PASS_REGULAR_EXPRESSION "Example successful")
+endif()
+
+# install
+install(
+  TARGETS
+  fairmq-ex-qc-sampler
+  fairmq-ex-qc-dispatcher
+  fairmq-ex-qc-task
+  fairmq-ex-qc-sink
+
+  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
+)
+
+# configure run script with different executable paths for build and for install directories
+set(BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
+set(DATA_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_DATADIR})
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-qc-topology.xml ${CMAKE_CURRENT_BINARY_DIR}/ex-qc-topology.xml_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-qc-env.sh ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-qc-env.sh_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-qc.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-qc.sh_install @ONLY)
+
+install(
+  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-qc-topology.xml_install
+  DESTINATION ${PROJECT_INSTALL_DATADIR}
+  RENAME ex-qc-topology.xml
+)
+
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-qc-env.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-ex-qc-env.sh
+)
+
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-qc.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-start-ex-qc.sh
+)

examples/qc/README.md

@@ -0,0 +1,4 @@
+QC
+==
+
+A topology consisting of 4 devices - Sampler, QCDispatcher, QCTask and Sink. The data flows from Sampler through QCDispatcher to Sink. On demand - by setting the corresponding configuration property - the QCDispatcher device will duplicate the data to the QCTask device. The property is set by the topology controller, in this example this is the `fairmq-dds-command-ui` utility.

examples/qc/ex-qc-topology.xml

@@ -0,0 +1,48 @@
+<topology name="ExampleQC">
+
+    <property name="fmqchan_data1" />
+    <property name="fmqchan_data2" />
+    <property name="fmqchan_qc" />
+
+    <decltask name="Sampler">
+        <exe>fairmq-ex-qc-sampler --color false --channel-config name=data1,type=push,method=bind -P dds --max-iterations 1000</exe>
+        <env reachable="false">fairmq-ex-qc-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_data1</name>
+        </properties>
+    </decltask>
+
+    <decltask name="QCDispatcher">
+        <exe>fairmq-ex-qc-dispatcher --color false --channel-config name=data1,type=pull,method=connect name=data2,type=push,method=connect name=qc,type=push,method=connect -P dds</exe>
+        <env reachable="false">fairmq-ex-qc-env.sh</env>
+        <properties>
+            <name access="read">fmqchan_data1</name>
+            <name access="read">fmqchan_data2</name>
+            <name access="read">fmqchan_qc</name>
+        </properties>
+    </decltask>
+
+    <decltask name="QCTask">
+        <exe>fairmq-ex-qc-task --color false --channel-config name=qc,type=pull,method=bind -P dds</exe>
+        <env reachable="false">fairmq-ex-qc-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_qc</name>
+        </properties>
+    </decltask>
+
+    <decltask name="Sink">
+        <exe>fairmq-ex-qc-sink --color false --channel-config name=data2,type=pull,method=bind -P dds --max-iterations 1000</exe>
+        <env reachable="false">fairmq-ex-qc-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_data2</name>
+        </properties>
+    </decltask>
+
+    <main name="main">
+        <task>Sampler</task>
+        <task>QCDispatcher</task>
+        <task>QCTask</task>
+        <task>Sink</task>
+    </main>
+
+</topology>

examples/qc/fairmq-ex-qc-env.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+export PATH=@BIN_DIR@:$PATH
+
+OS=$(uname -s 2>&1)
+if [ "$OS" == "Darwin" ]; then
+   export DYLD_LIBRARY_PATH=@LIB_DIR@:$DYLD_LIBRARY_PATH
+fi

examples/qc/fairmq-start-ex-qc.sh.in

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+# fairmq-start-ex-qc.sh -> submit agents with localhost plugin
+
+set -e
+
+cleanup() {
+  dds-session stop $1
+  echo "CLEANUP PERFORMED"
+}
+
+source @DDS_INSTALL_PREFIX@/DDS_env.sh
+export PATH=@BIN_DIR@:$PATH
+
+exec 5>&1
+output=$(dds-session start | tee >(cat - >&5))
+export DDS_SESSION_ID=$(echo ${output} | grep "DDS session ID: " | cut -d' ' -f4)
+echo "SESSION ID: ${DDS_SESSION_ID}"
+
+trap "cleanup ${DDS_SESSION_ID}" EXIT
+
+requiredNofSlots=4
+dds-submit -r localhost --slots ${requiredNofSlots}
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
+
+export FAIRMQ_DDS_TOPO_FILE=@DATA_DIR@/ex-qc-topology.xml
+echo "TOPOLOGY FILE: ${FAIRMQ_DDS_TOPO_FILE}"
+echo "TOPOLOGY NAME: $(dds-topology --disable-validation --topology-name ${FAIRMQ_DDS_TOPO_FILE})"
+
+dds-info --active-topology
+dds-topology --activate ${FAIRMQ_DDS_TOPO_FILE}
+dds-info --active-topology
+echo "...waiting for ${requiredNofSlots} executing slots..."
+dds-info --executing-count --wait ${requiredNofSlots}
+
+echo "------------------------"
+echo "...waiting for Topology to finish..."
+# TODO Retrieve number of devices from DDS topology API instead of having the user pass it explicitely
+fairmq-dds-command-ui -w "IDLE"
+fairmq-dds-command-ui -c i
+fairmq-dds-command-ui -c k
+fairmq-dds-command-ui -c b
+fairmq-dds-command-ui -c x
+fairmq-dds-command-ui -c j
+allexceptqctasks="main/(Sampler|QCDispatcher|Sink)"
+fairmq-dds-command-ui -c r -p $allexceptqctasks
+qctask="main/QCTask.*"
+qcdispatcher="main/QCDispatcher.*"
+fairmq-dds-command-ui -c p --property-key qc --property-value active -p $qcdispatcher
+fairmq-dds-command-ui -c r -p $qctask
+fairmq-dds-command-ui -w "RUNNING->READY" -p $qctask
+echo "...$qctask received data and transitioned to READY, sending shutdown..."
+fairmq-dds-command-ui -c s
+fairmq-dds-command-ui -c t
+fairmq-dds-command-ui -c d
+fairmq-dds-command-ui -c q
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
+echo "------------------------"
+
+dds-info --active-topology
+dds-topology --stop
+dds-info --active-topology
+
+dds-agent-cmd getlog -a
+logDir="${wrkDir}/logs"
+for file in $(find "${logDir}" -name "*.tar.gz"); do tar -xf ${file} -C "${logDir}" ; done
+echo "AGENT LOG FILES IN: ${logDir}"
+
+# This string is used by ctest to detect success
+echo "Example successful :)"
+
+# Cleanup function is called by EXIT trap

examples/qc/runQCDispatcher.cxx

@@ -0,0 +1,59 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "FairMQDevice.h"
+
+class QCDispatcher : public FairMQDevice
+{
+  public:
+    QCDispatcher()
+        : fDoQC(false)
+    {
+        OnData("data1", &QCDispatcher::HandleData);
+    }
+
+    void InitTask() override
+    {
+        GetConfig()->Subscribe<std::string>("qcdevice", [&](const std::string& key, std::string value) {
+            if (key == "qc") {
+                if (value == "active") {
+                    fDoQC.store(true);
+                } else if (value == "inactive") {
+                    fDoQC.store(false);
+                }
+            }
+        });
+    }
+
+  protected:
+    bool HandleData(FairMQMessagePtr& msg, int)
+    {
+        if (fDoQC.load() == true) {
+            FairMQMessagePtr msgCopy(NewMessage());
+            msgCopy->Copy(*msg);
+            if (Send(msg, "qc") < 0) {
+                return false;
+            }
+        }
+
+        if (Send(msg, "data2") < 0) {
+            return false;
+        }
+
+        return true;
+    }
+
+    void ResetTask() override { GetConfig()->Unsubscribe<std::string>("qcdevice"); }
+
+  private:
+    std::atomic<bool> fDoQC;
+};
+
+void addCustomOptions(boost::program_options::options_description& /*options*/) {}
+FairMQDevicePtr getDevice(const fair::mq::ProgOptions& /*config*/) { return new QCDispatcher(); }

examples/qc/runQCTask.cxx

@@ -0,0 +1,26 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "FairMQDevice.h"
+
+class QCTask : public FairMQDevice
+{
+  public:
+    QCTask()
+    {
+        OnData("qc", [](FairMQMessagePtr& /*msg*/, int) {
+            LOG(info) << "received data";
+            return false;
+        });
+    }
+};
+
+namespace bpo = boost::program_options;
+void addCustomOptions(bpo::options_description& /*options*/) {}
+FairMQDevicePtr getDevice(const fair::mq::ProgOptions& /*config*/) { return new QCTask(); }

examples/qc/runSampler.cxx

@@ -0,0 +1,36 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "FairMQDevice.h"
+
+#include <thread> // this_thread::sleep_for
+#include <chrono>
+
+class Sampler : public FairMQDevice
+{
+  public:
+    Sampler() {}
+
+  protected:
+    virtual bool ConditionalRun()
+    {
+        FairMQMessagePtr msg(NewMessage(1000));
+
+        if (Send(msg, "data1") < 0) {
+            return false;
+        }
+
+        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+        return true;
+    }
+};
+
+namespace bpo = boost::program_options;
+void addCustomOptions(bpo::options_description&) {}
+FairMQDevicePtr getDevice(const fair::mq::ProgOptions& /*config*/) { return new Sampler(); }

examples/qc/runSink.cxx

@@ -0,0 +1,25 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "FairMQDevice.h"
+
+#include <string>
+
+class Sink : public FairMQDevice
+{
+  public:
+    Sink() { OnData("data2", &Sink::HandleData); }
+
+  protected:
+    bool HandleData(FairMQMessagePtr& /*msg*/, int /*index*/) { return true; }
+};
+
+namespace bpo = boost::program_options;
+void addCustomOptions(bpo::options_description&) {}
+FairMQDevicePtr getDevice(const fair::mq::ProgOptions& /*config*/) { return new Sink(); }

examples/readout/fairmq-start-ex-readout-processing.sh.in

@@ -31,10 +31,12 @@ SENDER="fairmq-ex-readout-sender"
 SENDER+=" --id sender1"
 SENDER+=" --input-name ps"
 SENDER+=" --channel-config name=ps,type=pair,method=bind,address=tcp://localhost:7779,transport=shmem"
-SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7780,transport=ofi"
+#SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7780,transport=ofi"
+SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7780,transport=zeromq"
 xterm -geometry 80x23+1000+0 -hold -e @EX_BIN_DIR@/$SENDER &
 
 RECEIVER="fairmq-ex-readout-receiver"
 RECEIVER+=" --id receiver1"
-RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7780,transport=ofi"
+#RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7780,transport=ofi"
+RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7780,transport=zeromq"
 xterm -geometry 80x23+1500+0 -hold -e @EX_BIN_DIR@/$RECEIVER &

examples/readout/fairmq-start-ex-readout.sh.in

@@ -25,10 +25,12 @@ SENDER="fairmq-ex-readout-sender"
 SENDER+=" --id sender1"
 SENDER+=" --input-name bs"
 SENDER+=" --channel-config name=bs,type=pair,method=bind,address=tcp://localhost:7778,transport=shmem"
-SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7779,transport=ofi"
+# SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7779,transport=ofi"
+SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7779,transport=zeromq"
 xterm -geometry 80x23+1000+0 -hold -e @EX_BIN_DIR@/$SENDER &
 
 RECEIVER="fairmq-ex-readout-receiver"
 RECEIVER+=" --id receiver1"
-RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7779,transport=ofi"
+# RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7779,transport=ofi"
+RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7779,transport=zeromq"
 xterm -geometry 80x23+1500+0 -hold -e @EX_BIN_DIR@/$RECEIVER &

examples/region/Sampler.cxx

@@ -35,6 +35,14 @@ void Sampler::InitTask()
     fMsgSize = fConfig->GetProperty<int>("msg-size");
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
 
+    fChannels.at("data").at(0).Transport()->SubscribeToRegionEvents([](FairMQRegionInfo info) {
+        LOG(warn) << ">>>" << info.event;
+        LOG(warn) << "id: " << info.id;
+        LOG(warn) << "ptr: " << info.ptr;
+        LOG(warn) << "size: " << info.size;
+        LOG(warn) << "flags: " << info.flags;
+    });
+
     fRegion = FairMQUnmanagedRegionPtr(NewUnmanagedRegionFor("data",
                                                              0,
                                                              10000000,
@@ -82,6 +90,7 @@ void Sampler::ResetTask()
         LOG(debug) << "done, still unacked: " << fNumUnackedMsgs;
     }
     fRegion.reset();
+    fChannels.at("data").at(0).Transport()->UnsubscribeFromRegionEvents();
 }
 
 Sampler::~Sampler()

examples/region/Sink.cxx

@@ -29,6 +29,13 @@ void Sink::InitTask()
 {
     // Get the fMaxIterations value from the command line options (via fConfig)
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
+    fChannels.at("data").at(0).Transport()->SubscribeToRegionEvents([](FairMQRegionInfo info) {
+        LOG(warn) << ">>>" << info.event;
+        LOG(warn) << "id: " << info.id;
+        LOG(warn) << "ptr: " << info.ptr;
+        LOG(warn) << "size: " << info.size;
+        LOG(warn) << "flags: " << info.flags;
+    });
 }
 
 void Sink::Run()
@@ -50,6 +57,11 @@ void Sink::Run()
     }
 }
 
+void Sink::ResetTask()
+{
+    fChannels.at("data").at(0).Transport()->UnsubscribeFromRegionEvents();
+}
+
 Sink::~Sink()
 {
 }

examples/region/Sink.h

@@ -31,6 +31,7 @@ class Sink : public FairMQDevice
   protected:
     virtual void Run();
     virtual void InitTask();
+    virtual void ResetTask();
 
   private:
     uint64_t fMaxIterations;

examples/region/test-ex-region.sh.in

@@ -19,6 +19,7 @@ SAMPLER+=" --id sampler1"
 SAMPLER+=" --transport $transport"
 SAMPLER+=" --severity debug"
 SAMPLER+=" --session $SESSION"
+SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --control static --color false"
 SAMPLER+=" --max-iterations 1"
 SAMPLER+=" --msg-size $msgSize"

fairmq/CMakeLists.txt

@@ -6,16 +6,6 @@
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
-####################
-# external plugins #
-####################
-if(BUILD_DDS_PLUGIN)
-  add_subdirectory(plugins/DDS)
-endif()
-if(BUILD_PMIX_PLUGIN)
-  add_subdirectory(plugins/PMIx)
-endif()
-
 if(BUILD_FAIRMQ OR BUILD_SDK)
   ###########
   # Version #
@@ -58,6 +48,12 @@ if(BUILD_FAIRMQ OR BUILD_SDK)
     ${TOOLS_PUBLIC_HEADER_FILES}
   )
   target_compile_definitions(${target} PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
+  # workaround https://github.com/boostorg/asio/commit/43874d5497414c67655d901e48c939ef01337edb
+  if(    Boost_VERSION VERSION_LESS 1.69
+     AND CMAKE_CXX_COMPILER_ID STREQUAL AppleClang
+     AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 10.0.1)
+    target_compile_definitions(${target} PUBLIC BOOST_ASIO_HAS_STD_STRING_VIEW)
+  endif()
   target_include_directories(${target}
     PUBLIC
     $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}>
@@ -186,11 +182,11 @@ if(BUILD_FAIRMQ)
     plugins/Builtin.h
     plugins/config/Config.h
     plugins/Control.h
-    shmem/FairMQMessageSHM.h
-    shmem/FairMQPollerSHM.h
-    shmem/FairMQUnmanagedRegionSHM.h
-    shmem/FairMQSocketSHM.h
-    shmem/FairMQTransportFactorySHM.h
+    shmem/Message.h
+    shmem/Poller.h
+    shmem/UnmanagedRegion.h
+    shmem/Socket.h
+    shmem/TransportFactory.h
     shmem/Common.h
     shmem/Manager.h
     shmem/Region.h
@@ -247,11 +243,10 @@ if(BUILD_FAIRMQ)
     SuboptParser.cxx
     plugins/config/Config.cxx
     plugins/Control.cxx
-    shmem/FairMQMessageSHM.cxx
-    shmem/FairMQPollerSHM.cxx
-    shmem/FairMQUnmanagedRegionSHM.cxx
-    shmem/FairMQSocketSHM.cxx
-    shmem/FairMQTransportFactorySHM.cxx
+    shmem/Message.cxx
+    shmem/Poller.cxx
+    shmem/Socket.cxx
+    shmem/TransportFactory.cxx
     shmem/Manager.cxx
     shmem/Region.cxx
     zeromq/FairMQMessageZMQ.cxx
@@ -460,6 +455,20 @@ if(BUILD_FAIRMQ)
   endforeach()
 endif()
 
+if(BUILD_SDK_COMMANDS)
+  add_subdirectory(sdk/commands)
+endif()
+
 if(BUILD_SDK)
   add_subdirectory(sdk)
 endif()
+
+####################
+# external plugins #
+####################
+if(BUILD_DDS_PLUGIN)
+  add_subdirectory(plugins/DDS)
+endif()
+if(BUILD_PMIX_PLUGIN)
+  add_subdirectory(plugins/PMIx)
+endif()

fairmq/DeviceRunner.cxx

@@ -75,7 +75,7 @@ void DeviceRunner::SubscribeForConfigChange()
             fair::Logger::SetConsoleColor(val);
         }
     });
-    fConfig.Subscribe<string>("device-runner", [&](const std::string& key, const std::string val) {
+    fConfig.Subscribe<string>("device-runner", [&](const std::string& key, const std::string& val) {
         if (key == "severity") {
             fair::Logger::SetConsoleSeverity(val);
         } else if (key == "file-severity") {

fairmq/EventManager.h

@@ -12,12 +12,10 @@
 #include <memory>
 #include <mutex>
 #include <string>
-#include <tuple>
 #include <typeindex>
-#include <typeinfo>
 #include <unordered_map>
 #include <utility>
-#include <vector>
+#include <functional>
 
 #include <boost/any.hpp>
 #include <boost/functional/hash.hpp>

fairmq/FairMQChannel.cxx

@@ -7,7 +7,8 @@
  ********************************************************************************/
 
 #include "FairMQChannel.h"
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Strings.h>
+#include <fairmq/Properties.h>
 
 #include <boost/algorithm/string.hpp> // join/split
 
@@ -19,7 +20,7 @@ using namespace std;
 using namespace fair::mq;
 
 template<typename T>
-T GetPropertyOrDefault(const fair::mq::Properties& m, const string& k, const T& ifNotFound) noexcept
+T GetPropertyOrDefault(const fair::mq::Properties& m, const string& k, const T& ifNotFound)
 {
     if (m.count(k)) {
         return boost::any_cast<T>(m.at(k));
@@ -132,26 +133,37 @@ FairMQChannel::FairMQChannel(const FairMQChannel& chan, const string& newName)
 
 FairMQChannel& FairMQChannel::operator=(const FairMQChannel& chan)
 {
-    fTransportFactory = nullptr;
-    fTransportType = chan.fTransportType;
-    fSocket = nullptr;
-    fName = chan.fName;
-    fType = chan.fType;
-    fMethod = chan.fMethod;
-    fAddress = chan.fAddress;
-    fSndBufSize = chan.fSndBufSize;
-    fRcvBufSize = chan.fRcvBufSize;
-    fSndKernelSize = chan.fSndKernelSize;
-    fRcvKernelSize = chan.fRcvKernelSize;
-    fLinger = chan.fLinger;
-    fRateLogging = chan.fRateLogging;
-    fPortRangeMin = chan.fPortRangeMin;
-    fPortRangeMax = chan.fPortRangeMax;
-    fAutoBind = chan.fAutoBind;
-    fIsValid = false;
-    fMultipart = chan.fMultipart;
-    fModified = chan.fModified;
-    fReset = false;
+    if (this == &chan) {
+        return *this;
+    }
+
+    {
+        // TODO: replace this with std::scoped_lock (c++17)
+        lock(fMtx, chan.fMtx);
+        lock_guard<mutex> lock1(fMtx, adopt_lock);
+        lock_guard<mutex> lock2(chan.fMtx, adopt_lock);
+
+        fTransportFactory = nullptr;
+        fTransportType = chan.fTransportType;
+        fSocket = nullptr;
+        fName = chan.fName;
+        fType = chan.fType;
+        fMethod = chan.fMethod;
+        fAddress = chan.fAddress;
+        fSndBufSize = chan.fSndBufSize;
+        fRcvBufSize = chan.fRcvBufSize;
+        fSndKernelSize = chan.fSndKernelSize;
+        fRcvKernelSize = chan.fRcvKernelSize;
+        fLinger = chan.fLinger;
+        fRateLogging = chan.fRateLogging;
+        fPortRangeMin = chan.fPortRangeMin;
+        fPortRangeMax = chan.fPortRangeMax;
+        fAutoBind = chan.fAutoBind;
+        fIsValid = false;
+        fMultipart = chan.fMultipart;
+        fModified = chan.fModified;
+        fReset = false;
+    }
 
     return *this;
 }

fairmq/FairMQChannel.h

@@ -10,6 +10,7 @@
 #define FAIRMQCHANNEL_H_
 
 #include <FairMQTransportFactory.h>
+#include <FairMQUnmanagedRegion.h>
 #include <FairMQSocket.h>
 #include <fairmq/Transports.h>
 #include <FairMQLogger.h>
@@ -17,15 +18,14 @@
 #include <fairmq/Properties.h>
 #include <FairMQMessage.h>
 
-#include <boost/any.hpp>
-
 #include <string>
 #include <memory> // unique_ptr, shared_ptr
 #include <vector>
-#include <atomic>
 #include <mutex>
 #include <stdexcept>
 #include <utility> // std::move
+#include <cstddef> // size_t
+#include <cstdint> // int64_t
 
 class FairMQChannel
 {
@@ -340,6 +340,11 @@ class FairMQChannel
         return Transport()->CreateUnmanagedRegion(size, callback, path, flags);
     }
 
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0)
+    {
+        return Transport()->CreateUnmanagedRegion(size, userFlags, callback, path, flags);
+    }
+
     static constexpr fair::mq::Transport DefaultTransportType = fair::mq::Transport::DEFAULT;
     static constexpr const char* DefaultTransportName = "default";
     static constexpr const char* DefaultName = "";

fairmq/FairMQDevice.cxx

@@ -8,19 +8,15 @@
 
 #include <FairMQDevice.h>
 
+#include <fairmq/tools/RateLimit.h>
+#include <fairmq/tools/Network.h>
+
 #include <boost/algorithm/string.hpp> // join/split
 
-#include <boost/uuid/uuid.hpp>
-#include <boost/uuid/uuid_generators.hpp>
-#include <boost/uuid/uuid_io.hpp>
-
 #include <list>
-#include <cstdlib>
 #include <chrono>
 #include <mutex>
 #include <thread>
-#include <functional>
-#include <sstream>
 #include <iomanip>
 #include <future>
 #include <algorithm> // std::max
@@ -303,10 +299,15 @@ void FairMQDevice::InitWrapper()
                 // if binding address is not specified, try getting it from the configured network interface
                 if (subChannel.fAddress == "unspecified" || subChannel.fAddress == "") {
                     // if the configured network interface is default, get its name from the default route
-                    if (networkInterface == "default") {
-                        networkInterface = tools::getDefaultRouteNetworkInterface();
+                    try {
+                        if (networkInterface == "default") {
+                            networkInterface = tools::getDefaultRouteNetworkInterface();
+                        }
+                        subChannel.fAddress = "tcp://" + tools::getInterfaceIP(networkInterface) + ":1";
+                    } catch(const tools::DefaultRouteDetectionError& e) {
+                        LOG(debug) << "binding on tcp://*:1";
+                        subChannel.fAddress = "tcp://*:1";
                     }
-                    subChannel.fAddress = "tcp://" + tools::getInterfaceIP(networkInterface) + ":1";
                 }
                 // fill the uninitialized list
                 fUninitializedBindingChannels.push_back(&subChannel);

fairmq/FairMQDevice.h

@@ -14,7 +14,6 @@
 #include <fairmq/Transports.h>
 #include <fairmq/StateQueue.h>
 
-#include <FairMQSocket.h>
 #include <FairMQChannel.h>
 #include <FairMQMessage.h>
 #include <FairMQParts.h>
@@ -24,21 +23,19 @@
 
 #include <vector>
 #include <memory> // unique_ptr
-#include <algorithm> // std::sort()
+#include <algorithm> // find
 #include <string>
 #include <chrono>
 #include <iostream>
 #include <unordered_map>
 #include <functional>
-#include <assert.h> // static_assert
-#include <type_traits> // is_trivially_copyable
 #include <stdexcept>
-#include <queue>
-
 #include <mutex>
-#include <condition_variable>
+#include <atomic>
+#include <cstddef>
+#include <utility> // pair
 
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Version.h>
 
 using FairMQChannelMap = std::unordered_map<std::string, std::vector<FairMQChannel>>;
 
@@ -149,15 +146,6 @@ class FairMQDevice
         return GetChannel(channel, index).Receive(msg, rcvTimeoutInMs);
     }
 
-    int SendAsync(FairMQMessagePtr& msg, const std::string& channel, const int index = 0) __attribute__((deprecated("For non-blocking Send, use timeout version with timeout of 0: Send(msg, \"channelA\", subchannelIndex, timeout);")))
-    {
-        return GetChannel(channel, index).Send(msg, 0);
-    }
-    int ReceiveAsync(FairMQMessagePtr& msg, const std::string& channel, const int index = 0) __attribute__((deprecated("For non-blocking Receive, use timeout version with timeout of 0: Receive(msg, \"channelA\", subchannelIndex, timeout);")))
-    {
-        return GetChannel(channel, index).Receive(msg, 0);
-    }
-
     /// Shorthand method to send FairMQParts on `chan` at index `i`
     /// @param parts parts reference
     /// @param chan channel name
@@ -180,15 +168,6 @@ class FairMQDevice
         return GetChannel(channel, index).Receive(parts.fParts, rcvTimeoutInMs);
     }
 
-    int64_t SendAsync(FairMQParts& parts, const std::string& channel, const int index = 0) __attribute__((deprecated("For non-blocking Send, use timeout version with timeout of 0: Send(parts, \"channelA\", subchannelIndex, timeout);")))
-    {
-        return GetChannel(channel, index).Send(parts.fParts, 0);
-    }
-    int64_t ReceiveAsync(FairMQParts& parts, const std::string& channel, const int index = 0) __attribute__((deprecated("For non-blocking Receive, use timeout version with timeout of 0: Receive(parts, \"channelA\", subchannelIndex, timeout);")))
-    {
-        return GetChannel(channel, index).Receive(parts.fParts, 0);
-    }
-
     /// @brief Getter for default transport factory
     auto Transport() const -> FairMQTransportFactory*
     {
@@ -238,17 +217,47 @@ class FairMQDevice
     }
 
     // creates unamanaged region with the default device transport
-    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr)
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size,
+                                                FairMQRegionCallback callback = nullptr,
+                                                const std::string& path = "",
+                                                int flags = 0)
     {
-        return Transport()->CreateUnmanagedRegion(size, callback);
+        return Transport()->CreateUnmanagedRegion(size, callback, path, flags);
+    }
+
+    // creates unamanaged region with the default device transport
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size,
+                                                const int64_t userFlags,
+                                                FairMQRegionCallback callback = nullptr,
+                                                const std::string& path = "",
+                                                int flags = 0)
+    {
+        return Transport()->CreateUnmanagedRegion(size, userFlags, callback, path, flags);
     }
 
     // creates unmanaged region with the transport of the specified channel
-    FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel, int index, const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0)
+    FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel,
+                                                   int index,
+                                                   const size_t size,
+                                                   FairMQRegionCallback callback = nullptr,
+                                                   const std::string& path = "",
+                                                   int flags = 0)
     {
         return GetChannel(channel, index).NewUnmanagedRegion(size, callback, path, flags);
     }
 
+    // creates unmanaged region with the transport of the specified channel
+    FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel,
+                                                   int index,
+                                                   const size_t size,
+                                                   const int64_t userFlags,
+                                                   FairMQRegionCallback callback = nullptr,
+                                                   const std::string& path = "",
+                                                   int flags = 0)
+    {
+        return GetChannel(channel, index).NewUnmanagedRegion(size, userFlags, callback, path, flags);
+    }
+
     template<typename ...Ts>
     FairMQPollerPtr NewPoller(const Ts&... inputs)
     {
@@ -292,9 +301,6 @@ class FairMQDevice
         return channels.at(0)->Transport()->CreatePoller(channels);
     }
 
-    /// Waits for the first initialization run to finish
-    void WaitForInitialValidation() __attribute__((deprecated("This method will have no effect in future versions and will be removed. Instead subscribe for state changes and inspect configuration values."))) {}
-
     /// Adds a transport to the device if it doesn't exist
     /// @param transport  Transport string ("zeromq"/"nanomsg"/"shmem")
     std::shared_ptr<FairMQTransportFactory> AddTransport(const fair::mq::Transport transport);

fairmq/FairMQParts.h

@@ -33,7 +33,7 @@ class FairMQParts
     FairMQParts& operator=(const FairMQParts&) = delete;
     /// Constructor from argument pack of std::unique_ptr<FairMQMessage> rvalues
     template <typename... Ts>
-    FairMQParts(Ts&&... messages) : fParts() {AddPart(std::forward<Ts>(messages)...);}
+    FairMQParts(Ts&&... messages) : fParts() { AddPart(std::forward<Ts>(messages)...); }
     /// Default destructor
     ~FairMQParts() {};
 
@@ -63,10 +63,10 @@ class FairMQParts
     /// Add content of another object by move
     void AddPart(FairMQParts&& other)
     {
-      container parts = std::move(other.fParts);
-      for (auto& part : parts) {
-	fParts.push_back(std::move(part));
-      }
+        container parts = std::move(other.fParts);
+        for (auto& part : parts) {
+            fParts.push_back(std::move(part));
+        }
     }
 
     /// Get reference to part in the container at index (without bounds check)

fairmq/FairMQPoller.h

@@ -31,8 +31,8 @@ namespace fair
 namespace mq
 {
 
-using PollerPtr = std::unique_ptr<FairMQPoller>;
-
+using Poller = FairMQPoller;
+using PollerPtr = FairMQPollerPtr;
 struct PollerError : std::runtime_error { using std::runtime_error::runtime_error; };
 
 } /* namespace mq */

fairmq/FairMQSocket.h

@@ -22,7 +22,7 @@ class FairMQSocket
     FairMQSocket() {}
     FairMQSocket(FairMQTransportFactory* fac): fTransport(fac) {}
 
-    virtual std::string GetId() = 0;
+    virtual std::string GetId() const = 0;
 
     virtual bool Bind(const std::string& address) = 0;
     virtual bool Connect(const std::string& address) = 0;

fairmq/FairMQTransportFactory.cxx

@@ -7,8 +7,8 @@
  ********************************************************************************/
 
 #include <FairMQTransportFactory.h>
+#include <fairmq/shmem/TransportFactory.h>
 #include <zeromq/FairMQTransportFactoryZMQ.h>
-#include <shmem/FairMQTransportFactorySHM.h>
 #ifdef BUILD_NANOMSG_TRANSPORT
 #include <nanomsg/FairMQTransportFactoryNN.h>
 #endif /* BUILD_NANOMSG_TRANSPORT */
@@ -16,52 +16,47 @@
 #include <fairmq/ofi/TransportFactory.h>
 #endif
 #include <FairMQLogger.h>
-#include <fairmq/Tools.h>
-
+#include <fairmq/tools/Unique.h>
 #include <memory>
 #include <string>
-#include <sstream>
 
-FairMQTransportFactory::FairMQTransportFactory(const std::string& id)
+using namespace std;
+
+FairMQTransportFactory::FairMQTransportFactory(const string& id)
     : fkId(id)
-{
-}
+{}
 
-auto FairMQTransportFactory::CreateTransportFactory(const std::string& type, const std::string& id, const fair::mq::ProgOptions* config) -> std::shared_ptr<FairMQTransportFactory>
+auto FairMQTransportFactory::CreateTransportFactory(const string& type,
+                                                    const string& id,
+                                                    const fair::mq::ProgOptions* config)
+    -> shared_ptr<FairMQTransportFactory>
 {
-    using namespace std;
-
     auto finalId = id;
 
     // Generate uuid if empty
-    if (finalId == "")
-    {
+    if (finalId == "") {
         finalId = fair::mq::tools::Uuid();
     }
 
-    if (type == "zeromq")
-    {
+    if (type == "zeromq") {
         return make_shared<FairMQTransportFactoryZMQ>(finalId, config);
-    }
-    else if (type == "shmem")
-    {
-        return make_shared<FairMQTransportFactorySHM>(finalId, config);
+    } else if (type == "shmem") {
+        return make_shared<fair::mq::shmem::TransportFactory>(finalId, config);
     }
 #ifdef BUILD_NANOMSG_TRANSPORT
-    else if (type == "nanomsg")
-    {
+    else if (type == "nanomsg") {
         return make_shared<FairMQTransportFactoryNN>(finalId, config);
     }
 #endif /* BUILD_NANOMSG_TRANSPORT */
 #ifdef BUILD_OFI_TRANSPORT
-    else if (type == "ofi")
-    {
+    else if (type == "ofi") {
         return make_shared<fair::mq::ofi::TransportFactory>(finalId, config);
     }
 #endif /* BUILD_OFI_TRANSPORT */
-    else
-    {
-        LOG(error) << "Unavailable transport requested: " << "\"" << type << "\"" << ". Available are: "
+    else {
+        LOG(error) << "Unavailable transport requested: "
+                   << "\"" << type << "\""
+                   << ". Available are: "
                    << "\"zeromq\""
                    << "\"shmem\""
 #ifdef BUILD_NANOMSG_TRANSPORT

fairmq/FairMQTransportFactory.h

@@ -12,7 +12,6 @@
 #include <FairMQLogger.h>
 #include <FairMQMessage.h>
 #include <FairMQPoller.h>
-#include <fairmq/ProgOptionsFwd.h>
 #include <FairMQSocket.h>
 #include <FairMQUnmanagedRegion.h>
 #include <fairmq/MemoryResources.h>
@@ -22,8 +21,11 @@
 #include <memory>
 #include <vector>
 #include <unordered_map>
+#include <stdexcept>
+#include <cstddef> // size_t
 
 class FairMQChannel;
+namespace fair { namespace mq { class ProgOptions; } }
 
 class FairMQTransportFactory
 {
@@ -59,20 +61,46 @@ class FairMQTransportFactory
     /// @param obj optional helper pointer that can be used in the callback
     /// @return pointer to FairMQMessage
     virtual FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) = 0;
-
+    /// @brief create a message with the buffer located within the corresponding unmanaged region
+    /// @param unmanagedRegion the unmanaged region that this message buffer belongs to
+    /// @param data message buffer (must be within the region - checked at runtime by the transport)
+    /// @param size size of the message
+    /// @param hint optional parameter, returned to the user in the FairMQRegionCallback
     virtual FairMQMessagePtr CreateMessage(FairMQUnmanagedRegionPtr& unmanagedRegion, void* data, const size_t size, void* hint = 0) = 0;
 
-    /// Create a socket
+    /// @brief Create a socket
     virtual FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name) = 0;
 
-    /// Create a poller for a single channel (all subchannels)
+    /// @brief Create a poller for a single channel (all subchannels)
     virtual FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const = 0;
-    /// Create a poller for specific channels
+    /// @brief Create a poller for specific channels
     virtual FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel*>& channels) const = 0;
-    /// Create a poller for specific channels (all subchannels)
+    /// @brief Create a poller for specific channels (all subchannels)
     virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const = 0;
 
+    /// @brief Create new UnmanagedRegion
+    /// @param size size of the region
+    /// @param callback callback to be called when a message belonging to this region is no longer needed by the transport
+    /// @param path optional parameter to pass to the underlying transport
+    /// @param flags optional parameter to pass to the underlying transport
+    /// @return pointer to UnmanagedRegion
     virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const = 0;
+    /// @brief Create new UnmanagedRegion
+    /// @param size size of the region
+    /// @param userFlags flags to be stored with the region, have no effect on the transport, but can be retrieved from the region by the user
+    /// @param callback callback to be called when a message belonging to this region is no longer needed by the transport
+    /// @param path optional parameter to pass to the underlying transport
+    /// @param flags optional parameter to pass to the underlying transport
+    /// @return pointer to UnmanagedRegion
+    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const = 0;
+
+    /// @brief Subscribe to region events (creation, destruction, ...)
+    /// @param callback the callback that is called when a region event occurs
+    virtual void SubscribeToRegionEvents(FairMQRegionEventCallback callback) = 0;
+    /// @brief Unsubscribe from region events
+    virtual void UnsubscribeFromRegionEvents() = 0;
+
+    virtual std::vector<FairMQRegionInfo> GetRegionInfo() = 0;
 
     /// Get transport type
     virtual fair::mq::Transport GetType() const = 0;
@@ -135,6 +163,7 @@ namespace fair
 namespace mq
 {
 
+using TransportFactory = FairMQTransportFactory;
 struct TransportFactoryError : std::runtime_error { using std::runtime_error::runtime_error; };
 
 } /* namespace mq */

fairmq/FairMQUnmanagedRegion.h

@@ -12,8 +12,24 @@
 #include <cstddef> // size_t
 #include <memory> // std::unique_ptr
 #include <functional> // std::function
+#include <ostream> // std::ostream
+
+enum class FairMQRegionEvent : int
+{
+    created,
+    destroyed
+};
+
+struct FairMQRegionInfo {
+    uint64_t id;   // id of the region
+    void* ptr;     // pointer to the start of the region
+    size_t size;   // region size
+    int64_t flags; // custom flags set by the creator
+    FairMQRegionEvent event;
+};
 
 using FairMQRegionCallback = std::function<void(void*, size_t, void*)>;
+using FairMQRegionEventCallback = std::function<void(FairMQRegionInfo)>;
 
 class FairMQUnmanagedRegion
 {
@@ -26,12 +42,26 @@ class FairMQUnmanagedRegion
 
 using FairMQUnmanagedRegionPtr = std::unique_ptr<FairMQUnmanagedRegion>;
 
+inline std::ostream& operator<<(std::ostream& os, const FairMQRegionEvent& event)
+{
+    if (event == FairMQRegionEvent::created) {
+        return os << "created";
+    } else {
+        return os << "destroyed";
+    }
+}
+
 namespace fair
 {
 namespace mq
 {
 
-using UnmanagedRegionPtr = std::unique_ptr<FairMQUnmanagedRegion>;
+using RegionCallback = FairMQRegionCallback;
+using RegionEventCallback = FairMQRegionEventCallback;
+using RegionEvent = FairMQRegionEvent;
+using RegionInfo = FairMQRegionInfo;
+using UnmanagedRegion = FairMQUnmanagedRegion;
+using UnmanagedRegionPtr = FairMQUnmanagedRegionPtr;
 
 } /* namespace mq */
 } /* namespace fair */

fairmq/MemoryResourceTools.h

@@ -60,7 +60,7 @@ FairMQMessagePtr getMessage(ContainerT &&container_, FairMQMemoryResource *targe
         container.data(),
         containerSizeBytes);
     return message;
-};
+}
 
 } /* namespace mq */
 } /* namespace fair */

fairmq/MemoryResources.cxx

@@ -18,5 +18,5 @@
 void *fair::mq::ChannelResource::do_allocate(std::size_t bytes, std::size_t /*alignment*/)
 {
     return setMessage(factory->CreateMessage(bytes));
-};
+}
 

fairmq/MemoryResources.h

@@ -23,11 +23,7 @@ class FairMQTransportFactory;
 #include <boost/container/pmr/monotonic_buffer_resource.hpp>
 #include <boost/container/pmr/polymorphic_allocator.hpp>
 #include <cstring>
-#include <string>
-#include <type_traits>
-#include <unordered_map>
 #include <utility>
-#include <vector>
 
 namespace fair {
 namespace mq {

fairmq/Plugin.cxx

@@ -1,5 +1,5 @@
 /********************************************************************************
- *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ * Copyright (C) 2017-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -8,15 +8,20 @@
 
 #include <fairmq/Plugin.h>
 #include <FairMQLogger.h>
+#include <utility>
 
 using namespace std;
 
-fair::mq::Plugin::Plugin(const string name, const Version version, const string maintainer, const string homepage, PluginServices* pluginServices)
-    : fkName{name}
+fair::mq::Plugin::Plugin(string name,
+                         Version version,
+                         string maintainer,
+                         string homepage,
+                         PluginServices* pluginServices)
+    : fkName(std::move(name))
     , fkVersion(version)
-    , fkMaintainer{maintainer}
-    , fkHomepage{homepage}
-    , fPluginServices{pluginServices}
+    , fkMaintainer(std::move(maintainer))
+    , fkHomepage(std::move(homepage))
+    , fPluginServices(pluginServices)
 {
     LOG(debug) << "Loaded plugin: " << *this;
 }

fairmq/Plugin.h

@@ -1,5 +1,5 @@
 /********************************************************************************
- *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ * Copyright (C) 2017-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -9,7 +9,8 @@
 #ifndef FAIR_MQ_PLUGIN_H
 #define FAIR_MQ_PLUGIN_H
 
-#include <fairmq/Tools.h>
+#include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Version.h>
 #include <fairmq/PluginServices.h>
 
 #include <boost/dll/alias.hpp>
@@ -43,7 +44,11 @@ class Plugin
     using Version = tools::Version;
 
     Plugin() = delete;
-    Plugin(const std::string name, const Version version, const std::string maintainer, const std::string homepage, PluginServices* pluginServices);
+    Plugin(std::string name,
+           Version version,
+           std::string maintainer,
+           std::string homepage,
+           PluginServices* pluginServices);
 
     Plugin(const Plugin&) = delete;
     Plugin operator=(const Plugin&) = delete;
@@ -59,10 +64,10 @@ class Plugin
     friend auto operator!=(const Plugin& lhs, const Plugin& rhs) -> bool { return !(lhs == rhs); }
     friend auto operator<<(std::ostream& os, const Plugin& p) -> std::ostream&
     {
-        return os << "'" << p.GetName() << "', "
-                  << "version '" << p.GetVersion() << "', "
+        return os << "'"            << p.GetName()       << "', "
+                  << "version '"    << p.GetVersion()    << "', "
                   << "maintainer '" << p.GetMaintainer() << "', "
-                  << "homepage '" << p.GetHomepage() << "'";
+                  << "homepage '"   << p.GetHomepage()   << "'";
     }
     static auto NoProgramOptions() -> ProgOptions { return boost::none; }
 
@@ -79,7 +84,6 @@ class Plugin
     auto StealDeviceControl() -> void { fPluginServices->StealDeviceControl(fkName); };
     auto ReleaseDeviceControl() -> void { fPluginServices->ReleaseDeviceControl(fkName); };
     auto ChangeDeviceState(const DeviceStateTransition next) -> bool { return fPluginServices->ChangeDeviceState(fkName, next); }
-    void TransitionDeviceStateTo(const DeviceState state) { return fPluginServices->TransitionDeviceStateTo(fkName, state); }
     auto SubscribeToDeviceStateChange(std::function<void(DeviceState)> callback) -> void { fPluginServices->SubscribeToDeviceStateChange(fkName, callback); }
     auto UnsubscribeFromDeviceStateChange() -> void { fPluginServices->UnsubscribeFromDeviceStateChange(fkName); }
 

fairmq/PluginManager.cxx

@@ -8,7 +8,7 @@
 
 #include <fairmq/plugins/Builtin.h>
 #include <fairmq/PluginManager.h>
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Strings.h>
 #include <boost/program_options.hpp>
 #include <algorithm>
 #include <iterator>
@@ -28,6 +28,8 @@ using boost::optional;
 
 const std::string fair::mq::PluginManager::fgkLibPrefix = "FairMQPlugin_";
 
+std::vector<boost::dll::shared_library> fair::mq::PluginManager::fgDLLKeepAlive = std::vector<boost::dll::shared_library>();
+
 fair::mq::PluginManager::PluginManager()
     : fSearchPaths{}
     , fPluginFactories()

fairmq/PluginManager.h

@@ -11,8 +11,9 @@
 
 #include <fairmq/Plugin.h>
 #include <fairmq/PluginServices.h>
-#include <fairmq/Tools.h>
-#include <FairMQDevice.h>
+#include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Strings.h>
+
 #define BOOST_FILESYSTEM_VERSION 3
 #define BOOST_FILESYSTEM_NO_DEPRECATED
 #include <boost/filesystem.hpp>
@@ -21,13 +22,14 @@
 #include <boost/dll/import.hpp>
 #include <boost/dll/shared_library.hpp>
 #include <boost/dll/runtime_symbol_info.hpp>
+
 #include <functional>
 #include <map>
 #include <memory>
 #include <stdexcept>
 #include <string>
-#include <tuple>
 #include <vector>
+#include <utility> // forward
 
 namespace fair
 {
@@ -78,7 +80,7 @@ class PluginManager
     auto ForEachPluginProgOptions(std::function<void (boost::program_options::options_description)> func) const -> void { for(const auto& pair : fPluginProgOptions) { func(pair.second); } }
 
     template<typename... Args>
-    auto EmplacePluginServices(Args&&... args) -> void { fPluginServices = fair::mq::tools::make_unique<PluginServices>(std::forward<Args>(args)...); };
+    auto EmplacePluginServices(Args&&... args) -> void { fPluginServices = fair::mq::tools::make_unique<PluginServices>(std::forward<Args>(args)...); }
 
     auto WaitForPluginsToReleaseDeviceControl() -> void { fPluginServices->WaitForReleaseDeviceControl(); }
 
@@ -95,6 +97,7 @@ class PluginManager
         using fair::mq::tools::ToString;
 
         auto lib = shared_library{std::forward<Args>(args)...};
+        fgDLLKeepAlive.push_back(lib);
 
         fPluginFactories[pluginName] = import_alias<PluginFactory>(
             shared_library{lib},
@@ -115,6 +118,7 @@ class PluginManager
 
     static const std::string fgkLibPrefix;
     std::vector<boost::filesystem::path> fSearchPaths;
+    static std::vector<boost::dll::shared_library> fgDLLKeepAlive;
     std::map<std::string, std::function<PluginFactory>> fPluginFactories;
     std::unique_ptr<PluginServices> fPluginServices;
     std::map<std::string, std::unique_ptr<Plugin>> fPlugins;

fairmq/PluginServices.cxx

@@ -7,6 +7,7 @@
  ********************************************************************************/
 
 #include <fairmq/PluginServices.h>
+#include <fairmq/tools/Strings.h>
 
 using namespace fair::mq;
 using namespace std;
@@ -27,22 +28,6 @@ auto PluginServices::ChangeDeviceState(const string& controller, const DeviceSta
     }
 }
 
-void PluginServices::TransitionDeviceStateTo(const std::string& controller, DeviceState state)
-{
-    lock_guard<mutex> lock{fDeviceControllerMutex};
-
-    if (!fDeviceController) fDeviceController = controller;
-
-    if (fDeviceController == controller) {
-        fDevice.TransitionTo(state);
-    } else {
-        throw DeviceControlError{tools::ToString(
-            "Plugin '", controller, "' is not allowed to change device states. ",
-            "Currently, plugin '", *fDeviceController, "' has taken control."
-        )};
-    }
-}
-
 auto PluginServices::TakeDeviceControl(const string& controller) -> void
 {
     lock_guard<mutex> lock{fDeviceControllerMutex};

fairmq/PluginServices.h

@@ -9,7 +9,6 @@
 #ifndef FAIR_MQ_PLUGINSERVICES_H
 #define FAIR_MQ_PLUGINSERVICES_H
 
-#include <fairmq/Tools.h>
 #include <fairmq/States.h>
 #include <FairMQDevice.h>
 #include <fairmq/ProgOptions.h>
@@ -125,8 +124,6 @@ class PluginServices
     /// If the device control role has not been taken yet, calling this function will take over control implicitely.
     auto ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> bool;
 
-    void TransitionDeviceStateTo(const std::string& controller, DeviceState state);
-
     /// @brief Subscribe with a callback to device state changes
     /// @param subscriber id
     /// @param callback

fairmq/ProgOptions.cxx

@@ -15,7 +15,7 @@
 #include "FairMQLogger.h"
 #include <fairmq/ProgOptions.h>
 
-#include "tools/Unique.h"
+#include <fairmq/tools/Strings.h>
 
 #include <boost/any.hpp>
 #include <boost/regex.hpp>
@@ -62,7 +62,7 @@ ValInfo ConvertVarValToValInfo(const po::variable_value& v)
     } catch (out_of_range& oor) {
         return {string("[unidentified_type]"), string("[unidentified_type]"), origin};
     }
-};
+}
 
 string ConvertVarValToString(const po::variable_value& v)
 {
@@ -443,4 +443,4 @@ void ProgOptions::PrintOptionsRaw() const
 
 
 } // namespace mq
-} // namespace fair
+} // namespace fair

fairmq/ProgOptions.h

@@ -14,7 +14,7 @@
 #include <fairmq/EventManager.h>
 #include <fairmq/ProgOptionsFwd.h>
 #include <fairmq/Properties.h>
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Strings.h>
 
 #include <boost/program_options.hpp>
 

fairmq/StateMachine.cxx

@@ -79,7 +79,9 @@ struct Machine_ : public state_machine_def<Machine_>
 {
   public:
     Machine_()
-        : fLastTransitionResult(true)
+        : fState(State::Ok)
+        , fNewState(State::Ok)
+        , fLastTransitionResult(true)
         , fNewStatePending(false)
     {}
 

fairmq/Transports.h

@@ -9,7 +9,7 @@
 #ifndef FAIR_MQ_TRANSPORTS_H
 #define FAIR_MQ_TRANSPORTS_H
 
-#include <fairmq/Tools.h>
+#include <fairmq/tools/CppSTL.h>
 
 #include <memory>
 #include <string>

fairmq/Version.h.in

@@ -1,5 +1,5 @@
 /********************************************************************************
- *    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ * Copyright (C) 2018-2020 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -21,7 +21,7 @@
 #define FAIRMQ_GIT_DATE "@PROJECT_GIT_DATE@"
 #define FAIRMQ_REPO_URL "https://github.com/FairRootGroup/FairMQ"
 #define FAIRMQ_LICENSE "LGPL-3.0"
-#define FAIRMQ_COPYRIGHT "2012-2019 GSI"
+#define FAIRMQ_COPYRIGHT "2012-2020 GSI"
 #define FAIRMQ_BUILD_TYPE "@CMAKE_BUILD_TYPE@"
 
 #endif // FAIR_MQ_VERSION_H

fairmq/devices/FairMQBenchmarkSampler.cxx

@@ -8,10 +8,9 @@
 
 #include "FairMQBenchmarkSampler.h"
 
-#include <fairmq/Tools.h>
+#include "tools/RateLimit.h"
 #include "../FairMQLogger.h"
 
-#include <vector>
 #include <chrono>
 
 using namespace std;
@@ -27,10 +26,6 @@ FairMQBenchmarkSampler::FairMQBenchmarkSampler()
 {
 }
 
-FairMQBenchmarkSampler::~FairMQBenchmarkSampler()
-{
-}
-
 void FairMQBenchmarkSampler::InitTask()
 {
     fMultipart = fConfig->GetProperty<bool>("multipart");

fairmq/devices/FairMQBenchmarkSampler.h

@@ -10,8 +10,10 @@
 #define FAIRMQBENCHMARKSAMPLER_H_
 
 #include <string>
-#include <thread>
 #include <atomic>
+#include <cstddef> // size_t
+#include <cstdint> // uint64_t
+
 
 #include "FairMQDevice.h"
 
@@ -23,7 +25,7 @@ class FairMQBenchmarkSampler : public FairMQDevice
 {
   public:
     FairMQBenchmarkSampler();
-    virtual ~FairMQBenchmarkSampler();
+    virtual ~FairMQBenchmarkSampler() {}
 
   protected:
     bool fMultipart;

fairmq/nanomsg/FairMQPollerNN.h

@@ -17,7 +17,6 @@
 
 #include <vector>
 #include <unordered_map>
-#include <initializer_list>
 
 #include "FairMQPoller.h"
 #include "FairMQChannel.h"

fairmq/nanomsg/FairMQSocketNN.cxx

@@ -91,11 +91,6 @@ FairMQSocketNN::FairMQSocketNN(const string& type, const string& name, const str
     LOG(debug) << "Created socket " << GetId();
 }
 
-string FairMQSocketNN::GetId()
-{
-    return fId;
-}
-
 bool FairMQSocketNN::Bind(const string& address)
 {
     // LOG(info) << "bind socket " << fId << " on " << address;

fairmq/nanomsg/FairMQSocketNN.h

@@ -23,7 +23,7 @@ class FairMQSocketNN final : public FairMQSocket
     FairMQSocketNN(const FairMQSocketNN&) = delete;
     FairMQSocketNN operator=(const FairMQSocketNN&) = delete;
 
-    std::string GetId() override;
+    std::string GetId() const override { return fId; }
 
     bool Bind(const std::string& address) override;
     bool Connect(const std::string& address) override;

fairmq/nanomsg/FairMQTransportFactoryNN.cxx

@@ -70,6 +70,11 @@ FairMQUnmanagedRegionPtr FairMQTransportFactoryNN::CreateUnmanagedRegion(const s
     return unique_ptr<FairMQUnmanagedRegion>(new FairMQUnmanagedRegionNN(size, callback, path, flags));
 }
 
+FairMQUnmanagedRegionPtr FairMQTransportFactoryNN::CreateUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */) const
+{
+    return unique_ptr<FairMQUnmanagedRegion>(new FairMQUnmanagedRegionNN(size, userFlags, callback, path, flags));
+}
+
 fair::mq::Transport FairMQTransportFactoryNN::GetType() const
 {
     return fTransportType;

fairmq/nanomsg/FairMQTransportFactoryNN.h

@@ -37,6 +37,11 @@ class FairMQTransportFactoryNN final : public FairMQTransportFactory
     FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const override;
 
     FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback, const std::string& path = "", int flags = 0) const override;
+    FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const override;
+
+    void SubscribeToRegionEvents(FairMQRegionEventCallback /* callback */) override { LOG(error) << "SubscribeToRegionEvents not yet implemented for nanomsg"; }
+    void UnsubscribeFromRegionEvents() override { LOG(error) << "UnsubscribeFromRegionEvents not yet implemented for nanomsg"; }
+    std::vector<FairMQRegionInfo> GetRegionInfo() override { LOG(error) << "GetRegionInfo not yet implemented for nanomsg, returning empty vector"; return std::vector<FairMQRegionInfo>(); }
 
     fair::mq::Transport GetType() const override;
 

fairmq/nanomsg/FairMQUnmanagedRegionNN.cxx

@@ -18,6 +18,13 @@ FairMQUnmanagedRegionNN::FairMQUnmanagedRegionNN(const size_t size, FairMQRegion
 {
 }
 
+FairMQUnmanagedRegionNN::FairMQUnmanagedRegionNN(const size_t size, const int64_t /*userFlags*/, FairMQRegionCallback callback, const std::string& /*path = "" */, int /*flags = 0 */)
+    : fBuffer(malloc(size))
+    , fSize(size)
+    , fCallback(callback)
+{
+}
+
 void* FairMQUnmanagedRegionNN::GetData() const
 {
     return fBuffer;

fairmq/nanomsg/FairMQUnmanagedRegionNN.h

@@ -20,6 +20,8 @@ class FairMQUnmanagedRegionNN final : public FairMQUnmanagedRegion
 
   public:
     FairMQUnmanagedRegionNN(const size_t size, FairMQRegionCallback callback, const std::string& path = "", int flags = 0);
+    FairMQUnmanagedRegionNN(const size_t size, const int64_t userFlags, FairMQRegionCallback callback, const std::string& path = "", int flags = 0);
+
     FairMQUnmanagedRegionNN(const FairMQUnmanagedRegionNN&) = delete;
     FairMQUnmanagedRegionNN operator=(const FairMQUnmanagedRegionNN&) = delete;
 
@@ -34,4 +36,4 @@ class FairMQUnmanagedRegionNN final : public FairMQUnmanagedRegion
     FairMQRegionCallback fCallback;
 };
 
-#endif /* FAIRMQUNMANAGEDREGIONNN_H_ */
+#endif /* FAIRMQUNMANAGEDREGIONNN_H_ */

fairmq/ofi/Socket.h

@@ -43,7 +43,7 @@ class Socket final : public fair::mq::Socket
     Socket(const Socket&) = delete;
     Socket operator=(const Socket&) = delete;
 
-    auto GetId() -> std::string { return fId; }
+    auto GetId() const -> std::string override { return fId; }
 
     auto Bind(const std::string& address) -> bool override;
     auto Connect(const std::string& address) -> bool override;

fairmq/ofi/TransportFactory.cxx

@@ -90,6 +90,11 @@ auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, FairMQRegion
     throw runtime_error{"Not yet implemented UMR."};
 }
 
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, const int64_t /*userFlags*/, FairMQRegionCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) const -> UnmanagedRegionPtr
+{
+    throw runtime_error{"Not yet implemented UMR."};
+}
+
 auto TransportFactory::GetType() const -> Transport
 {
     return Transport::OFI;

fairmq/ofi/TransportFactory.h

@@ -46,7 +46,12 @@ class TransportFactory final : public FairMQTransportFactory
     auto CreatePoller(const std::vector<FairMQChannel*>& channels) const -> PollerPtr override;
     auto CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const -> PollerPtr override;
 
-    auto CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const -> UnmanagedRegionPtr override;
+    auto CreateUnmanagedRegion(const size_t size, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const -> UnmanagedRegionPtr override;
+    auto CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const -> UnmanagedRegionPtr override;
+
+    void SubscribeToRegionEvents(RegionEventCallback /* callback */) override { LOG(error) << "SubscribeToRegionEvents not yet implemented for OFI"; }
+    void UnsubscribeFromRegionEvents() override { LOG(error) << "UnsubscribeFromRegionEvents not yet implemented for OFI"; }
+    std::vector<FairMQRegionInfo> GetRegionInfo() override { LOG(error) << "GetRegionInfo not yet implemented for OFI, returning empty vector"; return std::vector<FairMQRegionInfo>(); }
 
     auto GetType() const -> Transport override;
 

fairmq/plugins/Control.cxx

@@ -427,6 +427,9 @@ auto Control::RunShutdownSequence() -> void
             case DeviceState::Idle:
                 ChangeDeviceState(DeviceStateTransition::End);
                 break;
+            case DeviceState::InitializingDevice:
+                ChangeDeviceState(DeviceStateTransition::CompleteInit);
+                break;
             case DeviceState::Initialized:
             case DeviceState::Bound:
             case DeviceState::DeviceReady:

fairmq/plugins/DDS/CMakeLists.txt

@@ -8,7 +8,7 @@
 
 set(plugin FairMQPlugin_dds)
 add_library(${plugin} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/DDS.cxx ${CMAKE_CURRENT_SOURCE_DIR}/DDS.h)
-target_link_libraries(${plugin} FairMQ DDS::dds_intercom_lib DDS::dds_protocol_lib)
+target_link_libraries(${plugin} PUBLIC FairMQ StateMachine DDS::dds_intercom_lib DDS::dds_protocol_lib Boost::boost PRIVATE Commands)
 target_include_directories(${plugin} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 set_target_properties(${plugin} PROPERTIES CXX_VISIBILITY_PRESET hidden)
 set_target_properties(${plugin} PROPERTIES
@@ -19,7 +19,7 @@ set_target_properties(${plugin} PROPERTIES
 
 set(exe fairmq-dds-command-ui)
 add_executable(${exe} ${CMAKE_CURRENT_SOURCE_DIR}/runDDSCommandUI.cxx)
-target_link_libraries(${exe} FairMQ DDS::dds_intercom_lib DDS::dds_protocol_lib)
+target_link_libraries(${exe} FairMQ Commands SDK StateMachine)
 target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 
 install(TARGETS ${plugin} ${exe}

fairmq/plugins/DDS/DDS.cxx

@@ -8,18 +8,16 @@
 
 #include "DDS.h"
 
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Strings.h>
 
 #include <boost/algorithm/string/join.hpp>
 #include <boost/algorithm/string/split.hpp>
 #include <boost/algorithm/string/classification.hpp>
+#include <boost/asio/post.hpp>
 
-#include <termios.h> // for the interactive mode
-#include <poll.h> // for the interactive mode
-#include <sstream>
-#include <iostream>
 #include <cstdlib>
 #include <stdexcept>
+#include <sstream>
 
 using namespace std;
 using fair::mq::tools::ToString;
@@ -37,94 +35,107 @@ DDS::DDS(const string& name,
          const string& homepage,
          PluginServices* pluginServices)
     : Plugin(name, version, maintainer, homepage, pluginServices)
-    , fTransitions({"INIT DEVICE",
-                    "COMPLETE INIT",
-                    "BIND",
-                    "CONNECT",
-                    "INIT TASK",
-                    "RUN",
-                    "STOP",
-                    "RESET TASK",
-                    "RESET DEVICE",
-                    "END"})
+    , fDDSTaskId(dds::env_prop<dds::task_id>())
     , fCurrentState(DeviceState::Idle)
     , fLastState(DeviceState::Idle)
     , fDeviceTerminationRequested(false)
     , fLastExternalController(0)
     , fExitingAckedByLastExternalController(false)
-    , fHeartbeatInterval(100)
     , fUpdatesAllowed(false)
+    , fWorkGuard(fWorkerQueue.get_executor())
 {
     try {
         TakeDeviceControl();
 
-        fHeartbeatThread = thread(&DDS::HeartbeatSender, this);
-
-        std::string deviceId(GetProperty<std::string>("id"));
+        string deviceId(GetProperty<string>("id"));
         if (deviceId.empty()) {
-            SetProperty<std::string>("id", dds::env_prop<dds::task_path>());
+            SetProperty<string>("id", dds::env_prop<dds::task_path>());
         }
-        std::string sessionId(GetProperty<std::string>("session"));
+        string sessionId(GetProperty<string>("session"));
         if (sessionId == "default") {
-            SetProperty<std::string>("session", dds::env_prop<dds::dds_session_id>());
+            SetProperty<string>("session", dds::env_prop<dds::dds_session_id>());
         }
 
         auto control = GetProperty<string>("control");
-        bool staticMode(false);
         if (control == "static") {
-            LOG(debug) << "Running DDS controller: static";
-            staticMode = true;
+            LOG(error) << "DDS Plugin: static mode is not supported";
+            throw invalid_argument("DDS Plugin: static mode is not supported");
         } else if (control == "dynamic" || control == "external" || control == "interactive") {
             LOG(debug) << "Running DDS controller: external";
         } else {
-            LOG(error) << "Unrecognized control mode '" << control << "' requested. " << "Ignoring and falling back to static control mode.";
-            staticMode = true;
+            LOG(error) << "Unrecognized control mode '" << control << "' requested. " << "Ignoring and starting in external control mode.";
         }
 
         SubscribeForCustomCommands();
         SubscribeForConnectingChannels();
-        fDDS.Start();
 
         // subscribe to device state changes, pushing new state changes into the event queue
         SubscribeToDeviceStateChange([&](DeviceState newState) {
-            fStateQueue.Push(newState);
             switch (newState) {
-                case DeviceState::Bound:
+                case DeviceState::Bound: {
                     // Receive addresses of connecting channels from DDS
                     // and propagate addresses of bound channels to DDS.
                     FillChannelContainers();
 
+                    // allow updates from key value after channel containers are filled
+                    {
+                        lock_guard<mutex> lk(fUpdateMutex);
+                        fUpdatesAllowed = true;
+                    }
+                    fUpdateCondition.notify_one();
+
                     // publish bound addresses via DDS at keys corresponding to the channel
                     // prefixes, e.g. 'data' in data[i]
                     PublishBoundChannels();
-                    break;
+                } break;
                 case DeviceState::ResettingDevice: {
-                    std::lock_guard<std::mutex> lk(fUpdateMutex);
-                    fUpdatesAllowed = false;
-                    break;
-                }
-                case DeviceState::Exiting:
+                    {
+                        lock_guard<mutex> lk(fUpdateMutex);
+                        fUpdatesAllowed = false;
+                    }
+
+                    EmptyChannelContainers();
+                } break;
+                case DeviceState::Exiting: {
+                    if (!fControllerThread.joinable()) {
+                        fControllerThread = thread(&DDS::WaitForExitingAck, this);
+                    }
+                    fWorkGuard.reset();
                     fDeviceTerminationRequested = true;
                     UnsubscribeFromDeviceStateChange();
                     ReleaseDeviceControl();
-                    break;
+                } break;
                 default:
                     break;
             }
 
-            lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+            using namespace sdk::cmd;
+            auto now = chrono::steady_clock::now();
             string id = GetProperty<string>("id");
             fLastState = fCurrentState;
             fCurrentState = newState;
-            for (auto subscriberId : fStateChangeSubscribers) {
-                LOG(debug) << "Publishing state-change: " << fLastState << "->" << newState << " to " << subscriberId;
-                fDDS.Send("state-change: " + id + "," + ToString(dds::env_prop<dds::task_id>()) + "," + ToStr(fLastState) + "->" + ToStr(newState), to_string(subscriberId));
+
+            lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+            for (auto it = fStateChangeSubscribers.cbegin(); it != fStateChangeSubscribers.end();) {
+                // if a subscriber did not send a heartbeat in more than 3 times the promised interval,
+                // remove it from the subscriber list
+                if (chrono::duration<double>(now - it->second.first).count() > 3 * it->second.second) {
+                    LOG(warn) << "Controller '" << it->first
+                              << "' did not send heartbeats since over 3 intervals ("
+                              << 3 * it->second.second << " ms), removing it.";
+                    fStateChangeSubscribers.erase(it++);
+                } else {
+                    LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState << " to " << it->first;
+                    Cmds cmds(make<StateChange>(id, fDDSTaskId, fLastState, fCurrentState));
+                    fDDS.Send(cmds.Serialize(), to_string(it->first));
+                    ++it;
+                }
             }
         });
 
-        if (staticMode) {
-            fControllerThread = thread(&DDS::StaticControl, this);
-        }
+        StartWorkerThread();
+
+        fDDS.Start();
     } catch (PluginServices::DeviceControlError& e) {
         LOG(debug) << e.what();
     } catch (exception& e) {
@@ -132,33 +143,26 @@ DDS::DDS(const string& name,
     }
 }
 
+void DDS::EmptyChannelContainers()
+{
+    fBindingChans.clear();
+    fConnectingChans.clear();
+}
+
+auto DDS::StartWorkerThread() -> void
+{
+    fWorkerThread = thread([this]() {
+        fWorkerQueue.run();
+    });
+}
+
 auto DDS::WaitForExitingAck() -> void
 {
     unique_lock<mutex> lock(fStateChangeSubscriberMutex);
-    fExitingAcked.wait_for(
-        lock,
-        chrono::milliseconds(GetProperty<unsigned int>("wait-for-exiting-ack-timeout")),
-        [this]() { return fExitingAckedByLastExternalController; });
-}
-
-auto DDS::StaticControl() -> void
-{
-    try {
-        TransitionDeviceStateTo(DeviceState::Running);
-
-        // wait until stop signal
-        unique_lock<mutex> lock(fStopMutex);
-        while (!fDeviceTerminationRequested) {
-            fStopCondition.wait_for(lock, chrono::seconds(1));
-        }
-        LOG(debug) << "Stopping DDS plugin static controller";
-    } catch (DeviceErrorState&) {
-        ReleaseDeviceControl();
-    } catch (exception& e) {
-        ReleaseDeviceControl();
-        LOG(error) << "Error: " << e.what() << endl;
-        return;
-    }
+    auto timeout = GetProperty<unsigned int>("wait-for-exiting-ack-timeout");
+    fExitingAcked.wait_for(lock, chrono::milliseconds(timeout), [this]() {
+        return fExitingAckedByLastExternalController || fStateChangeSubscribers.empty();
+    });
 }
 
 auto DDS::FillChannelContainers() -> void
@@ -177,9 +181,7 @@ auto DDS::FillChannelContainers() -> void
             } else if (GetProperty<string>(methodKey) == "connect") {
                 fConnectingChans.insert(make_pair(c.first, DDSConfig()));
                 LOG(debug) << "preparing to connect: " << c.first << " with " << c.second << " sub-channels.";
-                for (int i = 0; i < c.second; ++i) {
-                    fConnectingChans.at(c.first).fSubChannelAddresses.push_back(string());
-                }
+                fConnectingChans.at(c.first).fNumSubChannels = c.second;
             } else {
                 LOG(error) << "Cannot update address configuration. Channel method (bind/connect) not specified.";
                 return;
@@ -226,11 +228,6 @@ auto DDS::FillChannelContainers() -> void
             LOG(debug) << "dds-i-n: adding " << chanName << " -> i: " << i << " n: " << n;
             fIofN.insert(make_pair(chanName, IofN(i, n)));
         }
-        {
-            std::lock_guard<std::mutex> lk(fUpdateMutex);
-            fUpdatesAllowed = true;
-        }
-        fUpdateCondition.notify_one();
     } catch (const exception& e) {
         LOG(error) << "Error filling channel containers: " << e.what();
     }
@@ -240,60 +237,73 @@ auto DDS::SubscribeForConnectingChannels() -> void
 {
     LOG(debug) << "Subscribing for DDS properties.";
 
-    fDDS.SubscribeKeyValue([&] (const string& propertyId, const string& value, uint64_t senderTaskID) {
-        try {
-            LOG(debug) << "Received update for " << propertyId << ": value=" << value << ", senderTaskID=" << senderTaskID;
+    fDDS.SubscribeKeyValue([&] (const string& key, const string& value, uint64_t senderTaskID) {
+        LOG(debug) << "Received property: key=" << key << ", value=" << value << ", senderTaskID=" << senderTaskID;
 
-            std::unique_lock<std::mutex> lk(fUpdateMutex);
-            fUpdateCondition.wait(lk, [&]{ return fUpdatesAllowed; });
+        if (key.compare(0, 8, "fmqchan_") != 0) {
+            LOG(debug) << "property update is not a channel info update: " << key;
+            return;
+        }
+        string channelName = key.substr(8);
+        LOG(info) << "Update for channel name: " << channelName;
 
-            string val = value;
-            // check if it is to handle as one out of multiple values
-            auto it = fIofN.find(propertyId);
-            if (it != fIofN.end()) {
-                it->second.fEntries.push_back(value);
-                if (it->second.fEntries.size() == it->second.fN) {
-                    sort(it->second.fEntries.begin(), it->second.fEntries.end());
-                    val = it->second.fEntries.at(it->second.fI);
-                } else {
-                    LOG(debug) << "received " << it->second.fEntries.size() << " values for " << propertyId << ", expecting total of " << it->second.fN;
+        boost::asio::post(fWorkerQueue, [=]() {
+            try {
+                {
+                    unique_lock<mutex> lk(fUpdateMutex);
+                    fUpdateCondition.wait(lk, [&]{ return fUpdatesAllowed; });
+                }
+
+                if (fConnectingChans.find(channelName) == fConnectingChans.end()) {
+                    LOG(error) << "Received an update for a connecting channel, but either no channel with given channel name exists or it has already been configured: '" << channelName << "', ignoring...";
                     return;
                 }
-            }
 
-            vector<string> connectionStrings;
-            boost::algorithm::split(connectionStrings, val, boost::algorithm::is_any_of(","));
-            if (connectionStrings.size() > 1) { // multiple bound channels received
-                auto it2 = fI.find(propertyId);
-                if (it2 != fI.end()) {
-                    LOG(debug) << "adding connecting channel " << propertyId << " : " << connectionStrings.at(it2->second);
-                    fConnectingChans.at(propertyId).fDDSValues.insert({senderTaskID, connectionStrings.at(it2->second).c_str()});
-                } else {
-                    LOG(error) << "multiple bound channels received, but no task index specified, only assigning the first";
-                    fConnectingChans.at(propertyId).fDDSValues.insert({senderTaskID, connectionStrings.at(0).c_str()});
-                }
-            } else { // only one bound channel received
-                fConnectingChans.at(propertyId).fDDSValues.insert({senderTaskID, val.c_str()});
-            }
-
-            // update channels and remove them from unfinished container
-            for (auto mi = fConnectingChans.begin(); mi != fConnectingChans.end(); /* no increment */) {
-                if (mi->second.fSubChannelAddresses.size() == mi->second.fDDSValues.size()) {
-                    // when multiple subChannels are used, their order on every device should be the same, irregardless of arrival order from DDS.
-                    sort(mi->second.fSubChannelAddresses.begin(), mi->second.fSubChannelAddresses.end());
-                    auto it3 = mi->second.fDDSValues.begin();
-                    for (unsigned int i = 0; i < mi->second.fSubChannelAddresses.size(); ++i) {
-                        SetProperty<string>(string{"chans." + mi->first + "." + to_string(i) + ".address"}, it3->second);
-                        ++it3;
+                string val = value;
+                // check if it is to handle as one out of multiple values
+                auto it = fIofN.find(channelName);
+                if (it != fIofN.end()) {
+                    it->second.fEntries.push_back(value);
+                    if (it->second.fEntries.size() == it->second.fN) {
+                        sort(it->second.fEntries.begin(), it->second.fEntries.end());
+                        val = it->second.fEntries.at(it->second.fI);
+                    } else {
+                        LOG(debug) << "received " << it->second.fEntries.size() << " values for " << channelName << ", expecting total of " << it->second.fN;
+                        return;
                     }
-                    fConnectingChans.erase(mi++);
-                } else {
-                    ++mi;
                 }
+
+                vector<string> connectionStrings;
+                boost::algorithm::split(connectionStrings, val, boost::algorithm::is_any_of(","));
+                if (connectionStrings.size() > 1) { // multiple bound channels received
+                    auto it2 = fI.find(channelName);
+                    if (it2 != fI.end()) {
+                        LOG(debug) << "adding connecting channel " << channelName << " : " << connectionStrings.at(it2->second);
+                        fConnectingChans.at(channelName).fDDSValues.insert({senderTaskID, connectionStrings.at(it2->second).c_str()});
+                    } else {
+                        LOG(error) << "multiple bound channels received, but no task index specified, only assigning the first";
+                        fConnectingChans.at(channelName).fDDSValues.insert({senderTaskID, connectionStrings.at(0).c_str()});
+                    }
+                } else { // only one bound channel received
+                    fConnectingChans.at(channelName).fDDSValues.insert({senderTaskID, val.c_str()});
+                }
+
+                for (const auto& mi : fConnectingChans) {
+                    if (mi.second.fNumSubChannels == mi.second.fDDSValues.size()) {
+                        int i = 0;
+                        for (const auto& e : mi.second.fDDSValues) {
+                            auto result = UpdateProperty<string>(string{"chans." + mi.first + "." + to_string(i) + ".address"}, e.second);
+                            if (!result) {
+                                LOG(error) << "UpdateProperty failed for: " << "chans." << mi.first << "." << to_string(i) << ".address" << " - property does not exist";
+                            }
+                            ++i;
+                        }
+                    }
+                }
+            } catch (const exception& e) {
+                LOG(error) << "Error handling DDS property: key=" << key << ", value=" << value << ", senderTaskID=" << senderTaskID << ": " << e.what();
             }
-        } catch (const exception& e) {
-            LOG(error) << "Error on handling DDS property update for " << propertyId << ": value=" << value << ", senderTaskID=" << senderTaskID << ": " << e.what();
-        }
+        });
     });
 }
 
@@ -301,25 +311,8 @@ auto DDS::PublishBoundChannels() -> void
 {
     for (const auto& chan : fBindingChans) {
         string joined = boost::algorithm::join(chan.second, ",");
-        LOG(debug) << "Publishing " << chan.first << " bound addresses (" << chan.second.size() << ") to DDS under '" << chan.first << "' property name.";
-        fDDS.PutValue(chan.first, joined);
-    }
-}
-
-auto DDS::HeartbeatSender() -> void
-{
-    string id = GetProperty<string>("id");
-
-    while (!fDeviceTerminationRequested) {
-        {
-            lock_guard<mutex> lock{fHeartbeatSubscriberMutex};
-
-            for (const auto subscriberId : fHeartbeatSubscribers) {
-                fDDS.Send("heartbeat: " + id , to_string(subscriberId));
-            }
-        }
-
-        this_thread::sleep_for(chrono::milliseconds(fHeartbeatInterval));
+        LOG(debug) << "Publishing bound addresses (" << chan.second.size() << ") of channel '" << chan.first << "' to DDS under '" << "fmqchan_" + chan.first << "' property name.";
+        fDDS.PutValue("fmqchan_" + chan.first, joined);
     }
 }
 
@@ -329,45 +322,48 @@ auto DDS::SubscribeForCustomCommands() -> void
 
     string id = GetProperty<string>("id");
 
-    fDDS.SubscribeCustomCmd([id, this](const string& cmd, const string& cond, uint64_t senderId) {
-        LOG(info) << "Received command: '" << cmd << "' from " << senderId;
+    fDDS.SubscribeCustomCmd([id, this](const string& cmdStr, const string& cond, uint64_t senderId) {
+        // LOG(info) << "Received command: '" << cmdStr << "' from " << senderId;
+        sdk::cmd::Cmds inCmds;
+        inCmds.Deserialize(cmdStr);
+        for (const auto& cmd : inCmds) {
+            HandleCmd(id, *cmd, cond, senderId);
+        }
+    });
+}
 
-        if (cmd == "check-state") {
-            fDDS.Send(id + ": " + ToStr(GetCurrentDeviceState()), to_string(senderId));
-        } else if (fTransitions.find(cmd) != fTransitions.end()) {
-            if (ChangeDeviceState(ToDeviceStateTransition(cmd))) {
-                fDDS.Send(id + ": queued, " + cmd, to_string(senderId));
+auto DDS::HandleCmd(const string& id, sdk::cmd::Cmd& cmd, const string& cond, uint64_t senderId) -> void
+{
+    using namespace fair::mq::sdk;
+    using namespace fair::mq::sdk::cmd;
+    // LOG(info) << "Received command type: '" << cmd.GetType() << "' from " << senderId;
+    switch (cmd.GetType()) {
+        case Type::check_state: {
+            fDDS.Send(Cmds(make<CurrentState>(id, GetCurrentDeviceState())).Serialize(), to_string(senderId));
+        } break;
+        case Type::change_state: {
+            Transition transition = static_cast<ChangeState&>(cmd).GetTransition();
+            if (ChangeDeviceState(transition)) {
+                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Ok, transition));
+                fDDS.Send(outCmds.Serialize(), to_string(senderId));
             } else {
-                fDDS.Send(id + ": could not queue, " + cmd, to_string(senderId));
-            }
-            if (cmd == "END" && ToStr(GetCurrentDeviceState()) == "EXITING") {
-                unique_lock<mutex> lock(fStopMutex);
-                fStopCondition.notify_one();
+                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Failure, transition));
+                fDDS.Send(outCmds.Serialize(), to_string(senderId));
             }
             {
                 lock_guard<mutex> lock{fStateChangeSubscriberMutex};
                 fLastExternalController = senderId;
             }
-        } else if (cmd == "dump-config") {
+        } break;
+        case Type::dump_config: {
             stringstream ss;
-            for (const auto pKey: GetPropertyKeys()) {
-                ss << id << ": " << pKey << " -> " << GetPropertyAsString(pKey) << endl;
+            for (const auto pKey : GetPropertyKeys()) {
+                ss << id << ": " << pKey << " -> " << GetPropertyAsString(pKey) << "\n";
             }
-            fDDS.Send(ss.str(), to_string(senderId));
-        } else if (cmd == "subscribe-to-heartbeats") {
-            {
-                // auto size = fHeartbeatSubscribers.size();
-                lock_guard<mutex> lock{fHeartbeatSubscriberMutex};
-                fHeartbeatSubscribers.insert(senderId);
-            }
-            fDDS.Send("heartbeat-subscription: " + id + ",OK", to_string(senderId));
-        } else if (cmd == "unsubscribe-from-heartbeats") {
-            {
-                lock_guard<mutex> lock{fHeartbeatSubscriberMutex};
-                fHeartbeatSubscribers.erase(senderId);
-            }
-            fDDS.Send("heartbeat-unsubscription: " + id + ",OK", to_string(senderId));
-        } else if (cmd == "state-change-exiting-received") {
+            Cmds outCmds(make<Config>(id, ss.str()));
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        case Type::state_change_exiting_received: {
             {
                 lock_guard<mutex> lock{fStateChangeSubscriberMutex};
                 if (fLastExternalController == senderId) {
@@ -375,38 +371,76 @@ auto DDS::SubscribeForCustomCommands() -> void
                 }
             }
             fExitingAcked.notify_one();
-        } else if (cmd == "subscribe-to-state-changes") {
-            {
-                // auto size = fStateChangeSubscribers.size();
+        } break;
+        case Type::subscribe_to_state_change: {
+            auto _cmd = static_cast<cmd::SubscribeToStateChange&>(cmd);
+            lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+            fStateChangeSubscribers.emplace(senderId, make_pair(chrono::steady_clock::now(), _cmd.GetInterval()));
+
+            LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState << " to " << senderId;
+
+            Cmds outCmds(make<StateChangeSubscription>(id, fDDSTaskId, Result::Ok),
+                         make<StateChange>(id, fDDSTaskId, fLastState, fCurrentState));
+
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        case Type::subscription_heartbeat: {
+            try {
+                auto _cmd = static_cast<cmd::SubscriptionHeartbeat&>(cmd);
                 lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                fStateChangeSubscribers.insert(senderId);
-                if (!fControllerThread.joinable()) {
-                    fControllerThread = thread(&DDS::WaitForExitingAck, this);
-                }
+                fStateChangeSubscribers.at(senderId) = make_pair(chrono::steady_clock::now(), _cmd.GetInterval());
+            } catch(out_of_range& oor) {
+                LOG(warn) << "Received subscription heartbeat from an unknown controller with id '" << senderId << "'";
             }
-            fDDS.Send("state-changes-subscription: " + id + ",OK", to_string(senderId));
-            {
-                lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState << " to " << senderId;
-                // fDDSCustomCmd.send("state-change: " + id + "," + ToStr(fLastState) + "->" + ToStr(fCurrentState), to_string(senderId));
-                fDDS.Send("state-change: " + id + "," + ToString(dds::env_prop<dds::task_id>()) + "," + ToStr(fLastState) + "->" + ToStr(fCurrentState), to_string(senderId));
-            }
-        } else if (cmd == "unsubscribe-from-state-changes") {
+        } break;
+        case Type::unsubscribe_from_state_change: {
             {
                 lock_guard<mutex> lock{fStateChangeSubscriberMutex};
                 fStateChangeSubscribers.erase(senderId);
             }
-            fDDS.Send("state-changes-unsubscription: " + id + ",OK", to_string(senderId));
-        } else if (cmd == "SHUTDOWN") {
-            TransitionDeviceStateTo(DeviceState::Exiting);
-        } else if (cmd == "STARTUP") {
-            TransitionDeviceStateTo(DeviceState::Running);
-        } else {
-            LOG(warn) << "Unknown command: " << cmd;
+            Cmds outCmds(make<StateChangeUnsubscription>(id, fDDSTaskId, Result::Ok));
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        case Type::get_properties: {
+            auto _cmd = static_cast<cmd::GetProperties&>(cmd);
+            auto const request_id(_cmd.GetRequestId());
+            auto result(Result::Ok);
+            vector<pair<string, string>> props;
+            try {
+                for (auto const& prop : GetPropertiesAsString(_cmd.GetQuery())) {
+                    props.push_back({prop.first, prop.second});
+                }
+            } catch (exception const& e) {
+                LOG(warn) << "Getting properties (request id: " << request_id << ") failed: " << e.what();
+                result = Result::Failure;
+            }
+            Cmds const outCmds(make<cmd::Properties>(id, request_id, result, props));
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        case Type::set_properties: {
+            auto _cmd(static_cast<cmd::SetProperties&>(cmd));
+            auto const request_id(_cmd.GetRequestId());
+            auto result(Result::Ok);
+            try {
+                fair::mq::Properties props;
+                for (auto const& prop : _cmd.GetProps()) {
+                    props.insert({prop.first, fair::mq::Property(prop.second)});
+                }
+                // TODO Handle builtin keys with different value type than string
+                SetProperties(props);
+            } catch (exception const& e) {
+                LOG(warn) << "Setting properties (request id: " << request_id << ") failed: " << e.what();
+                result = Result::Failure;
+            }
+            Cmds const outCmds(make<PropertiesSet>(id, request_id, result));
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        default:
+            LOG(warn) << "Unexpected/unknown command received: " << cmd.GetType();
             LOG(warn) << "Origin: " << senderId;
             LOG(warn) << "Destination: " << cond;
-        }
-    });
+        break;
+            }
 }
 
 DDS::~DDS()
@@ -418,8 +452,9 @@ DDS::~DDS()
         fControllerThread.join();
     }
 
-    if (fHeartbeatThread.joinable()) {
-        fHeartbeatThread.join();
+    fWorkGuard.reset();
+    if (fWorkerThread.joinable()) {
+        fWorkerThread.join();
     }
 }
 

fairmq/plugins/DDS/DDS.h

@@ -9,21 +9,27 @@
 #ifndef FAIR_MQ_PLUGINS_DDS
 #define FAIR_MQ_PLUGINS_DDS
 
-#include <DDS/dds_env_prop.h>
-#include <DDS/dds_intercom.h>
-#include <cassert>
-#include <chrono>
-#include <condition_variable>
 #include <fairmq/Plugin.h>
 #include <fairmq/StateQueue.h>
 #include <fairmq/Version.h>
-#include <functional>
+#include <fairmq/sdk/commands/Commands.h>
+
+#include <dds/dds.h>
+
+#include <boost/asio/executor.hpp>
+#include <boost/asio/executor_work_guard.hpp>
+#include <boost/asio/io_context.hpp>
+
+#include <cassert>
+#include <chrono>
+#include <condition_variable>
 #include <mutex>
-#include <queue>
-#include <set>
 #include <string>
+#include <atomic>
 #include <thread>
+#include <map>
 #include <unordered_map>
+#include <utility> // pair
 #include <vector>
 
 namespace fair
@@ -36,16 +42,16 @@ namespace plugins
 struct DDSConfig
 {
     // container of sub channel addresses
-    std::vector<std::string> fSubChannelAddresses;
+    unsigned int fNumSubChannels;
     // dds values for the channel
-    std::unordered_map<uint64_t, std::string> fDDSValues;
+    std::map<uint64_t, std::string> fDDSValues;
 };
 
 struct DDSSubscription
 {
     DDSSubscription()
-    : fDDSCustomCmd(fService)
-    , fDDSKeyValue(fService)
+        : fDDSCustomCmd(fService)
+        , fDDSKeyValue(fService)
     {
         LOG(debug) << "$DDS_TASK_PATH: " << dds::env_prop<dds::task_path>();
         LOG(debug) << "$DDS_GROUP_NAME: " << dds::env_prop<dds::group_name>();
@@ -128,17 +134,19 @@ class DDS : public Plugin
     ~DDS();
 
   private:
-    auto StaticControl() -> void;
     auto WaitForExitingAck() -> void;
+    auto StartWorkerThread() -> void;
 
     auto FillChannelContainers() -> void;
+    auto EmptyChannelContainers() -> void;
+
     auto SubscribeForConnectingChannels() -> void;
     auto PublishBoundChannels() -> void;
     auto SubscribeForCustomCommands() -> void;
-
-    auto HeartbeatSender() -> void;
+    auto HandleCmd(const std::string& id, sdk::cmd::Cmd& cmd, const std::string& cond, uint64_t senderId) -> void;
 
     DDSSubscription fDDS;
+    size_t fDDSTaskId;
 
     std::unordered_map<std::string, std::vector<std::string>> fBindingChans;
     std::unordered_map<std::string, DDSConfig> fConnectingChans;
@@ -146,32 +154,24 @@ class DDS : public Plugin
     std::unordered_map<std::string, int> fI;
     std::unordered_map<std::string, IofN> fIofN;
 
-    std::mutex fStopMutex;
-    std::condition_variable fStopCondition;
-
-    const std::set<std::string> fTransitions;
-
     std::thread fControllerThread;
     DeviceState fCurrentState, fLastState;
-    fair::mq::StateQueue fStateQueue;
 
     std::atomic<bool> fDeviceTerminationRequested;
 
-    std::set<uint64_t> fHeartbeatSubscribers;
-    std::mutex fHeartbeatSubscriberMutex;
-
-    std::set<uint64_t> fStateChangeSubscribers;
+    std::unordered_map<uint64_t, std::pair<std::chrono::steady_clock::time_point, int64_t>> fStateChangeSubscribers;
     uint64_t fLastExternalController;
     bool fExitingAckedByLastExternalController;
     std::condition_variable fExitingAcked;
     std::mutex fStateChangeSubscriberMutex;
 
-    std::thread fHeartbeatThread;
-    std::chrono::milliseconds fHeartbeatInterval;
-
     bool fUpdatesAllowed;
     std::mutex fUpdateMutex;
     std::condition_variable fUpdateCondition;
+
+    std::thread fWorkerThread;
+    boost::asio::io_context fWorkerQueue;
+    boost::asio::executor_work_guard<boost::asio::executor> fWorkGuard;
 };
 
 Plugin::ProgOptions DDSProgramOptions()

fairmq/plugins/DDS/runDDSCommandUI.cxx

@@ -6,27 +6,26 @@
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#include <algorithm>
-#include <atomic>
-#include <boost/algorithm/string/classification.hpp>
-#include <boost/algorithm/string/predicate.hpp>
-#include <boost/algorithm/string/split.hpp>
-#include <boost/algorithm/string/trim.hpp>
+#include <fairmq/sdk/commands/Commands.h>
+#include <fairmq/States.h>
+#include <fairmq/SDK.h>
+
 #include <boost/program_options.hpp>
+
+#include <termios.h> // raw mode console input
+#include <unistd.h>
 #include <condition_variable>
 #include <cstdlib>
-#include <DDS/dds_intercom.h>
-#include <exception>
 #include <iostream>
-#include <unordered_map>
 #include <mutex>
-#include <termios.h>   // raw mode console input
-#include <thread>
 #include <string>
-#include <unistd.h>
+#include <thread>
+#include <utility>
 
 using namespace std;
-using namespace dds::intercom_api;
+using namespace fair::mq;
+using namespace fair::mq::sdk;
+using namespace fair::mq::sdk::cmd;
 namespace bpo = boost::program_options;
 
 struct TerminalConfig
@@ -50,247 +49,176 @@ struct TerminalConfig
     }
 };
 
-struct StateSubscription {
-    const string& fTopologyPath;
-    CCustomCmd& fDdsCustomCmd;
-
-    explicit StateSubscription(const string& topologyPath, CCustomCmd& ddsCustomCmd)
-        : fTopologyPath(topologyPath)
-        , fDdsCustomCmd(ddsCustomCmd)
-    {
-        fDdsCustomCmd.send("subscribe-to-state-changes", fTopologyPath);
-    }
-
-    ~StateSubscription() {
-        fDdsCustomCmd.send("unsubscribe-from-state-changes", fTopologyPath);
-        this_thread::sleep_for(chrono::milliseconds(100)); // give dds a chance to complete request
-    }
-};
-
 void printControlsHelp()
 {
     cout << "Use keys to control the devices:" << endl;
-    cout << "[c] check states, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device, [k] complete init, [b] bind, [x] connect" << endl;
+    cout << "[c] check states, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device, [k] complete init, [b] bind, [x] connect, [p] set property" << endl;
     cout << "To quit press Ctrl+C" << endl;
 }
 
-void commandMode(const string& commandIn, const string& topologyPath, CCustomCmd& ddsCustomCmd) {
+void handleCommand(const string& command, const string& path, unsigned int timeout, Topology& topo, const string& pKey, const string& pVal)
+{
+    if (command == "c") {
+        cout << "> checking state of the devices" << endl;
+        auto const result = topo.GetCurrentState();
+        for (const auto& d : result) {
+            cout << d.taskId << " : " << d.state << endl;
+        }
+    } else if (command == "o") {
+        cout << "> dumping config of the devices (" << path << ")" << endl;
+        // TODO: extend this regex to return all properties, once command size limitation is removed.
+        auto const result = topo.GetProperties("^(session|id)$", path, std::chrono::milliseconds(timeout));
+        for (const auto& d : result.second.devices) {
+            for (auto const& p : d.second.props) {
+                cout << d.first << ": " << p.first << " : " << p.second << endl;
+            }
+        }
+    } else if (command == "p") {
+        if (pKey == "" || pVal == "") {
+            cout << "cannot send property with empty key and/or value! given key: '" << pKey << "', value: '" << pVal << "'." << endl;
+            return;
+        }
+        const DeviceProperties props{{pKey, pVal}};
+        cout << "> sending property (" << path << ")" << endl;
+        topo.SetProperties(props, path);
+        // give dds time to complete request
+        this_thread::sleep_for(chrono::milliseconds(100));
+    } else if (command == "i") {
+        cout << "> init devices (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::InitDevice, path, std::chrono::milliseconds(timeout));
+    } else if (command == "k") {
+        cout << "> complete init (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::CompleteInit, path, std::chrono::milliseconds(timeout));
+    } else if (command == "b") {
+        cout << "> bind devices (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::Bind, path, std::chrono::milliseconds(timeout));
+    } else if (command == "x") {
+        cout << "> connect devices (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::Connect, path, std::chrono::milliseconds(timeout));
+    } else if (command == "j") {
+        cout << "> init tasks (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::InitTask, path, std::chrono::milliseconds(timeout));
+    } else if (command == "r") {
+        cout << "> run tasks (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::Run, path, std::chrono::milliseconds(timeout));
+    } else if (command == "s") {
+        cout << "> stop devices (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::Stop, path, std::chrono::milliseconds(timeout));
+    } else if (command == "t") {
+        cout << "> reset tasks (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::ResetTask, path, std::chrono::milliseconds(timeout));
+    } else if (command == "d") {
+        cout << "> reset devices (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::ResetDevice, path, std::chrono::milliseconds(timeout));
+    } else if (command == "q") {
+        cout << "> end (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::End, path, std::chrono::milliseconds(timeout));
+    } else if (command == "h") {
+        cout << "> help" << endl;
+        printControlsHelp();
+    } else {
+        cout << "\033[01;32mInvalid input: [" << command << "]\033[0m" << endl;
+        printControlsHelp();
+    }
+}
+
+void sendCommand(const string& commandIn, const string& path, unsigned int timeout, Topology& topo, const string& pKey, const string& pVal)
+{
+    if (commandIn != "") {
+        handleCommand(commandIn, path, timeout, topo, pKey, pVal);
+        return;
+    }
+
     char c;
-    string command(commandIn);
+    string command;
     TerminalConfig tconfig;
 
-    if (command == "") {
-        printControlsHelp();
+    printControlsHelp();
+    cin >> c;
+    command = c;
+
+    while (true) {
+        handleCommand(command, path, timeout, topo, pKey, pVal);
         cin >> c;
         command = c;
     }
-
-    while (true) {
-        if (command == "c") {
-            cout << "> checking state of the devices" << endl;
-            ddsCustomCmd.send("check-state", topologyPath);
-        } else if (command == "o") {
-            cout << "> dumping config of the devices" << endl;
-            ddsCustomCmd.send("dump-config", topologyPath);
-        } else if (command == "i") {
-            cout << "> init devices" << endl;
-            ddsCustomCmd.send("INIT DEVICE", topologyPath);
-        } else if (command == "k") {
-            cout << "> complete init" << endl;
-            ddsCustomCmd.send("COMPLETE INIT", topologyPath);
-        } else if (command == "b") {
-            cout << "> bind devices" << endl;
-            ddsCustomCmd.send("BIND", topologyPath);
-        } else if (command == "x") {
-            cout << "> connect devices" << endl;
-            ddsCustomCmd.send("CONNECT", topologyPath);
-        } else if (command == "j") {
-            cout << "> init tasks" << endl;
-            ddsCustomCmd.send("INIT TASK", topologyPath);
-        } else if (command == "p") {
-            cout << "> pause devices" << endl;
-            ddsCustomCmd.send("PAUSE", topologyPath);
-        } else if (command == "r") {
-            cout << "> run tasks" << endl;
-            ddsCustomCmd.send("RUN", topologyPath);
-        } else if (command == "s") {
-            cout << "> stop devices" << endl;
-            ddsCustomCmd.send("STOP", topologyPath);
-        } else if (command == "t") {
-            cout << "> reset tasks" << endl;
-            ddsCustomCmd.send("RESET TASK", topologyPath);
-        } else if (command == "d") {
-            cout << "> reset devices" << endl;
-            ddsCustomCmd.send("RESET DEVICE", topologyPath);
-        } else if (command == "h") {
-            cout << "> help" << endl;
-            printControlsHelp();
-        } else if (command == "q") {
-            cout << "> end" << endl;
-            ddsCustomCmd.send("END", topologyPath);
-        } else if (command == "q!") {
-            cout << "> shutdown" << endl;
-            ddsCustomCmd.send("SHUTDOWN", topologyPath);
-        } else if (command == "r!") {
-            cout << "> startup" << endl;
-            ddsCustomCmd.send("STARTUP", topologyPath);
-        } else {
-            cout << "\033[01;32mInvalid input: [" << c << "]\033[0m" << endl;
-            printControlsHelp();
-        }
-
-        if (commandIn != "") {
-            this_thread::sleep_for(chrono::milliseconds(100)); // give dds a chance to complete request
-            break;
-        } else {
-            cin >> c;
-            command = c;
-        }
-    }
 }
 
-struct WaitMode
-{
-    explicit WaitMode(const string& targetState)
-        : fTargetState(targetState)
-    {}
-
-    void Run(const chrono::milliseconds& timeout,
-             const string& topologyPath,
-             CCustomCmd& ddsCustomCmd,
-             unsigned int numberDevices,
-             const string& command = "")
-    {
-        StateSubscription stateSubscription(topologyPath, ddsCustomCmd);
-
-        if (command != "") {
-            commandMode(command, topologyPath, ddsCustomCmd);
-        }
-
-        // TODO once DDS provides an API to retrieve actual number of tasks, use it here
-        auto condition = [&] {
-            bool res(!fTargetStates.empty()
-                   && all_of(fTargetStates.cbegin(),
-                             fTargetStates.cend(),
-                             [&](unordered_map<uint64_t, string>::value_type i) {
-                                 return boost::algorithm::ends_with(i.second, fTargetState);
-                             }));
-            if (numberDevices > 0) {
-                res = res && (fTargetStates.size() == numberDevices);
-            }
-            cout << "waiting for " << numberDevices << " devices to reach " << fTargetState << ", condition check: " << res << endl;
-            return res;
-        };
-
-        unique_lock<mutex> lock(fMtx);
-
-        if (timeout > chrono::milliseconds(0)) {
-            if (!fCV.wait_for(lock, timeout, condition)) {
-                throw runtime_error("timeout");
-            }
-        } else {
-            fCV.wait(lock, condition);
-        }
-    }
-
-    void AddNewStateEntry(uint64_t senderId, const string& state)
-    {
-        {
-            unique_lock<mutex> lock(fMtx);
-            fTargetStates[senderId] = state;
-        }
-        fCV.notify_one();
-    }
-
-    mutex fMtx;
-    condition_variable fCV;
-    unordered_map<uint64_t, string> fTargetStates;
-    string fTargetState;
-};
-
 int main(int argc, char* argv[])
-{
-    try {
-        string sessionID;
-        string command;
-        string topologyPath;
-        string targetState;
-        unsigned int timeout;
-        unsigned int numberDevices(0);
+try {
+    string sessionID;
+    string topoFile;
 
-        bpo::options_description options("Common options");
+    string command;
+    string path;
+    string targetState;
+    string pKey;
+    string pVal;
+    unsigned int timeout;
 
-        auto envSessionId = getenv("DDS_SESSION_ID");
-        if (envSessionId) {
-            options.add_options()("session,s", bpo::value<string>(&sessionID)->default_value(envSessionId), "DDS Session ID (overrides any value in env var $DDS_SESSION_ID)");
+    fair::Logger::SetConsoleSeverity("debug");
+    fair::Logger::SetConsoleColor(true);
+
+    bpo::options_description options("Common options");
+
+    auto envSessionId = getenv("DDS_SESSION_ID");
+    if (envSessionId) {
+        options.add_options()("session,s", bpo::value<string>(&sessionID)->default_value(envSessionId), "DDS Session ID (overrides any value in env var $DDS_SESSION_ID)");
+    } else {
+        options.add_options()("session,s", bpo::value<string>(&sessionID)->required(), "DDS Session ID (overrides any value in env var $DDS_SESSION_ID)");
+    }
+
+    auto envTopoFile = getenv("FAIRMQ_DDS_TOPO_FILE");
+    if (envTopoFile) {
+        options.add_options()("topology-file,f", bpo::value<string>(&topoFile)->default_value(envTopoFile), "DDS topology file path");
+    } else {
+        options.add_options()("topology-file,f", bpo::value<string>(&topoFile)->required(), "DDS topology file path");
+    }
+
+    options.add_options()
+        ("command,c",        bpo::value<string>(&command)->default_value(""), "Command character")
+        ("path,p",           bpo::value<string>(&path)->default_value(""), "DDS Topology path to send command to (empty - send to all tasks)")
+        ("property-key",     bpo::value<string>(&pKey)->default_value(""), "property key to be used with 'p' command")
+        ("property-value",   bpo::value<string>(&pVal)->default_value(""), "property value to be used with 'p' command")
+        ("wait-for-state,w", bpo::value<string>(&targetState)->default_value(""), "Wait until targeted FairMQ devices reach the given state")
+        ("timeout,t",        bpo::value<unsigned int>(&timeout)->default_value(0), "Timeout in milliseconds when waiting for a device state (0 - wait infinitely)")
+        ("help,h",           "Produce help message");
+
+    bpo::variables_map vm;
+    bpo::store(bpo::command_line_parser(argc, argv).options(options).run(), vm);
+
+    if (vm.count("help")) {
+        cout << "FairMQ DDS Command UI" << endl << options << endl;
+        cout << "Commands: [c] check state, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device, [k] complete init, [b] bind, [x] connect, [p] set property" << endl;
+        return EXIT_SUCCESS;
+    }
+
+    bpo::notify(vm);
+
+    DDSEnvironment env;
+    DDSSession session(sessionID, env);
+    DDSTopology ddsTopo(DDSTopology::Path(topoFile), env);
+
+    Topology topo(ddsTopo, session);
+
+    if (targetState != "") {
+        if (command != "") {
+            sendCommand(command, path, timeout, topo, pKey, pVal);
+        }
+        size_t pos = targetState.find("->");
+        if (pos == string::npos) {
+            /* auto ec =  */topo.WaitForState(GetState(targetState), path, std::chrono::milliseconds(timeout));
+            // cout << "WaitForState(" << targetState << ") result: " << ec.message() << endl;
         } else {
-            options.add_options()("session,s", bpo::value<string>(&sessionID)->required(), "DDS Session ID (overrides any value in env var $DDS_SESSION_ID)");
+            /* auto ec =  */topo.WaitForState(GetState(targetState.substr(0, pos)), GetState(targetState.substr(pos + 2)), path, std::chrono::milliseconds(timeout));
+            // cout << "WaitForState(" << targetState << ") result: " << ec.message() << endl;
         }
-
-        options.add_options()
-            ("command,c",        bpo::value<string> (&command)->default_value(""), "Command character")
-            ("path,p",           bpo::value<string> (&topologyPath)->default_value(""), "DDS Topology path to send command to (empty - send to all tasks)")
-            ("wait-for-state,w", bpo::value<string> (&targetState)->default_value(""), "Wait until targeted FairMQ devices reach the given state")
-            ("timeout,t",        bpo::value<unsigned int> (&timeout)->default_value(0), "Timeout in milliseconds when waiting for a device state (0 - wait infinitely)")
-            ("number-devices,n", bpo::value<unsigned int> (&numberDevices)->default_value(0), "Number of devices (will be removed in the future)")
-            ("help,h", "Produce help message");
-
-        bpo::variables_map vm;
-        bpo::store(bpo::command_line_parser(argc, argv).options(options).run(), vm);
-
-        if (vm.count("help")) {
-            cout << "FairMQ DDS Command UI" << endl << options << endl;
-            cout << "Commands: [c] check state, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device, [k] complete init, [b] bind, [x] connect" << endl;
-            return EXIT_SUCCESS;
-        }
-
-        bpo::notify(vm);
-
-        WaitMode waitMode(targetState);
-
-        CIntercomService service;
-        CCustomCmd ddsCustomCmd(service);
-
-        service.subscribeOnError([](const EErrorCode errorCode, const string& errorMsg) {
-            cerr << "DDS error received: error code: " << errorCode << ", error message: " << errorMsg << endl;
-        });
-
-        // subscribe to receive messages from DDS
-        ddsCustomCmd.subscribe([&](const string& msg, const string& /*condition*/, uint64_t senderId) {
-            // cerr << "Received: " << msg << endl;
-            vector<string> parts;
-            boost::algorithm::split(parts, msg, boost::algorithm::is_any_of(":,"));
-            if (parts[0] == "state-change") {
-                // cerr << "Received: " << msg << endl;
-                boost::trim(parts[2]);
-                waitMode.AddNewStateEntry(senderId, parts[3]);
-                if(parts[3] == "IDLE->EXITING") {
-                    ddsCustomCmd.send("state-change-exiting-received", std::to_string(senderId));
-                }
-            } else if (parts[0] == "state-changes-subscription") {
-                if (parts[2] != "OK") {
-                    cerr << "state-changes-subscription failed with return code: " << parts[2];
-                }
-            } else if (parts[0] == "state-changes-unsubscription") {
-                if (parts[2] != "OK") {
-                    cerr << "state-changes-unsubscription failed with return code: " << parts[2];
-                }
-            } else {
-                cout << "Received: " << msg << endl;
-            }
-        });
-
-        service.start(sessionID);
-
-        if (targetState == "") {
-            commandMode(command, topologyPath, ddsCustomCmd);
-        } else {
-            waitMode.Run(chrono::milliseconds(timeout), topologyPath, ddsCustomCmd, numberDevices, command);
-        }
-    } catch (exception& e) {
-        cerr << "Error: " << e.what() << endl;
-        return EXIT_FAILURE;
+    } else {
+        sendCommand(command, path, timeout, topo, pKey, pVal);
     }
 
     return EXIT_SUCCESS;
+} catch (exception& e) {
+    cerr << "Error: " << e.what() << endl;
+    return EXIT_FAILURE;
 }

fairmq/plugins/PMIx/CMakeLists.txt

@@ -10,9 +10,10 @@ set(plugin FairMQPlugin_pmix)
 add_library(${plugin} SHARED
   ${CMAKE_CURRENT_SOURCE_DIR}/PMIxPlugin.cxx
   ${CMAKE_CURRENT_SOURCE_DIR}/PMIxPlugin.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/PMIxCommands.h
   ${CMAKE_CURRENT_SOURCE_DIR}/PMIx.hpp
 )
-target_link_libraries(${plugin} FairMQ PMIx::libpmix)
+target_link_libraries(${plugin} PUBLIC FairMQ PMIx::libpmix PRIVATE Commands)
 target_include_directories(${plugin} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 set_target_properties(${plugin} PROPERTIES
   CXX_VISIBILITY_PRESET hidden
@@ -20,7 +21,12 @@ set_target_properties(${plugin} PROPERTIES
   SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
 )
 
-install(TARGETS ${plugin}
+set(exe fairmq-pmix-command-ui)
+add_executable(${exe} ${CMAKE_CURRENT_SOURCE_DIR}/runPMIxCommandUI.cxx)
+target_link_libraries(${exe} FairMQ Commands StateMachine PMIx::libpmix)
+target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+
+install(TARGETS ${plugin} ${exe}
   EXPORT ${PROJECT_EXPORT_SET}
   LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
   RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}

fairmq/plugins/PMIx/PMIx.hpp

@@ -9,19 +9,20 @@
 #ifndef PMIX_HPP
 #define PMIX_HPP
 
+#include <array>
 #include <cstring>
+#include <functional>
 #include <limits>
 #include <memory>
 #include <ostream>
 #include <pmix.h>
+#include <sstream>
 #include <stdexcept>
-#include <string.h>
 #include <type_traits>
 #include <utility>
 #include <vector>
-#include <FairMQLogger.h>
 
-// C++ PMIx v2.1 API
+// C++ PMIx v2.2 API
 namespace pmix
 {
 
@@ -69,7 +70,7 @@ struct proc : pmix_proc_t
 
     friend std::ostream& operator<<(std::ostream& os, const proc& p)
     {
-        return os << "nspace=" << p.nspace << ",rank=" << p.rank;
+        return os << p.nspace << "_" << p.rank;
     }
 };
 
@@ -80,7 +81,6 @@ struct value : pmix_value_t
 
     value(const value& rhs)
     {
-      LOG(warn) << "copy ctor";
         status rc;
         auto lhs(static_cast<pmix_value_t*>(this));
         PMIX_VALUE_XFER(rc, lhs, static_cast<pmix_value_t*>(const_cast<value*>(&rhs)));
@@ -90,12 +90,6 @@ struct value : pmix_value_t
         }
     }
 
-    // template<typename T>
-    // value(const T* val, data_type dt)
-    // {
-        // PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), const_cast<void*>(val), dt);
-    // }
-
     template<typename T>
     explicit value(T)
     {
@@ -112,6 +106,16 @@ struct value : pmix_value_t
         PMIX_VALUE_LOAD(
             static_cast<pmix_value_t*>(this), const_cast<char*>(val.c_str()), PMIX_STRING);
     }
+
+    explicit value(int val)
+    {
+        PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), &val, PMIX_INT);
+    }
+
+    explicit value(pmix_data_array_t* val)
+    {
+        PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), val, PMIX_DATA_ARRAY);
+    }
 };
 
 struct info : pmix_info_t
@@ -135,9 +139,15 @@ struct info : pmix_info_t
         }
     }
 
-    friend std::ostream& operator<<(std::ostream& os, const info& p)
+    friend std::ostream& operator<<(std::ostream& os, const info& i)
     {
-        return os << "key=" << p.key << ",value='" << p.value.data.string << "'";
+        return os << "key=" << i.key << ",value='" << i.value.data.string << "'";
+    }
+
+    info(const info& rhs)
+    {
+        PMIX_INFO_XFER(static_cast<pmix_info_t*>(this),
+                       static_cast<pmix_info_t*>(const_cast<info*>(&rhs)));
     }
 };
 
@@ -145,6 +155,7 @@ struct pdata : pmix_pdata_t
 {
     pdata() { PMIX_PDATA_CONSTRUCT(static_cast<pmix_pdata_t*>(this)); }
     ~pdata() { PMIX_PDATA_DESTRUCT(static_cast<pmix_pdata_t*>(this)); }
+
     pdata(const pdata& rhs)
     {
         PMIX_PDATA_XFER(static_cast<pmix_pdata_t*>(this),
@@ -172,7 +183,7 @@ auto init(const std::vector<info>& info = {}) -> proc
 
 auto initialized() -> bool { return !!PMIx_Initialized(); }
 
-auto get_version() -> const char* { return PMIx_Get_version(); }
+auto get_version() -> std::string { return {PMIx_Get_version()}; }
 
 auto finalize(const std::vector<info>& info = {}) -> void
 {
@@ -214,6 +225,92 @@ auto lookup(std::vector<pdata>& pdata, const std::vector<info>& info = {}) -> vo
     }
 }
 
+std::string get_info(const std::string& name, pmix::proc& process)
+{
+    pmix_value_t* v;
+
+    pmix::status rc = PMIx_Get(&process, name.c_str(), nullptr, 0, &v);
+    if (rc == PMIX_SUCCESS) {
+        std::stringstream ss;
+
+        switch (v->type) {
+            case PMIX_SIZE:      ss << static_cast<size_t>(v->data.size)       << " (size_t)";       break;
+            case PMIX_INT:       ss << static_cast<int>(v->data.integer)       << " (int)";          break;
+            case PMIX_INT8:      ss << static_cast<int8_t>(v->data.int8)       << " (int8_t)";       break;
+            case PMIX_INT16:     ss << static_cast<int16_t>(v->data.int16)     << " (int16_t)";      break;
+            case PMIX_INT32:     ss << static_cast<int32_t>(v->data.int32)     << " (int32_t)";      break;
+            case PMIX_INT64:     ss << static_cast<int64_t>(v->data.int64)     << " (int64_t)";      break;
+            case PMIX_UINT:      ss << static_cast<unsigned int>(v->data.uint) << " (unsigned int)"; break;
+            case PMIX_UINT8:     ss << static_cast<uint8_t>(v->data.uint8)     << " (uint8_t)";      break;
+            case PMIX_UINT16:    ss << static_cast<uint16_t>(v->data.uint16)   << " (uint16_t)";     break;
+            case PMIX_UINT32:    ss << static_cast<uint32_t>(v->data.uint32)   << " (uint32_t)";     break;
+            case PMIX_UINT64:    ss << static_cast<uint64_t>(v->data.uint64)   << " (uint64_t)";     break;
+            case PMIX_FLOAT:     ss << static_cast<float>(v->data.fval)        << " (float)";        break;
+            case PMIX_DOUBLE:    ss << static_cast<double>(v->data.dval)       << " (double)";       break;
+            case PMIX_PID:       ss << static_cast<pid_t>(v->data.pid)         << " (pid_t)";        break;
+            case PMIX_STRING:    ss << static_cast<char*>(v->data.string)      << " (string)";       break;
+            case PMIX_PROC_RANK: ss << static_cast<uint32_t>(v->data.rank)     << " (pmix_rank_t)";  break;
+            case PMIX_PROC:      ss << "proc.nspace: " << static_cast<pmix_proc_t*>(v->data.proc)->nspace
+                                    << ", proc.rank: " << static_cast<pmix_proc_t*>(v->data.proc)->rank << " (pmix_proc_t*)"; break;
+            default:
+                ss << "unknown type: " << v->type;
+                break;
+        }
+
+        return ss.str();
+    } else if (rc == PMIX_ERR_NOT_FOUND) {
+        // LOG(error) << "PMIx_Get failed: PMIX_ERR_NOT_FOUND";
+        return "";
+    } else {
+        // LOG(error) << "PMIx_Get failed: " << rc;
+        return "<undefined>";
+    }
+}
+
+std::string get_value_str(const pmix_value_t& v)
+{
+    switch (v.type) {
+        case PMIX_BOOL:       return std::to_string(static_cast<bool>(v.data.flag));
+        case PMIX_SIZE:       return std::to_string(static_cast<size_t>(v.data.size));
+        case PMIX_INT:        return std::to_string(static_cast<int>(v.data.integer));
+        case PMIX_INT8:       return std::to_string(static_cast<int8_t>(v.data.int8));
+        case PMIX_INT16:      return std::to_string(static_cast<int16_t>(v.data.int16));
+        case PMIX_INT32:      return std::to_string(static_cast<int32_t>(v.data.int32));
+        case PMIX_INT64:      return std::to_string(static_cast<int64_t>(v.data.int64));
+        case PMIX_UINT:       return std::to_string(static_cast<unsigned int>(v.data.uint));
+        case PMIX_UINT8:      return std::to_string(static_cast<uint8_t>(v.data.uint8));
+        case PMIX_UINT16:     return std::to_string(static_cast<uint16_t>(v.data.uint16));
+        case PMIX_UINT32:     return std::to_string(static_cast<uint32_t>(v.data.uint32));
+        case PMIX_UINT64:     return std::to_string(static_cast<uint64_t>(v.data.uint64));
+        case PMIX_FLOAT:      return std::to_string(static_cast<float>(v.data.fval));
+        case PMIX_DOUBLE:     return std::to_string(static_cast<double>(v.data.dval));
+        case PMIX_PID:        return std::to_string(static_cast<pid_t>(v.data.pid));
+        case PMIX_STRING:     return static_cast<char*>(v.data.string);
+        case PMIX_PROC_RANK:  return std::to_string(static_cast<uint32_t>(v.data.rank));
+        case PMIX_POINTER:    { std::stringstream ss; ss << static_cast<void*>(v.data.ptr); return ss.str(); }
+        case PMIX_DATA_ARRAY: {
+            if (v.data.darray->type == PMIX_PROC) {
+                std::stringstream ss;
+                ss << "[";
+                for (size_t i = 0; i < v.data.darray->size; ++i) {
+                    ss << static_cast<pmix_proc_t*>(static_cast<pmix_data_array_t*>(v.data.darray)->array)[0].nspace;
+                    ss << "_";
+                    ss << static_cast<pmix_proc_t*>(static_cast<pmix_data_array_t*>(v.data.darray)->array)[0].rank;
+
+                    if (i < v.data.darray->size - 1) {
+                        ss << ",";
+                    }
+                }
+                ss << "]";
+                return ss.str();
+            } else {
+                return "UNKNOWN TYPE IN DATA ARRAY";
+            }
+        }
+        default:             return "UNKNOWN TYPE";
+    }
+}
+
 } /* namespace pmix */
 
 #endif /* PMIX_HPP */