Disable codecov.io checks

Sometimes devices shut down too fast when entering the EXITING state so
that the publication of that state-change will never be sent. The plugin
now waits for an acknowledgement by the external controller with a
configurable timeout.

.clang-tidy

@@ -1,3 +1,3 @@
 ---
 Checks:          '*,-google-*,-fuchsia-*,-cert-*,-llvm-header-guard,-readability-named-parameter,-misc-non-private-member-variables-in-classes,-*-magic-numbers,-llvm-include-order,-hicpp-no-array-decay,-performance-unnecessary-value-param,-cppcoreguidelines-pro-bounds-array-to-pointer-decay'
-HeaderFilterRegex: '/(fairmq/|FairMQ)'
+HeaderFilterRegex: '/(fairmq/)'

.codecov.yml

@@ -1,2 +1,3 @@
 comment:
   layout: "diff, files"
+  behavior: once

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "extern/googletest"]
+	path = extern/googletest
+	url = https://github.com/google/googletest
+[submodule "extern/asio"]
+	path = extern/asio
+	url = https://github.com/chriskohlhoff/asio

CMakeGraphVizOptions.cmake

@@ -0,0 +1,14 @@
+set(GRAPHVIZ_GRAPH_TYPE digraph)
+set(GRAPHVIZ_GRAPH_NAME FairMQ)
+set(GRAPHVIZ_EXECUTABLES ON)
+set(GRAPHVIZ_STATIC_LIBS OFF)
+set(GRAPHVIZ_SHARED_LIBS ON)
+set(GRAPHVIZ_MODULE_LIBS OFF)
+set(GRAPHVIZ_GENERATE_PER_TARGET OFF)
+set(GRAPHVIZ_GENERATE_DEPENDERS OFF)
+set(GRAPHVIZ_IGNORE_TARGETS
+  "fairmq-ex.*"
+  "testsuite_.*"
+  "testhelper_.*"
+  "FairMQPlugin_test_dummy"
+)

CMakeLists.txt

@@ -6,8 +6,8 @@
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
-cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
-cmake_policy(VERSION 3.10...3.14)
+cmake_minimum_required(VERSION 3.11 FATAL_ERROR)
+cmake_policy(VERSION 3.11...3.15)
 
 # Project ######################################################################
 set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
@@ -18,7 +18,7 @@ get_git_version()
 project(FairMQ VERSION ${PROJECT_VERSION} LANGUAGES CXX)
 message(STATUS "${BWhite}${PROJECT_NAME}${CR} ${PROJECT_GIT_VERSION} from ${PROJECT_DATE}")
 
-if(BUILD_OFI_TRANSPORT)
+if(BUILD_OFI_TRANSPORT OR BUILD_SDK)
   set(PROJECT_MIN_CXX_STANDARD 14)
 else()
   set(PROJECT_MIN_CXX_STANDARD 11)
@@ -31,17 +31,28 @@ include(CTest)
 
 
 # Build options ################################################################
-include(CMakeDependentOption)
-option(BUILD_FAIRMQ "Build FairMQ library and devices." ON)
-cmake_dependent_option(BUILD_TESTING "Build tests." OFF "BUILD_FAIRMQ" OFF)
-cmake_dependent_option(BUILD_NANOMSG_TRANSPORT "Build nanomsg transport." OFF "BUILD_FAIRMQ" OFF)
-cmake_dependent_option(BUILD_OFI_TRANSPORT "Build experimental OFI transport." OFF "BUILD_FAIRMQ" OFF)
-cmake_dependent_option(BUILD_DDS_PLUGIN "Build DDS plugin." OFF "BUILD_FAIRMQ" OFF)
-cmake_dependent_option(BUILD_PMIX_PLUGIN "Build PMIx plugin." OFF "BUILD_FAIRMQ" OFF)
-cmake_dependent_option(BUILD_EXAMPLES "Build FairMQ examples." ON "BUILD_FAIRMQ" OFF)
-option(BUILD_SDK "Build the FairMQ controller SDK." OFF)
-option(BUILD_DOCS "Build FairMQ documentation." OFF)
-option(FAST_BUILD "Fast production build. Not recommended for development." OFF)
+fairmq_build_option(BUILD_FAIRMQ            "Build FairMQ library and devices."
+                                             DEFAULT ON)
+fairmq_build_option(BUILD_TESTING           "Build tests."
+                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
+fairmq_build_option(BUILD_NANOMSG_TRANSPORT "Build nanomsg transport."
+                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
+fairmq_build_option(BUILD_OFI_TRANSPORT     "Build experimental OFI transport."
+                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
+fairmq_build_option(BUILD_SDK_COMMANDS      "Build the FairMQ SDK commands."
+                                             DEFAULT OFF)
+fairmq_build_option(BUILD_DDS_PLUGIN        "Build DDS plugin."
+                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
+fairmq_build_option(BUILD_PMIX_PLUGIN       "Build PMIx plugin."
+                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
+fairmq_build_option(BUILD_EXAMPLES          "Build FairMQ examples."
+                                             DEFAULT ON  REQUIRES "BUILD_FAIRMQ")
+fairmq_build_option(BUILD_SDK               "Build the FairMQ controller SDK."
+                                             DEFAULT OFF REQUIRES "BUILD_DDS_PLUGIN;BUILD_SDK_COMMANDS")
+fairmq_build_option(BUILD_DOCS              "Build FairMQ documentation."
+                                             DEFAULT OFF)
+fairmq_build_option(FAST_BUILD              "Fast production build. Not recommended for development."
+                                             DEFAULT OFF)
 ################################################################################
 
 
@@ -74,16 +85,15 @@ if(BUILD_NANOMSG_TRANSPORT)
   )
 endif()
 
-if(BUILD_SDK)
-  set(required_dds_version 2.5.22)
-else()
-  set(required_dds_version 2.4)
+if(BUILD_SDK_COMMANDS)
+  find_package2(PRIVATE Flatbuffers REQUIRED)
 endif()
+
 if(BUILD_DDS_PLUGIN OR BUILD_SDK)
   find_package2(PRIVATE DDS REQUIRED
-    VERSION ${required_dds_version}
+    VERSION 3.0
   )
-  set(DDS_Boost_COMPONENTS system log log_setup)
+  set(DDS_Boost_COMPONENTS system log log_setup regex filesystem thread)
   set(DDS_Boost_VERSION 1.67)
 endif()
 
@@ -98,6 +108,13 @@ if(BUILD_FAIRMQ OR BUILD_SDK)
     VERSION 1.2.0
   )
 
+  foreach(dep IN LISTS FairLogger_PACKAGE_DEPENDENCIES)
+    if(NOT dep STREQUAL "Boost")
+      find_package2(PUBLIC ${dep} REQUIRED VERSION ${FairLogger_${dep}_VERSION})
+      set(PROJECT_${dep}_VERSION ${FairLogger_${dep}_VERSION})
+    endif()
+  endforeach()
+
   if(NOT DEFINED Boost_NO_BOOST_CMAKE AND CMAKE_VERSION VERSION_LESS 3.15)
     # Since Boost 1.70 a CMake package is shipped by default. Unfortunately, it has a number
     # of problems that are only fixed in Boost 1.71 or CMake 3.15. By default we skip the
@@ -105,7 +122,7 @@ if(BUILD_FAIRMQ OR BUILD_SDK)
     set(Boost_NO_BOOST_CMAKE ON)
   endif()
   find_package2(PUBLIC Boost REQUIRED
-    VERSION 1.64
+    VERSION 1.66
 
     COMPONENTS
     container
@@ -119,18 +136,35 @@ if(BUILD_FAIRMQ OR BUILD_SDK)
     DDS
     FairLogger
   )
+
+  # Normalize Boost version
+  if(CMAKE_VERSION VERSION_LESS 3.15)
+    set(Boost_VERSION "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}")
+  endif()
+endif()
+
+if(BUILD_SDK)
+  find_package2(BUNDLED asio
+    VERSION 1.13.0
+  )
+  if(NOT asio_FOUND)
+    build_bundled(asio extern/asio)
+    find_package2(PRIVATE asio REQUIRED)
+  endif()
 endif()
 
 if(BUILD_FAIRMQ)
   find_package2(PRIVATE ZeroMQ REQUIRED
-    VERSION 4.1.5
+    VERSION 4.1.4
   )
 endif()
 
 if(BUILD_TESTING)
-  find_package2(PRIVATE GTest REQUIRED
-    VERSION 1.7.0
-  )
+  find_package2(PRIVATE GTest VERSION 1.7.0)
+  if(NOT GTest_FOUND)
+    build_bundled(GTest extern/googletest)
+    find_package2(PRIVATE GTest REQUIRED)
+  endif()
 endif()
 
 if(BUILD_DOCS)
@@ -196,6 +230,9 @@ endif()
 if(BUILD_SDK)
   list(APPEND PROJECT_PACKAGE_COMPONENTS sdk)
 endif()
+if(BUILD_SDK_COMMANDS)
+  list(APPEND PROJECT_PACKAGE_COMPONENTS sdk_commands)
+endif()
 ################################################################################
 
 
@@ -210,6 +247,22 @@ if(BUILD_DOCS)
     DESTINATION ${PROJECT_INSTALL_DATADIR}/docs
   )
 endif()
+if(BUILD_SDK)
+  install(FILES cmake/Findasio.cmake
+    DESTINATION ${PROJECT_INSTALL_CMAKEMODDIR}
+  )
+  if(asio_BUNDLED)
+    install(TARGETS bundled_asio_headers EXPORT ${PROJECT_EXPORT_SET})
+    install(DIRECTORY "${asio_BUILD_INCLUDE_DIR}/asio"
+      DESTINATION ${asio_INSTALL_INCLUDE_DIR}
+      PATTERN "Makefile.am" EXCLUDE
+      PATTERN ".gitignore" EXCLUDE
+    )
+    install(FILES "${asio_BUILD_INCLUDE_DIR}/asio.hpp"
+      DESTINATION ${asio_INSTALL_INCLUDE_DIR}
+    )
+  endif()
+endif()
 
 install_cmake_package()
 ################################################################################
@@ -245,16 +298,8 @@ if(PROJECT_PACKAGE_DEPENDENCIES)
   message(STATUS "  ")
   message(STATUS "  ${Cyan}DEPENDENCY FOUND     VERSION                   PREFIX${CR}")
   foreach(dep IN LISTS PROJECT_PACKAGE_DEPENDENCIES)
-    if(${dep}_VERSION)
-      if(${dep} STREQUAL Boost)
-        if(Boost_VERSION_MAJOR)
-          set(version_str "${BGreen}${${dep}_VERSION_MAJOR}.${${dep}_VERSION_MINOR}${CR}")
-        else()
-          set(version_str "${BGreen}${${dep}_MAJOR_VERSION}.${${dep}_MINOR_VERSION}${CR}")
-        endif()
-      else()
-        set(version_str "${BGreen}${${dep}_VERSION}${CR}")
-      endif()
+    if(${dep}_VERSION AND NOT ${dep}_VERSION STREQUAL "..")
+      set(version_str "${BGreen}${${dep}_VERSION}${CR}")
     else()
       set(version_str "${BYellow}unknown${CR}")
     endif()
@@ -298,10 +343,22 @@ if(PROJECT_PACKAGE_DEPENDENCIES)
       get_target_property(doxygen_bin Doxygen::doxygen INTERFACE_LOCATION)
       get_filename_component(prefix ${doxygen_bin} DIRECTORY)
       get_filename_component(prefix ${prefix}/.. ABSOLUTE)
+    elseif(${dep} STREQUAL fmt)
+      get_target_property(fmt_include fmt::fmt INTERFACE_INCLUDE_DIRECTORIES)
+      get_filename_component(prefix ${fmt_include}/.. ABSOLUTE)
+    elseif(${dep} STREQUAL Flatbuffers)
+      if(TARGET flatbuffers::flatbuffers)
+        get_target_property(flatbuffers_include flatbuffers::flatbuffers INTERFACE_INCLUDE_DIRECTORIES)
+      else()
+        get_target_property(flatbuffers_include flatbuffers::flatbuffers_shared INTERFACE_INCLUDE_DIRECTORIES)
+      endif()
+      get_filename_component(prefix ${flatbuffers_include}/.. ABSOLUTE)
     else()
       get_filename_component(prefix ${${dep}_INCLUDE_DIR}/.. ABSOLUTE)
     endif()
-    message(STATUS "  ${BWhite}${dep_padded}${CR}${version_padded}${prefix}")
+    if(NOT ${dep}_BUNDLED)
+      message(STATUS "  ${BWhite}${dep_padded}${CR}${version_padded}${prefix}")
+    endif()
     unset(version_str)
     unset(version_padded)
     unset(version_req_str)
@@ -358,11 +415,17 @@ else()
 endif()
 message(STATUS "  ${BWhite}docs${CR}               ${docs_summary}")
 if(BUILD_SDK)
-  set(sdk_summary "${BGreen}YES${CR}    EXPERIMENTAL (disable with ${BMagenta}-DBUILD_SDK=OFF${CR})")
+  set(sdk_summary "${BGreen}YES${CR}    EXPERIMENTAL (required C++14) (disable with ${BMagenta}-DBUILD_SDK=OFF${CR})")
 else()
-  set(sdk_summary "${BRed} NO${CR}    EXPERIMENTAL (default, enable with ${BMagenta}-DBUILD_SDK=ON${CR})")
+  set(sdk_summary "${BRed} NO${CR}    EXPERIMENTAL (required C++14) (default, enable with ${BMagenta}-DBUILD_SDK=ON${CR})")
 endif()
 message(STATUS "  ${BWhite}sdk${CR}                ${sdk_summary}")
+if(BUILD_SDK_COMMANDS)
+  set(sdk_commands_summary "${BGreen}YES${CR}    (disable with ${BMagenta}-DBUILD_SDK_COMMANDS=OFF${CR})")
+else()
+  set(sdk_commands_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_SDK_COMMANDS=ON${CR})")
+endif()
+message(STATUS "  ${BWhite}sdk_commands${CR}       ${sdk_commands_summary}")
 message(STATUS "  ")
 if(RUN_STATIC_ANALYSIS)
   list(LENGTH PROJECT_STATIC_ANALYSERS size)
@@ -384,6 +447,8 @@ if(RUN_STATIC_ANALYSIS)
 else()
   set(static_ana_summary "${BRed}OFF${CR}  (default, enable with ${BMagenta}-DRUN_STATIC_ANALYSIS=ON${CR})")
 endif()
+message(STATUS "  ${Cyan}INSTALL PREFIX${CR}     ${BGreen}${CMAKE_INSTALL_PREFIX}${CR} (change with ${BMagenta}-DCMAKE_INSTALL_PREFIX=...${CR})")
+message(STATUS "  ")
 message(STATUS "  ${Cyan}RUN STATIC ANALYSIS  ${static_ana_summary}")
 message(STATUS "  ")
 ################################################################################

COPYRIGHT

@@ -15,6 +15,14 @@ Files: cmake/cotire.cmake
 Copyright: 2012-2018 Sascha Kratky
 License: COTIRE
 
+Files: extern/googletest
+Copyright: 2008, Google Inc.
+License: GOOGLE
+
+Files: extern/asio
+Copyright: 2003-2019, Christopher M. Kohlhoff (chris at kohlhoff dot com)
+License: BSL-1.0
+
 License: LGPL-3.0-only
   [see LICENSE file]
 
@@ -39,3 +47,58 @@ License: COTIRE
   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
   OTHER DEALINGS IN THE SOFTWARE.
+
+License: GOOGLE
+  Copyright 2008, Google Inc.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are
+  met:
+
+      * Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+      * Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following disclaimer
+  in the documentation and/or other materials provided with the
+  distribution.
+      * Neither the name of Google Inc. nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+License: BSL-1.0
+  Boost Software License - Version 1.0 - August 17th, 2003
+
+  Permission is hereby granted, free of charge, to any person or organization
+  obtaining a copy of the software and accompanying documentation covered by
+  this license (the "Software") to use, reproduce, display, distribute,
+  execute, and transmit the Software, and to prepare derivative works of the
+  Software, and to permit third-parties to whom the Software is furnished to
+  do so, all subject to the following:
+
+  The copyright notices in the Software and this entire statement, including
+  the above license grant, this restriction and the following disclaimer,
+  must be included in all copies of the Software, in whole or in part, and
+  all derivative works of the Software, unless such copies or derivative
+  works are solely in the form of machine-executable object code generated by
+  a source language processor.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+  SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+  FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+  DEALINGS IN THE SOFTWARE.

Dart.sh

@@ -65,7 +65,7 @@ if [ "$1" == "alfa_ci" ]; then
   export ctest_model=Experimental
 elif [ "$1" == "codecov" ]; then
   export ctest_model=Profile
-  export do_codecov_upload=1
+  export do_codecov_upload=0
 else
   export ctest_model=$1
 fi

FairMQTest.cmake

@@ -30,12 +30,13 @@ Set(configure_options "${configure_options};-DBUILD_NANOMSG_TRANSPORT=ON")
 # Set(configure_options "${configure_options};-DBUILD_OFI_TRANSPORT=ON")
 Set(configure_options "${configure_options};-DBUILD_DDS_PLUGIN=ON")
 Set(configure_options "${configure_options};-DBUILD_SDK=ON")
+Set(configure_options "${configure_options};-DBUILD_SDK_COMMANDS=ON")
 Set(configure_options "${configure_options};-DFAST_BUILD=ON")
 Set(configure_options "${configure_options};-DCOTIRE_MAXIMUM_NUMBER_OF_UNITY_INCLUDES=-j$ENV{number_of_processors}")
 
 Set(EXTRA_FLAGS $ENV{EXTRA_FLAGS})
 If(EXTRA_FLAGS)
-  Set(configure_options "${configure_options};${EXTRA_FLAGS}") 
+  Set(configure_options "${configure_options};${EXTRA_FLAGS}")
 EndIf()
 
 If($ENV{ctest_model} MATCHES Profile)
@@ -43,6 +44,7 @@ If($ENV{ctest_model} MATCHES Profile)
   If(GCOV_COMMAND)
     Message("Found GCOV: ${GCOV_COMMAND}")
     Set(CTEST_COVERAGE_COMMAND ${GCOV_COMMAND})
+    set(CTEST_COVERAGE_EXTRA_FLAGS "-p")
   EndIf(GCOV_COMMAND)
 EndIf()
 
@@ -58,7 +60,7 @@ Ctest_Configure(BUILD "${CTEST_BINARY_DIRECTORY}"
 
 Ctest_Build(BUILD "${CTEST_BINARY_DIRECTORY}")
 
-Ctest_Test(BUILD "${CTEST_BINARY_DIRECTORY}" 
+Ctest_Test(BUILD "${CTEST_BINARY_DIRECTORY}"
            # PARALLEL_LEVEL $ENV{number_of_processors}
            PARALLEL_LEVEL $ENV{number_of_processors}
            RETURN_VALUE _ctest_test_ret_val
@@ -78,7 +80,7 @@ If("$ENV{do_codecov_upload}")
 EndIf()
 
 Ctest_Submit()
- 
+
 if (_ctest_test_ret_val)
   Message(FATAL_ERROR "Some tests failed.")
 endif()

Jenkinsfile

@@ -22,11 +22,11 @@ def jobMatrix(String prefix, List specs, Closure callback) {
             echo "export SIMPATH=\${SIMPATH_PREFIX}${fairsoft}" >> Dart.cfg
             echo "export FAIRSOFT_VERSION=${fairsoft}" >> Dart.cfg
           """
-          if (os =~ /Debian/ && compiler =~ /gcc8/) {
+          if (os =~ /Debian/ && compiler =~ /gcc9/) {
             sh '''\
               echo "source /etc/profile.d/modules.sh" >> Dart.cfg
               echo "module use /cvmfs/it.gsi.de/modulefiles" >> Dart.cfg
-              echo "module load compiler/gcc/8" >> Dart.cfg
+              echo "module load compiler/gcc/9.1.0" >> Dart.cfg
             '''
           }
           if (os =~ /MacOS/) {
@@ -49,6 +49,10 @@ def jobMatrix(String prefix, List specs, Closure callback) {
           deleteDir()
           githubNotify(context: "${prefix}/${label}", description: 'Success', status: 'SUCCESS')
         } catch (e) {
+          def tarball = "${prefix}_${label}_dds_logs.tar.gz"
+          sh "tar czvf ${tarball} -C \${WORKSPACE}/build/test/ .DDS/"
+          archiveArtifacts tarball
+
           deleteDir()
           githubNotify(context: "${prefix}/${label}", description: 'Error', status: 'ERROR')
           throw e
@@ -65,16 +69,16 @@ pipeline{
     stage("Run CI Matrix") {
       steps{
         script {
-          def build_jobs = jobMatrix('alfa-ci/build', [
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
+          def build_jobs = jobMatrix('build', [
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc9.1.0',        fairsoft: 'fairmq_dev'],
             [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
             [os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             sh './Dart.sh alfa_ci Dart.cfg'
           }
 
-          def profile_jobs = jobMatrix('alfa-ci/codecov', [
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
+          def profile_jobs = jobMatrix('codecov', [
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc9.1.0',        fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             withCredentials([string(credentialsId: 'fairmq_codecov_token', variable: 'CODECOV_TOKEN')]) {
               sh './Dart.sh codecov Dart.cfg'

Jenkinsfile.nightly

@@ -21,11 +21,11 @@ def buildMatrix(List specs, Closure callback) {
             echo "export SIMPATH=\${SIMPATH_PREFIX}${fairsoft}" >> Dart.cfg
             echo "export FAIRSOFT_VERSION=${fairsoft}" >> Dart.cfg
           """
-          if (os =~ /Debian/ && compiler =~ /gcc8/) {
+          if (os =~ /Debian/ && compiler =~ /gcc9/) {
             sh '''\
               echo "source /etc/profile.d/modules.sh" >> Dart.cfg
               echo "module use /cvmfs/it.gsi.de/modulefiles" >> Dart.cfg
-              echo "module load compiler/gcc/8" >> Dart.cfg
+              echo "module load compiler/gcc/9.1.0" >> Dart.cfg
             '''
           }
           if (os =~ /MacOS/) {
@@ -46,6 +46,10 @@ def buildMatrix(List specs, Closure callback) {
 
           deleteDir()
         } catch (e) {
+          def tarball = "${label}_dds_logs.tar.gz"
+          sh "tar czvf ${tarball} -C \${WORKSPACE}/build/test/ .DDS/"
+          archiveArtifacts tarball
+
           deleteDir()
           throw e
         }
@@ -63,7 +67,7 @@ pipeline{
       steps{
         script {
           parallel(buildMatrix([
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc9.1.0',        fairsoft: 'fairmq_dev'],
             [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
             [os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
           ]) { spec, label ->

README.md

@@ -33,25 +33,21 @@ configuration and control services.
 FairMQ has been developed in the context of its mother project [FairRoot](https://github.com/FairRootGroup/FairRoot) -
 a simulation, reconstruction and analysis framework.
 
-## Dependencies
-
-  * PUBLIC: [**Boost**](https://www.boost.org/), [**FairLogger**](https://github.com/FairRootGroup/FairLogger)
-  * BUILD: [CMake](https://cmake.org/), [GTest](https://github.com/google/googletest), [Doxygen](http://www.doxygen.org/)
-  * PRIVATE: [ZeroMQ](http://zeromq.org/), [Msgpack](https://msgpack.org/index.html), [nanomsg](http://nanomsg.org/),
-[asiofi](https://github.com/FairRootGroup/asiofi), [DDS](http://dds.gsi.de), [PMIx](https://pmix.org/)
-
-  Supported platforms: Linux and MacOS.
-
 ## Installation from Source
 
+Recommended:
+
 ```bash
-git clone https://github.com/FairRootGroup/FairMQ fairmq
-mkdir fairmq_build && cd fairmq_build
-cmake -DCMAKE_INSTALL_PREFIX=./fairmq_install ../fairmq
-cmake --build . --target install
+git clone https://github.com/FairRootGroup/FairMQ fairmq_source
+cmake -S fairmq_source -B fairmq_build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=fairmq_install
+cmake --build fairmq_build
+cd fairmq_build; ctest -j4; cd ..
+cmake --build fairmq_build --target install
 ```
 
-If dependencies are not installed in standard system directories, you can hint the installation location via `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...`. `{DEPENDENCY}` can be `GTEST`, `BOOST`, `FAIRLOGGER`, `ZEROMQ`, `MSGPACK`, `NANOMSG`, `OFI`, `PMIX`, `ASIOFI` or `DDS` (`*_ROOT` variables can also be environment variables).
+Please consult the [manpages of your CMake version](https://cmake.org/cmake/help/latest/manual/cmake.1.html) for more options.
+
+If dependencies are not installed in standard system directories, you can hint the installation location via `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...`. `{DEPENDENCY}` can be `GTEST`, `BOOST`, `FAIRLOGGER`, `ZEROMQ`, `MSGPACK`, `NANOMSG`, `OFI`, `PMIX`, `ASIO`, `ASIOFI` or `DDS` (`*_ROOT` variables can also be environment variables).
 
 ## Usage
 
@@ -70,30 +66,50 @@ find_package(FairMQ)
 
 `find_package(FairMQ)` will define an imported target `FairMQ::FairMQ`.
 
-In order to succesfully compile and link against the `FairMQ::FairMQ` target, you need to discover its public package dependencies, too.
+In order to succesfully compile and link against the `FairMQ::FairMQ` target, you need to discover its public package dependencies:
 
 ```cmake
 find_package(FairMQ)
 if(FairMQ_FOUND)
-  find_package(FairLogger ${FairMQ_FairLogger_VERSION})
-  find_package(Boost ${FairMQ_Boost_VERSION} COMPONENTS ${FairMQ_Boost_COMPONENTS})
+  foreach(dep IN LISTS FairMQ_PACKAGE_DEPENDENCIES)
+    if(FairMQ_${dep}_COMPONENTS)
+      find_package(${dep} ${FairMQ_${dep}_VERSION} COMPONENTS ${FairMQ_${dep}_COMPONENTS})
+    else()
+      find_package(${dep} ${FairMQ_${dep}_VERSION})
+    endif()
+  endforeach()
 endif()
 ```
 
-Of course, feel free to customize the above commands to your needs.
+If your project shares a dependency with FairMQ or if you want to omit a certain dependency, you may want to customize the above example code to your needs.
 
 Optionally, you can require certain FairMQ package components and a minimum version:
 
 ```cmake
 find_package(FairMQ 1.1.0 COMPONENTS nanomsg_transport dds_plugin)
-if(FairMQ_FOUND)
-  find_package(FairLogger ${FairMQ_FairLogger_VERSION})
-  find_package(Boost ${FairMQ_Boost_VERSION} COMPONENTS ${FairMQ_Boost_COMPONENTS})
-endif()
 ```
 
 When building FairMQ, CMake will print a summary table of all available package components.
 
+## Dependencies
+
+  * [asio](https://github.com/chriskohlhoff/asio) (optionally bundled)
+  * [asiofi](https://github.com/FairRootGroup/asiofi)
+  * [Boost](https://www.boost.org/)
+  * [CMake](https://cmake.org/)
+  * [DDS](http://dds.gsi.de)
+  * [Doxygen](http://www.doxygen.org/)
+  * [FairLogger](https://github.com/FairRootGroup/FairLogger)
+  * [GTest](https://github.com/google/googletest) (optionally bundled)
+  * [Msgpack](https://msgpack.org/index.html)
+  * [nanomsg](http://nanomsg.org/)
+  * [PMIx](https://pmix.org/)
+  * [ZeroMQ](http://zeromq.org/)
+
+  Which dependencies are required depends on which components are built.
+
+  Supported platforms: Linux and MacOS.
+
 ## CMake options
 
 On command line:
@@ -106,7 +122,7 @@ On command line:
   * `-DBUILD_DDS_PLUGIN=ON` enables building of the DDS plugin.
   * `-DBUILD_PMIX_PLUGIN=ON` enables building of the PMIx plugin.
   * `-DBUILD_DOCS=ON` enables building of API docs.
-  * You can hint non-system installations for dependent packages, see the #Installation section above
+  * You can hint non-system installations for dependent packages, see the #installation-from-source section above
 
 After the `find_package(FairMQ)` call the following CMake variables are defined:
 
@@ -165,4 +181,5 @@ After the `find_package(FairMQ)` call the following CMake variables are defined:
    3. [Provided Plugins](docs/Plugins.md#73-provided-plugins)
        1. [DDS](docs/Plugins.md#731-dds)
        2. [PMIx](docs/Plugins.md#732-pmix)
+8. [Controller SDK](docs/SDK.md)
 

cmake/FairMQConfig.cmake.in

@@ -36,4 +36,6 @@ set(CMAKE_MODULE_PATH ${@PROJECT_NAME@_CMAKEMODDIR} ${CMAKE_MODULE_PATH})
 ### Import targets
 include(@PACKAGE_CMAKE_INSTALL_PREFIX@/@PACKAGE_INSTALL_DESTINATION@/@PROJECT_EXPORT_SET@.cmake)
 
+@BUNDLED_PACKAGES@
+
 @PACKAGE_COMPONENTS@

cmake/FairMQLib.cmake

@@ -194,6 +194,13 @@ macro(set_fairmq_defaults)
     endif()
     list(APPEND PROJECT_STATIC_ANALYSERS "${analyser}")
   endif()
+
+  if(CMAKE_GENERATOR STREQUAL Ninja AND ENABLE_CCACHE)
+    find_program(CCACHE ccache)
+    if(CCACHE)
+      set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE})
+    endif()
+  endif()
 endmacro()
 
 function(join VALUES GLUE OUTPUT)
@@ -266,13 +273,30 @@ check_required_components(${PROJECT_NAME})
 set(PACKAGE_COMPONENTS ${PACKAGE_COMPONENTS} PARENT_SCOPE)
 endfunction()
 
+function(generate_bundled_packages)
+  if(asio_BUNDLED)
+  set(BUNDLED_PACKAGES "\
+####### Expanded from @BUNDLED_PACKAGES@ by configure_package_config_file() #########
+
+if(\"\${CMAKE_MAJOR_VERSION}.\${CMAKE_MINOR_VERSION}\" VERSION_LESS 3.11)
+   message(FATAL_ERROR \"CMake >= 3.11 required\")
+endif()
+set_target_properties(${PROJECT_NAME}::bundled_asio_headers PROPERTIES IMPORTED_GLOBAL TRUE)
+add_library(asio::headers ALIAS ${PROJECT_NAME}::bundled_asio_headers)
+set(asio_VERSION ${asio_VERSION})
+
+")
+  endif()
+set(BUNDLED_PACKAGES ${BUNDLED_PACKAGES} PARENT_SCOPE)
+endfunction()
+
 # Configure/Install CMake package
 macro(install_cmake_package)
   list(SORT PROJECT_PACKAGE_DEPENDENCIES)
   list(SORT PROJECT_INTERFACE_PACKAGE_DEPENDENCIES)
   include(CMakePackageConfigHelpers)
   set(PACKAGE_INSTALL_DESTINATION
-    ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}-${PROJECT_GIT_VERSION}
+    ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}-${PROJECT_VERSION}
   )
   if(BUILD_FAIRMQ)
     install(EXPORT ${PROJECT_EXPORT_SET}
@@ -288,6 +312,7 @@ macro(install_cmake_package)
   )
   generate_package_dependencies() # fills ${PACKAGE_DEPENDENCIES}
   generate_package_components() # fills ${PACKAGE_COMPONENTS}
+  generate_bundled_packages() # fills ${BUNDLED_PACKAGES}
   string(TOUPPER ${CMAKE_BUILD_TYPE} PROJECT_BUILD_TYPE_UPPER)
   set(PROJECT_CXX_FLAGS ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${PROJECT_BUILD_TYPE_UPPER}})
   configure_package_config_file(
@@ -312,10 +337,11 @@ endmacro()
 #
 # Wrapper around CMake's native find_package command to add some features and bookkeeping.
 #
-# The qualifier (PRIVATE|PUBLIC|INTERFACE) to the package to populate
+# The qualifier (PRIVATE|PUBLIC|INTERFACE|BUNDLED) to the package to populate
 # the variables PROJECT_[INTERFACE]_<pkgname>_([VERSION]|[COMPONENTS]|PACKAGE_DEPENDENCIES)
 # accordingly. This bookkeeping information is used to print our dependency found summary
-# table and to generate a part of our CMake package.
+# table and to generate a part of our CMake package. BUNDLED decays to PUBLIC if the variable
+# <pkgname>_BUNDLED is false and to PRIVATE otherwise.
 #
 # When a dependending package is listed with ADD_REQUIREMENTS_OF the variables
 # <dep_pkgname>_<pkgname>_VERSION|COMPONENTS are looked up to and added to the native
@@ -372,28 +398,149 @@ macro(find_package2 qualifier pkgname)
     find_package(${pkgname} ${__version__} QUIET ${ARGS_UNPARSED_ARGUMENTS})
   endif()
 
+  if(${qualifier} STREQUAL BUNDLED)
+    if(${pkgname}_BUNDLED)
+      set(__qualifier__ PRIVATE)
+    else()
+      set(__qualifier__ PUBLIC)
+    endif()
+  else()
+    set(__qualifier__ ${qualifier})
+  endif()
+
   if(${pkgname}_FOUND)
-    if(${qualifier} STREQUAL PRIVATE)
+    if(${__qualifier__} STREQUAL PRIVATE)
       set(PROJECT_${pkgname}_VERSION ${__version__})
       set(PROJECT_${pkgname}_COMPONENTS ${__components__})
       set(PROJECT_PACKAGE_DEPENDENCIES ${PROJECT_PACKAGE_DEPENDENCIES} ${pkgname})
-    elseif(${qualifier} STREQUAL PUBLIC)
+    elseif(${__qualifier__} STREQUAL PUBLIC)
       set(PROJECT_${pkgname}_VERSION ${__version__})
       set(PROJECT_${pkgname}_COMPONENTS ${__components__})
       set(PROJECT_PACKAGE_DEPENDENCIES ${PROJECT_PACKAGE_DEPENDENCIES} ${pkgname})
       set(PROJECT_INTERFACE_${pkgname}_VERSION ${__version__})
       set(PROJECT_INTERFACE_${pkgname}_COMPONENTS ${__components__})
       set(PROJECT_INTERFACE_PACKAGE_DEPENDENCIES ${PROJECT_INTERFACE_PACKAGE_DEPENDENCIES} ${pkgname})
-    elseif(${qualifier} STREQUAL INTERFACE)
+    elseif(${__qualifier__} STREQUAL INTERFACE)
       set(PROJECT_INTERFACE_${pkgname}_VERSION ${__version__})
       set(PROJECT_INTERFACE_${pkgname}_COMPONENTS ${__components__})
       set(PROJECT_INTERFACE_PACKAGE_DEPENDENCIES ${PROJECT_INTERFACE_PACKAGE_DEPENDENCIES} ${pkgname})
     endif()
   endif()
 
+  unset(__qualifier__)
   unset(__version__)
   unset(__components__)
   unset(__required_versions__)
   set(CMAKE_PREFIX_PATH ${__old_cpp__})
   unset(__old_cpp__)
 endmacro()
+
+macro(exec cmd)
+  join("${ARGN}" " " args)
+  file(APPEND ${${package}_BUILD_LOGFILE} ">>> ${cmd} ${args}\n")
+
+  execute_process(COMMAND ${cmd} ${ARGN}
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+    OUTPUT_VARIABLE log
+    ERROR_VARIABLE log
+    RESULT_VARIABLE res
+  )
+  file(APPEND ${${package}_BUILD_LOGFILE} ${log})
+
+  if(res)
+    message(FATAL_ERROR "${res} \nSee also \"${${package}_BUILD_LOGFILE}\"")
+  endif()
+endmacro()
+
+macro(exec_source cmd)
+  join("${ARGN}" " " args)
+  file(APPEND ${${package}_BUILD_LOGFILE} ">>> ${cmd} ${args}\n")
+
+  execute_process(COMMAND ${cmd} ${ARGN}
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE log
+    ERROR_VARIABLE log
+    RESULT_VARIABLE res
+  )
+  file(APPEND ${${package}_BUILD_LOGFILE} ${log})
+
+  if(res)
+    message(FATAL_ERROR "${res} \nSee also \"${${package}_BUILD_LOGFILE}\"")
+  endif()
+endmacro()
+
+function(build_bundled package bundle)
+  message(STATUS "Building bundled ${package}")
+
+  set(${package}_SOURCE_DIR ${CMAKE_SOURCE_DIR}/${bundle})
+  set(${package}_BINARY_DIR ${CMAKE_BINARY_DIR}/${bundle})
+  file(MAKE_DIRECTORY ${${package}_BINARY_DIR})
+  set(${package}_BUILD_LOGFILE ${${package}_BINARY_DIR}/build.log)
+  file(REMOVE ${${package}_BUILD_LOGFILE})
+
+  if(Git_FOUND)
+    exec_source(${GIT_EXECUTABLE} submodule update --init -- ${${package}_SOURCE_DIR})
+  endif()
+
+  if(${package} STREQUAL GTest)
+    set(${package}_INSTALL_DIR ${CMAKE_BINARY_DIR}/${bundle}_install)
+    file(MAKE_DIRECTORY ${${package}_INSTALL_DIR})
+    set(${package}_ROOT ${${package}_INSTALL_DIR})
+
+    exec(${CMAKE_COMMAND} -S ${${package}_SOURCE_DIR} -B ${${package}_BINARY_DIR} -G ${CMAKE_GENERATOR}
+      -DCMAKE_INSTALL_PREFIX=${${package}_INSTALL_DIR} -DBUILD_GMOCK=OFF
+    )
+    exec(${CMAKE_COMMAND} --build ${${package}_BINARY_DIR})
+    exec(${CMAKE_COMMAND} --build ${${package}_BINARY_DIR} --target install)
+  elseif(${package} STREQUAL asio)
+    set(${package}_BUILD_INCLUDE_DIR ${${package}_SOURCE_DIR}/asio/include CACHE PATH "Bundled ${package} build-interface include dir")
+    set(${package}_INSTALL_INCLUDE_DIR ${PROJECT_INSTALL_INCDIR}/bundled CACHE PATH "Bundled ${package} install-interface include dir")
+    set(${package}_ROOT ${${package}_SOURCE_DIR}/asio)
+  endif()
+
+  string(TOUPPER ${package} package_upper)
+  set(${package_upper}_ROOT "${${package}_ROOT}" CACHE PATH "Bundled ${package} install prefix search hint")
+  set(${package}_BUNDLED ON CACHE BOOL "Whether bundled ${package} was used")
+
+  message(STATUS "Building bundled ${package} - done")
+endfunction()
+
+macro(fairmq_build_option option description)
+  cmake_parse_arguments(ARGS "" "DEFAULT" "REQUIRES" ${ARGN})
+
+  if(ARGS_DEFAULT)
+    set(__default__ ON)
+  else()
+    set(__default__ OFF)
+  endif()
+
+  if(ARGS_REQUIRES)
+    include(CMakeDependentOption)
+    set(__requires__ ${ARGS_REQUIRES})
+    foreach(d ${__requires__})
+      string(REGEX REPLACE " +" ";" __requires_condition__ "${d}")
+      if(${__requires_condition__})
+      else()
+        if(${option})
+          message(WARNING "Cannot enable build option ${option}, depending option is not set: ${__requires_condition__}")
+        endif()
+      endif()
+    endforeach()
+  else()
+    set(__requires__)
+  endif()
+
+  if(__requires__)
+    cmake_dependent_option(${option} ${description} ${__default__} "${__requires__}" OFF)
+  else()
+    option(${option} ${description} ${__default__})
+  endif()
+
+  set(__default__)
+  set(__requires__)
+  set(__requires_condition__)
+  set(ARGS_DEFAULT)
+  set(ARGS_REQUIRES)
+  set(option)
+  set(description)
+endmacro()

cmake/Findasio.cmake

@@ -0,0 +1,53 @@
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+find_path(asio_INCLUDE_DIR
+  NAMES asio.hpp
+  PATH_SUFFIXES include
+)
+
+if(asio_INCLUDE_DIR)
+  find_file(asio_VERSION_HEADER "asio/version.hpp"
+    ${asio_INCLUDE_DIR}
+    NO_DEFAULT_PATH
+  )
+endif()
+
+if(asio_VERSION_HEADER)
+  file(READ "${asio_VERSION_HEADER}" _asio_VERSION_HEADER_CONTENT)
+  string(REGEX MATCH "#define ASIO_VERSION ([0-9]+)" _MATCH "${_asio_VERSION_HEADER_CONTENT}")
+  set(asio_VERSION_MACRO ${CMAKE_MATCH_1})
+  math(EXPR asio_VERSION_MAJOR "${asio_VERSION_MACRO} / 100000")
+  math(EXPR asio_VERSION_MINOR "${asio_VERSION_MACRO} / 100 % 1000")
+  math(EXPR asio_VERSION_PATCH "${asio_VERSION_MACRO} % 100")
+  set(asio_VERSION "${asio_VERSION_MAJOR}.${asio_VERSION_MINOR}.${asio_VERSION_PATCH}")
+endif()
+
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(asio
+  REQUIRED_VARS asio_INCLUDE_DIR
+  VERSION_VAR asio_VERSION
+  HANDLE_COMPONENTS
+)
+
+if(asio_FOUND AND asio_BUNDLED)
+  add_library(bundled_asio_headers INTERFACE)
+  target_include_directories(bundled_asio_headers INTERFACE
+    $<BUILD_INTERFACE:${asio_BUILD_INCLUDE_DIR}>
+    $<INSTALL_INTERFACE:${asio_INSTALL_INCLUDE_DIR}>
+  )
+  add_library(asio::headers ALIAS bundled_asio_headers)
+endif()
+if(asio_FOUND AND NOT TARGET asio::headers)
+  add_library(asio::headers INTERFACE IMPORTED)
+  set_target_properties(asio::headers PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${asio_INCLUDE_DIR}"
+  )
+endif()
+

docs/SDK.md

@@ -0,0 +1,34 @@
+← [Back](../README.md)
+
+# 8. Controller SDK
+
+The FairMQ Controller Software Development Kit (`-DBUILD_SDK=ON`) contains a (as of today still experimental) set of C++ APIs that provide essential functionality to the implementer of a global controller.
+
+The FairMQ core library only provides two local controllers - `static` (a fixed sequence of state transitions) and `interactive` (a read-eval-print-loop which reads keyboard commands from standard input). A local controller only knows how steer a single [FairMQ device](Device.md) - in fact, it runs in a thread within the device process.
+
+A global controller has knowledge about the full topology of connected FairMQ devices. Its responsibility is to facilitate the lifecycle of a distributed FairMQ-based application (*executing a topology*), such as
+
+* allocating/releasing compute resources from a resource management system,
+* launching/setting up the run-time environment and the FairMQ devices,
+* driving the device state machines in lock-step across the full topology,
+* pushing the device configuration,
+* monitoring (some aspects of the application's) operation,
+* and handling/reporting (some) error cases.
+
+The low-level hook to integrate FairMQ devices with such a global contoller is the [plugin mechanism](Plugins.md) in the FairMQ core library. The FairMQ Controller SDK provides C++ APIs that communicate to the endpoints exposed by such a FairMQ plugin.
+
+At the moment, the Controller SDK only supports [DDS](https://dds.gsi.de) as resource manager and run-time environment. A second implementation based on [PMIx](https://pmix.org/) (targeting its implementation in [Slurm](https://slurm.schedmd.com/documentation.html) and [OpenRTE](https://www-lb.open-mpi.org/papers/euro-pvmmpi-2005-orte/)) is in development.
+
+The following section give a short overview on the APIs provided.
+
+## RMS and run-time environment
+
+The classes [`fair::mq::sdk::DDSEnvironment`](../fairmq/sdk/DDSEnvironment.h), [`fair::mq::sdk::DDSSession`](../fairmq/sdk/DDSSession.h), and [`fair::mq::sdk::DDSTopology`](../fairmq/sdk/DDSTopology.h) are thin wrappers of most of the synchronous APIs exposed by DDS ([`dds::tools_api`](http://dds.gsi.de/doc/api-docs/DDS/html/namespacedds_1_1tools__api.html) and [`dds::topology_api`](http://dds.gsi.de/doc/api-docs/DDS/html/namespacedds_1_1topology__api.html)). E.g. they allow to [start a DDS session](https://github.com/FairRootGroup/FairMQ/blob/077eb0ef691940d764cfd1852bf3981dc812ddbd/main.cpp#L26-L28), [allocate resources](https://github.com/FairRootGroup/FairMQ/blob/077eb0ef691940d764cfd1852bf3981dc812ddbd/main.cpp#L34) and [launch a topology](https://github.com/FairRootGroup/FairMQ/blob/077eb0ef691940d764cfd1852bf3981dc812ddbd/main.cpp#L39) from a C++ program.
+
+## Driving the global state machine
+
+The class [`fair::mq::sdk::Topology`](../fairmq/sdk/Topology.h) adds a FairMQ-specific view on an existing DDS session that is executing a topology of FairMQ devices. One can e.g. [initiate a state transition on all devices in the topology simultaneously](https://github.com/FairRootGroup/FairMQ/blob/077eb0ef691940d764cfd1852bf3981dc812ddbd/main.cpp#L48-L49). This topology transition completes once a topology-wide barrier is passed (all devices completed the transition). This effectively exposes the device state machine as a topology state machine. The implementation is based on remote procedure calls over the [DDS intercom service](http://dds.gsi.de/doc/api-docs/DDS/html/namespacedds_1_1intercom__api.html) between the controller and the DDS plugin shipped with FairMQ (`-DBUILD_DDS_PLUGIN=ON`).
+
+For future versions of the SDK new APIs are planned to inspect and modify the device configurations and also operate only on subsets of a given topology.
+
+← [Back](../README.md)

examples/1-1/Sampler.cxx

@@ -6,9 +6,6 @@
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#include <thread> // this_thread::sleep_for
-#include <chrono>
-
 #include "Sampler.h"
 
 using namespace std;

examples/dds/CMakeLists.txt

@@ -37,6 +37,13 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-dds-env.sh ${CMAKE_CURRENT_
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-dds.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh @ONLY)
 
 # test
+if(DDS_FOUND)
+    add_test(NAME Example.DDS.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh localhost)
+    set_tests_properties(Example.DDS.localhost PROPERTIES
+    TIMEOUT 15
+    PASS_REGULAR_EXPRESSION "Example successful"
+    )
+endif()
 
 # install
 

examples/dds/ex-dds-topology-infinite.xml

@@ -1,43 +1,43 @@
 <topology name="ExampleDDS">
 
-    <property name="data1" />
-    <property name="data2" />
+    <property name="fmqchan_data1" />
+    <property name="fmqchan_data2" />
 
     <declrequirement name="SamplerWorker" type="wnname" value="sampler"/>
     <declrequirement name="ProcessorWorker" type="wnname" value="processor"/>
     <declrequirement name="SinkWorker" type="wnname" value="sink"/>
 
     <decltask name="Sampler">
+        <exe>fairmq-ex-dds-sampler --color false --channel-config name=data1,type=push,method=bind --rate 100 -P dds</exe>
         <env reachable="false">fairmq-ex-dds-env.sh</env>
-        <exe>fairmq-ex-dds-sampler --id sampler --color false --channel-config name=data1,type=push,method=bind --rate 100 -P dds</exe>
         <requirements>
             <name>SamplerWorker</name>
         </requirements>
         <properties>
-            <name access="write">data1</name>
+            <name access="write">fmqchan_data1</name>
         </properties>
     </decltask>
 
     <decltask name="Processor">
+        <exe>fairmq-ex-dds-processor --color false --channel-config name=data1,type=pull,method=connect name=data2,type=push,method=connect -P dds</exe>
         <env reachable="false">fairmq-ex-dds-env.sh</env>
-        <exe>fairmq-ex-dds-processor --id processor_%taskIndex% --color false --channel-config name=data1,type=pull,method=connect name=data2,type=push,method=connect -P dds</exe>
         <requirements>
-            <id>ProcessorWorker</id>
+            <name>ProcessorWorker</name>
         </requirements>
         <properties>
-            <name access="read">data1</name>
-            <name access="read">data2</name>
+            <name access="read">fmqchan_data1</name>
+            <name access="read">fmqchan_data2</name>
         </properties>
     </decltask>
 
     <decltask name="Sink">
+        <exe>fairmq-ex-dds-sink --color false --channel-config name=data2,type=pull,method=bind -P dds</exe>
         <env reachable="false">fairmq-ex-dds-env.sh</env>
-        <exe>fairmq-ex-dds-sink --id sink --color false --channel-config name=data2,type=pull,method=bind -P dds</exe>
         <requirements>
             <name>SinkWorker</name>
         </requirements>
         <properties>
-            <name access="write">data2</name>
+            <name access="write">fmqchan_data2</name>
         </properties>
     </decltask>
 

examples/dds/ex-dds-topology.xml

@@ -1,43 +1,43 @@
 <topology name="ExampleDDS">
 
-    <property name="data1" />
-    <property name="data2" />
+    <property name="fmqchan_data1" />
+    <property name="fmqchan_data2" />
 
     <declrequirement name="SamplerWorker" type="wnname" value="sampler"/>
     <declrequirement name="ProcessorWorker" type="wnname" value="processor"/>
     <declrequirement name="SinkWorker" type="wnname" value="sink"/>
 
     <decltask name="Sampler">
+        <exe>fairmq-ex-dds-sampler --color false --channel-config name=data1,type=push,method=bind -P dds --iterations 10</exe>
         <env reachable="false">fairmq-ex-dds-env.sh</env>
-        <exe>fairmq-ex-dds-sampler --id sampler --color false --channel-config name=data1,type=push,method=bind -P dds --iterations 10</exe>
         <requirements>
             <name>SamplerWorker</name>
         </requirements>
         <properties>
-            <name access="write">data1</name>
+            <name access="write">fmqchan_data1</name>
         </properties>
     </decltask>
 
     <decltask name="Processor">
+        <exe>fairmq-ex-dds-processor --color false --channel-config name=data1,type=pull,method=connect name=data2,type=push,method=connect -P dds</exe>
         <env reachable="false">fairmq-ex-dds-env.sh</env>
-        <exe>fairmq-ex-dds-processor --id processor_%taskIndex% --color false --channel-config name=data1,type=pull,method=connect name=data2,type=push,method=connect -P dds</exe>
         <requirements>
-            <id>ProcessorWorker</id>
+            <name>ProcessorWorker</name>
         </requirements>
         <properties>
-            <name access="read">data1</name>
-            <name access="read">data2</name>
+            <name access="read">fmqchan_data1</name>
+            <name access="read">fmqchan_data2</name>
         </properties>
     </decltask>
 
     <decltask name="Sink">
+        <exe>fairmq-ex-dds-sink --color false --channel-config name=data2,type=pull,method=bind -P dds --iterations 10</exe>
         <env reachable="false">fairmq-ex-dds-env.sh</env>
-        <exe>fairmq-ex-dds-sink --id sink --color false --channel-config name=data2,type=pull,method=bind -P dds --iterations 10</exe>
         <requirements>
             <name>SinkWorker</name>
         </requirements>
         <properties>
-            <name access="write">data2</name>
+            <name access="write">fmqchan_data2</name>
         </properties>
     </decltask>
 

examples/dds/fairmq-start-ex-dds.sh.in

@@ -30,56 +30,57 @@ echo "SESSION ID: ${DDS_SESSION_ID}"
 
 trap "cleanup ${DDS_SESSION_ID}" EXIT
 
-requiredNofAgents=12
+requiredNofSlots=12
 if [[ "$plugin" == "ssh" ]]; then
   dds-submit -r ${plugin} -c @DATA_DIR@/ex-dds-hosts.cfg
 else
-  dds-submit -r ${plugin} -n ${requiredNofAgents}
+  dds-submit -r ${plugin} --slots ${requiredNofSlots}
 fi
-echo "...waiting for ${requiredNofAgents} idle agents..."
-dds-info --wait-for-idle-agents ${requiredNofAgents}
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
 
 topologyFile=@DATA_DIR@/ex-dds-topology.xml
 echo "TOPOLOGY FILE: ${topologyFile}"
-# TODO Uncomment once DDS 2.6 is released
-# echo "TOPOLOGY NAME: $(dds-topology --disable-validation --topology-name ${topologyFile})"
+echo "TOPOLOGY NAME: $(dds-topology --disable-validation --topology-name ${topologyFile})"
 
-# TODO Uncomment once DDS 2.6 is released
-# dds-info --active-topology
-dds-topology --disable-validation --activate ${topologyFile}
-# dds-info --active-topology
-# dds-info --wait-for-executing-agents ${requiredNofAgents}
-sleep 1
+dds-info --active-topology
+dds-topology --activate ${topologyFile}
+dds-info --active-topology
+echo "...waiting for ${requiredNofSlots} executing slots..."
+dds-info --executing-count --wait ${requiredNofSlots}
 
 echo "------------------------"
 echo "...waiting for Topology to finish..."
 # TODO Retrieve number of devices from DDS topology API instead of having the user pass it explicitely
-fairmq-dds-command-ui -w "IDLE" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c i -w "INITIALIZING DEVICE" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c k -w "INITIALIZED" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c b -w "BOUND" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c x -w "DEVICE READY" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c j -w "READY" -n ${requiredNofAgents}
+fairmq-dds-command-ui -w "IDLE" -n ${requiredNofSlots}
+fairmq-dds-command-ui -c i -w "INITIALIZING DEVICE" -n ${requiredNofSlots}
+fairmq-dds-command-ui -c k -w "INITIALIZED" -n ${requiredNofSlots}
+fairmq-dds-command-ui -c b -w "BOUND" -n ${requiredNofSlots}
+fairmq-dds-command-ui -c x -w "DEVICE READY" -n ${requiredNofSlots}
+fairmq-dds-command-ui -c j -w "READY" -n ${requiredNofSlots}
 fairmq-dds-command-ui -c r
 sampler_and_sink="main/(Sampler|Sink)"
+# processors="main/ProcessorGroup/Processor"
 fairmq-dds-command-ui -p $sampler_and_sink -w "RUNNING->READY" -n 2
 echo "...$sampler_and_sink are READY, sending shutdown..."
-fairmq-dds-command-ui -c s -w "RUNNING->READY" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c t -w "DEVICE READY" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c d -w "IDLE" -n ${requiredNofAgents}
-fairmq-dds-command-ui -c q
-echo "...waiting for ${requiredNofAgents} idle agents..."
-dds-info --wait-for-idle-agents ${requiredNofAgents}
+fairmq-dds-command-ui -c s -w "RUNNING->READY" -n ${requiredNofSlots}
+fairmq-dds-command-ui -c t -w "DEVICE READY" -n ${requiredNofSlots}
+fairmq-dds-command-ui -c d -w "IDLE" -n ${requiredNofSlots}
+fairmq-dds-command-ui -c q -w "EXITING" -n ${requiredNofSlots}
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
 echo "------------------------"
 
-# TODO Uncomment once DDS 2.6 is released
-# dds-info --active-topology
+dds-info --active-topology
 dds-topology --stop
-# dds-info --active-topology
+dds-info --active-topology
 
 dds-agent-cmd getlog -a
 logDir="${wrkDir}/logs"
 for file in $(find "${logDir}" -name "*.tar.gz"); do tar -xf ${file} -C "${logDir}" ; done
 echo "AGENT LOG FILES IN: ${logDir}"
 
+# This string is used by ctest to detect success
+echo "Example successful :)"
+
 # Cleanup function is called by EXIT trap

examples/multipart/CMakeLists.txt

@@ -32,19 +32,19 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multipart.sh.in ${CMA
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multipart.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh)
 
 add_test(NAME Example.Multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh zeromq)
-set_tests_properties(Example.Multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 5 parts")
+set_tests_properties(Example.Multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
 
 if(BUILD_NANOMSG_TRANSPORT)
   add_test(NAME Example.Multipart.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh nanomsg)
-  set_tests_properties(Example.Multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 5 parts")
+  set_tests_properties(Example.Multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
 endif()
 
 add_test(NAME Example.Multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh shmem)
-set_tests_properties(Example.Multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 5 parts")
+set_tests_properties(Example.Multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
 
 if(BUILD_OFI_TRANSPORT)
   add_test(NAME Example.Multipart.ofi COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh ofi)
-  set_tests_properties(Example.Multipart.ofi PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 5 parts")
+  set_tests_properties(Example.Multipart.ofi PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
 endif()
 
 # install

examples/multipart/Sampler.cxx

@@ -40,8 +40,7 @@ bool Sampler::ConditionalRun()
     header.stopFlag = 0;
 
     // Set stopFlag to 1 for last message.
-    if (fMaxIterations > 0 && fNumIterations == fMaxIterations - 1)
-    {
+    if (fMaxIterations > 0 && fNumIterations == fMaxIterations - 1) {
         header.stopFlag = 1;
     }
     LOG(info) << "Sending header with stopFlag: " << header.stopFlag;
@@ -60,13 +59,20 @@ bool Sampler::ConditionalRun()
     assert(auxData.Size() == 0);
     assert(parts.Size() == 5);
 
+    parts.AddPart(NewMessage());
+
+    assert(parts.Size() == 6);
+
+    parts.AddPart(NewMessage(100));
+
+    assert(parts.Size() == 7);
+
     LOG(info) << "Sending body of size: " << parts.At(1)->GetSize();
 
     Send(parts, "data");
 
     // Go out of the sending loop if the stopFlag was sent.
-    if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations)
-    {
+    if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
         LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
         return false;
     }
@@ -77,8 +83,4 @@ bool Sampler::ConditionalRun()
     return true;
 }
 
-Sampler::~Sampler()
-{
-}
-
 } // namespace example_multipart

examples/multipart/Sampler.h

@@ -24,7 +24,7 @@ class Sampler : public FairMQDevice
 {
   public:
     Sampler();
-    virtual ~Sampler();
+    virtual ~Sampler() {}
 
   protected:
     virtual void InitTask();

examples/multipart/Sink.cxx

@@ -20,23 +20,28 @@ using namespace std;
 namespace example_multipart
 {
 
-Sink::Sink()
-{
-    OnData("data", &Sink::HandleData);
-}
-
 bool Sink::HandleData(FairMQParts& parts, int /*index*/)
 {
+    LOG(info) << "Received message with " << parts.Size() << " parts";
+
     Header header;
     header.stopFlag = (static_cast<Header*>(parts.At(0)->GetData()))->stopFlag;
 
-    LOG(info) << "Received message with " << parts.Size() << " parts";
+    LOG(info) << "Received part 1 (header) with stopFlag: " << header.stopFlag;
+    LOG(info) << "Received part 2 of size: " << parts.At(1)->GetSize();
+    assert(parts.At(1)->GetSize() == 1000);
+    LOG(info) << "Received part 3 of size: " << parts.At(2)->GetSize();
+    assert(parts.At(2)->GetSize() == 500);
+    LOG(info) << "Received part 4 of size: " << parts.At(3)->GetSize();
+    assert(parts.At(3)->GetSize() == 600);
+    LOG(info) << "Received part 5 of size: " << parts.At(4)->GetSize();
+    assert(parts.At(4)->GetSize() == 700);
+    LOG(info) << "Received part 6 of size: " << parts.At(5)->GetSize();
+    assert(parts.At(5)->GetSize() == 0);
+    LOG(info) << "Received part 7 of size: " << parts.At(6)->GetSize();
+    assert(parts.At(6)->GetSize() == 100);
 
-    LOG(info) << "Received header with stopFlag: " << header.stopFlag;
-    LOG(info) << "Received body of size: " << parts.At(1)->GetSize();
-
-    if (header.stopFlag == 1)
-    {
+    if (header.stopFlag == 1) {
         LOG(info) << "stopFlag is 1, going IDLE";
         return false;
     }
@@ -44,8 +49,4 @@ bool Sink::HandleData(FairMQParts& parts, int /*index*/)
     return true;
 }
 
-Sink::~Sink()
-{
-}
-
 }

examples/multipart/Sink.h

@@ -23,8 +23,8 @@ namespace example_multipart
 class Sink : public FairMQDevice
 {
   public:
-    Sink();
-    virtual ~Sink();
+    Sink() { OnData("data", &Sink::HandleData); }
+    virtual ~Sink() {}
 
   protected:
     bool HandleData(FairMQParts&, int);

examples/multipart/fairmq-start-ex-multipart.sh.in

@@ -2,12 +2,24 @@
 
 export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
 
+transport="zeromq"
+
+if [[ $1 =~ ^[a-z]+$ ]]; then
+    transport=$1
+fi
+
+SESSION="$(@CMAKE_BINARY_DIR@/fairmq/fairmq-uuid-gen -h)"
+
 SAMPLER="fairmq-ex-multipart-sampler"
 SAMPLER+=" --id sampler1"
+SAMPLER+=" --transport $transport"
+SAMPLER+=" --session $SESSION"
 SAMPLER+=" --channel-config name=data,type=push,method=connect,rateLogging=0,address=tcp://127.0.0.1:5555"
 xterm -geometry 80x23+0+0 -hold -e @EX_BIN_DIR@/$SAMPLER &
 
 SINK="fairmq-ex-multipart-sink"
 SINK+=" --id sink1"
+SINK+=" --transport $transport"
+SINK+=" --session $SESSION"
 SINK+=" --channel-config name=data,type=pull,method=bind,rateLogging=0,address=tcp://127.0.0.1:5555"
 xterm -geometry 80x23+500+0 -hold -e @EX_BIN_DIR@/$SINK &

examples/readout/fairmq-start-ex-readout-processing.sh.in

@@ -31,10 +31,12 @@ SENDER="fairmq-ex-readout-sender"
 SENDER+=" --id sender1"
 SENDER+=" --input-name ps"
 SENDER+=" --channel-config name=ps,type=pair,method=bind,address=tcp://localhost:7779,transport=shmem"
-SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7780,transport=ofi"
+#SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7780,transport=ofi"
+SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7780,transport=zeromq"
 xterm -geometry 80x23+1000+0 -hold -e @EX_BIN_DIR@/$SENDER &
 
 RECEIVER="fairmq-ex-readout-receiver"
 RECEIVER+=" --id receiver1"
-RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7780,transport=ofi"
+#RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7780,transport=ofi"
+RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7780,transport=zeromq"
 xterm -geometry 80x23+1500+0 -hold -e @EX_BIN_DIR@/$RECEIVER &

examples/readout/fairmq-start-ex-readout.sh.in

@@ -25,10 +25,12 @@ SENDER="fairmq-ex-readout-sender"
 SENDER+=" --id sender1"
 SENDER+=" --input-name bs"
 SENDER+=" --channel-config name=bs,type=pair,method=bind,address=tcp://localhost:7778,transport=shmem"
-SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7779,transport=ofi"
+# SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7779,transport=ofi"
+SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7779,transport=zeromq"
 xterm -geometry 80x23+1000+0 -hold -e @EX_BIN_DIR@/$SENDER &
 
 RECEIVER="fairmq-ex-readout-receiver"
 RECEIVER+=" --id receiver1"
-RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7779,transport=ofi"
+# RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7779,transport=ofi"
+RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7779,transport=zeromq"
 xterm -geometry 80x23+1500+0 -hold -e @EX_BIN_DIR@/$RECEIVER &

extern/bundled_asio.cmake

@@ -0,0 +1,17 @@
+################################################################################
+#     Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH   #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+if(NOT TARGET asio::headers)
+  if(Git_FOUND)
+    execute_process(
+      COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive --depth 1 -- extern/asio
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    )
+  endif()
+
+endif()

fairmq/CMakeLists.txt

@@ -58,6 +58,12 @@ if(BUILD_FAIRMQ OR BUILD_SDK)
     ${TOOLS_PUBLIC_HEADER_FILES}
   )
   target_compile_definitions(${target} PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
+  # workaround https://github.com/boostorg/asio/commit/43874d5497414c67655d901e48c939ef01337edb
+  if(    Boost_VERSION VERSION_LESS 1.69
+     AND CMAKE_CXX_COMPILER_ID STREQUAL AppleClang
+     AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 10.0.1)
+    target_compile_definitions(${target} PUBLIC BOOST_ASIO_HAS_STD_STRING_VIEW)
+  endif()
   target_include_directories(${target}
     PUBLIC
     $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}>
@@ -70,7 +76,7 @@ if(BUILD_FAIRMQ OR BUILD_SDK)
     Boost::boost
   )
   set_target_properties(${target} PROPERTIES
-    VERSION ${PROJECT_GIT_VERSION}
+    VERSION ${PROJECT_VERSION}
     SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
     OUTPUT_NAME FairMQ${target}
   )
@@ -124,7 +130,7 @@ if(BUILD_FAIRMQ OR BUILD_SDK)
     Tools
   )
   set_target_properties(${target} PROPERTIES
-    VERSION ${PROJECT_GIT_VERSION}
+    VERSION ${PROJECT_VERSION}
     SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
     OUTPUT_NAME FairMQ${target}
   )
@@ -371,7 +377,7 @@ if(BUILD_FAIRMQ)
     ${OFI_DEPS}
   )
   set_target_properties(${_target} PROPERTIES
-    VERSION ${PROJECT_GIT_VERSION}
+    VERSION ${PROJECT_VERSION}
     SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
   )
 
@@ -460,6 +466,10 @@ if(BUILD_FAIRMQ)
   endforeach()
 endif()
 
+if(BUILD_SDK_COMMANDS)
+  add_subdirectory(sdk/commands)
+endif()
+
 if(BUILD_SDK)
   add_subdirectory(sdk)
 endif()

fairmq/DeviceRunner.cxx

@@ -63,8 +63,6 @@ bool DeviceRunner::HandleGeneralOptions(const fair::mq::ProgOptions& config, boo
                 << "   / __/ / /_/ / / /    _  /  / / / /_/ /     " << FAIRMQ_REPO_URL << endl
                 << "  /_/    \\__,_/_/_/     /_/  /_/  \\___\\_\\     " << FAIRMQ_LICENSE << "  © " << FAIRMQ_COPYRIGHT << endl;
         }
-
-        config.PrintOptions();
     }
 
     return true;
@@ -169,6 +167,9 @@ auto DeviceRunner::Run() -> int
     // Instantiate and run plugins
     fPluginManager.InstantiatePlugins();
 
+    // Log IDLE configuration
+    fConfig.PrintOptions();
+
     // Run the device
     fDevice->RunStateMachine();
 

fairmq/EventManager.h

@@ -12,12 +12,10 @@
 #include <memory>
 #include <mutex>
 #include <string>
-#include <tuple>
 #include <typeindex>
-#include <typeinfo>
 #include <unordered_map>
 #include <utility>
-#include <vector>
+#include <functional>
 
 #include <boost/any.hpp>
 #include <boost/functional/hash.hpp>

fairmq/FairMQChannel.cxx

@@ -7,7 +7,8 @@
  ********************************************************************************/
 
 #include "FairMQChannel.h"
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Strings.h>
+#include <fairmq/Properties.h>
 
 #include <boost/algorithm/string.hpp> // join/split
 

fairmq/FairMQChannel.h

@@ -10,6 +10,7 @@
 #define FAIRMQCHANNEL_H_
 
 #include <FairMQTransportFactory.h>
+#include <FairMQUnmanagedRegion.h>
 #include <FairMQSocket.h>
 #include <fairmq/Transports.h>
 #include <FairMQLogger.h>
@@ -17,15 +18,14 @@
 #include <fairmq/Properties.h>
 #include <FairMQMessage.h>
 
-#include <boost/any.hpp>
-
 #include <string>
 #include <memory> // unique_ptr, shared_ptr
 #include <vector>
-#include <atomic>
 #include <mutex>
 #include <stdexcept>
 #include <utility> // std::move
+#include <cstddef> // size_t
+#include <cstdint> // int64_t
 
 class FairMQChannel
 {
@@ -68,13 +68,13 @@ class FairMQChannel
     FairMQChannel(const FairMQChannel&, const std::string& name);
 
     /// Move constructor
-    FairMQChannel(FairMQChannel&&) = default;
+    FairMQChannel(FairMQChannel&&) = delete;
 
     /// Assignment operator
     FairMQChannel& operator=(const FairMQChannel&);
 
     /// Move assignment operator
-    FairMQChannel& operator=(FairMQChannel&&) = default;
+    FairMQChannel& operator=(FairMQChannel&&) = delete;
 
     /// Destructor
     virtual ~FairMQChannel()

fairmq/FairMQDevice.cxx

@@ -8,19 +8,15 @@
 
 #include <FairMQDevice.h>
 
+#include <fairmq/tools/RateLimit.h>
+#include <fairmq/tools/Network.h>
+
 #include <boost/algorithm/string.hpp> // join/split
 
-#include <boost/uuid/uuid.hpp>
-#include <boost/uuid/uuid_generators.hpp>
-#include <boost/uuid/uuid_io.hpp>
-
 #include <list>
-#include <cstdlib>
 #include <chrono>
 #include <mutex>
 #include <thread>
-#include <functional>
-#include <sstream>
 #include <iomanip>
 #include <future>
 #include <algorithm> // std::max
@@ -482,32 +478,6 @@ void FairMQDevice::InitTaskWrapper()
     ChangeState(Transition::Auto);
 }
 
-bool FairMQDevice::SortSocketsByAddress(const FairMQChannel &lhs, const FairMQChannel &rhs)
-{
-    return lhs.fAddress < rhs.fAddress;
-}
-
-void FairMQDevice::SortChannel(const string& name, const bool reindex)
-{
-    if (fChannels.find(name) != fChannels.end())
-    {
-        sort(fChannels.at(name).begin(), fChannels.at(name).end(), SortSocketsByAddress);
-
-        if (reindex)
-        {
-            for (auto vi = fChannels.at(name).begin(); vi != fChannels.at(name).end(); ++vi)
-            {
-                // set channel name: name + vector index
-                vi->fName = tools::ToString(name, "[", vi - fChannels.at(name).begin(), "]");
-            }
-        }
-    }
-    else
-    {
-        LOG(error) << "Sorting failed: no channel with the name \"" << name << "\".";
-    }
-}
-
 void FairMQDevice::RunWrapper()
 {
     LOG(info) << "DEVICE: Running...";

fairmq/FairMQDevice.h

@@ -14,7 +14,6 @@
 #include <fairmq/Transports.h>
 #include <fairmq/StateQueue.h>
 
-#include <FairMQSocket.h>
 #include <FairMQChannel.h>
 #include <FairMQMessage.h>
 #include <FairMQParts.h>
@@ -24,21 +23,19 @@
 
 #include <vector>
 #include <memory> // unique_ptr
-#include <algorithm> // std::sort()
+#include <algorithm> // find
 #include <string>
 #include <chrono>
 #include <iostream>
 #include <unordered_map>
 #include <functional>
-#include <assert.h> // static_assert
-#include <type_traits> // is_trivially_copyable
 #include <stdexcept>
-#include <queue>
-
 #include <mutex>
-#include <condition_variable>
+#include <atomic>
+#include <cstddef>
+#include <utility> // pair
 
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Version.h>
 
 using FairMQChannelMap = std::unordered_map<std::string, std::vector<FairMQChannel>>;
 
@@ -115,11 +112,6 @@ class FairMQDevice
     /// Outputs the socket transfer rates
     virtual void LogSocketRates();
 
-    /// Sorts a channel by address, with optional reindexing of the sorted values
-    /// @param name    Channel name
-    /// @param reindex Should reindexing be done
-    void SortChannel(const std::string& name, const bool reindex = true);
-
     template<typename Serializer, typename DataType, typename... Args>
     void Serialize(FairMQMessage& msg, DataType&& data, Args&&... args) const
     {
@@ -154,15 +146,6 @@ class FairMQDevice
         return GetChannel(channel, index).Receive(msg, rcvTimeoutInMs);
     }
 
-    int SendAsync(FairMQMessagePtr& msg, const std::string& channel, const int index = 0) __attribute__((deprecated("For non-blocking Send, use timeout version with timeout of 0: Send(msg, \"channelA\", subchannelIndex, timeout);")))
-    {
-        return GetChannel(channel, index).Send(msg, 0);
-    }
-    int ReceiveAsync(FairMQMessagePtr& msg, const std::string& channel, const int index = 0) __attribute__((deprecated("For non-blocking Receive, use timeout version with timeout of 0: Receive(msg, \"channelA\", subchannelIndex, timeout);")))
-    {
-        return GetChannel(channel, index).Receive(msg, 0);
-    }
-
     /// Shorthand method to send FairMQParts on `chan` at index `i`
     /// @param parts parts reference
     /// @param chan channel name
@@ -185,15 +168,6 @@ class FairMQDevice
         return GetChannel(channel, index).Receive(parts.fParts, rcvTimeoutInMs);
     }
 
-    int64_t SendAsync(FairMQParts& parts, const std::string& channel, const int index = 0) __attribute__((deprecated("For non-blocking Send, use timeout version with timeout of 0: Send(parts, \"channelA\", subchannelIndex, timeout);")))
-    {
-        return GetChannel(channel, index).Send(parts.fParts, 0);
-    }
-    int64_t ReceiveAsync(FairMQParts& parts, const std::string& channel, const int index = 0) __attribute__((deprecated("For non-blocking Receive, use timeout version with timeout of 0: Receive(parts, \"channelA\", subchannelIndex, timeout);")))
-    {
-        return GetChannel(channel, index).Receive(parts.fParts, 0);
-    }
-
     /// @brief Getter for default transport factory
     auto Transport() const -> FairMQTransportFactory*
     {
@@ -297,9 +271,6 @@ class FairMQDevice
         return channels.at(0)->Transport()->CreatePoller(channels);
     }
 
-    /// Waits for the first initialization run to finish
-    void WaitForInitialValidation() __attribute__((deprecated("This method will have no effect in future versions and will be removed. Instead subscribe for state changes and inspect configuration values."))) {}
-
     /// Adds a transport to the device if it doesn't exist
     /// @param transport  Transport string ("zeromq"/"nanomsg"/"shmem")
     std::shared_ptr<FairMQTransportFactory> AddTransport(const fair::mq::Transport transport);
@@ -312,11 +283,6 @@ class FairMQDevice
         return fConfig;
     }
 
-    /// Implements the sort algorithm used in SortChannel()
-    /// @param lhs Right hand side value for comparison
-    /// @param rhs Left hand side value for comparison
-    static bool SortSocketsByAddress(const FairMQChannel &lhs, const FairMQChannel &rhs);
-
     // overload to easily bind member functions
     template<typename T>
     void OnData(const std::string& channelName, bool (T::* memberFunction)(FairMQMessagePtr& msg, int index))

fairmq/FairMQParts.h

@@ -33,7 +33,7 @@ class FairMQParts
     FairMQParts& operator=(const FairMQParts&) = delete;
     /// Constructor from argument pack of std::unique_ptr<FairMQMessage> rvalues
     template <typename... Ts>
-    FairMQParts(Ts&&... messages) : fParts() {AddPart(std::forward<Ts>(messages)...);}
+    FairMQParts(Ts&&... messages) : fParts() { AddPart(std::forward<Ts>(messages)...); }
     /// Default destructor
     ~FairMQParts() {};
 
@@ -63,10 +63,10 @@ class FairMQParts
     /// Add content of another object by move
     void AddPart(FairMQParts&& other)
     {
-      container parts = std::move(other.fParts);
-      for (auto& part : parts) {
-	fParts.push_back(std::move(part));
-      }
+        container parts = std::move(other.fParts);
+        for (auto& part : parts) {
+            fParts.push_back(std::move(part));
+        }
     }
 
     /// Get reference to part in the container at index (without bounds check)

fairmq/FairMQTransportFactory.cxx

@@ -16,11 +16,10 @@
 #include <fairmq/ofi/TransportFactory.h>
 #endif
 #include <FairMQLogger.h>
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Unique.h>
 
 #include <memory>
 #include <string>
-#include <sstream>
 
 FairMQTransportFactory::FairMQTransportFactory(const std::string& id)
     : fkId(id)

fairmq/FairMQTransportFactory.h

@@ -12,7 +12,6 @@
 #include <FairMQLogger.h>
 #include <FairMQMessage.h>
 #include <FairMQPoller.h>
-#include <fairmq/ProgOptionsFwd.h>
 #include <FairMQSocket.h>
 #include <FairMQUnmanagedRegion.h>
 #include <fairmq/MemoryResources.h>
@@ -22,8 +21,11 @@
 #include <memory>
 #include <vector>
 #include <unordered_map>
+#include <stdexcept>
+#include <cstddef> // size_t
 
 class FairMQChannel;
+namespace fair { namespace mq { class ProgOptions; } }
 
 class FairMQTransportFactory
 {

fairmq/MemoryResources.h

@@ -23,11 +23,7 @@ class FairMQTransportFactory;
 #include <boost/container/pmr/monotonic_buffer_resource.hpp>
 #include <boost/container/pmr/polymorphic_allocator.hpp>
 #include <cstring>
-#include <string>
-#include <type_traits>
-#include <unordered_map>
 #include <utility>
-#include <vector>
 
 namespace fair {
 namespace mq {

fairmq/Plugin.h

@@ -9,7 +9,8 @@
 #ifndef FAIR_MQ_PLUGIN_H
 #define FAIR_MQ_PLUGIN_H
 
-#include <fairmq/Tools.h>
+#include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Version.h>
 #include <fairmq/PluginServices.h>
 
 #include <boost/dll/alias.hpp>
@@ -59,10 +60,10 @@ class Plugin
     friend auto operator!=(const Plugin& lhs, const Plugin& rhs) -> bool { return !(lhs == rhs); }
     friend auto operator<<(std::ostream& os, const Plugin& p) -> std::ostream&
     {
-        return os << "'" << p.GetName() << "', "
-                  << "version '" << p.GetVersion() << "', "
+        return os << "'"            << p.GetName()       << "', "
+                  << "version '"    << p.GetVersion()    << "', "
                   << "maintainer '" << p.GetMaintainer() << "', "
-                  << "homepage '" << p.GetHomepage() << "'";
+                  << "homepage '"   << p.GetHomepage()   << "'";
     }
     static auto NoProgramOptions() -> ProgOptions { return boost::none; }
 
@@ -79,7 +80,6 @@ class Plugin
     auto StealDeviceControl() -> void { fPluginServices->StealDeviceControl(fkName); };
     auto ReleaseDeviceControl() -> void { fPluginServices->ReleaseDeviceControl(fkName); };
     auto ChangeDeviceState(const DeviceStateTransition next) -> bool { return fPluginServices->ChangeDeviceState(fkName, next); }
-    void TransitionDeviceStateTo(const DeviceState state) { return fPluginServices->TransitionDeviceStateTo(fkName, state); }
     auto SubscribeToDeviceStateChange(std::function<void(DeviceState)> callback) -> void { fPluginServices->SubscribeToDeviceStateChange(fkName, callback); }
     auto UnsubscribeFromDeviceStateChange() -> void { fPluginServices->UnsubscribeFromDeviceStateChange(fkName); }
 

fairmq/PluginManager.cxx

@@ -8,7 +8,7 @@
 
 #include <fairmq/plugins/Builtin.h>
 #include <fairmq/PluginManager.h>
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Strings.h>
 #include <boost/program_options.hpp>
 #include <algorithm>
 #include <iterator>

fairmq/PluginManager.h

@@ -11,8 +11,9 @@
 
 #include <fairmq/Plugin.h>
 #include <fairmq/PluginServices.h>
-#include <fairmq/Tools.h>
-#include <FairMQDevice.h>
+#include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Strings.h>
+
 #define BOOST_FILESYSTEM_VERSION 3
 #define BOOST_FILESYSTEM_NO_DEPRECATED
 #include <boost/filesystem.hpp>
@@ -21,13 +22,14 @@
 #include <boost/dll/import.hpp>
 #include <boost/dll/shared_library.hpp>
 #include <boost/dll/runtime_symbol_info.hpp>
+
 #include <functional>
 #include <map>
 #include <memory>
 #include <stdexcept>
 #include <string>
-#include <tuple>
 #include <vector>
+#include <utility> // forward
 
 namespace fair
 {

fairmq/PluginServices.cxx

@@ -7,6 +7,7 @@
  ********************************************************************************/
 
 #include <fairmq/PluginServices.h>
+#include <fairmq/tools/Strings.h>
 
 using namespace fair::mq;
 using namespace std;
@@ -27,22 +28,6 @@ auto PluginServices::ChangeDeviceState(const string& controller, const DeviceSta
     }
 }
 
-void PluginServices::TransitionDeviceStateTo(const std::string& controller, DeviceState state)
-{
-    lock_guard<mutex> lock{fDeviceControllerMutex};
-
-    if (!fDeviceController) fDeviceController = controller;
-
-    if (fDeviceController == controller) {
-        fDevice.TransitionTo(state);
-    } else {
-        throw DeviceControlError{tools::ToString(
-            "Plugin '", controller, "' is not allowed to change device states. ",
-            "Currently, plugin '", *fDeviceController, "' has taken control."
-        )};
-    }
-}
-
 auto PluginServices::TakeDeviceControl(const string& controller) -> void
 {
     lock_guard<mutex> lock{fDeviceControllerMutex};
@@ -74,7 +59,8 @@ auto PluginServices::ReleaseDeviceControl(const string& controller) -> void
         if (fDeviceController == controller) {
             fDeviceController = boost::none;
         } else {
-            throw DeviceControlError{tools::ToString("Plugin '", controller, "' cannot release control because it has not taken over control.")};
+            LOG(debug) << "Plugin '" << controller << "' cannot release control "
+                       << "because it has no control.";
         }
     }
 

fairmq/PluginServices.h

@@ -9,7 +9,6 @@
 #ifndef FAIR_MQ_PLUGINSERVICES_H
 #define FAIR_MQ_PLUGINSERVICES_H
 
-#include <fairmq/Tools.h>
 #include <fairmq/States.h>
 #include <FairMQDevice.h>
 #include <fairmq/ProgOptions.h>
@@ -125,8 +124,6 @@ class PluginServices
     /// If the device control role has not been taken yet, calling this function will take over control implicitely.
     auto ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> bool;
 
-    void TransitionDeviceStateTo(const std::string& controller, DeviceState state);
-
     /// @brief Subscribe with a callback to device state changes
     /// @param subscriber id
     /// @param callback

fairmq/ProgOptions.cxx

@@ -15,7 +15,7 @@
 #include "FairMQLogger.h"
 #include <fairmq/ProgOptions.h>
 
-#include "tools/Unique.h"
+#include <fairmq/tools/Strings.h>
 
 #include <boost/any.hpp>
 #include <boost/regex.hpp>

fairmq/ProgOptions.h

@@ -14,7 +14,7 @@
 #include <fairmq/EventManager.h>
 #include <fairmq/ProgOptionsFwd.h>
 #include <fairmq/Properties.h>
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Strings.h>
 
 #include <boost/program_options.hpp>
 

fairmq/SDK.h

@@ -10,11 +10,17 @@
 #define FAIR_MQ_SDK_H
 
 // IWYU pragma: begin_exports
-#include <fairmq/sdk/DDSInfo.h>
+#include <fairmq/sdk/AsioAsyncOp.h>
+#include <fairmq/sdk/AsioBase.h>
+#include <fairmq/sdk/DDSAgent.h>
 #include <fairmq/sdk/DDSEnvironment.h>
+#include <fairmq/sdk/DDSInfo.h>
 #include <fairmq/sdk/DDSSession.h>
+#include <fairmq/sdk/DDSTask.h>
 #include <fairmq/sdk/DDSTopology.h>
+#include <fairmq/sdk/Error.h>
 #include <fairmq/sdk/Topology.h>
+#include <fairmq/sdk/Traits.h>
 // IWYU pragma: end_exports
 
 #endif // FAIR_MQ_SDK_H

fairmq/Transports.h

@@ -9,7 +9,7 @@
 #ifndef FAIR_MQ_TRANSPORTS_H
 #define FAIR_MQ_TRANSPORTS_H
 
-#include <fairmq/Tools.h>
+#include <fairmq/tools/CppSTL.h>
 
 #include <memory>
 #include <string>

fairmq/devices/FairMQBenchmarkSampler.cxx

@@ -8,10 +8,9 @@
 
 #include "FairMQBenchmarkSampler.h"
 
-#include <fairmq/Tools.h>
+#include "tools/RateLimit.h"
 #include "../FairMQLogger.h"
 
-#include <vector>
 #include <chrono>
 
 using namespace std;
@@ -27,10 +26,6 @@ FairMQBenchmarkSampler::FairMQBenchmarkSampler()
 {
 }
 
-FairMQBenchmarkSampler::~FairMQBenchmarkSampler()
-{
-}
-
 void FairMQBenchmarkSampler::InitTask()
 {
     fMultipart = fConfig->GetProperty<bool>("multipart");

fairmq/devices/FairMQBenchmarkSampler.h

@@ -10,8 +10,10 @@
 #define FAIRMQBENCHMARKSAMPLER_H_
 
 #include <string>
-#include <thread>
 #include <atomic>
+#include <cstddef> // size_t
+#include <cstdint> // uint64_t
+
 
 #include "FairMQDevice.h"
 
@@ -23,7 +25,7 @@ class FairMQBenchmarkSampler : public FairMQDevice
 {
   public:
     FairMQBenchmarkSampler();
-    virtual ~FairMQBenchmarkSampler();
+    virtual ~FairMQBenchmarkSampler() {}
 
   protected:
     bool fMultipart;

fairmq/nanomsg/FairMQPollerNN.h

@@ -17,7 +17,6 @@
 
 #include <vector>
 #include <unordered_map>
-#include <initializer_list>
 
 #include "FairMQPoller.h"
 #include "FairMQChannel.h"

fairmq/plugins/Control.cxx

@@ -427,6 +427,9 @@ auto Control::RunShutdownSequence() -> void
             case DeviceState::Idle:
                 ChangeDeviceState(DeviceStateTransition::End);
                 break;
+            case DeviceState::InitializingDevice:
+                ChangeDeviceState(DeviceStateTransition::CompleteInit);
+                break;
             case DeviceState::Initialized:
             case DeviceState::Bound:
             case DeviceState::DeviceReady:

fairmq/plugins/DDS/CMakeLists.txt

@@ -8,18 +8,18 @@
 
 set(plugin FairMQPlugin_dds)
 add_library(${plugin} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/DDS.cxx ${CMAKE_CURRENT_SOURCE_DIR}/DDS.h)
-target_link_libraries(${plugin} FairMQ DDS::dds_intercom_lib DDS::dds_protocol_lib)
+target_link_libraries(${plugin} PUBLIC FairMQ StateMachine DDS::dds_intercom_lib DDS::dds_protocol_lib Boost::boost PRIVATE Commands)
 target_include_directories(${plugin} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 set_target_properties(${plugin} PROPERTIES CXX_VISIBILITY_PRESET hidden)
 set_target_properties(${plugin} PROPERTIES
-  VERSION ${PROJECT_GIT_VERSION}
+  VERSION ${PROJECT_VERSION}
   SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
   LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/fairmq
 )
 
 set(exe fairmq-dds-command-ui)
 add_executable(${exe} ${CMAKE_CURRENT_SOURCE_DIR}/runDDSCommandUI.cxx)
-target_link_libraries(${exe} FairMQ DDS::dds_intercom_lib DDS::dds_protocol_lib)
+target_link_libraries(${exe} FairMQ Commands StateMachine DDS::dds_intercom_lib DDS::dds_protocol_lib)
 target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 
 install(TARGETS ${plugin} ${exe}

fairmq/plugins/DDS/DDS.cxx

@@ -8,18 +8,18 @@
 
 #include "DDS.h"
 
-#include <fairmq/Tools.h>
+#include <fairmq/sdk/commands/Commands.h>
+
+#include <fairmq/tools/Strings.h>
 
 #include <boost/algorithm/string/join.hpp>
 #include <boost/algorithm/string/split.hpp>
 #include <boost/algorithm/string/classification.hpp>
+#include <boost/asio/post.hpp>
 
-#include <termios.h> // for the interactive mode
-#include <poll.h> // for the interactive mode
-#include <sstream>
-#include <iostream>
 #include <cstdlib>
 #include <stdexcept>
+#include <sstream>
 
 using namespace std;
 using fair::mq::tools::ToString;
@@ -37,25 +37,89 @@ DDS::DDS(const string& name,
          const string& homepage,
          PluginServices* pluginServices)
     : Plugin(name, version, maintainer, homepage, pluginServices)
-    , fTransitions({"INIT DEVICE",
-                    "COMPLETE INIT",
-                    "BIND",
-                    "CONNECT",
-                    "INIT TASK",
-                    "RUN",
-                    "STOP",
-                    "RESET TASK",
-                    "RESET DEVICE",
-                    "END"})
     , fCurrentState(DeviceState::Idle)
     , fLastState(DeviceState::Idle)
     , fDeviceTerminationRequested(false)
+    , fLastExternalController(0)
+    , fExitingAckedByLastExternalController(false)
     , fHeartbeatInterval(100)
+    , fUpdatesAllowed(false)
+    , fWorkGuard(fWorkerQueue.get_executor())
 {
     try {
         TakeDeviceControl();
-        fControllerThread = thread(&DDS::HandleControl, this);
+
         fHeartbeatThread = thread(&DDS::HeartbeatSender, this);
+
+        string deviceId(GetProperty<string>("id"));
+        if (deviceId.empty()) {
+            SetProperty<string>("id", dds::env_prop<dds::task_path>());
+        }
+        string sessionId(GetProperty<string>("session"));
+        if (sessionId == "default") {
+            SetProperty<string>("session", dds::env_prop<dds::dds_session_id>());
+        }
+
+        auto control = GetProperty<string>("control");
+        if (control == "static") {
+            LOG(error) << "DDS Plugin: static mode is not supported";
+            throw invalid_argument("DDS Plugin: static mode is not supported");
+        } else if (control == "dynamic" || control == "external" || control == "interactive") {
+            LOG(debug) << "Running DDS controller: external";
+        } else {
+            LOG(error) << "Unrecognized control mode '" << control << "' requested. " << "Ignoring and starting in external control mode.";
+        }
+
+        SubscribeForCustomCommands();
+        SubscribeForConnectingChannels();
+
+        // subscribe to device state changes, pushing new state changes into the event queue
+        SubscribeToDeviceStateChange([&](DeviceState newState) {
+            switch (newState) {
+                case DeviceState::Bound:
+                    // Receive addresses of connecting channels from DDS
+                    // and propagate addresses of bound channels to DDS.
+                    FillChannelContainers();
+
+                    // publish bound addresses via DDS at keys corresponding to the channel
+                    // prefixes, e.g. 'data' in data[i]
+                    PublishBoundChannels();
+                    break;
+                case DeviceState::ResettingDevice: {
+                    {
+                        lock_guard<mutex> lk(fUpdateMutex);
+                        fUpdatesAllowed = false;
+                    }
+
+                    EmptyChannelContainers();
+                    break;
+                }
+                case DeviceState::Exiting:
+                    fWorkGuard.reset();
+                    fDeviceTerminationRequested = true;
+                    UnsubscribeFromDeviceStateChange();
+                    ReleaseDeviceControl();
+                    break;
+                default:
+                    break;
+            }
+
+            lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+            string id = GetProperty<string>("id");
+            fLastState = fCurrentState;
+            fCurrentState = newState;
+            using namespace sdk::cmd;
+            for (auto subscriberId : fStateChangeSubscribers) {
+                LOG(debug) << "Publishing state-change: " << fLastState << "->" << newState << " to " << subscriberId;
+
+                Cmds cmds(make<StateChange>(id, dds::env_prop<dds::task_id>(), fLastState, fCurrentState));
+                fDDS.Send(cmds.Serialize(), to_string(subscriberId));
+            }
+        });
+
+        StartWorkerThread();
+
+        fDDS.Start();
     } catch (PluginServices::DeviceControlError& e) {
         LOG(debug) << e.what();
     } catch (exception& e) {
@@ -63,73 +127,26 @@ DDS::DDS(const string& name,
     }
 }
 
-auto DDS::HandleControl() -> void
+void DDS::EmptyChannelContainers()
 {
-    try {
-        auto control = GetProperty<string>("control");
-        bool staticMode(false);
-        if (control == "static") {
-            LOG(debug) << "Running DDS controller: static";
-            staticMode = true;
-        } else if (control == "dynamic" || control == "external" || control == "interactive") {
-            LOG(debug) << "Running DDS controller: external";
-        } else {
-            LOG(error) << "Unrecognized control mode '" << control << "' requested. " << "Ignoring and falling back to static control mode.";
-            staticMode = true;
-        }
+    fBindingChans.clear();
+    fConnectingChans.clear();
+}
 
-        SubscribeForCustomCommands();
-        SubscribeForConnectingChannels();
-        fDDS.Start();
+auto DDS::StartWorkerThread() -> void
+{
+    fWorkerThread = thread([this]() {
+        fWorkerQueue.run();
+    });
+}
 
-        // subscribe to device state changes, pushing new state changes into the event queue
-        SubscribeToDeviceStateChange([&](DeviceState newState) {
-            fStateQueue.Push(newState);
-            switch(newState) {
-              case DeviceState::Bound:
-                // Receive addresses of connecting channels from DDS
-                // and propagate addresses of bound channels to DDS.
-                FillChannelContainers();
-
-                // publish bound addresses via DDS at keys corresponding to the channel prefixes, e.g. 'data' in data[i]
-                PublishBoundChannels();
-                break;
-              case DeviceState::Exiting:
-                fDeviceTerminationRequested = true;
-                UnsubscribeFromDeviceStateChange();
-                ReleaseDeviceControl();
-                break;
-              default:
-                break;
-            }
-
-            lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-            string id = GetProperty<string>("id");
-            fLastState = fCurrentState;
-            fCurrentState = newState;
-            for (auto subscriberId : fStateChangeSubscribers) {
-                LOG(debug) << "Publishing state-change: " << fLastState << "->" << newState << " to " << subscriberId;
-                fDDS.Send("state-change: " + id + "," + ToString(dds::env_prop<dds::task_id>()) + "," + ToStr(fLastState) + "->" + ToStr(newState), to_string(subscriberId));
-            }
-        });
-
-        if (staticMode) {
-            TransitionDeviceStateTo(DeviceState::Running);
-
-            // wait until stop signal
-            unique_lock<mutex> lock(fStopMutex);
-            while (!fDeviceTerminationRequested) {
-                fStopCondition.wait_for(lock, chrono::seconds(1));
-            }
-            LOG(debug) << "Stopping DDS control plugin";
-        }
-    } catch (DeviceErrorState&) {
-        ReleaseDeviceControl();
-    } catch (exception& e) {
-        ReleaseDeviceControl();
-        LOG(error) << "Error: " << e.what() << endl;
-        return;
-    }
+auto DDS::WaitForExitingAck() -> void
+{
+    unique_lock<mutex> lock(fStateChangeSubscriberMutex);
+    fExitingAcked.wait_for(
+        lock,
+        chrono::milliseconds(GetProperty<unsigned int>("wait-for-exiting-ack-timeout")),
+        [this]() { return fExitingAckedByLastExternalController; });
 }
 
 auto DDS::FillChannelContainers() -> void
@@ -197,6 +214,11 @@ auto DDS::FillChannelContainers() -> void
             LOG(debug) << "dds-i-n: adding " << chanName << " -> i: " << i << " n: " << n;
             fIofN.insert(make_pair(chanName, IofN(i, n)));
         }
+        {
+            lock_guard<mutex> lk(fUpdateMutex);
+            fUpdatesAllowed = true;
+        }
+        fUpdateCondition.notify_one();
     } catch (const exception& e) {
         LOG(error) << "Error filling channel containers: " << e.what();
     }
@@ -206,56 +228,70 @@ auto DDS::SubscribeForConnectingChannels() -> void
 {
     LOG(debug) << "Subscribing for DDS properties.";
 
-    fDDS.SubscribeKeyValue([&] (const string& propertyId, const string& value, uint64_t senderTaskID) {
-        try {
-            LOG(debug) << "Received update for " << propertyId << ": value=" << value << ", senderTaskID=" << senderTaskID;
-            string val = value;
-            // check if it is to handle as one out of multiple values
-            auto it = fIofN.find(propertyId);
-            if (it != fIofN.end()) {
-                it->second.fEntries.push_back(value);
-                if (it->second.fEntries.size() == it->second.fN) {
-                    sort(it->second.fEntries.begin(), it->second.fEntries.end());
-                    val = it->second.fEntries.at(it->second.fI);
-                } else {
-                    LOG(debug) << "received " << it->second.fEntries.size() << " values for " << propertyId << ", expecting total of " << it->second.fN;
-                    return;
-                }
-            }
+    fDDS.SubscribeKeyValue([&] (const string& key, const string& value, uint64_t senderTaskID) {
+        LOG(debug) << "Received property: key=" << key << ", value=" << value << ", senderTaskID=" << senderTaskID;
 
-            vector<string> connectionStrings;
-            boost::algorithm::split(connectionStrings, val, boost::algorithm::is_any_of(","));
-            if (connectionStrings.size() > 1) { // multiple bound channels received
-                auto it2 = fI.find(propertyId);
-                if (it2 != fI.end()) {
-                    LOG(debug) << "adding connecting channel " << propertyId << " : " << connectionStrings.at(it2->second);
-                    fConnectingChans.at(propertyId).fDDSValues.insert({senderTaskID, connectionStrings.at(it2->second).c_str()});
-                } else {
-                    LOG(error) << "multiple bound channels received, but no task index specified, only assigning the first";
-                    fConnectingChans.at(propertyId).fDDSValues.insert({senderTaskID, connectionStrings.at(0).c_str()});
-                }
-            } else { // only one bound channel received
-                fConnectingChans.at(propertyId).fDDSValues.insert({senderTaskID, val.c_str()});
-            }
-
-            // update channels and remove them from unfinished container
-            for (auto mi = fConnectingChans.begin(); mi != fConnectingChans.end(); /* no increment */) {
-                if (mi->second.fSubChannelAddresses.size() == mi->second.fDDSValues.size()) {
-                    // when multiple subChannels are used, their order on every device should be the same, irregardless of arrival order from DDS.
-                    sort(mi->second.fSubChannelAddresses.begin(), mi->second.fSubChannelAddresses.end());
-                    auto it3 = mi->second.fDDSValues.begin();
-                    for (unsigned int i = 0; i < mi->second.fSubChannelAddresses.size(); ++i) {
-                        SetProperty<string>(string{"chans." + mi->first + "." + to_string(i) + ".address"}, it3->second);
-                        ++it3;
-                    }
-                    fConnectingChans.erase(mi++);
-                } else {
-                    ++mi;
-                }
-            }
-        } catch (const exception& e) {
-            LOG(error) << "Error on handling DDS property update for " << propertyId << ": value=" << value << ", senderTaskID=" << senderTaskID << ": " << e.what();
+        if (key.compare(0, 8, "fmqchan_") != 0) {
+            LOG(debug) << "property update is not a channel info update: " << key;
+            return;
         }
+        string channelName = key.substr(8);
+        LOG(info) << "Update for channel name: " << channelName;
+
+        boost::asio::post(fWorkerQueue, [=]() {
+            try {
+                {
+                    unique_lock<mutex> lk(fUpdateMutex);
+                    fUpdateCondition.wait(lk, [&]{ return fUpdatesAllowed; });
+                }
+                string val = value;
+                // check if it is to handle as one out of multiple values
+                auto it = fIofN.find(channelName);
+                if (it != fIofN.end()) {
+                    it->second.fEntries.push_back(value);
+                    if (it->second.fEntries.size() == it->second.fN) {
+                        sort(it->second.fEntries.begin(), it->second.fEntries.end());
+                        val = it->second.fEntries.at(it->second.fI);
+                    } else {
+                        LOG(debug) << "received " << it->second.fEntries.size() << " values for " << channelName << ", expecting total of " << it->second.fN;
+                        return;
+                    }
+                }
+
+                vector<string> connectionStrings;
+                boost::algorithm::split(connectionStrings, val, boost::algorithm::is_any_of(","));
+                if (connectionStrings.size() > 1) { // multiple bound channels received
+                    auto it2 = fI.find(channelName);
+                    if (it2 != fI.end()) {
+                        LOG(debug) << "adding connecting channel " << channelName << " : " << connectionStrings.at(it2->second);
+                        fConnectingChans.at(channelName).fDDSValues.insert({senderTaskID, connectionStrings.at(it2->second).c_str()});
+                    } else {
+                        LOG(error) << "multiple bound channels received, but no task index specified, only assigning the first";
+                        fConnectingChans.at(channelName).fDDSValues.insert({senderTaskID, connectionStrings.at(0).c_str()});
+                    }
+                } else { // only one bound channel received
+                    fConnectingChans.at(channelName).fDDSValues.insert({senderTaskID, val.c_str()});
+                }
+
+                // update channels and remove them from unfinished container
+                for (auto mi = fConnectingChans.begin(); mi != fConnectingChans.end(); /* no increment */) {
+                    if (mi->second.fSubChannelAddresses.size() == mi->second.fDDSValues.size()) {
+                        // when multiple subChannels are used, their order on every device should be the same, irregardless of arrival order from DDS.
+                        sort(mi->second.fSubChannelAddresses.begin(), mi->second.fSubChannelAddresses.end());
+                        auto it3 = mi->second.fDDSValues.begin();
+                        for (unsigned int i = 0; i < mi->second.fSubChannelAddresses.size(); ++i) {
+                            SetProperty<string>(string{"chans." + mi->first + "." + to_string(i) + ".address"}, it3->second);
+                            ++it3;
+                        }
+                        fConnectingChans.erase(mi++);
+                    } else {
+                        ++mi;
+                    }
+                }
+            } catch (const exception& e) {
+                LOG(error) << "Error handling DDS property: key=" << key << ", value=" << value << ", senderTaskID=" << senderTaskID << ": " << e.what();
+            }
+        });
     });
 }
 
@@ -263,13 +299,14 @@ auto DDS::PublishBoundChannels() -> void
 {
     for (const auto& chan : fBindingChans) {
         string joined = boost::algorithm::join(chan.second, ",");
-        LOG(debug) << "Publishing " << chan.first << " bound addresses (" << chan.second.size() << ") to DDS under '" << chan.first << "' property name.";
-        fDDS.PutValue(chan.first, joined);
+        LOG(debug) << "Publishing bound addresses (" << chan.second.size() << ") of channel '" << chan.first << "' to DDS under '" << "fmqchan_" + chan.first << "' property name.";
+        fDDS.PutValue("fmqchan_" + chan.first, joined);
     }
 }
 
 auto DDS::HeartbeatSender() -> void
 {
+    using namespace sdk::cmd;
     string id = GetProperty<string>("id");
 
     while (!fDeviceTerminationRequested) {
@@ -277,7 +314,7 @@ auto DDS::HeartbeatSender() -> void
             lock_guard<mutex> lock{fHeartbeatSubscriberMutex};
 
             for (const auto subscriberId : fHeartbeatSubscribers) {
-                fDDS.Send("heartbeat: " + id , to_string(subscriberId));
+                fDDS.Send(Cmds(make<Heartbeat>(id)).Serialize(), to_string(subscriberId));
             }
         }
 
@@ -287,71 +324,105 @@ auto DDS::HeartbeatSender() -> void
 
 auto DDS::SubscribeForCustomCommands() -> void
 {
+    using namespace sdk::cmd;
     LOG(debug) << "Subscribing for DDS custom commands.";
 
     string id = GetProperty<string>("id");
 
-    fDDS.SubscribeCustomCmd([id, this](const string& cmd, const string& cond, uint64_t senderId) {
-        LOG(info) << "Received command: '" << cmd << "' from " << senderId;
+    fDDS.SubscribeCustomCmd([id, this](const string& cmdStr, const string& cond, uint64_t senderId) {
+        // LOG(info) << "Received command: '" << cmdStr << "' from " << senderId;
 
-        if (cmd == "check-state") {
-            fDDS.Send(id + ": " + ToStr(GetCurrentDeviceState()), to_string(senderId));
-        } else if (fTransitions.find(cmd) != fTransitions.end()) {
-            if (ChangeDeviceState(ToDeviceStateTransition(cmd))) {
-                fDDS.Send(id + ": queued, " + cmd, to_string(senderId));
-            } else {
-                fDDS.Send(id + ": could not queue, " + cmd, to_string(senderId));
+        Cmds inCmds;
+        inCmds.Deserialize(cmdStr);
+
+        for (const auto& cmd : inCmds) {
+            // LOG(info) << "Received command type: '" << cmd->GetType() << "' from " << senderId;
+            switch (cmd->GetType()) {
+                case Type::check_state: {
+                    fDDS.Send(Cmds(make<CurrentState>(id, GetCurrentDeviceState())).Serialize(), to_string(senderId));
+                }
+                break;
+                case Type::change_state: {
+                    Transition transition = static_cast<ChangeState&>(*cmd).GetTransition();
+                    if (ChangeDeviceState(transition)) {
+                        Cmds outCmds(make<TransitionStatus>(id, Result::Ok, transition));
+                        fDDS.Send(outCmds.Serialize(), to_string(senderId));
+                    } else {
+                        Cmds outCmds(make<TransitionStatus>(id, Result::Failure, transition));
+                        fDDS.Send(outCmds.Serialize(), to_string(senderId));
+                    }
+                    {
+                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                        fLastExternalController = senderId;
+                    }
+                }
+                break;
+                case Type::dump_config: {
+                    stringstream ss;
+                    for (const auto pKey: GetPropertyKeys()) {
+                        ss << id << ": " << pKey << " -> " << GetPropertyAsString(pKey) << "\n";
+                    }
+                    Cmds outCmds(make<Config>(id, ss.str()));
+                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
+                }
+                break;
+                case Type::subscribe_to_heartbeats: {
+                    {
+                        lock_guard<mutex> lock{fHeartbeatSubscriberMutex};
+                        fHeartbeatSubscribers.insert(senderId);
+                    }
+                    Cmds outCmds(make<HeartbeatSubscription>(id, Result::Ok));
+                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
+                }
+                break;
+                case Type::unsubscribe_from_heartbeats: {
+                    {
+                        lock_guard<mutex> lock{fHeartbeatSubscriberMutex};
+                        fHeartbeatSubscribers.erase(senderId);
+                    }
+                    Cmds outCmds(make<HeartbeatUnsubscription>(id, Result::Ok));
+                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
+                }
+                break;
+                case Type::state_change_exiting_received: {
+                    {
+                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                        if (fLastExternalController == senderId) {
+                            fExitingAckedByLastExternalController = true;
+                        }
+                    }
+                    fExitingAcked.notify_one();
+                }
+                break;
+                case Type::subscribe_to_state_change: {
+                    lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                    fStateChangeSubscribers.insert(senderId);
+                    if (!fControllerThread.joinable()) {
+                        fControllerThread = thread(&DDS::WaitForExitingAck, this);
+                    }
+
+                    LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState << " to " << senderId;
+
+                    Cmds outCmds(make<StateChangeSubscription>(id, Result::Ok), make<StateChange>(id, dds::env_prop<dds::task_id>(), fLastState, fCurrentState));
+
+                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
+                }
+                break;
+                case Type::unsubscribe_from_state_change: {
+                    {
+                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                        fStateChangeSubscribers.erase(senderId);
+                    }
+                    Cmds outCmds(make<StateChangeUnsubscription>(id, Result::Ok));
+                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
+                }
+                break;
+                default:
+                    LOG(warn) << "Unexpected/unknown command received: " << cmdStr;
+                    LOG(warn) << "Origin: " << senderId;
+                    LOG(warn) << "Destination: " << cond;
+                break;
             }
-            if (cmd == "END" && ToStr(GetCurrentDeviceState()) == "EXITING") {
-                unique_lock<mutex> lock(fStopMutex);
-                fStopCondition.notify_one();
-            }
-        } else if (cmd == "dump-config") {
-            stringstream ss;
-            for (const auto pKey: GetPropertyKeys()) {
-                ss << id << ": " << pKey << " -> " << GetPropertyAsString(pKey) << endl;
-            }
-            fDDS.Send(ss.str(), to_string(senderId));
-        } else if (cmd == "subscribe-to-heartbeats") {
-            {
-                // auto size = fHeartbeatSubscribers.size();
-                lock_guard<mutex> lock{fHeartbeatSubscriberMutex};
-                fHeartbeatSubscribers.insert(senderId);
-            }
-            fDDS.Send("heartbeat-subscription: " + id + ",OK", to_string(senderId));
-        } else if (cmd == "unsubscribe-from-heartbeats") {
-            {
-                lock_guard<mutex> lock{fHeartbeatSubscriberMutex};
-                fHeartbeatSubscribers.erase(senderId);
-            }
-            fDDS.Send("heartbeat-unsubscription: " + id + ",OK", to_string(senderId));
-        } else if (cmd == "subscribe-to-state-changes") {
-            {
-                // auto size = fStateChangeSubscribers.size();
-                lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                fStateChangeSubscribers.insert(senderId);
-            }
-            fDDS.Send("state-changes-subscription: " + id + ",OK", to_string(senderId));
-            {
-                lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState << " to " << senderId;
-                // fDDSCustomCmd.send("state-change: " + id + "," + ToStr(fLastState) + "->" + ToStr(fCurrentState), to_string(senderId));
-                fDDS.Send("state-change: " + id + "," + ToString(dds::env_prop<dds::task_id>()) + "," + ToStr(fLastState) + "->" + ToStr(fCurrentState), to_string(senderId));
-            }
-        } else if (cmd == "unsubscribe-from-state-changes") {
-            {
-                lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                fStateChangeSubscribers.erase(senderId);
-            }
-            fDDS.Send("state-changes-unsubscription: " + id + ",OK", to_string(senderId));
-        } else if (cmd == "SHUTDOWN") {
-            TransitionDeviceStateTo(DeviceState::Exiting);
-        } else if (cmd == "STARTUP") {
-            TransitionDeviceStateTo(DeviceState::Running);
-        } else {
-            LOG(warn) << "Unknown command: " << cmd;
-            LOG(warn) << "Origin: " << senderId;
-            LOG(warn) << "Destination: " << cond;
         }
     });
 }
@@ -368,6 +439,11 @@ DDS::~DDS()
     if (fHeartbeatThread.joinable()) {
         fHeartbeatThread.join();
     }
+
+    fWorkGuard.reset();
+    if (fWorkerThread.joinable()) {
+        fWorkerThread.join();
+    }
 }
 
 } /* namespace plugins */

fairmq/plugins/DDS/DDS.h

@@ -9,19 +9,23 @@
 #ifndef FAIR_MQ_PLUGINS_DDS
 #define FAIR_MQ_PLUGINS_DDS
 
-#include <DDS/dds_env_prop.h>
-#include <DDS/dds_intercom.h>
-#include <cassert>
-#include <chrono>
-#include <condition_variable>
 #include <fairmq/Plugin.h>
 #include <fairmq/StateQueue.h>
 #include <fairmq/Version.h>
-#include <functional>
+
+#include <dds/dds.h>
+
+#include <boost/asio/executor.hpp>
+#include <boost/asio/executor_work_guard.hpp>
+#include <boost/asio/io_context.hpp>
+
+#include <cassert>
+#include <chrono>
+#include <condition_variable>
 #include <mutex>
-#include <queue>
 #include <set>
 #include <string>
+#include <atomic>
 #include <thread>
 #include <unordered_map>
 #include <vector>
@@ -63,6 +67,9 @@ struct DDSSubscription
             LOG(error) << "DDS Error received: error code: " << errorCode << ", error message: " << errorMsg;
         });
 
+        // fDDSCustomCmd.subscribe([](const std::string& cmd, const std::string& cond, uint64_t senderId) {
+            // LOG(debug) << "cmd: " << cmd << ", cond: " << cond << ", senderId: " << senderId;
+        // });
         assert(!dds_session_id.empty());
     }
 
@@ -125,9 +132,12 @@ class DDS : public Plugin
     ~DDS();
 
   private:
-    auto HandleControl() -> void;
+    auto WaitForExitingAck() -> void;
+    auto StartWorkerThread() -> void;
 
     auto FillChannelContainers() -> void;
+    auto EmptyChannelContainers() -> void;
+
     auto SubscribeForConnectingChannels() -> void;
     auto PublishBoundChannels() -> void;
     auto SubscribeForCustomCommands() -> void;
@@ -142,24 +152,30 @@ class DDS : public Plugin
     std::unordered_map<std::string, int> fI;
     std::unordered_map<std::string, IofN> fIofN;
 
-    std::mutex fStopMutex;
-    std::condition_variable fStopCondition;
-
-    const std::set<std::string> fTransitions;
-
     std::thread fControllerThread;
     DeviceState fCurrentState, fLastState;
-    fair::mq::StateQueue fStateQueue;
 
     std::atomic<bool> fDeviceTerminationRequested;
 
     std::set<uint64_t> fHeartbeatSubscribers;
     std::mutex fHeartbeatSubscriberMutex;
+
     std::set<uint64_t> fStateChangeSubscribers;
+    uint64_t fLastExternalController;
+    bool fExitingAckedByLastExternalController;
+    std::condition_variable fExitingAcked;
     std::mutex fStateChangeSubscriberMutex;
 
     std::thread fHeartbeatThread;
     std::chrono::milliseconds fHeartbeatInterval;
+
+    bool fUpdatesAllowed;
+    std::mutex fUpdateMutex;
+    std::condition_variable fUpdateCondition;
+
+    std::thread fWorkerThread;
+    boost::asio::io_context fWorkerQueue;
+    boost::asio::executor_work_guard<boost::asio::executor> fWorkGuard;
 };
 
 Plugin::ProgOptions DDSProgramOptions()
@@ -167,7 +183,8 @@ Plugin::ProgOptions DDSProgramOptions()
     boost::program_options::options_description options{"DDS Plugin"};
     options.add_options()
         ("dds-i", boost::program_options::value<std::vector<std::string>>()->multitoken()->composing(), "Task index for chosing connection target (single channel n to m). When all values come via same update.")
-        ("dds-i-n", boost::program_options::value<std::vector<std::string>>()->multitoken()->composing(), "Task index for chosing connection target (one out of n values to take). When values come as independent updates.");
+        ("dds-i-n", boost::program_options::value<std::vector<std::string>>()->multitoken()->composing(), "Task index for chosing connection target (one out of n values to take). When values come as independent updates.")
+        ("wait-for-exiting-ack-timeout", boost::program_options::value<unsigned int>()->default_value(1000), "Wait timeout for EXITING state-change acknowledgement by external controller in milliseconds.");
 
     return options;
 }

fairmq/plugins/DDS/runDDSCommandUI.cxx

@@ -6,27 +6,26 @@
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#include <algorithm>
-#include <atomic>
-#include <boost/algorithm/string/classification.hpp>
-#include <boost/algorithm/string/predicate.hpp>
-#include <boost/algorithm/string/split.hpp>
-#include <boost/algorithm/string/trim.hpp>
+#include <fairmq/sdk/commands/Commands.h>
+#include <fairmq/States.h>
+
+#include <dds/dds.h>
+
 #include <boost/program_options.hpp>
+
 #include <condition_variable>
 #include <cstdlib>
-#include <DDS/dds_intercom.h>
-#include <exception>
 #include <iostream>
-#include <unordered_map>
 #include <mutex>
-#include <termios.h>   // raw mode console input
-#include <thread>
 #include <string>
+#include <termios.h> // raw mode console input
+#include <thread>
+#include <utility>
 #include <unistd.h>
 
 using namespace std;
 using namespace dds::intercom_api;
+using namespace fair::mq::sdk::cmd;
 namespace bpo = boost::program_options;
 
 struct TerminalConfig
@@ -50,7 +49,8 @@ struct TerminalConfig
     }
 };
 
-struct StateSubscription {
+struct StateSubscription
+{
     const string& fTopologyPath;
     CCustomCmd& fDdsCustomCmd;
 
@@ -58,11 +58,11 @@ struct StateSubscription {
         : fTopologyPath(topologyPath)
         , fDdsCustomCmd(ddsCustomCmd)
     {
-        fDdsCustomCmd.send("subscribe-to-state-changes", fTopologyPath);
+        fDdsCustomCmd.send(Cmds(make<SubscribeToStateChange>()).Serialize(), fTopologyPath);
     }
 
     ~StateSubscription() {
-        fDdsCustomCmd.send("unsubscribe-from-state-changes", fTopologyPath);
+        fDdsCustomCmd.send(Cmds(make<UnsubscribeFromStateChange>()).Serialize(), fTopologyPath);
         this_thread::sleep_for(chrono::milliseconds(100)); // give dds a chance to complete request
     }
 };
@@ -74,7 +74,8 @@ void printControlsHelp()
     cout << "To quit press Ctrl+C" << endl;
 }
 
-void commandMode(const string& commandIn, const string& topologyPath, CCustomCmd& ddsCustomCmd) {
+void sendCommand(const string& commandIn, const string& topologyPath, CCustomCmd& ddsCustomCmd)
+{
     char c;
     string command(commandIn);
     TerminalConfig tconfig;
@@ -88,52 +89,43 @@ void commandMode(const string& commandIn, const string& topologyPath, CCustomCmd
     while (true) {
         if (command == "c") {
             cout << "> checking state of the devices" << endl;
-            ddsCustomCmd.send("check-state", topologyPath);
+            ddsCustomCmd.send(Cmds(make<CheckState>()).Serialize(), topologyPath);
         } else if (command == "o") {
             cout << "> dumping config of the devices" << endl;
-            ddsCustomCmd.send("dump-config", topologyPath);
+            ddsCustomCmd.send(Cmds(make<DumpConfig>()).Serialize(), topologyPath);
         } else if (command == "i") {
             cout << "> init devices" << endl;
-            ddsCustomCmd.send("INIT DEVICE", topologyPath);
+            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::InitDevice)).Serialize(), topologyPath);
         } else if (command == "k") {
             cout << "> complete init" << endl;
-            ddsCustomCmd.send("COMPLETE INIT", topologyPath);
+            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::CompleteInit)).Serialize(), topologyPath);
         } else if (command == "b") {
             cout << "> bind devices" << endl;
-            ddsCustomCmd.send("BIND", topologyPath);
+            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::Bind)).Serialize(), topologyPath);
         } else if (command == "x") {
             cout << "> connect devices" << endl;
-            ddsCustomCmd.send("CONNECT", topologyPath);
+            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::Connect)).Serialize(), topologyPath);
         } else if (command == "j") {
             cout << "> init tasks" << endl;
-            ddsCustomCmd.send("INIT TASK", topologyPath);
-        } else if (command == "p") {
-            cout << "> pause devices" << endl;
-            ddsCustomCmd.send("PAUSE", topologyPath);
+            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::InitTask)).Serialize(), topologyPath);
         } else if (command == "r") {
             cout << "> run tasks" << endl;
-            ddsCustomCmd.send("RUN", topologyPath);
+            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::Run)).Serialize(), topologyPath);
         } else if (command == "s") {
             cout << "> stop devices" << endl;
-            ddsCustomCmd.send("STOP", topologyPath);
+            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::Stop)).Serialize(), topologyPath);
         } else if (command == "t") {
             cout << "> reset tasks" << endl;
-            ddsCustomCmd.send("RESET TASK", topologyPath);
+            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::ResetTask)).Serialize(), topologyPath);
         } else if (command == "d") {
             cout << "> reset devices" << endl;
-            ddsCustomCmd.send("RESET DEVICE", topologyPath);
+            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::ResetDevice)).Serialize(), topologyPath);
         } else if (command == "h") {
             cout << "> help" << endl;
             printControlsHelp();
         } else if (command == "q") {
             cout << "> end" << endl;
-            ddsCustomCmd.send("END", topologyPath);
-        } else if (command == "q!") {
-            cout << "> shutdown" << endl;
-            ddsCustomCmd.send("SHUTDOWN", topologyPath);
-        } else if (command == "r!") {
-            cout << "> startup" << endl;
-            ddsCustomCmd.send("STARTUP", topologyPath);
+            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::End)).Serialize(), topologyPath);
         } else {
             cout << "\033[01;32mInvalid input: [" << c << "]\033[0m" << endl;
             printControlsHelp();
@@ -152,33 +144,34 @@ void commandMode(const string& commandIn, const string& topologyPath, CCustomCmd
 struct WaitMode
 {
     explicit WaitMode(const string& targetState)
-        : fTargetState(targetState)
-    {}
-
-    void Run(const chrono::milliseconds& timeout,
-             const string& topologyPath,
-             CCustomCmd& ddsCustomCmd,
-             unsigned int numberDevices,
-             const string& command = "")
+        : fTransitionedCount(0)
     {
-        StateSubscription stateSubscription(topologyPath, ddsCustomCmd);
+        if (targetState != "") {
+            size_t n = targetState.find("->");
+            if (n == string::npos) {
+                fTargetStatePair.first = fair::mq::State::Ok;
+                fTargetStatePair.second = fair::mq::GetState(targetState);
+            } else {
+                fTargetStatePair.first = fair::mq::GetState(targetState.substr(0, n));
+                fTargetStatePair.second = fair::mq::GetState(targetState.substr(n + 2));
+            }
+        }
+    }
 
+    void Run(const chrono::milliseconds& timeout, const string& topologyPath, CCustomCmd& ddsCustomCmd, unsigned int numDevices, const string& command = "")
+    {
         if (command != "") {
-            commandMode(command, topologyPath, ddsCustomCmd);
+            sendCommand(command, topologyPath, ddsCustomCmd);
         }
 
         // TODO once DDS provides an API to retrieve actual number of tasks, use it here
         auto condition = [&] {
-            bool res(!fTargetStates.empty()
-                   && all_of(fTargetStates.cbegin(),
-                             fTargetStates.cend(),
-                             [&](unordered_map<uint64_t, string>::value_type i) {
-                                 return boost::algorithm::ends_with(i.second, fTargetState);
-                             }));
-            if (numberDevices > 0) {
-                res = res && (fTargetStates.size() == numberDevices);
+            bool res = fTransitionedCount == numDevices;
+            if (fTargetStatePair.first == fair::mq::State::Ok) {
+                cout << "Waiting for " << numDevices << " devices to reach " << fTargetStatePair.second << ", condition check: " << res << endl;
+            } else {
+                cout << "Waiting for " << numDevices << " devices to reach " << fTargetStatePair.first << "->" << fTargetStatePair.second << ", condition check: " << res << endl;
             }
-            cout << "waiting for " << numberDevices << " devices to reach " << fTargetState << ", condition check: " << res << endl;
             return res;
         };
 
@@ -191,21 +184,33 @@ struct WaitMode
         } else {
             fCV.wait(lock, condition);
         }
+
+        // cout << "WaitMode.Run() finished" << endl;
     }
 
-    void AddNewStateEntry(uint64_t senderId, const string& state)
+    void CountStates(fair::mq::State lastState, fair::mq::State currentState)
     {
         {
             unique_lock<mutex> lock(fMtx);
-            fTargetStates[senderId] = state;
+            if (fTargetStatePair.first == fair::mq::State::Ok) {
+                if (fTargetStatePair.second == currentState) {
+                    fTransitionedCount++;
+                    // cout << "fTransitionedCount = " << fTransitionedCount << " for single value" << endl;
+                }
+            } else {
+                if (fTargetStatePair.first == lastState && fTargetStatePair.second == currentState) {
+                    fTransitionedCount++;
+                    // cout << "fTransitionedCount = " << fTransitionedCount << " for double value" << endl;
+                }
+            }
         }
         fCV.notify_one();
     }
 
     mutex fMtx;
     condition_variable fCV;
-    unordered_map<uint64_t, string> fTargetStates;
-    string fTargetState;
+    pair<fair::mq::State, fair::mq::State> fTargetStatePair;
+    unsigned int fTransitionedCount;
 };
 
 int main(int argc, char* argv[])
@@ -216,7 +221,7 @@ int main(int argc, char* argv[])
         string topologyPath;
         string targetState;
         unsigned int timeout;
-        unsigned int numberDevices(0);
+        unsigned int numDevices(0);
 
         bpo::options_description options("Common options");
 
@@ -232,7 +237,7 @@ int main(int argc, char* argv[])
             ("path,p",           bpo::value<string> (&topologyPath)->default_value(""), "DDS Topology path to send command to (empty - send to all tasks)")
             ("wait-for-state,w", bpo::value<string> (&targetState)->default_value(""), "Wait until targeted FairMQ devices reach the given state")
             ("timeout,t",        bpo::value<unsigned int> (&timeout)->default_value(0), "Timeout in milliseconds when waiting for a device state (0 - wait infinitely)")
-            ("number-devices,n", bpo::value<unsigned int> (&numberDevices)->default_value(0), "Number of devices (will be removed in the future)")
+            ("number-devices,n", bpo::value<unsigned int> (&numDevices)->default_value(0), "Number of devices (will be removed in the future)")
             ("help,h", "Produce help message");
 
         bpo::variables_map vm;
@@ -257,37 +262,66 @@ int main(int argc, char* argv[])
 
         // subscribe to receive messages from DDS
         ddsCustomCmd.subscribe([&](const string& msg, const string& /*condition*/, uint64_t senderId) {
-            // cerr << "Received: " << msg << endl;
-            vector<string> parts;
-            boost::algorithm::split(parts, msg, boost::algorithm::is_any_of(":,"));
-            if (parts[0] == "state-change") {
-                // cerr << "Received: " << msg << endl;
-                boost::trim(parts[2]);
-                waitMode.AddNewStateEntry(senderId, parts[3]);
-            } else if (parts[0] == "state-changes-subscription") {
-                if (parts[2] != "OK") {
-                    cerr << "state-changes-subscription failed with return code: " << parts[2];
+            Cmds cmds;
+            cmds.Deserialize(msg);
+            // cout << "Received " << cmds.Size() << " command(s) with total size of " << msg.length() << " bytes: " << endl;
+            for (const auto& cmd : cmds) {
+                // cout << " > " << cmd->GetType() << endl;
+                switch (cmd->GetType()) {
+                    case Type::state_change: {
+                        cout << "Received state_change from " << static_cast<StateChange&>(*cmd).GetDeviceId() << ": " << static_cast<StateChange&>(*cmd).GetLastState() << "->" << static_cast<StateChange&>(*cmd).GetCurrentState() << endl;
+                        if (static_cast<StateChange&>(*cmd).GetCurrentState() == fair::mq::State::Exiting) {
+                            ddsCustomCmd.send(Cmds(make<StateChangeExitingReceived>()).Serialize(), to_string(senderId));
+                        }
+                        waitMode.CountStates(static_cast<StateChange&>(*cmd).GetLastState(), static_cast<StateChange&>(*cmd).GetCurrentState());
+                    }
+                    break;
+                    case Type::state_change_subscription:
+                        if (static_cast<StateChangeSubscription&>(*cmd).GetResult() != Result::Ok) {
+                            cout << "State change subscription failed for " << static_cast<StateChangeSubscription&>(*cmd).GetDeviceId() << endl;
+                        }
+                    break;
+                    case Type::state_change_unsubscription:
+                        if (static_cast<StateChangeUnsubscription&>(*cmd).GetResult() != Result::Ok) {
+                            cout << "State change unsubscription failed for " << static_cast<StateChangeUnsubscription&>(*cmd).GetDeviceId() << endl;
+                        }
+                    break;
+                    case Type::transition_status: {
+                        if (static_cast<TransitionStatus&>(*cmd).GetResult() == Result::Ok) {
+                            cout << "Device " << static_cast<TransitionStatus&>(*cmd).GetDeviceId() << " started to transition with " << static_cast<TransitionStatus&>(*cmd).GetTransition() << endl;
+                        } else {
+                            cout << "Device " << static_cast<TransitionStatus&>(*cmd).GetDeviceId() << " cannot transition with " << static_cast<TransitionStatus&>(*cmd).GetTransition() << endl;
+                        }
+                    }
+                    break;
+                    case Type::current_state:
+                        cout << "Device " << static_cast<CurrentState&>(*cmd).GetDeviceId() << " is in " << static_cast<CurrentState&>(*cmd).GetCurrentState() << " state" << endl;
+                    break;
+                    case Type::config:
+                        cout << "Received config for device " << static_cast<Config&>(*cmd).GetDeviceId() << ":\n" << static_cast<Config&>(*cmd).GetConfig() << endl;
+                    break;
+                    default:
+                        cout << "Unexpected/unknown command received: " << cmd->GetType() << endl;
+                        cout << "Origin: " << senderId << endl;
+                    break;
                 }
-            } else if (parts[0] == "state-changes-unsubscription") {
-                if (parts[2] != "OK") {
-                    cerr << "state-changes-unsubscription failed with return code: " << parts[2];
-                }
-            } else {
-                // cout << "Received: " << msg << endl;
             }
         });
 
         service.start(sessionID);
 
+        StateSubscription stateSubscription(topologyPath, ddsCustomCmd);
+
         if (targetState == "") {
-            commandMode(command, topologyPath, ddsCustomCmd);
+            sendCommand(command, topologyPath, ddsCustomCmd);
         } else {
-            waitMode.Run(chrono::milliseconds(timeout), topologyPath, ddsCustomCmd, numberDevices, command);
+            waitMode.Run(chrono::milliseconds(timeout), topologyPath, ddsCustomCmd, numDevices, command);
         }
+
+        ddsCustomCmd.unsubscribe();
     } catch (exception& e) {
         cerr << "Error: " << e.what() << endl;
         return EXIT_FAILURE;
     }
-
     return EXIT_SUCCESS;
 }

fairmq/plugins/PMIx/CMakeLists.txt

@@ -16,7 +16,7 @@ target_link_libraries(${plugin} FairMQ PMIx::libpmix)
 target_include_directories(${plugin} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 set_target_properties(${plugin} PROPERTIES
   CXX_VISIBILITY_PRESET hidden
-  VERSION ${PROJECT_GIT_VERSION}
+  VERSION ${PROJECT_VERSION}
   SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
 )
 

fairmq/plugins/PMIx/PMIx.hpp

@@ -21,7 +21,7 @@
 #include <vector>
 #include <FairMQLogger.h>
 
-// C++ PMIx v2.1 API
+// C++ PMIx v2.2 API
 namespace pmix
 {
 
@@ -90,12 +90,6 @@ struct value : pmix_value_t
         }
     }
 
-    // template<typename T>
-    // value(const T* val, data_type dt)
-    // {
-        // PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), const_cast<void*>(val), dt);
-    // }
-
     template<typename T>
     explicit value(T)
     {
@@ -112,6 +106,11 @@ struct value : pmix_value_t
         PMIX_VALUE_LOAD(
             static_cast<pmix_value_t*>(this), const_cast<char*>(val.c_str()), PMIX_STRING);
     }
+
+    explicit value(int val)
+    {
+        PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), &val, PMIX_INT);
+    }
 };
 
 struct info : pmix_info_t

fairmq/plugins/PMIx/PMIxPlugin.cxx

@@ -1,5 +1,5 @@
 /********************************************************************************
- *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -8,7 +8,6 @@
 
 #include "PMIxPlugin.h"
 
-#include <boost/algorithm/string/join.hpp>
 #include <fairmq/Tools.h>
 #include <stdexcept>
 
@@ -27,14 +26,28 @@ PMIxPlugin::PMIxPlugin(const std::string& name,
     : Plugin(name, version, maintainer, homepage, pluginServices)
     , fPid(getpid())
 {
+    Init();
+    SetProperty<std::string>("id", std::string(fProc.nspace) + "_" + std::to_string(fProc.rank));
+    Fence();
+
     SubscribeToDeviceStateChange([&](DeviceState newState) {
         switch (newState) {
-            case DeviceState::Connecting:
-                Init();
-                Publish();
-                Fence();
-                Lookup();
-                break;
+        case DeviceState::Idle:
+            Fence();
+            break;
+        case DeviceState::Bound:
+            Publish();
+            Fence();
+            break;
+        case DeviceState::Connecting:
+            Lookup();
+            break;
+        case DeviceState::DeviceReady:
+            Fence();
+            break;
+        case DeviceState::Ready:
+            Fence();
+            break;
         case DeviceState::Exiting:
             UnsubscribeFromDeviceStateChange();
             break;
@@ -49,9 +62,9 @@ PMIxPlugin::~PMIxPlugin()
     while (pmix::initialized()) {
         try {
             pmix::finalize();
-            LOG(debug) << PMIxClient() << " pmix::finalize() OK";
+            LOG(debug) << PMIxClient() << "pmix::finalize() OK";
         } catch (const pmix::runtime_error& e) {
-            LOG(debug) << PMIxClient() << " pmix::finalize() failed: " << e.what();
+            LOG(debug) << PMIxClient() << "pmix::finalize() failed: " << e.what();
         }
     }
 }
@@ -59,7 +72,7 @@ PMIxPlugin::~PMIxPlugin()
 auto PMIxPlugin::PMIxClient() const -> std::string
 {
     std::stringstream ss;
-    ss << "PMIx client(pid=" << fPid << ")";
+    ss << "PMIx client(pid=" << fPid << ") ";
     return ss.str();
 }
 
@@ -67,7 +80,7 @@ auto PMIxPlugin::Init() -> void
 {
     if (!pmix::initialized()) {
         fProc = pmix::init();
-        LOG(debug) << PMIxClient() << " pmix::init() OK: " << fProc
+        LOG(debug) << PMIxClient() << "pmix::init() OK: " << fProc
                    << ",version=" << pmix::get_version();
     }
 }
@@ -90,7 +103,7 @@ auto PMIxPlugin::Publish() -> void
 
     if (info.size() > 0) {
         pmix::publish(info);
-        LOG(debug) << PMIxClient() << " pmix::publish() OK: published "
+        LOG(debug) << PMIxClient() << "pmix::publish() OK: published "
                    << info.size() << " binding channels.";
     }
 }
@@ -101,44 +114,43 @@ auto PMIxPlugin::Fence() -> void
     all.rank = pmix::rank::wildcard;
 
     pmix::fence({all});
+    LOG(debug) << PMIxClient() << "pmix::fence() OK";
 }
 
 auto PMIxPlugin::Lookup() -> void
 {
     auto channels(GetChannelInfo());
-    std::vector<pmix::pdata> pdata;
-
     for (const auto& c : channels) {
         std::string methodKey{"chans." + c.first + "." + std::to_string(c.second - 1) + ".method"};
         if (GetProperty<std::string>(methodKey) == "connect") {
             for (int i = 0; i < c.second; ++i) {
+                std::vector<pmix::pdata> pdata;
                 std::string addressKey{"chans." + c.first + "." + std::to_string(i) + ".address"};
                 pdata.emplace_back();
                 pdata.back().set_key(addressKey);
+                std::vector<pmix::info> info;
+                info.emplace_back(PMIX_WAIT, static_cast<int>(pdata.size()));
+
+                if (pdata.size() > 0) {
+                    pmix::lookup(pdata, info);
+                    LOG(debug) << PMIxClient() << "pmix::lookup() OK";
+                }
+
+                for (const auto& p : pdata) {
+                    if (p.value.type == PMIX_UNDEF) {
+                        LOG(debug) << PMIxClient() << "pmix::lookup() not found: key=" << p.key;
+                    } else if (p.value.type == PMIX_STRING) {
+                        LOG(debug) << PMIxClient() << "pmix::lookup() found:"
+				   << " key=" << p.key << ",value=" << p.value.data.string;
+                        SetProperty<std::string>(p.key, p.value.data.string);
+                    } else {
+                        LOG(debug) << PMIxClient() << "pmix::lookup() wrong type returned: "
+				   << "key=" << p.key << ",type=" << p.value.type;
+                    }
+                }
             }
         }
     }
-
-    if (pdata.size() > 0) {
-        pmix::lookup(pdata);
-        LOG(debug) << PMIxClient() << " pmix::lookup() OK";
-    }
-
-    LOG(info) << pdata.size();
-
-    for (const auto& p : pdata) {
-        if (p.value.type == PMIX_UNDEF) {
-            LOG(debug) << PMIxClient() << " pmix::lookup() not found: key=" << p.key;
-        } else if (p.value.type == PMIX_STRING) {
-            LOG(debug) << PMIxClient() << " pmix::lookup() found: key=" << p.key << ",value=" << p.value.data.string;
-            SetProperty<std::string>(p.key, p.value.data.string);
-            LOG(info) << GetProperty<std::string>(p.key);
-        } else {
-            LOG(debug) << PMIxClient() << " pmix::lookup() wrong type returned: key=" << p.key << ",type=" << p.value.type;
-        }
-    }
-
-    LOG(info) << pdata.size();
 }
 
 } /* namespace plugins */

fairmq/sdk/AsioAsyncOp.h

@@ -0,0 +1,223 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_SDK_ASIOASYNCOP_H
+#define FAIR_MQ_SDK_ASIOASYNCOP_H
+
+#include <asio/associated_allocator.hpp>
+#include <asio/associated_executor.hpp>
+#include <asio/executor_work_guard.hpp>
+#include <asio/system_executor.hpp>
+#include <chrono>
+#include <exception>
+#include <fairlogger/Logger.h>
+#include <fairmq/sdk/Error.h>
+#include <fairmq/sdk/Traits.h>
+#include <functional>
+#include <memory>
+#include <system_error>
+#include <type_traits>
+#include <utility>
+
+namespace fair {
+namespace mq {
+namespace sdk {
+
+template<typename... SignatureArgTypes>
+struct AsioAsyncOpImplBase
+{
+    virtual auto Complete(std::error_code, SignatureArgTypes...) -> void = 0;
+    virtual auto IsCompleted() const -> bool = 0;
+};
+
+/**
+ * @tparam Executor1 Associated I/O executor, see https://www.boost.org/doc/libs/1_70_0/doc/html/boost_asio/reference/asynchronous_operations.html#boost_asio.reference.asynchronous_operations.associated_i_o_executor
+ * @tparam Allocator1 Default allocation strategy, see https://www.boost.org/doc/libs/1_70_0/doc/html/boost_asio/reference/asynchronous_operations.html#boost_asio.reference.asynchronous_operations.allocation_of_intermediate_storage
+ */
+template<typename Executor1, typename Allocator1, typename Handler, typename... SignatureArgTypes>
+struct AsioAsyncOpImpl : AsioAsyncOpImplBase<SignatureArgTypes...>
+{
+    /// See https://www.boost.org/doc/libs/1_70_0/doc/html/boost_asio/reference/asynchronous_operations.html#boost_asio.reference.asynchronous_operations.allocation_of_intermediate_storage
+    using Allocator2 = typename asio::associated_allocator<Handler, Allocator1>::type;
+
+    /// See https://www.boost.org/doc/libs/1_70_0/doc/html/boost_asio/reference/asynchronous_operations.html#boost_asio.reference.asynchronous_operations.associated_completion_handler_executor
+    using Executor2 = typename asio::associated_executor<Handler, Executor1>::type;
+
+    /// Ctor
+    AsioAsyncOpImpl(const Executor1& ex1, Allocator1&& alloc1, Handler&& handler)
+        : fWork1(ex1)
+        , fWork2(asio::get_associated_executor(handler, ex1))
+        , fHandler(std::move(handler))
+        , fAlloc1(std::move(alloc1))
+    {}
+
+    auto GetAlloc2() const -> Allocator2 { return asio::get_associated_allocator(fHandler, fAlloc1); }
+    auto GetEx2() const -> Executor2 { return asio::get_associated_executor(fWork2); }
+
+    auto Complete(std::error_code ec, SignatureArgTypes... args) -> void override
+    {
+        if (IsCompleted()) {
+            throw RuntimeError("Async operation already completed");
+        }
+
+        GetEx2().dispatch(
+            [=, handler = std::move(fHandler)]() mutable {
+                try {
+                    handler(ec, args...);
+                } catch (const std::exception& e) {
+                    LOG(error) << "Uncaught exception in AsioAsyncOp completion handler: " << e.what();
+                } catch (...) {
+                    LOG(error) << "Unknown uncaught exception in AsioAsyncOp completion handler.";
+                }
+            },
+            GetAlloc2());
+
+        fWork1.reset();
+        fWork2.reset();
+    }
+
+    auto IsCompleted() const -> bool override
+    {
+        return !fWork1.owns_work() && !fWork2.owns_work();
+    }
+
+  private:
+    /// See https://www.boost.org/doc/libs/1_70_0/doc/html/boost_asio/reference/asynchronous_operations.html#boost_asio.reference.asynchronous_operations.outstanding_work
+    asio::executor_work_guard<Executor1> fWork1;
+    asio::executor_work_guard<Executor2> fWork2;
+    Handler fHandler;
+    Allocator1 fAlloc1;
+};
+
+/**
+ * @class AsioAsyncOp AsioAsyncOp.h <fairmq/sdk/AsioAsyncOp.h>
+ * @tparam Executor Associated I/O executor, see https://www.boost.org/doc/libs/1_70_0/doc/html/boost_asio/reference/asynchronous_operations.html#boost_asio.reference.asynchronous_operations.associated_i_o_executor
+ * @tparam Allocator Default allocation strategy, see https://www.boost.org/doc/libs/1_70_0/doc/html/boost_asio/reference/asynchronous_operations.html#boost_asio.reference.asynchronous_operations.allocation_of_intermediate_storage
+ * @tparam CompletionSignature
+ * @brief Interface for Asio-compliant asynchronous operation, see https://www.boost.org/doc/libs/1_70_0/doc/html/boost_asio/reference/asynchronous_operations.html
+ *
+ * @par Thread Safety
+ * @e Distinct @e objects: Safe.@n
+ * @e Shared @e objects: Unsafe.
+ *
+ * primary template
+ */
+template<typename Executor, typename Allocator, typename CompletionSignature>
+struct AsioAsyncOp
+{
+};
+
+/**
+ * @tparam Executor See primary template
+ * @tparam Allocator See primary template
+ * @tparam SignatureReturnType Return type of CompletionSignature, see primary template
+ * @tparam SignatureFirstArgType Type of first argument of CompletionSignature, see primary template
+ * @tparam SignatureArgTypes Types of the rest of arguments of CompletionSignature
+ *
+ * partial specialization to deconstruct CompletionSignature
+ */
+template<typename Executor,
+         typename Allocator,
+         typename SignatureReturnType,
+         typename SignatureFirstArgType,
+         typename... SignatureArgTypes>
+struct AsioAsyncOp<Executor,
+                   Allocator,
+                   SignatureReturnType(SignatureFirstArgType, SignatureArgTypes...)>
+{
+    static_assert(std::is_void<SignatureReturnType>::value,
+                  "return value of CompletionSignature must be void");
+    static_assert(std::is_same<SignatureFirstArgType, std::error_code>::value,
+                  "first argument of CompletionSignature must be std::error_code");
+    using Duration = std::chrono::milliseconds;
+
+  private:
+    using Impl = AsioAsyncOpImplBase<SignatureArgTypes...>;
+    using ImplPtr = std::unique_ptr<Impl, std::function<void(Impl*)>>;
+    ImplPtr fImpl;
+
+  public:
+    /// Default Ctor
+    AsioAsyncOp()
+        : fImpl(nullptr)
+    {}
+
+    /// Ctor with handler
+    template<typename Handler>
+    AsioAsyncOp(Executor&& ex1, Allocator&& alloc1, Handler&& handler)
+        : AsioAsyncOp()
+    {
+        // Async operation type to be allocated and constructed
+        using Op = AsioAsyncOpImpl<Executor, Allocator, Handler, SignatureArgTypes...>;
+
+        // Create allocator for concrete op type
+        // Allocator2, see https://www.boost.org/doc/libs/1_70_0/doc/html/boost_asio/reference/asynchronous_operations.html#boost_asio.reference.asynchronous_operations.allocation_of_intermediate_storage
+        using OpAllocator =
+            typename std::allocator_traits<typename Op::Allocator2>::template rebind_alloc<Op>;
+        OpAllocator opAlloc;
+
+        // Allocate memory
+        auto mem(std::allocator_traits<OpAllocator>::allocate(opAlloc, 1));
+
+        // Construct object
+        auto ptr(new (mem) Op(std::forward<Executor>(ex1),
+                              std::forward<Allocator>(alloc1),
+                              std::forward<Handler>(handler)));
+
+        // Assign ownership to this object
+        fImpl = ImplPtr(ptr, [opAlloc](Impl* p) mutable {
+            std::allocator_traits<OpAllocator>::deallocate(opAlloc, static_cast<Op*>(p), 1);
+        });
+    }
+
+    /// Ctor with handler #2
+    template<typename Handler>
+    AsioAsyncOp(Executor&& ex1, Handler&& handler)
+        : AsioAsyncOp(std::forward<Executor>(ex1), Allocator(), std::forward<Handler>(handler))
+    {}
+
+    /// Ctor with handler #3
+    template<typename Handler>
+    explicit AsioAsyncOp(Handler&& handler)
+        : AsioAsyncOp(asio::system_executor(), std::forward<Handler>(handler))
+    {}
+
+    auto IsCompleted() -> bool { return (fImpl == nullptr) || fImpl->IsCompleted(); }
+
+    auto Complete(std::error_code ec, SignatureArgTypes... args) -> void
+    {
+        if(IsCompleted()) {
+            throw RuntimeError("Async operation already completed");
+        }
+
+        fImpl->Complete(ec, args...);
+        fImpl.reset(nullptr);
+    }
+
+    auto Complete(SignatureArgTypes... args) -> void
+    {
+        Complete(std::error_code(), args...);
+    }
+
+    auto Cancel(SignatureArgTypes... args) -> void
+    {
+        Complete(MakeErrorCode(ErrorCode::OperationCanceled), args...);
+    }
+
+    auto Timeout(SignatureArgTypes... args) -> void
+    {
+        Complete(MakeErrorCode(ErrorCode::OperationTimeout), args...);
+    }
+};
+
+} /* namespace sdk */
+} /* namespace mq */
+} /* namespace fair */
+
+#endif /* FAIR_MQ_SDK_ASIOASYNCOP_H */
+

fairmq/sdk/AsioBase.h

@@ -0,0 +1,76 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_SDK_ASIOBASE_H
+#define FAIR_MQ_SDK_ASIOBASE_H
+
+#include <asio/executor.hpp>
+#include <fairmq/sdk/Traits.h>
+#include <memory>
+#include <utility>
+
+namespace fair {
+namespace mq {
+namespace sdk {
+
+using DefaultExecutor = asio::executor;
+using DefaultAllocator = std::allocator<int>;
+
+/**
+ * @class AsioBase AsioBase.h <fairmq/sdk/AsioBase.h>
+ * @tparam Executor Associated I/O executor
+ * @tparam Allocator Associated default allocator
+ * @brief Base for creating Asio-enabled I/O objects
+ *
+ * @par Thread Safety
+ * @e Distinct @e objects: Safe.@n
+ * @e Shared @e objects: Unsafe.
+ */
+template<typename Executor, typename Allocator>
+class AsioBase
+{
+  public:
+    /// Member type of associated I/O executor
+    using ExecutorType = Executor;
+    /// Get associated I/O executor
+    auto GetExecutor() const noexcept -> ExecutorType { return fExecutor; }
+
+    /// Member type of associated default allocator
+    using AllocatorType = Allocator;
+    /// Get associated default allocator
+    auto GetAllocator() const noexcept -> AllocatorType { return fAllocator; }
+
+    /// NO default ctor
+    AsioBase() = delete;
+
+    /// Construct with associated I/O executor
+    explicit AsioBase(Executor ex, Allocator alloc)
+        : fExecutor(std::move(ex))
+        , fAllocator(std::move(alloc))
+    {}
+
+    /// NOT copyable
+    AsioBase(const AsioBase&) = delete;
+    AsioBase& operator=(const AsioBase&) = delete;
+
+    /// movable
+    AsioBase(AsioBase&&) noexcept = default;
+    AsioBase& operator=(AsioBase&&) noexcept = default;
+
+    ~AsioBase() = default;
+
+  private:
+    ExecutorType fExecutor;
+    AllocatorType fAllocator;
+};
+
+} /* namespace sdk */
+} /* namespace mq */
+} /* namespace fair */
+
+#endif /* FAIR_MQ_SDK_ASIOBASE_H */

fairmq/sdk/CMakeLists.txt

@@ -15,10 +15,17 @@ set(target SDK)
 
 set(SDK_PUBLIC_HEADER_FILES
   ../SDK.h
+  AsioAsyncOp.h
+  AsioBase.h
+  DDSAgent.h
+  DDSCollection.h
   DDSEnvironment.h
   DDSSession.h
+  DDSTask.h
   DDSTopology.h
+  Error.h
   Topology.h
+  Traits.h
 )
 
 set(SDK_PRIVATE_HEADER_FILES
@@ -29,6 +36,7 @@ set(SDK_SOURCE_FILES
   DDSEnvironment.cxx
   DDSSession.cxx
   DDSTopology.cxx
+  Error.cxx
   Topology.cxx
 )
 
@@ -38,28 +46,30 @@ add_library(${target}
   ${SDK_PRIVATE_HEADER_FILES} # for IDE integration
 )
 set_target_properties(${target} PROPERTIES LABELS coverage)
-target_compile_definitions(${target} PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
 target_include_directories(${target}
   PUBLIC
   $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}>
   $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}>
-  $<INSTALL_INTERFACE:include>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
 )
 target_link_libraries(${target}
   PUBLIC
+  asio::headers
+  Boost::boost
   Boost::filesystem
   FairLogger::FairLogger
+  Threads::Threads
+  Tools
   StateMachine
+  Commands
 
   PRIVATE
-  Boost::boost
   DDS::dds_intercom_lib
   DDS::dds_tools_lib
   DDS::dds_topology_lib
-  Tools
 )
 set_target_properties(${target} PROPERTIES
-  VERSION ${PROJECT_GIT_VERSION}
+  VERSION ${PROJECT_VERSION}
   SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
   OUTPUT_NAME FairMQ_${target}
 )

fairmq/sdk/DDSAgent.h

@@ -0,0 +1,81 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_SDK_DDSSAGENT_H
+#define FAIR_MQ_SDK_DDSSAGENT_H
+
+#include <fairmq/sdk/DDSSession.h>
+
+#include <ostream>
+#include <string>
+#include <chrono>
+#include <cstdint>
+
+namespace fair {
+namespace mq {
+namespace sdk {
+
+/**
+ * @class DDSAgent <fairmq/sdk/DDSAgent.h>
+ * @brief Represents a DDS agent
+ */
+class DDSAgent
+{
+  public:
+    using Id = uint64_t;
+    using Pid = uint32_t;
+
+    explicit DDSAgent(DDSSession session,
+                      Id id,
+                      Pid pid,
+                      std::string path,
+                      std::string host,
+                      std::chrono::milliseconds startupTime,
+                      std::string username)
+        : fSession(std::move(session))
+        , fId(id)
+        , fPid(pid)
+        , fDDSPath(std::move(path))
+        , fHost(std::move(host))
+        , fStartupTime(startupTime)
+        , fUsername(std::move(username))
+    {}
+
+    DDSSession GetSession() const { return fSession; }
+    Id GetId() const { return fId; }
+    Pid GetPid() const { return fPid; }
+    std::string GetHost() const { return fHost; }
+    std::string GetDDSPath() const { return fDDSPath; }
+    std::chrono::milliseconds GetStartupTime() const { return fStartupTime; }
+    std::string GetUsername() const { return fUsername; }
+
+    friend auto operator<<(std::ostream& os, const DDSAgent& agent) -> std::ostream&
+    {
+        return os << "DDSAgent id: " << agent.fId
+                  << ", pid: " << agent.fPid
+                  << ", path: " << agent.fDDSPath
+                  << ", host: " << agent.fHost
+                  << ", startupTime: " << agent.fStartupTime.count()
+                  << ", username: " << agent.fUsername;
+    }
+
+  private:
+    DDSSession fSession;
+    Id fId;
+    Pid fPid;
+    std::string fDDSPath;
+    std::string fHost;
+    std::chrono::milliseconds fStartupTime;
+    std::string fUsername;
+};
+
+}   // namespace sdk
+}   // namespace mq
+}   // namespace fair
+
+#endif /* FAIR_MQ_SDK_DDSSAGENT_H */

fairmq/sdk/DDSCollection.h

@@ -0,0 +1,49 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_SDK_DDSCOLLECTION_H
+#define FAIR_MQ_SDK_DDSCOLLECTION_H
+
+// #include <fairmq/sdk/DDSAgent.h>
+
+#include <ostream>
+#include <cstdint>
+
+namespace fair {
+namespace mq {
+namespace sdk {
+
+/**
+ * @class DDSCollection <fairmq/sdk/DDSCollection.h>
+ * @brief Represents a DDS collection
+ */
+class DDSCollection
+{
+  public:
+    using Id = std::uint64_t;
+
+    explicit DDSCollection(Id id)
+        : fId(id)
+    {}
+
+    Id GetId() const { return fId; }
+
+    friend auto operator<<(std::ostream& os, const DDSCollection& collection) -> std::ostream&
+    {
+        return os << "DDSCollection id: " << collection.fId;
+    }
+
+  private:
+    Id fId;
+};
+
+}   // namespace sdk
+}   // namespace mq
+}   // namespace fair
+
+#endif /* FAIR_MQ_SDK_DDSCOLLECTION_H */

fairmq/sdk/DDSEnvironment.cxx

@@ -8,15 +8,11 @@
 
 #include "DDSEnvironment.h"
 
+#include <cstdlib>
+#include <dds/dds.h>
+#include <fairlogger/Logger.h>
 #include <fairmq/Tools.h>
 #include <fairmq/sdk/DDSInfo.h>
-
-#include <fairlogger/Logger.h>
-
-#include <DDS/Tools.h>
-#include <DDS/dds_intercom.h>
-
-#include <cstdlib>
 #include <sstream>
 #include <utility>
 

fairmq/sdk/DDSSession.cxx

@@ -8,20 +8,21 @@
 
 #include "DDSSession.h"
 
-#include <fairmq/sdk/DDSEnvironment.h>
-#include <fairmq/sdk/DDSTopology.h>
-#include <fairmq/Tools.h>
-
-#include <fairlogger/Logger.h>
-
-#include <DDS/Tools.h>
-
+#include <boost/process.hpp>
 #include <boost/uuid/uuid_io.hpp>
-
 #include <cassert>
 #include <cstdlib>
+#include <dds/dds.h>
+#include <fairlogger/Logger.h>
+#include <fairmq/Tools.h>
+#include <fairmq/sdk/DDSAgent.h>
+#include <fairmq/sdk/DDSEnvironment.h>
+#include <fairmq/sdk/DDSTopology.h>
+#include <mutex>
 #include <sstream>
+#include <unordered_map>
 #include <utility>
+#include <vector>
 
 namespace fair {
 namespace mq {
@@ -134,6 +135,8 @@ struct DDSSession::Impl
     dds::intercom_api::CCustomCmd fDDSCustomCmd;
     Id fId;
     bool fStopOnDestruction;
+    mutable std::mutex fMtx;
+    std::unordered_map<DDSChannel::Id, DDSTask::Id> fTaskIdByChannelIdMap;
 };
 
 DDSSession::DDSSession(DDSEnvironment env)
@@ -176,76 +179,111 @@ auto DDSSession::SubmitAgents(Quantity agents) -> void
     // Requesting to submit 0 agents is not meaningful
     assert(agents > 0);
 
-    dds::tools_api::SSubmitRequestData submitInfo;
+    using namespace dds::tools_api;
+
+    SSubmitRequestData submitInfo;
     submitInfo.m_rms = tools::ToString(GetRMSPlugin());
-    submitInfo.m_instances = agents;
+    submitInfo.m_instances = 1;
+    submitInfo.m_slots = agents; // TODO new api: get slots from agents
     submitInfo.m_config = GetRMSConfig().string();
 
-    tools::Semaphore blocker;
-    auto submitRequest = dds::tools_api::SSubmitRequest::makeRequest(submitInfo);
-    submitRequest->setMessageCallback(
-        [](const dds::tools_api::SMessageResponseData& message) { LOG(debug) << message; });
-    submitRequest->setDoneCallback([&]() {
+    tools::SharedSemaphore blocker;
+    auto submitRequest = SSubmitRequest::makeRequest(submitInfo);
+    submitRequest->setMessageCallback([](const SMessageResponseData& message){
+        LOG(debug) << message.m_msg;
+    });
+    submitRequest->setDoneCallback([agents, blocker]() mutable {
         LOG(debug) << agents << " Agents submitted";
         blocker.Signal();
     });
 
-    fImpl->fSession->sendRequest<dds::tools_api::SSubmitRequest>(submitRequest);
+    fImpl->fSession->sendRequest<SSubmitRequest>(submitRequest);
     blocker.Wait();
 
-    // perfect
     WaitForIdleAgents(agents);
 }
 
-auto DDSSession::RequestAgentInfo() -> AgentInfo
+auto DDSSession::RequestAgentCount() -> AgentCount
 {
-    dds::tools_api::SAgentInfoRequestData agentInfoInfo;
-    tools::Semaphore blocker;
-    AgentInfo info;
-    auto agentInfoRequest = dds::tools_api::SAgentInfoRequest::makeRequest(agentInfoInfo);
-    agentInfoRequest->setResponseCallback(
-        [this, &info](const dds::tools_api::SAgentInfoResponseData& _response) {
-            if (_response.m_index == 0) {
-                info.activeAgentsCount = _response.m_activeAgentsCount;
-                info.idleAgentsCount = _response.m_idleAgentsCount;
-                info.executingAgentsCount = _response.m_executingAgentsCount;
-                info.agents.reserve(_response.m_activeAgentsCount);
-            }
-            info.agents.emplace_back(*this, _response.m_agentInfo);
-        });
-    agentInfoRequest->setMessageCallback(
-        [](const dds::tools_api::SMessageResponseData& _message) { LOG(debug) << _message; });
-    agentInfoRequest->setDoneCallback([&]() { blocker.Signal(); });
-    fImpl->fSession->sendRequest<dds::tools_api::SAgentInfoRequest>(agentInfoRequest);
-    blocker.Wait();
+    using namespace dds::tools_api;
 
-    return info;
+    SAgentCountRequest::response_t res;
+    fImpl->fSession->syncSendRequest<SAgentCountRequest>(SAgentCountRequest::request_t(), res);
+
+    AgentCount count;
+    count.active = res.m_activeSlotsCount;
+    count.idle = res.m_idleSlotsCount;
+    count.executing = res.m_executingSlotsCount;
+
+    return count;
+}
+
+auto DDSSession::RequestAgentInfo() -> std::vector<DDSAgent>
+{
+    using namespace dds::tools_api;
+
+    SAgentInfoRequest::responseVector_t res;
+    fImpl->fSession->syncSendRequest<SAgentInfoRequest>(SAgentInfoRequest::request_t(), res);
+
+    std::vector<DDSAgent> agentInfo;
+    agentInfo.reserve(res.size());
+    for (const auto& a : res) {
+        agentInfo.emplace_back(
+            *this,
+            a.m_agentID,
+            a.m_agentPid,
+            a.m_DDSPath,
+            a.m_host,
+            a.m_startUpTime,
+            a.m_username
+            // a.m_nSlots
+        );
+    }
+
+    return agentInfo;
+}
+
+auto DDSSession::RequestTaskInfo() -> std::vector<DDSTask>
+{
+    using namespace dds::tools_api;
+
+    SAgentInfoRequest::responseVector_t res;
+    fImpl->fSession->syncSendRequest<SAgentInfoRequest>(SAgentInfoRequest::request_t(), res);
+
+    std::vector<DDSTask> taskInfo;
+    taskInfo.reserve(res.size());
+    for (auto& a : res) {
+        //taskInfo.emplace_back(a.m_taskID, 0);
+        (void)a;
+        taskInfo.emplace_back(0, 0);
+    }
+
+    return taskInfo;
 }
 
 auto DDSSession::RequestCommanderInfo() -> CommanderInfo
 {
-    dds::tools_api::SCommanderInfoRequestData commanderInfoInfo;
-    tools::Semaphore blocker;
+    using namespace dds::tools_api;
+
+    SCommanderInfoRequestData commanderInfo;
+    tools::SharedSemaphore blocker;
     std::string error;
-    auto commanderInfoRequest =
-        dds::tools_api::SCommanderInfoRequest::makeRequest(commanderInfoInfo);
+    auto commanderInfoRequest = SCommanderInfoRequest::makeRequest(commanderInfo);
     CommanderInfo info;
-    commanderInfoRequest->setResponseCallback(
-        [&info](const dds::tools_api::SCommanderInfoResponseData& _response) {
-            info.pid = _response.m_pid;
-            info.activeTopologyName = _response.m_activeTopologyName;
-        });
-    commanderInfoRequest->setMessageCallback(
-        [&](const dds::tools_api::SMessageResponseData& _message) {
-            if (_message.m_severity == dds::intercom_api::EMsgSeverity::error) {
-                error = _message.m_msg;
-                blocker.Signal();
-            } else {
-                LOG(debug) << _message;
-            }
-        });
-    commanderInfoRequest->setDoneCallback([&]() { blocker.Signal(); });
-    fImpl->fSession->sendRequest<dds::tools_api::SCommanderInfoRequest>(commanderInfoRequest);
+    commanderInfoRequest->setResponseCallback([&info](const SCommanderInfoResponseData& _response) {
+        info.pid = _response.m_pid;
+        info.activeTopologyName = _response.m_activeTopologyName;
+    });
+    commanderInfoRequest->setMessageCallback([&](const SMessageResponseData& _message) {
+        if (_message.m_severity == dds::intercom_api::EMsgSeverity::error) {
+            error = _message.m_msg;
+            blocker.Signal();
+        } else {
+            LOG(debug) << _message.m_msg;
+        }
+    });
+    commanderInfoRequest->setDoneCallback([blocker]() mutable { blocker.Signal(); });
+    fImpl->fSession->sendRequest<SCommanderInfoRequest>(commanderInfoRequest);
     blocker.Wait();
 
     if (!error.empty()) {
@@ -257,38 +295,41 @@ auto DDSSession::RequestCommanderInfo() -> CommanderInfo
 
 auto DDSSession::WaitForExecutingAgents(Quantity minCount) -> void
 {
-    auto info(RequestAgentInfo());
+    auto count(RequestAgentCount());
     int interval(8);
-    while (info.executingAgentsCount < minCount) {
+    while (count.executing < minCount) {
         std::this_thread::sleep_for(std::chrono::milliseconds(interval));
         interval = std::min(256, interval * 2);
-        info = RequestAgentInfo();
+        count = RequestAgentCount();
     }
 }
 
 auto DDSSession::WaitForIdleAgents(Quantity minCount) -> void
 {
-    auto info(RequestAgentInfo());
+    auto count(RequestAgentCount());
     int interval(8);
-    while (info.idleAgentsCount < minCount) {
+    while (count.idle < minCount) {
         std::this_thread::sleep_for(std::chrono::milliseconds(interval));
         interval = std::min(256, interval * 2);
-        info = RequestAgentInfo();
+        count = RequestAgentCount();
     }
 }
 
 auto DDSSession::ActivateTopology(DDSTopology topo) -> void
 {
-    dds::tools_api::STopologyRequestData topologyInfo;
-    topologyInfo.m_updateType = dds::tools_api::STopologyRequestData::EUpdateType::ACTIVATE;
+    using namespace dds::tools_api;
+
+    STopologyRequestData topologyInfo;
+    topologyInfo.m_updateType = STopologyRequestData::EUpdateType::ACTIVATE;
     topologyInfo.m_topologyFile = topo.GetTopoFile().string();
 
-    tools::Semaphore blocker;
-    auto topologyRequest = dds::tools_api::STopologyRequest::makeRequest(topologyInfo);
-    topologyRequest->setMessageCallback(
-        [](const dds::tools_api::SMessageResponseData& _message) { LOG(debug) << _message; });
-    topologyRequest->setDoneCallback([&]() { blocker.Signal(); });
-    fImpl->fSession->sendRequest<dds::tools_api::STopologyRequest>(topologyRequest);
+    tools::SharedSemaphore blocker;
+    auto topologyRequest = STopologyRequest::makeRequest(topologyInfo);
+    topologyRequest->setMessageCallback([](const SMessageResponseData& _message) {
+        LOG(debug) << _message.m_msg;
+    });
+    topologyRequest->setDoneCallback([blocker]() mutable { blocker.Signal(); });
+    fImpl->fSession->sendRequest<STopologyRequest>(topologyRequest);
     blocker.Wait();
 
     WaitForExecutingAgents(topo.GetNumRequiredAgents());
@@ -316,24 +357,59 @@ void DDSSession::UnsubscribeFromCommands()
 
 void DDSSession::SendCommand(const std::string& cmd) { fImpl->fDDSCustomCmd.send(cmd, ""); }
 
+void DDSSession::SendCommand(const std::string& cmd, DDSChannel::Id recipient)
+{
+    fImpl->fDDSCustomCmd.send(cmd, std::to_string(recipient));
+}
+
+auto DDSSession::UpdateChannelToTaskAssociation(DDSChannel::Id channelId, DDSTask::Id taskId) -> void
+{
+    std::lock_guard<std::mutex> lk(fImpl->fMtx);
+    fImpl->fTaskIdByChannelIdMap[channelId] = taskId;
+}
+
+auto DDSSession::GetTaskId(DDSChannel::Id channelId) const -> DDSTask::Id
+{
+    std::lock_guard<std::mutex> lk(fImpl->fMtx);
+    return fImpl->fTaskIdByChannelIdMap.at(channelId);
+}
+
 auto operator<<(std::ostream& os, const DDSSession& session) -> std::ostream&
 {
     return os << "$DDS_SESSION_ID: " << session.GetId();
 }
 
-auto DDSAgent::GetSession() const -> DDSSession
+auto getMostRecentRunningDDSSession(DDSEnv env) -> DDSSession
 {
-    return fSession;
-}
+    boost::process::ipstream pipeStream;
+    boost::process::child c("dds-session list all", boost::process::std_out > pipeStream);
+    std::string lastLine;
+    std::string currentLine;
 
-auto DDSAgent::GetInfoStr() const -> std::string
-{
-    return fInfoStr;
-}
+    while (pipeStream && std::getline(pipeStream, currentLine) && !currentLine.empty()) {
+        lastLine = currentLine;
+    }
+    c.wait();
+    std::string sessionId;
 
-auto operator<<(std::ostream& os, const DDSAgent& agent) -> std::ostream&
-{
-    return os << agent.GetInfoStr();
+    if (!lastLine.empty()) {
+        std::vector<std::string> words;
+        std::istringstream iss(lastLine);
+        for (std::string s; iss >> s;) {
+            if (s != "*") {
+                words.push_back(s);
+            }
+        }
+        if (words.back() == "RUNNING") {
+            sessionId = words.front();
+        }
+    }
+
+    if (sessionId.empty()) {
+        throw std::runtime_error("could not find most recent DDS session");
+    }
+
+    return DDSSession(DDSSession::Id(sessionId), std::move(env));
 }
 
 }   // namespace sdk

fairmq/sdk/DDSSession.h

@@ -11,6 +11,7 @@
 
 #include <fairmq/sdk/DDSEnvironment.h>
 #include <fairmq/sdk/DDSInfo.h>
+#include <fairmq/sdk/DDSTask.h>
 
 #include <boost/filesystem.hpp>
 
@@ -21,13 +22,12 @@
 #include <stdexcept>
 #include <string>
 #include <functional>
+#include <vector>
 
 namespace fair {
 namespace mq {
 namespace sdk {
 
-class DDSEnvironment;
-
 /**
  * @enum DDSRMSPlugin DDSSession.h <fairmq/sdk/DDSSession.h>
  * @brief Supported DDS resource management system plugins
@@ -43,6 +43,12 @@ auto operator>>(std::istream& is, DDSRMSPlugin& plugin) -> std::istream&;
 class DDSTopology;
 class DDSAgent;
 
+class DDSChannel
+{
+  public:
+    using Id = std::uint64_t;
+};
+
 /**
  * @class DDSSession DDSSession.h <fairmq/sdk/DDSSession.h>
  * @brief Represents a DDS session
@@ -72,13 +78,14 @@ class DDSSession
     auto StopOnDestruction(bool stop = true) -> void;
     auto IsRunning() const -> bool;
     auto SubmitAgents(Quantity agents) -> void;
-    struct AgentInfo {
-        Quantity idleAgentsCount = 0;
-        Quantity activeAgentsCount = 0;
-        Quantity executingAgentsCount = 0;
-        std::vector<DDSAgent> agents;
+    struct AgentCount {
+        Quantity idle = 0;
+        Quantity active = 0;
+        Quantity executing = 0;
     };
-    auto RequestAgentInfo() -> AgentInfo;
+    auto RequestAgentCount() -> AgentCount;
+    auto RequestAgentInfo() -> std::vector<DDSAgent>;
+    auto RequestTaskInfo() -> std::vector<DDSTask>;
     struct CommanderInfo {
         int pid = -1;
         std::string activeTopologyName;
@@ -95,6 +102,9 @@ class DDSSession
     void SubscribeToCommands(std::function<void(const std::string& msg, const std::string& condition, uint64_t senderId)>);
     void UnsubscribeFromCommands();
     void SendCommand(const std::string&);
+    void SendCommand(const std::string&, DDSChannel::Id);
+    auto UpdateChannelToTaskAssociation(DDSChannel::Id, DDSTask::Id) -> void;
+    auto GetTaskId(DDSChannel::Id) const -> DDSTask::Id;
 
     friend auto operator<<(std::ostream& os, const DDSSession& session) -> std::ostream&;
 
@@ -103,27 +113,8 @@ class DDSSession
     std::shared_ptr<Impl> fImpl;
 };
 
-/**
- * @class DDSAgent DDSSession.h <fairmq/sdk/DDSSession.h>
- * @brief Represents a DDS agent
- */
-class DDSAgent
-{
-  public:
-    explicit DDSAgent(DDSSession session, std::string infostr)
-        : fInfoStr(std::move(infostr))
-        , fSession(std::move(session))
-    {}
+auto getMostRecentRunningDDSSession(DDSEnv env = {}) -> DDSSession;
 
-    auto GetSession() const -> DDSSession;
-    auto GetInfoStr() const -> std::string;
-
-    friend auto operator<<(std::ostream& os, const DDSAgent& plugin) -> std::ostream&;
-
-  private:
-    std::string fInfoStr;
-    DDSSession fSession;
-};
 }   // namespace sdk
 }   // namespace mq
 }   // namespace fair

fairmq/sdk/DDSTask.h

@@ -0,0 +1,52 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_SDK_DDSTASK_H
+#define FAIR_MQ_SDK_DDSTASK_H
+
+#include <fairmq/sdk/DDSCollection.h>
+
+#include <ostream>
+#include <cstdint>
+
+namespace fair {
+namespace mq {
+namespace sdk {
+
+/**
+ * @class DDSTask <fairmq/sdk/DDSTask.h>
+ * @brief Represents a DDS task
+ */
+class DDSTask
+{
+  public:
+    using Id = std::uint64_t;
+
+    explicit DDSTask(Id id, Id collectionId)
+        : fId(id)
+        , fCollectionId(collectionId)
+    {}
+
+    Id GetId() const { return fId; }
+    DDSCollection::Id GetCollectionId() const { return fCollectionId; }
+
+    friend auto operator<<(std::ostream& os, const DDSTask& task) -> std::ostream&
+    {
+        return os << "DDSTask id: " << task.fId << ", collection id: " << task.fCollectionId;
+    }
+
+  private:
+    Id fId;
+    DDSCollection::Id fCollectionId;
+};
+
+}   // namespace sdk
+}   // namespace mq
+}   // namespace fair
+
+#endif /* FAIR_MQ_SDK_DDSTASK_H */

fairmq/sdk/DDSTopology.cxx

@@ -9,18 +9,14 @@
 #include "DDSTopology.h"
 
 #include <boost/range/iterator_range.hpp>
-
-#include <fairmq/sdk/DDSEnvironment.h>
-#include <fairmq/Tools.h>
-
+#include <dds/dds.h>
 #include <fairlogger/Logger.h>
-
-#include <DDS/Topology.h>
-
+#include <fairmq/Tools.h>
+#include <fairmq/sdk/DDSEnvironment.h>
+#include <memory>
 #include <sstream>
 #include <stdexcept>
 #include <utility>
-#include <memory>
 
 namespace fair {
 namespace mq {
@@ -63,29 +59,51 @@ auto DDSTopology::GetTopoFile() const -> Path
     return file;
 }
 
-int DDSTopology::GetNumRequiredAgents()
+auto DDSTopology::GetNumRequiredAgents() const -> int
 {
     return fImpl->fTopo.getRequiredNofAgents();
 }
 
-std::vector<uint64_t> DDSTopology::GetDeviceList()
+auto DDSTopology::GetTasks() const -> std::vector<DDSTask>
 {
-    std::vector<uint64_t> taskIDs;
-    taskIDs.reserve(GetNumRequiredAgents());
+    std::vector<DDSTask> list;
+    list.reserve(GetNumRequiredAgents());
 
-    // TODO make sure returned tasks are actually devices
     auto itPair = fImpl->fTopo.getRuntimeTaskIterator(
-        [](const dds::topology_api::STopoRuntimeTask::FilterIterator_t::value_type& /*value*/) -> bool { return true; });
+        [](const dds::topology_api::STopoRuntimeTask::FilterIterator_t::value_type&) -> bool {
+            return true;
+        });
     auto tasks = boost::make_iterator_range(itPair.first, itPair.second);
 
-    for (auto task : tasks) {
-        LOG(debug) << "Found task " << task.first << ": "
+    for (const auto& task : tasks) {
+        LOG(debug) << "Found task with id: " << task.first << ", "
                    << "Path: " << task.second.m_taskPath << ", "
+                   << "Collection id: " << task.second.m_taskCollectionId << ", "
                    << "Name: " << task.second.m_task->getName() << "_" << task.second.m_taskIndex;
-        taskIDs.push_back(task.first);
+        list.emplace_back(task.first, task.second.m_taskCollectionId);
     }
 
-    return taskIDs;
+    return list;
+}
+
+auto DDSTopology::GetCollections() const -> std::vector<DDSCollection>
+{
+    std::vector<DDSCollection> list;
+
+    auto itPair = fImpl->fTopo.getRuntimeCollectionIterator(
+        [](const dds::topology_api::STopoRuntimeCollection::FilterIterator_t::value_type&) -> bool {
+            return true;
+        });
+    auto collections = boost::make_iterator_range(itPair.first, itPair.second);
+
+    for (const auto& c : collections) {
+        LOG(debug) << "Found collection with id: " << c.first << ", "
+                   << "Index: " << c.second.m_collectionIndex << ", "
+                   << "Path: " << c.second.m_collectionPath;
+        list.emplace_back(c.first);
+    }
+
+    return list;
 }
 
 auto DDSTopology::GetName() const -> std::string { return fImpl->fTopo.getName(); }

fairmq/sdk/DDSTopology.h

@@ -10,10 +10,13 @@
 #define FAIR_MQ_SDK_DDSTOPOLOGY_H
 
 #include <boost/filesystem.hpp>
-#include <fairmq/sdk/DDSInfo.h>
+#include <fairmq/sdk/DDSCollection.h>
 #include <fairmq/sdk/DDSEnvironment.h>
+#include <fairmq/sdk/DDSInfo.h>
+#include <fairmq/sdk/DDSTask.h>
 #include <memory>
 #include <string>
+#include <vector>
 
 namespace fair {
 namespace mq {
@@ -48,10 +51,13 @@ class DDSTopology
     auto GetTopoFile() const -> Path;
 
     /// @brief Get number of required agents for this topology
-    int GetNumRequiredAgents();
+    auto GetNumRequiredAgents() const -> int;
 
-    /// @brief Get list of devices
-    std::vector<uint64_t> GetDeviceList();
+    /// @brief Get list of tasks in this topology
+    auto GetTasks() const -> std::vector<DDSTask>;
+
+    /// @brief Get list of tasks in this topology
+    auto GetCollections() const -> std::vector<DDSCollection>;
 
     /// @brief Get the name of the topology
     auto GetName() const -> std::string;

fairmq/sdk/Error.cxx

@@ -0,0 +1,40 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "Error.h"
+
+namespace fair {
+namespace mq {
+
+const char* ErrorCategory::name() const noexcept
+{
+    return "fairmq";
+}
+
+std::string ErrorCategory::message(int ev) const
+{
+    switch (static_cast<ErrorCode>(ev)) {
+        case ErrorCode::OperationInProgress:
+            return "async operation already in progress";
+        case ErrorCode::OperationTimeout:
+            return "async operation timed out";
+        case ErrorCode::OperationCanceled:
+            return "async operation canceled";
+        case ErrorCode::DeviceChangeStateFailed:
+            return "failed to change state of a fairmq device";
+        default:
+            return "(unrecognized error)";
+    }
+}
+
+const ErrorCategory errorCategory{};
+
+std::error_code MakeErrorCode(ErrorCode e) { return {static_cast<int>(e), errorCategory}; }
+
+}   // namespace mq
+}   // namespace fair

fairmq/sdk/Error.h

@@ -0,0 +1,62 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_SDK_ERROR_H
+#define FAIR_MQ_SDK_ERROR_H
+
+#include <fairmq/tools/Strings.h>
+#include <stdexcept>
+#include <system_error>
+
+namespace fair {
+namespace mq {
+namespace sdk {
+
+struct RuntimeError : ::std::runtime_error
+{
+    template<typename... T>
+    explicit RuntimeError(T&&... t)
+        : ::std::runtime_error::runtime_error(tools::ToString(std::forward<T>(t)...))
+    {}
+};
+
+struct MixedStateError : RuntimeError
+{
+    using RuntimeError::RuntimeError;
+};
+
+} /* namespace sdk */
+
+enum class ErrorCode
+{
+    OperationInProgress = 10,
+    OperationTimeout,
+    OperationCanceled,
+    DeviceChangeStateFailed
+};
+
+std::error_code MakeErrorCode(ErrorCode);
+
+struct ErrorCategory : std::error_category
+{
+  const char* name() const noexcept override;
+  std::string message(int ev) const override;
+};
+
+} /* namespace mq */
+} /* namespace fair */
+
+namespace std {
+
+template<>
+struct is_error_code_enum<fair::mq::ErrorCode> : true_type
+{};
+
+}   // namespace std
+
+#endif /* FAIR_MQ_SDK_ERROR_H */

fairmq/sdk/Topology.cxx

@@ -8,246 +8,21 @@
 
 #include "Topology.h"
 
-#include <DDS/Tools.h>
-#include <DDS/Topology.h>
-#include <boost/algorithm/string/split.hpp>
-#include <boost/algorithm/string/trim.hpp>
-#include <condition_variable>
-#include <fairlogger/Logger.h>
-#include <future>
-#include <mutex>
-#include <thread>
-#include <utility>
+#include <dds/dds.h>
 
 namespace fair {
 namespace mq {
-
-auto operator<<(std::ostream& os, AsyncOpResultCode v) -> std::ostream&
-{
-    switch (v) {
-        case AsyncOpResultCode::Aborted:
-            return os << "Aborted";
-        case AsyncOpResultCode::Timeout:
-            return os << "Timeout";
-        case AsyncOpResultCode::Error:
-            return os << "Error";
-        case AsyncOpResultCode::Ok:
-        default:
-            return os << "Ok";
-    }
-}
-
-auto operator<<(std::ostream& os, AsyncOpResult v) -> std::ostream&
-{
-    os << "[" << v.code << "]";
-    if (!v.msg.empty()) {
-        os << " " << v.msg;
-    }
-    return os;
-}
-
 namespace sdk {
 
-const std::unordered_map<DeviceTransition, DeviceState, tools::HashEnum<DeviceTransition>> expectedState =
+/// @brief Helper to (Re)Construct a FairMQ topology based on already existing native DDS API objects
+/// @param nativeSession Existing and initialized CSession (either via create() or attach())
+/// @param nativeTopo Existing CTopology that is activated on the given nativeSession
+/// @param env Optional DDSEnv (needed primarily for unit testing)
+auto MakeTopology(dds::topology_api::CTopology nativeTopo,
+                  std::shared_ptr<dds::tools_api::CSession> nativeSession,
+                  DDSEnv env) -> Topology
 {
-    { Transition::InitDevice,   DeviceState::InitializingDevice },
-    { Transition::CompleteInit, DeviceState::Initialized },
-    { Transition::Bind,         DeviceState::Bound },
-    { Transition::Connect,      DeviceState::DeviceReady },
-    { Transition::InitTask,     DeviceState::Ready },
-    { Transition::Run,          DeviceState::Running },
-    { Transition::Stop,         DeviceState::Ready },
-    { Transition::ResetTask,    DeviceState::DeviceReady },
-    { Transition::ResetDevice,  DeviceState::Idle },
-    { Transition::End,          DeviceState::Exiting }
-};
-
-Topology::Topology(DDSTopology topo, DDSSession session)
-    : fDDSSession(std::move(session))
-    , fDDSTopo(std::move(topo))
-    , fStateChangeOngoing(false)
-    , fTargetState(DeviceState::Idle)
-    , fStateChangeTimeout(0)
-    , fShutdown(false)
-{
-    std::vector<uint64_t> deviceList = fDDSTopo.GetDeviceList();
-    for (const auto& d : deviceList) {
-        // LOG(debug) << "Adding device " << d;
-        fState.emplace(d, DeviceStatus{ false, DeviceState::Ok });
-    }
-    fDDSSession.SubscribeToCommands([this](const std::string& msg, const std::string& /* condition */, uint64_t senderId) {
-        // LOG(debug) << "Received from " << senderId << ": " << msg;
-        std::vector<std::string> parts;
-        boost::algorithm::split(parts, msg, boost::algorithm::is_any_of(":,"));
-
-        for (unsigned int i = 0; i < parts.size(); ++i) {
-            boost::trim(parts.at(i));
-        }
-
-        if (parts[0] == "state-change") {
-            AddNewStateEntry(std::stoull(parts[2]), parts[3]);
-        } else if (parts[0] == "state-changes-subscription") {
-            LOG(debug) << "Received from " << senderId << ": " << msg;
-            if (parts[2] != "OK") {
-                LOG(error) << "state-changes-subscription failed with return code: " << parts[2];
-            }
-        } else if (parts[0] == "state-changes-unsubscription") {
-            if (parts[2] != "OK") {
-                LOG(error) << "state-changes-unsubscription failed with return code: " << parts[2];
-            }
-        } else if (parts[1] == "could not queue") {
-            LOG(warn) << "Could not queue " << parts[2] << " transition on " << senderId;
-        }
-    });
-    fDDSSession.StartDDSService();
-    LOG(debug) << "subscribe-to-state-changes";
-    fDDSSession.SendCommand("subscribe-to-state-changes");
-
-    fExecutionThread = std::thread(&Topology::WaitForState, this);
-}
-
-Topology::Topology(dds::topology_api::CTopology nativeTopo,
-                   std::shared_ptr<dds::tools_api::CSession> nativeSession,
-                   DDSEnv env)
-    : Topology(DDSTopo(std::move(nativeTopo), env), DDSSession(std::move(nativeSession), env))
-{
-    if (fDDSSession.RequestCommanderInfo().activeTopologyName != fDDSTopo.GetName()) {
-        throw std::runtime_error("Given topology must be activated");
-    }
-}
-
-auto Topology::ChangeState(TopologyTransition transition, ChangeStateCallback cb, Duration timeout) -> void
-{
-    {
-        std::unique_lock<std::mutex> lock(fMtx);
-        if (fStateChangeOngoing) {
-            throw std::runtime_error("A state change request is already in progress, concurrent requests are currently not supported");
-            lock.unlock();
-        }
-        LOG(debug) << "Initiating ChangeState with " << transition << " to " << expectedState.at(transition);
-        fStateChangeOngoing = true;
-        fChangeStateCallback = cb;
-        fStateChangeTimeout = timeout;
-        fTargetState = expectedState.at(transition);
-
-        fDDSSession.SendCommand(GetTransitionName(transition));
-    }
-    fExecutionCV.notify_one();
-}
-
-auto Topology::ChangeState(TopologyTransition t, Duration timeout) -> ChangeStateResult
-{
-    fair::mq::tools::Semaphore blocker;
-    ChangeStateResult res;
-    ChangeState(
-        t,
-        [&blocker, &res](Topology::ChangeStateResult _res) {
-            res = _res;
-            blocker.Signal();
-        },
-        timeout);
-    blocker.Wait();
-    return res;
-}
-
-void Topology::WaitForState()
-{
-    while (!fShutdown) {
-        if (fStateChangeOngoing) {
-            try {
-                auto condition = [&] {
-                    // LOG(info) << "checking condition";
-                    // LOG(info) << "fShutdown: " << fShutdown;
-                    // LOG(info) << "condition: " << std::all_of(fState.cbegin(), fState.cend(),
-                    // [&](TopologyState::value_type i) { return i.second.state == fTargetState; });
-                    return fShutdown
-                           || std::all_of(
-                               fState.cbegin(), fState.cend(), [&](TopologyState::value_type i) {
-                                   // TODO Check, if we can make sure that EXITING state change event are not missed
-                                   return (fTargetState == DeviceState::Exiting)
-                                          || ((i.second.state == fTargetState)
-                                              && i.second.initialized);
-                               });
-                };
-
-                std::unique_lock<std::mutex> lock(fMtx);
-
-                if (fStateChangeTimeout > std::chrono::milliseconds(0)) {
-                    if (!fCV.wait_for(lock, fStateChangeTimeout, condition)) {
-                        // LOG(debug) << "timeout";
-                        fStateChangeOngoing = false;
-                        TopologyState state = fState;
-                        lock.unlock();
-                        fChangeStateCallback(
-                            {{AsyncOpResultCode::Timeout, "timeout"}, std::move(state)});
-                        break;
-                    }
-                } else {
-                    fCV.wait(lock, condition);
-                }
-
-                fStateChangeOngoing = false;
-                if (fShutdown) {
-                    LOG(debug) << "Aborting because a shutdown was requested";
-                    TopologyState state = fState;
-                    lock.unlock();
-                    fChangeStateCallback(
-                        {{AsyncOpResultCode::Aborted, "Aborted because a shutdown was requested"},
-                         std::move(state)});
-                    break;
-                }
-            } catch (std::exception& e) {
-                fStateChangeOngoing = false;
-                LOG(error) << "Error while processing state request: " << e.what();
-                fChangeStateCallback(
-                    {{AsyncOpResultCode::Error, tools::ToString("Exception thrown: ", e.what())},
-                     fState});
-            }
-
-            fChangeStateCallback({{AsyncOpResultCode::Ok, "success"}, fState});
-        } else {
-            std::unique_lock<std::mutex> lock(fExecutionMtx);
-            fExecutionCV.wait(lock);
-        }
-    }
-    LOG(debug) << "WaitForState shutting down";
-};
-
-void Topology::AddNewStateEntry(uint64_t senderId, const std::string& state)
-{
-    std::size_t pos = state.find("->");
-    std::string endState = state.substr(pos + 2);
-    // LOG(debug) << "Adding new state entry: " << senderId << ", " << state << ", end state: " << endState;
-    {
-        try {
-            std::unique_lock<std::mutex> lock(fMtx);
-            fState[senderId] = DeviceStatus{ true, fair::mq::GetState(endState) };
-        } catch (const std::exception& e) {
-            LOG(error) << "Exception in AddNewStateEntry: " << e.what();
-        }
-
-        // LOG(info) << "fState after update: ";
-        // for (auto& e : fState) {
-        //     LOG(info) << e.first << ": " << e.second.state;
-        // }
-    }
-    fCV.notify_one();
-}
-
-Topology::~Topology()
-{
-    fDDSSession.UnsubscribeFromCommands();
-    {
-        std::lock_guard<std::mutex> guard(fExecutionMtx);
-        fShutdown = true;
-    }
-    fExecutionCV.notify_one();
-    fExecutionThread.join();
-}
-
-auto operator<<(std::ostream& os, Topology::ChangeStateResult v) -> std::ostream&
-{
-    return os << v.rc;
+    return {DDSTopo(std::move(nativeTopo), env), DDSSession(std::move(nativeSession), env)};
 }
 
 }   // namespace sdk

fairmq/sdk/Topology.h

@@ -9,70 +9,84 @@
 #ifndef FAIR_MQ_SDK_TOPOLOGY_H
 #define FAIR_MQ_SDK_TOPOLOGY_H
 
+#include <fairmq/sdk/AsioAsyncOp.h>
+#include <fairmq/sdk/AsioBase.h>
+#include <fairmq/sdk/commands/Commands.h>
+#include <fairmq/sdk/DDSCollection.h>
 #include <fairmq/sdk/DDSInfo.h>
 #include <fairmq/sdk/DDSSession.h>
+#include <fairmq/sdk/DDSTask.h>
 #include <fairmq/sdk/DDSTopology.h>
+#include <fairmq/sdk/Error.h>
 #include <fairmq/States.h>
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Semaphore.h>
 
-#include <functional>
-#include <unordered_map>
-#include <memory>
+#include <fairlogger/Logger.h>
+
+#include <asio/associated_executor.hpp>
+#include <asio/async_result.hpp>
+#include <asio/steady_timer.hpp>
+#include <asio/system_executor.hpp>
+
+#include <algorithm>
 #include <chrono>
+#include <functional>
+#include <map>
+#include <memory>
+#include <mutex>
 #include <ostream>
-#include <string>
-#include <vector>
 #include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
 
 namespace fair {
 namespace mq {
-
-enum class AsyncOpResultCode
-{
-    Ok,
-    Timeout,
-    Error,
-    Aborted
-};
-auto operator<<(std::ostream& os, AsyncOpResultCode v) -> std::ostream&;
-
-using AsyncOpResultMessage = std::string;
-
-struct AsyncOpResult {
-    AsyncOpResultCode code;
-    AsyncOpResultMessage msg;
-    operator AsyncOpResultCode() const { return code; }
-};
-auto operator<<(std::ostream& os, AsyncOpResult v) -> std::ostream&;
-
 namespace sdk {
 
 using DeviceState = fair::mq::State;
 using DeviceTransition = fair::mq::Transition;
 
+const std::map<DeviceTransition, DeviceState> expectedState =
+{
+    { DeviceTransition::InitDevice,   DeviceState::InitializingDevice },
+    { DeviceTransition::CompleteInit, DeviceState::Initialized },
+    { DeviceTransition::Bind,         DeviceState::Bound },
+    { DeviceTransition::Connect,      DeviceState::DeviceReady },
+    { DeviceTransition::InitTask,     DeviceState::Ready },
+    { DeviceTransition::Run,          DeviceState::Running },
+    { DeviceTransition::Stop,         DeviceState::Ready },
+    { DeviceTransition::ResetTask,    DeviceState::DeviceReady },
+    { DeviceTransition::ResetDevice,  DeviceState::Idle },
+    { DeviceTransition::End,          DeviceState::Exiting }
+};
+
 struct DeviceStatus
 {
     bool initialized;
     DeviceState state;
+    DDSTask::Id taskId;
+    DDSCollection::Id collectionId;
 };
 
-using TopologyState = std::unordered_map<uint64_t, DeviceStatus>;
+using TopologyState = std::vector<DeviceStatus>;
+using TopologyStateIndex = std::unordered_map<DDSTask::Id, int>; //  task id -> index in the data vector
+using TopologyStateByTask = std::unordered_map<DDSTask::Id, DeviceStatus>;
+using TopologyStateByCollection = std::unordered_map<DDSCollection::Id, std::vector<DeviceStatus>>;
 using TopologyTransition = fair::mq::Transition;
 
-struct MixedState : std::runtime_error { using std::runtime_error::runtime_error; };
-
 inline DeviceState AggregateState(const TopologyState& topologyState)
 {
-    DeviceState first = topologyState.begin()->second.state;
+    DeviceState first = topologyState.begin()->state;
 
     if (std::all_of(topologyState.cbegin(), topologyState.cend(), [&](TopologyState::value_type i) {
-            return i.second.state == first;
+            return i.state == first;
         })) {
         return first;
     }
 
-    throw MixedState("State is not uniform");
-
+    throw MixedStateError("State is not uniform");
 }
 
 inline bool StateEqualsTo(const TopologyState& topologyState, DeviceState state)
@@ -80,83 +94,398 @@ inline bool StateEqualsTo(const TopologyState& topologyState, DeviceState state)
     return AggregateState(topologyState) == state;
 }
 
+inline TopologyStateByCollection GroupByCollectionId(const TopologyState& topologyState)
+{
+    TopologyStateByCollection state;
+    for (const auto& ds : topologyState) {
+        if (ds.collectionId != 0) {
+            state[ds.collectionId].push_back(ds);
+        }
+    }
+
+    return state;
+}
+
+inline TopologyStateByTask GroupByTaskId(const TopologyState& topologyState)
+{
+    TopologyStateByTask state;
+    for (const auto& ds : topologyState) {
+        state[ds.taskId] = ds;
+    }
+
+    return state;
+}
+
 /**
- * @class Topology Topology.h <fairmq/sdk/Topology.h>
+ * @class BasicTopology Topology.h <fairmq/sdk/Topology.h>
+ * @tparam Executor Associated I/O executor
+ * @tparam Allocator Associated default allocator
  * @brief Represents a FairMQ topology
+ *
+ * @par Thread Safety
+ * @e Distinct @e objects: Safe.@n
+ * @e Shared @e objects: Safe.
  */
-class Topology
+template <typename Executor, typename Allocator>
+class BasicTopology : public AsioBase<Executor, Allocator>
 {
   public:
     /// @brief (Re)Construct a FairMQ topology from an existing DDS topology
     /// @param topo DDSTopology
     /// @param session DDSSession
-    explicit Topology(DDSTopology topo, DDSSession session = DDSSession());
+    BasicTopology(DDSTopology topo, DDSSession session)
+        : BasicTopology<Executor, Allocator>(asio::system_executor(), std::move(topo), std::move(session))
+    {}
 
-    /// @brief (Re)Construct a FairMQ topology based on already existing native DDS API objects
-    /// @param nativeSession Existing and initialized CSession (either via create() or attach())
-    /// @param nativeTopo Existing CTopology that is activated on the given nativeSession
-    /// @param env Optional DDSEnv (needed primarily for unit testing)
-    explicit Topology(dds::topology_api::CTopology nativeTopo,
-                      std::shared_ptr<dds::tools_api::CSession> nativeSession,
-                      DDSEnv env = {});
+    /// @brief (Re)Construct a FairMQ topology from an existing DDS topology
+    /// @param ex I/O executor to be associated
+    /// @param topo DDSTopology
+    /// @param session DDSSession
+    /// @throws RuntimeError
+    BasicTopology(const Executor& ex,
+                  DDSTopology topo,
+                  DDSSession session,
+                  Allocator alloc = DefaultAllocator())
+        : AsioBase<Executor, Allocator>(ex, std::move(alloc))
+        , fDDSSession(std::move(session))
+        , fDDSTopo(std::move(topo))
+        , fStateData()
+        , fStateIndex()
+        , fChangeStateOp()
+        , fChangeStateOpTimer(ex)
+        , fChangeStateTarget(DeviceState::Idle)
+    {
+        makeTopologyState();
 
-    explicit Topology(const Topology&) = delete;
-    Topology& operator=(const Topology&) = delete;
-    explicit Topology(Topology&&) = delete;
-    Topology& operator=(Topology&&) = delete;
+        std::string activeTopo(fDDSSession.RequestCommanderInfo().activeTopologyName);
+        std::string givenTopo(fDDSTopo.GetName());
+        if (activeTopo != givenTopo) {
+            throw RuntimeError("Given topology ", givenTopo, " is not activated (active: ", activeTopo, ")");
+        }
 
-    ~Topology();
+        using namespace fair::mq::sdk::cmd;
+        fDDSSession.SubscribeToCommands([&](const std::string& msg, const std::string& /* condition */, DDSChannel::Id senderId) {
+            Cmds inCmds;
+            inCmds.Deserialize(msg);
+            // LOG(debug) << "Received " << inCmds.Size() << " command(s) with total size of " << msg.length() << " bytes: ";
+            for (const auto& cmd : inCmds) {
+                // LOG(debug) << " > " << cmd->GetType();
+                switch (cmd->GetType()) {
+                    case Type::state_change: {
+                        DDSTask::Id taskId(static_cast<StateChange&>(*cmd).GetTaskId());
+                        fDDSSession.UpdateChannelToTaskAssociation(senderId, taskId);
+                        if (static_cast<StateChange&>(*cmd).GetCurrentState() == DeviceState::Exiting) {
+                            Cmds outCmds;
+                            outCmds.Add<StateChangeExitingReceived>();
+                            fDDSSession.SendCommand(outCmds.Serialize(), senderId);
+                        }
+                        UpdateStateEntry(taskId, static_cast<StateChange&>(*cmd).GetCurrentState());
+                    }
+                    break;
+                    case Type::state_change_subscription:
+                        if (static_cast<StateChangeSubscription&>(*cmd).GetResult() != Result::Ok) {
+                            LOG(error) << "State change subscription failed for " << static_cast<StateChangeSubscription&>(*cmd).GetDeviceId();
+                        }
+                    break;
+                    case Type::state_change_unsubscription:
+                        if (static_cast<StateChangeUnsubscription&>(*cmd).GetResult() != Result::Ok) {
+                            LOG(error) << "State change unsubscription failed for " << static_cast<StateChangeUnsubscription&>(*cmd).GetDeviceId();
+                        }
+                    break;
+                    case Type::transition_status: {
+                        if (static_cast<TransitionStatus&>(*cmd).GetResult() != Result::Ok) {
+                            LOG(error) << "Transition failed for " << static_cast<TransitionStatus&>(*cmd).GetDeviceId();
+                            std::lock_guard<std::mutex> lk(fMtx);
+                            if (!fChangeStateOp.IsCompleted() && fStateData.at(fStateIndex.at(fDDSSession.GetTaskId(senderId))).state != fChangeStateTarget) {
+                                fChangeStateOpTimer.cancel();
+                                fChangeStateOp.Complete(MakeErrorCode(ErrorCode::DeviceChangeStateFailed), fStateData);
+                            }
+                        }
+                    }
+                    break;
+                    default:
+                        LOG(warn) << "Unexpected/unknown command received: " << cmd->GetType();
+                        LOG(warn) << "Origin: " << senderId;
+                    break;
+                }
+            }
+        });
+
+        fDDSSession.StartDDSService();
+        LOG(debug) << "Subscribing to state change";
+        Cmds cmds(make<SubscribeToStateChange>());
+        fDDSSession.SendCommand(cmds.Serialize());
+    }
+
+    /// not copyable
+    BasicTopology(const BasicTopology&) = delete;
+    BasicTopology& operator=(const BasicTopology&) = delete;
+
+    /// movable
+    BasicTopology(BasicTopology&&) = default;
+    BasicTopology& operator=(BasicTopology&&) = default;
+
+    ~BasicTopology()
+    {
+        std::lock_guard<std::mutex> lk(fMtx);
+        fDDSSession.UnsubscribeFromCommands();
+        try {
+            fChangeStateOp.Cancel(fStateData);
+        } catch (...) {}
+    }
 
-    struct ChangeStateResult {
-        AsyncOpResult rc;
-        TopologyState state;
-        friend auto operator<<(std::ostream& os, ChangeStateResult v) -> std::ostream&;
-    };
-    using ChangeStateCallback = std::function<void(ChangeStateResult)>;
     using Duration = std::chrono::milliseconds;
+    using ChangeStateCompletionSignature = void(std::error_code, TopologyState);
 
     /// @brief Initiate state transition on all FairMQ devices in this topology
-    /// @param t FairMQ device state machine transition
-    /// @param cb Completion callback
+    /// @param transition FairMQ device state machine transition
     /// @param timeout Timeout in milliseconds, 0 means no timeout
-    auto ChangeState(TopologyTransition t, ChangeStateCallback cb, Duration timeout = std::chrono::milliseconds(0)) -> void;
+    /// @param token Asio completion token
+    /// @tparam CompletionToken Asio completion token type
+    /// @throws std::system_error
+    ///
+    /// @par Usage examples
+    /// With lambda:
+    /// @code
+    /// topo.AsyncChangeState(
+    ///     fair::mq::sdk::TopologyTransition::InitDevice,
+    ///     std::chrono::milliseconds(500),
+    ///     [](std::error_code ec, TopologyState state) {
+    ///         if (!ec) {
+    ///             // success
+    ///          } else if (ec.category().name() == "fairmq") {
+    ///             switch (static_cast<fair::mq::ErrorCode>(ec.value())) {
+    ///               case fair::mq::ErrorCode::OperationTimeout:
+    ///                 // async operation timed out
+    ///               case fair::mq::ErrorCode::OperationCanceled:
+    ///                 // async operation canceled
+    ///               case fair::mq::ErrorCode::DeviceChangeStateFailed:
+    ///                 // failed to change state of a fairmq device
+    ///               case fair::mq::ErrorCode::OperationInProgress:
+    ///                 // async operation already in progress
+    ///               default:
+    ///             }
+    ///         }
+    ///     }
+    /// );
+    /// @endcode
+    /// With future:
+    /// @code
+    /// auto fut = topo.AsyncChangeState(fair::mq::sdk::TopologyTransition::InitDevice,
+    ///                                  std::chrono::milliseconds(500),
+    ///                                  asio::use_future);
+    /// try {
+    ///     fair::mq::sdk::TopologyState state = fut.get();
+    ///     // success
+    /// } catch (const std::system_error& ex) {
+    ///     auto ec(ex.code());
+    ///     if (ec.category().name() == "fairmq") {
+    ///         switch (static_cast<fair::mq::ErrorCode>(ec.value())) {
+    ///           case fair::mq::ErrorCode::OperationTimeout:
+    ///             // async operation timed out
+    ///           case fair::mq::ErrorCode::OperationCanceled:
+    ///             // async operation canceled
+    ///           case fair::mq::ErrorCode::DeviceChangeStateFailed:
+    ///             // failed to change state of a fairmq device
+    ///           case fair::mq::ErrorCode::OperationInProgress:
+    ///             // async operation already in progress
+    ///           default:
+    ///         }
+    ///     }
+    /// }
+    /// @endcode
+    /// With coroutine (C++20, see https://en.cppreference.com/w/cpp/language/coroutines):
+    /// @code
+    /// try {
+    ///     fair::mq::sdk::TopologyState state = co_await
+    ///         topo.AsyncChangeState(fair::mq::sdk::TopologyTransition::InitDevice,
+    ///                               std::chrono::milliseconds(500),
+    ///                               asio::use_awaitable);
+    ///     // success
+    /// } catch (const std::system_error& ex) {
+    ///     auto ec(ex.code());
+    ///     if (ec.category().name() == "fairmq") {
+    ///         switch (static_cast<fair::mq::ErrorCode>(ec.value())) {
+    ///           case fair::mq::ErrorCode::OperationTimeout:
+    ///             // async operation timed out
+    ///           case fair::mq::ErrorCode::OperationCanceled:
+    ///             // async operation canceled
+    ///           case fair::mq::ErrorCode::DeviceChangeStateFailed:
+    ///             // failed to change state of a fairmq device
+    ///           case fair::mq::ErrorCode::OperationInProgress:
+    ///             // async operation already in progress
+    ///           default:
+    ///         }
+    ///     }
+    /// }
+    /// @endcode
+    template<typename CompletionToken>
+    auto AsyncChangeState(TopologyTransition transition,
+                          Duration timeout,
+                          CompletionToken&& token)
+    {
+        return asio::async_initiate<CompletionToken, ChangeStateCompletionSignature>(
+            [&](auto handler) {
+                std::lock_guard<std::mutex> lk(fMtx);
 
-    /// @brief Perform a state transition on all FairMQ devices in this topology
-    /// @param t FairMQ device state machine transition
+                if (fChangeStateOp.IsCompleted()) {
+                    fChangeStateOp = ChangeStateOp(AsioBase<Executor, Allocator>::GetExecutor(),
+                                                   AsioBase<Executor, Allocator>::GetAllocator(),
+                                                   std::move(handler));
+                    fChangeStateTarget = expectedState.at(transition);
+                    ResetTransitionedCount(fChangeStateTarget);
+                    cmd::Cmds cmds(cmd::make<cmd::ChangeState>(transition));
+                    fDDSSession.SendCommand(cmds.Serialize());
+                    if (timeout > std::chrono::milliseconds(0)) {
+                        fChangeStateOpTimer.expires_after(timeout);
+                        fChangeStateOpTimer.async_wait([&](std::error_code ec) {
+                            if (!ec) {
+                                std::lock_guard<std::mutex> lk2(fMtx);
+                                fChangeStateOp.Timeout(fStateData);
+                            }
+                        });
+                    }
+                } else {
+                    // TODO refactor to hide boiler plate
+                    auto ex2(asio::get_associated_executor(handler, AsioBase<Executor, Allocator>::GetExecutor()));
+                    auto alloc2(asio::get_associated_allocator(handler, AsioBase<Executor, Allocator>::GetAllocator()));
+                    auto state(GetCurrentStateUnsafe());
+
+                    ex2.post(
+                        [h = std::move(handler), s = std::move(state)]() mutable {
+                            try {
+                                h(MakeErrorCode(ErrorCode::OperationInProgress), s);
+                            } catch (const std::exception& e) {
+                                LOG(error) << "Uncaught exception in completion handler: " << e.what();
+                            } catch (...) {
+                                LOG(error) << "Unknown uncaught exception in completion handler.";
+                            }
+                        },
+                        alloc2);
+                }
+            },
+            token);
+    }
+
+    template<typename CompletionToken>
+    auto AsyncChangeState(TopologyTransition transition, CompletionToken&& token)
+    {
+        return AsyncChangeState(transition, Duration(0), std::move(token));
+    }
+
+    /// @brief Perform state transition on all FairMQ devices in this topology
+    /// @param transition FairMQ device state machine transition
     /// @param timeout Timeout in milliseconds, 0 means no timeout
-    /// @return The result of the state transition
-    auto ChangeState(TopologyTransition t, Duration timeout = std::chrono::milliseconds(0)) -> ChangeStateResult;
+    /// @tparam CompletionToken Asio completion token type
+    /// @throws std::system_error
+    auto ChangeState(TopologyTransition transition, Duration timeout = Duration(0))
+        -> std::pair<std::error_code, TopologyState>
+    {
+        tools::SharedSemaphore blocker;
+        std::error_code ec;
+        TopologyState state;
+        AsyncChangeState(
+            transition, timeout, [&, blocker](std::error_code _ec, TopologyState _state) mutable {
+                ec = _ec;
+                state = _state;
+                blocker.Signal();
+            });
+        blocker.Wait();
+        return {ec, state};
+    }
 
     /// @brief Returns the current state of the topology
     /// @return map of id : DeviceStatus (initialized, state)
-    TopologyState GetCurrentState() const { std::lock_guard<std::mutex> guard(fMtx); return fState; }
+    auto GetCurrentState() const -> TopologyState
+    {
+        std::lock_guard<std::mutex> lk(fMtx);
+        return fStateData;
+    }
 
-    DeviceState AggregateState() { return sdk::AggregateState(fState); }
+    auto AggregateState() const -> DeviceState { return sdk::AggregateState(GetCurrentState()); }
 
-    bool StateEqualsTo(DeviceState state) { return sdk::StateEqualsTo(fState, state); }
+    auto StateEqualsTo(DeviceState state) const -> bool { return sdk::StateEqualsTo(GetCurrentState(), state); }
 
   private:
+    using TransitionedCount = unsigned int;
+
     DDSSession fDDSSession;
     DDSTopology fDDSTopo;
-    TopologyState fState;
-    std::unordered_map<uint64_t, DeviceStatus> fStateChangesSubscriptions;
-    bool fStateChangeOngoing;
-    DeviceState fTargetState;
+    TopologyState fStateData;
+    TopologyStateIndex fStateIndex;
     mutable std::mutex fMtx;
-    mutable std::mutex fExecutionMtx;
-    std::condition_variable fCV;
-    std::condition_variable fExecutionCV;
-    std::thread fExecutionThread;
-    ChangeStateCallback fChangeStateCallback;
-    std::chrono::milliseconds fStateChangeTimeout;
-    bool fShutdown;
 
-    void WaitForState();
-    void AddNewStateEntry(uint64_t senderId, const std::string& state);
+    using ChangeStateOp = AsioAsyncOp<Executor, Allocator, ChangeStateCompletionSignature>;
+    ChangeStateOp fChangeStateOp;
+    asio::steady_timer fChangeStateOpTimer;
+    DeviceState fChangeStateTarget;
+    TransitionedCount fTransitionedCount;
+
+    auto makeTopologyState() -> void
+    {
+        fStateData.reserve(fDDSTopo.GetTasks().size());
+
+        int index = 0;
+
+        for (const auto& task : fDDSTopo.GetTasks()) {
+            fStateData.push_back(DeviceStatus{false, DeviceState::Ok, task.GetId(), task.GetCollectionId()});
+            fStateIndex.emplace(task.GetId(), index);
+            index++;
+        }
+    }
+
+    auto UpdateStateEntry(const DDSTask::Id taskId, const DeviceState state) -> void
+    {
+        try {
+            std::lock_guard<std::mutex> lk(fMtx);
+            DeviceStatus& task = fStateData.at(fStateIndex.at(taskId));
+            task.initialized = true;
+            task.state = state;
+            if (task.state == fChangeStateTarget) {
+                ++fTransitionedCount;
+            }
+            // LOG(debug) << "Updated state entry: taskId=" << taskId << ", state=" << state;
+            TryChangeStateCompletion();
+        } catch (const std::exception& e) {
+            LOG(error) << "Exception in UpdateStateEntry: " << e.what();
+        }
+    }
+
+    /// precodition: fMtx is locked.
+    auto TryChangeStateCompletion() -> void
+    {
+        if (!fChangeStateOp.IsCompleted() && fTransitionedCount == fStateData.size()) {
+            fChangeStateOpTimer.cancel();
+            fChangeStateOp.Complete(fStateData);
+        }
+    }
+
+    /// precodition: fMtx is locked.
+    auto ResetTransitionedCount(DeviceState targetState) -> void
+    {
+        fTransitionedCount = std::count_if(fStateIndex.cbegin(), fStateIndex.cend(), [=](const auto& s) {
+            return fStateData.at(s.second).state == targetState;
+        });
+    }
+
+    /// precodition: fMtx is locked.
+    auto GetCurrentStateUnsafe() const -> TopologyState
+    {
+        return fStateData;
+    }
 };
 
+using Topology = BasicTopology<DefaultExecutor, DefaultAllocator>;
 using Topo = Topology;
 
+/// @brief Helper to (Re)Construct a FairMQ topology based on already existing native DDS API objects
+/// @param nativeSession Existing and initialized CSession (either via create() or attach())
+/// @param nativeTopo Existing CTopology that is activated on the given nativeSession
+/// @param env Optional DDSEnv (needed primarily for unit testing)
+auto MakeTopology(dds::topology_api::CTopology nativeTopo,
+                  std::shared_ptr<dds::tools_api::CSession> nativeSession,
+                  DDSEnv env = {}) -> Topology;
+
 }   // namespace sdk
 }   // namespace mq
 }   // namespace fair

fairmq/sdk/Traits.h

@@ -0,0 +1,50 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_SDK_TRAITS_H
+#define FAIR_MQ_SDK_TRAITS_H
+
+#include <asio/associated_allocator.hpp>
+#include <asio/associated_executor.hpp>
+#include <type_traits>
+
+namespace asio {
+namespace detail {
+
+/// Specialize to match our coding conventions
+template<typename T, typename Executor>
+struct associated_executor_impl<T,
+                                Executor,
+                                std::enable_if_t<is_executor<typename T::ExecutorType>::value>>
+{
+    using type = typename T::ExecutorType;
+
+    static auto get(const T& obj, const Executor& /*ex = Executor()*/) noexcept -> type
+    {
+        return obj.GetExecutor();
+    }
+};
+
+/// Specialize to match our coding conventions
+template<typename T, typename Allocator>
+struct associated_allocator_impl<T,
+                                Allocator,
+                                std::enable_if_t<T::AllocatorType>>
+{
+    using type = typename T::AllocatorType;
+
+    static auto get(const T& obj, const Allocator& /*alloc = Allocator()*/) noexcept -> type
+    {
+        return obj.GetAllocator();
+    }
+};
+
+} /* namespace detail */
+} /* namespace asio */
+
+#endif /* FAIR_MQ_SDK_TRAITS_H */

fairmq/sdk/commands/CMakeLists.txt

@@ -0,0 +1,55 @@
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+set(target Commands)
+
+add_custom_command(
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/CommandsFormat.h
+  COMMAND $<TARGET_FILE:flatbuffers::flatc> -c -o ${CMAKE_CURRENT_BINARY_DIR} CommandsFormat.fbs
+  COMMAND ${CMAKE_COMMAND} -E rename ${CMAKE_CURRENT_BINARY_DIR}/CommandsFormat_generated.h ${CMAKE_CURRENT_BINARY_DIR}/CommandsFormat.h
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)
+
+# JSON serialization needs to see the .fbs file at run time, save it as constexpr string instead of locating/opening it every time
+file(STRINGS CommandsFormat.fbs tmp)
+list(JOIN tmp "\n" commands_format_def_fbs)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CommandsFormatDef.h.in ${CMAKE_CURRENT_BINARY_DIR}/CommandsFormatDef.h)
+
+add_library(${target} Commands.cxx Commands.h ${CMAKE_CURRENT_BINARY_DIR}/CommandsFormat.h ${CMAKE_CURRENT_BINARY_DIR}/CommandsFormatDef.h)
+add_library(FairMQ::${target} ALIAS ${target})
+
+# Some distros may not package the static library (e.g. Fedora), so we pick up the dynamic library instead
+set(_flatbuffers flatbuffers::flatbuffers)
+if(NOT TARGET flatbuffers::flatbuffers AND TARGET flatbuffers::flatbuffers_shared)
+  set(_flatbuffers flatbuffers::flatbuffers_shared)
+endif()
+
+target_link_libraries(${target}
+  PUBLIC
+  StateMachine
+  Tools
+
+  PRIVATE
+  ${_flatbuffers}
+)
+
+target_include_directories(${target}
+  PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}>
+  $<INSTALL_INTERFACE:include>
+)
+
+install(
+  TARGETS ${target}
+  EXPORT ${PROJECT_EXPORT_SET}
+  DESTINATION ${PROJECT_INSTALL_LIBDIR}
+)
+
+install(FILES Commands.h
+  DESTINATION ${PROJECT_INSTALL_INCDIR}/sdk/commands
+)

fairmq/sdk/commands/Commands.cxx

@@ -0,0 +1,431 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "Commands.h"
+
+#include <fairmq/sdk/commands/CommandsFormatDef.h>
+#include <fairmq/sdk/commands/CommandsFormat.h>
+
+#include <flatbuffers/idl.h>
+
+#include <array>
+
+using namespace std;
+
+namespace fair {
+namespace mq {
+namespace sdk {
+namespace cmd {
+
+array<Result, 2> fbResultToResult =
+{
+    {
+        Result::Ok,
+        Result::Failure
+    }
+};
+
+array<FBResult, 2> resultToFBResult =
+{
+    {
+        FBResult::FBResult_Ok,
+        FBResult::FBResult_Failure
+    }
+};
+
+array<string, 2> resultNames =
+{
+    {
+        "Ok",
+        "Failure"
+    }
+};
+
+array<string, 17> typeNames =
+{
+    {
+        "CheckState",
+        "ChangeState",
+        "DumpConfig",
+        "SubscribeToHeartbeats",
+        "UnsubscribeFromHeartbeats",
+        "SubscribeToStateChange",
+        "UnsubscribeFromStateChange",
+        "StateChangeExitingReceived",
+
+        "CurrentState",
+        "TransitionStatus",
+        "Config",
+        "HeartbeatSubscription",
+        "HeartbeatUnsubscription",
+        "Heartbeat",
+        "StateChangeSubscription",
+        "StateChangeUnsubscription",
+        "StateChange"
+    }
+};
+
+array<fair::mq::State, 15> fbStateToMQState =
+{
+    {
+        fair::mq::State::Ok,
+        fair::mq::State::Error,
+        fair::mq::State::Idle,
+        fair::mq::State::InitializingDevice,
+        fair::mq::State::Initialized,
+        fair::mq::State::Binding,
+        fair::mq::State::Bound,
+        fair::mq::State::Connecting,
+        fair::mq::State::DeviceReady,
+        fair::mq::State::InitializingTask,
+        fair::mq::State::Ready,
+        fair::mq::State::Running,
+        fair::mq::State::ResettingTask,
+        fair::mq::State::ResettingDevice,
+        fair::mq::State::Exiting
+    }
+};
+
+array<sdk::cmd::FBState, 15> mqStateToFBState =
+{
+    {
+        sdk::cmd::FBState_Ok,
+        sdk::cmd::FBState_Error,
+        sdk::cmd::FBState_Idle,
+        sdk::cmd::FBState_InitializingDevice,
+        sdk::cmd::FBState_Initialized,
+        sdk::cmd::FBState_Binding,
+        sdk::cmd::FBState_Bound,
+        sdk::cmd::FBState_Connecting,
+        sdk::cmd::FBState_DeviceReady,
+        sdk::cmd::FBState_InitializingTask,
+        sdk::cmd::FBState_Ready,
+        sdk::cmd::FBState_Running,
+        sdk::cmd::FBState_ResettingTask,
+        sdk::cmd::FBState_ResettingDevice,
+        sdk::cmd::FBState_Exiting
+    }
+};
+
+array<fair::mq::Transition, 12> fbTransitionToMQTransition =
+{
+    {
+        fair::mq::Transition::Auto,
+        fair::mq::Transition::InitDevice,
+        fair::mq::Transition::CompleteInit,
+        fair::mq::Transition::Bind,
+        fair::mq::Transition::Connect,
+        fair::mq::Transition::InitTask,
+        fair::mq::Transition::Run,
+        fair::mq::Transition::Stop,
+        fair::mq::Transition::ResetTask,
+        fair::mq::Transition::ResetDevice,
+        fair::mq::Transition::End,
+        fair::mq::Transition::ErrorFound
+    }
+};
+
+array<sdk::cmd::FBTransition, 12> mqTransitionToFBTransition =
+{
+    {
+        sdk::cmd::FBTransition_Auto,
+        sdk::cmd::FBTransition_InitDevice,
+        sdk::cmd::FBTransition_CompleteInit,
+        sdk::cmd::FBTransition_Bind,
+        sdk::cmd::FBTransition_Connect,
+        sdk::cmd::FBTransition_InitTask,
+        sdk::cmd::FBTransition_Run,
+        sdk::cmd::FBTransition_Stop,
+        sdk::cmd::FBTransition_ResetTask,
+        sdk::cmd::FBTransition_ResetDevice,
+        sdk::cmd::FBTransition_End,
+        sdk::cmd::FBTransition_ErrorFound
+    }
+};
+
+array<FBCmd, 17> typeToFBCmd =
+{
+    {
+        FBCmd::FBCmd_check_state,
+        FBCmd::FBCmd_change_state,
+        FBCmd::FBCmd_dump_config,
+        FBCmd::FBCmd_subscribe_to_heartbeats,
+        FBCmd::FBCmd_unsubscribe_from_heartbeats,
+        FBCmd::FBCmd_subscribe_to_state_change,
+        FBCmd::FBCmd_unsubscribe_from_state_change,
+        FBCmd::FBCmd_state_change_exiting_received,
+        FBCmd::FBCmd_current_state,
+        FBCmd::FBCmd_transition_status,
+        FBCmd::FBCmd_config,
+        FBCmd::FBCmd_heartbeat_subscription,
+        FBCmd::FBCmd_heartbeat_unsubscription,
+        FBCmd::FBCmd_heartbeat,
+        FBCmd::FBCmd_state_change_subscription,
+        FBCmd::FBCmd_state_change_unsubscription,
+        FBCmd::FBCmd_state_change
+    }
+};
+
+array<Type, 17> fbCmdToType =
+{
+    {
+        Type::check_state,
+        Type::change_state,
+        Type::dump_config,
+        Type::subscribe_to_heartbeats,
+        Type::unsubscribe_from_heartbeats,
+        Type::subscribe_to_state_change,
+        Type::unsubscribe_from_state_change,
+        Type::state_change_exiting_received,
+        Type::current_state,
+        Type::transition_status,
+        Type::config,
+        Type::heartbeat_subscription,
+        Type::heartbeat_unsubscription,
+        Type::heartbeat,
+        Type::state_change_subscription,
+        Type::state_change_unsubscription,
+        Type::state_change
+    }
+};
+
+fair::mq::State GetMQState(const FBState state) { return fbStateToMQState.at(state); }
+FBState GetFBState(const fair::mq::State state) { return mqStateToFBState.at(static_cast<int>(state)); }
+fair::mq::Transition GetMQTransition(const FBTransition transition) { return fbTransitionToMQTransition.at(transition); }
+FBTransition GetFBTransition(const fair::mq::Transition transition) { return mqTransitionToFBTransition.at(static_cast<int>(transition)); }
+
+Result GetResult(const FBResult result) { return fbResultToResult.at(result); }
+FBResult GetFBResult(const Result result) { return resultToFBResult.at(static_cast<int>(result)); }
+string GetResultName(const Result result) { return resultNames.at(static_cast<int>(result)); }
+string GetTypeName(const Type type) { return typeNames.at(static_cast<int>(type)); }
+
+FBCmd GetFBCmd(const Type type) { return typeToFBCmd.at(static_cast<int>(type)); }
+
+string Cmds::Serialize(const Format type) const
+{
+    flatbuffers::FlatBufferBuilder fbb;
+    vector<flatbuffers::Offset<FBCommand>> commandOffsets;
+
+    for (auto& cmd : fCmds) {
+        flatbuffers::Offset<FBCommand> cmdOffset;
+        unique_ptr<FBCommandBuilder> cmdBuilder; // delay the creation of the builder, because child strings need to be constructed first (which are conditional)
+
+        switch (cmd->GetType()) {
+            case Type::check_state: {
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+            }
+            break;
+            case Type::change_state: {
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_transition(GetFBTransition(static_cast<ChangeState&>(*cmd).GetTransition()));
+            }
+            break;
+            case Type::dump_config: {
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+            }
+            break;
+            case Type::subscribe_to_heartbeats: {
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+            }
+            break;
+            case Type::unsubscribe_from_heartbeats: {
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+            }
+            break;
+            case Type::subscribe_to_state_change: {
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+            }
+            break;
+            case Type::unsubscribe_from_state_change: {
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+            }
+            break;
+            case Type::state_change_exiting_received: {
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+            }
+            break;
+            case Type::current_state: {
+                auto deviceId = fbb.CreateString(static_cast<CurrentState&>(*cmd).GetDeviceId());
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_device_id(deviceId);
+                cmdBuilder->add_current_state(GetFBState(static_cast<CurrentState&>(*cmd).GetCurrentState()));
+            }
+            break;
+            case Type::transition_status: {
+                auto deviceId = fbb.CreateString(static_cast<TransitionStatus&>(*cmd).GetDeviceId());
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_device_id(deviceId);
+                cmdBuilder->add_result(GetFBResult(static_cast<TransitionStatus&>(*cmd).GetResult()));
+                cmdBuilder->add_transition(GetFBTransition(static_cast<TransitionStatus&>(*cmd).GetTransition()));
+            }
+            break;
+            case Type::config: {
+                auto deviceId = fbb.CreateString(static_cast<Config&>(*cmd).GetDeviceId());
+                auto config = fbb.CreateString(static_cast<Config&>(*cmd).GetConfig());
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_device_id(deviceId);
+                cmdBuilder->add_config_string(config);
+            }
+            break;
+            case Type::heartbeat_subscription: {
+                auto deviceId = fbb.CreateString(static_cast<HeartbeatSubscription&>(*cmd).GetDeviceId());
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_device_id(deviceId);
+                cmdBuilder->add_result(GetFBResult(static_cast<HeartbeatSubscription&>(*cmd).GetResult()));
+            }
+            break;
+            case Type::heartbeat_unsubscription: {
+                auto deviceId = fbb.CreateString(static_cast<HeartbeatUnsubscription&>(*cmd).GetDeviceId());
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_device_id(deviceId);
+                cmdBuilder->add_result(GetFBResult(static_cast<HeartbeatUnsubscription&>(*cmd).GetResult()));
+            }
+            break;
+            case Type::heartbeat: {
+                auto deviceId = fbb.CreateString(static_cast<Heartbeat&>(*cmd).GetDeviceId());
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_device_id(deviceId);
+            }
+            break;
+            case Type::state_change_subscription: {
+                auto deviceId = fbb.CreateString(static_cast<StateChangeSubscription&>(*cmd).GetDeviceId());
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_device_id(deviceId);
+                cmdBuilder->add_result(GetFBResult(static_cast<StateChangeSubscription&>(*cmd).GetResult()));
+            }
+            break;
+            case Type::state_change_unsubscription: {
+                auto deviceId = fbb.CreateString(static_cast<StateChangeUnsubscription&>(*cmd).GetDeviceId());
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_device_id(deviceId);
+                cmdBuilder->add_result(GetFBResult(static_cast<StateChangeUnsubscription&>(*cmd).GetResult()));
+            }
+            break;
+            case Type::state_change: {
+                auto deviceId = fbb.CreateString(static_cast<StateChange&>(*cmd).GetDeviceId());
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_device_id(deviceId);
+                cmdBuilder->add_task_id(static_cast<StateChange&>(*cmd).GetTaskId());
+                cmdBuilder->add_last_state(GetFBState(static_cast<StateChange&>(*cmd).GetLastState()));
+                cmdBuilder->add_current_state(GetFBState(static_cast<StateChange&>(*cmd).GetCurrentState()));
+            }
+            break;
+            default:
+                throw CommandFormatError("unrecognized command type given to fair::mq::cmd::Cmds::Serialize()");
+            break;
+        }
+
+        cmdBuilder->add_command_id(typeToFBCmd.at(static_cast<int>(cmd->GetType())));
+
+        cmdOffset = cmdBuilder->Finish();
+        commandOffsets.push_back(cmdOffset);
+    }
+
+    auto commands = fbb.CreateVector(commandOffsets);
+    auto cmds = CreateFBCommands(fbb, commands);
+    fbb.Finish(cmds);
+
+    if (type == Format::Binary) {
+        return string(reinterpret_cast<char*>(fbb.GetBufferPointer()), fbb.GetSize());
+    } else { // Type == Format::JSON
+        flatbuffers::Parser parser;
+        if (!parser.Parse(commandsFormatDefFbs)) {
+            throw CommandFormatError("Serialize couldn't parse commands format");
+        }
+        std::string json;
+        if (!flatbuffers::GenerateText(parser, fbb.GetBufferPointer(), &json)) {
+            throw CommandFormatError("Serialize couldn't serialize parsed data to JSON!");
+        }
+        return json;
+    }
+}
+
+void Cmds::Deserialize(const string& str, const Format type)
+{
+    fCmds.clear();
+
+    const flatbuffers::Vector<flatbuffers::Offset<FBCommand>>* cmds;
+
+    if (type == Format::Binary) {
+        cmds = cmd::GetFBCommands(const_cast<char*>(str.c_str()))->commands();
+    } else { // Type == Format::JSON
+        flatbuffers::Parser parser;
+        if (!parser.Parse(commandsFormatDefFbs)) {
+            throw CommandFormatError("Deserialize couldn't parse commands format");
+        }
+        if (!parser.Parse(str.c_str())) {
+            throw CommandFormatError("Deserialize couldn't parse incoming JSON string");
+        }
+        cmds = cmd::GetFBCommands(parser.builder_.GetBufferPointer())->commands();
+    }
+
+    for (unsigned int i = 0; i < cmds->size(); ++i) {
+        const fair::mq::sdk::cmd::FBCommand& cmdPtr = *(cmds->Get(i));
+        switch (cmdPtr.command_id()) {
+            case FBCmd_check_state:
+                fCmds.emplace_back(make<CheckState>());
+            break;
+            case FBCmd_change_state:
+                fCmds.emplace_back(make<ChangeState>(GetMQTransition(cmdPtr.transition())));
+            break;
+            case FBCmd_dump_config:
+                fCmds.emplace_back(make<DumpConfig>());
+            break;
+            case FBCmd_subscribe_to_heartbeats:
+                fCmds.emplace_back(make<SubscribeToHeartbeats>());
+            break;
+            case FBCmd_unsubscribe_from_heartbeats:
+                fCmds.emplace_back(make<UnsubscribeFromHeartbeats>());
+            break;
+            case FBCmd_subscribe_to_state_change:
+                fCmds.emplace_back(make<SubscribeToStateChange>());
+            break;
+            case FBCmd_unsubscribe_from_state_change:
+                fCmds.emplace_back(make<UnsubscribeFromStateChange>());
+            break;
+            case FBCmd_state_change_exiting_received:
+                fCmds.emplace_back(make<StateChangeExitingReceived>());
+            break;
+            case FBCmd_current_state:
+                fCmds.emplace_back(make<CurrentState>(cmdPtr.device_id()->str(), GetMQState(cmdPtr.current_state())));
+            break;
+            case FBCmd_transition_status:
+                fCmds.emplace_back(make<TransitionStatus>(cmdPtr.device_id()->str(), GetResult(cmdPtr.result()), GetMQTransition(cmdPtr.transition())));
+            break;
+            case FBCmd_config:
+                fCmds.emplace_back(make<Config>(cmdPtr.device_id()->str(), cmdPtr.config_string()->str()));
+            break;
+            case FBCmd_heartbeat_subscription:
+                fCmds.emplace_back(make<HeartbeatSubscription>(cmdPtr.device_id()->str(), GetResult(cmdPtr.result())));
+            break;
+            case FBCmd_heartbeat_unsubscription:
+                fCmds.emplace_back(make<HeartbeatUnsubscription>(cmdPtr.device_id()->str(), GetResult(cmdPtr.result())));
+            break;
+            case FBCmd_heartbeat:
+                fCmds.emplace_back(make<Heartbeat>(cmdPtr.device_id()->str()));
+            break;
+            case FBCmd_state_change_subscription:
+                fCmds.emplace_back(make<StateChangeSubscription>(cmdPtr.device_id()->str(), GetResult(cmdPtr.result())));
+            break;
+            case FBCmd_state_change_unsubscription:
+                fCmds.emplace_back(make<StateChangeUnsubscription>(cmdPtr.device_id()->str(), GetResult(cmdPtr.result())));
+            break;
+            case FBCmd_state_change:
+                fCmds.emplace_back(make<StateChange>(cmdPtr.device_id()->str(), cmdPtr.task_id(), GetMQState(cmdPtr.last_state()), GetMQState(cmdPtr.current_state())));
+            break;
+            default:
+                throw CommandFormatError("unrecognized command type given to fair::mq::cmd::Cmds::Deserialize()");
+            break;
+        }
+    }
+}
+
+}   // namespace cmd
+}   // namespace sdk
+}   // namespace mq
+}   // namespace fair

fairmq/sdk/commands/Commands.h

@@ -0,0 +1,361 @@
+/********************************************************************************
+ *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_SDK_COMMANDFACTORY
+#define FAIR_MQ_SDK_COMMANDFACTORY
+
+#include <fairmq/States.h>
+#include <fairmq/tools/CppSTL.h>
+
+#include <vector>
+#include <string>
+#include <memory>
+#include <type_traits>
+#include <stdexcept>
+
+namespace fair
+{
+namespace mq
+{
+namespace sdk
+{
+namespace cmd
+{
+
+enum class Format : int {
+    Binary,
+    JSON
+};
+
+enum class Result : int {
+    Ok,
+    Failure
+};
+
+enum class Type : int
+{
+    check_state,                   // args: { }
+    change_state,                  // args: { transition }
+    dump_config,                   // args: { }
+    subscribe_to_heartbeats,       // args: { }
+    unsubscribe_from_heartbeats,   // args: { }
+    subscribe_to_state_change,     // args: { }
+    unsubscribe_from_state_change, // args: { }
+    state_change_exiting_received, // args: { }
+
+    current_state,                 // args: { device_id, current_state }
+    transition_status,             // args: { device_id, Result, transition }
+    config,                        // args: { device_id, config_string }
+    heartbeat_subscription,        // args: { device_id, Result }
+    heartbeat_unsubscription,      // args: { device_id, Result }
+    heartbeat,                     // args: { device_id }
+    state_change_subscription,     // args: { device_id, Result }
+    state_change_unsubscription,   // args: { device_id, Result }
+    state_change                   // args: { device_id, task_id, last_state, current_state }
+};
+
+struct Cmd
+{
+    explicit Cmd(const Type type) : fType(type) {}
+
+    Type GetType() const { return fType; }
+
+  private:
+    Type fType;
+};
+
+struct CheckState : Cmd
+{
+    explicit CheckState() : Cmd(Type::check_state) {}
+};
+
+struct ChangeState : Cmd
+{
+    explicit ChangeState(Transition transition)
+        : Cmd(Type::change_state)
+        , fTransition(transition)
+    {}
+
+    Transition GetTransition() const { return fTransition; }
+    void SetTransition(Transition transition) { fTransition = transition; }
+
+  private:
+    Transition fTransition;
+};
+
+struct DumpConfig : Cmd
+{
+    explicit DumpConfig() : Cmd(Type::dump_config) {}
+};
+
+struct SubscribeToHeartbeats : Cmd
+{
+    explicit SubscribeToHeartbeats() : Cmd(Type::subscribe_to_heartbeats) {}
+};
+
+struct UnsubscribeFromHeartbeats : Cmd
+{
+    explicit UnsubscribeFromHeartbeats() : Cmd(Type::unsubscribe_from_heartbeats) {}
+};
+
+struct SubscribeToStateChange : Cmd
+{
+    explicit SubscribeToStateChange() : Cmd(Type::subscribe_to_state_change) {}
+};
+
+struct UnsubscribeFromStateChange : Cmd
+{
+    explicit UnsubscribeFromStateChange() : Cmd(Type::unsubscribe_from_state_change) {}
+};
+
+struct StateChangeExitingReceived : Cmd
+{
+    explicit StateChangeExitingReceived() : Cmd(Type::state_change_exiting_received) {}
+};
+
+struct CurrentState : Cmd
+{
+    explicit CurrentState(const std::string& id, State currentState)
+        : Cmd(Type::current_state)
+        , fDeviceId(id)
+        , fCurrentState(currentState)
+    {}
+
+    std::string GetDeviceId() const { return fDeviceId; }
+    void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    fair::mq::State GetCurrentState() const { return fCurrentState; }
+    void SetCurrentState(fair::mq::State state) { fCurrentState = state; }
+
+  private:
+    std::string fDeviceId;
+    fair::mq::State fCurrentState;
+};
+
+struct TransitionStatus : Cmd
+{
+    explicit TransitionStatus(const std::string& id, const Result result, const Transition transition)
+        : Cmd(Type::transition_status)
+        , fDeviceId(id)
+        , fResult(result)
+        , fTransition(transition)
+    {}
+
+    std::string GetDeviceId() const { return fDeviceId; }
+    void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    Result GetResult() const { return fResult; }
+    void SetResult(const Result result) { fResult = result; }
+    Transition GetTransition() const { return fTransition; }
+    void SetTransition(const Transition transition) { fTransition = transition; }
+
+  private:
+    std::string fDeviceId;
+    Result fResult;
+    Transition fTransition;
+};
+
+struct Config : Cmd
+{
+    explicit Config(const std::string& id, const std::string& config)
+        : Cmd(Type::config)
+        , fDeviceId(id)
+        , fConfig(config)
+    {}
+
+    std::string GetDeviceId() const { return fDeviceId; }
+    void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    std::string GetConfig() const { return fConfig; }
+    void SetConfig(const std::string& config) { fConfig = config; }
+
+  private:
+    std::string fDeviceId;
+    std::string fConfig;
+};
+
+struct HeartbeatSubscription : Cmd
+{
+    explicit HeartbeatSubscription(const std::string& id, const Result result)
+        : Cmd(Type::heartbeat_subscription)
+        , fDeviceId(id)
+        , fResult(result)
+    {}
+
+    std::string GetDeviceId() const { return fDeviceId; }
+    void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    Result GetResult() const { return fResult; }
+    void SetResult(const Result result) { fResult = result; }
+
+  private:
+    std::string fDeviceId;
+    Result fResult;
+};
+
+struct HeartbeatUnsubscription : Cmd
+{
+    explicit HeartbeatUnsubscription(const std::string& id, const Result result)
+        : Cmd(Type::heartbeat_unsubscription)
+        , fDeviceId(id)
+        , fResult(result)
+    {}
+
+    std::string GetDeviceId() const { return fDeviceId; }
+    void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    Result GetResult() const { return fResult; }
+    void SetResult(const Result result) { fResult = result; }
+
+  private:
+    std::string fDeviceId;
+    Result fResult;
+};
+
+struct Heartbeat : Cmd
+{
+    explicit Heartbeat(const std::string& id)
+        : Cmd(Type::heartbeat)
+        , fDeviceId(id)
+    {}
+
+    std::string GetDeviceId() const { return fDeviceId; }
+    void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+
+  private:
+    std::string fDeviceId;
+};
+
+struct StateChangeSubscription : Cmd
+{
+    explicit StateChangeSubscription(const std::string& id, const Result result)
+        : Cmd(Type::state_change_subscription)
+        , fDeviceId(id)
+        , fResult(result)
+    {}
+
+    std::string GetDeviceId() const { return fDeviceId; }
+    void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    Result GetResult() const { return fResult; }
+    void SetResult(const Result result) { fResult = result; }
+
+  private:
+    std::string fDeviceId;
+    Result fResult;
+};
+
+struct StateChangeUnsubscription : Cmd
+{
+    explicit StateChangeUnsubscription(const std::string& id, const Result result)
+        : Cmd(Type::state_change_unsubscription)
+        , fDeviceId(id)
+        , fResult(result)
+    {}
+
+    std::string GetDeviceId() const { return fDeviceId; }
+    void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    Result GetResult() const { return fResult; }
+    void SetResult(const Result result) { fResult = result; }
+
+  private:
+    std::string fDeviceId;
+    Result fResult;
+};
+
+struct StateChange : Cmd
+{
+    explicit StateChange(const std::string& deviceId, const uint64_t taskId, const State lastState, const State currentState)
+        : Cmd(Type::state_change)
+        , fDeviceId(deviceId)
+        , fTaskId(taskId)
+        , fLastState(lastState)
+        , fCurrentState(currentState)
+    {}
+
+    std::string GetDeviceId() const { return fDeviceId; }
+    void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    uint64_t GetTaskId() const { return fTaskId; }
+    void SetTaskId(const uint64_t taskId) { fTaskId = taskId; }
+    fair::mq::State GetLastState() const { return fLastState; }
+    void SetLastState(const fair::mq::State state) { fLastState = state; }
+    fair::mq::State GetCurrentState() const { return fCurrentState; }
+    void SetCurrentState(const fair::mq::State state) { fCurrentState = state; }
+
+  private:
+    std::string fDeviceId;
+    uint64_t fTaskId;
+    fair::mq::State fLastState;
+    fair::mq::State fCurrentState;
+};
+
+template<typename C, typename... Args>
+std::unique_ptr<Cmd> make(Args&&... args)
+{
+    return fair::mq::tools::make_unique<C>(std::forward<Args>(args)...);
+}
+
+struct Cmds
+{
+    using container = std::vector<std::unique_ptr<Cmd>>;
+    struct CommandFormatError : std::runtime_error { using std::runtime_error::runtime_error; };
+
+    explicit Cmds() {}
+
+    template<typename... Rest>
+    explicit Cmds(std::unique_ptr<Cmd>&& first, Rest&&... rest)
+    {
+        Unpack(std::forward<std::unique_ptr<Cmd>&&>(first), std::forward<Rest>(rest)...);
+    }
+
+
+    void Add(std::unique_ptr<Cmd>&& cmd) { fCmds.emplace_back(std::move(cmd)); }
+
+    template<typename C, typename... Args>
+    void Add(Args&&... args)
+    {
+        static_assert(std::is_base_of<Cmd, C>::value, "Only types derived from fair::mq::cmd::Cmd are allowed");
+        Add(make<C>(std::forward<Args>(args)...));
+    }
+
+    Cmd& At(size_t i) { return *(fCmds.at(i)); }
+
+    size_t Size() const { return fCmds.size(); }
+    void Reset() { fCmds.clear(); }
+
+    std::string Serialize(const Format type = Format::Binary) const;
+    void Deserialize(const std::string&, const Format type = Format::Binary);
+
+  private:
+    container fCmds;
+
+    void Unpack() {}
+
+    template <class... Rest>
+    void Unpack(std::unique_ptr<Cmd>&& first, Rest&&... rest)
+    {
+        fCmds.emplace_back(std::move(first));
+        Unpack(std::forward<Rest>(rest)...);
+    }
+
+  public:
+    using iterator = container::iterator;
+    using const_iterator = container::const_iterator;
+
+    auto begin() -> decltype(fCmds.begin()) { return fCmds.begin(); }
+    auto end() -> decltype(fCmds.end()) { return fCmds.end(); }
+    auto cbegin() -> decltype(fCmds.cbegin()) { return fCmds.cbegin(); }
+    auto cend() -> decltype(fCmds.cend()) { return fCmds.cend(); }
+};
+
+std::string GetResultName(const Result result);
+std::string GetTypeName(const Type type);
+
+inline std::ostream& operator<<(std::ostream& os, const Result& result) { return os << GetResultName(result); }
+inline std::ostream& operator<<(std::ostream& os, const Type& type) { return os << GetTypeName(type); }
+
+} /* namespace cmd */
+} /* namespace sdk */
+} /* namespace mq */
+} /* namespace fair */
+
+#endif /* FAIR_MQ_SDK_COMMANDFACTORY */

fairmq/sdk/commands/CommandsFormat.fbs

@@ -0,0 +1,79 @@
+namespace fair.mq.sdk.cmd;
+
+enum FBResult:byte {
+    Ok,
+    Failure
+}
+
+enum FBState:byte {
+    Ok,
+    Error,
+    Idle,
+    InitializingDevice,
+    Initialized,
+    Binding,
+    Bound,
+    Connecting,
+    DeviceReady,
+    InitializingTask,
+    Ready,
+    Running,
+    ResettingTask,
+    ResettingDevice,
+    Exiting
+}
+
+enum FBTransition:byte {
+    Auto,
+    InitDevice,
+    CompleteInit,
+    Bind,
+    Connect,
+    InitTask,
+    Run,
+    Stop,
+    ResetTask,
+    ResetDevice,
+    End,
+    ErrorFound
+}
+
+enum FBCmd:byte {
+    check_state,                   // args: { }
+    change_state,                  // args: { transition }
+    dump_config,                   // args: { }
+    subscribe_to_heartbeats,       // args: { }
+    unsubscribe_from_heartbeats,   // args: { }
+    subscribe_to_state_change,     // args: { }
+    unsubscribe_from_state_change, // args: { }
+    state_change_exiting_received, // args: { }
+
+    current_state,                 // args: { device_id, current_state }
+    transition_status,             // args: { device_id, Result, transition }
+    config,                        // args: { device_id, config_string }
+    heartbeat_subscription,        // args: { device_id, Result }
+    heartbeat_unsubscription,      // args: { device_id, Result }
+    heartbeat,                     // args: { device_id }
+    state_change_subscription,     // args: { device_id, Result }
+    state_change_unsubscription,   // args: { device_id, Result }
+    state_change                   // args: { device_id, task_id, last_state, current_state }
+}
+
+table FBCommand {
+    command_id:FBCmd;
+    device_id:string;
+    task_id:uint64;
+    state:FBState;
+    transition:FBTransition;
+    result:FBResult;
+    config_string:string;
+    last_state:FBState;
+    current_state:FBState;
+    debug:string;
+}
+
+table FBCommands {
+    commands:[FBCommand];
+}
+
+root_type FBCommands;

fairmq/sdk/commands/CommandsFormatDef.h.in

@@ -0,0 +1,21 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include <string>
+
+namespace fair {
+namespace mq {
+namespace sdk {
+namespace cmd {
+
+constexpr auto commandsFormatDefFbs = R"(@commands_format_def_fbs@)";
+
+}   // namespace cmd
+}   // namespace sdk
+}   // namespace mq
+}   // namespace fair

fairmq/shmem/Common.h

@@ -73,15 +73,6 @@ struct RegionCounter
     std::atomic<uint64_t> fCount;
 };
 
-struct MonitorStatus
-{
-    MonitorStatus()
-        : fActive(true)
-    {}
-
-    bool fActive;
-};
-
 struct MetaHeader
 {
     size_t fSize;

fairmq/shmem/FairMQMessageSHM.cxx

@@ -5,8 +5,8 @@
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
-#include <fairmq/shmem/Common.h>
-#include <fairmq/shmem/Region.h>
+#include "Common.h"
+#include "Region.h"
 
 #include "FairMQMessageSHM.h"
 #include "FairMQUnmanagedRegionSHM.h"
@@ -38,8 +38,7 @@ FairMQMessageSHM::FairMQMessageSHM(Manager& manager, FairMQTransportFactory* fac
     , fHint(0)
     , fLocalPtr(nullptr)
 {
-    if (zmq_msg_init(&fMessage) != 0)
-    {
+    if (zmq_msg_init(&fMessage) != 0) {
         LOG(error) << "failed initializing message, reason: " << zmq_strerror(errno);
     }
     fMetaCreated = true;
@@ -74,15 +73,11 @@ FairMQMessageSHM::FairMQMessageSHM(Manager& manager, void* data, const size_t si
     , fHint(0)
     , fLocalPtr(nullptr)
 {
-    if (InitializeChunk(size))
-    {
+    if (InitializeChunk(size)) {
         memcpy(fLocalPtr, data, size);
-        if (ffn)
-        {
+        if (ffn) {
             ffn(data, hint);
-        }
-        else
-        {
+        } else {
             free(data);
         }
     }
@@ -102,16 +97,12 @@ FairMQMessageSHM::FairMQMessageSHM(Manager& manager, FairMQUnmanagedRegionPtr& r
     , fLocalPtr(static_cast<char*>(data))
 {
     if (reinterpret_cast<const char*>(data) >= reinterpret_cast<const char*>(region->GetData()) ||
-        reinterpret_cast<const char*>(data) <= reinterpret_cast<const char*>(region->GetData()) + region->GetSize())
-    {
+        reinterpret_cast<const char*>(data) <= reinterpret_cast<const char*>(region->GetData()) + region->GetSize()) {
         fHandle = (bipc::managed_shared_memory::handle_t)(reinterpret_cast<const char*>(data) - reinterpret_cast<const char*>(region->GetData()));
 
-        if (zmq_msg_init_size(&fMessage, sizeof(MetaHeader)) != 0)
-        {
+        if (zmq_msg_init_size(&fMessage, sizeof(MetaHeader)) != 0) {
             LOG(error) << "failed initializing meta message, reason: " << zmq_strerror(errno);
-        }
-        else
-        {
+        } else {
             MetaHeader header;
             header.fSize = size;
             header.fHandle = fHandle;
@@ -121,9 +112,7 @@ FairMQMessageSHM::FairMQMessageSHM(Manager& manager, FairMQUnmanagedRegionPtr& r
 
             fMetaCreated = true;
         }
-    }
-    else
-    {
+    } else {
         LOG(error) << "trying to create region message with data from outside the region";
         throw runtime_error("trying to create region message with data from outside the region");
     }
@@ -131,24 +120,17 @@ FairMQMessageSHM::FairMQMessageSHM(Manager& manager, FairMQUnmanagedRegionPtr& r
 
 bool FairMQMessageSHM::InitializeChunk(const size_t size)
 {
-    while (fHandle < 0)
-    {
-        try
-        {
+    while (fHandle < 0) {
+        try {
             bipc::managed_shared_memory::size_type actualSize = size;
             char* hint = 0; // unused for bipc::allocate_new
             fLocalPtr = fManager.Segment().allocation_command<char>(bipc::allocate_new, size, actualSize, hint);
-        }
-        catch (bipc::bad_alloc& ba)
-        {
+        } catch (bipc::bad_alloc& ba) {
             // LOG(warn) << "Shared memory full...";
             this_thread::sleep_for(chrono::milliseconds(50));
-            if (fInterrupted)
-            {
+            if (fInterrupted) {
                 return false;
-            }
-            else
-            {
+            } else {
                 continue;
             }
         }
@@ -157,8 +139,7 @@ bool FairMQMessageSHM::InitializeChunk(const size_t size)
 
     fSize = size;
 
-    if (zmq_msg_init_size(&fMessage, sizeof(MetaHeader)) != 0)
-    {
+    if (zmq_msg_init_size(&fMessage, sizeof(MetaHeader)) != 0) {
         LOG(error) << "failed initializing meta message, reason: " << zmq_strerror(errno);
         return false;
     }
@@ -180,8 +161,7 @@ void FairMQMessageSHM::Rebuild()
 
     fQueued = false;
 
-    if (zmq_msg_init(&fMessage) != 0)
-    {
+    if (zmq_msg_init(&fMessage) != 0) {
         LOG(error) << "failed initializing message, reason: " << zmq_strerror(errno);
     }
     fMetaCreated = true;
@@ -190,9 +170,7 @@ void FairMQMessageSHM::Rebuild()
 void FairMQMessageSHM::Rebuild(const size_t size)
 {
     CloseMessage();
-
     fQueued = false;
-
     InitializeChunk(size);
 }
 
@@ -202,32 +180,25 @@ void FairMQMessageSHM::Rebuild(void* data, const size_t size, fairmq_free_fn* ff
 
     fQueued = false;
 
-    if (InitializeChunk(size))
-    {
+    if (InitializeChunk(size)) {
         memcpy(fLocalPtr, data, size);
-        if (ffn)
-        {
+        if (ffn) {
             ffn(data, hint);
-        }
-        else
-        {
+        } else {
             free(data);
         }
     }
 }
 
-zmq_msg_t* FairMQMessageSHM::GetMessage()
-{
-    return &fMessage;
-}
-
 void* FairMQMessageSHM::GetData() const
 {
-    if (fLocalPtr) {
-        return fLocalPtr;
-    } else {
+    if (!fLocalPtr) {
         if (fRegionId == 0) {
-            return fManager.Segment().get_address_from_handle(fHandle);
+            if (fSize > 0) {
+                fLocalPtr = reinterpret_cast<char*>(fManager.Segment().get_address_from_handle(fHandle));
+            } else {
+                fLocalPtr = nullptr;
+            }
         } else {
             fRegionPtr = fManager.GetRemoteRegion(fRegionId);
             if (fRegionPtr) {
@@ -236,14 +207,10 @@ void* FairMQMessageSHM::GetData() const
                 // LOG(warn) << "could not get pointer from a region message";
                 fLocalPtr = nullptr;
             }
-            return fLocalPtr;
         }
     }
-}
 
-size_t FairMQMessageSHM::GetSize() const
-{
-    return fSize;
+    return fLocalPtr;
 }
 
 bool FairMQMessageSHM::SetUsedSize(const size_t size)
@@ -270,72 +237,45 @@ bool FairMQMessageSHM::SetUsedSize(const size_t size)
     }
 }
 
-fair::mq::Transport FairMQMessageSHM::GetType() const
-{
-    return fTransportType;
-}
-
 void FairMQMessageSHM::Copy(const FairMQMessage& msg)
 {
-    if (fHandle < 0)
-    {
+    if (fHandle < 0) {
         bipc::managed_shared_memory::handle_t otherHandle = static_cast<const FairMQMessageSHM&>(msg).fHandle;
-        if (otherHandle)
-        {
-            if (InitializeChunk(msg.GetSize()))
-            {
+        if (otherHandle) {
+            if (InitializeChunk(msg.GetSize())) {
                 memcpy(GetData(), msg.GetData(), msg.GetSize());
             }
-        }
-        else
-        {
+        } else {
             LOG(error) << "copy fail: source message not initialized!";
         }
-    }
-    else
-    {
+    } else {
         LOG(error) << "copy fail: target message already initialized!";
     }
 }
 
 void FairMQMessageSHM::CloseMessage()
 {
-    if (fHandle >= 0 && !fQueued)
-    {
-        if (fRegionId == 0)
-        {
+    if (fHandle >= 0 && !fQueued) {
+        if (fRegionId == 0) {
             fManager.Segment().deallocate(fManager.Segment().get_address_from_handle(fHandle));
             fHandle = -1;
-        }
-        else
-        {
-            if (!fRegionPtr)
-            {
+        } else {
+            if (!fRegionPtr) {
                 fRegionPtr = fManager.GetRemoteRegion(fRegionId);
             }
 
-            if (fRegionPtr)
-            {
+            if (fRegionPtr) {
                 fRegionPtr->ReleaseBlock({fHandle, fSize, fHint});
-            }
-            else
-            {
+            } else {
                 LOG(warn) << "region ack queue for id " << fRegionId << " no longer exist. Not sending ack";
             }
         }
     }
 
-    if (fMetaCreated)
-    {
-        if (zmq_msg_close(&fMessage) != 0)
-        {
+    if (fMetaCreated) {
+        if (zmq_msg_close(&fMessage) != 0) {
             LOG(error) << "failed closing message, reason: " << zmq_strerror(errno);
         }
         fMetaCreated = false;
     }
 }
-
-FairMQMessageSHM::~FairMQMessageSHM()
-{
-    CloseMessage();
-}

fairmq/shmem/FairMQMessageSHM.h

@@ -40,15 +40,15 @@ class FairMQMessageSHM final : public FairMQMessage
     void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
 
     void* GetData() const override;
-    size_t GetSize() const override;
+    size_t GetSize() const override { return fSize; }
 
     bool SetUsedSize(const size_t size) override;
 
-    fair::mq::Transport GetType() const override;
+    fair::mq::Transport GetType() const override { return fTransportType; }
 
     void Copy(const FairMQMessage& msg) override;
 
-    ~FairMQMessageSHM() override;
+    ~FairMQMessageSHM() override { CloseMessage(); }
 
   private:
     fair::mq::shmem::Manager& fManager;
@@ -65,7 +65,7 @@ class FairMQMessageSHM final : public FairMQMessage
     mutable char* fLocalPtr;
 
     bool InitializeChunk(const size_t size);
-    zmq_msg_t* GetMessage();
+    zmq_msg_t* GetMessage() { return &fMessage; }
     void CloseMessage();
 };
 

fairmq/shmem/FairMQSocketSHM.cxx

@@ -5,7 +5,7 @@
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
-#include <fairmq/shmem/Common.h>
+#include "Common.h"
 
 #include "FairMQSocketSHM.h"
 #include "FairMQMessageSHM.h"
@@ -38,32 +38,27 @@ FairMQSocketSHM::FairMQSocketSHM(Manager& manager, const string& type, const str
     assert(context);
     fSocket = zmq_socket(context, GetConstant(type));
 
-    if (fSocket == nullptr)
-    {
+    if (fSocket == nullptr) {
         LOG(error) << "Failed creating socket " << fId << ", reason: " << zmq_strerror(errno);
         exit(EXIT_FAILURE);
     }
 
-    if (zmq_setsockopt(fSocket, ZMQ_IDENTITY, fId.c_str(), fId.length()) != 0)
-    {
+    if (zmq_setsockopt(fSocket, ZMQ_IDENTITY, fId.c_str(), fId.length()) != 0) {
         LOG(error) << "Failed setting ZMQ_IDENTITY socket option, reason: " << zmq_strerror(errno);
     }
 
     // Tell socket to try and send/receive outstanding messages for <linger> milliseconds before terminating.
     // Default value for ZeroMQ is -1, which is to wait forever.
     int linger = 1000;
-    if (zmq_setsockopt(fSocket, ZMQ_LINGER, &linger, sizeof(linger)) != 0)
-    {
+    if (zmq_setsockopt(fSocket, ZMQ_LINGER, &linger, sizeof(linger)) != 0) {
         LOG(error) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
     }
 
-    if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0)
-    {
+    if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0) {
         LOG(error) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
     }
 
-    if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0)
-    {
+    if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0) {
         LOG(error) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
     }
 
@@ -75,8 +70,7 @@ FairMQSocketSHM::FairMQSocketSHM(Manager& manager, const string& type, const str
     //     }
     // }
 
-    if (type == "sub" || type == "pub")
-    {
+    if (type == "sub" || type == "pub") {
         LOG(error) << "PUB/SUB socket type is not supported for shared memory transport";
         throw fair::mq::SocketError("PUB/SUB socket type is not supported for shared memory transport");
     }
@@ -86,10 +80,9 @@ FairMQSocketSHM::FairMQSocketSHM(Manager& manager, const string& type, const str
 
 bool FairMQSocketSHM::Bind(const string& address)
 {
-    // LOG(info) << "bind socket " << fId << " on " << address;
+    // LOG(info) << "binding socket " << fId << " on " << address;
 
-    if (zmq_bind(fSocket, address.c_str()) != 0)
-    {
+    if (zmq_bind(fSocket, address.c_str()) != 0) {
         if (errno == EADDRINUSE) {
             // do not print error in this case, this is handled by FairMQDevice in case no connection could be established after trying a number of random ports from a range.
             return false;
@@ -102,10 +95,9 @@ bool FairMQSocketSHM::Bind(const string& address)
 
 bool FairMQSocketSHM::Connect(const string& address)
 {
-    // LOG(info) << "connect socket " << fId << " on " << address;
+    // LOG(info) << "connecting socket " << fId << " on " << address;
 
-    if (zmq_connect(fSocket, address.c_str()) != 0)
-    {
+    if (zmq_connect(fSocket, address.c_str()) != 0) {
         LOG(error) << "Failed connecting socket " << fId << ", reason: " << zmq_strerror(errno);
         return false;
     }
@@ -116,55 +108,38 @@ bool FairMQSocketSHM::Connect(const string& address)
 int FairMQSocketSHM::Send(FairMQMessagePtr& msg, const int timeout)
 {
     int flags = 0;
-    if (timeout == 0)
-    {
+    if (timeout == 0) {
         flags = ZMQ_DONTWAIT;
     }
     int elapsed = 0;
 
-    while (true && !fInterrupted)
-    {
+    while (true && !fInterrupted) {
         int nbytes = zmq_msg_send(static_cast<FairMQMessageSHM*>(msg.get())->GetMessage(), fSocket, flags);
-        if (nbytes == 0)
-        {
+        if (nbytes == 0) {
+            ++fMessagesTx;
             return nbytes;
-        }
-        else if (nbytes > 0)
-        {
+        } else if (nbytes > 0) {
             static_cast<FairMQMessageSHM*>(msg.get())->fQueued = true;
-
             size_t size = msg->GetSize();
             fBytesTx += size;
             ++fMessagesTx;
-
             return size;
-        }
-        else if (zmq_errno() == EAGAIN)
-        {
-            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
-            {
-                if (timeout > 0)
-                {
+        } else if (zmq_errno() == EAGAIN) {
+            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0)) {
+                if (timeout > 0) {
                     elapsed += fSndTimeout;
-                    if (elapsed >= timeout)
-                    {
+                    if (elapsed >= timeout) {
                         return -2;
                     }
                 }
                 continue;
-            }
-            else
-            {
+            } else {
                 return -2;
             }
-        }
-        else if (zmq_errno() == ETERM)
-        {
+        } else if (zmq_errno() == ETERM) {
             LOG(info) << "terminating socket " << fId;
             return -1;
-        }
-        else
-        {
+        } else {
             LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
             return nbytes;
         }
@@ -176,72 +151,53 @@ int FairMQSocketSHM::Send(FairMQMessagePtr& msg, const int timeout)
 int FairMQSocketSHM::Receive(FairMQMessagePtr& msg, const int timeout)
 {
     int flags = 0;
-    if (timeout == 0)
-    {
+    if (timeout == 0) {
         flags = ZMQ_DONTWAIT;
     }
     int elapsed = 0;
 
-    zmq_msg_t* msgPtr = static_cast<FairMQMessageSHM*>(msg.get())->GetMessage();
-    while (true)
-    {
-        int nbytes = zmq_msg_recv(msgPtr, fSocket, flags);
-        if (nbytes == 0)
-        {
+    while (true) {
+        FairMQMessageSHM* shmMsg = static_cast<FairMQMessageSHM*>(msg.get());
+        zmq_msg_t* zmqMsg = shmMsg->GetMessage();
+        int nbytes = zmq_msg_recv(zmqMsg, fSocket, flags);
+        if (nbytes == 0) {
             ++fMessagesRx;
-
             return nbytes;
-        }
-        else if (nbytes > 0)
-        {
-            // check for number of receiving messages. must be 1
+        } else if (nbytes > 0) {
+            // check for number of received messages. must be 1
             const auto numMsgs = nbytes / sizeof(MetaHeader);
-            if (numMsgs > 1)
-            {
+            if (numMsgs > 1) {
                 LOG(error) << "Receiving SHM multipart with a single message receive call";
             }
 
-            assert (numMsgs == 1);
+            assert(numMsgs == 1);
 
-            MetaHeader* hdr = static_cast<MetaHeader*>(zmq_msg_data(msgPtr));
-            size_t size = 0;
-            static_cast<FairMQMessageSHM*>(msg.get())->fHandle = hdr->fHandle;
-            static_cast<FairMQMessageSHM*>(msg.get())->fSize = hdr->fSize;
-            static_cast<FairMQMessageSHM*>(msg.get())->fRegionId = hdr->fRegionId;
-            static_cast<FairMQMessageSHM*>(msg.get())->fHint = hdr->fHint;
-            size = msg->GetSize();
+            MetaHeader* hdr = static_cast<MetaHeader*>(zmq_msg_data(zmqMsg));
+            size_t size = hdr->fSize;
+            shmMsg->fHandle = hdr->fHandle;
+            shmMsg->fSize = size;
+            shmMsg->fRegionId = hdr->fRegionId;
+            shmMsg->fHint = hdr->fHint;
 
             fBytesRx += size;
             ++fMessagesRx;
-
             return size;
-        }
-        else if (zmq_errno() == EAGAIN)
-        {
-            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
-            {
-                if (timeout > 0)
-                {
+        } else if (zmq_errno() == EAGAIN) {
+            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0)) {
+                if (timeout > 0) {
                     elapsed += fRcvTimeout;
-                    if (elapsed >= timeout)
-                    {
+                    if (elapsed >= timeout) {
                         return -2;
                     }
                 }
                 continue;
-            }
-            else
-            {
+            } else {
                 return -2;
             }
-        }
-        else if (zmq_errno() == ETERM)
-        {
+        } else if (zmq_errno() == ETERM) {
             LOG(info) << "terminating socket " << fId;
             return -1;
-        }
-        else
-        {
+        } else {
             LOG(error) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
             return nbytes;
         }
@@ -251,8 +207,7 @@ int FairMQSocketSHM::Receive(FairMQMessagePtr& msg, const int timeout)
 int64_t FairMQSocketSHM::Send(vector<FairMQMessagePtr>& msgVec, const int timeout)
 {
     int flags = 0;
-    if (timeout == 0)
-    {
+    if (timeout == 0) {
         flags = ZMQ_DONTWAIT;
     }
     const unsigned int vecSize = msgVec.size();
@@ -269,27 +224,32 @@ int64_t FairMQSocketSHM::Send(vector<FairMQMessagePtr>& msgVec, const int timeou
     // prepare the message with shm metas
     MetaHeader* metas = static_cast<MetaHeader*>(zmq_msg_data(&zmqMsg));
 
-    for (auto& msg : msgVec)
-    {
+    for (auto& msg : msgVec) {
         zmq_msg_t* metaMsg = static_cast<FairMQMessageSHM*>(msg.get())->GetMessage();
-        memcpy(metas++, zmq_msg_data(metaMsg), sizeof(MetaHeader));
+        if (zmq_msg_size(metaMsg) > 0) {
+            memcpy(metas++, zmq_msg_data(metaMsg), sizeof(MetaHeader));
+        } else {
+            // if the message is empty, create meta data to reflect this
+            // (always creating meta data for empty messages would add an unnecessary allocation for the receive case, so we do it lazily here)
+            MetaHeader hdr;
+            hdr.fSize = 0;
+            hdr.fHandle = -1;
+            hdr.fRegionId = 0;
+            hdr.fHint = 0;
+            memcpy(metas++, &hdr, sizeof(MetaHeader));
+        }
     }
 
-    while (!fInterrupted)
-    {
+    while (!fInterrupted) {
         int64_t totalSize = 0;
         int nbytes = zmq_msg_send(&zmqMsg, fSocket, flags);
-        if (nbytes == 0)
-        {
+        if (nbytes == 0) {
             zmq_msg_close(&zmqMsg);
             return nbytes;
-        }
-        else if (nbytes > 0)
-        {
+        } else if (nbytes > 0) {
             assert(static_cast<unsigned int>(nbytes) == (vecSize * sizeof(MetaHeader))); // all or nothing
 
-            for (auto& msg : msgVec)
-            {
+            for (auto& msg : msgVec) {
                 FairMQMessageSHM* shmMsg = static_cast<FairMQMessageSHM*>(msg.get());
                 shmMsg->fQueued = true;
                 totalSize += shmMsg->fSize;
@@ -301,36 +261,25 @@ int64_t FairMQSocketSHM::Send(vector<FairMQMessagePtr>& msgVec, const int timeou
 
             zmq_msg_close(&zmqMsg);
             return totalSize;
-        }
-        else if (zmq_errno() == EAGAIN)
-        {
-            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
-            {
-                if (timeout > 0)
-                {
+        } else if (zmq_errno() == EAGAIN) {
+            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0)) {
+                if (timeout > 0) {
                     elapsed += fSndTimeout;
-                    if (elapsed >= timeout)
-                    {
+                    if (elapsed >= timeout) {
                         zmq_msg_close(&zmqMsg);
                         return -2;
                     }
                 }
                 continue;
-            }
-            else
-            {
+            } else {
                 zmq_msg_close(&zmqMsg);
                 return -2;
             }
-        }
-        else if (zmq_errno() == ETERM)
-        {
+        } else if (zmq_errno() == ETERM) {
             zmq_msg_close(&zmqMsg);
             LOG(info) << "terminating socket " << fId;
             return -1;
-        }
-        else
-        {
+        } else {
             zmq_msg_close(&zmqMsg);
             LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
             return nbytes;
@@ -344,8 +293,7 @@ int64_t FairMQSocketSHM::Send(vector<FairMQMessagePtr>& msgVec, const int timeou
 int64_t FairMQSocketSHM::Receive(vector<FairMQMessagePtr>& msgVec, const int timeout)
 {
     int flags = 0;
-    if (timeout == 0)
-    {
+    if (timeout == 0) {
         flags = ZMQ_DONTWAIT;
     }
     int elapsed = 0;
@@ -353,17 +301,13 @@ int64_t FairMQSocketSHM::Receive(vector<FairMQMessagePtr>& msgVec, const int tim
     zmq_msg_t zmqMsg;
     zmq_msg_init(&zmqMsg);
 
-    while (!fInterrupted)
-    {
+    while (!fInterrupted) {
         int64_t totalSize = 0;
         int nbytes = zmq_msg_recv(&zmqMsg, fSocket, flags);
-        if (nbytes == 0)
-        {
+        if (nbytes == 0) {
             zmq_msg_close(&zmqMsg);
             return 0;
-        }
-        else if (nbytes > 0)
-        {
+        } else if (nbytes > 0) {
             MetaHeader* hdrVec = static_cast<MetaHeader*>(zmq_msg_data(&zmqMsg));
             const auto hdrVecSize = zmq_msg_size(&zmqMsg);
             assert(hdrVecSize > 0);
@@ -373,24 +317,22 @@ int64_t FairMQSocketSHM::Receive(vector<FairMQMessagePtr>& msgVec, const int tim
 
             msgVec.reserve(numMessages);
 
-            for (size_t m = 0; m < numMessages; m++)
-            {
-                MetaHeader hdr;
-                memcpy(&hdr, &hdrVec[m], sizeof(MetaHeader));
+            for (size_t m = 0; m < numMessages; m++) {
+                // get the meta data pointer
+                MetaHeader* hdr = &hdrVec[m];
 
-                msgVec.emplace_back(fair::mq::tools::make_unique<FairMQMessageSHM>(fManager, GetTransport()));
+                // create new message (part)
+                msgVec.emplace_back(tools::make_unique<FairMQMessageSHM>(fManager, GetTransport()));
+                FairMQMessageSHM* shmMsg = static_cast<FairMQMessageSHM*>(msgVec.back().get());
+                // fill the zmq buffer with the delivered meta data
+                memcpy(zmq_msg_data(shmMsg->GetMessage()), hdr, sizeof(MetaHeader));
+                // set the message members with the meta data
+                shmMsg->fHandle = hdr->fHandle;
+                shmMsg->fSize = hdr->fSize;
+                shmMsg->fRegionId = hdr->fRegionId;
+                shmMsg->fHint = hdr->fHint;
 
-                FairMQMessageSHM* msg = static_cast<FairMQMessageSHM*>(msgVec.back().get());
-                MetaHeader* msgHdr = static_cast<MetaHeader*>(zmq_msg_data(msg->GetMessage()));
-
-                memcpy(msgHdr, &hdr, sizeof(MetaHeader));
-
-                msg->fHandle = hdr.fHandle;
-                msg->fSize = hdr.fSize;
-                msg->fRegionId = hdr.fRegionId;
-                msg->fHint = hdr.fHint;
-
-                totalSize += msg->GetSize();
+                totalSize += shmMsg->GetSize();
             }
 
             // store statistics on how many messages have been received (handle all parts as a single message)
@@ -399,30 +341,21 @@ int64_t FairMQSocketSHM::Receive(vector<FairMQMessagePtr>& msgVec, const int tim
 
             zmq_msg_close(&zmqMsg);
             return totalSize;
-        }
-        else if (zmq_errno() == EAGAIN)
-        {
-            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
-            {
-                if (timeout > 0)
-                {
+        } else if (zmq_errno() == EAGAIN) {
+            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0)) {
+                if (timeout > 0) {
                     elapsed += fRcvTimeout;
-                    if (elapsed >= timeout)
-                    {
+                    if (elapsed >= timeout) {
                         zmq_msg_close(&zmqMsg);
                         return -2;
                     }
                 }
                 continue;
-            }
-            else
-            {
+            } else {
                 zmq_msg_close(&zmqMsg);
                 return -2;
             }
-        }
-        else
-        {
+        } else {
             zmq_msg_close(&zmqMsg);
             LOG(error) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
             return nbytes;
@@ -437,13 +370,11 @@ void FairMQSocketSHM::Close()
 {
     // LOG(debug) << "Closing socket " << fId;
 
-    if (fSocket == nullptr)
-    {
+    if (fSocket == nullptr) {
         return;
     }
 
-    if (zmq_close(fSocket) != 0)
-    {
+    if (zmq_close(fSocket) != 0) {
         LOG(error) << "Failed closing socket " << fId << ", reason: " << zmq_strerror(errno);
     }
 
@@ -464,23 +395,16 @@ void FairMQSocketSHM::Resume()
     fInterrupted = false;
 }
 
-void* FairMQSocketSHM::GetSocket() const
-{
-    return fSocket;
-}
-
 void FairMQSocketSHM::SetOption(const string& option, const void* value, size_t valueSize)
 {
-    if (zmq_setsockopt(fSocket, GetConstant(option), value, valueSize) < 0)
-    {
+    if (zmq_setsockopt(fSocket, GetConstant(option), value, valueSize) < 0) {
         LOG(error) << "Failed setting socket option, reason: " << zmq_strerror(errno);
     }
 }
 
 void FairMQSocketSHM::GetOption(const string& option, void* value, size_t* valueSize)
 {
-    if (zmq_getsockopt(fSocket, GetConstant(option), value, valueSize) < 0)
-    {
+    if (zmq_getsockopt(fSocket, GetConstant(option), value, valueSize) < 0) {
         LOG(error) << "Failed getting socket option, reason: " << zmq_strerror(errno);
     }
 }
@@ -570,26 +494,6 @@ int FairMQSocketSHM::GetRcvKernelSize() const
     return value;
 }
 
-unsigned long FairMQSocketSHM::GetBytesTx() const
-{
-    return fBytesTx;
-}
-
-unsigned long FairMQSocketSHM::GetBytesRx() const
-{
-    return fBytesRx;
-}
-
-unsigned long FairMQSocketSHM::GetMessagesTx() const
-{
-    return fMessagesTx;
-}
-
-unsigned long FairMQSocketSHM::GetMessagesRx() const
-{
-    return fMessagesRx;
-}
-
 int FairMQSocketSHM::GetConstant(const string& constant)
 {
     if (constant == "") return 0;
@@ -618,8 +522,3 @@ int FairMQSocketSHM::GetConstant(const string& constant)
 
     return -1;
 }
-
-FairMQSocketSHM::~FairMQSocketSHM()
-{
-    Close();
-}

fairmq/shmem/FairMQSocketSHM.h

@@ -34,7 +34,7 @@ class FairMQSocketSHM final : public FairMQSocket
     int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int timeout = -1) override;
     int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int timeout = -1) override;
 
-    void* GetSocket() const;
+    void* GetSocket() const { return fSocket; }
 
     void Close() override;
 
@@ -55,14 +55,14 @@ class FairMQSocketSHM final : public FairMQSocket
     void SetRcvKernelSize(const int value) override;
     int GetRcvKernelSize() const override;
 
-    unsigned long GetBytesTx() const override;
-    unsigned long GetBytesRx() const override;
-    unsigned long GetMessagesTx() const override;
-    unsigned long GetMessagesRx() const override;
+    unsigned long GetBytesTx() const override { return fBytesTx; }
+    unsigned long GetBytesRx() const override { return fBytesRx; }
+    unsigned long GetMessagesTx() const override { return fMessagesTx; }
+    unsigned long GetMessagesRx() const override { return fMessagesRx; }
 
     static int GetConstant(const std::string& constant);
 
-    ~FairMQSocketSHM() override;
+    ~FairMQSocketSHM() override { Close(); }
 
   private:
     void* fSocket;

fairmq/shmem/FairMQTransportFactorySHM.cxx

@@ -77,11 +77,12 @@ FairMQTransportFactorySHM::FairMQTransportFactorySHM(const string& id, const fai
             LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
         }
 
+        if (autolaunchMonitor) {
+            Manager::StartMonitor(fShmId);
+        }
+
         fManager = fair::mq::tools::make_unique<Manager>(fShmId, segmentSize);
 
-        if (autolaunchMonitor) {
-            fManager->StartMonitor();
-        }
     } catch (bipc::interprocess_exception& e) {
         LOG(error) << "Could not initialize shared memory transport: " << e.what();
         throw runtime_error(fair::mq::tools::ToString("Could not initialize shared memory transport: ", e.what()));

fairmq/shmem/FairMQUnmanagedRegionSHM.cxx

@@ -6,7 +6,7 @@
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#include <fairmq/shmem/Common.h>
+#include "Common.h"
 
 #include "FairMQUnmanagedRegionSHM.h"
 
@@ -41,18 +41,3 @@ FairMQUnmanagedRegionSHM::FairMQUnmanagedRegionSHM(Manager& manager, const size_
         throw;
     }
 }
-
-void* FairMQUnmanagedRegionSHM::GetData() const
-{
-    return fRegion->get_address();
-}
-
-size_t FairMQUnmanagedRegionSHM::GetSize() const
-{
-    return fRegion->get_size();
-}
-
-FairMQUnmanagedRegionSHM::~FairMQUnmanagedRegionSHM()
-{
-    fManager.RemoveRegion(fRegionId);
-}

fairmq/shmem/FairMQUnmanagedRegionSHM.h

@@ -28,10 +28,10 @@ class FairMQUnmanagedRegionSHM final : public FairMQUnmanagedRegion
   public:
     FairMQUnmanagedRegionSHM(fair::mq::shmem::Manager& manager, const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0);
 
-    void* GetData() const override;
-    size_t GetSize() const override;
+    void* GetData() const override { return fRegion->get_address(); }
+    size_t GetSize() const override { return fRegion->get_size(); }
 
-    ~FairMQUnmanagedRegionSHM() override;
+    ~FairMQUnmanagedRegionSHM() override { fManager.RemoveRegion(fRegionId); }
 
   private:
     fair::mq::shmem::Manager& fManager;

fairmq/shmem/Manager.cxx

@@ -8,11 +8,14 @@
 
 #include <fairmq/shmem/Manager.h>
 #include <fairmq/shmem/Common.h>
+#include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Strings.h>
 
 #include <boost/process.hpp>
 #include <boost/filesystem.hpp>
 
 using namespace std;
+using bie = ::boost::interprocess::interprocess_exception;
 namespace bipc = ::boost::interprocess;
 namespace bfs = ::boost::filesystem;
 
@@ -51,57 +54,42 @@ Manager::Manager(const std::string& id, size_t size)
     }
 }
 
-bipc::managed_shared_memory& Manager::Segment()
-{
-    return fSegment;
-}
-
-bipc::managed_shared_memory& Manager::ManagementSegment()
-{
-    return fManagementSegment;
-}
-
-void Manager::StartMonitor()
+void Manager::StartMonitor(const std::string& id)
 {
     try {
-        MonitorStatus* monitorStatus = fManagementSegment.find<MonitorStatus>(bipc::unique_instance).first;
-        if (monitorStatus == nullptr) {
-            LOG(debug) << "no fairmq-shmmonitor found, starting...";
-            auto env = boost::this_process::environment();
+        bipc::named_mutex monitorStatus(bipc::open_only, string("fmq_" + id + "_ms").c_str());
+        LOG(debug) << "Found fairmq-shmmonitor for shared memory id " << id;
+    } catch (bie&) {
+        LOG(debug) << "no fairmq-shmmonitor found for shared memory id " << id << ", starting...";
+        auto env = boost::this_process::environment();
 
-            vector<bfs::path> ownPath = boost::this_process::path();
+        vector<bfs::path> ownPath = boost::this_process::path();
 
-            if (const char* fmqp = getenv("FAIRMQ_PATH")) {
-                ownPath.insert(ownPath.begin(), bfs::path(fmqp));
-            }
-
-            bfs::path p = boost::process::search_path("fairmq-shmmonitor", ownPath);
-
-            if (!p.empty()) {
-                boost::process::spawn(p, "-x", "--shmid", fShmId, "-d", "-t", "2000", env);
-                int numTries = 0;
-                do {
-                    monitorStatus = fManagementSegment.find<MonitorStatus>(bipc::unique_instance).first;
-                    if (monitorStatus) {
-                        LOG(debug) << "fairmq-shmmonitor started";
-                        break;
-                    } else {
-                        this_thread::sleep_for(chrono::milliseconds(10));
-                        if (++numTries > 1000) {
-                            LOG(error) << "Did not get response from fairmq-shmmonitor after " << 10 * 1000 << " milliseconds. Exiting.";
-                            throw runtime_error(fair::mq::tools::ToString("Did not get response from fairmq-shmmonitor after ", 10 * 1000, " milliseconds. Exiting."));
-                        }
-                    }
-                } while (true);
-            } else {
-                LOG(warn) << "could not find fairmq-shmmonitor in the path";
-            }
-        } else {
-            LOG(debug) << "found fairmq-shmmonitor.";
+        if (const char* fmqp = getenv("FAIRMQ_PATH")) {
+            ownPath.insert(ownPath.begin(), bfs::path(fmqp));
+        }
+
+        bfs::path p = boost::process::search_path("fairmq-shmmonitor", ownPath);
+
+        if (!p.empty()) {
+            boost::process::spawn(p, "-x", "--shmid", id, "-d", "-t", "2000", env);
+            int numTries = 0;
+            do {
+                try {
+                    bipc::named_mutex monitorStatus(bipc::open_only, string("fmq_" + id + "_ms").c_str());
+                    LOG(debug) << "Started fairmq-shmmonitor for shared memory id " << id;
+                    break;
+                } catch (bie&) {
+                    this_thread::sleep_for(chrono::milliseconds(10));
+                    if (++numTries > 1000) {
+                        LOG(error) << "Did not get response from fairmq-shmmonitor after " << 10 * 1000 << " milliseconds. Exiting.";
+                        throw runtime_error(fair::mq::tools::ToString("Did not get response from fairmq-shmmonitor after ", 10 * 1000, " milliseconds. Exiting."));
+                    }
+                }
+            } while (true);
+        } else {
+            LOG(warn) << "could not find fairmq-shmmonitor in the path";
         }
-    } catch (std::exception& e) {
-        LOG(error) << "Exception during fairmq-shmmonitor initialization: " << e.what() << ", application will now exit";
-        exit(EXIT_FAILURE);
     }
 }
 
@@ -132,9 +120,10 @@ bipc::mapped_region* Manager::CreateRegion(const size_t size, const uint64_t id,
         {
             bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
             VoidAlloc voidAlloc(fManagementSegment.get_segment_manager());
-            Uint64RegionInfoMap* m = fManagementSegment.find_or_construct<Uint64RegionInfoMap>(bipc::unique_instance)(voidAlloc);
-            m->emplace(id, RegionInfo(path.c_str(), flags, voidAlloc));
+            Uint64RegionInfoMap* infoMap = fManagementSegment.find_or_construct<Uint64RegionInfoMap>(bipc::unique_instance)(voidAlloc);
+            infoMap->emplace(id, RegionInfo(path.c_str(), flags, voidAlloc));
         }
+        // LOG(debug) << "Created region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
 
         auto r = fRegions.emplace(id, fair::mq::tools::make_unique<Region>(*this, id, size, false, callback, path, flags));
 
@@ -158,17 +147,20 @@ Region* Manager::GetRemoteRegion(const uint64_t id)
             // get region info
             {
                 bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-                VoidAlloc voidAlloc(fSegment.get_segment_manager());
-                Uint64RegionInfoMap* m = fManagementSegment.find<Uint64RegionInfoMap>(bipc::unique_instance).first;
-                RegionInfo ri = m->at(id);
-                path = ri.fPath.c_str();
-                flags = ri.fFlags;
-                // LOG(debug) << "path: " << path << ", flags: " << flags;
+                Uint64RegionInfoMap* infoMap = fManagementSegment.find<Uint64RegionInfoMap>(bipc::unique_instance).first;
+                if (infoMap == nullptr) {
+                    LOG(error) << "Unable to locate the region info";
+                    throw SharedMemoryError("Unable to locate remote region info");
+                }
+                RegionInfo regionInfo = infoMap->at(id);
+                path = regionInfo.fPath.c_str();
+                flags = regionInfo.fFlags;
             }
+            // LOG(debug) << "Located remote region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
 
             auto r = fRegions.emplace(id, fair::mq::tools::make_unique<Region>(*this, id, 0, true, nullptr, path, flags));
             return r.first->second.get();
-        } catch (bipc::interprocess_exception& e) {
+        } catch (bie& e) {
             LOG(warn) << "Could not get remote region for id: " << id;
             return nullptr;
         }

fairmq/shmem/Manager.h

@@ -15,7 +15,6 @@
 #ifndef FAIR_MQ_SHMEM_MANAGER_H_
 #define FAIR_MQ_SHMEM_MANAGER_H_
 
-#include <fairmq/Tools.h>
 #include <fairmq/shmem/Region.h>
 #include <fairmq/shmem/Common.h>
 
@@ -28,6 +27,7 @@
 
 #include <string>
 #include <unordered_map>
+#include <stdexcept>
 
 namespace fair
 {
@@ -36,6 +36,8 @@ namespace mq
 namespace shmem
 {
 
+struct SharedMemoryError : std::runtime_error { using std::runtime_error::runtime_error; };
+
 class Manager
 {
     friend struct Region;
@@ -50,10 +52,10 @@ class Manager
 
     ~Manager();
 
-    boost::interprocess::managed_shared_memory& Segment();
-    boost::interprocess::managed_shared_memory& ManagementSegment();
+    boost::interprocess::managed_shared_memory& Segment() { return fSegment; }
+    boost::interprocess::managed_shared_memory& ManagementSegment() { return fManagementSegment; }
 
-    void StartMonitor();
+    static void StartMonitor(const std::string&);
 
     static void Interrupt();
     static void Resume();

fairmq/shmem/Monitor.cxx

@@ -25,6 +25,7 @@
 #include <poll.h>
 
 using namespace std;
+using bie = ::boost::interprocess::interprocess_exception;
 namespace bipc = ::boost::interprocess;
 namespace bpt = ::boost::posix_time;
 
@@ -45,11 +46,12 @@ void signalHandler(int signal)
     gSignalStatus = signal;
 }
 
-Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, unsigned int timeoutInMS, bool runAsDaemon, bool cleanOnExit)
+Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, bool runAsDaemon, bool cleanOnExit)
     : fSelfDestruct(selfDestruct)
     , fInteractive(interactive)
-    , fSeenOnce(false)
+    , fViewOnly(viewOnly)
     , fIsDaemon(runAsDaemon)
+    , fSeenOnce(false)
     , fCleanOnExit(cleanOnExit)
     , fTimeoutInMS(timeoutInMS)
     , fShmId(shmId)
@@ -63,14 +65,14 @@ Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, unsig
     , fManagementSegment(bipc::open_or_create, fManagementSegmentName.c_str(), 65536)
     , fDeviceHeartbeats()
 {
-    MonitorStatus* monitorStatus = fManagementSegment.find<MonitorStatus>(bipc::unique_instance).first;
-    if (monitorStatus != nullptr) {
-        cout << "fairmq-shmmonitor already started or not properly exited. Try `fairmq-shmmonitor --cleanup`" << endl;
-        exit(EXIT_FAILURE);
+    if (!fViewOnly) {
+        try {
+            bipc::named_mutex monitorStatus(bipc::create_only, string("fmq_" + fShmId + "_ms").c_str());
+        } catch (bie&) {
+            cout << "fairmq-shmmonitor for shared memory id " << fShmId << " already started or not properly exited. Try `fairmq-shmmonitor --cleanup --shmid " << fShmId << "`" << endl;
+            throw DaemonPresent(tools::ToString("fairmq-shmmonitor for shared memory id ", fShmId, " already started or not properly exited."));
+        }
     }
-    fManagementSegment.construct<MonitorStatus>(bipc::unique_instance)();
-
-    RemoveQueue(fControlQueueName);
 }
 
 void Monitor::CatchSignals()
@@ -97,7 +99,11 @@ void Monitor::SignalMonitor()
 
 void Monitor::Run()
 {
-    thread heartbeatThread(&Monitor::MonitorHeartbeats, this);
+    thread heartbeatThread;
+    if (!fViewOnly) {
+        RemoveQueue(fControlQueueName);
+        heartbeatThread = thread(&Monitor::MonitorHeartbeats, this);
+    }
 
     if (fInteractive) {
         Interactive();
@@ -108,7 +114,9 @@ void Monitor::Run()
         }
     }
 
-    heartbeatThread.join();
+    if (!fViewOnly) {
+        heartbeatThread.join();
+    }
 }
 
 void Monitor::MonitorHeartbeats()
@@ -131,13 +139,34 @@ void Monitor::MonitorHeartbeats()
                 // cout << "control queue timeout" << endl;
             }
         }
-    } catch (bipc::interprocess_exception& ie) {
+    } catch (bie& ie) {
         cout << ie.what() << endl;
     }
 
     RemoveQueue(fControlQueueName);
 }
 
+struct TerminalConfig
+{
+    TerminalConfig()
+    {
+        termios t;
+        tcgetattr(STDIN_FILENO, &t); // get the current terminal I/O structure
+        t.c_lflag &= ~ICANON; // disable canonical input
+        t.c_lflag &= ~ECHO; // do not echo input chars
+        tcsetattr(STDIN_FILENO, TCSANOW, &t); // apply the new settings
+    }
+
+    ~TerminalConfig()
+    {
+        termios t;
+        tcgetattr(STDIN_FILENO, &t); // get the current terminal I/O structure
+        t.c_lflag |= ICANON; // re-enable canonical input
+        t.c_lflag |= ECHO; // echo input chars
+        tcsetattr(STDIN_FILENO, TCSANOW, &t); // apply the new settings
+    }
+};
+
 void Monitor::Interactive()
 {
     char c;
@@ -145,11 +174,7 @@ void Monitor::Interactive()
     cinfd[0].fd = fileno(stdin);
     cinfd[0].events = POLLIN;
 
-    struct termios t;
-    tcgetattr(STDIN_FILENO, &t); // get the current terminal I/O structure
-    t.c_lflag &= ~ICANON; // disable canonical input
-    t.c_lflag &= ~ECHO; // do not echo input chars
-    tcsetattr(STDIN_FILENO, TCSANOW, &t); // apply the new settings
+    TerminalConfig tcfg;
 
     cout << endl;
     PrintHelp();
@@ -175,7 +200,11 @@ void Monitor::Interactive()
                     break;
                 case 'x':
                     cout << "\n[x] --> closing shared memory:" << endl;
-                    Cleanup(fShmId);
+                    if (!fViewOnly) {
+                        Cleanup(fShmId);
+                    } else {
+                        cout << "cannot close because in view only mode" << endl;
+                    }
                     break;
                 case 'h':
                     cout << "\n[h] --> help:" << endl << endl;
@@ -207,11 +236,6 @@ void Monitor::Interactive()
             cout << "\r";
         }
     }
-
-    tcgetattr(STDIN_FILENO, &t); // get the current terminal I/O structure
-    t.c_lflag |= ICANON; // re-enable canonical input
-    t.c_lflag |= ECHO; // echo input chars
-    tcsetattr(STDIN_FILENO, TCSANOW, &t); // apply the new settings
 }
 
 void Monitor::CheckSegment()
@@ -250,9 +274,11 @@ void Monitor::CheckSegment()
 
         unsigned int numDevices = 0;
 
-        fair::mq::shmem::DeviceCounter* dc = managementSegment.find<fair::mq::shmem::DeviceCounter>(bipc::unique_instance).first;
-        if (dc) {
-            numDevices = dc->fCount;
+        if (fInteractive) {
+            fair::mq::shmem::DeviceCounter* dc = managementSegment.find<fair::mq::shmem::DeviceCounter>(bipc::unique_instance).first;
+            if (dc) {
+                numDevices = dc->fCount;
+            }
         }
 
         auto now = chrono::high_resolution_clock::now();
@@ -270,31 +296,23 @@ void Monitor::CheckSegment()
 
         if (fInteractive) {
             cout << "| "
-                << setw(18) << fSegmentName << " | "
-                << setw(10) << segment.get_size() << " | "
-                << setw(10) << segment.get_free_memory() << " | "
-                // << setw(15) << segment.all_memory_deallocated() << " | "
-                << setw(2) << segment.check_sanity() << " | "
-                // << setw(10) << segment.get_num_named_objects() << " | "
-                << setw(10) << numDevices << " | "
-                // << setw(10) << segment.get_num_unique_objects() << " |"
-                << setw(10) << duration << " |"
-                << c
-                << flush;
+                 << setw(18) << fSegmentName                                    << " | "
+                 << setw(10) << segment.get_size()                              << " | "
+                 << setw(10) << segment.get_free_memory()                       << " | "
+                 << setw(8)  << numDevices                                      << " | "
+                 << setw(10) << (fViewOnly ? "view only" : to_string(duration)) << " |"
+                 << c << flush;
         }
-    } catch (bipc::interprocess_exception& ie) {
+    } catch (bie&) {
         fHeartbeatTriggered = false;
         if (fInteractive) {
             cout << "| "
-                << setw(18) << "-" << " | "
-                << setw(10) << "-" << " | "
-                << setw(10) << "-" << " | "
-                // << setw(15) << "-" << " | "
-                << setw(2) << "-" << " | "
-                << setw(10) << "-" << " | "
-                << setw(10) << "-" << " |"
-                << c
-                << flush;
+                 << setw(18) << "-" << " | "
+                 << setw(10) << "-" << " | "
+                 << setw(10) << "-" << " | "
+                 << setw(8)  << "-" << " | "
+                 << setw(10) << "-" << " |"
+                 << c << flush;
         }
 
         auto now = chrono::high_resolution_clock::now();
@@ -318,50 +336,60 @@ void Monitor::CheckSegment()
     }
 }
 
-void Monitor::Cleanup(const string& shmId)
+void Monitor::PrintQueues()
 {
-    string managementSegmentName("fmq_" + shmId + "_mng");
+    cout << '\n';
+
     try {
-        bipc::managed_shared_memory managementSegment(bipc::open_only, managementSegmentName.c_str());
-        RegionCounter* rc = managementSegment.find<RegionCounter>(bipc::unique_instance).first;
-        if (rc) {
-            cout << "Region counter found: " << rc->fCount << endl;
-            uint64_t regionCount = rc->fCount;
+        bipc::managed_shared_memory segment(bipc::open_only, fSegmentName.c_str());
+        StrVector* queues = segment.find<StrVector>(string("fmq_" + fShmId + "_qs").c_str()).first;
+        if (queues) {
+            cout << "found " << queues->size() << " queue(s):" << endl;
 
-            Uint64RegionInfoMap* m = managementSegment.find<Uint64RegionInfoMap>(bipc::unique_instance).first;
-
-            for (uint64_t i = 1; i <= regionCount; ++i) {
-                if (m != nullptr) {
-                    RegionInfo ri = m->at(i);
-                    string path = ri.fPath.c_str();
-                    int flags = ri.fFlags;
-                    cout << "Found RegionInfo with path: '" << path << "', flags: " << flags << "'." << endl;
-                    if (path != "") {
-                        RemoveFileMapping(tools::ToString(path, "fmq_" + shmId + "_rg_" + to_string(i)));
-                    } else {
-                        RemoveObject("fmq_" + shmId + "_rg_" + to_string(i));
-                    }
+            for (const auto& queue : *queues) {
+                string name(queue.c_str());
+                cout << '\t' << name << " : ";
+                atomic<int>* queueSize = segment.find<atomic<int>>(name.c_str()).first;
+                if (queueSize) {
+                    cout << *queueSize << " messages" << endl;
                 } else {
-                    RemoveObject("fmq_" + shmId + "_rg_" + to_string(i));
+                    cout << "\tqueue does not have a queue size entry." << endl;
                 }
-
-                RemoveQueue(string("fmq_" + shmId + "_rgq_" + to_string(i)));
             }
         } else {
-            cout << "No region counter found. no regions to cleanup." << endl;
+            cout << "\tno queues found" << endl;
         }
-
-        RemoveObject(managementSegmentName.c_str());
-    } catch (bipc::interprocess_exception& ie) {
-        cout << "Did not find '" << managementSegmentName << "' shared memory segment. No regions to cleanup." << endl;
+    } catch (bie&) {
+        cout << "\tno queues found" << endl;
+    } catch (out_of_range&) {
+        cout << "\tno queues found" << endl;
     }
 
-    RemoveObject("fmq_" + shmId + "_main");
-    RemoveMutex("fmq_" + shmId + "_mtx");
+    cout << "\n    --> last heartbeats: " << endl << endl;
+    auto now = chrono::high_resolution_clock::now();
+    for (const auto& h : fDeviceHeartbeats)  {
+        cout << "\t" << h.first << " : " << chrono::duration<double, milli>(now - h.second).count() << "ms ago." << endl;
+    }
 
     cout << endl;
 }
 
+void Monitor::PrintHeader()
+{
+    cout << "| "
+         << setw(18) << "name"    << " | "
+         << setw(10) << "size"    << " | "
+         << setw(10) << "free"    << " | "
+         << setw(8)  << "devices" << " | "
+         << setw(10) << "last hb" << " |"
+         << endl;
+}
+
+void Monitor::PrintHelp()
+{
+    cout << "controls: [x] close memory, [p] print queues, [h] help, [q] quit." << endl;
+}
+
 void Monitor::RemoveObject(const string& name)
 {
     if (bipc::shared_memory_object::remove(name.c_str())) {
@@ -398,73 +426,61 @@ void Monitor::RemoveMutex(const string& name)
     }
 }
 
-void Monitor::PrintQueues()
+void Monitor::Cleanup(const string& shmId)
 {
-    cout << '\n';
-
+    string managementSegmentName("fmq_" + shmId + "_mng");
     try {
-        bipc::managed_shared_memory segment(bipc::open_only, fSegmentName.c_str());
-        StrVector* queues = segment.find<StrVector>(string("fmq_" + fShmId + "_qs").c_str()).first;
-        if (queues) {
-            cout << "found " << queues->size() << " queue(s):" << endl;
+        bipc::managed_shared_memory managementSegment(bipc::open_only, managementSegmentName.c_str());
+        RegionCounter* rc = managementSegment.find<RegionCounter>(bipc::unique_instance).first;
+        if (rc) {
+            cout << "Region counter found: " << rc->fCount << endl;
+            uint64_t regionCount = rc->fCount;
 
-            for (const auto& queue : *queues) {
-                string name(queue.c_str());
-                cout << '\t' << name << " : ";
-                atomic<int>* queueSize = segment.find<atomic<int>>(name.c_str()).first;
-                if (queueSize) {
-                    cout << *queueSize << " messages" << endl;
+            Uint64RegionInfoMap* m = managementSegment.find<Uint64RegionInfoMap>(bipc::unique_instance).first;
+
+            for (uint64_t i = 1; i <= regionCount; ++i) {
+                if (m != nullptr) {
+                    RegionInfo ri = m->at(i);
+                    string path = ri.fPath.c_str();
+                    int flags = ri.fFlags;
+                    cout << "Found RegionInfo with path: '" << path << "', flags: " << flags << "'." << endl;
+                    if (path != "") {
+                        RemoveFileMapping(tools::ToString(path, "fmq_" + shmId + "_rg_" + to_string(i)));
+                    } else {
+                        RemoveObject("fmq_" + shmId + "_rg_" + to_string(i));
+                    }
                 } else {
-                    cout << "\tqueue does not have a queue size entry." << endl;
+                    RemoveObject("fmq_" + shmId + "_rg_" + to_string(i));
                 }
+
+                RemoveQueue(string("fmq_" + shmId + "_rgq_" + to_string(i)));
             }
         } else {
-            cout << "\tno queues found" << endl;
+            cout << "No region counter found. no regions to cleanup." << endl;
         }
-    } catch (bipc::interprocess_exception& ie) {
-        cout << "\tno queues found" << endl;
-    } catch (out_of_range& ie) {
-        cout << "\tno queues found" << endl;
+
+        RemoveObject(managementSegmentName.c_str());
+    } catch (bie&) {
+        cout << "Did not find '" << managementSegmentName << "' shared memory segment. No regions to cleanup." << endl;
     }
 
-    cout << "\n    --> last heartbeats: " << endl << endl;
-    auto now = chrono::high_resolution_clock::now();
-    for (const auto& h : fDeviceHeartbeats)  {
-        cout << "\t" << h.first << " : " << chrono::duration<double, milli>(now - h.second).count() << "ms ago." << endl;
-    }
+    RemoveObject("fmq_" + shmId + "_main");
+    RemoveMutex("fmq_" + shmId + "_mtx");
 
     cout << endl;
 }
 
-void Monitor::PrintHeader()
-{
-    cout << "| "
-        << "\033[01;32m" << setw(18) << "name"            << "\033[0m" << " | "
-        << "\033[01;32m" << setw(10) << "size"            << "\033[0m" << " | "
-        << "\033[01;32m" << setw(10) << "free"            << "\033[0m" << " | "
-        // << "\033[01;32m" << setw(15) << "all deallocated" << "\033[0m" << " | "
-        << "\033[01;32m" << setw(2)  << "ok"              << "\033[0m" << " | "
-        // << "\033[01;32m" << setw(10) << "# named"         << "\033[0m" << " | "
-        << "\033[01;32m" << setw(10) << "# devices"       << "\033[0m" << " | "
-        // << "\033[01;32m" << setw(10) << "# unique"        << "\033[0m" << " |"
-        << "\033[01;32m" << setw(10) << "ms since"        << "\033[0m" << " |"
-        << endl;
-}
-
-void Monitor::PrintHelp()
-{
-    cout << "controls: [x] close memory, [p] print queues, [h] help, [q] quit." << endl;
-}
-
 Monitor::~Monitor()
 {
-    fManagementSegment.destroy<MonitorStatus>(bipc::unique_instance);
     if (fSignalThread.joinable()) {
         fSignalThread.join();
     }
     if (fCleanOnExit) {
         Cleanup(fShmId);
     }
+    if (!fViewOnly) {
+        RemoveMutex("fmq_" + fShmId + "_ms");
+    }
 }
 
 } // namespace shmem

fairmq/shmem/Monitor.h

@@ -14,6 +14,7 @@
 #include <chrono>
 #include <atomic>
 #include <string>
+#include <stdexcept>
 #include <unordered_map>
 
 namespace fair
@@ -26,22 +27,24 @@ namespace shmem
 class Monitor
 {
   public:
-    Monitor(const std::string& shmId, bool selfDestruct, bool interactive, unsigned int timeoutInMS, bool runAsDaemon, bool cleanOnExit);
+    Monitor(const std::string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, bool runAsDaemon, bool cleanOnExit);
 
     Monitor(const Monitor&) = delete;
     Monitor operator=(const Monitor&) = delete;
 
+    virtual ~Monitor();
+
     void CatchSignals();
     void Run();
 
-    virtual ~Monitor();
-
     static void Cleanup(const std::string& shmId);
     static void RemoveObject(const std::string&);
     static void RemoveFileMapping(const std::string&);
     static void RemoveQueue(const std::string&);
     static void RemoveMutex(const std::string&);
 
+    struct DaemonPresent : std::runtime_error { using std::runtime_error::runtime_error; };
+
   private:
     void PrintHeader();
     void PrintHelp();
@@ -53,8 +56,9 @@ class Monitor
 
     bool fSelfDestruct; // will self-destruct after the memory has been closed
     bool fInteractive; // running in interactive mode
-    bool fSeenOnce; // true is segment has been opened successfully at least once
+    bool fViewOnly; // view only mode
     bool fIsDaemon;
+    bool fSeenOnce; // true is segment has been opened successfully at least once
     bool fCleanOnExit;
     unsigned int fTimeoutInMS;
     std::string fShmId;

fairmq/shmem/README.md

@@ -32,5 +32,6 @@ FairMQ Shared Memory currently uses following names to register shared memory on
 `fmq_<shmId>_mng` - management segment name, used for storing management data.
 `fmq_<shmId>_cq` - message queue for communicating between shm transport and shm monitor (exists independent of above segments).
 `fmq_<shmId>_mtx` - boost::interprocess::named_mutex for management purposes (exists independent of above segments).
+`fmq_<shmId>_ms` - shmmonitor status used to signal if it is active or not (exists independent of above segments).
 `fmq_<shmId>_rg_<index>` - names of unmanaged regions.
 `fmq_<shmId>_rgq_<index>` - names of queues for the unmanaged regions.

fairmq/shmem/Region.cxx

@@ -9,6 +9,8 @@
 #include <fairmq/shmem/Region.h>
 #include <fairmq/shmem/Common.h>
 #include <fairmq/shmem/Manager.h>
+#include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Strings.h>
 
 #include <boost/filesystem.hpp>
 #include <boost/process.hpp>

fairmq/shmem/Region.h

@@ -18,7 +18,6 @@
 #include "FairMQLogger.h"
 #include "FairMQUnmanagedRegion.h"
 
-#include <fairmq/Tools.h>
 #include <fairmq/shmem/Common.h>
 
 #include <boost/interprocess/managed_shared_memory.hpp>
@@ -45,8 +44,8 @@ struct Region
 
     Region() = delete;
 
-    Region(const Region&) = default;
-    Region(Region&&) = default;
+    Region(const Region&) = delete;
+    Region(Region&&) = delete;
 
     void InitializeQueues();
 

fairmq/shmem/runMonitor.cxx

@@ -26,19 +26,25 @@ using namespace fair::mq::shmem;
 static void daemonize()
 {
     // already a daemon?
-    // if (getppid() == 1) return;
+    if (getppid() == 1) {
+        return;
+    }
 
-    // Fork off the parent process
-    // pid_t pid = fork();
-    // if (pid < 0) exit(1);
+    // Fork
+    pid_t pid = fork();
+    if (pid < 0) {
+        exit(1);
+    }
 
     // If we got a good PID, then we can exit the parent process.
-    // if (pid > 0) exit(0);
+    if (pid > 0) {
+        exit(0);
+    }
 
     // Change the file mode mask
     umask(0);
 
-    // Create a new SID for the child process
+    // Create a new session with the calling process as its leader.
     if (setsid() < 0) {
         exit(1);
     }
@@ -68,20 +74,22 @@ int main(int argc, char** argv)
         bool cleanup = false;
         bool selfDestruct = false;
         bool interactive = false;
+        bool viewOnly = false;
         unsigned int timeoutInMS = 5000;
         bool runAsDaemon = false;
         bool cleanOnExit = false;
 
         options_description desc("Options");
         desc.add_options()
-            ("session,s", value<string>(&sessionName)->default_value("default"), "session id which to monitor")
-            ("shmid", value<string>(&shmId)->default_value(""), "Shmem Id to monitor (if not provided, it is generated out of session id and user id)")
-            ("cleanup,c", value<bool>(&cleanup)->implicit_value(true), "Perform cleanup and quit")
-            ("self-destruct,x", value<bool>(&selfDestruct)->implicit_value(true), "Quit after first closing of the memory")
-            ("interactive,i", value<bool>(&interactive)->implicit_value(true), "Interactive run")
-            ("timeout,t", value<unsigned int>(&timeoutInMS)->default_value(5000), "Heartbeat timeout in milliseconds")
-            ("daemonize,d", value<bool>(&runAsDaemon)->implicit_value(true), "Daemonize the monitor")
-            ("clean-on-exit,e", value<bool>(&cleanOnExit)->implicit_value(true), "Perform cleanup on exit")
+            ("session,s"      , value<string>(&sessionName)->default_value("default"),  "Session id")
+            ("shmid"          , value<string>(&shmId)->default_value(""),               "Shmem id (if not provided, it is generated out of session id and user id)")
+            ("cleanup,c"      , value<bool>(&cleanup)->implicit_value(true),            "Perform cleanup and quit")
+            ("self-destruct,x", value<bool>(&selfDestruct)->implicit_value(true),       "Quit after first closing of the memory")
+            ("interactive,i"  , value<bool>(&interactive)->implicit_value(true),        "Interactive run")
+            ("view,v"         , value<bool>(&viewOnly)->implicit_value(true),           "Run in view only mode")
+            ("timeout,t"      , value<unsigned int>(&timeoutInMS)->default_value(5000), "Heartbeat timeout in milliseconds")
+            ("daemonize,d"    , value<bool>(&runAsDaemon)->implicit_value(true),        "Daemonize the monitor")
+            ("clean-on-exit,e", value<bool>(&cleanOnExit)->implicit_value(true),        "Perform cleanup on exit")
             ("help,h", "Print help");
 
         variables_map vm;
@@ -111,10 +119,12 @@ int main(int argc, char** argv)
 
         cout << "Starting shared memory monitor for session: \"" << sessionName << "\" (shmId: " << shmId << ")..." << endl;
 
-        Monitor monitor{shmId, selfDestruct, interactive, timeoutInMS, runAsDaemon, cleanOnExit};
+        Monitor monitor(shmId, selfDestruct, interactive, viewOnly, timeoutInMS, runAsDaemon, cleanOnExit);
 
         monitor.CatchSignals();
         monitor.Run();
+    } catch (Monitor::DaemonPresent& dp) {
+        return 0;
     } catch (exception& e) {
         cerr << "Unhandled Exception reached the top of main: " << e.what() << ", application will now exit" << endl;
         return 2;

fairmq/tools/Semaphore.cxx

@@ -39,12 +39,35 @@ auto Semaphore::Signal() -> void
     fCv.notify_one();
 }
 
-auto Semaphore::GetCount() -> std::size_t
+auto Semaphore::GetCount() const -> std::size_t
 {
     std::unique_lock<std::mutex> lk(fMutex);
     return fCount;
 }
 
+SharedSemaphore::SharedSemaphore()
+    : fSemaphore(std::make_shared<Semaphore>())
+{}
+
+SharedSemaphore::SharedSemaphore(std::size_t initial_count)
+    : fSemaphore(std::make_shared<Semaphore>(initial_count))
+{}
+
+auto SharedSemaphore::Wait() -> void
+{
+    fSemaphore->Wait();
+}
+
+auto SharedSemaphore::Signal() -> void
+{
+    fSemaphore->Signal();
+}
+
+auto SharedSemaphore::GetCount() const -> std::size_t
+{
+    return fSemaphore->GetCount();
+}
+
 } /* namespace tools */
 } /* namespace mq */
 } /* namespace fair */