Shmem: Send acks also for local regions

Resolves #244

.clang-tidy

@@ -1,3 +1,3 @@
 ---
-Checks:          '*,-google-*,-fuchsia-*,-cert-*,-llvm-header-guard,-readability-named-parameter,-misc-non-private-member-variables-in-classes,-*-magic-numbers,-llvm-include-order,-hicpp-no-array-decay,-performance-unnecessary-value-param,-cppcoreguidelines-pro-bounds-array-to-pointer-decay'
+Checks:          '*,-google-*,-fuchsia-*,-cert-*,-llvm-header-guard,-readability-named-parameter,-misc-non-private-member-variables-in-classes,-*-magic-numbers,-llvm-include-order,-hicpp-no-array-decay,-performance-unnecessary-value-param,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-modernize-use-trailing-return-type,-readability-redundant-member-init'
 HeaderFilterRegex: '/(fairmq/)'

CMakeLists.txt

@@ -6,8 +6,8 @@
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
-cmake_minimum_required(VERSION 3.11 FATAL_ERROR)
-cmake_policy(VERSION 3.11...3.15)
+cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
+cmake_policy(VERSION 3.12...3.15)
 
 # Project ######################################################################
 set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
@@ -18,7 +18,7 @@ get_git_version()
 project(FairMQ VERSION ${PROJECT_VERSION} LANGUAGES CXX)
 message(STATUS "${BWhite}${PROJECT_NAME}${CR} ${PROJECT_GIT_VERSION} from ${PROJECT_DATE}")
 
-if(BUILD_OFI_TRANSPORT OR BUILD_SDK)
+if(BUILD_OFI_TRANSPORT OR BUILD_SDK OR BUILD_PMIX_PLUGIN)
   set(PROJECT_MIN_CXX_STANDARD 14)
 else()
   set(PROJECT_MIN_CXX_STANDARD 11)
@@ -44,7 +44,7 @@ fairmq_build_option(BUILD_SDK_COMMANDS      "Build the FairMQ SDK commands."
 fairmq_build_option(BUILD_DDS_PLUGIN        "Build DDS plugin."
                                              DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
 fairmq_build_option(BUILD_PMIX_PLUGIN       "Build PMIx plugin."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
+                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
 fairmq_build_option(BUILD_EXAMPLES          "Build FairMQ examples."
                                              DEFAULT ON  REQUIRES "BUILD_FAIRMQ")
 fairmq_build_option(BUILD_SDK               "Build the FairMQ controller SDK."
@@ -397,9 +397,9 @@ else()
 endif()
 message(STATUS "  ${BWhite}dds_plugin${CR}         ${dds_summary}")
 if(BUILD_PMIX_PLUGIN)
-  set(pmix_summary "${BGreen}YES${CR}    (disable with ${BMagenta}-DBUILD_PMIX_PLUGIN=OFF${CR})")
+  set(pmix_summary "${BGreen}YES${CR}    EXPERIMENTAL (requires C++14) (disable with ${BMagenta}-DBUILD_PMIX_PLUGIN=OFF${CR})")
 else()
-  set(pmix_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_PMIX_PLUGIN=ON${CR})")
+  set(pmix_summary "${BRed} NO${CR}    EXPERIMENTAL (requires C++14) (default, enable with ${BMagenta}-DBUILD_PMIX_PLUGIN=ON${CR})")
 endif()
 message(STATUS "  ${BWhite}pmix_plugin${CR}        ${pmix_summary}")
 if(BUILD_EXAMPLES)

CONTRIBUTORS

@@ -4,6 +4,7 @@ Eulisse, Giulio
 Karabowicz, Radoslaw
 Kretz, Matthias <kretz@kde.org>
 Krzewicki, Mikolaj 
+Lebedev, Andrey
 Mrnjavac, Teo <teo.m@cern.ch>
 Neskovic, Gvozden
 Richter, Matthias

cmake/FairMQLib.cmake

@@ -143,15 +143,17 @@ macro(set_fairmq_defaults)
   set(PROJECT_EXPORT_SET ${PROJECT_NAME}Targets)
 
   # Configure build types
-  set(CMAKE_CONFIGURATION_TYPES "Debug" "Release" "RelWithDebInfo" "Nightly" "Profile" "Experimental" "AdressSan" "ThreadSan")
-  set(CMAKE_CXX_FLAGS_DEBUG          "-g -Wshadow -Wall -Wextra")
+  set(CMAKE_CONFIGURATION_TYPES "Debug" "Release" "RelWithDebInfo" "Nightly" "Profile" "Experimental" "AddressSan" "ThreadSan")
+  set(_warnings "-Wshadow -Wall -Wextra -Wpedantic")
+  set(CMAKE_CXX_FLAGS_DEBUG          "-g ${_warnings}")
   set(CMAKE_CXX_FLAGS_RELEASE        "-O2 -DNDEBUG")
-  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -Wshadow -Wall -Wextra -DNDEBUG")
-  set(CMAKE_CXX_FLAGS_NIGHTLY        "-O2 -g -Wshadow -Wall -Wextra")
-  set(CMAKE_CXX_FLAGS_PROFILE        "-g3 -Wshadow -Wall -Wextra -fno-inline -ftest-coverage -fprofile-arcs")
-  set(CMAKE_CXX_FLAGS_EXPERIMENTAL   "-O2 -g -Wshadow -Wall -Wextra -DNDEBUG")
-  set(CMAKE_CXX_FLAGS_ADRESSSAN      "-O2 -g -Wshadow -Wall -Wextra -fsanitize=address -fno-omit-frame-pointer")
-  set(CMAKE_CXX_FLAGS_THREADSAN      "-O2 -g -Wshadow -Wall -Wextra -fsanitize=thread")
+  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g ${_warnings} -DNDEBUG")
+  set(CMAKE_CXX_FLAGS_NIGHTLY        "-O2 -g ${_warnings}")
+  set(CMAKE_CXX_FLAGS_PROFILE        "-g3 ${_warnings} -fno-inline -ftest-coverage -fprofile-arcs")
+  set(CMAKE_CXX_FLAGS_EXPERIMENTAL   "-O2 -g ${_warnings} -DNDEBUG")
+  set(CMAKE_CXX_FLAGS_ADDRESSSAN     "-O2 -g ${_warnings} -fsanitize=address -fno-omit-frame-pointer")
+  set(CMAKE_CXX_FLAGS_THREADSAN      "-O2 -g ${_warnings} -fsanitize=thread")
+  unset(_warnings)
 
   if(CMAKE_GENERATOR STREQUAL "Ninja" AND
      ((CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9) OR
@@ -522,7 +524,7 @@ macro(fairmq_build_option option description)
       if(${__requires_condition__})
       else()
         if(${option})
-          message(WARNING "Cannot enable build option ${option}, depending option is not set: ${__requires_condition__}")
+          message(FATAL_ERROR "Cannot enable build option ${option}, depending option is not set: ${__requires_condition__}")
         endif()
       endif()
     endforeach()

examples/CMakeLists.txt

@@ -16,6 +16,8 @@ add_subdirectory(multiple-channels)
 if(BUILD_NANOMSG_TRANSPORT)
   add_subdirectory(multiple-transports)
 endif()
+add_subdirectory(n-m)
+add_subdirectory(qc)
 add_subdirectory(readout)
 add_subdirectory(region)
 add_subdirectory(req-rep)

examples/README.md

@@ -6,41 +6,50 @@ Set of FairMQ examples. More examples that combine FairMQ with FairRoot can be f
 
 A simple topology of two devices - **Sampler** and **Sink**. **Sampler** sends data to **Sink** with the **PUSH-PULL** pattern.
 
-
 ## 1-n-1
 
 A simple topology of three device types - **Sampler**, **Processor** and **Sink**. **Sampler** sends data to one or more **Processor**s, who modify the data and send it to one **Sink**. Transport with the **PUSH-PULL** pattern. The example also shows the configuration via JSON files, as oposed to `--channel-config` that is used by other examples.
 
+## Built-in devices
 
-## DDS
-
-This example demonstrates usage of the Dynamic Deployment System ([DDS](http://dds.gsi.de/)) to dynamically deploy and configure a topology of devices. The topology is similar to those of Example 2, but now it can be easily distributed on different computing nodes without the need for manual reconfiguration of the devices.
-
+Usage of generic devies provided with FairMQ.
 
 ## Copy & Push
 
 A topology consisting of one **Sampler** and two **Sink**s. The **Sampler** uses the `Copy` method to send the same data to both sinks with the **PUSH-PULL** pattern. In countrary to the **PUB-SUB** pattern, this ensures that all receivers are connected and no data is lost, but requires additional channels to be configured.
 
+## DDS
 
-## Request & Reply
+This example demonstrates usage of the Dynamic Deployment System ([DDS](http://dds.gsi.de/)) to dynamically deploy and configure a topology of devices. The topology is similar to those of Example 2, but now it can be easily distributed on different computing nodes without the need for manual reconfiguration of the devices.
 
-This topology contains two devices that communicate with each other via the **REQ-REP** pettern. Bidirectional communication via a single socket.
+## Multipart
 
+This example shows how to send a multipart message from one device to the other. (two parts message parts - header and body).
 
 ## Multiple Channels
 
 This example demonstrates how to work with multiple channels and multiplex between them.
 
-
-## Sending Multipart messages
-
-This example shows how to send a multipart message from one device to the other. (two parts message parts - header and body).
-
-
-## Multiple Transports example
+## Multiple Transports
 
 This examples shows how to combine different channel transports (zeromq/nanomsg/shmem) inside of one device and/or topology.
 
-## Region example
+## n-m
+
+A topology consisting of three layers of devices: synchronizer -> n * senders -> m * receivers.
+
+## QC
+
+A topology consisting of 4 devices - Sampler, QCDispatcher, QCTask and Sink. The data flows from Sampler through QCDispatcher to Sink. On demand - by setting the corresponding configuration property - the QCDispatcher device will duplicate the data to the QCTask device. The property is set by the topology controller, in this example this is the `fairmq-dds-command-ui` utility.
+
+## Readout
+
+Two example topologies of setups to be distributed to two kinds of nodes - detector readout node and processing node. Detector readout node contains readout process, data builder and data sender (and optionally an additional processor), while processing node contains data receiver devices. communication within readout nodes is done via unmanaged region through shared memory transport.
+
+## Region
 
 This example demonstrates the use of a more advanced feature - UnmanagedRegion, that can be used to create a buffer through one of FairMQ transports. The contents of this buffer are managed by the user, who can also create messages out of sub-buffers of the created buffer. Such feature can be interesting in environments that have special requirements by the hardware that writes the data, to keep the transfer efficient (e.g. shared memory).
+
+## Request & Reply
+
+This topology contains two devices that communicate with each other via the **REQ-REP** pettern. Bidirectional communication via a single socket.

examples/dds/CMakeLists.txt

@@ -7,12 +7,12 @@
 ################################################################################
 
 add_library(ExampleDDSLib STATIC
-    "Sampler.cxx"
-    "Sampler.h"
-    "Processor.cxx"
-    "Processor.h"
-    "Sink.cxx"
-    "Sink.h"
+  "Sampler.cxx"
+  "Sampler.h"
+  "Processor.cxx"
+  "Processor.h"
+  "Sink.cxx"
+  "Sink.h"
 )
 
 target_link_libraries(ExampleDDSLib PUBLIC FairMQ)
@@ -28,6 +28,7 @@ target_link_libraries(fairmq-ex-dds-sink PRIVATE ExampleDDSLib)
 
 add_custom_target(ExampleDDS DEPENDS fairmq-ex-dds-sampler fairmq-ex-dds-processor fairmq-ex-dds-sink)
 
+list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
 set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
 set(DATA_DIR ${CMAKE_CURRENT_BINARY_DIR})
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-dds-topology.xml ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology.xml @ONLY)
@@ -38,23 +39,24 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-dds.sh.in ${CMAKE_CUR
 
 # test
 if(DDS_FOUND)
-    add_test(NAME Example.DDS.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh localhost)
-    set_tests_properties(Example.DDS.localhost PROPERTIES
+  add_test(NAME Example.DDS.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh localhost)
+  set_tests_properties(Example.DDS.localhost PROPERTIES
     TIMEOUT 15
+    RUN_SERIAL true
     PASS_REGULAR_EXPRESSION "Example successful"
-    )
+  )
 endif()
 
 # install
 
 install(
-    TARGETS
-    fairmq-ex-dds-sampler
-    fairmq-ex-dds-processor
-    fairmq-ex-dds-sink
+  TARGETS
+  fairmq-ex-dds-sampler
+  fairmq-ex-dds-processor
+  fairmq-ex-dds-sink
 
-    LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-    RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
+  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
 )
 
 # configure run script with different executable paths for build and for install directories
@@ -66,30 +68,30 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-dds-env.sh ${CMAKE_CURRENT_
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-dds.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh_install @ONLY)
 
 install(
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology.xml_install
-    DESTINATION ${PROJECT_INSTALL_DATADIR}
-    RENAME ex-dds-topology.xml
+  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology.xml_install
+  DESTINATION ${PROJECT_INSTALL_DATADIR}
+  RENAME ex-dds-topology.xml
 )
 
 install(
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology-infinite.xml_install
-    DESTINATION ${PROJECT_INSTALL_DATADIR}
-    RENAME ex-dds-topology-infinite.xml
+  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology-infinite.xml_install
+  DESTINATION ${PROJECT_INSTALL_DATADIR}
+  RENAME ex-dds-topology-infinite.xml
 )
 
 install(
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-hosts.cfg
-    DESTINATION ${PROJECT_INSTALL_DATADIR}
+  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-hosts.cfg
+  DESTINATION ${PROJECT_INSTALL_DATADIR}
 )
 
 install(
-    PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-dds-env.sh_install
-    DESTINATION ${PROJECT_INSTALL_BINDIR}
-    RENAME fairmq-ex-dds-env.sh
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-dds-env.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-ex-dds-env.sh
 )
 
 install(
-    PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh_install
-    DESTINATION ${PROJECT_INSTALL_BINDIR}
-    RENAME fairmq-start-ex-dds.sh
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-dds.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-start-ex-dds.sh
 )

examples/dds/fairmq-ex-dds-env.sh

@@ -9,3 +9,8 @@
 ################################################################################
 
 export PATH=@BIN_DIR@:$PATH
+
+OS=$(uname -s 2>&1)
+if [ "$OS" == "Darwin" ]; then
+   export DYLD_LIBRARY_PATH=@LIB_DIR@:$DYLD_LIBRARY_PATH
+fi

examples/dds/fairmq-start-ex-dds.sh.in

@@ -39,12 +39,12 @@ fi
 echo "...waiting for ${requiredNofSlots} idle slots..."
 dds-info --idle-count --wait ${requiredNofSlots}
 
-topologyFile=@DATA_DIR@/ex-dds-topology.xml
-echo "TOPOLOGY FILE: ${topologyFile}"
-echo "TOPOLOGY NAME: $(dds-topology --disable-validation --topology-name ${topologyFile})"
+export FAIRMQ_DDS_TOPO_FILE=@DATA_DIR@/ex-dds-topology.xml
+echo "TOPOLOGY FILE: ${FAIRMQ_DDS_TOPO_FILE}"
+echo "TOPOLOGY NAME: $(dds-topology --disable-validation --topology-name ${FAIRMQ_DDS_TOPO_FILE})"
 
 dds-info --active-topology
-dds-topology --activate ${topologyFile}
+dds-topology --activate ${FAIRMQ_DDS_TOPO_FILE}
 dds-info --active-topology
 echo "...waiting for ${requiredNofSlots} executing slots..."
 dds-info --executing-count --wait ${requiredNofSlots}
@@ -52,21 +52,20 @@ dds-info --executing-count --wait ${requiredNofSlots}
 echo "------------------------"
 echo "...waiting for Topology to finish..."
 # TODO Retrieve number of devices from DDS topology API instead of having the user pass it explicitely
-fairmq-dds-command-ui -w "IDLE" -n ${requiredNofSlots}
-fairmq-dds-command-ui -c i -w "INITIALIZING DEVICE" -n ${requiredNofSlots}
-fairmq-dds-command-ui -c k -w "INITIALIZED" -n ${requiredNofSlots}
-fairmq-dds-command-ui -c b -w "BOUND" -n ${requiredNofSlots}
-fairmq-dds-command-ui -c x -w "DEVICE READY" -n ${requiredNofSlots}
-fairmq-dds-command-ui -c j -w "READY" -n ${requiredNofSlots}
+fairmq-dds-command-ui -w "IDLE"
+fairmq-dds-command-ui -c i
+fairmq-dds-command-ui -c k
+fairmq-dds-command-ui -c b
+fairmq-dds-command-ui -c x
+fairmq-dds-command-ui -c j
 fairmq-dds-command-ui -c r
-sampler_and_sink="main/(Sampler|Sink)"
-# processors="main/ProcessorGroup/Processor"
-fairmq-dds-command-ui -p $sampler_and_sink -w "RUNNING->READY" -n 2
+sampler_and_sink="main/(Sampler|Sink).*"
+fairmq-dds-command-ui -w "RUNNING->READY" -p $sampler_and_sink
 echo "...$sampler_and_sink are READY, sending shutdown..."
-fairmq-dds-command-ui -c s -w "RUNNING->READY" -n ${requiredNofSlots}
-fairmq-dds-command-ui -c t -w "DEVICE READY" -n ${requiredNofSlots}
-fairmq-dds-command-ui -c d -w "IDLE" -n ${requiredNofSlots}
-fairmq-dds-command-ui -c q -w "EXITING" -n ${requiredNofSlots}
+fairmq-dds-command-ui -c s
+fairmq-dds-command-ui -c t
+fairmq-dds-command-ui -c d
+fairmq-dds-command-ui -c q
 echo "...waiting for ${requiredNofSlots} idle slots..."
 dds-info --idle-count --wait ${requiredNofSlots}
 echo "------------------------"

examples/multipart/test-ex-multipart.sh.in

@@ -20,7 +20,7 @@ SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --session $SESSION"
 SAMPLER+=" --max-iterations 1"
 SAMPLER+=" --control static --color false"
-SAMPLER+=" --channel-config name=data,type=pair,method=connect,rateLogging=0,address=tcp://127.0.0.1:5555"
+SAMPLER+=" --channel-config name=data,type=pair,method=connect,rateLogging=0,address=tcp://127.0.0.1:5555,linger=1000"
 @CMAKE_CURRENT_BINARY_DIR@/$SAMPLER &
 SAMPLER_PID=$!
 

examples/n-m/CMakeLists.txt

@@ -0,0 +1,99 @@
+################################################################################
+#    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+add_executable(fairmq-ex-n-m-synchronizer runSynchronizer.cxx)
+target_link_libraries(fairmq-ex-n-m-synchronizer PRIVATE FairMQ)
+
+add_executable(fairmq-ex-n-m-sender runSender.cxx)
+target_link_libraries(fairmq-ex-n-m-sender PRIVATE FairMQ)
+
+add_executable(fairmq-ex-n-m-receiver runReceiver.cxx)
+target_link_libraries(fairmq-ex-n-m-receiver PRIVATE FairMQ)
+
+add_custom_target(ExampleNM DEPENDS fairmq-ex-n-m-synchronizer fairmq-ex-n-m-sender fairmq-ex-n-m-receiver)
+
+list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
+set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
+set(DATA_DIR ${CMAKE_CURRENT_BINARY_DIR})
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-n-m-topology.xml                ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-topology.xml @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-n-m-pair-topology.xml           ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-pair-topology.xml @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-n-m-env.sh               ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-n-m-env.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m.sh.in          ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-pair.sh.in     ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-dds.sh.in      ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-dds.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-pair-dds.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair-dds.sh @ONLY)
+
+# test
+if(DDS_FOUND)
+  add_test(NAME Example.N-M.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-dds.sh localhost)
+  set_tests_properties(Example.N-M.localhost PROPERTIES TIMEOUT 15 RUN_SERIAL true PASS_REGULAR_EXPRESSION "Example successful")
+  add_test(NAME Example.N-M-pair.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair-dds.sh localhost)
+  set_tests_properties(Example.N-M-pair.localhost PROPERTIES TIMEOUT 15 RUN_SERIAL true PASS_REGULAR_EXPRESSION "Example successful")
+endif()
+
+# install
+install(
+  TARGETS
+  fairmq-ex-n-m-synchronizer
+  fairmq-ex-n-m-sender
+  fairmq-ex-n-m-receiver
+
+  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
+)
+
+# configure run script with different executable paths for build and for install directories
+set(BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
+set(DATA_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_DATADIR})
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-n-m-topology.xml                ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-topology.xml_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-n-m-pair-topology.xml           ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-pair-topology.xml_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-n-m-env.sh               ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-n-m-env.sh_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m.sh.in          ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m.sh_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-pair.sh.in     ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair.sh_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-dds.sh.in      ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-dds.sh_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-n-m-pair-dds.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair-dds.sh_install @ONLY)
+
+install(
+  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-topology.xml_install
+  DESTINATION ${PROJECT_INSTALL_DATADIR}
+  RENAME ex-n-m-topology.xml
+)
+install(
+  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-pair-topology.xml_install
+  DESTINATION ${PROJECT_INSTALL_DATADIR}
+  RENAME ex-n-m-pair-topology.xml
+)
+
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-n-m-env.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-ex-n-m-env.sh
+)
+
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-start-ex-n-m.sh
+)
+
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-start-ex-n-m-pair.sh
+)
+
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-dds.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-start-ex-n-m-dds.sh
+)
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-n-m-pair-dds.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-start-ex-n-m-pair-dds.sh
+)

examples/n-m/Header.h

@@ -0,0 +1,24 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+#ifndef FAIR_MQ_EXAMPLE_N_M_HEADER_H
+#define FAIR_MQ_EXAMPLE_N_M_HEADER_H
+
+#include <cstdint>
+
+namespace example_n_m
+{
+
+struct Header
+{
+    std::uint16_t id;
+    int senderIndex;
+};
+
+} // namespace example_n_m
+
+#endif /* FAIR_MQ_EXAMPLE_N_M_HEADER_H */

examples/n-m/README.md

@@ -0,0 +1,4 @@
+N-M
+==========================
+
+A topology consisting of three layers of devices: synchronizer -> sender(s) -> receiver(s). Senders distribute data to receivers based on the data id contained in the message from the synchronizer (same id goes to the same receiver from every sender). The senders send the data in a non-blocking fashion - if queue is full or receiver is down, data is discarded. Two configurations are provided - one using push/pull channels between senders/receivers, another using pair channels. In push/pull case there is only one receiving channel on the receiver device. In pair case there are as many receiver (sub-)channels as there are senders.

examples/n-m/ex-n-m-pair-topology.xml

@@ -0,0 +1,44 @@
+<topology name="myTopology">
+
+    <var name="numSenders" value="3" />
+    <var name="numReceivers" value="4" />
+
+    <property name="fmqchan_sync" />
+    <property name="fmqchan_data" />
+
+    <decltask name="Synchronizer">
+        <exe reachable="true">fairmq-ex-n-m-synchronizer --id sync --rate 100 --color false -P dds --channel-config name=sync,type=pub,method=bind</exe>
+        <env reachable="false">fairmq-ex-n-m-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_sync</id>
+        </properties>
+    </decltask>
+
+    <decltask name="Sender">
+        <exe reachable="true">fairmq-ex-n-m-sender --id sender%taskIndex% --timeframe-size 100000 --num-receivers ${numReceivers} --color false -P dds --channel-config name=sync,type=sub,method=connect name=data,type=pair,method=connect,numSockets=${numReceivers} --dds-i data:%taskIndex%</exe>
+        <env reachable="false">fairmq-ex-n-m-env.sh</env>
+        <properties>
+            <name access="read">fmqchan_sync</id>
+            <name access="read">fmqchan_data</id>
+        </properties>
+    </decltask>
+
+    <decltask name="Receiver">
+        <exe reachable="true">fairmq-ex-n-m-receiver --id receiver%taskIndex% --num-senders ${numSenders} --color false -P dds --max-timeframes 10 --channel-config name=data,type=pair,method=bind,numSockets=${numSenders}</exe>
+        <env reachable="false">fairmq-ex-n-m-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_data</id>
+        </properties>
+    </decltask>
+
+    <main name="main">
+        <task>Synchronizer</task>
+        <group name="Senders" n="${numSenders}">
+            <task>Sender</task>
+        </group>
+        <group name="Receivers" n="${numReceivers}">
+            <task>Receiver</task>
+        </group>
+    </main>
+
+</topology>

examples/n-m/ex-n-m-topology.xml

@@ -0,0 +1,44 @@
+<topology name="myTopology">
+
+    <var name="numSenders" value="3" />
+    <var name="numReceivers" value="4" />
+
+    <property name="fmqchan_sync" />
+    <property name="fmqchan_data" />
+
+    <decltask name="Synchronizer">
+        <exe reachable="true">fairmq-ex-n-m-synchronizer --id sync --rate 100 --color false -P dds --channel-config name=sync,type=pub,method=bind</exe>
+        <env reachable="false">fairmq-ex-n-m-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_sync</id>
+        </properties>
+    </decltask>
+
+    <decltask name="Sender">
+        <exe reachable="true">fairmq-ex-n-m-sender --id sender%taskIndex% --timeframe-size 100000 --num-receivers ${numReceivers} --color false -P dds --channel-config name=sync,type=sub,method=connect name=data,type=push,method=connect,numSockets=${numReceivers}</exe>
+        <env reachable="false">fairmq-ex-n-m-env.sh</env>
+        <properties>
+            <name access="read">fmqchan_sync</id>
+            <name access="read">fmqchan_data</id>
+        </properties>
+    </decltask>
+
+    <decltask name="Receiver">
+        <exe reachable="true">fairmq-ex-n-m-receiver --id receiver%taskIndex% --num-senders ${numSenders} --color false -P dds --max-timeframes 10 --channel-config name=data,type=pull,method=bind</exe>
+        <env reachable="false">fairmq-ex-n-m-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_data</id>
+        </properties>
+    </decltask>
+
+    <main name="main">
+        <task>Synchronizer</task>
+        <group name="Senders" n="${numSenders}">
+            <task>Sender</task>
+        </group>
+        <group name="Receivers" n="${numReceivers}">
+            <task>Receiver</task>
+        </group>
+    </main>
+
+</topology>

examples/n-m/fairmq-ex-n-m-env.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+export PATH=@BIN_DIR@:$PATH
+
+OS=$(uname -s 2>&1)
+if [ "$OS" == "Darwin" ]; then
+   export DYLD_LIBRARY_PATH=@LIB_DIR@:$DYLD_LIBRARY_PATH
+fi

examples/n-m/fairmq-start-ex-n-m-dds.sh.in

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+set -e
+
+cleanup() {
+  dds-session stop $1
+  echo "CLEANUP PERFORMED"
+}
+
+source @DDS_INSTALL_PREFIX@/DDS_env.sh
+export PATH=@BIN_DIR@:$PATH
+
+exec 5>&1
+output=$(dds-session start | tee >(cat - >&5))
+export DDS_SESSION_ID=$(echo ${output} | grep "DDS session ID: " | cut -d' ' -f4)
+echo "SESSION ID: ${DDS_SESSION_ID}"
+
+trap "cleanup ${DDS_SESSION_ID}" EXIT
+
+requiredNofSlots=8
+dds-submit -r localhost --slots ${requiredNofSlots}
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
+
+export FAIRMQ_DDS_TOPO_FILE=@DATA_DIR@/ex-n-m-topology.xml
+echo "TOPOLOGY FILE: ${FAIRMQ_DDS_TOPO_FILE}"
+echo "TOPOLOGY NAME: $(dds-topology --disable-validation --topology-name ${FAIRMQ_DDS_TOPO_FILE})"
+
+dds-info --active-topology
+dds-topology --activate ${FAIRMQ_DDS_TOPO_FILE}
+dds-info --active-topology
+echo "...waiting for ${requiredNofSlots} executing slots..."
+dds-info --executing-count --wait ${requiredNofSlots}
+
+echo "------------------------"
+echo "...waiting for Topology to finish..."
+# TODO Retrieve number of devices from DDS topology API instead of having the user pass it explicitely
+fairmq-dds-command-ui -w "IDLE"
+fairmq-dds-command-ui -c i
+fairmq-dds-command-ui -c k
+fairmq-dds-command-ui -c b
+fairmq-dds-command-ui -c x
+fairmq-dds-command-ui -c j
+fairmq-dds-command-ui -c r
+receivers="main/Receivers.*"
+fairmq-dds-command-ui -w "RUNNING->READY" -p $receivers
+echo "All receivers transitioned from RUNNING to READY"
+fairmq-dds-command-ui -c s
+fairmq-dds-command-ui -c t
+fairmq-dds-command-ui -c d
+fairmq-dds-command-ui -c q
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
+echo "------------------------"
+
+dds-info --active-topology
+dds-topology --stop
+dds-info --active-topology
+
+dds-agent-cmd getlog -a
+logDir="${wrkDir}/logs"
+for file in $(find "${logDir}" -name "*.tar.gz"); do tar -xf ${file} -C "${logDir}" ; done
+echo "AGENT LOG FILES IN: ${logDir}"
+
+# This string is used by ctest to detect success
+echo "Example successful :)"
+
+# Cleanup function is called by EXIT trap

examples/n-m/fairmq-start-ex-n-m-pair-dds.sh.in

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+set -e
+
+cleanup() {
+  dds-session stop $1
+  echo "CLEANUP PERFORMED"
+}
+
+source @DDS_INSTALL_PREFIX@/DDS_env.sh
+export PATH=@BIN_DIR@:$PATH
+
+exec 5>&1
+output=$(dds-session start | tee >(cat - >&5))
+export DDS_SESSION_ID=$(echo ${output} | grep "DDS session ID: " | cut -d' ' -f4)
+echo "SESSION ID: ${DDS_SESSION_ID}"
+
+trap "cleanup ${DDS_SESSION_ID}" EXIT
+
+requiredNofSlots=8
+dds-submit -r localhost --slots ${requiredNofSlots}
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
+
+export FAIRMQ_DDS_TOPO_FILE=@DATA_DIR@/ex-n-m-pair-topology.xml
+echo "TOPOLOGY FILE: ${FAIRMQ_DDS_TOPO_FILE}"
+echo "TOPOLOGY NAME: $(dds-topology --disable-validation --topology-name ${FAIRMQ_DDS_TOPO_FILE})"
+
+dds-info --active-topology
+dds-topology --activate ${FAIRMQ_DDS_TOPO_FILE}
+dds-info --active-topology
+echo "...waiting for ${requiredNofSlots} executing slots..."
+dds-info --executing-count --wait ${requiredNofSlots}
+
+echo "------------------------"
+echo "...waiting for Topology to finish..."
+# TODO Retrieve number of devices from DDS topology API instead of having the user pass it explicitely
+fairmq-dds-command-ui -w "IDLE"
+fairmq-dds-command-ui -c i
+fairmq-dds-command-ui -c k
+fairmq-dds-command-ui -c b
+fairmq-dds-command-ui -c x
+fairmq-dds-command-ui -c j
+fairmq-dds-command-ui -c r
+receivers="main/Receivers.*"
+fairmq-dds-command-ui -w "RUNNING->READY" -p $receivers
+echo "All receivers transitioned from RUNNING to READY"
+fairmq-dds-command-ui -c s
+fairmq-dds-command-ui -c t
+fairmq-dds-command-ui -c d
+fairmq-dds-command-ui -c q
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
+echo "------------------------"
+
+dds-info --active-topology
+dds-topology --stop
+dds-info --active-topology
+
+dds-agent-cmd getlog -a
+logDir="${wrkDir}/logs"
+for file in $(find "${logDir}" -name "*.tar.gz"); do tar -xf ${file} -C "${logDir}" ; done
+echo "AGENT LOG FILES IN: ${logDir}"
+
+# This string is used by ctest to detect success
+echo "Example successful :)"
+
+# Cleanup function is called by EXIT trap

examples/n-m/fairmq-start-ex-n-m-pair.sh.in

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+export PATH=@BIN_DIR@:$PATH
+
+SYNC="fairmq-ex-n-m-synchronizer"
+SYNC+=" --id Sync"
+SYNC+=" --channel-config name=sync,type=pub,method=bind,address=tcp://localhost:8010"
+SYNC+=" --rate 100"
+xterm -geometry 80x25+0+0 -hold -e $SYNC &
+
+SENDER0="fairmq-ex-n-m-sender"
+SENDER0+=" --id Sender1"
+SENDER0+=" --channel-config name=sync,type=sub,method=connect,address=tcp://localhost:8010"
+SENDER0+="                  name=data,type=pair,method=connect,address=tcp://localhost:8021,address=tcp://localhost:8022,address=tcp://localhost:8023,address=tcp://localhost:8024"
+SENDER0+=" --sender-index 0"
+SENDER0+=" --subtimeframe-size 1000000"
+SENDER0+=" --num-receivers 4"
+xterm -geometry 80x25+500+0 -hold -e $SENDER0 &
+
+SENDER1="fairmq-ex-n-m-sender"
+SENDER1+=" --id Sender2"
+SENDER1+=" --channel-config name=sync,type=sub,method=connect,address=tcp://localhost:8010"
+SENDER1+="                  name=data,type=pair,method=connect,address=tcp://localhost:8031,address=tcp://localhost:8032,address=tcp://localhost:8033,address=tcp://localhost:8034"
+SENDER1+=" --sender-index 1"
+SENDER1+=" --subtimeframe-size 1000000"
+SENDER1+=" --num-receivers 4"
+xterm -geometry 80x25+500+350 -hold -e $SENDER1 &
+
+SENDER2="fairmq-ex-n-m-sender"
+SENDER2+=" --id Sender3"
+SENDER2+=" --channel-config name=sync,type=sub,method=connect,address=tcp://localhost:8010"
+SENDER2+="                  name=data,type=pair,method=connect,address=tcp://localhost:8041,address=tcp://localhost:8042,address=tcp://localhost:8043,address=tcp://localhost:8044"
+SENDER2+=" --sender-index 2"
+SENDER2+=" --subtimeframe-size 1000000"
+SENDER2+=" --num-receivers 4"
+xterm -geometry 80x25+500+700 -hold -e $SENDER2 &
+
+RECEIVER0="fairmq-ex-n-m-receiver"
+RECEIVER0+=" --id Receiver1"
+RECEIVER0+=" --channel-config name=data,type=pair,method=bind,address=tcp://localhost:8021,address=tcp://localhost:8031,address=tcp://localhost:8041"
+RECEIVER0+=" --num-senders 3"
+xterm -geometry 80x25+1000+0 -hold -e $RECEIVER0 &
+
+RECEIVER1="fairmq-ex-n-m-receiver"
+RECEIVER1+=" --id Receiver2"
+RECEIVER1+=" --channel-config name=data,type=pair,method=bind,address=tcp://localhost:8022,address=tcp://localhost:8032,address=tcp://localhost:8042"
+RECEIVER1+=" --num-senders 3"
+xterm -geometry 80x25+1000+350 -hold -e $RECEIVER1 &
+
+RECEIVER2="fairmq-ex-n-m-receiver"
+RECEIVER2+=" --id Receiver3"
+RECEIVER2+=" --channel-config name=data,type=pair,method=bind,address=tcp://localhost:8023,address=tcp://localhost:8033,address=tcp://localhost:8043"
+RECEIVER2+=" --num-senders 3"
+xterm -geometry 80x25+1000+700 -hold -e $RECEIVER2 &
+
+RECEIVER3="fairmq-ex-n-m-receiver"
+RECEIVER3+=" --id Receiver4"
+RECEIVER3+=" --channel-config name=data,type=pair,method=bind,address=tcp://localhost:8024,address=tcp://localhost:8034,address=tcp://localhost:8044"
+RECEIVER3+=" --num-senders 3"
+xterm -geometry 80x25+1000+1050 -hold -e $RECEIVER3 &

examples/n-m/fairmq-start-ex-n-m.sh.in

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+export PATH=@BIN_DIR@:$PATH
+
+SYNC="fairmq-ex-n-m-synchronizer"
+SYNC+=" --id Sync"
+SYNC+=" --channel-config name=sync,type=pub,method=bind,address=tcp://localhost:8010"
+SYNC+=" --rate 100"
+xterm -geometry 80x25+0+0 -hold -e $SYNC &
+
+SENDER0="fairmq-ex-n-m-sender"
+SENDER0+=" --id Sender1"
+SENDER0+=" --channel-config name=sync,type=sub,method=connect,address=tcp://localhost:8010"
+SENDER0+="                  name=data,type=push,method=connect,address=tcp://localhost:8021,address=tcp://localhost:8022,address=tcp://localhost:8023,address=tcp://localhost:8024"
+SENDER0+=" --sender-index 0"
+SENDER0+=" --subtimeframe-size 1000000"
+SENDER0+=" --num-receivers 4"
+xterm -geometry 80x25+500+0 -hold -e $SENDER0 &
+
+SENDER1="fairmq-ex-n-m-sender"
+SENDER1+=" --id Sender2"
+SENDER1+=" --channel-config name=sync,type=sub,method=connect,address=tcp://localhost:8010"
+SENDER1+="                  name=data,type=push,method=connect,address=tcp://localhost:8021,address=tcp://localhost:8022,address=tcp://localhost:8023,address=tcp://localhost:8024"
+SENDER1+=" --sender-index 1"
+SENDER1+=" --subtimeframe-size 1000000"
+SENDER1+=" --num-receivers 4"
+xterm -geometry 80x25+500+350 -hold -e $SENDER1 &
+
+SENDER2="fairmq-ex-n-m-sender"
+SENDER2+=" --id Sender3"
+SENDER2+=" --channel-config name=sync,type=sub,method=connect,address=tcp://localhost:8010"
+SENDER2+="                  name=data,type=push,method=connect,address=tcp://localhost:8021,address=tcp://localhost:8022,address=tcp://localhost:8023,address=tcp://localhost:8024"
+SENDER2+=" --sender-index 2"
+SENDER2+=" --subtimeframe-size 1000000"
+SENDER2+=" --num-receivers 4"
+xterm -geometry 80x25+500+700 -hold -e $SENDER2 &
+
+RECEIVER0="fairmq-ex-n-m-receiver"
+RECEIVER0+=" --id Receiver1"
+RECEIVER0+=" --channel-config name=data,type=pull,method=bind,address=tcp://localhost:8021"
+RECEIVER0+=" --num-senders 3"
+xterm -geometry 80x25+1000+0 -hold -e $RECEIVER0 &
+
+RECEIVER1="fairmq-ex-n-m-receiver"
+RECEIVER1+=" --id Receiver2"
+RECEIVER1+=" --channel-config name=data,type=pull,method=bind,address=tcp://localhost:8022"
+RECEIVER1+=" --num-senders 3"
+xterm -geometry 80x25+1000+350 -hold -e $RECEIVER1 &
+
+RECEIVER2="fairmq-ex-n-m-receiver"
+RECEIVER2+=" --id Receiver3"
+RECEIVER2+=" --channel-config name=data,type=pull,method=bind,address=tcp://localhost:8023"
+RECEIVER2+=" --num-senders 3"
+xterm -geometry 80x25+1000+700 -hold -e $RECEIVER2 &
+
+RECEIVER3="fairmq-ex-n-m-receiver"
+RECEIVER3+=" --id Receiver4"
+RECEIVER3+=" --channel-config name=data,type=pull,method=bind,address=tcp://localhost:8024"
+RECEIVER3+=" --num-senders 3"
+xterm -geometry 80x25+1000+1050 -hold -e $RECEIVER3 &

examples/n-m/runReceiver.cxx

@@ -0,0 +1,119 @@
+/********************************************************************************
+ *    Copyright (C) 2020 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "Header.h"
+
+#include <FairMQDevice.h>
+#include <runFairMQDevice.h>
+
+#include <string>
+#include <iomanip>
+#include <chrono>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+using namespace std;
+using namespace example_n_m;
+namespace bpo = boost::program_options;
+
+struct TFBuffer
+{
+    FairMQParts parts;
+    chrono::steady_clock::time_point start;
+    chrono::steady_clock::time_point end;
+};
+
+class Receiver : public FairMQDevice
+{
+  public:
+    Receiver()
+        : fBuffer()
+        , fDiscardedSet()
+        , fNumSenders(0)
+        , fBufferTimeoutInMs(5000)
+        , fMaxTimeframes(0)
+        , fTimeframeCounter(0)
+    {
+        OnData("data", &Receiver::HandleData);
+    }
+
+    ~Receiver() = default;
+
+    void InitTask() override
+    {
+        fNumSenders = GetConfig()->GetValue<int>("num-senders");
+        fBufferTimeoutInMs = GetConfig()->GetValue<int>("buffer-timeout");
+        fMaxTimeframes = GetConfig()->GetValue<int>("max-timeframes");
+    }
+
+  protected:
+    bool HandleData(FairMQParts& parts, int /* index */)
+    {
+        Header& h = *(static_cast<Header*>(parts.At(0)->GetData()));
+        // LOG(info) << "Received sub-time frame #" << h.id << " from Sender" << h.senderIndex;
+
+        if (fDiscardedSet.find(h.id) == fDiscardedSet.end()) {
+            if (fBuffer.find(h.id) == fBuffer.end()) {
+                // if this is the first part with this ID, save the receive time.
+                fBuffer[h.id].start = chrono::steady_clock::now();
+            }
+            // if the received ID has not previously been discarded, store the data part in the buffer
+            fBuffer[h.id].parts.AddPart(move(parts.At(1)));
+        } else {
+            // if received ID has been previously discarded.
+            LOG(debug) << "Received part from an already discarded timeframe with id " << h.id;
+        }
+
+        if (fBuffer[h.id].parts.Size() == fNumSenders) {
+            LOG(info) << "Successfully completed timeframe #" << h.id;
+            fBuffer.erase(h.id);
+
+            if (fMaxTimeframes > 0 && ++fTimeframeCounter >= fMaxTimeframes) {
+                LOG(info) << "Reached configured maximum number of timeframes (" << fMaxTimeframes << "). Exiting RUNNING state.";
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    void DiscardIncompleteTimeframes()
+    {
+        auto it = fBuffer.begin();
+        while (it != fBuffer.end()) {
+            if (chrono::duration_cast<chrono::milliseconds>(chrono::steady_clock::now() - (it->second).start).count() > fBufferTimeoutInMs) {
+                LOG(debug) << "Timeframe #" << it->first << " incomplete after " << fBufferTimeoutInMs << " milliseconds, discarding";
+                fDiscardedSet.insert(it->first);
+                fBuffer.erase(it++);
+                LOG(debug) << "Number of discarded timeframes: " << fDiscardedSet.size();
+            } else {
+                // LOG(info) << "Timeframe #" << it->first << " within timeout, buffering...";
+                ++it;
+            }
+        }
+    }
+
+    unordered_map<uint16_t, TFBuffer> fBuffer;
+    unordered_set<uint16_t> fDiscardedSet;
+
+    int fNumSenders;
+    int fBufferTimeoutInMs;
+    int fMaxTimeframes;
+    int fTimeframeCounter;
+};
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()
+        ("buffer-timeout", bpo::value<int>()->default_value(1000), "Buffer timeout in milliseconds")
+        ("num-senders", bpo::value<int>()->required(), "Number of senders")
+        ("max-timeframes", bpo::value<int>()->default_value(0), "Maximum number of timeframes to receive (0 - unlimited)");
+}
+
+FairMQDevice* getDevice(const FairMQProgOptions& /* config */) { return new Receiver(); }

examples/n-m/runSender.cxx

@@ -0,0 +1,79 @@
+/********************************************************************************
+ *    Copyright (C) 2020 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "Header.h"
+
+#include <FairMQDevice.h>
+#include <runFairMQDevice.h>
+
+#include <string>
+
+using namespace std;
+using namespace example_n_m;
+namespace bpo = boost::program_options;
+
+class Sender : public FairMQDevice
+{
+  public:
+    Sender()
+        : fNumReceivers(0)
+        , fIndex(0)
+        , fSubtimeframeSize(10000)
+    {}
+
+    ~Sender() = default;
+
+  protected:
+    void InitTask() override
+    {
+        fIndex = GetConfig()->GetProperty<int>("sender-index");
+        fSubtimeframeSize = GetConfig()->GetProperty<int>("subtimeframe-size");
+        fNumReceivers = GetConfig()->GetProperty<int>("num-receivers");
+    }
+
+    void Run() override
+    {
+        FairMQChannel& dataInChannel = fChannels.at("sync").at(0);
+
+        while (!NewStatePending()) {
+            Header h;
+            FairMQMessagePtr id(NewMessage());
+            if (dataInChannel.Receive(id) > 0) {
+                h.id = *(static_cast<uint16_t*>(id->GetData()));
+                h.senderIndex = fIndex;
+            } else {
+                continue;
+            }
+
+            FairMQParts parts;
+            parts.AddPart(NewSimpleMessage(h));
+            parts.AddPart(NewMessage(fSubtimeframeSize));
+
+            uint64_t currentDataId = h.id;
+            int direction = currentDataId % fNumReceivers;
+
+            if (Send(parts, "data", direction, 0) < 0) {
+                LOG(debug) << "Failed to queue Subtimeframe #" << currentDataId << " to Receiver[" << direction << "]";
+            }
+        }
+    }
+
+  private:
+    int fNumReceivers;
+    unsigned int fIndex;
+    int fSubtimeframeSize;
+};
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()
+        ("sender-index", bpo::value<int>()->default_value(0), "Sender Index")
+        ("subtimeframe-size", bpo::value<int>()->default_value(1000), "Subtimeframe size in bytes")
+        ("num-receivers", bpo::value<int>()->required(), "Number of EPNs");
+}
+FairMQDevice* getDevice(const FairMQProgOptions& /* config */) { return new Sender(); }

examples/n-m/runSynchronizer.cxx

@@ -0,0 +1,46 @@
+/********************************************************************************
+ *    Copyright (C) 2020 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include <FairMQDevice.h>
+#include <runFairMQDevice.h>
+
+#include <string>
+#include <cstdint>
+
+using namespace std;
+namespace bpo = boost::program_options;
+
+class Synchronizer : public FairMQDevice
+{
+  public:
+    Synchronizer()
+        : fTimeframeId(0)
+    {}
+    ~Synchronizer() = default;
+
+  protected:
+    bool ConditionalRun() override
+    {
+        FairMQMessagePtr msg(NewSimpleMessage(fTimeframeId));
+
+        if (Send(msg, "sync") > 0) {
+            if (++fTimeframeId == UINT16_MAX - 1) {
+                fTimeframeId = 0;
+            }
+        } else {
+            return false;
+        }
+
+        return true;
+    }
+
+    uint16_t fTimeframeId;
+};
+
+void addCustomOptions(bpo::options_description& /* options */) {}
+FairMQDevice* getDevice(const FairMQProgOptions& /* config */) { return new Synchronizer(); }

examples/qc/CMakeLists.txt

@@ -0,0 +1,71 @@
+################################################################################
+#    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+add_executable(fairmq-ex-qc-sampler runSampler.cxx)
+target_link_libraries(fairmq-ex-qc-sampler PRIVATE FairMQ)
+
+add_executable(fairmq-ex-qc-dispatcher runQCDispatcher.cxx)
+target_link_libraries(fairmq-ex-qc-dispatcher PRIVATE FairMQ)
+
+add_executable(fairmq-ex-qc-task runQCTask.cxx)
+target_link_libraries(fairmq-ex-qc-task PRIVATE FairMQ)
+
+add_executable(fairmq-ex-qc-sink runSink.cxx)
+target_link_libraries(fairmq-ex-qc-sink PRIVATE FairMQ)
+
+add_custom_target(ExampleQC DEPENDS fairmq-ex-qc-sampler fairmq-ex-qc-dispatcher fairmq-ex-qc-task fairmq-ex-qc-sink)
+
+list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
+set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
+set(DATA_DIR ${CMAKE_CURRENT_BINARY_DIR})
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-qc-topology.xml ${CMAKE_CURRENT_BINARY_DIR}/ex-qc-topology.xml @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-qc-env.sh ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-qc-env.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-qc.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-qc.sh @ONLY)
+
+# test
+if(DDS_FOUND)
+  add_test(NAME Example.QC.localhost COMMAND ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-qc.sh localhost)
+  set_tests_properties(Example.QC.localhost PROPERTIES TIMEOUT 15 RUN_SERIAL true PASS_REGULAR_EXPRESSION "Example successful")
+endif()
+
+# install
+install(
+  TARGETS
+  fairmq-ex-qc-sampler
+  fairmq-ex-qc-dispatcher
+  fairmq-ex-qc-task
+  fairmq-ex-qc-sink
+
+  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
+)
+
+# configure run script with different executable paths for build and for install directories
+set(BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
+set(DATA_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_DATADIR})
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-qc-topology.xml ${CMAKE_CURRENT_BINARY_DIR}/ex-qc-topology.xml_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-qc-env.sh ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-qc-env.sh_install @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-qc.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-qc.sh_install @ONLY)
+
+install(
+  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-qc-topology.xml_install
+  DESTINATION ${PROJECT_INSTALL_DATADIR}
+  RENAME ex-qc-topology.xml
+)
+
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-qc-env.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-ex-qc-env.sh
+)
+
+install(
+  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-qc.sh_install
+  DESTINATION ${PROJECT_INSTALL_BINDIR}
+  RENAME fairmq-start-ex-qc.sh
+)

examples/qc/README.md

@@ -0,0 +1,4 @@
+QC
+==
+
+A topology consisting of 4 devices - Sampler, QCDispatcher, QCTask and Sink. The data flows from Sampler through QCDispatcher to Sink. On demand - by setting the corresponding configuration property - the QCDispatcher device will duplicate the data to the QCTask device. The property is set by the topology controller, in this example this is the `fairmq-dds-command-ui` utility.

examples/qc/ex-qc-topology.xml

@@ -0,0 +1,48 @@
+<topology name="ExampleQC">
+
+    <property name="fmqchan_data1" />
+    <property name="fmqchan_data2" />
+    <property name="fmqchan_qc" />
+
+    <decltask name="Sampler">
+        <exe>fairmq-ex-qc-sampler --color false --channel-config name=data1,type=push,method=bind -P dds --max-iterations 1000</exe>
+        <env reachable="false">fairmq-ex-qc-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_data1</name>
+        </properties>
+    </decltask>
+
+    <decltask name="QCDispatcher">
+        <exe>fairmq-ex-qc-dispatcher --color false --channel-config name=data1,type=pull,method=connect name=data2,type=push,method=connect name=qc,type=push,method=connect -P dds</exe>
+        <env reachable="false">fairmq-ex-qc-env.sh</env>
+        <properties>
+            <name access="read">fmqchan_data1</name>
+            <name access="read">fmqchan_data2</name>
+            <name access="read">fmqchan_qc</name>
+        </properties>
+    </decltask>
+
+    <decltask name="QCTask">
+        <exe>fairmq-ex-qc-task --color false --channel-config name=qc,type=pull,method=bind -P dds</exe>
+        <env reachable="false">fairmq-ex-qc-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_qc</name>
+        </properties>
+    </decltask>
+
+    <decltask name="Sink">
+        <exe>fairmq-ex-qc-sink --color false --channel-config name=data2,type=pull,method=bind -P dds --max-iterations 1000</exe>
+        <env reachable="false">fairmq-ex-qc-env.sh</env>
+        <properties>
+            <name access="write">fmqchan_data2</name>
+        </properties>
+    </decltask>
+
+    <main name="main">
+        <task>Sampler</task>
+        <task>QCDispatcher</task>
+        <task>QCTask</task>
+        <task>Sink</task>
+    </main>
+
+</topology>

examples/qc/fairmq-ex-qc-env.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+export PATH=@BIN_DIR@:$PATH
+
+OS=$(uname -s 2>&1)
+if [ "$OS" == "Darwin" ]; then
+   export DYLD_LIBRARY_PATH=@LIB_DIR@:$DYLD_LIBRARY_PATH
+fi

examples/qc/fairmq-start-ex-qc.sh.in

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+# fairmq-start-ex-qc.sh -> submit agents with localhost plugin
+
+set -e
+
+cleanup() {
+  dds-session stop $1
+  echo "CLEANUP PERFORMED"
+}
+
+source @DDS_INSTALL_PREFIX@/DDS_env.sh
+export PATH=@BIN_DIR@:$PATH
+
+exec 5>&1
+output=$(dds-session start | tee >(cat - >&5))
+export DDS_SESSION_ID=$(echo ${output} | grep "DDS session ID: " | cut -d' ' -f4)
+echo "SESSION ID: ${DDS_SESSION_ID}"
+
+trap "cleanup ${DDS_SESSION_ID}" EXIT
+
+requiredNofSlots=4
+dds-submit -r localhost --slots ${requiredNofSlots}
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
+
+export FAIRMQ_DDS_TOPO_FILE=@DATA_DIR@/ex-qc-topology.xml
+echo "TOPOLOGY FILE: ${FAIRMQ_DDS_TOPO_FILE}"
+echo "TOPOLOGY NAME: $(dds-topology --disable-validation --topology-name ${FAIRMQ_DDS_TOPO_FILE})"
+
+dds-info --active-topology
+dds-topology --activate ${FAIRMQ_DDS_TOPO_FILE}
+dds-info --active-topology
+echo "...waiting for ${requiredNofSlots} executing slots..."
+dds-info --executing-count --wait ${requiredNofSlots}
+
+echo "------------------------"
+echo "...waiting for Topology to finish..."
+# TODO Retrieve number of devices from DDS topology API instead of having the user pass it explicitely
+fairmq-dds-command-ui -w "IDLE"
+fairmq-dds-command-ui -c i
+fairmq-dds-command-ui -c k
+fairmq-dds-command-ui -c b
+fairmq-dds-command-ui -c x
+fairmq-dds-command-ui -c j
+allexceptqctasks="main/(Sampler|QCDispatcher|Sink)"
+fairmq-dds-command-ui -c r -p $allexceptqctasks
+qctask="main/QCTask.*"
+qcdispatcher="main/QCDispatcher.*"
+fairmq-dds-command-ui -c p --property-key qc --property-value active -p $qcdispatcher
+fairmq-dds-command-ui -c r -p $qctask
+fairmq-dds-command-ui -w "RUNNING->READY" -p $qctask
+echo "...$qctask received data and transitioned to READY, sending shutdown..."
+fairmq-dds-command-ui -c s
+fairmq-dds-command-ui -c t
+fairmq-dds-command-ui -c d
+fairmq-dds-command-ui -c q
+echo "...waiting for ${requiredNofSlots} idle slots..."
+dds-info --idle-count --wait ${requiredNofSlots}
+echo "------------------------"
+
+dds-info --active-topology
+dds-topology --stop
+dds-info --active-topology
+
+dds-agent-cmd getlog -a
+logDir="${wrkDir}/logs"
+for file in $(find "${logDir}" -name "*.tar.gz"); do tar -xf ${file} -C "${logDir}" ; done
+echo "AGENT LOG FILES IN: ${logDir}"
+
+# This string is used by ctest to detect success
+echo "Example successful :)"
+
+# Cleanup function is called by EXIT trap

examples/qc/runQCDispatcher.cxx

@@ -0,0 +1,59 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "FairMQDevice.h"
+
+class QCDispatcher : public FairMQDevice
+{
+  public:
+    QCDispatcher()
+        : fDoQC(false)
+    {
+        OnData("data1", &QCDispatcher::HandleData);
+    }
+
+    void InitTask() override
+    {
+        GetConfig()->Subscribe<std::string>("qcdevice", [&](const std::string& key, std::string value) {
+            if (key == "qc") {
+                if (value == "active") {
+                    fDoQC.store(true);
+                } else if (value == "inactive") {
+                    fDoQC.store(false);
+                }
+            }
+        });
+    }
+
+  protected:
+    bool HandleData(FairMQMessagePtr& msg, int)
+    {
+        if (fDoQC.load() == true) {
+            FairMQMessagePtr msgCopy(NewMessage());
+            msgCopy->Copy(*msg);
+            if (Send(msg, "qc") < 0) {
+                return false;
+            }
+        }
+
+        if (Send(msg, "data2") < 0) {
+            return false;
+        }
+
+        return true;
+    }
+
+    void ResetTask() override { GetConfig()->Unsubscribe<std::string>("qcdevice"); }
+
+  private:
+    std::atomic<bool> fDoQC;
+};
+
+void addCustomOptions(boost::program_options::options_description& /*options*/) {}
+FairMQDevicePtr getDevice(const fair::mq::ProgOptions& /*config*/) { return new QCDispatcher(); }

examples/qc/runQCTask.cxx

@@ -0,0 +1,26 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "FairMQDevice.h"
+
+class QCTask : public FairMQDevice
+{
+  public:
+    QCTask()
+    {
+        OnData("qc", [](FairMQMessagePtr& /*msg*/, int) {
+            LOG(info) << "received data";
+            return false;
+        });
+    }
+};
+
+namespace bpo = boost::program_options;
+void addCustomOptions(bpo::options_description& /*options*/) {}
+FairMQDevicePtr getDevice(const fair::mq::ProgOptions& /*config*/) { return new QCTask(); }

examples/qc/runSampler.cxx

@@ -0,0 +1,36 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "FairMQDevice.h"
+
+#include <thread> // this_thread::sleep_for
+#include <chrono>
+
+class Sampler : public FairMQDevice
+{
+  public:
+    Sampler() {}
+
+  protected:
+    virtual bool ConditionalRun()
+    {
+        FairMQMessagePtr msg(NewMessage(1000));
+
+        if (Send(msg, "data1") < 0) {
+            return false;
+        }
+
+        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+        return true;
+    }
+};
+
+namespace bpo = boost::program_options;
+void addCustomOptions(bpo::options_description&) {}
+FairMQDevicePtr getDevice(const fair::mq::ProgOptions& /*config*/) { return new Sampler(); }

examples/qc/runSink.cxx

@@ -0,0 +1,25 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "FairMQDevice.h"
+
+#include <string>
+
+class Sink : public FairMQDevice
+{
+  public:
+    Sink() { OnData("data2", &Sink::HandleData); }
+
+  protected:
+    bool HandleData(FairMQMessagePtr& /*msg*/, int /*index*/) { return true; }
+};
+
+namespace bpo = boost::program_options;
+void addCustomOptions(bpo::options_description&) {}
+FairMQDevicePtr getDevice(const fair::mq::ProgOptions& /*config*/) { return new Sink(); }

examples/region/Sampler.cxx

@@ -35,6 +35,14 @@ void Sampler::InitTask()
     fMsgSize = fConfig->GetProperty<int>("msg-size");
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
 
+    fChannels.at("data").at(0).Transport()->SubscribeToRegionEvents([](FairMQRegionInfo info) {
+        LOG(warn) << ">>>" << info.event;
+        LOG(warn) << "id: " << info.id;
+        LOG(warn) << "ptr: " << info.ptr;
+        LOG(warn) << "size: " << info.size;
+        LOG(warn) << "flags: " << info.flags;
+    });
+
     fRegion = FairMQUnmanagedRegionPtr(NewUnmanagedRegionFor("data",
                                                              0,
                                                              10000000,
@@ -82,6 +90,7 @@ void Sampler::ResetTask()
         LOG(debug) << "done, still unacked: " << fNumUnackedMsgs;
     }
     fRegion.reset();
+    fChannels.at("data").at(0).Transport()->UnsubscribeFromRegionEvents();
 }
 
 Sampler::~Sampler()

examples/region/Sink.cxx

@@ -29,6 +29,13 @@ void Sink::InitTask()
 {
     // Get the fMaxIterations value from the command line options (via fConfig)
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
+    fChannels.at("data").at(0).Transport()->SubscribeToRegionEvents([](FairMQRegionInfo info) {
+        LOG(warn) << ">>>" << info.event;
+        LOG(warn) << "id: " << info.id;
+        LOG(warn) << "ptr: " << info.ptr;
+        LOG(warn) << "size: " << info.size;
+        LOG(warn) << "flags: " << info.flags;
+    });
 }
 
 void Sink::Run()
@@ -50,6 +57,11 @@ void Sink::Run()
     }
 }
 
+void Sink::ResetTask()
+{
+    fChannels.at("data").at(0).Transport()->UnsubscribeFromRegionEvents();
+}
+
 Sink::~Sink()
 {
 }

examples/region/Sink.h

@@ -31,6 +31,7 @@ class Sink : public FairMQDevice
   protected:
     virtual void Run();
     virtual void InitTask();
+    virtual void ResetTask();
 
   private:
     uint64_t fMaxIterations;

examples/region/test-ex-region.sh.in

@@ -19,6 +19,7 @@ SAMPLER+=" --id sampler1"
 SAMPLER+=" --transport $transport"
 SAMPLER+=" --severity debug"
 SAMPLER+=" --session $SESSION"
+SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --control static --color false"
 SAMPLER+=" --max-iterations 1"
 SAMPLER+=" --msg-size $msgSize"

fairmq/CMakeLists.txt

@@ -6,16 +6,6 @@
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
-####################
-# external plugins #
-####################
-if(BUILD_DDS_PLUGIN)
-  add_subdirectory(plugins/DDS)
-endif()
-if(BUILD_PMIX_PLUGIN)
-  add_subdirectory(plugins/PMIx)
-endif()
-
 if(BUILD_FAIRMQ OR BUILD_SDK)
   ###########
   # Version #
@@ -192,11 +182,11 @@ if(BUILD_FAIRMQ)
     plugins/Builtin.h
     plugins/config/Config.h
     plugins/Control.h
-    shmem/FairMQMessageSHM.h
-    shmem/FairMQPollerSHM.h
-    shmem/FairMQUnmanagedRegionSHM.h
-    shmem/FairMQSocketSHM.h
-    shmem/FairMQTransportFactorySHM.h
+    shmem/Message.h
+    shmem/Poller.h
+    shmem/UnmanagedRegion.h
+    shmem/Socket.h
+    shmem/TransportFactory.h
     shmem/Common.h
     shmem/Manager.h
     shmem/Region.h
@@ -253,11 +243,10 @@ if(BUILD_FAIRMQ)
     SuboptParser.cxx
     plugins/config/Config.cxx
     plugins/Control.cxx
-    shmem/FairMQMessageSHM.cxx
-    shmem/FairMQPollerSHM.cxx
-    shmem/FairMQUnmanagedRegionSHM.cxx
-    shmem/FairMQSocketSHM.cxx
-    shmem/FairMQTransportFactorySHM.cxx
+    shmem/Message.cxx
+    shmem/Poller.cxx
+    shmem/Socket.cxx
+    shmem/TransportFactory.cxx
     shmem/Manager.cxx
     shmem/Region.cxx
     zeromq/FairMQMessageZMQ.cxx
@@ -473,3 +462,13 @@ endif()
 if(BUILD_SDK)
   add_subdirectory(sdk)
 endif()
+
+####################
+# external plugins #
+####################
+if(BUILD_DDS_PLUGIN)
+  add_subdirectory(plugins/DDS)
+endif()
+if(BUILD_PMIX_PLUGIN)
+  add_subdirectory(plugins/PMIx)
+endif()

fairmq/DeviceRunner.cxx

@@ -75,7 +75,7 @@ void DeviceRunner::SubscribeForConfigChange()
             fair::Logger::SetConsoleColor(val);
         }
     });
-    fConfig.Subscribe<string>("device-runner", [&](const std::string& key, const std::string val) {
+    fConfig.Subscribe<string>("device-runner", [&](const std::string& key, const std::string& val) {
         if (key == "severity") {
             fair::Logger::SetConsoleSeverity(val);
         } else if (key == "file-severity") {

fairmq/FairMQChannel.cxx

@@ -20,7 +20,7 @@ using namespace std;
 using namespace fair::mq;
 
 template<typename T>
-T GetPropertyOrDefault(const fair::mq::Properties& m, const string& k, const T& ifNotFound) noexcept
+T GetPropertyOrDefault(const fair::mq::Properties& m, const string& k, const T& ifNotFound)
 {
     if (m.count(k)) {
         return boost::any_cast<T>(m.at(k));
@@ -133,26 +133,37 @@ FairMQChannel::FairMQChannel(const FairMQChannel& chan, const string& newName)
 
 FairMQChannel& FairMQChannel::operator=(const FairMQChannel& chan)
 {
-    fTransportFactory = nullptr;
-    fTransportType = chan.fTransportType;
-    fSocket = nullptr;
-    fName = chan.fName;
-    fType = chan.fType;
-    fMethod = chan.fMethod;
-    fAddress = chan.fAddress;
-    fSndBufSize = chan.fSndBufSize;
-    fRcvBufSize = chan.fRcvBufSize;
-    fSndKernelSize = chan.fSndKernelSize;
-    fRcvKernelSize = chan.fRcvKernelSize;
-    fLinger = chan.fLinger;
-    fRateLogging = chan.fRateLogging;
-    fPortRangeMin = chan.fPortRangeMin;
-    fPortRangeMax = chan.fPortRangeMax;
-    fAutoBind = chan.fAutoBind;
-    fIsValid = false;
-    fMultipart = chan.fMultipart;
-    fModified = chan.fModified;
-    fReset = false;
+    if (this == &chan) {
+        return *this;
+    }
+
+    {
+        // TODO: replace this with std::scoped_lock (c++17)
+        lock(fMtx, chan.fMtx);
+        lock_guard<mutex> lock1(fMtx, adopt_lock);
+        lock_guard<mutex> lock2(chan.fMtx, adopt_lock);
+
+        fTransportFactory = nullptr;
+        fTransportType = chan.fTransportType;
+        fSocket = nullptr;
+        fName = chan.fName;
+        fType = chan.fType;
+        fMethod = chan.fMethod;
+        fAddress = chan.fAddress;
+        fSndBufSize = chan.fSndBufSize;
+        fRcvBufSize = chan.fRcvBufSize;
+        fSndKernelSize = chan.fSndKernelSize;
+        fRcvKernelSize = chan.fRcvKernelSize;
+        fLinger = chan.fLinger;
+        fRateLogging = chan.fRateLogging;
+        fPortRangeMin = chan.fPortRangeMin;
+        fPortRangeMax = chan.fPortRangeMax;
+        fAutoBind = chan.fAutoBind;
+        fIsValid = false;
+        fMultipart = chan.fMultipart;
+        fModified = chan.fModified;
+        fReset = false;
+    }
 
     return *this;
 }

fairmq/FairMQChannel.h

@@ -340,6 +340,11 @@ class FairMQChannel
         return Transport()->CreateUnmanagedRegion(size, callback, path, flags);
     }
 
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0)
+    {
+        return Transport()->CreateUnmanagedRegion(size, userFlags, callback, path, flags);
+    }
+
     static constexpr fair::mq::Transport DefaultTransportType = fair::mq::Transport::DEFAULT;
     static constexpr const char* DefaultTransportName = "default";
     static constexpr const char* DefaultName = "";

fairmq/FairMQDevice.cxx

@@ -299,10 +299,15 @@ void FairMQDevice::InitWrapper()
                 // if binding address is not specified, try getting it from the configured network interface
                 if (subChannel.fAddress == "unspecified" || subChannel.fAddress == "") {
                     // if the configured network interface is default, get its name from the default route
-                    if (networkInterface == "default") {
-                        networkInterface = tools::getDefaultRouteNetworkInterface();
+                    try {
+                        if (networkInterface == "default") {
+                            networkInterface = tools::getDefaultRouteNetworkInterface();
+                        }
+                        subChannel.fAddress = "tcp://" + tools::getInterfaceIP(networkInterface) + ":1";
+                    } catch(const tools::DefaultRouteDetectionError& e) {
+                        LOG(debug) << "binding on tcp://*:1";
+                        subChannel.fAddress = "tcp://*:1";
                     }
-                    subChannel.fAddress = "tcp://" + tools::getInterfaceIP(networkInterface) + ":1";
                 }
                 // fill the uninitialized list
                 fUninitializedBindingChannels.push_back(&subChannel);

fairmq/FairMQDevice.h

@@ -217,17 +217,47 @@ class FairMQDevice
     }
 
     // creates unamanaged region with the default device transport
-    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr)
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size,
+                                                FairMQRegionCallback callback = nullptr,
+                                                const std::string& path = "",
+                                                int flags = 0)
     {
-        return Transport()->CreateUnmanagedRegion(size, callback);
+        return Transport()->CreateUnmanagedRegion(size, callback, path, flags);
+    }
+
+    // creates unamanaged region with the default device transport
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size,
+                                                const int64_t userFlags,
+                                                FairMQRegionCallback callback = nullptr,
+                                                const std::string& path = "",
+                                                int flags = 0)
+    {
+        return Transport()->CreateUnmanagedRegion(size, userFlags, callback, path, flags);
     }
 
     // creates unmanaged region with the transport of the specified channel
-    FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel, int index, const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0)
+    FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel,
+                                                   int index,
+                                                   const size_t size,
+                                                   FairMQRegionCallback callback = nullptr,
+                                                   const std::string& path = "",
+                                                   int flags = 0)
     {
         return GetChannel(channel, index).NewUnmanagedRegion(size, callback, path, flags);
     }
 
+    // creates unmanaged region with the transport of the specified channel
+    FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel,
+                                                   int index,
+                                                   const size_t size,
+                                                   const int64_t userFlags,
+                                                   FairMQRegionCallback callback = nullptr,
+                                                   const std::string& path = "",
+                                                   int flags = 0)
+    {
+        return GetChannel(channel, index).NewUnmanagedRegion(size, userFlags, callback, path, flags);
+    }
+
     template<typename ...Ts>
     FairMQPollerPtr NewPoller(const Ts&... inputs)
     {

fairmq/FairMQPoller.h

@@ -31,8 +31,8 @@ namespace fair
 namespace mq
 {
 
-using PollerPtr = std::unique_ptr<FairMQPoller>;
-
+using Poller = FairMQPoller;
+using PollerPtr = FairMQPollerPtr;
 struct PollerError : std::runtime_error { using std::runtime_error::runtime_error; };
 
 } /* namespace mq */

fairmq/FairMQSocket.h

@@ -22,7 +22,7 @@ class FairMQSocket
     FairMQSocket() {}
     FairMQSocket(FairMQTransportFactory* fac): fTransport(fac) {}
 
-    virtual std::string GetId() = 0;
+    virtual std::string GetId() const = 0;
 
     virtual bool Bind(const std::string& address) = 0;
     virtual bool Connect(const std::string& address) = 0;

fairmq/FairMQTransportFactory.cxx

@@ -7,8 +7,8 @@
  ********************************************************************************/
 
 #include <FairMQTransportFactory.h>
+#include <fairmq/shmem/TransportFactory.h>
 #include <zeromq/FairMQTransportFactoryZMQ.h>
-#include <shmem/FairMQTransportFactorySHM.h>
 #ifdef BUILD_NANOMSG_TRANSPORT
 #include <nanomsg/FairMQTransportFactoryNN.h>
 #endif /* BUILD_NANOMSG_TRANSPORT */
@@ -17,50 +17,46 @@
 #endif
 #include <FairMQLogger.h>
 #include <fairmq/tools/Unique.h>
-
 #include <memory>
 #include <string>
 
-FairMQTransportFactory::FairMQTransportFactory(const std::string& id)
+using namespace std;
+
+FairMQTransportFactory::FairMQTransportFactory(const string& id)
     : fkId(id)
-{
-}
+{}
 
-auto FairMQTransportFactory::CreateTransportFactory(const std::string& type, const std::string& id, const fair::mq::ProgOptions* config) -> std::shared_ptr<FairMQTransportFactory>
+auto FairMQTransportFactory::CreateTransportFactory(const string& type,
+                                                    const string& id,
+                                                    const fair::mq::ProgOptions* config)
+    -> shared_ptr<FairMQTransportFactory>
 {
-    using namespace std;
-
     auto finalId = id;
 
     // Generate uuid if empty
-    if (finalId == "")
-    {
+    if (finalId == "") {
         finalId = fair::mq::tools::Uuid();
     }
 
-    if (type == "zeromq")
-    {
+    if (type == "zeromq") {
         return make_shared<FairMQTransportFactoryZMQ>(finalId, config);
-    }
-    else if (type == "shmem")
-    {
-        return make_shared<FairMQTransportFactorySHM>(finalId, config);
+    } else if (type == "shmem") {
+        return make_shared<fair::mq::shmem::TransportFactory>(finalId, config);
     }
 #ifdef BUILD_NANOMSG_TRANSPORT
-    else if (type == "nanomsg")
-    {
+    else if (type == "nanomsg") {
         return make_shared<FairMQTransportFactoryNN>(finalId, config);
     }
 #endif /* BUILD_NANOMSG_TRANSPORT */
 #ifdef BUILD_OFI_TRANSPORT
-    else if (type == "ofi")
-    {
+    else if (type == "ofi") {
         return make_shared<fair::mq::ofi::TransportFactory>(finalId, config);
     }
 #endif /* BUILD_OFI_TRANSPORT */
-    else
-    {
-        LOG(error) << "Unavailable transport requested: " << "\"" << type << "\"" << ". Available are: "
+    else {
+        LOG(error) << "Unavailable transport requested: "
+                   << "\"" << type << "\""
+                   << ". Available are: "
                    << "\"zeromq\""
                    << "\"shmem\""
 #ifdef BUILD_NANOMSG_TRANSPORT

fairmq/FairMQTransportFactory.h

@@ -61,20 +61,46 @@ class FairMQTransportFactory
     /// @param obj optional helper pointer that can be used in the callback
     /// @return pointer to FairMQMessage
     virtual FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) = 0;
-
+    /// @brief create a message with the buffer located within the corresponding unmanaged region
+    /// @param unmanagedRegion the unmanaged region that this message buffer belongs to
+    /// @param data message buffer (must be within the region - checked at runtime by the transport)
+    /// @param size size of the message
+    /// @param hint optional parameter, returned to the user in the FairMQRegionCallback
     virtual FairMQMessagePtr CreateMessage(FairMQUnmanagedRegionPtr& unmanagedRegion, void* data, const size_t size, void* hint = 0) = 0;
 
-    /// Create a socket
+    /// @brief Create a socket
     virtual FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name) = 0;
 
-    /// Create a poller for a single channel (all subchannels)
+    /// @brief Create a poller for a single channel (all subchannels)
     virtual FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const = 0;
-    /// Create a poller for specific channels
+    /// @brief Create a poller for specific channels
     virtual FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel*>& channels) const = 0;
-    /// Create a poller for specific channels (all subchannels)
+    /// @brief Create a poller for specific channels (all subchannels)
     virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const = 0;
 
+    /// @brief Create new UnmanagedRegion
+    /// @param size size of the region
+    /// @param callback callback to be called when a message belonging to this region is no longer needed by the transport
+    /// @param path optional parameter to pass to the underlying transport
+    /// @param flags optional parameter to pass to the underlying transport
+    /// @return pointer to UnmanagedRegion
     virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const = 0;
+    /// @brief Create new UnmanagedRegion
+    /// @param size size of the region
+    /// @param userFlags flags to be stored with the region, have no effect on the transport, but can be retrieved from the region by the user
+    /// @param callback callback to be called when a message belonging to this region is no longer needed by the transport
+    /// @param path optional parameter to pass to the underlying transport
+    /// @param flags optional parameter to pass to the underlying transport
+    /// @return pointer to UnmanagedRegion
+    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const = 0;
+
+    /// @brief Subscribe to region events (creation, destruction, ...)
+    /// @param callback the callback that is called when a region event occurs
+    virtual void SubscribeToRegionEvents(FairMQRegionEventCallback callback) = 0;
+    /// @brief Unsubscribe from region events
+    virtual void UnsubscribeFromRegionEvents() = 0;
+
+    virtual std::vector<FairMQRegionInfo> GetRegionInfo() = 0;
 
     /// Get transport type
     virtual fair::mq::Transport GetType() const = 0;
@@ -137,6 +163,7 @@ namespace fair
 namespace mq
 {
 
+using TransportFactory = FairMQTransportFactory;
 struct TransportFactoryError : std::runtime_error { using std::runtime_error::runtime_error; };
 
 } /* namespace mq */

fairmq/FairMQUnmanagedRegion.h

@@ -12,8 +12,24 @@
 #include <cstddef> // size_t
 #include <memory> // std::unique_ptr
 #include <functional> // std::function
+#include <ostream> // std::ostream
+
+enum class FairMQRegionEvent : int
+{
+    created,
+    destroyed
+};
+
+struct FairMQRegionInfo {
+    uint64_t id;   // id of the region
+    void* ptr;     // pointer to the start of the region
+    size_t size;   // region size
+    int64_t flags; // custom flags set by the creator
+    FairMQRegionEvent event;
+};
 
 using FairMQRegionCallback = std::function<void(void*, size_t, void*)>;
+using FairMQRegionEventCallback = std::function<void(FairMQRegionInfo)>;
 
 class FairMQUnmanagedRegion
 {
@@ -26,12 +42,26 @@ class FairMQUnmanagedRegion
 
 using FairMQUnmanagedRegionPtr = std::unique_ptr<FairMQUnmanagedRegion>;
 
+inline std::ostream& operator<<(std::ostream& os, const FairMQRegionEvent& event)
+{
+    if (event == FairMQRegionEvent::created) {
+        return os << "created";
+    } else {
+        return os << "destroyed";
+    }
+}
+
 namespace fair
 {
 namespace mq
 {
 
-using UnmanagedRegionPtr = std::unique_ptr<FairMQUnmanagedRegion>;
+using RegionCallback = FairMQRegionCallback;
+using RegionEventCallback = FairMQRegionEventCallback;
+using RegionEvent = FairMQRegionEvent;
+using RegionInfo = FairMQRegionInfo;
+using UnmanagedRegion = FairMQUnmanagedRegion;
+using UnmanagedRegionPtr = FairMQUnmanagedRegionPtr;
 
 } /* namespace mq */
 } /* namespace fair */

fairmq/MemoryResourceTools.h

@@ -60,7 +60,7 @@ FairMQMessagePtr getMessage(ContainerT &&container_, FairMQMemoryResource *targe
         container.data(),
         containerSizeBytes);
     return message;
-};
+}
 
 } /* namespace mq */
 } /* namespace fair */

fairmq/MemoryResources.cxx

@@ -18,5 +18,5 @@
 void *fair::mq::ChannelResource::do_allocate(std::size_t bytes, std::size_t /*alignment*/)
 {
     return setMessage(factory->CreateMessage(bytes));
-};
+}
 

fairmq/Plugin.cxx

@@ -1,5 +1,5 @@
 /********************************************************************************
- *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ * Copyright (C) 2017-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -8,15 +8,20 @@
 
 #include <fairmq/Plugin.h>
 #include <FairMQLogger.h>
+#include <utility>
 
 using namespace std;
 
-fair::mq::Plugin::Plugin(const string name, const Version version, const string maintainer, const string homepage, PluginServices* pluginServices)
-    : fkName{name}
+fair::mq::Plugin::Plugin(string name,
+                         Version version,
+                         string maintainer,
+                         string homepage,
+                         PluginServices* pluginServices)
+    : fkName(std::move(name))
     , fkVersion(version)
-    , fkMaintainer{maintainer}
-    , fkHomepage{homepage}
-    , fPluginServices{pluginServices}
+    , fkMaintainer(std::move(maintainer))
+    , fkHomepage(std::move(homepage))
+    , fPluginServices(pluginServices)
 {
     LOG(debug) << "Loaded plugin: " << *this;
 }

fairmq/Plugin.h

@@ -1,5 +1,5 @@
 /********************************************************************************
- *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ * Copyright (C) 2017-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -44,7 +44,11 @@ class Plugin
     using Version = tools::Version;
 
     Plugin() = delete;
-    Plugin(const std::string name, const Version version, const std::string maintainer, const std::string homepage, PluginServices* pluginServices);
+    Plugin(std::string name,
+           Version version,
+           std::string maintainer,
+           std::string homepage,
+           PluginServices* pluginServices);
 
     Plugin(const Plugin&) = delete;
     Plugin operator=(const Plugin&) = delete;

fairmq/PluginManager.cxx

@@ -28,6 +28,8 @@ using boost::optional;
 
 const std::string fair::mq::PluginManager::fgkLibPrefix = "FairMQPlugin_";
 
+std::vector<boost::dll::shared_library> fair::mq::PluginManager::fgDLLKeepAlive = std::vector<boost::dll::shared_library>();
+
 fair::mq::PluginManager::PluginManager()
     : fSearchPaths{}
     , fPluginFactories()

fairmq/PluginManager.h

@@ -80,7 +80,7 @@ class PluginManager
     auto ForEachPluginProgOptions(std::function<void (boost::program_options::options_description)> func) const -> void { for(const auto& pair : fPluginProgOptions) { func(pair.second); } }
 
     template<typename... Args>
-    auto EmplacePluginServices(Args&&... args) -> void { fPluginServices = fair::mq::tools::make_unique<PluginServices>(std::forward<Args>(args)...); };
+    auto EmplacePluginServices(Args&&... args) -> void { fPluginServices = fair::mq::tools::make_unique<PluginServices>(std::forward<Args>(args)...); }
 
     auto WaitForPluginsToReleaseDeviceControl() -> void { fPluginServices->WaitForReleaseDeviceControl(); }
 
@@ -97,6 +97,7 @@ class PluginManager
         using fair::mq::tools::ToString;
 
         auto lib = shared_library{std::forward<Args>(args)...};
+        fgDLLKeepAlive.push_back(lib);
 
         fPluginFactories[pluginName] = import_alias<PluginFactory>(
             shared_library{lib},
@@ -117,6 +118,7 @@ class PluginManager
 
     static const std::string fgkLibPrefix;
     std::vector<boost::filesystem::path> fSearchPaths;
+    static std::vector<boost::dll::shared_library> fgDLLKeepAlive;
     std::map<std::string, std::function<PluginFactory>> fPluginFactories;
     std::unique_ptr<PluginServices> fPluginServices;
     std::map<std::string, std::unique_ptr<Plugin>> fPlugins;

fairmq/ProgOptions.cxx

@@ -62,7 +62,7 @@ ValInfo ConvertVarValToValInfo(const po::variable_value& v)
     } catch (out_of_range& oor) {
         return {string("[unidentified_type]"), string("[unidentified_type]"), origin};
     }
-};
+}
 
 string ConvertVarValToString(const po::variable_value& v)
 {
@@ -443,4 +443,4 @@ void ProgOptions::PrintOptionsRaw() const
 
 
 } // namespace mq
-} // namespace fair
+} // namespace fair

fairmq/StateMachine.cxx

@@ -79,7 +79,9 @@ struct Machine_ : public state_machine_def<Machine_>
 {
   public:
     Machine_()
-        : fLastTransitionResult(true)
+        : fState(State::Ok)
+        , fNewState(State::Ok)
+        , fLastTransitionResult(true)
         , fNewStatePending(false)
     {}
 

fairmq/Version.h.in

@@ -1,5 +1,5 @@
 /********************************************************************************
- *    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ * Copyright (C) 2018-2020 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -21,7 +21,7 @@
 #define FAIRMQ_GIT_DATE "@PROJECT_GIT_DATE@"
 #define FAIRMQ_REPO_URL "https://github.com/FairRootGroup/FairMQ"
 #define FAIRMQ_LICENSE "LGPL-3.0"
-#define FAIRMQ_COPYRIGHT "2012-2019 GSI"
+#define FAIRMQ_COPYRIGHT "2012-2020 GSI"
 #define FAIRMQ_BUILD_TYPE "@CMAKE_BUILD_TYPE@"
 
 #endif // FAIR_MQ_VERSION_H

fairmq/nanomsg/FairMQSocketNN.cxx

@@ -91,11 +91,6 @@ FairMQSocketNN::FairMQSocketNN(const string& type, const string& name, const str
     LOG(debug) << "Created socket " << GetId();
 }
 
-string FairMQSocketNN::GetId()
-{
-    return fId;
-}
-
 bool FairMQSocketNN::Bind(const string& address)
 {
     // LOG(info) << "bind socket " << fId << " on " << address;

fairmq/nanomsg/FairMQSocketNN.h

@@ -23,7 +23,7 @@ class FairMQSocketNN final : public FairMQSocket
     FairMQSocketNN(const FairMQSocketNN&) = delete;
     FairMQSocketNN operator=(const FairMQSocketNN&) = delete;
 
-    std::string GetId() override;
+    std::string GetId() const override { return fId; }
 
     bool Bind(const std::string& address) override;
     bool Connect(const std::string& address) override;

fairmq/nanomsg/FairMQTransportFactoryNN.cxx

@@ -70,6 +70,11 @@ FairMQUnmanagedRegionPtr FairMQTransportFactoryNN::CreateUnmanagedRegion(const s
     return unique_ptr<FairMQUnmanagedRegion>(new FairMQUnmanagedRegionNN(size, callback, path, flags));
 }
 
+FairMQUnmanagedRegionPtr FairMQTransportFactoryNN::CreateUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */) const
+{
+    return unique_ptr<FairMQUnmanagedRegion>(new FairMQUnmanagedRegionNN(size, userFlags, callback, path, flags));
+}
+
 fair::mq::Transport FairMQTransportFactoryNN::GetType() const
 {
     return fTransportType;

fairmq/nanomsg/FairMQTransportFactoryNN.h

@@ -37,6 +37,11 @@ class FairMQTransportFactoryNN final : public FairMQTransportFactory
     FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const override;
 
     FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback, const std::string& path = "", int flags = 0) const override;
+    FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const override;
+
+    void SubscribeToRegionEvents(FairMQRegionEventCallback /* callback */) override { LOG(error) << "SubscribeToRegionEvents not yet implemented for nanomsg"; }
+    void UnsubscribeFromRegionEvents() override { LOG(error) << "UnsubscribeFromRegionEvents not yet implemented for nanomsg"; }
+    std::vector<FairMQRegionInfo> GetRegionInfo() override { LOG(error) << "GetRegionInfo not yet implemented for nanomsg, returning empty vector"; return std::vector<FairMQRegionInfo>(); }
 
     fair::mq::Transport GetType() const override;
 

fairmq/nanomsg/FairMQUnmanagedRegionNN.cxx

@@ -18,6 +18,13 @@ FairMQUnmanagedRegionNN::FairMQUnmanagedRegionNN(const size_t size, FairMQRegion
 {
 }
 
+FairMQUnmanagedRegionNN::FairMQUnmanagedRegionNN(const size_t size, const int64_t /*userFlags*/, FairMQRegionCallback callback, const std::string& /*path = "" */, int /*flags = 0 */)
+    : fBuffer(malloc(size))
+    , fSize(size)
+    , fCallback(callback)
+{
+}
+
 void* FairMQUnmanagedRegionNN::GetData() const
 {
     return fBuffer;

fairmq/nanomsg/FairMQUnmanagedRegionNN.h

@@ -20,6 +20,8 @@ class FairMQUnmanagedRegionNN final : public FairMQUnmanagedRegion
 
   public:
     FairMQUnmanagedRegionNN(const size_t size, FairMQRegionCallback callback, const std::string& path = "", int flags = 0);
+    FairMQUnmanagedRegionNN(const size_t size, const int64_t userFlags, FairMQRegionCallback callback, const std::string& path = "", int flags = 0);
+
     FairMQUnmanagedRegionNN(const FairMQUnmanagedRegionNN&) = delete;
     FairMQUnmanagedRegionNN operator=(const FairMQUnmanagedRegionNN&) = delete;
 
@@ -34,4 +36,4 @@ class FairMQUnmanagedRegionNN final : public FairMQUnmanagedRegion
     FairMQRegionCallback fCallback;
 };
 
-#endif /* FAIRMQUNMANAGEDREGIONNN_H_ */
+#endif /* FAIRMQUNMANAGEDREGIONNN_H_ */

fairmq/ofi/Socket.h

@@ -43,7 +43,7 @@ class Socket final : public fair::mq::Socket
     Socket(const Socket&) = delete;
     Socket operator=(const Socket&) = delete;
 
-    auto GetId() -> std::string { return fId; }
+    auto GetId() const -> std::string override { return fId; }
 
     auto Bind(const std::string& address) -> bool override;
     auto Connect(const std::string& address) -> bool override;

fairmq/ofi/TransportFactory.cxx

@@ -90,6 +90,11 @@ auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, FairMQRegion
     throw runtime_error{"Not yet implemented UMR."};
 }
 
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, const int64_t /*userFlags*/, FairMQRegionCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) const -> UnmanagedRegionPtr
+{
+    throw runtime_error{"Not yet implemented UMR."};
+}
+
 auto TransportFactory::GetType() const -> Transport
 {
     return Transport::OFI;

fairmq/ofi/TransportFactory.h

@@ -46,7 +46,12 @@ class TransportFactory final : public FairMQTransportFactory
     auto CreatePoller(const std::vector<FairMQChannel*>& channels) const -> PollerPtr override;
     auto CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const -> PollerPtr override;
 
-    auto CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const -> UnmanagedRegionPtr override;
+    auto CreateUnmanagedRegion(const size_t size, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const -> UnmanagedRegionPtr override;
+    auto CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const -> UnmanagedRegionPtr override;
+
+    void SubscribeToRegionEvents(RegionEventCallback /* callback */) override { LOG(error) << "SubscribeToRegionEvents not yet implemented for OFI"; }
+    void UnsubscribeFromRegionEvents() override { LOG(error) << "UnsubscribeFromRegionEvents not yet implemented for OFI"; }
+    std::vector<FairMQRegionInfo> GetRegionInfo() override { LOG(error) << "GetRegionInfo not yet implemented for OFI, returning empty vector"; return std::vector<FairMQRegionInfo>(); }
 
     auto GetType() const -> Transport override;
 

fairmq/plugins/DDS/CMakeLists.txt

@@ -19,7 +19,7 @@ set_target_properties(${plugin} PROPERTIES
 
 set(exe fairmq-dds-command-ui)
 add_executable(${exe} ${CMAKE_CURRENT_SOURCE_DIR}/runDDSCommandUI.cxx)
-target_link_libraries(${exe} FairMQ Commands StateMachine DDS::dds_intercom_lib DDS::dds_protocol_lib)
+target_link_libraries(${exe} FairMQ Commands SDK StateMachine)
 target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 
 install(TARGETS ${plugin} ${exe}

fairmq/plugins/DDS/DDS.cxx

@@ -8,8 +8,6 @@
 
 #include "DDS.h"
 
-#include <fairmq/sdk/commands/Commands.h>
-
 #include <fairmq/tools/Strings.h>
 
 #include <boost/algorithm/string/join.hpp>
@@ -37,20 +35,18 @@ DDS::DDS(const string& name,
          const string& homepage,
          PluginServices* pluginServices)
     : Plugin(name, version, maintainer, homepage, pluginServices)
+    , fDDSTaskId(dds::env_prop<dds::task_id>())
     , fCurrentState(DeviceState::Idle)
     , fLastState(DeviceState::Idle)
     , fDeviceTerminationRequested(false)
     , fLastExternalController(0)
     , fExitingAckedByLastExternalController(false)
-    , fHeartbeatInterval(100)
     , fUpdatesAllowed(false)
     , fWorkGuard(fWorkerQueue.get_executor())
 {
     try {
         TakeDeviceControl();
 
-        fHeartbeatThread = thread(&DDS::HeartbeatSender, this);
-
         string deviceId(GetProperty<string>("id"));
         if (deviceId.empty()) {
             SetProperty<string>("id", dds::env_prop<dds::task_path>());
@@ -76,15 +72,22 @@ DDS::DDS(const string& name,
         // subscribe to device state changes, pushing new state changes into the event queue
         SubscribeToDeviceStateChange([&](DeviceState newState) {
             switch (newState) {
-                case DeviceState::Bound:
+                case DeviceState::Bound: {
                     // Receive addresses of connecting channels from DDS
                     // and propagate addresses of bound channels to DDS.
                     FillChannelContainers();
 
+                    // allow updates from key value after channel containers are filled
+                    {
+                        lock_guard<mutex> lk(fUpdateMutex);
+                        fUpdatesAllowed = true;
+                    }
+                    fUpdateCondition.notify_one();
+
                     // publish bound addresses via DDS at keys corresponding to the channel
                     // prefixes, e.g. 'data' in data[i]
                     PublishBoundChannels();
-                    break;
+                } break;
                 case DeviceState::ResettingDevice: {
                     {
                         lock_guard<mutex> lk(fUpdateMutex);
@@ -92,28 +95,41 @@ DDS::DDS(const string& name,
                     }
 
                     EmptyChannelContainers();
-                    break;
-                }
-                case DeviceState::Exiting:
+                } break;
+                case DeviceState::Exiting: {
+                    if (!fControllerThread.joinable()) {
+                        fControllerThread = thread(&DDS::WaitForExitingAck, this);
+                    }
                     fWorkGuard.reset();
                     fDeviceTerminationRequested = true;
                     UnsubscribeFromDeviceStateChange();
                     ReleaseDeviceControl();
-                    break;
+                } break;
                 default:
                     break;
             }
 
-            lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+            using namespace sdk::cmd;
+            auto now = chrono::steady_clock::now();
             string id = GetProperty<string>("id");
             fLastState = fCurrentState;
             fCurrentState = newState;
-            using namespace sdk::cmd;
-            for (auto subscriberId : fStateChangeSubscribers) {
-                LOG(debug) << "Publishing state-change: " << fLastState << "->" << newState << " to " << subscriberId;
 
-                Cmds cmds(make<StateChange>(id, dds::env_prop<dds::task_id>(), fLastState, fCurrentState));
-                fDDS.Send(cmds.Serialize(), to_string(subscriberId));
+            lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+            for (auto it = fStateChangeSubscribers.cbegin(); it != fStateChangeSubscribers.end();) {
+                // if a subscriber did not send a heartbeat in more than 3 times the promised interval,
+                // remove it from the subscriber list
+                if (chrono::duration<double>(now - it->second.first).count() > 3 * it->second.second) {
+                    LOG(warn) << "Controller '" << it->first
+                              << "' did not send heartbeats since over 3 intervals ("
+                              << 3 * it->second.second << " ms), removing it.";
+                    fStateChangeSubscribers.erase(it++);
+                } else {
+                    LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState << " to " << it->first;
+                    Cmds cmds(make<StateChange>(id, fDDSTaskId, fLastState, fCurrentState));
+                    fDDS.Send(cmds.Serialize(), to_string(it->first));
+                    ++it;
+                }
             }
         });
 
@@ -143,10 +159,10 @@ auto DDS::StartWorkerThread() -> void
 auto DDS::WaitForExitingAck() -> void
 {
     unique_lock<mutex> lock(fStateChangeSubscriberMutex);
-    fExitingAcked.wait_for(
-        lock,
-        chrono::milliseconds(GetProperty<unsigned int>("wait-for-exiting-ack-timeout")),
-        [this]() { return fExitingAckedByLastExternalController; });
+    auto timeout = GetProperty<unsigned int>("wait-for-exiting-ack-timeout");
+    fExitingAcked.wait_for(lock, chrono::milliseconds(timeout), [this]() {
+        return fExitingAckedByLastExternalController || fStateChangeSubscribers.empty();
+    });
 }
 
 auto DDS::FillChannelContainers() -> void
@@ -165,9 +181,7 @@ auto DDS::FillChannelContainers() -> void
             } else if (GetProperty<string>(methodKey) == "connect") {
                 fConnectingChans.insert(make_pair(c.first, DDSConfig()));
                 LOG(debug) << "preparing to connect: " << c.first << " with " << c.second << " sub-channels.";
-                for (int i = 0; i < c.second; ++i) {
-                    fConnectingChans.at(c.first).fSubChannelAddresses.push_back(string());
-                }
+                fConnectingChans.at(c.first).fNumSubChannels = c.second;
             } else {
                 LOG(error) << "Cannot update address configuration. Channel method (bind/connect) not specified.";
                 return;
@@ -214,11 +228,6 @@ auto DDS::FillChannelContainers() -> void
             LOG(debug) << "dds-i-n: adding " << chanName << " -> i: " << i << " n: " << n;
             fIofN.insert(make_pair(chanName, IofN(i, n)));
         }
-        {
-            lock_guard<mutex> lk(fUpdateMutex);
-            fUpdatesAllowed = true;
-        }
-        fUpdateCondition.notify_one();
     } catch (const exception& e) {
         LOG(error) << "Error filling channel containers: " << e.what();
     }
@@ -244,6 +253,12 @@ auto DDS::SubscribeForConnectingChannels() -> void
                     unique_lock<mutex> lk(fUpdateMutex);
                     fUpdateCondition.wait(lk, [&]{ return fUpdatesAllowed; });
                 }
+
+                if (fConnectingChans.find(channelName) == fConnectingChans.end()) {
+                    LOG(error) << "Received an update for a connecting channel, but either no channel with given channel name exists or it has already been configured: '" << channelName << "', ignoring...";
+                    return;
+                }
+
                 string val = value;
                 // check if it is to handle as one out of multiple values
                 auto it = fIofN.find(channelName);
@@ -273,19 +288,16 @@ auto DDS::SubscribeForConnectingChannels() -> void
                     fConnectingChans.at(channelName).fDDSValues.insert({senderTaskID, val.c_str()});
                 }
 
-                // update channels and remove them from unfinished container
-                for (auto mi = fConnectingChans.begin(); mi != fConnectingChans.end(); /* no increment */) {
-                    if (mi->second.fSubChannelAddresses.size() == mi->second.fDDSValues.size()) {
-                        // when multiple subChannels are used, their order on every device should be the same, irregardless of arrival order from DDS.
-                        sort(mi->second.fSubChannelAddresses.begin(), mi->second.fSubChannelAddresses.end());
-                        auto it3 = mi->second.fDDSValues.begin();
-                        for (unsigned int i = 0; i < mi->second.fSubChannelAddresses.size(); ++i) {
-                            SetProperty<string>(string{"chans." + mi->first + "." + to_string(i) + ".address"}, it3->second);
-                            ++it3;
+                for (const auto& mi : fConnectingChans) {
+                    if (mi.second.fNumSubChannels == mi.second.fDDSValues.size()) {
+                        int i = 0;
+                        for (const auto& e : mi.second.fDDSValues) {
+                            auto result = UpdateProperty<string>(string{"chans." + mi.first + "." + to_string(i) + ".address"}, e.second);
+                            if (!result) {
+                                LOG(error) << "UpdateProperty failed for: " << "chans." << mi.first << "." << to_string(i) << ".address" << " - property does not exist";
+                            }
+                            ++i;
                         }
-                        fConnectingChans.erase(mi++);
-                    } else {
-                        ++mi;
                     }
                 }
             } catch (const exception& e) {
@@ -304,129 +316,133 @@ auto DDS::PublishBoundChannels() -> void
     }
 }
 
-auto DDS::HeartbeatSender() -> void
-{
-    using namespace sdk::cmd;
-    string id = GetProperty<string>("id");
-
-    while (!fDeviceTerminationRequested) {
-        {
-            lock_guard<mutex> lock{fHeartbeatSubscriberMutex};
-
-            for (const auto subscriberId : fHeartbeatSubscribers) {
-                fDDS.Send(Cmds(make<Heartbeat>(id)).Serialize(), to_string(subscriberId));
-            }
-        }
-
-        this_thread::sleep_for(chrono::milliseconds(fHeartbeatInterval));
-    }
-}
-
 auto DDS::SubscribeForCustomCommands() -> void
 {
-    using namespace sdk::cmd;
     LOG(debug) << "Subscribing for DDS custom commands.";
 
     string id = GetProperty<string>("id");
 
     fDDS.SubscribeCustomCmd([id, this](const string& cmdStr, const string& cond, uint64_t senderId) {
         // LOG(info) << "Received command: '" << cmdStr << "' from " << senderId;
-
-        Cmds inCmds;
+        sdk::cmd::Cmds inCmds;
         inCmds.Deserialize(cmdStr);
-
         for (const auto& cmd : inCmds) {
-            // LOG(info) << "Received command type: '" << cmd->GetType() << "' from " << senderId;
-            switch (cmd->GetType()) {
-                case Type::check_state: {
-                    fDDS.Send(Cmds(make<CurrentState>(id, GetCurrentDeviceState())).Serialize(), to_string(senderId));
-                }
-                break;
-                case Type::change_state: {
-                    Transition transition = static_cast<ChangeState&>(*cmd).GetTransition();
-                    if (ChangeDeviceState(transition)) {
-                        Cmds outCmds(make<TransitionStatus>(id, Result::Ok, transition));
-                        fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                    } else {
-                        Cmds outCmds(make<TransitionStatus>(id, Result::Failure, transition));
-                        fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                    }
-                    {
-                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                        fLastExternalController = senderId;
-                    }
-                }
-                break;
-                case Type::dump_config: {
-                    stringstream ss;
-                    for (const auto pKey: GetPropertyKeys()) {
-                        ss << id << ": " << pKey << " -> " << GetPropertyAsString(pKey) << "\n";
-                    }
-                    Cmds outCmds(make<Config>(id, ss.str()));
-                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                }
-                break;
-                case Type::subscribe_to_heartbeats: {
-                    {
-                        lock_guard<mutex> lock{fHeartbeatSubscriberMutex};
-                        fHeartbeatSubscribers.insert(senderId);
-                    }
-                    Cmds outCmds(make<HeartbeatSubscription>(id, Result::Ok));
-                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                }
-                break;
-                case Type::unsubscribe_from_heartbeats: {
-                    {
-                        lock_guard<mutex> lock{fHeartbeatSubscriberMutex};
-                        fHeartbeatSubscribers.erase(senderId);
-                    }
-                    Cmds outCmds(make<HeartbeatUnsubscription>(id, Result::Ok));
-                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                }
-                break;
-                case Type::state_change_exiting_received: {
-                    {
-                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                        if (fLastExternalController == senderId) {
-                            fExitingAckedByLastExternalController = true;
-                        }
-                    }
-                    fExitingAcked.notify_one();
-                }
-                break;
-                case Type::subscribe_to_state_change: {
-                    lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                    fStateChangeSubscribers.insert(senderId);
-                    if (!fControllerThread.joinable()) {
-                        fControllerThread = thread(&DDS::WaitForExitingAck, this);
-                    }
-
-                    LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState << " to " << senderId;
-
-                    Cmds outCmds(make<StateChangeSubscription>(id, Result::Ok), make<StateChange>(id, dds::env_prop<dds::task_id>(), fLastState, fCurrentState));
-
-                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                }
-                break;
-                case Type::unsubscribe_from_state_change: {
-                    {
-                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                        fStateChangeSubscribers.erase(senderId);
-                    }
-                    Cmds outCmds(make<StateChangeUnsubscription>(id, Result::Ok));
-                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                }
-                break;
-                default:
-                    LOG(warn) << "Unexpected/unknown command received: " << cmdStr;
-                    LOG(warn) << "Origin: " << senderId;
-                    LOG(warn) << "Destination: " << cond;
-                break;
-            }
+            HandleCmd(id, *cmd, cond, senderId);
         }
     });
 }
 
+auto DDS::HandleCmd(const string& id, sdk::cmd::Cmd& cmd, const string& cond, uint64_t senderId) -> void
+{
+    using namespace fair::mq::sdk;
+    using namespace fair::mq::sdk::cmd;
+    // LOG(info) << "Received command type: '" << cmd.GetType() << "' from " << senderId;
+    switch (cmd.GetType()) {
+        case Type::check_state: {
+            fDDS.Send(Cmds(make<CurrentState>(id, GetCurrentDeviceState())).Serialize(), to_string(senderId));
+        } break;
+        case Type::change_state: {
+            Transition transition = static_cast<ChangeState&>(cmd).GetTransition();
+            if (ChangeDeviceState(transition)) {
+                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Ok, transition));
+                fDDS.Send(outCmds.Serialize(), to_string(senderId));
+            } else {
+                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Failure, transition));
+                fDDS.Send(outCmds.Serialize(), to_string(senderId));
+            }
+            {
+                lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                fLastExternalController = senderId;
+            }
+        } break;
+        case Type::dump_config: {
+            stringstream ss;
+            for (const auto pKey : GetPropertyKeys()) {
+                ss << id << ": " << pKey << " -> " << GetPropertyAsString(pKey) << "\n";
+            }
+            Cmds outCmds(make<Config>(id, ss.str()));
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        case Type::state_change_exiting_received: {
+            {
+                lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                if (fLastExternalController == senderId) {
+                    fExitingAckedByLastExternalController = true;
+                }
+            }
+            fExitingAcked.notify_one();
+        } break;
+        case Type::subscribe_to_state_change: {
+            auto _cmd = static_cast<cmd::SubscribeToStateChange&>(cmd);
+            lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+            fStateChangeSubscribers.emplace(senderId, make_pair(chrono::steady_clock::now(), _cmd.GetInterval()));
+
+            LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState << " to " << senderId;
+
+            Cmds outCmds(make<StateChangeSubscription>(id, fDDSTaskId, Result::Ok),
+                         make<StateChange>(id, fDDSTaskId, fLastState, fCurrentState));
+
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        case Type::subscription_heartbeat: {
+            try {
+                auto _cmd = static_cast<cmd::SubscriptionHeartbeat&>(cmd);
+                lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                fStateChangeSubscribers.at(senderId) = make_pair(chrono::steady_clock::now(), _cmd.GetInterval());
+            } catch(out_of_range& oor) {
+                LOG(warn) << "Received subscription heartbeat from an unknown controller with id '" << senderId << "'";
+            }
+        } break;
+        case Type::unsubscribe_from_state_change: {
+            {
+                lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                fStateChangeSubscribers.erase(senderId);
+            }
+            Cmds outCmds(make<StateChangeUnsubscription>(id, fDDSTaskId, Result::Ok));
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        case Type::get_properties: {
+            auto _cmd = static_cast<cmd::GetProperties&>(cmd);
+            auto const request_id(_cmd.GetRequestId());
+            auto result(Result::Ok);
+            vector<pair<string, string>> props;
+            try {
+                for (auto const& prop : GetPropertiesAsString(_cmd.GetQuery())) {
+                    props.push_back({prop.first, prop.second});
+                }
+            } catch (exception const& e) {
+                LOG(warn) << "Getting properties (request id: " << request_id << ") failed: " << e.what();
+                result = Result::Failure;
+            }
+            Cmds const outCmds(make<cmd::Properties>(id, request_id, result, props));
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        case Type::set_properties: {
+            auto _cmd(static_cast<cmd::SetProperties&>(cmd));
+            auto const request_id(_cmd.GetRequestId());
+            auto result(Result::Ok);
+            try {
+                fair::mq::Properties props;
+                for (auto const& prop : _cmd.GetProps()) {
+                    props.insert({prop.first, fair::mq::Property(prop.second)});
+                }
+                // TODO Handle builtin keys with different value type than string
+                SetProperties(props);
+            } catch (exception const& e) {
+                LOG(warn) << "Setting properties (request id: " << request_id << ") failed: " << e.what();
+                result = Result::Failure;
+            }
+            Cmds const outCmds(make<PropertiesSet>(id, request_id, result));
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        default:
+            LOG(warn) << "Unexpected/unknown command received: " << cmd.GetType();
+            LOG(warn) << "Origin: " << senderId;
+            LOG(warn) << "Destination: " << cond;
+        break;
+            }
+}
+
 DDS::~DDS()
 {
     UnsubscribeFromDeviceStateChange();
@@ -436,10 +452,6 @@ DDS::~DDS()
         fControllerThread.join();
     }
 
-    if (fHeartbeatThread.joinable()) {
-        fHeartbeatThread.join();
-    }
-
     fWorkGuard.reset();
     if (fWorkerThread.joinable()) {
         fWorkerThread.join();

fairmq/plugins/DDS/DDS.h

@@ -12,6 +12,7 @@
 #include <fairmq/Plugin.h>
 #include <fairmq/StateQueue.h>
 #include <fairmq/Version.h>
+#include <fairmq/sdk/commands/Commands.h>
 
 #include <dds/dds.h>
 
@@ -23,11 +24,12 @@
 #include <chrono>
 #include <condition_variable>
 #include <mutex>
-#include <set>
 #include <string>
 #include <atomic>
 #include <thread>
+#include <map>
 #include <unordered_map>
+#include <utility> // pair
 #include <vector>
 
 namespace fair
@@ -40,16 +42,16 @@ namespace plugins
 struct DDSConfig
 {
     // container of sub channel addresses
-    std::vector<std::string> fSubChannelAddresses;
+    unsigned int fNumSubChannels;
     // dds values for the channel
-    std::unordered_map<uint64_t, std::string> fDDSValues;
+    std::map<uint64_t, std::string> fDDSValues;
 };
 
 struct DDSSubscription
 {
     DDSSubscription()
-    : fDDSCustomCmd(fService)
-    , fDDSKeyValue(fService)
+        : fDDSCustomCmd(fService)
+        , fDDSKeyValue(fService)
     {
         LOG(debug) << "$DDS_TASK_PATH: " << dds::env_prop<dds::task_path>();
         LOG(debug) << "$DDS_GROUP_NAME: " << dds::env_prop<dds::group_name>();
@@ -141,10 +143,10 @@ class DDS : public Plugin
     auto SubscribeForConnectingChannels() -> void;
     auto PublishBoundChannels() -> void;
     auto SubscribeForCustomCommands() -> void;
-
-    auto HeartbeatSender() -> void;
+    auto HandleCmd(const std::string& id, sdk::cmd::Cmd& cmd, const std::string& cond, uint64_t senderId) -> void;
 
     DDSSubscription fDDS;
+    size_t fDDSTaskId;
 
     std::unordered_map<std::string, std::vector<std::string>> fBindingChans;
     std::unordered_map<std::string, DDSConfig> fConnectingChans;
@@ -157,18 +159,12 @@ class DDS : public Plugin
 
     std::atomic<bool> fDeviceTerminationRequested;
 
-    std::set<uint64_t> fHeartbeatSubscribers;
-    std::mutex fHeartbeatSubscriberMutex;
-
-    std::set<uint64_t> fStateChangeSubscribers;
+    std::unordered_map<uint64_t, std::pair<std::chrono::steady_clock::time_point, int64_t>> fStateChangeSubscribers;
     uint64_t fLastExternalController;
     bool fExitingAckedByLastExternalController;
     std::condition_variable fExitingAcked;
     std::mutex fStateChangeSubscriberMutex;
 
-    std::thread fHeartbeatThread;
-    std::chrono::milliseconds fHeartbeatInterval;
-
     bool fUpdatesAllowed;
     std::mutex fUpdateMutex;
     std::condition_variable fUpdateCondition;

fairmq/plugins/DDS/runDDSCommandUI.cxx

@@ -8,23 +8,23 @@
 
 #include <fairmq/sdk/commands/Commands.h>
 #include <fairmq/States.h>
-
-#include <dds/dds.h>
+#include <fairmq/SDK.h>
 
 #include <boost/program_options.hpp>
 
+#include <termios.h> // raw mode console input
+#include <unistd.h>
 #include <condition_variable>
 #include <cstdlib>
 #include <iostream>
 #include <mutex>
 #include <string>
-#include <termios.h> // raw mode console input
 #include <thread>
 #include <utility>
-#include <unistd.h>
 
 using namespace std;
-using namespace dds::intercom_api;
+using namespace fair::mq;
+using namespace fair::mq::sdk;
 using namespace fair::mq::sdk::cmd;
 namespace bpo = boost::program_options;
 
@@ -49,279 +49,176 @@ struct TerminalConfig
     }
 };
 
-struct StateSubscription
-{
-    const string& fTopologyPath;
-    CCustomCmd& fDdsCustomCmd;
-
-    explicit StateSubscription(const string& topologyPath, CCustomCmd& ddsCustomCmd)
-        : fTopologyPath(topologyPath)
-        , fDdsCustomCmd(ddsCustomCmd)
-    {
-        fDdsCustomCmd.send(Cmds(make<SubscribeToStateChange>()).Serialize(), fTopologyPath);
-    }
-
-    ~StateSubscription() {
-        fDdsCustomCmd.send(Cmds(make<UnsubscribeFromStateChange>()).Serialize(), fTopologyPath);
-        this_thread::sleep_for(chrono::milliseconds(100)); // give dds a chance to complete request
-    }
-};
-
 void printControlsHelp()
 {
     cout << "Use keys to control the devices:" << endl;
-    cout << "[c] check states, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device, [k] complete init, [b] bind, [x] connect" << endl;
+    cout << "[c] check states, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device, [k] complete init, [b] bind, [x] connect, [p] set property" << endl;
     cout << "To quit press Ctrl+C" << endl;
 }
 
-void sendCommand(const string& commandIn, const string& topologyPath, CCustomCmd& ddsCustomCmd)
+void handleCommand(const string& command, const string& path, unsigned int timeout, Topology& topo, const string& pKey, const string& pVal)
 {
+    if (command == "c") {
+        cout << "> checking state of the devices" << endl;
+        auto const result = topo.GetCurrentState();
+        for (const auto& d : result) {
+            cout << d.taskId << " : " << d.state << endl;
+        }
+    } else if (command == "o") {
+        cout << "> dumping config of the devices (" << path << ")" << endl;
+        // TODO: extend this regex to return all properties, once command size limitation is removed.
+        auto const result = topo.GetProperties("^(session|id)$", path, std::chrono::milliseconds(timeout));
+        for (const auto& d : result.second.devices) {
+            for (auto const& p : d.second.props) {
+                cout << d.first << ": " << p.first << " : " << p.second << endl;
+            }
+        }
+    } else if (command == "p") {
+        if (pKey == "" || pVal == "") {
+            cout << "cannot send property with empty key and/or value! given key: '" << pKey << "', value: '" << pVal << "'." << endl;
+            return;
+        }
+        const DeviceProperties props{{pKey, pVal}};
+        cout << "> sending property (" << path << ")" << endl;
+        topo.SetProperties(props, path);
+        // give dds time to complete request
+        this_thread::sleep_for(chrono::milliseconds(100));
+    } else if (command == "i") {
+        cout << "> init devices (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::InitDevice, path, std::chrono::milliseconds(timeout));
+    } else if (command == "k") {
+        cout << "> complete init (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::CompleteInit, path, std::chrono::milliseconds(timeout));
+    } else if (command == "b") {
+        cout << "> bind devices (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::Bind, path, std::chrono::milliseconds(timeout));
+    } else if (command == "x") {
+        cout << "> connect devices (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::Connect, path, std::chrono::milliseconds(timeout));
+    } else if (command == "j") {
+        cout << "> init tasks (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::InitTask, path, std::chrono::milliseconds(timeout));
+    } else if (command == "r") {
+        cout << "> run tasks (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::Run, path, std::chrono::milliseconds(timeout));
+    } else if (command == "s") {
+        cout << "> stop devices (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::Stop, path, std::chrono::milliseconds(timeout));
+    } else if (command == "t") {
+        cout << "> reset tasks (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::ResetTask, path, std::chrono::milliseconds(timeout));
+    } else if (command == "d") {
+        cout << "> reset devices (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::ResetDevice, path, std::chrono::milliseconds(timeout));
+    } else if (command == "q") {
+        cout << "> end (" << path << ")" << endl;
+        topo.ChangeState(TopologyTransition::End, path, std::chrono::milliseconds(timeout));
+    } else if (command == "h") {
+        cout << "> help" << endl;
+        printControlsHelp();
+    } else {
+        cout << "\033[01;32mInvalid input: [" << command << "]\033[0m" << endl;
+        printControlsHelp();
+    }
+}
+
+void sendCommand(const string& commandIn, const string& path, unsigned int timeout, Topology& topo, const string& pKey, const string& pVal)
+{
+    if (commandIn != "") {
+        handleCommand(commandIn, path, timeout, topo, pKey, pVal);
+        return;
+    }
+
     char c;
-    string command(commandIn);
+    string command;
     TerminalConfig tconfig;
 
-    if (command == "") {
-        printControlsHelp();
+    printControlsHelp();
+    cin >> c;
+    command = c;
+
+    while (true) {
+        handleCommand(command, path, timeout, topo, pKey, pVal);
         cin >> c;
         command = c;
     }
-
-    while (true) {
-        if (command == "c") {
-            cout << "> checking state of the devices" << endl;
-            ddsCustomCmd.send(Cmds(make<CheckState>()).Serialize(), topologyPath);
-        } else if (command == "o") {
-            cout << "> dumping config of the devices" << endl;
-            ddsCustomCmd.send(Cmds(make<DumpConfig>()).Serialize(), topologyPath);
-        } else if (command == "i") {
-            cout << "> init devices" << endl;
-            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::InitDevice)).Serialize(), topologyPath);
-        } else if (command == "k") {
-            cout << "> complete init" << endl;
-            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::CompleteInit)).Serialize(), topologyPath);
-        } else if (command == "b") {
-            cout << "> bind devices" << endl;
-            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::Bind)).Serialize(), topologyPath);
-        } else if (command == "x") {
-            cout << "> connect devices" << endl;
-            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::Connect)).Serialize(), topologyPath);
-        } else if (command == "j") {
-            cout << "> init tasks" << endl;
-            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::InitTask)).Serialize(), topologyPath);
-        } else if (command == "r") {
-            cout << "> run tasks" << endl;
-            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::Run)).Serialize(), topologyPath);
-        } else if (command == "s") {
-            cout << "> stop devices" << endl;
-            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::Stop)).Serialize(), topologyPath);
-        } else if (command == "t") {
-            cout << "> reset tasks" << endl;
-            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::ResetTask)).Serialize(), topologyPath);
-        } else if (command == "d") {
-            cout << "> reset devices" << endl;
-            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::ResetDevice)).Serialize(), topologyPath);
-        } else if (command == "h") {
-            cout << "> help" << endl;
-            printControlsHelp();
-        } else if (command == "q") {
-            cout << "> end" << endl;
-            ddsCustomCmd.send(Cmds(make<ChangeState>(fair::mq::Transition::End)).Serialize(), topologyPath);
-        } else {
-            cout << "\033[01;32mInvalid input: [" << c << "]\033[0m" << endl;
-            printControlsHelp();
-        }
-
-        if (commandIn != "") {
-            this_thread::sleep_for(chrono::milliseconds(100)); // give dds a chance to complete request
-            break;
-        } else {
-            cin >> c;
-            command = c;
-        }
-    }
 }
 
-struct WaitMode
-{
-    explicit WaitMode(const string& targetState)
-        : fTransitionedCount(0)
-    {
-        if (targetState != "") {
-            size_t n = targetState.find("->");
-            if (n == string::npos) {
-                fTargetStatePair.first = fair::mq::State::Ok;
-                fTargetStatePair.second = fair::mq::GetState(targetState);
-            } else {
-                fTargetStatePair.first = fair::mq::GetState(targetState.substr(0, n));
-                fTargetStatePair.second = fair::mq::GetState(targetState.substr(n + 2));
-            }
-        }
-    }
-
-    void Run(const chrono::milliseconds& timeout, const string& topologyPath, CCustomCmd& ddsCustomCmd, unsigned int numDevices, const string& command = "")
-    {
-        if (command != "") {
-            sendCommand(command, topologyPath, ddsCustomCmd);
-        }
-
-        // TODO once DDS provides an API to retrieve actual number of tasks, use it here
-        auto condition = [&] {
-            bool res = fTransitionedCount == numDevices;
-            if (fTargetStatePair.first == fair::mq::State::Ok) {
-                cout << "Waiting for " << numDevices << " devices to reach " << fTargetStatePair.second << ", condition check: " << res << endl;
-            } else {
-                cout << "Waiting for " << numDevices << " devices to reach " << fTargetStatePair.first << "->" << fTargetStatePair.second << ", condition check: " << res << endl;
-            }
-            return res;
-        };
-
-        unique_lock<mutex> lock(fMtx);
-
-        if (timeout > chrono::milliseconds(0)) {
-            if (!fCV.wait_for(lock, timeout, condition)) {
-                throw runtime_error("timeout");
-            }
-        } else {
-            fCV.wait(lock, condition);
-        }
-
-        // cout << "WaitMode.Run() finished" << endl;
-    }
-
-    void CountStates(fair::mq::State lastState, fair::mq::State currentState)
-    {
-        {
-            unique_lock<mutex> lock(fMtx);
-            if (fTargetStatePair.first == fair::mq::State::Ok) {
-                if (fTargetStatePair.second == currentState) {
-                    fTransitionedCount++;
-                    // cout << "fTransitionedCount = " << fTransitionedCount << " for single value" << endl;
-                }
-            } else {
-                if (fTargetStatePair.first == lastState && fTargetStatePair.second == currentState) {
-                    fTransitionedCount++;
-                    // cout << "fTransitionedCount = " << fTransitionedCount << " for double value" << endl;
-                }
-            }
-        }
-        fCV.notify_one();
-    }
-
-    mutex fMtx;
-    condition_variable fCV;
-    pair<fair::mq::State, fair::mq::State> fTargetStatePair;
-    unsigned int fTransitionedCount;
-};
-
 int main(int argc, char* argv[])
-{
-    try {
-        string sessionID;
-        string command;
-        string topologyPath;
-        string targetState;
-        unsigned int timeout;
-        unsigned int numDevices(0);
+try {
+    string sessionID;
+    string topoFile;
 
-        bpo::options_description options("Common options");
+    string command;
+    string path;
+    string targetState;
+    string pKey;
+    string pVal;
+    unsigned int timeout;
 
-        auto envSessionId = getenv("DDS_SESSION_ID");
-        if (envSessionId) {
-            options.add_options()("session,s", bpo::value<string>(&sessionID)->default_value(envSessionId), "DDS Session ID (overrides any value in env var $DDS_SESSION_ID)");
-        } else {
-            options.add_options()("session,s", bpo::value<string>(&sessionID)->required(), "DDS Session ID (overrides any value in env var $DDS_SESSION_ID)");
-        }
+    fair::Logger::SetConsoleSeverity("debug");
+    fair::Logger::SetConsoleColor(true);
 
-        options.add_options()
-            ("command,c",        bpo::value<string> (&command)->default_value(""), "Command character")
-            ("path,p",           bpo::value<string> (&topologyPath)->default_value(""), "DDS Topology path to send command to (empty - send to all tasks)")
-            ("wait-for-state,w", bpo::value<string> (&targetState)->default_value(""), "Wait until targeted FairMQ devices reach the given state")
-            ("timeout,t",        bpo::value<unsigned int> (&timeout)->default_value(0), "Timeout in milliseconds when waiting for a device state (0 - wait infinitely)")
-            ("number-devices,n", bpo::value<unsigned int> (&numDevices)->default_value(0), "Number of devices (will be removed in the future)")
-            ("help,h", "Produce help message");
+    bpo::options_description options("Common options");
 
-        bpo::variables_map vm;
-        bpo::store(bpo::command_line_parser(argc, argv).options(options).run(), vm);
-
-        if (vm.count("help")) {
-            cout << "FairMQ DDS Command UI" << endl << options << endl;
-            cout << "Commands: [c] check state, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device, [k] complete init, [b] bind, [x] connect" << endl;
-            return EXIT_SUCCESS;
-        }
-
-        bpo::notify(vm);
-
-        WaitMode waitMode(targetState);
-
-        CIntercomService service;
-        CCustomCmd ddsCustomCmd(service);
-
-        service.subscribeOnError([](const EErrorCode errorCode, const string& errorMsg) {
-            cerr << "DDS error received: error code: " << errorCode << ", error message: " << errorMsg << endl;
-        });
-
-        // subscribe to receive messages from DDS
-        ddsCustomCmd.subscribe([&](const string& msg, const string& /*condition*/, uint64_t senderId) {
-            Cmds cmds;
-            cmds.Deserialize(msg);
-            // cout << "Received " << cmds.Size() << " command(s) with total size of " << msg.length() << " bytes: " << endl;
-            for (const auto& cmd : cmds) {
-                // cout << " > " << cmd->GetType() << endl;
-                switch (cmd->GetType()) {
-                    case Type::state_change: {
-                        cout << "Received state_change from " << static_cast<StateChange&>(*cmd).GetDeviceId() << ": " << static_cast<StateChange&>(*cmd).GetLastState() << "->" << static_cast<StateChange&>(*cmd).GetCurrentState() << endl;
-                        if (static_cast<StateChange&>(*cmd).GetCurrentState() == fair::mq::State::Exiting) {
-                            ddsCustomCmd.send(Cmds(make<StateChangeExitingReceived>()).Serialize(), to_string(senderId));
-                        }
-                        waitMode.CountStates(static_cast<StateChange&>(*cmd).GetLastState(), static_cast<StateChange&>(*cmd).GetCurrentState());
-                    }
-                    break;
-                    case Type::state_change_subscription:
-                        if (static_cast<StateChangeSubscription&>(*cmd).GetResult() != Result::Ok) {
-                            cout << "State change subscription failed for " << static_cast<StateChangeSubscription&>(*cmd).GetDeviceId() << endl;
-                        }
-                    break;
-                    case Type::state_change_unsubscription:
-                        if (static_cast<StateChangeUnsubscription&>(*cmd).GetResult() != Result::Ok) {
-                            cout << "State change unsubscription failed for " << static_cast<StateChangeUnsubscription&>(*cmd).GetDeviceId() << endl;
-                        }
-                    break;
-                    case Type::transition_status: {
-                        if (static_cast<TransitionStatus&>(*cmd).GetResult() == Result::Ok) {
-                            cout << "Device " << static_cast<TransitionStatus&>(*cmd).GetDeviceId() << " started to transition with " << static_cast<TransitionStatus&>(*cmd).GetTransition() << endl;
-                        } else {
-                            cout << "Device " << static_cast<TransitionStatus&>(*cmd).GetDeviceId() << " cannot transition with " << static_cast<TransitionStatus&>(*cmd).GetTransition() << endl;
-                        }
-                    }
-                    break;
-                    case Type::current_state:
-                        cout << "Device " << static_cast<CurrentState&>(*cmd).GetDeviceId() << " is in " << static_cast<CurrentState&>(*cmd).GetCurrentState() << " state" << endl;
-                    break;
-                    case Type::config:
-                        cout << "Received config for device " << static_cast<Config&>(*cmd).GetDeviceId() << ":\n" << static_cast<Config&>(*cmd).GetConfig() << endl;
-                    break;
-                    default:
-                        cout << "Unexpected/unknown command received: " << cmd->GetType() << endl;
-                        cout << "Origin: " << senderId << endl;
-                    break;
-                }
-            }
-        });
-
-        service.start(sessionID);
-
-        StateSubscription stateSubscription(topologyPath, ddsCustomCmd);
-
-        if (targetState == "") {
-            sendCommand(command, topologyPath, ddsCustomCmd);
-        } else {
-            waitMode.Run(chrono::milliseconds(timeout), topologyPath, ddsCustomCmd, numDevices, command);
-        }
-
-        ddsCustomCmd.unsubscribe();
-    } catch (exception& e) {
-        cerr << "Error: " << e.what() << endl;
-        return EXIT_FAILURE;
+    auto envSessionId = getenv("DDS_SESSION_ID");
+    if (envSessionId) {
+        options.add_options()("session,s", bpo::value<string>(&sessionID)->default_value(envSessionId), "DDS Session ID (overrides any value in env var $DDS_SESSION_ID)");
+    } else {
+        options.add_options()("session,s", bpo::value<string>(&sessionID)->required(), "DDS Session ID (overrides any value in env var $DDS_SESSION_ID)");
     }
+
+    auto envTopoFile = getenv("FAIRMQ_DDS_TOPO_FILE");
+    if (envTopoFile) {
+        options.add_options()("topology-file,f", bpo::value<string>(&topoFile)->default_value(envTopoFile), "DDS topology file path");
+    } else {
+        options.add_options()("topology-file,f", bpo::value<string>(&topoFile)->required(), "DDS topology file path");
+    }
+
+    options.add_options()
+        ("command,c",        bpo::value<string>(&command)->default_value(""), "Command character")
+        ("path,p",           bpo::value<string>(&path)->default_value(""), "DDS Topology path to send command to (empty - send to all tasks)")
+        ("property-key",     bpo::value<string>(&pKey)->default_value(""), "property key to be used with 'p' command")
+        ("property-value",   bpo::value<string>(&pVal)->default_value(""), "property value to be used with 'p' command")
+        ("wait-for-state,w", bpo::value<string>(&targetState)->default_value(""), "Wait until targeted FairMQ devices reach the given state")
+        ("timeout,t",        bpo::value<unsigned int>(&timeout)->default_value(0), "Timeout in milliseconds when waiting for a device state (0 - wait infinitely)")
+        ("help,h",           "Produce help message");
+
+    bpo::variables_map vm;
+    bpo::store(bpo::command_line_parser(argc, argv).options(options).run(), vm);
+
+    if (vm.count("help")) {
+        cout << "FairMQ DDS Command UI" << endl << options << endl;
+        cout << "Commands: [c] check state, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device, [k] complete init, [b] bind, [x] connect, [p] set property" << endl;
+        return EXIT_SUCCESS;
+    }
+
+    bpo::notify(vm);
+
+    DDSEnvironment env;
+    DDSSession session(sessionID, env);
+    DDSTopology ddsTopo(DDSTopology::Path(topoFile), env);
+
+    Topology topo(ddsTopo, session);
+
+    if (targetState != "") {
+        if (command != "") {
+            sendCommand(command, path, timeout, topo, pKey, pVal);
+        }
+        size_t pos = targetState.find("->");
+        if (pos == string::npos) {
+            /* auto ec =  */topo.WaitForState(GetState(targetState), path, std::chrono::milliseconds(timeout));
+            // cout << "WaitForState(" << targetState << ") result: " << ec.message() << endl;
+        } else {
+            /* auto ec =  */topo.WaitForState(GetState(targetState.substr(0, pos)), GetState(targetState.substr(pos + 2)), path, std::chrono::milliseconds(timeout));
+            // cout << "WaitForState(" << targetState << ") result: " << ec.message() << endl;
+        }
+    } else {
+        sendCommand(command, path, timeout, topo, pKey, pVal);
+    }
+
     return EXIT_SUCCESS;
+} catch (exception& e) {
+    cerr << "Error: " << e.what() << endl;
+    return EXIT_FAILURE;
 }

fairmq/plugins/PMIx/CMakeLists.txt

@@ -10,9 +10,10 @@ set(plugin FairMQPlugin_pmix)
 add_library(${plugin} SHARED
   ${CMAKE_CURRENT_SOURCE_DIR}/PMIxPlugin.cxx
   ${CMAKE_CURRENT_SOURCE_DIR}/PMIxPlugin.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/PMIxCommands.h
   ${CMAKE_CURRENT_SOURCE_DIR}/PMIx.hpp
 )
-target_link_libraries(${plugin} FairMQ PMIx::libpmix)
+target_link_libraries(${plugin} PUBLIC FairMQ PMIx::libpmix PRIVATE Commands)
 target_include_directories(${plugin} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 set_target_properties(${plugin} PROPERTIES
   CXX_VISIBILITY_PRESET hidden
@@ -20,7 +21,12 @@ set_target_properties(${plugin} PROPERTIES
   SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
 )
 
-install(TARGETS ${plugin}
+set(exe fairmq-pmix-command-ui)
+add_executable(${exe} ${CMAKE_CURRENT_SOURCE_DIR}/runPMIxCommandUI.cxx)
+target_link_libraries(${exe} FairMQ Commands StateMachine PMIx::libpmix)
+target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+
+install(TARGETS ${plugin} ${exe}
   EXPORT ${PROJECT_EXPORT_SET}
   LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
   RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}

fairmq/plugins/PMIx/PMIx.hpp

@@ -9,17 +9,18 @@
 #ifndef PMIX_HPP
 #define PMIX_HPP
 
+#include <array>
 #include <cstring>
+#include <functional>
 #include <limits>
 #include <memory>
 #include <ostream>
 #include <pmix.h>
+#include <sstream>
 #include <stdexcept>
-#include <string.h>
 #include <type_traits>
 #include <utility>
 #include <vector>
-#include <FairMQLogger.h>
 
 // C++ PMIx v2.2 API
 namespace pmix
@@ -69,7 +70,7 @@ struct proc : pmix_proc_t
 
     friend std::ostream& operator<<(std::ostream& os, const proc& p)
     {
-        return os << "nspace=" << p.nspace << ",rank=" << p.rank;
+        return os << p.nspace << "_" << p.rank;
     }
 };
 
@@ -80,7 +81,6 @@ struct value : pmix_value_t
 
     value(const value& rhs)
     {
-      LOG(warn) << "copy ctor";
         status rc;
         auto lhs(static_cast<pmix_value_t*>(this));
         PMIX_VALUE_XFER(rc, lhs, static_cast<pmix_value_t*>(const_cast<value*>(&rhs)));
@@ -111,6 +111,11 @@ struct value : pmix_value_t
     {
         PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), &val, PMIX_INT);
     }
+
+    explicit value(pmix_data_array_t* val)
+    {
+        PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), val, PMIX_DATA_ARRAY);
+    }
 };
 
 struct info : pmix_info_t
@@ -134,9 +139,15 @@ struct info : pmix_info_t
         }
     }
 
-    friend std::ostream& operator<<(std::ostream& os, const info& p)
+    friend std::ostream& operator<<(std::ostream& os, const info& i)
     {
-        return os << "key=" << p.key << ",value='" << p.value.data.string << "'";
+        return os << "key=" << i.key << ",value='" << i.value.data.string << "'";
+    }
+
+    info(const info& rhs)
+    {
+        PMIX_INFO_XFER(static_cast<pmix_info_t*>(this),
+                       static_cast<pmix_info_t*>(const_cast<info*>(&rhs)));
     }
 };
 
@@ -144,6 +155,7 @@ struct pdata : pmix_pdata_t
 {
     pdata() { PMIX_PDATA_CONSTRUCT(static_cast<pmix_pdata_t*>(this)); }
     ~pdata() { PMIX_PDATA_DESTRUCT(static_cast<pmix_pdata_t*>(this)); }
+
     pdata(const pdata& rhs)
     {
         PMIX_PDATA_XFER(static_cast<pmix_pdata_t*>(this),
@@ -171,7 +183,7 @@ auto init(const std::vector<info>& info = {}) -> proc
 
 auto initialized() -> bool { return !!PMIx_Initialized(); }
 
-auto get_version() -> const char* { return PMIx_Get_version(); }
+auto get_version() -> std::string { return {PMIx_Get_version()}; }
 
 auto finalize(const std::vector<info>& info = {}) -> void
 {
@@ -213,6 +225,92 @@ auto lookup(std::vector<pdata>& pdata, const std::vector<info>& info = {}) -> vo
     }
 }
 
+std::string get_info(const std::string& name, pmix::proc& process)
+{
+    pmix_value_t* v;
+
+    pmix::status rc = PMIx_Get(&process, name.c_str(), nullptr, 0, &v);
+    if (rc == PMIX_SUCCESS) {
+        std::stringstream ss;
+
+        switch (v->type) {
+            case PMIX_SIZE:      ss << static_cast<size_t>(v->data.size)       << " (size_t)";       break;
+            case PMIX_INT:       ss << static_cast<int>(v->data.integer)       << " (int)";          break;
+            case PMIX_INT8:      ss << static_cast<int8_t>(v->data.int8)       << " (int8_t)";       break;
+            case PMIX_INT16:     ss << static_cast<int16_t>(v->data.int16)     << " (int16_t)";      break;
+            case PMIX_INT32:     ss << static_cast<int32_t>(v->data.int32)     << " (int32_t)";      break;
+            case PMIX_INT64:     ss << static_cast<int64_t>(v->data.int64)     << " (int64_t)";      break;
+            case PMIX_UINT:      ss << static_cast<unsigned int>(v->data.uint) << " (unsigned int)"; break;
+            case PMIX_UINT8:     ss << static_cast<uint8_t>(v->data.uint8)     << " (uint8_t)";      break;
+            case PMIX_UINT16:    ss << static_cast<uint16_t>(v->data.uint16)   << " (uint16_t)";     break;
+            case PMIX_UINT32:    ss << static_cast<uint32_t>(v->data.uint32)   << " (uint32_t)";     break;
+            case PMIX_UINT64:    ss << static_cast<uint64_t>(v->data.uint64)   << " (uint64_t)";     break;
+            case PMIX_FLOAT:     ss << static_cast<float>(v->data.fval)        << " (float)";        break;
+            case PMIX_DOUBLE:    ss << static_cast<double>(v->data.dval)       << " (double)";       break;
+            case PMIX_PID:       ss << static_cast<pid_t>(v->data.pid)         << " (pid_t)";        break;
+            case PMIX_STRING:    ss << static_cast<char*>(v->data.string)      << " (string)";       break;
+            case PMIX_PROC_RANK: ss << static_cast<uint32_t>(v->data.rank)     << " (pmix_rank_t)";  break;
+            case PMIX_PROC:      ss << "proc.nspace: " << static_cast<pmix_proc_t*>(v->data.proc)->nspace
+                                    << ", proc.rank: " << static_cast<pmix_proc_t*>(v->data.proc)->rank << " (pmix_proc_t*)"; break;
+            default:
+                ss << "unknown type: " << v->type;
+                break;
+        }
+
+        return ss.str();
+    } else if (rc == PMIX_ERR_NOT_FOUND) {
+        // LOG(error) << "PMIx_Get failed: PMIX_ERR_NOT_FOUND";
+        return "";
+    } else {
+        // LOG(error) << "PMIx_Get failed: " << rc;
+        return "<undefined>";
+    }
+}
+
+std::string get_value_str(const pmix_value_t& v)
+{
+    switch (v.type) {
+        case PMIX_BOOL:       return std::to_string(static_cast<bool>(v.data.flag));
+        case PMIX_SIZE:       return std::to_string(static_cast<size_t>(v.data.size));
+        case PMIX_INT:        return std::to_string(static_cast<int>(v.data.integer));
+        case PMIX_INT8:       return std::to_string(static_cast<int8_t>(v.data.int8));
+        case PMIX_INT16:      return std::to_string(static_cast<int16_t>(v.data.int16));
+        case PMIX_INT32:      return std::to_string(static_cast<int32_t>(v.data.int32));
+        case PMIX_INT64:      return std::to_string(static_cast<int64_t>(v.data.int64));
+        case PMIX_UINT:       return std::to_string(static_cast<unsigned int>(v.data.uint));
+        case PMIX_UINT8:      return std::to_string(static_cast<uint8_t>(v.data.uint8));
+        case PMIX_UINT16:     return std::to_string(static_cast<uint16_t>(v.data.uint16));
+        case PMIX_UINT32:     return std::to_string(static_cast<uint32_t>(v.data.uint32));
+        case PMIX_UINT64:     return std::to_string(static_cast<uint64_t>(v.data.uint64));
+        case PMIX_FLOAT:      return std::to_string(static_cast<float>(v.data.fval));
+        case PMIX_DOUBLE:     return std::to_string(static_cast<double>(v.data.dval));
+        case PMIX_PID:        return std::to_string(static_cast<pid_t>(v.data.pid));
+        case PMIX_STRING:     return static_cast<char*>(v.data.string);
+        case PMIX_PROC_RANK:  return std::to_string(static_cast<uint32_t>(v.data.rank));
+        case PMIX_POINTER:    { std::stringstream ss; ss << static_cast<void*>(v.data.ptr); return ss.str(); }
+        case PMIX_DATA_ARRAY: {
+            if (v.data.darray->type == PMIX_PROC) {
+                std::stringstream ss;
+                ss << "[";
+                for (size_t i = 0; i < v.data.darray->size; ++i) {
+                    ss << static_cast<pmix_proc_t*>(static_cast<pmix_data_array_t*>(v.data.darray)->array)[0].nspace;
+                    ss << "_";
+                    ss << static_cast<pmix_proc_t*>(static_cast<pmix_data_array_t*>(v.data.darray)->array)[0].rank;
+
+                    if (i < v.data.darray->size - 1) {
+                        ss << ",";
+                    }
+                }
+                ss << "]";
+                return ss.str();
+            } else {
+                return "UNKNOWN TYPE IN DATA ARRAY";
+            }
+        }
+        default:             return "UNKNOWN TYPE";
+    }
+}
+
 } /* namespace pmix */
 
 #endif /* PMIX_HPP */

fairmq/plugins/PMIx/PMIxCommands.h

@@ -0,0 +1,291 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef PMIXCOMMANDS_H
+#define PMIXCOMMANDS_H
+
+#include "PMIx.hpp"
+
+#include <FairMQLogger.h>
+#include <fairmq/tools/Semaphore.h>
+#include <fairmq/tools/CppSTL.h>
+#include <string>
+
+namespace pmix
+{
+
+std::array<std::string, 47> typeNames =
+{
+    {
+        "PMIX_UNDEF",
+        "PMIX_BOOL",
+        "PMIX_BYTE",
+        "PMIX_STRING",
+        "PMIX_SIZE",
+        "PMIX_PID",
+        "PMIX_INT",
+        "PMIX_INT8",
+        "PMIX_INT16",
+        "PMIX_INT32",
+        "PMIX_INT64",
+        "PMIX_UINT",
+        "PMIX_UINT8",
+        "PMIX_UINT16",
+        "PMIX_UINT32",
+        "PMIX_UINT64",
+        "PMIX_FLOAT",
+        "PMIX_DOUBLE",
+        "PMIX_TIMEVAL",
+        "PMIX_TIME",
+        "PMIX_STATUS",
+        "PMIX_VALUE",
+        "PMIX_PROC",
+        "PMIX_APP",
+        "PMIX_INFO",
+        "PMIX_PDATA",
+        "PMIX_BUFFER",
+        "PMIX_BYTE_OBJECT",
+        "PMIX_KVAL",
+        "PMIX_MODEX",
+        "PMIX_PERSIST",
+        "PMIX_POINTER",
+        "PMIX_SCOPE",
+        "PMIX_DATA_RANGE",
+        "PMIX_COMMAND",
+        "PMIX_INFO_DIRECTIVES",
+        "PMIX_DATA_TYPE",
+        "PMIX_PROC_STATE",
+        "PMIX_PROC_INFO",
+        "PMIX_DATA_ARRAY",
+        "PMIX_PROC_RANK",
+        "PMIX_QUERY",
+        "PMIX_COMPRESSED_STRING",
+        "PMIX_ALLOC_DIRECTIVE",
+        "PMIX_INFO_ARRAY",
+        "PMIX_IOF_CHANNEL",
+        "PMIX_ENVAR"
+    }
+};
+
+enum class Command : int
+{
+    general = PMIX_EXTERNAL_ERR_BASE,
+    error = PMIX_EXTERNAL_ERR_BASE - 1
+};
+
+
+class Commands
+{
+  public:
+    Commands(const proc& process)
+        : fProcess(process)
+        , fSubscribed(false)
+    {
+    }
+
+    ~Commands()
+    {
+        Unsubscribe();
+    }
+
+    void Subscribe(std::function<void(const std::string& msg, const proc& sender)> callback)
+    {
+        using namespace std::placeholders;
+
+        LOG(debug) << "PMIxCommands: Subscribing...";
+
+        fCallback = callback;
+        std::array<pmix::status, 1> codes;
+        codes[0] = static_cast<int>(pmix::Command::general);
+
+        PMIX_INFO_LOAD(&(fInfos[0]), PMIX_EVENT_RETURN_OBJECT, this, PMIX_POINTER);
+
+        PMIx_Register_event_handler(codes.data(), codes.size(),
+                                    fInfos.data(), fInfos.size(),
+                                    &Commands::Handler,
+                                    &Commands::EventHandlerRegistration,
+                                    this);
+        fBlocker.Wait();
+        LOG(debug) << "PMIxCommands: Subscribing complete!";
+    }
+
+    void Unsubscribe()
+    {
+        if (fSubscribed) {
+            LOG(debug) << "PMIxCommands: Unsubscribing...";
+            PMIx_Deregister_event_handler(fHandlerRef, &Commands::EventHandlerDeregistration, this);
+            fBlocker.Wait();
+            LOG(debug) << "PMIxCommands: Unsubscribing complete!";
+        } else {
+            LOG(debug) << "Unsubscribe() is called while no subscription is active";
+        }
+    }
+
+    struct Holder
+    {
+        Holder() : fData(nullptr) {}
+        ~Holder() { PMIX_DATA_ARRAY_FREE(fData); }
+
+        std::vector<pmix::info> fInfos;
+        pmix_data_array_t* fData;
+    };
+
+    void Send(const std::string& msg)
+    {
+        std::vector<pmix::info>* infos = new std::vector<pmix::info>();
+        infos->emplace_back("fairmq.cmd", msg);
+        PMIx_Notify_event(static_cast<int>(pmix::Command::general),
+                          &fProcess,
+                          PMIX_RANGE_NAMESPACE,
+                          infos->data(), infos->size(),
+                          &Commands::OpCompleteCallback<std::vector<pmix::info>>,
+                          infos);
+    }
+
+    void Send(const std::string& msg, rank rank)
+    {
+        pmix::proc destination(fProcess);
+        destination.rank = rank;
+        Send(msg, {destination});
+    }
+
+    void Send(const std::string& msg, const std::vector<proc>& destination)
+    {
+        std::unique_ptr<Holder> holder = fair::mq::tools::make_unique<Holder>();
+
+        PMIX_DATA_ARRAY_CREATE(holder->fData, destination.size(), PMIX_PROC);
+        memcpy(holder->fData->array, destination.data(), destination.size() * sizeof(pmix_proc_t));
+        // LOG(warn) << "OLOG: " << msg << " > " << static_cast<pmix_proc_t*>(holder->fData->array)[0].nspace << ": " << static_cast<pmix_proc_t*>(holder->fData->array)[0].rank;
+        holder->fInfos.emplace_back(PMIX_EVENT_CUSTOM_RANGE, holder->fData);
+        // LOG(warn) << msg << " // packed range: " << static_cast<pmix_proc_t*>(static_cast<pmix_data_array_t*>(holder->fInfos.at(0).value.data.darray)->array)[0].nspace << "_" << static_cast<pmix_proc_t*>(static_cast<pmix_data_array_t*>(holder->fInfos.at(0).value.data.darray)->array)[0].rank;
+        // LOG(warn) << msg << " // packed range.type: " << pmix::typeNames.at(holder->fInfos.at(0).value.type);
+        // LOG(warn) << msg << " // packed range.array.type: " << pmix::typeNames.at(static_cast<pmix_data_array_t*>(holder->fInfos.at(0).value.data.darray)->type);
+        // LOG(warn) << msg << " // packed range.array.size: " << static_cast<pmix_data_array_t*>(holder->fInfos.at(0).value.data.darray)->size;
+        // LOG(warn) << holder->fInfos.size();
+        holder->fInfos.emplace_back("fairmq.cmd", msg);
+        // LOG(warn) << msg << " // packed msg: " << holder->fInfos.at(1).value.data.string;
+        // LOG(warn) << msg << " // packed msg.type: " << pmix::typeNames.at(holder->fInfos.at(1).value.type);
+        // LOG(warn) << holder->fInfos.size();
+
+        PMIx_Notify_event(static_cast<int>(pmix::Command::general),
+                          &fProcess,
+                          PMIX_RANGE_CUSTOM,
+                          holder->fInfos.data(), holder->fInfos.size(),
+                          &Commands::OpCompleteCallback<Holder>,
+                          holder.get());
+        holder.release();
+    }
+
+  private:
+    static void EventHandlerRegistration(pmix_status_t s, size_t handlerRef, void* obj)
+    {
+        if (s == PMIX_SUCCESS) {
+            LOG(debug) << "Successfully registered event handler, reference = " << static_cast<unsigned long>(handlerRef);
+            static_cast<Commands*>(obj)->fHandlerRef = handlerRef;
+            static_cast<Commands*>(obj)->fSubscribed = true;
+        } else {
+            LOG(error) << "Could not register PMIx event handler, status = " << s;
+        }
+        static_cast<Commands*>(obj)->fBlocker.Signal();
+    }
+
+    static void EventHandlerDeregistration(pmix_status_t s, void* obj)
+    {
+        if (s == PMIX_SUCCESS) {
+            LOG(debug) << "Successfully deregistered event handler, reference = " << static_cast<Commands*>(obj)->fHandlerRef;
+            static_cast<Commands*>(obj)->fSubscribed = false;
+        } else {
+            LOG(error) << "Could not deregister PMIx event handler, reference = " << static_cast<Commands*>(obj)->fHandlerRef << ", status = " << s;
+        }
+        static_cast<Commands*>(obj)->fBlocker.Signal();
+    }
+
+    template<typename T>
+    static void OpCompleteCallback(pmix_status_t s, void* data)
+    {
+        if (s == PMIX_SUCCESS) {
+            // LOG(info) << "Operation completed successfully";
+        } else {
+            LOG(error) << "Could not complete operation, status = " << s;
+        }
+        if (data) {
+            // LOG(warn) << "Destroying event data...";
+            delete static_cast<T*>(data);
+        }
+    }
+
+    static void Handler(size_t handlerId,
+                        pmix_status_t s,
+                        const pmix_proc_t* src,
+                        pmix_info_t info[], size_t ninfo,
+                        pmix_info_t[] /* results */, size_t nresults,
+                        pmix_event_notification_cbfunc_fn_t cbfunc,
+                        void* cbdata)
+    {
+        std::stringstream ss;
+        ss << "Event handler called with "
+           << "status: "    << s                               << ", "
+           << "source: "    << src->nspace << "_" << src->rank << ", "
+           << "ninfo: "     << ninfo                           << ", "
+           << "nresults: "  << nresults                        << ", "
+           << "handlerId: " << handlerId;
+
+        std::string msg;
+
+        Commands* obj = nullptr;
+
+        if (ninfo > 0) {
+            ss << ":\n";
+            for (size_t i = 0; i < ninfo; ++i) {
+                ss << "   [" << i << "]: key: '"        << info[i].key
+                                  << "', value: '"      << pmix::get_value_str(info[i].value)
+                                  << "', value.type: '" << pmix::typeNames.at(info[i].value.type)
+                                  << "', flags: "       << info[i].flags;
+
+                if (std::strcmp(info[i].key, "fairmq.cmd") == 0) {
+                    msg = pmix::get_value_str(info[i].value);
+                }
+
+                if (std::strcmp(info[i].key, PMIX_EVENT_RETURN_OBJECT) == 0) {
+                    obj = static_cast<Commands*>(info[i].value.data.ptr);
+                }
+
+                if (i < ninfo - 1) {
+                    ss << "\n";
+                }
+            }
+        }
+
+
+        if (obj != nullptr) {
+            if (static_cast<Commands*>(obj)->fProcess.rank != src->rank) {
+                // LOG(warn) << ss.str();
+                static_cast<Commands*>(obj)->fCallback(msg, proc(const_cast<char*>(src->nspace), rank(src->rank)));
+            } else {
+                // LOG(trace) << "suppressing message from itself";
+            }
+        } else {
+            LOG(ERROR) << "ERROR";
+        }
+
+        if (cbfunc != nullptr) {
+            cbfunc(PMIX_SUCCESS, nullptr, 0, nullptr, nullptr, cbdata);
+        }
+    }
+
+    const proc& fProcess;
+    size_t fHandlerRef;
+    std::function<void(const std::string& msg, const proc& sender)> fCallback;
+    std::array<pmix_info_t, 1> fInfos;
+    bool fSubscribed;
+    fair::mq::tools::SharedSemaphore fBlocker;
+};
+
+} /* namespace pmix */
+
+#endif /* PMIXCOMMANDS_H */

fairmq/plugins/PMIx/PMIxPlugin.cxx

@@ -8,8 +8,15 @@
 
 #include "PMIxPlugin.h"
 
+#include <fairmq/sdk/commands/Commands.h>
 #include <fairmq/Tools.h>
+
+#include <sstream>
 #include <stdexcept>
+#include <cstdint> // UINT32_MAX
+
+using namespace std;
+using namespace fair::mq::sdk::cmd;
 
 namespace fair
 {
@@ -18,47 +25,99 @@ namespace mq
 namespace plugins
 {
 
-PMIxPlugin::PMIxPlugin(const std::string& name,
+PMIxPlugin::PMIxPlugin(const string& name,
                        const Plugin::Version version,
-                       const std::string& maintainer,
-                       const std::string& homepage,
+                       const string& maintainer,
+                       const string& homepage,
                        PluginServices* pluginServices)
     : Plugin(name, version, maintainer, homepage, pluginServices)
+    , fProcess(Init())
     , fPid(getpid())
+    , fPMIxClient(tools::ToString("PMIx client(pid=", fPid, ") "))
+    , fDeviceId(string(fProcess.nspace) + "_" + to_string(fProcess.rank))
+    , fCommands(fProcess)
+    , fLastExternalController(UINT32_MAX)
+    , fExitingAckedByLastExternalController(false)
+    , fCurrentState(DeviceState::Idle)
+    , fLastState(DeviceState::Idle)
 {
-    Init();
-    SetProperty<std::string>("id", std::string(fProc.nspace) + "_" + std::to_string(fProc.rank));
-    Fence();
+    TakeDeviceControl();
+    LOG(debug) << PMIxClient() << "pmix::init() OK: " << fProcess << ", version=" << pmix::get_version();
+    SetProperty<string>("id", fDeviceId);
 
-    SubscribeToDeviceStateChange([&](DeviceState newState) {
+    Fence("pmix::init");
+    SubscribeForCommands();
+    Fence("subscribed");
+
+    // fCommands.Send("test1");
+    // fCommands.Send("test2", 0);
+    // fCommands.Send("test3", 0);
+
+    // LOG(info) << "PMIX_EXTERNAL_ERR_BASE: " << PMIX_EXTERNAL_ERR_BASE;
+
+    // job level infos
+    // LOG(info) << "PMIX_SESSION_ID: "         << pmix::getInfo(PMIX_SESSION_ID, fProcess);
+    // LOG(info) << "PMIX_UNIV_SIZE: "          << pmix::getInfo(PMIX_UNIV_SIZE, fProcess);
+    // LOG(info) << "PMIX_JOB_SIZE: "           << pmix::getInfo(PMIX_JOB_SIZE, fProcess);
+    // LOG(info) << "PMIX_JOB_NUM_APPS: "       << pmix::getInfo(PMIX_JOB_NUM_APPS, fProcess);
+    // LOG(info) << "PMIX_APP_SIZE: "           << pmix::getInfo(PMIX_APP_SIZE, fProcess);
+    // LOG(info) << "PMIX_MAX_PROCS: "          << pmix::getInfo(PMIX_MAX_PROCS, fProcess);
+    // LOG(info) << "PMIX_NUM_NODES: "          << pmix::getInfo(PMIX_NUM_NODES, fProcess);
+    // LOG(info) << "PMIX_CLUSTER_ID: "         << pmix::getInfo(PMIX_CLUSTER_ID, fProcess);
+    // LOG(info) << "PMIX_NSPACE: "             << pmix::getInfo(PMIX_NSPACE, fProcess);
+    // LOG(info) << "PMIX_JOBID: "              << pmix::getInfo(PMIX_JOBID, fProcess);
+    // LOG(info) << "PMIX_NODE_LIST: "          << pmix::getInfo(PMIX_NODE_LIST, fProcess);
+    // LOG(info) << "PMIX_ALLOCATED_NODELIST: " << pmix::getInfo(PMIX_ALLOCATED_NODELIST, fProcess);
+    // LOG(info) << "PMIX_NPROC_OFFSET: "       << pmix::getInfo(PMIX_NPROC_OFFSET, fProcess);
+    // LOG(info) << "PMIX_LOCALLDR: "           << pmix::getInfo(PMIX_LOCALLDR, fProcess);
+    // LOG(info) << "PMIX_APPLDR: "             << pmix::getInfo(PMIX_APPLDR, fProcess);
+
+    // // per-node information
+    // LOG(info) << "PMIX_NODE_SIZE: "          << pmix::getInfo(PMIX_NODE_SIZE, fProcess);
+    // LOG(info) << "PMIX_LOCAL_SIZE: "         << pmix::getInfo(PMIX_LOCAL_SIZE, fProcess);
+    // LOG(info) << "PMIX_AVAIL_PHYS_MEMORY: "  << pmix::getInfo(PMIX_AVAIL_PHYS_MEMORY, fProcess);
+
+    // // per-process information
+    // LOG(info) << "PMIX_PROCID: "             << pmix::getInfo(PMIX_PROCID, fProcess);
+    // LOG(info) << "PMIX_APPNUM: "             << pmix::getInfo(PMIX_APPNUM, fProcess);
+    // LOG(info) << "PMIX_LOCAL_RANK: "         << pmix::getInfo(PMIX_LOCAL_RANK, fProcess);
+    // LOG(info) << "PMIX_NODE_RANK: "          << pmix::getInfo(PMIX_NODE_RANK, fProcess);
+    // LOG(info) << "PMIX_RANK: "               << pmix::getInfo(PMIX_RANK, fProcess);
+    // LOG(info) << "PMIX_GLOBAL_RANK: "        << pmix::getInfo(PMIX_GLOBAL_RANK, fProcess);
+    // LOG(info) << "PMIX_APP_RANK: "           << pmix::getInfo(PMIX_APP_RANK, fProcess);
+
+    SubscribeToDeviceStateChange([this](DeviceState newState) {
         switch (newState) {
-        case DeviceState::Idle:
-            Fence();
-            break;
-        case DeviceState::Bound:
-            Publish();
-            Fence();
-            break;
-        case DeviceState::Connecting:
-            Lookup();
-            break;
-        case DeviceState::DeviceReady:
-            Fence();
-            break;
-        case DeviceState::Ready:
-            Fence();
-            break;
-        case DeviceState::Exiting:
-            UnsubscribeFromDeviceStateChange();
-            break;
-        default:
-            break;
+            case DeviceState::Bound:
+                Publish();
+                break;
+            case DeviceState::Connecting:
+                Lookup();
+                break;
+            case DeviceState::Exiting:
+                ReleaseDeviceControl();
+                UnsubscribeFromDeviceStateChange();
+                break;
+            default:
+                break;
+        }
+
+        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+        fLastState = fCurrentState;
+        fCurrentState = newState;
+        for (auto subscriberId : fStateChangeSubscribers) {
+            LOG(debug) << "Publishing state-change: " << fLastState << "->" << newState << " to " << subscriberId;
+            Cmds cmds(make<StateChange>(fDeviceId, 0, fLastState, fCurrentState));
+            fCommands.Send(cmds.Serialize(Format::JSON), static_cast<pmix::rank>(subscriberId));
         }
     });
 }
 
 PMIxPlugin::~PMIxPlugin()
 {
+    LOG(debug) << "Destroying PMIxPlugin";
+    ReleaseDeviceControl();
+    fCommands.Unsubscribe();
     while (pmix::initialized()) {
         try {
             pmix::finalize();
@@ -69,33 +128,112 @@ PMIxPlugin::~PMIxPlugin()
     }
 }
 
-auto PMIxPlugin::PMIxClient() const -> std::string
+auto PMIxPlugin::SubscribeForCommands() -> void
 {
-    std::stringstream ss;
-    ss << "PMIx client(pid=" << fPid << ") ";
-    return ss.str();
+    fCommands.Subscribe([this](const string& cmdStr, const pmix::proc& sender) {
+        // LOG(info) << "PMIx Plugin received message: '" << cmdStr << "', from " << sender;
+
+        Cmds inCmds;
+        inCmds.Deserialize(cmdStr, Format::JSON);
+
+        for (const auto& cmd : inCmds) {
+            LOG(info) << "Received command type: '" << cmd->GetType() << "' from " << sender;
+            switch (cmd->GetType()) {
+                case Type::check_state:
+                    fCommands.Send(Cmds(make<CurrentState>(fDeviceId, GetCurrentDeviceState()))
+                                       .Serialize(Format::JSON),
+                                   {sender});
+                    break;
+                case Type::change_state: {
+                    Transition transition = static_cast<ChangeState&>(*cmd).GetTransition();
+                    if (ChangeDeviceState(transition)) {
+                        fCommands.Send(
+                            Cmds(make<TransitionStatus>(fDeviceId, 0, Result::Ok, transition))
+                                .Serialize(Format::JSON),
+                            {sender});
+                    } else {
+                        fCommands.Send(
+                            Cmds(make<TransitionStatus>(fDeviceId, 0, Result::Failure, transition))
+                                .Serialize(Format::JSON),
+                            {sender});
+                    }
+                    {
+                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                        fLastExternalController = sender.rank;
+                    }
+                }
+                break;
+                case Type::subscribe_to_state_change: {
+                    {
+                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                        fStateChangeSubscribers.insert(sender.rank);
+                    }
+
+                    LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState
+                               << " to " << sender;
+                    Cmds outCmds(make<StateChangeSubscription>(fDeviceId, fProcess.rank, Result::Ok),
+                                 make<StateChange>(fDeviceId, 0, fLastState, fCurrentState));
+                    fCommands.Send(outCmds.Serialize(Format::JSON), {sender});
+                }
+                break;
+                case Type::unsubscribe_from_state_change: {
+                    {
+                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                        fStateChangeSubscribers.erase(sender.rank);
+                    }
+                    fCommands.Send(Cmds(make<StateChangeUnsubscription>(fDeviceId, fProcess.rank, Result::Ok))
+                                       .Serialize(Format::JSON),
+                                   {sender});
+                }
+                break;
+                case Type::state_change_exiting_received: {
+                    {
+                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                        if (fLastExternalController == sender.rank) {
+                            fExitingAckedByLastExternalController = true;
+                        }
+                    }
+                    fExitingAcked.notify_one();
+                }
+                break;
+                case Type::dump_config: {
+                    stringstream ss;
+                    for (const auto& k: GetPropertyKeys()) {
+                        ss << fDeviceId << ": " << k << " -> " << GetPropertyAsString(k) << "\n";
+                    }
+                    fCommands.Send(Cmds(make<Config>(fDeviceId, ss.str())).Serialize(Format::JSON),
+                                   {sender});
+                }
+                break;
+                default:
+                    LOG(warn) << "Unexpected/unknown command received: " << cmdStr;
+                    LOG(warn) << "Origin: " << sender;
+                break;
+            }
+        }
+    });
 }
 
-auto PMIxPlugin::Init() -> void
+auto PMIxPlugin::Init() -> pmix::proc
 {
     if (!pmix::initialized()) {
-        fProc = pmix::init();
-        LOG(debug) << PMIxClient() << "pmix::init() OK: " << fProc
-                   << ",version=" << pmix::get_version();
+        return pmix::init();
+    } else {
+        throw runtime_error("trying to initialize PMIx while it is already initialized");
     }
 }
 
 auto PMIxPlugin::Publish() -> void
 {
     auto channels(GetChannelInfo());
-    std::vector<pmix::info> info;
+    vector<pmix::info> info;
 
     for (const auto& c : channels) {
-        std::string methodKey{"chans." + c.first + "." + std::to_string(c.second - 1) + ".method"};
-        if (GetProperty<std::string>(methodKey) == "bind") {
+        string methodKey("chans." + c.first + "." + to_string(c.second - 1) + ".method");
+        if (GetProperty<string>(methodKey) == "bind") {
             for (int i = 0; i < c.second; ++i) {
-                std::string addressKey{"chans." + c.first + "." + std::to_string(i) + ".address"};
-                info.emplace_back(addressKey, GetProperty<std::string>(addressKey));
+                string addressKey("chans." + c.first + "." + to_string(i) + ".address");
+                info.emplace_back(addressKey, GetProperty<string>(addressKey));
                 LOG(debug) << PMIxClient() << info.back();
             }
         }
@@ -103,32 +241,37 @@ auto PMIxPlugin::Publish() -> void
 
     if (info.size() > 0) {
         pmix::publish(info);
-        LOG(debug) << PMIxClient() << "pmix::publish() OK: published "
-                   << info.size() << " binding channels.";
+        LOG(debug) << PMIxClient() << "pmix::publish() OK: published " << info.size()
+                   << " binding channels.";
     }
 }
 
 auto PMIxPlugin::Fence() -> void
 {
-    pmix::proc all(fProc);
+    pmix::proc all(fProcess);
     all.rank = pmix::rank::wildcard;
 
     pmix::fence({all});
-    LOG(debug) << PMIxClient() << "pmix::fence() OK";
+}
+
+auto PMIxPlugin::Fence(const std::string& label) -> void
+{
+    Fence(label);
+    LOG(debug) << PMIxClient() << "pmix::fence() [" << label << "] OK";
 }
 
 auto PMIxPlugin::Lookup() -> void
 {
     auto channels(GetChannelInfo());
     for (const auto& c : channels) {
-        std::string methodKey{"chans." + c.first + "." + std::to_string(c.second - 1) + ".method"};
-        if (GetProperty<std::string>(methodKey) == "connect") {
+        string methodKey("chans." + c.first + "." + to_string(c.second - 1) + ".method");
+        if (GetProperty<string>(methodKey) == "connect") {
             for (int i = 0; i < c.second; ++i) {
-                std::vector<pmix::pdata> pdata;
-                std::string addressKey{"chans." + c.first + "." + std::to_string(i) + ".address"};
+                vector<pmix::pdata> pdata;
+                string addressKey("chans." + c.first + "." + to_string(i) + ".address");
                 pdata.emplace_back();
                 pdata.back().set_key(addressKey);
-                std::vector<pmix::info> info;
+                vector<pmix::info> info;
                 info.emplace_back(PMIX_WAIT, static_cast<int>(pdata.size()));
 
                 if (pdata.size() > 0) {
@@ -141,11 +284,11 @@ auto PMIxPlugin::Lookup() -> void
                         LOG(debug) << PMIxClient() << "pmix::lookup() not found: key=" << p.key;
                     } else if (p.value.type == PMIX_STRING) {
                         LOG(debug) << PMIxClient() << "pmix::lookup() found:"
-				   << " key=" << p.key << ",value=" << p.value.data.string;
-                        SetProperty<std::string>(p.key, p.value.data.string);
+                                   << " key=" << p.key << ",value=" << p.value.data.string;
+                        SetProperty<string>(p.key, p.value.data.string);
                     } else {
                         LOG(debug) << PMIxClient() << "pmix::lookup() wrong type returned: "
-				   << "key=" << p.key << ",type=" << p.value.type;
+                                   << "key=" << p.key << ",type=" << p.value.type;
                     }
                 }
             }
@@ -153,6 +296,14 @@ auto PMIxPlugin::Lookup() -> void
     }
 }
 
+auto PMIxPlugin::WaitForExitingAck() -> void
+{
+    unique_lock<mutex> lock(fStateChangeSubscriberMutex);
+    fExitingAcked.wait_for(lock, chrono::milliseconds(1000), [this]() {
+        return fExitingAckedByLastExternalController;
+    });
+}
+
 } /* namespace plugins */
 } /* namespace mq */
 } /* namespace fair */

fairmq/plugins/PMIx/PMIxPlugin.h

@@ -10,6 +10,7 @@
 #define FAIR_MQ_PLUGINS_PMIX
 
 #include "PMIx.hpp"
+#include "PMIxCommands.h"
 
 #include <fairmq/Plugin.h>
 #include <fairmq/Version.h>
@@ -39,23 +40,40 @@ class PMIxPlugin : public Plugin
          const std::string& homepage,
          PluginServices* pluginServices);
     ~PMIxPlugin();
-    auto PMIxClient() const -> std::string;
+
+    auto PMIxClient() const -> std::string { return fPMIxClient; };
 
   private:
-    pmix::proc fProc;
+    pmix::proc fProcess;
     pid_t fPid;
+    std::string fPMIxClient;
+    std::string fDeviceId;
+    pmix::Commands fCommands;
 
-    auto Init() -> void;
+    std::set<uint32_t> fStateChangeSubscribers;
+    uint32_t fLastExternalController;
+    bool fExitingAckedByLastExternalController;
+    std::condition_variable fExitingAcked;
+    std::mutex fStateChangeSubscriberMutex;
+
+    DeviceState fCurrentState;
+    DeviceState fLastState;
+
+    auto Init() -> pmix::proc;
     auto Publish() -> void;
     auto Fence() -> void;
+    auto Fence(const std::string& label) -> void;
     auto Lookup() -> void;
+
+    auto SubscribeForCommands() -> void;
+    auto WaitForExitingAck() -> void;
 };
 
 Plugin::ProgOptions PMIxProgramOptions()
 {
-    boost::program_options::options_description options{"PMIx Plugin"};
-    options.add_options()(
-        "pmix-dummy", boost::program_options::value<int>()->default_value(0), "Dummy.");
+    boost::program_options::options_description options("PMIx Plugin");
+    options.add_options()
+        ("pmix-dummy", boost::program_options::value<int>()->default_value(0), "Dummy.");
     return options;
 }
 

fairmq/plugins/PMIx/runPMIxCommandUI.cxx

@@ -0,0 +1,218 @@
+/********************************************************************************
+ *  Copyright (C) 2014-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include <fairmq/sdk/commands/Commands.h>
+#include <fairmq/States.h>
+
+#include <fairlogger/Logger.h>
+
+#include "PMIx.hpp"
+#include "PMIxCommands.h"
+
+#include <boost/program_options.hpp>
+
+#include <condition_variable>
+#include <algorithm>
+#include <cstdlib>
+#include <iostream>
+#include <mutex>
+#include <string>
+#include <termios.h> // raw mode console input
+#include <thread>
+#include <utility>
+#include <unistd.h>
+#include <vector>
+
+using namespace std;
+using namespace fair::mq::sdk::cmd;
+namespace bpo = boost::program_options;
+
+const std::map<fair::mq::Transition, fair::mq::State> expected =
+{
+    { fair::mq::Transition::InitDevice,   fair::mq::State::InitializingDevice },
+    { fair::mq::Transition::CompleteInit, fair::mq::State::Initialized },
+    { fair::mq::Transition::Bind,         fair::mq::State::Bound },
+    { fair::mq::Transition::Connect,      fair::mq::State::DeviceReady },
+    { fair::mq::Transition::InitTask,     fair::mq::State::Ready },
+    { fair::mq::Transition::Run,          fair::mq::State::Running },
+    { fair::mq::Transition::Stop,         fair::mq::State::Ready },
+    { fair::mq::Transition::ResetTask,    fair::mq::State::DeviceReady },
+    { fair::mq::Transition::ResetDevice,  fair::mq::State::Idle },
+    { fair::mq::Transition::End,          fair::mq::State::Exiting }
+};
+
+struct StateSubscription
+{
+    pmix::Commands& fCommands;
+
+    explicit StateSubscription(pmix::Commands& commands)
+        : fCommands(commands)
+    {
+        fCommands.Send(Cmds(make<SubscribeToStateChange>(600000)).Serialize(Format::JSON));
+    }
+
+    ~StateSubscription()
+    {
+        fCommands.Send(Cmds(make<UnsubscribeFromStateChange>()).Serialize(Format::JSON));
+        this_thread::sleep_for(chrono::milliseconds(100)); // give PMIx a chance to complete request
+    }
+};
+
+struct MiniTopo
+{
+    explicit MiniTopo(unsigned int n)
+        : fState(n, fair::mq::State::Ok)
+    {}
+
+    void WaitFor(const fair::mq::State state)
+    {
+        std::unique_lock<std::mutex> lk(fMtx);
+
+        fCV.wait(lk, [&](){
+            unsigned int count = std::count_if(fState.cbegin(), fState.cend(), [=](const auto& s) {
+                return s == state;
+            });
+
+            bool result = count == fState.size();
+            cout << "expecting " << state << " for " << fState.size() << " devices, found " << count << ", condition: " << result << endl;
+            return result;
+        });
+    }
+
+    void Update(uint32_t rank, const fair::mq::State state)
+    {
+        try {
+            {
+                std::lock_guard<std::mutex> lk(fMtx);
+                fState.at(rank - 1) = state;
+            }
+            fCV.notify_one();
+        } catch (const std::exception& e) {
+            LOG(error) << "Exception in Update: " << e.what();
+        }
+    }
+
+  private:
+    vector<fair::mq::State> fState;
+    std::mutex fMtx;
+    std::condition_variable fCV;
+};
+
+int main(int argc, char* argv[])
+{
+    try {
+        unsigned int numDevices;
+
+        bpo::options_description options("Common options");
+        options.add_options()
+            ("number-devices,n", bpo::value<unsigned int>(&numDevices)->default_value(0), "Number of devices (will be removed in the future)")
+            ("help,h", "Produce help message");
+
+        bpo::variables_map vm;
+        bpo::store(bpo::command_line_parser(argc, argv).options(options).run(), vm);
+
+        if (vm.count("help")) {
+            cout << "FairMQ DDS Command UI" << endl << options << endl;
+            cout << "Commands: [c] check state, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device, [k] complete init, [b] bind, [x] connect" << endl;
+            return EXIT_SUCCESS;
+        }
+
+        bpo::notify(vm);
+
+        fair::Logger::SetConsoleSeverity(fair::Severity::debug);
+        fair::Logger::SetConsoleColor(true);
+        fair::Logger::SetVerbosity(fair::Verbosity::low);
+
+        pmix::proc process;
+
+        if (!pmix::initialized()) {
+            process = pmix::init();
+            LOG(warn) << "pmix::init() OK: " << process << ", version=" << pmix::get_version();
+        }
+
+        pmix::proc all(process);
+        all.rank = pmix::rank::wildcard;
+        pmix::fence({all});
+        LOG(warn) << "pmix::fence() [pmix::init] OK";
+
+        MiniTopo topo(numDevices);
+        pmix::Commands commands(process);
+
+        commands.Subscribe([&](const string& msg, const pmix::proc& sender) {
+            // LOG(info) << "Received '" << msg << "' from " << sender;
+            Cmds cmds;
+            cmds.Deserialize(msg, Format::JSON);
+            // cout << "Received " << cmds.Size() << " command(s) with total size of " << msg.length() << " bytes: " << endl;
+            for (const auto& cmd : cmds) {
+                // cout << " > " << cmd->GetType() << endl;
+                switch (cmd->GetType()) {
+                    case Type::state_change: {
+                        cout << "Received state_change from " << static_cast<StateChange&>(*cmd).GetDeviceId() << ": " << static_cast<StateChange&>(*cmd).GetLastState() << "->" << static_cast<StateChange&>(*cmd).GetCurrentState() << endl;
+                        topo.Update(sender.rank, static_cast<StateChange&>(*cmd).GetCurrentState());
+                        if (static_cast<StateChange&>(*cmd).GetCurrentState() == fair::mq::State::Exiting) {
+                            commands.Send(Cmds(make<StateChangeExitingReceived>()).Serialize(Format::JSON), {sender});
+                        }
+                    }
+                    break;
+                    case Type::state_change_subscription:
+                        if (static_cast<StateChangeSubscription&>(*cmd).GetResult() != Result::Ok) {
+                            cout << "State change subscription failed for " << static_cast<StateChangeSubscription&>(*cmd).GetDeviceId() << endl;
+                        }
+                    break;
+                    case Type::state_change_unsubscription:
+                        if (static_cast<StateChangeUnsubscription&>(*cmd).GetResult() != Result::Ok) {
+                            cout << "State change unsubscription failed for " << static_cast<StateChangeUnsubscription&>(*cmd).GetDeviceId() << endl;
+                        }
+                    break;
+                    case Type::transition_status: {
+                        if (static_cast<TransitionStatus&>(*cmd).GetResult() == Result::Ok) {
+                            cout << "Device " << static_cast<TransitionStatus&>(*cmd).GetDeviceId() << " started to transition with " << static_cast<TransitionStatus&>(*cmd).GetTransition() << endl;
+                        } else {
+                            cout << "Device " << static_cast<TransitionStatus&>(*cmd).GetDeviceId() << " cannot transition with " << static_cast<TransitionStatus&>(*cmd).GetTransition() << endl;
+                        }
+                    }
+                    break;
+                    case Type::current_state:
+                        cout << "Device " << static_cast<CurrentState&>(*cmd).GetDeviceId() << " is in " << static_cast<CurrentState&>(*cmd).GetCurrentState() << " state" << endl;
+                    break;
+                    case Type::config:
+                        cout << "Received config for device " << static_cast<Config&>(*cmd).GetDeviceId() << ":\n" << static_cast<Config&>(*cmd).GetConfig() << endl;
+                    break;
+                    default:
+                        cout << "Unexpected/unknown command received: " << cmd->GetType() << endl;
+                        cout << "Origin: " << sender << endl;
+                    break;
+                }
+            }
+        });
+
+        pmix::fence({all});
+        LOG(warn) << "pmix::fence() [subscribed] OK";
+
+        StateSubscription stateSubscription(commands);
+
+        for (auto transition : { fair::mq::Transition::InitDevice,
+                                 fair::mq::Transition::CompleteInit,
+                                 fair::mq::Transition::Bind,
+                                 fair::mq::Transition::Connect,
+                                 fair::mq::Transition::InitTask,
+                                 fair::mq::Transition::Run,
+                                 fair::mq::Transition::Stop,
+                                 fair::mq::Transition::ResetTask,
+                                 fair::mq::Transition::ResetDevice,
+                                 fair::mq::Transition::End }) {
+            commands.Send(Cmds(make<ChangeState>(transition)).Serialize(Format::JSON));
+            topo.WaitFor(expected.at(transition));
+        }
+    } catch (exception& e) {
+        LOG(error) << "Error: " << e.what();
+        return EXIT_FAILURE;
+    }
+    LOG(warn) << "exiting";
+    return EXIT_SUCCESS;
+}

fairmq/run/startMQBenchmark.sh.in

@@ -6,6 +6,8 @@ maxIterations="0"
 msgSize="1000000"
 transport="zeromq"
 affinity="false"
+multipart="false"
+numParts="1"
 affinitySamp=""
 affinitySink=""
 
@@ -26,6 +28,14 @@ if [[ $4 =~ ^[a-z]+$ ]]; then
     affinity=$4
 fi
 
+if [[ $5 =~ ^[a-z]+$ ]]; then
+    multipart=$5
+fi
+
+if [[ $6 =~ ^[0-9]+$ ]]; then
+    numParts=$6
+fi
+
 
 echo "Starting benchmark with following settings:"
 
@@ -58,7 +68,8 @@ SAMPLER+=" --id bsampler1"
 SAMPLER+=" --transport $transport"
 SAMPLER+=" --severity debug"
 SAMPLER+=" --msg-size $msgSize"
-SAMPLER+=" --num-parts 1"
+SAMPLER+=" --multipart $multipart"
+SAMPLER+=" --num-parts $numParts"
 # SAMPLER+=" --msg-rate 1000"
 SAMPLER+=" --max-iterations $maxIterations"
 SAMPLER+=" --channel-config name=data,type=pair,method=bind,address=tcp://127.0.0.1:5555"
@@ -73,7 +84,7 @@ SINK+=" --id sink1"
 #SINK+=" --control static"
 SINK+=" --transport $transport"
 SINK+=" --severity debug"
-SINK+=" --multipart false"
+SINK+=" --multipart $multipart"
 SINK+=" --max-iterations $maxIterations"
 SINK+=" --channel-config name=data,type=pair,method=connect,address=tcp://127.0.0.1:5555"
 xterm -geometry 90x50+550+0 -hold -e $affinitySink @CMAKE_CURRENT_BINARY_DIR@/$SINK &

fairmq/sdk/AsioAsyncOp.h

@@ -15,7 +15,6 @@
 #include <asio/system_executor.hpp>
 #include <chrono>
 #include <exception>
-#include <fairlogger/Logger.h>
 #include <fairmq/sdk/Error.h>
 #include <fairmq/sdk/Traits.h>
 #include <functional>
@@ -24,6 +23,11 @@
 #include <type_traits>
 #include <utility>
 
+#include <fairlogger/Logger.h>
+#ifndef FAIR_LOG
+#define FAIR_LOG LOG
+#endif /* ifndef FAIR_LOG */
+
 namespace fair {
 namespace mq {
 namespace sdk {
@@ -49,7 +53,7 @@ struct AsioAsyncOpImpl : AsioAsyncOpImplBase<SignatureArgTypes...>
     using Executor2 = typename asio::associated_executor<Handler, Executor1>::type;
 
     /// Ctor
-    AsioAsyncOpImpl(const Executor1& ex1, Allocator1&& alloc1, Handler&& handler)
+    AsioAsyncOpImpl(const Executor1& ex1, Allocator1 alloc1, Handler&& handler)
         : fWork1(ex1)
         , fWork2(asio::get_associated_executor(handler, ex1))
         , fHandler(std::move(handler))
@@ -70,9 +74,9 @@ struct AsioAsyncOpImpl : AsioAsyncOpImplBase<SignatureArgTypes...>
                 try {
                     handler(ec, args...);
                 } catch (const std::exception& e) {
-                    LOG(error) << "Uncaught exception in AsioAsyncOp completion handler: " << e.what();
+                    FAIR_LOG(error) << "Uncaught exception in AsioAsyncOp completion handler: " << e.what();
                 } catch (...) {
-                    LOG(error) << "Unknown uncaught exception in AsioAsyncOp completion handler.";
+                    FAIR_LOG(error) << "Unknown uncaught exception in AsioAsyncOp completion handler.";
                 }
             },
             GetAlloc2());
@@ -149,7 +153,7 @@ struct AsioAsyncOp<Executor,
 
     /// Ctor with handler
     template<typename Handler>
-    AsioAsyncOp(Executor&& ex1, Allocator&& alloc1, Handler&& handler)
+    AsioAsyncOp(Executor ex1, Allocator alloc1, Handler&& handler)
         : AsioAsyncOp()
     {
         // Async operation type to be allocated and constructed
@@ -165,8 +169,8 @@ struct AsioAsyncOp<Executor,
         auto mem(std::allocator_traits<OpAllocator>::allocate(opAlloc, 1));
 
         // Construct object
-        auto ptr(new (mem) Op(std::forward<Executor>(ex1),
-                              std::forward<Allocator>(alloc1),
+        auto ptr(new (mem) Op(std::move(ex1),
+                              std::move(alloc1),
                               std::forward<Handler>(handler)));
 
         // Assign ownership to this object
@@ -177,8 +181,8 @@ struct AsioAsyncOp<Executor,
 
     /// Ctor with handler #2
     template<typename Handler>
-    AsioAsyncOp(Executor&& ex1, Handler&& handler)
-        : AsioAsyncOp(std::forward<Executor>(ex1), Allocator(), std::forward<Handler>(handler))
+    AsioAsyncOp(Executor ex1, Handler&& handler)
+        : AsioAsyncOp(std::move(ex1), Allocator(), std::forward<Handler>(handler))
     {}
 
     /// Ctor with handler #3

fairmq/sdk/DDSEnvironment.cxx

@@ -55,12 +55,17 @@ struct DDSEnvironment::Impl
             setenv("HOME", fConfigHome.c_str(), 1);
         }
 
-        std::istringstream cmd;
-        cmd.str("DDS_CFG=`dds-user-defaults --ignore-default-sid -p`\n"
-                "if [ -z \"$DDS_CFG\" ]; then\n"
-                "  mkdir -p \"$HOME/.DDS\"\n"
-                "  dds-user-defaults --ignore-default-sid -d -c \"$HOME/.DDS/DDS.cfg\"\n"
-                "fi");
+        std::stringstream cmd;
+#ifdef __APPLE__
+        // On macOS System Integrity Protection might filter out the DYLD_LIBRARY_PATH, so we pass it
+        // through explicitely here.
+        cmd << "export " << fgLdVar << "=" << GetEnv(fgLdVar) << "\n";
+#endif
+        cmd << "DDS_CFG=`dds-user-defaults --ignore-default-sid -p`\n"
+               "if [ -z \"$DDS_CFG\" ]; then\n"
+               "  mkdir -p \"$HOME/.DDS\"\n"
+               "  dds-user-defaults --ignore-default-sid -d -c \"$HOME/.DDS/DDS.cfg\"\n"
+               "fi\n";
         std::system(cmd.str().c_str());
     }
 
@@ -72,20 +77,23 @@ struct DDSEnvironment::Impl
         setenv("PATH", path.c_str(), 1);
     }
 
-    auto SetupDynamicLoader() -> void
+    auto GenerateDDSLibDir() const -> Path
     {
-#ifdef __APPLE__
-        std::string ldVar("DYLD_LIBRARY_PATH");
-#else
-        std::string ldVar("LD_LIBRARY_PATH");
-#endif
-        std::string ld(GetEnv(ldVar));
-        Path ddsLibDir = (fLocation == DDSInstallPrefix) ? DDSLibraryDir : fLocation / Path("lib");
-        ld = ddsLibDir.string() + std::string(":") + ld;
-        setenv(ldVar.c_str(), ld.c_str(), 1);
+        return {(fLocation == DDSInstallPrefix) ? DDSLibraryDir : fLocation / Path("lib")};
     }
 
-    auto GetEnv(const std::string& key) -> std::string
+    auto SetupDynamicLoader() -> void
+    {
+        std::string ld(GetEnv(fgLdVar));
+        if (ld.empty()) {
+            ld = GenerateDDSLibDir().string();
+        } else {
+            ld = GenerateDDSLibDir().string() + std::string(":") + ld;
+        }
+        setenv(fgLdVar.c_str(), ld.c_str(), 1);
+    }
+
+    auto GetEnv(const std::string& key) const -> std::string
     {
         auto value = std::getenv(key.c_str());
         if (value) {
@@ -100,6 +108,11 @@ struct DDSEnvironment::Impl
 
     Path fLocation;
     Path fConfigHome;
+#ifdef __APPLE__
+    std::string const fgLdVar = "DYLD_LIBRARY_PATH";
+#else
+    std::string const fgLdVar = "LD_LIBRARY_PATH";
+#endif
 };
 
 DDSEnvironment::DDSEnvironment()

fairmq/sdk/DDSSession.cxx

@@ -135,8 +135,6 @@ struct DDSSession::Impl
     dds::intercom_api::CCustomCmd fDDSCustomCmd;
     Id fId;
     bool fStopOnDestruction;
-    mutable std::mutex fMtx;
-    std::unordered_map<DDSChannel::Id, DDSTask::Id> fTaskIdByChannelIdMap;
 };
 
 DDSSession::DDSSession(DDSEnvironment env)
@@ -355,25 +353,13 @@ void DDSSession::UnsubscribeFromCommands()
     fImpl->fDDSCustomCmd.unsubscribe();
 }
 
-void DDSSession::SendCommand(const std::string& cmd) { fImpl->fDDSCustomCmd.send(cmd, ""); }
+void DDSSession::SendCommand(const std::string& cmd, const std::string& path /* = "" */) { fImpl->fDDSCustomCmd.send(cmd, path); }
 
 void DDSSession::SendCommand(const std::string& cmd, DDSChannel::Id recipient)
 {
     fImpl->fDDSCustomCmd.send(cmd, std::to_string(recipient));
 }
 
-auto DDSSession::UpdateChannelToTaskAssociation(DDSChannel::Id channelId, DDSTask::Id taskId) -> void
-{
-    std::lock_guard<std::mutex> lk(fImpl->fMtx);
-    fImpl->fTaskIdByChannelIdMap[channelId] = taskId;
-}
-
-auto DDSSession::GetTaskId(DDSChannel::Id channelId) const -> DDSTask::Id
-{
-    std::lock_guard<std::mutex> lk(fImpl->fMtx);
-    return fImpl->fTaskIdByChannelIdMap.at(channelId);
-}
-
 auto operator<<(std::ostream& os, const DDSSession& session) -> std::ostream&
 {
     return os << "$DDS_SESSION_ID: " << session.GetId();

fairmq/sdk/DDSSession.h

@@ -101,9 +101,8 @@ class DDSSession
     void StartDDSService();
     void SubscribeToCommands(std::function<void(const std::string& msg, const std::string& condition, uint64_t senderId)>);
     void UnsubscribeFromCommands();
-    void SendCommand(const std::string&);
+    void SendCommand(const std::string&, const std::string& = "");
     void SendCommand(const std::string&, DDSChannel::Id);
-    auto UpdateChannelToTaskAssociation(DDSChannel::Id, DDSTask::Id) -> void;
     auto GetTaskId(DDSChannel::Id) const -> DDSTask::Id;
 
     friend auto operator<<(std::ostream& os, const DDSSession& session) -> std::ostream&;

fairmq/sdk/DDSTopology.cxx

@@ -64,22 +64,23 @@ auto DDSTopology::GetNumRequiredAgents() const -> int
     return fImpl->fTopo.getRequiredNofAgents();
 }
 
-auto DDSTopology::GetTasks() const -> std::vector<DDSTask>
+auto DDSTopology::GetTasks(const std::string& path /* = "" */) const -> std::vector<DDSTask>
 {
     std::vector<DDSTask> list;
-    list.reserve(GetNumRequiredAgents());
 
-    auto itPair = fImpl->fTopo.getRuntimeTaskIterator(
-        [](const dds::topology_api::STopoRuntimeTask::FilterIterator_t::value_type&) -> bool {
-            return true;
-        });
+    dds::topology_api::STopoRuntimeTask::FilterIteratorPair_t itPair;
+    if (path == "") {
+        itPair = fImpl->fTopo.getRuntimeTaskIterator(nullptr); // passing nullptr will get all tasks
+    } else {
+        itPair = fImpl->fTopo.getRuntimeTaskIteratorMatchingPath(path);
+    }
     auto tasks = boost::make_iterator_range(itPair.first, itPair.second);
 
     for (const auto& task : tasks) {
-        LOG(debug) << "Found task with id: " << task.first << ", "
-                   << "Path: " << task.second.m_taskPath << ", "
-                   << "Collection id: " << task.second.m_taskCollectionId << ", "
-                   << "Name: " << task.second.m_task->getName() << "_" << task.second.m_taskIndex;
+        // LOG(debug) << "Found task with id: " << task.first << ", "
+        //            << "Path: " << task.second.m_taskPath << ", "
+        //            << "Collection id: " << task.second.m_taskCollectionId << ", "
+        //            << "Name: " << task.second.m_task->getName() << "_" << task.second.m_taskIndex;
         list.emplace_back(task.first, task.second.m_taskCollectionId);
     }
 
@@ -90,10 +91,7 @@ auto DDSTopology::GetCollections() const -> std::vector<DDSCollection>
 {
     std::vector<DDSCollection> list;
 
-    auto itPair = fImpl->fTopo.getRuntimeCollectionIterator(
-        [](const dds::topology_api::STopoRuntimeCollection::FilterIterator_t::value_type&) -> bool {
-            return true;
-        });
+    auto itPair = fImpl->fTopo.getRuntimeCollectionIterator(nullptr); // passing nullptr will get all collections
     auto collections = boost::make_iterator_range(itPair.first, itPair.second);
 
     for (const auto& c : collections) {

fairmq/sdk/DDSTopology.h

@@ -53,8 +53,8 @@ class DDSTopology
     /// @brief Get number of required agents for this topology
     auto GetNumRequiredAgents() const -> int;
 
-    /// @brief Get list of tasks in this topology
-    auto GetTasks() const -> std::vector<DDSTask>;
+    /// @brief Get list of tasks in this topology, optionally matching provided path
+    auto GetTasks(const std::string& = "") const -> std::vector<DDSTask>;
 
     /// @brief Get list of tasks in this topology
     auto GetCollections() const -> std::vector<DDSCollection>;

fairmq/sdk/Error.cxx

@@ -27,6 +27,10 @@ std::string ErrorCategory::message(int ev) const
             return "async operation canceled";
         case ErrorCode::DeviceChangeStateFailed:
             return "failed to change state of a fairmq device";
+        case ErrorCode::DeviceGetPropertiesFailed:
+            return "failed to get fairmq device properties";
+        case ErrorCode::DeviceSetPropertiesFailed:
+            return "failed to set fairmq device properties";
         default:
             return "(unrecognized error)";
     }

fairmq/sdk/Error.h

@@ -37,7 +37,9 @@ enum class ErrorCode
     OperationInProgress = 10,
     OperationTimeout,
     OperationCanceled,
-    DeviceChangeStateFailed
+    DeviceChangeStateFailed,
+    DeviceGetPropertiesFailed,
+    DeviceSetPropertiesFailed
 };
 
 std::error_code MakeErrorCode(ErrorCode);

fairmq/sdk/commands/CMakeLists.txt

@@ -37,6 +37,11 @@ target_link_libraries(${target}
   PRIVATE
   ${_flatbuffers}
 )
+set_target_properties(${target} PROPERTIES
+  VERSION ${PROJECT_VERSION}
+  SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
+  OUTPUT_NAME FairMQ${target}
+)
 
 target_include_directories(${target}
   PUBLIC

fairmq/sdk/commands/Commands.cxx

@@ -52,21 +52,21 @@ array<string, 17> typeNames =
         "CheckState",
         "ChangeState",
         "DumpConfig",
-        "SubscribeToHeartbeats",
-        "UnsubscribeFromHeartbeats",
         "SubscribeToStateChange",
         "UnsubscribeFromStateChange",
         "StateChangeExitingReceived",
+        "GetProperties",
+        "SetProperties",
+        "SubscriptionHeartbeat",
 
         "CurrentState",
         "TransitionStatus",
         "Config",
-        "HeartbeatSubscription",
-        "HeartbeatUnsubscription",
-        "Heartbeat",
         "StateChangeSubscription",
         "StateChangeUnsubscription",
-        "StateChange"
+        "StateChange",
+        "Properties",
+        "PropertiesSet"
     }
 };
 
@@ -154,20 +154,20 @@ array<FBCmd, 17> typeToFBCmd =
         FBCmd::FBCmd_check_state,
         FBCmd::FBCmd_change_state,
         FBCmd::FBCmd_dump_config,
-        FBCmd::FBCmd_subscribe_to_heartbeats,
-        FBCmd::FBCmd_unsubscribe_from_heartbeats,
         FBCmd::FBCmd_subscribe_to_state_change,
         FBCmd::FBCmd_unsubscribe_from_state_change,
         FBCmd::FBCmd_state_change_exiting_received,
+        FBCmd::FBCmd_get_properties,
+        FBCmd::FBCmd_set_properties,
+        FBCmd::FBCmd_subscription_heartbeat,
         FBCmd::FBCmd_current_state,
         FBCmd::FBCmd_transition_status,
         FBCmd::FBCmd_config,
-        FBCmd::FBCmd_heartbeat_subscription,
-        FBCmd::FBCmd_heartbeat_unsubscription,
-        FBCmd::FBCmd_heartbeat,
         FBCmd::FBCmd_state_change_subscription,
         FBCmd::FBCmd_state_change_unsubscription,
-        FBCmd::FBCmd_state_change
+        FBCmd::FBCmd_state_change,
+        FBCmd::FBCmd_properties,
+        FBCmd::FBCmd_properties_set
     }
 };
 
@@ -177,20 +177,20 @@ array<Type, 17> fbCmdToType =
         Type::check_state,
         Type::change_state,
         Type::dump_config,
-        Type::subscribe_to_heartbeats,
-        Type::unsubscribe_from_heartbeats,
         Type::subscribe_to_state_change,
         Type::unsubscribe_from_state_change,
         Type::state_change_exiting_received,
+        Type::get_properties,
+        Type::set_properties,
+        Type::subscription_heartbeat,
         Type::current_state,
         Type::transition_status,
         Type::config,
-        Type::heartbeat_subscription,
-        Type::heartbeat_unsubscription,
-        Type::heartbeat,
         Type::state_change_subscription,
         Type::state_change_unsubscription,
-        Type::state_change
+        Type::state_change,
+        Type::properties,
+        Type::properties_set
     }
 };
 
@@ -229,16 +229,11 @@ string Cmds::Serialize(const Format type) const
                 cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
             }
             break;
-            case Type::subscribe_to_heartbeats: {
-                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
-            }
-            break;
-            case Type::unsubscribe_from_heartbeats: {
-                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
-            }
             break;
             case Type::subscribe_to_state_change: {
+                auto _cmd = static_cast<SubscribeToStateChange&>(*cmd);
                 cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_interval(_cmd.GetInterval());
             }
             break;
             case Type::unsubscribe_from_state_change: {
@@ -249,70 +244,115 @@ string Cmds::Serialize(const Format type) const
                 cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
             }
             break;
+            case Type::get_properties: {
+                auto _cmd = static_cast<GetProperties&>(*cmd);
+                auto query = fbb.CreateString(_cmd.GetQuery());
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_request_id(_cmd.GetRequestId());
+                cmdBuilder->add_property_query(query);
+            }
+            break;
+            case Type::set_properties: {
+                auto _cmd = static_cast<SetProperties&>(*cmd);
+                std::vector<flatbuffers::Offset<FBProperty>> propsVector;
+                for (auto const& e : _cmd.GetProps()) {
+                    auto const key(fbb.CreateString(e.first));
+                    auto const val(fbb.CreateString(e.second));
+                    propsVector.push_back(CreateFBProperty(fbb, key, val));
+                }
+                auto props = fbb.CreateVector(propsVector);
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_request_id(_cmd.GetRequestId());
+                cmdBuilder->add_properties(props);
+            }
+            break;
+            case Type::subscription_heartbeat: {
+                auto _cmd = static_cast<SubscriptionHeartbeat&>(*cmd);
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_interval(_cmd.GetInterval());
+            }
+            break;
             case Type::current_state: {
-                auto deviceId = fbb.CreateString(static_cast<CurrentState&>(*cmd).GetDeviceId());
+                auto _cmd = static_cast<CurrentState&>(*cmd);
+                auto deviceId = fbb.CreateString(_cmd.GetDeviceId());
                 cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
                 cmdBuilder->add_device_id(deviceId);
-                cmdBuilder->add_current_state(GetFBState(static_cast<CurrentState&>(*cmd).GetCurrentState()));
+                cmdBuilder->add_current_state(GetFBState(_cmd.GetCurrentState()));
             }
             break;
             case Type::transition_status: {
-                auto deviceId = fbb.CreateString(static_cast<TransitionStatus&>(*cmd).GetDeviceId());
+                auto _cmd = static_cast<TransitionStatus&>(*cmd);
+                auto deviceId = fbb.CreateString(_cmd.GetDeviceId());
                 cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
                 cmdBuilder->add_device_id(deviceId);
-                cmdBuilder->add_result(GetFBResult(static_cast<TransitionStatus&>(*cmd).GetResult()));
-                cmdBuilder->add_transition(GetFBTransition(static_cast<TransitionStatus&>(*cmd).GetTransition()));
+                cmdBuilder->add_task_id(_cmd.GetTaskId());
+                cmdBuilder->add_result(GetFBResult(_cmd.GetResult()));
+                cmdBuilder->add_transition(GetFBTransition(_cmd.GetTransition()));
             }
             break;
             case Type::config: {
-                auto deviceId = fbb.CreateString(static_cast<Config&>(*cmd).GetDeviceId());
-                auto config = fbb.CreateString(static_cast<Config&>(*cmd).GetConfig());
+                auto _cmd = static_cast<Config&>(*cmd);
+                auto deviceId = fbb.CreateString(_cmd.GetDeviceId());
+                auto config = fbb.CreateString(_cmd.GetConfig());
                 cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
                 cmdBuilder->add_device_id(deviceId);
                 cmdBuilder->add_config_string(config);
             }
             break;
-            case Type::heartbeat_subscription: {
-                auto deviceId = fbb.CreateString(static_cast<HeartbeatSubscription&>(*cmd).GetDeviceId());
-                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
-                cmdBuilder->add_device_id(deviceId);
-                cmdBuilder->add_result(GetFBResult(static_cast<HeartbeatSubscription&>(*cmd).GetResult()));
-            }
-            break;
-            case Type::heartbeat_unsubscription: {
-                auto deviceId = fbb.CreateString(static_cast<HeartbeatUnsubscription&>(*cmd).GetDeviceId());
-                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
-                cmdBuilder->add_device_id(deviceId);
-                cmdBuilder->add_result(GetFBResult(static_cast<HeartbeatUnsubscription&>(*cmd).GetResult()));
-            }
-            break;
-            case Type::heartbeat: {
-                auto deviceId = fbb.CreateString(static_cast<Heartbeat&>(*cmd).GetDeviceId());
-                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
-                cmdBuilder->add_device_id(deviceId);
-            }
-            break;
             case Type::state_change_subscription: {
-                auto deviceId = fbb.CreateString(static_cast<StateChangeSubscription&>(*cmd).GetDeviceId());
+                auto _cmd = static_cast<StateChangeSubscription&>(*cmd);
+                auto deviceId = fbb.CreateString(_cmd.GetDeviceId());
                 cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
                 cmdBuilder->add_device_id(deviceId);
-                cmdBuilder->add_result(GetFBResult(static_cast<StateChangeSubscription&>(*cmd).GetResult()));
+                cmdBuilder->add_task_id(_cmd.GetTaskId());
+                cmdBuilder->add_result(GetFBResult(_cmd.GetResult()));
             }
             break;
             case Type::state_change_unsubscription: {
-                auto deviceId = fbb.CreateString(static_cast<StateChangeUnsubscription&>(*cmd).GetDeviceId());
+                auto _cmd = static_cast<StateChangeUnsubscription&>(*cmd);
+                auto deviceId = fbb.CreateString(_cmd.GetDeviceId());
                 cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
                 cmdBuilder->add_device_id(deviceId);
-                cmdBuilder->add_result(GetFBResult(static_cast<StateChangeUnsubscription&>(*cmd).GetResult()));
+                cmdBuilder->add_task_id(_cmd.GetTaskId());
+                cmdBuilder->add_result(GetFBResult(_cmd.GetResult()));
             }
             break;
             case Type::state_change: {
-                auto deviceId = fbb.CreateString(static_cast<StateChange&>(*cmd).GetDeviceId());
+                auto _cmd = static_cast<StateChange&>(*cmd);
+                auto deviceId = fbb.CreateString(_cmd.GetDeviceId());
                 cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
                 cmdBuilder->add_device_id(deviceId);
-                cmdBuilder->add_task_id(static_cast<StateChange&>(*cmd).GetTaskId());
-                cmdBuilder->add_last_state(GetFBState(static_cast<StateChange&>(*cmd).GetLastState()));
-                cmdBuilder->add_current_state(GetFBState(static_cast<StateChange&>(*cmd).GetCurrentState()));
+                cmdBuilder->add_task_id(_cmd.GetTaskId());
+                cmdBuilder->add_last_state(GetFBState(_cmd.GetLastState()));
+                cmdBuilder->add_current_state(GetFBState(_cmd.GetCurrentState()));
+            }
+            break;
+            case Type::properties: {
+                auto _cmd = static_cast<Properties&>(*cmd);
+                auto deviceId = fbb.CreateString(_cmd.GetDeviceId());
+
+                std::vector<flatbuffers::Offset<FBProperty>> propsVector;
+                for (const auto& e : _cmd.GetProps()) {
+                    auto key = fbb.CreateString(e.first);
+                    auto val = fbb.CreateString(e.second);
+                    auto prop = CreateFBProperty(fbb, key, val);
+                    propsVector.push_back(prop);
+                }
+                auto props = fbb.CreateVector(propsVector);
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_device_id(deviceId);
+                cmdBuilder->add_request_id(_cmd.GetRequestId());
+                cmdBuilder->add_result(GetFBResult(_cmd.GetResult()));
+                cmdBuilder->add_properties(props);
+            }
+            break;
+            case Type::properties_set: {
+                auto _cmd = static_cast<PropertiesSet&>(*cmd);
+                auto deviceId = fbb.CreateString(_cmd.GetDeviceId());
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_device_id(deviceId);
+                cmdBuilder->add_request_id(_cmd.GetRequestId());
+                cmdBuilder->add_result(GetFBResult(_cmd.GetResult()));
             }
             break;
             default:
@@ -376,14 +416,8 @@ void Cmds::Deserialize(const string& str, const Format type)
             case FBCmd_dump_config:
                 fCmds.emplace_back(make<DumpConfig>());
             break;
-            case FBCmd_subscribe_to_heartbeats:
-                fCmds.emplace_back(make<SubscribeToHeartbeats>());
-            break;
-            case FBCmd_unsubscribe_from_heartbeats:
-                fCmds.emplace_back(make<UnsubscribeFromHeartbeats>());
-            break;
             case FBCmd_subscribe_to_state_change:
-                fCmds.emplace_back(make<SubscribeToStateChange>());
+                fCmds.emplace_back(make<SubscribeToStateChange>(cmdPtr.interval()));
             break;
             case FBCmd_unsubscribe_from_state_change:
                 fCmds.emplace_back(make<UnsubscribeFromStateChange>());
@@ -391,33 +425,49 @@ void Cmds::Deserialize(const string& str, const Format type)
             case FBCmd_state_change_exiting_received:
                 fCmds.emplace_back(make<StateChangeExitingReceived>());
             break;
+            case FBCmd_get_properties:
+                fCmds.emplace_back(make<GetProperties>(cmdPtr.request_id(), cmdPtr.property_query()->str()));
+            break;
+            case FBCmd_set_properties: {
+                std::vector<std::pair<std::string, std::string>> properties;
+                auto props = cmdPtr.properties();
+                for (unsigned int j = 0; j < props->size(); ++j) {
+                    properties.emplace_back(props->Get(j)->key()->str(), props->Get(j)->value()->str());
+                }
+                fCmds.emplace_back(make<SetProperties>(cmdPtr.request_id(), properties));
+            } break;
+            case FBCmd_subscription_heartbeat:
+                fCmds.emplace_back(make<SubscriptionHeartbeat>(cmdPtr.interval()));
+            break;
             case FBCmd_current_state:
                 fCmds.emplace_back(make<CurrentState>(cmdPtr.device_id()->str(), GetMQState(cmdPtr.current_state())));
             break;
             case FBCmd_transition_status:
-                fCmds.emplace_back(make<TransitionStatus>(cmdPtr.device_id()->str(), GetResult(cmdPtr.result()), GetMQTransition(cmdPtr.transition())));
+                fCmds.emplace_back(make<TransitionStatus>(cmdPtr.device_id()->str(), cmdPtr.task_id(), GetResult(cmdPtr.result()), GetMQTransition(cmdPtr.transition())));
             break;
             case FBCmd_config:
                 fCmds.emplace_back(make<Config>(cmdPtr.device_id()->str(), cmdPtr.config_string()->str()));
             break;
-            case FBCmd_heartbeat_subscription:
-                fCmds.emplace_back(make<HeartbeatSubscription>(cmdPtr.device_id()->str(), GetResult(cmdPtr.result())));
-            break;
-            case FBCmd_heartbeat_unsubscription:
-                fCmds.emplace_back(make<HeartbeatUnsubscription>(cmdPtr.device_id()->str(), GetResult(cmdPtr.result())));
-            break;
-            case FBCmd_heartbeat:
-                fCmds.emplace_back(make<Heartbeat>(cmdPtr.device_id()->str()));
-            break;
             case FBCmd_state_change_subscription:
-                fCmds.emplace_back(make<StateChangeSubscription>(cmdPtr.device_id()->str(), GetResult(cmdPtr.result())));
+                fCmds.emplace_back(make<StateChangeSubscription>(cmdPtr.device_id()->str(), cmdPtr.task_id(), GetResult(cmdPtr.result())));
             break;
             case FBCmd_state_change_unsubscription:
-                fCmds.emplace_back(make<StateChangeUnsubscription>(cmdPtr.device_id()->str(), GetResult(cmdPtr.result())));
+                fCmds.emplace_back(make<StateChangeUnsubscription>(cmdPtr.device_id()->str(), cmdPtr.task_id(), GetResult(cmdPtr.result())));
             break;
             case FBCmd_state_change:
                 fCmds.emplace_back(make<StateChange>(cmdPtr.device_id()->str(), cmdPtr.task_id(), GetMQState(cmdPtr.last_state()), GetMQState(cmdPtr.current_state())));
             break;
+            case FBCmd_properties: {
+                std::vector<std::pair<std::string, std::string>> properties;
+                auto props = cmdPtr.properties();
+                for (unsigned int j = 0; j < props->size(); ++j) {
+                    properties.emplace_back(props->Get(j)->key()->str(), props->Get(j)->value()->str());
+                }
+                fCmds.emplace_back(make<Properties>(cmdPtr.device_id()->str(), cmdPtr.request_id(), GetResult(cmdPtr.result()), properties));
+            } break;
+            case FBCmd_properties_set:
+                fCmds.emplace_back(make<PropertiesSet>(cmdPtr.device_id()->str(), cmdPtr.request_id(), GetResult(cmdPtr.result())));
+            break;
             default:
                 throw CommandFormatError("unrecognized command type given to fair::mq::cmd::Cmds::Deserialize()");
             break;

fairmq/sdk/commands/Commands.h

@@ -42,26 +42,27 @@ enum class Type : int
     check_state,                   // args: { }
     change_state,                  // args: { transition }
     dump_config,                   // args: { }
-    subscribe_to_heartbeats,       // args: { }
-    unsubscribe_from_heartbeats,   // args: { }
     subscribe_to_state_change,     // args: { }
     unsubscribe_from_state_change, // args: { }
     state_change_exiting_received, // args: { }
+    get_properties,                // args: { request_id, property_query }
+    set_properties,                // args: { request_id, properties }
+    subscription_heartbeat,        // args: { interval }
 
     current_state,                 // args: { device_id, current_state }
-    transition_status,             // args: { device_id, Result, transition }
+    transition_status,             // args: { device_id, task_id, Result, transition }
     config,                        // args: { device_id, config_string }
-    heartbeat_subscription,        // args: { device_id, Result }
-    heartbeat_unsubscription,      // args: { device_id, Result }
-    heartbeat,                     // args: { device_id }
-    state_change_subscription,     // args: { device_id, Result }
-    state_change_unsubscription,   // args: { device_id, Result }
-    state_change                   // args: { device_id, task_id, last_state, current_state }
+    state_change_subscription,     // args: { device_id, task_id, Result }
+    state_change_unsubscription,   // args: { device_id, task_id, Result }
+    state_change,                  // args: { device_id, task_id, last_state, current_state }
+    properties,                    // args: { device_id, request_id, Result, properties }
+    properties_set                 // args: { device_id, request_id, Result }
 };
 
 struct Cmd
 {
     explicit Cmd(const Type type) : fType(type) {}
+    virtual ~Cmd() = default;
 
     Type GetType() const { return fType; }
 
@@ -93,19 +94,18 @@ struct DumpConfig : Cmd
     explicit DumpConfig() : Cmd(Type::dump_config) {}
 };
 
-struct SubscribeToHeartbeats : Cmd
-{
-    explicit SubscribeToHeartbeats() : Cmd(Type::subscribe_to_heartbeats) {}
-};
-
-struct UnsubscribeFromHeartbeats : Cmd
-{
-    explicit UnsubscribeFromHeartbeats() : Cmd(Type::unsubscribe_from_heartbeats) {}
-};
-
 struct SubscribeToStateChange : Cmd
 {
-    explicit SubscribeToStateChange() : Cmd(Type::subscribe_to_state_change) {}
+    explicit SubscribeToStateChange(int64_t interval)
+        : Cmd(Type::subscribe_to_state_change)
+        , fInterval(interval)
+    {}
+
+    int64_t GetInterval() const { return fInterval; }
+    void SetInterval(int64_t interval) { fInterval = interval; }
+
+  private:
+    int64_t fInterval;
 };
 
 struct UnsubscribeFromStateChange : Cmd
@@ -118,6 +118,56 @@ struct StateChangeExitingReceived : Cmd
     explicit StateChangeExitingReceived() : Cmd(Type::state_change_exiting_received) {}
 };
 
+struct GetProperties : Cmd
+{
+    GetProperties(std::size_t request_id, std::string query)
+        : Cmd(Type::get_properties)
+        , fRequestId(request_id)
+        , fQuery(std::move(query))
+    {}
+
+    auto GetRequestId() const -> std::size_t { return fRequestId; }
+    auto SetRequestId(std::size_t requestId) -> void { fRequestId = requestId; }
+    auto GetQuery() const -> std::string { return fQuery; }
+    auto SetQuery(std::string query) -> void { fQuery = std::move(query); }
+
+  private:
+    std::size_t fRequestId;
+    std::string fQuery;
+};
+
+struct SetProperties : Cmd
+{
+    SetProperties(std::size_t request_id, std::vector<std::pair<std::string, std::string>> properties)
+        : Cmd(Type::set_properties)
+        , fRequestId(request_id)
+        , fProperties(std::move(properties))
+    {}
+
+    auto GetRequestId() const -> std::size_t { return fRequestId; }
+    auto SetRequestId(std::size_t requestId) -> void { fRequestId = requestId; }
+    auto GetProps() const -> std::vector<std::pair<std::string, std::string>> { return fProperties; }
+    auto SetProps(std::vector<std::pair<std::string, std::string>> properties) -> void { fProperties = std::move(properties); }
+
+  private:
+    std::size_t fRequestId;
+    std::vector<std::pair<std::string, std::string>> fProperties;
+};
+
+struct SubscriptionHeartbeat : Cmd
+{
+    explicit SubscriptionHeartbeat(int64_t interval)
+        : Cmd(Type::subscription_heartbeat)
+        , fInterval(interval)
+    {}
+
+    int64_t GetInterval() const { return fInterval; }
+    void SetInterval(int64_t interval) { fInterval = interval; }
+
+  private:
+    int64_t fInterval;
+};
+
 struct CurrentState : Cmd
 {
     explicit CurrentState(const std::string& id, State currentState)
@@ -138,15 +188,18 @@ struct CurrentState : Cmd
 
 struct TransitionStatus : Cmd
 {
-    explicit TransitionStatus(const std::string& id, const Result result, const Transition transition)
+    explicit TransitionStatus(const std::string& deviceId, const uint64_t taskId, const Result result, const Transition transition)
         : Cmd(Type::transition_status)
-        , fDeviceId(id)
+        , fDeviceId(deviceId)
+        , fTaskId(taskId)
         , fResult(result)
         , fTransition(transition)
     {}
 
     std::string GetDeviceId() const { return fDeviceId; }
     void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    uint64_t GetTaskId() const { return fTaskId; }
+    void SetTaskId(const uint64_t taskId) { fTaskId = taskId; }
     Result GetResult() const { return fResult; }
     void SetResult(const Result result) { fResult = result; }
     Transition GetTransition() const { return fTransition; }
@@ -154,6 +207,7 @@ struct TransitionStatus : Cmd
 
   private:
     std::string fDeviceId;
+    uint64_t fTaskId;
     Result fResult;
     Transition fTransition;
 };
@@ -176,89 +230,47 @@ struct Config : Cmd
     std::string fConfig;
 };
 
-struct HeartbeatSubscription : Cmd
-{
-    explicit HeartbeatSubscription(const std::string& id, const Result result)
-        : Cmd(Type::heartbeat_subscription)
-        , fDeviceId(id)
-        , fResult(result)
-    {}
-
-    std::string GetDeviceId() const { return fDeviceId; }
-    void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
-    Result GetResult() const { return fResult; }
-    void SetResult(const Result result) { fResult = result; }
-
-  private:
-    std::string fDeviceId;
-    Result fResult;
-};
-
-struct HeartbeatUnsubscription : Cmd
-{
-    explicit HeartbeatUnsubscription(const std::string& id, const Result result)
-        : Cmd(Type::heartbeat_unsubscription)
-        , fDeviceId(id)
-        , fResult(result)
-    {}
-
-    std::string GetDeviceId() const { return fDeviceId; }
-    void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
-    Result GetResult() const { return fResult; }
-    void SetResult(const Result result) { fResult = result; }
-
-  private:
-    std::string fDeviceId;
-    Result fResult;
-};
-
-struct Heartbeat : Cmd
-{
-    explicit Heartbeat(const std::string& id)
-        : Cmd(Type::heartbeat)
-        , fDeviceId(id)
-    {}
-
-    std::string GetDeviceId() const { return fDeviceId; }
-    void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
-
-  private:
-    std::string fDeviceId;
-};
-
 struct StateChangeSubscription : Cmd
 {
-    explicit StateChangeSubscription(const std::string& id, const Result result)
+    explicit StateChangeSubscription(const std::string& id, const uint64_t taskId, const Result result)
         : Cmd(Type::state_change_subscription)
         , fDeviceId(id)
+        , fTaskId(taskId)
         , fResult(result)
     {}
 
     std::string GetDeviceId() const { return fDeviceId; }
     void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    uint64_t GetTaskId() const { return fTaskId; }
+    void SetTaskId(const uint64_t taskId) { fTaskId = taskId; }
     Result GetResult() const { return fResult; }
     void SetResult(const Result result) { fResult = result; }
 
   private:
     std::string fDeviceId;
+    uint64_t fTaskId;
     Result fResult;
 };
 
 struct StateChangeUnsubscription : Cmd
 {
-    explicit StateChangeUnsubscription(const std::string& id, const Result result)
+    explicit StateChangeUnsubscription(const std::string& id, const uint64_t taskId, const Result result)
         : Cmd(Type::state_change_unsubscription)
         , fDeviceId(id)
+        , fTaskId(taskId)
         , fResult(result)
     {}
 
     std::string GetDeviceId() const { return fDeviceId; }
     void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    uint64_t GetTaskId() const { return fTaskId; }
+    void SetTaskId(const uint64_t taskId) { fTaskId = taskId; }
     Result GetResult() const { return fResult; }
     void SetResult(const Result result) { fResult = result; }
 
   private:
     std::string fDeviceId;
+    uint64_t fTaskId;
     Result fResult;
 };
 
@@ -288,6 +300,53 @@ struct StateChange : Cmd
     fair::mq::State fCurrentState;
 };
 
+struct Properties : Cmd
+{
+    Properties(std::string deviceId, std::size_t requestId, const Result result, std::vector<std::pair<std::string, std::string>> properties)
+        : Cmd(Type::properties)
+        , fDeviceId(std::move(deviceId))
+        , fRequestId(requestId)
+        , fResult(result)
+        , fProperties(std::move(properties))
+    {}
+
+    auto GetDeviceId() const -> std::string { return fDeviceId; }
+    auto SetDeviceId(std::string deviceId) -> void { fDeviceId = std::move(deviceId); }
+    auto GetRequestId() const -> std::size_t { return fRequestId; }
+    auto SetRequestId(std::size_t requestId) -> void { fRequestId = requestId; }
+    auto GetResult() const -> Result { return fResult; }
+    auto SetResult(Result result) -> void { fResult = result; }
+    auto GetProps() const -> std::vector<std::pair<std::string, std::string>> { return fProperties; }
+    auto SetProps(std::vector<std::pair<std::string, std::string>> properties) -> void { fProperties = std::move(properties); }
+
+  private:
+    std::string fDeviceId;
+    std::size_t fRequestId;
+    Result fResult;
+    std::vector<std::pair<std::string, std::string>> fProperties;
+};
+
+struct PropertiesSet : Cmd {
+    PropertiesSet(std::string deviceId, std::size_t requestId, Result result)
+        : Cmd(Type::properties_set)
+        , fDeviceId(std::move(deviceId))
+        , fRequestId(requestId)
+        , fResult(result)
+    {}
+
+    auto GetDeviceId() const -> std::string { return fDeviceId; }
+    auto SetDeviceId(std::string deviceId) -> void { fDeviceId = std::move(deviceId); }
+    auto GetRequestId() const -> std::size_t { return fRequestId; }
+    auto SetRequestId(std::size_t requestId) -> void { fRequestId = requestId; }
+    auto GetResult() const -> Result { return fResult; }
+    auto SetResult(Result result) -> void { fResult = result; }
+
+  private:
+    std::string fDeviceId;
+    std::size_t fRequestId;
+    Result fResult;
+};
+
 template<typename C, typename... Args>
 std::unique_ptr<Cmd> make(Args&&... args)
 {
@@ -307,7 +366,6 @@ struct Cmds
         Unpack(std::forward<std::unique_ptr<Cmd>&&>(first), std::forward<Rest>(rest)...);
     }
 
-
     void Add(std::unique_ptr<Cmd>&& cmd) { fCmds.emplace_back(std::move(cmd)); }
 
     template<typename C, typename... Args>

fairmq/sdk/commands/CommandsFormat.fbs

@@ -38,31 +38,38 @@ enum FBTransition:byte {
     ErrorFound
 }
 
+table FBProperty {
+    key:string;
+    value:string;
+}
+
 enum FBCmd:byte {
     check_state,                   // args: { }
     change_state,                  // args: { transition }
     dump_config,                   // args: { }
-    subscribe_to_heartbeats,       // args: { }
-    unsubscribe_from_heartbeats,   // args: { }
-    subscribe_to_state_change,     // args: { }
+    subscribe_to_state_change,     // args: { interval }
     unsubscribe_from_state_change, // args: { }
     state_change_exiting_received, // args: { }
+    get_properties,                // args: { request_id, property_query }
+    set_properties,                // args: { request_id, properties }
+    subscription_heartbeat,        // args: { interval }
 
     current_state,                 // args: { device_id, current_state }
-    transition_status,             // args: { device_id, Result, transition }
+    transition_status,             // args: { device_id, task_id, Result, transition }
     config,                        // args: { device_id, config_string }
-    heartbeat_subscription,        // args: { device_id, Result }
-    heartbeat_unsubscription,      // args: { device_id, Result }
-    heartbeat,                     // args: { device_id }
-    state_change_subscription,     // args: { device_id, Result }
-    state_change_unsubscription,   // args: { device_id, Result }
-    state_change                   // args: { device_id, task_id, last_state, current_state }
+    state_change_subscription,     // args: { device_id, task_id, Result }
+    state_change_unsubscription,   // args: { device_id, task_id, Result }
+    state_change,                  // args: { device_id, task_id, last_state, current_state }
+    properties,                    // args: { device_id, request_id, Result, properties }
+    properties_set                 // args: { device_id, request_id, Result }
 }
 
 table FBCommand {
     command_id:FBCmd;
     device_id:string;
     task_id:uint64;
+    request_id:uint64;
+    interval:int64;
     state:FBState;
     transition:FBTransition;
     result:FBResult;
@@ -70,6 +77,8 @@ table FBCommand {
     last_state:FBState;
     current_state:FBState;
     debug:string;
+    properties:[FBProperty];
+    property_query:string;
 }
 
 table FBCommands {

fairmq/shmem/Common.h

@@ -41,15 +41,21 @@ struct RegionInfo
     RegionInfo(const VoidAlloc& alloc)
         : fPath("", alloc)
         , fFlags(0)
+        , fUserFlags(0)
+        , fDestroyed(false)
     {}
 
-    RegionInfo(const char* path, int flags, const VoidAlloc& alloc)
+    RegionInfo(const char* path, const int flags, const uint64_t userFlags, const VoidAlloc& alloc)
         : fPath(path, alloc)
         , fFlags(flags)
+        , fUserFlags(userFlags)
+        , fDestroyed(false)
     {}
 
     Str fPath;
     int fFlags;
+    uint64_t fUserFlags;
+    bool fDestroyed;
 };
 
 using Uint64RegionInfoPairAlloc = boost::interprocess::allocator<std::pair<const uint64_t, RegionInfo>, SegmentManager>;
@@ -77,8 +83,8 @@ struct MetaHeader
 {
     size_t fSize;
     size_t fRegionId;
-    boost::interprocess::managed_shared_memory::handle_t fHandle;
     size_t fHint;
+    boost::interprocess::managed_shared_memory::handle_t fHandle;
 };
 
 struct RegionBlock

fairmq/shmem/FairMQMessageSHM.cxx

@@ -1,281 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-#include "Common.h"
-#include "Region.h"
-
-#include "FairMQMessageSHM.h"
-#include "FairMQUnmanagedRegionSHM.h"
-#include "FairMQLogger.h"
-
-#include <boost/date_time/posix_time/posix_time.hpp>
-
-#include <cstdlib>
-
-using namespace std;
-using namespace fair::mq::shmem;
-
-namespace bipc = ::boost::interprocess;
-namespace bpt = ::boost::posix_time;
-
-atomic<bool> FairMQMessageSHM::fInterrupted(false);
-fair::mq::Transport FairMQMessageSHM::fTransportType = fair::mq::Transport::SHM;
-
-FairMQMessageSHM::FairMQMessageSHM(Manager& manager, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fManager(manager)
-    , fMessage()
-    , fQueued(false)
-    , fMetaCreated(false)
-    , fRegionId(0)
-    , fRegionPtr(nullptr)
-    , fHandle(-1)
-    , fSize(0)
-    , fHint(0)
-    , fLocalPtr(nullptr)
-{
-    if (zmq_msg_init(&fMessage) != 0) {
-        LOG(error) << "failed initializing message, reason: " << zmq_strerror(errno);
-    }
-    fMetaCreated = true;
-}
-
-FairMQMessageSHM::FairMQMessageSHM(Manager& manager, const size_t size, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fManager(manager)
-    , fMessage()
-    , fQueued(false)
-    , fMetaCreated(false)
-    , fRegionId(0)
-    , fRegionPtr(nullptr)
-    , fHandle(-1)
-    , fSize(0)
-    , fHint(0)
-    , fLocalPtr(nullptr)
-{
-    InitializeChunk(size);
-}
-
-FairMQMessageSHM::FairMQMessageSHM(Manager& manager, void* data, const size_t size, fairmq_free_fn* ffn, void* hint, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fManager(manager)
-    , fMessage()
-    , fQueued(false)
-    , fMetaCreated(false)
-    , fRegionId(0)
-    , fRegionPtr(nullptr)
-    , fHandle(-1)
-    , fSize(0)
-    , fHint(0)
-    , fLocalPtr(nullptr)
-{
-    if (InitializeChunk(size)) {
-        memcpy(fLocalPtr, data, size);
-        if (ffn) {
-            ffn(data, hint);
-        } else {
-            free(data);
-        }
-    }
-}
-
-FairMQMessageSHM::FairMQMessageSHM(Manager& manager, FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fManager(manager)
-    , fMessage()
-    , fQueued(false)
-    , fMetaCreated(false)
-    , fRegionId(static_cast<FairMQUnmanagedRegionSHM*>(region.get())->fRegionId)
-    , fRegionPtr(nullptr)
-    , fHandle(-1)
-    , fSize(size)
-    , fHint(reinterpret_cast<size_t>(hint))
-    , fLocalPtr(static_cast<char*>(data))
-{
-    if (reinterpret_cast<const char*>(data) >= reinterpret_cast<const char*>(region->GetData()) ||
-        reinterpret_cast<const char*>(data) <= reinterpret_cast<const char*>(region->GetData()) + region->GetSize()) {
-        fHandle = (bipc::managed_shared_memory::handle_t)(reinterpret_cast<const char*>(data) - reinterpret_cast<const char*>(region->GetData()));
-
-        if (zmq_msg_init_size(&fMessage, sizeof(MetaHeader)) != 0) {
-            LOG(error) << "failed initializing meta message, reason: " << zmq_strerror(errno);
-        } else {
-            MetaHeader header;
-            header.fSize = size;
-            header.fHandle = fHandle;
-            header.fRegionId = fRegionId;
-            header.fHint = fHint;
-            memcpy(zmq_msg_data(&fMessage), &header, sizeof(MetaHeader));
-
-            fMetaCreated = true;
-        }
-    } else {
-        LOG(error) << "trying to create region message with data from outside the region";
-        throw runtime_error("trying to create region message with data from outside the region");
-    }
-}
-
-bool FairMQMessageSHM::InitializeChunk(const size_t size)
-{
-    while (fHandle < 0) {
-        try {
-            bipc::managed_shared_memory::size_type actualSize = size;
-            char* hint = 0; // unused for bipc::allocate_new
-            fLocalPtr = fManager.Segment().allocation_command<char>(bipc::allocate_new, size, actualSize, hint);
-        } catch (bipc::bad_alloc& ba) {
-            // LOG(warn) << "Shared memory full...";
-            this_thread::sleep_for(chrono::milliseconds(50));
-            if (fInterrupted) {
-                return false;
-            } else {
-                continue;
-            }
-        }
-        fHandle = fManager.Segment().get_handle_from_address(fLocalPtr);
-    }
-
-    fSize = size;
-
-    if (zmq_msg_init_size(&fMessage, sizeof(MetaHeader)) != 0) {
-        LOG(error) << "failed initializing meta message, reason: " << zmq_strerror(errno);
-        return false;
-    }
-    MetaHeader header;
-    header.fSize = size;
-    header.fHandle = fHandle;
-    header.fRegionId = fRegionId;
-    header.fHint = fHint;
-    memcpy(zmq_msg_data(&fMessage), &header, sizeof(MetaHeader));
-
-    fMetaCreated = true;
-
-    return true;
-}
-
-void FairMQMessageSHM::Rebuild()
-{
-    CloseMessage();
-
-    fQueued = false;
-
-    if (zmq_msg_init(&fMessage) != 0) {
-        LOG(error) << "failed initializing message, reason: " << zmq_strerror(errno);
-    }
-    fMetaCreated = true;
-}
-
-void FairMQMessageSHM::Rebuild(const size_t size)
-{
-    CloseMessage();
-    fQueued = false;
-    InitializeChunk(size);
-}
-
-void FairMQMessageSHM::Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
-{
-    CloseMessage();
-
-    fQueued = false;
-
-    if (InitializeChunk(size)) {
-        memcpy(fLocalPtr, data, size);
-        if (ffn) {
-            ffn(data, hint);
-        } else {
-            free(data);
-        }
-    }
-}
-
-void* FairMQMessageSHM::GetData() const
-{
-    if (!fLocalPtr) {
-        if (fRegionId == 0) {
-            if (fSize > 0) {
-                fLocalPtr = reinterpret_cast<char*>(fManager.Segment().get_address_from_handle(fHandle));
-            } else {
-                fLocalPtr = nullptr;
-            }
-        } else {
-            fRegionPtr = fManager.GetRemoteRegion(fRegionId);
-            if (fRegionPtr) {
-                fLocalPtr = reinterpret_cast<char*>(fRegionPtr->fRegion.get_address()) + fHandle;
-            } else {
-                // LOG(warn) << "could not get pointer from a region message";
-                fLocalPtr = nullptr;
-            }
-        }
-    }
-
-    return fLocalPtr;
-}
-
-bool FairMQMessageSHM::SetUsedSize(const size_t size)
-{
-    if (size == fSize) {
-        return true;
-    } else if (size <= fSize) {
-        try {
-            bipc::managed_shared_memory::size_type shrunkSize = size;
-            fLocalPtr = fManager.Segment().allocation_command<char>(bipc::shrink_in_place, fSize + 128, shrunkSize, fLocalPtr);
-            fSize = size;
-
-            // update meta header
-            MetaHeader* hdrPtr = static_cast<MetaHeader*>(zmq_msg_data(&fMessage));
-            hdrPtr->fSize = fSize;
-            return true;
-        } catch (bipc::interprocess_exception& e) {
-            LOG(info) << "could not set used size: " << e.what();
-            return false;
-        }
-    } else {
-        LOG(error) << "cannot set used size higher than original.";
-        return false;
-    }
-}
-
-void FairMQMessageSHM::Copy(const FairMQMessage& msg)
-{
-    if (fHandle < 0) {
-        bipc::managed_shared_memory::handle_t otherHandle = static_cast<const FairMQMessageSHM&>(msg).fHandle;
-        if (otherHandle) {
-            if (InitializeChunk(msg.GetSize())) {
-                memcpy(GetData(), msg.GetData(), msg.GetSize());
-            }
-        } else {
-            LOG(error) << "copy fail: source message not initialized!";
-        }
-    } else {
-        LOG(error) << "copy fail: target message already initialized!";
-    }
-}
-
-void FairMQMessageSHM::CloseMessage()
-{
-    if (fHandle >= 0 && !fQueued) {
-        if (fRegionId == 0) {
-            fManager.Segment().deallocate(fManager.Segment().get_address_from_handle(fHandle));
-            fHandle = -1;
-        } else {
-            if (!fRegionPtr) {
-                fRegionPtr = fManager.GetRemoteRegion(fRegionId);
-            }
-
-            if (fRegionPtr) {
-                fRegionPtr->ReleaseBlock({fHandle, fSize, fHint});
-            } else {
-                LOG(warn) << "region ack queue for id " << fRegionId << " no longer exist. Not sending ack";
-            }
-        }
-    }
-
-    if (fMetaCreated) {
-        if (zmq_msg_close(&fMessage) != 0) {
-            LOG(error) << "failed closing message, reason: " << zmq_strerror(errno);
-        }
-        fMetaCreated = false;
-    }
-}

fairmq/shmem/FairMQMessageSHM.h

@@ -1,72 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-#ifndef FAIRMQMESSAGESHM_H_
-#define FAIRMQMESSAGESHM_H_
-
-#include <fairmq/shmem/Manager.h>
-
-#include "FairMQMessage.h"
-#include "FairMQUnmanagedRegion.h"
-
-#include <zmq.h>
-
-#include <boost/interprocess/mapped_region.hpp>
-
-#include <cstddef> // size_t
-#include <atomic>
-
-class FairMQSocketSHM;
-
-class FairMQMessageSHM final : public FairMQMessage
-{
-    friend class FairMQSocketSHM;
-
-  public:
-    FairMQMessageSHM(fair::mq::shmem::Manager& manager, FairMQTransportFactory* factory = nullptr);
-    FairMQMessageSHM(fair::mq::shmem::Manager& manager, const size_t size, FairMQTransportFactory* factory = nullptr);
-    FairMQMessageSHM(fair::mq::shmem::Manager& manager, void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr, FairMQTransportFactory* factory = nullptr);
-    FairMQMessageSHM(fair::mq::shmem::Manager& manager, FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0, FairMQTransportFactory* factory = nullptr);
-
-    FairMQMessageSHM(const FairMQMessageSHM&) = delete;
-    FairMQMessageSHM operator=(const FairMQMessageSHM&) = delete;
-
-    void Rebuild() override;
-    void Rebuild(const size_t size) override;
-    void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
-
-    void* GetData() const override;
-    size_t GetSize() const override { return fSize; }
-
-    bool SetUsedSize(const size_t size) override;
-
-    fair::mq::Transport GetType() const override { return fTransportType; }
-
-    void Copy(const FairMQMessage& msg) override;
-
-    ~FairMQMessageSHM() override { CloseMessage(); }
-
-  private:
-    fair::mq::shmem::Manager& fManager;
-    zmq_msg_t fMessage;
-    bool fQueued;
-    bool fMetaCreated;
-    static std::atomic<bool> fInterrupted;
-    static fair::mq::Transport fTransportType;
-    size_t fRegionId;
-    mutable fair::mq::shmem::Region* fRegionPtr;
-    boost::interprocess::managed_shared_memory::handle_t fHandle;
-    size_t fSize;
-    size_t fHint;
-    mutable char* fLocalPtr;
-
-    bool InitializeChunk(const size_t size);
-    zmq_msg_t* GetMessage() { return &fMessage; }
-    void CloseMessage();
-};
-
-#endif /* FAIRMQMESSAGESHM_H_ */

fairmq/shmem/FairMQTransportFactorySHM.h

@@ -1,67 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2016-2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIRMQTRANSPORTFACTORYSHM_H_
-#define FAIRMQTRANSPORTFACTORYSHM_H_
-
-#include <fairmq/shmem/Manager.h>
-#include <fairmq/shmem/Common.h>
-
-#include "FairMQTransportFactory.h"
-#include "FairMQMessageSHM.h"
-#include "FairMQSocketSHM.h"
-#include "FairMQPollerSHM.h"
-#include "FairMQUnmanagedRegionSHM.h"
-#include <fairmq/ProgOptions.h>
-
-#include <vector>
-#include <string>
-#include <thread>
-#include <atomic>
-
-class FairMQTransportFactorySHM final : public FairMQTransportFactory
-{
-  public:
-    FairMQTransportFactorySHM(const std::string& id = "", const fair::mq::ProgOptions* config = nullptr);
-    FairMQTransportFactorySHM(const FairMQTransportFactorySHM&) = delete;
-    FairMQTransportFactorySHM operator=(const FairMQTransportFactorySHM&) = delete;
-
-    FairMQMessagePtr CreateMessage() override;
-    FairMQMessagePtr CreateMessage(const size_t size) override;
-    FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
-    FairMQMessagePtr CreateMessage(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0) override;
-
-    FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name) override;
-
-    FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const override;
-    FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel*>& channels) const override;
-    FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const override;
-
-    FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const override;
-
-    fair::mq::Transport GetType() const override;
-
-    void Interrupt() override { FairMQSocketSHM::Interrupt(); }
-    void Resume() override { FairMQSocketSHM::Resume(); }
-    void Reset() override {}
-
-    ~FairMQTransportFactorySHM() override;
-
-  private:
-    void SendHeartbeats();
-
-    static fair::mq::Transport fTransportType;
-    std::string fDeviceId;
-    std::string fShmId;
-    void* fZMQContext;
-    std::unique_ptr<fair::mq::shmem::Manager> fManager;
-    std::thread fHeartbeatThread;
-    std::atomic<bool> fSendHeartbeats;
-};
-
-#endif /* FAIRMQTRANSPORTFACTORYSHM_H_ */

fairmq/shmem/FairMQUnmanagedRegionSHM.cxx

@@ -1,43 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "Common.h"
-
-#include "FairMQUnmanagedRegionSHM.h"
-
-using namespace std;
-using namespace fair::mq::shmem;
-
-namespace bipc = ::boost::interprocess;
-
-FairMQUnmanagedRegionSHM::FairMQUnmanagedRegionSHM(Manager& manager, const size_t size, FairMQRegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */)
-    : fManager(manager)
-    , fRegion(nullptr)
-    , fRegionId(0)
-{
-    try {
-        RegionCounter* rc = fManager.ManagementSegment().find<RegionCounter>(bipc::unique_instance).first;
-        if (rc) {
-            LOG(debug) << "region counter found, with value of " << rc->fCount << ". incrementing.";
-            (rc->fCount)++;
-            LOG(debug) << "incremented region counter, now: " << rc->fCount;
-        } else {
-            LOG(debug) << "no region counter found, creating one and initializing with 1";
-            rc = fManager.ManagementSegment().construct<RegionCounter>(bipc::unique_instance)(1);
-            LOG(debug) << "initialized region counter with: " << rc->fCount;
-        }
-
-        fRegionId = rc->fCount;
-
-        fRegion = fManager.CreateRegion(size, fRegionId, callback, path, flags);
-    } catch (bipc::interprocess_exception& e) {
-        LOG(error) << "cannot create region. Already created/not cleaned up?";
-        LOG(error) << e.what();
-        throw;
-    }
-}

fairmq/shmem/FairMQUnmanagedRegionSHM.h

@@ -1,42 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIRMQUNMANAGEDREGIONSHM_H_
-#define FAIRMQUNMANAGEDREGIONSHM_H_
-
-#include <fairmq/shmem/Manager.h>
-
-#include "FairMQUnmanagedRegion.h"
-#include "FairMQLogger.h"
-
-#include <boost/interprocess/shared_memory_object.hpp>
-#include <boost/interprocess/mapped_region.hpp>
-
-#include <cstddef> // size_t
-#include <string>
-
-class FairMQUnmanagedRegionSHM final : public FairMQUnmanagedRegion
-{
-    friend class FairMQSocketSHM;
-    friend class FairMQMessageSHM;
-
-  public:
-    FairMQUnmanagedRegionSHM(fair::mq::shmem::Manager& manager, const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0);
-
-    void* GetData() const override { return fRegion->get_address(); }
-    size_t GetSize() const override { return fRegion->get_size(); }
-
-    ~FairMQUnmanagedRegionSHM() override { fManager.RemoveRegion(fRegionId); }
-
-  private:
-    fair::mq::shmem::Manager& fManager;
-    boost::interprocess::mapped_region* fRegion;
-    uint64_t fRegionId;
-};
-
-#endif /* FAIRMQUNMANAGEDREGIONSHM_H_ */

fairmq/shmem/Manager.cxx

@@ -6,8 +6,8 @@
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#include <fairmq/shmem/Manager.h>
-#include <fairmq/shmem/Common.h>
+#include "Manager.h"
+
 #include <fairmq/tools/CppSTL.h>
 #include <fairmq/tools/Strings.h>
 
@@ -18,6 +18,7 @@ using namespace std;
 using bie = ::boost::interprocess::interprocess_exception;
 namespace bipc = ::boost::interprocess;
 namespace bfs = ::boost::filesystem;
+namespace bpt = ::boost::posix_time;
 
 namespace fair
 {
@@ -26,19 +27,26 @@ namespace mq
 namespace shmem
 {
 
-std::unordered_map<uint64_t, std::unique_ptr<Region>> Manager::fRegions;
-
-Manager::Manager(const std::string& id, size_t size)
+Manager::Manager(const string& id, size_t size)
     : fShmId(id)
     , fSegmentName("fmq_" + fShmId + "_main")
     , fManagementSegmentName("fmq_" + fShmId + "_mng")
     , fSegment(bipc::open_or_create, fSegmentName.c_str(), size)
-    , fManagementSegment(bipc::open_or_create, fManagementSegmentName.c_str(), 65536)
+    , fManagementSegment(bipc::open_or_create, fManagementSegmentName.c_str(), 655360)
+    , fShmVoidAlloc(fManagementSegment.get_segment_manager())
     , fShmMtx(bipc::open_or_create, string("fmq_" + fShmId + "_mtx").c_str())
+    , fRegionEventsCV(bipc::open_or_create, string("fmq_" + fShmId + "_cv").c_str())
+    , fRegionEventsSubscriptionActive(false)
     , fDeviceCounter(nullptr)
+    , fRegionInfos(nullptr)
+    , fInterrupted(false)
 {
     LOG(debug) << "created/opened shared memory segment '" << "fmq_" << fShmId << "_main" << "' of " << size << " bytes. Available are " << fSegment.get_free_memory() << " bytes.";
 
+    fRegionInfos = fManagementSegment.find_or_construct<Uint64RegionInfoMap>(bipc::unique_instance)(fShmVoidAlloc);
+    // store info about the managed segment as region with id 0
+    fRegionInfos->emplace(0, RegionInfo("", 0, 0, fShmVoidAlloc));
+
     bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
 
     fDeviceCounter = fManagementSegment.find<DeviceCounter>(bipc::unique_instance).first;
@@ -54,7 +62,7 @@ Manager::Manager(const std::string& id, size_t size)
     }
 }
 
-void Manager::StartMonitor(const std::string& id)
+void Manager::StartMonitor(const string& id)
 {
     try {
         bipc::named_mutex monitorStatus(bipc::open_only, string("fmq_" + id + "_ms").c_str());
@@ -83,7 +91,7 @@ void Manager::StartMonitor(const std::string& id)
                     this_thread::sleep_for(chrono::milliseconds(10));
                     if (++numTries > 1000) {
                         LOG(error) << "Did not get response from fairmq-shmmonitor after " << 10 * 1000 << " milliseconds. Exiting.";
-                        throw runtime_error(fair::mq::tools::ToString("Did not get response from fairmq-shmmonitor after ", 10 * 1000, " milliseconds. Exiting."));
+                        throw runtime_error(tools::ToString("Did not get response from fairmq-shmmonitor after ", 10 * 1000, " milliseconds. Exiting."));
                     }
                 }
             } while (true);
@@ -93,47 +101,74 @@ void Manager::StartMonitor(const std::string& id)
     }
 }
 
-void Manager::Interrupt()
+pair<bipc::mapped_region*, uint64_t> Manager::CreateRegion(const size_t size, const int64_t userFlags, RegionCallback callback, const string& path /* = "" */, int flags /* = 0 */)
 {
-}
+    try {
 
-void Manager::Resume()
-{
-    // close remote regions before processing new transfers
-    for (auto it = fRegions.begin(); it != fRegions.end(); /**/) {
-        if (it->second->fRemote) {
-           it = fRegions.erase(it);
-        } else {
-            ++it;
-        }
-    }
-}
+        pair<bipc::mapped_region*, uint64_t> result;
 
-bipc::mapped_region* Manager::CreateRegion(const size_t size, const uint64_t id, FairMQRegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */)
-{
-    auto it = fRegions.find(id);
-    if (it != fRegions.end()) {
-        LOG(error) << "Trying to create a region that already exists";
-        return nullptr;
-    } else {
-        // create region info
         {
+            uint64_t id = 0;
             bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-            VoidAlloc voidAlloc(fManagementSegment.get_segment_manager());
-            Uint64RegionInfoMap* infoMap = fManagementSegment.find_or_construct<Uint64RegionInfoMap>(bipc::unique_instance)(voidAlloc);
-            infoMap->emplace(id, RegionInfo(path.c_str(), flags, voidAlloc));
+
+            RegionCounter* rc = fManagementSegment.find<RegionCounter>(bipc::unique_instance).first;
+
+            if (rc) {
+                LOG(debug) << "region counter found, with value of " << rc->fCount << ". incrementing.";
+                (rc->fCount)++;
+                LOG(debug) << "incremented region counter, now: " << rc->fCount;
+            } else {
+                LOG(debug) << "no region counter found, creating one and initializing with 1";
+                rc = fManagementSegment.construct<RegionCounter>(bipc::unique_instance)(1);
+                LOG(debug) << "initialized region counter with: " << rc->fCount;
+            }
+
+            id = rc->fCount;
+
+            auto it = fRegions.find(id);
+            if (it != fRegions.end()) {
+                LOG(error) << "Trying to create a region that already exists";
+                return {nullptr, id};
+            }
+
+            // create region info
+            fRegionInfos->emplace(id, RegionInfo(path.c_str(), flags, userFlags, fShmVoidAlloc));
+
+            auto r = fRegions.emplace(id, tools::make_unique<Region>(*this, id, size, false, callback, path, flags));
+            // LOG(debug) << "Created region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
+
+            r.first->second->StartReceivingAcks();
+            result.first = &(r.first->second->fRegion);
+            result.second = id;
         }
-        // LOG(debug) << "Created region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
+        fRegionEventsCV.notify_all();
 
-        auto r = fRegions.emplace(id, fair::mq::tools::make_unique<Region>(*this, id, size, false, callback, path, flags));
+        return result;
 
-        r.first->second->StartReceivingAcks();
-
-        return &(r.first->second->fRegion);
+    } catch (bipc::interprocess_exception& e) {
+        LOG(error) << "cannot create region. Already created/not cleaned up?";
+        LOG(error) << e.what();
+        throw;
     }
 }
 
-Region* Manager::GetRemoteRegion(const uint64_t id)
+void Manager::RemoveRegion(const uint64_t id)
+{
+    {
+        bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
+        fRegions.erase(id);
+        fRegionInfos->at(id).fDestroyed = true;
+    }
+    fRegionEventsCV.notify_all();
+}
+
+Region* Manager::GetRegion(const uint64_t id)
+{
+    bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
+    return GetRegionUnsafe(id);
+}
+
+Region* Manager::GetRegionUnsafe(const uint64_t id)
 {
     // remote region could actually be a local one if a message originates from this device (has been sent out and returned)
     auto it = fRegions.find(id);
@@ -141,36 +176,107 @@ Region* Manager::GetRemoteRegion(const uint64_t id)
         return it->second.get();
     } else {
         try {
-            string path;
-            int flags;
-
             // get region info
-            {
-                bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-                Uint64RegionInfoMap* infoMap = fManagementSegment.find<Uint64RegionInfoMap>(bipc::unique_instance).first;
-                if (infoMap == nullptr) {
-                    LOG(error) << "Unable to locate the region info";
-                    throw SharedMemoryError("Unable to locate remote region info");
-                }
-                RegionInfo regionInfo = infoMap->at(id);
-                path = regionInfo.fPath.c_str();
-                flags = regionInfo.fFlags;
-            }
+            RegionInfo regionInfo = fRegionInfos->at(id);
+            string path = regionInfo.fPath.c_str();
+            int flags = regionInfo.fFlags;
             // LOG(debug) << "Located remote region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
 
-            auto r = fRegions.emplace(id, fair::mq::tools::make_unique<Region>(*this, id, 0, true, nullptr, path, flags));
+            auto r = fRegions.emplace(id, tools::make_unique<Region>(*this, id, 0, true, nullptr, path, flags));
             return r.first->second.get();
         } catch (bie& e) {
             LOG(warn) << "Could not get remote region for id: " << id;
             return nullptr;
         }
-
     }
 }
 
-void Manager::RemoveRegion(const uint64_t id)
+vector<fair::mq::RegionInfo> Manager::GetRegionInfo()
 {
-    fRegions.erase(id);
+    bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
+    return GetRegionInfoUnsafe();
+}
+
+vector<fair::mq::RegionInfo> Manager::GetRegionInfoUnsafe()
+{
+    vector<fair::mq::RegionInfo> result;
+
+    for (const auto& e : *fRegionInfos) {
+        fair::mq::RegionInfo info;
+        info.id = e.first;
+        info.flags = e.second.fUserFlags;
+        info.event = e.second.fDestroyed ? RegionEvent::destroyed : RegionEvent::created;
+        if (info.id != 0) {
+            if (!e.second.fDestroyed) {
+                auto region = GetRegionUnsafe(info.id);
+                info.ptr = region->fRegion.get_address();
+                info.size = region->fRegion.get_size();
+            } else {
+                info.ptr = nullptr;
+                info.size = 0;
+            }
+            result.push_back(info);
+        } else {
+            if (!e.second.fDestroyed) {
+                info.ptr = fSegment.get_address();
+                info.size = fSegment.get_size();
+            } else {
+                info.ptr = nullptr;
+                info.size = 0;
+            }
+            result.push_back(info);
+        }
+    }
+
+    return result;
+}
+
+void Manager::SubscribeToRegionEvents(RegionEventCallback callback)
+{
+    bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
+    if (fRegionEventThread.joinable()) {
+        fRegionEventsSubscriptionActive.store(false);
+        fRegionEventThread.join();
+    }
+    fRegionEventCallback = callback;
+    fRegionEventsSubscriptionActive.store(true);
+    fRegionEventThread = thread(&Manager::RegionEventsSubscription, this);
+}
+
+void Manager::UnsubscribeFromRegionEvents()
+{
+    if (fRegionEventThread.joinable()) {
+        fRegionEventsSubscriptionActive.store(false);
+        fRegionEventsCV.notify_all();
+        fRegionEventThread.join();
+    }
+    bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
+    fRegionEventCallback = nullptr;
+}
+
+void Manager::RegionEventsSubscription()
+{
+    while (fRegionEventsSubscriptionActive.load()) {
+        bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
+        auto infos = GetRegionInfoUnsafe();
+        for (const auto& i : infos) {
+            auto el = fObservedRegionEvents.find(i.id);
+            if (el == fObservedRegionEvents.end()) {
+                fRegionEventCallback(i);
+                fObservedRegionEvents.emplace(i.id, i.event);
+            } else {
+                if (el->second == RegionEvent::created && i.event == RegionEvent::destroyed) {
+                    fRegionEventCallback(i);
+                    el->second = i.event;
+                } else {
+                    // LOG(debug) << "ignoring event for id" << i.id << ":";
+                    // LOG(debug) << "incoming event: " << i.event;
+                    // LOG(debug) << "stored event: " << el->second;
+                }
+            }
+        }
+        fRegionEventsCV.wait(lock);
+    }
 }
 
 void Manager::RemoveSegments()
@@ -192,7 +298,13 @@ Manager::~Manager()
 {
     bool lastRemoved = false;
 
-    {
+    if (fRegionEventThread.joinable()) {
+        fRegionEventsSubscriptionActive.store(false);
+        fRegionEventsCV.notify_all();
+        fRegionEventThread.join();
+    }
+
+    try {
         bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
 
         (fDeviceCounter->fCount)--;
@@ -205,10 +317,13 @@ Manager::~Manager()
         } else {
             LOG(debug) << "other segment users present (" << fDeviceCounter->fCount << "), not removing it.";
         }
+    } catch(bie& e) {
+        LOG(error) << "error while acquiring lock in Manager destructor: " << e.what();
     }
 
     if (lastRemoved) {
         bipc::named_mutex::remove(string("fmq_" + fShmId + "_mtx").c_str());
+        bipc::named_condition::remove(string("fmq_" + fShmId + "_cv").c_str());
     }
 }
 

fairmq/shmem/Manager.h

@@ -6,7 +6,7 @@
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**
- * FairMQShmManager.h
+ * Manager.h
  *
  * @since 2016-04-08
  * @author A. Rybalchenko
@@ -15,19 +15,24 @@
 #ifndef FAIR_MQ_SHMEM_MANAGER_H_
 #define FAIR_MQ_SHMEM_MANAGER_H_
 
-#include <fairmq/shmem/Region.h>
-#include <fairmq/shmem/Common.h>
+#include "Common.h"
+#include "Region.h"
 
-#include "FairMQLogger.h"
-#include "FairMQMessage.h"
+#include <FairMQLogger.h>
+#include <FairMQUnmanagedRegion.h>
 
-#include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/ipc/message_queue.hpp>
+#include <boost/interprocess/managed_shared_memory.hpp>
+#include <boost/interprocess/sync/named_condition.hpp>
 #include <boost/interprocess/sync/named_mutex.hpp>
 
-#include <string>
-#include <unordered_map>
+#include <set>
 #include <stdexcept>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <utility>
+#include <vector>
 
 namespace fair
 {
@@ -57,17 +62,25 @@ class Manager
 
     static void StartMonitor(const std::string&);
 
-    static void Interrupt();
-    static void Resume();
+    void Interrupt() { fInterrupted.store(true); }
+    void Resume() { fInterrupted.store(false); }
+    bool Interrupted() { return fInterrupted.load(); }
 
     int GetDeviceCounter();
     int IncrementDeviceCounter();
     int DecrementDeviceCounter();
 
-    boost::interprocess::mapped_region* CreateRegion(const size_t size, const uint64_t id, FairMQRegionCallback callback, const std::string& path = "", int flags = 0);
-    Region* GetRemoteRegion(const uint64_t id);
+    std::pair<boost::interprocess::mapped_region*, uint64_t> CreateRegion(const size_t size, const int64_t userFlags, RegionCallback callback, const std::string& path = "", int flags = 0);
+    Region* GetRegion(const uint64_t id);
+    Region* GetRegionUnsafe(const uint64_t id);
     void RemoveRegion(const uint64_t id);
 
+    std::vector<fair::mq::RegionInfo> GetRegionInfo();
+    std::vector<fair::mq::RegionInfo> GetRegionInfoUnsafe();
+    void SubscribeToRegionEvents(RegionEventCallback callback);
+    void UnsubscribeFromRegionEvents();
+    void RegionEventsSubscription();
+
     void RemoveSegments();
 
   private:
@@ -76,9 +89,20 @@ class Manager
     std::string fManagementSegmentName;
     boost::interprocess::managed_shared_memory fSegment;
     boost::interprocess::managed_shared_memory fManagementSegment;
+    VoidAlloc fShmVoidAlloc;
     boost::interprocess::named_mutex fShmMtx;
-    fair::mq::shmem::DeviceCounter* fDeviceCounter;
-    static std::unordered_map<uint64_t, std::unique_ptr<Region>> fRegions;
+
+    boost::interprocess::named_condition fRegionEventsCV;
+    std::thread fRegionEventThread;
+    std::atomic<bool> fRegionEventsSubscriptionActive;
+    std::function<void(fair::mq::RegionInfo)> fRegionEventCallback;
+    std::unordered_map<uint64_t, RegionEvent> fObservedRegionEvents;
+
+    DeviceCounter* fDeviceCounter;
+    Uint64RegionInfoMap* fRegionInfos;
+    std::unordered_map<uint64_t, std::unique_ptr<Region>> fRegions;
+
+    std::atomic<bool> fInterrupted;
 };
 
 } // namespace shmem

fairmq/shmem/Message.cxx

@@ -0,0 +1,250 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "Region.h"
+#include "Message.h"
+#include "UnmanagedRegion.h"
+#include "TransportFactory.h"
+
+#include <FairMQLogger.h>
+
+#include <boost/date_time/posix_time/posix_time.hpp>
+
+#include <cstdlib>
+
+using namespace std;
+
+namespace bipc = ::boost::interprocess;
+namespace bpt = ::boost::posix_time;
+
+namespace fair
+{
+namespace mq
+{
+namespace shmem
+{
+
+Transport Message::fTransportType = Transport::SHM;
+
+Message::Message(Manager& manager, FairMQTransportFactory* factory)
+    : fair::mq::Message{factory}
+    , fManager(manager)
+    , fQueued(false)
+    , fMeta{0, 0, 0, -1}
+    , fRegionPtr(nullptr)
+    , fLocalPtr(nullptr)
+{
+    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
+}
+
+Message::Message(Manager& manager, const size_t size, FairMQTransportFactory* factory)
+    : fair::mq::Message{factory}
+    , fManager(manager)
+    , fQueued(false)
+    , fMeta{0, 0, 0, -1}
+    , fRegionPtr(nullptr)
+    , fLocalPtr(nullptr)
+{
+    InitializeChunk(size);
+    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
+}
+
+Message::Message(Manager& manager, MetaHeader& hdr, FairMQTransportFactory* factory)
+    : fair::mq::Message{factory}
+    , fManager(manager)
+    , fQueued(false)
+    , fMeta{hdr}
+    , fRegionPtr(nullptr)
+    , fLocalPtr(nullptr)
+{
+    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
+}
+
+Message::Message(Manager& manager, void* data, const size_t size, fairmq_free_fn* ffn, void* hint, FairMQTransportFactory* factory)
+    : fair::mq::Message{factory}
+    , fManager(manager)
+    , fQueued(false)
+    , fMeta{0, 0, 0, -1}
+    , fRegionPtr(nullptr)
+    , fLocalPtr(nullptr)
+{
+    if (InitializeChunk(size)) {
+        std::memcpy(fLocalPtr, data, size);
+        if (ffn) {
+            ffn(data, hint);
+        } else {
+            free(data);
+        }
+    }
+    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
+}
+
+Message::Message(Manager& manager, UnmanagedRegionPtr& region, void* data, const size_t size, void* hint, FairMQTransportFactory* factory)
+    : fair::mq::Message{factory}
+    , fManager(manager)
+    , fQueued(false)
+    , fMeta{size, static_cast<UnmanagedRegion*>(region.get())->fRegionId, reinterpret_cast<size_t>(hint), -1}
+    , fRegionPtr(nullptr)
+    , fLocalPtr(static_cast<char*>(data))
+{
+    if (reinterpret_cast<const char*>(data) >= reinterpret_cast<const char*>(region->GetData()) ||
+        reinterpret_cast<const char*>(data) <= reinterpret_cast<const char*>(region->GetData()) + region->GetSize()) {
+        fMeta.fHandle = (bipc::managed_shared_memory::handle_t)(reinterpret_cast<const char*>(data) - reinterpret_cast<const char*>(region->GetData()));
+    } else {
+        LOG(error) << "trying to create region message with data from outside the region";
+        throw runtime_error("trying to create region message with data from outside the region");
+    }
+    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
+}
+
+bool Message::InitializeChunk(const size_t size)
+{
+    while (fMeta.fHandle < 0) {
+        try {
+            bipc::managed_shared_memory::size_type actualSize = size;
+            char* hint = 0; // unused for bipc::allocate_new
+            fLocalPtr = fManager.Segment().allocation_command<char>(bipc::allocate_new, size, actualSize, hint);
+        } catch (bipc::bad_alloc& ba) {
+            // LOG(warn) << "Shared memory full...";
+            this_thread::sleep_for(chrono::milliseconds(50));
+            if (fManager.Interrupted()) {
+                return false;
+            } else {
+                continue;
+            }
+        }
+        fMeta.fHandle = fManager.Segment().get_handle_from_address(fLocalPtr);
+    }
+
+    fMeta.fSize = size;
+    return true;
+}
+
+void Message::Rebuild()
+{
+    CloseMessage();
+    fQueued = false;
+}
+
+void Message::Rebuild(const size_t size)
+{
+    CloseMessage();
+    fQueued = false;
+    InitializeChunk(size);
+}
+
+void Message::Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
+{
+    CloseMessage();
+    fQueued = false;
+
+    if (InitializeChunk(size)) {
+        std::memcpy(fLocalPtr, data, size);
+        if (ffn) {
+            ffn(data, hint);
+        } else {
+            free(data);
+        }
+    }
+}
+
+void* Message::GetData() const
+{
+    if (!fLocalPtr) {
+        if (fMeta.fRegionId == 0) {
+            if (fMeta.fSize > 0) {
+                fLocalPtr = reinterpret_cast<char*>(fManager.Segment().get_address_from_handle(fMeta.fHandle));
+            } else {
+                fLocalPtr = nullptr;
+            }
+        } else {
+            fRegionPtr = fManager.GetRegion(fMeta.fRegionId);
+            if (fRegionPtr) {
+                fLocalPtr = reinterpret_cast<char*>(fRegionPtr->fRegion.get_address()) + fMeta.fHandle;
+            } else {
+                // LOG(warn) << "could not get pointer from a region message";
+                fLocalPtr = nullptr;
+            }
+        }
+    }
+
+    return fLocalPtr;
+}
+
+bool Message::SetUsedSize(const size_t size)
+{
+    if (size == fMeta.fSize) {
+        return true;
+    } else if (size <= fMeta.fSize) {
+        try {
+            bipc::managed_shared_memory::size_type shrunkSize = size;
+            fLocalPtr = fManager.Segment().allocation_command<char>(bipc::shrink_in_place, fMeta.fSize + 128, shrunkSize, fLocalPtr);
+            fMeta.fSize = size;
+            return true;
+        } catch (bipc::interprocess_exception& e) {
+            LOG(info) << "could not set used size: " << e.what();
+            return false;
+        }
+    } else {
+        LOG(error) << "cannot set used size higher than original.";
+        return false;
+    }
+}
+
+void Message::Copy(const fair::mq::Message& msg)
+{
+    if (fMeta.fHandle < 0) {
+        bipc::managed_shared_memory::handle_t otherHandle = static_cast<const Message&>(msg).fMeta.fHandle;
+        if (otherHandle) {
+            if (InitializeChunk(msg.GetSize())) {
+                std::memcpy(GetData(), msg.GetData(), msg.GetSize());
+            }
+        } else {
+            LOG(error) << "copy fail: source message not initialized!";
+        }
+    } else {
+        LOG(error) << "copy fail: target message already initialized!";
+    }
+}
+
+void Message::CloseMessage()
+{
+    if (fMeta.fHandle >= 0 && !fQueued) {
+        if (fMeta.fRegionId == 0) {
+            fManager.Segment().deallocate(fManager.Segment().get_address_from_handle(fMeta.fHandle));
+            fMeta.fHandle = -1;
+        } else {
+            if (!fRegionPtr) {
+                fRegionPtr = fManager.GetRegion(fMeta.fRegionId);
+            }
+
+            if (fRegionPtr) {
+                fRegionPtr->ReleaseBlock({fMeta.fHandle, fMeta.fSize, fMeta.fHint});
+            } else {
+                LOG(warn) << "region ack queue for id " << fMeta.fRegionId << " no longer exist. Not sending ack";
+            }
+        }
+    }
+
+    static_cast<TransportFactory*>(GetTransport())->DecrementMsgCounter();
+}
+
+Message::~Message()
+{
+    try {
+        CloseMessage();
+    } catch(SharedMemoryError& sme) {
+        LOG(error) << "error closing message: " << sme.what();
+    } catch(bipc::lock_exception& le) {
+        LOG(error) << "error closing message: " << le.what();
+    }
+}
+
+}
+}
+}

fairmq/shmem/Message.h

@@ -0,0 +1,78 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+#ifndef FAIR_MQ_SHMEM_MESSAGE_H_
+#define FAIR_MQ_SHMEM_MESSAGE_H_
+
+#include "Common.h"
+#include "Manager.h"
+
+#include <FairMQMessage.h>
+#include <FairMQUnmanagedRegion.h>
+
+#include <boost/interprocess/mapped_region.hpp>
+
+#include <cstddef> // size_t
+#include <atomic>
+
+namespace fair
+{
+namespace mq
+{
+namespace shmem
+{
+
+class Socket;
+
+class Message final : public fair::mq::Message
+{
+    friend class Socket;
+
+  public:
+    Message(Manager& manager, FairMQTransportFactory* factory = nullptr);
+    Message(Manager& manager, const size_t size, FairMQTransportFactory* factory = nullptr);
+    Message(Manager& manager, void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr, FairMQTransportFactory* factory = nullptr);
+    Message(Manager& manager, UnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0, FairMQTransportFactory* factory = nullptr);
+
+    Message(Manager& manager, MetaHeader& hdr, FairMQTransportFactory* factory = nullptr);
+
+    Message(const Message&) = delete;
+    Message operator=(const Message&) = delete;
+
+    void Rebuild() override;
+    void Rebuild(const size_t size) override;
+    void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
+
+    void* GetData() const override;
+    size_t GetSize() const override { return fMeta.fSize; }
+
+    bool SetUsedSize(const size_t size) override;
+
+    Transport GetType() const override { return fTransportType; }
+
+    void Copy(const fair::mq::Message& msg) override;
+
+    ~Message() override;
+
+  private:
+    Manager& fManager;
+    bool fQueued;
+    MetaHeader fMeta;
+    mutable Region* fRegionPtr;
+    mutable char* fLocalPtr;
+
+    static Transport fTransportType;
+
+    bool InitializeChunk(const size_t size);
+    void CloseMessage();
+};
+
+}
+}
+}
+
+#endif /* FAIR_MQ_SHMEM_MESSAGE_H_ */

fairmq/shmem/Monitor.cxx

@@ -6,14 +6,16 @@
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#include <fairmq/shmem/Monitor.h>
-#include <fairmq/shmem/Common.h>
+#include "Monitor.h"
+#include "Common.h"
+
 #include <fairmq/Tools.h>
 
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/file_mapping.hpp>
 
 #include <boost/interprocess/sync/named_mutex.hpp>
+#include <boost/interprocess/sync/named_condition.hpp>
 #include <boost/interprocess/ipc/message_queue.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
 
@@ -275,7 +277,7 @@ void Monitor::CheckSegment()
         unsigned int numDevices = 0;
 
         if (fInteractive) {
-            fair::mq::shmem::DeviceCounter* dc = managementSegment.find<fair::mq::shmem::DeviceCounter>(bipc::unique_instance).first;
+            DeviceCounter* dc = managementSegment.find<DeviceCounter>(bipc::unique_instance).first;
             if (dc) {
                 numDevices = dc->fCount;
             }
@@ -426,6 +428,15 @@ void Monitor::RemoveMutex(const string& name)
     }
 }
 
+void Monitor::RemoveCondition(const string& name)
+{
+    if (bipc::named_condition::remove(name.c_str())) {
+        cout << "Successfully removed \"" << name << "\"." << endl;
+    } else {
+        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
+    }
+}
+
 void Monitor::Cleanup(const string& shmId)
 {
     string managementSegmentName("fmq_" + shmId + "_mng");
@@ -462,10 +473,13 @@ void Monitor::Cleanup(const string& shmId)
         RemoveObject(managementSegmentName.c_str());
     } catch (bie&) {
         cout << "Did not find '" << managementSegmentName << "' shared memory segment. No regions to cleanup." << endl;
+    } catch(std::out_of_range& oor) {
+        cout << "Could not locate element in the region map, out of range: " << oor.what() << endl;
     }
 
     RemoveObject("fmq_" + shmId + "_main");
     RemoveMutex("fmq_" + shmId + "_mtx");
+    RemoveCondition("fmq_" + shmId + "_cv");
 
     cout << endl;
 }