shm: do mlock/zeroing under shmem lock

Debian's libpmix-dev package installs everything below
/usr/lib/ARCH-TRIPLET/pmix2.
For some reasoning behind this,
see: https://bugs.debian.org/882033

CMake does not search there by default, so help it.

Also add myself to CONTRIBUTORS.

CMakeLists.txt

@@ -31,28 +31,30 @@ include(CTest)
 
 
 # Build options ################################################################
-fairmq_build_option(BUILD_FAIRMQ            "Build FairMQ library and devices."
-                                             DEFAULT ON)
-fairmq_build_option(BUILD_TESTING           "Build tests."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
-fairmq_build_option(BUILD_OFI_TRANSPORT     "Build experimental OFI transport."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
-fairmq_build_option(BUILD_SDK_COMMANDS      "Build the FairMQ SDK commands."
-                                             DEFAULT OFF)
-fairmq_build_option(BUILD_DDS_PLUGIN        "Build DDS plugin."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
-fairmq_build_option(BUILD_PMIX_PLUGIN       "Build PMIx plugin."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
-fairmq_build_option(BUILD_EXAMPLES          "Build FairMQ examples."
-                                             DEFAULT ON  REQUIRES "BUILD_FAIRMQ")
-fairmq_build_option(BUILD_SDK               "Build the FairMQ controller SDK."
-                                             DEFAULT OFF REQUIRES "BUILD_DDS_PLUGIN;BUILD_SDK_COMMANDS")
-fairmq_build_option(BUILD_DOCS              "Build FairMQ documentation."
-                                             DEFAULT OFF)
-fairmq_build_option(FAST_BUILD              "Fast production build. Not recommended for development."
-                                             DEFAULT OFF)
-fairmq_build_option(USE_EXTERNAL_GTEST      "Do not use bundled GTest. Not recommended."
-                                             DEFAULT OFF)
+fairmq_build_option(BUILD_FAIRMQ        "Build FairMQ library and devices."
+                                         DEFAULT ON)
+fairmq_build_option(BUILD_TESTING       "Build tests."
+                                         DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
+fairmq_build_option(BUILD_OFI_TRANSPORT "Build experimental OFI transport."
+                                         DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
+fairmq_build_option(BUILD_SDK_COMMANDS  "Build the FairMQ SDK commands."
+                                         DEFAULT OFF)
+fairmq_build_option(BUILD_DDS_PLUGIN    "Build DDS plugin."
+                                         DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
+fairmq_build_option(BUILD_PMIX_PLUGIN   "Build PMIx plugin."
+                                         DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
+fairmq_build_option(BUILD_EXAMPLES      "Build FairMQ examples."
+                                         DEFAULT ON  REQUIRES "BUILD_FAIRMQ")
+fairmq_build_option(BUILD_SDK           "Build the FairMQ controller SDK."
+                                         DEFAULT OFF REQUIRES "BUILD_DDS_PLUGIN;BUILD_SDK_COMMANDS")
+fairmq_build_option(BUILD_DOCS          "Build FairMQ documentation."
+                                         DEFAULT OFF)
+fairmq_build_option(FAST_BUILD          "Fast production build. Not recommended for development."
+                                         DEFAULT OFF)
+fairmq_build_option(USE_EXTERNAL_GTEST  "Do not use bundled GTest. Not recommended."
+                                         DEFAULT OFF)
+fairmq_build_option(FAIRMQ_DEBUG_MODE   "Compile in debug mode (may decrease performance)."
+                                         DEFAULT OFF)
 ################################################################################
 
 
@@ -80,7 +82,7 @@ endif()
 
 if(BUILD_DDS_PLUGIN OR BUILD_SDK)
   find_package2(PRIVATE DDS REQUIRED
-    VERSION 3.0
+    VERSION 3.5.3
   )
   set(DDS_Boost_COMPONENTS system log log_setup regex filesystem thread)
   set(DDS_Boost_VERSION 1.67)
@@ -94,7 +96,7 @@ endif()
 
 if(BUILD_FAIRMQ OR BUILD_SDK)
   find_package2(PUBLIC FairLogger REQUIRED
-    VERSION 1.2.0
+    VERSION 1.6.0
   )
 
   foreach(dep IN LISTS FairLogger_PACKAGE_DEPENDENCIES)
@@ -134,7 +136,7 @@ endif()
 
 if(BUILD_SDK)
   find_package2(BUNDLED asio
-    VERSION 1.13.0
+    VERSION 1.18.0
   )
   if(NOT asio_FOUND)
     build_bundled(asio extern/asio)
@@ -429,4 +431,10 @@ message(STATUS "  ${Cyan}INSTALL PREFIX${CR}     ${BGreen}${CMAKE_INSTALL_PREFIX
 message(STATUS "  ")
 message(STATUS "  ${Cyan}RUN STATIC ANALYSIS  ${static_ana_summary}")
 message(STATUS "  ")
+if(FAIRMQ_DEBUG_MODE)
+  message(STATUS "  ${Cyan}DEBUG_MODE${CR}         ${BGreen}${FAIRMQ_DEBUG_MODE}${CR} (disable with ${BMagenta}-DFAIRMQ_DEBUG_MODE=OFF${CR})")
+else()
+  message(STATUS "  ${Cyan}DEBUG_MODE${CR}         ${BRed}${FAIRMQ_DEBUG_MODE}${CR} (enable with ${BMagenta}-DFAIRMQ_DEBUG_MODE=ON${CR})")
+endif()
+message(STATUS "  ")
 ################################################################################

CONTRIBUTORS

@@ -8,5 +8,6 @@ Lebedev, Andrey
 Mrnjavac, Teo <teo.m@cern.ch>
 Neskovic, Gvozden
 Richter, Matthias
+Tacke, Christian
 Uhlig, Florian
 Wenzel, Sandro

FairMQTest.cmake

@@ -30,8 +30,6 @@ Set(configure_options "${configure_options};-DCMAKE_PREFIX_PATH=$ENV{SIMPATH}")
 Set(configure_options "${configure_options};-DBUILD_DDS_PLUGIN=ON")
 Set(configure_options "${configure_options};-DBUILD_SDK=ON")
 Set(configure_options "${configure_options};-DBUILD_SDK_COMMANDS=ON")
-Set(configure_options "${configure_options};-DFAST_BUILD=ON")
-Set(configure_options "${configure_options};-DCOTIRE_MAXIMUM_NUMBER_OF_UNITY_INCLUDES=-j$ENV{number_of_processors}")
 
 Set(EXTRA_FLAGS $ENV{EXTRA_FLAGS})
 If(EXTRA_FLAGS)
@@ -59,8 +57,13 @@ Ctest_Configure(BUILD "${CTEST_BINARY_DIRECTORY}"
 
 Ctest_Build(BUILD "${CTEST_BINARY_DIRECTORY}")
 
+unset(exclude_tests)
+if($ENV{EXCLUDE_UNSTABLE_DDS_TESTS})
+  set(exclude_tests EXCLUDE ".*\\.localhost$")
+endif()
 Ctest_Test(BUILD "${CTEST_BINARY_DIRECTORY}"
            # PARALLEL_LEVEL $ENV{number_of_processors}
+           ${exclude_tests}
            PARALLEL_LEVEL $ENV{number_of_processors}
            RETURN_VALUE _ctest_test_ret_val
           )

Jenkinsfile

@@ -8,11 +8,15 @@ def jobMatrix(String prefix, List specs, Closure callback) {
   def nodes = [:]
   for (spec in specs) {
     def label = specToLabel(spec)
+    def node_tag = label
+    if (spec.os =~ /macOS/) {
+      node_tag = spec.os
+    }
     def fairsoft = spec.fairsoft
     def os = spec.os
     def compiler = spec.compiler
     nodes["${prefix}/${label}"] = {
-      node(label) {
+      node(node_tag) {
         githubNotify(context: "${prefix}/${label}", description: 'Building ...', status: 'PENDING')
         try {
           deleteDir()
@@ -29,7 +33,7 @@ def jobMatrix(String prefix, List specs, Closure callback) {
               echo "module load compiler/gcc/9.1.0" >> Dart.cfg
             '''
           }
-          if (os =~ /MacOS/) {
+          if (os =~ /[Mm]acOS/) {
             sh "echo \"export EXTRA_FLAGS='-DCMAKE_CXX_COMPILER=clang++'\" >> Dart.cfg"
           } else {
             sh "echo \"export EXTRA_FLAGS='-DCMAKE_CXX_COMPILER=g++'\" >> Dart.cfg"
@@ -42,6 +46,11 @@ def jobMatrix(String prefix, List specs, Closure callback) {
             echo "export GIT_BRANCH=$JOB_BASE_NAME" >> Dart.cfg
             echo "echo \\\$PATH" >> Dart.cfg
           '''
+
+          if (os =~ /macOS10.14/) {
+            sh "echo \"export EXCLUDE_UNSTABLE_DDS_TESTS=1\" >> Dart.cfg"
+          }
+
           sh 'cat Dart.cfg'
 
           callback.call(spec, label)
@@ -70,9 +79,9 @@ pipeline{
       steps{
         script {
           def build_jobs = jobMatrix('build', [
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc9.1.0',        fairsoft: 'fairmq_dev'],
-            [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
-            [os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc9.1.0',       fairsoft: 'fairmq_dev'],
+            [os: 'macOS10.14', arch: 'x86_64', compiler: 'AppleClang11.0', fairsoft: 'fairmq_dev'],
+            [os: 'macOS10.15', arch: 'x86_64', compiler: 'AppleClang12.0', fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             sh './Dart.sh alfa_ci Dart.cfg'
           }

cmake/FairMQLib.cmake

@@ -145,7 +145,7 @@ macro(set_fairmq_defaults)
   # Configure build types
   set(CMAKE_CONFIGURATION_TYPES "Debug" "Release" "RelWithDebInfo" "Nightly" "Profile" "Experimental" "AddressSan" "ThreadSan")
   set(_warnings "-Wshadow -Wall -Wextra -Wpedantic")
-  set(CMAKE_CXX_FLAGS_DEBUG          "-g ${_warnings}")
+  set(CMAKE_CXX_FLAGS_DEBUG          "-Og -g ${_warnings}")
   set(CMAKE_CXX_FLAGS_RELEASE        "-O2 -DNDEBUG")
   set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g ${_warnings} -DNDEBUG")
   set(CMAKE_CXX_FLAGS_NIGHTLY        "-O2 -g ${_warnings}")

cmake/FindPMIx.cmake

@@ -6,28 +6,35 @@
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
+
+# The "lib/${CMAKE_LIBRARY_ARCHITECTURE}/pmix" part in all
+# the PATH_SUFFIXES is here to be able to find Debian's
+# libpmix-dev package. It installs everything below
+# /usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}/pmix
+
+
 find_path(PMIx_INCLUDE_DIR
   NAMES pmix.h
   HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
-  PATH_SUFFIXES include
+  PATH_SUFFIXES include lib/${CMAKE_LIBRARY_ARCHITECTURE}/pmix/include
 )
 
 find_path(PMIx_LIBRARY_DIR
   NAMES libpmix.dylib libpmix.so
   HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
-  PATH_SUFFIXES lib lib64
+  PATH_SUFFIXES lib lib64 lib/${CMAKE_LIBRARY_ARCHITECTURE}/pmix/lib
 )
 
 find_library(PMIx_LIBRARY_SHARED
   NAMES libpmix.dylib libpmix.so
   HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
-  PATH_SUFFIXES lib lib64
+  PATH_SUFFIXES lib lib64 lib/${CMAKE_LIBRARY_ARCHITECTURE}/pmix/lib
 )
 
 find_file(PMIx_VERSION_FILE
   NAMES pmix_version.h
   HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
-  PATH_SUFFIXES include
+  PATH_SUFFIXES include lib/${CMAKE_LIBRARY_ARCHITECTURE}/pmix/include
 )
 
 file(READ "${PMIx_VERSION_FILE}" __version_raw)

examples/1-1/Sampler.cxx

@@ -17,8 +17,7 @@ Sampler::Sampler()
     : fText()
     , fMaxIterations(0)
     , fNumIterations(0)
-{
-}
+{}
 
 void Sampler::InitTask()
 {
@@ -32,25 +31,22 @@ bool Sampler::ConditionalRun()
     // create a copy of the data with new(), that will be deleted after the transfer is complete
     string* text = new string(fText);
 
-    // create message object with a pointer to the data buffer,
-    // its size,
+    // create message object with a pointer to the data buffer, its size,
     // custom deletion function (called when transfer is done),
     // and pointer to the object managing the data buffer
-    FairMQMessagePtr msg(NewMessage(const_cast<char*>(text->c_str()),
-                                    text->length(),
-                                    [](void* /*data*/, void* object) { delete static_cast<string*>(object); },
-                                    text));
+    FairMQMessagePtr msg(NewMessage(
+        const_cast<char*>(text->c_str()),
+        text->length(),
+        [](void* /*data*/, void* object) { delete static_cast<string*>(object); },
+        text));
 
     LOG(info) << "Sending \"" << fText << "\"";
 
-    // in case of error or transfer interruption, return false to go to IDLE state
+    // in case of error or transfer interruption, return false to go to the Ready state
     // successfull transfer will return number of bytes transfered (can be 0 if sending an empty message).
-    if (Send(msg, "data") < 0)
-    {
+    if (Send(msg, "data") < 0) {
         return false;
-    }
-    else if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations)
-    {
+    } else if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
         LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
         return false;
     }
@@ -58,8 +54,6 @@ bool Sampler::ConditionalRun()
     return true;
 }
 
-Sampler::~Sampler()
-{
-}
+Sampler::~Sampler() {}
 
 } // namespace example_1_1

examples/1-1/Sink.cxx

@@ -33,23 +33,22 @@ void Sink::InitTask()
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
 }
 
-// handler is called whenever a message arrives on "data", with a reference to the message and a sub-channel index (here 0)
+// handler is called whenever a message arrives on "data", with a reference to the message and a
+// sub-channel index (here 0)
 bool Sink::HandleData(FairMQMessagePtr& msg, int /*index*/)
 {
     LOG(info) << "Received: \"" << string(static_cast<char*>(msg->GetData()), msg->GetSize()) << "\"";
 
-    if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations)
-    {
+    if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
         LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
         return false;
     }
 
-    // return true if want to be called again (otherwise return false go to IDLE state)
+    // return true if you want the handler to be called again (otherwise return false go to the
+    // Ready state)
     return true;
 }
 
-Sink::~Sink()
-{
-}
+Sink::~Sink() {}
 
 } // namespace example_1_1

examples/dds/CMakeLists.txt

@@ -29,7 +29,7 @@ target_link_libraries(fairmq-ex-dds-sink PRIVATE ExampleDDSLib)
 add_custom_target(ExampleDDS DEPENDS fairmq-ex-dds-sampler fairmq-ex-dds-processor fairmq-ex-dds-sink)
 
 list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
-set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
+set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/sdk)
 set(DATA_DIR ${CMAKE_CURRENT_BINARY_DIR})
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-dds-topology.xml ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology.xml @ONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-dds-topology-infinite.xml ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology-infinite.xml @ONLY)

examples/n-m/CMakeLists.txt

@@ -18,7 +18,7 @@ target_link_libraries(fairmq-ex-n-m-receiver PRIVATE FairMQ)
 add_custom_target(ExampleNM DEPENDS fairmq-ex-n-m-synchronizer fairmq-ex-n-m-sender fairmq-ex-n-m-receiver)
 
 list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
-set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
+set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/sdk)
 set(DATA_DIR ${CMAKE_CURRENT_BINARY_DIR})
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-n-m-topology.xml                ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-topology.xml @ONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-n-m-pair-topology.xml           ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-pair-topology.xml @ONLY)

examples/n-m/ex-n-m-pair-topology.xml

@@ -10,7 +10,7 @@
         <exe reachable="true">fairmq-ex-n-m-synchronizer --id sync --rate 100 --color false -P dds --channel-config name=sync,type=pub,method=bind</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="write">fmqchan_sync</id>
+            <name access="write">fmqchan_sync</name>
         </properties>
     </decltask>
 
@@ -18,8 +18,8 @@
         <exe reachable="true">fairmq-ex-n-m-sender --id sender%taskIndex% --timeframe-size 100000 --num-receivers ${numReceivers} --color false -P dds --channel-config name=sync,type=sub,method=connect name=data,type=pair,method=connect,numSockets=${numReceivers} --dds-i data:%taskIndex%</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="read">fmqchan_sync</id>
-            <name access="read">fmqchan_data</id>
+            <name access="read">fmqchan_sync</name>
+            <name access="read">fmqchan_data</name>
         </properties>
     </decltask>
 
@@ -27,7 +27,7 @@
         <exe reachable="true">fairmq-ex-n-m-receiver --id receiver%taskIndex% --num-senders ${numSenders} --color false -P dds --max-timeframes 10 --channel-config name=data,type=pair,method=bind,numSockets=${numSenders}</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="write">fmqchan_data</id>
+            <name access="write">fmqchan_data</name>
         </properties>
     </decltask>
 

examples/n-m/ex-n-m-topology.xml

@@ -10,7 +10,7 @@
         <exe reachable="true">fairmq-ex-n-m-synchronizer --id sync --rate 100 --color false -P dds --channel-config name=sync,type=pub,method=bind</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="write">fmqchan_sync</id>
+            <name access="write">fmqchan_sync</name>
         </properties>
     </decltask>
 
@@ -18,8 +18,8 @@
         <exe reachable="true">fairmq-ex-n-m-sender --id sender%taskIndex% --timeframe-size 100000 --num-receivers ${numReceivers} --color false -P dds --channel-config name=sync,type=sub,method=connect name=data,type=push,method=connect,numSockets=${numReceivers}</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="read">fmqchan_sync</id>
-            <name access="read">fmqchan_data</id>
+            <name access="read">fmqchan_sync</name>
+            <name access="read">fmqchan_data</name>
         </properties>
     </decltask>
 
@@ -27,7 +27,7 @@
         <exe reachable="true">fairmq-ex-n-m-receiver --id receiver%taskIndex% --num-senders ${numSenders} --color false -P dds --max-timeframes 10 --channel-config name=data,type=pull,method=bind</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="write">fmqchan_data</id>
+            <name access="write">fmqchan_data</name>
         </properties>
     </decltask>
 

examples/qc/CMakeLists.txt

@@ -21,7 +21,7 @@ target_link_libraries(fairmq-ex-qc-sink PRIVATE FairMQ)
 add_custom_target(ExampleQC DEPENDS fairmq-ex-qc-sampler fairmq-ex-qc-dispatcher fairmq-ex-qc-task fairmq-ex-qc-sink)
 
 list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
-set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
+set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/sdk)
 set(DATA_DIR ${CMAKE_CURRENT_BINARY_DIR})
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-qc-topology.xml ${CMAKE_CURRENT_BINARY_DIR}/ex-qc-topology.xml @ONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-qc-env.sh ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-qc-env.sh @ONLY)

examples/qc/fairmq-start-ex-qc.sh.in

@@ -51,7 +51,7 @@ fairmq-dds-command-ui -c k
 fairmq-dds-command-ui -c b
 fairmq-dds-command-ui -c x
 fairmq-dds-command-ui -c j
-allexceptqctasks="main/(Sampler|QCDispatcher|Sink)"
+allexceptqctasks="main/(Sampler|QCDispatcher|Sink).*"
 fairmq-dds-command-ui -c r -p $allexceptqctasks
 qctask="main/QCTask.*"
 qcdispatcher="main/QCDispatcher.*"

examples/region/Sampler.cxx

@@ -23,36 +23,41 @@ namespace example_region
 
 Sampler::Sampler()
     : fMsgSize(10000)
+    , fLinger(100)
     , fMaxIterations(0)
     , fNumIterations(0)
     , fRegion(nullptr)
     , fNumUnackedMsgs(0)
-{
-}
+{}
 
 void Sampler::InitTask()
 {
     fMsgSize = fConfig->GetProperty<int>("msg-size");
+    fLinger = fConfig->GetProperty<uint32_t>("region-linger");
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
 
     fChannels.at("data").at(0).Transport()->SubscribeToRegionEvents([](FairMQRegionInfo info) {
-        LOG(warn) << ">>>" << info.event;
-        LOG(warn) << "id: " << info.id;
-        LOG(warn) << "ptr: " << info.ptr;
-        LOG(warn) << "size: " << info.size;
-        LOG(warn) << "flags: " << info.flags;
+        LOG(info) << "Region event: " << info.event
+                  << ", managed: " << info.managed
+                  << ", id: " << info.id
+                  << ", ptr: " << info.ptr
+                  << ", size: " << info.size
+                  << ", flags: " << info.flags;
     });
 
     fRegion = FairMQUnmanagedRegionPtr(NewUnmanagedRegionFor("data",
                                                              0,
                                                              10000000,
                                                              [this](const std::vector<fair::mq::RegionBlock>& blocks) { // callback to be called when message buffers no longer needed by transport
+                                                                 lock_guard<mutex> lock(fMtx);
                                                                  fNumUnackedMsgs -= blocks.size();
+
                                                                  if (fMaxIterations > 0) {
-                                                                    LOG(debug) << "Received " << blocks.size() << " acks";
+                                                                    LOG(info) << "Received " << blocks.size() << " acks";
                                                                  }
                                                              }
                                                              ));
+    fRegion->SetLinger(fLinger);
 }
 
 bool Sampler::ConditionalRun()
@@ -69,27 +74,30 @@ bool Sampler::ConditionalRun()
     // LOG(info) << "check: " << static_cast<char*>(fRegion->GetData())[3];
     // std::this_thread::sleep_for(std::chrono::seconds(1));
 
+    lock_guard<mutex> lock(fMtx);
     if (Send(msg, "data", 0) > 0) {
-        ++fNumUnackedMsgs;
-
         if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
             LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
             return false;
         }
     }
+    ++fNumUnackedMsgs;
 
     return true;
 }
 
 void Sampler::ResetTask()
 {
-    // if not all messages acknowledged, wait for a bit. But only once, since receiver could be already dead.
-    if (fNumUnackedMsgs != 0) {
-        LOG(debug) << "waiting for all acknowledgements... (" << fNumUnackedMsgs << ")";
-        this_thread::sleep_for(chrono::milliseconds(500));
-        LOG(debug) << "done, still unacked: " << fNumUnackedMsgs;
-    }
+    // On destruction UnmanagedRegion will try to TODO
     fRegion.reset();
+    {
+        lock_guard<mutex> lock(fMtx);
+        if (fNumUnackedMsgs != 0) {
+            LOG(info) << "Done, still not acknowledged: " << fNumUnackedMsgs;
+        } else {
+            LOG(info) << "All acknowledgements received.";
+        }
+    }
     fChannels.at("data").at(0).Transport()->UnsubscribeFromRegionEvents();
 }
 

examples/region/Sampler.h

@@ -15,7 +15,8 @@
 #ifndef FAIRMQEXAMPLEREGIONSAMPLER_H
 #define FAIRMQEXAMPLEREGIONSAMPLER_H
 
-#include <atomic>
+#include <mutex>
+#include <cstdint>
 
 #include "FairMQDevice.h"
 
@@ -35,10 +36,12 @@ class Sampler : public FairMQDevice
 
   private:
     int fMsgSize;
+    uint32_t fLinger;
     uint64_t fMaxIterations;
     uint64_t fNumIterations;
     FairMQUnmanagedRegionPtr fRegion;
-    std::atomic<uint64_t> fNumUnackedMsgs;
+    std::mutex fMtx;
+    uint64_t fNumUnackedMsgs;
 };
 
 } // namespace example_region

examples/region/Sink.cxx

@@ -30,11 +30,12 @@ void Sink::InitTask()
     // Get the fMaxIterations value from the command line options (via fConfig)
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
     fChannels.at("data").at(0).Transport()->SubscribeToRegionEvents([](FairMQRegionInfo info) {
-        LOG(warn) << ">>>" << info.event;
-        LOG(warn) << "id: " << info.id;
-        LOG(warn) << "ptr: " << info.ptr;
-        LOG(warn) << "size: " << info.size;
-        LOG(warn) << "flags: " << info.flags;
+        LOG(info) << "Region event: " << info.event
+                  << ", managed: " << info.managed
+                  << ", id: " << info.id
+                  << ", ptr: " << info.ptr
+                  << ", size: " << info.size
+                  << ", flags: " << info.flags;
     });
 }
 

examples/region/runSampler.cxx

@@ -15,6 +15,7 @@ void addCustomOptions(bpo::options_description& options)
 {
     options.add_options()
         ("msg-size", bpo::value<int>()->default_value(1000), "Message size in bytes")
+        ("region-linger", bpo::value<uint32_t>()->default_value(100), "Linger period for regions")
         ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
 }
 

examples/region/test-ex-region.sh.in

@@ -23,6 +23,7 @@ SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --control static --color false"
 SAMPLER+=" --max-iterations 1"
 SAMPLER+=" --msg-size $msgSize"
+SAMPLER+=" --region-linger 500"
 SAMPLER+=" --channel-config name=data,type=push,method=bind,address=tcp://127.0.0.1:7777"
 @CMAKE_CURRENT_BINARY_DIR@/$SAMPLER &
 SAMPLER_PID=$!

fairmq/CMakeLists.txt

@@ -170,15 +170,10 @@ if(BUILD_FAIRMQ)
     PluginManager.h
     PluginServices.h
     runFairMQDevice.h
+    shmem/Monitor.h
   )
 
   set(FAIRMQ_PRIVATE_HEADER_FILES
-    devices/FairMQBenchmarkSampler.h
-    devices/FairMQMerger.h
-    devices/FairMQMultiplier.h
-    devices/FairMQProxy.h
-    devices/FairMQSink.h
-    devices/FairMQSplitter.h
     plugins/Builtin.h
     plugins/config/Config.h
     plugins/Control.h
@@ -220,10 +215,6 @@ if(BUILD_FAIRMQ)
     FairMQPoller.cxx
     FairMQSocket.cxx
     FairMQTransportFactory.cxx
-    devices/FairMQMerger.cxx
-    devices/FairMQMultiplier.cxx
-    devices/FairMQProxy.cxx
-    devices/FairMQSplitter.cxx
     Plugin.cxx
     PluginManager.cxx
     PluginServices.cxx
@@ -234,6 +225,7 @@ if(BUILD_FAIRMQ)
     plugins/config/Config.cxx
     plugins/Control.cxx
     MemoryResources.cxx
+    shmem/Monitor.cxx
   )
 
   if(BUILD_OFI_TRANSPORT)
@@ -276,6 +268,9 @@ if(BUILD_FAIRMQ)
   # preprocessor definitions #
   ############################
   target_compile_definitions(${_target} PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
+  if(FAIRMQ_DEBUG_MODE)
+    target_compile_definitions(${_target} PUBLIC FAIRMQ_DEBUG_MODE)
+  endif()
   if(BUILD_OFI_TRANSPORT)
     target_compile_definitions(${_target} PRIVATE BUILD_OFI_TRANSPORT)
   endif()
@@ -372,12 +367,16 @@ if(BUILD_FAIRMQ)
 
   add_executable(fairmq-shmmonitor shmem/Monitor.cxx shmem/Monitor.h shmem/runMonitor.cxx)
   target_compile_definitions(fairmq-shmmonitor PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
+  if(FAIRMQ_DEBUG_MODE)
+    target_compile_definitions(fairmq-shmmonitor PUBLIC FAIRMQ_DEBUG_MODE)
+  endif()
   target_link_libraries(fairmq-shmmonitor PUBLIC
     Threads::Threads
     $<$<PLATFORM_ID:Linux>:rt>
     Boost::boost
     Boost::date_time
     Boost::program_options
+    FairLogger::FairLogger
     PicoSHA2
   )
   target_include_directories(fairmq-shmmonitor PUBLIC

fairmq/DeviceRunner.cxx

@@ -8,10 +8,14 @@
 
 #include "DeviceRunner.h"
 
-#include <exception>
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Strings.h>
+#include <fairmq/tools/Version.h>
 #include <fairmq/Version.h>
 
+#include <fairlogger/Logger.h>
+
+#include <exception>
+
 using namespace std;
 using namespace fair::mq;
 
@@ -52,7 +56,9 @@ bool DeviceRunner::HandleGeneralOptions(const fair::mq::ProgOptions& config, boo
             fair::Logger::SetConsoleSeverity("nolog");
         } else {
             fair::Logger::SetConsoleColor(color);
-            fair::Logger::SetConsoleSeverity(severity);
+            if (severity != "") {
+                fair::Logger::SetConsoleSeverity(severity);
+            }
         }
 
         if (printLogo) {
@@ -150,8 +156,8 @@ auto DeviceRunner::Run() -> int
 
     // Handle --version
     if (fConfig.Count("version")) {
-        cout << "FairMQ version: " << FAIRMQ_GIT_VERSION << endl;
-        cout << "User device version: " << fDevice->GetVersion() << endl;
+        LOGV(info, verylow) << "FairMQ version: " << FAIRMQ_GIT_VERSION;
+        LOGV(info, verylow) << "User device version: " << fDevice->GetVersion();
         fDevice->ChangeState(fair::mq::Transition::End);
         return 0;
     }

fairmq/DeviceRunner.h

@@ -13,7 +13,6 @@
 #include <fairmq/PluginManager.h>
 #include <fairmq/ProgOptions.h>
 #include <FairMQDevice.h>
-#include <FairMQLogger.h>
 
 #include <functional>
 #include <memory>

fairmq/FairMQChannel.cxx

@@ -7,11 +7,15 @@
  ********************************************************************************/
 
 #include "FairMQChannel.h"
+
 #include <fairmq/tools/Strings.h>
 #include <fairmq/Properties.h>
 
+#include <fairlogger/Logger.h>
+
 #include <boost/algorithm/string.hpp> // join/split
 
+#include <cstddef> // size_t
 #include <regex>
 #include <set>
 #include <random>
@@ -77,31 +81,30 @@ FairMQChannel::FairMQChannel(const string& name, const string& type, const strin
     , fPortRangeMin(DefaultPortRangeMin)
     , fPortRangeMax(DefaultPortRangeMax)
     , fAutoBind(DefaultAutoBind)
-    , fIsValid(false)
+    , fValid(false)
     , fMultipart(false)
-    , fModified(true)
-    , fReset(false)
-    , fMtx()
-{}
+{
+    // LOG(warn) << "Constructing channel '" << fName << "'";
+}
 
 FairMQChannel::FairMQChannel(const string& name, int index, const fair::mq::Properties& properties)
     : FairMQChannel(tools::ToString(name, "[", index, "]"), "unspecified", "unspecified", "unspecified", nullptr)
 {
     string prefix(tools::ToString("chans.", name, ".", index, "."));
 
-    fType = GetPropertyOrDefault(properties, string(prefix + "type"), fType);
-    fMethod = GetPropertyOrDefault(properties, string(prefix + "method"), fMethod);
-    fAddress = GetPropertyOrDefault(properties, string(prefix + "address"), fAddress);
-    fTransportType = TransportTypes.at(GetPropertyOrDefault(properties, string(prefix + "transport"), TransportNames.at(fTransportType)));
-    fSndBufSize = GetPropertyOrDefault(properties, string(prefix + "sndBufSize"), fSndBufSize);
-    fRcvBufSize = GetPropertyOrDefault(properties, string(prefix + "rcvBufSize"), fRcvBufSize);
-    fSndKernelSize = GetPropertyOrDefault(properties, string(prefix + "sndKernelSize"), fSndKernelSize);
-    fRcvKernelSize = GetPropertyOrDefault(properties, string(prefix + "rcvKernelSize"), fRcvKernelSize);
-    fLinger = GetPropertyOrDefault(properties, string(prefix + "linger"), fLinger);
-    fRateLogging = GetPropertyOrDefault(properties, string(prefix + "rateLogging"), fRateLogging);
-    fPortRangeMin = GetPropertyOrDefault(properties, string(prefix + "portRangeMin"), fPortRangeMin);
-    fPortRangeMax = GetPropertyOrDefault(properties, string(prefix + "portRangeMax"), fPortRangeMax);
-    fAutoBind = GetPropertyOrDefault(properties, string(prefix + "autoBind"), fAutoBind);
+    fType = GetPropertyOrDefault(properties, string(prefix + "type"), std::string(DefaultType));
+    fMethod = GetPropertyOrDefault(properties, string(prefix + "method"), std::string(DefaultMethod));
+    fAddress = GetPropertyOrDefault(properties, string(prefix + "address"), std::string(DefaultAddress));
+    fTransportType = TransportType(GetPropertyOrDefault(properties, string(prefix + "transport"), std::string(DefaultTransportName)));
+    fSndBufSize = GetPropertyOrDefault(properties, string(prefix + "sndBufSize"), DefaultSndBufSize);
+    fRcvBufSize = GetPropertyOrDefault(properties, string(prefix + "rcvBufSize"), DefaultRcvBufSize);
+    fSndKernelSize = GetPropertyOrDefault(properties, string(prefix + "sndKernelSize"), DefaultSndKernelSize);
+    fRcvKernelSize = GetPropertyOrDefault(properties, string(prefix + "rcvKernelSize"), DefaultRcvKernelSize);
+    fLinger = GetPropertyOrDefault(properties, string(prefix + "linger"), DefaultLinger);
+    fRateLogging = GetPropertyOrDefault(properties, string(prefix + "rateLogging"), DefaultRateLogging);
+    fPortRangeMin = GetPropertyOrDefault(properties, string(prefix + "portRangeMin"), DefaultPortRangeMin);
+    fPortRangeMax = GetPropertyOrDefault(properties, string(prefix + "portRangeMax"), DefaultPortRangeMax);
+    fAutoBind = GetPropertyOrDefault(properties, string(prefix + "autoBind"), DefaultAutoBind);
 }
 
 FairMQChannel::FairMQChannel(const FairMQChannel& chan)
@@ -125,10 +128,8 @@ FairMQChannel::FairMQChannel(const FairMQChannel& chan, const string& newName)
     , fPortRangeMin(chan.fPortRangeMin)
     , fPortRangeMax(chan.fPortRangeMax)
     , fAutoBind(chan.fAutoBind)
-    , fIsValid(false)
+    , fValid(false)
     , fMultipart(chan.fMultipart)
-    , fModified(chan.fModified)
-    , fReset(false)
 {}
 
 FairMQChannel& FairMQChannel::operator=(const FairMQChannel& chan)
@@ -137,372 +138,34 @@ FairMQChannel& FairMQChannel::operator=(const FairMQChannel& chan)
         return *this;
     }
 
-    {
-        // TODO: replace this with std::scoped_lock (c++17)
-        lock(fMtx, chan.fMtx);
-        lock_guard<mutex> lock1(fMtx, adopt_lock);
-        lock_guard<mutex> lock2(chan.fMtx, adopt_lock);
-
-        fTransportFactory = nullptr;
-        fTransportType = chan.fTransportType;
-        fSocket = nullptr;
-        fName = chan.fName;
-        fType = chan.fType;
-        fMethod = chan.fMethod;
-        fAddress = chan.fAddress;
-        fSndBufSize = chan.fSndBufSize;
-        fRcvBufSize = chan.fRcvBufSize;
-        fSndKernelSize = chan.fSndKernelSize;
-        fRcvKernelSize = chan.fRcvKernelSize;
-        fLinger = chan.fLinger;
-        fRateLogging = chan.fRateLogging;
-        fPortRangeMin = chan.fPortRangeMin;
-        fPortRangeMax = chan.fPortRangeMax;
-        fAutoBind = chan.fAutoBind;
-        fIsValid = false;
-        fMultipart = chan.fMultipart;
-        fModified = chan.fModified;
-        fReset = false;
-    }
+    fTransportFactory = nullptr;
+    fTransportType = chan.fTransportType;
+    fSocket = nullptr;
+    fName = chan.fName;
+    fType = chan.fType;
+    fMethod = chan.fMethod;
+    fAddress = chan.fAddress;
+    fSndBufSize = chan.fSndBufSize;
+    fRcvBufSize = chan.fRcvBufSize;
+    fSndKernelSize = chan.fSndKernelSize;
+    fRcvKernelSize = chan.fRcvKernelSize;
+    fLinger = chan.fLinger;
+    fRateLogging = chan.fRateLogging;
+    fPortRangeMin = chan.fPortRangeMin;
+    fPortRangeMax = chan.fPortRangeMax;
+    fAutoBind = chan.fAutoBind;
+    fValid = false;
+    fMultipart = chan.fMultipart;
 
     return *this;
 }
 
-FairMQSocket & FairMQChannel::GetSocket() const
-{
-    assert(fSocket);
-    return *fSocket;
-}
-
-string FairMQChannel::GetName() const
-{
-    lock_guard<mutex> lock(fMtx);
-    return fName;
-}
-
-string FairMQChannel::GetPrefix() const
-{
-    lock_guard<mutex> lock(fMtx);
-    string prefix = fName;
-    prefix = prefix.erase(fName.rfind('['));
-    return prefix;
-}
-
-string FairMQChannel::GetIndex() const
-{
-    lock_guard<mutex> lock(fMtx);
-    string indexStr = fName;
-    indexStr.erase(indexStr.rfind(']'));
-    indexStr.erase(0, indexStr.rfind('[') + 1);
-    return indexStr;
-}
-
-string FairMQChannel::GetType() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fType;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetType: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-string FairMQChannel::GetMethod() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fMethod;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetMethod: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-string FairMQChannel::GetAddress() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fAddress;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetAddress: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-string FairMQChannel::GetTransportName() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return TransportNames.at(fTransportType);
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetTransportName: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-Transport FairMQChannel::GetTransportType() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fTransportType;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetTransportType: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-
-int FairMQChannel::GetSndBufSize() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fSndBufSize;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetSndBufSize: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-int FairMQChannel::GetRcvBufSize() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fRcvBufSize;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetRcvBufSize: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-int FairMQChannel::GetSndKernelSize() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fSndKernelSize;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetSndKernelSize: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-int FairMQChannel::GetRcvKernelSize() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fRcvKernelSize;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetRcvKernelSize: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-int FairMQChannel::GetLinger() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fLinger;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetLinger: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-int FairMQChannel::GetRateLogging() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fRateLogging;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetRateLogging: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-int FairMQChannel::GetPortRangeMin() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fPortRangeMin;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetPortRangeMin: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-int FairMQChannel::GetPortRangeMax() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fPortRangeMax;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetPortRangeMax: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-bool FairMQChannel::GetAutoBind() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fAutoBind;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::GetAutoBind: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdateType(const string& type)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fType = type;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdateType: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdateMethod(const string& method)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fMethod = method;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdateMethod: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdateAddress(const string& address)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fAddress = address;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdateAddress: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdateTransport(const string& transport)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fTransportType = TransportTypes.at(transport);
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdateTransport: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdateSndBufSize(const int sndBufSize)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fSndBufSize = sndBufSize;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdateSndBufSize: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdateRcvBufSize(const int rcvBufSize)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fRcvBufSize = rcvBufSize;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdateRcvBufSize: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdateSndKernelSize(const int sndKernelSize)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fSndKernelSize = sndKernelSize;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdateSndKernelSize: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdateRcvKernelSize(const int rcvKernelSize)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fRcvKernelSize = rcvKernelSize;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdateRcvKernelSize: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdateLinger(const int duration)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fLinger = duration;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdateLinger: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdateRateLogging(const int rateLogging)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fRateLogging = rateLogging;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdateRateLogging: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdatePortRangeMin(const int minPort)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fPortRangeMin = minPort;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdatePortRangeMin: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdatePortRangeMax(const int maxPort)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fPortRangeMax = maxPort;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdatePortRangeMax: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdateAutoBind(const bool autobind)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fAutoBind = autobind;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdateAutoBind: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-auto FairMQChannel::SetModified(const bool modified) -> void
-try {
-    lock_guard<mutex> lock(fMtx);
-    fModified = modified;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::SetModified: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-void FairMQChannel::UpdateName(const string& name)
-try {
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    fName = name;
-    fModified = true;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::UpdateName: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
-bool FairMQChannel::IsValid() const
-try {
-    lock_guard<mutex> lock(fMtx);
-    return fIsValid;
-} catch (exception& e) {
-    LOG(error) << "Exception caught in FairMQChannel::IsValid: " << e.what();
-    throw ChannelConfigurationError(tools::ToString("failed to acquire lock: ", e.what()));
-}
-
 bool FairMQChannel::Validate()
 try {
-    lock_guard<mutex> lock(fMtx);
     stringstream ss;
     ss << "Validating channel '" << fName << "'... ";
 
-    if (fIsValid) {
+    if (fValid) {
         ss << "ALREADY VALID";
         LOG(debug) << ss.str();
         return true;
@@ -535,7 +198,7 @@ try {
     } else {
         vector<string> endpoints;
         boost::algorithm::split(endpoints, fAddress, boost::algorithm::is_any_of(";"));
-        for (const auto endpoint : endpoints) {
+        for (const auto& endpoint : endpoints) {
             string address;
             if (endpoint[0] == '@' || endpoint[0] == '+' || endpoint[0] == '>') {
                 address = endpoint.substr(1);
@@ -637,7 +300,7 @@ try {
         throw ChannelConfigurationError(tools::ToString("invalid socket rate logging interval (cannot be negative): '", fRateLogging, "'"));
     }
 
-    fIsValid = true;
+    fValid = true;
     ss << "VALID";
     LOG(debug) << ss.str();
     return true;
@@ -648,8 +311,6 @@ try {
 
 void FairMQChannel::Init()
 {
-    lock_guard<mutex> lock(fMtx);
-
     fSocket = fTransportFactory->CreateSocket(fType, fName);
 
     // set linger duration (how long socket should wait for outstanding transfers before shutdown)
@@ -670,19 +331,22 @@ void FairMQChannel::Init()
 
 bool FairMQChannel::ConnectEndpoint(const string& endpoint)
 {
-    lock_guard<mutex> lock(fMtx);
-
     return fSocket->Connect(endpoint);
 }
 
 bool FairMQChannel::BindEndpoint(string& endpoint)
 {
-    lock_guard<mutex> lock(fMtx);
-
     // try to bind to the configured port. If it fails, try random one (if AutoBind is on).
     if (fSocket->Bind(endpoint)) {
         return true;
     } else {
+        // auto-bind only implemented for TCP
+        size_t protocolPos = endpoint.find(':');
+        string protocol = endpoint.substr(0, protocolPos);
+        if (protocol != "tcp") {
+            return false;
+        }
+
         if (fAutoBind) {
             // number of attempts when choosing a random port
             int numAttempts = 0;
@@ -712,10 +376,3 @@ bool FairMQChannel::BindEndpoint(string& endpoint)
     }
 
 }
-
-void FairMQChannel::ResetChannel()
-{
-    lock_guard<mutex> lock(fMtx);
-    fIsValid = false;
-    // TODO: implement channel resetting
-}

fairmq/FairMQChannel.h

@@ -13,7 +13,6 @@
 #include <FairMQUnmanagedRegion.h>
 #include <FairMQSocket.h>
 #include <fairmq/Transports.h>
-#include <FairMQLogger.h>
 #include <FairMQParts.h>
 #include <fairmq/Properties.h>
 #include <FairMQMessage.h>
@@ -24,9 +23,14 @@
 #include <mutex>
 #include <stdexcept>
 #include <utility> // std::move
-#include <cstddef> // size_t
 #include <cstdint> // int64_t
 
+/**
+ * @class FairMQChannel FairMQChannel.h <FairMQChannel.h>
+ * @brief Wrapper class for FairMQSocket and related methods
+ *
+ * The class is not thread-safe.
+ */
 class FairMQChannel
 {
     friend class FairMQDevice;
@@ -68,23 +72,20 @@ class FairMQChannel
     FairMQChannel(const FairMQChannel&, const std::string& name);
 
     /// Move constructor
-    FairMQChannel(FairMQChannel&&) = delete;
+    // FairMQChannel(FairMQChannel&&) = delete;
 
     /// Assignment operator
     FairMQChannel& operator=(const FairMQChannel&);
 
     /// Move assignment operator
-    FairMQChannel& operator=(FairMQChannel&&) = delete;
+    // FairMQChannel& operator=(FairMQChannel&&) = delete;
 
     /// Destructor
-    virtual ~FairMQChannel()
-    {
-        // LOG(debug) << "Destroying channel " << fName;
-    }
+    virtual ~FairMQChannel() { /* LOG(warn) << "Destroying channel '" << fName << "'"; */ }
 
     struct ChannelConfigurationError : std::runtime_error { using std::runtime_error::runtime_error; };
 
-    FairMQSocket& GetSocket() const;
+    FairMQSocket& GetSocket() const { assert(fSocket); return *fSocket; }
 
     bool Bind(const std::string& address)
     {
@@ -102,139 +103,142 @@ class FairMQChannel
 
     /// Get channel name
     /// @return Returns full channel name (e.g. "data[0]")
-    std::string GetChannelName() const __attribute__((deprecated("Use GetName()"))) { return GetName(); }
-    std::string GetName() const ;
+    std::string GetName() const { return fName; }
 
     /// Get channel prefix
     /// @return Returns channel prefix (e.g. "data" in "data[0]")
-    std::string GetChannelPrefix() const  __attribute__((deprecated("Use GetPrefix()"))) { return GetPrefix(); }
-    std::string GetPrefix() const;
+    std::string GetPrefix() const
+    {
+        std::string prefix = fName;
+        prefix = prefix.erase(fName.rfind('['));
+        return prefix;
+    }
 
     /// Get channel index
     /// @return Returns channel index (e.g. 0 in "data[0]")
-    std::string GetChannelIndex() const __attribute__((deprecated("Use GetIndex()"))) { return GetIndex(); }
-    std::string GetIndex() const;
+    std::string GetIndex() const
+    {
+        std::string indexStr = fName;
+        indexStr.erase(indexStr.rfind(']'));
+        indexStr.erase(0, indexStr.rfind('[') + 1);
+        return indexStr;
+    }
 
     /// Get socket type
     /// @return Returns socket type (push/pull/pub/sub/spub/xsub/pair/req/rep/dealer/router/)
-    std::string GetType() const;
+    std::string GetType() const { return fType; }
 
     /// Get socket method
     /// @return Returns socket method (bind/connect)
-    std::string GetMethod() const;
+    std::string GetMethod() const { return fMethod; }
 
     /// Get socket address (e.g. "tcp://127.0.0.1:5555" or "ipc://abc")
     /// @return Returns socket address (e.g. "tcp://127.0.0.1:5555" or "ipc://abc")
-    std::string GetAddress() const;
+    std::string GetAddress() const { return fAddress; }
 
     /// Get channel transport name ("default", "zeromq" or "shmem")
     /// @return Returns channel transport name (e.g. "default", "zeromq" or "shmem")
-    std::string GetTransportName() const;
+    std::string GetTransportName() const { return fair::mq::TransportName(fTransportType); }
 
     /// Get channel transport type
     /// @return Returns channel transport type
-    fair::mq::Transport GetTransportType() const;
+    fair::mq::Transport GetTransportType() const { return fTransportType; }
 
     /// Get socket send buffer size (in number of messages)
     /// @return Returns socket send buffer size (in number of messages)
-    int GetSndBufSize() const;
+    int GetSndBufSize() const { return fSndBufSize; }
 
     /// Get socket receive buffer size (in number of messages)
     /// @return Returns socket receive buffer size (in number of messages)
-    int GetRcvBufSize() const;
+    int GetRcvBufSize() const { return fRcvBufSize; }
 
     /// Get socket kernel transmit send buffer size (in bytes)
     /// @return Returns socket kernel transmit send buffer size (in bytes)
-    int GetSndKernelSize() const;
+    int GetSndKernelSize() const { return fSndKernelSize; }
 
     /// Get socket kernel transmit receive buffer size (in bytes)
     /// @return Returns socket kernel transmit receive buffer size (in bytes)
-    int GetRcvKernelSize() const;
+    int GetRcvKernelSize() const { return fRcvKernelSize; }
 
     /// Get linger duration (in milliseconds)
     /// @return Returns linger duration (in milliseconds)
-    int GetLinger() const;
+    int GetLinger() const { return fLinger; }
 
     /// Get socket rate logging interval (in seconds)
     /// @return Returns socket rate logging interval (in seconds)
-    int GetRateLogging() const;
+    int GetRateLogging() const { return fRateLogging; }
 
     /// Get start of the port range for automatic binding
     /// @return start of the port range
-    int GetPortRangeMin() const;
+    int GetPortRangeMin() const { return fPortRangeMin; }
 
     /// Get end of the port range for automatic binding
     /// @return end of the port range
-    int GetPortRangeMax() const;
+    int GetPortRangeMax() const { return fPortRangeMax; }
 
     /// Set automatic binding (pick random port if bind fails)
     /// @return true/false, true if automatic binding is enabled
-    bool GetAutoBind() const;
-
-    /// Set socket type
-    /// @param type Socket type (push/pull/pub/sub/spub/xsub/pair/req/rep/dealer/router/)
-    void UpdateType(const std::string& type);
-
-    /// Set socket method
-    /// @param method Socket method (bind/connect)
-    void UpdateMethod(const std::string& method);
-
-    /// Set socket address
-    /// @param Socket address (e.g. "tcp://127.0.0.1:5555" or "ipc://abc")
-    void UpdateAddress(const std::string& address);
-
-    /// Set channel transport
-    /// @param transport transport string ("default", "zeromq" or "shmem")
-    void UpdateTransport(const std::string& transport);
-
-    /// Set socket send buffer size
-    /// @param sndBufSize Socket send buffer size (in number of messages)
-    void UpdateSndBufSize(const int sndBufSize);
-
-    /// Set socket receive buffer size
-    /// @param rcvBufSize Socket receive buffer size (in number of messages)
-    void UpdateRcvBufSize(const int rcvBufSize);
-
-    /// Set socket kernel transmit send buffer size (in bytes)
-    /// @param sndKernelSize Socket send buffer size (in bytes)
-    void UpdateSndKernelSize(const int sndKernelSize);
-
-    /// Set socket kernel transmit receive buffer size (in bytes)
-    /// @param rcvKernelSize Socket receive buffer size (in bytes)
-    void UpdateRcvKernelSize(const int rcvKernelSize);
-
-    /// Set linger duration (in milliseconds)
-    /// @param duration linger duration (in milliseconds)
-    void UpdateLinger(const int duration);
-
-    /// Set socket rate logging interval (in seconds)
-    /// @param rateLogging Socket rate logging interval (in seconds)
-    void UpdateRateLogging(const int rateLogging);
-
-    /// Set start of the port range for automatic binding
-    /// @param minPort start of the port range
-    void UpdatePortRangeMin(const int minPort);
-
-    /// Set end of the port range for automatic binding
-    /// @param maxPort end of the port range
-    void UpdatePortRangeMax(const int maxPort);
-
-    /// Set automatic binding (pick random port if bind fails)
-    /// @param autobind true/false, true to enable automatic binding
-    void UpdateAutoBind(const bool autobind);
+    bool GetAutoBind() const { return fAutoBind; }
 
     /// Set channel name
     /// @param name Arbitrary channel name
-    void UpdateChannelName(const std::string& name) __attribute__((deprecated("Use UpdateName()"))) { UpdateName(name); }
-    void UpdateName(const std::string& name);
+    void UpdateName(const std::string& name) { fName = name; Invalidate(); }
+
+    /// Set socket type
+    /// @param type Socket type (push/pull/pub/sub/spub/xsub/pair/req/rep/dealer/router/)
+    void UpdateType(const std::string& type) { fType = type; Invalidate(); }
+
+    /// Set socket method
+    /// @param method Socket method (bind/connect)
+    void UpdateMethod(const std::string& method) { fMethod = method; Invalidate(); }
+
+    /// Set socket address
+    /// @param Socket address (e.g. "tcp://127.0.0.1:5555" or "ipc://abc")
+    void UpdateAddress(const std::string& address) { fAddress = address; Invalidate(); }
+
+    /// Set channel transport
+    /// @param transport transport string ("default", "zeromq" or "shmem")
+    void UpdateTransport(const std::string& transport) { fTransportType = fair::mq::TransportType(transport); Invalidate(); }
+
+    /// Set socket send buffer size
+    /// @param sndBufSize Socket send buffer size (in number of messages)
+    void UpdateSndBufSize(const int sndBufSize) { fSndBufSize = sndBufSize; Invalidate(); }
+
+    /// Set socket receive buffer size
+    /// @param rcvBufSize Socket receive buffer size (in number of messages)
+    void UpdateRcvBufSize(const int rcvBufSize) { fRcvBufSize = rcvBufSize; Invalidate(); }
+
+    /// Set socket kernel transmit send buffer size (in bytes)
+    /// @param sndKernelSize Socket send buffer size (in bytes)
+    void UpdateSndKernelSize(const int sndKernelSize) { fSndKernelSize = sndKernelSize; Invalidate(); }
+
+    /// Set socket kernel transmit receive buffer size (in bytes)
+    /// @param rcvKernelSize Socket receive buffer size (in bytes)
+    void UpdateRcvKernelSize(const int rcvKernelSize) { fRcvKernelSize = rcvKernelSize; Invalidate(); }
+
+    /// Set linger duration (in milliseconds)
+    /// @param duration linger duration (in milliseconds)
+    void UpdateLinger(const int duration) { fLinger = duration; Invalidate(); }
+
+    /// Set socket rate logging interval (in seconds)
+    /// @param rateLogging Socket rate logging interval (in seconds)
+    void UpdateRateLogging(const int rateLogging) { fRateLogging = rateLogging; Invalidate(); }
+
+    /// Set start of the port range for automatic binding
+    /// @param minPort start of the port range
+    void UpdatePortRangeMin(const int minPort) { fPortRangeMin = minPort; Invalidate(); }
+
+    /// Set end of the port range for automatic binding
+    /// @param maxPort end of the port range
+    void UpdatePortRangeMax(const int maxPort) { fPortRangeMax = maxPort; Invalidate(); }
+
+    /// Set automatic binding (pick random port if bind fails)
+    /// @param autobind true/false, true to enable automatic binding
+    void UpdateAutoBind(const bool autobind) { fAutoBind = autobind; Invalidate(); }
 
     /// Checks if the configured channel settings are valid (checks the validity parameter, without running full validation (as oposed to ValidateChannel()))
     /// @return true if channel settings are valid, false otherwise.
-    bool IsValid() const;
-
-    /// Validates channel configuration
-    /// @return true if channel settings are valid, false otherwise.
-    bool ValidateChannel() __attribute__((deprecated("Use Validate()"))) { return Validate(); }
+    bool IsValid() const { return fValid; }
 
     /// Validates channel configuration
     /// @return true if channel settings are valid, false otherwise.
@@ -246,14 +250,14 @@ class FairMQChannel
 
     bool BindEndpoint(std::string& endpoint);
 
-    /// Resets the channel (requires validation to be used again).
-    void ResetChannel();
+    /// invalidates the channel (requires validation to be used again).
+    void Invalidate() { fValid = false; }
 
     /// Sends a message to the socket queue.
     /// @param msg Constant reference of unique_ptr to a FairMQMessage
     /// @param sndTimeoutInMs send timeout in ms. -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot send)
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out. -1 if there was an error.
-    int Send(FairMQMessagePtr& msg, int sndTimeoutInMs = -1)
+    /// @return Number of bytes that have been queued, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
+    int64_t Send(FairMQMessagePtr& msg, int sndTimeoutInMs = -1)
     {
         CheckSendCompatibility(msg);
         return fSocket->Send(msg, sndTimeoutInMs);
@@ -262,8 +266,8 @@ class FairMQChannel
     /// Receives a message from the socket queue.
     /// @param msg Constant reference of unique_ptr to a FairMQMessage
     /// @param rcvTimeoutInMs receive timeout in ms. -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot receive)
-    /// @return Number of bytes that have been received. -2 if reading from the queue was not possible or timed out. -1 if there was an error.
-    int Receive(FairMQMessagePtr& msg, int rcvTimeoutInMs = -1)
+    /// @return Number of bytes that have been received, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
+    int64_t Receive(FairMQMessagePtr& msg, int rcvTimeoutInMs = -1)
     {
         CheckReceiveCompatibility(msg);
         return fSocket->Receive(msg, rcvTimeoutInMs);
@@ -272,7 +276,7 @@ class FairMQChannel
     /// Send a vector of messages
     /// @param msgVec message vector reference
     /// @param sndTimeoutInMs send timeout in ms. -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot send)
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been queued, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int64_t Send(std::vector<FairMQMessagePtr>& msgVec, int sndTimeoutInMs = -1)
     {
         CheckSendCompatibility(msgVec);
@@ -282,7 +286,7 @@ class FairMQChannel
     /// Receive a vector of messages
     /// @param msgVec message vector reference
     /// @param rcvTimeoutInMs receive timeout in ms. -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot receive)
-    /// @return Number of bytes that have been received. -2 if reading from the queue was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been received, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int64_t Receive(std::vector<FairMQMessagePtr>& msgVec, int rcvTimeoutInMs = -1)
     {
         CheckReceiveCompatibility(msgVec);
@@ -292,7 +296,7 @@ class FairMQChannel
     /// Send FairMQParts
     /// @param parts FairMQParts reference
     /// @param sndTimeoutInMs send timeout in ms. -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot send)
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been queued, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int64_t Send(FairMQParts& parts, int sndTimeoutInMs = -1)
     {
         return Send(parts.fParts, sndTimeoutInMs);
@@ -301,7 +305,7 @@ class FairMQChannel
     /// Receive FairMQParts
     /// @param parts FairMQParts reference
     /// @param rcvTimeoutInMs receive timeout in ms. -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot receive)
-    /// @return Number of bytes that have been received. -2 if reading from the queue was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been received, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int64_t Receive(FairMQParts& parts, int rcvTimeoutInMs = -1)
     {
         return Receive(parts.fParts, rcvTimeoutInMs);
@@ -312,10 +316,7 @@ class FairMQChannel
     unsigned long GetMessagesTx() const { return fSocket->GetMessagesTx(); }
     unsigned long GetMessagesRx() const { return fSocket->GetMessagesRx(); }
 
-    auto Transport() -> FairMQTransportFactory*
-    {
-        return fTransportFactory.get();
-    };
+    auto Transport() -> FairMQTransportFactory* { return fTransportFactory.get(); };
 
     template<typename... Args>
     FairMQMessagePtr NewMessage(Args&&... args)
@@ -376,13 +377,9 @@ class FairMQChannel
     int fPortRangeMax;
     bool fAutoBind;
 
-    bool fIsValid;
+    bool fValid;
 
     bool fMultipart;
-    bool fModified;
-    bool fReset;
-
-    mutable std::mutex fMtx;
 
     void CheckSendCompatibility(FairMQMessagePtr& msg)
     {
@@ -439,8 +436,6 @@ class FairMQChannel
         fTransportFactory = factory;
         fTransportType = factory->GetType();
     }
-
-    auto SetModified(const bool modified) -> void;
 };
 
 #endif /* FAIRMQCHANNEL_H_ */

fairmq/FairMQDevice.cxx

@@ -390,7 +390,6 @@ void FairMQDevice::AttachChannels(vector<FairMQChannel*>& chans)
         if ((*itr)->Validate()) {
             (*itr)->Init();
             if (AttachChannel(**itr)) {
-                (*itr)->SetModified(false);
                 // remove the channel from the uninitialized container
                 itr = chans.erase(itr);
             } else {

fairmq/FairMQDevice.h

@@ -26,7 +26,6 @@
 #include <algorithm> // find
 #include <string>
 #include <chrono>
-#include <iostream>
 #include <unordered_map>
 #include <functional>
 #include <stdexcept>
@@ -129,8 +128,8 @@ class FairMQDevice
     /// @param chan channel name
     /// @param i channel index
     /// @param sndTimeoutInMs send timeout in ms, -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot send)
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out. -1 if there was an error.
-    int Send(FairMQMessagePtr& msg, const std::string& channel, const int index = 0, int sndTimeoutInMs = -1)
+    /// @return Number of bytes that have been queued, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
+    int64_t Send(FairMQMessagePtr& msg, const std::string& channel, const int index = 0, int sndTimeoutInMs = -1)
     {
         return GetChannel(channel, index).Send(msg, sndTimeoutInMs);
     }
@@ -140,8 +139,8 @@ class FairMQDevice
     /// @param chan channel name
     /// @param i channel index
     /// @param rcvTimeoutInMs receive timeout in ms, -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot receive)
-    /// @return Number of bytes that have been received. -2 if reading from the queue was not possible or timed out. -1 if there was an error.
-    int Receive(FairMQMessagePtr& msg, const std::string& channel, const int index = 0, int rcvTimeoutInMs = -1)
+    /// @return Number of bytes that have been received, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
+    int64_t Receive(FairMQMessagePtr& msg, const std::string& channel, const int index = 0, int rcvTimeoutInMs = -1)
     {
         return GetChannel(channel, index).Receive(msg, rcvTimeoutInMs);
     }
@@ -151,7 +150,7 @@ class FairMQDevice
     /// @param chan channel name
     /// @param i channel index
     /// @param sndTimeoutInMs send timeout in ms, -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot send)
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been queued, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int64_t Send(FairMQParts& parts, const std::string& channel, const int index = 0, int sndTimeoutInMs = -1)
     {
         return GetChannel(channel, index).Send(parts.fParts, sndTimeoutInMs);
@@ -162,7 +161,7 @@ class FairMQDevice
     /// @param chan channel name
     /// @param i channel index
     /// @param rcvTimeoutInMs receive timeout in ms, -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot receive)
-    /// @return Number of bytes that have been received. -2 if reading from the queue was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been received, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int64_t Receive(FairMQParts& parts, const std::string& channel, const int index = 0, int rcvTimeoutInMs = -1)
     {
         return GetChannel(channel, index).Receive(parts.fParts, rcvTimeoutInMs);
@@ -363,10 +362,10 @@ class FairMQDevice
     void PrintRegisteredChannels()
     {
         if (fChannelRegistry.size() < 1) {
-            std::cout << "no channels registered." << std::endl;
+            LOGV(info, verylow) << "no channels registered.";
         } else {
             for (const auto& c : fChannelRegistry) {
-                std::cout << c.first << ":" << c.second.first << ":" << c.second.second << std::endl;
+                LOGV(info, verylow) << c.first << ":" << c.second.first << ":" << c.second.second;
             }
         }
     }

fairmq/FairMQMessage.h

@@ -11,6 +11,7 @@
 
 #include <cstddef> // for size_t
 #include <memory> // unique_ptr
+#include <stdexcept>
 
 #include <fairmq/Transports.h>
 

fairmq/FairMQSocket.h

@@ -9,13 +9,31 @@
 #ifndef FAIRMQSOCKET_H_
 #define FAIRMQSOCKET_H_
 
+#include "FairMQMessage.h"
+
+#include <memory>
+#include <ostream>
+#include <stdexcept>
 #include <string>
 #include <vector>
-#include <memory>
 
-#include "FairMQMessage.h"
 class FairMQTransportFactory;
 
+namespace fair
+{
+namespace mq
+{
+
+enum class TransferResult : int
+{
+    error = -1,
+    timeout = -2,
+    interrupted = -3
+};
+
+} // namespace mq
+} // namespace fair
+
 class FairMQSocket
 {
   public:
@@ -27,8 +45,8 @@ class FairMQSocket
     virtual bool Bind(const std::string& address) = 0;
     virtual bool Connect(const std::string& address) = 0;
 
-    virtual int Send(FairMQMessagePtr& msg, int timeout = -1) = 0;
-    virtual int Receive(FairMQMessagePtr& msg, int timeout = -1) = 0;
+    virtual int64_t Send(FairMQMessagePtr& msg, int timeout = -1) = 0;
+    virtual int64_t Receive(FairMQMessagePtr& msg, int timeout = -1) = 0;
     virtual int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, int timeout = -1) = 0;
     virtual int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, int timeout = -1) = 0;
 
@@ -37,6 +55,10 @@ class FairMQSocket
     virtual void SetOption(const std::string& option, const void* value, size_t valueSize) = 0;
     virtual void GetOption(const std::string& option, void* value, size_t* valueSize) = 0;
 
+    /// If the backend supports it, fills the unsigned integer @a events with the ZMQ_EVENTS value
+    /// DISCLAIMER: this API is experimental and unsupported and might be dropped / refactored in
+    /// the future.
+    virtual void Events(uint32_t* events) = 0;
     virtual void SetLinger(const int value) = 0;
     virtual int GetLinger() const = 0;
     virtual void SetSndBufSize(const int value) = 0;

fairmq/FairMQTransportFactory.cxx

@@ -15,6 +15,8 @@
 #include <FairMQLogger.h>
 #include <fairmq/Tools.h>
 
+#include <fairlogger/Logger.h>
+
 #include <memory>
 #include <string>
 

fairmq/FairMQTransportFactory.h

@@ -9,7 +9,6 @@
 #ifndef FAIRMQTRANSPORTFACTORY_H_
 #define FAIRMQTRANSPORTFACTORY_H_
 
-#include <FairMQLogger.h>
 #include <FairMQMessage.h>
 #include <FairMQPoller.h>
 #include <FairMQSocket.h>

fairmq/FairMQUnmanagedRegion.h

@@ -10,6 +10,7 @@
 #define FAIRMQUNMANAGEDREGION_H_
 
 #include <cstddef> // size_t
+#include <cstdint> // uint32_t
 #include <memory> // std::unique_ptr
 #include <functional> // std::function
 #include <ostream> // std::ostream
@@ -27,21 +28,24 @@ enum class FairMQRegionEvent : int
 struct FairMQRegionInfo
 {
     FairMQRegionInfo()
-        : id(0)
+        : managed(true)
+        , id(0)
         , ptr(nullptr)
         , size(0)
         , flags(0)
         , event(FairMQRegionEvent::created)
     {}
 
-    FairMQRegionInfo(uint64_t _id, void* _ptr, size_t _size, int64_t _flags, FairMQRegionEvent _event)
-        : id(_id)
+    FairMQRegionInfo(bool _managed, uint64_t _id, void* _ptr, size_t _size, int64_t _flags, FairMQRegionEvent _event)
+        : managed(_managed)
+        , id(_id)
         , ptr(_ptr)
         , size(_size)
         , flags(_flags)
-        , event (_event)
+        , event(_event)
     {}
 
+    bool managed;  // managed/unmanaged
     uint64_t id;   // id of the region
     void* ptr;     // pointer to the start of the region
     size_t size;   // region size
@@ -71,7 +75,9 @@ class FairMQUnmanagedRegion
 
     virtual void* GetData() const = 0;
     virtual size_t GetSize() const = 0;
-    virtual uint64_t GetId() const = 0;
+    virtual uint16_t GetId() const = 0;
+    virtual void SetLinger(uint32_t linger) = 0;
+    virtual uint32_t GetLinger() const = 0;
 
     FairMQTransportFactory* GetTransport() { return fTransport; }
     void SetTransport(FairMQTransportFactory* transport) { fTransport = transport; }

fairmq/JSONParser.cxx

@@ -12,16 +12,19 @@
  * Created on May 14, 2015, 5:01 PM
  */
 
-#include <fairmq/PropertyOutput.h>
 #include "JSONParser.h"
-#include "FairMQLogger.h"
-#include <fairmq/Tools.h>
+#include "FairMQChannel.h"
+#include <fairmq/PropertyOutput.h>
+#include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Strings.h>
+
+#include <fairlogger/Logger.h>
 
 #include <boost/property_tree/json_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <boost/any.hpp>
 
-#include <ios>
+#include <iomanip>
 
 using namespace std;
 using namespace fair::mq;

fairmq/JSONParser.h

@@ -15,15 +15,11 @@
 #ifndef FAIR_MQ_JSONPARSER_H
 #define FAIR_MQ_JSONPARSER_H
 
-#include <string>
-#include <vector>
-#include <unordered_map>
-#include <exception>
-
+#include <fairmq/Properties.h>
 #include <boost/property_tree/ptree_fwd.hpp>
 
-#include "FairMQChannel.h"
-#include <fairmq/Properties.h>
+#include <stdexcept>
+#include <string>
 
 namespace fair
 {

fairmq/MemoryResources.h

@@ -18,11 +18,12 @@
 #include <fairmq/FairMQMessage.h>
 class FairMQTransportFactory;
 
+#include <boost/container/container_fwd.hpp>
 #include <boost/container/flat_map.hpp>
 #include <boost/container/pmr/memory_resource.hpp>
-#include <boost/container/pmr/monotonic_buffer_resource.hpp>
-#include <boost/container/pmr/polymorphic_allocator.hpp>
+
 #include <cstring>
+#include <stdexcept>
 #include <utility>
 
 namespace fair {

fairmq/PluginServices.h

@@ -17,13 +17,14 @@
 #include <boost/optional.hpp>
 #include <boost/optional/optional_io.hpp>
 
+#include <condition_variable>
 #include <functional>
+#include <map>
+#include <mutex>
+#include <stdexcept>
 #include <string>
 #include <unordered_map>
-#include <mutex>
-#include <map>
-#include <condition_variable>
-#include <stdexcept>
+#include <vector>
 
 namespace fair
 {

fairmq/ProgOptions.cxx

@@ -79,7 +79,7 @@ ProgOptions::ProgOptions()
     fAllOptions.add_options()
         ("help,h",                                                      "Print help")
         ("version,v",                                                   "Print version")
-        ("severity",      po::value<string>()->default_value("debug"),  "Log severity level (console): trace, debug, info, state, warn, error, fatal, nolog")
+        ("severity",      po::value<string>()->default_value(""),       "Log severity level (console): trace, debug, info, state, warn, error, fatal, nolog.")
         ("file-severity", po::value<string>()->default_value("debug"),  "Log severity level (file): trace, debug, info, state, warn, error, fatal, nolog")
         ("verbosity",     po::value<string>()->default_value("medium"), "Log verbosity level: veryhigh, high, medium, low")
         ("color",         po::value<bool  >()->default_value(true),     "Log color (true/false)")

fairmq/Properties.cxx

@@ -76,9 +76,11 @@ unordered_map<type_index, function<pair<string, string>(const Property&)>> Prope
     { type_index(typeid(const char*)),                     [](const Property& p) { return pair<string, string>{ string(any_cast<const char*>(p)), "string" }; } },
     { type_index(typeid(string)),                          [](const Property& p) { return pair<string, string>{ any_cast<string>(p), "string" }; } },
     { type_index(typeid(int)),                             [](const Property& p) { return getString<int>(p, "int"); } },
+    { type_index(typeid(short)),                           [](const Property& p) { return getString<short>(p, "short"); } },
     { type_index(typeid(long)),                            [](const Property& p) { return getString<long>(p, "long"); } },
     { type_index(typeid(long long)),                       [](const Property& p) { return getString<long long>(p, "long long"); } },
     { type_index(typeid(unsigned)),                        [](const Property& p) { return getString<unsigned>(p, "unsigned"); } },
+    { type_index(typeid(unsigned short)),                  [](const Property& p) { return getString<unsigned short>(p, "unsigned short"); } },
     { type_index(typeid(unsigned long)),                   [](const Property& p) { return getString<unsigned long>(p, "unsigned long"); } },
     { type_index(typeid(unsigned long long)),              [](const Property& p) { return getString<unsigned long long>(p, "unsigned long long"); } },
     { type_index(typeid(float)),                           [](const Property& p) { return getStringPair<float>(p, "float"); } },
@@ -92,9 +94,11 @@ unordered_map<type_index, function<pair<string, string>(const Property&)>> Prope
     { type_index(typeid(vector<unsigned char>)),           [](const Property& p) { return getStringPair<vector<unsigned char>>(p, "vector<unsigned char>"); } },
     { type_index(typeid(vector<string>)),                  [](const Property& p) { return getStringPair<vector<string>>(p, "vector<string>"); } },
     { type_index(typeid(vector<int>)),                     [](const Property& p) { return getStringPair<vector<int>>(p, "vector<int>"); } },
+    { type_index(typeid(vector<short>)),                   [](const Property& p) { return getStringPair<vector<short>>(p, "vector<short>"); } },
     { type_index(typeid(vector<long>)),                    [](const Property& p) { return getStringPair<vector<long>>(p, "vector<long>"); } },
     { type_index(typeid(vector<long long>)),               [](const Property& p) { return getStringPair<vector<long long>>(p, "vector<long long>"); } },
     { type_index(typeid(vector<unsigned>)),                [](const Property& p) { return getStringPair<vector<unsigned>>(p, "vector<unsigned>"); } },
+    { type_index(typeid(vector<unsigned short>)),          [](const Property& p) { return getStringPair<vector<unsigned short>>(p, "vector<unsigned short>"); } },
     { type_index(typeid(vector<unsigned long>)),           [](const Property& p) { return getStringPair<vector<unsigned long>>(p, "vector<unsigned long>"); } },
     { type_index(typeid(vector<unsigned long long>)),      [](const Property& p) { return getStringPair<vector<unsigned long long>>(p, "vector<unsigned long long>"); } },
     { type_index(typeid(vector<float>)),                   [](const Property& p) { return getStringPair<vector<float>>(p, "vector<float>"); } },
@@ -110,9 +114,11 @@ unordered_map<type_index, void(*)(const EventManager&, const string&, const Prop
     { type_index(typeid(const char*)),                     [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, string>(k, string(any_cast<const char*>(p))); } },
     { type_index(typeid(string)),                          [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, string>(k, any_cast<string>(p)); } },
     { type_index(typeid(int)),                             [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, int>(k, any_cast<int>(p)); } },
+    { type_index(typeid(short)),                           [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, short>(k, any_cast<short>(p)); } },
     { type_index(typeid(long)),                            [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, long>(k, any_cast<long>(p)); } },
     { type_index(typeid(long long)),                       [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, long long>(k, any_cast<long long>(p)); } },
     { type_index(typeid(unsigned)),                        [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, unsigned>(k, any_cast<unsigned>(p)); } },
+    { type_index(typeid(unsigned short)),                  [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, unsigned short>(k, any_cast<unsigned short>(p)); } },
     { type_index(typeid(unsigned long)),                   [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, unsigned long>(k, any_cast<unsigned long>(p)); } },
     { type_index(typeid(unsigned long long)),              [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, unsigned long long>(k, any_cast<unsigned long long>(p)); } },
     { type_index(typeid(float)),                           [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, float>(k, any_cast<float>(p)); } },
@@ -126,9 +132,11 @@ unordered_map<type_index, void(*)(const EventManager&, const string&, const Prop
     { type_index(typeid(vector<unsigned char>)),           [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<unsigned char>>(k, any_cast<vector<unsigned char>>(p)); } },
     { type_index(typeid(vector<string>)),                  [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<string>>(k, any_cast<vector<string>>(p)); } },
     { type_index(typeid(vector<int>)),                     [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<int>>(k, any_cast<vector<int>>(p)); } },
+    { type_index(typeid(vector<short>)),                   [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<short>>(k, any_cast<vector<short>>(p)); } },
     { type_index(typeid(vector<long>)),                    [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<long>>(k, any_cast<vector<long>>(p)); } },
     { type_index(typeid(vector<long long>)),               [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<long long>>(k, any_cast<vector<long long>>(p)); } },
     { type_index(typeid(vector<unsigned>)),                [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<unsigned>>(k, any_cast<vector<unsigned>>(p)); } },
+    { type_index(typeid(vector<unsigned short>)),          [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<unsigned short>>(k, any_cast<vector<unsigned short>>(p)); } },
     { type_index(typeid(vector<unsigned long>)),           [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<unsigned long>>(k, any_cast<vector<unsigned long>>(p)); } },
     { type_index(typeid(vector<unsigned long long>)),      [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<unsigned long long>>(k, any_cast<vector<unsigned long long>>(p)); } },
     { type_index(typeid(vector<float>)),                   [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<float>>(k, any_cast<vector<float>>(p)); } },

fairmq/Properties.h

@@ -16,6 +16,7 @@
 #include <functional>
 #include <map>
 #include <unordered_map>
+#include <stdexcept>
 #include <string>
 #include <typeindex>
 #include <typeinfo>

fairmq/StateMachine.cxx

@@ -7,7 +7,8 @@
  ********************************************************************************/
 
 #include <fairmq/StateMachine.h>
-#include <fairmq/Tools.h>
+
+#include <fairlogger/Logger.h>
 
 // Increase maximum number of boost::msm states (default is 10)
 // This #define has to be before any msm header includes

fairmq/StateMachine.h

@@ -11,8 +11,6 @@
 
 #include <fairmq/States.h>
 
-#include <fairlogger/Logger.h>
-
 #include <string>
 #include <memory>
 #include <functional>

fairmq/States.cxx

@@ -18,9 +18,10 @@ namespace fair
 namespace mq
 {
 
-array<string, 15> stateNames =
+array<string, 16> stateNames =
 {
     {
+        "UNDEFINED",
         "OK",
         "ERROR",
         "IDLE",
@@ -41,6 +42,7 @@ array<string, 15> stateNames =
 
 unordered_map<string, State> states =
 {
+    { "UNDEFINED",           State::Undefined },
     { "OK",                  State::Ok },
     { "ERROR",               State::Error },
     { "IDLE",                State::Idle },

fairmq/States.h

@@ -20,6 +20,7 @@ namespace mq
 
 enum class State : int
 {
+    Undefined = 0,
     Ok,
     Error,
     Idle,
@@ -39,7 +40,7 @@ enum class State : int
 
 enum class Transition : int
 {
-    Auto,
+    Auto = 0,
     InitDevice,
     CompleteInit,
     Bind,

fairmq/SuboptParser.cxx

@@ -12,9 +12,12 @@
 /// @brief  Parser implementation for key-value subopt format
 
 #include <fairmq/SuboptParser.h>
+#include <fairmq/JSONParser.h>
+
+#include <fairlogger/Logger.h>
 
 #include <boost/property_tree/ptree.hpp>
-#include <cstring>
+
 #include <utility> // make_pair
 
 using boost::property_tree::ptree;

fairmq/SuboptParser.h

@@ -14,7 +14,7 @@
 #ifndef FAIR_MQ_SUBOPTPARSER_H
 #define FAIR_MQ_SUBOPTPARSER_H
 
-#include <fairmq/JSONParser.h>
+#include <fairmq/Properties.h>
 
 #include <vector>
 #include <string>

fairmq/Transports.h

@@ -10,8 +10,10 @@
 #define FAIR_MQ_TRANSPORTS_H
 
 #include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Strings.h>
 
 #include <memory>
+#include <stdexcept>
 #include <string>
 #include <unordered_map>
 
@@ -28,6 +30,8 @@ enum class Transport
     OFI
 };
 
+struct TransportError : std::runtime_error { using std::runtime_error::runtime_error; };
+
 } /* namespace mq */
 } /* namespace fair */
 
@@ -58,6 +62,18 @@ static std::unordered_map<Transport, std::string> TransportNames {
     { Transport::OFI, "ofi" }
 };
 
+inline std::string TransportName(Transport transport)
+{
+    return TransportNames[transport];
+}
+
+inline Transport TransportType(const std::string& transport)
+try {
+    return TransportTypes.at(transport);
+} catch (std::out_of_range&) {
+    throw TransportError(tools::ToString("Unknown transport provided: ", transport));
+}
+
 } /* namespace mq */
 } /* namespace fair */
 

fairmq/devices/FairMQBenchmarkSampler.h

@@ -17,6 +17,7 @@
 #include <chrono>
 #include <cstddef>   // size_t
 #include <cstdint>   // uint64_t
+#include <cstring>   // memset
 #include <string>
 
 /**
@@ -28,8 +29,10 @@ class FairMQBenchmarkSampler : public FairMQDevice
   public:
     FairMQBenchmarkSampler()
         : fMultipart(false)
+        , fMemSet(false)
         , fNumParts(1)
         , fMsgSize(10000)
+        , fMsgAlignment(0)
         , fMsgRate(0)
         , fNumIterations(0)
         , fMaxIterations(0)
@@ -39,8 +42,10 @@ class FairMQBenchmarkSampler : public FairMQDevice
     void InitTask() override
     {
         fMultipart = fConfig->GetProperty<bool>("multipart");
+        fMemSet = fConfig->GetProperty<bool>("memset");
         fNumParts = fConfig->GetProperty<size_t>("num-parts");
         fMsgSize = fConfig->GetProperty<size_t>("msg-size");
+        fMsgAlignment = fConfig->GetProperty<size_t>("msg-alignment");
         fMsgRate = fConfig->GetProperty<float>("msg-rate");
         fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
         fOutChannelName = fConfig->GetProperty<std::string>("out-channel");
@@ -51,8 +56,6 @@ class FairMQBenchmarkSampler : public FairMQDevice
         // store the channel reference to avoid traversing the map on every loop iteration
         FairMQChannel& dataOutChannel = fChannels.at(fOutChannelName).at(0);
 
-        FairMQMessagePtr baseMsg(dataOutChannel.NewMessage(fMsgSize));
-
         LOG(info) << "Starting the benchmark with message size of " << fMsgSize << " and " << fMaxIterations << " iterations.";
         auto tStart = std::chrono::high_resolution_clock::now();
 
@@ -63,7 +66,10 @@ class FairMQBenchmarkSampler : public FairMQDevice
                 FairMQParts parts;
 
                 for (size_t i = 0; i < fNumParts; ++i) {
-                    parts.AddPart(dataOutChannel.NewMessage(fMsgSize));
+                    parts.AddPart(dataOutChannel.NewMessage(fMsgSize, fair::mq::Alignment{fMsgAlignment}));
+                    if (fMemSet) {
+                        std::memset(parts.At(i)->GetData(), 0, parts.At(i)->GetSize());
+                    }
                 }
 
                 if (dataOutChannel.Send(parts) >= 0) {
@@ -75,7 +81,10 @@ class FairMQBenchmarkSampler : public FairMQDevice
                     ++fNumIterations;
                 }
             } else {
-                FairMQMessagePtr msg(dataOutChannel.NewMessage(fMsgSize));
+                FairMQMessagePtr msg(dataOutChannel.NewMessage(fMsgSize, fair::mq::Alignment{fMsgAlignment}));
+                if (fMemSet) {
+                    std::memset(msg->GetData(), 0, msg->GetSize());
+                }
 
                 if (dataOutChannel.Send(msg) >= 0) {
                     if (fMaxIterations > 0) {
@@ -101,8 +110,10 @@ class FairMQBenchmarkSampler : public FairMQDevice
 
   protected:
     bool fMultipart;
+    bool fMemSet;
     size_t fNumParts;
     size_t fMsgSize;
+    size_t fMsgAlignment;
     std::atomic<int> fMsgCounter;
     float fMsgRate;
     uint64_t fNumIterations;

fairmq/devices/FairMQMerger.cxx

@@ -1,122 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQMerger.cxx
- *
- * @since 2012-12-06
- * @author D. Klein, A. Rybalchenko
- */
-
-#include "FairMQMerger.h"
-#include "../FairMQLogger.h"
-#include "../FairMQPoller.h"
-
-using namespace std;
-
-FairMQMerger::FairMQMerger()
-    : fMultipart(true)
-    , fInChannelName("data-in")
-    , fOutChannelName("data-out")
-{
-}
-
-void FairMQMerger::RegisterChannelEndpoints()
-{
-    RegisterChannelEndpoint(fInChannelName, 1, 10000);
-    RegisterChannelEndpoint(fOutChannelName, 1, 1);
-
-    PrintRegisteredChannels();
-}
-
-FairMQMerger::~FairMQMerger()
-{
-}
-
-void FairMQMerger::InitTask()
-{
-    fMultipart = fConfig->GetProperty<bool>("multipart");
-    fInChannelName = fConfig->GetProperty<string>("in-channel");
-    fOutChannelName = fConfig->GetProperty<string>("out-channel");
-}
-
-void FairMQMerger::Run()
-{
-    int numInputs = fChannels.at(fInChannelName).size();
-
-    vector<FairMQChannel*> chans;
-
-    for (auto& chan : fChannels.at(fInChannelName))
-    {
-        chans.push_back(&chan);
-    }
-
-    FairMQPollerPtr poller(NewPoller(chans));
-
-    if (fMultipart)
-    {
-        while (!NewStatePending())
-        {
-            poller->Poll(100);
-
-            // Loop over the data input channels.
-            for (int i = 0; i < numInputs; ++i)
-            {
-                // Check if the channel has data ready to be received.
-                if (poller->CheckInput(i))
-                {
-                    FairMQParts payload;
-
-                    if (Receive(payload, fInChannelName, i) >= 0)
-                    {
-                        if (Send(payload, fOutChannelName) < 0)
-                        {
-                            LOG(debug) << "Transfer interrupted";
-                            break;
-                        }
-                    }
-                    else
-                    {
-                        LOG(debug) << "Transfer interrupted";
-                        break;
-                    }
-                }
-            }
-        }
-    }
-    else
-    {
-        while (!NewStatePending())
-        {
-            poller->Poll(100);
-
-            // Loop over the data input channels.
-            for (int i = 0; i < numInputs; ++i)
-            {
-                // Check if the channel has data ready to be received.
-                if (poller->CheckInput(i))
-                {
-                    FairMQMessagePtr payload(fTransportFactory->CreateMessage());
-
-                    if (Receive(payload, fInChannelName, i) >= 0)
-                    {
-                        if (Send(payload, fOutChannelName) < 0)
-                        {
-                            LOG(debug) << "Transfer interrupted";
-                            break;
-                        }
-                    }
-                    else
-                    {
-                        LOG(debug) << "Transfer interrupted";
-                        break;
-                    }
-                }
-            }
-        }
-    }
-}

fairmq/devices/FairMQMerger.h

@@ -16,23 +16,100 @@
 #define FAIRMQMERGER_H_
 
 #include "FairMQDevice.h"
+#include "../FairMQPoller.h"
+#include "../FairMQLogger.h"
 
 #include <string>
+#include <vector>
 
 class FairMQMerger : public FairMQDevice
 {
   public:
-    FairMQMerger();
-    virtual ~FairMQMerger();
+    FairMQMerger()
+        : fMultipart(true)
+        , fInChannelName("data-in")
+        , fOutChannelName("data-out")
+    {}
+    ~FairMQMerger() {}
 
   protected:
     bool fMultipart;
     std::string fInChannelName;
     std::string fOutChannelName;
 
-    virtual void RegisterChannelEndpoints() override;
-    virtual void Run() override;
-    virtual void InitTask() override;
+    void InitTask() override
+    {
+        fMultipart = fConfig->GetProperty<bool>("multipart");
+        fInChannelName = fConfig->GetProperty<std::string>("in-channel");
+        fOutChannelName = fConfig->GetProperty<std::string>("out-channel");
+    }
+
+    void RegisterChannelEndpoints() override
+    {
+        RegisterChannelEndpoint(fInChannelName, 1, 10000);
+        RegisterChannelEndpoint(fOutChannelName, 1, 1);
+
+        PrintRegisteredChannels();
+    }
+
+    void Run() override
+    {
+        int numInputs = fChannels.at(fInChannelName).size();
+
+        std::vector<FairMQChannel*> chans;
+
+        for (auto& chan : fChannels.at(fInChannelName)) {
+            chans.push_back(&chan);
+        }
+
+        FairMQPollerPtr poller(NewPoller(chans));
+
+        if (fMultipart) {
+            while (!NewStatePending()) {
+                poller->Poll(100);
+
+                // Loop over the data input channels.
+                for (int i = 0; i < numInputs; ++i) {
+                    // Check if the channel has data ready to be received.
+                    if (poller->CheckInput(i)) {
+                        FairMQParts payload;
+
+                        if (Receive(payload, fInChannelName, i) >= 0) {
+                            if (Send(payload, fOutChannelName) < 0) {
+                                LOG(debug) << "Transfer interrupted";
+                                break;
+                            }
+                        } else {
+                            LOG(debug) << "Transfer interrupted";
+                            break;
+                        }
+                    }
+                }
+            }
+        } else {
+            while (!NewStatePending()) {
+                poller->Poll(100);
+
+                // Loop over the data input channels.
+                for (int i = 0; i < numInputs; ++i) {
+                    // Check if the channel has data ready to be received.
+                    if (poller->CheckInput(i)) {
+                        FairMQMessagePtr payload(fTransportFactory->CreateMessage());
+
+                        if (Receive(payload, fInChannelName, i) >= 0) {
+                            if (Send(payload, fOutChannelName) < 0) {
+                                LOG(debug) << "Transfer interrupted";
+                                break;
+                            }
+                        } else {
+                            LOG(debug) << "Transfer interrupted";
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+    }
 };
 
 #endif /* FAIRMQMERGER_H_ */

fairmq/devices/FairMQMultiplier.cxx

@@ -1,110 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "FairMQMultiplier.h"
-
-#include "../FairMQLogger.h"
-
-using namespace std;
-
-FairMQMultiplier::FairMQMultiplier()
-    : fMultipart(true)
-    , fNumOutputs(0)
-    , fInChannelName()
-    , fOutChannelNames()
-{
-}
-
-FairMQMultiplier::~FairMQMultiplier()
-{
-}
-
-void FairMQMultiplier::InitTask()
-{
-    fMultipart = fConfig->GetProperty<bool>("multipart");
-    fInChannelName = fConfig->GetProperty<string>("in-channel");
-    fOutChannelNames = fConfig->GetProperty<vector<string>>("out-channel");
-    fNumOutputs = fChannels.at(fOutChannelNames.at(0)).size();
-
-    if (fMultipart)
-    {
-        OnData(fInChannelName, &FairMQMultiplier::HandleMultipartData);
-    }
-    else
-    {
-        OnData(fInChannelName, &FairMQMultiplier::HandleSingleData);
-    }
-}
-
-bool FairMQMultiplier::HandleSingleData(std::unique_ptr<FairMQMessage>& payload, int /*index*/)
-{
-    for (unsigned int i = 0; i < fOutChannelNames.size() - 1; ++i) // all except last channel
-    {
-        for (unsigned int j = 0; j < fChannels.at(fOutChannelNames.at(i)).size(); ++j) // all subChannels in a channel
-        {
-            FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
-            msgCopy->Copy(*payload);
-
-            Send(msgCopy, fOutChannelNames.at(i), j);
-        }
-    }
-
-    unsigned int lastChannelSize = fChannels.at(fOutChannelNames.back()).size();
-
-    for (unsigned int i = 0; i < lastChannelSize - 1; ++i) // iterate over all except last subChannels of the last channel
-    {
-        FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
-        msgCopy->Copy(*payload);
-
-        Send(msgCopy, fOutChannelNames.back(), i);
-    }
-
-    Send(payload, fOutChannelNames.back(), lastChannelSize - 1); // send final message to last subChannel of last channel
-
-    return true;
-}
-
-bool FairMQMultiplier::HandleMultipartData(FairMQParts& payload, int /*index*/)
-{
-    for (unsigned int i = 0; i < fOutChannelNames.size() - 1; ++i) // all except last channel
-    {
-        for (unsigned int j = 0; j < fChannels.at(fOutChannelNames.at(i)).size(); ++j) // all subChannels in a channel
-        {
-            FairMQParts parts;
-
-            for (int k = 0; k < payload.Size(); ++k)
-            {
-                FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
-                msgCopy->Copy(payload.AtRef(k));
-                parts.AddPart(std::move(msgCopy));
-            }
-
-            Send(parts, fOutChannelNames.at(i), j);
-        }
-    }
-
-    unsigned int lastChannelSize = fChannels.at(fOutChannelNames.back()).size();
-
-    for (unsigned int i = 0; i < lastChannelSize - 1; ++i) // iterate over all except last subChannels of the last channel
-    {
-        FairMQParts parts;
-
-        for (int k = 0; k < payload.Size(); ++k)
-        {
-            FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
-            msgCopy->Copy(payload.AtRef(k));
-            parts.AddPart(std::move(msgCopy));
-        }
-
-        Send(parts, fOutChannelNames.back(), i);
-    }
-
-    Send(payload, fOutChannelNames.back(), lastChannelSize - 1); // send final message to last subChannel of last channel
-
-    return true;
-}

fairmq/devices/FairMQMultiplier.h

@@ -12,12 +12,18 @@
 #include "FairMQDevice.h"
 
 #include <string>
+#include <vector>
 
 class FairMQMultiplier : public FairMQDevice
 {
   public:
-    FairMQMultiplier();
-    virtual ~FairMQMultiplier();
+    FairMQMultiplier()
+        : fMultipart(true)
+        , fNumOutputs(0)
+        , fInChannelName()
+        , fOutChannelNames()
+    {}
+    ~FairMQMultiplier() {}
 
   protected:
     bool fMultipart;
@@ -25,10 +31,80 @@ class FairMQMultiplier : public FairMQDevice
     std::string fInChannelName;
     std::vector<std::string> fOutChannelNames;
 
-    virtual void InitTask();
+    void InitTask() override
+    {
+        fMultipart = fConfig->GetProperty<bool>("multipart");
+        fInChannelName = fConfig->GetProperty<std::string>("in-channel");
+        fOutChannelNames = fConfig->GetProperty<std::vector<std::string>>("out-channel");
+        fNumOutputs = fChannels.at(fOutChannelNames.at(0)).size();
 
-    bool HandleSingleData(std::unique_ptr<FairMQMessage>&, int);
-    bool HandleMultipartData(FairMQParts&, int);
+        if (fMultipart) {
+            OnData(fInChannelName, &FairMQMultiplier::HandleMultipartData);
+        } else {
+            OnData(fInChannelName, &FairMQMultiplier::HandleSingleData);
+        }
+    }
+
+
+    bool HandleSingleData(std::unique_ptr<FairMQMessage>& payload, int)
+    {
+        for (unsigned int i = 0; i < fOutChannelNames.size() - 1; ++i) { // all except last channel
+            for (unsigned int j = 0; j < fChannels.at(fOutChannelNames.at(i)).size(); ++j) { // all subChannels in a channel
+                FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
+                msgCopy->Copy(*payload);
+
+                Send(msgCopy, fOutChannelNames.at(i), j);
+            }
+        }
+
+        unsigned int lastChannelSize = fChannels.at(fOutChannelNames.back()).size();
+
+        for (unsigned int i = 0; i < lastChannelSize - 1; ++i) { // iterate over all except last subChannels of the last channel
+            FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
+            msgCopy->Copy(*payload);
+
+            Send(msgCopy, fOutChannelNames.back(), i);
+        }
+
+        Send(payload, fOutChannelNames.back(), lastChannelSize - 1); // send final message to last subChannel of last channel
+
+        return true;
+    }
+
+    bool HandleMultipartData(FairMQParts& payload, int)
+    {
+        for (unsigned int i = 0; i < fOutChannelNames.size() - 1; ++i) { // all except last channel
+            for (unsigned int j = 0; j < fChannels.at(fOutChannelNames.at(i)).size(); ++j) { // all subChannels in a channel
+                FairMQParts parts;
+
+                for (int k = 0; k < payload.Size(); ++k) {
+                    FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
+                    msgCopy->Copy(payload.AtRef(k));
+                    parts.AddPart(std::move(msgCopy));
+                }
+
+                Send(parts, fOutChannelNames.at(i), j);
+            }
+        }
+
+        unsigned int lastChannelSize = fChannels.at(fOutChannelNames.back()).size();
+
+        for (unsigned int i = 0; i < lastChannelSize - 1; ++i) { // iterate over all except last subChannels of the last channel
+            FairMQParts parts;
+
+            for (int k = 0; k < payload.Size(); ++k) {
+                FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
+                msgCopy->Copy(payload.AtRef(k));
+                parts.AddPart(std::move(msgCopy));
+            }
+
+            Send(parts, fOutChannelNames.back(), i);
+        }
+
+        Send(payload, fOutChannelNames.back(), lastChannelSize - 1); // send final message to last subChannel of last channel
+
+        return true;
+    }
 };
 
 #endif /* FAIRMQMULTIPLIER_H_ */

fairmq/devices/FairMQProxy.cxx

@@ -1,81 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQProxy.cxx
- *
- * @since 2013-10-02
- * @author A. Rybalchenko
- */
-
-#include "FairMQProxy.h"
-
-#include "../FairMQLogger.h"
-
-using namespace std;
-
-FairMQProxy::FairMQProxy()
-    : fMultipart(true)
-    , fInChannelName()
-    , fOutChannelName()
-{
-}
-
-FairMQProxy::~FairMQProxy()
-{
-}
-
-void FairMQProxy::InitTask()
-{
-    fMultipart = fConfig->GetProperty<bool>("multipart");
-    fInChannelName = fConfig->GetProperty<string>("in-channel");
-    fOutChannelName = fConfig->GetProperty<string>("out-channel");
-}
-
-void FairMQProxy::Run()
-{
-    if (fMultipart)
-    {
-        while (!NewStatePending())
-        {
-            FairMQParts payload;
-            if (Receive(payload, fInChannelName) >= 0)
-            {
-                if (Send(payload, fOutChannelName) < 0)
-                {
-                    LOG(debug) << "Transfer interrupted";
-                    break;
-                }
-            }
-            else
-            {
-                LOG(debug) << "Transfer interrupted";
-                break;
-            }
-        }
-    }
-    else
-    {
-        while (!NewStatePending())
-        {
-            unique_ptr<FairMQMessage> payload(fTransportFactory->CreateMessage());
-            if (Receive(payload, fInChannelName) >= 0)
-            {
-                if (Send(payload, fOutChannelName) < 0)
-                {
-                    LOG(debug) << "Transfer interrupted";
-                    break;
-                }
-            }
-            else
-            {
-                LOG(debug) << "Transfer interrupted";
-                break;
-            }
-        }
-    }
-}

fairmq/devices/FairMQProxy.h

@@ -22,16 +22,55 @@
 class FairMQProxy : public FairMQDevice
 {
   public:
-    FairMQProxy();
-    virtual ~FairMQProxy();
+    FairMQProxy()
+        : fMultipart(true)
+        , fInChannelName()
+        , fOutChannelName()
+    {}
+    ~FairMQProxy() {}
 
   protected:
     bool fMultipart;
     std::string fInChannelName;
     std::string fOutChannelName;
 
-    virtual void Run();
-    virtual void InitTask();
+    void InitTask() override
+    {
+        fMultipart = fConfig->GetProperty<bool>("multipart");
+        fInChannelName = fConfig->GetProperty<std::string>("in-channel");
+        fOutChannelName = fConfig->GetProperty<std::string>("out-channel");
+    }
+
+    void Run() override
+    {
+        if (fMultipart) {
+            while (!NewStatePending()) {
+                FairMQParts payload;
+                if (Receive(payload, fInChannelName) >= 0) {
+                    if (Send(payload, fOutChannelName) < 0) {
+                        LOG(debug) << "Transfer interrupted";
+                        break;
+                    }
+                } else {
+                    LOG(debug) << "Transfer interrupted";
+                    break;
+                }
+            }
+        } else {
+            while (!NewStatePending()) {
+                FairMQMessagePtr payload(fTransportFactory->CreateMessage());
+                if (Receive(payload, fInChannelName) >= 0) {
+                    if (Send(payload, fOutChannelName) < 0) {
+                        LOG(debug) << "Transfer interrupted";
+                        break;
+                    }
+                } else {
+                    LOG(debug) << "Transfer interrupted";
+                    break;
+                }
+            }
+        }
+    }
 };
 
 #endif /* FAIRMQPROXY_H_ */

fairmq/devices/FairMQSink.h

@@ -32,7 +32,7 @@ class FairMQSink : public FairMQDevice   //, public OutputPolicy
         , fInChannelName()
     {}
 
-    virtual ~FairMQSink() {}
+    ~FairMQSink() {}
 
   protected:
     bool fMultipart;
@@ -40,14 +40,14 @@ class FairMQSink : public FairMQDevice   //, public OutputPolicy
     uint64_t fNumIterations;
     std::string fInChannelName;
 
-    virtual void InitTask()
+    void InitTask() override
     {
         fMultipart = fConfig->GetProperty<bool>("multipart");
         fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
         fInChannelName = fConfig->GetProperty<std::string>("in-channel");
     }
 
-    virtual void Run()
+    void Run() override
     {
         // store the channel reference to avoid traversing the map on every loop iteration
         FairMQChannel& dataInChannel = fChannels.at(fInChannelName).at(0);

fairmq/devices/FairMQSplitter.cxx

@@ -1,74 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQSplitter.cxx
- *
- * @since 2012-12-06
- * @author D. Klein, A. Rybalchenko
- */
-
-#include "FairMQSplitter.h"
-
-#include "../FairMQLogger.h"
-
-using namespace std;
-
-FairMQSplitter::FairMQSplitter()
-    : fMultipart(true)
-    , fNumOutputs(0)
-    , fDirection(0)
-    , fInChannelName()
-    , fOutChannelName()
-{
-}
-
-FairMQSplitter::~FairMQSplitter()
-{
-}
-
-void FairMQSplitter::InitTask()
-{
-    fMultipart = fConfig->GetProperty<bool>("multipart");
-    fInChannelName = fConfig->GetProperty<string>("in-channel");
-    fOutChannelName = fConfig->GetProperty<string>("out-channel");
-    fNumOutputs = fChannels.at(fOutChannelName).size();
-    fDirection = 0;
-
-    if (fMultipart)
-    {
-        OnData(fInChannelName, &FairMQSplitter::HandleMultipartData);
-    }
-    else
-    {
-        OnData(fInChannelName, &FairMQSplitter::HandleSingleData);
-    }
-}
-
-bool FairMQSplitter::HandleSingleData(FairMQMessagePtr& payload, int /*index*/)
-{
-    Send(payload, fOutChannelName, fDirection);
-
-    if (++fDirection >= fNumOutputs)
-    {
-        fDirection = 0;
-    }
-
-    return true;
-}
-
-bool FairMQSplitter::HandleMultipartData(FairMQParts& payload, int /*index*/)
-{
-    Send(payload, fOutChannelName, fDirection);
-
-    if (++fDirection >= fNumOutputs)
-    {
-        fDirection = 0;
-    }
-
-    return true;
-}

fairmq/devices/FairMQSplitter.h

@@ -22,8 +22,14 @@
 class FairMQSplitter : public FairMQDevice
 {
   public:
-    FairMQSplitter();
-    virtual ~FairMQSplitter();
+    FairMQSplitter()
+        : fMultipart(true)
+        , fNumOutputs(0)
+        , fDirection(0)
+        , fInChannelName()
+        , fOutChannelName()
+    {}
+    ~FairMQSplitter() {}
 
   protected:
     bool fMultipart;
@@ -32,10 +38,32 @@ class FairMQSplitter : public FairMQDevice
     std::string fInChannelName;
     std::string fOutChannelName;
 
-    virtual void InitTask();
+    void InitTask() override
+    {
+        fMultipart = fConfig->GetProperty<bool>("multipart");
+        fInChannelName = fConfig->GetProperty<std::string>("in-channel");
+        fOutChannelName = fConfig->GetProperty<std::string>("out-channel");
+        fNumOutputs = fChannels.at(fOutChannelName).size();
+        fDirection = 0;
 
-    bool HandleSingleData(std::unique_ptr<FairMQMessage>&, int);
-    bool HandleMultipartData(FairMQParts&, int);
+        if (fMultipart) {
+            OnData(fInChannelName, &FairMQSplitter::HandleData<FairMQParts>);
+        } else {
+            OnData(fInChannelName, &FairMQSplitter::HandleData<FairMQMessagePtr>);
+        }
+    }
+
+    template<typename T>
+    bool HandleData(T& payload, int)
+    {
+        Send(payload, fOutChannelName, fDirection);
+
+        if (++fDirection >= fNumOutputs) {
+            fDirection = 0;
+        }
+
+        return true;
+    }
 };
 
 #endif /* FAIRMQSPLITTER_H_ */

fairmq/ofi/Socket.cxx

@@ -254,7 +254,7 @@ auto Socket::ConnectEndpoint(std::unique_ptr<asiofi::connected_endpoint>& endpoi
     }
 }
 
-auto Socket::Send(MessagePtr& msg, const int /*timeout*/) -> int
+auto Socket::Send(MessagePtr& msg, const int /*timeout*/) -> int64_t
 {
     // timeout argument not yet implemented
 
@@ -272,7 +272,7 @@ try {
     int size(0);
     for (auto& msg : msgVec) {
         size += msg->GetSize();
-    } 
+    }
 
     fSendPushSem.wait();
     {
@@ -284,7 +284,7 @@ try {
     return size;
 } catch (const std::exception& e) {
     LOG(error) << e.what();
-    return -1;
+    return static_cast<int64_t>(TransferResult::error);
 }
 
 auto Socket::SendQueueReader() -> void
@@ -412,7 +412,7 @@ auto Socket::SendQueueReaderStatic() -> void
     });
 }
 
-auto Socket::Receive(MessagePtr& msg, const int /*timeout*/) -> int
+auto Socket::Receive(MessagePtr& msg, const int /*timeout*/) -> int64_t
 try {
     // timeout argument not yet implemented
 
@@ -431,7 +431,7 @@ try {
     return size;
 } catch (const std::exception& e) {
     LOG(error) << e.what();
-    return -1;
+    return static_cast<int>(TransferResult::error);
 }
 
 auto Socket::Receive(std::vector<MessagePtr>& msgVec, const int /*timeout*/) -> int64_t
@@ -449,14 +449,14 @@ try {
     int64_t size(0);
     for (auto& msg : msgVec) {
         size += msg->GetSize();
-    } 
+    }
     fBytesRx += size;
     ++fMessagesRx;
 
-    return size; 
+    return size;
 } catch (const std::exception& e) {
     LOG(error) << e.what();
-    return -1;
+    return static_cast<int64_t>(TransferResult::error);
 }
 
 auto Socket::RecvControlQueueReader() -> void

fairmq/ofi/Socket.h

@@ -45,11 +45,12 @@ class Socket final : public fair::mq::Socket
 
     auto GetId() const -> std::string override { return fId; }
 
+    auto Events(uint32_t *events) -> void override { *events = 0; }
     auto Bind(const std::string& address) -> bool override;
     auto Connect(const std::string& address) -> bool override;
 
-    auto Send(MessagePtr& msg, int timeout = 0) -> int override;
-    auto Receive(MessagePtr& msg, int timeout = 0) -> int override;
+    auto Send(MessagePtr& msg, int timeout = 0) -> int64_t override;
+    auto Receive(MessagePtr& msg, int timeout = 0) -> int64_t override;
     auto Send(std::vector<MessagePtr>& msgVec, int timeout = 0) -> int64_t override;
     auto Receive(std::vector<MessagePtr>& msgVec, int timeout = 0) -> int64_t override;
 

fairmq/plugins/Control.cxx

@@ -8,14 +8,16 @@
 
 #include "Control.h"
 
-#include <termios.h> // for the interactive mode
-#include <poll.h> // for the interactive mode
+#include <atomic>
+#include <chrono>
 #include <csignal> // catching system signals
 #include <cstdlib>
 #include <functional>
-#include <atomic>
+#include <iostream>
 #include <thread>
-#include <chrono>
+
+#include <poll.h> // for the interactive mode
+#include <termios.h> // for the interactive mode
 
 using namespace std;
 

fairmq/plugins/DDS/CMakeLists.txt

@@ -17,13 +17,7 @@ set_target_properties(${plugin} PROPERTIES
   LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/fairmq
 )
 
-set(exe fairmq-dds-command-ui)
-add_executable(${exe} ${CMAKE_CURRENT_SOURCE_DIR}/runDDSCommandUI.cxx)
-target_link_libraries(${exe} FairMQ Commands SDK StateMachine)
-target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
-
-install(TARGETS ${plugin} ${exe}
+install(TARGETS ${plugin}
   EXPORT ${PROJECT_EXPORT_SET}
   LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
 )

fairmq/plugins/DDS/DDS.cxx

@@ -344,10 +344,10 @@ auto DDS::HandleCmd(const string& id, sdk::cmd::Cmd& cmd, const string& cond, ui
         case Type::change_state: {
             Transition transition = static_cast<ChangeState&>(cmd).GetTransition();
             if (ChangeDeviceState(transition)) {
-                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Ok, transition));
+                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Ok, transition, GetCurrentDeviceState()));
                 fDDS.Send(outCmds.Serialize(), to_string(senderId));
             } else {
-                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Failure, transition));
+                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Failure, transition, GetCurrentDeviceState()));
                 fDDS.Send(outCmds.Serialize(), to_string(senderId));
             }
             {
@@ -357,7 +357,7 @@ auto DDS::HandleCmd(const string& id, sdk::cmd::Cmd& cmd, const string& cond, ui
         } break;
         case Type::dump_config: {
             stringstream ss;
-            for (const auto pKey : GetPropertyKeys()) {
+            for (const auto& pKey : GetPropertyKeys()) {
                 ss << id << ": " << pKey << " -> " << GetPropertyAsString(pKey) << "\n";
             }
             Cmds outCmds(make<Config>(id, ss.str()));

fairmq/plugins/PMIx/PMIxPlugin.cxx

@@ -148,12 +148,12 @@ auto PMIxPlugin::SubscribeForCommands() -> void
                     Transition transition = static_cast<ChangeState&>(*cmd).GetTransition();
                     if (ChangeDeviceState(transition)) {
                         fCommands.Send(
-                            Cmds(make<TransitionStatus>(fDeviceId, 0, Result::Ok, transition))
+                            Cmds(make<TransitionStatus>(fDeviceId, 0, Result::Ok, transition, GetCurrentDeviceState()))
                                 .Serialize(Format::JSON),
                             {sender});
                     } else {
                         fCommands.Send(
-                            Cmds(make<TransitionStatus>(fDeviceId, 0, Result::Failure, transition))
+                            Cmds(make<TransitionStatus>(fDeviceId, 0, Result::Failure, transition, GetCurrentDeviceState()))
                                 .Serialize(Format::JSON),
                             {sender});
                     }

fairmq/plugins/config/Config.cxx

@@ -61,22 +61,26 @@ Plugin::ProgOptions ConfigPluginProgramOptions()
     namespace po = boost::program_options;
     auto pluginOptions = po::options_description{"FairMQ device options"};
     pluginOptions.add_options()
-        ("id",                  po::value<string        >()->default_value(""),         "Device ID.")
-        ("io-threads",          po::value<int           >()->default_value(1),          "Number of I/O threads.")
-        ("transport",           po::value<string        >()->default_value("zeromq"),   "Transport ('zeromq'/'shmem').")
-        ("network-interface",   po::value<string        >()->default_value("default"),  "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).")
-        ("init-timeout",        po::value<int           >()->default_value(120),        "Timeout for the initialization in seconds (when expecting dynamic initialization).")
-        ("max-run-time",        po::value<uint64_t      >()->default_value(0),          "Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).")
-        ("print-channels",      po::value<bool          >()->implicit_value(true),      "Print registered channel endpoints in a machine-readable format (<channel name>:<min num subchannels>:<max num subchannels>)")
-        ("shm-segment-size",    po::value<size_t        >()->default_value(2ULL << 30), "Shared memory: size of the shared memory segment (in bytes).")
-        ("shm-throw-bad-alloc", po::value<bool          >()->default_value(true),       "Throw a fair::mq::MessageBadAlloc if cannot allocate a message (retry if false).")
-        ("shm-monitor",         po::value<bool          >()->default_value(true),       "Shared memory: run monitor daemon.")
-        ("ofi-size-hint",       po::value<size_t        >()->default_value(0),          "EXPERIMENTAL: OFI size hint for the allocator.")
-        ("rate",                po::value<float         >()->default_value(0.),         "Rate for conditional run loop (Hz).")
-        ("session",             po::value<string        >()->default_value("default"),  "Session name.")
-        ("config-key",          po::value<string        >(),                            "Use provided value instead of device id for fetching the configuration from JSON file.")
-        ("mq-config",           po::value<string        >(),                            "JSON input as file.")
-        ("channel-config",      po::value<vector<string>>()->multitoken()->composing(), "Configuration of single or multiple channel(s) by comma separated key=value list");
+        ("id",                  po::value<string        >()->default_value(""),                "Device ID.")
+        ("io-threads",          po::value<int           >()->default_value(1),                 "Number of I/O threads.")
+        ("transport",           po::value<string        >()->default_value("zeromq"),          "Transport ('zeromq'/'shmem').")
+        ("network-interface",   po::value<string        >()->default_value("default"),         "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).")
+        ("init-timeout",        po::value<int           >()->default_value(120),               "Timeout for the initialization in seconds (when expecting dynamic initialization).")
+        ("max-run-time",        po::value<uint64_t      >()->default_value(0),                 "Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).")
+        ("print-channels",      po::value<bool          >()->implicit_value(true),             "Print registered channel endpoints in a machine-readable format (<channel name>:<min num subchannels>:<max num subchannels>)")
+        ("shm-segment-size",    po::value<size_t        >()->default_value(2ULL << 30),        "Shared memory: size of the shared memory segment (in bytes).")
+        ("shm-allocation",      po::value<string        >()->default_value("rbtree_best_fit"), "Shared memory allocation algorithm: rbtree_best_fit/simple_seq_fit.")
+        ("shm-segment-id",      po::value<uint16_t      >()->default_value(0),                 "EXPERIMENTAL: Shared memory segment id for message creation.")
+        ("shm-mlock-segment",   po::value<bool          >()->default_value(false),             "Shared memory: mlock the shared memory segment after initialization.")
+        ("shm-zero-segment",    po::value<bool          >()->default_value(false),             "Shared memory: zero the shared memory segment memory after initialization.")
+        ("shm-throw-bad-alloc", po::value<bool          >()->default_value(true),              "Throw a fair::mq::MessageBadAlloc if cannot allocate a message (retry if false).")
+        ("shm-monitor",         po::value<bool          >()->default_value(true),              "Shared memory: run monitor daemon.")
+        ("ofi-size-hint",       po::value<size_t        >()->default_value(0),                 "EXPERIMENTAL: OFI size hint for the allocator.")
+        ("rate",                po::value<float         >()->default_value(0.),                "Rate for conditional run loop (Hz).")
+        ("session",             po::value<string        >()->default_value("default"),         "Session name.")
+        ("config-key",          po::value<string        >(),                                   "Use provided value instead of device id for fetching the configuration from JSON file.")
+        ("mq-config",           po::value<string        >(),                                   "JSON input as file.")
+        ("channel-config",      po::value<vector<string>>()->multitoken()->composing(),        "Configuration of single or multiple channel(s) by comma separated key=value list");
     return pluginOptions;
 }
 

fairmq/run/runBenchmarkSampler.cxx

@@ -15,10 +15,11 @@ void addCustomOptions(bpo::options_description& options)
 {
     options.add_options()
         ("out-channel", bpo::value<std::string>()->default_value("data"), "Name of the output channel")
-        ("same-msg", bpo::value<bool>()->default_value(false), "Re-send the same message, or recreate for each iteration")
         ("multipart", bpo::value<bool>()->default_value(false), "Handle multipart payloads")
+        ("memset", bpo::value<bool>()->default_value(false), "Memset allocated buffers to 0")
         ("num-parts", bpo::value<size_t>()->default_value(1), "Number of parts to send. 1 will send single messages, not parts")
         ("msg-size", bpo::value<size_t>()->default_value(1000000), "Message size in bytes")
+        ("msg-alignment", bpo::value<size_t>()->default_value(0), "Message alignment")
         ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Number of run iterations (0 - infinite)")
         ("msg-rate", bpo::value<float>()->default_value(0), "Msg rate limit in maximum number of messages per second");
 }

fairmq/sdk/AsioAsyncOp.h

@@ -12,6 +12,7 @@
 #include <asio/associated_allocator.hpp>
 #include <asio/associated_executor.hpp>
 #include <asio/executor_work_guard.hpp>
+#include <asio/dispatch.hpp>
 #include <asio/system_executor.hpp>
 #include <chrono>
 #include <exception>
@@ -69,17 +70,16 @@ struct AsioAsyncOpImpl : AsioAsyncOpImplBase<SignatureArgTypes...>
             throw RuntimeError("Async operation already completed");
         }
 
-        GetEx2().dispatch(
-            [=, handler = std::move(fHandler)]() mutable {
-                try {
-                    handler(ec, args...);
-                } catch (const std::exception& e) {
-                    FAIR_LOG(error) << "Uncaught exception in AsioAsyncOp completion handler: " << e.what();
-                } catch (...) {
-                    FAIR_LOG(error) << "Unknown uncaught exception in AsioAsyncOp completion handler.";
-                }
-            },
-            GetAlloc2());
+        asio::dispatch(GetEx2(),
+          [=, handler = std::move(fHandler)]() mutable {
+              try {
+                  handler(ec, args...);
+              } catch (const std::exception& e) {
+                  FAIR_LOG(error) << "Uncaught exception in AsioAsyncOp completion handler: " << e.what();
+              } catch (...) {
+                  FAIR_LOG(error) << "Unknown uncaught exception in AsioAsyncOp completion handler.";
+              }
+          });
 
         fWork1.reset();
         fWork2.reset();

fairmq/sdk/AsioBase.h

@@ -9,7 +9,7 @@
 #ifndef FAIR_MQ_SDK_ASIOBASE_H
 #define FAIR_MQ_SDK_ASIOBASE_H
 
-#include <asio/executor.hpp>
+#include <asio/any_io_executor.hpp>
 #include <fairmq/sdk/Traits.h>
 #include <memory>
 #include <utility>
@@ -18,7 +18,7 @@ namespace fair {
 namespace mq {
 namespace sdk {
 
-using DefaultExecutor = asio::executor;
+using DefaultExecutor = asio::any_io_executor;
 using DefaultAllocator = std::allocator<int>;
 
 /**

fairmq/sdk/CMakeLists.txt

@@ -84,10 +84,19 @@ target_link_libraries(fairmq
   Boost::program_options
 )
 
+add_executable(fairmq-dds-command-ui ${CMAKE_CURRENT_SOURCE_DIR}/runDDSCommandUI.cxx)
+target_link_libraries(fairmq-dds-command-ui
+  FairMQ
+  Commands
+  SDK
+  StateMachine
+)
+
 install(
   TARGETS
   SDK
   fairmq
+  fairmq-dds-command-ui
 
   EXPORT ${PROJECT_EXPORT_SET}
   RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}

fairmq/sdk/DDSSession.cxx

@@ -65,8 +65,6 @@ struct DDSSession::Impl
         , fId(to_string(fSession->create()))
         , fStopOnDestruction(false)
     {
-        setenv("DDS_SESSION_ID", fId.c_str(), 1);
-
         fDDSService.subscribeOnError([](const dds::intercom_api::EErrorCode errorCode, const std::string& msg) {
             std::cerr << "DDS error, error code: " << errorCode << ", error message: " << msg << std::endl;
         });
@@ -81,10 +79,6 @@ struct DDSSession::Impl
         , fStopOnDestruction(false)
     {
         fSession->attach(fId);
-        auto envId(std::getenv("DDS_SESSION_ID"));
-        if (envId != nullptr && std::string(envId) != fId) {
-            setenv("DDS_SESSION_ID", fId.c_str(), 1);
-        }
 
         fDDSService.subscribeOnError([](const dds::intercom_api::EErrorCode errorCode, const std::string& msg) {
             std::cerr << "DDS error, error code: " << errorCode << ", error message: " << msg << std::endl;
@@ -99,11 +93,6 @@ struct DDSSession::Impl
         , fId(to_string(fSession->getSessionID()))
         , fStopOnDestruction(false)
     {
-        auto envId(std::getenv("DDS_SESSION_ID"));
-        if (envId != nullptr && std::string(envId) != fId) {
-            setenv("DDS_SESSION_ID", fId.c_str(), 1);
-        }
-
         // Sanity check
         if (!fSession->IsRunning()) {
             throw std::runtime_error("Given CSession must be running");
@@ -123,10 +112,6 @@ struct DDSSession::Impl
     Impl(Impl&&) = delete;
     Impl& operator=(Impl&&) = delete;
 
-    struct Tag {};
-    friend auto operator<<(std::ostream& os, Tag) -> std::ostream& { return os << "DDSSession"; }
-    tools::InstanceLimiter<Tag, 1> fCount;
-
     DDSEnvironment fEnv;
     DDSRMSPlugin fRMSPlugin;
     Path fRMSConfig;

fairmq/sdk/Error.h

@@ -25,11 +25,6 @@ struct RuntimeError : ::std::runtime_error
     {}
 };
 
-struct MixedStateError : RuntimeError
-{
-    using RuntimeError::RuntimeError;
-};
-
 } /* namespace sdk */
 
 enum class ErrorCode

fairmq/sdk/Topology.cxx

@@ -18,11 +18,13 @@ namespace sdk {
 /// @param nativeSession Existing and initialized CSession (either via create() or attach())
 /// @param nativeTopo Existing CTopology that is activated on the given nativeSession
 /// @param env Optional DDSEnv (needed primarily for unit testing)
+/// @param blockUntilConnected if true, ctor will wait for all tasks to confirm subscriptions
 auto MakeTopology(dds::topology_api::CTopology nativeTopo,
                   std::shared_ptr<dds::tools_api::CSession> nativeSession,
-                  DDSEnv env) -> Topology
+                  DDSEnv env,
+                  bool blockUntilConnected) -> Topology
 {
-    return {DDSTopo(std::move(nativeTopo), env), DDSSession(std::move(nativeSession), env)};
+    return {DDSTopo(std::move(nativeTopo), env), DDSSession(std::move(nativeSession), env), blockUntilConnected};
 }
 
 }   // namespace sdk

fairmq/sdk/Topology.h

@@ -70,6 +70,65 @@ const std::map<DeviceTransition, DeviceState> expectedState =
     { DeviceTransition::End,          DeviceState::Exiting }
 };
 
+// mirrors DeviceState, but adds a "Mixed" state that represents a topology where devices are currently not in the same state.
+enum class AggregatedTopologyState : int
+{
+    Undefined = static_cast<int>(fair::mq::State::Undefined),
+    Ok = static_cast<int>(fair::mq::State::Ok),
+    Error = static_cast<int>(fair::mq::State::Error),
+    Idle = static_cast<int>(fair::mq::State::Idle),
+    InitializingDevice = static_cast<int>(fair::mq::State::InitializingDevice),
+    Initialized = static_cast<int>(fair::mq::State::Initialized),
+    Binding = static_cast<int>(fair::mq::State::Binding),
+    Bound = static_cast<int>(fair::mq::State::Bound),
+    Connecting = static_cast<int>(fair::mq::State::Connecting),
+    DeviceReady = static_cast<int>(fair::mq::State::DeviceReady),
+    InitializingTask = static_cast<int>(fair::mq::State::InitializingTask),
+    Ready = static_cast<int>(fair::mq::State::Ready),
+    Running = static_cast<int>(fair::mq::State::Running),
+    ResettingTask = static_cast<int>(fair::mq::State::ResettingTask),
+    ResettingDevice = static_cast<int>(fair::mq::State::ResettingDevice),
+    Exiting = static_cast<int>(fair::mq::State::Exiting),
+    Mixed
+};
+
+inline auto operator==(DeviceState lhs, AggregatedTopologyState rhs) -> bool
+{
+    return static_cast<int>(lhs) == static_cast<int>(rhs);
+}
+
+inline auto operator==(AggregatedTopologyState lhs, DeviceState rhs) -> bool
+{
+    return static_cast<int>(lhs) == static_cast<int>(rhs);
+}
+
+inline std::ostream& operator<<(std::ostream& os, const AggregatedTopologyState& state)
+{
+    if (state == AggregatedTopologyState::Mixed) {
+        return os << "MIXED";
+    } else {
+        return os << static_cast<DeviceState>(state);
+    }
+}
+
+inline std::string GetAggregatedTopologyStateName(AggregatedTopologyState s)
+{
+    if (s == AggregatedTopologyState::Mixed) {
+        return "MIXED";
+    } else {
+        return GetStateName(static_cast<State>(s));
+    }
+}
+
+inline AggregatedTopologyState GetAggregatedTopologyState(const std::string& state)
+{
+    if (state == "MIXED") {
+        return AggregatedTopologyState::Mixed;
+    } else {
+        return static_cast<AggregatedTopologyState>(GetState(state));
+    }
+}
+
 struct DeviceStatus
 {
     bool subscribed_to_state_changes;
@@ -100,22 +159,22 @@ using TopologyStateByTask = std::unordered_map<DDSTask::Id, DeviceStatus>;
 using TopologyStateByCollection = std::unordered_map<DDSCollection::Id, std::vector<DeviceStatus>>;
 using TopologyTransition = fair::mq::Transition;
 
-inline DeviceState AggregateState(const TopologyState& topologyState)
+inline AggregatedTopologyState AggregateState(const TopologyState& topologyState)
 {
     DeviceState first = topologyState.begin()->state;
 
     if (std::all_of(topologyState.cbegin(), topologyState.cend(), [&](TopologyState::value_type i) {
             return i.state == first;
         })) {
-        return first;
+        return static_cast<AggregatedTopologyState>(first);
     }
 
-    throw MixedStateError("State is not uniform");
+    return AggregatedTopologyState::Mixed;
 }
 
 inline bool StateEqualsTo(const TopologyState& topologyState, DeviceState state)
 {
-    return AggregateState(topologyState) == state;
+    return AggregateState(topologyState) == static_cast<AggregatedTopologyState>(state);
 }
 
 inline TopologyStateByCollection GroupByCollectionId(const TopologyState& topologyState)
@@ -157,24 +216,30 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     /// @brief (Re)Construct a FairMQ topology from an existing DDS topology
     /// @param topo DDSTopology
     /// @param session DDSSession
-    BasicTopology(DDSTopology topo, DDSSession session)
-        : BasicTopology<Executor, Allocator>(asio::system_executor(), std::move(topo), std::move(session))
+    /// @param blockUntilConnected if true, ctor will wait for all tasks to confirm subscriptions
+    BasicTopology(DDSTopology topo, DDSSession session, bool blockUntilConnected = false)
+        : BasicTopology<Executor, Allocator>(asio::system_executor(), std::move(topo), std::move(session), blockUntilConnected)
     {}
 
     /// @brief (Re)Construct a FairMQ topology from an existing DDS topology
     /// @param ex I/O executor to be associated
     /// @param topo DDSTopology
     /// @param session DDSSession
+    /// @param blockUntilConnected if true, ctor will wait for all tasks to confirm subscriptions
     /// @throws RuntimeError
     BasicTopology(const Executor& ex,
                   DDSTopology topo,
                   DDSSession session,
+                  bool blockUntilConnected = false,
                   Allocator alloc = DefaultAllocator())
         : AsioBase<Executor, Allocator>(ex, std::move(alloc))
         , fDDSSession(std::move(session))
         , fDDSTopo(std::move(topo))
         , fStateData()
         , fStateIndex()
+        , fMtx(std::make_unique<std::mutex>())
+        , fStateChangeSubscriptionsCV(std::make_unique<std::condition_variable>())
+        , fNumStateChangePublishers(0)
         , fHeartbeatsTimer(asio::system_executor())
         , fHeartbeatInterval(600000)
     {
@@ -190,6 +255,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
 
         fDDSSession.StartDDSService();
         SubscribeToStateChanges();
+        if (blockUntilConnected) {
+            WaitForPublisherCount(fStateIndex.size());
+        }
     }
 
     /// not copyable
@@ -204,7 +272,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     {
         UnsubscribeFromStateChanges();
 
-        std::lock_guard<std::mutex> lk(fMtx);
+        std::lock_guard<std::mutex> lk(*fMtx);
         fDDSSession.UnsubscribeFromCommands();
         try {
             for (auto& op : fChangeStateOps) {
@@ -223,6 +291,14 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         fHeartbeatsTimer.async_wait(std::bind(&BasicTopology::SendSubscriptionHeartbeats, this, std::placeholders::_1));
     }
 
+    void WaitForPublisherCount(unsigned int number)
+    {
+        std::unique_lock<std::mutex> lk(*fMtx);
+        fStateChangeSubscriptionsCV->wait(lk, [&](){
+            return fNumStateChangePublishers == number;
+        });
+    }
+
     void SendSubscriptionHeartbeats(const std::error_code& ec)
     {
         if (!ec) {
@@ -247,13 +323,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         fDDSSession.SendCommand(cmd::Cmds(cmd::make<cmd::UnsubscribeFromStateChange>()).Serialize());
 
         // wait for all tasks to confirm unsubscription
-        std::unique_lock<std::mutex> lk(fMtx);
-        fStateChangeUnsubscriptionCV.wait(lk, [&](){
-            unsigned int count = std::count_if(fStateIndex.cbegin(), fStateIndex.cend(), [=](const auto& s) {
-                return fStateData.at(s.second).subscribed_to_state_changes == false;
-            });
-            return count == fStateIndex.size();
-        });
+        WaitForPublisherCount(0);
     }
 
     void SubscribeToCommands()
@@ -299,11 +369,19 @@ class BasicTopology : public AsioBase<Executor, Allocator>
             DDSTask::Id taskId(cmd.GetTaskId());
 
             try {
-                std::lock_guard<std::mutex> lk(fMtx);
+                std::unique_lock<std::mutex> lk(*fMtx);
                 DeviceStatus& task = fStateData.at(fStateIndex.at(taskId));
-                task.subscribed_to_state_changes = true;
+                if (!task.subscribed_to_state_changes) {
+                    task.subscribed_to_state_changes = true;
+                    ++fNumStateChangePublishers;
+                } else {
+                    FAIR_LOG(warn) << "Task '" << task.taskId << "' sent subscription confirmation more than once";
+                }
+                lk.unlock();
+                fStateChangeSubscriptionsCV->notify_one();
             } catch (const std::exception& e) {
                 FAIR_LOG(error) << "Exception in HandleCmd(cmd::StateChangeSubscription const&): " << e.what();
+                FAIR_LOG(error) << "Possibly no task with id '" << taskId << "'?";
             }
         } else {
             FAIR_LOG(error) << "State change subscription failed for device: " << cmd.GetDeviceId() << ", task id: " << cmd.GetTaskId();
@@ -316,11 +394,16 @@ class BasicTopology : public AsioBase<Executor, Allocator>
             DDSTask::Id taskId(cmd.GetTaskId());
 
             try {
-                std::unique_lock<std::mutex> lk(fMtx);
+                std::unique_lock<std::mutex> lk(*fMtx);
                 DeviceStatus& task = fStateData.at(fStateIndex.at(taskId));
-                task.subscribed_to_state_changes = false;
+                if (task.subscribed_to_state_changes) {
+                    task.subscribed_to_state_changes = false;
+                    --fNumStateChangePublishers;
+                } else {
+                    FAIR_LOG(warn) << "Task '" << task.taskId << "' sent unsubscription confirmation more than once";
+                }
                 lk.unlock();
-                fStateChangeUnsubscriptionCV.notify_one();
+                fStateChangeSubscriptionsCV->notify_one();
             } catch (const std::exception& e) {
                 FAIR_LOG(error) << "Exception in HandleCmd(cmd::StateChangeUnsubscription const&): " << e.what();
             }
@@ -338,13 +421,14 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         DDSTask::Id taskId(cmd.GetTaskId());
 
         try {
-            std::lock_guard<std::mutex> lk(fMtx);
+            std::lock_guard<std::mutex> lk(*fMtx);
             DeviceStatus& task = fStateData.at(fStateIndex.at(taskId));
             task.lastState = cmd.GetLastState();
             task.state = cmd.GetCurrentState();
             // if the task is exiting, it will not respond to unsubscription request anymore, set it to false now.
             if (task.state == DeviceState::Exiting) {
                 task.subscribed_to_state_changes = false;
+                --fNumStateChangePublishers;
             }
             // FAIR_LOG(debug) << "Updated state entry: taskId=" << taskId << ", state=" << state;
 
@@ -362,13 +446,16 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     auto HandleCmd(cmd::TransitionStatus const& cmd) -> void
     {
         if (cmd.GetResult() != cmd::Result::Ok) {
-            FAIR_LOG(error) << cmd.GetTransition() << " transition failed for " << cmd.GetDeviceId();
             DDSTask::Id taskId(cmd.GetTaskId());
-            std::lock_guard<std::mutex> lk(fMtx);
+            std::lock_guard<std::mutex> lk(*fMtx);
             for (auto& op : fChangeStateOps) {
-                if (!op.second.IsCompleted() && op.second.ContainsTask(taskId) &&
-                    fStateData.at(fStateIndex.at(taskId)).state != op.second.GetTargetState()) {
-                    op.second.Complete(MakeErrorCode(ErrorCode::DeviceChangeStateFailed));
+                if (!op.second.IsCompleted() && op.second.ContainsTask(taskId)) {
+                    if (fStateData.at(fStateIndex.at(taskId)).state != op.second.GetTargetState()) {
+                        FAIR_LOG(error) << cmd.GetTransition() << " transition failed for " << cmd.GetDeviceId() << ", device is in " << cmd.GetCurrentState() << " state.";
+                        op.second.Complete(MakeErrorCode(ErrorCode::DeviceChangeStateFailed));
+                    } else {
+                        FAIR_LOG(debug) << cmd.GetTransition() << " transition failed for " << cmd.GetDeviceId() << ", device is already in " << cmd.GetCurrentState() << " state.";
+                    }
                 }
             }
         }
@@ -376,7 +463,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
 
     auto HandleCmd(cmd::Properties const& cmd) -> void
     {
-        std::unique_lock<std::mutex> lk(fMtx);
+        std::unique_lock<std::mutex> lk(*fMtx);
         try {
             auto& op(fGetPropertiesOps.at(cmd.GetRequestId()));
             lk.unlock();
@@ -390,7 +477,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
 
     auto HandleCmd(cmd::PropertiesSet const& cmd) -> void
     {
-        std::unique_lock<std::mutex> lk(fMtx);
+        std::unique_lock<std::mutex> lk(*fMtx);
         try {
             auto& op(fSetPropertiesOps.at(cmd.GetRequestId()));
             lk.unlock();
@@ -402,7 +489,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         }
     }
 
-    using Duration = std::chrono::milliseconds;
+    using Duration = std::chrono::microseconds;
     using ChangeStateCompletionSignature = void(std::error_code, TopologyState);
 
   private:
@@ -439,6 +526,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     }
                 });
             }
+            if (fTasks.empty()) {
+                FAIR_LOG(warn) << "ChangeState initiated on an empty set of tasks, check the path argument.";
+            }
         }
         ChangeStateOp() = delete;
         ChangeStateOp(const ChangeStateOp&) = delete;
@@ -594,7 +684,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         return asio::async_initiate<CompletionToken, ChangeStateCompletionSignature>([&](auto handler) {
             typename ChangeStateOp::Id const id(tools::UuidHash());
 
-            std::lock_guard<std::mutex> lk(fMtx);
+            std::lock_guard<std::mutex> lk(*fMtx);
 
             for (auto it = begin(fChangeStateOps); it != end(fChangeStateOps);) {
                 if (it->second.IsCompleted()) {
@@ -612,7 +702,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                                       fDDSTopo.GetTasks(path),
                                       fStateData,
                                       timeout,
-                                      fMtx,
+                                      *fMtx,
                                       AsioBase<Executor, Allocator>::GetExecutor(),
                                       AsioBase<Executor, Allocator>::GetAllocator(),
                                       std::move(handler)));
@@ -697,7 +787,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     /// @return map of id : DeviceStatus
     auto GetCurrentState() const -> TopologyState
     {
-        std::lock_guard<std::mutex> lk(fMtx);
+        std::lock_guard<std::mutex> lk(*fMtx);
         return fStateData;
     }
 
@@ -741,6 +831,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     }
                 });
             }
+            if (fTasks.empty()) {
+                FAIR_LOG(warn) << "WaitForState initiated on an empty set of tasks, check the path argument.";
+            }
         }
         WaitForStateOp() = delete;
         WaitForStateOp(const WaitForStateOp&) = delete;
@@ -754,9 +847,8 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         {
             fCount = std::count_if(stateIndex.cbegin(), stateIndex.cend(), [=](const auto& s) {
                 if (ContainsTask(stateData.at(s.second).taskId)) {
-                    return stateData.at(s.second).state == fTargetCurrentState
-                           &&
-                           (stateData.at(s.second).lastState == fTargetLastState || fTargetLastState == DeviceState::Ok);
+                    return stateData.at(s.second).state == fTargetCurrentState &&
+                          (stateData.at(s.second).lastState == fTargetLastState || fTargetLastState == DeviceState::Undefined);
                 } else {
                     return false;
                 }
@@ -768,8 +860,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         {
             if (!fOp.IsCompleted() && ContainsTask(taskId)) {
                 if (currentState == fTargetCurrentState &&
-                    (lastState == fTargetLastState ||
-                     fTargetLastState == DeviceState::Ok)) {
+                    (lastState == fTargetLastState || fTargetLastState == DeviceState::Undefined)) {
                     ++fCount;
                 }
                 TryCompletion();
@@ -824,7 +915,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         return asio::async_initiate<CompletionToken, WaitForStateCompletionSignature>([&](auto handler) {
             typename GetPropertiesOp::Id const id(tools::UuidHash());
 
-            std::lock_guard<std::mutex> lk(fMtx);
+            std::lock_guard<std::mutex> lk(*fMtx);
 
             for (auto it = begin(fWaitForStateOps); it != end(fWaitForStateOps);) {
                 if (it->second.IsCompleted()) {
@@ -842,7 +933,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                                       targetCurrentState,
                                       fDDSTopo.GetTasks(path),
                                       timeout,
-                                      fMtx,
+                                      *fMtx,
                                       AsioBase<Executor, Allocator>::GetExecutor(),
                                       AsioBase<Executor, Allocator>::GetAllocator(),
                                       std::move(handler)));
@@ -873,7 +964,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     template<typename CompletionToken>
     auto AsyncWaitForState(const DeviceState targetCurrentState, CompletionToken&& token)
     {
-        return AsyncWaitForState(DeviceState::Ok, targetCurrentState, "", Duration(0), std::move(token));
+        return AsyncWaitForState(DeviceState::Undefined, targetCurrentState, "", Duration(0), std::move(token));
     }
 
     /// @brief Wait for selected FairMQ devices to reach given last & current state in this topology
@@ -903,7 +994,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     auto WaitForState(const DeviceState targetCurrentState, const std::string& path = "", Duration timeout = Duration(0))
         -> std::error_code
     {
-        return WaitForState(DeviceState::Ok, targetCurrentState, path, timeout);
+        return WaitForState(DeviceState::Undefined, targetCurrentState, path, timeout);
     }
 
     using GetPropertiesCompletionSignature = void(std::error_code, GetPropertiesResult);
@@ -938,6 +1029,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     }
                 });
             }
+            if (expectedCount == 0) {
+                FAIR_LOG(warn) << "GetProperties initiated on an empty set of tasks, check the path argument.";
+            }
             // FAIR_LOG(debug) << "GetProperties " << fId << " with expected count of " << fExpectedCount << " started.";
         }
         GetPropertiesOp() = delete;
@@ -1002,7 +1096,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
             [&](auto handler) {
                 typename GetPropertiesOp::Id const id(tools::UuidHash());
 
-                std::lock_guard<std::mutex> lk(fMtx);
+                std::lock_guard<std::mutex> lk(*fMtx);
 
                 for (auto it = begin(fGetPropertiesOps); it != end(fGetPropertiesOps);) {
                     if (it->second.IsCompleted()) {
@@ -1018,7 +1112,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     std::forward_as_tuple(id,
                                           fDDSTopo.GetTasks(path).size(),
                                           timeout,
-                                          fMtx,
+                                          *fMtx,
                                           AsioBase<Executor, Allocator>::GetExecutor(),
                                           AsioBase<Executor, Allocator>::GetAllocator(),
                                           std::move(handler)));
@@ -1093,6 +1187,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     }
                 });
             }
+            if (expectedCount == 0) {
+                FAIR_LOG(warn) << "SetProperties initiated on an empty set of tasks, check the path argument.";
+            }
             // FAIR_LOG(debug) << "SetProperties " << fId << " with expected count of " << fExpectedCount << " started.";
         }
         SetPropertiesOp() = delete;
@@ -1155,7 +1252,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
             [&](auto handler) {
                 typename SetPropertiesOp::Id const id(tools::UuidHash());
 
-                std::lock_guard<std::mutex> lk(fMtx);
+                std::lock_guard<std::mutex> lk(*fMtx);
 
                 for (auto it = begin(fGetPropertiesOps); it != end(fGetPropertiesOps);) {
                     if (it->second.IsCompleted()) {
@@ -1171,7 +1268,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     std::forward_as_tuple(id,
                                           fDDSTopo.GetTasks(path).size(),
                                           timeout,
-                                          fMtx,
+                                          *fMtx,
                                           AsioBase<Executor, Allocator>::GetExecutor(),
                                           AsioBase<Executor, Allocator>::GetAllocator(),
                                           std::move(handler)));
@@ -1224,9 +1321,10 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     TopologyState fStateData;
     TopologyStateIndex fStateIndex;
 
-    mutable std::mutex fMtx;
+    mutable std::unique_ptr<std::mutex> fMtx;
 
-    std::condition_variable fStateChangeUnsubscriptionCV;
+    std::unique_ptr<std::condition_variable> fStateChangeSubscriptionsCV;
+    unsigned int fNumStateChangePublishers;
     asio::steady_timer fHeartbeatsTimer;
     Duration fHeartbeatInterval;
 
@@ -1242,7 +1340,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         int index = 0;
 
         for (const auto& task : fDDSTopo.GetTasks()) {
-            fStateData.push_back(DeviceStatus{false, DeviceState::Ok, DeviceState::Ok, task.GetId(), task.GetCollectionId()});
+            fStateData.push_back(DeviceStatus{false, DeviceState::Undefined, DeviceState::Undefined, task.GetId(), task.GetCollectionId()});
             fStateIndex.emplace(task.GetId(), index);
             index++;
         }
@@ -1262,9 +1360,11 @@ using Topo = Topology;
 /// @param nativeSession Existing and initialized CSession (either via create() or attach())
 /// @param nativeTopo Existing CTopology that is activated on the given nativeSession
 /// @param env Optional DDSEnv (needed primarily for unit testing)
+/// @param blockUntilConnected if true, ctor will wait for all tasks to confirm subscriptions
 auto MakeTopology(dds::topology_api::CTopology nativeTopo,
                   std::shared_ptr<dds::tools_api::CSession> nativeSession,
-                  DDSEnv env = {}) -> Topology;
+                  DDSEnv env = {},
+                  bool blockUntilConnected = false) -> Topology;
 
 }   // namespace sdk
 }   // namespace mq

fairmq/sdk/commands/Commands.cxx

@@ -70,9 +70,10 @@ array<string, 17> typeNames =
     }
 };
 
-array<fair::mq::State, 15> fbStateToMQState =
+array<fair::mq::State, 16> fbStateToMQState =
 {
     {
+        fair::mq::State::Undefined,
         fair::mq::State::Ok,
         fair::mq::State::Error,
         fair::mq::State::Idle,
@@ -91,9 +92,10 @@ array<fair::mq::State, 15> fbStateToMQState =
     }
 };
 
-array<sdk::cmd::FBState, 15> mqStateToFBState =
+array<sdk::cmd::FBState, 16> mqStateToFBState =
 {
     {
+        sdk::cmd::FBState_Undefined,
         sdk::cmd::FBState_Ok,
         sdk::cmd::FBState_Error,
         sdk::cmd::FBState_Idle,
@@ -288,6 +290,7 @@ string Cmds::Serialize(const Format type) const
                 cmdBuilder->add_task_id(_cmd.GetTaskId());
                 cmdBuilder->add_result(GetFBResult(_cmd.GetResult()));
                 cmdBuilder->add_transition(GetFBTransition(_cmd.GetTransition()));
+                cmdBuilder->add_current_state(GetFBState(_cmd.GetCurrentState()));
             }
             break;
             case Type::config: {
@@ -443,7 +446,7 @@ void Cmds::Deserialize(const string& str, const Format type)
                 fCmds.emplace_back(make<CurrentState>(cmdPtr.device_id()->str(), GetMQState(cmdPtr.current_state())));
             break;
             case FBCmd_transition_status:
-                fCmds.emplace_back(make<TransitionStatus>(cmdPtr.device_id()->str(), cmdPtr.task_id(), GetResult(cmdPtr.result()), GetMQTransition(cmdPtr.transition())));
+                fCmds.emplace_back(make<TransitionStatus>(cmdPtr.device_id()->str(), cmdPtr.task_id(), GetResult(cmdPtr.result()), GetMQTransition(cmdPtr.transition()), GetMQState(cmdPtr.current_state())));
             break;
             case FBCmd_config:
                 fCmds.emplace_back(make<Config>(cmdPtr.device_id()->str(), cmdPtr.config_string()->str()));

fairmq/sdk/commands/Commands.h

@@ -50,7 +50,7 @@ enum class Type : int
     subscription_heartbeat,        // args: { interval }
 
     current_state,                 // args: { device_id, current_state }
-    transition_status,             // args: { device_id, task_id, Result, transition }
+    transition_status,             // args: { device_id, task_id, Result, transition, current_state }
     config,                        // args: { device_id, config_string }
     state_change_subscription,     // args: { device_id, task_id, Result }
     state_change_unsubscription,   // args: { device_id, task_id, Result }
@@ -188,12 +188,13 @@ struct CurrentState : Cmd
 
 struct TransitionStatus : Cmd
 {
-    explicit TransitionStatus(const std::string& deviceId, const uint64_t taskId, const Result result, const Transition transition)
+    explicit TransitionStatus(const std::string& deviceId, const uint64_t taskId, const Result result, const Transition transition, State currentState)
         : Cmd(Type::transition_status)
         , fDeviceId(deviceId)
         , fTaskId(taskId)
         , fResult(result)
         , fTransition(transition)
+        , fCurrentState(currentState)
     {}
 
     std::string GetDeviceId() const { return fDeviceId; }
@@ -204,12 +205,15 @@ struct TransitionStatus : Cmd
     void SetResult(const Result result) { fResult = result; }
     Transition GetTransition() const { return fTransition; }
     void SetTransition(const Transition transition) { fTransition = transition; }
+    fair::mq::State GetCurrentState() const { return fCurrentState; }
+    void SetCurrentState(fair::mq::State state) { fCurrentState = state; }
 
   private:
     std::string fDeviceId;
     uint64_t fTaskId;
     Result fResult;
     Transition fTransition;
+    fair::mq::State fCurrentState;
 };
 
 struct Config : Cmd

fairmq/sdk/commands/CommandsFormat.fbs

@@ -6,6 +6,7 @@ enum FBResult:byte {
 }
 
 enum FBState:byte {
+    Undefined,
     Ok,
     Error,
     Idle,
@@ -55,7 +56,7 @@ enum FBCmd:byte {
     subscription_heartbeat,        // args: { interval }
 
     current_state,                 // args: { device_id, current_state }
-    transition_status,             // args: { device_id, task_id, Result, transition }
+    transition_status,             // args: { device_id, task_id, Result, transition, current_state }
     config,                        // args: { device_id, config_string }
     state_change_subscription,     // args: { device_id, task_id, Result }
     state_change_unsubscription,   // args: { device_id, task_id, Result }

fairmq/sdk/runDDSCommandUI.cxx

@@ -9,6 +9,7 @@
 #include <fairmq/sdk/commands/Commands.h>
 #include <fairmq/States.h>
 #include <fairmq/SDK.h>
+#include <fairmq/Tools.h>
 
 #include <boost/program_options.hpp>
 
@@ -58,67 +59,93 @@ void printControlsHelp()
 
 void handleCommand(const string& command, const string& path, unsigned int timeout, Topology& topo, const string& pKey, const string& pVal)
 {
+    std::pair<std::error_code, fair::mq::sdk::TopologyState> changeStateResult;
+
     if (command == "c") {
         cout << "> checking state of the devices" << endl;
         auto const result = topo.GetCurrentState();
+        bool error = false;
         for (const auto& d : result) {
             cout << d.taskId << " : " << d.state << endl;
+            if (d.state == sdk::DeviceState::Error) {
+                error = true;
+            }
         }
+        if (error) {
+            throw runtime_error("Some of the devices are in the Error state");
+        }
+        return;
     } else if (command == "o") {
-        cout << "> dumping config of the devices (" << path << ")" << endl;
+        cout << "> dumping config of " << (path == "" ? "all" : path) << endl;
         // TODO: extend this regex to return all properties, once command size limitation is removed.
         auto const result = topo.GetProperties("^(session|id)$", path, std::chrono::milliseconds(timeout));
+        if (result.first != std::error_code()) {
+            cout << "ERROR: GetProperties failed for '" << path << "': " << result.first.message() << endl;
+            throw runtime_error(tools::ToString("GetProperties failed for '", path, "': ", result.first.message()));
+        }
         for (const auto& d : result.second.devices) {
             for (auto const& p : d.second.props) {
                 cout << d.first << ": " << p.first << " : " << p.second << endl;
             }
         }
+        return;
     } else if (command == "p") {
         if (pKey == "" || pVal == "") {
             cout << "cannot send property with empty key and/or value! given key: '" << pKey << "', value: '" << pVal << "'." << endl;
-            return;
+            throw runtime_error(tools::ToString("cannot send property with empty key and/or value! given key: '", pKey, "', value: '", pVal, "'."));
         }
         const DeviceProperties props{{pKey, pVal}};
-        cout << "> sending property (" << path << ")" << endl;
-        topo.SetProperties(props, path);
+        cout << "> setting properties --> " << (path == "" ? "all" : path) << endl;
+        auto const result = topo.SetProperties(props, path);
+        if (result.first != std::error_code()) {
+            cout << "ERROR: SetProperties failed for '" << path << "': " << result.first.message() << endl;
+            throw runtime_error(tools::ToString("SetProperties failed for '", path, "': ", result.first.message()));
+        }
         // give dds time to complete request
         this_thread::sleep_for(chrono::milliseconds(100));
+        return;
     } else if (command == "i") {
-        cout << "> init devices (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::InitDevice, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating InitDevice transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::InitDevice, path, std::chrono::milliseconds(timeout));
     } else if (command == "k") {
-        cout << "> complete init (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::CompleteInit, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating CompleteInit transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::CompleteInit, path, std::chrono::milliseconds(timeout));
     } else if (command == "b") {
-        cout << "> bind devices (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::Bind, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating Bind transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::Bind, path, std::chrono::milliseconds(timeout));
     } else if (command == "x") {
-        cout << "> connect devices (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::Connect, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating Connect transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::Connect, path, std::chrono::milliseconds(timeout));
     } else if (command == "j") {
-        cout << "> init tasks (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::InitTask, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating InitTask transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::InitTask, path, std::chrono::milliseconds(timeout));
     } else if (command == "r") {
-        cout << "> run tasks (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::Run, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating Run transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::Run, path, std::chrono::milliseconds(timeout));
     } else if (command == "s") {
-        cout << "> stop devices (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::Stop, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating Stop transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::Stop, path, std::chrono::milliseconds(timeout));
     } else if (command == "t") {
-        cout << "> reset tasks (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::ResetTask, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating ResetTask transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::ResetTask, path, std::chrono::milliseconds(timeout));
     } else if (command == "d") {
-        cout << "> reset devices (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::ResetDevice, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating ResetDevice transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::ResetDevice, path, std::chrono::milliseconds(timeout));
     } else if (command == "q") {
-        cout << "> end (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::End, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating End transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::End, path, std::chrono::milliseconds(timeout));
     } else if (command == "h") {
         cout << "> help" << endl;
         printControlsHelp();
+        return;
     } else {
         cout << "\033[01;32mInvalid input: [" << command << "]\033[0m" << endl;
         printControlsHelp();
+        throw runtime_error(tools::ToString("\033[01;32mInvalid input: [", command, "]\033[0m"));
+    }
+    if (changeStateResult.first != std::error_code()) {
+        cout << "ERROR: ChangeState failed for '" << path << "': " << changeStateResult.first.message() << endl;
+        throw runtime_error(tools::ToString("ERROR: ChangeState failed for '", path, "': ", changeStateResult.first.message()));
     }
 }
 
@@ -138,7 +165,11 @@ void sendCommand(const string& commandIn, const string& path, unsigned int timeo
     command = c;
 
     while (true) {
-        handleCommand(command, path, timeout, topo, pKey, pVal);
+        try {
+            handleCommand(command, path, timeout, topo, pKey, pVal);
+        } catch(exception& e) {
+            cout << "Error: " << e.what() << endl;
+        }
         cin >> c;
         command = c;
     }
@@ -199,13 +230,14 @@ try {
     DDSSession session(sessionID, env);
     DDSTopology ddsTopo(DDSTopology::Path(topoFile), env);
 
-    Topology topo(ddsTopo, session);
+    Topology topo(ddsTopo, session, true);
 
     if (targetState != "") {
         if (command != "") {
             sendCommand(command, path, timeout, topo, pKey, pVal);
         }
         size_t pos = targetState.find("->");
+        cout << "> waiting for " << (path == "" ? "all" : path) << " to reach " << targetState << endl;
         if (pos == string::npos) {
             /* auto ec =  */topo.WaitForState(GetState(targetState), path, std::chrono::milliseconds(timeout));
             // cout << "WaitForState(" << targetState << ") result: " << ec.message() << endl;

fairmq/shmem/Common.h

@@ -12,14 +12,18 @@
 
 #include <atomic>
 #include <string>
-#include <unordered_map>
+#include <functional> // std::equal_to
 
-#include <boost/interprocess/managed_shared_memory.hpp>
+#include <boost/functional/hash.hpp>
 #include <boost/interprocess/allocators/allocator.hpp>
 #include <boost/interprocess/containers/map.hpp>
 #include <boost/interprocess/containers/string.hpp>
 #include <boost/interprocess/containers/vector.hpp>
-#include <boost/functional/hash.hpp>
+#include <boost/interprocess/indexes/null_index.hpp>
+#include <boost/interprocess/managed_shared_memory.hpp>
+#include <boost/interprocess/mem_algo/simple_seq_fit.hpp>
+#include <boost/unordered_map.hpp>
+#include <boost/variant.hpp>
 
 #include <unistd.h>
 #include <sys/types.h>
@@ -31,6 +35,17 @@ namespace mq
 namespace shmem
 {
 
+struct SharedMemoryError : std::runtime_error { using std::runtime_error::runtime_error; };
+
+using SimpleSeqFitSegment = boost::interprocess::basic_managed_shared_memory<char,
+    boost::interprocess::simple_seq_fit<boost::interprocess::mutex_family>,
+    boost::interprocess::null_index>;
+    // boost::interprocess::iset_index>;
+using RBTreeBestFitSegment = boost::interprocess::basic_managed_shared_memory<char,
+    boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family>,
+    boost::interprocess::null_index>;
+    // boost::interprocess::iset_index>;
+
 using SegmentManager = boost::interprocess::managed_shared_memory::segment_manager;
 using VoidAlloc      = boost::interprocess::allocator<void, SegmentManager>;
 using CharAlloc      = boost::interprocess::allocator<char, SegmentManager>;
@@ -38,6 +53,12 @@ using Str            = boost::interprocess::basic_string<char, std::char_traits<
 using StrAlloc       = boost::interprocess::allocator<Str, SegmentManager>;
 using StrVector      = boost::interprocess::vector<Str, StrAlloc>;
 
+enum class AllocationAlgorithm : int
+{
+    rbtree_best_fit,
+    simple_seq_fit
+};
+
 struct RegionInfo
 {
     RegionInfo(const VoidAlloc& alloc)
@@ -60,8 +81,22 @@ struct RegionInfo
     bool fDestroyed;
 };
 
-using Uint64RegionInfoPairAlloc = boost::interprocess::allocator<std::pair<const uint64_t, RegionInfo>, SegmentManager>;
-using Uint64RegionInfoMap = boost::interprocess::map<uint64_t, RegionInfo, std::less<uint64_t>, Uint64RegionInfoPairAlloc>;
+using Uint16RegionInfoPairAlloc = boost::interprocess::allocator<std::pair<const uint16_t, RegionInfo>, SegmentManager>;
+using Uint16RegionInfoMap = boost::interprocess::map<uint16_t, RegionInfo, std::less<uint16_t>, Uint16RegionInfoPairAlloc>;
+using Uint16RegionInfoHashMap = boost::unordered_map<uint16_t, RegionInfo, boost::hash<uint16_t>, std::equal_to<uint16_t>, Uint16RegionInfoPairAlloc>;
+
+struct SegmentInfo
+{
+    SegmentInfo(AllocationAlgorithm aa)
+        : fAllocationAlgorithm(aa)
+    {}
+
+    AllocationAlgorithm fAllocationAlgorithm;
+};
+
+using Uint16SegmentInfoPairAlloc = boost::interprocess::allocator<std::pair<const uint16_t, SegmentInfo>, SegmentManager>;
+using Uint16SegmentInfoHashMap = boost::unordered_map<uint16_t, SegmentInfo, boost::hash<uint16_t>, std::equal_to<uint16_t>, Uint16SegmentInfoPairAlloc>;
+// using Uint16SegmentInfoMap = boost::interprocess::map<uint16_t, SegmentInfo, std::less<uint16_t>, Uint16SegmentInfoPairAlloc>;
 
 struct DeviceCounter
 {
@@ -74,21 +109,65 @@ struct DeviceCounter
 
 struct RegionCounter
 {
-    RegionCounter(uint64_t c)
+    RegionCounter(uint16_t c)
         : fCount(c)
     {}
 
-    std::atomic<uint64_t> fCount;
+    std::atomic<uint16_t> fCount;
 };
 
 struct MetaHeader
 {
     size_t fSize;
-    size_t fRegionId;
     size_t fHint;
+    uint16_t fRegionId;
+    uint16_t fSegmentId;
     boost::interprocess::managed_shared_memory::handle_t fHandle;
 };
 
+#ifdef FAIRMQ_DEBUG_MODE
+struct MsgCounter
+{
+    MsgCounter()
+        : fCount(0)
+    {}
+
+    MsgCounter(unsigned int c)
+        : fCount(c)
+    {}
+
+    std::atomic<unsigned int> fCount;
+};
+
+using Uint16MsgCounterPairAlloc = boost::interprocess::allocator<std::pair<const uint16_t, MsgCounter>, SegmentManager>;
+using Uint16MsgCounterHashMap = boost::unordered_map<uint16_t, MsgCounter, boost::hash<uint16_t>, std::equal_to<uint16_t>, Uint16MsgCounterPairAlloc>;
+
+struct MsgDebug
+{
+    MsgDebug()
+        : fPid(0)
+        , fSize(0)
+        , fCreationTime(0)
+    {}
+
+    MsgDebug(pid_t pid, size_t size, const uint64_t creationTime)
+        : fPid(pid)
+        , fSize(size)
+        , fCreationTime(creationTime)
+    {}
+
+    pid_t fPid;
+    size_t fSize;
+    uint64_t fCreationTime;
+};
+
+using SizetMsgDebugPairAlloc = boost::interprocess::allocator<std::pair<const size_t, MsgDebug>, SegmentManager>;
+// using SizetMsgDebugHashMap = boost::unordered_map<size_t, MsgDebug, boost::hash<size_t>, std::equal_to<size_t>, SizetMsgDebugPairAlloc>;
+using SizetMsgDebugMap = boost::interprocess::map<size_t, MsgDebug, std::less<size_t>, SizetMsgDebugPairAlloc>;
+using Uint16MsgDebugMapPairAlloc = boost::interprocess::allocator<std::pair<const uint16_t, SizetMsgDebugMap>, SegmentManager>;
+using Uint16MsgDebugMapHashMap = boost::unordered_map<uint16_t, SizetMsgDebugMap, boost::hash<uint16_t>, std::equal_to<uint16_t>, Uint16MsgDebugMapPairAlloc>;
+#endif
+
 struct RegionBlock
 {
     RegionBlock()
@@ -113,11 +192,109 @@ struct RegionBlock
 inline std::string buildShmIdFromSessionIdAndUserId(const std::string& sessionId)
 {
     std::string seed((std::to_string(geteuid()) + sessionId));
-    std::string shmId = picosha2::hash256_hex_string(seed);
-    shmId.resize(10, '_');
+    // generate a 8-digit hex value out of sha256 hash
+    std::vector<unsigned char> hash(4);
+    picosha2::hash256(seed.begin(), seed.end(), hash.begin(), hash.end());
+    std::string shmId = picosha2::bytes_to_hex_string(hash.begin(), hash.end());
+
     return shmId;
 }
 
+struct SegmentSize : public boost::static_visitor<size_t>
+{
+    template<typename S>
+    size_t operator()(S& s) const { return s.get_size(); }
+};
+
+struct SegmentAddress : public boost::static_visitor<void*>
+{
+    template<typename S>
+    void* operator()(S& s) const { return s.get_address(); }
+};
+
+struct SegmentMemoryZeroer : public boost::static_visitor<>
+{
+    template<typename S>
+    void operator()(S& s) const { s.zero_free_memory(); }
+};
+
+struct SegmentFreeMemory : public boost::static_visitor<size_t>
+{
+    template<typename S>
+    size_t operator()(S& s) const { return s.get_free_memory(); }
+};
+
+struct SegmentHandleFromAddress : public boost::static_visitor<boost::interprocess::managed_shared_memory::handle_t>
+{
+    SegmentHandleFromAddress(const void* _ptr) : ptr(_ptr) {}
+
+    template<typename S>
+    boost::interprocess::managed_shared_memory::handle_t operator()(S& s) const { return s.get_handle_from_address(ptr); }
+
+    const void* ptr;
+};
+
+struct SegmentAddressFromHandle : public boost::static_visitor<void*>
+{
+    SegmentAddressFromHandle(const boost::interprocess::managed_shared_memory::handle_t _handle) : handle(_handle) {}
+
+    template<typename S>
+    void* operator()(S& s) const { return s.get_address_from_handle(handle); }
+
+    const boost::interprocess::managed_shared_memory::handle_t handle;
+};
+
+struct SegmentAllocate : public boost::static_visitor<void*>
+{
+    SegmentAllocate(const size_t _size) : size(_size) {}
+
+    template<typename S>
+    void* operator()(S& s) const { return s.allocate(size); }
+
+    const size_t size;
+};
+
+struct SegmentAllocateAligned : public boost::static_visitor<void*>
+{
+    SegmentAllocateAligned(const size_t _size, const size_t _alignment) : size(_size), alignment(_alignment) {}
+
+    template<typename S>
+    void* operator()(S& s) const { return s.allocate_aligned(size, alignment); }
+
+    const size_t size;
+    const size_t alignment;
+};
+
+struct SegmentBufferShrink : public boost::static_visitor<char*>
+{
+    SegmentBufferShrink(const size_t _old_size, const size_t _new_size, char* _local_ptr)
+        : old_size(_old_size)
+        , new_size(_new_size)
+        , local_ptr(_local_ptr)
+    {}
+
+    template<typename S>
+    char* operator()(S& s) const
+    {
+        boost::interprocess::managed_shared_memory::size_type shrunk_size = new_size;
+        return s.template allocation_command<char>(boost::interprocess::shrink_in_place, old_size + 128, shrunk_size, local_ptr);
+    }
+
+    const size_t old_size;
+    const size_t new_size;
+    mutable char* local_ptr;
+};
+
+struct SegmentDeallocate : public boost::static_visitor<>
+{
+    SegmentDeallocate(void* _ptr) : ptr(_ptr) {}
+
+    template<typename S>
+    void operator()(S& s) const { return s.deallocate(ptr); }
+
+    void* ptr;
+};
+
 } // namespace shmem
 } // namespace mq
 } // namespace fair

fairmq/shmem/Manager.h

@@ -17,22 +17,28 @@
 
 #include "Common.h"
 #include "Region.h"
+#include "Monitor.h"
 
 #include <FairMQLogger.h>
 #include <FairMQMessage.h>
+#include <fairmq/ProgOptions.h>
 #include <fairmq/tools/CppSTL.h>
 #include <fairmq/tools/Strings.h>
 
-#include <boost/process.hpp>
-#include <boost/filesystem.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/interprocess/ipc/message_queue.hpp>
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/sync/named_condition.hpp>
 #include <boost/interprocess/sync/named_mutex.hpp>
-#include <boost/interprocess/ipc/message_queue.hpp>
+#include <boost/process.hpp>
+#include <boost/variant.hpp>
 
 #include <cstdlib> // getenv
+#include <condition_variable>
+#include <mutex>
 #include <set>
+#include <sstream>
 #include <stdexcept>
 #include <string>
 #include <thread>
@@ -40,6 +46,8 @@
 #include <utility> // pair
 #include <vector>
 
+#include <sys/mman.h> // mlock
+
 namespace fair
 {
 namespace mq
@@ -47,49 +55,126 @@ namespace mq
 namespace shmem
 {
 
-struct SharedMemoryError : std::runtime_error { using std::runtime_error::runtime_error; };
-
 class Manager
 {
   public:
-    Manager(std::string id, std::string deviceId, size_t size, bool throwOnBadAlloc)
-        : fShmId(std::move(id))
+    Manager(std::string shmId, std::string deviceId, size_t size, const ProgOptions* config)
+        : fShmId(std::move(shmId))
+        , fSegmentId(config ? config->GetProperty<uint16_t>("shm-segment-id", 0) : 0)
         , fDeviceId(std::move(deviceId))
-        , fSegmentName("fmq_" + fShmId + "_main")
-        , fManagementSegmentName("fmq_" + fShmId + "_mng")
-        , fSegment(boost::interprocess::open_or_create, fSegmentName.c_str(), size)
-        , fManagementSegment(boost::interprocess::open_or_create, fManagementSegmentName.c_str(), 655360)
+        , fSegments()
+        , fManagementSegment(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_mng").c_str(), 6553600)
         , fShmVoidAlloc(fManagementSegment.get_segment_manager())
         , fShmMtx(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_mtx").c_str())
         , fRegionEventsCV(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_cv").c_str())
         , fRegionEventsSubscriptionActive(false)
         , fDeviceCounter(nullptr)
-        , fRegionInfos(nullptr)
+        , fShmSegments(nullptr)
+        , fShmRegions(nullptr)
         , fInterrupted(false)
         , fMsgCounter(0)
+#ifdef FAIRMQ_DEBUG_MODE
+        , fMsgDebug(nullptr)
+        , fShmMsgCounters(nullptr)
+#endif
         , fHeartbeatThread()
         , fSendHeartbeats(true)
-        , fThrowOnBadAlloc(throwOnBadAlloc)
+        , fThrowOnBadAlloc(config ? config->GetProperty<bool>("shm-throw-bad-alloc", true) : true)
     {
         using namespace boost::interprocess;
-        LOG(debug) << "created/opened shared memory segment '" << "fmq_" << fShmId << "_main" << "' of " << size << " bytes. Available are " << fSegment.get_free_memory() << " bytes.";
 
-        fRegionInfos = fManagementSegment.find_or_construct<Uint64RegionInfoMap>(unique_instance)(fShmVoidAlloc);
-        // store info about the managed segment as region with id 0
-        fRegionInfos->emplace(0, RegionInfo("", 0, 0, fShmVoidAlloc));
-
-        boost::interprocess::scoped_lock<named_mutex> lock(fShmMtx);
-
-        fDeviceCounter = fManagementSegment.find<DeviceCounter>(unique_instance).first;
-
-        if (fDeviceCounter) {
-            LOG(debug) << "device counter found, with value of " << fDeviceCounter->fCount << ". incrementing.";
-            (fDeviceCounter->fCount)++;
-            LOG(debug) << "incremented device counter, now: " << fDeviceCounter->fCount;
+        bool mlockSegment = false;
+        bool zeroSegment = false;
+        bool autolaunchMonitor = false;
+        std::string allocationAlgorithm("rbtree_best_fit");
+        if (config) {
+            mlockSegment = config->GetProperty<bool>("shm-mlock-segment", mlockSegment);
+            zeroSegment = config->GetProperty<bool>("shm-zero-segment", zeroSegment);
+            autolaunchMonitor = config->GetProperty<bool>("shm-monitor", autolaunchMonitor);
+            allocationAlgorithm = config->GetProperty<std::string>("shm-allocation", allocationAlgorithm);
         } else {
-            LOG(debug) << "no device counter found, creating one and initializing with 1";
-            fDeviceCounter = fManagementSegment.construct<DeviceCounter>(unique_instance)(1);
-            LOG(debug) << "initialized device counter with: " << fDeviceCounter->fCount;
+            LOG(debug) << "ProgOptions not available! Using defaults.";
+        }
+
+        if (autolaunchMonitor) {
+            StartMonitor(fShmId);
+        }
+
+        {
+            std::stringstream ss;
+            boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+
+            fShmSegments = fManagementSegment.find_or_construct<Uint16SegmentInfoHashMap>(unique_instance)(fShmVoidAlloc);
+
+            try {
+                auto it = fShmSegments->find(fSegmentId);
+                if (it == fShmSegments->end()) {
+                    // no segment with given id exists, creating
+                    if (allocationAlgorithm == "rbtree_best_fit") {
+                        fSegments.emplace(fSegmentId, RBTreeBestFitSegment(create_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(fSegmentId)).c_str(), size));
+                        fShmSegments->emplace(fSegmentId, AllocationAlgorithm::rbtree_best_fit);
+                    } else if (allocationAlgorithm == "simple_seq_fit") {
+                        fSegments.emplace(fSegmentId, SimpleSeqFitSegment(create_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(fSegmentId)).c_str(), size));
+                        fShmSegments->emplace(fSegmentId, AllocationAlgorithm::simple_seq_fit);
+                    }
+                    ss << "Created ";
+                } else {
+                    // found segment with the given id, opening
+                    if (it->second.fAllocationAlgorithm == AllocationAlgorithm::rbtree_best_fit) {
+                        fSegments.emplace(fSegmentId, RBTreeBestFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(fSegmentId)).c_str()));
+                        if (allocationAlgorithm != "rbtree_best_fit") {
+                            LOG(warn) << "Allocation algorithm of the opened segment is rbtree_best_fit, but requested is " << allocationAlgorithm << ". Ignoring requested setting.";
+                            allocationAlgorithm = "rbtree_best_fit";
+                        }
+                    } else {
+                        fSegments.emplace(fSegmentId, SimpleSeqFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(fSegmentId)).c_str()));
+                        if (allocationAlgorithm != "simple_seq_fit") {
+                            LOG(warn) << "Allocation algorithm of the opened segment is simple_seq_fit, but requested is " << allocationAlgorithm << ". Ignoring requested setting.";
+                            allocationAlgorithm = "simple_seq_fit";
+                        }
+                    }
+                    ss << "Opened ";
+                }
+                ss << "shared memory segment '" << "fmq_" << fShmId << "_m_" << fSegmentId << "'."
+                << " Size: " << boost::apply_visitor(SegmentSize{}, fSegments.at(fSegmentId)) << " bytes."
+                << " Available: " << boost::apply_visitor(SegmentFreeMemory{}, fSegments.at(fSegmentId)) << " bytes."
+                << " Allocation algorithm: " << allocationAlgorithm;
+                LOG(debug) << ss.str();
+            } catch(interprocess_exception& bie) {
+                LOG(error) << "something went wrong: " << bie.what();
+            }
+
+            if (mlockSegment) {
+                LOG(debug) << "Locking the managed segment memory pages...";
+                if (mlock(boost::apply_visitor(SegmentAddress{}, fSegments.at(fSegmentId)), boost::apply_visitor(SegmentSize{}, fSegments.at(fSegmentId))) == -1) {
+                    LOG(error) << "Could not lock the managed segment memory. Code: " << errno << ", reason: " << strerror(errno);
+                }
+                LOG(debug) << "Successfully locked the managed segment memory pages.";
+            }
+            if (zeroSegment) {
+                LOG(debug) << "Zeroing the managed segment free memory...";
+                boost::apply_visitor(SegmentMemoryZeroer{}, fSegments.at(fSegmentId));
+                LOG(debug) << "Successfully zeroed the managed segment free memory.";
+            }
+
+            fShmRegions = fManagementSegment.find_or_construct<Uint16RegionInfoHashMap>(unique_instance)(fShmVoidAlloc);
+
+            fDeviceCounter = fManagementSegment.find<DeviceCounter>(unique_instance).first;
+
+            if (fDeviceCounter) {
+                LOG(debug) << "device counter found, with value of " << fDeviceCounter->fCount << ". incrementing.";
+                (fDeviceCounter->fCount)++;
+                LOG(debug) << "incremented device counter, now: " << fDeviceCounter->fCount;
+            } else {
+                LOG(debug) << "no device counter found, creating one and initializing with 1";
+                fDeviceCounter = fManagementSegment.construct<DeviceCounter>(unique_instance)(1);
+                LOG(debug) << "initialized device counter with: " << fDeviceCounter->fCount;
+            }
+
+#ifdef FAIRMQ_DEBUG_MODE
+            fMsgDebug = fManagementSegment.find_or_construct<Uint16MsgDebugMapHashMap>(unique_instance)(fShmVoidAlloc);
+            fShmMsgCounters = fManagementSegment.find_or_construct<Uint16MsgCounterHashMap>(unique_instance)(fShmVoidAlloc);
+#endif
         }
 
         fHeartbeatThread = std::thread(&Manager::SendHeartbeats, this);
@@ -100,42 +185,6 @@ class Manager
     Manager(const Manager&) = delete;
     Manager operator=(const Manager&) = delete;
 
-    ~Manager()
-    {
-        using namespace boost::interprocess;
-        bool lastRemoved = false;
-
-        UnsubscribeFromRegionEvents();
-
-        fSendHeartbeats = false;
-        fHeartbeatThread.join();
-
-        try {
-            boost::interprocess::scoped_lock<named_mutex> lock(fShmMtx);
-
-            (fDeviceCounter->fCount)--;
-
-            if (fDeviceCounter->fCount == 0) {
-                LOG(debug) << "last segment user, removing segment.";
-
-                RemoveSegments();
-                lastRemoved = true;
-            } else {
-                LOG(debug) << "other segment users present (" << fDeviceCounter->fCount << "), not removing it.";
-            }
-        } catch(interprocess_exception& e) {
-            LOG(error) << "error while acquiring lock in Manager destructor: " << e.what();
-        }
-
-        if (lastRemoved) {
-            named_mutex::remove(std::string("fmq_" + fShmId + "_mtx").c_str());
-            named_condition::remove(std::string("fmq_" + fShmId + "_cv").c_str());
-        }
-    }
-
-    boost::interprocess::managed_shared_memory& Segment() { return fSegment; }
-    boost::interprocess::managed_shared_memory& ManagementSegment() { return fManagementSegment; }
-
     static void StartMonitor(const std::string& id)
     {
         using namespace boost::interprocess;
@@ -187,7 +236,7 @@ class Manager
     }
     bool Interrupted() { return fInterrupted.load(); }
 
-    std::pair<boost::interprocess::mapped_region*, uint64_t> CreateRegion(const size_t size,
+    std::pair<boost::interprocess::mapped_region*, uint16_t> CreateRegion(const size_t size,
                                                                           const int64_t userFlags,
                                                                           RegionCallback callback,
                                                                           RegionBulkCallback bulkCallback,
@@ -196,10 +245,10 @@ class Manager
     {
         using namespace boost::interprocess;
         try {
-            std::pair<mapped_region*, uint64_t> result;
+            std::pair<mapped_region*, uint16_t> result;
 
             {
-                uint64_t id = 0;
+                uint16_t id = 0;
                 boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
 
                 RegionCounter* rc = fManagementSegment.find<RegionCounter>(unique_instance).first;
@@ -223,7 +272,7 @@ class Manager
                 }
 
                 // create region info
-                fRegionInfos->emplace(id, RegionInfo(path.c_str(), flags, userFlags, fShmVoidAlloc));
+                fShmRegions->emplace(id, RegionInfo(path.c_str(), flags, userFlags, fShmVoidAlloc));
 
                 auto r = fRegions.emplace(id, tools::make_unique<Region>(fShmId, id, size, false, callback, bulkCallback, path, flags));
                 // LOG(debug) << "Created region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
@@ -243,13 +292,13 @@ class Manager
         }
     }
 
-    Region* GetRegion(const uint64_t id)
+    Region* GetRegion(const uint16_t id)
     {
         boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
         return GetRegionUnsafe(id);
     }
 
-    Region* GetRegionUnsafe(const uint64_t id)
+    Region* GetRegionUnsafe(const uint16_t id)
     {
         // remote region could actually be a local one if a message originates from this device (has been sent out and returned)
         auto it = fRegions.find(id);
@@ -258,26 +307,30 @@ class Manager
         } else {
             try {
                 // get region info
-                RegionInfo regionInfo = fRegionInfos->at(id);
+                RegionInfo regionInfo = fShmRegions->at(id);
                 std::string path = regionInfo.fPath.c_str();
                 int flags = regionInfo.fFlags;
                 // LOG(debug) << "Located remote region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
 
                 auto r = fRegions.emplace(id, tools::make_unique<Region>(fShmId, id, 0, true, nullptr, nullptr, path, flags));
                 return r.first->second.get();
+            } catch (std::out_of_range& oor) {
+                LOG(error) << "Could not get remote region with id '" << id << "'. Does the region creator run with the same session id?";
+                LOG(error) << oor.what();
+                return nullptr;
             } catch (boost::interprocess::interprocess_exception& e) {
-                LOG(warn) << "Could not get remote region for id: " << id;
+                LOG(warn) << "Could not get remote region for id '" << id << "'";
                 return nullptr;
             }
         }
     }
 
-    void RemoveRegion(const uint64_t id)
+    void RemoveRegion(const uint16_t id)
     {
+        fRegions.erase(id);
         {
             boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
-            fRegions.erase(id);
-            fRegionInfos->at(id).fDestroyed = true;
+            fShmRegions->at(id).fDestroyed = true;
         }
         fRegionEventsCV.notify_all();
     }
@@ -292,30 +345,37 @@ class Manager
     {
         std::vector<fair::mq::RegionInfo> result;
 
-        for (const auto& e : *fRegionInfos) {
+        for (const auto& e : *fShmRegions) {
             fair::mq::RegionInfo info;
+            info.managed = false;
             info.id = e.first;
             info.flags = e.second.fUserFlags;
             info.event = e.second.fDestroyed ? RegionEvent::destroyed : RegionEvent::created;
-            if (info.id != 0) {
-                if (!e.second.fDestroyed) {
-                    auto region = GetRegionUnsafe(info.id);
-                    info.ptr = region->fRegion.get_address();
-                    info.size = region->fRegion.get_size();
-                } else {
-                    info.ptr = nullptr;
-                    info.size = 0;
-                }
-                result.push_back(info);
+            if (!e.second.fDestroyed) {
+                auto region = GetRegionUnsafe(info.id);
+                info.ptr = region->fRegion.get_address();
+                info.size = region->fRegion.get_size();
             } else {
-                if (!e.second.fDestroyed) {
-                    info.ptr = fSegment.get_address();
-                    info.size = fSegment.get_size();
-                } else {
-                    info.ptr = nullptr;
-                    info.size = 0;
-                }
+                info.ptr = nullptr;
+                info.size = 0;
+            }
+            result.push_back(info);
+        }
+
+        for (const auto& e : *fShmSegments) {
+            // make sure any segments in the session are found
+            GetSegment(e.first);
+            try {
+                fair::mq::RegionInfo info;
+                info.managed = true;
+                info.id = e.first;
+                info.event = RegionEvent::created;
+                info.ptr = boost::apply_visitor(SegmentAddress{}, fSegments.at(e.first));
+                info.size = boost::apply_visitor(SegmentSize{}, fSegments.at(e.first));
                 result.push_back(info);
+            } catch (const std::out_of_range& oor) {
+                LOG(error) << "could not find segment with id " << e.first;
+                LOG(error) << oor.what();
             }
         }
 
@@ -378,52 +438,181 @@ class Manager
         }
     }
 
-    void IncrementMsgCounter() { ++fMsgCounter; }
-    void DecrementMsgCounter() { --fMsgCounter; }
+    void IncrementMsgCounter() { fMsgCounter.fetch_add(1, std::memory_order_relaxed); }
+    void DecrementMsgCounter() { fMsgCounter.fetch_sub(1, std::memory_order_relaxed); }
 
-    void RemoveSegments()
-    {
-        using namespace boost::interprocess;
-        if (shared_memory_object::remove(fSegmentName.c_str())) {
-            LOG(debug) << "successfully removed '" << fSegmentName << "' segment after the device has stopped.";
-        } else {
-            LOG(debug) << "did not remove " << fSegmentName << " segment after the device stopped. Already removed?";
-        }
+#ifdef FAIRMQ_DEBUG_MODE
+    void IncrementShmMsgCounter(uint16_t segmentId) { ++((*fShmMsgCounters)[segmentId].fCount); }
+    void DecrementShmMsgCounter(uint16_t segmentId) { --((*fShmMsgCounters)[segmentId].fCount); }
+#endif
 
-        if (shared_memory_object::remove(fManagementSegmentName.c_str())) {
-            LOG(debug) << "successfully removed '" << fManagementSegmentName << "' segment after the device has stopped.";
-        } else {
-            LOG(debug) << "did not remove '" << fManagementSegmentName << "' segment after the device stopped. Already removed?";
-        }
-    }
+    boost::interprocess::named_mutex& GetMtx() { return fShmMtx; }
 
     void SendHeartbeats()
     {
         std::string controlQueueName("fmq_" + fShmId + "_cq");
+        std::unique_lock<std::mutex> lock(fHeartbeatsMtx);
         while (fSendHeartbeats) {
             try {
                 boost::interprocess::message_queue mq(boost::interprocess::open_only, controlQueueName.c_str());
                 boost::posix_time::ptime sndTill = boost::posix_time::microsec_clock::universal_time() + boost::posix_time::milliseconds(100);
                 if (mq.timed_send(fDeviceId.c_str(), fDeviceId.size(), 0, sndTill)) {
-                    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+                    fHeartbeatsCV.wait_for(lock, std::chrono::milliseconds(100), [&]() { return !fSendHeartbeats; });
                 } else {
                     LOG(debug) << "control queue timeout";
                 }
             } catch (boost::interprocess::interprocess_exception& ie) {
-                std::this_thread::sleep_for(std::chrono::milliseconds(500));
-                // LOG(warn) << "no " << controlQueueName << " found";
+                fHeartbeatsCV.wait_for(lock, std::chrono::milliseconds(500), [&]() { return !fSendHeartbeats; });
+                // LOG(debug) << "no " << controlQueueName << " found";
             }
         }
     }
 
     bool ThrowingOnBadAlloc() const { return fThrowOnBadAlloc; }
 
+    void GetSegment(uint16_t id)
+    {
+        auto it = fSegments.find(id);
+        if (it == fSegments.end()) {
+            try {
+                // get region info
+                SegmentInfo segmentInfo = fShmSegments->at(id);
+                LOG(debug) << "Located segment with id '" << id << "'";
+
+                using namespace boost::interprocess;
+
+                if (segmentInfo.fAllocationAlgorithm == AllocationAlgorithm::rbtree_best_fit) {
+                    fSegments.emplace(id, RBTreeBestFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(id)).c_str()));
+                } else {
+                    fSegments.emplace(id, SimpleSeqFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(id)).c_str()));
+                }
+            } catch (std::out_of_range& oor) {
+                LOG(error) << "Could not get segment with id '" << id << "': " << oor.what();
+            } catch (boost::interprocess::interprocess_exception& bie) {
+                LOG(error) << "Could not get segment with id '" << id << "': " << bie.what();
+            }
+        }
+    }
+
+    boost::interprocess::managed_shared_memory::handle_t GetHandleFromAddress(const void* ptr, uint16_t segmentId) const
+    {
+        return boost::apply_visitor(SegmentHandleFromAddress{ptr}, fSegments.at(segmentId));
+    }
+    void* GetAddressFromHandle(const boost::interprocess::managed_shared_memory::handle_t handle, uint16_t segmentId) const
+    {
+        return boost::apply_visitor(SegmentAddressFromHandle{handle}, fSegments.at(segmentId));
+    }
+
+    char* Allocate(const size_t size, size_t alignment = 0)
+    {
+        char* ptr = nullptr;
+        // tools::RateLimiter rateLimiter(20);
+
+        while (ptr == nullptr) {
+            try {
+                // boost::interprocess::managed_shared_memory::size_type actualSize = size;
+                // char* hint = 0; // unused for boost::interprocess::allocate_new
+                // ptr = fSegments.at(fSegmentId).allocation_command<char>(boost::interprocess::allocate_new, size, actualSize, hint);
+                size_t segmentSize = boost::apply_visitor(SegmentSize{}, fSegments.at(fSegmentId));
+                if (size > segmentSize) {
+                    throw MessageBadAlloc(tools::ToString("Requested message size (", size, ") exceeds segment size (", segmentSize, ")"));
+                }
+                if (alignment == 0) {
+                    ptr = reinterpret_cast<char*>(boost::apply_visitor(SegmentAllocate{size}, fSegments.at(fSegmentId)));
+                } else {
+                    ptr = reinterpret_cast<char*>(boost::apply_visitor(SegmentAllocateAligned{size, alignment}, fSegments.at(fSegmentId)));
+                }
+            } catch (boost::interprocess::bad_alloc& ba) {
+                // LOG(warn) << "Shared memory full...";
+                if (ThrowingOnBadAlloc()) {
+                    throw MessageBadAlloc(tools::ToString("shmem: could not create a message of size ", size, ", alignment: ", (alignment != 0) ? std::to_string(alignment) : "default", ", free memory: ", boost::apply_visitor(SegmentFreeMemory{}, fSegments.at(fSegmentId))));
+                }
+                // rateLimiter.maybe_sleep();
+                std::this_thread::sleep_for(std::chrono::milliseconds(50));
+                if (Interrupted()) {
+                    return ptr;
+                } else {
+                    continue;
+                }
+            }
+#ifdef FAIRMQ_DEBUG_MODE
+            boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+            IncrementShmMsgCounter(fSegmentId);
+            if (fMsgDebug->count(fSegmentId) == 0) {
+                (*fMsgDebug).emplace(fSegmentId, fShmVoidAlloc);
+            }
+            (*fMsgDebug).at(fSegmentId).emplace(
+                static_cast<size_t>(GetHandleFromAddress(ptr, fSegmentId)),
+                MsgDebug(getpid(), size, std::chrono::system_clock::now().time_since_epoch().count())
+            );
+#endif
+        }
+
+        return ptr;
+    }
+
+    void Deallocate(boost::interprocess::managed_shared_memory::handle_t handle, uint16_t segmentId)
+    {
+        boost::apply_visitor(SegmentDeallocate{GetAddressFromHandle(handle, segmentId)}, fSegments.at(segmentId));
+#ifdef FAIRMQ_DEBUG_MODE
+        boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+        DecrementShmMsgCounter(segmentId);
+        try {
+            (*fMsgDebug).at(segmentId).erase(handle);
+        } catch(const std::out_of_range& oor) {
+            LOG(debug) << "could not locate debug container for " << segmentId << ": " << oor.what();
+        }
+#endif
+    }
+
+    char* ShrinkInPlace(size_t oldSize, size_t newSize, char* localPtr, uint16_t segmentId)
+    {
+        return boost::apply_visitor(SegmentBufferShrink{oldSize, newSize, localPtr}, fSegments.at(segmentId));
+    }
+
+    uint16_t GetSegmentId() const { return fSegmentId; }
+
+    ~Manager()
+    {
+        using namespace boost::interprocess;
+        bool lastRemoved = false;
+
+        UnsubscribeFromRegionEvents();
+
+        {
+            std::unique_lock<std::mutex> lock(fHeartbeatsMtx);
+            fSendHeartbeats = false;
+        }
+        fHeartbeatsCV.notify_one();
+        if (fHeartbeatThread.joinable()) {
+            fHeartbeatThread.join();
+        }
+
+        try {
+            boost::interprocess::scoped_lock<named_mutex> lock(fShmMtx);
+
+            (fDeviceCounter->fCount)--;
+
+            if (fDeviceCounter->fCount == 0) {
+                LOG(debug) << "Last segment user, removing segment.";
+                lastRemoved = true;
+            } else {
+                LOG(debug) << "Other segment users present (" << fDeviceCounter->fCount << "), skipping removal.";
+            }
+        } catch (interprocess_exception& e) {
+            LOG(error) << "Manager could not acquire lock: " << e.what();
+        }
+
+        if (lastRemoved) {
+            Monitor::Cleanup(ShmId{fShmId});
+        }
+    }
+
   private:
     std::string fShmId;
+    uint16_t fSegmentId;
     std::string fDeviceId;
-    std::string fSegmentName;
-    std::string fManagementSegmentName;
-    boost::interprocess::managed_shared_memory fSegment;
+    std::unordered_map<uint16_t, boost::variant<RBTreeBestFitSegment, SimpleSeqFitSegment>> fSegments;
     boost::interprocess::managed_shared_memory fManagementSegment;
     VoidAlloc fShmVoidAlloc;
     boost::interprocess::named_mutex fShmMtx;
@@ -432,17 +621,25 @@ class Manager
     std::thread fRegionEventThread;
     bool fRegionEventsSubscriptionActive;
     std::function<void(fair::mq::RegionInfo)> fRegionEventCallback;
-    std::unordered_map<uint64_t, RegionEvent> fObservedRegionEvents;
+    std::unordered_map<uint16_t, RegionEvent> fObservedRegionEvents;
 
     DeviceCounter* fDeviceCounter;
-    Uint64RegionInfoMap* fRegionInfos;
-    std::unordered_map<uint64_t, std::unique_ptr<Region>> fRegions;
+    Uint16SegmentInfoHashMap* fShmSegments;
+    Uint16RegionInfoHashMap* fShmRegions;
+    std::unordered_map<uint16_t, std::unique_ptr<Region>> fRegions;
 
     std::atomic<bool> fInterrupted;
     std::atomic<int32_t> fMsgCounter; // TODO: find a better lifetime solution instead of the counter
+#ifdef FAIRMQ_DEBUG_MODE
+    Uint16MsgDebugMapHashMap* fMsgDebug;
+    Uint16MsgCounterHashMap* fShmMsgCounters;
+#endif
 
     std::thread fHeartbeatThread;
-    std::atomic<bool> fSendHeartbeats;
+    bool fSendHeartbeats;
+    std::mutex fHeartbeatsMtx;
+    std::condition_variable fHeartbeatsCV;
+
     bool fThrowOnBadAlloc;
 };
 

fairmq/shmem/Message.h

@@ -22,6 +22,9 @@
 #include <cstddef> // size_t
 #include <atomic>
 
+#include <sys/types.h> // getpid
+#include <unistd.h> // pid_t
+
 namespace fair
 {
 namespace mq
@@ -40,7 +43,7 @@ class Message final : public fair::mq::Message
         : fair::mq::Message(factory)
         , fManager(manager)
         , fQueued(false)
-        , fMeta{0, 0, 0, -1}
+        , fMeta{0, 0, 0, fManager.GetSegmentId(), -1}
         , fRegionPtr(nullptr)
         , fLocalPtr(nullptr)
     {
@@ -51,7 +54,7 @@ class Message final : public fair::mq::Message
         : fair::mq::Message(factory)
         , fManager(manager)
         , fQueued(false)
-        , fMeta{0, 0, 0, -1}
+        , fMeta{0, 0, 0, fManager.GetSegmentId(), -1}
         , fRegionPtr(nullptr)
         , fLocalPtr(nullptr)
     {
@@ -62,7 +65,7 @@ class Message final : public fair::mq::Message
         : fair::mq::Message(factory)
         , fManager(manager)
         , fQueued(false)
-        , fMeta{0, 0, 0, -1}
+        , fMeta{0, 0, 0, fManager.GetSegmentId(), -1}
         , fRegionPtr(nullptr)
         , fLocalPtr(nullptr)
     {
@@ -74,7 +77,7 @@ class Message final : public fair::mq::Message
         : fair::mq::Message(factory)
         , fManager(manager)
         , fQueued(false)
-        , fMeta{0, 0, 0, -1}
+        , fMeta{0, 0, 0, fManager.GetSegmentId(), -1}
         , fRegionPtr(nullptr)
         , fLocalPtr(nullptr)
     {
@@ -86,7 +89,7 @@ class Message final : public fair::mq::Message
         : fair::mq::Message(factory)
         , fManager(manager)
         , fQueued(false)
-        , fMeta{0, 0, 0, -1}
+        , fMeta{0, 0, 0, fManager.GetSegmentId(), -1}
         , fRegionPtr(nullptr)
         , fLocalPtr(nullptr)
     {
@@ -105,7 +108,7 @@ class Message final : public fair::mq::Message
         : fair::mq::Message(factory)
         , fManager(manager)
         , fQueued(false)
-        , fMeta{size, static_cast<UnmanagedRegion*>(region.get())->fRegionId, reinterpret_cast<size_t>(hint), -1}
+        , fMeta{size, reinterpret_cast<size_t>(hint), static_cast<UnmanagedRegion*>(region.get())->fRegionId, fManager.GetSegmentId(), -1}
         , fRegionPtr(nullptr)
         , fLocalPtr(static_cast<char*>(data))
     {
@@ -166,7 +169,8 @@ class Message final : public fair::mq::Message
         if (!fLocalPtr) {
             if (fMeta.fRegionId == 0) {
                 if (fMeta.fSize > 0) {
-                    fLocalPtr = reinterpret_cast<char*>(fManager.Segment().get_address_from_handle(fMeta.fHandle));
+                    fManager.GetSegment(fMeta.fSegmentId);
+                    fLocalPtr = reinterpret_cast<char*>(fManager.GetAddressFromHandle(fMeta.fHandle, fMeta.fSegmentId));
                 } else {
                     fLocalPtr = nullptr;
                 }
@@ -186,15 +190,14 @@ class Message final : public fair::mq::Message
 
     size_t GetSize() const override { return fMeta.fSize; }
 
-    bool SetUsedSize(const size_t size) override
+    bool SetUsedSize(const size_t newSize) override
     {
-        if (size == fMeta.fSize) {
+        if (newSize == fMeta.fSize) {
             return true;
-        } else if (size <= fMeta.fSize) {
+        } else if (newSize <= fMeta.fSize) {
             try {
-                boost::interprocess::managed_shared_memory::size_type shrunkSize = size;
-                fLocalPtr = fManager.Segment().allocation_command<char>(boost::interprocess::shrink_in_place, fMeta.fSize + 128, shrunkSize, fLocalPtr);
-                fMeta.fSize = size;
+                fLocalPtr = fManager.ShrinkInPlace(fMeta.fSize, newSize, fLocalPtr, fMeta.fSegmentId);
+                fMeta.fSize = newSize;
                 return true;
             } catch (boost::interprocess::interprocess_exception& e) {
                 LOG(info) << "could not set used size: " << e.what();
@@ -242,44 +245,22 @@ class Message final : public fair::mq::Message
     mutable Region* fRegionPtr;
     mutable char* fLocalPtr;
 
-    bool InitializeChunk(const size_t size, size_t alignment = 0)
+    char* InitializeChunk(const size_t size, size_t alignment = 0)
     {
-        tools::RateLimiter rateLimiter(20);
-
-        while (fMeta.fHandle < 0) {
-            try {
-                // boost::interprocess::managed_shared_memory::size_type actualSize = size;
-                // char* hint = 0; // unused for boost::interprocess::allocate_new
-                // fLocalPtr = fManager.Segment().allocation_command<char>(boost::interprocess::allocate_new, size, actualSize, hint);
-                if (alignment == 0) {
-                    fLocalPtr = reinterpret_cast<char*>(fManager.Segment().allocate(size));
-                } else {
-                    fLocalPtr = reinterpret_cast<char*>(fManager.Segment().allocate_aligned(size, alignment));
-                }
-            } catch (boost::interprocess::bad_alloc& ba) {
-                // LOG(warn) << "Shared memory full...";
-                if (fManager.ThrowingOnBadAlloc()) {
-                    throw MessageBadAlloc(tools::ToString("shmem: could not create a message of size ", size, ", alignment: ", (alignment != 0) ? std::to_string(alignment) : "default"));
-                }
-                rateLimiter.maybe_sleep();
-                if (fManager.Interrupted()) {
-                    return false;
-                } else {
-                    continue;
-                }
-            }
-            fMeta.fHandle = fManager.Segment().get_handle_from_address(fLocalPtr);
+        fLocalPtr = fManager.Allocate(size, alignment);
+        if (fLocalPtr) {
+            fMeta.fHandle = fManager.GetHandleFromAddress(fLocalPtr, fMeta.fSegmentId);
+            fMeta.fSize = size;
         }
-
-        fMeta.fSize = size;
-        return true;
+        return fLocalPtr;
     }
 
     void CloseMessage()
     {
         if (fMeta.fHandle >= 0 && !fQueued) {
             if (fMeta.fRegionId == 0) {
-                fManager.Segment().deallocate(fManager.Segment().get_address_from_handle(fMeta.fHandle));
+                fManager.GetSegment(fMeta.fSegmentId);
+                fManager.Deallocate(fMeta.fHandle, fMeta.fSegmentId);
                 fMeta.fHandle = -1;
             } else {
                 if (!fRegionPtr) {
@@ -293,6 +274,8 @@ class Message final : public fair::mq::Message
                 }
             }
         }
+        fLocalPtr = nullptr;
+        fMeta.fSize = 0;
 
         fManager.DecrementMsgCounter();
     }

fairmq/shmem/Monitor.cxx

@@ -10,6 +10,7 @@
 #include "Common.h"
 
 #include <fairmq/Tools.h>
+#include <fairlogger/Logger.h>
 
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/file_mapping.hpp>
@@ -22,6 +23,11 @@
 #include <csignal>
 #include <iostream>
 #include <iomanip>
+#include <chrono>
+#include <ctime>
+#include <time.h>
+#include <iomanip>
+#include <sstream>
 
 #include <termios.h>
 #include <poll.h>
@@ -48,7 +54,7 @@ void signalHandler(int signal)
     gSignalStatus = signal;
 }
 
-Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, bool runAsDaemon, bool cleanOnExit)
+Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, unsigned int intervalInMS, bool runAsDaemon, bool cleanOnExit)
     : fSelfDestruct(selfDestruct)
     , fInteractive(interactive)
     , fViewOnly(viewOnly)
@@ -56,15 +62,15 @@ Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool
     , fSeenOnce(false)
     , fCleanOnExit(cleanOnExit)
     , fTimeoutInMS(timeoutInMS)
+    , fIntervalInMS(intervalInMS)
     , fShmId(shmId)
-    , fSegmentName("fmq_" + fShmId + "_main")
+    , fSegmentName("fmq_" + fShmId + "_m_0")
     , fManagementSegmentName("fmq_" + fShmId + "_mng")
     , fControlQueueName("fmq_" + fShmId + "_cq")
     , fTerminating(false)
     , fHeartbeatTriggered(false)
     , fLastHeartbeat(chrono::high_resolution_clock::now())
     , fSignalThread()
-    , fManagementSegment(bipc::open_or_create, fManagementSegmentName.c_str(), 65536)
     , fDeviceHeartbeats()
 {
     if (!fViewOnly) {
@@ -75,6 +81,10 @@ Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool
             throw DaemonPresent(tools::ToString("fairmq-shmmonitor for shared memory id ", fShmId, " already started or not properly exited."));
         }
     }
+
+    Logger::SetConsoleColor(false);
+    Logger::DefineVerbosity(Verbosity::user1, VerbositySpec::Make(VerbositySpec::Info::timestamp_us));
+    Logger::SetVerbosity(Verbosity::verylow);
 }
 
 void Monitor::CatchSignals()
@@ -111,7 +121,7 @@ void Monitor::Run()
         Interactive();
     } else {
         while (!fTerminating) {
-            this_thread::sleep_for(chrono::milliseconds(100));
+            this_thread::sleep_for(chrono::milliseconds(fIntervalInMS));
             CheckSegment();
         }
     }
@@ -180,11 +190,9 @@ void Monitor::Interactive()
 
     cout << endl;
     PrintHelp();
-    cout << endl;
-    PrintHeader();
 
     while (!fTerminating) {
-        if (poll(cinfd, 1, 100)) {
+        if (poll(cinfd, 1, fIntervalInMS)) {
             if (fTerminating || gSignalStatus != 0) {
                 break;
             }
@@ -196,14 +204,10 @@ void Monitor::Interactive()
                     cout << "\n[q] --> quitting." << endl;
                     fTerminating = true;
                     break;
-                case 'p':
-                    cout << "\n[p] --> active queues:" << endl;
-                    PrintQueues();
-                    break;
                 case 'x':
                     cout << "\n[x] --> closing shared memory:" << endl;
                     if (!fViewOnly) {
-                        Cleanup(fShmId);
+                        Cleanup(ShmId{fShmId});
                     } else {
                         cout << "cannot close because in view only mode" << endl;
                     }
@@ -216,6 +220,9 @@ void Monitor::Interactive()
                 case '\n':
                     cout << "\n[\\n] --> invalid input." << endl;
                     break;
+                case 'b':
+                    PrintDebugInfo(ShmId{fShmId});
+                    break;
                 default:
                     cout << "\n[" << c << "] --> invalid input." << endl;
                     break;
@@ -224,8 +231,6 @@ void Monitor::Interactive()
             if (fTerminating) {
                 break;
             }
-
-            PrintHeader();
         }
 
         if (fTerminating) {
@@ -242,45 +247,43 @@ void Monitor::Interactive()
 
 void Monitor::CheckSegment()
 {
-    char c = '#';
-
-    if (fInteractive) {
-        static uint64_t counter = 0;
-        int mod = counter++ % 5;
-        switch (mod) {
-            case 0:
-                c = '-';
-                break;
-            case 1:
-                c = '\\';
-                break;
-            case 2:
-                c = '|';
-                break;
-            case 3:
-                c = '-';
-                break;
-            case 4:
-                c = '/';
-                break;
-            default:
-                break;
-        }
-    }
+    using namespace boost::interprocess;
 
     try {
-        bipc::managed_shared_memory segment(bipc::open_only, fSegmentName.c_str());
-        bipc::managed_shared_memory managementSegment(bipc::open_only, fManagementSegmentName.c_str());
+        managed_shared_memory managementSegment(open_only, fManagementSegmentName.c_str());
+        VoidAlloc allocInstance(managementSegment.get_segment_manager());
+
+        Uint16SegmentInfoHashMap* segmentInfos = managementSegment.find<Uint16SegmentInfoHashMap>(unique_instance).first;
+        std::unordered_map<uint16_t, boost::variant<RBTreeBestFitSegment, SimpleSeqFitSegment>> segments;
+
+        if (!segmentInfos) {
+            cout << "Found management segment, but cannot locate segment info, something went wrong..." << endl;
+            return;
+        }
+
+        for (const auto& s : *segmentInfos) {
+            if (s.second.fAllocationAlgorithm == AllocationAlgorithm::rbtree_best_fit) {
+                segments.emplace(s.first, RBTreeBestFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + to_string(s.first)).c_str()));
+            } else {
+                segments.emplace(s.first, SimpleSeqFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + to_string(s.first)).c_str()));
+            }
+        }
 
         fSeenOnce = true;
 
         unsigned int numDevices = 0;
+#ifdef FAIRMQ_DEBUG_MODE
+        Uint16MsgCounterHashMap* msgCounters = nullptr;
+#endif
 
-        if (fInteractive) {
-            DeviceCounter* dc = managementSegment.find<DeviceCounter>(bipc::unique_instance).first;
+        if (fInteractive || fViewOnly) {
+            DeviceCounter* dc = managementSegment.find<DeviceCounter>(unique_instance).first;
             if (dc) {
                 numDevices = dc->fCount;
             }
+#ifdef FAIRMQ_DEBUG_MODE
+            msgCounters = managementSegment.find_or_construct<Uint16MsgCounterHashMap>(unique_instance)(allocInstance);
+#endif
         }
 
         auto now = chrono::high_resolution_clock::now();
@@ -288,7 +291,7 @@ void Monitor::CheckSegment()
 
         if (fHeartbeatTriggered && duration > fTimeoutInMS) {
             cout << "no heartbeats since over " << fTimeoutInMS << " milliseconds, cleaning..." << endl;
-            Cleanup(fShmId);
+            Cleanup(ShmId{fShmId});
             fHeartbeatTriggered = false;
             if (fSelfDestruct) {
                 cout << "\nself destructing" << endl;
@@ -296,32 +299,43 @@ void Monitor::CheckSegment()
             }
         }
 
-        if (fInteractive) {
-            cout << "| "
-                 << setw(18) << fSegmentName                                    << " | "
-                 << setw(10) << segment.get_size()                              << " | "
-                 << setw(10) << segment.get_free_memory()                       << " | "
-                 << setw(8)  << numDevices                                      << " | "
-                 << setw(10) << (fViewOnly ? "view only" : to_string(duration)) << " |"
-                 << c << flush;
+        if (fInteractive || fViewOnly) {
+            stringstream ss;
+            size_t mfree = managementSegment.get_free_memory();
+            size_t mtotal = managementSegment.get_size();
+            size_t mused = mtotal - mfree;
+
+            ss << "shm id: " << fShmId
+               << ", devices: " << numDevices << ", segments:\n";
+            for (const auto& s : segments) {
+                size_t free = boost::apply_visitor(SegmentFreeMemory{}, s.second);
+                size_t total = boost::apply_visitor(SegmentSize{}, s.second);
+                size_t used = total - free;
+                ss << "   [" << s.first
+                   << "]: total: " << total
+#ifdef FAIRMQ_DEBUG_MODE
+                   << ", msgs: " << (*msgCounters)[s.first].fCount
+#else
+                   << ", msgs: NODEBUG"
+#endif
+                   << ", free: " << free
+                   << ", used: " << used
+                   << "\n";
+            }
+            ss << "   [m]: "
+               << "total: " << mtotal
+               << ", free: " << mfree
+               << ", used: " << mused;
+            LOGV(info, user1) << ss.str();
         }
     } catch (bie&) {
         fHeartbeatTriggered = false;
-        if (fInteractive) {
-            cout << "| "
-                 << setw(18) << "-" << " | "
-                 << setw(10) << "-" << " | "
-                 << setw(10) << "-" << " | "
-                 << setw(8)  << "-" << " | "
-                 << setw(10) << "-" << " |"
-                 << c << flush;
-        }
 
         auto now = chrono::high_resolution_clock::now();
         unsigned int duration = chrono::duration_cast<chrono::milliseconds>(now - fLastHeartbeat).count();
 
         if (fIsDaemon && duration > fTimeoutInMS * 2) {
-            Cleanup(fShmId);
+            Cleanup(ShmId{fShmId});
             fHeartbeatTriggered = false;
             if (fSelfDestruct) {
                 cout << "\nself destructing" << endl;
@@ -338,150 +352,214 @@ void Monitor::CheckSegment()
     }
 }
 
-void Monitor::PrintQueues()
+void Monitor::PrintDebugInfo(const ShmId& shmId __attribute__((unused)))
 {
-    cout << '\n';
-
+#ifdef FAIRMQ_DEBUG_MODE
+    string managementSegmentName("fmq_" + shmId.shmId + "_mng");
     try {
-        bipc::managed_shared_memory segment(bipc::open_only, fSegmentName.c_str());
-        StrVector* queues = segment.find<StrVector>(string("fmq_" + fShmId + "_qs").c_str()).first;
-        if (queues) {
-            cout << "found " << queues->size() << " queue(s):" << endl;
+        bipc::managed_shared_memory managementSegment(bipc::open_only, managementSegmentName.c_str());
+        boost::interprocess::named_mutex mtx(boost::interprocess::open_only, string("fmq_" + shmId.shmId + "_mtx").c_str());
+        boost::interprocess::scoped_lock<bipc::named_mutex> lock(mtx);
 
-            for (const auto& queue : *queues) {
-                string name(queue.c_str());
-                cout << '\t' << name << " : ";
-                atomic<int>* queueSize = segment.find<atomic<int>>(name.c_str()).first;
-                if (queueSize) {
-                    cout << *queueSize << " messages" << endl;
-                } else {
-                    cout << "\tqueue does not have a queue size entry." << endl;
-                }
-            }
-        } else {
-            cout << "\tno queues found" << endl;
+        Uint16MsgDebugMapHashMap* debug = managementSegment.find<Uint16MsgDebugMapHashMap>(bipc::unique_instance).first;
+
+        size_t numMessages = 0;
+
+        for (const auto& e : *debug) {
+            numMessages += e.second.size();
         }
+        cout << endl << "found " << numMessages << " messages." << endl;
+
+        for (const auto& s : *debug) {
+            for (const auto& e : s.second) {
+                using time_point = chrono::system_clock::time_point;
+                time_point tmpt{chrono::duration_cast<time_point::duration>(chrono::nanoseconds(e.second.fCreationTime))};
+                time_t t = chrono::system_clock::to_time_t(tmpt);
+                uint64_t ms = e.second.fCreationTime % 1000000;
+                auto tm = localtime(&t);
+                cout << "segment: " << setw(3) << setfill(' ') << s.first
+                     << ", offset: " << setw(12) << setfill(' ') << e.first
+                     << ", size: " << setw(10) << setfill(' ') << e.second.fSize
+                     << ", creator PID: " << e.second.fPid << setfill('0')
+                     << ", at: " << setw(2) << tm->tm_hour << ":" << setw(2) << tm->tm_min << ":" << setw(2) << tm->tm_sec << "." << setw(6) << ms << endl;
+            }
+        }
+        cout << setfill(' ');
     } catch (bie&) {
-        cout << "\tno queues found" << endl;
-    } catch (out_of_range&) {
-        cout << "\tno queues found" << endl;
+        cout << "no segments found" << endl;
     }
-
-    cout << "\n    --> last heartbeats: " << endl << endl;
-    auto now = chrono::high_resolution_clock::now();
-    for (const auto& h : fDeviceHeartbeats)  {
-        cout << "\t" << h.first << " : " << chrono::duration<double, milli>(now - h.second).count() << "ms ago." << endl;
-    }
-
-    cout << endl;
+#else
+    cout << "FairMQ was not compiled in debug mode (FAIRMQ_DEBUG_MODE)" << endl;
+#endif
 }
 
-void Monitor::PrintHeader()
+void Monitor::PrintDebugInfo(const SessionId& sessionId)
 {
-    cout << "| "
-         << setw(18) << "name"    << " | "
-         << setw(10) << "size"    << " | "
-         << setw(10) << "free"    << " | "
-         << setw(8)  << "devices" << " | "
-         << setw(10) << "last hb" << " |"
-         << endl;
+    ShmId shmId{buildShmIdFromSessionIdAndUserId(sessionId.sessionId)};
+    PrintDebugInfo(shmId);
+}
+
+unordered_map<uint16_t, std::vector<BufferDebugInfo>> Monitor::GetDebugInfo(const ShmId& shmId __attribute__((unused)))
+{
+    unordered_map<uint16_t, std::vector<BufferDebugInfo>> result;
+
+#ifdef FAIRMQ_DEBUG_MODE
+    string managementSegmentName("fmq_" + shmId.shmId + "_mng");
+    try {
+        bipc::managed_shared_memory managementSegment(bipc::open_only, managementSegmentName.c_str());
+        boost::interprocess::named_mutex mtx(boost::interprocess::open_only, string("fmq_" + shmId.shmId + "_mtx").c_str());
+        boost::interprocess::scoped_lock<bipc::named_mutex> lock(mtx);
+
+        Uint16MsgDebugMapHashMap* debug = managementSegment.find<Uint16MsgDebugMapHashMap>(bipc::unique_instance).first;
+
+        result.reserve(debug->size());
+
+        for (const auto& s : *debug) {
+            result[s.first].reserve(s.second.size());
+            for (const auto& e : s.second) {
+                result[s.first][e.first] = BufferDebugInfo(e.first, e.second.fPid, e.second.fSize, e.second.fCreationTime);
+            }
+        }
+    } catch (bie&) {
+        cout << "no segments found" << endl;
+    }
+#else
+    cout << "FairMQ was not compiled in debug mode (FAIRMQ_DEBUG_MODE)" << endl;
+#endif
+
+    return result;
+}
+unordered_map<uint16_t, std::vector<BufferDebugInfo>> Monitor::GetDebugInfo(const SessionId& sessionId)
+{
+    ShmId shmId{buildShmIdFromSessionIdAndUserId(sessionId.sessionId)};
+    return GetDebugInfo(shmId);
 }
 
 void Monitor::PrintHelp()
 {
-    cout << "controls: [x] close memory, [p] print queues, [h] help, [q] quit." << endl;
+    cout << "controls: [x] close memory, "
+         << "[b] print a list of allocated messages (only available when compiled with FAIMQ_DEBUG_MODE=ON), "
+         << "[h] help, "
+         << "[q] quit." << endl;
 }
 
-void Monitor::RemoveObject(const string& name)
+
+std::pair<std::string, bool> RunRemoval(std::function<bool(const std::string&)> f, std::string name, bool verbose)
 {
-    if (bipc::shared_memory_object::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
+    if (f(name)) {
+        if (verbose) {
+            cout << "Successfully removed '" << name << "'." << endl;
+        }
+        return {name, true};
     } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
+        if (verbose) {
+            cout << "Did not remove '" << name << "'. Already removed?" << endl;
+        }
+        return {name, false};
     }
 }
 
-void Monitor::RemoveFileMapping(const string& name)
-{
-    if (bipc::file_mapping::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
-    } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
-    }
-}
+bool Monitor::RemoveObject(const string& name)      { return bipc::shared_memory_object::remove(name.c_str()); }
+bool Monitor::RemoveFileMapping(const string& name) { return bipc::file_mapping::remove(name.c_str()); }
+bool Monitor::RemoveQueue(const string& name)       { return bipc::message_queue::remove(name.c_str()); }
+bool Monitor::RemoveMutex(const string& name)       { return bipc::named_mutex::remove(name.c_str()); }
+bool Monitor::RemoveCondition(const string& name)   { return bipc::named_condition::remove(name.c_str()); }
 
-void Monitor::RemoveQueue(const string& name)
+std::vector<std::pair<std::string, bool>> Monitor::Cleanup(const ShmId& shmId, bool verbose /* = true */)
 {
-    if (bipc::message_queue::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
-    } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
-    }
-}
+    std::vector<std::pair<std::string, bool>> result;
 
-void Monitor::RemoveMutex(const string& name)
-{
-    if (bipc::named_mutex::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
-    } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
+    if (verbose) {
+        cout << "Cleaning up for shared memory id '" << shmId.shmId << "'..." << endl;
     }
-}
 
-void Monitor::RemoveCondition(const string& name)
-{
-    if (bipc::named_condition::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
-    } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
-    }
-}
-
-void Monitor::Cleanup(const string& shmId)
-{
-    string managementSegmentName("fmq_" + shmId + "_mng");
+    string managementSegmentName("fmq_" + shmId.shmId + "_mng");
     try {
         bipc::managed_shared_memory managementSegment(bipc::open_only, managementSegmentName.c_str());
-        RegionCounter* rc = managementSegment.find<RegionCounter>(bipc::unique_instance).first;
-        if (rc) {
-            cout << "Region counter found: " << rc->fCount << endl;
-            uint64_t regionCount = rc->fCount;
 
-            Uint64RegionInfoMap* m = managementSegment.find<Uint64RegionInfoMap>(bipc::unique_instance).first;
-
-            for (uint64_t i = 1; i <= regionCount; ++i) {
-                if (m != nullptr) {
-                    RegionInfo ri = m->at(i);
-                    string path = ri.fPath.c_str();
-                    int flags = ri.fFlags;
-                    cout << "Found RegionInfo with path: '" << path << "', flags: " << flags << "'." << endl;
-                    if (path != "") {
-                        RemoveFileMapping(tools::ToString(path, "fmq_" + shmId + "_rg_" + to_string(i)));
-                    } else {
-                        RemoveObject("fmq_" + shmId + "_rg_" + to_string(i));
-                    }
-                } else {
-                    RemoveObject("fmq_" + shmId + "_rg_" + to_string(i));
+        try {
+            RegionCounter* rc = managementSegment.find<RegionCounter>(bipc::unique_instance).first;
+            if (rc) {
+                if (verbose) {
+                    cout << "Region counter found: " << rc->fCount << endl;
                 }
+                uint16_t regionCount = rc->fCount;
 
-                RemoveQueue(string("fmq_" + shmId + "_rgq_" + to_string(i)));
+                Uint16RegionInfoMap* m = managementSegment.find<Uint16RegionInfoMap>(bipc::unique_instance).first;
+
+                for (uint16_t i = 1; i <= regionCount; ++i) {
+                    if (m != nullptr) {
+                        RegionInfo ri = m->at(i);
+                        string path = ri.fPath.c_str();
+                        int flags = ri.fFlags;
+                        if (verbose) {
+                            cout << "Found RegionInfo with path: '" << path << "', flags: " << flags << ", fDestroyed: " << ri.fDestroyed << "." << endl;
+                        }
+                        if (path != "") {
+                            result.emplace_back(RunRemoval(Monitor::RemoveFileMapping, path + "fmq_" + shmId.shmId + "_rg_" + to_string(i), verbose));
+                        } else {
+                            result.emplace_back(RunRemoval(Monitor::RemoveObject, "fmq_" + shmId.shmId + "_rg_" + to_string(i), verbose));
+                        }
+                    } else {
+                        result.emplace_back(RunRemoval(Monitor::RemoveObject, "fmq_" + shmId.shmId + "_rg_" + to_string(i), verbose));
+                    }
+
+                    result.emplace_back(RunRemoval(Monitor::RemoveQueue, string("fmq_" + shmId.shmId + "_rgq_" + to_string(i)), verbose));
+                }
+            } else {
+                if (verbose) {
+                    cout << "No region counter found. No regions to cleanup." << endl;
+                }
+            }
+        } catch(out_of_range& oor) {
+            if (verbose) {
+                cout << "Could not locate element in the region map, out of range: " << oor.what() << endl;
             }
-        } else {
-            cout << "No region counter found. no regions to cleanup." << endl;
         }
 
-        RemoveObject(managementSegmentName.c_str());
+        Uint16SegmentInfoHashMap* segmentInfos = managementSegment.find<Uint16SegmentInfoHashMap>(bipc::unique_instance).first;
+
+        for (const auto& s : *segmentInfos) {
+            result.emplace_back(RunRemoval(Monitor::RemoveObject, "fmq_" + shmId.shmId + "_m_" + to_string(s.first), verbose));
+        }
+
+        result.emplace_back(RunRemoval(Monitor::RemoveObject, managementSegmentName.c_str(), verbose));
     } catch (bie&) {
-        cout << "Did not find '" << managementSegmentName << "' shared memory segment. No regions to cleanup." << endl;
-    } catch(std::out_of_range& oor) {
-        cout << "Could not locate element in the region map, out of range: " << oor.what() << endl;
+        if (verbose) {
+            cout << "Did not find '" << managementSegmentName << "' shared memory segment. No regions to cleanup." << endl;
+        }
     }
 
-    RemoveObject("fmq_" + shmId + "_main");
-    RemoveMutex("fmq_" + shmId + "_mtx");
-    RemoveCondition("fmq_" + shmId + "_cv");
+    result.emplace_back(RunRemoval(Monitor::RemoveMutex, "fmq_" + shmId.shmId + "_mtx", verbose));
+    result.emplace_back(RunRemoval(Monitor::RemoveCondition, "fmq_" + shmId.shmId + "_cv", verbose));
 
-    cout << endl;
+    return result;
+}
+
+std::vector<std::pair<std::string, bool>> Monitor::Cleanup(const SessionId& sessionId, bool verbose /* = true */)
+{
+    ShmId shmId{buildShmIdFromSessionIdAndUserId(sessionId.sessionId)};
+    if (verbose) {
+        cout << "Cleanup called with session id '" << sessionId.sessionId << "', translating to shared memory id '" << shmId.shmId << "'" << endl;
+    }
+    return Cleanup(shmId, verbose);
+}
+
+std::vector<std::pair<std::string, bool>> Monitor::CleanupFull(const ShmId& shmId, bool verbose /* = true */)
+{
+    auto result = Cleanup(shmId, verbose);
+    result.emplace_back(RunRemoval(Monitor::RemoveMutex, "fmq_" + shmId.shmId + "_ms", verbose));
+    result.emplace_back(RunRemoval(Monitor::RemoveQueue, "fmq_" + shmId.shmId + "_cq", verbose));
+    return result;
+}
+
+std::vector<std::pair<std::string, bool>> Monitor::CleanupFull(const SessionId& sessionId, bool verbose /* = true */)
+{
+    ShmId shmId{buildShmIdFromSessionIdAndUserId(sessionId.sessionId)};
+    if (verbose) {
+        cout << "Cleanup called with session id '" << sessionId.sessionId << "', translating to shared memory id '" << shmId.shmId << "'" << endl;
+    }
+    return CleanupFull(shmId, verbose);
 }
 
 Monitor::~Monitor()
@@ -490,7 +568,7 @@ Monitor::~Monitor()
         fSignalThread.join();
     }
     if (fCleanOnExit) {
-        Cleanup(fShmId);
+        Cleanup(ShmId{fShmId});
     }
     if (!fViewOnly) {
         RemoveMutex("fmq_" + fShmId + "_ms");

fairmq/shmem/Monitor.h

@@ -8,14 +8,14 @@
 #ifndef FAIR_MQ_SHMEM_MONITOR_H_
 #define FAIR_MQ_SHMEM_MONITOR_H_
 
-#include <boost/interprocess/managed_shared_memory.hpp>
-
 #include <thread>
 #include <chrono>
 #include <atomic>
 #include <string>
 #include <stdexcept>
 #include <unordered_map>
+#include <utility> // pair
+#include <vector>
 
 namespace fair
 {
@@ -24,10 +24,37 @@ namespace mq
 namespace shmem
 {
 
+struct SessionId
+{
+    std::string sessionId;
+    explicit operator std::string() const { return sessionId; }
+};
+
+struct ShmId
+{
+    std::string shmId;
+    explicit operator std::string() const { return shmId; }
+};
+
+struct BufferDebugInfo
+{
+    BufferDebugInfo(size_t offset, pid_t pid, size_t size, uint64_t creationTime)
+        : fOffset(offset)
+        , fPid(pid)
+        , fSize(size)
+        , fCreationTime(creationTime)
+    {}
+
+    size_t fOffset;
+    pid_t fPid;
+    size_t fSize;
+    uint64_t fCreationTime;
+};
+
 class Monitor
 {
   public:
-    Monitor(const std::string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, bool runAsDaemon, bool cleanOnExit);
+    Monitor(const std::string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, unsigned int intervalInMS, bool runAsDaemon, bool cleanOnExit);
 
     Monitor(const Monitor&) = delete;
     Monitor operator=(const Monitor&) = delete;
@@ -37,19 +64,38 @@ class Monitor
     void CatchSignals();
     void Run();
 
-    static void Cleanup(const std::string& shmId);
-    static void RemoveObject(const std::string&);
-    static void RemoveFileMapping(const std::string&);
-    static void RemoveQueue(const std::string&);
-    static void RemoveMutex(const std::string&);
-    static void RemoveCondition(const std::string&);
+    /// @brief Cleanup all shared memory artifacts created by devices
+    /// @param shmId shared memory id
+    /// @param verbose output cleanup results to stdout
+    static std::vector<std::pair<std::string, bool>> Cleanup(const ShmId& shmId, bool verbose = true);
+    /// @brief Cleanup all shared memory artifacts created by devices
+    /// @param sessionId session id
+    /// @param verbose output cleanup results to stdout
+    static std::vector<std::pair<std::string, bool>> Cleanup(const SessionId& sessionId, bool verbose = true);
+    /// @brief Cleanup all shared memory artifacts created by devices and monitors
+    /// @param shmId shared memory id
+    /// @param verbose output cleanup results to stdout
+    static std::vector<std::pair<std::string, bool>> CleanupFull(const ShmId& shmId, bool verbose = true);
+    /// @brief Cleanup all shared memory artifacts created by devices and monitors
+    /// @param sessionId session id
+    /// @param verbose output cleanup results to stdout
+    static std::vector<std::pair<std::string, bool>> CleanupFull(const SessionId& sessionId, bool verbose = true);
+
+    static void PrintDebugInfo(const ShmId& shmId);
+    static void PrintDebugInfo(const SessionId& shmId);
+    static std::unordered_map<uint16_t, std::vector<BufferDebugInfo>> GetDebugInfo(const ShmId& shmId);
+    static std::unordered_map<uint16_t, std::vector<BufferDebugInfo>> GetDebugInfo(const SessionId& shmId);
+
+    static bool RemoveObject(const std::string& name);
+    static bool RemoveFileMapping(const std::string& name);
+    static bool RemoveQueue(const std::string& name);
+    static bool RemoveMutex(const std::string& name);
+    static bool RemoveCondition(const std::string& name);
 
     struct DaemonPresent : std::runtime_error { using std::runtime_error::runtime_error; };
 
   private:
-    void PrintHeader();
     void PrintHelp();
-    void PrintQueues();
     void MonitorHeartbeats();
     void CheckSegment();
     void Interactive();
@@ -62,6 +108,7 @@ class Monitor
     bool fSeenOnce; // true is segment has been opened successfully at least once
     bool fCleanOnExit;
     unsigned int fTimeoutInMS;
+    unsigned int fIntervalInMS;
     std::string fShmId;
     std::string fSegmentName;
     std::string fManagementSegmentName;
@@ -70,7 +117,6 @@ class Monitor
     std::atomic<bool> fHeartbeatTriggered;
     std::chrono::high_resolution_clock::time_point fLastHeartbeat;
     std::thread fSignalThread;
-    boost::interprocess::managed_shared_memory fManagementSegment;
     std::unordered_map<std::string, std::chrono::high_resolution_clock::time_point> fDeviceHeartbeats;
 };
 

fairmq/shmem/Poller.h

@@ -127,13 +127,20 @@ class Poller final : public fair::mq::Poller
 
     void Poll(const int timeout) override
     {
-        if (zmq_poll(fItems, fNumItems, timeout) < 0) {
-            if (errno == ETERM) {
-                LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
-            } else {
-                LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
-                throw fair::mq::PollerError(fair::mq::tools::ToString("Polling failed, reason: ", zmq_strerror(errno)));
+        while (true) {
+            if (zmq_poll(fItems, fNumItems, timeout) < 0) {
+                if (errno == ETERM) {
+                    LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
+                    return;
+                } else if (errno == EINTR) {
+                    LOG(debug) << "polling interrupted by system call";
+                    continue;
+                } else {
+                    LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
+                    throw fair::mq::PollerError(fair::mq::tools::ToString("Polling failed, reason: ", zmq_strerror(errno)));
+                }
             }
+            break;
         }
     }
 

fairmq/shmem/README.md

@@ -6,6 +6,23 @@ The transport manages shared memory via boost::interprocess library. The transfe
 
 Devices track and cleanup shared memory on shutdown. For more information on the current shared memory segment and additional cleanup options, see following section.
 
+# Shared Memory objects / files
+
+FairMQ Shared Memory currently uses the following names to register shared memory on the system:
+
+| name                        | info                                           | created by         | used by                        |
+| --------------------------- | ---------------------------------------------- | ------------------ | ------------------------------ |
+| `fmq_<shmId>_m_<segmentId>` | managed segment(s) (user data)                 | one of the devices | devices                        |
+| `fmq_<shmId>_mng`           | management segment (management data)           | one of the devices | devices                        |
+| `fmq_<shmId>_mtx`           | mutex                                          | one of the devices | devices                        |
+| `fmq_<shmId>_cv`            | condition variable                             | one of the devices | devices with unmanaged regions |
+| `fmq_<shmId>_rg_<index>`    | unmanaged region(s)                            | one of the devices | devices with unmanaged regions |
+| `fmq_<shmId>_rgq_<index>`   | unmanaged region queue(s)                      | one of the devices | devices with unmanaged regions |
+| `fmq_<shmId>_ms`            | shmmonitor status                              | shmmonitor         | devices, shmmonitor            |
+| `fmq_<shmId>_cq`            | message queue between transport and shmmonitor | shmmonitor         | devices, shmmonitor            |
+
+The shmId is generated out of session id and user id.
+
 ## Shared memory monitor
 
 The shared memory monitor tool, supplied with the shared memory transport can be used to monitor shared memory use and automatically cleanup shared memory in case of device crashes.
@@ -25,13 +42,3 @@ Without the `--self-destruct` option, the monitor will run continuously, moitori
 Possible further implementation would be to run the monitor with `--self-destruct` with each topology.
 
 The Monitor class can also be used independently from the supplied executable (built from `runMonitor.cxx`), allowing integration on any level. For example invoking the monitor could be a functionality that a device offers.
-
-FairMQ Shared Memory currently uses following names to register shared memory on the system:
-
-`fmq_<shmId>_main` - main segment name, used for user data (the shmId is generated out of session id and user id).
-`fmq_<shmId>_mng` - management segment name, used for storing management data.
-`fmq_<shmId>_cq` - message queue for communicating between shm transport and shm monitor (exists independent of above segments).
-`fmq_<shmId>_mtx` - boost::interprocess::named_mutex for management purposes (exists independent of above segments).
-`fmq_<shmId>_ms` - shmmonitor status used to signal if it is active or not (exists independent of above segments).
-`fmq_<shmId>_rg_<index>` - names of unmanaged regions.
-`fmq_<shmId>_rgq_<index>` - names of queues for the unmanaged regions.

fairmq/shmem/Region.h

@@ -30,6 +30,7 @@
 #include <boost/interprocess/ipc/message_queue.hpp>
 
 #include <algorithm> // min
+#include <atomic>
 #include <thread>
 #include <mutex>
 #include <condition_variable>
@@ -47,8 +48,9 @@ namespace shmem
 
 struct Region
 {
-    Region(const std::string& shmId, uint64_t id, uint64_t size, bool remote, RegionCallback callback, RegionBulkCallback bulkCallback, const std::string& path, int flags)
+    Region(const std::string& shmId, uint16_t id, uint64_t size, bool remote, RegionCallback callback, RegionBulkCallback bulkCallback, const std::string& path, int flags)
         : fRemote(remote)
+        , fLinger(100)
         , fStop(false)
         , fName("fmq_" + shmId + "_rg_" + std::to_string(id))
         , fQueueName("fmq_" + shmId + "_rgq_" + std::to_string(id))
@@ -56,8 +58,8 @@ struct Region
         , fFile(nullptr)
         , fFileMapping()
         , fQueue(nullptr)
-        , fReceiveAcksWorker()
-        , fSendAcksWorker()
+        , fAcksReceiver()
+        , fAcksSender()
         , fCallback(callback)
         , fBulkCallback(bulkCallback)
     {
@@ -118,38 +120,35 @@ struct Region
         LOG(debug) << "shmem: initialized region queue: " << fQueueName;
     }
 
-    void StartSendingAcks()
-    {
-        fSendAcksWorker = std::thread(&Region::SendAcks, this);
-    }
-
+    void StartSendingAcks() { fAcksSender = std::thread(&Region::SendAcks, this); }
     void SendAcks()
     {
         std::unique_ptr<RegionBlock[]> blocks = tools::make_unique<RegionBlock[]>(fAckBunchSize);
+        size_t blocksToSend = 0;
 
-        while (true) { // we'll try to send all acks before stopping
-            size_t blocksToSend = 0;
-
-            {   // mutex locking block
+        while (true) {
+            blocksToSend = 0;
+            {
                 std::unique_lock<std::mutex> lock(fBlockMtx);
 
-                // try to get more blocks without waiting (we can miss a notify from CloseMessage())
-                if (!fStop && (fBlocksToFree.size() < fAckBunchSize)) {
-                    // cv.wait() timeout: send whatever blocks we have
+                // try to get <fAckBunchSize> blocks
+                if (fBlocksToFree.size() < fAckBunchSize) {
                     fBlockSendCV.wait_for(lock, std::chrono::milliseconds(500));
                 }
 
+                // send whatever blocks we have
                 blocksToSend = std::min(fBlocksToFree.size(), fAckBunchSize);
 
                 copy_n(fBlocksToFree.end() - blocksToSend, blocksToSend, blocks.get());
                 fBlocksToFree.resize(fBlocksToFree.size() - blocksToSend);
-            } // unlock the block mutex here while sending over IPC
+            }
 
             if (blocksToSend > 0) {
                 while (!fQueue->try_send(blocks.get(), blocksToSend * sizeof(RegionBlock), 0) && !fStop) {
                     // receiver slow? yield and try again...
                     std::this_thread::yield();
                 }
+                // LOG(debug) << "Sent " << blocksToSend << " blocks.";
             } else { // blocksToSend == 0
                 if (fStop) {
                     break;
@@ -157,14 +156,11 @@ struct Region
             }
         }
 
-        LOG(debug) << "send ack worker for " << fName << " leaving.";
-    }
-
-    void StartReceivingAcks()
-    {
-        fReceiveAcksWorker = std::thread(&Region::ReceiveAcks, this);
+        LOG(debug) << "AcksSender for " << fName << " leaving " << "(blocks left to free: " << fBlocksToFree.size() << ", "
+                                                                << " blocks left to send: " << blocksToSend << ").";
     }
 
+    void StartReceivingAcks() { fAcksReceiver = std::thread(&Region::ReceiveAcks, this); }
     void ReceiveAcks()
     {
         unsigned int priority;
@@ -173,12 +169,18 @@ struct Region
         std::vector<fair::mq::RegionBlock> result;
         result.reserve(fAckBunchSize);
 
-        while (!fStop) { // end thread condition (should exist until region is destroyed)
-            auto rcvTill = boost::posix_time::microsec_clock::universal_time() + boost::posix_time::milliseconds(500);
+        while (true) {
+            uint32_t timeout = 100;
+            bool leave = false;
+            if (fStop) {
+                timeout = fLinger;
+                leave = true;
+            }
+            auto rcvTill = boost::posix_time::microsec_clock::universal_time() + boost::posix_time::milliseconds(timeout);
 
             while (fQueue->timed_receive(blocks.get(), fAckBunchSize * sizeof(RegionBlock), recvdSize, priority, rcvTill)) {
-                // LOG(debug) << "received: " << block.fHandle << " " << block.fSize << " " << block.fMessageId;
                 const auto numBlocks = recvdSize / sizeof(RegionBlock);
+                // LOG(debug) << "Received " << numBlocks << " blocks (recvdSize: " << recvdSize << "). (remaining queue size: " << fQueue->get_num_msg() << ").";
                 if (fBulkCallback) {
                     result.clear();
                     for (size_t i = 0; i < numBlocks; i++) {
@@ -191,9 +193,13 @@ struct Region
                     }
                 }
             }
-        } // while !fStop
 
-        LOG(debug) << "ReceiveAcks() worker for " << fName << " leaving.";
+            if (leave) {
+                break;
+            }
+        }
+
+        LOG(debug) << "AcksReceiver for " << fName << " leaving (remaining queue size: " << fQueue->get_num_msg() << ").";
     }
 
     void ReleaseBlock(const RegionBlock& block)
@@ -203,31 +209,34 @@ struct Region
         fBlocksToFree.emplace_back(block);
 
         if (fBlocksToFree.size() >= fAckBunchSize) {
-            lock.unlock(); // reduces contention on fBlockMtx
+            lock.unlock();
             fBlockSendCV.notify_one();
         }
     }
 
+    void SetLinger(uint32_t linger) { fLinger = linger; }
+    uint32_t GetLinger() const { return fLinger; }
+
     ~Region()
     {
         fStop = true;
 
-        if (fSendAcksWorker.joinable()) {
+        if (fAcksSender.joinable()) {
             fBlockSendCV.notify_one();
-            fSendAcksWorker.join();
+            fAcksSender.join();
         }
 
         if (!fRemote) {
-            if (fReceiveAcksWorker.joinable()) {
-                fReceiveAcksWorker.join();
+            if (fAcksReceiver.joinable()) {
+                fAcksReceiver.join();
             }
 
             if (boost::interprocess::shared_memory_object::remove(fName.c_str())) {
-                LOG(debug) << "shmem: destroyed region " << fName;
+                LOG(debug) << "Region '" << fName << "' destroyed.";
             }
 
             if (boost::interprocess::file_mapping::remove(fName.c_str())) {
-                LOG(debug) << "shmem: destroyed file mapping " << fName;
+                LOG(debug) << "File mapping '" << fName << "' destroyed.";
             }
 
             if (fFile) {
@@ -235,16 +244,19 @@ struct Region
             }
 
             if (boost::interprocess::message_queue::remove(fQueueName.c_str())) {
-                LOG(debug) << "shmem: removed region queue " << fQueueName;
+                LOG(debug) << "Region queue '" << fQueueName << "' destroyed.";
             }
         } else {
             // LOG(debug) << "shmem: region '" << fName << "' is remote, no cleanup necessary.";
-            LOG(debug) << "shmem: region queue '" << fQueueName << "' is remote, no cleanup necessary";
+            LOG(debug) << "Region queue '" << fQueueName << "' is remote, no cleanup necessary";
         }
+
+        LOG(debug) << "Region '" << fName << "' (" << (fRemote ? "remote" : "local") << ") destructed.";
     }
 
     bool fRemote;
-    bool fStop;
+    uint32_t fLinger;
+    std::atomic<bool> fStop;
     std::string fName;
     std::string fQueueName;
     boost::interprocess::shared_memory_object fShmemObject;
@@ -258,8 +270,8 @@ struct Region
     const std::size_t fAckBunchSize = 256;
     std::unique_ptr<boost::interprocess::message_queue> fQueue;
 
-    std::thread fReceiveAcksWorker;
-    std::thread fSendAcksWorker;
+    std::thread fAcksReceiver;
+    std::thread fAcksSender;
     RegionCallback fCallback;
     RegionBulkCallback fBulkCallback;
 };

fairmq/shmem/Socket.h

@@ -55,10 +55,15 @@ class Socket final : public fair::mq::Socket
         , fBytesRx(0)
         , fMessagesTx(0)
         , fMessagesRx(0)
-        , fSndTimeout(100)
-        , fRcvTimeout(100)
+        , fTimeout(100)
     {
         assert(context);
+
+        if (type == "sub" || type == "pub") {
+            LOG(error) << "PUB/SUB socket type is not supported for shared memory transport";
+            throw SocketError("PUB/SUB socket type is not supported for shared memory transport");
+        }
+
         fSocket = zmq_socket(context, GetConstant(type));
 
         if (fSocket == nullptr) {
@@ -77,11 +82,11 @@ class Socket final : public fair::mq::Socket
             LOG(error) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
         }
 
-        if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0) {
+        if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fTimeout, sizeof(fTimeout)) != 0) {
             LOG(error) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
         }
 
-        if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0) {
+        if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fTimeout, sizeof(fTimeout)) != 0) {
             LOG(error) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
         }
 
@@ -92,11 +97,6 @@ class Socket final : public fair::mq::Socket
         //         LOG(error) << "Failed setting ZMQ_SUBSCRIBE socket option, reason: " << zmq_strerror(errno);
         //     }
         // }
-
-        if (type == "sub" || type == "pub") {
-            LOG(error) << "PUB/SUB socket type is not supported for shared memory transport";
-            throw SocketError("PUB/SUB socket type is not supported for shared memory transport");
-        }
         LOG(debug) << "Created socket " << GetId();
     }
 
@@ -129,7 +129,33 @@ class Socket final : public fair::mq::Socket
         return true;
     }
 
-    int Send(MessagePtr& msg, const int timeout = -1) override
+    bool ShouldRetry(int flags, int timeout, int& elapsed) const
+    {
+        if ((flags & ZMQ_DONTWAIT) == 0) {
+            if (timeout > 0) {
+                elapsed += fTimeout;
+                if (elapsed >= timeout) {
+                    return false;
+                }
+            }
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    int HandleErrors() const
+    {
+        if (zmq_errno() == ETERM) {
+            LOG(debug) << "Terminating socket " << fId;
+            return static_cast<int>(TransferResult::error);
+        } else {
+            LOG(error) << "Failed transfer on socket " << fId << ", reason: " << zmq_strerror(errno);
+            return static_cast<int>(TransferResult::error);
+        }
+    }
+
+    int64_t Send(MessagePtr& msg, const int timeout = -1) override
     {
         int flags = 0;
         if (timeout == 0) {
@@ -141,7 +167,7 @@ class Socket final : public fair::mq::Socket
         ZMsg zmqMsg(sizeof(MetaHeader));
         std::memcpy(zmqMsg.Data(), &(shmMsg->fMeta), sizeof(MetaHeader));
 
-        while (true && !fManager.Interrupted()) {
+        while (true) {
             int nbytes = zmq_msg_send(zmqMsg.Msg(), fSocket, flags);
             if (nbytes > 0) {
                 shmMsg->fQueued = true;
@@ -149,34 +175,23 @@ class Socket final : public fair::mq::Socket
                 size_t size = msg->GetSize();
                 fBytesTx += size;
                 return size;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fSndTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
+            } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                if (fManager.Interrupted()) {
+                    return static_cast<int>(TransferResult::interrupted);
+                } else if (ShouldRetry(flags, timeout, elapsed)) {
                     continue;
                 } else {
-                    return -2;
+                    return static_cast<int>(TransferResult::timeout);
                 }
-            } else if (zmq_errno() == ETERM) {
-                LOG(info) << "terminating socket " << fId;
-                return -1;
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Send interrupted by system call";
-                return nbytes;
-            }else {
-                LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
-                return nbytes;
+            } else {
+                return HandleErrors();
             }
         }
 
-        return -1;
+        return static_cast<int>(TransferResult::error);
     }
 
-    int Receive(MessagePtr& msg, const int timeout = -1) override
+    int64_t Receive(MessagePtr& msg, const int timeout = -1) override
     {
         int flags = 0;
         if (timeout == 0) {
@@ -205,27 +220,16 @@ class Socket final : public fair::mq::Socket
                 fBytesRx += size;
                 ++fMessagesRx;
                 return size;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fRcvTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
+            } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                if (fManager.Interrupted()) {
+                    return static_cast<int>(TransferResult::interrupted);
+                } else if (ShouldRetry(flags, timeout, elapsed)) {
                     continue;
                 } else {
-                    return -2;
+                    return static_cast<int>(TransferResult::timeout);
                 }
-            } else if (zmq_errno() == ETERM) {
-                LOG(info) << "terminating socket " << fId;
-                return -1;
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Receive interrupted by system call";
-                return nbytes;
-            }else {
-                LOG(error) << "Failed receiving on socket " << fId << ", errno: " << errno << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
-                return nbytes;
+            } else {
+                return HandleErrors();
             }
         }
     }
@@ -250,7 +254,7 @@ class Socket final : public fair::mq::Socket
             std::memcpy(metas++, &(shmMsg->fMeta), sizeof(MetaHeader));
         }
 
-        while (!fManager.Interrupted()) {
+        while (true) {
             int64_t totalSize = 0;
             int nbytes = zmq_msg_send(zmqMsg.Msg(), fSocket, flags);
             if (nbytes > 0) {
@@ -267,31 +271,20 @@ class Socket final : public fair::mq::Socket
                 fBytesTx += totalSize;
 
                 return totalSize;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fSndTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
+            } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                if (fManager.Interrupted()) {
+                    return static_cast<int>(TransferResult::interrupted);
+                } else if (ShouldRetry(flags, timeout, elapsed)) {
                     continue;
                 } else {
-                    return -2;
+                    return static_cast<int>(TransferResult::timeout);
                 }
-            } else if (zmq_errno() == ETERM) {
-                LOG(info) << "terminating socket " << fId;
-                return -1;
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Send interrupted by system call";
-                return nbytes;
-            }else {
-                LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
-                return nbytes;
+            } else {
+                return HandleErrors();
             }
         }
 
-        return -1;
+        return static_cast<int>(TransferResult::error);
     }
 
     int64_t Receive(std::vector<MessagePtr>& msgVec, const int timeout = -1) override
@@ -304,7 +297,7 @@ class Socket final : public fair::mq::Socket
 
         ZMsg zmqMsg;
 
-        while (!fManager.Interrupted()) {
+        while (true) {
             int64_t totalSize = 0;
             int nbytes = zmq_msg_recv(zmqMsg.Msg(), fSocket, flags);
             if (nbytes > 0) {
@@ -334,28 +327,20 @@ class Socket final : public fair::mq::Socket
                 fBytesRx += totalSize;
 
                 return totalSize;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fRcvTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
+            } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                if (fManager.Interrupted()) {
+                    return static_cast<int>(TransferResult::interrupted);
+                } else if (ShouldRetry(flags, timeout, elapsed)) {
                     continue;
                 } else {
-                    return -2;
+                    return static_cast<int>(TransferResult::timeout);
                 }
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Receive interrupted by system call";
-                return nbytes;
             } else {
-                LOG(error) << "Failed receiving on socket " << fId << ", errno: " << errno << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
-                return nbytes;
+                return HandleErrors();
             }
         }
 
-        return -1;
+        return static_cast<int>(TransferResult::error);
     }
 
     void* GetSocket() const { return fSocket; }
@@ -396,6 +381,15 @@ class Socket final : public fair::mq::Socket
         }
     }
 
+    void Events(uint32_t* events) override
+    {
+        size_t eventsSize = sizeof(uint32_t);
+        if (zmq_getsockopt(fSocket, ZMQ_EVENTS, events, &eventsSize) < 0) {
+            throw SocketError(
+                tools::ToString("failed setting ZMQ_EVENTS, reason: ", zmq_strerror(errno)));
+        }
+    }
+
     int GetLinger() const override
     {
         int value = 0;
@@ -506,8 +500,14 @@ class Socket final : public fair::mq::Socket
         if (constant == "snd-more no-block") return ZMQ_DONTWAIT|ZMQ_SNDMORE;
 
         if (constant == "fd") return ZMQ_FD;
+        if (constant == "events")
+            return ZMQ_EVENTS;
+        if (constant == "pollin")
+            return ZMQ_POLLIN;
+        if (constant == "pollout")
+            return ZMQ_POLLOUT;
 
-        return -1;
+        throw SocketError(tools::ToString("GetConstant called with an invalid argument: ", constant));
     }
 
     ~Socket() override { Close(); }
@@ -521,8 +521,7 @@ class Socket final : public fair::mq::Socket
     std::atomic<unsigned long> fMessagesTx;
     std::atomic<unsigned long> fMessagesRx;
 
-    int fSndTimeout;
-    int fRcvTimeout;
+    int fTimeout;
 };
 
 }

fairmq/shmem/TransportFactory.h

@@ -43,7 +43,7 @@ class TransportFactory final : public fair::mq::TransportFactory
         : fair::mq::TransportFactory(id)
         , fDeviceId(id)
         , fShmId()
-        , fZMQContext(zmq_ctx_new())
+        , fZmqCtx(zmq_ctx_new())
         , fManager(nullptr)
     {
         int major, minor, patch;
@@ -51,42 +51,42 @@ class TransportFactory final : public fair::mq::TransportFactory
         LOG(debug) << "Transport: Using ZeroMQ (" << major << "." << minor << "." << patch << ") & "
                 << "boost::interprocess (" << (BOOST_VERSION / 100000) << "." << (BOOST_VERSION / 100 % 1000) << "." << (BOOST_VERSION % 100) << ")";
 
-        if (!fZMQContext) {
+        if (!fZmqCtx) {
             throw std::runtime_error(tools::ToString("failed creating context, reason: ", zmq_strerror(errno)));
         }
 
         int numIoThreads = 1;
         std::string sessionName = "default";
-        size_t segmentSize = 2000000000;
-        bool autolaunchMonitor = false;
-        bool throwOnBadAlloc = true;
+        size_t segmentSize = 2ULL << 30;
+        std::string allocationAlgorithm("rbtree_best_fit");
         if (config) {
             numIoThreads = config->GetProperty<int>("io-threads", numIoThreads);
             sessionName = config->GetProperty<std::string>("session", sessionName);
             segmentSize = config->GetProperty<size_t>("shm-segment-size", segmentSize);
-            autolaunchMonitor = config->GetProperty<bool>("shm-monitor", autolaunchMonitor);
-            throwOnBadAlloc = config->GetProperty<bool>("shm-throw-bad-alloc", throwOnBadAlloc);
+            allocationAlgorithm = config->GetProperty<std::string>("shm-allocation", allocationAlgorithm);
         } else {
             LOG(debug) << "ProgOptions not available! Using defaults.";
         }
 
+        if (allocationAlgorithm != "rbtree_best_fit" && allocationAlgorithm != "simple_seq_fit") {
+            LOG(error) << "Provided shared memory allocation algorithm '" << allocationAlgorithm << "' is not supported. Supported are 'rbtree_best_fit'/'simple_seq_fit'";
+            throw SharedMemoryError(tools::ToString("Provided shared memory allocation algorithm '", allocationAlgorithm, "' is not supported. Supported are 'rbtree_best_fit'/'simple_seq_fit'"));
+        }
+
         fShmId = buildShmIdFromSessionIdAndUserId(sessionName);
+        LOG(debug) << "Generated shmid '" << fShmId << "' out of session id '" << sessionName << "'.";
 
         try {
-            if (zmq_ctx_set(fZMQContext, ZMQ_IO_THREADS, numIoThreads) != 0) {
+            if (zmq_ctx_set(fZmqCtx, ZMQ_IO_THREADS, numIoThreads) != 0) {
                 LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
             }
 
             // Set the maximum number of allowed sockets on the context.
-            if (zmq_ctx_set(fZMQContext, ZMQ_MAX_SOCKETS, 10000) != 0) {
+            if (zmq_ctx_set(fZmqCtx, ZMQ_MAX_SOCKETS, 10000) != 0) {
                 LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
             }
 
-            if (autolaunchMonitor) {
-                Manager::StartMonitor(fShmId);
-            }
-
-            fManager = tools::make_unique<Manager>(fShmId, fDeviceId, segmentSize, throwOnBadAlloc);
+            fManager = tools::make_unique<Manager>(fShmId, fDeviceId, segmentSize, config);
         } catch (boost::interprocess::interprocess_exception& e) {
             LOG(error) << "Could not initialize shared memory transport: " << e.what();
             throw std::runtime_error(tools::ToString("Could not initialize shared memory transport: ", e.what()));
@@ -128,7 +128,7 @@ class TransportFactory final : public fair::mq::TransportFactory
 
     SocketPtr CreateSocket(const std::string& type, const std::string& name) override
     {
-        return tools::make_unique<Socket>(*fManager, type, name, GetId(), fZMQContext, this);
+        return tools::make_unique<Socket>(*fManager, type, name, GetId(), fZmqCtx, this);
     }
 
     PollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const override
@@ -186,14 +186,17 @@ class TransportFactory final : public fair::mq::TransportFactory
     {
         LOG(debug) << "Destroying Shared Memory transport...";
 
-        if (fZMQContext) {
-            if (zmq_ctx_term(fZMQContext) != 0) {
-                if (errno == EINTR) {
-                    LOG(error) << "failed closing context, reason: " << zmq_strerror(errno);
-                } else {
-                    fZMQContext = nullptr;
-                    return;
+        if (fZmqCtx) {
+            while (true) {
+                if (zmq_ctx_term(fZmqCtx) != 0) {
+                    if (errno == EINTR) {
+                        LOG(debug) << "zmq_ctx_term interrupted by system call, retrying";
+                        continue;
+                    } else {
+                        fZmqCtx = nullptr;
+                    }
                 }
+                break;
             }
         } else {
             LOG(error) << "context not available for shutdown";
@@ -203,7 +206,7 @@ class TransportFactory final : public fair::mq::TransportFactory
   private:
     std::string fDeviceId;
     std::string fShmId;
-    void* fZMQContext;
+    void* fZmqCtx;
     std::unique_ptr<Manager> fManager;
 };
 

fairmq/shmem/UnmanagedRegion.h

@@ -56,14 +56,16 @@ class UnmanagedRegion final : public fair::mq::UnmanagedRegion
 
     void* GetData() const override { return fRegion->get_address(); }
     size_t GetSize() const override { return fRegion->get_size(); }
-    uint64_t GetId() const override { return fRegionId; }
+    uint16_t GetId() const override { return fRegionId; }
+    void SetLinger(uint32_t linger) override { fManager.GetRegion(fRegionId)->SetLinger(linger); }
+    uint32_t GetLinger() const override { return fManager.GetRegion(fRegionId)->GetLinger(); }
 
     ~UnmanagedRegion() override { fManager.RemoveRegion(fRegionId); }
 
   private:
     Manager& fManager;
     boost::interprocess::mapped_region* fRegion;
-    uint64_t fRegionId;
+    uint16_t fRegionId;
 };
 
 }

fairmq/shmem/runMonitor.cxx

@@ -76,7 +76,9 @@ int main(int argc, char** argv)
         bool interactive = false;
         bool viewOnly = false;
         unsigned int timeoutInMS = 5000;
+        unsigned int intervalInMS = 100;
         bool runAsDaemon = false;
+        bool debug = false;
         bool cleanOnExit = false;
 
         options_description desc("Options");
@@ -89,7 +91,9 @@ int main(int argc, char** argv)
             ("view,v"         , value<bool>(&viewOnly)->implicit_value(true),           "Run in view only mode")
             ("timeout,t"      , value<unsigned int>(&timeoutInMS)->default_value(5000), "Heartbeat timeout in milliseconds")
             ("daemonize,d"    , value<bool>(&runAsDaemon)->implicit_value(true),        "Daemonize the monitor")
+            ("debug,b"        , value<bool>(&debug)->implicit_value(true),             "Debug - Print a list of messages)")
             ("clean-on-exit,e", value<bool>(&cleanOnExit)->implicit_value(true),        "Perform cleanup on exit")
+            ("interval"       , value<unsigned int>(&intervalInMS)->default_value(100), "Output interval for interactive/view-only mode")
             ("help,h", "Print help");
 
         variables_map vm;
@@ -111,15 +115,21 @@ int main(int argc, char** argv)
         }
 
         if (cleanup) {
-            cout << "Cleaning up \"" << shmId << "\"..." << endl;
-            Monitor::Cleanup(shmId);
-            Monitor::RemoveQueue("fmq_" + shmId + "_cq");
+            Monitor::CleanupFull(ShmId{shmId});
+            return 0;
+        }
+
+        if (debug) {
+            Monitor::PrintDebugInfo(ShmId{shmId});
             return 0;
         }
 
         cout << "Starting shared memory monitor for session: \"" << sessionName << "\" (shmId: " << shmId << ")..." << endl;
+        if (viewOnly && !interactive) {
+            cout << "running in non-interactive view-only mode, outputting with interval of " << intervalInMS << "ms. (change with --interval), press ctrl+C to exit." << endl;
+        }
 
-        Monitor monitor(shmId, selfDestruct, interactive, viewOnly, timeoutInMS, runAsDaemon, cleanOnExit);
+        Monitor monitor(shmId, selfDestruct, interactive, viewOnly, timeoutInMS, intervalInMS, runAsDaemon, cleanOnExit);
 
         monitor.CatchSignals();
         monitor.Run();

fairmq/tools/RateLimit.h

@@ -11,7 +11,7 @@
 
 #include <cassert>
 #include <string>
-#include <iostream>
+// #include <iostream>
 #include <iomanip>
 #include <thread>
 #include <chrono>

fairmq/zeromq/Context.h

@@ -56,7 +56,7 @@ class Context
             throw ContextError(tools::ToString("failed configuring context, reason: ", zmq_strerror(errno)));
         }
 
-        fRegionEvents.emplace(0, nullptr, 0, 0, RegionEvent::local_only);
+        fRegionEvents.emplace(true, 0, nullptr, 0, 0, RegionEvent::local_only);
     }
 
     Context(const Context&) = delete;
@@ -114,24 +114,24 @@ class Context
         return fRegionInfos;
     }
 
-    uint64_t RegionCount() const
+    uint16_t RegionCount() const
     {
         std::lock_guard<std::mutex> lock(fMtx);
         return fRegionCounter;
     }
 
-    void AddRegion(uint64_t id, void* ptr, size_t size, int64_t userFlags, RegionEvent event)
+    void AddRegion(bool managed, uint16_t id, void* ptr, size_t size, int64_t userFlags, RegionEvent event)
     {
         {
             std::lock_guard<std::mutex> lock(fMtx);
             ++fRegionCounter;
-            fRegionInfos.emplace_back(id, ptr, size, userFlags, event);
-            fRegionEvents.emplace(id, ptr, size, userFlags, event);
+            fRegionInfos.emplace_back(managed, id, ptr, size, userFlags, event);
+            fRegionEvents.emplace(managed, id, ptr, size, userFlags, event);
         }
         fRegionEventsCV.notify_one();
     }
 
-    void RemoveRegion(uint64_t id)
+    void RemoveRegion(uint16_t id)
     {
         {
             std::lock_guard<std::mutex> lock(fMtx);
@@ -161,13 +161,16 @@ class Context
         UnsubscribeFromRegionEvents();
 
         if (fZmqCtx) {
-            if (zmq_ctx_term(fZmqCtx) != 0) {
-                if (errno == EINTR) {
-                    LOG(error) << " failed closing context, reason: " << zmq_strerror(errno);
-                } else {
-                    fZmqCtx = nullptr;
-                    return;
+            while (true) {
+                if (zmq_ctx_term(fZmqCtx) != 0) {
+                    if (errno == EINTR) {
+                        LOG(debug) << "zmq_ctx_term interrupted by system call, retrying";
+                        continue;
+                    } else {
+                        fZmqCtx = nullptr;
+                    }
                 }
+                break;
             }
         } else {
             LOG(error) << "context not available for shutdown";
@@ -179,7 +182,7 @@ class Context
     mutable std::mutex fMtx;
     std::atomic<bool> fInterrupted;
 
-    uint64_t fRegionCounter;
+    uint16_t fRegionCounter;
     std::condition_variable fRegionEventsCV;
     std::vector<RegionInfo> fRegionInfos;
     std::queue<RegionInfo> fRegionEvents;

fairmq/zeromq/Poller.h

@@ -130,13 +130,20 @@ class Poller final : public fair::mq::Poller
 
     void Poll(const int timeout) override
     {
-        if (zmq_poll(fItems, fNumItems, timeout) < 0) {
-            if (errno == ETERM) {
-                LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
-            } else {
-                LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
-                throw fair::mq::PollerError(fair::mq::tools::ToString("Polling failed, reason: ", zmq_strerror(errno)));
+        while (true) {
+            if (zmq_poll(fItems, fNumItems, timeout) < 0) {
+                if (errno == ETERM) {
+                    LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
+                    return;
+                } else if (errno == EINTR) {
+                    LOG(debug) << "polling interrupted by system call";
+                    continue;
+                } else {
+                    LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
+                    throw fair::mq::PollerError(fair::mq::tools::ToString("Polling failed, reason: ", zmq_strerror(errno)));
+                }
             }
+            break;
         }
     }
 

fairmq/zeromq/Socket.h

@@ -12,13 +12,15 @@
 #include <FairMQLogger.h>
 #include <FairMQMessage.h>
 #include <FairMQSocket.h>
-#include <atomic>
 #include <fairmq/Tools.h>
 #include <fairmq/zeromq/Context.h>
 #include <fairmq/zeromq/Message.h>
-#include <memory>   // unique_ptr
+
 #include <zmq.h>
 
+#include <atomic>
+#include <memory> // unique_ptr
+
 namespace fair {
 namespace mq {
 namespace zmq {
@@ -35,8 +37,7 @@ class Socket final : public fair::mq::Socket
         , fBytesRx(0)
         , fMessagesTx(0)
         , fMessagesRx(0)
-        , fSndTimeout(100)
-        , fRcvTimeout(100)
+        , fTimeout(100)
     {
         if (fSocket == nullptr) {
             LOG(error) << "Failed creating socket " << fId << ", reason: " << zmq_strerror(errno);
@@ -54,11 +55,11 @@ class Socket final : public fair::mq::Socket
             LOG(error) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
         }
 
-        if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0) {
+        if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fTimeout, sizeof(fTimeout)) != 0) {
             LOG(error) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
         }
 
-        if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0) {
+        if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fTimeout, sizeof(fTimeout)) != 0) {
             LOG(error) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
         }
 
@@ -78,13 +79,12 @@ class Socket final : public fair::mq::Socket
 
     bool Bind(const std::string& address) override
     {
-        // LOG(info) << "bind socket " << fId << " on " << address;
+        // LOG(debug) << "Binding socket " << fId << " on " << address;
 
         if (zmq_bind(fSocket, address.c_str()) != 0) {
             if (errno == EADDRINUSE) {
                 // do not print error in this case, this is handled by FairMQDevice in case no
-                // connection could be established after trying a number of random ports from a
-                // range.
+                // connection could be established after trying a number of random ports from a range.
                 return false;
             }
             LOG(error) << "Failed binding socket " << fId << ", address: " << address << ", reason: " << zmq_strerror(errno);
@@ -96,7 +96,7 @@ class Socket final : public fair::mq::Socket
 
     bool Connect(const std::string& address) override
     {
-        // LOG(info) << "connect socket " << fId << " on " << address;
+        // LOG(debug) << "Connecting socket " << fId << " on " << address;
 
         if (zmq_connect(fSocket, address.c_str()) != 0) {
             LOG(error) << "Failed connecting socket " << fId << ", address: " << address << ", reason: " << zmq_strerror(errno);
@@ -106,7 +106,33 @@ class Socket final : public fair::mq::Socket
         return true;
     }
 
-    int Send(MessagePtr& msg, const int timeout = -1) override
+    bool ShouldRetry(int flags, int timeout, int& elapsed) const
+    {
+        if ((flags & ZMQ_DONTWAIT) == 0) {
+            if (timeout > 0) {
+                elapsed += fTimeout;
+                if (elapsed >= timeout) {
+                    return false;
+                }
+            }
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    int HandleErrors() const
+    {
+        if (zmq_errno() == ETERM) {
+            LOG(debug) << "Terminating socket " << fId;
+            return static_cast<int>(TransferResult::error);
+        } else {
+            LOG(error) << "Failed transfer on socket " << fId << ", errno: " << errno << ", reason: " << zmq_strerror(errno);
+            return static_cast<int>(TransferResult::error);
+        }
+    }
+
+    int64_t Send(MessagePtr& msg, const int timeout = -1) override
     {
         int flags = 0;
         if (timeout == 0) {
@@ -121,34 +147,22 @@ class Socket final : public fair::mq::Socket
             if (nbytes >= 0) {
                 fBytesTx += nbytes;
                 ++fMessagesTx;
-
                 return nbytes;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fCtx.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fSndTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
+            } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                if (fCtx.Interrupted()) {
+                    return static_cast<int>(TransferResult::interrupted);
+                } else if (ShouldRetry(flags, timeout, elapsed)) {
                     continue;
                 } else {
-                    return -2;
+                    return static_cast<int>(TransferResult::timeout);
                 }
-            } else if (zmq_errno() == ETERM) {
-                LOG(info) << "terminating socket " << fId;
-                return -1;
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Send interrupted by system call";
-                return nbytes;
             } else {
-                LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
-                return nbytes;
+                return HandleErrors();
             }
         }
     }
 
-    int Receive(MessagePtr& msg, const int timeout = -1) override
+    int64_t Receive(MessagePtr& msg, const int timeout = -1) override
     {
         int flags = 0;
         if (timeout == 0) {
@@ -162,27 +176,16 @@ class Socket final : public fair::mq::Socket
                 fBytesRx += nbytes;
                 ++fMessagesRx;
                 return nbytes;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fCtx.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fRcvTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
+            } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                if (fCtx.Interrupted()) {
+                    return static_cast<int>(TransferResult::interrupted);
+                } else if (ShouldRetry(flags, timeout, elapsed)) {
                     continue;
                 } else {
-                    return -2;
+                    return static_cast<int>(TransferResult::timeout);
                 }
-            } else if (zmq_errno() == ETERM) {
-                LOG(info) << "terminating socket " << fId;
-                return -1;
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Receive interrupted by system call";
-                return nbytes;
             } else {
-                LOG(error) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
-                return nbytes;
+                return HandleErrors();
             }
         }
     }
@@ -210,32 +213,17 @@ class Socket final : public fair::mq::Socket
                     int nbytes = zmq_msg_send(static_cast<Message*>(msgVec[i].get())->GetMessage(), fSocket, (i < vecSize - 1) ? ZMQ_SNDMORE | flags : flags);
                     if (nbytes >= 0) {
                         totalSize += nbytes;
-                    } else {
-                        // according to ZMQ docs, this can only occur for the first part
-                        if (zmq_errno() == EAGAIN) {
-                            if (!fCtx.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                                if (timeout > 0) {
-                                    elapsed += fSndTimeout;
-                                    if (elapsed >= timeout) {
-                                        return -2;
-                                    }
-                                }
-                                repeat = true;
-                                break;
-                            } else {
-                                return -2;
-                            }
-                        }
-                        if (zmq_errno() == ETERM) {
-                            LOG(info) << "terminating socket " << fId;
-                            return -1;
-                        } else if (zmq_errno() == EINTR) {
-                            LOG(debug) << "Receive interrupted by system call";
-                            return nbytes;
+                    } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                        if (fCtx.Interrupted()) {
+                            return static_cast<int>(TransferResult::interrupted);
+                        } else if (ShouldRetry(flags, timeout, elapsed)) {
+                            repeat = true;
+                            break;
                         } else {
-                            LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
-                            return nbytes;
+                            return static_cast<int>(TransferResult::timeout);
                         }
+                    } else {
+                        return HandleErrors();
                     }
                 }
 
@@ -243,18 +231,16 @@ class Socket final : public fair::mq::Socket
                     continue;
                 }
 
-                // store statistics on how many messages have been sent (handle all parts as a
-                // single message)
+                // store statistics on how many messages have been sent (handle all parts as a single message)
                 ++fMessagesTx;
                 fBytesTx += totalSize;
                 return totalSize;
             }
-        }   // If there's only one part, send it as a regular message
-        else if (vecSize == 1) {
+        } else if (vecSize == 1) { // If there's only one part, send it as a regular message
             return Send(msgVec.back(), timeout);
-        } else {   // if the vector is empty, something might be wrong
+        } else { // if the vector is empty, something might be wrong
             LOG(warn) << "Will not send empty vector";
-            return -1;
+            return static_cast<int>(TransferResult::error);
         }
     }
 
@@ -278,24 +264,17 @@ class Socket final : public fair::mq::Socket
                 if (nbytes >= 0) {
                     msgVec.push_back(move(part));
                     totalSize += nbytes;
-                } else if (zmq_errno() == EAGAIN) {
-                    if (!fCtx.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                        if (timeout > 0) {
-                            elapsed += fRcvTimeout;
-                            if (elapsed >= timeout) {
-                                return -2;
-                            }
-                        }
+                } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                    if (fCtx.Interrupted()) {
+                        return static_cast<int>(TransferResult::interrupted);
+                    } else if (ShouldRetry(flags, timeout, elapsed)) {
                         repeat = true;
                         break;
                     } else {
-                        return -2;
+                        return static_cast<int>(TransferResult::timeout);
                     }
-                } else if (zmq_errno() == EINTR) {
-                    LOG(debug) << "Receive interrupted by system call";
-                    return nbytes;
                 } else {
-                    return nbytes;
+                    return HandleErrors();
                 }
 
                 size_t moreSize = sizeof(more);
@@ -306,8 +285,7 @@ class Socket final : public fair::mq::Socket
                 continue;
             }
 
-            // store statistics on how many messages have been received (handle all parts as a
-            // single message)
+            // store statistics on how many messages have been received (handle all parts as a single message)
             ++fMessagesRx;
             fBytesRx += totalSize;
             return totalSize;
@@ -345,6 +323,15 @@ class Socket final : public fair::mq::Socket
         }
     }
 
+    void Events(uint32_t* events) override
+    {
+        size_t eventsSize = sizeof(uint32_t);
+        if (zmq_getsockopt(fSocket, ZMQ_EVENTS, events, &eventsSize) < 0) {
+            throw SocketError(
+                tools::ToString("failed setting ZMQ_EVENTS, reason: ", zmq_strerror(errno)));
+        }
+    }
+
     void SetLinger(const int value) override
     {
         if (zmq_setsockopt(fSocket, ZMQ_LINGER, &value, sizeof(value)) < 0) {
@@ -460,8 +447,14 @@ class Socket final : public fair::mq::Socket
         if (constant == "linger") return ZMQ_LINGER;
 
         if (constant == "fd") return ZMQ_FD;
+        if (constant == "events")
+            return ZMQ_EVENTS;
+        if (constant == "pollin")
+            return ZMQ_POLLIN;
+        if (constant == "pollout")
+            return ZMQ_POLLOUT;
 
-        return -1;
+        throw SocketError(tools::ToString("GetConstant called with an invalid argument: ", constant));
     }
 
     ~Socket() override { Close(); }
@@ -475,8 +468,7 @@ class Socket final : public fair::mq::Socket
     std::atomic<unsigned long> fMessagesTx;
     std::atomic<unsigned long> fMessagesRx;
 
-    int fSndTimeout;
-    int fRcvTimeout;
+    int fTimeout;
 };
 
 }   // namespace zmq

fairmq/zeromq/TransportFactory.h

@@ -55,7 +55,7 @@ class TransportFactory final : public FairMQTransportFactory
     {
         return tools::make_unique<Message>(this);
     }
-    
+
     MessagePtr CreateMessage(Alignment alignment) override
     {
         return tools::make_unique<Message>(alignment, this);
@@ -125,7 +125,7 @@ class TransportFactory final : public FairMQTransportFactory
     {
         UnmanagedRegionPtr ptr = tools::make_unique<UnmanagedRegion>(*fCtx, size, userFlags, callback, bulkCallback, this);
         auto zPtr = static_cast<UnmanagedRegion*>(ptr.get());
-        fCtx->AddRegion(zPtr->GetId(), zPtr->GetData(), zPtr->GetSize(), zPtr->GetUserFlags(), RegionEvent::created);
+        fCtx->AddRegion(false, zPtr->GetId(), zPtr->GetData(), zPtr->GetSize(), zPtr->GetUserFlags(), RegionEvent::created);
         return ptr;
     }
 

fairmq/zeromq/UnmanagedRegion.h

@@ -50,8 +50,10 @@ class UnmanagedRegion final : public fair::mq::UnmanagedRegion
 
     virtual void* GetData() const override { return fBuffer; }
     virtual size_t GetSize() const override { return fSize; }
-    uint64_t GetId() const override { return fId; }
+    uint16_t GetId() const override { return fId; }
     int64_t GetUserFlags() const { return fUserFlags; }
+    void SetLinger(uint32_t /* linger */) override { LOG(debug) << "ZeroMQ UnmanagedRegion linger option not implemented. Acknowledgements are local."; }
+    uint32_t GetLinger() const override { LOG(debug) << "ZeroMQ UnmanagedRegion linger option not implemented. Acknowledgements are local."; return 0; }
 
     virtual ~UnmanagedRegion()
     {
@@ -62,7 +64,7 @@ class UnmanagedRegion final : public fair::mq::UnmanagedRegion
 
   private:
     Context& fCtx;
-    uint64_t fId;
+    uint16_t fId;
     void* fBuffer;
     size_t fSize;
     int64_t fUserFlags;

test/CMakeLists.txt

@@ -12,6 +12,18 @@ include(GTestHelper)
 # FairMQ Testsuites/helpers #
 #############################
 
+if(FairLogger_VERSION VERSION_LESS 1.9.0 AND FairLogger_VERSION VERSION_GREATER_EQUAL 1.7.0)
+  LIST(APPEND definitions FAIR_MIN_SEVERITY=trace)
+endif()
+
+if(BUILD_OFI_TRANSPORT)
+  LIST(APPEND definitions BUILD_OFI_TRANSPORT)
+endif()
+
+if(definitions)
+  set(definitions DEFINITIONS ${definitions})
+endif()
+
 add_testhelper(runTestDevice
     SOURCES
     helper/runTestDevice.cxx
@@ -30,16 +42,9 @@ add_testhelper(runTestDevice
     helper/devices/TestExceptions.h
 
     LINKS FairMQ
+    ${definitions}
 )
 
-if(BUILD_OFI_TRANSPORT)
-  LIST(APPEND definitions BUILD_OFI_TRANSPORT)
-endif()
-
-if(definitions)
-  set(definitions DEFINITIONS ${definitions})
-endif()
-
 set(MQ_CONFIG "${CMAKE_BINARY_DIR}/test/testsuite_FairMQ.IOPatterns_config.json")
 set(RUN_TEST_DEVICE "${CMAKE_BINARY_DIR}/test/testhelper_runTestDevice")
 set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)

test/channel/_channel.cxx

@@ -88,7 +88,7 @@ TEST(Channel, Validation)
     channel2.UpdateName("Kanal");
     ASSERT_EQ(channel2.GetName(), "Kanal");
 
-    channel2.ResetChannel();
+    channel2.Invalidate();
     ASSERT_EQ(channel2.IsValid(), false);
     ASSERT_EQ(channel2.Validate(), true);
 }

test/commands/_commands.cxx

@@ -31,7 +31,7 @@ TEST(Format, Construction)
     Cmds setPropertiesCmds(make<SetProperties>(42, props));
     Cmds subscriptionHeartbeatCmds(make<SubscriptionHeartbeat>(60000));
     Cmds currentStateCmds(make<CurrentState>("somedeviceid", State::Running));
-    Cmds transitionStatusCmds(make<TransitionStatus>("somedeviceid", 123456, Result::Ok, Transition::Stop));
+    Cmds transitionStatusCmds(make<TransitionStatus>("somedeviceid", 123456, Result::Ok, Transition::Stop, State::Running));
     Cmds configCmds(make<Config>("somedeviceid", "someconfig"));
     Cmds stateChangeSubscriptionCmds(make<StateChangeSubscription>("somedeviceid", 123456, Result::Ok));
     Cmds stateChangeUnsubscriptionCmds(make<StateChangeUnsubscription>("somedeviceid", 123456, Result::Ok));
@@ -63,6 +63,7 @@ TEST(Format, Construction)
     ASSERT_EQ(static_cast<TransitionStatus&>(transitionStatusCmds.At(0)).GetTaskId(), 123456);
     ASSERT_EQ(static_cast<TransitionStatus&>(transitionStatusCmds.At(0)).GetResult(), Result::Ok);
     ASSERT_EQ(static_cast<TransitionStatus&>(transitionStatusCmds.At(0)).GetTransition(), Transition::Stop);
+    ASSERT_EQ(static_cast<TransitionStatus&>(transitionStatusCmds.At(0)).GetCurrentState(), State::Running);
     ASSERT_EQ(configCmds.At(0).GetType(), Type::config);
     ASSERT_EQ(static_cast<Config&>(configCmds.At(0)).GetDeviceId(), "somedeviceid");
     ASSERT_EQ(static_cast<Config&>(configCmds.At(0)).GetConfig(), "someconfig");
@@ -104,7 +105,7 @@ void fillCommands(Cmds& cmds)
     cmds.Add<SetProperties>(42, props);
     cmds.Add<SubscriptionHeartbeat>(60000);
     cmds.Add<CurrentState>("somedeviceid", State::Running);
-    cmds.Add<TransitionStatus>("somedeviceid", 123456, Result::Ok, Transition::Stop);
+    cmds.Add<TransitionStatus>("somedeviceid", 123456, Result::Ok, Transition::Stop, State::Running);
     cmds.Add<Config>("somedeviceid", "someconfig");
     cmds.Add<StateChangeSubscription>("somedeviceid", 123456, Result::Ok);
     cmds.Add<StateChangeUnsubscription>("somedeviceid", 123456, Result::Ok);
@@ -167,6 +168,7 @@ void checkCommands(Cmds& cmds)
                 ASSERT_EQ(static_cast<TransitionStatus&>(*cmd).GetTaskId(), 123456);
                 ASSERT_EQ(static_cast<TransitionStatus&>(*cmd).GetResult(), Result::Ok);
                 ASSERT_EQ(static_cast<TransitionStatus&>(*cmd).GetTransition(), Transition::Stop);
+                ASSERT_EQ(static_cast<TransitionStatus&>(*cmd).GetCurrentState(), State::Running);
             break;
             case Type::config:
                 ++count;

test/helper/devices/TestErrorState.h

@@ -28,7 +28,7 @@ class ErrorState : public FairMQDevice
     {
         std::string state("Init");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "going to change to Error state from " << state << "()";
+            LOG(info) << "going to change to Error state from " << state << "()";
             ChangeState(fair::mq::Transition::ErrorFound);
         }
     }
@@ -37,7 +37,7 @@ class ErrorState : public FairMQDevice
     {
         std::string state("Bind");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "going to change to Error state from " << state << "()";
+            LOG(info) << "going to change to Error state from " << state << "()";
             ChangeState(fair::mq::Transition::ErrorFound);
         }
     }
@@ -46,7 +46,7 @@ class ErrorState : public FairMQDevice
     {
         std::string state("Connect");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "going to change to Error state from " << state << "()";
+            LOG(info) << "going to change to Error state from " << state << "()";
             ChangeState(fair::mq::Transition::ErrorFound);
         }
     }
@@ -55,7 +55,7 @@ class ErrorState : public FairMQDevice
     {
         std::string state("InitTask");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "going to change to Error state from " << state << "()";
+            LOG(info) << "going to change to Error state from " << state << "()";
             ChangeState(fair::mq::Transition::ErrorFound);
         }
     }
@@ -64,7 +64,7 @@ class ErrorState : public FairMQDevice
     {
         std::string state("PreRun");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "going to change to Error state from " << state << "()";
+            LOG(info) << "going to change to Error state from " << state << "()";
             ChangeState(fair::mq::Transition::ErrorFound);
         }
     }
@@ -73,7 +73,7 @@ class ErrorState : public FairMQDevice
     {
         std::string state("Run");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "going to change to Error state from " << state << "()";
+            LOG(info) << "going to change to Error state from " << state << "()";
             ChangeState(fair::mq::Transition::ErrorFound);
         }
     }
@@ -82,7 +82,7 @@ class ErrorState : public FairMQDevice
     {
         std::string state("PostRun");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "going to change to Error state from " << state << "()";
+            LOG(info) << "going to change to Error state from " << state << "()";
             ChangeState(fair::mq::Transition::ErrorFound);
         }
     }
@@ -91,7 +91,7 @@ class ErrorState : public FairMQDevice
     {
         std::string state("ResetTask");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "going to change to Error state from " << state << "()";
+            LOG(info) << "going to change to Error state from " << state << "()";
             ChangeState(fair::mq::Transition::ErrorFound);
         }
     }
@@ -100,7 +100,7 @@ class ErrorState : public FairMQDevice
     {
         std::string state("Reset");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "going to change to Error state from " << state << "()";
+            LOG(info) << "going to change to Error state from " << state << "()";
             ChangeState(fair::mq::Transition::ErrorFound);
         }
     }

test/helper/devices/TestSignals.h

@@ -29,7 +29,7 @@ class Signals : public FairMQDevice
     {
         std::string state("Init");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "raising SIGINT from " << state << "()";
+            LOG(info) << "raising SIGINT from " << state << "()";
             raise(SIGINT);
         }
     }
@@ -37,7 +37,7 @@ class Signals : public FairMQDevice
     {
         std::string state("Bind");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "raising SIGINT from " << state << "()";
+            LOG(info) << "raising SIGINT from " << state << "()";
             raise(SIGINT);
         }
     }
@@ -45,7 +45,7 @@ class Signals : public FairMQDevice
     {
         std::string state("Connect");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "raising SIGINT from " << state << "()";
+            LOG(info) << "raising SIGINT from " << state << "()";
             raise(SIGINT);
         }
     }
@@ -54,7 +54,7 @@ class Signals : public FairMQDevice
     {
         std::string state("InitTask");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "raising SIGINT from " << state << "()";
+            LOG(info) << "raising SIGINT from " << state << "()";
             raise(SIGINT);
         }
     }
@@ -63,7 +63,7 @@ class Signals : public FairMQDevice
     {
         std::string state("PreRun");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "raising SIGINT from " << state << "()";
+            LOG(info) << "raising SIGINT from " << state << "()";
             raise(SIGINT);
         }
     }
@@ -72,7 +72,7 @@ class Signals : public FairMQDevice
     {
         std::string state("Run");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "raising SIGINT from " << state << "()";
+            LOG(info) << "raising SIGINT from " << state << "()";
             raise(SIGINT);
         }
     }
@@ -81,7 +81,7 @@ class Signals : public FairMQDevice
     {
         std::string state("PostRun");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "raising SIGINT from " << state << "()";
+            LOG(info) << "raising SIGINT from " << state << "()";
             raise(SIGINT);
         }
     }
@@ -90,7 +90,7 @@ class Signals : public FairMQDevice
     {
         std::string state("ResetTask");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "raising SIGINT from " << state << "()";
+            LOG(info) << "raising SIGINT from " << state << "()";
             raise(SIGINT);
         }
     }
@@ -99,7 +99,7 @@ class Signals : public FairMQDevice
     {
         std::string state("Reset");
         if (std::string::npos != GetId().find("_" + state + "_")) {
-            LOG(debug) << "raising SIGINT from " << state << "()";
+            LOG(info) << "raising SIGINT from " << state << "()";
             raise(SIGINT);
         }
     }

test/helper/devices/TestTransferTimeout.h

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2015-2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2015static_cast<int>(TransferResult::timeout017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH ) *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -24,20 +24,20 @@ class TransferTimeout : public FairMQDevice
   protected:
     auto Run() -> void override
     {
-        bool sendMsgCancelingAfter100ms = false;
-        bool receiveMsgCancelingAfter100ms = false;
+        bool sendMsgCancelingAfter200ms = false;
+        bool receiveMsgCancelingAfter200ms = false;
 
         bool sendMsgCancelingAfter0ms = false;
         bool receiveMsgCancelingAfter0ms = false;
 
-        bool send1PartCancelingAfter100ms = false;
-        bool receive1PartCancelingAfter100ms = false;
+        bool send1PartCancelingAfter200ms = false;
+        bool receive1PartCancelingAfter200ms = false;
 
         bool send1PartCancelingAfter0ms = false;
         bool receive1PartCancelingAfter0ms = false;
 
-        bool send2PartsCancelingAfter100ms = false;
-        bool receive2PartsCancelingAfter100ms = false;
+        bool send2PartsCancelingAfter200ms = false;
+        bool receive2PartsCancelingAfter200ms = false;
 
         bool send2PartsCancelingAfter0ms = false;
         bool receive2PartsCancelingAfter0ms = false;
@@ -45,28 +45,28 @@ class TransferTimeout : public FairMQDevice
         FairMQMessagePtr msg1(NewMessage());
         FairMQMessagePtr msg2(NewMessage());
 
-        if (Send(msg1, "data-out", 0, 100) == -2) {
-            LOG(info) << "send msg canceled (100ms)";
-            sendMsgCancelingAfter100ms = true;
+        if (Send(msg1, "data-out", 0, 200) == static_cast<int>(TransferResult::timeout)) {
+            LOG(info) << "send msg canceled (200ms)";
+            sendMsgCancelingAfter200ms = true;
         } else {
-            LOG(error) << "send msg did not cancel (100ms)";
+            LOG(error) << "send msg did not cancel (200ms)";
         }
 
-        if (Receive(msg2, "data-in", 0, 100) == -2) {
-            LOG(info) << "receive msg canceled (100ms)";
-            receiveMsgCancelingAfter100ms = true;
+        if (Receive(msg2, "data-in", 0, 200) == static_cast<int>(TransferResult::timeout)) {
+            LOG(info) << "receive msg canceled (200ms)";
+            receiveMsgCancelingAfter200ms = true;
         } else {
-            LOG(error) << "receive msg did not cancel (100ms)";
+            LOG(error) << "receive msg did not cancel (200ms)";
         }
 
-        if (Send(msg1, "data-out", 0, 0) == -2) {
+        if (Send(msg1, "data-out", 0, 0) == static_cast<int>(TransferResult::timeout)) {
             LOG(info) << "send msg canceled (0ms)";
             sendMsgCancelingAfter0ms = true;
         } else {
             LOG(error) << "send msg did not cancel (0ms)";
         }
 
-        if (Receive(msg2, "data-in", 0, 0) == -2) {
+        if (Receive(msg2, "data-in", 0, 0) == static_cast<int>(TransferResult::timeout)) {
             LOG(info) << "receive msg canceled (0ms)";
             receiveMsgCancelingAfter0ms = true;
         } else {
@@ -77,28 +77,28 @@ class TransferTimeout : public FairMQDevice
         parts1.AddPart(NewMessage(10));
         FairMQParts parts2;
 
-        if (Send(parts1, "data-out", 0, 100) == -2) {
-            LOG(info) << "send 1 part canceled (100ms)";
-            send1PartCancelingAfter100ms = true;
+        if (Send(parts1, "data-out", 0, 200) == static_cast<int>(TransferResult::timeout)) {
+            LOG(info) << "send 1 part canceled (200ms)";
+            send1PartCancelingAfter200ms = true;
         } else {
-            LOG(error) << "send 1 part did not cancel (100ms)";
+            LOG(error) << "send 1 part did not cancel (200ms)";
         }
 
-        if (Receive(parts2, "data-in", 0, 100) == -2) {
-            LOG(info) << "receive 1 part canceled (100ms)";
-            receive1PartCancelingAfter100ms = true;
+        if (Receive(parts2, "data-in", 0, 200) == static_cast<int>(TransferResult::timeout)) {
+            LOG(info) << "receive 1 part canceled (200ms)";
+            receive1PartCancelingAfter200ms = true;
         } else {
-            LOG(error) << "receive 1 part did not cancel (100ms)";
+            LOG(error) << "receive 1 part did not cancel (200ms)";
         }
 
-        if (Send(parts1, "data-out", 0, 0) == -2) {
+        if (Send(parts1, "data-out", 0, 0) == static_cast<int>(TransferResult::timeout)) {
             LOG(info) << "send 1 part canceled (0ms)";
             send1PartCancelingAfter0ms = true;
         } else {
             LOG(error) << "send 1 part did not cancel (0ms)";
         }
 
-        if (Receive(parts2, "data-in", 0, 0) == -2) {
+        if (Receive(parts2, "data-in", 0, 0) == static_cast<int>(TransferResult::timeout)) {
             LOG(info) << "receive 1 part canceled (0ms)";
             receive1PartCancelingAfter0ms = true;
         } else {
@@ -110,44 +110,44 @@ class TransferTimeout : public FairMQDevice
         parts3.AddPart(NewMessage(10));
         FairMQParts parts4;
 
-        if (Send(parts3, "data-out", 0, 100) == -2) {
-            LOG(info) << "send 2 parts canceled (100ms)";
-            send2PartsCancelingAfter100ms = true;
+        if (Send(parts3, "data-out", 0, 200) == static_cast<int>(TransferResult::timeout)) {
+            LOG(info) << "send 2 parts canceled (200ms)";
+            send2PartsCancelingAfter200ms = true;
         } else {
-            LOG(error) << "send 2 parts did not cancel (100ms)";
+            LOG(error) << "send 2 parts did not cancel (200ms)";
         }
 
-        if (Receive(parts4, "data-in", 0, 100) == -2) {
-            LOG(info) << "receive 2 parts canceled (100ms)";
-            receive2PartsCancelingAfter100ms = true;
+        if (Receive(parts4, "data-in", 0, 200) == static_cast<int>(TransferResult::timeout)) {
+            LOG(info) << "receive 2 parts canceled (200ms)";
+            receive2PartsCancelingAfter200ms = true;
         } else {
-            LOG(error) << "receive 2 parts did not cancel (100ms)";
+            LOG(error) << "receive 2 parts did not cancel (200ms)";
         }
 
-        if (Send(parts3, "data-out", 0, 0) == -2) {
+        if (Send(parts3, "data-out", 0, 0) == static_cast<int>(TransferResult::timeout)) {
             LOG(info) << "send 2 parts canceled (0ms)";
             send2PartsCancelingAfter0ms = true;
         } else {
             LOG(error) << "send 2 parts did not cancel (0ms)";
         }
 
-        if (Receive(parts4, "data-in", 0, 0) == -2) {
+        if (Receive(parts4, "data-in", 0, 0) == static_cast<int>(TransferResult::timeout)) {
             LOG(info) << "receive 2 parts canceled (0ms)";
             receive2PartsCancelingAfter0ms = true;
         } else {
             LOG(error) << "receive 2 parts did not cancel (0ms)";
         }
 
-        if (sendMsgCancelingAfter100ms &&
-            receiveMsgCancelingAfter100ms &&
+        if (sendMsgCancelingAfter200ms &&
+            receiveMsgCancelingAfter200ms &&
             sendMsgCancelingAfter0ms &&
             receiveMsgCancelingAfter0ms &&
-            send1PartCancelingAfter100ms &&
-            receive1PartCancelingAfter100ms &&
+            send1PartCancelingAfter200ms &&
+            receive1PartCancelingAfter200ms &&
             send1PartCancelingAfter0ms &&
             receive1PartCancelingAfter0ms &&
-            send2PartsCancelingAfter100ms &&
-            receive2PartsCancelingAfter100ms &&
+            send2PartsCancelingAfter200ms &&
+            receive2PartsCancelingAfter200ms &&
             send2PartsCancelingAfter0ms &&
             receive2PartsCancelingAfter0ms)
         {

test/memory_resources/_memory_resources.cxx

@@ -6,66 +6,83 @@
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#include <cstring>
 #include <fairmq/FairMQTransportFactory.h>
 #include <fairmq/MemoryResourceTools.h>
+#include <fairmq/ProgOptions.h>
+#include <fairmq/Tools.h>
+
+#include <boost/container/pmr/polymorphic_allocator.hpp>
+
 #include <gtest/gtest.h>
+
+#include <cstring>
 #include <vector>
 
-namespace {
+namespace
+{
 
 using namespace std;
 using namespace fair::mq;
-using factoryType = std::shared_ptr<FairMQTransportFactory>;
 
-factoryType factoryZMQ = FairMQTransportFactory::CreateTransportFactory("zeromq");
-factoryType factorySHM = FairMQTransportFactory::CreateTransportFactory("shmem");
+using FactoryType = shared_ptr<FairMQTransportFactory>;
 
-struct testData
+struct TestData
 {
     int i{1};
     static int nallocated;
     static int nallocations;
     static int ndeallocations;
-    testData()
+
+    TestData()
     {
         ++nallocated;
         ++nallocations;
     }
-    testData(const testData& in)
+
+    TestData(const TestData& in)
         : i{in.i}
     {
         ++nallocated;
         ++nallocations;
     }
-    testData(const testData&& in)
+
+    TestData(const TestData&& in)
         : i{in.i}
     {
         ++nallocated;
         ++nallocations;
     }
-    testData(int in)
+
+    TestData(int in)
         : i{in}
     {
         ++nallocated;
         ++nallocations;
     }
-    ~testData()
+
+    ~TestData()
     {
         --nallocated;
         ++ndeallocations;
     }
 };
 
-int testData::nallocated = 0;
-int testData::nallocations = 0;
-int testData::ndeallocations = 0;
+int TestData::nallocated = 0;
+int TestData::nallocations = 0;
+int TestData::ndeallocations = 0;
 
-auto allocZMQ = factoryZMQ -> GetMemoryResource();
-auto allocSHM = factorySHM -> GetMemoryResource();
-
-TEST(MemoryResources, transportallocatormap)
+TEST(MemoryResources, transportAllocatorMap)
 {
+    size_t session{tools::UuidHash()};
+    ProgOptions config;
+    config.SetProperty<string>("session", to_string(session));
+
+    FactoryType factoryZMQ = FairMQTransportFactory::CreateTransportFactory("zeromq", fair::mq::tools::Uuid(), &config);
+    FactoryType factorySHM = FairMQTransportFactory::CreateTransportFactory("shmem", fair::mq::tools::Uuid(), &config);
+
+    auto allocZMQ = factoryZMQ->GetMemoryResource();
+    auto allocSHM = factorySHM->GetMemoryResource();
+
     EXPECT_TRUE(allocZMQ != nullptr && allocSHM != allocZMQ);
     auto _tmp = factoryZMQ->GetMemoryResource();
     EXPECT_TRUE(_tmp == allocZMQ);
@@ -75,29 +92,45 @@ using namespace fair::mq::pmr;
 
 TEST(MemoryResources, allocator)
 {
-    testData::nallocations = 0;
-    testData::ndeallocations = 0;
+    TestData::nallocations = 0;
+    TestData::ndeallocations = 0;
+
+    size_t session{tools::UuidHash()};
+    ProgOptions config;
+    config.SetProperty<string>("session", to_string(session));
+
+    FactoryType factoryZMQ = FairMQTransportFactory::CreateTransportFactory("zeromq", fair::mq::tools::Uuid(), &config);
+
+    auto allocZMQ = factoryZMQ->GetMemoryResource();
 
     {
-        std::vector<testData, polymorphic_allocator<testData>> v(
-            polymorphic_allocator<testData>{allocZMQ});
+        std::vector<TestData, polymorphic_allocator<TestData>> v(polymorphic_allocator<TestData>{allocZMQ});
         v.reserve(3);
         EXPECT_TRUE(v.capacity() == 3);
         EXPECT_TRUE(allocZMQ->getNumberOfMessages() == 1);
         v.emplace_back(1);
         v.emplace_back(2);
         v.emplace_back(3);
-        EXPECT_TRUE((fair::mq::byte*)&(*v.end()) - (fair::mq::byte*)&(*v.begin()) == 3 * sizeof(testData));
-        EXPECT_TRUE(testData::nallocated == 3);
+        EXPECT_TRUE((fair::mq::byte*)&(*v.end()) - (fair::mq::byte*)&(*v.begin()) == 3 * sizeof(TestData));
+        EXPECT_TRUE(TestData::nallocated == 3);
     }
-    EXPECT_TRUE(testData::nallocated == 0);
-    EXPECT_TRUE(testData::nallocations == testData::ndeallocations);
+    EXPECT_TRUE(TestData::nallocated == 0);
+    EXPECT_TRUE(TestData::nallocations == TestData::ndeallocations);
 }
 
 TEST(MemoryResources, getMessage)
 {
-    testData::nallocations = 0;
-    testData::ndeallocations = 0;
+    TestData::nallocations = 0;
+    TestData::ndeallocations = 0;
+
+    size_t session{tools::UuidHash()};
+    ProgOptions config;
+    config.SetProperty<string>("session", to_string(session));
+
+    FactoryType factoryZMQ = FairMQTransportFactory::CreateTransportFactory("zeromq", fair::mq::tools::Uuid(), &config);
+    FactoryType factorySHM = FairMQTransportFactory::CreateTransportFactory("shmem", fair::mq::tools::Uuid(), &config);
+
+    auto allocZMQ = factoryZMQ->GetMemoryResource();
 
     FairMQMessagePtr message{nullptr};
 
@@ -105,8 +138,7 @@ TEST(MemoryResources, getMessage)
 
     // test message creation on the same channel it was allocated with
     {
-        std::vector<testData, polymorphic_allocator<testData>> v(
-            polymorphic_allocator<testData>{allocZMQ});
+        std::vector<TestData, polymorphic_allocator<TestData>> v(polymorphic_allocator<TestData>{allocZMQ});
         v.emplace_back(1);
         v.emplace_back(2);
         v.emplace_back(3);
@@ -115,14 +147,13 @@ TEST(MemoryResources, getMessage)
         EXPECT_TRUE(message != nullptr);
         EXPECT_TRUE(message->GetData() == vectorBeginPtr);
     }
-    EXPECT_TRUE(message->GetSize() == 3 * sizeof(testData));
+    EXPECT_TRUE(message->GetSize() == 3 * sizeof(TestData));
     messageArray = static_cast<int*>(message->GetData());
     EXPECT_TRUE(messageArray[0] == 1 && messageArray[1] == 2 && messageArray[2] == 3);
 
     // test message creation on a different channel than it was allocated with
     {
-        std::vector<testData, polymorphic_allocator<testData>> v(
-            polymorphic_allocator<testData>{allocZMQ});
+        std::vector<TestData, polymorphic_allocator<TestData>> v(polymorphic_allocator<TestData>{allocZMQ});
         v.emplace_back(4);
         v.emplace_back(5);
         v.emplace_back(6);
@@ -132,7 +163,7 @@ TEST(MemoryResources, getMessage)
         EXPECT_TRUE(message->GetData() != vectorBeginPtr);
     }
 
-    EXPECT_TRUE(message->GetSize() == 3 * sizeof(testData));
+    EXPECT_TRUE(message->GetSize() == 3 * sizeof(TestData));
     messageArray = static_cast<int*>(message->GetData());
     EXPECT_TRUE(messageArray[0] == 4 && messageArray[1] == 5 && messageArray[2] == 6);
 }