Retry on EINTR in blocking zmq calls

To allow potential optimizations (e.g. huge pages)

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "extern/asio"]
 	path = extern/asio
 	url = https://github.com/chriskohlhoff/asio
+[submodule "extern/PicoSHA2"]
+	path = extern/PicoSHA2
+	url = https://github.com/okdshin/PicoSHA2

CMakeLists.txt

@@ -35,8 +35,6 @@ fairmq_build_option(BUILD_FAIRMQ            "Build FairMQ library and devices."
                                              DEFAULT ON)
 fairmq_build_option(BUILD_TESTING           "Build tests."
                                              DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
-fairmq_build_option(BUILD_NANOMSG_TRANSPORT "Build nanomsg transport."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
 fairmq_build_option(BUILD_OFI_TRANSPORT     "Build experimental OFI transport."
                                              DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
 fairmq_build_option(BUILD_SDK_COMMANDS      "Build the FairMQ SDK commands."
@@ -53,6 +51,8 @@ fairmq_build_option(BUILD_DOCS              "Build FairMQ documentation."
                                              DEFAULT OFF)
 fairmq_build_option(FAST_BUILD              "Fast production build. Not recommended for development."
                                              DEFAULT OFF)
+fairmq_build_option(USE_EXTERNAL_GTEST      "Do not use bundled GTest. Not recommended."
+                                             DEFAULT OFF)
 ################################################################################
 
 
@@ -65,11 +65,6 @@ set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
 set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_package(Threads REQUIRED)
 
-if(BUILD_NANOMSG_TRANSPORT)
-  find_package2(PRIVATE nanomsg REQUIRED)
-  set(PROJECT_nanomsg_VERSION 1.1.3) # Once upstream releases 1.1.5, we should bump again and use version check
-endif()
-
 if(BUILD_OFI_TRANSPORT)
   find_package2(PRIVATE asiofi REQUIRED
     VERSION 0.3.1
@@ -79,12 +74,6 @@ if(BUILD_OFI_TRANSPORT)
   )
 endif()
 
-if(BUILD_NANOMSG_TRANSPORT)
-  find_package2(PRIVATE msgpack REQUIRED
-    VERSION 3.1.0
-  )
-endif()
-
 if(BUILD_SDK_COMMANDS)
   find_package2(PRIVATE Flatbuffers REQUIRED)
 endif()
@@ -105,7 +94,7 @@ endif()
 
 if(BUILD_FAIRMQ OR BUILD_SDK)
   find_package2(PUBLIC FairLogger REQUIRED
-    VERSION 1.2.0
+    VERSION 1.6.0
   )
 
   foreach(dep IN LISTS FairLogger_PACKAGE_DEPENDENCIES)
@@ -157,10 +146,14 @@ if(BUILD_FAIRMQ)
   find_package2(PRIVATE ZeroMQ REQUIRED
     VERSION 4.1.4
   )
+  build_bundled(PicoSHA2 extern/PicoSHA2)
+  find_package2(PRIVATE PicoSHA2 REQUIRED)
 endif()
 
 if(BUILD_TESTING)
-  find_package2(PRIVATE GTest VERSION 1.7.0)
+  if(USE_EXTERNAL_GTEST)
+    find_package2(PRIVATE GTest VERSION 1.7.0)
+  endif()
   if(NOT GTest_FOUND)
     build_bundled(GTest extern/googletest)
     find_package2(PRIVATE GTest REQUIRED)
@@ -215,9 +208,6 @@ endif()
 if(BUILD_PMIX_PLUGIN)
   list(APPEND PROJECT_PACKAGE_COMPONENTS pmix_plugin)
 endif()
-if(BUILD_NANOMSG_TRANSPORT)
-  list(APPEND PROJECT_PACKAGE_COMPONENTS nanomsg_transport)
-endif()
 if(BUILD_OFI_TRANSPORT)
   list(APPEND PROJECT_PACKAGE_COMPONENTS ofi_transport)
 endif()
@@ -320,16 +310,10 @@ if(PROJECT_PACKAGE_DEPENDENCIES)
       endif()
     elseif(${dep} STREQUAL GTest)
       get_filename_component(prefix ${GTEST_INCLUDE_DIRS}/.. ABSOLUTE)
-    elseif(${dep} STREQUAL msgpack)
-      get_target_property(msgpack_include msgpackc-cxx INTERFACE_INCLUDE_DIRECTORIES)
-      get_filename_component(prefix ${msgpack_include}/.. ABSOLUTE)
     elseif(${dep} STREQUAL asiofi)
       set(prefix ${asiofi_ROOT})
     elseif(${dep} STREQUAL OFI)
       get_filename_component(prefix ${${dep}_INCLUDE_DIRS}/.. ABSOLUTE)
-    elseif(${dep} STREQUAL nanomsg)
-      get_target_property(nn_include nanomsg INTERFACE_INCLUDE_DIRECTORIES)
-      get_filename_component(prefix ${nn_include}/.. ABSOLUTE)
     elseif(${dep} STREQUAL DDS)
       set(prefix "${DDS_INSTALL_PREFIX}")
     elseif(${dep} STREQUAL Boost)
@@ -378,12 +362,6 @@ else()
   set(tests_summary "${BRed} NO${CR}    (enable with ${BMagenta}-DBUILD_TESTING=ON${CR})")
 endif()
 message(STATUS "  ${BWhite}tests${CR}              ${tests_summary}")
-if(BUILD_NANOMSG_TRANSPORT)
-  set(nn_summary "${BGreen}YES${CR}    (disable with ${BMagenta}-DBUILD_NANOMSG_TRANSPORT=OFF${CR})")
-else()
-  set(nn_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_NANOMSG_TRANSPORT=ON${CR})")
-endif()
-message(STATUS "  ${BWhite}nanomsg_transport${CR}  ${nn_summary}")
 if(BUILD_OFI_TRANSPORT)
   set(ofi_summary "${BGreen}YES${CR}    EXPERIMENTAL (requires C++14) (disable with ${BMagenta}-DBUILD_OFI_TRANSPORT=OFF${CR})")
 else()

COPYRIGHT

@@ -23,6 +23,10 @@ Files: extern/asio
 Copyright: 2003-2019, Christopher M. Kohlhoff (chris at kohlhoff dot com)
 License: BSL-1.0
 
+Files: extern/PicoSHA2
+Copyright: 2017 okdshin
+License: MIT
+
 License: LGPL-3.0-only
   [see LICENSE file]
 
@@ -102,3 +106,22 @@ License: BSL-1.0
   FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
   ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
   DEALINGS IN THE SOFTWARE.
+
+License: MIT
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.

FairMQTest.cmake

@@ -26,13 +26,13 @@ Set(configure_options "${configure_options};-DCTEST_USE_LAUNCHERS=${CTEST_USE_LA
 
 Set(configure_options "${configure_options};-DDISABLE_COLOR=ON")
 Set(configure_options "${configure_options};-DCMAKE_PREFIX_PATH=$ENV{SIMPATH}")
-Set(configure_options "${configure_options};-DBUILD_NANOMSG_TRANSPORT=ON")
 # Set(configure_options "${configure_options};-DBUILD_OFI_TRANSPORT=ON")
 Set(configure_options "${configure_options};-DBUILD_DDS_PLUGIN=ON")
 Set(configure_options "${configure_options};-DBUILD_SDK=ON")
 Set(configure_options "${configure_options};-DBUILD_SDK_COMMANDS=ON")
 Set(configure_options "${configure_options};-DFAST_BUILD=ON")
 Set(configure_options "${configure_options};-DCOTIRE_MAXIMUM_NUMBER_OF_UNITY_INCLUDES=-j$ENV{number_of_processors}")
+Set(configure_options "${configure_options};-DBoost_NO_BOOST_CMAKE=ON")
 
 Set(EXTRA_FLAGS $ENV{EXTRA_FLAGS})
 If(EXTRA_FLAGS)

README.md

@@ -21,7 +21,7 @@ FairMQ is designed to help implementing large-scale data processing workflows ne
 The core of FairMQ provides an abstract asynchronous message passing API with scalability protocols
 inspired by [ZeroMQ](https://github.com/zeromq/libzmq) (e.g. PUSH/PULL, PUB/SUB).
 FairMQ provides multiple implementations for its API (so-called "transports",
-e.g. `zeromq`, `shmem`, `nanomsg`, and `ofi` (in development)) to cover a variety of use cases
+e.g. `zeromq`, `shmem` and `ofi` (in development)) to cover a variety of use cases
 (e.g. inter-thread, inter-process, inter-node communication) and machines (e.g. Ethernet, Infiniband).
 In addition to this core functionality FairMQ provides a framework for creating "devices" - actors which
 are communicating through message passing. FairMQ does not only allow the user to use different transport but also to mix them; i.e: A Device can communicate using different transport on different channels at the same time. Device execution is modelled as a simple state machine that
@@ -47,7 +47,7 @@ cmake --build fairmq_build --target install
 
 Please consult the [manpages of your CMake version](https://cmake.org/cmake/help/latest/manual/cmake.1.html) for more options.
 
-If dependencies are not installed in standard system directories, you can hint the installation location via `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...`. `{DEPENDENCY}` can be `GTEST`, `BOOST`, `FAIRLOGGER`, `ZEROMQ`, `MSGPACK`, `NANOMSG`, `OFI`, `PMIX`, `ASIO`, `ASIOFI` or `DDS` (`*_ROOT` variables can also be environment variables).
+If dependencies are not installed in standard system directories, you can hint the installation location via `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...`. `{DEPENDENCY}` can be `GTEST`, `BOOST`, `FAIRLOGGER`, `ZEROMQ`, `OFI`, `PMIX`, `ASIO`, `ASIOFI` or `DDS` (`*_ROOT` variables can also be environment variables).
 
 ## Usage
 
@@ -86,7 +86,7 @@ If your project shares a dependency with FairMQ or if you want to omit a certain
 Optionally, you can require certain FairMQ package components and a minimum version:
 
 ```cmake
-find_package(FairMQ 1.1.0 COMPONENTS nanomsg_transport dds_plugin)
+find_package(FairMQ 1.1.0 COMPONENTS dds_plugin)
 ```
 
 When building FairMQ, CMake will print a summary table of all available package components.
@@ -101,8 +101,6 @@ When building FairMQ, CMake will print a summary table of all available package
   * [Doxygen](http://www.doxygen.org/)
   * [FairLogger](https://github.com/FairRootGroup/FairLogger)
   * [GTest](https://github.com/google/googletest) (optionally bundled)
-  * [Msgpack](https://msgpack.org/index.html)
-  * [nanomsg](http://nanomsg.org/)
   * [PMIx](https://pmix.org/)
   * [ZeroMQ](http://zeromq.org/)
 
@@ -117,7 +115,6 @@ On command line:
   * `-DDISABLE_COLOR=ON` disables coloured console output.
   * `-DBUILD_TESTING=OFF` disables building of tests.
   * `-DBUILD_EXAMPLES=OFF` disables building of examples.
-  * `-DBUILD_NANOMSG_TRANSPORT=ON` enables building of nanomsg transport.
   * `-DBUILD_OFI_TRANSPORT=ON` enables building of the experimental OFI transport.
   * `-DBUILD_DDS_PLUGIN=ON` enables building of the DDS plugin.
   * `-DBUILD_PMIX_PLUGIN=ON` enables building of the PMIx plugin.
@@ -182,4 +179,3 @@ After the `find_package(FairMQ)` call the following CMake variables are defined:
        1. [DDS](docs/Plugins.md#731-dds)
        2. [PMIx](docs/Plugins.md#732-pmix)
 8. [Controller SDK](docs/SDK.md)
-

cmake/FairMQLib.cmake

@@ -143,15 +143,15 @@ macro(set_fairmq_defaults)
   set(PROJECT_EXPORT_SET ${PROJECT_NAME}Targets)
 
   # Configure build types
-  set(CMAKE_CONFIGURATION_TYPES "Debug" "Release" "RelWithDebInfo" "Nightly" "Profile" "Experimental" "AdressSan" "ThreadSan")
+  set(CMAKE_CONFIGURATION_TYPES "Debug" "Release" "RelWithDebInfo" "Nightly" "Profile" "Experimental" "AddressSan" "ThreadSan")
   set(_warnings "-Wshadow -Wall -Wextra -Wpedantic")
-  set(CMAKE_CXX_FLAGS_DEBUG          "-g ${_warnings}")
+  set(CMAKE_CXX_FLAGS_DEBUG          "-Og -g ${_warnings}")
   set(CMAKE_CXX_FLAGS_RELEASE        "-O2 -DNDEBUG")
   set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g ${_warnings} -DNDEBUG")
   set(CMAKE_CXX_FLAGS_NIGHTLY        "-O2 -g ${_warnings}")
   set(CMAKE_CXX_FLAGS_PROFILE        "-g3 ${_warnings} -fno-inline -ftest-coverage -fprofile-arcs")
   set(CMAKE_CXX_FLAGS_EXPERIMENTAL   "-O2 -g ${_warnings} -DNDEBUG")
-  set(CMAKE_CXX_FLAGS_ADRESSSAN      "-O2 -g ${_warnings} -fsanitize=address -fno-omit-frame-pointer")
+  set(CMAKE_CXX_FLAGS_ADDRESSSAN     "-O2 -g ${_warnings} -fsanitize=address -fno-omit-frame-pointer")
   set(CMAKE_CXX_FLAGS_THREADSAN      "-O2 -g ${_warnings} -fsanitize=thread")
   unset(_warnings)
 
@@ -498,6 +498,8 @@ function(build_bundled package bundle)
     set(${package}_BUILD_INCLUDE_DIR ${${package}_SOURCE_DIR}/asio/include CACHE PATH "Bundled ${package} build-interface include dir")
     set(${package}_INSTALL_INCLUDE_DIR ${PROJECT_INSTALL_INCDIR}/bundled CACHE PATH "Bundled ${package} install-interface include dir")
     set(${package}_ROOT ${${package}_SOURCE_DIR}/asio)
+  elseif(${package} STREQUAL PicoSHA2)
+    set(${package}_ROOT ${${package}_SOURCE_DIR})
   endif()
 
   string(TOUPPER ${package} package_upper)

cmake/FindPicoSHA2.cmake

@@ -0,0 +1,21 @@
+################################################################################
+#    Copyright (C) 2020 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+find_path(PicoSHA2_INCLUDE_DIR NAMES picosha2.h)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(PicoSHA2
+  REQUIRED_VARS PicoSHA2_INCLUDE_DIR
+)
+
+if(PicoSHA2_FOUND)
+  add_library(PicoSHA2 INTERFACE IMPORTED)
+  set_target_properties(PicoSHA2 PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${PicoSHA2_INCLUDE_DIR}"
+  )
+endif()

docs/Device.md

@@ -18,7 +18,7 @@ Topology configuration is currently happening via setup scripts. This is very ru
 
 ## 1.2 Communication Patterns
 
-FairMQ devices communicate via the communication patterns offered by ZeroMQ (or nanomsg): PUSH-PULL, PUB-SUB, REQ-REP, PAIR, [more info here](http://api.zeromq.org/4-0:zmq-socket). Each transport may provide further patterns.
+FairMQ devices communicate via the communication patterns offered by ZeroMQ: PUSH-PULL, PUB-SUB, REQ-REP, PAIR, [more info here](http://api.zeromq.org/4-0:zmq-socket). Each transport may provide further patterns.
 
 ## 1.3 State Machine
 

docs/Transport.md

@@ -2,7 +2,7 @@
 
 # 2. Transport Interface
 
-The communication layer is available through the transport interface. Three interface implementations are currently available. Main implementation uses the [ZeroMQ](http://zeromq.org) library. Alternative implementation relies on the [nanomsg](http://nanomsg.org) library. Third transport implementation is using shared memory via boost::interprocess & ZeroMQ combination.
+The communication layer is available through the transport interface. Three interface implementations are currently available. Main implementation uses the [ZeroMQ](http://zeromq.org) library. Second transport implementation is using shared memory via boost::interprocess & ZeroMQ combination.
 
 Here is an overview to give an idea how the interface is implemented:
 
@@ -10,20 +10,20 @@ Here is an overview to give an idea how the interface is implemented:
 
 Currently, the transports have been tested to work with these communication patterns:
 
-|               | zeromq | nanomsg | shmem |
-| ------------- |--------| ------- | ----- |
-| PAIR          | yes    | yes     | yes   |
-| PUSH/PULL     | yes    | yes     | yes   |
-| PUB/SUB       | yes    | yes     | no    |
-| REQ/REP       | yes    | yes     | yes   |
+|               | zeromq | shmem |
+| ------------- |--------| ----- |
+| PAIR          | yes    | yes   |
+| PUSH/PULL     | yes    | yes   |
+| PUB/SUB       | yes    | no    |
+| REQ/REP       | yes    | yes   |
 
 The next table shows the supported address types for each transport implementation:
 
-|             | zeromq | nanomsg | shmem | comment                                       |
-| ----------- | ------ | ------- | ----- | --------------------------------------------- |
-| `inproc://` | yes    | yes     | yes   | in process: useful for unit testing           |
-| `ipc://`    | yes    | yes     | yes   | inter process comm: useful on single machine  |
-| `tcp://`    | yes    | yes     | yes   | useful for any communication, local or remote |
+|             | zeromq | shmem | comment                                       |
+| ----------- | ------ | ----- | --------------------------------------------- |
+| `inproc://` | yes    | yes   | in process: useful for unit testing           |
+| `ipc://`    | yes    | yes   | inter process comm: useful on single machine  |
+| `tcp://`    | yes    | yes   | useful for any communication, local or remote |
 
 ## 2.1 Message
 

examples/1-1/CMakeLists.txt

@@ -34,11 +34,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-1-1.sh.in ${CMAKE_CURRENT_BIN
 add_test(NAME Example.1-1.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh zeromq)
 set_tests_properties(Example.1-1.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.1-1.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh nanomsg)
-  set_tests_properties(Example.1-1.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
-endif()
-
 add_test(NAME Example.1-1.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh shmem)
 set_tests_properties(Example.1-1.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 

examples/1-1/Sampler.cxx

@@ -17,8 +17,7 @@ Sampler::Sampler()
     : fText()
     , fMaxIterations(0)
     , fNumIterations(0)
-{
-}
+{}
 
 void Sampler::InitTask()
 {
@@ -32,25 +31,22 @@ bool Sampler::ConditionalRun()
     // create a copy of the data with new(), that will be deleted after the transfer is complete
     string* text = new string(fText);
 
-    // create message object with a pointer to the data buffer,
-    // its size,
+    // create message object with a pointer to the data buffer, its size,
     // custom deletion function (called when transfer is done),
     // and pointer to the object managing the data buffer
-    FairMQMessagePtr msg(NewMessage(const_cast<char*>(text->c_str()),
-                                    text->length(),
-                                    [](void* /*data*/, void* object) { delete static_cast<string*>(object); },
-                                    text));
+    FairMQMessagePtr msg(NewMessage(
+        const_cast<char*>(text->c_str()),
+        text->length(),
+        [](void* /*data*/, void* object) { delete static_cast<string*>(object); },
+        text));
 
     LOG(info) << "Sending \"" << fText << "\"";
 
-    // in case of error or transfer interruption, return false to go to IDLE state
+    // in case of error or transfer interruption, return false to go to the Ready state
     // successfull transfer will return number of bytes transfered (can be 0 if sending an empty message).
-    if (Send(msg, "data") < 0)
-    {
+    if (Send(msg, "data") < 0) {
         return false;
-    }
-    else if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations)
-    {
+    } else if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
         LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
         return false;
     }
@@ -58,8 +54,6 @@ bool Sampler::ConditionalRun()
     return true;
 }
 
-Sampler::~Sampler()
-{
-}
+Sampler::~Sampler() {}
 
 } // namespace example_1_1

examples/1-1/Sink.cxx

@@ -33,23 +33,22 @@ void Sink::InitTask()
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
 }
 
-// handler is called whenever a message arrives on "data", with a reference to the message and a sub-channel index (here 0)
+// handler is called whenever a message arrives on "data", with a reference to the message and a
+// sub-channel index (here 0)
 bool Sink::HandleData(FairMQMessagePtr& msg, int /*index*/)
 {
     LOG(info) << "Received: \"" << string(static_cast<char*>(msg->GetData()), msg->GetSize()) << "\"";
 
-    if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations)
-    {
+    if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
         LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
         return false;
     }
 
-    // return true if want to be called again (otherwise return false go to IDLE state)
+    // return true if you want the handler to be called again (otherwise return false go to the
+    // Ready state)
     return true;
 }
 
-Sink::~Sink()
-{
-}
+Sink::~Sink() {}
 
 } // namespace example_1_1

examples/1-n-1/CMakeLists.txt

@@ -42,11 +42,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-1-n-1.sh.in ${CMAKE_CURRENT_B
 add_test(NAME Example.1-n-1.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh zeromq)
 set_tests_properties(Example.1-n-1.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.1-n-1.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh nanomsg)
-  set_tests_properties(Example.1-n-1.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
-endif()
-
 add_test(NAME Example.1-n-1.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh shmem)
 set_tests_properties(Example.1-n-1.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 

examples/CMakeLists.txt

@@ -13,9 +13,7 @@ add_subdirectory(copypush)
 add_subdirectory(dds)
 add_subdirectory(multipart)
 add_subdirectory(multiple-channels)
-if(BUILD_NANOMSG_TRANSPORT)
-  add_subdirectory(multiple-transports)
-endif()
+add_subdirectory(multiple-transports)
 add_subdirectory(n-m)
 add_subdirectory(qc)
 add_subdirectory(readout)

examples/README.md

@@ -32,7 +32,7 @@ This example demonstrates how to work with multiple channels and multiplex betwe
 
 ## Multiple Transports
 
-This examples shows how to combine different channel transports (zeromq/nanomsg/shmem) inside of one device and/or topology.
+This examples shows how to combine different channel transports (zeromq/shmem) inside of one device and/or topology.
 
 ## n-m
 
@@ -40,7 +40,7 @@ A topology consisting of three layers of devices: synchronizer -> n * senders ->
 
 ## QC
 
-A topology consisting of 4 devices - Sampler, QCProducer, QCConsumer and Sink. The data flows from Sampler through QCProducer to Sink. On demand - by setting the corresponding configuration property - the QCProducer device will duplicate the data to the QCConsumer device. The property is set by the topology controller, in this example this is the `fairmq-dds-command-ui` utility.
+A topology consisting of 4 devices - Sampler, QCDispatcher, QCTask and Sink. The data flows from Sampler through QCDispatcher to Sink. On demand - by setting the corresponding configuration property - the QCDispatcher device will duplicate the data to the QCTask device. The property is set by the topology controller, in this example this is the `fairmq-dds-command-ui` utility.
 
 ## Readout
 
@@ -52,4 +52,4 @@ This example demonstrates the use of a more advanced feature - UnmanagedRegion,
 
 ## Request & Reply
 
-This topology contains two devices that communicate with each other via the **REQ-REP** pettern. Bidirectional communication via a single socket.
+This topology contains two devices that communicate with each other via the **REQ-REP** pettern. Bidirectional communication via a single socket.

examples/builtin-devices/CMakeLists.txt

@@ -17,22 +17,12 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-builtin-devices.sh.in ${CMAKE
 add_test(NAME Example.BuiltinDevices.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq)
 set_tests_properties(Example.BuiltinDevices.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.BuiltinDevices.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg)
-  set_tests_properties(Example.BuiltinDevices.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
-endif()
-
 add_test(NAME Example.BuiltinDevices.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem)
 set_tests_properties(Example.BuiltinDevices.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
 add_test(NAME Example.BuiltinDevices.multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq true 2)
 set_tests_properties(Example.BuiltinDevices.multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.BuiltinDevices.multipart.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg true 2)
-  set_tests_properties(Example.BuiltinDevices.multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
-endif()
-
 add_test(NAME Example.BuiltinDevices.multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem true 2)
 set_tests_properties(Example.BuiltinDevices.multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 

examples/copypush/CMakeLists.txt

@@ -35,11 +35,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-copypush.sh.in ${CMAKE_CURREN
 add_test(NAME Example.CopyPush.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh zeromq)
 set_tests_properties(Example.CopyPush.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.CopyPush.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh nanomsg)
-  set_tests_properties(Example.CopyPush.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
-endif()
-
 add_test(NAME Example.CopyPush.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh shmem)
 set_tests_properties(Example.CopyPush.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 

examples/multipart/CMakeLists.txt

@@ -34,11 +34,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multipart.sh.in ${CMAKE_CURRE
 add_test(NAME Example.Multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh zeromq)
 set_tests_properties(Example.Multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.Multipart.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh nanomsg)
-  set_tests_properties(Example.Multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
-endif()
-
 add_test(NAME Example.Multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh shmem)
 set_tests_properties(Example.Multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
 

examples/multiple-channels/CMakeLists.txt

@@ -39,11 +39,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multiple-channels.sh.in ${CMA
 add_test(NAME Example.MultipleChannels.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh zeromq)
 set_tests_properties(Example.MultipleChannels.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.MultipleChannels.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh nanomsg)
-  set_tests_properties(Example.MultipleChannels.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
-endif()
-
 # install
 
 install(

examples/multiple-transports/README.md

@@ -1,12 +1,12 @@
 Multiple Transports example
 ===========================
 
-This example demonstrates use of multiple transports (zeromq/nanomsg/shmem) within the same topology and/or device. It is a simple topology consisting of two samplers and a sink. The devices are connected via 3 channels:
+This example demonstrates use of multiple transports (zeromq/shmem) within the same topology and/or device. It is a simple topology consisting of two samplers and a sink. The devices are connected via 3 channels:
 
 ![Multiple Transports example](../../docs/images/example_multiple_transports.png?raw=true "Multiple Transports example")
 
 Each device has main transport that it uses. By default it is ZeroMQ, and can be overriden via the `--transport` cmd option. The device will initialize additional transports if any of the channels have them configured (e.g. as an option to `--channel-config`).
 
-In this example sampler1 and sink are started with `--transport shmem`, making shared memory their main transport, sampler2 with `--transport nanomsg`. Additionally, the ack channel is configured to use zeromq as its transport.
+In this example sampler1 and sink are started with `--transport shmem`, making shared memory their main transport, sampler2 with `--transport zeromq`. Additionally, the ack channel is configured to use zeromq as its transport.
 
 The main two things that a transport does is transfer of data and allocation of memory for the messages. By default, new messages are created via the main device transport. If a message has been created with one transport and is to be transferred with another, it has to be copied into a new message of the target transport. This happens automatically behind the scenes. To avoid this copy the device can create messages via `NewMessageFor(const string& channelName, int subChannelIndex, ...)` method, that creates the messages via the transport of the given channel (check sampler1 and sink for an example) or as the channel directly to create a message.

examples/multiple-transports/fairmq-start-ex-multiple-transports.sh.in

@@ -13,7 +13,7 @@ xterm -geometry 80x30+0+0 -hold -e @EX_BIN_DIR@/$SAMPLER1 &
 SAMPLER2="fairmq-ex-multiple-transports-sampler2"
 SAMPLER2+=" --id sampler2"
 SAMPLER2+=" --severity debug"
-SAMPLER2+=" --transport nanomsg"
+SAMPLER2+=" --transport shmem"
 SAMPLER2+=" --channel-config name=data2,type=push,method=bind,address=tcp://127.0.0.1:5556"
 xterm -geometry 80x30+0+450 -hold -e @EX_BIN_DIR@/$SAMPLER2 &
 
@@ -22,6 +22,6 @@ SINK+=" --id sink1"
 SINK+=" --severity debug"
 SINK+=" --transport shmem"
 SINK+=" --channel-config name=data1,type=pull,method=connect,address=tcp://127.0.0.1:5555"
-SINK+="                  name=data2,type=pull,method=connect,address=tcp://127.0.0.1:5556,transport=nanomsg"
+SINK+="                  name=data2,type=pull,method=connect,address=tcp://127.0.0.1:5556,transport=shmem"
 SINK+="                  name=ack,type=pub,method=connect,address=tcp://127.0.0.1:5557,transport=zeromq"
 xterm -geometry 80x30+500+225 -hold -e @EX_BIN_DIR@/$SINK &

examples/multiple-transports/test-ex-multiple-transports.sh.in

@@ -14,7 +14,7 @@ SINK+=" --max-iterations 1"
 SINK+=" --control static --color false"
 SINK+=" --transport shmem"
 SINK+=" --channel-config name=data1,type=pull,method=connect,address=tcp://127.0.0.1:5555"
-SINK+="                  name=data2,type=pull,method=connect,address=tcp://127.0.0.1:5556,transport=nanomsg"
+SINK+="                  name=data2,type=pull,method=connect,address=tcp://127.0.0.1:5556,transport=zeromq"
 SINK+="                  name=ack,type=pub,method=connect,address=tcp://127.0.0.1:5557,transport=zeromq"
 @CMAKE_CURRENT_BINARY_DIR@/$SINK &
 SINK_PID=$!
@@ -37,7 +37,7 @@ SAMPLER2+=" --session $SESSION"
 SAMPLER2+=" --verbosity veryhigh"
 SAMPLER2+=" --max-iterations 1"
 SAMPLER2+=" --control static --color false"
-SAMPLER2+=" --transport nanomsg"
+SAMPLER2+=" --transport zeromq"
 SAMPLER2+=" --channel-config name=data2,type=push,method=bind,address=tcp://127.0.0.1:5556"
 @CMAKE_CURRENT_BINARY_DIR@/$SAMPLER2 &
 SAMPLER2_PID=$!

examples/n-m/ex-n-m-pair-topology.xml

@@ -10,7 +10,7 @@
         <exe reachable="true">fairmq-ex-n-m-synchronizer --id sync --rate 100 --color false -P dds --channel-config name=sync,type=pub,method=bind</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="write">fmqchan_sync</id>
+            <name access="write">fmqchan_sync</name>
         </properties>
     </decltask>
 
@@ -18,8 +18,8 @@
         <exe reachable="true">fairmq-ex-n-m-sender --id sender%taskIndex% --timeframe-size 100000 --num-receivers ${numReceivers} --color false -P dds --channel-config name=sync,type=sub,method=connect name=data,type=pair,method=connect,numSockets=${numReceivers} --dds-i data:%taskIndex%</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="read">fmqchan_sync</id>
-            <name access="read">fmqchan_data</id>
+            <name access="read">fmqchan_sync</name>
+            <name access="read">fmqchan_data</name>
         </properties>
     </decltask>
 
@@ -27,7 +27,7 @@
         <exe reachable="true">fairmq-ex-n-m-receiver --id receiver%taskIndex% --num-senders ${numSenders} --color false -P dds --max-timeframes 10 --channel-config name=data,type=pair,method=bind,numSockets=${numSenders}</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="write">fmqchan_data</id>
+            <name access="write">fmqchan_data</name>
         </properties>
     </decltask>
 

examples/n-m/ex-n-m-topology.xml

@@ -10,7 +10,7 @@
         <exe reachable="true">fairmq-ex-n-m-synchronizer --id sync --rate 100 --color false -P dds --channel-config name=sync,type=pub,method=bind</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="write">fmqchan_sync</id>
+            <name access="write">fmqchan_sync</name>
         </properties>
     </decltask>
 
@@ -18,8 +18,8 @@
         <exe reachable="true">fairmq-ex-n-m-sender --id sender%taskIndex% --timeframe-size 100000 --num-receivers ${numReceivers} --color false -P dds --channel-config name=sync,type=sub,method=connect name=data,type=push,method=connect,numSockets=${numReceivers}</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="read">fmqchan_sync</id>
-            <name access="read">fmqchan_data</id>
+            <name access="read">fmqchan_sync</name>
+            <name access="read">fmqchan_data</name>
         </properties>
     </decltask>
 
@@ -27,7 +27,7 @@
         <exe reachable="true">fairmq-ex-n-m-receiver --id receiver%taskIndex% --num-senders ${numSenders} --color false -P dds --max-timeframes 10 --channel-config name=data,type=pull,method=bind</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="write">fmqchan_data</id>
+            <name access="write">fmqchan_data</name>
         </properties>
     </decltask>
 

examples/qc/CMakeLists.txt

@@ -9,16 +9,16 @@
 add_executable(fairmq-ex-qc-sampler runSampler.cxx)
 target_link_libraries(fairmq-ex-qc-sampler PRIVATE FairMQ)
 
-add_executable(fairmq-ex-qc-producer runQCProducer.cxx)
-target_link_libraries(fairmq-ex-qc-producer PRIVATE FairMQ)
+add_executable(fairmq-ex-qc-dispatcher runQCDispatcher.cxx)
+target_link_libraries(fairmq-ex-qc-dispatcher PRIVATE FairMQ)
 
-add_executable(fairmq-ex-qc-consumer runQCConsumer.cxx)
-target_link_libraries(fairmq-ex-qc-consumer PRIVATE FairMQ)
+add_executable(fairmq-ex-qc-task runQCTask.cxx)
+target_link_libraries(fairmq-ex-qc-task PRIVATE FairMQ)
 
 add_executable(fairmq-ex-qc-sink runSink.cxx)
 target_link_libraries(fairmq-ex-qc-sink PRIVATE FairMQ)
 
-add_custom_target(ExampleQC DEPENDS fairmq-ex-qc-sampler fairmq-ex-qc-producer fairmq-ex-qc-consumer fairmq-ex-qc-sink)
+add_custom_target(ExampleQC DEPENDS fairmq-ex-qc-sampler fairmq-ex-qc-dispatcher fairmq-ex-qc-task fairmq-ex-qc-sink)
 
 list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
 set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
@@ -37,8 +37,8 @@ endif()
 install(
   TARGETS
   fairmq-ex-qc-sampler
-  fairmq-ex-qc-producer
-  fairmq-ex-qc-consumer
+  fairmq-ex-qc-dispatcher
+  fairmq-ex-qc-task
   fairmq-ex-qc-sink
 
   LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}

examples/qc/README.md

@@ -1,4 +1,4 @@
 QC
 ==
 
-A topology consisting of 4 devices - Sampler, QCProducer, QCConsumer and Sink. The data flows from Sampler through QCProducer to Sink. On demand - by setting the corresponding configuration property - the QCProducer device will duplicate the data to the QCConsumer device. The property is set by the topology controller, in this example this is the `fairmq-dds-command-ui` utility.
+A topology consisting of 4 devices - Sampler, QCDispatcher, QCTask and Sink. The data flows from Sampler through QCDispatcher to Sink. On demand - by setting the corresponding configuration property - the QCDispatcher device will duplicate the data to the QCTask device. The property is set by the topology controller, in this example this is the `fairmq-dds-command-ui` utility.

examples/qc/ex-qc-topology.xml

@@ -12,8 +12,8 @@
         </properties>
     </decltask>
 
-    <decltask name="QCProducer">
-        <exe>fairmq-ex-qc-producer --color false --channel-config name=data1,type=pull,method=connect name=data2,type=push,method=connect name=qc,type=push,method=connect -P dds</exe>
+    <decltask name="QCDispatcher">
+        <exe>fairmq-ex-qc-dispatcher --color false --channel-config name=data1,type=pull,method=connect name=data2,type=push,method=connect name=qc,type=push,method=connect -P dds</exe>
         <env reachable="false">fairmq-ex-qc-env.sh</env>
         <properties>
             <name access="read">fmqchan_data1</name>
@@ -22,8 +22,8 @@
         </properties>
     </decltask>
 
-    <decltask name="QCConsumer">
-        <exe>fairmq-ex-qc-consumer --color false --channel-config name=qc,type=pull,method=bind -P dds</exe>
+    <decltask name="QCTask">
+        <exe>fairmq-ex-qc-task --color false --channel-config name=qc,type=pull,method=bind -P dds</exe>
         <env reachable="false">fairmq-ex-qc-env.sh</env>
         <properties>
             <name access="write">fmqchan_qc</name>
@@ -40,8 +40,8 @@
 
     <main name="main">
         <task>Sampler</task>
-        <task>QCProducer</task>
-        <task>QCConsumer</task>
+        <task>QCDispatcher</task>
+        <task>QCTask</task>
         <task>Sink</task>
     </main>
 

examples/qc/fairmq-start-ex-qc.sh.in

@@ -51,12 +51,14 @@ fairmq-dds-command-ui -c k
 fairmq-dds-command-ui -c b
 fairmq-dds-command-ui -c x
 fairmq-dds-command-ui -c j
-fairmq-dds-command-ui -c r
-qcconsumer="main/QCConsumer.*"
-qcproducer="main/QCProducer.*"
-fairmq-dds-command-ui -c p --property-key qc --property-value active -p $qcproducer
-fairmq-dds-command-ui -w "RUNNING->READY" -p $qcconsumer
-echo "...$qcconsumer received data and transitioned to READY, sending shutdown..."
+allexceptqctasks="main/(Sampler|QCDispatcher|Sink).*"
+fairmq-dds-command-ui -c r -p $allexceptqctasks
+qctask="main/QCTask.*"
+qcdispatcher="main/QCDispatcher.*"
+fairmq-dds-command-ui -c p --property-key qc --property-value active -p $qcdispatcher
+fairmq-dds-command-ui -c r -p $qctask
+fairmq-dds-command-ui -w "RUNNING->READY" -p $qctask
+echo "...$qctask received data and transitioned to READY, sending shutdown..."
 fairmq-dds-command-ui -c s
 fairmq-dds-command-ui -c t
 fairmq-dds-command-ui -c d

examples/qc/runQCDispatcher.cxx

@@ -9,13 +9,13 @@
 #include "runFairMQDevice.h"
 #include "FairMQDevice.h"
 
-class QCProducer : public FairMQDevice
+class QCDispatcher : public FairMQDevice
 {
   public:
-    QCProducer()
+    QCDispatcher()
         : fDoQC(false)
     {
-        OnData("data1", &QCProducer::HandleData);
+        OnData("data1", &QCDispatcher::HandleData);
     }
 
     void InitTask() override
@@ -56,4 +56,4 @@ class QCProducer : public FairMQDevice
 };
 
 void addCustomOptions(boost::program_options::options_description& /*options*/) {}
-FairMQDevicePtr getDevice(const fair::mq::ProgOptions& /*config*/) { return new QCProducer(); }
+FairMQDevicePtr getDevice(const fair::mq::ProgOptions& /*config*/) { return new QCDispatcher(); }

examples/qc/runQCTask.cxx

@@ -9,10 +9,10 @@
 #include "runFairMQDevice.h"
 #include "FairMQDevice.h"
 
-class QCConsumer : public FairMQDevice
+class QCTask : public FairMQDevice
 {
   public:
-    QCConsumer()
+    QCTask()
     {
         OnData("qc", [](FairMQMessagePtr& /*msg*/, int) {
             LOG(info) << "received data";
@@ -23,4 +23,4 @@ class QCConsumer : public FairMQDevice
 
 namespace bpo = boost::program_options;
 void addCustomOptions(bpo::options_description& /*options*/) {}
-FairMQDevicePtr getDevice(const fair::mq::ProgOptions& /*config*/) { return new QCConsumer(); }
+FairMQDevicePtr getDevice(const fair::mq::ProgOptions& /*config*/) { return new QCTask(); }

examples/readout/Readout.h

@@ -37,10 +37,10 @@ class Readout : public FairMQDevice
         fRegion = FairMQUnmanagedRegionPtr(NewUnmanagedRegionFor("rb",
                                                                 0,
                                                                 10000000,
-                                                                [this](void* /*data*/, size_t /*size*/, void* /*hint*/) { // callback to be called when message buffers no longer needed by transport
-                                                                    --fNumUnackedMsgs;
+                                                                [this](const std::vector<fair::mq::RegionBlock>& blocks) { // callback to be called when message buffers no longer needed by transport
+                                                                    fNumUnackedMsgs -= blocks.size();
                                                                     if (fMaxIterations > 0) {
-                                                                        LOG(debug) << "Received ack";
+                                                                        LOG(debug) << "Received " << blocks.size() << " acks";
                                                                     }
                                                                 }
                                                                 ));

examples/region/CMakeLists.txt

@@ -32,15 +32,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-region.sh.in ${CMAKE_
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-region.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh)
 
 add_test(NAME Example.Region.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh zeromq)
-set_tests_properties(Example.Region.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
-
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.Region.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh nanomsg)
-  set_tests_properties(Example.Region.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
-endif()
+set_tests_properties(Example.Region.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received [0-9*] acks")
 
 add_test(NAME Example.Region.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh shmem)
-set_tests_properties(Example.Region.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
+set_tests_properties(Example.Region.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received [0-9*] acks")
 
 # install
 

examples/region/Sampler.cxx

@@ -23,28 +23,40 @@ namespace example_region
 
 Sampler::Sampler()
     : fMsgSize(10000)
+    , fLinger(100)
     , fMaxIterations(0)
     , fNumIterations(0)
     , fRegion(nullptr)
     , fNumUnackedMsgs(0)
-{
-}
+{}
 
 void Sampler::InitTask()
 {
     fMsgSize = fConfig->GetProperty<int>("msg-size");
+    fLinger = fConfig->GetProperty<uint32_t>("region-linger");
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
 
+    fChannels.at("data").at(0).Transport()->SubscribeToRegionEvents([](FairMQRegionInfo info) {
+        LOG(info) << "Region event: " << info.event
+                  << ", id: " << info.id
+                  << ", ptr: " << info.ptr
+                  << ", size: " << info.size
+                  << ", flags: " << info.flags;
+    });
+
     fRegion = FairMQUnmanagedRegionPtr(NewUnmanagedRegionFor("data",
                                                              0,
                                                              10000000,
-                                                             [this](void* /*data*/, size_t /*size*/, void* /*hint*/) { // callback to be called when message buffers no longer needed by transport
-                                                                 --fNumUnackedMsgs;
+                                                             [this](const std::vector<fair::mq::RegionBlock>& blocks) { // callback to be called when message buffers no longer needed by transport
+                                                                 lock_guard<mutex> lock(fMtx);
+                                                                 fNumUnackedMsgs -= blocks.size();
+
                                                                  if (fMaxIterations > 0) {
-                                                                     LOG(debug) << "Received ack";
+                                                                    LOG(info) << "Received " << blocks.size() << " acks";
                                                                  }
                                                              }
                                                              ));
+    fRegion->SetLinger(fLinger);
 }
 
 bool Sampler::ConditionalRun()
@@ -61,27 +73,31 @@ bool Sampler::ConditionalRun()
     // LOG(info) << "check: " << static_cast<char*>(fRegion->GetData())[3];
     // std::this_thread::sleep_for(std::chrono::seconds(1));
 
+    lock_guard<mutex> lock(fMtx);
     if (Send(msg, "data", 0) > 0) {
-        ++fNumUnackedMsgs;
-
         if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
             LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
             return false;
         }
     }
+    ++fNumUnackedMsgs;
 
     return true;
 }
 
 void Sampler::ResetTask()
 {
-    // if not all messages acknowledged, wait for a bit. But only once, since receiver could be already dead.
-    if (fNumUnackedMsgs != 0) {
-        LOG(debug) << "waiting for all acknowledgements... (" << fNumUnackedMsgs << ")";
-        this_thread::sleep_for(chrono::milliseconds(500));
-        LOG(debug) << "done, still unacked: " << fNumUnackedMsgs;
-    }
+    // On destruction UnmanagedRegion will try to TODO
     fRegion.reset();
+    {
+        lock_guard<mutex> lock(fMtx);
+        if (fNumUnackedMsgs != 0) {
+            LOG(info) << "Done, still not acknowledged: " << fNumUnackedMsgs;
+        } else {
+            LOG(info) << "All acknowledgements received.";
+        }
+    }
+    fChannels.at("data").at(0).Transport()->UnsubscribeFromRegionEvents();
 }
 
 Sampler::~Sampler()

examples/region/Sampler.h

@@ -15,7 +15,8 @@
 #ifndef FAIRMQEXAMPLEREGIONSAMPLER_H
 #define FAIRMQEXAMPLEREGIONSAMPLER_H
 
-#include <atomic>
+#include <mutex>
+#include <cstdint>
 
 #include "FairMQDevice.h"
 
@@ -35,10 +36,12 @@ class Sampler : public FairMQDevice
 
   private:
     int fMsgSize;
+    uint32_t fLinger;
     uint64_t fMaxIterations;
     uint64_t fNumIterations;
     FairMQUnmanagedRegionPtr fRegion;
-    std::atomic<uint64_t> fNumUnackedMsgs;
+    std::mutex fMtx;
+    uint64_t fNumUnackedMsgs;
 };
 
 } // namespace example_region

examples/region/Sink.cxx

@@ -29,6 +29,13 @@ void Sink::InitTask()
 {
     // Get the fMaxIterations value from the command line options (via fConfig)
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
+    fChannels.at("data").at(0).Transport()->SubscribeToRegionEvents([](FairMQRegionInfo info) {
+        LOG(info) << "Region event: " << info.event
+                  << ", id: " << info.id
+                  << ", ptr: " << info.ptr
+                  << ", size: " << info.size
+                  << ", flags: " << info.flags;
+    });
 }
 
 void Sink::Run()
@@ -50,6 +57,11 @@ void Sink::Run()
     }
 }
 
+void Sink::ResetTask()
+{
+    fChannels.at("data").at(0).Transport()->UnsubscribeFromRegionEvents();
+}
+
 Sink::~Sink()
 {
 }

examples/region/Sink.h

@@ -31,6 +31,7 @@ class Sink : public FairMQDevice
   protected:
     virtual void Run();
     virtual void InitTask();
+    virtual void ResetTask();
 
   private:
     uint64_t fMaxIterations;

examples/region/runSampler.cxx

@@ -15,6 +15,7 @@ void addCustomOptions(bpo::options_description& options)
 {
     options.add_options()
         ("msg-size", bpo::value<int>()->default_value(1000), "Message size in bytes")
+        ("region-linger", bpo::value<uint32_t>()->default_value(100), "Linger period for regions")
         ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
 }
 

examples/region/test-ex-region.sh.in

@@ -19,9 +19,11 @@ SAMPLER+=" --id sampler1"
 SAMPLER+=" --transport $transport"
 SAMPLER+=" --severity debug"
 SAMPLER+=" --session $SESSION"
+SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --control static --color false"
 SAMPLER+=" --max-iterations 1"
 SAMPLER+=" --msg-size $msgSize"
+SAMPLER+=" --region-linger 500"
 SAMPLER+=" --channel-config name=data,type=push,method=bind,address=tcp://127.0.0.1:7777"
 @CMAKE_CURRENT_BINARY_DIR@/$SAMPLER &
 SAMPLER_PID=$!

examples/req-rep/CMakeLists.txt

@@ -35,11 +35,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-req-rep.sh.in ${CMAKE_CURRENT
 add_test(NAME Example.ReqRep.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh zeromq)
 set_tests_properties(Example.ReqRep.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.ReqRep.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh nanomsg)
-  set_tests_properties(Example.ReqRep.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
-endif()
-
 add_test(NAME Example.ReqRep.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh shmem)
 set_tests_properties(Example.ReqRep.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 

fairmq/CMakeLists.txt

@@ -170,6 +170,7 @@ if(BUILD_FAIRMQ)
     PluginManager.h
     PluginServices.h
     runFairMQDevice.h
+    shmem/Monitor.h
   )
 
   set(FAIRMQ_PRIVATE_HEADER_FILES
@@ -190,23 +191,14 @@ if(BUILD_FAIRMQ)
     shmem/Common.h
     shmem/Manager.h
     shmem/Region.h
-    zeromq/FairMQMessageZMQ.h
-    zeromq/FairMQPollerZMQ.h
-    zeromq/FairMQUnmanagedRegionZMQ.h
-    zeromq/FairMQSocketZMQ.h
-    zeromq/FairMQTransportFactoryZMQ.h
+    zeromq/Context.h
+    zeromq/Message.h
+    zeromq/Poller.h
+    zeromq/UnmanagedRegion.h
+    zeromq/Socket.h
+    zeromq/TransportFactory.h
   )
 
-  if(BUILD_NANOMSG_TRANSPORT)
-    set(FAIRMQ_PRIVATE_HEADER_FILES ${FAIRMQ_PRIVATE_HEADER_FILES}
-      nanomsg/FairMQMessageNN.h
-      nanomsg/FairMQPollerNN.h
-      nanomsg/FairMQUnmanagedRegionNN.h
-      nanomsg/FairMQSocketNN.h
-      nanomsg/FairMQTransportFactoryNN.h
-    )
-  endif()
-
   if(BUILD_OFI_TRANSPORT)
     set(FAIRMQ_PRIVATE_HEADER_FILES ${FAIRMQ_PRIVATE_HEADER_FILES}
       ofi/Context.h
@@ -229,7 +221,6 @@ if(BUILD_FAIRMQ)
     FairMQPoller.cxx
     FairMQSocket.cxx
     FairMQTransportFactory.cxx
-    devices/FairMQBenchmarkSampler.cxx
     devices/FairMQMerger.cxx
     devices/FairMQMultiplier.cxx
     devices/FairMQProxy.cxx
@@ -243,31 +234,10 @@ if(BUILD_FAIRMQ)
     SuboptParser.cxx
     plugins/config/Config.cxx
     plugins/Control.cxx
-    shmem/Message.cxx
-    shmem/Poller.cxx
-    shmem/UnmanagedRegion.cxx
-    shmem/Socket.cxx
-    shmem/TransportFactory.cxx
-    shmem/Manager.cxx
-    shmem/Region.cxx
-    zeromq/FairMQMessageZMQ.cxx
-    zeromq/FairMQPollerZMQ.cxx
-    zeromq/FairMQUnmanagedRegionZMQ.cxx
-    zeromq/FairMQSocketZMQ.cxx
-    zeromq/FairMQTransportFactoryZMQ.cxx
     MemoryResources.cxx
+    shmem/Monitor.cxx
   )
 
-  if(BUILD_NANOMSG_TRANSPORT)
-    set(FAIRMQ_SOURCE_FILES ${FAIRMQ_SOURCE_FILES}
-      nanomsg/FairMQMessageNN.cxx
-      nanomsg/FairMQPollerNN.cxx
-      nanomsg/FairMQUnmanagedRegionNN.cxx
-      nanomsg/FairMQSocketNN.cxx
-      nanomsg/FairMQTransportFactoryNN.cxx
-    )
-  endif()
-
   if(BUILD_OFI_TRANSPORT)
     set(FAIRMQ_SOURCE_FILES ${FAIRMQ_SOURCE_FILES}
       ofi/Context.cxx
@@ -308,9 +278,6 @@ if(BUILD_FAIRMQ)
   # preprocessor definitions #
   ############################
   target_compile_definitions(${_target} PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
-  if(BUILD_NANOMSG_TRANSPORT)
-    target_compile_definitions(${_target} PRIVATE BUILD_NANOMSG_TRANSPORT)
-  endif()
   if(BUILD_OFI_TRANSPORT)
     target_compile_definitions(${_target} PRIVATE BUILD_OFI_TRANSPORT)
   endif()
@@ -331,16 +298,13 @@ if(BUILD_FAIRMQ)
   ##################
   # link libraries #
   ##################
-  if(BUILD_NANOMSG_TRANSPORT)
-    set(NANOMSG_DEPS nanomsg msgpackc-cxx)
-  endif()
   if(BUILD_OFI_TRANSPORT)
     set(OFI_DEPS
       asiofi::asiofi
       Boost::container
     )
   endif()
-  set(optional_deps ${NANOMSG_DEPS} ${OFI_DEPS})
+  set(optional_deps ${OFI_DEPS})
   if(optional_deps)
     list(REMOVE_DUPLICATES optional_deps)
   endif()
@@ -363,7 +327,7 @@ if(BUILD_FAIRMQ)
 
     PRIVATE # only libFairMQ links against private dependencies
     libzmq
-    ${NANOMSG_DEPS}
+    PicoSHA2
     ${OFI_DEPS}
   )
   set_target_properties(${_target} PROPERTIES
@@ -416,6 +380,8 @@ if(BUILD_FAIRMQ)
     Boost::boost
     Boost::date_time
     Boost::program_options
+    FairLogger::FairLogger
+    PicoSHA2
   )
   target_include_directories(fairmq-shmmonitor PUBLIC
     $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}>

fairmq/FairMQChannel.cxx

@@ -671,7 +671,6 @@ void FairMQChannel::Init()
 bool FairMQChannel::ConnectEndpoint(const string& endpoint)
 {
     lock_guard<mutex> lock(fMtx);
-
     return fSocket->Connect(endpoint);
 }
 
@@ -683,6 +682,13 @@ bool FairMQChannel::BindEndpoint(string& endpoint)
     if (fSocket->Bind(endpoint)) {
         return true;
     } else {
+        // auto-bind only implemented for TCP
+        size_t protocolPos = endpoint.find(':');
+        string protocol = endpoint.substr(0, protocolPos);
+        if (protocol != "tcp") {
+            return false;
+        }
+
         if (fAutoBind) {
             // number of attempts when choosing a random port
             int numAttempts = 0;

fairmq/FairMQChannel.h

@@ -127,8 +127,8 @@ class FairMQChannel
     /// @return Returns socket address (e.g. "tcp://127.0.0.1:5555" or "ipc://abc")
     std::string GetAddress() const;
 
-    /// Get channel transport name ("default", "zeromq", "nanomsg" or "shmem")
-    /// @return Returns channel transport name (e.g. "default", "zeromq", "nanomsg" or "shmem")
+    /// Get channel transport name ("default", "zeromq" or "shmem")
+    /// @return Returns channel transport name (e.g. "default", "zeromq" or "shmem")
     std::string GetTransportName() const;
 
     /// Get channel transport type
@@ -184,7 +184,7 @@ class FairMQChannel
     void UpdateAddress(const std::string& address);
 
     /// Set channel transport
-    /// @param transport transport string ("default", "zeromq", "nanomsg" or "shmem")
+    /// @param transport transport string ("default", "zeromq" or "shmem")
     void UpdateTransport(const std::string& transport);
 
     /// Set socket send buffer size
@@ -335,9 +335,10 @@ class FairMQChannel
         return Transport()->NewStaticMessage(data);
     }
 
-    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0)
+    template<typename... Args>
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(Args&&... args)
     {
-        return Transport()->CreateUnmanagedRegion(size, callback, path, flags);
+        return Transport()->CreateUnmanagedRegion(std::forward<Args>(args)...);
     }
 
     static constexpr fair::mq::Transport DefaultTransportType = fair::mq::Transport::DEFAULT;

fairmq/FairMQDevice.h

@@ -217,15 +217,17 @@ class FairMQDevice
     }
 
     // creates unamanaged region with the default device transport
-    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr)
+    template<typename... Args>
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(Args&&... args)
     {
-        return Transport()->CreateUnmanagedRegion(size, callback);
+        return Transport()->CreateUnmanagedRegion(std::forward<Args>(args)...);
     }
 
     // creates unmanaged region with the transport of the specified channel
-    FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel, int index, const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0)
+    template<typename... Args>
+    FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel, int index, Args&&... args)
     {
-        return GetChannel(channel, index).NewUnmanagedRegion(size, callback, path, flags);
+        return GetChannel(channel, index).NewUnmanagedRegion(std::forward<Args>(args)...);
     }
 
     template<typename ...Ts>
@@ -272,7 +274,7 @@ class FairMQDevice
     }
 
     /// Adds a transport to the device if it doesn't exist
-    /// @param transport  Transport string ("zeromq"/"nanomsg"/"shmem")
+    /// @param transport  Transport string ("zeromq"/"shmem")
     std::shared_ptr<FairMQTransportFactory> AddTransport(const fair::mq::Transport transport);
 
     /// Assigns config to the device
@@ -387,7 +389,7 @@ class FairMQDevice
     int GetInitTimeoutInS() const { return fConfig->GetProperty<int>("init-timeout", DefaultInitTimeout); }
 
     /// Sets the default transport for the device
-    /// @param transport  Transport string ("zeromq"/"nanomsg"/"shmem")
+    /// @param transport  Transport string ("zeromq"/"shmem")
     void SetTransport(const std::string& transport) { fConfig->SetProperty("transport", transport); }
     /// Gets the default transport name
     std::string GetTransportName() const { return fConfig->GetProperty<std::string>("transport", DefaultTransportName); }

fairmq/FairMQMessage.h

@@ -11,17 +11,33 @@
 
 #include <cstddef> // for size_t
 #include <memory> // unique_ptr
+#include <stdexcept>
 
 #include <fairmq/Transports.h>
 
 using fairmq_free_fn = void(void* data, void* hint);
 class FairMQTransportFactory;
 
+namespace fair
+{
+namespace mq
+{
+
+struct Alignment
+{
+    size_t alignment;
+    explicit operator size_t() const { return alignment; }
+};
+
+} /* namespace mq */
+} /* namespace fair */
+
 class FairMQMessage
 {
   public:
     FairMQMessage() = default;
-    FairMQMessage(FairMQTransportFactory* factory):fTransport{factory} {}
+    FairMQMessage(FairMQTransportFactory* factory) : fTransport(factory) {}
+
     virtual void Rebuild() = 0;
     virtual void Rebuild(const size_t size) = 0;
     virtual void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) = 0;
@@ -33,7 +49,7 @@ class FairMQMessage
 
     virtual fair::mq::Transport GetType() const = 0;
     FairMQTransportFactory* GetTransport() { return fTransport; }
-    //void SetTransport(FairMQTransportFactory* transport) { fTransport = transport; }
+    void SetTransport(FairMQTransportFactory* transport) { fTransport = transport; }
 
     virtual void Copy(const FairMQMessage& msg) = 0;
 
@@ -53,6 +69,7 @@ namespace mq
 using Message = FairMQMessage;
 using MessagePtr = FairMQMessagePtr;
 struct MessageError : std::runtime_error { using std::runtime_error::runtime_error; };
+struct MessageBadAlloc : std::runtime_error { using std::runtime_error::runtime_error; };
 
 } /* namespace mq */
 } /* namespace fair */

fairmq/FairMQSocket.h

@@ -9,9 +9,10 @@
 #ifndef FAIRMQSOCKET_H_
 #define FAIRMQSOCKET_H_
 
+#include <memory>
+#include <stdexcept>
 #include <string>
 #include <vector>
-#include <memory>
 
 #include "FairMQMessage.h"
 class FairMQTransportFactory;
@@ -20,7 +21,7 @@ class FairMQSocket
 {
   public:
     FairMQSocket() {}
-    FairMQSocket(FairMQTransportFactory* fac): fTransport(fac) {}
+    FairMQSocket(FairMQTransportFactory* fac) : fTransport(fac) {}
 
     virtual std::string GetId() const = 0;
 
@@ -37,6 +38,10 @@ class FairMQSocket
     virtual void SetOption(const std::string& option, const void* value, size_t valueSize) = 0;
     virtual void GetOption(const std::string& option, void* value, size_t* valueSize) = 0;
 
+    /// If the backend supports it, fills the unsigned integer @a events with the ZMQ_EVENTS value
+    /// DISCLAIMER: this API is experimental and unsupported and might be dropped / refactored in
+    /// the future.
+    virtual void Events(uint32_t* events) = 0;
     virtual void SetLinger(const int value) = 0;
     virtual int GetLinger() const = 0;
     virtual void SetSndBufSize(const int value) = 0;
@@ -54,7 +59,7 @@ class FairMQSocket
     virtual unsigned long GetMessagesRx() const = 0;
 
     FairMQTransportFactory* GetTransport() { return fTransport; }
-    void SetTransport(FairMQTransportFactory* transport) { fTransport=transport; }
+    void SetTransport(FairMQTransportFactory* transport) { fTransport = transport; }
 
     virtual ~FairMQSocket() {};
 

fairmq/FairMQTransportFactory.cxx

@@ -7,69 +7,55 @@
  ********************************************************************************/
 
 #include <FairMQTransportFactory.h>
-#include <zeromq/FairMQTransportFactoryZMQ.h>
 #include <fairmq/shmem/TransportFactory.h>
-#ifdef BUILD_NANOMSG_TRANSPORT
-#include <nanomsg/FairMQTransportFactoryNN.h>
-#endif /* BUILD_NANOMSG_TRANSPORT */
+#include <fairmq/zeromq/TransportFactory.h>
 #ifdef BUILD_OFI_TRANSPORT
 #include <fairmq/ofi/TransportFactory.h>
 #endif
 #include <FairMQLogger.h>
-#include <fairmq/tools/Unique.h>
+#include <fairmq/Tools.h>
 
 #include <memory>
 #include <string>
 
-FairMQTransportFactory::FairMQTransportFactory(const std::string& id)
+using namespace std;
+
+FairMQTransportFactory::FairMQTransportFactory(const string& id)
     : fkId(id)
-{
-}
+{}
 
-auto FairMQTransportFactory::CreateTransportFactory(const std::string& type, const std::string& id, const fair::mq::ProgOptions* config) -> std::shared_ptr<FairMQTransportFactory>
+auto FairMQTransportFactory::CreateTransportFactory(const string& type,
+                                                    const string& id,
+                                                    const fair::mq::ProgOptions* config)
+    -> shared_ptr<FairMQTransportFactory>
 {
-    using namespace std;
-
     auto finalId = id;
 
     // Generate uuid if empty
-    if (finalId == "")
-    {
+    if (finalId == "") {
         finalId = fair::mq::tools::Uuid();
     }
 
-    if (type == "zeromq")
-    {
-        return make_shared<FairMQTransportFactoryZMQ>(finalId, config);
-    }
-    else if (type == "shmem")
-    {
+    if (type == "zeromq") {
+        return make_shared<fair::mq::zmq::TransportFactory>(finalId, config);
+    } else if (type == "shmem") {
         return make_shared<fair::mq::shmem::TransportFactory>(finalId, config);
     }
-#ifdef BUILD_NANOMSG_TRANSPORT
-    else if (type == "nanomsg")
-    {
-        return make_shared<FairMQTransportFactoryNN>(finalId, config);
-    }
-#endif /* BUILD_NANOMSG_TRANSPORT */
 #ifdef BUILD_OFI_TRANSPORT
-    else if (type == "ofi")
-    {
+    else if (type == "ofi") {
         return make_shared<fair::mq::ofi::TransportFactory>(finalId, config);
     }
 #endif /* BUILD_OFI_TRANSPORT */
-    else
-    {
-        LOG(error) << "Unavailable transport requested: " << "\"" << type << "\"" << ". Available are: "
-                   << "\"zeromq\""
+    else {
+        LOG(error) << "Unavailable transport requested: "
+                   << "\"" << type << "\""
+                   << ". Available are: "
+                   << "\"zeromq\","
                    << "\"shmem\""
-#ifdef BUILD_NANOMSG_TRANSPORT
-                   << ", \"nanomsg\""
-#endif /* BUILD_NANOMSG_TRANSPORT */
 #ifdef BUILD_OFI_TRANSPORT
                    << ", and \"ofi\""
 #endif /* BUILD_OFI_TRANSPORT */
                    << ". Exiting.";
-        exit(EXIT_FAILURE);
+        throw fair::mq::TransportFactoryError(fair::mq::tools::ToString("Unavailable transport requested: ", type));
     }
 }

fairmq/FairMQTransportFactory.h

@@ -18,7 +18,7 @@
 #include <fairmq/Transports.h>
 
 #include <string>
-#include <memory>
+#include <memory> // shared_ptr
 #include <vector>
 #include <unordered_map>
 #include <stdexcept>
@@ -47,13 +47,22 @@ class FairMQTransportFactory
     fair::mq::ChannelResource* GetMemoryResource() { return &fMemoryResource; }
     operator fair::mq::ChannelResource*() { return &fMemoryResource; }
 
-    /// @brief Create empty FairMQMessage
+    /// @brief Create empty FairMQMessage (for receiving)
     /// @return pointer to FairMQMessage
     virtual FairMQMessagePtr CreateMessage() = 0;
+    /// @brief Create empty FairMQMessage (for receiving), align received buffer to specified alignment
+    /// @param alignment alignment to align received buffer to
+    /// @return pointer to FairMQMessage
+    virtual FairMQMessagePtr CreateMessage(fair::mq::Alignment alignment) = 0;
     /// @brief Create new FairMQMessage of specified size
     /// @param size message size
     /// @return pointer to FairMQMessage
     virtual FairMQMessagePtr CreateMessage(const size_t size) = 0;
+    /// @brief Create new FairMQMessage of specified size and alignment
+    /// @param size message size
+    /// @param alignment message alignment
+    /// @return pointer to FairMQMessage
+    virtual FairMQMessagePtr CreateMessage(const size_t size, fair::mq::Alignment alignment) = 0;
     /// @brief Create new FairMQMessage with user provided buffer and size
     /// @param data pointer to user provided buffer
     /// @param size size of the user provided buffer
@@ -61,20 +70,51 @@ class FairMQTransportFactory
     /// @param obj optional helper pointer that can be used in the callback
     /// @return pointer to FairMQMessage
     virtual FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) = 0;
-
+    /// @brief create a message with the buffer located within the corresponding unmanaged region
+    /// @param unmanagedRegion the unmanaged region that this message buffer belongs to
+    /// @param data message buffer (must be within the region - checked at runtime by the transport)
+    /// @param size size of the message
+    /// @param hint optional parameter, returned to the user in the FairMQRegionCallback
     virtual FairMQMessagePtr CreateMessage(FairMQUnmanagedRegionPtr& unmanagedRegion, void* data, const size_t size, void* hint = 0) = 0;
 
-    /// Create a socket
+    /// @brief Create a socket
     virtual FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name) = 0;
 
-    /// Create a poller for a single channel (all subchannels)
+    /// @brief Create a poller for a single channel (all subchannels)
     virtual FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const = 0;
-    /// Create a poller for specific channels
+    /// @brief Create a poller for specific channels
     virtual FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel*>& channels) const = 0;
-    /// Create a poller for specific channels (all subchannels)
+    /// @brief Create a poller for specific channels (all subchannels)
     virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const = 0;
 
-    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const = 0;
+    /// @brief Create new UnmanagedRegion
+    /// @param size size of the region
+    /// @param callback callback to be called when a message belonging to this region is no longer needed by the transport
+    /// @param path optional parameter to pass to the underlying transport
+    /// @param flags optional parameter to pass to the underlying transport
+    /// @return pointer to UnmanagedRegion
+    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) = 0;
+    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionBulkCallback callback = nullptr, const std::string& path = "", int flags = 0) = 0;
+    /// @brief Create new UnmanagedRegion
+    /// @param size size of the region
+    /// @param userFlags flags to be stored with the region, have no effect on the transport, but can be retrieved from the region by the user
+    /// @param callback callback to be called when a message belonging to this region is no longer needed by the transport
+    /// @param path optional parameter to pass to the underlying transport
+    /// @param flags optional parameter to pass to the underlying transport
+    /// @return pointer to UnmanagedRegion
+    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) = 0;
+    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionBulkCallback callback = nullptr, const std::string& path = "", int flags = 0) = 0;
+
+    /// @brief Subscribe to region events (creation, destruction, ...)
+    /// @param callback the callback that is called when a region event occurs
+    virtual void SubscribeToRegionEvents(FairMQRegionEventCallback callback) = 0;
+    /// @brief Check if there is an active subscription to region events
+    /// @return true/false
+    virtual bool SubscribedToRegionEvents() = 0;
+    /// @brief Unsubscribe from region events
+    virtual void UnsubscribeFromRegionEvents() = 0;
+
+    virtual std::vector<FairMQRegionInfo> GetRegionInfo() = 0;
 
     /// Get transport type
     virtual fair::mq::Transport GetType() const = 0;

fairmq/FairMQUnmanagedRegion.h

@@ -10,28 +10,108 @@
 #define FAIRMQUNMANAGEDREGION_H_
 
 #include <cstddef> // size_t
+#include <cstdint> // uint32_t
 #include <memory> // std::unique_ptr
 #include <functional> // std::function
+#include <ostream> // std::ostream
+#include <vector>
+
+class FairMQTransportFactory;
+
+enum class FairMQRegionEvent : int
+{
+    created,
+    destroyed,
+    local_only
+};
+
+struct FairMQRegionInfo
+{
+    FairMQRegionInfo()
+        : id(0)
+        , ptr(nullptr)
+        , size(0)
+        , flags(0)
+        , event(FairMQRegionEvent::created)
+    {}
+
+    FairMQRegionInfo(uint64_t _id, void* _ptr, size_t _size, int64_t _flags, FairMQRegionEvent _event)
+        : id(_id)
+        , ptr(_ptr)
+        , size(_size)
+        , flags(_flags)
+        , event (_event)
+    {}
+
+    uint64_t id;   // id of the region
+    void* ptr;     // pointer to the start of the region
+    size_t size;   // region size
+    int64_t flags; // custom flags set by the creator
+    FairMQRegionEvent event;
+};
+
+struct FairMQRegionBlock {
+    void* ptr;
+    size_t size;
+    void* hint;
+
+    FairMQRegionBlock(void* p, size_t s, void* h)
+        : ptr(p), size(s), hint(h)
+    {}
+};
 
 using FairMQRegionCallback = std::function<void(void*, size_t, void*)>;
+using FairMQRegionBulkCallback = std::function<void(const std::vector<FairMQRegionBlock>&)>;
+using FairMQRegionEventCallback = std::function<void(FairMQRegionInfo)>;
 
 class FairMQUnmanagedRegion
 {
   public:
+    FairMQUnmanagedRegion() {}
+    FairMQUnmanagedRegion(FairMQTransportFactory* factory) : fTransport(factory) {}
+
     virtual void* GetData() const = 0;
     virtual size_t GetSize() const = 0;
+    virtual uint64_t GetId() const = 0;
+    virtual void SetLinger(uint32_t linger) = 0;
+    virtual uint32_t GetLinger() const = 0;
+
+    FairMQTransportFactory* GetTransport() { return fTransport; }
+    void SetTransport(FairMQTransportFactory* transport) { fTransport = transport; }
 
     virtual ~FairMQUnmanagedRegion() {};
+
+  private:
+    FairMQTransportFactory* fTransport{nullptr};
 };
 
 using FairMQUnmanagedRegionPtr = std::unique_ptr<FairMQUnmanagedRegion>;
 
+inline std::ostream& operator<<(std::ostream& os, const FairMQRegionEvent& event)
+{
+    switch (event) {
+        case FairMQRegionEvent::created:
+            return os << "created";
+        case FairMQRegionEvent::destroyed:
+            return os << "destroyed";
+        case FairMQRegionEvent::local_only:
+            return os << "local_only";
+        default:
+            return os << "unrecognized event";
+    }
+}
+
 namespace fair
 {
 namespace mq
 {
 
 using RegionCallback = FairMQRegionCallback;
+using RegionBulkCallback = FairMQRegionBulkCallback;
+using RegionEventCallback = FairMQRegionEventCallback;
+using RegionEvent = FairMQRegionEvent;
+using RegionInfo = FairMQRegionInfo;
+using RegionBlock = FairMQRegionBlock;
 using UnmanagedRegion = FairMQUnmanagedRegion;
 using UnmanagedRegionPtr = FairMQUnmanagedRegionPtr;
 

fairmq/MemoryResources.cxx

@@ -15,8 +15,7 @@
 #include <fairmq/FairMQTransportFactory.h>
 #include <fairmq/MemoryResources.h>
 
-void *fair::mq::ChannelResource::do_allocate(std::size_t bytes, std::size_t /*alignment*/)
+void *fair::mq::ChannelResource::do_allocate(std::size_t bytes, std::size_t alignment)
 {
-    return setMessage(factory->CreateMessage(bytes));
+    return setMessage(factory->CreateMessage(bytes, fair::mq::Alignment{alignment}));
 }
-

fairmq/PluginManager.cxx

@@ -28,6 +28,8 @@ using boost::optional;
 
 const std::string fair::mq::PluginManager::fgkLibPrefix = "FairMQPlugin_";
 
+std::vector<boost::dll::shared_library> fair::mq::PluginManager::fgDLLKeepAlive = std::vector<boost::dll::shared_library>();
+
 fair::mq::PluginManager::PluginManager()
     : fSearchPaths{}
     , fPluginFactories()

fairmq/PluginManager.h

@@ -97,6 +97,7 @@ class PluginManager
         using fair::mq::tools::ToString;
 
         auto lib = shared_library{std::forward<Args>(args)...};
+        fgDLLKeepAlive.push_back(lib);
 
         fPluginFactories[pluginName] = import_alias<PluginFactory>(
             shared_library{lib},
@@ -117,6 +118,7 @@ class PluginManager
 
     static const std::string fgkLibPrefix;
     std::vector<boost::filesystem::path> fSearchPaths;
+    static std::vector<boost::dll::shared_library> fgDLLKeepAlive;
     std::map<std::string, std::function<PluginFactory>> fPluginFactories;
     std::unique_ptr<PluginServices> fPluginServices;
     std::map<std::string, std::unique_ptr<Plugin>> fPlugins;

fairmq/ProgOptions.cxx

@@ -117,16 +117,24 @@ void ProgOptions::ParseAll(const int argc, char const* const* argv, bool allowUn
     // clear the container because it was filled with default values and subsequent calls to store() do not overwrite the existing values
     fVarMap.clear();
 
-    if (allowUnregistered) {
-        po::command_line_parser parser(argc, argv);
-        parser.options(fAllOptions).allow_unregistered();
-        po::parsed_options parsed = parser.run();
-        fUnregisteredOptions = po::collect_unrecognized(parsed.options, po::include_positional);
 
-        po::store(parsed, fVarMap);
-    } else {
-        po::store(po::parse_command_line(argc, argv, fAllOptions), fVarMap);
+    po::command_line_parser parser(argc, argv);
+
+    parser.options(fAllOptions);
+
+    if (allowUnregistered) {
+        parser.allow_unregistered();
     }
+
+    using namespace po::command_line_style;
+    style_t style = style_t(allow_short | short_allow_adjacent | short_allow_next | allow_long | long_allow_adjacent | long_allow_next | allow_sticky | allow_dash_for_short);
+    parser.style(style);
+
+    po::parsed_options parsed = parser.run();
+
+    fUnregisteredOptions = po::collect_unrecognized(parsed.options, po::include_positional);
+
+    po::store(parsed, fVarMap);
 }
 
 void ProgOptions::Notify()

fairmq/States.cxx

@@ -18,9 +18,10 @@ namespace fair
 namespace mq
 {
 
-array<string, 15> stateNames =
+array<string, 16> stateNames =
 {
     {
+        "UNDEFINED",
         "OK",
         "ERROR",
         "IDLE",
@@ -41,6 +42,7 @@ array<string, 15> stateNames =
 
 unordered_map<string, State> states =
 {
+    { "UNDEFINED",           State::Undefined },
     { "OK",                  State::Ok },
     { "ERROR",               State::Error },
     { "IDLE",                State::Idle },

fairmq/States.h

@@ -20,6 +20,7 @@ namespace mq
 
 enum class State : int
 {
+    Undefined = 0,
     Ok,
     Error,
     Idle,
@@ -39,7 +40,7 @@ enum class State : int
 
 enum class Transition : int
 {
-    Auto,
+    Auto = 0,
     InitDevice,
     CompleteInit,
     Bind,

fairmq/Transports.h

@@ -24,7 +24,6 @@ enum class Transport
 {
     DEFAULT,
     ZMQ,
-    NN,
     SHM,
     OFI
 };
@@ -48,7 +47,6 @@ namespace mq
 static std::unordered_map<std::string, Transport> TransportTypes {
     { "default", Transport::DEFAULT },
     { "zeromq", Transport::ZMQ },
-    { "nanomsg", Transport::NN },
     { "shmem", Transport::SHM },
     { "ofi", Transport::OFI }
 };
@@ -56,7 +54,6 @@ static std::unordered_map<std::string, Transport> TransportTypes {
 static std::unordered_map<Transport, std::string> TransportNames {
     { Transport::DEFAULT, "default" },
     { Transport::ZMQ, "zeromq" },
-    { Transport::NN, "nanomsg" },
     { Transport::SHM, "shmem" },
     { Transport::OFI, "ofi" }
 };

fairmq/devices/FairMQBenchmarkSampler.cxx

@@ -1,100 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "FairMQBenchmarkSampler.h"
-
-#include "tools/RateLimit.h"
-#include "../FairMQLogger.h"
-
-#include <chrono>
-
-using namespace std;
-
-FairMQBenchmarkSampler::FairMQBenchmarkSampler()
-    : fMultipart(false)
-    , fNumParts(1)
-    , fMsgSize(10000)
-    , fMsgRate(0)
-    , fNumIterations(0)
-    , fMaxIterations(0)
-    , fOutChannelName()
-{
-}
-
-void FairMQBenchmarkSampler::InitTask()
-{
-    fMultipart = fConfig->GetProperty<bool>("multipart");
-    fNumParts = fConfig->GetProperty<size_t>("num-parts");
-    fMsgSize = fConfig->GetProperty<size_t>("msg-size");
-    fMsgRate = fConfig->GetProperty<float>("msg-rate");
-    fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
-    fOutChannelName = fConfig->GetProperty<string>("out-channel");
-}
-
-void FairMQBenchmarkSampler::Run()
-{
-    // store the channel reference to avoid traversing the map on every loop iteration
-    FairMQChannel& dataOutChannel = fChannels.at(fOutChannelName).at(0);
-
-    FairMQMessagePtr baseMsg(dataOutChannel.NewMessage(fMsgSize));
-
-    LOG(info) << "Starting the benchmark with message size of " << fMsgSize << " and " << fMaxIterations << " iterations.";
-    auto tStart = chrono::high_resolution_clock::now();
-
-    fair::mq::tools::RateLimiter rateLimiter(fMsgRate);
-
-    while (!NewStatePending())
-    {
-        if (fMultipart)
-        {
-            FairMQParts parts;
-
-            for (size_t i = 0; i < fNumParts; ++i)
-            {
-                parts.AddPart(dataOutChannel.NewMessage(fMsgSize));
-            }
-
-            if (dataOutChannel.Send(parts) >= 0)
-            {
-                if (fMaxIterations > 0)
-                {
-                    if (fNumIterations >= fMaxIterations)
-                    {
-                        break;
-                    }
-                }
-                ++fNumIterations;
-            }
-        }
-        else
-        {
-            FairMQMessagePtr msg(dataOutChannel.NewMessage(fMsgSize));
-
-            if (dataOutChannel.Send(msg) >= 0)
-            {
-                if (fMaxIterations > 0)
-                {
-                    if (fNumIterations >= fMaxIterations)
-                    {
-                        break;
-                    }
-                }
-                ++fNumIterations;
-            }
-        }
-
-        if (fMsgRate > 0)
-        {
-            rateLimiter.maybe_sleep();
-        }
-    }
-
-    auto tEnd = chrono::high_resolution_clock::now();
-
-    LOG(info) << "Done " << fNumIterations << " iterations in " << chrono::duration<double, milli>(tEnd - tStart).count() << "ms.";
-}

fairmq/devices/FairMQBenchmarkSampler.h

@@ -9,13 +9,16 @@
 #ifndef FAIRMQBENCHMARKSAMPLER_H_
 #define FAIRMQBENCHMARKSAMPLER_H_
 
-#include <string>
-#include <atomic>
-#include <cstddef> // size_t
-#include <cstdint> // uint64_t
-
-
+#include "../FairMQLogger.h"
 #include "FairMQDevice.h"
+#include "tools/RateLimit.h"
+
+#include <atomic>
+#include <chrono>
+#include <cstddef>   // size_t
+#include <cstdint>   // uint64_t
+#include <cstring>   // memset
+#include <string>
 
 /**
  * Sampler to generate traffic for benchmarking.
@@ -24,21 +27,98 @@
 class FairMQBenchmarkSampler : public FairMQDevice
 {
   public:
-    FairMQBenchmarkSampler();
+    FairMQBenchmarkSampler()
+        : fMultipart(false)
+        , fMemSet(false)
+        , fNumParts(1)
+        , fMsgSize(10000)
+        , fMsgAlignment(0)
+        , fMsgRate(0)
+        , fNumIterations(0)
+        , fMaxIterations(0)
+        , fOutChannelName()
+    {}
+
+    void InitTask() override
+    {
+        fMultipart = fConfig->GetProperty<bool>("multipart");
+        fMemSet = fConfig->GetProperty<bool>("memset");
+        fNumParts = fConfig->GetProperty<size_t>("num-parts");
+        fMsgSize = fConfig->GetProperty<size_t>("msg-size");
+        fMsgAlignment = fConfig->GetProperty<size_t>("msg-alignment");
+        fMsgRate = fConfig->GetProperty<float>("msg-rate");
+        fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
+        fOutChannelName = fConfig->GetProperty<std::string>("out-channel");
+    }
+
+    void Run() override
+    {
+        // store the channel reference to avoid traversing the map on every loop iteration
+        FairMQChannel& dataOutChannel = fChannels.at(fOutChannelName).at(0);
+
+        LOG(info) << "Starting the benchmark with message size of " << fMsgSize << " and " << fMaxIterations << " iterations.";
+        auto tStart = std::chrono::high_resolution_clock::now();
+
+        fair::mq::tools::RateLimiter rateLimiter(fMsgRate);
+
+        while (!NewStatePending()) {
+            if (fMultipart) {
+                FairMQParts parts;
+
+                for (size_t i = 0; i < fNumParts; ++i) {
+                    parts.AddPart(dataOutChannel.NewMessage(fMsgSize, fair::mq::Alignment{fMsgAlignment}));
+                    if (fMemSet) {
+                        std::memset(parts.At(i)->GetData(), 0, parts.At(i)->GetSize());
+                    }
+                }
+
+                if (dataOutChannel.Send(parts) >= 0) {
+                    if (fMaxIterations > 0) {
+                        if (fNumIterations >= fMaxIterations) {
+                            break;
+                        }
+                    }
+                    ++fNumIterations;
+                }
+            } else {
+                FairMQMessagePtr msg(dataOutChannel.NewMessage(fMsgSize, fair::mq::Alignment{fMsgAlignment}));
+                if (fMemSet) {
+                    std::memset(msg->GetData(), 0, msg->GetSize());
+                }
+
+                if (dataOutChannel.Send(msg) >= 0) {
+                    if (fMaxIterations > 0) {
+                        if (fNumIterations >= fMaxIterations) {
+                            break;
+                        }
+                    }
+                    ++fNumIterations;
+                }
+            }
+
+            if (fMsgRate > 0) {
+                rateLimiter.maybe_sleep();
+            }
+        }
+
+        auto tEnd = std::chrono::high_resolution_clock::now();
+
+        LOG(info) << "Done " << fNumIterations << " iterations in " << std::chrono::duration<double, std::milli>(tEnd - tStart).count() << "ms.";
+    }
+
     virtual ~FairMQBenchmarkSampler() {}
 
   protected:
     bool fMultipart;
+    bool fMemSet;
     size_t fNumParts;
     size_t fMsgSize;
+    size_t fMsgAlignment;
     std::atomic<int> fMsgCounter;
     float fMsgRate;
     uint64_t fNumIterations;
     uint64_t fMaxIterations;
     std::string fOutChannelName;
-
-    virtual void InitTask() override;
-    virtual void Run() override;
 };
 
 #endif /* FAIRMQBENCHMARKSAMPLER_H_ */

fairmq/devices/FairMQSink.h

@@ -15,14 +15,14 @@
 #ifndef FAIRMQSINK_H_
 #define FAIRMQSINK_H_
 
-#include <string>
-#include <chrono>
-
 #include "../FairMQDevice.h"
 #include "../FairMQLogger.h"
 
+#include <chrono>
+#include <string>
+
 // template<typename OutputPolicy>
-class FairMQSink : public FairMQDevice//, public OutputPolicy
+class FairMQSink : public FairMQDevice   //, public OutputPolicy
 {
   public:
     FairMQSink()
@@ -32,8 +32,7 @@ class FairMQSink : public FairMQDevice//, public OutputPolicy
         , fInChannelName()
     {}
 
-    virtual ~FairMQSink()
-    {}
+    virtual ~FairMQSink() {}
 
   protected:
     bool fMultipart;
@@ -56,35 +55,25 @@ class FairMQSink : public FairMQDevice//, public OutputPolicy
         LOG(info) << "Starting the benchmark and expecting to receive " << fMaxIterations << " messages.";
         auto tStart = std::chrono::high_resolution_clock::now();
 
-        while (!NewStatePending())
-        {
-            if (fMultipart)
-            {
+        while (!NewStatePending()) {
+            if (fMultipart) {
                 FairMQParts parts;
 
-                if (dataInChannel.Receive(parts) >= 0)
-                {
-                    if (fMaxIterations > 0)
-                    {
-                        if (fNumIterations >= fMaxIterations)
-                        {
+                if (dataInChannel.Receive(parts) >= 0) {
+                    if (fMaxIterations > 0) {
+                        if (fNumIterations >= fMaxIterations) {
                             LOG(info) << "Configured maximum number of iterations reached.";
                             break;
                         }
                     }
                     fNumIterations++;
                 }
-            }
-            else
-            {
+            } else {
                 FairMQMessagePtr msg(dataInChannel.NewMessage());
 
-                if (dataInChannel.Receive(msg) >= 0)
-                {
-                    if (fMaxIterations > 0)
-                    {
-                        if (fNumIterations >= fMaxIterations)
-                        {
+                if (dataInChannel.Receive(msg) >= 0) {
+                    if (fMaxIterations > 0) {
+                        if (fNumIterations >= fMaxIterations) {
                             LOG(info) << "Configured maximum number of iterations reached.";
                             break;
                         }
@@ -96,7 +85,8 @@ class FairMQSink : public FairMQDevice//, public OutputPolicy
 
         auto tEnd = std::chrono::high_resolution_clock::now();
 
-        LOG(info) << "Leaving RUNNING state. Received " << fNumIterations << " messages in " << std::chrono::duration<double, std::milli>(tEnd - tStart).count() << "ms.";
+        LOG(info) << "Leaving RUNNING state. Received " << fNumIterations << " messages in "
+                  << std::chrono::duration<double, std::milli>(tEnd - tStart).count() << "ms.";
     }
 };
 

fairmq/nanomsg/FairMQMessageNN.cxx

@@ -1,227 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQMessageNN.cxx
- *
- * @since 2013-12-05
- * @author A. Rybalchenko
- */
-
-#include <cstring>
-#include <stdlib.h>
-
-#include <nanomsg/nn.h>
-
-#include "FairMQMessageNN.h"
-#include "FairMQLogger.h"
-
-using namespace std;
-
-fair::mq::Transport FairMQMessageNN::fTransportType = fair::mq::Transport::NN;
-
-FairMQMessageNN::FairMQMessageNN(FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fMessage(nullptr)
-    , fSize(0)
-    , fHint(0)
-    , fReceiving(false)
-    , fRegionPtr(nullptr)
-{
-    fMessage = nn_allocmsg(0, 0);
-    if (!fMessage)
-    {
-        LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
-    }
-}
-
-FairMQMessageNN::FairMQMessageNN(const size_t size, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fMessage(nullptr)
-    , fSize(0)
-    , fHint(0)
-    , fReceiving(false)
-    , fRegionPtr(nullptr)
-{
-    fMessage = nn_allocmsg(size, 0);
-    if (!fMessage)
-    {
-        LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
-    }
-    fSize = size;
-}
-
-
-/* nanomsg does not offer support for creating a message out of an existing buffer,
- * therefore the following method is using memcpy. For more efficient handling,
- * create FairMQMessage object only with size parameter and fill it with data.
- * possible TODO: make this zero copy (will should then be as efficient as ZeroMQ).
-*/
-FairMQMessageNN::FairMQMessageNN(void* data, const size_t size, fairmq_free_fn* ffn, void* hint, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fMessage(nullptr)
-    , fSize(0)
-    , fHint(0)
-    , fReceiving(false)
-    , fRegionPtr(nullptr)
-{
-    fMessage = nn_allocmsg(size, 0);
-    if (!fMessage)
-    {
-        LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
-    }
-    else
-    {
-        memcpy(fMessage, data, size);
-        fSize = size;
-        if (ffn)
-        {
-            ffn(data, hint);
-        }
-        else
-        {
-            free(data);
-        }
-    }
-}
-
-FairMQMessageNN::FairMQMessageNN(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fMessage(data)
-    , fSize(size)
-    , fHint(reinterpret_cast<size_t>(hint))
-    , fReceiving(false)
-    , fRegionPtr(region.get())
-{
-    // currently nanomsg will copy the buffer (data) inside nn_sendmsg()
-}
-
-void FairMQMessageNN::Rebuild()
-{
-    CloseMessage();
-    fReceiving = false;
-}
-
-void FairMQMessageNN::Rebuild(const size_t size)
-{
-    CloseMessage();
-    fMessage = nn_allocmsg(size, 0);
-    if (!fMessage)
-    {
-        LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
-    }
-    fSize = size;
-    fReceiving = false;
-}
-
-void FairMQMessageNN::Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
-{
-    CloseMessage();
-    fMessage = nn_allocmsg(size, 0);
-    if (!fMessage)
-    {
-        LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
-    }
-    else
-    {
-        memcpy(fMessage, data, size);
-        fSize = size;
-        fReceiving = false;
-
-        if (ffn)
-        {
-            ffn(data, hint);
-        }
-    }
-}
-
-void* FairMQMessageNN::GetMessage() const
-{
-    return fMessage;
-}
-
-void* FairMQMessageNN::GetData() const
-{
-    return fMessage;
-}
-
-size_t FairMQMessageNN::GetSize() const
-{
-    return fSize;
-}
-
-bool FairMQMessageNN::SetUsedSize(const size_t size)
-{
-    if (size <= fSize)
-    {
-        // with size smaller than original nanomsg will simply "chop" the data, avoiding reallocation
-        fMessage = nn_reallocmsg(fMessage, size);
-        fSize = size;
-        return true;
-    }
-    else
-    {
-        LOG(error) << "cannot set used size higher than original.";
-        return false;
-    }
-}
-
-void FairMQMessageNN::SetMessage(void* data, const size_t size)
-{
-    fMessage = data;
-    fSize = size;
-}
-
-fair::mq::Transport FairMQMessageNN::GetType() const
-{
-    return fTransportType;
-}
-
-void FairMQMessageNN::Copy(const FairMQMessage& msg)
-{
-    if (fMessage)
-    {
-        if (nn_freemsg(fMessage) < 0)
-        {
-            LOG(error) << "failed freeing message, reason: " << nn_strerror(errno);
-        }
-    }
-
-    size_t size = msg.GetSize();
-
-    fMessage = nn_allocmsg(size, 0);
-    if (!fMessage)
-    {
-        LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
-    }
-    else
-    {
-        memcpy(fMessage, static_cast<const FairMQMessageNN&>(msg).GetMessage(), size);
-        fSize = size;
-    }
-}
-
-void FairMQMessageNN::CloseMessage()
-{
-    if (nn_freemsg(fMessage) < 0)
-    {
-        LOG(error) << "failed freeing message, reason: " << nn_strerror(errno);
-    }
-    else
-    {
-        fMessage = nullptr;
-        fSize = 0;
-    }
-}
-
-FairMQMessageNN::~FairMQMessageNN()
-{
-    if (fReceiving)
-    {
-        CloseMessage();
-    }
-}

fairmq/nanomsg/FairMQMessageNN.h

@@ -1,68 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQMessageNN.h
- *
- * @since 2013-12-05
- * @author A. Rybalchenko
- */
-
-#ifndef FAIRMQMESSAGENN_H_
-#define FAIRMQMESSAGENN_H_
-
-#include <cstddef>
-#include <string>
-#include <memory>
-
-#include "FairMQMessage.h"
-#include "FairMQUnmanagedRegion.h"
-
-class FairMQSocketNN;
-
-class FairMQMessageNN final : public FairMQMessage
-{
-    friend class FairMQSocketNN;
-
-  public:
-    FairMQMessageNN(FairMQTransportFactory* factory = nullptr);
-    FairMQMessageNN(const size_t size, FairMQTransportFactory* factory = nullptr);
-    FairMQMessageNN(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr, FairMQTransportFactory* factory = nullptr);
-    FairMQMessageNN(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0, FairMQTransportFactory* factory = nullptr);
-
-    FairMQMessageNN(const FairMQMessageNN&) = delete;
-    FairMQMessageNN operator=(const FairMQMessageNN&) = delete;
-
-    void Rebuild() override;
-    void Rebuild(const size_t size) override;
-    void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
-
-    void* GetData() const override;
-    size_t GetSize() const override;
-
-    bool SetUsedSize(const size_t size) override;
-
-    fair::mq::Transport GetType() const override;
-
-    void Copy(const FairMQMessage& msg) override;
-
-    ~FairMQMessageNN() override;
-
-  private:
-    void* fMessage;
-    size_t fSize;
-    size_t fHint;
-    bool fReceiving;
-    FairMQUnmanagedRegion* fRegionPtr;
-    static fair::mq::Transport fTransportType;
-
-    void* GetMessage() const;
-    void CloseMessage();
-    void SetMessage(void* data, const size_t size);
-};
-
-#endif /* FAIRMQMESSAGENN_H_ */

fairmq/nanomsg/FairMQPollerNN.cxx

@@ -1,207 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQPollerNN.cxx
- *
- * @since 2014-01-23
- * @author A. Rybalchenko
- */
-
-#include <nanomsg/nn.h>
-#include <nanomsg/pipeline.h>
-#include <nanomsg/pubsub.h>
-#include <nanomsg/reqrep.h>
-#include <nanomsg/pair.h>
-
-#include "FairMQPollerNN.h"
-#include "FairMQSocketNN.h"
-#include "FairMQLogger.h"
-
-using namespace std;
-
-FairMQPollerNN::FairMQPollerNN(const vector<FairMQChannel>& channels)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    fNumItems = channels.size();
-    fItems = new nn_pollfd[fNumItems];
-
-    for (int i = 0; i < fNumItems; ++i)
-    {
-        fItems[i].fd = static_cast<const FairMQSocketNN*>(&(channels.at(i).GetSocket()))->GetSocket();
-
-        int type = 0;
-        size_t sz = sizeof(type);
-        nn_getsockopt(static_cast<const FairMQSocketNN*>(&(channels.at(i).GetSocket()))->GetSocket(), NN_SOL_SOCKET, NN_PROTOCOL, &type, &sz);
-
-        SetItemEvents(fItems[i], type);
-    }
-}
-
-FairMQPollerNN::FairMQPollerNN(const vector<FairMQChannel*>& channels)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    fNumItems = channels.size();
-    fItems = new nn_pollfd[fNumItems];
-
-    for (int i = 0; i < fNumItems; ++i)
-    {
-        fItems[i].fd = static_cast<const FairMQSocketNN*>(&(channels.at(i)->GetSocket()))->GetSocket();
-
-        int type = 0;
-        size_t sz = sizeof(type);
-        nn_getsockopt(static_cast<const FairMQSocketNN*>(&(channels.at(i)->GetSocket()))->GetSocket(), NN_SOL_SOCKET, NN_PROTOCOL, &type, &sz);
-
-        SetItemEvents(fItems[i], type);
-    }
-}
-
-FairMQPollerNN::FairMQPollerNN(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    try
-    {
-        int offset = 0;
-        // calculate offsets and the total size of the poll item set
-        for (string channel : channelList)
-        {
-            fOffsetMap[channel] = offset;
-            offset += channelsMap.at(channel).size();
-            fNumItems += channelsMap.at(channel).size();
-        }
-
-        fItems = new nn_pollfd[fNumItems];
-
-        int index = 0;
-        for (string channel : channelList)
-        {
-            for (unsigned int i = 0; i < channelsMap.at(channel).size(); ++i)
-            {
-                index = fOffsetMap[channel] + i;
-                fItems[index].fd = static_cast<const FairMQSocketNN*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
-
-                int type = 0;
-                size_t sz = sizeof(type);
-                nn_getsockopt(static_cast<const FairMQSocketNN*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), NN_SOL_SOCKET, NN_PROTOCOL, &type, &sz);
-
-                SetItemEvents(fItems[index], type);
-            }
-        }
-    }
-    catch (const std::out_of_range& oor)
-    {
-        LOG(error) << "at least one of the provided channel keys for poller initialization is invalid";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-void FairMQPollerNN::SetItemEvents(nn_pollfd& item, const int type)
-{
-    if (type == NN_REQ || type == NN_REP || type == NN_PAIR)
-    {
-        item.events = NN_POLLIN|NN_POLLOUT;
-    }
-    else if (type == NN_PUSH || type == NN_PUB)
-    {
-        item.events = NN_POLLOUT;
-    }
-    else if (type == NN_PULL || type == NN_SUB)
-    {
-        item.events = NN_POLLIN;
-    }
-    else
-    {
-        LOG(error) << "invalid poller configuration, exiting.";
-        exit(EXIT_FAILURE);
-    }
-}
-
-void FairMQPollerNN::Poll(const int timeout)
-{
-    if (nn_poll(fItems, fNumItems, timeout) < 0)
-    {
-        if (errno == ETERM)
-        {
-            LOG(debug) << "polling exited, reason: " << nn_strerror(errno);
-        }
-        else
-        {
-            LOG(error) << "polling failed, reason: " << nn_strerror(errno);
-            throw std::runtime_error("polling failed");
-        }
-    }
-}
-
-bool FairMQPollerNN::CheckInput(const int index)
-{
-    if (fItems[index].revents & (NN_POLLIN | NN_POLLOUT))
-    {
-        return true;
-    }
-
-    return false;
-}
-
-bool FairMQPollerNN::CheckOutput(const int index)
-{
-    if (fItems[index].revents & NN_POLLOUT)
-    {
-        return true;
-    }
-
-    return false;
-}
-
-bool FairMQPollerNN::CheckInput(const string& channelKey, const int index)
-{
-    try
-    {
-        if (fItems[fOffsetMap.at(channelKey) + index].revents & (NN_POLLIN | NN_POLLOUT))
-        {
-            return true;
-        }
-
-        return false;
-    }
-    catch (const std::out_of_range& oor)
-    {
-        LOG(error) << "invalid channel key: \"" << channelKey << "\"";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-bool FairMQPollerNN::CheckOutput(const string& channelKey, const int index)
-{
-    try
-    {
-        if (fItems[fOffsetMap.at(channelKey) + index].revents & NN_POLLOUT)
-        {
-            return true;
-        }
-
-        return false;
-    }
-    catch (const std::out_of_range& oor)
-    {
-        LOG(error) << "invalid channel key: \"" << channelKey << "\"";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-FairMQPollerNN::~FairMQPollerNN()
-{
-    delete[] fItems;
-}

fairmq/nanomsg/FairMQPollerNN.h

@@ -1,58 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQPollerNN.h
- *
- * @since 2014-01-23
- * @author A. Rybalchenko
- */
-
-#ifndef FAIRMQPOLLERNN_H_
-#define FAIRMQPOLLERNN_H_
-
-#include <vector>
-#include <unordered_map>
-
-#include "FairMQPoller.h"
-#include "FairMQChannel.h"
-#include "FairMQTransportFactoryNN.h"
-
-class FairMQChannel;
-struct nn_pollfd;
-
-class FairMQPollerNN final : public FairMQPoller
-{
-    friend class FairMQChannel;
-    friend class FairMQTransportFactoryNN;
-
-  public:
-    FairMQPollerNN(const std::vector<FairMQChannel>& channels);
-    FairMQPollerNN(const std::vector<FairMQChannel*>& channels);
-    FairMQPollerNN(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList);
-
-    FairMQPollerNN(const FairMQPollerNN&) = delete;
-    FairMQPollerNN operator=(const FairMQPollerNN&) = delete;
-
-    void SetItemEvents(nn_pollfd& item, const int type);
-
-    void Poll(const int timeout) override;
-    bool CheckInput(const int index) override;
-    bool CheckOutput(const int index) override;
-    bool CheckInput(const std::string& channelKey, const int index) override;
-    bool CheckOutput(const std::string& channelKey, const int index) override;
-
-    ~FairMQPollerNN() override;
-
-  private:
-    nn_pollfd* fItems;
-    int fNumItems;
-
-    std::unordered_map<std::string, int> fOffsetMap;
-};
-
-#endif /* FAIRMQPOLLERNN_H_ */

fairmq/nanomsg/FairMQSocketNN.cxx

@@ -1,628 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQSocketNN.cxx
- *
- * @since 2012-12-05
- * @author A. Rybalchenko
- */
-
-#include "FairMQSocketNN.h"
-#include "FairMQMessageNN.h"
-#include "FairMQLogger.h"
-#include "FairMQUnmanagedRegionNN.h"
-#include <fairmq/Tools.h>
-
-#include <nanomsg/nn.h>
-#include <nanomsg/pipeline.h>
-#include <nanomsg/pubsub.h>
-#include <nanomsg/reqrep.h>
-#include <nanomsg/pair.h>
-
-#include <sstream>
-#include <msgpack.hpp>
-
-using namespace std;
-using namespace fair::mq;
-
-atomic<bool> FairMQSocketNN::fInterrupted(false);
-
-FairMQSocketNN::FairMQSocketNN(const string& type, const string& name, const string& id /*= ""*/, FairMQTransportFactory* fac /*=nullptr*/)
-    : FairMQSocket{fac}
-    , fSocket(-1)
-    , fId(id + "." + name + "." + type)
-    , fBytesTx(0)
-    , fBytesRx(0)
-    , fMessagesTx(0)
-    , fMessagesRx(0)
-    , fSndTimeout(100)
-    , fRcvTimeout(100)
-    , fLinger(500)
-{
-    if (type == "router" || type == "dealer")
-    {
-        // Additional info about using the sockets ROUTER and DEALER with nanomsg can be found in:
-        // http://250bpm.com/blog:14
-        // http://www.freelists.org/post/nanomsg/a-stupid-load-balancing-question,1
-        fSocket = nn_socket(AF_SP_RAW, GetConstant(type));
-        if (fSocket == -1)
-        {
-            LOG(error) << "failed creating socket " << fId << ", reason: " << nn_strerror(errno);
-            exit(EXIT_FAILURE);
-        }
-    }
-    else
-    {
-        fSocket = nn_socket(AF_SP, GetConstant(type));
-        if (fSocket == -1)
-        {
-            LOG(error) << "failed creating socket " << fId << ", reason: " << nn_strerror(errno);
-            exit(EXIT_FAILURE);
-        }
-        if (type == "sub")
-        {
-            nn_setsockopt(fSocket, NN_SUB, NN_SUB_SUBSCRIBE, nullptr, 0);
-        }
-    }
-
-    if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0)
-    {
-        LOG(error) << "Failed setting NN_SNDTIMEO socket option, reason: " << nn_strerror(errno);
-    }
-
-    if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0)
-    {
-        LOG(error) << "Failed setting NN_RCVTIMEO socket option, reason: " << nn_strerror(errno);
-    }
-
-#ifdef NN_RCVMAXSIZE
-    int rcvSize = -1;
-    if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_RCVMAXSIZE, &rcvSize, sizeof(rcvSize)) != 0)
-    {
-        LOG(error) << "Failed setting NN_RCVMAXSIZE socket option, reason: " << nn_strerror(errno);
-    }
-#endif
-
-    LOG(debug) << "Created socket " << GetId();
-}
-
-bool FairMQSocketNN::Bind(const string& address)
-{
-    // LOG(info) << "bind socket " << fId << " on " << address;
-
-    if (nn_bind(fSocket, address.c_str()) < 0)
-    {
-        LOG(error) << "failed binding socket " << fId << ", reason: " << nn_strerror(errno);
-        return false;
-    }
-
-    return true;
-}
-
-bool FairMQSocketNN::Connect(const string& address)
-{
-    // LOG(info) << "connect socket " << fId << " to " << address;
-
-    if (nn_connect(fSocket, address.c_str()) < 0)
-    {
-        LOG(error) << "failed connecting socket " << fId << ", reason: " << nn_strerror(errno);
-        return false;
-    }
-
-    return true;
-}
-
-int FairMQSocketNN::Send(FairMQMessagePtr& msg, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0)
-    {
-        flags = NN_DONTWAIT;
-    }
-    int nbytes = -1;
-    int elapsed = 0;
-
-    FairMQMessageNN* msgPtr = static_cast<FairMQMessageNN*>(msg.get());
-    void* bufPtr = msgPtr->GetMessage();
-
-    while (true)
-    {
-        if (msgPtr->fRegionPtr == nullptr)
-        {
-            nbytes = nn_send(fSocket, &bufPtr, NN_MSG, flags);
-        }
-        else
-        {
-            nbytes = nn_send(fSocket, bufPtr, msg->GetSize(), flags);
-            // nn_send copies the data, safe to call region callback here
-            static_cast<FairMQUnmanagedRegionNN*>(msgPtr->fRegionPtr)->fCallback(bufPtr, msg->GetSize(), reinterpret_cast<void*>(msgPtr->fHint));
-        }
-
-        if (nbytes >= 0)
-        {
-            fBytesTx += nbytes;
-            ++fMessagesTx;
-            static_cast<FairMQMessageNN*>(msg.get())->fReceiving = false;
-
-            return nbytes;
-        }
-        else if (nn_errno() == ETIMEDOUT)
-        {
-            if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
-            {
-                if (timeout > 0)
-                {
-                    elapsed += fSndTimeout;
-                    if (elapsed >= timeout)
-                    {
-                        return -2;
-                    }
-                }
-                continue;
-            }
-            else
-            {
-                return -2;
-            }
-        }
-        else if (nn_errno() == EAGAIN)
-        {
-            return -2;
-        }
-        else if (nn_errno() == ETERM)
-        {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        }
-        else
-        {
-            LOG(error) << "Failed sending on socket " << fId << ", reason: " << nn_strerror(errno);
-            return nbytes;
-        }
-    }
-}
-
-int FairMQSocketNN::Receive(FairMQMessagePtr& msg, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0)
-    {
-        flags = NN_DONTWAIT;
-    }
-    int elapsed = 0;
-
-    FairMQMessageNN* msgPtr = static_cast<FairMQMessageNN*>(msg.get());
-
-    while (true)
-    {
-        void* ptr = nullptr;
-        int nbytes = nn_recv(fSocket, &ptr, NN_MSG, flags);
-        if (nbytes >= 0)
-        {
-            fBytesRx += nbytes;
-            ++fMessagesRx;
-            msgPtr->SetMessage(ptr, nbytes);
-            msgPtr->fReceiving = true;
-            return nbytes;
-        }
-        else if (nn_errno() == ETIMEDOUT)
-        {
-            if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
-            {
-                if (timeout > 0)
-                {
-                    elapsed += fRcvTimeout;
-                    if (elapsed >= timeout)
-                    {
-                        return -2;
-                    }
-                }
-                continue;
-            }
-            else
-            {
-                return -2;
-            }
-        }
-        else if (nn_errno() == EAGAIN)
-        {
-            return -2;
-        }
-        else if (nn_errno() == ETERM)
-        {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        }
-        else
-        {
-            LOG(error) << "Failed receiving on socket " << fId << ", reason: " << nn_strerror(errno);
-            return nbytes;
-        }
-    }
-}
-
-int64_t FairMQSocketNN::Send(vector<FairMQMessagePtr>& msgVec, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0)
-    {
-        flags = NN_DONTWAIT;
-    }
-    const unsigned int vecSize = msgVec.size();
-    int elapsed = 0;
-
-    // create msgpack simple buffer
-    msgpack::sbuffer sbuf;
-    // create msgpack packer
-    msgpack::packer<msgpack::sbuffer> packer(&sbuf);
-
-    // pack all parts into a single msgpack simple buffer
-    for (unsigned int i = 0; i < vecSize; ++i)
-    {
-        FairMQMessageNN* partPtr = static_cast<FairMQMessageNN*>(msgVec[i].get());
-
-        partPtr->fReceiving = false;
-        packer.pack_bin(msgVec[i]->GetSize());
-        packer.pack_bin_body(static_cast<char*>(msgVec[i]->GetData()), msgVec[i]->GetSize());
-        // call region callback
-        if (partPtr->fRegionPtr)
-        {
-            static_cast<FairMQUnmanagedRegionNN*>(partPtr->fRegionPtr)->fCallback(partPtr->GetMessage(), partPtr->GetSize(), reinterpret_cast<void*>(partPtr->fHint));
-        }
-    }
-
-    while (true)
-    {
-        int64_t nbytes = nn_send(fSocket, sbuf.data(), sbuf.size(), flags);
-        if (nbytes >= 0)
-        {
-            fBytesTx += nbytes;
-            ++fMessagesTx;
-            return nbytes;
-        }
-        else if (nn_errno() == ETIMEDOUT)
-        {
-            if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
-            {
-                if (timeout > 0)
-                {
-                    elapsed += fSndTimeout;
-                    if (elapsed >= timeout)
-                    {
-                        return -2;
-                    }
-                }
-                continue;
-            }
-            else
-            {
-                return -2;
-            }
-        }
-        else if (nn_errno() == EAGAIN)
-        {
-            return -2;
-        }
-        else if (nn_errno() == ETERM)
-        {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        }
-        else
-        {
-            LOG(error) << "Failed sending on socket " << fId << ", reason: " << nn_strerror(errno);
-            return nbytes;
-        }
-    }
-}
-
-int64_t FairMQSocketNN::Receive(vector<FairMQMessagePtr>& msgVec, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0)
-    {
-        flags = NN_DONTWAIT;
-    }
-    // Warn if the vector is filled before Receive() and empty it.
-    // if (msgVec.size() > 0)
-    // {
-    //     LOG(warn) << "Message vector contains elements before Receive(), they will be deleted!";
-    //     msgVec.clear();
-    // }
-
-    int elapsed = 0;
-
-    while (true)
-    {
-        // pointer to point to received message buffer
-        char* ptr = nullptr;
-        // receive the message into a buffer allocated by nanomsg and let ptr point to it
-        int nbytes = nn_recv(fSocket, &ptr, NN_MSG, flags);
-        if (nbytes >= 0) // if no errors or non-blocking timeouts
-        {
-            // store statistics on how many bytes received
-            fBytesRx += nbytes;
-            // store statistics on how many messages received (count messages instead of parts)
-            ++fMessagesRx;
-
-            // offset to be used by msgpack to handle separate chunks
-            size_t offset = 0;
-            while (offset != static_cast<size_t>(nbytes)) // continue until all parts have been read
-            {
-                // vector of chars to hold blob (unlike char*/void* this type can be converted to by msgpack)
-                vector<char> buf;
-
-                // unpack and convert chunk
-                msgpack::unpacked result;
-                unpack(result, ptr, nbytes, offset);
-                msgpack::object object(result.get());
-                object.convert(buf);
-                // get the single message size
-                size_t size = buf.size() * sizeof(char);
-                FairMQMessagePtr part(new FairMQMessageNN(size, GetTransport()));
-                static_cast<FairMQMessageNN*>(part.get())->fReceiving = true;
-                memcpy(part->GetData(), buf.data(), size);
-                msgVec.push_back(move(part));
-            }
-
-            nn_freemsg(ptr);
-            return nbytes;
-        }
-        else if (nn_errno() == ETIMEDOUT)
-        {
-            if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
-            {
-                if (timeout > 0)
-                {
-                    elapsed += fRcvTimeout;
-                    if (elapsed >= timeout)
-                    {
-                        return -2;
-                    }
-                }
-                continue;
-            }
-            else
-            {
-                return -2;
-            }
-        }
-        else if (nn_errno() == EAGAIN)
-        {
-            return -2;
-        }
-        else if (nn_errno() == ETERM)
-        {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        }
-        else
-        {
-            LOG(error) << "Failed receiving on socket " << fId << ", reason: " << nn_strerror(errno);
-            return nbytes;
-        }
-    }
-}
-
-void FairMQSocketNN::Close()
-{
-    nn_close(fSocket);
-}
-
-void FairMQSocketNN::Interrupt()
-{
-    fInterrupted = true;
-}
-
-void FairMQSocketNN::Resume()
-{
-    fInterrupted = false;
-}
-
-int FairMQSocketNN::GetSocket() const
-{
-    return fSocket;
-}
-
-void FairMQSocketNN::SetOption(const string& option, const void* value, size_t valueSize)
-{
-    if (option == "snd-size" || option == "rcv-size")
-    {
-        int val = *(static_cast<int*>(const_cast<void*>(value)));
-        if (val <= 0)
-        {
-            LOG(warn) << "value for sndKernelSize/rcvKernelSize should be greater than 0, leaving unchanged.";
-            return;
-        }
-    }
-
-    if (option == "snd-hwm" || option == "rcv-hwm")
-    {
-        return;
-    }
-
-    if (option == "linger")
-    {
-        fLinger = *static_cast<int*>(const_cast<void*>(value));
-        return;
-    }
-
-    int rc = nn_setsockopt(fSocket, NN_SOL_SOCKET, GetConstant(option), value, valueSize);
-    if (rc < 0)
-    {
-        LOG(error) << "failed setting socket option, reason: " << nn_strerror(errno);
-    }
-}
-
-void FairMQSocketNN::GetOption(const string& option, void* value, size_t* valueSize)
-{
-    if (option == "linger")
-    {
-        *static_cast<int*>(value) = fLinger;
-        return;
-    }
-
-    if (option == "snd-hwm" || option == "rcv-hwm")
-    {
-        *static_cast<int*>(value) = -1;
-        return;
-    }
-
-
-    int rc = nn_getsockopt(fSocket, NN_SOL_SOCKET, GetConstant(option), value, valueSize);
-    if (rc < 0)
-    {
-        LOG(error) << "failed getting socket option, reason: " << nn_strerror(errno);
-    }
-}
-
-void FairMQSocketNN::SetLinger(const int value)
-{
-    fLinger = value;
-}
-
-int FairMQSocketNN::GetLinger() const
-{
-    return fLinger;
-}
-
-void FairMQSocketNN::SetSndBufSize(const int /* value */)
-{
-    // not used in nanomsg
-}
-
-int FairMQSocketNN::GetSndBufSize() const
-{
-    // not used in nanomsg
-    return -1;
-}
-
-void FairMQSocketNN::SetRcvBufSize(const int /* value */)
-{
-    // not used in nanomsg
-}
-
-int FairMQSocketNN::GetRcvBufSize() const
-{
-    // not used in nanomsg
-    return -1;
-}
-
-void FairMQSocketNN::SetSndKernelSize(const int value)
-{
-    if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_SNDBUF, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed setting NN_SNDBUF, reason: ", nn_strerror(errno)));
-    }
-}
-
-int FairMQSocketNN::GetSndKernelSize() const
-{
-    int value = 0;
-    size_t valueSize;
-    if (nn_getsockopt(fSocket, NN_SOL_SOCKET, NN_SNDBUF, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting NN_SNDBUF, reason: ", nn_strerror(errno)));
-    }
-    return value;
-}
-
-void FairMQSocketNN::SetRcvKernelSize(const int value)
-{
-    if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_RCVBUF, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed setting NN_RCVBUF, reason: ", nn_strerror(errno)));
-    }
-}
-
-int FairMQSocketNN::GetRcvKernelSize() const
-{
-    int value = 0;
-    size_t valueSize;
-    if (nn_getsockopt(fSocket, NN_SOL_SOCKET, NN_RCVBUF, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting NN_RCVBUF, reason: ", nn_strerror(errno)));
-    }
-    return value;
-}
-
-
-unsigned long FairMQSocketNN::GetBytesTx() const
-{
-    return fBytesTx;
-}
-
-unsigned long FairMQSocketNN::GetBytesRx() const
-{
-    return fBytesRx;
-}
-
-unsigned long FairMQSocketNN::GetMessagesTx() const
-{
-    return fMessagesTx;
-}
-
-unsigned long FairMQSocketNN::GetMessagesRx() const
-{
-    return fMessagesRx;
-}
-
-int FairMQSocketNN::GetConstant(const string& constant)
-{
-    if (constant == "")
-        return 0;
-    if (constant == "sub")
-        return NN_SUB;
-    if (constant == "pub")
-        return NN_PUB;
-    if (constant == "xsub")
-        return NN_SUB;
-    if (constant == "xpub")
-        return NN_PUB;
-    if (constant == "push")
-        return NN_PUSH;
-    if (constant == "pull")
-        return NN_PULL;
-    if (constant == "req")
-        return NN_REQ;
-    if (constant == "rep")
-        return NN_REP;
-    if (constant == "dealer")
-        return NN_REQ;
-    if (constant == "router")
-        return NN_REP;
-    if (constant == "pair")
-        return NN_PAIR;
-
-    if (constant == "snd-hwm")
-        return NN_SNDBUF;
-    if (constant == "rcv-hwm")
-        return NN_RCVBUF;
-    if (constant == "snd-size")
-        return NN_SNDBUF;
-    if (constant == "rcv-size")
-        return NN_RCVBUF;
-    if (constant == "snd-more")
-    {
-        LOG(error) << "Multipart messages functionality currently not supported by nanomsg!";
-        return -1;
-    }
-    if (constant == "rcv-more")
-    {
-        LOG(error) << "Multipart messages functionality currently not supported by nanomsg!";
-        return -1;
-    }
-
-    if (constant == "linger")
-        return NN_LINGER;
-    if (constant == "no-block")
-        return NN_DONTWAIT;
-
-    return -1;
-}
-
-FairMQSocketNN::~FairMQSocketNN()
-{
-    Close();
-}

fairmq/nanomsg/FairMQSocketNN.h

@@ -1,81 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIRMQSOCKETNN_H_
-#define FAIRMQSOCKETNN_H_
-
-#include <vector>
-#include <atomic>
-
-#include "FairMQSocket.h"
-#include "FairMQMessage.h"
-class FairMQTransportFactory;
-
-class FairMQSocketNN final : public FairMQSocket
-{
-  public:
-    FairMQSocketNN(const std::string& type, const std::string& name, const std::string& id = "", FairMQTransportFactory* fac = nullptr);
-    FairMQSocketNN(const FairMQSocketNN&) = delete;
-    FairMQSocketNN operator=(const FairMQSocketNN&) = delete;
-
-    std::string GetId() const override { return fId; }
-
-    bool Bind(const std::string& address) override;
-    bool Connect(const std::string& address) override;
-
-    int Send(FairMQMessagePtr& msg, const int timeout = -1) override;
-    int Receive(FairMQMessagePtr& msg, const int timeout = -1) override;
-    int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int timeout = -1) override;
-    int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int timeout = -1) override;
-
-    int GetSocket() const;
-
-    void Close() override;
-
-    static void Interrupt();
-    static void Resume();
-
-    void SetOption(const std::string& option, const void* value, size_t valueSize) override;
-    void GetOption(const std::string& option, void* value, size_t* valueSize) override;
-
-    void SetLinger(const int value) override;
-    int GetLinger() const override;
-    void SetSndBufSize(const int value) override;
-    int GetSndBufSize() const override;
-    void SetRcvBufSize(const int value) override;
-    int GetRcvBufSize() const override;
-    void SetSndKernelSize(const int value) override;
-    int GetSndKernelSize() const override;
-    void SetRcvKernelSize(const int value) override;
-    int GetRcvKernelSize() const override;
-
-    unsigned long GetBytesTx() const override;
-    unsigned long GetBytesRx() const override;
-    unsigned long GetMessagesTx() const override;
-    unsigned long GetMessagesRx() const override;
-
-    static int GetConstant(const std::string& constant);
-
-    ~FairMQSocketNN() override;
-
-  private:
-    int fSocket;
-    std::string fId;
-    std::atomic<unsigned long> fBytesTx;
-    std::atomic<unsigned long> fBytesRx;
-    std::atomic<unsigned long> fMessagesTx;
-    std::atomic<unsigned long> fMessagesRx;
-
-    static std::atomic<bool> fInterrupted;
-
-    int fSndTimeout;
-    int fRcvTimeout;
-    int fLinger;
-};
-
-#endif /* FAIRMQSOCKETNN_H_ */

fairmq/nanomsg/FairMQTransportFactoryNN.cxx

@@ -1,94 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2014-2017 GSI Helmholtzzentrum fuer Schwerionenforschung Gmb   *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "FairMQTransportFactoryNN.h"
-
-#include <nanomsg/nn.h>
-#include <algorithm>
-#include <thread>
-#include <chrono>
-
-using namespace std;
-
-fair::mq::Transport FairMQTransportFactoryNN::fTransportType = fair::mq::Transport::NN;
-
-FairMQTransportFactoryNN::FairMQTransportFactoryNN(const string& id, const fair::mq::ProgOptions* /*config*/)
-    : FairMQTransportFactory(id)
-{
-    LOG(debug) << "Transport: Using nanomsg library";
-}
-
-FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage()
-{
-    return unique_ptr<FairMQMessage>(new FairMQMessageNN(this));
-}
-
-FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage(const size_t size)
-{
-    return unique_ptr<FairMQMessage>(new FairMQMessageNN(size, this));
-}
-
-FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
-{
-    return unique_ptr<FairMQMessage>(new FairMQMessageNN(data, size, ffn, hint, this));
-}
-
-FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint)
-{
-    return unique_ptr<FairMQMessage>(new FairMQMessageNN(region, data, size, hint, this));
-}
-
-FairMQSocketPtr FairMQTransportFactoryNN::CreateSocket(const string& type, const string& name)
-{
-    unique_ptr<FairMQSocket> socket(new FairMQSocketNN(type, name, GetId(), this));
-    fSockets.push_back(socket.get());
-    return socket;
-}
-
-FairMQPollerPtr FairMQTransportFactoryNN::CreatePoller(const vector<FairMQChannel>& channels) const
-{
-    return unique_ptr<FairMQPoller>(new FairMQPollerNN(channels));
-}
-
-FairMQPollerPtr FairMQTransportFactoryNN::CreatePoller(const std::vector<FairMQChannel*>& channels) const
-{
-    return unique_ptr<FairMQPoller>(new FairMQPollerNN(channels));
-}
-
-FairMQPollerPtr FairMQTransportFactoryNN::CreatePoller(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList) const
-{
-    return unique_ptr<FairMQPoller>(new FairMQPollerNN(channelsMap, channelList));
-}
-
-FairMQUnmanagedRegionPtr FairMQTransportFactoryNN::CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */) const
-{
-    return unique_ptr<FairMQUnmanagedRegion>(new FairMQUnmanagedRegionNN(size, callback, path, flags));
-}
-
-fair::mq::Transport FairMQTransportFactoryNN::GetType() const
-{
-    return fTransportType;
-}
-
-void FairMQTransportFactoryNN::Reset()
-{
-    auto it = max_element(fSockets.begin(), fSockets.end(), [](FairMQSocket* s1, FairMQSocket* s2) {
-        return static_cast<FairMQSocketNN*>(s1)->GetLinger() < static_cast<FairMQSocketNN*>(s2)->GetLinger();
-    });
-    if (it != fSockets.end()) {
-        this_thread::sleep_for(chrono::milliseconds(static_cast<FairMQSocketNN*>(*it)->GetLinger()));
-    }
-    fSockets.clear();
-}
-
-FairMQTransportFactoryNN::~FairMQTransportFactoryNN()
-{
-    LOG(debug) << "Destroying Shared Memory transport...";
-    // nn_term();
-    // see https://www.freelists.org/post/nanomsg/Getting-rid-of-nn-init-and-nn-term,8
-}

fairmq/nanomsg/FairMQTransportFactoryNN.h

@@ -1,52 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2014-2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIRMQTRANSPORTFACTORYNN_H_
-#define FAIRMQTRANSPORTFACTORYNN_H_
-
-#include "FairMQTransportFactory.h"
-#include "FairMQMessageNN.h"
-#include "FairMQSocketNN.h"
-#include "FairMQPollerNN.h"
-#include "FairMQUnmanagedRegionNN.h"
-#include <fairmq/ProgOptions.h>
-
-#include <vector>
-#include <string>
-
-class FairMQTransportFactoryNN final : public FairMQTransportFactory
-{
-  public:
-    FairMQTransportFactoryNN(const std::string& id = "", const fair::mq::ProgOptions* config = nullptr);
-    ~FairMQTransportFactoryNN() override;
-
-    FairMQMessagePtr CreateMessage() override;
-    FairMQMessagePtr CreateMessage(const size_t size) override;
-    FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
-    FairMQMessagePtr CreateMessage(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0) override;
-
-    FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name) override;
-
-    FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const override;
-    FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel*>& channels) const override;
-    FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const override;
-
-    FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback, const std::string& path = "", int flags = 0) const override;
-
-    fair::mq::Transport GetType() const override;
-
-    void Interrupt() override { FairMQSocketNN::Interrupt(); }
-    void Resume() override { FairMQSocketNN::Resume(); }
-    void Reset() override;
-
-  private:
-    static fair::mq::Transport fTransportType;
-    mutable std::vector<FairMQSocket*> fSockets;
-};
-
-#endif /* FAIRMQTRANSPORTFACTORYNN_H_ */

fairmq/nanomsg/FairMQUnmanagedRegionNN.cxx

@@ -1,35 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "FairMQUnmanagedRegionNN.h"
-#include "FairMQLogger.h"
-
-using namespace std;
-
-FairMQUnmanagedRegionNN::FairMQUnmanagedRegionNN(const size_t size, FairMQRegionCallback callback, const std::string& /*path = "" */, int /*flags = 0 */)
-    : fBuffer(malloc(size))
-    , fSize(size)
-    , fCallback(callback)
-{
-}
-
-void* FairMQUnmanagedRegionNN::GetData() const
-{
-    return fBuffer;
-}
-
-size_t FairMQUnmanagedRegionNN::GetSize() const
-{
-    return fSize;
-}
-
-FairMQUnmanagedRegionNN::~FairMQUnmanagedRegionNN()
-{
-    LOG(debug) << "destroying region";
-    free(fBuffer);
-}

fairmq/nanomsg/FairMQUnmanagedRegionNN.h

@@ -1,37 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIRMQUNMANAGEDREGIONNN_H_
-#define FAIRMQUNMANAGEDREGIONNN_H_
-
-#include "FairMQUnmanagedRegion.h"
-
-#include <cstddef> // size_t
-#include <string>
-
-class FairMQUnmanagedRegionNN final : public FairMQUnmanagedRegion
-{
-    friend class FairMQSocketNN;
-
-  public:
-    FairMQUnmanagedRegionNN(const size_t size, FairMQRegionCallback callback, const std::string& path = "", int flags = 0);
-    FairMQUnmanagedRegionNN(const FairMQUnmanagedRegionNN&) = delete;
-    FairMQUnmanagedRegionNN operator=(const FairMQUnmanagedRegionNN&) = delete;
-
-    virtual void* GetData() const override;
-    virtual size_t GetSize() const override;
-
-    virtual ~FairMQUnmanagedRegionNN();
-
-  private:
-    void* fBuffer;
-    size_t fSize;
-    FairMQRegionCallback fCallback;
-};
-
-#endif /* FAIRMQUNMANAGEDREGIONNN_H_ */

fairmq/ofi/Message.cxx

@@ -34,6 +34,16 @@ Message::Message(boost::container::pmr::memory_resource* pmr)
 {
 }
 
+Message::Message(boost::container::pmr::memory_resource* pmr, Alignment /* alignment */)
+    : fInitialSize(0)
+    , fSize(0)
+    , fData(nullptr)
+    , fFreeFunction(nullptr)
+    , fHint(nullptr)
+    , fPmr(pmr)
+{
+}
+
 Message::Message(boost::container::pmr::memory_resource* pmr, const size_t size)
     : fInitialSize(size)
     , fSize(size)
@@ -48,6 +58,20 @@ Message::Message(boost::container::pmr::memory_resource* pmr, const size_t size)
     }
 }
 
+Message::Message(boost::container::pmr::memory_resource* pmr, const size_t size, Alignment /* alignment */)
+    : fInitialSize(size)
+    , fSize(size)
+    , fData(nullptr)
+    , fFreeFunction(nullptr)
+    , fHint(nullptr)
+    , fPmr(pmr)
+{
+    if (size) {
+        fData = fPmr->allocate(size);
+        assert(fData);
+    }
+}
+
 Message::Message(boost::container::pmr::memory_resource* pmr,
                  void* data,
                  const size_t size,

fairmq/ofi/Message.h

@@ -34,7 +34,9 @@ class Message final : public fair::mq::Message
 {
   public:
     Message(boost::container::pmr::memory_resource* pmr);
+    Message(boost::container::pmr::memory_resource* pmr, Alignment alignment);
     Message(boost::container::pmr::memory_resource* pmr, const size_t size);
+    Message(boost::container::pmr::memory_resource* pmr, const size_t size, Alignment alignment);
     Message(boost::container::pmr::memory_resource* pmr,
             void* data,
             const size_t size,

fairmq/ofi/Socket.h

@@ -45,6 +45,7 @@ class Socket final : public fair::mq::Socket
 
     auto GetId() const -> std::string override { return fId; }
 
+    auto Events(uint32_t *events) -> void override { *events = 0; }
     auto Bind(const std::string& address) -> bool override;
     auto Connect(const std::string& address) -> bool override;
 

fairmq/ofi/TransportFactory.cxx

@@ -41,11 +41,23 @@ auto TransportFactory::CreateMessage() -> MessagePtr
     return MessagePtr{new Message(&fMemoryResource)};
 }
 
+auto TransportFactory::CreateMessage(Alignment /* alignment */) -> MessagePtr
+{
+    // TODO Do not ignore alignment
+    return MessagePtr{new Message(&fMemoryResource)};
+}
+
 auto TransportFactory::CreateMessage(const size_t size) -> MessagePtr
 {
     return MessagePtr{new Message(&fMemoryResource, size)};
 }
 
+auto TransportFactory::CreateMessage(const size_t size, Alignment /* alignment */) -> MessagePtr
+{
+    // TODO Do not ignore alignment
+    return MessagePtr{new Message(&fMemoryResource, size)};
+}
+
 auto TransportFactory::CreateMessage(void* data,
                                      const size_t size,
                                      fairmq_free_fn* ffn,
@@ -85,7 +97,22 @@ auto TransportFactory::CreatePoller(const unordered_map<string, vector<FairMQCha
     // return PollerPtr{new Poller(channelsMap, channelList)};
 }
 
-auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, FairMQRegionCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) const -> UnmanagedRegionPtr
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, FairMQRegionCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) -> UnmanagedRegionPtr
+{
+    throw runtime_error{"Not yet implemented UMR."};
+}
+
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, FairMQRegionBulkCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) -> UnmanagedRegionPtr
+{
+    throw runtime_error{"Not yet implemented UMR."};
+}
+
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, const int64_t /*userFlags*/, FairMQRegionCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) -> UnmanagedRegionPtr
+{
+    throw runtime_error{"Not yet implemented UMR."};
+}
+
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, const int64_t /*userFlags*/, FairMQRegionBulkCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) -> UnmanagedRegionPtr
 {
     throw runtime_error{"Not yet implemented UMR."};
 }

fairmq/ofi/TransportFactory.h

@@ -36,7 +36,9 @@ class TransportFactory final : public FairMQTransportFactory
     TransportFactory operator=(const TransportFactory&) = delete;
 
     auto CreateMessage() -> MessagePtr override;
+    auto CreateMessage(Alignment alignment) -> MessagePtr override;
     auto CreateMessage(const std::size_t size) -> MessagePtr override;
+    auto CreateMessage(const std::size_t size, Alignment alignment) -> MessagePtr override;
     auto CreateMessage(void* data, const std::size_t size, fairmq_free_fn* ffn, void* hint = nullptr) -> MessagePtr override;
     auto CreateMessage(UnmanagedRegionPtr& region, void* data, const std::size_t size, void* hint = nullptr) -> MessagePtr override;
 
@@ -46,7 +48,15 @@ class TransportFactory final : public FairMQTransportFactory
     auto CreatePoller(const std::vector<FairMQChannel*>& channels) const -> PollerPtr override;
     auto CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const -> PollerPtr override;
 
-    auto CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const -> UnmanagedRegionPtr override;
+    auto CreateUnmanagedRegion(const size_t size, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) -> UnmanagedRegionPtr override;
+    auto CreateUnmanagedRegion(const size_t size, RegionBulkCallback callback = nullptr, const std::string& path = "", int flags = 0) -> UnmanagedRegionPtr override;
+    auto CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) -> UnmanagedRegionPtr override;
+    auto CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionBulkCallback callback = nullptr, const std::string& path = "", int flags = 0) -> UnmanagedRegionPtr override;
+
+    void SubscribeToRegionEvents(RegionEventCallback /* callback */) override { LOG(error) << "SubscribeToRegionEvents not yet implemented for OFI"; }
+    bool SubscribedToRegionEvents() override { LOG(error) << "Region event subscriptions not yet implemented for OFI"; return false; }
+    void UnsubscribeFromRegionEvents() override { LOG(error) << "UnsubscribeFromRegionEvents not yet implemented for OFI"; }
+    std::vector<FairMQRegionInfo> GetRegionInfo() override { LOG(error) << "GetRegionInfo not yet implemented for OFI, returning empty vector"; return std::vector<FairMQRegionInfo>(); }
 
     auto GetType() const -> Transport override;
 

fairmq/plugins/Control.cxx

@@ -24,7 +24,7 @@ namespace
     std::atomic<sig_atomic_t> gLastSignal(0);
     std::atomic<int> gSignalCount(0);
 
-    extern "C" auto signal_handler(int signal) -> void
+    extern "C" auto sigint_handler(int signal) -> void
     {
         ++gSignalCount;
         gLastSignal = signal;
@@ -33,6 +33,12 @@ namespace
             std::abort();
         }
     }
+
+    extern "C" auto sigterm_handler(int signal) -> void
+    {
+        ++gSignalCount;
+        gLastSignal = signal;
+    }
 }
 
 namespace fair
@@ -85,8 +91,8 @@ Control::Control(const string& name, const Plugin::Version version, const string
     if (GetProperty<int>("catch-signals") > 0) {
         LOG(debug) << "Plugin '" << name << "' is setting up signal handling for SIGINT and SIGTERM";
         fSignalHandlerThread = thread(&Control::SignalHandler, this);
-        signal(SIGINT, signal_handler);
-        signal(SIGTERM, signal_handler);
+        signal(SIGINT, sigint_handler);
+        signal(SIGTERM, sigterm_handler);
     } else {
         LOG(warn) << "Signal handling (e.g. Ctrl-C) has been deactivated.";
     }

fairmq/plugins/DDS/DDS.cxx

@@ -8,8 +8,6 @@
 
 #include "DDS.h"
 
-#include <fairmq/sdk/commands/Commands.h>
-
 #include <fairmq/tools/Strings.h>
 
 #include <boost/algorithm/string/join.hpp>
@@ -37,6 +35,7 @@ DDS::DDS(const string& name,
          const string& homepage,
          PluginServices* pluginServices)
     : Plugin(name, version, maintainer, homepage, pluginServices)
+    , fDDSTaskId(dds::env_prop<dds::task_id>())
     , fCurrentState(DeviceState::Idle)
     , fLastState(DeviceState::Idle)
     , fDeviceTerminationRequested(false)
@@ -73,15 +72,22 @@ DDS::DDS(const string& name,
         // subscribe to device state changes, pushing new state changes into the event queue
         SubscribeToDeviceStateChange([&](DeviceState newState) {
             switch (newState) {
-                case DeviceState::Bound:
+                case DeviceState::Bound: {
                     // Receive addresses of connecting channels from DDS
                     // and propagate addresses of bound channels to DDS.
                     FillChannelContainers();
 
+                    // allow updates from key value after channel containers are filled
+                    {
+                        lock_guard<mutex> lk(fUpdateMutex);
+                        fUpdatesAllowed = true;
+                    }
+                    fUpdateCondition.notify_one();
+
                     // publish bound addresses via DDS at keys corresponding to the channel
                     // prefixes, e.g. 'data' in data[i]
                     PublishBoundChannels();
-                    break;
+                } break;
                 case DeviceState::ResettingDevice: {
                     {
                         lock_guard<mutex> lk(fUpdateMutex);
@@ -89,9 +95,8 @@ DDS::DDS(const string& name,
                     }
 
                     EmptyChannelContainers();
-                    break;
-                }
-                case DeviceState::Exiting:
+                } break;
+                case DeviceState::Exiting: {
                     if (!fControllerThread.joinable()) {
                         fControllerThread = thread(&DDS::WaitForExitingAck, this);
                     }
@@ -99,21 +104,32 @@ DDS::DDS(const string& name,
                     fDeviceTerminationRequested = true;
                     UnsubscribeFromDeviceStateChange();
                     ReleaseDeviceControl();
-                    break;
+                } break;
                 default:
                     break;
             }
 
-            lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+            using namespace sdk::cmd;
+            auto now = chrono::steady_clock::now();
             string id = GetProperty<string>("id");
             fLastState = fCurrentState;
             fCurrentState = newState;
-            using namespace sdk::cmd;
-            for (auto subscriberId : fStateChangeSubscribers) {
-                LOG(debug) << "Publishing state-change: " << fLastState << "->" << newState << " to " << subscriberId;
 
-                Cmds cmds(make<StateChange>(id, dds::env_prop<dds::task_id>(), fLastState, fCurrentState));
-                fDDS.Send(cmds.Serialize(), to_string(subscriberId));
+            lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+            for (auto it = fStateChangeSubscribers.cbegin(); it != fStateChangeSubscribers.end();) {
+                // if a subscriber did not send a heartbeat in more than 3 times the promised interval,
+                // remove it from the subscriber list
+                if (chrono::duration<double>(now - it->second.first).count() > 3 * it->second.second) {
+                    LOG(warn) << "Controller '" << it->first
+                              << "' did not send heartbeats since over 3 intervals ("
+                              << 3 * it->second.second << " ms), removing it.";
+                    fStateChangeSubscribers.erase(it++);
+                } else {
+                    LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState << " to " << it->first;
+                    Cmds cmds(make<StateChange>(id, fDDSTaskId, fLastState, fCurrentState));
+                    fDDS.Send(cmds.Serialize(), to_string(it->first));
+                    ++it;
+                }
             }
         });
 
@@ -143,10 +159,10 @@ auto DDS::StartWorkerThread() -> void
 auto DDS::WaitForExitingAck() -> void
 {
     unique_lock<mutex> lock(fStateChangeSubscriberMutex);
-    fExitingAcked.wait_for(
-        lock,
-        chrono::milliseconds(GetProperty<unsigned int>("wait-for-exiting-ack-timeout")),
-        [this]() { return fExitingAckedByLastExternalController; });
+    auto timeout = GetProperty<unsigned int>("wait-for-exiting-ack-timeout");
+    fExitingAcked.wait_for(lock, chrono::milliseconds(timeout), [this]() {
+        return fExitingAckedByLastExternalController || fStateChangeSubscribers.empty();
+    });
 }
 
 auto DDS::FillChannelContainers() -> void
@@ -212,11 +228,6 @@ auto DDS::FillChannelContainers() -> void
             LOG(debug) << "dds-i-n: adding " << chanName << " -> i: " << i << " n: " << n;
             fIofN.insert(make_pair(chanName, IofN(i, n)));
         }
-        {
-            lock_guard<mutex> lk(fUpdateMutex);
-            fUpdatesAllowed = true;
-        }
-        fUpdateCondition.notify_one();
     } catch (const exception& e) {
         LOG(error) << "Error filling channel containers: " << e.what();
     }
@@ -242,6 +253,12 @@ auto DDS::SubscribeForConnectingChannels() -> void
                     unique_lock<mutex> lk(fUpdateMutex);
                     fUpdateCondition.wait(lk, [&]{ return fUpdatesAllowed; });
                 }
+
+                if (fConnectingChans.find(channelName) == fConnectingChans.end()) {
+                    LOG(error) << "Received an update for a connecting channel, but either no channel with given channel name exists or it has already been configured: '" << channelName << "', ignoring...";
+                    return;
+                }
+
                 string val = value;
                 // check if it is to handle as one out of multiple values
                 auto it = fIofN.find(channelName);
@@ -275,7 +292,10 @@ auto DDS::SubscribeForConnectingChannels() -> void
                     if (mi.second.fNumSubChannels == mi.second.fDDSValues.size()) {
                         int i = 0;
                         for (const auto& e : mi.second.fDDSValues) {
-                            SetProperty<string>(string{"chans." + mi.first + "." + to_string(i) + ".address"}, e.second);
+                            auto result = UpdateProperty<string>(string{"chans." + mi.first + "." + to_string(i) + ".address"}, e.second);
+                            if (!result) {
+                                LOG(error) << "UpdateProperty failed for: " << "chans." << mi.first << "." << to_string(i) << ".address" << " - property does not exist";
+                            }
                             ++i;
                         }
                     }
@@ -304,119 +324,125 @@ auto DDS::SubscribeForCustomCommands() -> void
 
     fDDS.SubscribeCustomCmd([id, this](const string& cmdStr, const string& cond, uint64_t senderId) {
         // LOG(info) << "Received command: '" << cmdStr << "' from " << senderId;
-
-        using namespace fair::mq::sdk;
-        cmd::Cmds inCmds;
+        sdk::cmd::Cmds inCmds;
         inCmds.Deserialize(cmdStr);
-
         for (const auto& cmd : inCmds) {
-            // LOG(info) << "Received command type: '" << cmd->GetType() << "' from " << senderId;
-            switch (cmd->GetType()) {
-                case cmd::Type::check_state: {
-                    fDDS.Send(cmd::Cmds(cmd::make<cmd::CurrentState>(id, GetCurrentDeviceState()))
-                                  .Serialize(),
-                              to_string(senderId));
-                } break;
-                case cmd::Type::change_state: {
-                    Transition transition = static_cast<cmd::ChangeState&>(*cmd).GetTransition();
-                    if (ChangeDeviceState(transition)) {
-                        cmd::Cmds outCmds(
-                            cmd::make<cmd::TransitionStatus>(id, dds::env_prop<dds::task_id>(), cmd::Result::Ok, transition));
-                        fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                    } else {
-                        sdk::cmd::Cmds outCmds(
-                            cmd::make<cmd::TransitionStatus>(id, dds::env_prop<dds::task_id>(), cmd::Result::Failure, transition));
-                        fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                    }
-                    {
-                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                        fLastExternalController = senderId;
-                    }
-                } break;
-                case cmd::Type::dump_config: {
-                    stringstream ss;
-                    for (const auto pKey : GetPropertyKeys()) {
-                        ss << id << ": " << pKey << " -> " << GetPropertyAsString(pKey) << "\n";
-                    }
-                    cmd::Cmds outCmds(cmd::make<cmd::Config>(id, ss.str()));
-                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                } break;
-                case cmd::Type::state_change_exiting_received: {
-                    {
-                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                        if (fLastExternalController == senderId) {
-                            fExitingAckedByLastExternalController = true;
-                        }
-                    }
-                    fExitingAcked.notify_one();
-                } break;
-                case cmd::Type::subscribe_to_state_change: {
-                    lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                    fStateChangeSubscribers.insert(senderId);
-
-                    LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState
-                               << " to " << senderId;
-
-                    cmd::Cmds outCmds(
-                        cmd::make<cmd::StateChangeSubscription>(id, cmd::Result::Ok),
-                        cmd::make<cmd::StateChange>(
-                            id, dds::env_prop<dds::task_id>(), fLastState, fCurrentState));
-
-                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                } break;
-                case cmd::Type::unsubscribe_from_state_change: {
-                    {
-                        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-                        fStateChangeSubscribers.erase(senderId);
-                    }
-                    cmd::Cmds outCmds(
-                        cmd::make<cmd::StateChangeUnsubscription>(id, cmd::Result::Ok));
-                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                } break;
-                case cmd::Type::get_properties: {
-                    auto _cmd = static_cast<cmd::GetProperties&>(*cmd);
-                    auto const request_id(_cmd.GetRequestId());
-                    auto result(cmd::Result::Ok);
-                    std::vector<std::pair<std::string, std::string>> props;
-                    try {
-                        for (auto const& prop : GetPropertiesAsString(_cmd.GetQuery())) {
-                            props.push_back({prop.first, prop.second});
-                        }
-                    } catch (std::exception const& e) {
-                        LOG(warn) << "Getting properties (request id: " << request_id << ") failed: " << e.what();
-                        result = cmd::Result::Failure;
-                    }
-                    cmd::Cmds const outCmds(cmd::make<cmd::Properties>(id, request_id, result, props));
-                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                } break;
-                case cmd::Type::set_properties: {
-                    auto _cmd(static_cast<cmd::SetProperties&>(*cmd));
-                    auto const request_id(_cmd.GetRequestId());
-                    auto result(cmd::Result::Ok);
-                    try {
-                        fair::mq::Properties props;
-                        for (auto const& prop : _cmd.GetProps()) {
-                            props.insert({prop.first, fair::mq::Property(prop.second)});
-                        }
-                        // TODO Handle builtin keys with different value type than string
-                        SetProperties(props);
-                    } catch (std::exception const& e) {
-                        LOG(warn) << "Setting properties (request id: " << request_id << ") failed: " << e.what();
-                        result = cmd::Result::Failure;
-                    }
-                    cmd::Cmds const outCmds(cmd::make<cmd::PropertiesSet>(id, request_id, result));
-                    fDDS.Send(outCmds.Serialize(), to_string(senderId));
-                } break;
-                default:
-                    LOG(warn) << "Unexpected/unknown command received: " << cmdStr;
-                    LOG(warn) << "Origin: " << senderId;
-                    LOG(warn) << "Destination: " << cond;
-                break;
-            }
+            HandleCmd(id, *cmd, cond, senderId);
         }
     });
 }
 
+auto DDS::HandleCmd(const string& id, sdk::cmd::Cmd& cmd, const string& cond, uint64_t senderId) -> void
+{
+    using namespace fair::mq::sdk;
+    using namespace fair::mq::sdk::cmd;
+    // LOG(info) << "Received command type: '" << cmd.GetType() << "' from " << senderId;
+    switch (cmd.GetType()) {
+        case Type::check_state: {
+            fDDS.Send(Cmds(make<CurrentState>(id, GetCurrentDeviceState())).Serialize(), to_string(senderId));
+        } break;
+        case Type::change_state: {
+            Transition transition = static_cast<ChangeState&>(cmd).GetTransition();
+            if (ChangeDeviceState(transition)) {
+                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Ok, transition));
+                fDDS.Send(outCmds.Serialize(), to_string(senderId));
+            } else {
+                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Failure, transition));
+                fDDS.Send(outCmds.Serialize(), to_string(senderId));
+            }
+            {
+                lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                fLastExternalController = senderId;
+            }
+        } break;
+        case Type::dump_config: {
+            stringstream ss;
+            for (const auto pKey : GetPropertyKeys()) {
+                ss << id << ": " << pKey << " -> " << GetPropertyAsString(pKey) << "\n";
+            }
+            Cmds outCmds(make<Config>(id, ss.str()));
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        case Type::state_change_exiting_received: {
+            {
+                lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                if (fLastExternalController == senderId) {
+                    fExitingAckedByLastExternalController = true;
+                }
+            }
+            fExitingAcked.notify_one();
+        } break;
+        case Type::subscribe_to_state_change: {
+            auto _cmd = static_cast<cmd::SubscribeToStateChange&>(cmd);
+            lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+            fStateChangeSubscribers.emplace(senderId, make_pair(chrono::steady_clock::now(), _cmd.GetInterval()));
+
+            LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState << " to " << senderId;
+
+            Cmds outCmds(make<StateChangeSubscription>(id, fDDSTaskId, Result::Ok),
+                         make<StateChange>(id, fDDSTaskId, fLastState, fCurrentState));
+
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        case Type::subscription_heartbeat: {
+            try {
+                auto _cmd = static_cast<cmd::SubscriptionHeartbeat&>(cmd);
+                lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                fStateChangeSubscribers.at(senderId) = make_pair(chrono::steady_clock::now(), _cmd.GetInterval());
+            } catch(out_of_range& oor) {
+                LOG(warn) << "Received subscription heartbeat from an unknown controller with id '" << senderId << "'";
+            }
+        } break;
+        case Type::unsubscribe_from_state_change: {
+            {
+                lock_guard<mutex> lock{fStateChangeSubscriberMutex};
+                fStateChangeSubscribers.erase(senderId);
+            }
+            Cmds outCmds(make<StateChangeUnsubscription>(id, fDDSTaskId, Result::Ok));
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        case Type::get_properties: {
+            auto _cmd = static_cast<cmd::GetProperties&>(cmd);
+            auto const request_id(_cmd.GetRequestId());
+            auto result(Result::Ok);
+            vector<pair<string, string>> props;
+            try {
+                for (auto const& prop : GetPropertiesAsString(_cmd.GetQuery())) {
+                    props.push_back({prop.first, prop.second});
+                }
+            } catch (exception const& e) {
+                LOG(warn) << "Getting properties (request id: " << request_id << ") failed: " << e.what();
+                result = Result::Failure;
+            }
+            Cmds const outCmds(make<cmd::Properties>(id, request_id, result, props));
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        case Type::set_properties: {
+            auto _cmd(static_cast<cmd::SetProperties&>(cmd));
+            auto const request_id(_cmd.GetRequestId());
+            auto result(Result::Ok);
+            try {
+                fair::mq::Properties props;
+                for (auto const& prop : _cmd.GetProps()) {
+                    props.insert({prop.first, fair::mq::Property(prop.second)});
+                }
+                // TODO Handle builtin keys with different value type than string
+                SetProperties(props);
+            } catch (exception const& e) {
+                LOG(warn) << "Setting properties (request id: " << request_id << ") failed: " << e.what();
+                result = Result::Failure;
+            }
+            Cmds const outCmds(make<PropertiesSet>(id, request_id, result));
+            fDDS.Send(outCmds.Serialize(), to_string(senderId));
+        } break;
+        default:
+            LOG(warn) << "Unexpected/unknown command received: " << cmd.GetType();
+            LOG(warn) << "Origin: " << senderId;
+            LOG(warn) << "Destination: " << cond;
+        break;
+            }
+}
+
 DDS::~DDS()
 {
     UnsubscribeFromDeviceStateChange();

fairmq/plugins/DDS/DDS.h

@@ -12,6 +12,7 @@
 #include <fairmq/Plugin.h>
 #include <fairmq/StateQueue.h>
 #include <fairmq/Version.h>
+#include <fairmq/sdk/commands/Commands.h>
 
 #include <dds/dds.h>
 
@@ -23,12 +24,12 @@
 #include <chrono>
 #include <condition_variable>
 #include <mutex>
-#include <set>
 #include <string>
 #include <atomic>
 #include <thread>
 #include <map>
 #include <unordered_map>
+#include <utility> // pair
 #include <vector>
 
 namespace fair
@@ -142,8 +143,10 @@ class DDS : public Plugin
     auto SubscribeForConnectingChannels() -> void;
     auto PublishBoundChannels() -> void;
     auto SubscribeForCustomCommands() -> void;
+    auto HandleCmd(const std::string& id, sdk::cmd::Cmd& cmd, const std::string& cond, uint64_t senderId) -> void;
 
     DDSSubscription fDDS;
+    size_t fDDSTaskId;
 
     std::unordered_map<std::string, std::vector<std::string>> fBindingChans;
     std::unordered_map<std::string, DDSConfig> fConnectingChans;
@@ -156,7 +159,7 @@ class DDS : public Plugin
 
     std::atomic<bool> fDeviceTerminationRequested;
 
-    std::set<uint64_t> fStateChangeSubscribers;
+    std::unordered_map<uint64_t, std::pair<std::chrono::steady_clock::time_point, int64_t>> fStateChangeSubscribers;
     uint64_t fLastExternalController;
     bool fExitingAckedByLastExternalController;
     std::condition_variable fExitingAcked;

fairmq/plugins/DDS/runDDSCommandUI.cxx

@@ -65,7 +65,7 @@ void handleCommand(const string& command, const string& path, unsigned int timeo
             cout << d.taskId << " : " << d.state << endl;
         }
     } else if (command == "o") {
-        cout << "> dumping config of the devices" << endl;
+        cout << "> dumping config of " << (path == "" ? "all" : path) << endl;
         // TODO: extend this regex to return all properties, once command size limitation is removed.
         auto const result = topo.GetProperties("^(session|id)$", path, std::chrono::milliseconds(timeout));
         for (const auto& d : result.second.devices) {
@@ -79,43 +79,43 @@ void handleCommand(const string& command, const string& path, unsigned int timeo
             return;
         }
         const DeviceProperties props{{pKey, pVal}};
-        cout << "> sending property" << endl;
+        cout << "> setting properties --> " << (path == "" ? "all" : path) << endl;
         topo.SetProperties(props, path);
         // give dds time to complete request
         this_thread::sleep_for(chrono::milliseconds(100));
     } else if (command == "i") {
-        cout << "> init devices" << endl;
-        topo.ChangeState(TopologyTransition::InitDevice, std::chrono::milliseconds(timeout));
+        cout << "> initiating InitDevice transition --> " << (path == "" ? "all" : path) << endl;
+        topo.ChangeState(TopologyTransition::InitDevice, path, std::chrono::milliseconds(timeout));
     } else if (command == "k") {
-        cout << "> complete init" << endl;
-        topo.ChangeState(TopologyTransition::CompleteInit, std::chrono::milliseconds(timeout));
+        cout << "> initiating CompleteInit transition --> " << (path == "" ? "all" : path) << endl;
+        topo.ChangeState(TopologyTransition::CompleteInit, path, std::chrono::milliseconds(timeout));
     } else if (command == "b") {
-        cout << "> bind devices" << endl;
-        topo.ChangeState(TopologyTransition::Bind, std::chrono::milliseconds(timeout));
+        cout << "> initiating Bind transition --> " << (path == "" ? "all" : path) << endl;
+        topo.ChangeState(TopologyTransition::Bind, path, std::chrono::milliseconds(timeout));
     } else if (command == "x") {
-        cout << "> connect devices" << endl;
-        topo.ChangeState(TopologyTransition::Connect, std::chrono::milliseconds(timeout));
+        cout << "> initiating Connect transition --> " << (path == "" ? "all" : path) << endl;
+        topo.ChangeState(TopologyTransition::Connect, path, std::chrono::milliseconds(timeout));
     } else if (command == "j") {
-        cout << "> init tasks" << endl;
-        topo.ChangeState(TopologyTransition::InitTask, std::chrono::milliseconds(timeout));
+        cout << "> initiating InitTask transition --> " << (path == "" ? "all" : path) << endl;
+        topo.ChangeState(TopologyTransition::InitTask, path, std::chrono::milliseconds(timeout));
     } else if (command == "r") {
-        cout << "> run tasks" << endl;
-        topo.ChangeState(TopologyTransition::Run, std::chrono::milliseconds(timeout));
+        cout << "> initiating Run transition --> " << (path == "" ? "all" : path) << endl;
+        topo.ChangeState(TopologyTransition::Run, path, std::chrono::milliseconds(timeout));
     } else if (command == "s") {
-        cout << "> stop devices" << endl;
-        topo.ChangeState(TopologyTransition::Stop, std::chrono::milliseconds(timeout));
+        cout << "> initiating Stop transition --> " << (path == "" ? "all" : path) << endl;
+        topo.ChangeState(TopologyTransition::Stop, path, std::chrono::milliseconds(timeout));
     } else if (command == "t") {
-        cout << "> reset tasks" << endl;
-        topo.ChangeState(TopologyTransition::ResetTask, std::chrono::milliseconds(timeout));
+        cout << "> initiating ResetTask transition --> " << (path == "" ? "all" : path) << endl;
+        topo.ChangeState(TopologyTransition::ResetTask, path, std::chrono::milliseconds(timeout));
     } else if (command == "d") {
-        cout << "> reset devices" << endl;
-        topo.ChangeState(TopologyTransition::ResetDevice, std::chrono::milliseconds(timeout));
+        cout << "> initiating ResetDevice transition --> " << (path == "" ? "all" : path) << endl;
+        topo.ChangeState(TopologyTransition::ResetDevice, path, std::chrono::milliseconds(timeout));
+    } else if (command == "q") {
+        cout << "> initiating End transition --> " << (path == "" ? "all" : path) << endl;
+        topo.ChangeState(TopologyTransition::End, path, std::chrono::milliseconds(timeout));
     } else if (command == "h") {
         cout << "> help" << endl;
         printControlsHelp();
-    } else if (command == "q") {
-        cout << "> end" << endl;
-        topo.ChangeState(TopologyTransition::End, std::chrono::milliseconds(timeout));
     } else {
         cout << "\033[01;32mInvalid input: [" << command << "]\033[0m" << endl;
         printControlsHelp();
@@ -206,6 +206,7 @@ try {
             sendCommand(command, path, timeout, topo, pKey, pVal);
         }
         size_t pos = targetState.find("->");
+        cout << "> waiting for " << (path == "" ? "all" : path) << " to reach " << targetState << endl;
         if (pos == string::npos) {
             /* auto ec =  */topo.WaitForState(GetState(targetState), path, std::chrono::milliseconds(timeout));
             // cout << "WaitForState(" << targetState << ") result: " << ec.message() << endl;

fairmq/plugins/PMIx/PMIxPlugin.cxx

@@ -171,7 +171,7 @@ auto PMIxPlugin::SubscribeForCommands() -> void
 
                     LOG(debug) << "Publishing state-change: " << fLastState << "->" << fCurrentState
                                << " to " << sender;
-                    Cmds outCmds(make<StateChangeSubscription>(fDeviceId, Result::Ok),
+                    Cmds outCmds(make<StateChangeSubscription>(fDeviceId, fProcess.rank, Result::Ok),
                                  make<StateChange>(fDeviceId, 0, fLastState, fCurrentState));
                     fCommands.Send(outCmds.Serialize(Format::JSON), {sender});
                 }
@@ -181,7 +181,7 @@ auto PMIxPlugin::SubscribeForCommands() -> void
                         lock_guard<mutex> lock{fStateChangeSubscriberMutex};
                         fStateChangeSubscribers.erase(sender.rank);
                     }
-                    fCommands.Send(Cmds(make<StateChangeUnsubscription>(fDeviceId, Result::Ok))
+                    fCommands.Send(Cmds(make<StateChangeUnsubscription>(fDeviceId, fProcess.rank, Result::Ok))
                                        .Serialize(Format::JSON),
                                    {sender});
                 }

fairmq/plugins/PMIx/runPMIxCommandUI.cxx

@@ -53,7 +53,7 @@ struct StateSubscription
     explicit StateSubscription(pmix::Commands& commands)
         : fCommands(commands)
     {
-        fCommands.Send(Cmds(make<SubscribeToStateChange>()).Serialize(Format::JSON));
+        fCommands.Send(Cmds(make<SubscribeToStateChange>(600000)).Serialize(Format::JSON));
     }
 
     ~StateSubscription()

fairmq/plugins/config/Config.cxx

@@ -61,21 +61,24 @@ Plugin::ProgOptions ConfigPluginProgramOptions()
     namespace po = boost::program_options;
     auto pluginOptions = po::options_description{"FairMQ device options"};
     pluginOptions.add_options()
-        ("id",                po::value<string        >()->default_value(""),         "Device ID.")
-        ("io-threads",        po::value<int           >()->default_value(1),          "Number of I/O threads.")
-        ("transport",         po::value<string        >()->default_value("zeromq"),   "Transport ('zeromq'/'nanomsg'/'shmem').")
-        ("network-interface", po::value<string        >()->default_value("default"),  "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).")
-        ("init-timeout",      po::value<int           >()->default_value(120),        "Timeout for the initialization in seconds (when expecting dynamic initialization).")
-        ("max-run-time",      po::value<uint64_t      >()->default_value(0),          "Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).")
-        ("print-channels",    po::value<bool          >()->implicit_value(true),      "Print registered channel endpoints in a machine-readable format (<channel name>:<min num subchannels>:<max num subchannels>)")
-        ("shm-segment-size",  po::value<size_t        >()->default_value(2000000000), "Shared memory: size of the shared memory segment (in bytes).")
-        ("shm-monitor",       po::value<bool          >()->default_value(true),       "Shared memory: run monitor daemon.")
-        ("ofi-size-hint",     po::value<size_t        >()->default_value(0),          "EXPERIMENTAL: OFI size hint for the allocator.")
-        ("rate",              po::value<float         >()->default_value(0.),         "Rate for conditional run loop (Hz).")
-        ("session",           po::value<string        >()->default_value("default"),  "Session name.")
-        ("config-key",        po::value<string        >(),                            "Use provided value instead of device id for fetching the configuration from JSON file.")
-        ("mq-config",         po::value<string        >(),                            "JSON input as file.")
-        ("channel-config",    po::value<vector<string>>()->multitoken()->composing(), "Configuration of single or multiple channel(s) by comma separated key=value list");
+        ("id",                  po::value<string        >()->default_value(""),         "Device ID.")
+        ("io-threads",          po::value<int           >()->default_value(1),          "Number of I/O threads.")
+        ("transport",           po::value<string        >()->default_value("zeromq"),   "Transport ('zeromq'/'shmem').")
+        ("network-interface",   po::value<string        >()->default_value("default"),  "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).")
+        ("init-timeout",        po::value<int           >()->default_value(120),        "Timeout for the initialization in seconds (when expecting dynamic initialization).")
+        ("max-run-time",        po::value<uint64_t      >()->default_value(0),          "Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).")
+        ("print-channels",      po::value<bool          >()->implicit_value(true),      "Print registered channel endpoints in a machine-readable format (<channel name>:<min num subchannels>:<max num subchannels>)")
+        ("shm-segment-size",    po::value<size_t        >()->default_value(2ULL << 30), "Shared memory: size of the shared memory segment (in bytes).")
+        ("shm-mlock-segment",   po::value<bool          >()->default_value(false),      "Shared memory: mlock the shared memory segment after initialization.")
+        ("shm-zero-segment",    po::value<bool          >()->default_value(false),      "Shared memory: zero the shared memory segment memory after initialization.")
+        ("shm-throw-bad-alloc", po::value<bool          >()->default_value(true),       "Throw a fair::mq::MessageBadAlloc if cannot allocate a message (retry if false).")
+        ("shm-monitor",         po::value<bool          >()->default_value(true),       "Shared memory: run monitor daemon.")
+        ("ofi-size-hint",       po::value<size_t        >()->default_value(0),          "EXPERIMENTAL: OFI size hint for the allocator.")
+        ("rate",                po::value<float         >()->default_value(0.),         "Rate for conditional run loop (Hz).")
+        ("session",             po::value<string        >()->default_value("default"),  "Session name.")
+        ("config-key",          po::value<string        >(),                            "Use provided value instead of device id for fetching the configuration from JSON file.")
+        ("mq-config",           po::value<string        >(),                            "JSON input as file.")
+        ("channel-config",      po::value<vector<string>>()->multitoken()->composing(), "Configuration of single or multiple channel(s) by comma separated key=value list");
     return pluginOptions;
 }
 

fairmq/run/runBenchmarkSampler.cxx

@@ -15,10 +15,11 @@ void addCustomOptions(bpo::options_description& options)
 {
     options.add_options()
         ("out-channel", bpo::value<std::string>()->default_value("data"), "Name of the output channel")
-        ("same-msg", bpo::value<bool>()->default_value(false), "Re-send the same message, or recreate for each iteration")
         ("multipart", bpo::value<bool>()->default_value(false), "Handle multipart payloads")
+        ("memset", bpo::value<bool>()->default_value(false), "Memset allocated buffers to 0")
         ("num-parts", bpo::value<size_t>()->default_value(1), "Number of parts to send. 1 will send single messages, not parts")
         ("msg-size", bpo::value<size_t>()->default_value(1000000), "Message size in bytes")
+        ("msg-alignment", bpo::value<size_t>()->default_value(0), "Message alignment")
         ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Number of run iterations (0 - infinite)")
         ("msg-rate", bpo::value<float>()->default_value(0), "Msg rate limit in maximum number of messages per second");
 }

fairmq/run/startMQBenchmark.sh.in

@@ -59,7 +59,7 @@ else
 fi
 
 echo ""
-echo "Usage: startBenchmark [message size=1000000] [number of iterations=0] [transport=zeromq/nanomsg/shmem] [affinity=false]"
+echo "Usage: startBenchmark [message size=1000000] [number of iterations=0] [transport=zeromq/shmem] [affinity=false]"
 
 SAMPLER="fairmq-bsampler"
 SAMPLER+=" --id bsampler1"
@@ -70,6 +70,7 @@ SAMPLER+=" --severity debug"
 SAMPLER+=" --msg-size $msgSize"
 SAMPLER+=" --multipart $multipart"
 SAMPLER+=" --num-parts $numParts"
+SAMPLER+=" --shm-throw-bad-alloc false"
 # SAMPLER+=" --msg-rate 1000"
 SAMPLER+=" --max-iterations $maxIterations"
 SAMPLER+=" --channel-config name=data,type=pair,method=bind,address=tcp://127.0.0.1:5555"

fairmq/sdk/Error.h

@@ -25,11 +25,6 @@ struct RuntimeError : ::std::runtime_error
     {}
 };
 
-struct MixedStateError : RuntimeError
-{
-    using RuntimeError::RuntimeError;
-};
-
 } /* namespace sdk */
 
 enum class ErrorCode

fairmq/sdk/Topology.h

@@ -34,6 +34,7 @@
 
 #include <algorithm>
 #include <chrono>
+#include <condition_variable>
 #include <functional>
 #include <map>
 #include <memory>
@@ -69,9 +70,68 @@ const std::map<DeviceTransition, DeviceState> expectedState =
     { DeviceTransition::End,          DeviceState::Exiting }
 };
 
+// mirrors DeviceState, but adds a "Mixed" state that represents a topology where devices are currently not in the same state.
+enum class AggregatedTopologyState : int
+{
+    Undefined = static_cast<int>(fair::mq::State::Undefined),
+    Ok = static_cast<int>(fair::mq::State::Ok),
+    Error = static_cast<int>(fair::mq::State::Error),
+    Idle = static_cast<int>(fair::mq::State::Idle),
+    InitializingDevice = static_cast<int>(fair::mq::State::InitializingDevice),
+    Initialized = static_cast<int>(fair::mq::State::Initialized),
+    Binding = static_cast<int>(fair::mq::State::Binding),
+    Bound = static_cast<int>(fair::mq::State::Bound),
+    Connecting = static_cast<int>(fair::mq::State::Connecting),
+    DeviceReady = static_cast<int>(fair::mq::State::DeviceReady),
+    InitializingTask = static_cast<int>(fair::mq::State::InitializingTask),
+    Ready = static_cast<int>(fair::mq::State::Ready),
+    Running = static_cast<int>(fair::mq::State::Running),
+    ResettingTask = static_cast<int>(fair::mq::State::ResettingTask),
+    ResettingDevice = static_cast<int>(fair::mq::State::ResettingDevice),
+    Exiting = static_cast<int>(fair::mq::State::Exiting),
+    Mixed
+};
+
+inline auto operator==(DeviceState lhs, AggregatedTopologyState rhs) -> bool
+{
+    return static_cast<int>(lhs) == static_cast<int>(rhs);
+}
+
+inline auto operator==(AggregatedTopologyState lhs, DeviceState rhs) -> bool
+{
+    return static_cast<int>(lhs) == static_cast<int>(rhs);
+}
+
+inline std::ostream& operator<<(std::ostream& os, const AggregatedTopologyState& state)
+{
+    if (state == AggregatedTopologyState::Mixed) {
+        return os << "MIXED";
+    } else {
+        return os << static_cast<DeviceState>(state);
+    }
+}
+
+inline std::string GetAggregatedTopologyStateName(AggregatedTopologyState s)
+{
+    if (s == AggregatedTopologyState::Mixed) {
+        return "MIXED";
+    } else {
+        return GetStateName(static_cast<State>(s));
+    }
+}
+
+inline AggregatedTopologyState GetAggregatedTopologyState(const std::string& state)
+{
+    if (state == "MIXED") {
+        return AggregatedTopologyState::Mixed;
+    } else {
+        return static_cast<AggregatedTopologyState>(GetState(state));
+    }
+}
+
 struct DeviceStatus
 {
-    bool initialized;
+    bool subscribed_to_state_changes;
     DeviceState lastState;
     DeviceState state;
     DDSTask::Id taskId;
@@ -99,22 +159,22 @@ using TopologyStateByTask = std::unordered_map<DDSTask::Id, DeviceStatus>;
 using TopologyStateByCollection = std::unordered_map<DDSCollection::Id, std::vector<DeviceStatus>>;
 using TopologyTransition = fair::mq::Transition;
 
-inline DeviceState AggregateState(const TopologyState& topologyState)
+inline AggregatedTopologyState AggregateState(const TopologyState& topologyState)
 {
     DeviceState first = topologyState.begin()->state;
 
     if (std::all_of(topologyState.cbegin(), topologyState.cend(), [&](TopologyState::value_type i) {
             return i.state == first;
         })) {
-        return first;
+        return static_cast<AggregatedTopologyState>(first);
     }
 
-    throw MixedStateError("State is not uniform");
+    return AggregatedTopologyState::Mixed;
 }
 
 inline bool StateEqualsTo(const TopologyState& topologyState, DeviceState state)
 {
-    return AggregateState(topologyState) == state;
+    return AggregateState(topologyState) == static_cast<AggregatedTopologyState>(state);
 }
 
 inline TopologyStateByCollection GroupByCollectionId(const TopologyState& topologyState)
@@ -174,6 +234,8 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         , fDDSTopo(std::move(topo))
         , fStateData()
         , fStateIndex()
+        , fHeartbeatsTimer(asio::system_executor())
+        , fHeartbeatInterval(600000)
     {
         makeTopologyState();
 
@@ -183,67 +245,10 @@ class BasicTopology : public AsioBase<Executor, Allocator>
             throw RuntimeError("Given topology ", givenTopo, " is not activated (active: ", activeTopo, ")");
         }
 
-        using namespace fair::mq::sdk::cmd;
-        fDDSSession.SubscribeToCommands([&](const std::string& msg, const std::string& /* condition */, DDSChannel::Id senderId) {
-            Cmds inCmds;
-            inCmds.Deserialize(msg);
-            // FAIR_LOG(debug) << "Received " << inCmds.Size() << " command(s) with total size of " << msg.length() << " bytes: ";
-
-            for (const auto& cmd : inCmds) {
-                // FAIR_LOG(debug) << " > " << cmd->GetType();
-                switch (cmd->GetType()) {
-                    case Type::state_change: {
-                        auto _cmd = static_cast<StateChange&>(*cmd);
-                        if (_cmd.GetCurrentState() == DeviceState::Exiting) {
-                            fDDSSession.SendCommand(Cmds(make<StateChangeExitingReceived>()).Serialize(), senderId);
-                        }
-                        HandleCmd(_cmd);
-                    } break;
-                    case Type::state_change_subscription:
-                        if (static_cast<StateChangeSubscription&>(*cmd).GetResult() != Result::Ok) {
-                            FAIR_LOG(error) << "State change subscription failed for " << static_cast<StateChangeSubscription&>(*cmd).GetDeviceId();
-                        }
-                    break;
-                    case Type::state_change_unsubscription:
-                        if (static_cast<StateChangeUnsubscription&>(*cmd).GetResult() != Result::Ok) {
-                            FAIR_LOG(error) << "State change unsubscription failed for " << static_cast<StateChangeUnsubscription&>(*cmd).GetDeviceId();
-                        }
-                    break;
-                    case Type::transition_status: {
-                        auto _cmd = static_cast<TransitionStatus&>(*cmd);
-                        if (_cmd.GetResult() != Result::Ok) {
-                            FAIR_LOG(error) << _cmd.GetTransition() << " transition failed for " << _cmd.GetDeviceId();
-                            DDSTask::Id id(_cmd.GetTaskId());
-                            std::lock_guard<std::mutex> lk(fMtx);
-                            for (auto& op : fChangeStateOps) {
-                                if (!op.second.IsCompleted() && op.second.ContainsTask(id) &&
-                                    fStateData.at(fStateIndex.at(id)).state != op.second.GetTargetState()) {
-                                    op.second.Complete(MakeErrorCode(ErrorCode::DeviceChangeStateFailed));
-                                }
-                            }
-                        }
-                    }
-                    break;
-                    case Type::properties: {
-                        HandleCmd(static_cast<cmd::Properties&>(*cmd));
-                    }
-                    break;
-                    case Type::properties_set: {
-                        HandleCmd(static_cast<cmd::PropertiesSet&>(*cmd));
-                    }
-                    break;
-                    default:
-                        FAIR_LOG(warn) << "Unexpected/unknown command received: " << cmd->GetType();
-                        FAIR_LOG(warn) << "Origin: " << senderId;
-                    break;
-                }
-            }
-        });
+        SubscribeToCommands();
 
         fDDSSession.StartDDSService();
-        // FAIR_LOG(debug) << "Subscribing to state change";
-        Cmds cmds(make<SubscribeToStateChange>());
-        fDDSSession.SendCommand(cmds.Serialize());
+        SubscribeToStateChanges();
     }
 
     /// not copyable
@@ -254,15 +259,6 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     BasicTopology(BasicTopology&&) = default;
     BasicTopology& operator=(BasicTopology&&) = default;
 
-    void UnsubscribeFromStateChanges()
-    {
-        using namespace fair::mq::sdk::cmd;
-
-        fDDSSession.SendCommand(Cmds(make<UnsubscribeFromStateChange>()).Serialize());
-        // give dds a chance to complete request, TODO: track each individual task and its subscription status
-        std::this_thread::sleep_for(std::chrono::milliseconds(100));
-    }
-
     ~BasicTopology()
     {
         UnsubscribeFromStateChanges();
@@ -276,16 +272,139 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         } catch (...) {}
     }
 
-    auto HandleCmd(cmd::StateChange const& cmd) -> void
+    void SubscribeToStateChanges()
     {
+        // FAIR_LOG(debug) << "Subscribing to state change";
+        cmd::Cmds cmds(cmd::make<cmd::SubscribeToStateChange>(fHeartbeatInterval.count()));
+        fDDSSession.SendCommand(cmds.Serialize());
+
+        fHeartbeatsTimer.expires_after(fHeartbeatInterval);
+        fHeartbeatsTimer.async_wait(std::bind(&BasicTopology::SendSubscriptionHeartbeats, this, std::placeholders::_1));
+    }
+
+    void SendSubscriptionHeartbeats(const std::error_code& ec)
+    {
+        if (!ec) {
+            // Timer expired.
+            fDDSSession.SendCommand(cmd::Cmds(cmd::make<cmd::SubscriptionHeartbeat>(fHeartbeatInterval.count())).Serialize());
+            // schedule again
+            fHeartbeatsTimer.expires_after(fHeartbeatInterval);
+            fHeartbeatsTimer.async_wait(std::bind(&BasicTopology::SendSubscriptionHeartbeats, this, std::placeholders::_1));
+        } else if (ec == asio::error::operation_aborted) {
+            // FAIR_LOG(debug) << "Heartbeats timer canceled";
+        } else {
+            FAIR_LOG(error) << "Timer error: " << ec;
+        }
+    }
+
+    void UnsubscribeFromStateChanges()
+    {
+        // stop sending heartbeats
+        fHeartbeatsTimer.cancel();
+
+        // unsubscribe from state changes
+        fDDSSession.SendCommand(cmd::Cmds(cmd::make<cmd::UnsubscribeFromStateChange>()).Serialize());
+
+        // wait for all tasks to confirm unsubscription
+        std::unique_lock<std::mutex> lk(fMtx);
+        fStateChangeUnsubscriptionCV.wait(lk, [&](){
+            unsigned int count = std::count_if(fStateIndex.cbegin(), fStateIndex.cend(), [=](const auto& s) {
+                return fStateData.at(s.second).subscribed_to_state_changes == false;
+            });
+            return count == fStateIndex.size();
+        });
+    }
+
+    void SubscribeToCommands()
+    {
+        fDDSSession.SubscribeToCommands([&](const std::string& msg, const std::string& /* condition */, DDSChannel::Id senderId) {
+            cmd::Cmds inCmds;
+            inCmds.Deserialize(msg);
+            // FAIR_LOG(debug) << "Received " << inCmds.Size() << " command(s) with total size of " << msg.length() << " bytes: ";
+
+            for (const auto& cmd : inCmds) {
+                // FAIR_LOG(debug) << " > " << cmd->GetType();
+                switch (cmd->GetType()) {
+                    case cmd::Type::state_change_subscription:
+                        HandleCmd(static_cast<cmd::StateChangeSubscription&>(*cmd));
+                    break;
+                    case cmd::Type::state_change_unsubscription:
+                        HandleCmd(static_cast<cmd::StateChangeUnsubscription&>(*cmd));
+                    break;
+                    case cmd::Type::state_change:
+                        HandleCmd(static_cast<cmd::StateChange&>(*cmd), senderId);
+                    break;
+                    case cmd::Type::transition_status:
+                        HandleCmd(static_cast<cmd::TransitionStatus&>(*cmd));
+                    break;
+                    case cmd::Type::properties:
+                        HandleCmd(static_cast<cmd::Properties&>(*cmd));
+                    break;
+                    case cmd::Type::properties_set:
+                        HandleCmd(static_cast<cmd::PropertiesSet&>(*cmd));
+                    break;
+                    default:
+                        FAIR_LOG(warn) << "Unexpected/unknown command received: " << cmd->GetType();
+                        FAIR_LOG(warn) << "Origin: " << senderId;
+                    break;
+                }
+            }
+        });
+    }
+
+    auto HandleCmd(cmd::StateChangeSubscription const& cmd) -> void
+    {
+        if (cmd.GetResult() == cmd::Result::Ok) {
+            DDSTask::Id taskId(cmd.GetTaskId());
+
+            try {
+                std::lock_guard<std::mutex> lk(fMtx);
+                DeviceStatus& task = fStateData.at(fStateIndex.at(taskId));
+                task.subscribed_to_state_changes = true;
+            } catch (const std::exception& e) {
+                FAIR_LOG(error) << "Exception in HandleCmd(cmd::StateChangeSubscription const&): " << e.what();
+            }
+        } else {
+            FAIR_LOG(error) << "State change subscription failed for device: " << cmd.GetDeviceId() << ", task id: " << cmd.GetTaskId();
+        }
+    }
+
+    auto HandleCmd(cmd::StateChangeUnsubscription const& cmd) -> void
+    {
+        if (cmd.GetResult() == cmd::Result::Ok) {
+            DDSTask::Id taskId(cmd.GetTaskId());
+
+            try {
+                std::unique_lock<std::mutex> lk(fMtx);
+                DeviceStatus& task = fStateData.at(fStateIndex.at(taskId));
+                task.subscribed_to_state_changes = false;
+                lk.unlock();
+                fStateChangeUnsubscriptionCV.notify_one();
+            } catch (const std::exception& e) {
+                FAIR_LOG(error) << "Exception in HandleCmd(cmd::StateChangeUnsubscription const&): " << e.what();
+            }
+        } else {
+            FAIR_LOG(error) << "State change unsubscription failed for device: " << cmd.GetDeviceId() << ", task id: " << cmd.GetTaskId();
+        }
+    }
+
+    auto HandleCmd(cmd::StateChange const& cmd, DDSChannel::Id const& senderId) -> void
+    {
+        if (cmd.GetCurrentState() == DeviceState::Exiting) {
+            fDDSSession.SendCommand(cmd::Cmds(cmd::make<cmd::StateChangeExitingReceived>()).Serialize(), senderId);
+        }
+
         DDSTask::Id taskId(cmd.GetTaskId());
-        std::lock_guard<std::mutex> lk(fMtx);
 
         try {
+            std::lock_guard<std::mutex> lk(fMtx);
             DeviceStatus& task = fStateData.at(fStateIndex.at(taskId));
-            task.initialized = true;
             task.lastState = cmd.GetLastState();
             task.state = cmd.GetCurrentState();
+            // if the task is exiting, it will not respond to unsubscription request anymore, set it to false now.
+            if (task.state == DeviceState::Exiting) {
+                task.subscribed_to_state_changes = false;
+            }
             // FAIR_LOG(debug) << "Updated state entry: taskId=" << taskId << ", state=" << state;
 
             for (auto& op : fChangeStateOps) {
@@ -299,6 +418,21 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         }
     }
 
+    auto HandleCmd(cmd::TransitionStatus const& cmd) -> void
+    {
+        if (cmd.GetResult() != cmd::Result::Ok) {
+            FAIR_LOG(error) << cmd.GetTransition() << " transition failed for " << cmd.GetDeviceId();
+            DDSTask::Id taskId(cmd.GetTaskId());
+            std::lock_guard<std::mutex> lk(fMtx);
+            for (auto& op : fChangeStateOps) {
+                if (!op.second.IsCompleted() && op.second.ContainsTask(taskId) &&
+                    fStateData.at(fStateIndex.at(taskId)).state != op.second.GetTargetState()) {
+                    op.second.Complete(MakeErrorCode(ErrorCode::DeviceChangeStateFailed));
+                }
+            }
+        }
+    }
+
     auto HandleCmd(cmd::Properties const& cmd) -> void
     {
         std::unique_lock<std::mutex> lk(fMtx);
@@ -364,6 +498,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     }
                 });
             }
+            if (fTasks.empty()) {
+                FAIR_LOG(warn) << "ChangeState initiated on an empty set of tasks, check the path argument.";
+            }
         }
         ChangeStateOp() = delete;
         ChangeStateOp(const ChangeStateOp&) = delete;
@@ -619,7 +756,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     }
 
     /// @brief Returns the current state of the topology
-    /// @return map of id : DeviceStatus (initialized, state)
+    /// @return map of id : DeviceStatus
     auto GetCurrentState() const -> TopologyState
     {
         std::lock_guard<std::mutex> lk(fMtx);
@@ -666,6 +803,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     }
                 });
             }
+            if (fTasks.empty()) {
+                FAIR_LOG(warn) << "WaitForState initiated on an empty set of tasks, check the path argument.";
+            }
         }
         WaitForStateOp() = delete;
         WaitForStateOp(const WaitForStateOp&) = delete;
@@ -679,9 +819,8 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         {
             fCount = std::count_if(stateIndex.cbegin(), stateIndex.cend(), [=](const auto& s) {
                 if (ContainsTask(stateData.at(s.second).taskId)) {
-                    return stateData.at(s.second).state == fTargetCurrentState
-                           &&
-                           (stateData.at(s.second).lastState == fTargetLastState || fTargetLastState == DeviceState::Ok);
+                    return stateData.at(s.second).state == fTargetCurrentState &&
+                          (stateData.at(s.second).lastState == fTargetLastState || fTargetLastState == DeviceState::Undefined);
                 } else {
                     return false;
                 }
@@ -693,8 +832,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         {
             if (!fOp.IsCompleted() && ContainsTask(taskId)) {
                 if (currentState == fTargetCurrentState &&
-                    (lastState == fTargetLastState ||
-                     fTargetLastState == DeviceState::Ok)) {
+                    (lastState == fTargetLastState || fTargetLastState == DeviceState::Undefined)) {
                     ++fCount;
                 }
                 TryCompletion();
@@ -798,7 +936,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     template<typename CompletionToken>
     auto AsyncWaitForState(const DeviceState targetCurrentState, CompletionToken&& token)
     {
-        return AsyncWaitForState(DeviceState::Ok, targetCurrentState, "", Duration(0), std::move(token));
+        return AsyncWaitForState(DeviceState::Undefined, targetCurrentState, "", Duration(0), std::move(token));
     }
 
     /// @brief Wait for selected FairMQ devices to reach given last & current state in this topology
@@ -828,7 +966,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     auto WaitForState(const DeviceState targetCurrentState, const std::string& path = "", Duration timeout = Duration(0))
         -> std::error_code
     {
-        return WaitForState(DeviceState::Ok, targetCurrentState, path, timeout);
+        return WaitForState(DeviceState::Undefined, targetCurrentState, path, timeout);
     }
 
     using GetPropertiesCompletionSignature = void(std::error_code, GetPropertiesResult);
@@ -863,6 +1001,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     }
                 });
             }
+            if (expectedCount == 0) {
+                FAIR_LOG(warn) << "GetProperties initiated on an empty set of tasks, check the path argument.";
+            }
             // FAIR_LOG(debug) << "GetProperties " << fId << " with expected count of " << fExpectedCount << " started.";
         }
         GetPropertiesOp() = delete;
@@ -1018,6 +1159,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     }
                 });
             }
+            if (expectedCount == 0) {
+                FAIR_LOG(warn) << "SetProperties initiated on an empty set of tasks, check the path argument.";
+            }
             // FAIR_LOG(debug) << "SetProperties " << fId << " with expected count of " << fExpectedCount << " started.";
         }
         SetPropertiesOp() = delete;
@@ -1138,6 +1282,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         return {ec, failed};
     }
 
+    Duration GetHeartbeatInterval() const { return fHeartbeatInterval; }
+    void SetHeartbeatInterval(Duration duration) { fHeartbeatInterval = duration; }
+
   private:
     using TransitionedCount = unsigned int;
 
@@ -1145,8 +1292,13 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     DDSTopology fDDSTopo;
     TopologyState fStateData;
     TopologyStateIndex fStateIndex;
+
     mutable std::mutex fMtx;
 
+    std::condition_variable fStateChangeUnsubscriptionCV;
+    asio::steady_timer fHeartbeatsTimer;
+    Duration fHeartbeatInterval;
+
     std::unordered_map<typename ChangeStateOp::Id, ChangeStateOp> fChangeStateOps;
     std::unordered_map<typename WaitForStateOp::Id, WaitForStateOp> fWaitForStateOps;
     std::unordered_map<typename SetPropertiesOp::Id, SetPropertiesOp> fSetPropertiesOps;
@@ -1159,7 +1311,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         int index = 0;
 
         for (const auto& task : fDDSTopo.GetTasks()) {
-            fStateData.push_back(DeviceStatus{false, DeviceState::Ok, DeviceState::Ok, task.GetId(), task.GetCollectionId()});
+            fStateData.push_back(DeviceStatus{false, DeviceState::Undefined, DeviceState::Undefined, task.GetId(), task.GetCollectionId()});
             fStateIndex.emplace(task.GetId(), index);
             index++;
         }

fairmq/sdk/commands/Commands.cxx

@@ -46,7 +46,7 @@ array<string, 2> resultNames =
     }
 };
 
-array<string, 16> typeNames =
+array<string, 17> typeNames =
 {
     {
         "CheckState",
@@ -57,6 +57,7 @@ array<string, 16> typeNames =
         "StateChangeExitingReceived",
         "GetProperties",
         "SetProperties",
+        "SubscriptionHeartbeat",
 
         "CurrentState",
         "TransitionStatus",
@@ -69,9 +70,10 @@ array<string, 16> typeNames =
     }
 };
 
-array<fair::mq::State, 15> fbStateToMQState =
+array<fair::mq::State, 16> fbStateToMQState =
 {
     {
+        fair::mq::State::Undefined,
         fair::mq::State::Ok,
         fair::mq::State::Error,
         fair::mq::State::Idle,
@@ -90,9 +92,10 @@ array<fair::mq::State, 15> fbStateToMQState =
     }
 };
 
-array<sdk::cmd::FBState, 15> mqStateToFBState =
+array<sdk::cmd::FBState, 16> mqStateToFBState =
 {
     {
+        sdk::cmd::FBState_Undefined,
         sdk::cmd::FBState_Ok,
         sdk::cmd::FBState_Error,
         sdk::cmd::FBState_Idle,
@@ -147,7 +150,7 @@ array<sdk::cmd::FBTransition, 12> mqTransitionToFBTransition =
     }
 };
 
-array<FBCmd, 16> typeToFBCmd =
+array<FBCmd, 17> typeToFBCmd =
 {
     {
         FBCmd::FBCmd_check_state,
@@ -158,6 +161,7 @@ array<FBCmd, 16> typeToFBCmd =
         FBCmd::FBCmd_state_change_exiting_received,
         FBCmd::FBCmd_get_properties,
         FBCmd::FBCmd_set_properties,
+        FBCmd::FBCmd_subscription_heartbeat,
         FBCmd::FBCmd_current_state,
         FBCmd::FBCmd_transition_status,
         FBCmd::FBCmd_config,
@@ -169,7 +173,7 @@ array<FBCmd, 16> typeToFBCmd =
     }
 };
 
-array<Type, 16> fbCmdToType =
+array<Type, 17> fbCmdToType =
 {
     {
         Type::check_state,
@@ -180,6 +184,7 @@ array<Type, 16> fbCmdToType =
         Type::state_change_exiting_received,
         Type::get_properties,
         Type::set_properties,
+        Type::subscription_heartbeat,
         Type::current_state,
         Type::transition_status,
         Type::config,
@@ -228,7 +233,9 @@ string Cmds::Serialize(const Format type) const
             break;
             break;
             case Type::subscribe_to_state_change: {
+                auto _cmd = static_cast<SubscribeToStateChange&>(*cmd);
                 cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_interval(_cmd.GetInterval());
             }
             break;
             case Type::unsubscribe_from_state_change: {
@@ -261,6 +268,12 @@ string Cmds::Serialize(const Format type) const
                 cmdBuilder->add_properties(props);
             }
             break;
+            case Type::subscription_heartbeat: {
+                auto _cmd = static_cast<SubscriptionHeartbeat&>(*cmd);
+                cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
+                cmdBuilder->add_interval(_cmd.GetInterval());
+            }
+            break;
             case Type::current_state: {
                 auto _cmd = static_cast<CurrentState&>(*cmd);
                 auto deviceId = fbb.CreateString(_cmd.GetDeviceId());
@@ -293,6 +306,7 @@ string Cmds::Serialize(const Format type) const
                 auto deviceId = fbb.CreateString(_cmd.GetDeviceId());
                 cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
                 cmdBuilder->add_device_id(deviceId);
+                cmdBuilder->add_task_id(_cmd.GetTaskId());
                 cmdBuilder->add_result(GetFBResult(_cmd.GetResult()));
             }
             break;
@@ -301,6 +315,7 @@ string Cmds::Serialize(const Format type) const
                 auto deviceId = fbb.CreateString(_cmd.GetDeviceId());
                 cmdBuilder = tools::make_unique<FBCommandBuilder>(fbb);
                 cmdBuilder->add_device_id(deviceId);
+                cmdBuilder->add_task_id(_cmd.GetTaskId());
                 cmdBuilder->add_result(GetFBResult(_cmd.GetResult()));
             }
             break;
@@ -404,7 +419,7 @@ void Cmds::Deserialize(const string& str, const Format type)
                 fCmds.emplace_back(make<DumpConfig>());
             break;
             case FBCmd_subscribe_to_state_change:
-                fCmds.emplace_back(make<SubscribeToStateChange>());
+                fCmds.emplace_back(make<SubscribeToStateChange>(cmdPtr.interval()));
             break;
             case FBCmd_unsubscribe_from_state_change:
                 fCmds.emplace_back(make<UnsubscribeFromStateChange>());
@@ -423,6 +438,9 @@ void Cmds::Deserialize(const string& str, const Format type)
                 }
                 fCmds.emplace_back(make<SetProperties>(cmdPtr.request_id(), properties));
             } break;
+            case FBCmd_subscription_heartbeat:
+                fCmds.emplace_back(make<SubscriptionHeartbeat>(cmdPtr.interval()));
+            break;
             case FBCmd_current_state:
                 fCmds.emplace_back(make<CurrentState>(cmdPtr.device_id()->str(), GetMQState(cmdPtr.current_state())));
             break;
@@ -433,10 +451,10 @@ void Cmds::Deserialize(const string& str, const Format type)
                 fCmds.emplace_back(make<Config>(cmdPtr.device_id()->str(), cmdPtr.config_string()->str()));
             break;
             case FBCmd_state_change_subscription:
-                fCmds.emplace_back(make<StateChangeSubscription>(cmdPtr.device_id()->str(), GetResult(cmdPtr.result())));
+                fCmds.emplace_back(make<StateChangeSubscription>(cmdPtr.device_id()->str(), cmdPtr.task_id(), GetResult(cmdPtr.result())));
             break;
             case FBCmd_state_change_unsubscription:
-                fCmds.emplace_back(make<StateChangeUnsubscription>(cmdPtr.device_id()->str(), GetResult(cmdPtr.result())));
+                fCmds.emplace_back(make<StateChangeUnsubscription>(cmdPtr.device_id()->str(), cmdPtr.task_id(), GetResult(cmdPtr.result())));
             break;
             case FBCmd_state_change:
                 fCmds.emplace_back(make<StateChange>(cmdPtr.device_id()->str(), cmdPtr.task_id(), GetMQState(cmdPtr.last_state()), GetMQState(cmdPtr.current_state())));

fairmq/sdk/commands/Commands.h

@@ -47,12 +47,13 @@ enum class Type : int
     state_change_exiting_received, // args: { }
     get_properties,                // args: { request_id, property_query }
     set_properties,                // args: { request_id, properties }
+    subscription_heartbeat,        // args: { interval }
 
     current_state,                 // args: { device_id, current_state }
     transition_status,             // args: { device_id, task_id, Result, transition }
     config,                        // args: { device_id, config_string }
-    state_change_subscription,     // args: { device_id, Result }
-    state_change_unsubscription,   // args: { device_id, Result }
+    state_change_subscription,     // args: { device_id, task_id, Result }
+    state_change_unsubscription,   // args: { device_id, task_id, Result }
     state_change,                  // args: { device_id, task_id, last_state, current_state }
     properties,                    // args: { device_id, request_id, Result, properties }
     properties_set                 // args: { device_id, request_id, Result }
@@ -95,7 +96,16 @@ struct DumpConfig : Cmd
 
 struct SubscribeToStateChange : Cmd
 {
-    explicit SubscribeToStateChange() : Cmd(Type::subscribe_to_state_change) {}
+    explicit SubscribeToStateChange(int64_t interval)
+        : Cmd(Type::subscribe_to_state_change)
+        , fInterval(interval)
+    {}
+
+    int64_t GetInterval() const { return fInterval; }
+    void SetInterval(int64_t interval) { fInterval = interval; }
+
+  private:
+    int64_t fInterval;
 };
 
 struct UnsubscribeFromStateChange : Cmd
@@ -144,6 +154,20 @@ struct SetProperties : Cmd
     std::vector<std::pair<std::string, std::string>> fProperties;
 };
 
+struct SubscriptionHeartbeat : Cmd
+{
+    explicit SubscriptionHeartbeat(int64_t interval)
+        : Cmd(Type::subscription_heartbeat)
+        , fInterval(interval)
+    {}
+
+    int64_t GetInterval() const { return fInterval; }
+    void SetInterval(int64_t interval) { fInterval = interval; }
+
+  private:
+    int64_t fInterval;
+};
+
 struct CurrentState : Cmd
 {
     explicit CurrentState(const std::string& id, State currentState)
@@ -208,37 +232,45 @@ struct Config : Cmd
 
 struct StateChangeSubscription : Cmd
 {
-    explicit StateChangeSubscription(const std::string& id, const Result result)
+    explicit StateChangeSubscription(const std::string& id, const uint64_t taskId, const Result result)
         : Cmd(Type::state_change_subscription)
         , fDeviceId(id)
+        , fTaskId(taskId)
         , fResult(result)
     {}
 
     std::string GetDeviceId() const { return fDeviceId; }
     void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    uint64_t GetTaskId() const { return fTaskId; }
+    void SetTaskId(const uint64_t taskId) { fTaskId = taskId; }
     Result GetResult() const { return fResult; }
     void SetResult(const Result result) { fResult = result; }
 
   private:
     std::string fDeviceId;
+    uint64_t fTaskId;
     Result fResult;
 };
 
 struct StateChangeUnsubscription : Cmd
 {
-    explicit StateChangeUnsubscription(const std::string& id, const Result result)
+    explicit StateChangeUnsubscription(const std::string& id, const uint64_t taskId, const Result result)
         : Cmd(Type::state_change_unsubscription)
         , fDeviceId(id)
+        , fTaskId(taskId)
         , fResult(result)
     {}
 
     std::string GetDeviceId() const { return fDeviceId; }
     void SetDeviceId(const std::string& deviceId) { fDeviceId = deviceId; }
+    uint64_t GetTaskId() const { return fTaskId; }
+    void SetTaskId(const uint64_t taskId) { fTaskId = taskId; }
     Result GetResult() const { return fResult; }
     void SetResult(const Result result) { fResult = result; }
 
   private:
     std::string fDeviceId;
+    uint64_t fTaskId;
     Result fResult;
 };
 

fairmq/sdk/commands/CommandsFormat.fbs

@@ -6,6 +6,7 @@ enum FBResult:byte {
 }
 
 enum FBState:byte {
+    Undefined,
     Ok,
     Error,
     Idle,
@@ -47,17 +48,18 @@ enum FBCmd:byte {
     check_state,                   // args: { }
     change_state,                  // args: { transition }
     dump_config,                   // args: { }
-    subscribe_to_state_change,     // args: { }
+    subscribe_to_state_change,     // args: { interval }
     unsubscribe_from_state_change, // args: { }
     state_change_exiting_received, // args: { }
     get_properties,                // args: { request_id, property_query }
     set_properties,                // args: { request_id, properties }
+    subscription_heartbeat,        // args: { interval }
 
     current_state,                 // args: { device_id, current_state }
-    transition_status,             // args: { device_id, Result, transition }
+    transition_status,             // args: { device_id, task_id, Result, transition }
     config,                        // args: { device_id, config_string }
-    state_change_subscription,     // args: { device_id, Result }
-    state_change_unsubscription,   // args: { device_id, Result }
+    state_change_subscription,     // args: { device_id, task_id, Result }
+    state_change_unsubscription,   // args: { device_id, task_id, Result }
     state_change,                  // args: { device_id, task_id, last_state, current_state }
     properties,                    // args: { device_id, request_id, Result, properties }
     properties_set                 // args: { device_id, request_id, Result }
@@ -68,6 +70,7 @@ table FBCommand {
     device_id:string;
     task_id:uint64;
     request_id:uint64;
+    interval:int64;
     state:FBState;
     transition:FBTransition;
     result:FBResult;

fairmq/shmem/Common.h

@@ -8,9 +8,10 @@
 #ifndef FAIR_MQ_SHMEM_COMMON_H_
 #define FAIR_MQ_SHMEM_COMMON_H_
 
+#include <picosha2.h>
+
 #include <atomic>
 #include <string>
-#include <unordered_map>
 
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/allocators/allocator.hpp>
@@ -41,15 +42,21 @@ struct RegionInfo
     RegionInfo(const VoidAlloc& alloc)
         : fPath("", alloc)
         , fFlags(0)
+        , fUserFlags(0)
+        , fDestroyed(false)
     {}
 
-    RegionInfo(const char* path, int flags, const VoidAlloc& alloc)
+    RegionInfo(const char* path, const int flags, const uint64_t userFlags, const VoidAlloc& alloc)
         : fPath(path, alloc)
         , fFlags(flags)
+        , fUserFlags(userFlags)
+        , fDestroyed(false)
     {}
 
     Str fPath;
     int fFlags;
+    uint64_t fUserFlags;
+    bool fDestroyed;
 };
 
 using Uint64RegionInfoPairAlloc = boost::interprocess::allocator<std::pair<const uint64_t, RegionInfo>, SegmentManager>;
@@ -104,9 +111,12 @@ struct RegionBlock
 // a hash of user id + session id, truncated to 8 characters (to accommodate for name size limit on some systems (MacOS)).
 inline std::string buildShmIdFromSessionIdAndUserId(const std::string& sessionId)
 {
-    boost::hash<std::string> stringHash;
-    std::string shmId(std::to_string(stringHash(std::string((std::to_string(geteuid()) + sessionId)))));
-    shmId.resize(8, '_');
+    std::string seed((std::to_string(geteuid()) + sessionId));
+    // generate a 8-digit hex value out of sha256 hash
+    std::vector<unsigned char> hash(4);
+    picosha2::hash256(seed.begin(), seed.end(), hash.begin(), hash.end());
+    std::string shmId = picosha2::bytes_to_hex_string(hash.begin(), hash.end());
+
     return shmId;
 }
 

fairmq/shmem/Manager.cxx

@@ -1,220 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "Manager.h"
-#include "Common.h"
-
-#include <fairmq/tools/CppSTL.h>
-#include <fairmq/tools/Strings.h>
-
-#include <boost/process.hpp>
-#include <boost/filesystem.hpp>
-
-using namespace std;
-using bie = ::boost::interprocess::interprocess_exception;
-namespace bipc = ::boost::interprocess;
-namespace bfs = ::boost::filesystem;
-
-namespace fair
-{
-namespace mq
-{
-namespace shmem
-{
-
-std::unordered_map<uint64_t, std::unique_ptr<Region>> Manager::fRegions;
-
-Manager::Manager(const std::string& id, size_t size)
-    : fShmId(id)
-    , fSegmentName("fmq_" + fShmId + "_main")
-    , fManagementSegmentName("fmq_" + fShmId + "_mng")
-    , fSegment(bipc::open_or_create, fSegmentName.c_str(), size)
-    , fManagementSegment(bipc::open_or_create, fManagementSegmentName.c_str(), 65536)
-    , fShmMtx(bipc::open_or_create, string("fmq_" + fShmId + "_mtx").c_str())
-    , fDeviceCounter(nullptr)
-{
-    LOG(debug) << "created/opened shared memory segment '" << "fmq_" << fShmId << "_main" << "' of " << size << " bytes. Available are " << fSegment.get_free_memory() << " bytes.";
-
-    bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-
-    fDeviceCounter = fManagementSegment.find<DeviceCounter>(bipc::unique_instance).first;
-
-    if (fDeviceCounter) {
-        LOG(debug) << "device counter found, with value of " << fDeviceCounter->fCount << ". incrementing.";
-        (fDeviceCounter->fCount)++;
-        LOG(debug) << "incremented device counter, now: " << fDeviceCounter->fCount;
-    } else {
-        LOG(debug) << "no device counter found, creating one and initializing with 1";
-        fDeviceCounter = fManagementSegment.construct<DeviceCounter>(bipc::unique_instance)(1);
-        LOG(debug) << "initialized device counter with: " << fDeviceCounter->fCount;
-    }
-}
-
-void Manager::StartMonitor(const std::string& id)
-{
-    try {
-        bipc::named_mutex monitorStatus(bipc::open_only, string("fmq_" + id + "_ms").c_str());
-        LOG(debug) << "Found fairmq-shmmonitor for shared memory id " << id;
-    } catch (bie&) {
-        LOG(debug) << "no fairmq-shmmonitor found for shared memory id " << id << ", starting...";
-        auto env = boost::this_process::environment();
-
-        vector<bfs::path> ownPath = boost::this_process::path();
-
-        if (const char* fmqp = getenv("FAIRMQ_PATH")) {
-            ownPath.insert(ownPath.begin(), bfs::path(fmqp));
-        }
-
-        bfs::path p = boost::process::search_path("fairmq-shmmonitor", ownPath);
-
-        if (!p.empty()) {
-            boost::process::spawn(p, "-x", "--shmid", id, "-d", "-t", "2000", env);
-            int numTries = 0;
-            do {
-                try {
-                    bipc::named_mutex monitorStatus(bipc::open_only, string("fmq_" + id + "_ms").c_str());
-                    LOG(debug) << "Started fairmq-shmmonitor for shared memory id " << id;
-                    break;
-                } catch (bie&) {
-                    this_thread::sleep_for(chrono::milliseconds(10));
-                    if (++numTries > 1000) {
-                        LOG(error) << "Did not get response from fairmq-shmmonitor after " << 10 * 1000 << " milliseconds. Exiting.";
-                        throw runtime_error(tools::ToString("Did not get response from fairmq-shmmonitor after ", 10 * 1000, " milliseconds. Exiting."));
-                    }
-                }
-            } while (true);
-        } else {
-            LOG(warn) << "could not find fairmq-shmmonitor in the path";
-        }
-    }
-}
-
-void Manager::Interrupt()
-{
-}
-
-void Manager::Resume()
-{
-    // close remote regions before processing new transfers
-    for (auto it = fRegions.begin(); it != fRegions.end(); /**/) {
-        if (it->second->fRemote) {
-           it = fRegions.erase(it);
-        } else {
-            ++it;
-        }
-    }
-}
-
-bipc::mapped_region* Manager::CreateRegion(const size_t size, const uint64_t id, RegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */)
-{
-    auto it = fRegions.find(id);
-    if (it != fRegions.end()) {
-        LOG(error) << "Trying to create a region that already exists";
-        return nullptr;
-    } else {
-        // create region info
-        {
-            bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-            VoidAlloc voidAlloc(fManagementSegment.get_segment_manager());
-            Uint64RegionInfoMap* infoMap = fManagementSegment.find_or_construct<Uint64RegionInfoMap>(bipc::unique_instance)(voidAlloc);
-            infoMap->emplace(id, RegionInfo(path.c_str(), flags, voidAlloc));
-        }
-        // LOG(debug) << "Created region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
-
-        auto r = fRegions.emplace(id, tools::make_unique<Region>(*this, id, size, false, callback, path, flags));
-
-        r.first->second->StartReceivingAcks();
-
-        return &(r.first->second->fRegion);
-    }
-}
-
-Region* Manager::GetRemoteRegion(const uint64_t id)
-{
-    // remote region could actually be a local one if a message originates from this device (has been sent out and returned)
-    auto it = fRegions.find(id);
-    if (it != fRegions.end()) {
-        return it->second.get();
-    } else {
-        try {
-            string path;
-            int flags;
-
-            // get region info
-            {
-                bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-                Uint64RegionInfoMap* infoMap = fManagementSegment.find<Uint64RegionInfoMap>(bipc::unique_instance).first;
-                if (infoMap == nullptr) {
-                    LOG(error) << "Unable to locate the region info";
-                    throw SharedMemoryError("Unable to locate remote region info");
-                }
-                RegionInfo regionInfo = infoMap->at(id);
-                path = regionInfo.fPath.c_str();
-                flags = regionInfo.fFlags;
-            }
-            // LOG(debug) << "Located remote region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
-
-            auto r = fRegions.emplace(id, tools::make_unique<Region>(*this, id, 0, true, nullptr, path, flags));
-            return r.first->second.get();
-        } catch (bie& e) {
-            LOG(warn) << "Could not get remote region for id: " << id;
-            return nullptr;
-        }
-
-    }
-}
-
-void Manager::RemoveRegion(const uint64_t id)
-{
-    fRegions.erase(id);
-}
-
-void Manager::RemoveSegments()
-{
-    if (bipc::shared_memory_object::remove(fSegmentName.c_str())) {
-        LOG(debug) << "successfully removed '" << fSegmentName << "' segment after the device has stopped.";
-    } else {
-        LOG(debug) << "did not remove " << fSegmentName << " segment after the device stopped. Already removed?";
-    }
-
-    if (bipc::shared_memory_object::remove(fManagementSegmentName.c_str())) {
-        LOG(debug) << "successfully removed '" << fManagementSegmentName << "' segment after the device has stopped.";
-    } else {
-        LOG(debug) << "did not remove '" << fManagementSegmentName << "' segment after the device stopped. Already removed?";
-    }
-}
-
-Manager::~Manager()
-{
-    bool lastRemoved = false;
-
-    try {
-        bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-
-        (fDeviceCounter->fCount)--;
-
-        if (fDeviceCounter->fCount == 0) {
-            LOG(debug) << "last segment user, removing segment.";
-
-            RemoveSegments();
-            lastRemoved = true;
-        } else {
-            LOG(debug) << "other segment users present (" << fDeviceCounter->fCount << "), not removing it.";
-        }
-    } catch(bie& e) {
-        LOG(error) << "error while acquiring lock in Manager destructor: " << e.what();
-    }
-
-    if (lastRemoved) {
-        bipc::named_mutex::remove(string("fmq_" + fShmId + "_mtx").c_str());
-    }
-}
-
-} // namespace shmem
-} // namespace mq
-} // namespace fair

fairmq/shmem/Manager.h

@@ -15,18 +15,38 @@
 #ifndef FAIR_MQ_SHMEM_MANAGER_H_
 #define FAIR_MQ_SHMEM_MANAGER_H_
 
-#include "Region.h"
 #include "Common.h"
+#include "Region.h"
+#include "Monitor.h"
 
 #include <FairMQLogger.h>
+#include <FairMQMessage.h>
+#include <fairmq/ProgOptions.h>
+#include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Strings.h>
 
-#include <boost/interprocess/managed_shared_memory.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/interprocess/indexes/null_index.hpp>
 #include <boost/interprocess/ipc/message_queue.hpp>
+#include <boost/interprocess/managed_shared_memory.hpp>
+#include <boost/interprocess/sync/named_condition.hpp>
 #include <boost/interprocess/sync/named_mutex.hpp>
+#include <boost/interprocess/mem_algo/simple_seq_fit.hpp>
+#include <boost/process.hpp>
 
-#include <string>
-#include <unordered_map>
+#include <cstdlib> // getenv
+#include <condition_variable>
+#include <mutex>
+#include <set>
 #include <stdexcept>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <utility> // pair
+#include <vector>
+
+#include <sys/mman.h> // mlock
 
 namespace fair
 {
@@ -37,47 +57,429 @@ namespace shmem
 
 struct SharedMemoryError : std::runtime_error { using std::runtime_error::runtime_error; };
 
+using SimpleSeqFitSegment = boost::interprocess::basic_managed_shared_memory<char,
+    boost::interprocess::simple_seq_fit<boost::interprocess::mutex_family>,
+    boost::interprocess::iset_index>;
+using RBTreeBestFitSegment = boost::interprocess::basic_managed_shared_memory<char,
+    boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family>,
+    boost::interprocess::iset_index>;
+
 class Manager
 {
-    friend struct Region;
-
   public:
-    Manager(const std::string& id, size_t size);
+    Manager(std::string shmId, std::string deviceId, size_t size, const ProgOptions* config)
+        : fShmId(std::move(shmId))
+        , fDeviceId(std::move(deviceId))
+        , fSegment(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_main").c_str(), size)
+        , fManagementSegment(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_mng").c_str(), 655360)
+        , fShmVoidAlloc(fManagementSegment.get_segment_manager())
+        , fShmMtx(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_mtx").c_str())
+        , fRegionEventsCV(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_cv").c_str())
+        , fRegionEventsSubscriptionActive(false)
+        , fDeviceCounter(nullptr)
+        , fRegionInfos(nullptr)
+        , fInterrupted(false)
+        , fMsgCounter(0)
+        , fHeartbeatThread()
+        , fSendHeartbeats(true)
+        , fThrowOnBadAlloc(true)
+    {
+        using namespace boost::interprocess;
+
+        bool mlockSegment = false;
+        bool zeroSegment = false;
+        bool autolaunchMonitor = false;
+        if (config) {
+            mlockSegment = config->GetProperty<bool>("shm-mlock-segment", mlockSegment);
+            zeroSegment = config->GetProperty<bool>("shm-zero-segment", zeroSegment);
+            autolaunchMonitor = config->GetProperty<bool>("shm-monitor", autolaunchMonitor);
+            fThrowOnBadAlloc = config->GetProperty<bool>("shm-throw-bad-alloc", fThrowOnBadAlloc);
+        } else {
+            LOG(debug) << "ProgOptions not available! Using defaults.";
+        }
+
+        if (autolaunchMonitor) {
+            StartMonitor(fShmId);
+        }
+
+        LOG(debug) << "created/opened shared memory segment '" << "fmq_" << fShmId << "_main" << "' of " << fSegment.get_size() << " bytes. Available are " << fSegment.get_free_memory() << " bytes.";
+        if (mlockSegment) {
+            LOG(debug) << "Locking the managed segment memory pages...";
+            if (mlock(fSegment.get_address(), fSegment.get_size()) == -1) {
+                LOG(error) << "Could not lock the managed segment memory. Code: " << errno << ", reason: " << strerror(errno);
+            }
+            LOG(debug) << "Successfully locked the managed segment memory pages.";
+        }
+        if (zeroSegment) {
+            LOG(debug) << "Zeroing the managed segment free memory...";
+            fSegment.zero_free_memory();
+            LOG(debug) << "Successfully zeroed the managed segment free memory.";
+        }
+
+        fRegionInfos = fManagementSegment.find_or_construct<Uint64RegionInfoMap>(unique_instance)(fShmVoidAlloc);
+        // store info about the managed segment as region with id 0
+        fRegionInfos->emplace(0, RegionInfo("", 0, 0, fShmVoidAlloc));
+
+        boost::interprocess::scoped_lock<named_mutex> lock(fShmMtx);
+
+        fDeviceCounter = fManagementSegment.find<DeviceCounter>(unique_instance).first;
+
+        if (fDeviceCounter) {
+            LOG(debug) << "device counter found, with value of " << fDeviceCounter->fCount << ". incrementing.";
+            (fDeviceCounter->fCount)++;
+            LOG(debug) << "incremented device counter, now: " << fDeviceCounter->fCount;
+        } else {
+            LOG(debug) << "no device counter found, creating one and initializing with 1";
+            fDeviceCounter = fManagementSegment.construct<DeviceCounter>(unique_instance)(1);
+            LOG(debug) << "initialized device counter with: " << fDeviceCounter->fCount;
+        }
+
+        fHeartbeatThread = std::thread(&Manager::SendHeartbeats, this);
+    }
 
     Manager() = delete;
 
     Manager(const Manager&) = delete;
     Manager operator=(const Manager&) = delete;
 
-    ~Manager();
-
-    boost::interprocess::managed_shared_memory& Segment() { return fSegment; }
+    RBTreeBestFitSegment& Segment() { return fSegment; }
     boost::interprocess::managed_shared_memory& ManagementSegment() { return fManagementSegment; }
 
-    static void StartMonitor(const std::string&);
+    static void StartMonitor(const std::string& id)
+    {
+        using namespace boost::interprocess;
+        try {
+            named_mutex monitorStatus(open_only, std::string("fmq_" + id + "_ms").c_str());
+            LOG(debug) << "Found fairmq-shmmonitor for shared memory id " << id;
+        } catch (interprocess_exception&) {
+            LOG(debug) << "no fairmq-shmmonitor found for shared memory id " << id << ", starting...";
+            auto env = boost::this_process::environment();
 
-    static void Interrupt();
-    static void Resume();
+            std::vector<boost::filesystem::path> ownPath = boost::this_process::path();
 
-    int GetDeviceCounter();
-    int IncrementDeviceCounter();
-    int DecrementDeviceCounter();
+            if (const char* fmqp = getenv("FAIRMQ_PATH")) {
+                ownPath.insert(ownPath.begin(), boost::filesystem::path(fmqp));
+            }
 
-    boost::interprocess::mapped_region* CreateRegion(const size_t size, const uint64_t id, RegionCallback callback, const std::string& path = "", int flags = 0);
-    Region* GetRemoteRegion(const uint64_t id);
-    void RemoveRegion(const uint64_t id);
+            boost::filesystem::path p = boost::process::search_path("fairmq-shmmonitor", ownPath);
 
-    void RemoveSegments();
+            if (!p.empty()) {
+                boost::process::spawn(p, "-x", "--shmid", id, "-d", "-t", "2000", env);
+                int numTries = 0;
+                do {
+                    try {
+                        named_mutex monitorStatus(open_only, std::string("fmq_" + id + "_ms").c_str());
+                        LOG(debug) << "Started fairmq-shmmonitor for shared memory id " << id;
+                        break;
+                    } catch (interprocess_exception&) {
+                        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+                        if (++numTries > 1000) {
+                            LOG(error) << "Did not get response from fairmq-shmmonitor after " << 10 * 1000 << " milliseconds. Exiting.";
+                            throw std::runtime_error(tools::ToString("Did not get response from fairmq-shmmonitor after ", 10 * 1000, " milliseconds. Exiting."));
+                        }
+                    }
+                } while (true);
+            } else {
+                LOG(warn) << "could not find fairmq-shmmonitor in the path";
+            }
+        }
+    }
+
+    void Interrupt() { fInterrupted.store(true); }
+    void Resume() { fInterrupted.store(false); }
+    void Reset()
+    {
+        if (fMsgCounter.load() != 0) {
+            LOG(error) << "Message counter during Reset expected to be 0, found: " << fMsgCounter.load();
+            throw MessageError(tools::ToString("Message counter during Reset expected to be 0, found: ", fMsgCounter.load()));
+        }
+    }
+    bool Interrupted() { return fInterrupted.load(); }
+
+    std::pair<boost::interprocess::mapped_region*, uint64_t> CreateRegion(const size_t size,
+                                                                          const int64_t userFlags,
+                                                                          RegionCallback callback,
+                                                                          RegionBulkCallback bulkCallback,
+                                                                          const std::string& path = "",
+                                                                          int flags = 0)
+    {
+        using namespace boost::interprocess;
+        try {
+            std::pair<mapped_region*, uint64_t> result;
+
+            {
+                uint64_t id = 0;
+                boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+
+                RegionCounter* rc = fManagementSegment.find<RegionCounter>(unique_instance).first;
+
+                if (rc) {
+                    LOG(debug) << "region counter found, with value of " << rc->fCount << ". incrementing.";
+                    (rc->fCount)++;
+                    LOG(debug) << "incremented region counter, now: " << rc->fCount;
+                } else {
+                    LOG(debug) << "no region counter found, creating one and initializing with 1";
+                    rc = fManagementSegment.construct<RegionCounter>(unique_instance)(1);
+                    LOG(debug) << "initialized region counter with: " << rc->fCount;
+                }
+
+                id = rc->fCount;
+
+                auto it = fRegions.find(id);
+                if (it != fRegions.end()) {
+                    LOG(error) << "Trying to create a region that already exists";
+                    return {nullptr, id};
+                }
+
+                // create region info
+                fRegionInfos->emplace(id, RegionInfo(path.c_str(), flags, userFlags, fShmVoidAlloc));
+
+                auto r = fRegions.emplace(id, tools::make_unique<Region>(fShmId, id, size, false, callback, bulkCallback, path, flags));
+                // LOG(debug) << "Created region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
+
+                r.first->second->StartReceivingAcks();
+                result.first = &(r.first->second->fRegion);
+                result.second = id;
+            }
+            fRegionEventsCV.notify_all();
+
+            return result;
+
+        } catch (interprocess_exception& e) {
+            LOG(error) << "cannot create region. Already created/not cleaned up?";
+            LOG(error) << e.what();
+            throw;
+        }
+    }
+
+    Region* GetRegion(const uint64_t id)
+    {
+        boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+        return GetRegionUnsafe(id);
+    }
+
+    Region* GetRegionUnsafe(const uint64_t id)
+    {
+        // remote region could actually be a local one if a message originates from this device (has been sent out and returned)
+        auto it = fRegions.find(id);
+        if (it != fRegions.end()) {
+            return it->second.get();
+        } else {
+            try {
+                // get region info
+                RegionInfo regionInfo = fRegionInfos->at(id);
+                std::string path = regionInfo.fPath.c_str();
+                int flags = regionInfo.fFlags;
+                // LOG(debug) << "Located remote region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
+
+                auto r = fRegions.emplace(id, tools::make_unique<Region>(fShmId, id, 0, true, nullptr, nullptr, path, flags));
+                return r.first->second.get();
+            } catch (std::out_of_range& oor) {
+                LOG(error) << "Could not get remote region with id '" << id << "'. Does the region creator run with the same session id?";
+                LOG(error) << oor.what();
+                return nullptr;
+            } catch (boost::interprocess::interprocess_exception& e) {
+                LOG(warn) << "Could not get remote region for id '" << id << "'";
+                return nullptr;
+            }
+        }
+    }
+
+    void RemoveRegion(const uint64_t id)
+    {
+        fRegions.erase(id);
+        {
+            boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+            fRegionInfos->at(id).fDestroyed = true;
+        }
+        fRegionEventsCV.notify_all();
+    }
+
+    std::vector<fair::mq::RegionInfo> GetRegionInfo()
+    {
+        boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+        return GetRegionInfoUnsafe();
+    }
+
+    std::vector<fair::mq::RegionInfo> GetRegionInfoUnsafe()
+    {
+        std::vector<fair::mq::RegionInfo> result;
+
+        for (const auto& e : *fRegionInfos) {
+            fair::mq::RegionInfo info;
+            info.id = e.first;
+            info.flags = e.second.fUserFlags;
+            info.event = e.second.fDestroyed ? RegionEvent::destroyed : RegionEvent::created;
+            if (info.id != 0) {
+                if (!e.second.fDestroyed) {
+                    auto region = GetRegionUnsafe(info.id);
+                    info.ptr = region->fRegion.get_address();
+                    info.size = region->fRegion.get_size();
+                } else {
+                    info.ptr = nullptr;
+                    info.size = 0;
+                }
+                result.push_back(info);
+            } else {
+                if (!e.second.fDestroyed) {
+                    info.ptr = fSegment.get_address();
+                    info.size = fSegment.get_size();
+                } else {
+                    info.ptr = nullptr;
+                    info.size = 0;
+                }
+                result.push_back(info);
+            }
+        }
+
+        return result;
+    }
+
+    void SubscribeToRegionEvents(RegionEventCallback callback)
+    {
+        if (fRegionEventThread.joinable()) {
+            LOG(debug) << "Already subscribed. Overwriting previous subscription.";
+            boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+            fRegionEventsSubscriptionActive = false;
+            lock.unlock();
+            fRegionEventsCV.notify_all();
+            fRegionEventThread.join();
+        }
+        boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+        fRegionEventCallback = callback;
+        fRegionEventsSubscriptionActive = true;
+        fRegionEventThread = std::thread(&Manager::RegionEventsSubscription, this);
+    }
+
+    bool SubscribedToRegionEvents() { return fRegionEventThread.joinable(); }
+
+    void UnsubscribeFromRegionEvents()
+    {
+        if (fRegionEventThread.joinable()) {
+            boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+            fRegionEventsSubscriptionActive = false;
+            lock.unlock();
+            fRegionEventsCV.notify_all();
+            fRegionEventThread.join();
+            lock.lock();
+            fRegionEventCallback = nullptr;
+        }
+    }
+
+    void RegionEventsSubscription()
+    {
+        boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+        while (fRegionEventsSubscriptionActive) {
+            auto infos = GetRegionInfoUnsafe();
+            for (const auto& i : infos) {
+                auto el = fObservedRegionEvents.find(i.id);
+                if (el == fObservedRegionEvents.end()) {
+                    fRegionEventCallback(i);
+                    fObservedRegionEvents.emplace(i.id, i.event);
+                } else {
+                    if (el->second == RegionEvent::created && i.event == RegionEvent::destroyed) {
+                        fRegionEventCallback(i);
+                        el->second = i.event;
+                    } else {
+                        // LOG(debug) << "ignoring event for id" << i.id << ":";
+                        // LOG(debug) << "incoming event: " << i.event;
+                        // LOG(debug) << "stored event: " << el->second;
+                    }
+                }
+            }
+            fRegionEventsCV.wait(lock);
+        }
+    }
+
+    void IncrementMsgCounter() { fMsgCounter.fetch_add(1, std::memory_order_relaxed); }
+    void DecrementMsgCounter() { fMsgCounter.fetch_sub(1, std::memory_order_relaxed); }
+
+    void SendHeartbeats()
+    {
+        std::string controlQueueName("fmq_" + fShmId + "_cq");
+        std::unique_lock<std::mutex> lock(fHeartbeatsMtx);
+        while (fSendHeartbeats) {
+            try {
+                boost::interprocess::message_queue mq(boost::interprocess::open_only, controlQueueName.c_str());
+                boost::posix_time::ptime sndTill = boost::posix_time::microsec_clock::universal_time() + boost::posix_time::milliseconds(100);
+                if (mq.timed_send(fDeviceId.c_str(), fDeviceId.size(), 0, sndTill)) {
+                    fHeartbeatsCV.wait_for(lock, std::chrono::milliseconds(100), [&]() { return !fSendHeartbeats; });
+                } else {
+                    LOG(debug) << "control queue timeout";
+                }
+            } catch (boost::interprocess::interprocess_exception& ie) {
+                fHeartbeatsCV.wait_for(lock, std::chrono::milliseconds(500), [&]() { return !fSendHeartbeats; });
+                // LOG(debug) << "no " << controlQueueName << " found";
+            }
+        }
+    }
+
+    bool ThrowingOnBadAlloc() const { return fThrowOnBadAlloc; }
+
+    ~Manager()
+    {
+        using namespace boost::interprocess;
+        bool lastRemoved = false;
+
+        UnsubscribeFromRegionEvents();
+
+        {
+            std::unique_lock<std::mutex> lock(fHeartbeatsMtx);
+            fSendHeartbeats = false;
+        }
+        fHeartbeatsCV.notify_one();
+        if (fHeartbeatThread.joinable()) {
+            fHeartbeatThread.join();
+        }
+
+        try {
+            boost::interprocess::scoped_lock<named_mutex> lock(fShmMtx);
+
+            (fDeviceCounter->fCount)--;
+
+            if (fDeviceCounter->fCount == 0) {
+                LOG(debug) << "Last segment user, removing segment.";
+                lastRemoved = true;
+            } else {
+                LOG(debug) << "Other segment users present (" << fDeviceCounter->fCount << "), skipping removal.";
+            }
+        } catch (interprocess_exception& e) {
+            LOG(error) << "Manager could not acquire lock: " << e.what();
+        }
+
+        if (lastRemoved) {
+            Monitor::Cleanup(ShmId{fShmId});
+        }
+    }
 
   private:
     std::string fShmId;
-    std::string fSegmentName;
-    std::string fManagementSegmentName;
-    boost::interprocess::managed_shared_memory fSegment;
+    std::string fDeviceId;
+    // boost::interprocess::managed_shared_memory fSegment;
+    RBTreeBestFitSegment fSegment;
     boost::interprocess::managed_shared_memory fManagementSegment;
+    VoidAlloc fShmVoidAlloc;
     boost::interprocess::named_mutex fShmMtx;
+
+    boost::interprocess::named_condition fRegionEventsCV;
+    std::thread fRegionEventThread;
+    bool fRegionEventsSubscriptionActive;
+    std::function<void(fair::mq::RegionInfo)> fRegionEventCallback;
+    std::unordered_map<uint64_t, RegionEvent> fObservedRegionEvents;
+
     DeviceCounter* fDeviceCounter;
-    static std::unordered_map<uint64_t, std::unique_ptr<Region>> fRegions;
+    Uint64RegionInfoMap* fRegionInfos;
+    std::unordered_map<uint64_t, std::unique_ptr<Region>> fRegions;
+
+    std::atomic<bool> fInterrupted;
+    std::atomic<int32_t> fMsgCounter; // TODO: find a better lifetime solution instead of the counter
+
+    std::thread fHeartbeatThread;
+    bool fSendHeartbeats;
+    std::mutex fHeartbeatsMtx;
+    std::condition_variable fHeartbeatsCV;
+
+    bool fThrowOnBadAlloc;
 };
 
 } // namespace shmem

fairmq/shmem/Message.cxx

@@ -1,251 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "Region.h"
-#include "Message.h"
-#include "UnmanagedRegion.h"
-#include "TransportFactory.h"
-
-#include <FairMQLogger.h>
-
-#include <boost/date_time/posix_time/posix_time.hpp>
-
-#include <cstdlib>
-
-using namespace std;
-
-namespace bipc = ::boost::interprocess;
-namespace bpt = ::boost::posix_time;
-
-namespace fair
-{
-namespace mq
-{
-namespace shmem
-{
-
-atomic<bool> Message::fInterrupted(false);
-Transport Message::fTransportType = Transport::SHM;
-
-Message::Message(Manager& manager, FairMQTransportFactory* factory)
-    : fair::mq::Message{factory}
-    , fManager(manager)
-    , fQueued(false)
-    , fMeta{0, 0, 0, -1}
-    , fRegionPtr(nullptr)
-    , fLocalPtr(nullptr)
-{
-    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
-}
-
-Message::Message(Manager& manager, const size_t size, FairMQTransportFactory* factory)
-    : fair::mq::Message{factory}
-    , fManager(manager)
-    , fQueued(false)
-    , fMeta{0, 0, 0, -1}
-    , fRegionPtr(nullptr)
-    , fLocalPtr(nullptr)
-{
-    InitializeChunk(size);
-    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
-}
-
-Message::Message(Manager& manager, MetaHeader& hdr, FairMQTransportFactory* factory)
-    : fair::mq::Message{factory}
-    , fManager(manager)
-    , fQueued(false)
-    , fMeta{hdr}
-    , fRegionPtr(nullptr)
-    , fLocalPtr(nullptr)
-{
-    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
-}
-
-Message::Message(Manager& manager, void* data, const size_t size, fairmq_free_fn* ffn, void* hint, FairMQTransportFactory* factory)
-    : fair::mq::Message{factory}
-    , fManager(manager)
-    , fQueued(false)
-    , fMeta{0, 0, 0, -1}
-    , fRegionPtr(nullptr)
-    , fLocalPtr(nullptr)
-{
-    if (InitializeChunk(size)) {
-        std::memcpy(fLocalPtr, data, size);
-        if (ffn) {
-            ffn(data, hint);
-        } else {
-            free(data);
-        }
-    }
-    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
-}
-
-Message::Message(Manager& manager, UnmanagedRegionPtr& region, void* data, const size_t size, void* hint, FairMQTransportFactory* factory)
-    : fair::mq::Message{factory}
-    , fManager(manager)
-    , fQueued(false)
-    , fMeta{size, static_cast<UnmanagedRegion*>(region.get())->fRegionId, reinterpret_cast<size_t>(hint), -1}
-    , fRegionPtr(nullptr)
-    , fLocalPtr(static_cast<char*>(data))
-{
-    if (reinterpret_cast<const char*>(data) >= reinterpret_cast<const char*>(region->GetData()) ||
-        reinterpret_cast<const char*>(data) <= reinterpret_cast<const char*>(region->GetData()) + region->GetSize()) {
-        fMeta.fHandle = (bipc::managed_shared_memory::handle_t)(reinterpret_cast<const char*>(data) - reinterpret_cast<const char*>(region->GetData()));
-    } else {
-        LOG(error) << "trying to create region message with data from outside the region";
-        throw runtime_error("trying to create region message with data from outside the region");
-    }
-    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
-}
-
-bool Message::InitializeChunk(const size_t size)
-{
-    while (fMeta.fHandle < 0) {
-        try {
-            bipc::managed_shared_memory::size_type actualSize = size;
-            char* hint = 0; // unused for bipc::allocate_new
-            fLocalPtr = fManager.Segment().allocation_command<char>(bipc::allocate_new, size, actualSize, hint);
-        } catch (bipc::bad_alloc& ba) {
-            // LOG(warn) << "Shared memory full...";
-            this_thread::sleep_for(chrono::milliseconds(50));
-            if (fInterrupted) {
-                return false;
-            } else {
-                continue;
-            }
-        }
-        fMeta.fHandle = fManager.Segment().get_handle_from_address(fLocalPtr);
-    }
-
-    fMeta.fSize = size;
-    return true;
-}
-
-void Message::Rebuild()
-{
-    CloseMessage();
-    fQueued = false;
-}
-
-void Message::Rebuild(const size_t size)
-{
-    CloseMessage();
-    fQueued = false;
-    InitializeChunk(size);
-}
-
-void Message::Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
-{
-    CloseMessage();
-    fQueued = false;
-
-    if (InitializeChunk(size)) {
-        std::memcpy(fLocalPtr, data, size);
-        if (ffn) {
-            ffn(data, hint);
-        } else {
-            free(data);
-        }
-    }
-}
-
-void* Message::GetData() const
-{
-    if (!fLocalPtr) {
-        if (fMeta.fRegionId == 0) {
-            if (fMeta.fSize > 0) {
-                fLocalPtr = reinterpret_cast<char*>(fManager.Segment().get_address_from_handle(fMeta.fHandle));
-            } else {
-                fLocalPtr = nullptr;
-            }
-        } else {
-            fRegionPtr = fManager.GetRemoteRegion(fMeta.fRegionId);
-            if (fRegionPtr) {
-                fLocalPtr = reinterpret_cast<char*>(fRegionPtr->fRegion.get_address()) + fMeta.fHandle;
-            } else {
-                // LOG(warn) << "could not get pointer from a region message";
-                fLocalPtr = nullptr;
-            }
-        }
-    }
-
-    return fLocalPtr;
-}
-
-bool Message::SetUsedSize(const size_t size)
-{
-    if (size == fMeta.fSize) {
-        return true;
-    } else if (size <= fMeta.fSize) {
-        try {
-            bipc::managed_shared_memory::size_type shrunkSize = size;
-            fLocalPtr = fManager.Segment().allocation_command<char>(bipc::shrink_in_place, fMeta.fSize + 128, shrunkSize, fLocalPtr);
-            fMeta.fSize = size;
-            return true;
-        } catch (bipc::interprocess_exception& e) {
-            LOG(info) << "could not set used size: " << e.what();
-            return false;
-        }
-    } else {
-        LOG(error) << "cannot set used size higher than original.";
-        return false;
-    }
-}
-
-void Message::Copy(const fair::mq::Message& msg)
-{
-    if (fMeta.fHandle < 0) {
-        bipc::managed_shared_memory::handle_t otherHandle = static_cast<const Message&>(msg).fMeta.fHandle;
-        if (otherHandle) {
-            if (InitializeChunk(msg.GetSize())) {
-                std::memcpy(GetData(), msg.GetData(), msg.GetSize());
-            }
-        } else {
-            LOG(error) << "copy fail: source message not initialized!";
-        }
-    } else {
-        LOG(error) << "copy fail: target message already initialized!";
-    }
-}
-
-void Message::CloseMessage()
-{
-    if (fMeta.fHandle >= 0 && !fQueued) {
-        if (fMeta.fRegionId == 0) {
-            fManager.Segment().deallocate(fManager.Segment().get_address_from_handle(fMeta.fHandle));
-            fMeta.fHandle = -1;
-        } else {
-            if (!fRegionPtr) {
-                fRegionPtr = fManager.GetRemoteRegion(fMeta.fRegionId);
-            }
-
-            if (fRegionPtr) {
-                fRegionPtr->ReleaseBlock({fMeta.fHandle, fMeta.fSize, fMeta.fHint});
-            } else {
-                LOG(warn) << "region ack queue for id " << fMeta.fRegionId << " no longer exist. Not sending ack";
-            }
-        }
-    }
-
-    static_cast<TransportFactory*>(GetTransport())->DecrementMsgCounter();
-}
-
-Message::~Message()
-{
-    try {
-        CloseMessage();
-    } catch(SharedMemoryError& sme) {
-        LOG(error) << "error closing message: " << sme.what();
-    } catch(bipc::lock_exception& le) {
-        LOG(error) << "error closing message: " << le.what();
-    }
-}
-
-}
-}
-}

fairmq/shmem/Message.h

@@ -10,7 +10,10 @@
 
 #include "Common.h"
 #include "Manager.h"
-
+#include "Region.h"
+#include "UnmanagedRegion.h"
+#include <fairmq/Tools.h>
+#include <FairMQLogger.h>
 #include <FairMQMessage.h>
 #include <FairMQUnmanagedRegion.h>
 
@@ -33,30 +36,204 @@ class Message final : public fair::mq::Message
     friend class Socket;
 
   public:
-    Message(Manager& manager, FairMQTransportFactory* factory = nullptr);
-    Message(Manager& manager, const size_t size, FairMQTransportFactory* factory = nullptr);
-    Message(Manager& manager, void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr, FairMQTransportFactory* factory = nullptr);
-    Message(Manager& manager, UnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0, FairMQTransportFactory* factory = nullptr);
+    Message(Manager& manager, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{0, 0, 0, -1}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(nullptr)
+    {
+        fManager.IncrementMsgCounter();
+    }
 
-    Message(Manager& manager, MetaHeader& hdr, FairMQTransportFactory* factory = nullptr);
+    Message(Manager& manager, Alignment /* alignment */, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{0, 0, 0, -1}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(nullptr)
+    {
+        fManager.IncrementMsgCounter();
+    }
+
+    Message(Manager& manager, const size_t size, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{0, 0, 0, -1}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(nullptr)
+    {
+        InitializeChunk(size);
+        fManager.IncrementMsgCounter();
+    }
+
+    Message(Manager& manager, const size_t size, Alignment alignment, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{0, 0, 0, -1}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(nullptr)
+    {
+        InitializeChunk(size, static_cast<size_t>(alignment));
+        fManager.IncrementMsgCounter();
+    }
+
+    Message(Manager& manager, void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{0, 0, 0, -1}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(nullptr)
+    {
+        if (InitializeChunk(size)) {
+            std::memcpy(fLocalPtr, data, size);
+            if (ffn) {
+                ffn(data, hint);
+            } else {
+                free(data);
+            }
+        }
+        fManager.IncrementMsgCounter();
+    }
+
+    Message(Manager& manager, UnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{size, static_cast<UnmanagedRegion*>(region.get())->fRegionId, reinterpret_cast<size_t>(hint), -1}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(static_cast<char*>(data))
+    {
+        if (reinterpret_cast<const char*>(data) >= reinterpret_cast<const char*>(region->GetData()) &&
+            reinterpret_cast<const char*>(data) <= reinterpret_cast<const char*>(region->GetData()) + region->GetSize()) {
+            fMeta.fHandle = (boost::interprocess::managed_shared_memory::handle_t)(reinterpret_cast<const char*>(data) - reinterpret_cast<const char*>(region->GetData()));
+        } else {
+            LOG(error) << "trying to create region message with data from outside the region";
+            throw std::runtime_error("trying to create region message with data from outside the region");
+        }
+        fManager.IncrementMsgCounter();
+    }
+
+    Message(Manager& manager, MetaHeader& hdr, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{hdr}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(nullptr)
+    {
+        fManager.IncrementMsgCounter();
+    }
 
     Message(const Message&) = delete;
     Message operator=(const Message&) = delete;
 
-    void Rebuild() override;
-    void Rebuild(const size_t size) override;
-    void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
+    void Rebuild() override
+    {
+        CloseMessage();
+        fQueued = false;
+    }
+
+    void Rebuild(const size_t size) override
+    {
+        CloseMessage();
+        fQueued = false;
+        InitializeChunk(size);
+    }
+
+    void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override
+    {
+        CloseMessage();
+        fQueued = false;
+
+        if (InitializeChunk(size)) {
+            std::memcpy(fLocalPtr, data, size);
+            if (ffn) {
+                ffn(data, hint);
+            } else {
+                free(data);
+            }
+        }
+    }
+
+    void* GetData() const override
+    {
+        if (!fLocalPtr) {
+            if (fMeta.fRegionId == 0) {
+                if (fMeta.fSize > 0) {
+                    fLocalPtr = reinterpret_cast<char*>(fManager.Segment().get_address_from_handle(fMeta.fHandle));
+                } else {
+                    fLocalPtr = nullptr;
+                }
+            } else {
+                fRegionPtr = fManager.GetRegion(fMeta.fRegionId);
+                if (fRegionPtr) {
+                    fLocalPtr = reinterpret_cast<char*>(fRegionPtr->fRegion.get_address()) + fMeta.fHandle;
+                } else {
+                    // LOG(warn) << "could not get pointer from a region message";
+                    fLocalPtr = nullptr;
+                }
+            }
+        }
+
+        return fLocalPtr;
+    }
 
-    void* GetData() const override;
     size_t GetSize() const override { return fMeta.fSize; }
 
-    bool SetUsedSize(const size_t size) override;
+    bool SetUsedSize(const size_t size) override
+    {
+        if (size == fMeta.fSize) {
+            return true;
+        } else if (size <= fMeta.fSize) {
+            try {
+                boost::interprocess::managed_shared_memory::size_type shrunkSize = size;
+                fLocalPtr = fManager.Segment().allocation_command<char>(boost::interprocess::shrink_in_place, fMeta.fSize + 128, shrunkSize, fLocalPtr);
+                fMeta.fSize = size;
+                return true;
+            } catch (boost::interprocess::interprocess_exception& e) {
+                LOG(info) << "could not set used size: " << e.what();
+                return false;
+            }
+        } else {
+            LOG(error) << "cannot set used size higher than original.";
+            return false;
+        }
+    }
 
-    Transport GetType() const override { return fTransportType; }
+    Transport GetType() const override { return fair::mq::Transport::SHM; }
 
-    void Copy(const fair::mq::Message& msg) override;
+    void Copy(const fair::mq::Message& msg) override
+    {
+        if (fMeta.fHandle < 0) {
+            boost::interprocess::managed_shared_memory::handle_t otherHandle = static_cast<const Message&>(msg).fMeta.fHandle;
+            if (otherHandle) {
+                if (InitializeChunk(msg.GetSize())) {
+                    std::memcpy(GetData(), msg.GetData(), msg.GetSize());
+                }
+            } else {
+                LOG(error) << "copy fail: source message not initialized!";
+            }
+        } else {
+            LOG(error) << "copy fail: target message already initialized!";
+        }
+    }
 
-    ~Message() override;
+    ~Message() override
+    {
+        try {
+            CloseMessage();
+        } catch(SharedMemoryError& sme) {
+            LOG(error) << "error closing message: " << sme.what();
+        } catch(boost::interprocess::lock_exception& le) {
+            LOG(error) << "error closing message: " << le.what();
+        }
+    }
 
   private:
     Manager& fManager;
@@ -65,11 +242,65 @@ class Message final : public fair::mq::Message
     mutable Region* fRegionPtr;
     mutable char* fLocalPtr;
 
-    static std::atomic<bool> fInterrupted;
-    static Transport fTransportType;
+    bool InitializeChunk(const size_t size, size_t alignment = 0)
+    {
+        // tools::RateLimiter rateLimiter(20);
 
-    bool InitializeChunk(const size_t size);
-    void CloseMessage();
+        while (fMeta.fHandle < 0) {
+            try {
+                // boost::interprocess::managed_shared_memory::size_type actualSize = size;
+                // char* hint = 0; // unused for boost::interprocess::allocate_new
+                // fLocalPtr = fManager.Segment().allocation_command<char>(boost::interprocess::allocate_new, size, actualSize, hint);
+                size_t segmentSize = fManager.Segment().get_size();
+                if (size > segmentSize) {
+                    throw MessageBadAlloc(tools::ToString("Requested message size (", size, ") exceeds segment size (", segmentSize, ")"));
+                }
+                if (alignment == 0) {
+                    fLocalPtr = reinterpret_cast<char*>(fManager.Segment().allocate(size));
+                } else {
+                    fLocalPtr = reinterpret_cast<char*>(fManager.Segment().allocate_aligned(size, alignment));
+                }
+            } catch (boost::interprocess::bad_alloc& ba) {
+                // LOG(warn) << "Shared memory full...";
+                if (fManager.ThrowingOnBadAlloc()) {
+                    throw MessageBadAlloc(tools::ToString("shmem: could not create a message of size ", size, ", alignment: ", (alignment != 0) ? std::to_string(alignment) : "default", ", free memory: ", fManager.Segment().get_free_memory()));
+                }
+                // rateLimiter.maybe_sleep();
+                std::this_thread::sleep_for(std::chrono::milliseconds(50));
+                if (fManager.Interrupted()) {
+                    return false;
+                } else {
+                    continue;
+                }
+            }
+            fMeta.fHandle = fManager.Segment().get_handle_from_address(fLocalPtr);
+        }
+
+        fMeta.fSize = size;
+        return true;
+    }
+
+    void CloseMessage()
+    {
+        if (fMeta.fHandle >= 0 && !fQueued) {
+            if (fMeta.fRegionId == 0) {
+                fManager.Segment().deallocate(fManager.Segment().get_address_from_handle(fMeta.fHandle));
+                fMeta.fHandle = -1;
+            } else {
+                if (!fRegionPtr) {
+                    fRegionPtr = fManager.GetRegion(fMeta.fRegionId);
+                }
+
+                if (fRegionPtr) {
+                    fRegionPtr->ReleaseBlock({fMeta.fHandle, fMeta.fSize, fMeta.fHint});
+                } else {
+                    LOG(warn) << "region ack queue for id " << fMeta.fRegionId << " no longer exist. Not sending ack";
+                }
+            }
+        }
+
+        fManager.DecrementMsgCounter();
+    }
 };
 
 }

fairmq/shmem/Monitor.cxx

@@ -10,17 +10,23 @@
 #include "Common.h"
 
 #include <fairmq/Tools.h>
+#include <fairlogger/Logger.h>
 
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/file_mapping.hpp>
 
 #include <boost/interprocess/sync/named_mutex.hpp>
+#include <boost/interprocess/sync/named_condition.hpp>
 #include <boost/interprocess/ipc/message_queue.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
 
 #include <csignal>
 #include <iostream>
 #include <iomanip>
+#include <chrono>
+#include <ctime>
+#include <time.h>
+#include <iomanip>
 
 #include <termios.h>
 #include <poll.h>
@@ -47,7 +53,7 @@ void signalHandler(int signal)
     gSignalStatus = signal;
 }
 
-Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, bool runAsDaemon, bool cleanOnExit)
+Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, unsigned int intervalInMS, bool runAsDaemon, bool cleanOnExit)
     : fSelfDestruct(selfDestruct)
     , fInteractive(interactive)
     , fViewOnly(viewOnly)
@@ -55,6 +61,7 @@ Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool
     , fSeenOnce(false)
     , fCleanOnExit(cleanOnExit)
     , fTimeoutInMS(timeoutInMS)
+    , fIntervalInMS(intervalInMS)
     , fShmId(shmId)
     , fSegmentName("fmq_" + fShmId + "_main")
     , fManagementSegmentName("fmq_" + fShmId + "_mng")
@@ -63,7 +70,6 @@ Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool
     , fHeartbeatTriggered(false)
     , fLastHeartbeat(chrono::high_resolution_clock::now())
     , fSignalThread()
-    , fManagementSegment(bipc::open_or_create, fManagementSegmentName.c_str(), 65536)
     , fDeviceHeartbeats()
 {
     if (!fViewOnly) {
@@ -74,6 +80,10 @@ Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool
             throw DaemonPresent(tools::ToString("fairmq-shmmonitor for shared memory id ", fShmId, " already started or not properly exited."));
         }
     }
+
+    Logger::SetConsoleColor(false);
+    Logger::DefineVerbosity(Verbosity::user1, VerbositySpec::Make(VerbositySpec::Info::timestamp_us));
+    Logger::SetVerbosity(Verbosity::verylow);
 }
 
 void Monitor::CatchSignals()
@@ -110,7 +120,7 @@ void Monitor::Run()
         Interactive();
     } else {
         while (!fTerminating) {
-            this_thread::sleep_for(chrono::milliseconds(100));
+            this_thread::sleep_for(chrono::milliseconds(fIntervalInMS));
             CheckSegment();
         }
     }
@@ -183,7 +193,7 @@ void Monitor::Interactive()
     PrintHeader();
 
     while (!fTerminating) {
-        if (poll(cinfd, 1, 100)) {
+        if (poll(cinfd, 1, fIntervalInMS)) {
             if (fTerminating || gSignalStatus != 0) {
                 break;
             }
@@ -202,7 +212,7 @@ void Monitor::Interactive()
                 case 'x':
                     cout << "\n[x] --> closing shared memory:" << endl;
                     if (!fViewOnly) {
-                        Cleanup(fShmId);
+                        Cleanup(ShmId{fShmId});
                     } else {
                         cout << "cannot close because in view only mode" << endl;
                     }
@@ -275,7 +285,7 @@ void Monitor::CheckSegment()
 
         unsigned int numDevices = 0;
 
-        if (fInteractive) {
+        if (fInteractive || fViewOnly) {
             DeviceCounter* dc = managementSegment.find<DeviceCounter>(bipc::unique_instance).first;
             if (dc) {
                 numDevices = dc->fCount;
@@ -287,7 +297,7 @@ void Monitor::CheckSegment()
 
         if (fHeartbeatTriggered && duration > fTimeoutInMS) {
             cout << "no heartbeats since over " << fTimeoutInMS << " milliseconds, cleaning..." << endl;
-            Cleanup(fShmId);
+            Cleanup(ShmId{fShmId});
             fHeartbeatTriggered = false;
             if (fSelfDestruct) {
                 cout << "\nself destructing" << endl;
@@ -303,6 +313,15 @@ void Monitor::CheckSegment()
                  << setw(8)  << numDevices                                      << " | "
                  << setw(10) << (fViewOnly ? "view only" : to_string(duration)) << " |"
                  << c << flush;
+        } else if (fViewOnly) {
+            size_t free = segment.get_free_memory();
+            size_t total = segment.get_size();
+            size_t used = total - free;
+            LOGV(info, user1) << "[" << fSegmentName
+                              << "] devices: " << numDevices
+                              << ", total: " << total
+                              << ", free: " << free
+                              << ", used: " << used;
         }
     } catch (bie&) {
         fHeartbeatTriggered = false;
@@ -320,7 +339,7 @@ void Monitor::CheckSegment()
         unsigned int duration = chrono::duration_cast<chrono::milliseconds>(now - fLastHeartbeat).count();
 
         if (fIsDaemon && duration > fTimeoutInMS * 2) {
-            Cleanup(fShmId);
+            Cleanup(ShmId{fShmId});
             fHeartbeatTriggered = false;
             if (fSelfDestruct) {
                 cout << "\nself destructing" << endl;
@@ -394,42 +413,52 @@ void Monitor::PrintHelp()
 void Monitor::RemoveObject(const string& name)
 {
     if (bipc::shared_memory_object::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
+        cout << "Successfully removed '" << name << "'." << endl;
     } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
+        cout << "Did not remove '" << name << "'. Already removed?" << endl;
     }
 }
 
 void Monitor::RemoveFileMapping(const string& name)
 {
     if (bipc::file_mapping::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
+        cout << "Successfully removed '" << name << "'." << endl;
     } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
+        cout << "Did not remove '" << name << "'. Already removed?" << endl;
     }
 }
 
 void Monitor::RemoveQueue(const string& name)
 {
     if (bipc::message_queue::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
+        cout << "Successfully removed '" << name << "'." << endl;
     } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
+        cout << "Did not remove '" << name << "'. Already removed?" << endl;
     }
 }
 
 void Monitor::RemoveMutex(const string& name)
 {
     if (bipc::named_mutex::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
+        cout << "Successfully removed '" << name << "'." << endl;
     } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
+        cout << "Did not remove '" << name << "'. Already removed?" << endl;
     }
 }
 
-void Monitor::Cleanup(const string& shmId)
+void Monitor::RemoveCondition(const string& name)
 {
-    string managementSegmentName("fmq_" + shmId + "_mng");
+    if (bipc::named_condition::remove(name.c_str())) {
+        cout << "Successfully removed '" << name << "'." << endl;
+    } else {
+        cout << "Did not remove '" << name << "'. Already removed?" << endl;
+    }
+}
+
+void Monitor::Cleanup(const ShmId& shmId)
+{
+    cout << "Cleaning up for shared memory id '" << shmId.shmId << "'..." << endl;
+    string managementSegmentName("fmq_" + shmId.shmId + "_mng");
     try {
         bipc::managed_shared_memory managementSegment(bipc::open_only, managementSegmentName.c_str());
         RegionCounter* rc = managementSegment.find<RegionCounter>(bipc::unique_instance).first;
@@ -444,42 +473,64 @@ void Monitor::Cleanup(const string& shmId)
                     RegionInfo ri = m->at(i);
                     string path = ri.fPath.c_str();
                     int flags = ri.fFlags;
-                    cout << "Found RegionInfo with path: '" << path << "', flags: " << flags << "'." << endl;
+                    cout << "Found RegionInfo with path: '" << path << "', flags: " << flags << ", fDestroyed: " << ri.fDestroyed << "." << endl;
                     if (path != "") {
-                        RemoveFileMapping(tools::ToString(path, "fmq_" + shmId + "_rg_" + to_string(i)));
+                        RemoveFileMapping(tools::ToString(path, "fmq_" + shmId.shmId + "_rg_" + to_string(i)));
                     } else {
-                        RemoveObject("fmq_" + shmId + "_rg_" + to_string(i));
+                        RemoveObject("fmq_" + shmId.shmId + "_rg_" + to_string(i));
                     }
                 } else {
-                    RemoveObject("fmq_" + shmId + "_rg_" + to_string(i));
+                    RemoveObject("fmq_" + shmId.shmId + "_rg_" + to_string(i));
                 }
 
-                RemoveQueue(string("fmq_" + shmId + "_rgq_" + to_string(i)));
+                RemoveQueue(string("fmq_" + shmId.shmId + "_rgq_" + to_string(i)));
             }
         } else {
-            cout << "No region counter found. no regions to cleanup." << endl;
+            cout << "No region counter found. No regions to cleanup." << endl;
         }
 
         RemoveObject(managementSegmentName.c_str());
     } catch (bie&) {
         cout << "Did not find '" << managementSegmentName << "' shared memory segment. No regions to cleanup." << endl;
-    } catch(std::out_of_range& oor) {
+    } catch(out_of_range& oor) {
         cout << "Could not locate element in the region map, out of range: " << oor.what() << endl;
     }
 
-    RemoveObject("fmq_" + shmId + "_main");
-    RemoveMutex("fmq_" + shmId + "_mtx");
+    RemoveObject("fmq_" + shmId.shmId + "_main");
+    RemoveMutex("fmq_" + shmId.shmId + "_mtx");
+    RemoveCondition("fmq_" + shmId.shmId + "_cv");
 
     cout << endl;
 }
 
+void Monitor::Cleanup(const SessionId& sessionId)
+{
+    ShmId shmId{buildShmIdFromSessionIdAndUserId(sessionId.sessionId)};
+    cout << "Cleanup called with session id '" << sessionId.sessionId << "', translating to shared memory id '" << shmId.shmId << "'" << endl;
+    Cleanup(shmId);
+}
+
+void Monitor::CleanupFull(const ShmId& shmId)
+{
+    Cleanup(shmId);
+    RemoveMutex("fmq_" + shmId.shmId + "_ms");
+    RemoveQueue("fmq_" + shmId.shmId + "_cq");
+}
+
+void Monitor::CleanupFull(const SessionId& sessionId)
+{
+    ShmId shmId{buildShmIdFromSessionIdAndUserId(sessionId.sessionId)};
+    cout << "Cleanup called with session id '" << sessionId.sessionId << "', translating to shared memory id '" << shmId.shmId << "'" << endl;
+    CleanupFull(shmId);
+}
+
 Monitor::~Monitor()
 {
     if (fSignalThread.joinable()) {
         fSignalThread.join();
     }
     if (fCleanOnExit) {
-        Cleanup(fShmId);
+        Cleanup(ShmId{fShmId});
     }
     if (!fViewOnly) {
         RemoveMutex("fmq_" + fShmId + "_ms");

fairmq/shmem/Monitor.h

@@ -8,8 +8,6 @@
 #ifndef FAIR_MQ_SHMEM_MONITOR_H_
 #define FAIR_MQ_SHMEM_MONITOR_H_
 
-#include <boost/interprocess/managed_shared_memory.hpp>
-
 #include <thread>
 #include <chrono>
 #include <atomic>
@@ -24,10 +22,22 @@ namespace mq
 namespace shmem
 {
 
+struct SessionId
+{
+    std::string sessionId;
+    explicit operator std::string() const { return sessionId; }
+};
+
+struct ShmId
+{
+    std::string shmId;
+    explicit operator std::string() const { return shmId; }
+};
+
 class Monitor
 {
   public:
-    Monitor(const std::string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, bool runAsDaemon, bool cleanOnExit);
+    Monitor(const std::string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, unsigned int intervalInMS, bool runAsDaemon, bool cleanOnExit);
 
     Monitor(const Monitor&) = delete;
     Monitor operator=(const Monitor&) = delete;
@@ -37,11 +47,24 @@ class Monitor
     void CatchSignals();
     void Run();
 
-    static void Cleanup(const std::string& shmId);
+    /// @brief Cleanup all shared memory artifacts created by devices
+    /// @param shmId shared memory id
+    static void Cleanup(const ShmId& shmId);
+    /// @brief Cleanup all shared memory artifacts created by devices
+    /// @param sessionId session id
+    static void Cleanup(const SessionId& sessionId);
+    /// @brief Cleanup all shared memory artifacts created by devices and monitors
+    /// @param shmId shared memory id
+    static void CleanupFull(const ShmId& shmId);
+    /// @brief Cleanup all shared memory artifacts created by devices and monitors
+    /// @param sessionId session id
+    static void CleanupFull(const SessionId& sessionId);
+
     static void RemoveObject(const std::string&);
     static void RemoveFileMapping(const std::string&);
     static void RemoveQueue(const std::string&);
     static void RemoveMutex(const std::string&);
+    static void RemoveCondition(const std::string&);
 
     struct DaemonPresent : std::runtime_error { using std::runtime_error::runtime_error; };
 
@@ -61,6 +84,7 @@ class Monitor
     bool fSeenOnce; // true is segment has been opened successfully at least once
     bool fCleanOnExit;
     unsigned int fTimeoutInMS;
+    unsigned int fIntervalInMS;
     std::string fShmId;
     std::string fSegmentName;
     std::string fManagementSegmentName;
@@ -69,7 +93,6 @@ class Monitor
     std::atomic<bool> fHeartbeatTriggered;
     std::chrono::high_resolution_clock::time_point fLastHeartbeat;
     std::thread fSignalThread;
-    boost::interprocess::managed_shared_memory fManagementSegment;
     std::unordered_map<std::string, std::chrono::high_resolution_clock::time_point> fDeviceHeartbeats;
 };
 

fairmq/shmem/Poller.cxx

@@ -1,222 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * Poller.cxx
- *
- * @since 2014-01-23
- * @author A. Rybalchenko
- */
-
-#include "Poller.h"
-#include "Socket.h"
-
-#include <FairMQLogger.h>
-
-#include <zmq.h>
-
-using namespace std;
-
-namespace fair
-{
-namespace mq
-{
-namespace shmem
-{
-
-Poller::Poller(const vector<FairMQChannel>& channels)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    fNumItems = channels.size();
-    fItems = new zmq_pollitem_t[fNumItems];
-
-    for (int i = 0; i < fNumItems; ++i)
-    {
-        fItems[i].socket = static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket();
-        fItems[i].fd = 0;
-        fItems[i].revents = 0;
-
-        int type = 0;
-        size_t size = sizeof(type);
-        zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
-
-        SetItemEvents(fItems[i], type);
-    }
-}
-
-Poller::Poller(const vector<FairMQChannel*>& channels)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    fNumItems = channels.size();
-    fItems = new zmq_pollitem_t[fNumItems];
-
-    for (int i = 0; i < fNumItems; ++i)
-    {
-        fItems[i].socket = static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket();
-        fItems[i].fd = 0;
-        fItems[i].revents = 0;
-
-        int type = 0;
-        size_t size = sizeof(type);
-        zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
-
-        SetItemEvents(fItems[i], type);
-    }
-}
-
-Poller::Poller(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    try
-    {
-        int offset = 0;
-        // calculate offsets and the total size of the poll item set
-        for (string channel : channelList)
-        {
-            fOffsetMap[channel] = offset;
-            offset += channelsMap.at(channel).size();
-            fNumItems += channelsMap.at(channel).size();
-        }
-
-        fItems = new zmq_pollitem_t[fNumItems];
-
-        int index = 0;
-        for (string channel : channelList)
-        {
-            for (unsigned int i = 0; i < channelsMap.at(channel).size(); ++i)
-            {
-                index = fOffsetMap[channel] + i;
-
-                fItems[index].socket = static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
-                fItems[index].fd = 0;
-                fItems[index].revents = 0;
-
-                int type = 0;
-                size_t size = sizeof(type);
-                zmq_getsockopt(static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
-
-                SetItemEvents(fItems[index], type);
-            }
-        }
-    }
-    catch (const out_of_range& oor)
-    {
-        LOG(error) << "at least one of the provided channel keys for poller initialization is invalid";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-void Poller::SetItemEvents(zmq_pollitem_t& item, const int type)
-{
-    if (type == ZMQ_REQ || type == ZMQ_REP || type == ZMQ_PAIR || type == ZMQ_DEALER || type == ZMQ_ROUTER)
-    {
-        item.events = ZMQ_POLLIN|ZMQ_POLLOUT;
-    }
-    else if (type == ZMQ_PUSH || type == ZMQ_PUB || type == ZMQ_XPUB)
-    {
-        item.events = ZMQ_POLLOUT;
-    }
-    else if (type == ZMQ_PULL || type == ZMQ_SUB || type == ZMQ_XSUB)
-    {
-        item.events = ZMQ_POLLIN;
-    }
-    else
-    {
-        LOG(error) << "invalid poller configuration, exiting.";
-        exit(EXIT_FAILURE);
-    }
-}
-
-void Poller::Poll(const int timeout)
-{
-    if (zmq_poll(fItems, fNumItems, timeout) < 0)
-    {
-        if (errno == ETERM)
-        {
-            LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
-        }
-        else
-        {
-            LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
-            throw runtime_error("polling failed");
-        }
-    }
-}
-
-bool Poller::CheckInput(const int index)
-{
-    if (fItems[index].revents & ZMQ_POLLIN)
-    {
-        return true;
-    }
-
-    return false;
-}
-
-bool Poller::CheckOutput(const int index)
-{
-    if (fItems[index].revents & ZMQ_POLLOUT)
-    {
-        return true;
-    }
-
-    return false;
-}
-
-bool Poller::CheckInput(const string& channelKey, const int index)
-{
-    try
-    {
-        if (fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLIN)
-        {
-            return true;
-        }
-
-        return false;
-    }
-    catch (const out_of_range& oor)
-    {
-        LOG(error) << "invalid channel key: \"" << channelKey << "\"";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-bool Poller::CheckOutput(const string& channelKey, const int index)
-{
-    try
-    {
-        if (fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLOUT)
-        {
-            return true;
-        }
-
-        return false;
-    }
-    catch (const out_of_range& oor)
-    {
-        LOG(error) << "Invalid channel key: \"" << channelKey << "\"";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-Poller::~Poller()
-{
-    delete[] fItems;
-}
-
-}
-}
-}

fairmq/shmem/Poller.h

@@ -8,42 +8,191 @@
 #ifndef FAIR_MQ_SHMEM_POLLER_H_
 #define FAIR_MQ_SHMEM_POLLER_H_
 
+#include "Socket.h"
+#include <fairmq/Tools.h>
+#include <FairMQChannel.h>
+#include <FairMQLogger.h>
+#include <FairMQPoller.h>
+
 #include <zmq.h>
 
-#include <FairMQPoller.h>
-#include <FairMQChannel.h>
-
-#include <vector>
 #include <unordered_map>
+#include <vector>
 
 class FairMQChannel;
 
-namespace fair
-{
-namespace mq
-{
-namespace shmem
-{
+namespace fair {
+namespace mq {
+namespace shmem {
 
 class Poller final : public fair::mq::Poller
 {
   public:
-    Poller(const std::vector<FairMQChannel>& channels);
-    Poller(const std::vector<FairMQChannel*>& channels);
-    Poller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList);
+    Poller(const std::vector<FairMQChannel>& channels)
+        : fItems()
+        , fNumItems(0)
+        , fOffsetMap()
+    {
+        fNumItems = channels.size();
+        fItems = new zmq_pollitem_t[fNumItems];
+
+        for (int i = 0; i < fNumItems; ++i) {
+            fItems[i].socket = static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket();
+            fItems[i].fd = 0;
+            fItems[i].revents = 0;
+
+            int type = 0;
+            size_t size = sizeof(type);
+            zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+
+            SetItemEvents(fItems[i], type);
+        }
+    }
+
+    Poller(const std::vector<FairMQChannel*>& channels)
+        : fItems()
+        , fNumItems(0)
+        , fOffsetMap()
+    {
+        fNumItems = channels.size();
+        fItems = new zmq_pollitem_t[fNumItems];
+
+        for (int i = 0; i < fNumItems; ++i) {
+            fItems[i].socket = static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket();
+            fItems[i].fd = 0;
+            fItems[i].revents = 0;
+
+            int type = 0;
+            size_t size = sizeof(type);
+            zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+
+            SetItemEvents(fItems[i], type);
+        }
+    }
+
+    Poller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList)
+        : fItems()
+        , fNumItems(0)
+        , fOffsetMap()
+    {
+        try {
+            int offset = 0;
+            // calculate offsets and the total size of the poll item set
+            for (std::string channel : channelList) {
+                fOffsetMap[channel] = offset;
+                offset += channelsMap.at(channel).size();
+                fNumItems += channelsMap.at(channel).size();
+            }
+
+            fItems = new zmq_pollitem_t[fNumItems];
+
+            int index = 0;
+            for (std::string channel : channelList) {
+                for (unsigned int i = 0; i < channelsMap.at(channel).size(); ++i) {
+                    index = fOffsetMap[channel] + i;
+
+                    fItems[index].socket = static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
+                    fItems[index].fd = 0;
+                    fItems[index].revents = 0;
+
+                    int type = 0;
+                    size_t size = sizeof(type);
+                    zmq_getsockopt(static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+
+                    SetItemEvents(fItems[index], type);
+                }
+            }
+        } catch (const std::out_of_range& oor) {
+            LOG(error) << "At least one of the provided channel keys for poller initialization is invalid." << " Out of range error: " << oor.what();
+            throw fair::mq::PollerError(fair::mq::tools::ToString("At least one of the provided channel keys for poller initialization is invalid. ", "Out of range error: ", oor.what()));
+        }
+    }
 
     Poller(const Poller&) = delete;
     Poller operator=(const Poller&) = delete;
 
-    void SetItemEvents(zmq_pollitem_t& item, const int type);
+    void SetItemEvents(zmq_pollitem_t& item, const int type)
+    {
+        if (type == ZMQ_REQ || type == ZMQ_REP || type == ZMQ_PAIR || type == ZMQ_DEALER || type == ZMQ_ROUTER) {
+            item.events = ZMQ_POLLIN | ZMQ_POLLOUT;
+        } else if (type == ZMQ_PUSH || type == ZMQ_PUB || type == ZMQ_XPUB) {
+            item.events = ZMQ_POLLOUT;
+        } else if (type == ZMQ_PULL || type == ZMQ_SUB || type == ZMQ_XSUB) {
+            item.events = ZMQ_POLLIN;
+        } else {
+            LOG(error) << "invalid poller configuration, exiting.";
+            throw fair::mq::PollerError("Invalid poller configuration, exiting.");
+        }
+    }
 
-    void Poll(const int timeout) override;
-    bool CheckInput(const int index) override;
-    bool CheckOutput(const int index) override;
-    bool CheckInput(const std::string& channelKey, const int index) override;
-    bool CheckOutput(const std::string& channelKey, const int index) override;
+    void Poll(const int timeout) override
+    {
+        while (true) {
+            if (zmq_poll(fItems, fNumItems, timeout) < 0) {
+                if (errno == ETERM) {
+                    LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
+                    return;
+                } else if (errno == EINTR) {
+                    LOG(debug) << "polling interrupted by system call";
+                    continue;
+                } else {
+                    LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
+                    throw fair::mq::PollerError(fair::mq::tools::ToString("Polling failed, reason: ", zmq_strerror(errno)));
+                }
+            }
+            break;
+        }
+    }
 
-    ~Poller() override;
+    bool CheckInput(const int index) override
+    {
+        if (fItems[index].revents & ZMQ_POLLIN) {
+            return true;
+        }
+
+        return false;
+    }
+
+    bool CheckOutput(const int index) override
+    {
+        if (fItems[index].revents & ZMQ_POLLOUT) {
+            return true;
+        }
+
+        return false;
+    }
+
+    bool CheckInput(const std::string& channelKey, const int index) override
+    {
+        try {
+            if (fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLIN) {
+                return true;
+            }
+
+            return false;
+        } catch (const std::out_of_range& oor) {
+            LOG(error) << "invalid channel key: '" << channelKey << "'";
+            LOG(error) << "out of range error: " << oor.what();
+            throw fair::mq::PollerError(fair::mq::tools::ToString("Invalid channel key '", channelKey, "'. Out of range error: ", oor.what()));
+        }
+    }
+
+    bool CheckOutput(const std::string& channelKey, const int index) override
+    {
+        try {
+            if (fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLOUT) {
+                return true;
+            }
+
+            return false;
+        } catch (const std::out_of_range& oor) {
+            LOG(error) << "invalid channel key: '" << channelKey << "'";
+            LOG(error) << "out of range error: " << oor.what();
+            throw fair::mq::PollerError(fair::mq::tools::ToString("Invalid channel key '", channelKey, "'. Out of range error: ", oor.what()));
+        }
+    }
+
+    ~Poller() override { delete[] fItems; }
 
   private:
     zmq_pollitem_t* fItems;
@@ -52,8 +201,8 @@ class Poller final : public fair::mq::Poller
     std::unordered_map<std::string, int> fOffsetMap;
 };
 
-}
-}
-}
+}   // namespace shmem
+}   // namespace mq
+}   // namespace fair
 
 #endif /* FAIR_MQ_SHMEM_POLLER_H_ */

fairmq/shmem/README.md

@@ -6,6 +6,23 @@ The transport manages shared memory via boost::interprocess library. The transfe
 
 Devices track and cleanup shared memory on shutdown. For more information on the current shared memory segment and additional cleanup options, see following section.
 
+# Shared Memory objects / files
+
+FairMQ Shared Memory currently uses the following names to register shared memory on the system:
+
+| name                      | info                                           | created by         | used by                        |
+| ------------------------- | ---------------------------------------------- | ------------------ | ------------------------------ |
+| `fmq_<shmId>_main`        | main segment (user data)                       | one of the devices | devices                        |
+| `fmq_<shmId>_mng`         | management segment (management data)           | one of the devices | devices                        |
+| `fmq_<shmId>_mtx`         | mutex                                          | one of the devices | devices                        |
+| `fmq_<shmId>_cv`          | condition variable                             | one of the devices | devices with unmanaged regions |
+| `fmq_<shmId>_rg_<index>`  | unmanaged region(s)                            | one of the devices | devices with unmanaged regions |
+| `fmq_<shmId>_rgq_<index>` | unmanaged region queue(s)                      | one of the devices | devices with unmanaged regions |
+| `fmq_<shmId>_ms`          | shmmonitor status                              | shmmonitor         | devices, shmmonitor            |
+| `fmq_<shmId>_cq`          | message queue between transport and shmmonitor | shmmonitor         | devices, shmmonitor            |
+
+The shmId is generated out of session id and user id.
+
 ## Shared memory monitor
 
 The shared memory monitor tool, supplied with the shared memory transport can be used to monitor shared memory use and automatically cleanup shared memory in case of device crashes.
@@ -25,13 +42,3 @@ Without the `--self-destruct` option, the monitor will run continuously, moitori
 Possible further implementation would be to run the monitor with `--self-destruct` with each topology.
 
 The Monitor class can also be used independently from the supplied executable (built from `runMonitor.cxx`), allowing integration on any level. For example invoking the monitor could be a functionality that a device offers.
-
-FairMQ Shared Memory currently uses following names to register shared memory on the system:
-
-`fmq_<shmId>_main` - main segment name, used for user data (the shmId is generated out of session id and user id).
-`fmq_<shmId>_mng` - management segment name, used for storing management data.
-`fmq_<shmId>_cq` - message queue for communicating between shm transport and shm monitor (exists independent of above segments).
-`fmq_<shmId>_mtx` - boost::interprocess::named_mutex for management purposes (exists independent of above segments).
-`fmq_<shmId>_ms` - shmmonitor status used to signal if it is active or not (exists independent of above segments).
-`fmq_<shmId>_rg_<index>` - names of unmanaged regions.
-`fmq_<shmId>_rgq_<index>` - names of queues for the unmanaged regions.

fairmq/shmem/Region.cxx

@@ -1,200 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "Region.h"
-#include "Common.h"
-#include "Manager.h"
-
-#include <fairmq/tools/CppSTL.h>
-#include <fairmq/tools/Strings.h>
-
-#include <boost/filesystem.hpp>
-#include <boost/process.hpp>
-#include <boost/date_time/posix_time/posix_time.hpp>
-
-#include <cerrno>
-#include <chrono>
-
-using namespace std;
-
-namespace bipc = ::boost::interprocess;
-namespace bpt = ::boost::posix_time;
-
-namespace fair
-{
-namespace mq
-{
-namespace shmem
-{
-
-Region::Region(Manager& manager, uint64_t id, uint64_t size, bool remote, RegionCallback callback, const string& path /* = "" */, int flags /* = 0 */)
-    : fManager(manager)
-    , fRemote(remote)
-    , fStop(false)
-    , fName("fmq_" + fManager.fShmId + "_rg_" + to_string(id))
-    , fQueueName("fmq_" + fManager.fShmId + "_rgq_" + to_string(id))
-    , fShmemObject()
-    , fFile(nullptr)
-    , fFileMapping()
-    , fQueue(nullptr)
-    , fReceiveAcksWorker()
-    , fSendAcksWorker()
-    , fCallback(callback)
-{
-    if (path != "") {
-        fName = string(path + fName);
-
-        fFile = fopen(fName.c_str(), fRemote ? "r+" : "w+");
-
-        if (!fFile) {
-            LOG(error) << "Failed to initialize file: " << fName;
-            LOG(error) << "errno: " << errno << ": " << strerror(errno);
-            throw runtime_error(tools::ToString("Failed to initialize file for shared memory region: ", strerror(errno)));
-        }
-        fFileMapping = bipc::file_mapping(fName.c_str(), bipc::read_write);
-        LOG(debug) << "shmem: initialized file: " << fName;
-        fRegion = bipc::mapped_region(fFileMapping, bipc::read_write, 0, size, 0, flags);
-    } else {
-        if (fRemote) {
-            fShmemObject = bipc::shared_memory_object(bipc::open_only, fName.c_str(), bipc::read_write);
-        } else {
-            fShmemObject = bipc::shared_memory_object(bipc::create_only, fName.c_str(), bipc::read_write);
-            fShmemObject.truncate(size);
-        }
-        fRegion = bipc::mapped_region(fShmemObject, bipc::read_write, 0, 0, 0, flags);
-    }
-
-    InitializeQueues();
-    LOG(debug) << "shmem: initialized region: " << fName;
-    fSendAcksWorker = thread(&Region::SendAcks, this);
-}
-
-void Region::InitializeQueues()
-{
-    if (fRemote) {
-        fQueue = tools::make_unique<bipc::message_queue>(bipc::open_only, fQueueName.c_str());
-    } else {
-        fQueue = tools::make_unique<bipc::message_queue>(bipc::create_only, fQueueName.c_str(), 1024, fAckBunchSize * sizeof(RegionBlock));
-    }
-    LOG(debug) << "shmem: initialized region queue: " << fQueueName;
-}
-
-void Region::StartReceivingAcks()
-{
-    fReceiveAcksWorker = thread(&Region::ReceiveAcks, this);
-}
-
-void Region::ReceiveAcks()
-{
-    unsigned int priority;
-    bipc::message_queue::size_type recvdSize;
-    unique_ptr<RegionBlock[]> blocks = tools::make_unique<RegionBlock[]>(fAckBunchSize);
-
-    while (!fStop) { // end thread condition (should exist until region is destroyed)
-        auto rcvTill = bpt::microsec_clock::universal_time() + bpt::milliseconds(500);
-
-        while (fQueue->timed_receive(blocks.get(), fAckBunchSize * sizeof(RegionBlock), recvdSize, priority, rcvTill)) {
-            // LOG(debug) << "received: " << block.fHandle << " " << block.fSize << " " << block.fMessageId;
-            if (fCallback) {
-                const auto numBlocks = recvdSize / sizeof(RegionBlock);
-                for (size_t i = 0; i < numBlocks; i++) {
-                    fCallback(reinterpret_cast<char*>(fRegion.get_address()) + blocks[i].fHandle, blocks[i].fSize, reinterpret_cast<void*>(blocks[i].fHint));
-                }
-            }
-        }
-    } // while !fStop
-
-    LOG(debug) << "receive ack worker for " << fName << " leaving.";
-}
-
-void Region::ReleaseBlock(const RegionBlock &block)
-{
-    unique_lock<mutex> lock(fBlockLock);
-
-    fBlocksToFree.emplace_back(block);
-
-    if (fBlocksToFree.size() >= fAckBunchSize) {
-        lock.unlock(); // reduces contention on fBlockLock
-        fBlockSendCV.notify_one();
-    }
-}
-
-void Region::SendAcks()
-{
-    unique_ptr<RegionBlock[]> blocks = tools::make_unique<RegionBlock[]>(fAckBunchSize);
-
-    while (true) { // we'll try to send all acks before stopping
-        size_t blocksToSend = 0;
-
-        {   // mutex locking block
-            unique_lock<mutex> lock(fBlockLock);
-
-            // try to get more blocks without waiting (we can miss a notify from CloseMessage())
-            if (!fStop && (fBlocksToFree.size() < fAckBunchSize)) {
-                // cv.wait() timeout: send whatever blocks we have
-                fBlockSendCV.wait_for(lock, chrono::milliseconds(500));
-            }
-
-            blocksToSend = min(fBlocksToFree.size(), fAckBunchSize);
-
-            copy_n(fBlocksToFree.end() - blocksToSend, blocksToSend, blocks.get());
-            fBlocksToFree.resize(fBlocksToFree.size() - blocksToSend);
-        } // unlock the block mutex here while sending over IPC
-
-        if (blocksToSend > 0) {
-            while (!fQueue->try_send(blocks.get(), blocksToSend * sizeof(RegionBlock), 0) && !fStop) {
-                // receiver slow? yield and try again...
-                this_thread::yield();
-            }
-        } else { // blocksToSend == 0
-            if (fStop) {
-                break;
-            }
-        }
-    }
-
-    LOG(debug) << "send ack worker for " << fName << " leaving.";
-}
-
-Region::~Region()
-{
-    fStop = true;
-
-    if (fSendAcksWorker.joinable()) {
-        fSendAcksWorker.join();
-    }
-
-    if (!fRemote) {
-        if (fReceiveAcksWorker.joinable()) {
-            fReceiveAcksWorker.join();
-        }
-
-        if (bipc::shared_memory_object::remove(fName.c_str())) {
-            LOG(debug) << "shmem: destroyed region " << fName;
-        }
-
-        if (bipc::file_mapping::remove(fName.c_str())) {
-            LOG(debug) << "shmem: destroyed file mapping " << fName;
-        }
-
-        if (fFile) {
-            fclose(fFile);
-        }
-
-        if (bipc::message_queue::remove(fQueueName.c_str())) {
-            LOG(debug) << "shmem: removed region queue " << fQueueName;
-        }
-    } else {
-        // LOG(debug) << "shmem: region '" << fName << "' is remote, no cleanup necessary.";
-        LOG(debug) << "shmem: region queue '" << fQueueName << "' is remote, no cleanup necessary";
-    }
-}
-
-} // namespace shmem
-} // namespace mq
-} // namespace fair

fairmq/shmem/Region.h

@@ -19,15 +19,25 @@
 
 #include <FairMQLogger.h>
 #include <FairMQUnmanagedRegion.h>
+#include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Strings.h>
 
+#include <boost/filesystem.hpp>
+#include <boost/process.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp>
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/file_mapping.hpp>
 #include <boost/interprocess/ipc/message_queue.hpp>
 
+#include <algorithm> // min
+#include <atomic>
 #include <thread>
 #include <mutex>
 #include <condition_variable>
 #include <unordered_map>
+#include <cerrno>
+#include <chrono>
+#include <ios>
 
 namespace fair
 {
@@ -36,30 +46,217 @@ namespace mq
 namespace shmem
 {
 
-class Manager;
-
 struct Region
 {
-    Region(Manager& manager, uint64_t id, uint64_t size, bool remote, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0);
+    Region(const std::string& shmId, uint64_t id, uint64_t size, bool remote, RegionCallback callback, RegionBulkCallback bulkCallback, const std::string& path, int flags)
+        : fRemote(remote)
+        , fLinger(100)
+        , fStop(false)
+        , fName("fmq_" + shmId + "_rg_" + std::to_string(id))
+        , fQueueName("fmq_" + shmId + "_rgq_" + std::to_string(id))
+        , fShmemObject()
+        , fFile(nullptr)
+        , fFileMapping()
+        , fQueue(nullptr)
+        , fAcksReceiver()
+        , fAcksSender()
+        , fCallback(callback)
+        , fBulkCallback(bulkCallback)
+    {
+        using namespace boost::interprocess;
+
+        if (path != "") {
+            fName = std::string(path + fName);
+
+            if (!fRemote) {
+                // create a file
+                std::filebuf fbuf;
+                if (fbuf.open(fName, std::ios_base::in | std::ios_base::out | std::ios_base::trunc | std::ios_base::binary)) {
+                    // set the size
+                    fbuf.pubseekoff(size - 1, std::ios_base::beg);
+                    fbuf.sputc(0);
+                }
+            }
+
+            fFile = fopen(fName.c_str(), "r+");
+
+            if (!fFile) {
+                LOG(error) << "Failed to initialize file: " << fName;
+                LOG(error) << "errno: " << errno << ": " << strerror(errno);
+                throw std::runtime_error(tools::ToString("Failed to initialize file for shared memory region: ", strerror(errno)));
+            }
+            fFileMapping = file_mapping(fName.c_str(), read_write);
+            LOG(debug) << "shmem: initialized file: " << fName;
+            fRegion = mapped_region(fFileMapping, read_write, 0, size, 0, flags);
+        } else {
+            if (fRemote) {
+                fShmemObject = shared_memory_object(open_only, fName.c_str(), read_write);
+            } else {
+                fShmemObject = shared_memory_object(create_only, fName.c_str(), read_write);
+                fShmemObject.truncate(size);
+            }
+            fRegion = mapped_region(fShmemObject, read_write, 0, 0, 0, flags);
+        }
+
+        InitializeQueues();
+        StartSendingAcks();
+        LOG(debug) << "shmem: initialized region: " << fName;
+    }
 
     Region() = delete;
 
     Region(const Region&) = delete;
     Region(Region&&) = delete;
 
-    void InitializeQueues();
+    void InitializeQueues()
+    {
+        using namespace boost::interprocess;
 
-    void StartReceivingAcks();
-    void ReceiveAcks();
+        if (fRemote) {
+            fQueue = tools::make_unique<message_queue>(open_only, fQueueName.c_str());
+        } else {
+            fQueue = tools::make_unique<message_queue>(create_only, fQueueName.c_str(), 1024, fAckBunchSize * sizeof(RegionBlock));
+        }
+        LOG(debug) << "shmem: initialized region queue: " << fQueueName;
+    }
 
-    void ReleaseBlock(const RegionBlock &);
-    void SendAcks();
+    void StartSendingAcks() { fAcksSender = std::thread(&Region::SendAcks, this); }
+    void SendAcks()
+    {
+        std::unique_ptr<RegionBlock[]> blocks = tools::make_unique<RegionBlock[]>(fAckBunchSize);
+        size_t blocksToSend = 0;
 
-    ~Region();
+        while (true) {
+            blocksToSend = 0;
+            {
+                std::unique_lock<std::mutex> lock(fBlockMtx);
+
+                // try to get <fAckBunchSize> blocks
+                if (fBlocksToFree.size() < fAckBunchSize) {
+                    fBlockSendCV.wait_for(lock, std::chrono::milliseconds(500));
+                }
+
+                // send whatever blocks we have
+                blocksToSend = std::min(fBlocksToFree.size(), fAckBunchSize);
+
+                copy_n(fBlocksToFree.end() - blocksToSend, blocksToSend, blocks.get());
+                fBlocksToFree.resize(fBlocksToFree.size() - blocksToSend);
+            }
+
+            if (blocksToSend > 0) {
+                while (!fQueue->try_send(blocks.get(), blocksToSend * sizeof(RegionBlock), 0) && !fStop) {
+                    // receiver slow? yield and try again...
+                    std::this_thread::yield();
+                }
+                // LOG(debug) << "Sent " << blocksToSend << " blocks.";
+            } else { // blocksToSend == 0
+                if (fStop) {
+                    break;
+                }
+            }
+        }
+
+        LOG(debug) << "AcksSender for " << fName << " leaving " << "(blocks left to free: " << fBlocksToFree.size() << ", "
+                                                                << " blocks left to send: " << blocksToSend << ").";
+    }
+
+    void StartReceivingAcks() { fAcksReceiver = std::thread(&Region::ReceiveAcks, this); }
+    void ReceiveAcks()
+    {
+        unsigned int priority;
+        boost::interprocess::message_queue::size_type recvdSize;
+        std::unique_ptr<RegionBlock[]> blocks = tools::make_unique<RegionBlock[]>(fAckBunchSize);
+        std::vector<fair::mq::RegionBlock> result;
+        result.reserve(fAckBunchSize);
+
+        while (true) {
+            uint32_t timeout = 100;
+            bool leave = false;
+            if (fStop) {
+                timeout = fLinger;
+                leave = true;
+            }
+            auto rcvTill = boost::posix_time::microsec_clock::universal_time() + boost::posix_time::milliseconds(timeout);
+
+            while (fQueue->timed_receive(blocks.get(), fAckBunchSize * sizeof(RegionBlock), recvdSize, priority, rcvTill)) {
+                const auto numBlocks = recvdSize / sizeof(RegionBlock);
+                // LOG(debug) << "Received " << numBlocks << " blocks (recvdSize: " << recvdSize << "). (remaining queue size: " << fQueue->get_num_msg() << ").";
+                if (fBulkCallback) {
+                    result.clear();
+                    for (size_t i = 0; i < numBlocks; i++) {
+                        result.emplace_back(reinterpret_cast<char*>(fRegion.get_address()) + blocks[i].fHandle, blocks[i].fSize, reinterpret_cast<void*>(blocks[i].fHint));
+                    }
+                    fBulkCallback(result);
+                } else if (fCallback) {
+                    for (size_t i = 0; i < numBlocks; i++) {
+                        fCallback(reinterpret_cast<char*>(fRegion.get_address()) + blocks[i].fHandle, blocks[i].fSize, reinterpret_cast<void*>(blocks[i].fHint));
+                    }
+                }
+            }
+
+            if (leave) {
+                break;
+            }
+        }
+
+        LOG(debug) << "AcksReceiver for " << fName << " leaving (remaining queue size: " << fQueue->get_num_msg() << ").";
+    }
+
+    void ReleaseBlock(const RegionBlock& block)
+    {
+        std::unique_lock<std::mutex> lock(fBlockMtx);
+
+        fBlocksToFree.emplace_back(block);
+
+        if (fBlocksToFree.size() >= fAckBunchSize) {
+            lock.unlock();
+            fBlockSendCV.notify_one();
+        }
+    }
+
+    void SetLinger(uint32_t linger) { fLinger = linger; }
+    uint32_t GetLinger() const { return fLinger; }
+
+    ~Region()
+    {
+        fStop = true;
+
+        if (fAcksSender.joinable()) {
+            fBlockSendCV.notify_one();
+            fAcksSender.join();
+        }
+
+        if (!fRemote) {
+            if (fAcksReceiver.joinable()) {
+                fAcksReceiver.join();
+            }
+
+            if (boost::interprocess::shared_memory_object::remove(fName.c_str())) {
+                LOG(debug) << "Region '" << fName << "' destroyed.";
+            }
+
+            if (boost::interprocess::file_mapping::remove(fName.c_str())) {
+                LOG(debug) << "File mapping '" << fName << "' destroyed.";
+            }
+
+            if (fFile) {
+                fclose(fFile);
+            }
+
+            if (boost::interprocess::message_queue::remove(fQueueName.c_str())) {
+                LOG(debug) << "Region queue '" << fQueueName << "' destroyed.";
+            }
+        } else {
+            // LOG(debug) << "shmem: region '" << fName << "' is remote, no cleanup necessary.";
+            LOG(debug) << "Region queue '" << fQueueName << "' is remote, no cleanup necessary";
+        }
+
+        LOG(debug) << "Region '" << fName << "' (" << (fRemote ? "remote" : "local") << ") destructed.";
+    }
 
-    Manager& fManager;
     bool fRemote;
-    bool fStop;
+    uint32_t fLinger;
+    std::atomic<bool> fStop;
     std::string fName;
     std::string fQueueName;
     boost::interprocess::shared_memory_object fShmemObject;
@@ -67,15 +264,16 @@ struct Region
     boost::interprocess::file_mapping fFileMapping;
     boost::interprocess::mapped_region fRegion;
 
-    std::mutex fBlockLock;
+    std::mutex fBlockMtx;
     std::condition_variable fBlockSendCV;
     std::vector<RegionBlock> fBlocksToFree;
     const std::size_t fAckBunchSize = 256;
     std::unique_ptr<boost::interprocess::message_queue> fQueue;
 
-    std::thread fReceiveAcksWorker;
-    std::thread fSendAcksWorker;
+    std::thread fAcksReceiver;
+    std::thread fAcksSender;
     RegionCallback fCallback;
+    RegionBulkCallback fBulkCallback;
 };
 
 } // namespace shmem