find_package2: Fix list sub-command REMOVE_DUPLICATES requires list to be present

Fixes #175

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,6 @@
 Describe your proposal.
 
-Mention any issue this PR is resolves or is related to.
+Mention any issue this PR resolves or is related to.
 
 ---
 

CMakeLists.txt

@@ -6,8 +6,8 @@
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
-cmake_minimum_required(VERSION 3.9.4 FATAL_ERROR)
-cmake_policy(VERSION 3.9...3.15)
+cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
+cmake_policy(VERSION 3.10...3.14)
 
 # Project ######################################################################
 set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
@@ -18,6 +18,12 @@ get_git_version()
 project(FairMQ VERSION ${PROJECT_VERSION} LANGUAGES CXX)
 message(STATUS "${BWhite}${PROJECT_NAME}${CR} ${PROJECT_GIT_VERSION} from ${PROJECT_DATE}")
 
+if(BUILD_OFI_TRANSPORT)
+  set(PROJECT_MIN_CXX_STANDARD 14)
+else()
+  set(PROJECT_MIN_CXX_STANDARD 11)
+endif()
+
 set_fairmq_defaults()
 
 include(CTest)
@@ -31,6 +37,7 @@ cmake_dependent_option(BUILD_TESTING "Build tests." OFF "BUILD_FAIRMQ" OFF)
 cmake_dependent_option(BUILD_NANOMSG_TRANSPORT "Build nanomsg transport." OFF "BUILD_FAIRMQ" OFF)
 cmake_dependent_option(BUILD_OFI_TRANSPORT "Build experimental OFI transport." OFF "BUILD_FAIRMQ" OFF)
 cmake_dependent_option(BUILD_DDS_PLUGIN "Build DDS plugin." OFF "BUILD_FAIRMQ" OFF)
+cmake_dependent_option(BUILD_PMIX_PLUGIN "Build PMIx plugin." OFF "BUILD_FAIRMQ" OFF)
 cmake_dependent_option(BUILD_EXAMPLES "Build FairMQ examples." ON "BUILD_FAIRMQ" OFF)
 option(BUILD_DOCS "Build FairMQ documentation." OFF)
 option(FAST_BUILD "Fast production build. Not recommended for development." OFF)
@@ -46,39 +53,83 @@ set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
 set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_package(Threads REQUIRED)
 
-if(BUILD_FAIRMQ)
-  find_package2(PUBLIC Boost VERSION 1.64 REQUIRED
-    COMPONENTS container program_options thread system filesystem regex date_time
-  )
-  # Normalize Boost version
-  if(CMAKE_VERSION VERSION_LESS 3.15)
-    set(Boost_VERSION "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}")
-  endif()
-  find_package2(PUBLIC FairLogger VERSION 1.2.0 REQUIRED)
-  find_package2(PRIVATE ZeroMQ VERSION 4.1.5 REQUIRED)
-endif()
-
 if(BUILD_NANOMSG_TRANSPORT)
   find_package2(PRIVATE nanomsg REQUIRED)
   set(PROJECT_nanomsg_VERSION 1.1.3) # Once upstream releases 1.1.5, we should bump again and use version check
-  find_package2(PRIVATE msgpack VERSION 3.1.0 REQUIRED)
 endif()
 
 if(BUILD_OFI_TRANSPORT)
-  find_package2(PRIVATE OFI VERSION 1.6.0 REQUIRED COMPONENTS fi_sockets fi_verbs)
-  find_package2(PRIVATE Protobuf VERSION 3.4.0 REQUIRED)
+  find_package2(PRIVATE asiofi REQUIRED
+    VERSION 0.3.1
+  )
+  find_package2(PRIVATE OFI REQUIRED
+    ADD_REQUIREMENTS_OF asiofi
+  )
+endif()
+
+if(BUILD_NANOMSG_TRANSPORT)
+  find_package2(PRIVATE msgpack REQUIRED
+    VERSION 3.1.0
+  )
 endif()
 
 if(BUILD_DDS_PLUGIN)
-  find_package2(PRIVATE DDS VERSION 2.2 REQUIRED)
+  find_package2(PRIVATE DDS REQUIRED
+    VERSION 2.4
+  )
+  set(DDS_Boost_COMPONENTS system log log_setup)
+  set(DDS_Boost_VERSION 1.67)
+endif()
+
+if(BUILD_PMIX_PLUGIN)
+  find_package2(PRIVATE PMIx REQUIRED
+    VERSION 2.1.4
+  )
+endif()
+
+if(BUILD_FAIRMQ)
+  find_package2(PUBLIC FairLogger REQUIRED
+    VERSION 1.2.0
+  )
+
+  if(NOT DEFINED Boost_NO_BOOST_CMAKE AND CMAKE_VERSION VERSION_LESS 3.15)
+    # Since Boost 1.70 a CMake package is shipped by default. Unfortunately, it has a number
+    # of problems that are only fixed in Boost 1.71 or CMake 3.15. By default we skip the
+    # BoostConfig lookup. This can be overridden on the command line via -DBoost_NO_BOOST_CMAKE=OFF
+    set(Boost_NO_BOOST_CMAKE ON)
+  endif()
+  find_package2(PUBLIC Boost REQUIRED
+    VERSION 1.64
+
+    COMPONENTS
+    container
+    program_options
+    filesystem
+    date_time
+    regex
+
+    ADD_REQUIREMENTS_OF
+    asiofi
+    DDS
+    FairLogger
+  )
+  find_package2(PRIVATE ZeroMQ REQUIRED
+    VERSION 4.1.5
+  )
 endif()
 
 if(BUILD_TESTING)
-  find_package2(PRIVATE GTest VERSION 1.7.0 REQUIRED)
+  find_package2(PRIVATE GTest REQUIRED
+    VERSION 1.7.0
+  )
 endif()
 
 if(BUILD_DOCS)
-  find_package2(PRIVATE Doxygen VERSION 1.8.8 REQUIRED COMPONENTS dot OPTIONAL_COMPONENTS mscgen dia)
+  find_package2(PRIVATE Doxygen REQUIRED
+    VERSION 1.8.8
+    COMPONENTS dot
+    OPTIONAL_COMPONENTS mscgen dia
+  )
 endif()
 ################################################################################
 
@@ -104,7 +155,7 @@ endif()
 if(BUILD_DOCS)
   set(DOXYGEN_OUTPUT_DIRECTORY doxygen)
   set(DOXYGEN_PROJECT_NUMBER ${PROJECT_GIT_VERSION})
-  set(DOXYGEN_PROJECT_BRIEF "C++ Message Passing Framework")
+  set(DOXYGEN_PROJECT_BRIEF "C++ Message Queuing Library and Framework")
   set(DOXYGEN_USE_MDFILE_AS_MAINPAGE README.md)
   set(DOXYGEN_HTML_FOOTER docs/footer.html)
   doxygen_add_docs(doxygen README.md fairmq)
@@ -123,6 +174,9 @@ endif()
 if(BUILD_DDS_PLUGIN)
   list(APPEND PROJECT_PACKAGE_COMPONENTS dds_plugin)
 endif()
+if(BUILD_PMIX_PLUGIN)
+  list(APPEND PROJECT_PACKAGE_COMPONENTS pmix_plugin)
+endif()
 if(BUILD_NANOMSG_TRANSPORT)
   list(APPEND PROJECT_PACKAGE_COMPONENTS nanomsg_transport)
 endif()
@@ -148,16 +202,6 @@ if(BUILD_FAIRMQ)
     DESTINATION ${PROJECT_INSTALL_CMAKEMODDIR}
   )
 endif()
-if(BUILD_DDS_PLUGIN)
-  install(FILES cmake/FindDDS.cmake
-    DESTINATION ${PROJECT_INSTALL_CMAKEMODDIR}
-  )
-endif()
-if(BUILD_OFI_TRANSPORT)
-  install(FILES cmake/FindOFI.cmake
-    DESTINATION ${PROJECT_INSTALL_CMAKEMODDIR}
-  )
-endif()
 if(BUILD_DOCS)
   install(DIRECTORY ${CMAKE_BINARY_DIR}/doxygen/html
     DESTINATION ${PROJECT_INSTALL_DATADIR}/docs
@@ -169,6 +213,8 @@ install_cmake_package()
 
 
 # Summary ######################################################################
+message(STATUS "  ")
+message(STATUS "  ${Cyan}CXX STANDARD${CR}       ${BGreen}C++${CMAKE_CXX_STANDARD}${CR} (>= C++${PROJECT_MIN_CXX_STANDARD}, change with ${BMagenta}-DCMAKE_CXX_STANDARD=17${CR})")
 if(CMAKE_CXX_FLAGS)
   message(STATUS "  ")
   message(STATUS "  ${Cyan}GLOBAL CXX FLAGS${CR}  ${BGreen}${CMAKE_CXX_FLAGS}${CR}")
@@ -198,7 +244,11 @@ if(PROJECT_PACKAGE_DEPENDENCIES)
   foreach(dep IN LISTS PROJECT_PACKAGE_DEPENDENCIES)
     if(${dep}_VERSION)
       if(${dep} STREQUAL Boost)
-        set(version_str "${BGreen}${${dep}_MAJOR_VERSION}.${${dep}_MINOR_VERSION}${CR}")
+        if(Boost_VERSION_MAJOR)
+          set(version_str "${BGreen}${${dep}_VERSION_MAJOR}.${${dep}_VERSION_MINOR}${CR}")
+        else()
+          set(version_str "${BGreen}${${dep}_MAJOR_VERSION}.${${dep}_MINOR_VERSION}${CR}")
+        endif()
       else()
         set(version_str "${BGreen}${${dep}_VERSION}${CR}")
       endif()
@@ -216,7 +266,7 @@ if(PROJECT_PACKAGE_DEPENDENCIES)
     endif()
     if(${dep} STREQUAL FairLogger)
       if(FairLogger_PREFIX)
-       set(prefix ${FairLogger_PREFIX})
+        set(prefix ${FairLogger_PREFIX})
       else()
         set(prefix ${FairLogger_ROOT})
       endif()
@@ -225,11 +275,23 @@ if(PROJECT_PACKAGE_DEPENDENCIES)
     elseif(${dep} STREQUAL msgpack)
       get_target_property(msgpack_include msgpackc-cxx INTERFACE_INCLUDE_DIRECTORIES)
       get_filename_component(prefix ${msgpack_include}/.. ABSOLUTE)
+    elseif(${dep} STREQUAL asiofi)
+      set(prefix ${asiofi_ROOT})
     elseif(${dep} STREQUAL OFI)
       get_filename_component(prefix ${${dep}_INCLUDE_DIRS}/.. ABSOLUTE)
     elseif(${dep} STREQUAL nanomsg)
       get_target_property(nn_include nanomsg INTERFACE_INCLUDE_DIRECTORIES)
       get_filename_component(prefix ${nn_include}/.. ABSOLUTE)
+    elseif(${dep} STREQUAL DDS)
+      get_target_property(dds_include DDS::dds_intercom_lib INTERFACE_INCLUDE_DIRECTORIES)
+      get_filename_component(prefix ${dds_include}/.. ABSOLUTE)
+    elseif(${dep} STREQUAL Boost)
+      if(TARGET Boost::headers)
+        get_target_property(boost_include Boost::headers INTERFACE_INCLUDE_DIRECTORIES)
+      else()
+        get_target_property(boost_include Boost::boost INTERFACE_INCLUDE_DIRECTORIES)
+      endif()
+      get_filename_component(prefix ${boost_include}/.. ABSOLUTE)
     elseif(${dep} STREQUAL Doxygen)
       get_target_property(doxygen_bin Doxygen::doxygen INTERFACE_LOCATION)
       get_filename_component(prefix ${doxygen_bin} DIRECTORY)
@@ -264,9 +326,9 @@ else()
 endif()
 message(STATUS "  ${BWhite}nanomsg_transport${CR}  ${nn_summary}")
 if(BUILD_OFI_TRANSPORT)
-  set(ofi_summary "${BGreen}YES${CR}    EXPERIMENTAL (disable with ${BMagenta}-DBUILD_OFI_TRANSPORT=OFF${CR})")
+  set(ofi_summary "${BGreen}YES${CR}    EXPERIMENTAL (requires C++14) (disable with ${BMagenta}-DBUILD_OFI_TRANSPORT=OFF${CR})")
 else()
-  set(ofi_summary "${BRed} NO${CR}    EXPERIMENTAL (default, enable with ${BMagenta}-DBUILD_OFI_TRANSPORT=ON${CR})")
+  set(ofi_summary "${BRed} NO${CR}    EXPERIMENTAL (requires C++14) (default, enable with ${BMagenta}-DBUILD_OFI_TRANSPORT=ON${CR})")
 endif()
 message(STATUS "  ${BWhite}ofi_transport${CR}      ${ofi_summary}")
 if(BUILD_DDS_PLUGIN)
@@ -275,6 +337,12 @@ else()
   set(dds_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_DDS_PLUGIN=ON${CR})")
 endif()
 message(STATUS "  ${BWhite}dds_plugin${CR}         ${dds_summary}")
+if(BUILD_PMIX_PLUGIN)
+  set(pmix_summary "${BGreen}YES${CR}    (disable with ${BMagenta}-DBUILD_PMIX_PLUGIN=OFF${CR})")
+else()
+  set(pmix_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_PMIX_PLUGIN=ON${CR})")
+endif()
+message(STATUS "  ${BWhite}pmix_plugin${CR}        ${pmix_summary}")
 if(BUILD_EXAMPLES)
   set(examples_summary "${BGreen}YES${CR}    (default, disable with ${BMagenta}-DBUILD_EXAMPLES=OFF${CR})")
 else()
@@ -288,4 +356,26 @@ else()
 endif()
 message(STATUS "  ${BWhite}docs${CR}               ${docs_summary}")
 message(STATUS "  ")
+if(RUN_STATIC_ANALYSIS)
+  list(LENGTH PROJECT_STATIC_ANALYSERS size)
+  unset(analyser_list)
+  set(count 0)
+  foreach(analyser IN LISTS PROJECT_STATIC_ANALYSERS)
+    if(${analyser}_FOUND)
+      set(${analyser}_status "${analyser} ${BGreen}YES${CR}")
+    else()
+      set(${analyser}_status "${analyser} ${BRed}NO${CR}")
+    endif()
+    math(EXPR count "${count} + 1")
+    string(APPEND analyser_list "${${analyser}_status}")
+    if(count LESS size)
+      string(APPEND analyser_list "${BWhite},${CR} ")
+    endif()
+  endforeach()
+  set(static_ana_summary "${BWhite}(${CR}${analyser_list}${BWhite})${CR}  (disable with ${BMagenta}-DRUN_STATIC_ANALYSIS=OFF${CR})")
+else()
+  set(static_ana_summary "${BRed}OFF${CR}  (default, enable with ${BMagenta}-DRUN_STATIC_ANALYSIS=ON${CR})")
+endif()
+message(STATUS "  ${Cyan}RUN STATIC ANALYSIS  ${static_ana_summary}")
+message(STATUS "  ")
 ################################################################################

FairMQTest.cmake

@@ -27,6 +27,7 @@ Set(configure_options "${configure_options};-DCTEST_USE_LAUNCHERS=${CTEST_USE_LA
 Set(configure_options "${configure_options};-DDISABLE_COLOR=ON")
 Set(configure_options "${configure_options};-DCMAKE_PREFIX_PATH=$ENV{SIMPATH}")
 Set(configure_options "${configure_options};-DBUILD_NANOMSG_TRANSPORT=ON")
+# Set(configure_options "${configure_options};-DBUILD_OFI_TRANSPORT=ON")
 Set(configure_options "${configure_options};-DBUILD_DDS_PLUGIN=ON")
 Set(configure_options "${configure_options};-DFAST_BUILD=ON")
 Set(configure_options "${configure_options};-DCOTIRE_MAXIMUM_NUMBER_OF_UNITY_INCLUDES=-j$ENV{number_of_processors}")
@@ -57,7 +58,8 @@ Ctest_Configure(BUILD "${CTEST_BINARY_DIRECTORY}"
 Ctest_Build(BUILD "${CTEST_BINARY_DIRECTORY}")
 
 Ctest_Test(BUILD "${CTEST_BINARY_DIRECTORY}" 
-           PARALLEL_LEVEL $ENV{number_of_processors}
+           # PARALLEL_LEVEL $ENV{number_of_processors}
+           PARALLEL_LEVEL 1
            RETURN_VALUE _ctest_test_ret_val
           )
 If("$ENV{do_codecov_upload}")

Jenkinsfile

@@ -66,14 +66,15 @@ pipeline{
       steps{
         script {
           def build_jobs = jobMatrix('alfa-ci/build', [
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',          fairsoft: 'fairmq_dev'],
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
             [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
+            [os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             sh './Dart.sh alfa_ci Dart.cfg'
           }
 
           def profile_jobs = jobMatrix('alfa-ci/codecov', [
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',          fairsoft: 'fairmq_dev'],
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             withCredentials([string(credentialsId: 'fairmq_codecov_token', variable: 'CODECOV_TOKEN')]) {
               sh './Dart.sh codecov Dart.cfg'

Jenkinsfile.nightly

@@ -63,8 +63,9 @@ pipeline{
       steps{
         script {
           parallel(buildMatrix([
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',         fairsoft: 'fairmq_dev'],
-            [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0',  fairsoft: 'fairmq_dev'],
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
+            [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
+            [os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             sh './Dart.sh Nightly Dart.cfg'
             sh './Dart.sh Profile Dart.cfg'

README.md

@@ -38,7 +38,7 @@ a simulation, reconstruction and analysis framework.
   * PUBLIC: [**Boost**](https://www.boost.org/), [**FairLogger**](https://github.com/FairRootGroup/FairLogger)
   * BUILD: [CMake](https://cmake.org/), [GTest](https://github.com/google/googletest), [Doxygen](http://www.doxygen.org/)
   * PRIVATE: [ZeroMQ](http://zeromq.org/), [Msgpack](https://msgpack.org/index.html), [nanomsg](http://nanomsg.org/),
-[OFI](https://ofiwg.github.io/libfabric/), [Protobuf](https://developers.google.com/protocol-buffers/), [DDS](http://dds.gsi.de)
+[asiofi](https://github.com/FairRootGroup/asiofi), [DDS](http://dds.gsi.de), [PMIx](https://pmix.org/)
 
   Supported platforms: Linux and MacOS.
 
@@ -51,7 +51,7 @@ cmake -DCMAKE_INSTALL_PREFIX=./fairmq_install ../fairmq
 cmake --build . --target install
 ```
 
-If dependencies are not installed in standard system directories, you can hint the installation location via `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...`. `{DEPENDENCY}` can be `GTEST`, `BOOST`, `FAIRLOGGER`, `ZEROMQ`, `MSGPACK`, `NANOMSG`, `OFI`, `PROTOBUF`, or `DDS` (`*_ROOT` variables can also be environment variables).
+If dependencies are not installed in standard system directories, you can hint the installation location via `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...`. `{DEPENDENCY}` can be `GTEST`, `BOOST`, `FAIRLOGGER`, `ZEROMQ`, `MSGPACK`, `NANOMSG`, `OFI`, `PMIX`, `ASIOFI` or `DDS` (`*_ROOT` variables can also be environment variables).
 
 ## Usage
 
@@ -104,6 +104,7 @@ On command line:
   * `-DBUILD_NANOMSG_TRANSPORT=ON` enables building of nanomsg transport.
   * `-DBUILD_OFI_TRANSPORT=ON` enables building of the experimental OFI transport.
   * `-DBUILD_DDS_PLUGIN=ON` enables building of the DDS plugin.
+  * `-DBUILD_PMIX_PLUGIN=ON` enables building of the PMIx plugin.
   * `-DBUILD_DOCS=ON` enables building of API docs.
   * You can hint non-system installations for dependent packages, see the #Installation section above
 
@@ -119,15 +120,17 @@ After the `find_package(FairMQ)` call the following CMake variables are defined:
 | `${FairMQ_#COMPONENT#_FOUND}` | `TRUE` if this component was built |
 | `${FairMQ_VERSION}` | the version in format `MAJOR.MINOR.PATCH` |
 | `${FairMQ_GIT_VERSION}` | the version in the format returned by `git describe --tags --dirty --match "v*"` |
-| `${FairMQ_ROOT}` | the actual installation prefix, notice the difference to the hint variable `FAIRMQ_ROOT` |
+| `${FairMQ_PREFIX}` | the actual installation prefix |
 | `${FairMQ_BINDIR}` | the installation bin directory |
 | `${FairMQ_INCDIR}` | the installation include directory |
 | `${FairMQ_LIBDIR}` | the installation lib directory |
 | `${FairMQ_DATADIR}` | the installation data directory (`../share/fairmq`) |
 | `${FairMQ_CMAKEMODDIR}` | the installation directory of shipped CMake find modules |
-| `${FairMQ_CXX_STANDARD_REQUIRED}` | the value of `CMAKE_CXX_STANDARD_REQUIRED` at built-time |
-| `${FairMQ_CXX_STANDARD}` | the value of `CMAKE_CXX_STANDARD` at built-time |
-| `${FairMQ_CXX_EXTENSIONS}` | the values of `CMAKE_CXX_EXTENSIONS` at built-time |
+| `${FairMQ_CXX_STANDARD_REQUIRED}` | the value of `CMAKE_CXX_STANDARD_REQUIRED` at build-time |
+| `${FairMQ_CXX_STANDARD}` | the value of `CMAKE_CXX_STANDARD` at build-time |
+| `${FairMQ_CXX_EXTENSIONS}` | the values of `CMAKE_CXX_EXTENSIONS` at build-time |
+| `${FairMQ_BUILD_TYPE}` | the value of `CMAKE_BUILD_TYPE` at build-time |
+| `${FairMQ_CXX_FLAGS}` | the values of `CMAKE_CXX_FLAGS` and `CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}` at build-time |
 
 ## Documentation
 
@@ -156,3 +159,10 @@ After the `find_package(FairMQ)` call the following CMake variables are defined:
    4. [File output](docs/Logging.md#54-file-output)
    5. [Custom sinks](docs/Logging.md#55-custom-sinks)
 6. [Examples](docs/Examples.md#6-examples)
+7. [Plugins](docs/Plugins.md#7-plugins)
+   1. [Usage](docs/Plugins.md#71-usage)
+   2. [Development](docs/Plugins.md#72-development)
+   3. [Provided Plugins](docs/Plugins.md#73-provided-plugins)
+       1. [DDS](docs/Plugins.md#731-dds)
+       2. [PMIx](docs/Plugins.md#732-pmix)
+

cmake/FairMQConfig.cmake.in

@@ -13,7 +13,7 @@ set(@PROJECT_NAME@_VERSION @PROJECT_VERSION@)
 set(@PROJECT_NAME@_GIT_VERSION @PROJECT_GIT_VERSION@)
 set(@PROJECT_NAME@_GIT_DATE @PROJECT_GIT_DATE@)
 
-set_and_check(@PROJECT_NAME@_ROOT @PACKAGE_CMAKE_INSTALL_PREFIX@)
+set_and_check(@PROJECT_NAME@_PREFIX @PACKAGE_CMAKE_INSTALL_PREFIX@)
 set_and_check(@PROJECT_NAME@_BINDIR @PACKAGE_CMAKE_INSTALL_PREFIX@/@PROJECT_INSTALL_BINDIR@)
 set_and_check(@PROJECT_NAME@_INCDIR @PACKAGE_CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@)
 set_and_check(@PROJECT_NAME@_LIBDIR @PACKAGE_CMAKE_INSTALL_PREFIX@/@PROJECT_INSTALL_LIBDIR@)
@@ -24,6 +24,9 @@ set(@PROJECT_NAME@_CXX_STANDARD_REQUIRED @CMAKE_CXX_STANDARD_REQUIRED@)
 set(@PROJECT_NAME@_CXX_STANDARD @CMAKE_CXX_STANDARD@)
 set(@PROJECT_NAME@_CXX_EXTENSIONS @CMAKE_CXX_EXTENSIONS@)
 set(@PROJECT_NAME@_VERSION_HOTFIX @PROJECT_VERSION_HOTFIX@)
+set(@PROJECT_NAME@_BUILD_TYPE @CMAKE_BUILD_TYPE@)
+set(@PROJECT_NAME@_BUILD_TYPE_UPPER @PROJECT_BUILD_TYPE_UPPER@)
+set(@PROJECT_NAME@_CXX_FLAGS @PROJECT_CXX_FLAGS@)
 
 ### Import cmake modules
 set(CMAKE_MODULE_PATH ${@PROJECT_NAME@_CMAKEMODDIR} ${CMAKE_MODULE_PATH})

cmake/FairMQLib.cmake

@@ -96,11 +96,9 @@ macro(set_fairmq_defaults)
   # Handle C++ standard level
   set(CMAKE_CXX_STANDARD_REQUIRED ON)
   if(NOT CMAKE_CXX_STANDARD)
-    set(CMAKE_CXX_STANDARD 11)
-  elseif(${CMAKE_CXX_STANDARD} LESS 11)
-    message(FATAL_ERROR "A minimum CMAKE_CXX_STANDARD of 11 is required.")
-  elseif(${CMAKE_CXX_STANDARD} GREATER 11)
-    message(WARNING "A CMAKE_CXX_STANDARD of ${CMAKE_CXX_STANDARD} (greater than 11) is not tested. Use on your own risk.")
+    set(CMAKE_CXX_STANDARD ${PROJECT_MIN_CXX_STANDARD})
+  elseif(${CMAKE_CXX_STANDARD} LESS ${PROJECT_MIN_CXX_STANDARD})
+    message(FATAL_ERROR "A minimum CMAKE_CXX_STANDARD of ${PROJECT_MIN_CXX_STANDARD} is required.")
   endif()
   set(CMAKE_CXX_EXTENSIONS OFF)
 
@@ -168,6 +166,34 @@ macro(set_fairmq_defaults)
   else()
     set(PROJECT_VERSION_HOTFIX ${PROJECT_VERSION_TWEAK})
   endif()
+
+  if(NOT DEFINED RUN_STATIC_ANALYSIS)
+    set(RUN_STATIC_ANALYSIS OFF)
+  endif()
+
+  unset(PROJECT_STATIC_ANALYSERS)
+  if(RUN_STATIC_ANALYSIS)
+    set(analyser "clang-tidy")
+    find_program(${analyser}_FOUND "${analyser}")
+    if(${analyser}_FOUND)
+      set(CMAKE_CXX_CLANG_TIDY "${${analyser}_FOUND}" "-color")
+    endif()
+    list(APPEND PROJECT_STATIC_ANALYSERS "${analyser}")
+
+    set(analyser "iwyu")
+    find_program(${analyser}_FOUND "${analyser}")
+    if(${analyser}_FOUND)
+      set(CMAKE_CXX_IWYU "${${analyser}_FOUND}")
+    endif()
+    list(APPEND PROJECT_STATIC_ANALYSERS "${analyser}")
+
+    set(analyser "cpplint")
+    find_program(${analyser}_FOUND "${analyser}")
+    if(${analyser}_FOUND)
+      set(CMAKE_CXX_CPPLINT "${${analyser}_FOUND}")
+    endif()
+    list(APPEND PROJECT_STATIC_ANALYSERS "${analyser}")
+  endif()
 endmacro()
 
 function(join VALUES GLUE OUTPUT)
@@ -242,6 +268,8 @@ endfunction()
 
 # Configure/Install CMake package
 macro(install_cmake_package)
+  list(SORT PROJECT_PACKAGE_DEPENDENCIES)
+  list(SORT PROJECT_INTERFACE_PACKAGE_DEPENDENCIES)
   include(CMakePackageConfigHelpers)
   set(PACKAGE_INSTALL_DESTINATION
     ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}-${PROJECT_GIT_VERSION}
@@ -260,6 +288,8 @@ macro(install_cmake_package)
   )
   generate_package_dependencies() # fills ${PACKAGE_DEPENDENCIES}
   generate_package_components() # fills ${PACKAGE_COMPONENTS}
+  string(TOUPPER ${CMAKE_BUILD_TYPE} PROJECT_BUILD_TYPE_UPPER)
+  set(PROJECT_CXX_FLAGS ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${PROJECT_BUILD_TYPE_UPPER}})
   configure_package_config_file(
     ${CMAKE_SOURCE_DIR}/cmake/${PROJECT_NAME}Config.cmake.in
     ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake
@@ -273,36 +303,97 @@ macro(install_cmake_package)
   )
 endmacro()
 
+#
+# find_package2(PRIVATE|PUBLIC|INTERFACE <pkgname>
+#               [VERSION <version>]
+#               [COMPONENTS <list of components>]
+#               [ADD_REQUIREMENTS_OF <list of dep_pgkname>]
+#               [any other option the native find_package supports]...)
+#
+# Wrapper around CMake's native find_package command to add some features and bookkeeping.
+#
+# The qualifier (PRIVATE|PUBLIC|INTERFACE) to the package to populate
+# the variables PROJECT_[INTERFACE]_<pkgname>_([VERSION]|[COMPONENTS]|PACKAGE_DEPENDENCIES)
+# accordingly. This bookkeeping information is used to print our dependency found summary
+# table and to generate a part of our CMake package.
+#
+# When a dependending package is listed with ADD_REQUIREMENTS_OF the variables
+# <dep_pkgname>_<pkgname>_VERSION|COMPONENTS are looked up to and added to the native
+# VERSION (selected highest version) and COMPONENTS (deduplicated) args.
+#
+# COMPONENTS and VERSION args are then just passed to the native find_package.
+#
 macro(find_package2 qualifier pkgname)
-  cmake_parse_arguments(ARGS "" "VERSION" "COMPONENTS" ${ARGN})
+  cmake_parse_arguments(ARGS "" "VERSION" "COMPONENTS;ADD_REQUIREMENTS_OF" ${ARGN})
 
   string(TOUPPER ${pkgname} pkgname_upper)
-  set(old_CPP ${CMAKE_PREFIX_PATH})
+  set(__old_cpp__ ${CMAKE_PREFIX_PATH})
   set(CMAKE_PREFIX_PATH ${${pkgname_upper}_ROOT} $ENV{${pkgname_upper}_ROOT} ${CMAKE_PREFIX_PATH})
-  if(ARGS_COMPONENTS)
-    find_package(${pkgname} ${ARGS_VERSION} QUIET COMPONENTS ${ARGS_COMPONENTS} ${ARGS_UNPARSED_ARGUMENTS})
-  else()
-    find_package(${pkgname} ${ARGS_VERSION} QUIET ${ARGS_UNPARSED_ARGUMENTS})
+
+  # build lists of required versions and components
+  unset(__required_versions__)
+  unset(__components__)
+  if(ARGS_VERSION)
+    list(APPEND __required_versions__ ${ARGS_VERSION})
+  endif()
+  if(ARGS_COMPONENTS)
+    list(APPEND __components__ ${ARGS_COMPONENTS})
+  endif()
+  if(ARGS_ADD_REQUIREMENTS_OF)
+    foreach(dep_pkgname IN LISTS ARGS_ADD_REQUIREMENTS_OF)
+      if(${dep_pkgname}_${pkgname}_VERSION)
+        list(APPEND __required_versions__ ${${dep_pkgname}_${pkgname}_VERSION})
+      endif()
+      if(${dep_pkgname}_${pkgname}_COMPONENTS)
+        list(APPEND __components__ ${${dep_pkgname}_${pkgname}_COMPONENTS})
+      endif()
+    endforeach()
+  endif()
+
+  # select highest required version
+  unset(__version__)
+  if(__required_versions__)
+    list(GET __required_versions__ 0 __version__)
+    foreach(v IN LISTS __required_versions__)
+      if(${v} VERSION_GREATER ${__version__})
+        set(__version__ ${v})
+      endif()
+    endforeach()
+  endif()
+  # deduplicate required component list
+  if(__components__)
+    list(REMOVE_DUPLICATES __components__)
+  endif()
+
+  # call native find_package
+  if(__components__)
+    find_package(${pkgname} ${__version__} QUIET COMPONENTS ${__components__} ${ARGS_UNPARSED_ARGUMENTS})
+  else()
+    find_package(${pkgname} ${__version__} QUIET ${ARGS_UNPARSED_ARGUMENTS})
   endif()
-  set(CMAKE_PREFIX_PATH ${old_CPP})
-  unset(old_CPP)
 
   if(${pkgname}_FOUND)
     if(${qualifier} STREQUAL PRIVATE)
-      set(PROJECT_${pkgname}_VERSION ${ARGS_VERSION})
+      set(PROJECT_${pkgname}_VERSION ${__version__})
       set(PROJECT_${pkgname}_COMPONENTS ${ARGS_COMPONENTS})
       set(PROJECT_PACKAGE_DEPENDENCIES ${PROJECT_PACKAGE_DEPENDENCIES} ${pkgname})
     elseif(${qualifier} STREQUAL PUBLIC)
-      set(PROJECT_${pkgname}_VERSION ${ARGS_VERSION})
+      set(PROJECT_${pkgname}_VERSION ${__version__})
       set(PROJECT_${pkgname}_COMPONENTS ${ARGS_COMPONENTS})
       set(PROJECT_PACKAGE_DEPENDENCIES ${PROJECT_PACKAGE_DEPENDENCIES} ${pkgname})
-      set(PROJECT_INTERFACE_${pkgname}_VERSION ${ARGS_VERSION})
+      set(PROJECT_INTERFACE_${pkgname}_VERSION ${__version__})
       set(PROJECT_INTERFACE_${pkgname}_COMPONENTS ${ARGS_COMPONENTS})
       set(PROJECT_INTERFACE_PACKAGE_DEPENDENCIES ${PROJECT_INTERFACE_PACKAGE_DEPENDENCIES} ${pkgname})
     elseif(${qualifier} STREQUAL INTERFACE)
-      set(PROJECT_INTERFACE_${pkgname}_VERSION ${ARGS_VERSION})
+      set(PROJECT_INTERFACE_${pkgname}_VERSION ${__version__})
       set(PROJECT_INTERFACE_${pkgname}_COMPONENTS ${ARGS_COMPONENTS})
       set(PROJECT_INTERFACE_PACKAGE_DEPENDENCIES ${PROJECT_INTERFACE_PACKAGE_DEPENDENCIES} ${pkgname})
     endif()
   endif()
+
+  unset(__version__)
+  unset(__components__)
+  unset(__required_versions__)
+  set(CMAKE_PREFIX_PATH ${__old_cpp__})
+  unset(__old_cpp__)
 endmacro()

cmake/FindDDS.cmake

@@ -1,95 +0,0 @@
-################################################################################
-# Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
-#                                                                              #
-#              This software is distributed under the terms of the             #
-#              GNU Lesser General Public Licence (LGPL) version 3,             #
-#                  copied verbatim in the file "LICENSE"                       #
-################################################################################
-
-find_path(DDS_INCLUDE_DIR
-  NAMES dds_intercom.h
-  HINTS ${DDS_ROOT} $ENV{DDS_ROOT}
-  PATH_SUFFIXES include
-)
-if(NOT DDS_INCLUDE_DIR)
-find_path(DDS_INCLUDE_DIR
-  NAMES dds_intercom.h
-  HINTS ${DDS_ROOT} $ENV{DDS_ROOT}
-  PATH_SUFFIXES include/DDS
-)
-endif()
-
-find_path(DDS_LIBRARY_DIR
-  NAMES libdds_intercom_lib.dylib libdds_intercom_lib.so
-  HINTS ${DDS_ROOT} $ENV{DDS_ROOT}
-  PATH_SUFFIXES lib
-)
-
-find_library(DDS_INTERCOM_LIBRARY_SHARED
-  NAMES libdds_intercom_lib.dylib libdds_intercom_lib.so
-  HINTS ${DDS_ROOT} $ENV{DDS_ROOT}
-  PATH_SUFFIXES lib
-  DOC "Path to libdds_intercom_lib.dylib libdds_intercom_lib.so."
-)
-
-find_library(DDS_PROTOCOL_LIBRARY_SHARED
-  NAMES libdds_protocol_lib.dylib libdds_protocol_lib.so
-  HINTS ${DDS_ROOT} $ENV{DDS_ROOT}
-  PATH_SUFFIXES lib
-  DOC "Path to libdds_protocol_lib.dylib libdds_protocol_lib.so."
-)
-
-find_library(DDS_USER_DEFAULTS_LIBRARY_SHARED
-  NAMES libdds-user-defaults.dylib libdds-user-defaults.so
-  HINTS ${DDS_ROOT} $ENV{DDS_ROOT}
-  PATH_SUFFIXES lib
-  DOC "Path to libdds-user-defaults.dylib libdds-user-defaults.so."
-)
-
-find_file(DDS_VERSION_FILE
-  NAMES version
-  HINTS ${DDS_ROOT} $ENV{DDS_ROOT}
-  PATH_SUFFIXES etc
-)
-
-if(DDS_VERSION_FILE AND NOT DDS_VERSION)
-  file(READ ${DDS_VERSION_FILE} DDS_VERSION)
-  string(STRIP "${DDS_VERSION}" DDS_VERSION)
-  set(DDS_VERSION ${DDS_VERSION} CACHE string "DDS version.")
-endif()
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(DDS
-  REQUIRED_VARS
-  DDS_INCLUDE_DIR
-  DDS_LIBRARY_DIR
-  DDS_INTERCOM_LIBRARY_SHARED
-  DDS_PROTOCOL_LIBRARY_SHARED
-  DDS_USER_DEFAULTS_LIBRARY_SHARED
-
-  VERSION_VAR DDS_VERSION
-)
-
-if(NOT TARGET DDS::dds_intercom_lib AND DDS_FOUND)
-  add_library(DDS::dds_intercom_lib SHARED IMPORTED)
-  set_target_properties(DDS::dds_intercom_lib PROPERTIES
-    IMPORTED_LOCATION ${DDS_INTERCOM_LIBRARY_SHARED}
-    INTERFACE_INCLUDE_DIRECTORIES ${DDS_INCLUDE_DIR}
-  )
-endif()
-
-if(NOT TARGET DDS::dds_protocol_lib AND DDS_FOUND)
-  add_library(DDS::dds_protocol_lib SHARED IMPORTED)
-  set_target_properties(DDS::dds_protocol_lib PROPERTIES
-    IMPORTED_LOCATION ${DDS_PROTOCOL_LIBRARY_SHARED}
-    INTERFACE_INCLUDE_DIRECTORIES ${DDS_INCLUDE_DIR}
-  )
-endif()
-
-if(NOT TARGET DDS::dds-user-defaults AND DDS_FOUND)
-  add_library(DDS::dds-user-defaults SHARED IMPORTED)
-  set_target_properties(DDS::dds-user-defaults PROPERTIES
-    IMPORTED_LOCATION ${DDS_USER_DEFAULTS_LIBRARY_SHARED}
-    INTERFACE_INCLUDE_DIRECTORIES ${DDS_INCLUDE_DIR}
-  )
-endif()

cmake/FindFlatBuffers.cmake

@@ -1,67 +0,0 @@
-################################################################################
-# Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
-#                                                                              #
-#              This software is distributed under the terms of the             #
-#              GNU Lesser General Public Licence (LGPL) version 3,             #
-#                  copied verbatim in the file "LICENSE"                       #
-################################################################################
-
-find_path(FLATBUFFERS_INCLUDE_DIR
-  NAMES flatbuffers/flatbuffers.h
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES include
-)
-
-find_path(FLATBUFFERS_LIBRARY_DIR
-  NAMES libflatbuffers.a
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES lib
-)
-
-find_library(FLATBUFFERS_STATIC_LIBRARY
-  NAMES libflatbuffers.a
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES lib
-)
-
-find_path(FLATBUFFERS_BINARY_DIR
-  NAMES flatc
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES bin
-)
-
-find_program(FLATBUFFERS_BINARY_FLATC
-  NAMES flatc
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES bin
-)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(FlatBuffers
-  REQUIRED_VARS
-  FLATBUFFERS_INCLUDE_DIR
-  FLATBUFFERS_LIBRARY_DIR
-  FLATBUFFERS_BINARY_DIR
-)
-
-# idempotently import targets
-if(NOT TARGET FlatBuffers)
-  if(FLATBUFFERS_FOUND)
-    # import target
-    add_library(FlatBuffers STATIC IMPORTED)
-    set_target_properties(FlatBuffers PROPERTIES
-      INTERFACE_INCLUDE_DIRECTORIES ${FLATBUFFERS_INCLUDE_DIR}
-      IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIBRARY}
-    )
-  endif()
-endif()
-
-if(NOT TARGET FlatBuffers::flatc)
-  if(FLATBUFFERS_FOUND)
-  # import target
-  add_executable(FlatBuffers::flatc IMPORTED)
-  set_target_properties(FlatBuffers::flatc PROPERTIES
-    IMPORTED_LOCATION ${FLATBUFFERS_BINARY_FLATC}
-  )
-  endif()
-endif()

cmake/FindOFI.cmake

@@ -1,89 +0,0 @@
-################################################################################
-#    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
-#                                                                              #
-#              This software is distributed under the terms of the             #
-#              GNU Lesser General Public Licence (LGPL) version 3,             #
-#                  copied verbatim in the file "LICENSE"                       #
-################################################################################
-
-# According to the docs the modification of the PKG_CONFIG_PATH environment should
-# not be necessary, but it does not work otherwise.
-if(OFI_ROOT)
-  list(APPEND CMAKE_PREFIX_PATH "${OFI_ROOT}/lib/pkgconfig")
-  set(ENV{PKG_CONFIG_PATH} "${OFI_ROOT}/lib/pkgconfig:" $ENV{PKG_CONFIG_PATH})
-endif()
-
-if(ENV{OFI_ROOT})
-  list(APPEND CMAKE_PREFIX_PATH "$ENV{OFI_ROOT}/lib/pkgconfig")
-  set(ENV{PKG_CONFIG_PATH} "$ENV{OFI_ROOT}/lib/pkgconfig:" $ENV{PKG_CONFIG_PATH})
-endif()
-
-# This should be the default as of CMake 3.1, but it is not set. BUG? Also, it does not work
-set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH 1)
-find_package(PkgConfig QUIET)
-
-if(PKG_CONFIG_FOUND)
-  # Find include dir and dependencies from pkgconfig
-  pkg_check_modules(_OFI libfabric QUIET)
-
-  # Retrieve version from pkgconfig
-  execute_process(
-    COMMAND ${PKG_CONFIG_EXECUTABLE} libfabric --modversion
-    OUTPUT_VARIABLE OFI_VERSION
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
-
-  # The IMPORTED_TARGET option of the pkg_check_modules() function is useless,
-  # so let's build it ourselves
-  find_library(OFI_LIBFABRIC
-    NAMES libfabric.so libfabric.dylib
-    HINTS ${OFI_ROOT} $ENV{OFI_ROOT}
-    PATH_SUFFIXES lib
-  )
-
-  # Just take the include dirs found by the PkgConfig module
-  set(OFI_INCLUDE_DIRS ${_OFI_INCLUDE_DIRS})
-
-  # Find fi_info command to be able to check required features of the OFI installation
-  find_program(OFI_INFO_EXECUTABLE
-    NAMES fi_info
-    HINTS ${OFI_ROOT} $ENV{OFI_ROOT}
-    PATH_SUFFIXES bin
-  )
-
-  # Detect ofi providers, they can be required via the COMPONENTS argument of find_package
-  if(OFI_INFO_EXECUTABLE)
-    execute_process(
-      COMMAND ${OFI_INFO_EXECUTABLE} -l
-      OUTPUT_VARIABLE output
-    )
-    string(REPLACE "\n" ";" lines ${output})
-    foreach(line IN LISTS lines)
-      string(REGEX
-        MATCH "^([a-zA-Z0-9_]+):"
-        found "${line}"
-      )
-      if(found)
-        string(TOLOWER "${CMAKE_MATCH_1}" provider)
-        set(OFI_fi_${provider}_FOUND TRUE)
-      endif()
-    endforeach()
-  endif()
-
-  # Check search result, check version constraints and print status
-  include(FindPackageHandleStandardArgs)
-  find_package_handle_standard_args(OFI
-    REQUIRED_VARS OFI_LIBFABRIC OFI_INCLUDE_DIRS OFI_INFO_EXECUTABLE
-    VERSION_VAR OFI_VERSION
-    HANDLE_COMPONENTS
-  )
-endif()
-
-if(NOT TARGET OFI::libfabric AND OFI_FOUND)
-  # Define an imported target
-  add_library(OFI::libfabric SHARED IMPORTED GLOBAL)
-  set_target_properties(OFI::libfabric PROPERTIES
-    IMPORTED_LOCATION ${OFI_LIBFABRIC}
-    INTERFACE_INCLUDE_DIRECTORIES ${OFI_INCLUDE_DIRS}
-  )
-endif()

cmake/FindPMIx.cmake

@@ -0,0 +1,67 @@
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+find_path(PMIx_INCLUDE_DIR
+  NAMES pmix.h
+  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
+  PATH_SUFFIXES include
+)
+
+find_path(PMIx_LIBRARY_DIR
+  NAMES libpmix.dylib libpmix.so
+  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
+  PATH_SUFFIXES lib lib64
+)
+
+find_library(PMIx_LIBRARY_SHARED
+  NAMES libpmix.dylib libpmix.so
+  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
+  PATH_SUFFIXES lib lib64
+)
+
+find_file(PMIx_VERSION_FILE
+  NAMES pmix_version.h
+  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
+  PATH_SUFFIXES include
+)
+
+file(READ "${PMIx_VERSION_FILE}" __version_raw)
+string(REGEX MATCH "#define PMIX_VERSION_MAJOR ([0-9]?)L?"
+  __version_major_raw "${__version_raw}"
+)
+set(PMIx_VERSION_MAJOR "${CMAKE_MATCH_1}")
+
+string(REGEX MATCH "#define PMIX_VERSION_MINOR ([0-9]?)L?"
+  __version_minor_raw "${__version_raw}"
+)
+set(PMIx_VERSION_MINOR "${CMAKE_MATCH_1}")
+
+string(REGEX MATCH "#define PMIX_VERSION_RELEASE ([0-9]?)L?"
+  __version_patch_raw "${__version_raw}"
+)
+set(PMIx_VERSION_PATCH "${CMAKE_MATCH_1}")
+
+set(PMIx_VERSION "${PMIx_VERSION_MAJOR}.${PMIx_VERSION_MINOR}.${PMIx_VERSION_PATCH}")
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(PMIx
+  REQUIRED_VARS
+  PMIx_INCLUDE_DIR
+  PMIx_LIBRARY_DIR
+  PMIx_LIBRARY_SHARED
+
+  VERSION_VAR PMIx_VERSION
+)
+
+if(NOT TARGET PMIx::libpmix AND PMIx_FOUND)
+  add_library(PMIx::libpmix SHARED IMPORTED)
+  set_target_properties(PMIx::libpmix PROPERTIES
+    IMPORTED_LOCATION ${PMIx_LIBRARY_SHARED}
+    INTERFACE_INCLUDE_DIRECTORIES ${PMIx_INCLUDE_DIR}
+  )
+endif()

cmake/GTestHelper.cmake

@@ -47,6 +47,8 @@
 #
 #
 
+include(GoogleTest)
+
 function(add_testsuite suitename)
     cmake_parse_arguments(testsuite
         ""
@@ -73,13 +75,23 @@ function(add_testsuite suitename)
         target_compile_definitions("${target}" PUBLIC ${testsuite_DEFINITIONS})
     endif()
 
-    add_test(NAME "${suitename}" WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMAND ${target})
+    # add_test(NAME "${suitename}" WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMAND ${target})
     if(testsuite_TIMEOUT)
-        set_tests_properties("${suitename}" PROPERTIES TIMEOUT ${testsuite_TIMEOUT})
+        # set_tests_properties("${suitename}" PROPERTIES TIMEOUT ${testsuite_TIMEOUT})
+    else()
+        set(testsuite_TIMEOUT 10)
     endif()
     if(testsuite_RUN_SERIAL)
-        set_tests_properties("${suitename}" PROPERTIES RUN_SERIAL ${testsuite_RUN_SERIAL})
+        # set_tests_properties("${suitename}" PROPERTIES RUN_SERIAL ${testsuite_RUN_SERIAL})
+    else()
+        set(testsuite_RUN_SERIAL OFF)
     endif()
+    gtest_discover_tests(${target}
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+      TEST_PREFIX ${suitename}.
+      PROPERTIES RUN_SERIAL ${testsuite_RUN_SERIAL}
+                 TIMEOUT ${testsuite_TIMEOUT}
+    )
 
     list(APPEND ALL_TEST_TARGETS ${target})
     set(ALL_TEST_TARGETS ${ALL_TEST_TARGETS} PARENT_SCOPE)

docs/Device.md

@@ -24,21 +24,22 @@ FairMQ devices communicate via the communication patterns offered by ZeroMQ (or
 
 Each FairMQ device has an internal state machine:
 
-![FairMQ state machine](images/device_states.png?raw=true "FairMQ state machine")
+![FairMQ state machine](images/device_states.svg "FairMQ state machine")
 
 The state machine can be querried and controlled via `GetCurrentStateName()` and `ChangeState("<state name>")` methods. Only legal state transitions are allowed (see image above). Illegal transitions will fail with an error.
 
 If the device is running in interactive mode (default), states can be changed via keyboard input:
 
  - `'h'` - help
- - `'p'` - pause
  - `'r'` - run
  - `'s'` - stop
  - `'t'` - reset task
  - `'d'` - reset device
  - `'q'` - end
  - `'j'` - init task
- - `'i'` - init device
+ - `'i'` - initialize
+ - `'b'` - bind
+ - `'x'` - connect
 
 Without the interactive mode, for example for a run in background, two other control mechanisms are available:
 

docs/Plugins.md

@@ -0,0 +1,93 @@
+← [Back](../README.md)
+
+# 7. Plugins
+
+FairMQ devices can be integrated with external configuration and control systems through its plugin system. FairMQ plugins are special dynamic libraries that can be loaded at runtime. Plugins have access to the Plugin API which includes the capability to control/monitor the device [state machine](Device.md#13-state-machine) and change/monitor configuration properties.
+
+A simple plugin may add the feature to read configuration from a certain desired file format once at the start of a device. A more complex plugin may create a long-running thread that integrates a network client to an external API of a central experiment control system.
+
+Because plugins are loaded dynamically, they can be developed in separate repositories/projects and also have their own set of runtime dependency that are not needed to be known at compile-time of the FairMQ device.
+
+## 7.1 Usage
+
+To load a plugin pass the `-P <name>[,<name>]` (or long `--plugin`) command line option. Multiple plugins can be loaded at the same time. The load order is as specified at the command line. This determines the order in which the plugins are instantiated (ctor call order) and in which order they are notified, should they subscribe to any notifications.
+
+When passing `-h/--help` on the command line one can find more detailed information:
+
+```
+Plugin Manager:
+  -S [ --plugin-search-path ] arg       List of plugin search paths.
+                                        * Override default search path, e.g.
+                                          -S /home/user/lib /lib
+                                        * Append(>) or prepend(<) to default
+                                        search path, e.g.
+                                          -S >/lib </home/user/lib
+                                        * If you mix the overriding and
+                                        appending/prepending syntaxes, the
+                                        overriding paths act as default search
+                                        path, e.g.
+                                          -S /usr/lib >/lib </home/user/lib
+                                        /usr/local/lib results in
+                                        /home/user/lib,/usr/local/lib,/usr/lib/
+                                        ,/lib
+                                        If nothing is found, the default
+                                        dynamic library lookup is performed,
+                                        see man ld.so(8) for details.
+  -P [ --plugin ] arg                   List of plugin names to load in
+                                        order,e.g. if the file is called
+                                        'libFairMQPlugin_example.so', just list
+                                        'example' or 'd:example' here.To load a
+                                        prelinked plugin, list 'p:example'
+                                        here.
+```
+
+## 7.2 Development
+
+To develop a custom FairMQ plugin, one simply needs to inherit from the `fair::mq::Plugin` base class (`#include <fairmq/Plugin.h>`) and call the `REGISTER_FAIRMQ_PLUGIN` macro. It is possible to introduce new command line option together with a plugin.
+
+The Plugin API includes:
+  * `Take/Steal/ReleaseDeviceControl()`/`GetCurrent/ChangeDeviceState()`/`SubscribeTo/UnsubscribeFromDeviceStateChange()` APIs enable controlling the device state machine. Only one plugin is authorized to control at the same time. Which one is determined by which plugin calls `TakeDeviceControl()` first.
+  * `Set/GetProperty()`/`GetPropertyKeys()`/`SubscribeTo/UnsubscribeFromPropertyChange()` APIs enable configuration of device properties.
+See [`<fairmq/Plugin.h>`](/fairmq/Plugin.h) for the full API.
+
+A more complete example which may serve as a start including example CMake code can be found here: [FairRootGroup/FairMQPlugin_example](https://github.com/FairRootGroup/FairMQPlugin_example).
+
+## 7.3 Provided Plugins
+
+### 7.3.1 DDS
+
+When launching a FairMQ topology via [DDS](http://dds.gsi.de/) the DDS plugin enables FairMQ devices to interact with DDS' custom command and property subsystems - enable the plugin by passing `-P dds` on the command line.
+
+Via the property subsystem a FairMQ topology may exchange channel connection data (essentially to do service discovery) needed to connect/bind all FairMQ channels appropriately. DDS is highly optimized for this use case. See [examples/dds](examples/dds/README.md) for more details.
+
+Via the custom command subsystem a FairMQ device can receive a number of commands. FairMQ provides a convenient command line tool `fairmq-dds-command-ui` that allows interactive or scripted control of a running FairMQ topology managed via DDS. If one develops directly against the custom command DDS API, the following table lists all the commands the DDS plugin currently understands:
+
+| Custom Command | Response | Error | Description |
+| --- | --- | --- | --- |
+| `check-state` | `<ID>: <STATE> (pid: <PID>)` | n/a | Query current device state, see state machine for possible states |
+| `dump-config` | `(<ID>: <PKEY> -> <PVALUE>\n)+` | n/a | Query current device config (list property key/value pairs) |
+| `INIT DEVICE` | `<ID>: queued <CMD> transition` | `<ID>: could not queue <CMD> transition` | Initiate state transition |
+| `BIND` | `<ID>: queued <CMD> transition` | `<ID>: could not queue <CMD> transition` | Initiate state transition |
+| `CONNECT` | `<ID>: queued <CMD> transition` | `<ID>: could not queue <CMD> transition` | Initiate state transition |
+| `INIT TASK` | `<ID>: queued <CMD> transition` | `<ID>: could not queue <CMD> transition` | Initiate state transition |
+| `RUN` | `<ID>: queued <CMD> transition` | `<ID>: could not queue <CMD> transition` | Initiate state transition |
+| `STOP` | `<ID>: queued <CMD> transition` | `<ID>: could not queue <CMD> transition` | Initiate state transition |
+| `RESET TASK` | `<ID>: queued <CMD> transition` | `<ID>: could not queue <CMD> transition` | Initiate state transition |
+| `RESET DEVICE` | `<ID>: queued <CMD> transition` | `<ID>: could not queue <CMD> transition` | Initiate state transition |
+| `END` | `<ID>: queued <CMD> transition` | `<ID>: could not queue <CMD> transition` | Initiate state transition |
+| `subscribe-to-heartbeats` | `heartbeat-subscription: <ID>,OK` | n/a | Subscribe to heartbeats |
+| on heartbeat subscription | `heartbeat: <ID>,<PID>` | n/a | Heartbeat every 100ms |
+| `unsubscribe-from-heartbeats` | `heartbeat-unsubscription: <ID>,OK` | n/a | Unsubscribe from heartbeats |
+| `subscribe-to-state-changes` | `state-changes-subscription: <ID>,OK` | n/a | Subscribe to state changes |
+| on state changes subscription | `state-change: <ID>,<STATE>` | n/a | State change notification |
+| `unsubscribe-from-state-changes` | `state-changes-unsubscription: <ID>,OK` | n/a | Unsubscribe from state changes |
+
+If unknown commands are received the plugin will print a warning.
+
+### 7.3.2 PMIx
+
+The [PMIx](https://pmix.org/) plugin enables launching a FairMQ topology with any PMIx capable launcher, e.g. the [Open Run-Time Environment (ORTE) of OpenMPI](https://www.open-mpi.org/doc/v4.0/man1/mpirun.1.php) or the [Slurm workload manager](https://slurm.schedmd.com/srun.html). This plugin is not (yet) very mature and serves as a proof of concept at the moment.
+
+TODO example usage
+
+← [Back](../README.md)

examples/1-1/CMakeLists.txt

@@ -31,16 +31,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-1-1.sh.in ${CMAKE_CUR
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-1-1.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh)
 
-add_test(NAME Example-1-1-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh zeromq)
-set_tests_properties(Example-1-1-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+add_test(NAME Example.1-1.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh zeromq)
+set_tests_properties(Example.1-1.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-1-1-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh nanomsg)
-  set_tests_properties(Example-1-1-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+  add_test(NAME Example.1-1.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh nanomsg)
+  set_tests_properties(Example.1-1.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 endif()
 
-add_test(NAME Example-1-1-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh shmem)
-set_tests_properties(Example-1-1-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+add_test(NAME Example.1-1.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh shmem)
+set_tests_properties(Example.1-1.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
 # install
 

examples/1-n-1/CMakeLists.txt

@@ -39,16 +39,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-1-n-1.json ${CMAKE_CURRENT_BINARY_
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-1-n-1.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh)
 
-add_test(NAME Example-1-n-1-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh zeromq)
-set_tests_properties(Example-1-n-1-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+add_test(NAME Example.1-n-1.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh zeromq)
+set_tests_properties(Example.1-n-1.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-1-n-1-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh nanomsg)
-  set_tests_properties(Example-1-n-1-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+  add_test(NAME Example.1-n-1.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh nanomsg)
+  set_tests_properties(Example.1-n-1.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 endif()
 
-add_test(NAME Example-1-n-1-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh shmem)
-set_tests_properties(Example-1-n-1-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+add_test(NAME Example.1-n-1.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh shmem)
+set_tests_properties(Example.1-n-1.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
 # install
 

examples/CMakeLists.txt

@@ -16,5 +16,6 @@ add_subdirectory(multiple-channels)
 if(BUILD_NANOMSG_TRANSPORT)
   add_subdirectory(multiple-transports)
 endif()
+add_subdirectory(readout)
 add_subdirectory(region)
 add_subdirectory(req-rep)

examples/builtin-devices/CMakeLists.txt

@@ -14,27 +14,27 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-builtin-devices.sh.in
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-builtin-devices.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh)
 
-add_test(NAME Example-Builtin-Devices-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq)
-set_tests_properties(Example-Builtin-Devices-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+add_test(NAME Example.BuiltinDevices.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq)
+set_tests_properties(Example.BuiltinDevices.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Builtin-Devices-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg)
-  set_tests_properties(Example-Builtin-Devices-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+  add_test(NAME Example.BuiltinDevices.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg)
+  set_tests_properties(Example.BuiltinDevices.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 endif()
 
-add_test(NAME Example-Builtin-Devices-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem)
-set_tests_properties(Example-Builtin-Devices-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+add_test(NAME Example.BuiltinDevices.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem)
+set_tests_properties(Example.BuiltinDevices.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
-add_test(NAME Example-Builtin-Devices-zeromq-multipart COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq true 2)
-set_tests_properties(Example-Builtin-Devices-zeromq-multipart PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+add_test(NAME Example.BuiltinDevices.multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq true 2)
+set_tests_properties(Example.BuiltinDevices.multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Builtin-Devices-nanomsg-multipart COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg true 2)
-  set_tests_properties(Example-Builtin-Devices-nanomsg-multipart PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+  add_test(NAME Example.BuiltinDevices.multipart.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg true 2)
+  set_tests_properties(Example.BuiltinDevices.multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 endif()
 
-add_test(NAME Example-Builtin-Devices-shmem-multipart COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem true 2)
-set_tests_properties(Example-Builtin-Devices-shmem-multipart PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+add_test(NAME Example.BuiltinDevices.multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem true 2)
+set_tests_properties(Example.BuiltinDevices.multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
 # install
 

examples/copypush/CMakeLists.txt

@@ -32,16 +32,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-copypush.sh.in ${CMAK
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-copypush.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh)
 
-add_test(NAME Example-CopyPush-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh zeromq)
-set_tests_properties(Example-CopyPush-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
+add_test(NAME Example.CopyPush.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh zeromq)
+set_tests_properties(Example.CopyPush.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-CopyPush-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh nanomsg)
-  set_tests_properties(Example-CopyPush-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
+  add_test(NAME Example.CopyPush.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh nanomsg)
+  set_tests_properties(Example.CopyPush.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 endif()
 
-add_test(NAME Example-CopyPush-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh shmem)
-set_tests_properties(Example-CopyPush-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
+add_test(NAME Example.CopyPush.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh shmem)
+set_tests_properties(Example.CopyPush.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 
 # install
 

examples/dds/Processor.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

examples/dds/README.md

@@ -40,9 +40,9 @@ The configuration of the channel connection addresses is done by the DDS plugin
 
 **If you chose step 2b earlier**, then modify the provided `ex-dds-topology.xml` in the top that the following lines read as following:
 ```xml
-<declrequirement id="SamplerWorker" type="wnname" value=".*"/>
-<declrequirement id="ProcessorWorker" type="wnname" value=".*"/>
-<declrequirement id="SinkWorker" type="wnname" value=".*"/>
+<declrequirement name="SamplerWorker" type="wnname" value=".*"/>
+<declrequirement name="ProcessorWorker" type="wnname" value=".*"/>
+<declrequirement name="SinkWorker" type="wnname" value=".*"/>
 ```
 
 Note that the attributes `value` contain a different value.

examples/dds/Sampler.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

examples/dds/Sink.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

examples/dds/ex-dds-hosts.cfg

@@ -2,6 +2,6 @@
 # source setup.sh
 @bash_end@
 
-sampler, username@localhost, , /path/to/dds-work-dir/, 1
-processor, username@localhost, , /path/to/dds-work-dir/, 10
-sink, username@localhost, , /path/to/dds-work-dir/, 1
+sampler, username@localhost, , /path/to/dds-work/, 1
+processor, username@localhost, , /path/to/dds-work/, 10
+sink, username@localhost, , /path/to/dds-work/, 1

examples/dds/ex-dds-topology.xml

@@ -1,47 +1,47 @@
-<topology id="ExampleDDS">
+<topology name="ExampleDDS">
 
-    <property id="data1" />
-    <property id="data2" />
+    <property name="data1" />
+    <property name="data2" />
 
-    <declrequirement id="SamplerWorker" type="wnname" value="sampler"/>
-    <declrequirement id="ProcessorWorker" type="wnname" value="processor"/>
-    <declrequirement id="SinkWorker" type="wnname" value="sink"/>
+    <declrequirement name="SamplerWorker" type="wnname" value="sampler"/>
+    <declrequirement name="ProcessorWorker" type="wnname" value="processor"/>
+    <declrequirement name="SinkWorker" type="wnname" value="sink"/>
 
-    <decltask id="Sampler">
+    <decltask name="Sampler">
         <exe reachable="true">@EX_BIN_DIR@/fairmq-ex-dds-sampler --id sampler --color false --channel-config name=data1,type=push,method=bind -S "&lt;@DDS_PLUGIN_LIB_DIR@/" -P dds</exe>
         <requirements>
-            <id>SamplerWorker</id>
+            <name>SamplerWorker</name>
         </requirements>
         <properties>
-            <id access="write">data1</id>
+            <name access="write">data1</name>
         </properties>
     </decltask>
 
-    <decltask id="Processor">
+    <decltask name="Processor">
         <exe reachable="true">@EX_BIN_DIR@/fairmq-ex-dds-processor --id processor_%taskIndex% --config-key processor --color false --channel-config name=data1,type=pull,method=connect name=data2,type=push,method=connect -S "&lt;@DDS_PLUGIN_LIB_DIR@/" -P dds</exe>
         <requirements>
-            <id>ProcessorWorker</id>
+            <name>ProcessorWorker</name>
         </requirements>
         <properties>
-            <id access="read">data1</id>
-            <id access="read">data2</id>
+            <name access="read">data1</name>
+            <name access="read">data2</name>
         </properties>
     </decltask>
 
-    <decltask id="Sink">
+    <decltask name="Sink">
         <exe reachable="true">@EX_BIN_DIR@/fairmq-ex-dds-sink --id sink --color false --channel-config name=data2,type=pull,method=bind -S "&lt;@DDS_PLUGIN_LIB_DIR@/" -P dds</exe>
         <requirements>
-            <id>SinkWorker</id>
+            <name>SinkWorker</name>
         </requirements>
         <properties>
-            <id access="write">data2</id>
+            <name access="write">data2</name>
         </properties>
     </decltask>
 
-    <main id="main">
+    <main name="main">
         <task>Sampler</task>
         <task>Sink</task>
-        <group id="ProcessorGroup" n="10">
+        <group name="ProcessorGroup" n="10">
             <task>Processor</task>
         </group>
     </main>

examples/multipart/CMakeLists.txt

@@ -31,16 +31,21 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multipart.sh.in ${CMA
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multipart.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh)
 
-add_test(NAME Example-Multipart-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh zeromq)
-set_tests_properties(Example-Multipart-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
+add_test(NAME Example.Multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh zeromq)
+set_tests_properties(Example.Multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 5 parts")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Multipart-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh nanomsg)
-  set_tests_properties(Example-Multipart-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
+  add_test(NAME Example.Multipart.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh nanomsg)
+  set_tests_properties(Example.Multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 5 parts")
 endif()
 
-add_test(NAME Example-Multipart-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh shmem)
-set_tests_properties(Example-Multipart-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
+add_test(NAME Example.Multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh shmem)
+set_tests_properties(Example.Multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 5 parts")
+
+if(BUILD_OFI_TRANSPORT)
+  add_test(NAME Example.Multipart.ofi COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh ofi)
+  set_tests_properties(Example.Multipart.ofi PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 5 parts")
+endif()
 
 # install
 

examples/multipart/Sampler.cxx

@@ -53,6 +53,13 @@ bool Sampler::ConditionalRun()
     parts.AddPart(NewSimpleMessage(header));
     parts.AddPart(NewMessage(1000));
 
+    // create more data parts, testing the FairMQParts in-place constructor
+    FairMQParts auxData{ NewMessage(500), NewMessage(600), NewMessage(700) };
+    assert(auxData.Size() == 3);
+    parts.AddPart(std::move(auxData));
+    assert(auxData.Size() == 0);
+    assert(parts.Size() == 5);
+
     LOG(info) << "Sending body of size: " << parts.At(1)->GetSize();
 
     Send(parts, "data");
@@ -74,4 +81,4 @@ Sampler::~Sampler()
 {
 }
 
-} // namespace example_multipart
+} // namespace example_multipart

examples/multipart/test-ex-multipart.sh.in

@@ -20,7 +20,7 @@ SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --session $SESSION"
 SAMPLER+=" --max-iterations 1"
 SAMPLER+=" --control static --color false"
-SAMPLER+=" --channel-config name=data,type=push,method=connect,rateLogging=0,address=tcp://127.0.0.1:5555"
+SAMPLER+=" --channel-config name=data,type=pair,method=connect,rateLogging=0,address=tcp://127.0.0.1:5555"
 @CMAKE_CURRENT_BINARY_DIR@/$SAMPLER &
 SAMPLER_PID=$!
 
@@ -30,7 +30,7 @@ SINK+=" --transport $transport"
 SINK+=" --verbosity veryhigh"
 SINK+=" --session $SESSION"
 SINK+=" --control static --color false"
-SINK+=" --channel-config name=data,type=pull,method=bind,rateLogging=0,address=tcp://127.0.0.1:5555"
+SINK+=" --channel-config name=data,type=pair,method=bind,rateLogging=0,address=tcp://127.0.0.1:5555"
 @CMAKE_CURRENT_BINARY_DIR@/$SINK &
 SINK_PID=$!
 

examples/multiple-channels/CMakeLists.txt

@@ -36,12 +36,12 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multiple-channels.sh.
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multiple-channels.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh)
 
-add_test(NAME Example-Multiple-Channels-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh zeromq)
-set_tests_properties(Example-Multiple-Channels-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
+add_test(NAME Example.MultipleChannels.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh zeromq)
+set_tests_properties(Example.MultipleChannels.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Multiple-Channels-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh nanomsg)
-  set_tests_properties(Example-Multiple-Channels-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
+  add_test(NAME Example.MultipleChannels.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh nanomsg)
+  set_tests_properties(Example.MultipleChannels.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
 endif()
 
 # install

examples/multiple-channels/Sampler.cxx

@@ -41,7 +41,7 @@ void Sampler::Run()
 {
     FairMQPollerPtr poller(NewPoller("data", "broadcast"));
 
-    while (CheckCurrentState(RUNNING))
+    while (!NewStatePending())
     {
         poller->Poll(100);
 

examples/multiple-transports/CMakeLists.txt

@@ -35,8 +35,8 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multiple-transports.s
 # test
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multiple-transports.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-transports.sh)
-add_test(NAME Example-Multiple-Transports COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-transports.sh)
-set_tests_properties(Example-Multiple-Transports PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
+add_test(NAME Example.MultipleTransports COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-transports.sh)
+set_tests_properties(Example.MultipleTransports PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
 
 # install
 

examples/multiple-transports/Sampler1.cxx

@@ -60,7 +60,7 @@ void Sampler1::ListenForAcks()
 {
     uint64_t numAcks = 0;
 
-    while (CheckCurrentState(RUNNING))
+    while (!NewStatePending())
     {
         FairMQMessagePtr ack(NewMessageFor("ack", 0));
         if (Receive(ack, "ack") < 0)

examples/readout/Builder.h

@@ -0,0 +1,46 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+#ifndef FAIRMQEXAMPLEREGIONBUILDER_H
+#define FAIRMQEXAMPLEREGIONBUILDER_H
+
+#include "FairMQDevice.h"
+
+#include <string>
+
+namespace example_readout
+{
+
+class Builder : public FairMQDevice
+{
+  public:
+    Builder()
+        : fOutputChannelName()
+    {}
+
+    void Init() override
+    {
+        fOutputChannelName = fConfig->GetValue<std::string>("output-name");
+        OnData("rb", &Builder::HandleData);
+    }
+
+    bool HandleData(FairMQMessagePtr& msg, int /*index*/)
+    {
+        if (Send(msg, fOutputChannelName) < 0) {
+            return false;
+        }
+
+        return true;
+    }
+
+  private:
+    std::string fOutputChannelName;
+};
+
+} // namespace example_readout
+
+#endif /* FAIRMQEXAMPLEREGIONBUILDER_H */

examples/readout/CMakeLists.txt

@@ -0,0 +1,59 @@
+ ################################################################################
+ #    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+ #                                                                              #
+ #              This software is distributed under the terms of the             #
+ #              GNU Lesser General Public Licence (LGPL) version 3,             #
+ #                  copied verbatim in the file "LICENSE"                       #
+ ################################################################################
+
+add_executable(fairmq-ex-readout-readout runReadout.cxx)
+target_link_libraries(fairmq-ex-readout-readout PRIVATE FairMQ)
+
+add_executable(fairmq-ex-readout-builder runBuilder.cxx)
+target_link_libraries(fairmq-ex-readout-builder PRIVATE FairMQ)
+
+add_executable(fairmq-ex-readout-processor runProcessor.cxx)
+target_link_libraries(fairmq-ex-readout-processor PRIVATE FairMQ)
+
+add_executable(fairmq-ex-readout-sender runSender.cxx)
+target_link_libraries(fairmq-ex-readout-sender PRIVATE FairMQ)
+
+add_executable(fairmq-ex-readout-receiver runReceiver.cxx)
+target_link_libraries(fairmq-ex-readout-receiver PRIVATE FairMQ)
+
+set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
+set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-readout.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout.sh)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-readout-processing.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout-processing.sh)
+
+# install
+
+install(
+    TARGETS
+    fairmq-ex-readout-readout
+    fairmq-ex-readout-builder
+    fairmq-ex-readout-processor
+    fairmq-ex-readout-sender
+    fairmq-ex-readout-receiver
+
+    LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
+    RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
+)
+
+# configure run script with different executable paths for build and for install directories
+set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
+set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-readout.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout.sh_install)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-readout-processing.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout-processing.sh_install)
+
+install(
+    PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout.sh_install
+    DESTINATION ${PROJECT_INSTALL_BINDIR}
+    RENAME fairmq-start-ex-readout.sh
+)
+
+install(
+    PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout-processing.sh_install
+    DESTINATION ${PROJECT_INSTALL_BINDIR}
+    RENAME fairmq-start-ex-readout-processing.sh
+)

examples/readout/Processor.h

@@ -0,0 +1,37 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+#ifndef FAIRMQEXAMPLEREGIONPROCESSOR_H
+#define FAIRMQEXAMPLEREGIONPROCESSOR_H
+
+#include "FairMQDevice.h"
+
+namespace example_readout
+{
+
+class Processor : public FairMQDevice
+{
+  public:
+    Processor() {
+        OnData("bp", &Processor::HandleData);
+    }
+
+  protected:
+    bool HandleData(FairMQMessagePtr& msg, int /*index*/)
+    {
+        FairMQMessagePtr msg2(NewMessageFor("ps", 0, msg->GetSize()));
+        if (Send(msg2, "ps") < 0) {
+            return false;
+        }
+
+        return true;
+    }
+};
+
+} // namespace example_readout
+
+#endif /* FAIRMQEXAMPLEREGIONPROCESSOR_H */

examples/readout/README.md

@@ -0,0 +1,27 @@
+# Readout example
+
+This examples shows two possible topologies (out of many) for a node connected to a detector readout (followed by a processing node).
+
+## Setup without new data generation
+
+```
+|------------------------------- Readout Node ---------------------------|     |- Processing Node -|
+| Readout            -->            Builder          -->          Sender | --> |     Receiver      |
+| [# shared memory segment (unused in this topology) ##################] | ofi |                   |
+| [# shmem unmanaged region (readout writes here, others read) ########] |     |                   |
+|------------------------------------------------------------------------|     |-------------------|
+```
+
+The devices one the Readout Node communicate via shared memory transport. Readout device writes into shared memory unmanaged region. The data is then forwarded through Builder to Sender process, which sends it out via OFI transport.
+
+## Setup with generating new data on the Readout node
+
+```
+|------------------------------- Readout Node ---------------------------|     |- Processing Node -|
+| Readout     -->     Builder      -->      Processor     -->     Sender | --> |     Receiver      |
+| [# shared memory segment (used between Proccessor and Sender) #######] | ofi |                   |
+| [# shmem unmanaged region (readout writes here, builder & proc read) ] |     |                   |
+|------------------------------------------------------------------------|     |-------------------|
+```
+
+In this topology one more device is added - Processor. It examines the arriving data and creates new data in shared memory. This data is not part of the unmanaged region, but lives in the general shared memory segment (unused in the previous setup). This new data is then forwarded to Sender and the Readout device is notified that the corresponding data piece in the unmanaged region is no longer used.

examples/readout/Readout.h

@@ -0,0 +1,91 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+#ifndef FAIRMQEXAMPLEREADOUTREADOUT_H
+#define FAIRMQEXAMPLEREADOUTREADOUT_H
+
+#include <atomic>
+#include <thread>
+#include <chrono>
+
+#include "FairMQDevice.h"
+
+namespace example_readout
+{
+
+class Readout : public FairMQDevice
+{
+  public:
+    Readout()
+        : fMsgSize(10000)
+        , fMaxIterations(0)
+        , fNumIterations(0)
+        , fRegion(nullptr)
+        , fNumUnackedMsgs(0)
+    {}
+
+  protected:
+    void InitTask() override
+    {
+        fMsgSize = fConfig->GetValue<int>("msg-size");
+        fMaxIterations = fConfig->GetValue<uint64_t>("max-iterations");
+
+        fRegion = FairMQUnmanagedRegionPtr(NewUnmanagedRegionFor("rb",
+                                                                0,
+                                                                10000000,
+                                                                [this](void* /*data*/, size_t /*size*/, void* /*hint*/) { // callback to be called when message buffers no longer needed by transport
+                                                                    --fNumUnackedMsgs;
+                                                                    if (fMaxIterations > 0) {
+                                                                        LOG(debug) << "Received ack";
+                                                                    }
+                                                                }
+                                                                ));
+    }
+
+    bool ConditionalRun() override
+    {
+        FairMQMessagePtr msg(NewMessageFor("rb", // channel
+                                            0, // sub-channel
+                                            fRegion, // region
+                                            fRegion->GetData(), // ptr within region
+                                            fMsgSize, // offset from ptr
+                                            nullptr // hint
+                                            ));
+
+        if (Send(msg, "rb", 0) > 0) {
+            ++fNumUnackedMsgs;
+
+            if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
+                LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
+                return false;
+            }
+        }
+
+        return true;
+    }
+    void ResetTask() override
+    {
+        // if not all messages acknowledged, wait for a bit. But only once, since receiver could be already dead.
+        if (fNumUnackedMsgs != 0) {
+            LOG(debug) << "waiting for all acknowledgements... (" << fNumUnackedMsgs << ")";
+            std::this_thread::sleep_for(std::chrono::milliseconds(500));
+            LOG(debug) << "done, still unacked: " << fNumUnackedMsgs;
+        }
+        fRegion.reset();
+    }
+
+  private:
+    int fMsgSize;
+    uint64_t fMaxIterations;
+    uint64_t fNumIterations;
+    FairMQUnmanagedRegionPtr fRegion;
+    std::atomic<uint64_t> fNumUnackedMsgs;
+};
+
+} // namespace example_readout
+
+#endif /* FAIRMQEXAMPLEREADOUTREADOUT_H */

examples/readout/Receiver.h

@@ -0,0 +1,54 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+#ifndef FAIRMQEXAMPLEREGIONRECEIVER_H
+#define FAIRMQEXAMPLEREGIONRECEIVER_H
+
+#include "FairMQDevice.h"
+
+namespace example_readout
+{
+
+class Receiver : public FairMQDevice
+{
+  public:
+    Receiver()
+        : fMaxIterations(0)
+        , fNumIterations(0)
+    {}
+
+  protected:
+    void InitTask() override
+    {
+        // Get the fMaxIterations value from the command line options (via fConfig)
+        fMaxIterations = fConfig->GetValue<uint64_t>("max-iterations");
+    }
+
+    void Run() override
+    {
+        FairMQChannel& dataInChannel = fChannels.at("sr").at(0);
+
+        while (!NewStatePending()) {
+            FairMQMessagePtr msg(dataInChannel.Transport()->CreateMessage());
+            dataInChannel.Receive(msg);
+            // void* ptr = msg->GetData();
+
+            if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
+                LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
+                break;
+            }
+        }
+    }
+
+  private:
+    uint64_t fMaxIterations;
+    uint64_t fNumIterations;
+};
+
+} // namespace example_readout
+
+#endif /* FAIRMQEXAMPLEREGIONRECEIVER_H */

examples/readout/Sender.h

@@ -0,0 +1,46 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+#ifndef FAIRMQEXAMPLEREGIONSENDER_H
+#define FAIRMQEXAMPLEREGIONSENDER_H
+
+#include "FairMQDevice.h"
+
+#include <string>
+
+namespace example_readout
+{
+
+class Sender : public FairMQDevice
+{
+  public:
+    Sender()
+        : fInputChannelName()
+    {}
+
+    void Init() override
+    {
+        fInputChannelName = fConfig->GetValue<std::string>("input-name");
+        OnData(fInputChannelName, &Sender::HandleData);
+    }
+
+    bool HandleData(FairMQMessagePtr& msg, int /*index*/)
+    {
+        if (Send(msg, "sr") < 0) {
+            return false;
+        }
+
+        return true;
+    }
+
+  private:
+    std::string fInputChannelName;
+};
+
+} // namespace example_readout
+
+#endif /* FAIRMQEXAMPLEREGIONSENDER_H */

examples/readout/fairmq-start-ex-readout-processing.sh.in

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
+
+msgSize="1000000"
+
+if [[ $1 =~ ^[0-9]+$ ]]; then
+    msgSize=$1
+fi
+
+READOUT="fairmq-ex-readout-readout"
+READOUT+=" --id readout1"
+READOUT+=" --msg-size $msgSize"
+READOUT+=" --channel-config name=rb,type=pair,method=bind,address=tcp://localhost:7777,transport=shmem"
+xterm -geometry 80x23+0+0 -hold -e @EX_BIN_DIR@/$READOUT &
+
+BUILDER="fairmq-ex-readout-builder"
+BUILDER+=" --id builder1"
+BUILDER+=" --output-name bp"
+BUILDER+=" --channel-config name=rb,type=pair,method=connect,address=tcp://localhost:7777,transport=shmem"
+BUILDER+="                  name=bp,type=pair,method=connect,address=tcp://localhost:7778,transport=shmem"
+xterm -geometry 80x23+500+0 -hold -e @EX_BIN_DIR@/$BUILDER &
+
+PROCESSOR="fairmq-ex-readout-processor"
+PROCESSOR+=" --id processor1"
+PROCESSOR+=" --channel-config name=bp,type=pair,method=bind,address=tcp://localhost:7778,transport=shmem"
+PROCESSOR+="                  name=ps,type=pair,method=connect,address=tcp://localhost:7779,transport=shmem"
+xterm -geometry 80x23+750+500 -hold -e @EX_BIN_DIR@/$PROCESSOR &
+
+SENDER="fairmq-ex-readout-sender"
+SENDER+=" --id sender1"
+SENDER+=" --input-name ps"
+SENDER+=" --channel-config name=ps,type=pair,method=bind,address=tcp://localhost:7779,transport=shmem"
+SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7780,transport=ofi"
+xterm -geometry 80x23+1000+0 -hold -e @EX_BIN_DIR@/$SENDER &
+
+RECEIVER="fairmq-ex-readout-receiver"
+RECEIVER+=" --id receiver1"
+RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7780,transport=ofi"
+xterm -geometry 80x23+1500+0 -hold -e @EX_BIN_DIR@/$RECEIVER &

examples/readout/fairmq-start-ex-readout.sh.in

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
+
+msgSize="1000000"
+
+if [[ $1 =~ ^[0-9]+$ ]]; then
+    msgSize=$1
+fi
+
+READOUT="fairmq-ex-readout-readout"
+READOUT+=" --id readout1"
+READOUT+=" --msg-size $msgSize"
+READOUT+=" --channel-config name=rb,type=pair,method=bind,address=tcp://localhost:7777,transport=shmem"
+xterm -geometry 80x23+0+0 -hold -e @EX_BIN_DIR@/$READOUT &
+
+BUILDER="fairmq-ex-readout-builder"
+BUILDER+=" --id builder1"
+BUILDER+=" --output-name bs"
+BUILDER+=" --channel-config name=rb,type=pair,method=connect,address=tcp://localhost:7777,transport=shmem"
+BUILDER+="                  name=bs,type=pair,method=connect,address=tcp://localhost:7778,transport=shmem"
+xterm -geometry 80x23+500+0 -hold -e @EX_BIN_DIR@/$BUILDER &
+
+SENDER="fairmq-ex-readout-sender"
+SENDER+=" --id sender1"
+SENDER+=" --input-name bs"
+SENDER+=" --channel-config name=bs,type=pair,method=bind,address=tcp://localhost:7778,transport=shmem"
+SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7779,transport=ofi"
+xterm -geometry 80x23+1000+0 -hold -e @EX_BIN_DIR@/$SENDER &
+
+RECEIVER="fairmq-ex-readout-receiver"
+RECEIVER+=" --id receiver1"
+RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7779,transport=ofi"
+xterm -geometry 80x23+1500+0 -hold -e @EX_BIN_DIR@/$RECEIVER &

examples/readout/runBuilder.cxx

@@ -0,0 +1,23 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "Builder.h"
+
+namespace bpo = boost::program_options;
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()
+        ("output-name", bpo::value<std::string>()->default_value("bs"), "Output channel name");
+}
+
+FairMQDevicePtr getDevice(const FairMQProgOptions& /*config*/)
+{
+    return new example_readout::Builder();
+}

examples/readout/runProcessor.cxx

@@ -0,0 +1,20 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "Processor.h"
+
+namespace bpo = boost::program_options;
+
+void addCustomOptions(bpo::options_description& /* options */)
+{}
+
+FairMQDevicePtr getDevice(const FairMQProgOptions& /*config*/)
+{
+    return new example_readout::Processor();
+}

examples/readout/runReadout.cxx

@@ -0,0 +1,24 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "Readout.h"
+
+namespace bpo = boost::program_options;
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()
+        ("msg-size", bpo::value<int>()->default_value(1000), "Message size in bytes")
+        ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
+}
+
+FairMQDevicePtr getDevice(const FairMQProgOptions& /*config*/)
+{
+    return new example_readout::Readout();
+}

examples/readout/runReceiver.cxx

@@ -0,0 +1,23 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "Receiver.h"
+
+namespace bpo = boost::program_options;
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()
+        ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
+}
+
+FairMQDevicePtr getDevice(const FairMQProgOptions& /*config*/)
+{
+    return new example_readout::Receiver();
+}

examples/readout/runSender.cxx

@@ -0,0 +1,23 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "Sender.h"
+
+namespace bpo = boost::program_options;
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()
+        ("input-name", bpo::value<std::string>()->default_value("bs"), "Input channel name");
+}
+
+FairMQDevicePtr getDevice(const FairMQProgOptions& /*config*/)
+{
+    return new example_readout::Sender();
+}

examples/region/CMakeLists.txt

@@ -18,7 +18,6 @@ target_link_libraries(ExampleRegionLib PUBLIC FairMQ)
 add_executable(fairmq-ex-region-sampler runSampler.cxx)
 target_link_libraries(fairmq-ex-region-sampler PRIVATE ExampleRegionLib)
 
-
 add_executable(fairmq-ex-region-sink runSink.cxx)
 target_link_libraries(fairmq-ex-region-sink PRIVATE ExampleRegionLib)
 
@@ -32,16 +31,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-region.sh.in ${CMAKE_
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-region.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh)
 
-add_test(NAME Example-Region-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh zeromq)
-set_tests_properties(Example-Region-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
+add_test(NAME Example.Region.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh zeromq)
+set_tests_properties(Example.Region.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Region-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh nanomsg)
-  set_tests_properties(Example-Region-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
+  add_test(NAME Example.Region.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh nanomsg)
+  set_tests_properties(Example.Region.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
 endif()
 
-add_test(NAME Example-Region-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh shmem)
-set_tests_properties(Example-Region-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
+add_test(NAME Example.Region.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh shmem)
+set_tests_properties(Example.Region.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
 
 # install
 

examples/region/Sink.cxx

@@ -35,7 +35,7 @@ void Sink::Run()
 {
     FairMQChannel& dataInChannel = fChannels.at("data").at(0);
 
-    while (CheckCurrentState(RUNNING))
+    while (!NewStatePending())
     {
         FairMQMessagePtr msg(dataInChannel.Transport()->CreateMessage());
         dataInChannel.Receive(msg);

examples/req-rep/CMakeLists.txt

@@ -32,16 +32,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-req-rep.sh.in ${CMAKE
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-req-rep.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh)
 
-add_test(NAME Example-ReqRep-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh zeromq)
-set_tests_properties(Example-ReqRep-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
+add_test(NAME Example.ReqRep.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh zeromq)
+set_tests_properties(Example.ReqRep.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-ReqRep-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh nanomsg)
-  set_tests_properties(Example-ReqRep-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
+  add_test(NAME Example.ReqRep.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh nanomsg)
+  set_tests_properties(Example.ReqRep.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 endif()
 
-add_test(NAME Example-ReqRep-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh shmem)
-set_tests_properties(Example-ReqRep-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
+add_test(NAME Example.ReqRep.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh shmem)
+set_tests_properties(Example.ReqRep.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 
 # install
 

fairmq/CMakeLists.txt

@@ -12,24 +12,8 @@
 if(BUILD_DDS_PLUGIN)
   add_subdirectory(plugins/DDS)
 endif()
-
-
-############################
-# preprocessor definitions #
-############################
-if(BUILD_NANOMSG_TRANSPORT)
-  add_definitions(-DBUILD_NANOMSG_TRANSPORT)
-endif()
-if(BUILD_OFI_TRANSPORT)
-  add_definitions(-DBUILD_OFI_TRANSPORT)
-endif()
-
-
-##################
-# subdirectories #
-##################
-if(BUILD_OFI_TRANSPORT)
-  add_subdirectory(ofi)
+if(BUILD_PMIX_PLUGIN)
+  add_subdirectory(plugins/PMIx)
 endif()
 
 
@@ -47,7 +31,7 @@ set(FAIRMQ_PUBLIC_HEADER_FILES
   FairMQPoller.h
   FairMQUnmanagedRegion.h
   FairMQSocket.h
-  FairMQStateMachine.h
+  StateMachine.h
   FairMQTransportFactory.h
   MemoryResources.h
   MemoryResourceTools.h
@@ -80,7 +64,6 @@ set(FAIRMQ_PRIVATE_HEADER_FILES
   options/FairProgOptionsHelper.h
   plugins/Builtin.h
   plugins/Control.h
-  StateMachine.h
   shmem/FairMQMessageSHM.h
   shmem/FairMQPollerSHM.h
   shmem/FairMQUnmanagedRegionSHM.h
@@ -127,7 +110,7 @@ set(FAIRMQ_SOURCE_FILES
   FairMQMessage.cxx
   FairMQPoller.cxx
   FairMQSocket.cxx
-  FairMQStateMachine.cxx
+  StateMachine.cxx
   FairMQTransportFactory.cxx
   devices/FairMQBenchmarkSampler.cxx
   devices/FairMQMerger.cxx
@@ -141,7 +124,6 @@ set(FAIRMQ_SOURCE_FILES
   PluginManager.cxx
   PluginServices.cxx
   plugins/Control.cxx
-  StateMachine.cxx
   shmem/FairMQMessageSHM.cxx
   shmem/FairMQPollerSHM.cxx
   shmem/FairMQUnmanagedRegionSHM.cxx
@@ -206,9 +188,19 @@ set_target_properties(${_target} PROPERTIES LABELS coverage)
 if(FAST_BUILD)
   set_target_properties(${_target} PROPERTIES OUTPUT_NAME FairMQ)
 endif()
-if(Boost_VERSION VERSION_LESS 1.66)
-  target_compile_definitions(${_target} PUBLIC FAIR_MQ_HAS_NO_ASIO_IO_CONTEXT)
+
+
+############################
+# preprocessor definitions #
+############################
+target_compile_definitions(${_target} PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
+if(BUILD_NANOMSG_TRANSPORT)
+  target_compile_definitions(${_target} PRIVATE BUILD_NANOMSG_TRANSPORT)
 endif()
+if(BUILD_OFI_TRANSPORT)
+  target_compile_definitions(${_target} PRIVATE BUILD_OFI_TRANSPORT)
+endif()
+
 
 #######################
 # include directories #
@@ -229,7 +221,10 @@ if(BUILD_NANOMSG_TRANSPORT)
   set(NANOMSG_DEPS nanomsg msgpackc-cxx)
 endif()
 if(BUILD_OFI_TRANSPORT)
-  set(OFI_DEPS OFI::libfabric protobuf::libprotobuf $<TARGET_OBJECTS:OfiTransport>)
+  set(OFI_DEPS
+    asiofi::asiofi
+    Boost::container
+  )
 endif()
 set(optional_deps ${NANOMSG_DEPS} ${OFI_DEPS})
 if(optional_deps)
@@ -246,11 +241,8 @@ target_link_libraries(${_target}
   $<$<PLATFORM_ID:Linux>:rt>
   Boost::boost
   Boost::program_options
-  Boost::thread
-  Boost::system
   Boost::filesystem
   Boost::regex
-  Boost::date_time
   FairLogger::FairLogger
 
   PRIVATE # only libFairMQ links against private dependencies

fairmq/DeviceRunner.cxx

@@ -74,15 +74,14 @@ auto DeviceRunner::Run() -> int
     fDevice->RegisterChannelEndpoints();
     if (fConfig.Count("print-channels")) {
         fDevice->PrintRegisteredChannels();
-        fDevice->ChangeState(FairMQDevice::END);
+        fDevice->ChangeState(fair::mq::Transition::End);
         return 0;
     }
 
     // Handle --version
     if (fConfig.Count("version")) {
         std::cout << "User device version: " << fDevice->GetVersion() << std::endl;
-        std::cout << "FAIRMQ_INTERFACE_VERSION: " << FAIRMQ_INTERFACE_VERSION << std::endl;
-        fDevice->ChangeState(FairMQDevice::END);
+        fDevice->ChangeState(fair::mq::Transition::End);
         return 0;
     }
 

fairmq/FairMQChannel.h

@@ -345,6 +345,11 @@ class FairMQChannel
         return Transport()->NewStaticMessage(data);
     }
 
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr)
+    {
+        return Transport()->CreateUnmanagedRegion(size, callback);
+    }
+
   private:
     std::shared_ptr<FairMQTransportFactory> fTransportFactory;
     fair::mq::Transport fTransportType;

fairmq/FairMQDevice.cxx

@@ -26,6 +26,35 @@
 #include <algorithm> // std::max
 
 using namespace std;
+using namespace fair::mq;
+
+static map<Transition, State> backwardsCompatibilityWaitForEndOfStateHelper =
+{
+    { Transition::InitDevice,   State::InitializingDevice },
+    { Transition::CompleteInit, State::Initialized },
+    { Transition::Bind,         State::Bound },
+    { Transition::Connect,      State::DeviceReady },
+    { Transition::InitTask,     State::Ready },
+    { Transition::Run,          State::Ready },
+    { Transition::Stop,         State::Ready },
+    { Transition::ResetTask,    State::DeviceReady },
+    { Transition::ResetDevice,  State::Idle }
+};
+
+static map<int, Transition> backwardsCompatibilityChangeStateHelper =
+{
+    { FairMQDevice::Event::INIT_DEVICE,           Transition::InitDevice },
+    { FairMQDevice::Event::internal_DEVICE_READY, Transition::Auto },
+    { FairMQDevice::Event::INIT_TASK,             Transition::InitTask },
+    { FairMQDevice::Event::internal_READY,        Transition::Auto },
+    { FairMQDevice::Event::RUN,                   Transition::Run },
+    { FairMQDevice::Event::STOP,                  Transition::Stop },
+    { FairMQDevice::Event::RESET_TASK,            Transition::ResetTask },
+    { FairMQDevice::Event::RESET_DEVICE,          Transition::ResetDevice },
+    { FairMQDevice::Event::internal_IDLE,         Transition::Auto },
+    { FairMQDevice::Event::END,                   Transition::End },
+    { FairMQDevice::Event::ERROR_FOUND,           Transition::ErrorFound }
+};
 
 FairMQDevice::FairMQDevice()
     : FairMQDevice(nullptr, {0, 0, 0})
@@ -37,24 +66,27 @@ FairMQDevice::FairMQDevice(FairMQProgOptions& config)
 {
 }
 
-FairMQDevice::FairMQDevice(const fair::mq::tools::Version version)
+FairMQDevice::FairMQDevice(const tools::Version version)
     : FairMQDevice(nullptr, version)
 {
 }
 
-FairMQDevice::FairMQDevice(FairMQProgOptions& config, const fair::mq::tools::Version version)
+FairMQDevice::FairMQDevice(FairMQProgOptions& config, const tools::Version version)
     : FairMQDevice(&config, version)
 {
 }
 
-FairMQDevice::FairMQDevice(FairMQProgOptions* config, const fair::mq::tools::Version version)
+FairMQDevice::FairMQDevice(FairMQProgOptions* config, const tools::Version version)
     : fTransportFactory(nullptr)
     , fTransports()
     , fChannels()
-    , fInternalConfig(config ? nullptr : fair::mq::tools::make_unique<FairMQProgOptions>())
+    , fInternalConfig(config ? nullptr : tools::make_unique<FairMQProgOptions>())
     , fConfig(config ? config : fInternalConfig.get())
     , fId()
-    , fDefaultTransportType(fair::mq::Transport::DEFAULT)
+    , fDefaultTransportType(fair::mq::Transport::ZMQ)
+    , fStateMachine()
+    , fUninitializedBindingChannels()
+    , fUninitializedConnectingChannels()
     , fDataCallbacks(false)
     , fMsgInputs()
     , fMultipartInputs()
@@ -65,18 +97,107 @@ FairMQDevice::FairMQDevice(FairMQProgOptions* config, const fair::mq::tools::Ver
     , fMultitransportProceed(false)
     , fVersion(version)
     , fRate(0.)
+    , fMaxRunRuntimeInS(0)
     , fRawCmdLineArgs()
-    , fInterrupted(false)
-    , fInterruptedCV()
-    , fInterruptedMtx()
-    , fRateLogging(true)
 {
+    SubscribeToNewTransition("device", [&](Transition transition) {
+        LOG(trace) << "device notified on new transition: " << transition;
+
+        switch (transition) {
+            case Transition::Stop:
+                UnblockTransports();
+                break;
+            default:
+                break;
+        }
+    });
+
+    fStateMachine.HandleStates([&](fair::mq::State state) {
+        LOG(trace) << "device notified on new state: " << state;
+
+        {
+            lock_guard<mutex> lock(fStatesMtx);
+            fStates.push(state);
+        }
+        fStatesCV.notify_all();
+
+        switch (state) {
+            case fair::mq::State::InitializingDevice:
+                InitWrapper();
+                break;
+            case fair::mq::State::Binding:
+                BindWrapper();
+                break;
+            case fair::mq::State::Connecting:
+                ConnectWrapper();
+                break;
+            case fair::mq::State::InitializingTask:
+                InitTaskWrapper();
+                break;
+            case fair::mq::State::Running:
+                RunWrapper();
+                break;
+            case fair::mq::State::ResettingTask:
+                ResetTaskWrapper();
+                break;
+            case fair::mq::State::ResettingDevice:
+                ResetWrapper();
+                break;
+            case fair::mq::State::Exiting:
+                Exit();
+                break;
+            default:
+                LOG(trace) << "device notified on new state without a matching handler: " << state;
+                break;
+        }
+    });
+
+    fStateMachine.Start();
+}
+
+fair::mq::State FairMQDevice::WaitForNextState()
+{
+    unique_lock<mutex> lock(fStatesMtx);
+    while (fStates.empty()) {
+        fStatesCV.wait_for(lock, chrono::milliseconds(50));
+    }
+
+    auto result = fStates.front();
+
+    if (result == fair::mq::State::Error) {
+        throw DeviceStateError("Device transitioned to error state.");
+    }
+
+    fStates.pop();
+
+    return result;
+}
+
+void FairMQDevice::WaitForState(fair::mq::State state)
+{
+    while (WaitForNextState() != state) {}
+}
+
+bool FairMQDevice::ChangeState(const int transition)
+{
+    return ChangeState(backwardsCompatibilityChangeStateHelper.at(transition));
+}
+
+void FairMQDevice::WaitForEndOfState(Transition transition)
+{
+    WaitForState(backwardsCompatibilityWaitForEndOfStateHelper.at(transition));
 }
 
 void FairMQDevice::InitWrapper()
 {
+    fStateMachine.WaitForPendingState();
+
     fId = fConfig->GetValue<string>("id");
+
+    Init();
+
     fRate = fConfig->GetValue<float>("rate");
+    fMaxRunRuntimeInS = fConfig->GetValue<uint64_t>("max-run-time");
 
     try {
         fDefaultTransportType = fair::mq::TransportTypes.at(fConfig->GetValue<string>("transport"));
@@ -96,13 +217,9 @@ void FairMQDevice::InitWrapper()
         }
     }
 
-    LOG(debug) << "Requesting '" << fair::mq::TransportNames.at(fDefaultTransportType) << "' as default transport for the device";
+    LOG(debug) << "Setting '" << fair::mq::TransportNames.at(fDefaultTransportType) << "' as default transport for the device";
     fTransportFactory = AddTransport(fDefaultTransportType);
 
-    // Containers to store the uninitialized channels.
-    vector<FairMQChannel*> uninitializedBindingChannels;
-    vector<FairMQChannel*> uninitializedConnectingChannels;
-
     string networkInterface = fConfig->GetValue<string>("network-interface");
 
     // Fill the uninitialized channel containers
@@ -110,54 +227,59 @@ void FairMQDevice::InitWrapper()
         int subChannelIndex = 0;
         for (auto& vi : mi.second) {
             // set channel name: name + vector index
-            vi.fName = fair::mq::tools::ToString(mi.first, "[", subChannelIndex, "]");
+            vi.fName = tools::ToString(mi.first, "[", subChannelIndex, "]");
 
             // set channel transport
-            if (vi.fTransportType == fair::mq::Transport::DEFAULT || vi.fTransportType == fTransportFactory->GetType()) {
-                LOG(debug) << vi.fName << ": using default transport";
-                vi.InitTransport(fTransportFactory);
-            } else {
-                LOG(debug) << vi.fName << ": channel transport (" << fair::mq::TransportNames.at(fDefaultTransportType) << ") overriden to " << fair::mq::TransportNames.at(vi.fTransportType);
-                vi.InitTransport(AddTransport(vi.fTransportType));
-            }
+            LOG(debug) << "Initializing transport for channel " << vi.fName << ": " << fair::mq::TransportNames.at(vi.fTransportType);
+            vi.InitTransport(AddTransport(vi.fTransportType));
 
             if (vi.fMethod == "bind") {
                 // if binding address is not specified, try getting it from the configured network interface
                 if (vi.fAddress == "unspecified" || vi.fAddress == "") {
                     // if the configured network interface is default, get its name from the default route
                     if (networkInterface == "default") {
-                        networkInterface = fair::mq::tools::getDefaultRouteNetworkInterface();
+                        networkInterface = tools::getDefaultRouteNetworkInterface();
                     }
-                    vi.fAddress = "tcp://" + fair::mq::tools::getInterfaceIP(networkInterface) + ":1";
+                    vi.fAddress = "tcp://" + tools::getInterfaceIP(networkInterface) + ":1";
                 }
                 // fill the uninitialized list
-                uninitializedBindingChannels.push_back(&vi);
+                fUninitializedBindingChannels.push_back(&vi);
             } else if (vi.fMethod == "connect") {
                 // fill the uninitialized list
-                uninitializedConnectingChannels.push_back(&vi);
+                fUninitializedConnectingChannels.push_back(&vi);
             } else if (vi.fAddress.find_first_of("@+>") != string::npos) {
                 // fill the uninitialized list
-                uninitializedConnectingChannels.push_back(&vi);
+                fUninitializedConnectingChannels.push_back(&vi);
             } else {
                 LOG(error) << "Cannot update configuration. Socket method (bind/connect) for channel '" << vi.fName << "' not specified.";
-                throw runtime_error(fair::mq::tools::ToString("Cannot update configuration. Socket method (bind/connect) for channel ", vi.fName, " not specified."));
+                throw runtime_error(tools::ToString("Cannot update configuration. Socket method (bind/connect) for channel ", vi.fName, " not specified."));
             }
 
             subChannelIndex++;
         }
     }
 
+    // ChangeState(Transition::Auto);
+}
+
+void FairMQDevice::BindWrapper()
+{
     // Bind channels. Here one run is enough, because bind settings should be available locally
     // If necessary this could be handled in the same way as the connecting channels
-    AttachChannels(uninitializedBindingChannels);
+    AttachChannels(fUninitializedBindingChannels);
 
-    if (!uninitializedBindingChannels.empty()) {
-        LOG(error) << uninitializedBindingChannels.size() << " of the binding channels could not initialize. Initial configuration incomplete.";
-        throw runtime_error(fair::mq::tools::ToString(uninitializedBindingChannels.size(), " of the binding channels could not initialize. Initial configuration incomplete."));
+    if (!fUninitializedBindingChannels.empty()) {
+        LOG(error) << fUninitializedBindingChannels.size() << " of the binding channels could not initialize. Initial configuration incomplete.";
+        throw runtime_error(tools::ToString(fUninitializedBindingChannels.size(), " of the binding channels could not initialize. Initial configuration incomplete."));
     }
 
-    CallStateChangeCallbacks(INITIALIZING_DEVICE);
+    Bind();
 
+    ChangeState(Transition::Auto);
+}
+
+void FairMQDevice::ConnectWrapper()
+{
     int initializationTimeoutInS = fConfig->GetValue<int>("initialization-timeout");
 
     // go over the list of channels until all are initialized (and removed from the uninitialized list)
@@ -165,12 +287,12 @@ void FairMQDevice::InitWrapper()
     auto sleepTimeInMS = 50;
     auto maxAttempts = initializationTimeoutInS * 1000 / sleepTimeInMS;
     // first attempt
-    AttachChannels(uninitializedConnectingChannels);
+    AttachChannels(fUninitializedConnectingChannels);
     // if not all channels could be connected, update their address values from config and retry
-    while (!uninitializedConnectingChannels.empty()) {
+    while (!fUninitializedConnectingChannels.empty()) {
         this_thread::sleep_for(chrono::milliseconds(sleepTimeInMS));
 
-        for (auto& chan : uninitializedConnectingChannels) {
+        for (auto& chan : fUninitializedConnectingChannels) {
             string key{"chans." + chan->GetChannelPrefix() + "." + chan->GetChannelIndex() + ".address"};
             string newAddress = fConfig->GetValue<string>(key);
             if (newAddress != chan->GetAddress()) {
@@ -180,23 +302,19 @@ void FairMQDevice::InitWrapper()
 
         if (numAttempts++ > maxAttempts) {
             LOG(error) << "could not connect all channels after " << initializationTimeoutInS << " attempts";
-            throw runtime_error(fair::mq::tools::ToString("could not connect all channels after ", initializationTimeoutInS, " attempts"));
+            throw runtime_error(tools::ToString("could not connect all channels after ", initializationTimeoutInS, " attempts"));
         }
 
-        AttachChannels(uninitializedConnectingChannels);
+        AttachChannels(fUninitializedConnectingChannels);
     }
 
-    Init();
-
     if (fChannels.empty()) {
         LOG(warn) << "No channels created after finishing initialization";
     }
 
-    ChangeState(internal_DEVICE_READY);
-}
+    Connect();
 
-void FairMQDevice::Init()
-{
+    ChangeState(Transition::Auto);
 }
 
 void FairMQDevice::AttachChannels(vector<FairMQChannel*>& chans)
@@ -249,7 +367,7 @@ bool FairMQDevice::AttachChannel(FairMQChannel& chan)
             string hostPart = addressString.substr(0, pos);
             if (!(bind && hostPart == "*")) {
                 string portPart = addressString.substr(pos + 1);
-                string resolvedHost = fair::mq::tools::getIpFromHostname(hostPart);
+                string resolvedHost = tools::getIpFromHostname(hostPart);
                 if (resolvedHost == "") {
                     return false;
                 }
@@ -295,15 +413,9 @@ bool FairMQDevice::AttachChannel(FairMQChannel& chan)
 
 void FairMQDevice::InitTaskWrapper()
 {
-    CallStateChangeCallbacks(INITIALIZING_TASK);
-
     InitTask();
 
-    ChangeState(internal_READY);
-}
-
-void FairMQDevice::InitTask()
-{
+    ChangeState(Transition::Auto);
 }
 
 bool FairMQDevice::SortSocketsByAddress(const FairMQChannel &lhs, const FairMQChannel &rhs)
@@ -322,7 +434,7 @@ void FairMQDevice::SortChannel(const string& name, const bool reindex)
             for (auto vi = fChannels.at(name).begin(); vi != fChannels.at(name).end(); ++vi)
             {
                 // set channel name: name + vector index
-                vi->fName = fair::mq::tools::ToString(name, "[", vi - fChannels.at(name).begin(), "]");
+                vi->fName = tools::ToString(name, "[", vi - fChannels.at(name).begin(), "]");
             }
         }
     }
@@ -334,21 +446,13 @@ void FairMQDevice::SortChannel(const string& name, const bool reindex)
 
 void FairMQDevice::RunWrapper()
 {
-    CallStateChangeCallbacks(RUNNING);
-
     LOG(info) << "DEVICE: Running...";
 
     // start the rate logger thread
-    fRateLogging = true;
     future<void> rateLogger = async(launch::async, &FairMQDevice::LogSocketRates, this);
 
     // notify transports to resume transfers
-    {
-        lock_guard<mutex> guard(fInterruptedMtx);
-        fInterrupted = false;
-    }
-    for (auto& t : fTransports)
-    {
+    for (auto& t : fTransports) {
         t.second->Resume();
     }
 
@@ -356,50 +460,43 @@ void FairMQDevice::RunWrapper()
         PreRun();
 
         // process either data callbacks or ConditionalRun/Run
-        if (fDataCallbacks)
-        {
+        if (fDataCallbacks) {
             // if only one input channel, do lightweight handling without additional polling.
-            if (fInputChannelKeys.size() == 1 && fChannels.at(fInputChannelKeys.at(0)).size() == 1)
-            {
+            if (fInputChannelKeys.size() == 1 && fChannels.at(fInputChannelKeys.at(0)).size() == 1) {
                 HandleSingleChannelInput();
-            }
-            else // otherwise do full handling with polling
-            {
+            } else {// otherwise do full handling with polling
                 HandleMultipleChannelInput();
             }
-        }
-        else
-        {
-            fair::mq::tools::RateLimiter rateLimiter(fRate);
+        } else {
+            tools::RateLimiter rateLimiter(fRate);
 
-            while (CheckCurrentState(RUNNING) && ConditionalRun())
-            {
-                if (fRate > 0.001)
-                {
+            while (!NewStatePending() && ConditionalRun()) {
+                if (fRate > 0.001) {
                     rateLimiter.maybe_sleep();
                 }
             }
 
             Run();
         }
+
+        // if Run() exited and the state is still RUNNING, transition to READY.
+        if (!NewStatePending()) {
+            UnblockTransports();
+            ChangeState(Transition::Stop);
+        }
+
+        PostRun();
+
     } catch (const out_of_range& oor) {
         LOG(error) << "out of range: " << oor.what();
         LOG(error) << "incorrect/incomplete channel configuration?";
-        fRateLogging = false;
+        ChangeState(Transition::ErrorFound);
         throw;
     } catch (...) {
-        fRateLogging = false;
+        ChangeState(Transition::ErrorFound);
         throw;
     }
 
-    // if Run() exited and the state is still RUNNING, transition to READY.
-    if (CheckCurrentState(RUNNING))
-    {
-        ChangeState(internal_READY);
-    }
-
-    PostRun();
-
     rateLogger.get();
 }
 
@@ -409,14 +506,14 @@ void FairMQDevice::HandleSingleChannelInput()
 
     if (fMsgInputs.size() > 0)
     {
-        while (CheckCurrentState(RUNNING) && proceed)
+        while (!NewStatePending() && proceed)
         {
             proceed = HandleMsgInput(fInputChannelKeys.at(0), fMsgInputs.begin()->second, 0);
         }
     }
     else if (fMultipartInputs.size() > 0)
     {
-        while (CheckCurrentState(RUNNING) && proceed)
+        while (!NewStatePending() && proceed)
         {
             proceed = HandleMultipartInput(fInputChannelKeys.at(0), fMultipartInputs.begin()->second, 0);
         }
@@ -468,7 +565,7 @@ void FairMQDevice::HandleMultipleChannelInput()
 
         FairMQPollerPtr poller(fChannels.at(fInputChannelKeys.at(0)).at(0).fTransportFactory->CreatePoller(fChannels, fInputChannelKeys));
 
-        while (CheckCurrentState(RUNNING) && proceed)
+        while (!NewStatePending() && proceed)
         {
             poller->Poll(200);
 
@@ -526,7 +623,7 @@ void FairMQDevice::PollForTransport(const FairMQTransportFactory* factory, const
     {
         FairMQPollerPtr poller(factory->CreatePoller(fChannels, channelKeys));
 
-        while (CheckCurrentState(RUNNING) && fMultitransportProceed)
+        while (!NewStatePending() && fMultitransportProceed)
         {
             poller->Poll(500);
 
@@ -568,7 +665,7 @@ void FairMQDevice::PollForTransport(const FairMQTransportFactory* factory, const
     catch (exception& e)
     {
         LOG(error) << "FairMQDevice::PollForTransport() failed: " << e.what() << ", going to ERROR state.";
-        throw runtime_error(fair::mq::tools::ToString("FairMQDevice::PollForTransport() failed: ", e.what(), ", going to ERROR state."));
+        throw runtime_error(tools::ToString("FairMQDevice::PollForTransport() failed: ", e.what(), ", going to ERROR state."));
     }
 }
 
@@ -600,58 +697,21 @@ bool FairMQDevice::HandleMultipartInput(const string& chName, const InputMultipa
     }
 }
 
-void FairMQDevice::Run()
+shared_ptr<FairMQTransportFactory> FairMQDevice::AddTransport(fair::mq::Transport transport)
 {
-}
-
-void FairMQDevice::PreRun()
-{
-}
-
-bool FairMQDevice::ConditionalRun()
-{
-    return false;
-}
-
-void FairMQDevice::PostRun()
-{
-}
-
-void FairMQDevice::PauseWrapper()
-{
-    CallStateChangeCallbacks(PAUSED);
-
-    Pause();
-}
-
-void FairMQDevice::Pause()
-{
-    while (CheckCurrentState(PAUSED))
-    {
-        this_thread::sleep_for(chrono::milliseconds(500));
-        LOG(debug) << "paused...";
+    if (transport == fair::mq::Transport::DEFAULT) {
+        transport = fDefaultTransportType;
     }
-    LOG(debug) << "Unpausing";
-}
 
-shared_ptr<FairMQTransportFactory> FairMQDevice::AddTransport(const fair::mq::Transport transport)
-{
     auto i = fTransports.find(transport);
 
-    if (i == fTransports.end())
-    {
+    if (i == fTransports.end()) {
+        LOG(debug) << "Adding '" << fair::mq::TransportNames.at(transport) << "' transport";
         auto tr = FairMQTransportFactory::CreateTransportFactory(fair::mq::TransportNames.at(transport), fId, fConfig);
-
-        LOG(debug) << "Adding '" << fair::mq::TransportNames.at(transport) << "' transport to the device.";
-
-        pair<fair::mq::Transport, shared_ptr<FairMQTransportFactory>> trPair(transport, tr);
-        fTransports.insert(trPair);
-
+        fTransports.insert({transport, tr});
         return tr;
-    }
-    else
-    {
-        LOG(debug) << "Reusing existing '" << fair::mq::TransportNames.at(transport) << "' transport.";
+    } else {
+        LOG(debug) << "Reusing existing '" << fair::mq::TransportNames.at(transport) << "' transport";
         return i->second;
     }
 }
@@ -664,9 +724,6 @@ void FairMQDevice::SetConfig(FairMQProgOptions& config)
 
 void FairMQDevice::LogSocketRates()
 {
-    chrono::time_point<chrono::high_resolution_clock> t0;
-    chrono::time_point<chrono::high_resolution_clock> t1;
-
     vector<FairMQSocket*> filteredSockets;
     vector<string> filteredChannelNames;
     vector<int> logIntervals;
@@ -675,71 +732,63 @@ void FairMQDevice::LogSocketRates()
     size_t chanNameLen = 0;
 
     // iterate over the channels map
-    for (const auto& mi : fChannels)
-    {
+    for (const auto& mi : fChannels) {
         // iterate over the channels vector
-        for (auto vi = (mi.second).begin(); vi != (mi.second).end(); ++vi)
-        {
-            if (vi->fRateLogging > 0)
-            {
+        for (auto vi = (mi.second).begin(); vi != (mi.second).end(); ++vi) {
+            if (vi->fRateLogging > 0) {
                 filteredSockets.push_back(vi->fSocket.get());
                 logIntervals.push_back(vi->fRateLogging);
                 intervalCounters.push_back(0);
-                filteredChannelNames.push_back(fair::mq::tools::ToString(mi.first, "[", vi - (mi.second).begin(), "]"));
+                filteredChannelNames.push_back(tools::ToString(mi.first, "[", vi - (mi.second).begin(), "]"));
                 chanNameLen = max(chanNameLen, filteredChannelNames.back().length());
             }
         }
     }
 
-    unsigned int numFilteredSockets = filteredSockets.size();
+    vector<unsigned long> bytesIn(filteredSockets.size());
+    vector<unsigned long> msgIn(filteredSockets.size());
+    vector<unsigned long> bytesOut(filteredSockets.size());
+    vector<unsigned long> msgOut(filteredSockets.size());
 
-    if (numFilteredSockets > 0)
-    {
-        vector<unsigned long> bytesIn(numFilteredSockets);
-        vector<unsigned long> msgIn(numFilteredSockets);
-        vector<unsigned long> bytesOut(numFilteredSockets);
-        vector<unsigned long> msgOut(numFilteredSockets);
+    vector<unsigned long> bytesInNew(filteredSockets.size());
+    vector<unsigned long> msgInNew(filteredSockets.size());
+    vector<unsigned long> bytesOutNew(filteredSockets.size());
+    vector<unsigned long> msgOutNew(filteredSockets.size());
 
-        vector<unsigned long> bytesInNew(numFilteredSockets);
-        vector<unsigned long> msgInNew(numFilteredSockets);
-        vector<unsigned long> bytesOutNew(numFilteredSockets);
-        vector<unsigned long> msgOutNew(numFilteredSockets);
+    vector<double> mbPerSecIn(filteredSockets.size());
+    vector<double> msgPerSecIn(filteredSockets.size());
+    vector<double> mbPerSecOut(filteredSockets.size());
+    vector<double> msgPerSecOut(filteredSockets.size());
 
-        vector<double> mbPerSecIn(numFilteredSockets);
-        vector<double> msgPerSecIn(numFilteredSockets);
-        vector<double> mbPerSecOut(numFilteredSockets);
-        vector<double> msgPerSecOut(numFilteredSockets);
+    int i = 0;
+    for (const auto& vi : filteredSockets) {
+        bytesIn.at(i) = vi->GetBytesRx();
+        bytesOut.at(i) = vi->GetBytesTx();
+        msgIn.at(i) = vi->GetMessagesRx();
+        msgOut.at(i) = vi->GetMessagesTx();
+        ++i;
+    }
 
-        int i = 0;
-        for (const auto& vi : filteredSockets)
-        {
-            bytesIn.at(i) = vi->GetBytesRx();
-            bytesOut.at(i) = vi->GetBytesTx();
-            msgIn.at(i) = vi->GetMessagesRx();
-            msgOut.at(i) = vi->GetMessagesTx();
-            ++i;
-        }
+    chrono::time_point<chrono::high_resolution_clock> t0(chrono::high_resolution_clock::now());
+    chrono::time_point<chrono::high_resolution_clock> t1;
+    uint64_t secondsElapsed = 0;
 
-        t0 = chrono::high_resolution_clock::now();
+    while (!NewStatePending()) {
+        WaitFor(chrono::seconds(1));
 
-        LOG(debug) << "<channel>: in: <#msgs> (<MB>) out: <#msgs> (<MB>)";
+        t1 = chrono::high_resolution_clock::now();
 
-        while (fRateLogging)
-        {
-            t1 = chrono::high_resolution_clock::now();
+        uint64_t msSinceLastLog = chrono::duration_cast<chrono::milliseconds>(t1 - t0).count();
 
-            unsigned long long msSinceLastLog = chrono::duration_cast<chrono::milliseconds>(t1 - t0).count();
+        i = 0;
 
-            i = 0;
+        for (const auto& vi : filteredSockets) {
+            intervalCounters.at(i)++;
 
-            for (const auto& vi : filteredSockets)
-            {
-                intervalCounters.at(i)++;
-
-                if (intervalCounters.at(i) == logIntervals.at(i))
-                {
-                    intervalCounters.at(i) = 0;
+            if (intervalCounters.at(i) == logIntervals.at(i)) {
+                intervalCounters.at(i) = 0;
 
+                if (msSinceLastLog > 0) {
                     bytesInNew.at(i) = vi->GetBytesRx();
                     msgInNew.at(i) = vi->GetMessagesRx();
                     bytesOutNew.at(i) = vi->GetBytesTx();
@@ -759,58 +808,42 @@ void FairMQDevice::LogSocketRates()
                               << "in: " << msgPerSecIn.at(i) << " (" << mbPerSecIn.at(i) << " MB) "
                               << "out: " << msgPerSecOut.at(i) << " (" << mbPerSecOut.at(i) << " MB)";
                 }
-
-                ++i;
             }
 
-            t0 = t1;
-            WaitFor(chrono::milliseconds(1000));
+            ++i;
+        }
+
+        t0 = t1;
+        if (fMaxRunRuntimeInS > 0 && ++secondsElapsed >= fMaxRunRuntimeInS) {
+            ChangeState(Transition::Stop);
         }
     }
 }
 
-void FairMQDevice::Unblock()
+void FairMQDevice::UnblockTransports()
 {
-    for (auto& t : fTransports)
-    {
+    for (auto& t : fTransports) {
         t.second->Interrupt();
     }
-    {
-        lock_guard<mutex> guard(fInterruptedMtx);
-        fInterrupted = true;
-        fRateLogging = false;
-    }
-    fInterruptedCV.notify_all();
 }
 
 void FairMQDevice::ResetTaskWrapper()
 {
-    CallStateChangeCallbacks(RESETTING_TASK);
-
     ResetTask();
 
-    ChangeState(internal_DEVICE_READY);
-}
-
-void FairMQDevice::ResetTask()
-{
+    ChangeState(Transition::Auto);
 }
 
 void FairMQDevice::ResetWrapper()
 {
-    CallStateChangeCallbacks(RESETTING_DEVICE);
-
-    for (auto& t : fTransports)
-    {
+    for (auto& t : fTransports) {
         t.second->Reset();
     }
 
     // iterate over the channels map
-    for (auto& mi : fChannels)
-    {
+    for (auto& mi : fChannels) {
         // iterate over the channels vector
-        for (auto& vi : mi.second)
-        {
+        for (auto& vi : mi.second) {
             // vi.fReset = true;
             vi.fSocket.reset(); // destroy FairMQSocket
         }
@@ -818,18 +851,12 @@ void FairMQDevice::ResetWrapper()
 
     Reset();
 
-    ChangeState(internal_IDLE);
-}
-
-void FairMQDevice::Reset()
-{
-}
-
-void FairMQDevice::Exit()
-{
+    ChangeState(Transition::Auto);
 }
 
 FairMQDevice::~FairMQDevice()
 {
-    LOG(debug) << "Destructing device " << fId;
+    UnsubscribeFromNewTransition("device");
+    fStateMachine.StopHandlingStates();
+    LOG(debug) << "Shutting down device " << fId;
 }

fairmq/FairMQDevice.h

@@ -9,7 +9,7 @@
 #ifndef FAIRMQDEVICE_H_
 #define FAIRMQDEVICE_H_
 
-#include <FairMQStateMachine.h>
+#include <StateMachine.h>
 #include <FairMQTransportFactory.h>
 #include <fairmq/Transports.h>
 
@@ -32,6 +32,7 @@
 #include <assert.h> // static_assert
 #include <type_traits> // is_trivially_copyable
 #include <stdexcept>
+#include <queue>
 
 #include <mutex>
 #include <condition_variable>
@@ -43,11 +44,43 @@ using FairMQChannelMap = std::unordered_map<std::string, std::vector<FairMQChann
 using InputMsgCallback = std::function<bool(FairMQMessagePtr&, int)>;
 using InputMultipartCallback = std::function<bool(FairMQParts&, int)>;
 
-class FairMQDevice : public FairMQStateMachine
+class FairMQDevice
 {
     friend class FairMQChannel;
 
   public:
+    // backwards-compatibility enum for old state machine interface, todo: delete this
+    enum Event
+    {
+        INIT_DEVICE,
+        internal_DEVICE_READY,
+        INIT_TASK,
+        internal_READY,
+        RUN,
+        STOP,
+        RESET_TASK,
+        RESET_DEVICE,
+        internal_IDLE,
+        END,
+        ERROR_FOUND
+    };
+
+    // backwards-compatibility enum for old state machine interface, todo: delete this
+    enum State
+    {
+        OK,
+        Error,
+        IDLE,
+        INITIALIZING_DEVICE,
+        DEVICE_READY,
+        INITIALIZING_TASK,
+        READY,
+        RUNNING,
+        RESETTING_TASK,
+        RESETTING_DEVICE,
+        EXITING
+    };
+
     /// Default constructor
     FairMQDevice();
     /// Constructor with external FairMQProgOptions
@@ -70,9 +103,6 @@ class FairMQDevice : public FairMQStateMachine
     /// Default destructor
     virtual ~FairMQDevice();
 
-    /// Catches interrupt signals (SIGINT, SIGTERM)
-    void CatchSignals();
-
     /// Outputs the socket transfer rates
     virtual void LogSocketRates();
 
@@ -161,50 +191,58 @@ class FairMQDevice : public FairMQStateMachine
         return fTransportFactory.get();
     }
 
+    // creates message with the default device transport
     template<typename... Args>
     FairMQMessagePtr NewMessage(Args&&... args)
     {
         return Transport()->CreateMessage(std::forward<Args>(args)...);
     }
 
+    // creates message with the transport of the specified channel
     template<typename... Args>
     FairMQMessagePtr NewMessageFor(const std::string& channel, int index, Args&&... args)
     {
         return GetChannel(channel, index).NewMessage(std::forward<Args>(args)...);
     }
 
+    // creates a message that will not be cleaned up after transfer, with the default device transport
     template<typename T>
     FairMQMessagePtr NewStaticMessage(const T& data)
     {
         return Transport()->NewStaticMessage(data);
     }
 
+    // creates a message that will not be cleaned up after transfer, with the transport of the specified channel
     template<typename T>
     FairMQMessagePtr NewStaticMessageFor(const std::string& channel, int index, const T& data)
     {
         return GetChannel(channel, index).NewStaticMessage(data);
     }
 
+    // creates a message with a copy of the provided data, with the default device transport
     template<typename T>
     FairMQMessagePtr NewSimpleMessage(const T& data)
     {
         return Transport()->NewSimpleMessage(data);
     }
 
+    // creates a message with a copy of the provided data, with the transport of the specified channel
     template<typename T>
     FairMQMessagePtr NewSimpleMessageFor(const std::string& channel, int index, const T& data)
     {
         return GetChannel(channel, index).NewSimpleMessage(data);
     }
 
-    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size)
+    // creates unamanaged region with the default device transport
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr)
     {
-        return Transport()->CreateUnmanagedRegion(size);
+        return Transport()->CreateUnmanagedRegion(size, callback);
     }
 
+    // creates unmanaged region with the transport of the specified channel
     FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel, int index, const size_t size, FairMQRegionCallback callback = nullptr)
     {
-        return GetChannel(channel, index).Transport()->CreateUnmanagedRegion(size, callback);
+        return GetChannel(channel, index).NewUnmanagedRegion(size, callback);
     }
 
     template<typename ...Ts>
@@ -328,9 +366,9 @@ class FairMQDevice : public FairMQStateMachine
     try {
         return fChannels.at(channelName).at(index);
     } catch (const std::out_of_range& oor) {
-        LOG(error) << "out of range: " << oor.what();
         LOG(error) << "requested channel has not been configured? check channel names/configuration.";
-        fRateLogging = false;
+        LOG(error) << "channel: " << channelName << ", index: " << index;
+        LOG(error) << "out of range: " << oor.what();
         throw;
     }
 
@@ -339,8 +377,7 @@ class FairMQDevice : public FairMQStateMachine
     bool RegisterChannelEndpoint(const std::string& channelName, uint16_t minNumSubChannels = 1, uint16_t maxNumSubChannels = 1)
     {
         bool ok = fChannelRegistry.insert(std::make_pair(channelName, std::make_pair(minNumSubChannels, maxNumSubChannels))).second;
-        if (!ok)
-        {
+        if (!ok) {
             LOG(warn) << "Registering channel: name already registered: \"" << channelName << "\"";
         }
         return ok;
@@ -348,14 +385,10 @@ class FairMQDevice : public FairMQStateMachine
 
     void PrintRegisteredChannels()
     {
-        if (fChannelRegistry.size() < 1)
-        {
+        if (fChannelRegistry.size() < 1) {
             std::cout << "no channels registered." << std::endl;
-        }
-        else
-        {
-            for (const auto& c : fChannelRegistry)
-            {
+        } else {
+            for (const auto& c : fChannelRegistry) {
                 std::cout << c.first << ":" << c.second.first << ":" << c.second.second << std::endl;
             }
         }
@@ -389,8 +422,7 @@ class FairMQDevice : public FairMQStateMachine
 
     void RunStateMachine()
     {
-      CallStateChangeCallbacks(FairMQStateMachine::IDLE);
-      ProcessWork();
+        fStateMachine.ProcessWork();
     };
 
     /// Wait for the supplied amount of time or for interruption.
@@ -399,8 +431,7 @@ class FairMQDevice : public FairMQStateMachine
     template<class Rep, class Period>
     bool WaitFor(std::chrono::duration<Rep, Period> const& duration)
     {
-        std::unique_lock<std::mutex> lock(fInterruptedMtx);
-        return !fInterruptedCV.wait_for(lock, duration, [&] { return fInterrupted.load(); }); // return true if no interruption happened
+        return !fStateMachine.WaitForPendingStateFor(std::chrono::duration_cast<std::chrono::milliseconds>(duration).count());
     }
 
   protected:
@@ -421,53 +452,92 @@ class FairMQDevice : public FairMQStateMachine
     std::string fId; ///< Device ID
 
     /// Additional user initialization (can be overloaded in child classes). Prefer to use InitTask().
-    virtual void Init();
+    virtual void Init() {}
+
+    virtual void Bind() {}
+
+    virtual void Connect() {}
 
     /// Task initialization (can be overloaded in child classes)
-    virtual void InitTask();
+    virtual void InitTask() {}
 
     /// Runs the device (to be overloaded in child classes)
-    virtual void Run();
+    virtual void Run() {}
 
     /// Called in the RUNNING state once before executing the Run()/ConditionalRun() method
-    virtual void PreRun();
+    virtual void PreRun() {}
 
     /// Called during RUNNING state repeatedly until it returns false or device state changes
-    virtual bool ConditionalRun();
+    virtual bool ConditionalRun() { return false; }
 
     /// Called in the RUNNING state once after executing the Run()/ConditionalRun() method
-    virtual void PostRun();
+    virtual void PostRun() {}
 
-    /// Handles the PAUSE state
-    virtual void Pause();
+    virtual void Pause() __attribute__((deprecated("PAUSE state is removed. This method is never called. To pause Run, go to READY with STOP transition and back to RUNNING with RUN to resume."))) {}
 
     /// Resets the user task (to be overloaded in child classes)
-    virtual void ResetTask();
+    virtual void ResetTask() {}
 
     /// Resets the device (can be overloaded in child classes)
-    virtual void Reset();
+    virtual void Reset() {}
+
+  public:
+    bool ChangeState(const fair::mq::Transition transition) { return fStateMachine.ChangeState(transition); }
+    bool ChangeState(const std::string& transition) { return fStateMachine.ChangeState(fair::mq::StateMachine::GetTransition(transition)); }
+
+    bool ChangeState(const int transition) __attribute__((deprecated("Use ChangeState(const fair::mq::Transition transition).")));
+
+    void WaitForEndOfState(const fair::mq::Transition transition) __attribute__((deprecated("Use WaitForState(fair::mq::State expectedState).")));
+    void WaitForEndOfState(const std::string& transition) __attribute__((deprecated("Use WaitForState(fair::mq::State expectedState)."))) { WaitForState(transition); }
+
+    fair::mq::State WaitForNextState();
+    void WaitForState(fair::mq::State state);
+    void WaitForState(const std::string& state) { fair::mq::StateMachine::GetState(state); }
+
+    void SubscribeToStateChange(const std::string& key, std::function<void(const fair::mq::State)> callback) { fStateMachine.SubscribeToStateChange(key, callback); }
+    void UnsubscribeFromStateChange(const std::string& key) { fStateMachine.UnsubscribeFromStateChange(key); }
+
+    void SubscribeToNewTransition(const std::string& key, std::function<void(const fair::mq::Transition)> callback) { fStateMachine.SubscribeToNewTransition(key, callback); }
+    void UnsubscribeFromNewTransition(const std::string& key) { fStateMachine.UnsubscribeFromNewTransition(key); }
+
+    bool CheckCurrentState(const int /* state */) const __attribute__((deprecated("Use NewStatePending()."))) { return !fStateMachine.NewStatePending(); }
+    bool CheckCurrentState(const std::string& /* state */) const __attribute__((deprecated("Use NewStatePending()."))) { return !fStateMachine.NewStatePending(); }
+
+    /// Returns true is a new state has been requested, signaling the current handler to stop.
+    bool NewStatePending() const { return fStateMachine.NewStatePending(); }
+
+    fair::mq::State GetCurrentState() const { return fStateMachine.GetCurrentState(); }
+    std::string GetCurrentStateName() const { return fStateMachine.GetCurrentStateName(); }
+
+    static std::string GetStateName(const fair::mq::State state) { return fair::mq::StateMachine::GetStateName(state); }
+    static std::string GetTransitionName(const fair::mq::Transition transition) { return fair::mq::StateMachine::GetTransitionName(transition); }
+
+    struct DeviceStateError : std::runtime_error { using std::runtime_error::runtime_error; };
 
   private:
     fair::mq::Transport fDefaultTransportType; ///< Default transport for the device
+    fair::mq::StateMachine fStateMachine;
 
-    /// Handles the initialization and the Init() method
+    /// Handles the initialization
     void InitWrapper();
+    /// Initializes binding channels
+    void BindWrapper();
+    /// Initializes connecting channels
+    void ConnectWrapper();
     /// Handles the InitTask() method
     void InitTaskWrapper();
     /// Handles the Run() method
     void RunWrapper();
-    /// Handles the Pause() method
-    void PauseWrapper();
     /// Handles the ResetTask() method
     void ResetTaskWrapper();
     /// Handles the Reset() method
     void ResetWrapper();
 
-    /// Unblocks blocking channel send/receive calls
-    void Unblock();
+    /// Notifies transports to cease any blocking activity
+    void UnblockTransports();
 
     /// Shuts down the transports and the device
-    void Exit();
+    void Exit() {}
 
     /// Attach (bind/connect) channels in the list
     void AttachChannels(std::vector<FairMQChannel*>& chans);
@@ -481,7 +551,8 @@ class FairMQDevice : public FairMQStateMachine
     bool HandleMsgInput(const std::string& chName, const InputMsgCallback& callback, int i);
     bool HandleMultipartInput(const std::string& chName, const InputMultipartCallback& callback, int i);
 
-    void CreateOwnConfig();
+    std::vector<FairMQChannel*> fUninitializedBindingChannels;
+    std::vector<FairMQChannel*> fUninitializedConnectingChannels;
 
     bool fDataCallbacks;
     std::unordered_map<std::string, InputMsgCallback> fMsgInputs;
@@ -494,12 +565,12 @@ class FairMQDevice : public FairMQStateMachine
 
     const fair::mq::tools::Version fVersion;
     float fRate; ///< Rate limiting for ConditionalRun
+    uint64_t fMaxRunRuntimeInS; ///< Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).
     std::vector<std::string> fRawCmdLineArgs;
 
-    std::atomic<bool> fInterrupted;
-    std::condition_variable fInterruptedCV;
-    std::mutex fInterruptedMtx;
-    mutable std::atomic<bool> fRateLogging;
+    std::queue<fair::mq::State> fStates;
+    std::mutex fStatesMtx;
+    std::condition_variable fStatesCV;
 };
 
 #endif /* FAIRMQDEVICE_H_ */

fairmq/FairMQMessage.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/FairMQParts.h

@@ -31,6 +31,9 @@ class FairMQParts
     FairMQParts(FairMQParts&& p) = default;
     /// Assignment operator
     FairMQParts& operator=(const FairMQParts&) = delete;
+    /// Constructor from argument pack of std::unique_ptr<FairMQMessage> rvalues
+    template <typename... Ts>
+    FairMQParts(Ts&&... messages) : fParts() {AddPart(std::forward<Ts>(messages)...);}
     /// Default destructor
     ~FairMQParts() {};
 
@@ -41,14 +44,6 @@ class FairMQParts
         fParts.push_back(std::unique_ptr<FairMQMessage>(msg));
     }
 
-    /// Adds part (std::unique_ptr<FairMQMessage>&) to the container (move)
-    /// @param msg unique pointer to FairMQMessage
-    /// lvalue ref (move not required when passing argument)
-    // inline void AddPart(std::unique_ptr<FairMQMessage>& msg)
-    // {
-    //     fParts.push_back(std::move(msg));
-    // }
-
     /// Adds part (std::unique_ptr<FairMQMessage>&) to the container (move)
     /// @param msg unique pointer to FairMQMessage
     /// rvalue ref (move required when passing argument)
@@ -57,6 +52,23 @@ class FairMQParts
         fParts.push_back(std::move(msg));
     }
 
+    /// Add variable list of parts to the container (move)
+    template <typename... Ts>
+    void AddPart(std::unique_ptr<FairMQMessage>&& first, Ts&&... remaining)
+    {
+        AddPart(std::move(first));
+        AddPart(std::forward<Ts>(remaining)...);
+    }
+
+    /// Add content of another object by move
+    void AddPart(FairMQParts&& other)
+    {
+      container parts = std::move(other.fParts);
+      for (auto& part : parts) {
+	fParts.push_back(std::move(part));
+      }
+    }
+
     /// Get reference to part in the container at index (without bounds check)
     /// @param index container index
     FairMQMessage& operator[](const int index) { return *(fParts[index]); }

fairmq/FairMQPoller.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/FairMQPoller.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/FairMQSocket.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/FairMQSocket.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/FairMQStateMachine.cxx

@@ -1,672 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQStateMachine.cxx
- *
- * @since 2012-10-25
- * @author D. Klein, A. Rybalchenko
- */
-
-#include "FairMQStateMachine.h"
-#include <fairmq/Tools.h>
-
-// Increase maximum number of boost::msm states (default is 10)
-// This #define has to be before any msm header includes
-#define FUSION_MAX_VECTOR_SIZE 20
-
-#include <boost/mpl/for_each.hpp>
-#include <boost/msm/back/state_machine.hpp>
-#include <boost/msm/back/tools.hpp>
-#include <boost/msm/back/metafunctions.hpp>
-#include <boost/msm/front/state_machine_def.hpp>
-#include <boost/msm/front/functor_row.hpp>
-#include <boost/core/demangle.hpp>
-#include <boost/signals2.hpp> // signal/slot for onStateChange callbacks
-
-#include <atomic>
-#include <condition_variable>
-#include <chrono>
-#include <array>
-#include <unordered_map>
-
-using namespace std;
-using namespace boost::msm::front;
-
-namespace std
-{
-
-template<>
-struct hash<FairMQStateMachine::Event> : fair::mq::tools::HashEnum<FairMQStateMachine::Event> {};
-
-} /* namespace std */
-
-namespace fair
-{
-namespace mq
-{
-namespace fsm
-{
-
-// list of FSM states
-struct OK_FSM_STATE                  : public state<> { static string Name() { return "OK"; }                  static FairMQStateMachine::State Type() { return FairMQStateMachine::State::OK; } };
-struct ERROR_FSM_STATE               : public terminate_state<> { static string Name() { return "ERROR"; }     static FairMQStateMachine::State Type() { return FairMQStateMachine::State::Error; } };
-
-struct IDLE_FSM_STATE                : public state<> { static string Name() { return "IDLE"; }                static FairMQStateMachine::State Type() { return FairMQStateMachine::State::IDLE; } };
-struct INITIALIZING_DEVICE_FSM_STATE : public state<> { static string Name() { return "INITIALIZING_DEVICE"; } static FairMQStateMachine::State Type() { return FairMQStateMachine::State::INITIALIZING_DEVICE; } };
-struct DEVICE_READY_FSM_STATE        : public state<> { static string Name() { return "DEVICE_READY"; }        static FairMQStateMachine::State Type() { return FairMQStateMachine::State::DEVICE_READY; } };
-struct INITIALIZING_TASK_FSM_STATE   : public state<> { static string Name() { return "INITIALIZING_TASK"; }   static FairMQStateMachine::State Type() { return FairMQStateMachine::State::INITIALIZING_TASK; } };
-struct READY_FSM_STATE               : public state<> { static string Name() { return "READY"; }               static FairMQStateMachine::State Type() { return FairMQStateMachine::State::READY; } };
-struct RUNNING_FSM_STATE             : public state<> { static string Name() { return "RUNNING"; }             static FairMQStateMachine::State Type() { return FairMQStateMachine::State::RUNNING; } };
-struct PAUSED_FSM_STATE              : public state<> { static string Name() { return "PAUSED"; }              static FairMQStateMachine::State Type() { return FairMQStateMachine::State::PAUSED; } };
-struct RESETTING_TASK_FSM_STATE      : public state<> { static string Name() { return "RESETTING_TASK"; }      static FairMQStateMachine::State Type() { return FairMQStateMachine::State::RESETTING_TASK; } };
-struct RESETTING_DEVICE_FSM_STATE    : public state<> { static string Name() { return "RESETTING_DEVICE"; }    static FairMQStateMachine::State Type() { return FairMQStateMachine::State::RESETTING_DEVICE; } };
-struct EXITING_FSM_STATE             : public state<> { static string Name() { return "EXITING"; }             static FairMQStateMachine::State Type() { return FairMQStateMachine::State::EXITING; } };
-
-// list of FSM events
-struct INIT_DEVICE_FSM_EVENT           { static string Name() { return "INIT_DEVICE"; }           static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::INIT_DEVICE; } };
-struct internal_DEVICE_READY_FSM_EVENT { static string Name() { return "internal_DEVICE_READY"; } static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::internal_DEVICE_READY; } };
-struct INIT_TASK_FSM_EVENT             { static string Name() { return "INIT_TASK"; }             static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::INIT_TASK; } };
-struct internal_READY_FSM_EVENT        { static string Name() { return "internal_READY"; }        static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::internal_READY; } };
-struct RUN_FSM_EVENT                   { static string Name() { return "RUN"; }                   static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::RUN; } };
-struct PAUSE_FSM_EVENT                 { static string Name() { return "PAUSE"; }                 static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::PAUSE; } };
-struct STOP_FSM_EVENT                  { static string Name() { return "STOP"; }                  static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::STOP; } };
-struct RESET_TASK_FSM_EVENT            { static string Name() { return "RESET_TASK"; }            static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::RESET_TASK; } };
-struct RESET_DEVICE_FSM_EVENT          { static string Name() { return "RESET_DEVICE"; }          static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::RESET_DEVICE; } };
-struct internal_IDLE_FSM_EVENT         { static string Name() { return "internal_IDLE"; }         static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::internal_IDLE; } };
-struct END_FSM_EVENT                   { static string Name() { return "END"; }                   static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::END; } };
-struct ERROR_FOUND_FSM_EVENT           { static string Name() { return "ERROR_FOUND"; }           static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::ERROR_FOUND; } };
-
-static array<string, 12> stateNames =
-{
-    {
-        "OK",
-        "Error",
-        "IDLE",
-        "INITIALIZING_DEVICE",
-        "DEVICE_READY",
-        "INITIALIZING_TASK",
-        "READY",
-        "RUNNING",
-        "PAUSED",
-        "RESETTING_TASK",
-        "RESETTING_DEVICE",
-        "EXITING"
-    }
-};
-
-static array<string, 12> eventNames =
-{
-    {
-        "INIT_DEVICE",
-        "internal_DEVICE_READY",
-        "INIT_TASK",
-        "internal_READY",
-        "RUN",
-        "PAUSE",
-        "STOP",
-        "RESET_TASK",
-        "RESET_DEVICE",
-        "internal_IDLE",
-        "END",
-        "ERROR_FOUND"
-    }
-};
-
-static map<string, int> stateNumbers =
-{
-    { "OK",                  FairMQStateMachine::State::OK },
-    { "Error",               FairMQStateMachine::State::Error },
-    { "IDLE",                FairMQStateMachine::State::IDLE },
-    { "INITIALIZING_DEVICE", FairMQStateMachine::State::INITIALIZING_DEVICE },
-    { "DEVICE_READY",        FairMQStateMachine::State::DEVICE_READY },
-    { "INITIALIZING_TASK",   FairMQStateMachine::State::INITIALIZING_TASK },
-    { "READY",               FairMQStateMachine::State::READY },
-    { "RUNNING",             FairMQStateMachine::State::RUNNING },
-    { "PAUSED",              FairMQStateMachine::State::PAUSED },
-    { "RESETTING_TASK",      FairMQStateMachine::State::RESETTING_TASK },
-    { "RESETTING_DEVICE",    FairMQStateMachine::State::RESETTING_DEVICE },
-    { "EXITING",             FairMQStateMachine::State::EXITING }
-};
-
-static map<string, int> eventNumbers =
-{
-    { "INIT_DEVICE",           FairMQStateMachine::Event::INIT_DEVICE },
-    { "internal_DEVICE_READY", FairMQStateMachine::Event::internal_DEVICE_READY },
-    { "INIT_TASK",             FairMQStateMachine::Event::INIT_TASK },
-    { "internal_READY",        FairMQStateMachine::Event::internal_READY },
-    { "RUN",                   FairMQStateMachine::Event::RUN },
-    { "PAUSE",                 FairMQStateMachine::Event::PAUSE },
-    { "STOP",                  FairMQStateMachine::Event::STOP },
-    { "RESET_TASK",            FairMQStateMachine::Event::RESET_TASK },
-    { "RESET_DEVICE",          FairMQStateMachine::Event::RESET_DEVICE },
-    { "internal_IDLE",         FairMQStateMachine::Event::internal_IDLE },
-    { "END",                   FairMQStateMachine::Event::END },
-    { "ERROR_FOUND",           FairMQStateMachine::Event::ERROR_FOUND }
-};
-
-// defining the boost MSM state machine
-struct Machine_ : public state_machine_def<Machine_>
-{
-  public:
-    Machine_()
-        : fUnblockHandler()
-        , fStateHandlers()
-        , fWork()
-        , fWorkAvailableCondition()
-        , fWorkDoneCondition()
-        , fWorkMutex()
-        , fWorkerTerminated(false)
-        , fWorkActive(false)
-        , fWorkAvailable(false)
-        , fStateChangeSignal()
-        , fStateChangeSignalsMap()
-        , fState()
-    {}
-
-    virtual ~Machine_()
-    {}
-
-    // initial states
-    using initial_state = boost::mpl::vector<IDLE_FSM_STATE, OK_FSM_STATE>;
-
-    template<typename Event, typename FSM>
-    void on_entry(Event const&, FSM& /*fsm*/)
-    {
-        LOG(state) << "Starting FairMQ state machine";
-        fState = FairMQStateMachine::IDLE;
-        LOG(state) << "Entering IDLE state";
-        // fsm.CallStateChangeCallbacks(FairMQStateMachine::IDLE);
-        // we call this for now in FairMQDevice::RunStateMachine()
-    }
-
-    template<typename Event, typename FSM>
-    void on_exit(Event const&, FSM& /*fsm*/)
-    {
-        LOG(state) << "Exiting FairMQ state machine";
-    }
-
-    // actions
-    struct AutomaticFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-            LOG(state) << "Entering " << ts.Name() << " state";
-        }
-    };
-
-    struct DefaultFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const& e, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-
-            unique_lock<mutex> lock(fsm.fWorkMutex);
-            while (fsm.fWorkActive)
-            {
-                fsm.fWorkDoneCondition.wait(lock);
-            }
-            fsm.fWorkAvailable = true;
-            LOG(state) << "Entering " << ts.Name() << " state";
-            fsm.fWork = fsm.fStateHandlers.at(e.Type());
-            fsm.fWorkAvailableCondition.notify_one();
-        }
-    };
-
-    struct PauseFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-
-            fsm.fUnblockHandler();
-            unique_lock<mutex> lock(fsm.fWorkMutex);
-            while (fsm.fWorkActive)
-            {
-                fsm.fWorkDoneCondition.wait(lock);
-            }
-            fsm.fWorkAvailable = true;
-            LOG(state) << "Entering " << ts.Name() << " state";
-            fsm.fWork = fsm.fPauseWrapperHandler;
-            fsm.fWorkAvailableCondition.notify_one();
-        }
-    };
-
-    struct StopFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-
-            fsm.fUnblockHandler();
-            unique_lock<mutex> lock(fsm.fWorkMutex);
-            while (fsm.fWorkActive)
-            {
-                fsm.fWorkDoneCondition.wait(lock);
-            }
-            LOG(state) << "Entering " << ts.Name() << " state";
-        }
-    };
-
-    struct InternalStopFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-            fsm.fUnblockHandler();
-            LOG(state) << "RUNNING state finished without an external event, entering " << ts.Name() << " state";
-        }
-    };
-
-    struct ExitingFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const& e, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            LOG(state) << "Entering " << ts.Name() << " state";
-            fsm.fState = ts.Type();
-            fsm.CallStateChangeCallbacks(FairMQStateMachine::EXITING);
-
-            // Stop ProcessWork()
-            {
-                lock_guard<mutex> lock(fsm.fWorkMutex);
-                fsm.fWorkerTerminated = true;
-                fsm.fWorkAvailableCondition.notify_one();
-            }
-
-            fsm.fStateHandlers.at(e.Type())();
-        }
-    };
-
-    struct ErrorFoundFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-            LOG(state) << "Entering " << ts.Name() << " state";
-            fsm.CallStateChangeCallbacks(FairMQStateMachine::Error);
-        }
-    };
-
-    // Transition table for Machine_
-    struct transition_table : boost::mpl::vector<
-        // Start                           Event                            Next                           Action           Guard
-        Row<IDLE_FSM_STATE,                INIT_DEVICE_FSM_EVENT,           INITIALIZING_DEVICE_FSM_STATE, DefaultFct,      none>,
-        Row<IDLE_FSM_STATE,                END_FSM_EVENT,                   EXITING_FSM_STATE,             ExitingFct,      none>,
-        Row<INITIALIZING_DEVICE_FSM_STATE, internal_DEVICE_READY_FSM_EVENT, DEVICE_READY_FSM_STATE,        AutomaticFct,    none>,
-        Row<DEVICE_READY_FSM_STATE,        INIT_TASK_FSM_EVENT,             INITIALIZING_TASK_FSM_STATE,   DefaultFct,      none>,
-        Row<DEVICE_READY_FSM_STATE,        RESET_DEVICE_FSM_EVENT,          RESETTING_DEVICE_FSM_STATE,    DefaultFct,      none>,
-        Row<INITIALIZING_TASK_FSM_STATE,   internal_READY_FSM_EVENT,        READY_FSM_STATE,               AutomaticFct,    none>,
-        Row<READY_FSM_STATE,               RUN_FSM_EVENT,                   RUNNING_FSM_STATE,             DefaultFct,      none>,
-        Row<READY_FSM_STATE,               RESET_TASK_FSM_EVENT,            RESETTING_TASK_FSM_STATE,      DefaultFct,      none>,
-        Row<RUNNING_FSM_STATE,             PAUSE_FSM_EVENT,                 PAUSED_FSM_STATE,              DefaultFct,      none>,
-        Row<RUNNING_FSM_STATE,             STOP_FSM_EVENT,                  READY_FSM_STATE,               StopFct,         none>,
-        Row<RUNNING_FSM_STATE,             internal_READY_FSM_EVENT,        READY_FSM_STATE,               InternalStopFct, none>,
-        Row<PAUSED_FSM_STATE,              RUN_FSM_EVENT,                   RUNNING_FSM_STATE,             DefaultFct,      none>,
-        Row<RESETTING_TASK_FSM_STATE,      internal_DEVICE_READY_FSM_EVENT, DEVICE_READY_FSM_STATE,        AutomaticFct,    none>,
-        Row<RESETTING_DEVICE_FSM_STATE,    internal_IDLE_FSM_EVENT,         IDLE_FSM_STATE,                AutomaticFct,    none>,
-        Row<OK_FSM_STATE,                  ERROR_FOUND_FSM_EVENT,           ERROR_FSM_STATE,               ErrorFoundFct,   none>>
-    {};
-
-    // replaces the default no-transition response.
-    template<typename FSM, typename Event>
-    void no_transition(Event const& e, FSM&, int state)
-    {
-        using recursive_stt = typename boost::msm::back::recursive_get_transition_table<FSM>::type;
-        using all_states = typename boost::msm::back::generate_state_set<recursive_stt>::type;
-
-        string stateName;
-
-        boost::mpl::for_each<all_states, boost::msm::wrap<boost::mpl::placeholders::_1>>(boost::msm::back::get_state_name<recursive_stt>(stateName, state));
-
-        stateName = boost::core::demangle(stateName.c_str());
-        size_t pos = stateName.rfind(":");
-        if (pos != string::npos)
-        {
-            stateName = stateName.substr(pos + 1);
-            stateName = stateName.substr(0, stateName.size() - 10);
-        }
-
-        if (stateName != "OK")
-        {
-            LOG(state) << "No transition from state " << stateName << " on event " << e.Name();
-        }
-    }
-
-    void CallStateChangeCallbacks(const FairMQStateMachine::State state) const
-    {
-        if (!fStateChangeSignal.empty())
-        {
-            fStateChangeSignal(state);
-        }
-    }
-
-    function<void(void)> fUnblockHandler;
-    unordered_map<FairMQStateMachine::Event, function<void(void)>> fStateHandlers;
-
-    // function to execute user states in a worker thread
-    function<void(void)> fWork;
-    condition_variable fWorkAvailableCondition;
-    condition_variable fWorkDoneCondition;
-    mutex fWorkMutex;
-    bool fWorkerTerminated;
-    bool fWorkActive;
-    bool fWorkAvailable;
-
-    boost::signals2::signal<void(const FairMQStateMachine::State)> fStateChangeSignal;
-    unordered_map<string, boost::signals2::connection> fStateChangeSignalsMap;
-
-    atomic<FairMQStateMachine::State> fState;
-
-    void ProcessWork()
-    {
-        while (true)
-        {
-            {
-                unique_lock<mutex> lock(fWorkMutex);
-                // Wait for work to be done.
-                while (!fWorkAvailable && !fWorkerTerminated)
-                {
-                    fWorkAvailableCondition.wait_for(lock, chrono::milliseconds(100));
-                }
-
-                if (fWorkerTerminated)
-                {
-                    break;
-                }
-
-                fWorkActive = true;
-            }
-
-            fWork();
-
-            {
-                lock_guard<mutex> lock(fWorkMutex);
-                fWorkActive = false;
-                fWorkAvailable = false;
-                fWorkDoneCondition.notify_one();
-            }
-            CallStateChangeCallbacks(fState);
-        }
-    }
-}; // Machine_
-
-using FairMQFSM = boost::msm::back::state_machine<Machine_>;
-
-} // namespace fsm
-} // namespace mq
-} // namespace fair
-
-using namespace fair::mq::fsm;
-
-FairMQStateMachine::FairMQStateMachine()
-    : fChangeStateMutex()
-    , fFsm(new FairMQFSM)
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(INIT_DEVICE, bind(&FairMQStateMachine::InitWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(INIT_TASK, bind(&FairMQStateMachine::InitTaskWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(RUN, bind(&FairMQStateMachine::RunWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(PAUSE, bind(&FairMQStateMachine::PauseWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(RESET_TASK, bind(&FairMQStateMachine::ResetTaskWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(RESET_DEVICE, bind(&FairMQStateMachine::ResetWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(END, bind(&FairMQStateMachine::Exit, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fUnblockHandler = bind(&FairMQStateMachine::Unblock, this);
-
-    static_pointer_cast<FairMQFSM>(fFsm)->start();
-}
-
-FairMQStateMachine::~FairMQStateMachine()
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->stop();
-}
-
-int FairMQStateMachine::GetInterfaceVersion() const
-{
-    return FAIRMQ_INTERFACE_VERSION;
-}
-
-bool FairMQStateMachine::ChangeState(int event)
-{
-    try
-    {
-        switch (event)
-        {
-            case INIT_DEVICE:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(INIT_DEVICE_FSM_EVENT());
-                return true;
-            }
-            case internal_DEVICE_READY:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(internal_DEVICE_READY_FSM_EVENT());
-                return true;
-            }
-            case INIT_TASK:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(INIT_TASK_FSM_EVENT());
-                return true;
-            }
-            case internal_READY:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(internal_READY_FSM_EVENT());
-                return true;
-            }
-            case RUN:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(RUN_FSM_EVENT());
-                return true;
-            }
-            case PAUSE:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(PAUSE_FSM_EVENT());
-                return true;
-            }
-            case STOP:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(STOP_FSM_EVENT());
-                return true;
-            }
-            case RESET_DEVICE:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(RESET_DEVICE_FSM_EVENT());
-                return true;
-            }
-            case RESET_TASK:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(RESET_TASK_FSM_EVENT());
-                return true;
-            }
-            case internal_IDLE:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(internal_IDLE_FSM_EVENT());
-                return true;
-            }
-            case END:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(END_FSM_EVENT());
-                return true;
-            }
-            case ERROR_FOUND:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(ERROR_FOUND_FSM_EVENT());
-                return true;
-            }
-            default:
-            {
-                LOG(error) << "Requested state transition with an unsupported event: " << event << endl
-                           << "Supported are: INIT_DEVICE, INIT_TASK, RUN, PAUSE, STOP, RESET_TASK, RESET_DEVICE, END, ERROR_FOUND";
-                return false;
-            }
-        }
-    }
-    catch (exception& e)
-    {
-        LOG(error) << "Exception in FairMQStateMachine::ChangeState(): " << e.what();
-        exit(EXIT_FAILURE);
-    }
-    return false;
-}
-
-bool FairMQStateMachine::ChangeState(const string& event)
-{
-    return ChangeState(GetEventNumber(event));
-}
-
-void FairMQStateMachine::WaitForEndOfState(int event)
-{
-    try
-    {
-        switch (event)
-        {
-            case INIT_DEVICE:
-            case INIT_TASK:
-            case RUN:
-            case RESET_TASK:
-            case RESET_DEVICE:
-            {
-                unique_lock<mutex> lock(static_pointer_cast<FairMQFSM>(fFsm)->fWorkMutex);
-                while (static_pointer_cast<FairMQFSM>(fFsm)->fWorkActive || static_pointer_cast<FairMQFSM>(fFsm)->fWorkAvailable)
-                {
-                    static_pointer_cast<FairMQFSM>(fFsm)->fWorkDoneCondition.wait_for(lock, chrono::seconds(1));
-                }
-
-                break;
-            }
-            default:
-                LOG(error) << "Requested state is either synchronous or does not exist.";
-                break;
-        }
-    }
-    catch (exception& e)
-    {
-        LOG(error) << "Exception in FairMQStateMachine::WaitForEndOfState(): " << e.what();
-    }
-}
-
-void FairMQStateMachine::WaitForEndOfState(const string& event)
-{
-    return WaitForEndOfState(GetEventNumber(event));
-}
-
-bool FairMQStateMachine::WaitForEndOfStateForMs(int event, int durationInMs)
-{
-    try
-    {
-        switch (event)
-        {
-            case INIT_DEVICE:
-            case INIT_TASK:
-            case RUN:
-            case RESET_TASK:
-            case RESET_DEVICE:
-            {
-                unique_lock<mutex> lock(static_pointer_cast<FairMQFSM>(fFsm)->fWorkMutex);
-                while (static_pointer_cast<FairMQFSM>(fFsm)->fWorkActive || static_pointer_cast<FairMQFSM>(fFsm)->fWorkAvailable)
-                {
-                    static_pointer_cast<FairMQFSM>(fFsm)->fWorkDoneCondition.wait_for(lock, chrono::milliseconds(durationInMs));
-                    if (static_pointer_cast<FairMQFSM>(fFsm)->fWorkActive)
-                    {
-                        return false;
-                    }
-                }
-                return true;
-            }
-            default:
-                LOG(error) << "Requested state is either synchronous or does not exist.";
-                return false;
-        }
-    }
-    catch (exception& e)
-    {
-        LOG(error) << "Exception in FairMQStateMachine::WaitForEndOfStateForMs(): " << e.what();
-    }
-    return false;
-}
-
-bool FairMQStateMachine::WaitForEndOfStateForMs(const string& event, int durationInMs)
-{
-    return WaitForEndOfStateForMs(GetEventNumber(event), durationInMs);
-}
-
-void FairMQStateMachine::SubscribeToStateChange(const string& key, function<void(const State)> callback)
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.insert({key, static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignal.connect(callback)});
-}
-void FairMQStateMachine::UnsubscribeFromStateChange(const string& key)
-{
-    if (static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.count(key))
-    {
-        static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.at(key).disconnect();
-        static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.erase(key);
-    }
-}
-
-void FairMQStateMachine::CallStateChangeCallbacks(const State state) const
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->CallStateChangeCallbacks(state);
-}
-
-string FairMQStateMachine::GetCurrentStateName() const
-{
-    return GetStateName(static_pointer_cast<FairMQFSM>(fFsm)->fState);
-}
-string FairMQStateMachine::GetStateName(const State state)
-{
-    return stateNames.at(state);
-}
-int FairMQStateMachine::GetCurrentState() const
-{
-    return static_pointer_cast<FairMQFSM>(fFsm)->fState;
-}
-bool FairMQStateMachine::CheckCurrentState(int state) const
-{
-    return state == static_pointer_cast<FairMQFSM>(fFsm)->fState;
-}
-bool FairMQStateMachine::CheckCurrentState(const string& state) const
-{
-    return state == GetCurrentStateName();
-}
-
-void FairMQStateMachine::ProcessWork()
-try
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->ProcessWork();
-} catch(...) {
-    {
-        lock_guard<mutex> lock(static_pointer_cast<FairMQFSM>(fFsm)->fWorkMutex);
-        static_pointer_cast<FairMQFSM>(fFsm)->fWorkActive = false;
-        static_pointer_cast<FairMQFSM>(fFsm)->fWorkAvailable = false;
-        static_pointer_cast<FairMQFSM>(fFsm)->fWorkDoneCondition.notify_one();
-    }
-    ChangeState(ERROR_FOUND);
-    throw;
-}
-
-
-int FairMQStateMachine::GetEventNumber(const string& event)
-{
-    return eventNumbers.at(event);
-}

fairmq/FairMQStateMachine.h

@@ -1,107 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQStateMachine.h
- *
- * @since 2012-10-25
- * @author D. Klein, A. Rybalchenko
- */
-
-#ifndef FAIRMQSTATEMACHINE_H_
-#define FAIRMQSTATEMACHINE_H_
-
-#define FAIRMQ_INTERFACE_VERSION 3
-
-#include "FairMQLogger.h"
-
-#include <string>
-#include <memory>
-#include <functional>
-#include <mutex>
-
-class FairMQStateMachine
-{
-  public:
-    enum Event
-    {
-        INIT_DEVICE,
-        internal_DEVICE_READY,
-        INIT_TASK,
-        internal_READY,
-        RUN,
-        PAUSE,
-        STOP,
-        RESET_TASK,
-        RESET_DEVICE,
-        internal_IDLE,
-        END,
-        ERROR_FOUND
-    };
-
-    enum State
-    {
-        OK,
-        Error,
-        IDLE,
-        INITIALIZING_DEVICE,
-        DEVICE_READY,
-        INITIALIZING_TASK,
-        READY,
-        RUNNING,
-        PAUSED,
-        RESETTING_TASK,
-        RESETTING_DEVICE,
-        EXITING
-    };
-
-    FairMQStateMachine();
-    virtual ~FairMQStateMachine();
-
-    int GetInterfaceVersion() const;
-
-    bool ChangeState(int event);
-    bool ChangeState(const std::string& event);
-
-    void WaitForEndOfState(int event);
-    void WaitForEndOfState(const std::string& event);
-
-    bool WaitForEndOfStateForMs(int event, int durationInMs);
-    bool WaitForEndOfStateForMs(const std::string& event, int durationInMs);
-
-    void SubscribeToStateChange(const std::string& key, std::function<void(const State)> callback);
-    void UnsubscribeFromStateChange(const std::string& key);
-
-    void CallStateChangeCallbacks(const State state) const;
-
-    std::string GetCurrentStateName() const;
-    static std::string GetStateName(const State);
-    int GetCurrentState() const;
-    bool CheckCurrentState(int state) const;
-    bool CheckCurrentState(const std::string& state) const;
-
-    // actions to be overwritten by derived classes
-    virtual void InitWrapper() {}
-    virtual void InitTaskWrapper() {}
-    virtual void RunWrapper() {}
-    virtual void PauseWrapper() {}
-    virtual void ResetWrapper() {}
-    virtual void ResetTaskWrapper() {}
-    virtual void Exit() {}
-    virtual void Unblock() {}
-
-    void ProcessWork();
-
-  private:
-    static int GetEventNumber(const std::string& event);
-
-    std::mutex fChangeStateMutex;
-
-    std::shared_ptr<void> fFsm;
-};
-
-#endif /* FAIRMQSTATEMACHINE_H_ */

fairmq/Plugin.h

@@ -78,7 +78,7 @@ class Plugin
     auto TakeDeviceControl() -> void { fPluginServices->TakeDeviceControl(fkName); };
     auto StealDeviceControl() -> void { fPluginServices->StealDeviceControl(fkName); };
     auto ReleaseDeviceControl() -> void { fPluginServices->ReleaseDeviceControl(fkName); };
-    auto ChangeDeviceState(const DeviceStateTransition next) -> void { fPluginServices->ChangeDeviceState(fkName, next); }
+    auto ChangeDeviceState(const DeviceStateTransition next) -> bool { return fPluginServices->ChangeDeviceState(fkName, next); }
     auto SubscribeToDeviceStateChange(std::function<void(DeviceState)> callback) -> void { fPluginServices->SubscribeToDeviceStateChange(fkName, callback); }
     auto UnsubscribeFromDeviceStateChange() -> void { fPluginServices->UnsubscribeFromDeviceStateChange(fkName); }
 

fairmq/PluginManager.cxx

@@ -29,7 +29,7 @@ using boost::optional;
 const std::string fair::mq::PluginManager::fgkLibPrefix = "FairMQPlugin_";
 
 fair::mq::PluginManager::PluginManager()
-    : fSearchPaths{{"."}}
+    : fSearchPaths{}
     , fPluginFactories()
     , fPluginServices()
     , fPlugins()
@@ -39,7 +39,7 @@ fair::mq::PluginManager::PluginManager()
 }
 
 fair::mq::PluginManager::PluginManager(const vector<string> args)
-    : fSearchPaths{{"."}}
+    : fSearchPaths{}
     , fPluginFactories()
     , fPluginServices()
     , fPlugins()
@@ -115,7 +115,8 @@ auto fair::mq::PluginManager::ProgramOptions() -> po::options_description
                                                                             "* Append(>) or prepend(<) to default search path, e.g.\n"
                                                                             "  -S >/lib </home/user/lib\n"
                                                                             "* If you mix the overriding and appending/prepending syntaxes, the overriding paths act as default search path, e.g.\n"
-                                                                            "  -S /usr/lib >/lib </home/user/lib /usr/local/lib results in /home/user/lib,/usr/local/lib,/usr/lib/,/lib")
+                                                                            "  -S /usr/lib >/lib </home/user/lib /usr/local/lib results in /home/user/lib,/usr/local/lib,/usr/lib/,/lib\n"
+                                                                            "If nothing is found, the default dynamic library lookup is performed, see man ld.so(8) for details.")
         ("plugin,P", po::value<vector<string>>(), "List of plugin names to load in order,"
                                                   "e.g. if the file is called 'libFairMQPlugin_example.so', just list 'example' or 'd:example' here."
                                                   "To load a prelinked plugin, list 'p:example' here.");
@@ -170,29 +171,42 @@ auto fair::mq::PluginManager::LoadPluginDynamic(const string& pluginName) -> voi
     if (fPluginFactories.find(pluginName) == fPluginFactories.end())
     {
         auto success = false;
-        for(const auto& searchPath : SearchPaths())
-        {
-            try
-            {
-                LoadSymbols(
-                    pluginName,
-                    searchPath / ToString(LibPrefix(), pluginName),
-                    dll::load_mode::append_decorations
-                );
-
+        for (const auto& searchPath : SearchPaths()) {
+            try {
+                LoadSymbols(pluginName,
+                            searchPath / ToString(LibPrefix(), pluginName),
+                            dll::load_mode::append_decorations | dll::load_mode::rtld_global);
                 fPluginOrder.push_back(pluginName);
                 success = true;
                 break;
-            }
-            catch (boost::system::system_error& e)
-            {
-                if(string{e.what()}.find("No such file or directory") == string::npos)
-                {
-                    throw PluginLoadError(ToString("An error occurred while loading dynamic plugin ", pluginName, ": ", e.what()));
+            } catch (boost::system::system_error& e) {
+                if (string{e.what()}.find("No such file or directory") == string::npos) {
+                    throw PluginLoadError(
+                        ToString("An error occurred while loading dynamic plugin ",
+                                 pluginName, ": ", e.what()));
                 }
             }
         }
-        if(!success) { throw PluginLoadError(ToString("The plugin ", pluginName, " could not be found in the plugin search paths.")); }
+
+        if (!success) {
+            try {
+                // LoadSymbols(pluginName,
+                            // ToString(LibPrefix(), pluginName),
+                            // dll::load_mode::search_system_folders | dll::load_mode::append_decorations);
+                // Not sure, why the above does not work. Workaround for now:
+                LoadSymbols(pluginName,
+                            ToString("lib",
+                                     LibPrefix(),
+                                     pluginName,
+                                     boost::dll::detail::shared_library_impl::suffix().native()),
+                            dll::load_mode::search_system_folders | dll::load_mode::rtld_global);
+                fPluginOrder.push_back(pluginName);
+            } catch (boost::system::system_error& e) {
+                throw PluginLoadError(
+                    ToString("An error occurred while loading dynamic plugin ",
+                             pluginName, ": ", e.what()));
+            }
+        }
     }
 }
 

fairmq/PluginServices.cxx

@@ -16,11 +16,14 @@ const std::unordered_map<std::string, PluginServices::DeviceState> PluginService
     {"ERROR",               DeviceState::Error},
     {"IDLE",                DeviceState::Idle},
     {"INITIALIZING DEVICE", DeviceState::InitializingDevice},
+    {"INITIALIZED",         DeviceState::Initialized},
+    {"BINDING",             DeviceState::Binding},
+    {"BOUND",               DeviceState::Bound},
+    {"CONNECTING",          DeviceState::Connecting},
     {"DEVICE READY",        DeviceState::DeviceReady},
     {"INITIALIZING TASK",   DeviceState::InitializingTask},
     {"READY",               DeviceState::Ready},
     {"RUNNING",             DeviceState::Running},
-    {"PAUSED",              DeviceState::Paused},
     {"RESETTING TASK",      DeviceState::ResettingTask},
     {"RESETTING DEVICE",    DeviceState::ResettingDevice},
     {"EXITING",             DeviceState::Exiting}
@@ -30,83 +33,96 @@ const std::unordered_map<PluginServices::DeviceState, std::string, tools::HashEn
     {DeviceState::Error,              "ERROR"},
     {DeviceState::Idle,               "IDLE"},
     {DeviceState::InitializingDevice, "INITIALIZING DEVICE"},
+    {DeviceState::Initialized,        "INITIALIZED"},
+    {DeviceState::Binding,            "BINDING"},
+    {DeviceState::Bound,              "BOUND"},
+    {DeviceState::Connecting,         "CONNECTING"},
     {DeviceState::DeviceReady,        "DEVICE READY"},
     {DeviceState::InitializingTask,   "INITIALIZING TASK"},
     {DeviceState::Ready,              "READY"},
     {DeviceState::Running,            "RUNNING"},
-    {DeviceState::Paused,             "PAUSED"},
     {DeviceState::ResettingTask,      "RESETTING TASK"},
     {DeviceState::ResettingDevice,    "RESETTING DEVICE"},
     {DeviceState::Exiting,            "EXITING"}
 };
 const std::unordered_map<std::string, PluginServices::DeviceStateTransition> PluginServices::fkDeviceStateTransitionStrMap = {
-    {"INIT DEVICE",  DeviceStateTransition::InitDevice},
-    {"INIT TASK",    DeviceStateTransition::InitTask},
-    {"RUN",          DeviceStateTransition::Run},
-    {"PAUSE",        DeviceStateTransition::Pause},
-    {"RESUME",       DeviceStateTransition::Resume},
-    {"STOP",         DeviceStateTransition::Stop},
-    {"RESET TASK",   DeviceStateTransition::ResetTask},
-    {"RESET DEVICE", DeviceStateTransition::ResetDevice},
-    {"END",          DeviceStateTransition::End},
-    {"ERROR FOUND",  DeviceStateTransition::ErrorFound},
+    {"AUTO",          DeviceStateTransition::Auto},
+    {"INIT DEVICE",   DeviceStateTransition::InitDevice},
+    {"COMPLETE INIT", DeviceStateTransition::CompleteInit},
+    {"BIND",          DeviceStateTransition::Bind},
+    {"CONNECT",       DeviceStateTransition::Connect},
+    {"INIT TASK",     DeviceStateTransition::InitTask},
+    {"RUN",           DeviceStateTransition::Run},
+    {"STOP",          DeviceStateTransition::Stop},
+    {"RESET TASK",    DeviceStateTransition::ResetTask},
+    {"RESET DEVICE",  DeviceStateTransition::ResetDevice},
+    {"END",           DeviceStateTransition::End},
+    {"ERROR FOUND",   DeviceStateTransition::ErrorFound},
 };
 const std::unordered_map<PluginServices::DeviceStateTransition, std::string, tools::HashEnum<PluginServices::DeviceStateTransition>> PluginServices::fkStrDeviceStateTransitionMap = {
-    {DeviceStateTransition::InitDevice,  "INIT DEVICE"},
-    {DeviceStateTransition::InitTask,    "INIT TASK"},
-    {DeviceStateTransition::Run,         "RUN"},
-    {DeviceStateTransition::Pause,       "PAUSE"},
-    {DeviceStateTransition::Resume,      "RESUME"},
-    {DeviceStateTransition::Stop,        "STOP"},
-    {DeviceStateTransition::ResetTask,   "RESET TASK"},
-    {DeviceStateTransition::ResetDevice, "RESET DEVICE"},
-    {DeviceStateTransition::End,         "END"},
-    {DeviceStateTransition::ErrorFound,  "ERROR FOUND"},
+    {DeviceStateTransition::Auto,         "Auto"},
+    {DeviceStateTransition::InitDevice,   "INIT DEVICE"},
+    {DeviceStateTransition::CompleteInit, "COMPLETE INIT"},
+    {DeviceStateTransition::Bind,         "BIND"},
+    {DeviceStateTransition::Connect,      "CONNECT"},
+    {DeviceStateTransition::InitTask,     "INIT TASK"},
+    {DeviceStateTransition::Run,          "RUN"},
+    {DeviceStateTransition::Stop,         "STOP"},
+    {DeviceStateTransition::ResetTask,    "RESET TASK"},
+    {DeviceStateTransition::ResetDevice,  "RESET DEVICE"},
+    {DeviceStateTransition::End,          "END"},
+    {DeviceStateTransition::ErrorFound,   "ERROR FOUND"},
 };
-const std::unordered_map<FairMQDevice::State, PluginServices::DeviceState, fair::mq::tools::HashEnum<FairMQDevice::State>> PluginServices::fkDeviceStateMap = {
-    {FairMQDevice::OK,                  DeviceState::Ok},
-    {FairMQDevice::Error,               DeviceState::Error},
-    {FairMQDevice::IDLE,                DeviceState::Idle},
-    {FairMQDevice::INITIALIZING_DEVICE, DeviceState::InitializingDevice},
-    {FairMQDevice::DEVICE_READY,        DeviceState::DeviceReady},
-    {FairMQDevice::INITIALIZING_TASK,   DeviceState::InitializingTask},
-    {FairMQDevice::READY,               DeviceState::Ready},
-    {FairMQDevice::RUNNING,             DeviceState::Running},
-    {FairMQDevice::PAUSED,              DeviceState::Paused},
-    {FairMQDevice::RESETTING_TASK,      DeviceState::ResettingTask},
-    {FairMQDevice::RESETTING_DEVICE,    DeviceState::ResettingDevice},
-    {FairMQDevice::EXITING,             DeviceState::Exiting}
+const std::unordered_map<fair::mq::State, PluginServices::DeviceState, fair::mq::tools::HashEnum<fair::mq::State>> PluginServices::fkDeviceStateMap = {
+    {fair::mq::State::Ok,                 DeviceState::Ok},
+    {fair::mq::State::Error,              DeviceState::Error},
+    {fair::mq::State::Idle,               DeviceState::Idle},
+    {fair::mq::State::InitializingDevice, DeviceState::InitializingDevice},
+    {fair::mq::State::Initialized,        DeviceState::Initialized},
+    {fair::mq::State::Binding,            DeviceState::Binding},
+    {fair::mq::State::Bound,              DeviceState::Bound},
+    {fair::mq::State::Connecting,         DeviceState::Connecting},
+    {fair::mq::State::DeviceReady,        DeviceState::DeviceReady},
+    {fair::mq::State::InitializingTask,   DeviceState::InitializingTask},
+    {fair::mq::State::Ready,              DeviceState::Ready},
+    {fair::mq::State::Running,            DeviceState::Running},
+    {fair::mq::State::ResettingTask,      DeviceState::ResettingTask},
+    {fair::mq::State::ResettingDevice,    DeviceState::ResettingDevice},
+    {fair::mq::State::Exiting,            DeviceState::Exiting}
 };
-const std::unordered_map<PluginServices::DeviceStateTransition, FairMQDevice::Event, tools::HashEnum<PluginServices::DeviceStateTransition>> PluginServices::fkDeviceStateTransitionMap = {
-    {DeviceStateTransition::InitDevice,  FairMQDevice::INIT_DEVICE},
-    {DeviceStateTransition::InitTask,    FairMQDevice::INIT_TASK},
-    {DeviceStateTransition::Run,         FairMQDevice::RUN},
-    {DeviceStateTransition::Pause,       FairMQDevice::PAUSE},
-    {DeviceStateTransition::Resume,      FairMQDevice::RUN},
-    {DeviceStateTransition::Stop,        FairMQDevice::STOP},
-    {DeviceStateTransition::ResetTask,   FairMQDevice::RESET_TASK},
-    {DeviceStateTransition::ResetDevice, FairMQDevice::RESET_DEVICE},
-    {DeviceStateTransition::End,         FairMQDevice::END},
-    {DeviceStateTransition::ErrorFound,  FairMQDevice::ERROR_FOUND}
+const std::unordered_map<PluginServices::DeviceStateTransition, fair::mq::Transition, tools::HashEnum<PluginServices::DeviceStateTransition>> PluginServices::fkDeviceStateTransitionMap = {
+    {DeviceStateTransition::Auto,         fair::mq::Transition::Auto},
+    {DeviceStateTransition::InitDevice,   fair::mq::Transition::InitDevice},
+    {DeviceStateTransition::CompleteInit, fair::mq::Transition::CompleteInit},
+    {DeviceStateTransition::Bind,         fair::mq::Transition::Bind},
+    {DeviceStateTransition::Connect,      fair::mq::Transition::Connect},
+    {DeviceStateTransition::InitTask,     fair::mq::Transition::InitTask},
+    {DeviceStateTransition::Run,          fair::mq::Transition::Run},
+    {DeviceStateTransition::Stop,         fair::mq::Transition::Stop},
+    {DeviceStateTransition::ResetTask,    fair::mq::Transition::ResetTask},
+    {DeviceStateTransition::ResetDevice,  fair::mq::Transition::ResetDevice},
+    {DeviceStateTransition::End,          fair::mq::Transition::End},
+    {DeviceStateTransition::ErrorFound,   fair::mq::Transition::ErrorFound}
 };
 
-auto PluginServices::ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> void
+auto PluginServices::ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> bool
 {
     lock_guard<mutex> lock{fDeviceControllerMutex};
 
     if (!fDeviceController) fDeviceController = controller;
 
-    if (fDeviceController == controller)
-    {
-        fDevice.ChangeState(fkDeviceStateTransitionMap.at(next));
-    }
-    else
-    {
+    bool result = false;
+
+    if (fDeviceController == controller) {
+        result = fDevice.ChangeState(fkDeviceStateTransitionMap.at(next));
+    } else {
         throw DeviceControlError{tools::ToString(
             "Plugin '", controller, "' is not allowed to change device states. ",
             "Currently, plugin '", *fDeviceController, "' has taken control."
         )};
     }
+
+    return result;
 }
 
 auto PluginServices::TakeDeviceControl(const std::string& controller) -> void

fairmq/PluginServices.h

@@ -63,11 +63,14 @@ class PluginServices
         Error,
         Idle,
         InitializingDevice,
+        Initialized,
+        Binding,
+        Bound,
+        Connecting,
         DeviceReady,
         InitializingTask,
         Ready,
         Running,
-        Paused,
         ResettingTask,
         ResettingDevice,
         Exiting
@@ -75,11 +78,13 @@ class PluginServices
 
     enum class DeviceStateTransition : int // transition event between DeviceStates
     {
+        Auto,
         InitDevice,
+        CompleteInit,
+        Bind,
+        Connect,
         InitTask,
         Run,
-        Pause,
-        Resume,
         Stop,
         ResetTask,
         ResetDevice,
@@ -115,7 +120,7 @@ class PluginServices
     friend auto operator<<(std::ostream& os, const DeviceStateTransition& transition) -> std::ostream& { return os << ToStr(transition); }
 
     /// @return current device state
-    auto GetCurrentDeviceState() const -> DeviceState { return fkDeviceStateMap.at(static_cast<FairMQDevice::State>(fDevice.GetCurrentState())); }
+    auto GetCurrentDeviceState() const -> DeviceState { return fkDeviceStateMap.at(static_cast<fair::mq::State>(fDevice.GetCurrentState())); }
 
     /// @brief Become device controller
     /// @param controller id
@@ -151,7 +156,7 @@ class PluginServices
     /// The state transition may not happen immediately, but when the current state evaluates the
     /// pending transition event and terminates. In other words, the device states are scheduled cooperatively.
     /// If the device control role has not been taken yet, calling this function will take over control implicitely.
-    auto ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> void;
+    auto ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> bool;
 
     /// @brief Subscribe with a callback to device state changes
     /// @param subscriber id
@@ -161,7 +166,7 @@ class PluginServices
     /// the state is running in.
     auto SubscribeToDeviceStateChange(const std::string& subscriber, std::function<void(DeviceState /*newState*/)> callback) -> void
     {
-        fDevice.SubscribeToStateChange(subscriber, [&,callback](FairMQDevice::State newState){
+        fDevice.SubscribeToStateChange(subscriber, [&,callback](fair::mq::State newState){
             callback(fkDeviceStateMap.at(newState));
         });
     }
@@ -187,12 +192,14 @@ class PluginServices
     {
         auto currentState = GetCurrentDeviceState();
         if (   (currentState == DeviceState::InitializingDevice)
-            || ((currentState == DeviceState::Idle) && (key == "channel-config")))
-        {
+            || (currentState == DeviceState::Initialized)
+            || (currentState == DeviceState::Binding)
+            || (currentState == DeviceState::Bound)
+            || (currentState == DeviceState::Connecting)
+            || (currentState == DeviceState::Ready)
+            || (currentState == DeviceState::Idle && key == "channel-config")) {
             fConfig.SetValue(key, val);
-        }
-        else
-        {
+        } else {
             throw InvalidStateError{
                 tools::ToString("PluginServices::SetProperty is not supported in device state ", currentState, ". ",
                                 "Supported state is ", DeviceState::InitializingDevice, ".")};
@@ -271,8 +278,8 @@ class PluginServices
     static const std::unordered_map<DeviceState, std::string, tools::HashEnum<DeviceState>> fkStrDeviceStateMap;
     static const std::unordered_map<std::string, DeviceStateTransition> fkDeviceStateTransitionStrMap;
     static const std::unordered_map<DeviceStateTransition, std::string, tools::HashEnum<DeviceStateTransition>> fkStrDeviceStateTransitionMap;
-    static const std::unordered_map<FairMQDevice::State, DeviceState, tools::HashEnum<FairMQDevice::State>> fkDeviceStateMap;
-    static const std::unordered_map<DeviceStateTransition, FairMQDevice::Event, tools::HashEnum<DeviceStateTransition>> fkDeviceStateTransitionMap;
+    static const std::unordered_map<fair::mq::State, DeviceState, tools::HashEnum<fair::mq::State>> fkDeviceStateMap;
+    static const std::unordered_map<DeviceStateTransition, fair::mq::Transition, tools::HashEnum<DeviceStateTransition>> fkDeviceStateTransitionMap;
 
   private:
     FairMQProgOptions& fConfig;

fairmq/StateMachine.cxx

@@ -1,5 +1,5 @@
 /********************************************************************************
- *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -7,189 +7,474 @@
  ********************************************************************************/
 
 #include "StateMachine.h"
+#include <fairmq/Tools.h>
+
+// Increase maximum number of boost::msm states (default is 10)
+// This #define has to be before any msm header includes
+#define FUSION_MAX_VECTOR_SIZE 20
+
+#include <boost/mpl/for_each.hpp>
+#include <boost/msm/back/state_machine.hpp>
+#include <boost/msm/back/tools.hpp>
+#include <boost/msm/back/metafunctions.hpp>
+#include <boost/msm/front/state_machine_def.hpp>
+#include <boost/msm/front/functor_row.hpp>
+#include <boost/core/demangle.hpp>
+#include <boost/signals2.hpp> // signal/slot for onStateChange callbacks
+
+#include <atomic>
+#include <condition_variable>
+#include <chrono>
+#include <array>
+#include <unordered_map>
+#include <mutex>
+#include <stdexcept>
 
-using namespace fair::mq;
 using namespace std;
+using namespace boost::msm;
+using namespace boost::msm::front;
+using namespace boost::msm::back;
+namespace bmpl = boost::mpl;
 
-const std::unordered_map<std::string, StateMachine::State> StateMachine::fkStateStrMap = {
-    {"OK",                  State::Ok},
-    {"ERROR",               State::Error},
-    {"IDLE",                State::Idle},
-    {"INITIALIZING DEVICE", State::InitializingDevice},
-    {"DEVICE READY",        State::DeviceReady},
-    {"INITIALIZING TASK",   State::InitializingTask},
-    {"READY",               State::Ready},
-    {"RUNNING",             State::Running},
-    {"RESETTING TASK",      State::ResettingTask},
-    {"RESETTING DEVICE",    State::ResettingDevice},
-    {"EXITING",             State::Exiting}
-};
-const std::unordered_map<StateMachine::State, std::string, tools::HashEnum<StateMachine::State>> StateMachine::fkStrStateMap = {
-    {State::Ok,                 "OK"},
-    {State::Error,              "ERROR"},
-    {State::Idle,               "IDLE"},
-    {State::InitializingDevice, "INITIALIZING DEVICE"},
-    {State::DeviceReady,        "DEVICE READY"},
-    {State::InitializingTask,   "INITIALIZING TASK"},
-    {State::Ready,              "READY"},
-    {State::Running,            "RUNNING"},
-    {State::ResettingTask,      "RESETTING TASK"},
-    {State::ResettingDevice,    "RESETTING DEVICE"},
-    {State::Exiting,            "EXITING"}
-};
-const std::unordered_map<std::string, StateMachine::StateTransition> StateMachine::fkStateTransitionStrMap = {
-    {"INIT DEVICE",  StateTransition::InitDevice},
-    {"INIT TASK",    StateTransition::InitTask},
-    {"RUN",          StateTransition::Run},
-    {"STOP",         StateTransition::Stop},
-    {"RESET TASK",   StateTransition::ResetTask},
-    {"RESET DEVICE", StateTransition::ResetDevice},
-    {"END",          StateTransition::End},
-    {"ERROR FOUND",  StateTransition::ErrorFound},
-    {"AUTOMATIC",    StateTransition::Automatic},
-};
-const std::unordered_map<StateMachine::StateTransition, std::string, tools::HashEnum<StateMachine::StateTransition>> StateMachine::fkStrStateTransitionMap = {
-    {StateTransition::InitDevice,  "INIT DEVICE"},
-    {StateTransition::InitTask,    "INIT TASK"},
-    {StateTransition::Run,         "RUN"},
-    {StateTransition::Stop,        "STOP"},
-    {StateTransition::ResetTask,   "RESET TASK"},
-    {StateTransition::ResetDevice, "RESET DEVICE"},
-    {StateTransition::End,         "END"},
-    {StateTransition::ErrorFound,  "ERROR FOUND"},
-    {StateTransition::Automatic,   "AUTOMATIC"},
-};
-
-auto StateMachine::Run() -> void
+namespace std
 {
-    LOG(state) << "Starting FairMQ state machine";
 
-    LOG(debug) << "Entering initial " << fErrorState << " state (orthogonal error state machine)";
-    LOG(state) << "Entering initial " << fState << " state";
+template<>
+struct hash<fair::mq::Transition> : fair::mq::tools::HashEnum<fair::mq::Transition> {};
 
-    std::unique_lock<std::mutex> lock{fMutex};
-    while (true)
+template<>
+struct hash<fair::mq::State> : fair::mq::tools::HashEnum<fair::mq::State> {};
+
+} /* namespace std */
+
+namespace fair
+{
+namespace mq
+{
+namespace fsm
+{
+
+// list of FSM states
+struct OK_S                  : public state<> { static string Name() { return "OK"; }                  static State Type() { return State::Ok; } };
+
+struct IDLE_S                : public state<> { static string Name() { return "IDLE"; }                static State Type() { return State::Idle; } };
+struct INITIALIZING_DEVICE_S : public state<> { static string Name() { return "INITIALIZING_DEVICE"; } static State Type() { return State::InitializingDevice; } };
+struct INITIALIZED_S         : public state<> { static string Name() { return "INITIALIZED"; }         static State Type() { return State::Initialized; } };
+struct BINDING_S             : public state<> { static string Name() { return "BINDING"; }             static State Type() { return State::Binding; } };
+struct BOUND_S               : public state<> { static string Name() { return "BOUND"; }               static State Type() { return State::Bound; } };
+struct CONNECTING_S          : public state<> { static string Name() { return "CONNECTING"; }          static State Type() { return State::Connecting; } };
+struct DEVICE_READY_S        : public state<> { static string Name() { return "DEVICE_READY"; }        static State Type() { return State::DeviceReady; } };
+struct INITIALIZING_TASK_S   : public state<> { static string Name() { return "INITIALIZING_TASK"; }   static State Type() { return State::InitializingTask; } };
+struct READY_S               : public state<> { static string Name() { return "READY"; }               static State Type() { return State::Ready; } };
+struct RUNNING_S             : public state<> { static string Name() { return "RUNNING"; }             static State Type() { return State::Running; } };
+struct RESETTING_TASK_S      : public state<> { static string Name() { return "RESETTING_TASK"; }      static State Type() { return State::ResettingTask; } };
+struct RESETTING_DEVICE_S    : public state<> { static string Name() { return "RESETTING_DEVICE"; }    static State Type() { return State::ResettingDevice; } };
+struct EXITING_S             : public state<> { static string Name() { return "EXITING"; }             static State Type() { return State::Exiting; } };
+
+struct ERROR_S               : public terminate_state<> { static string Name() { return "ERROR"; }     static State Type() { return State::Error; } };
+
+// list of FSM transitions (events)
+struct AUTO_E          { static string Name() { return "AUTO"; }          static Transition Type() { return Transition::Auto; } };
+struct INIT_DEVICE_E   { static string Name() { return "INIT_DEVICE"; }   static Transition Type() { return Transition::InitDevice; } };
+struct COMPLETE_INIT_E { static string Name() { return "COMPLETE_INIT"; } static Transition Type() { return Transition::CompleteInit; } };
+struct BIND_E          { static string Name() { return "BIND"; }          static Transition Type() { return Transition::Bind; } };
+struct CONNECT_E       { static string Name() { return "CONNECT"; }       static Transition Type() { return Transition::Connect; } };
+struct INIT_TASK_E     { static string Name() { return "INIT_TASK"; }     static Transition Type() { return Transition::InitTask; } };
+struct RUN_E           { static string Name() { return "RUN"; }           static Transition Type() { return Transition::Run; } };
+struct STOP_E          { static string Name() { return "STOP"; }          static Transition Type() { return Transition::Stop; } };
+struct RESET_TASK_E    { static string Name() { return "RESET_TASK"; }    static Transition Type() { return Transition::ResetTask; } };
+struct RESET_DEVICE_E  { static string Name() { return "RESET_DEVICE"; }  static Transition Type() { return Transition::ResetDevice; } };
+struct END_E           { static string Name() { return "END"; }           static Transition Type() { return Transition::End; } };
+struct ERROR_FOUND_E   { static string Name() { return "ERROR_FOUND"; }   static Transition Type() { return Transition::ErrorFound; } };
+
+static array<string, 15> stateNames =
+{
     {
-        while (fNextStates.empty())
+        "OK",
+        "Error",
+        "IDLE",
+        "INITIALIZING_DEVICE",
+        "INITIALIZED",
+        "BINDING",
+        "BOUND",
+        "CONNECTING",
+        "DEVICE_READY",
+        "INITIALIZING_TASK",
+        "READY",
+        "RUNNING",
+        "RESETTING_TASK",
+        "RESETTING_DEVICE",
+        "EXITING"
+    }
+};
+
+static array<string, 12> transitionNames =
+{
+    {
+        "AUTO",
+        "INIT_DEVICE",
+        "COMPLETE_INIT",
+        "BIND",
+        "CONNECT",
+        "INIT_TASK",
+        "RUN",
+        "STOP",
+        "RESET_TASK",
+        "RESET_DEVICE",
+        "END",
+        "ERROR_FOUND"
+    }
+};
+
+static map<string, State> stateNumbers =
+{
+    { "OK",                  State::Ok },
+    { "Error",               State::Error },
+    { "IDLE",                State::Idle },
+    { "INITIALIZING_DEVICE", State::InitializingDevice },
+    { "INITIALIZED",         State::Initialized },
+    { "BINDING",             State::Binding },
+    { "BOUND",               State::Bound },
+    { "CONNECTING",          State::Connecting },
+    { "DEVICE_READY",        State::DeviceReady },
+    { "INITIALIZING_TASK",   State::InitializingTask },
+    { "READY",               State::Ready },
+    { "RUNNING",             State::Running },
+    { "RESETTING_TASK",      State::ResettingTask },
+    { "RESETTING_DEVICE",    State::ResettingDevice },
+    { "EXITING",             State::Exiting }
+};
+
+static map<string, Transition> transitionNumbers =
+{
+    { "AUTO",          Transition::Auto },
+    { "INIT_DEVICE",   Transition::InitDevice },
+    { "COMPLETE_INIT", Transition::CompleteInit },
+    { "BIND",          Transition::Bind },
+    { "CONNECT",       Transition::Connect },
+    { "INIT_TASK",     Transition::InitTask },
+    { "RUN",           Transition::Run },
+    { "STOP",          Transition::Stop },
+    { "RESET_TASK",    Transition::ResetTask },
+    { "RESET_DEVICE",  Transition::ResetDevice },
+    { "END",           Transition::End },
+    { "ERROR_FOUND",   Transition::ErrorFound }
+};
+
+// defining the boost MSM state machine
+struct Machine_ : public state_machine_def<Machine_>
+{
+  public:
+    Machine_()
+        : fLastTransitionResult(true)
+        , fNewStatePending(false)
+    {}
+
+    virtual ~Machine_() {}
+
+    // initial states
+    using initial_state = bmpl::vector<IDLE_S, OK_S>;
+
+    template<typename Transition, typename FSM>
+    void on_entry(Transition const&, FSM& /* fsm */)
+    {
+        LOG(state) << "Starting FairMQ state machine --> IDLE";
+        fState = State::Idle;
+    }
+
+    template<typename Transition, typename FSM>
+    void on_exit(Transition const&, FSM& /*fsm*/)
+    {
+        LOG(state) << "Exiting FairMQ state machine";
+    }
+
+    struct DefaultFct
+    {
+        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
+        void operator()(EVT const& e, FSM& fsm, SourceState& /* ss */, TargetState& ts)
         {
-            fNewState.wait(lock);
+            fsm.fNewState = ts.Type();
+            fsm.fLastTransitionResult = true;
+            fsm.CallNewTransitionCallbacks(e.Type());
+            fsm.fNewStatePending = true;
+            fsm.fNewStatePendingCV.notify_all();
+        }
+    };
+
+    struct transition_table : bmpl::vector<
+        //  Start                  Transition       Next                   Action      Guard
+        Row<IDLE_S,                END_E,           EXITING_S,             DefaultFct, none>,
+        Row<IDLE_S,                INIT_DEVICE_E,   INITIALIZING_DEVICE_S, DefaultFct, none>,
+
+        Row<INITIALIZING_DEVICE_S, COMPLETE_INIT_E, INITIALIZED_S,         DefaultFct, none>,
+        Row<INITIALIZED_S,         BIND_E,          BINDING_S,             DefaultFct, none>,
+        Row<INITIALIZED_S,         RESET_DEVICE_E,  RESETTING_DEVICE_S,    DefaultFct, none>,
+
+        Row<BINDING_S,             AUTO_E,          BOUND_S,               DefaultFct, none>,
+        Row<BOUND_S,               CONNECT_E,       CONNECTING_S,          DefaultFct, none>,
+        Row<BOUND_S,               RESET_DEVICE_E,  RESETTING_DEVICE_S,    DefaultFct, none>,
+
+        Row<CONNECTING_S,          AUTO_E,          DEVICE_READY_S,        DefaultFct, none>,
+        Row<DEVICE_READY_S,        INIT_TASK_E,     INITIALIZING_TASK_S,   DefaultFct, none>,
+        Row<DEVICE_READY_S,        RESET_DEVICE_E,  RESETTING_DEVICE_S,    DefaultFct, none>,
+
+        Row<INITIALIZING_TASK_S,   AUTO_E,          READY_S,               DefaultFct, none>,
+
+        Row<READY_S,               RUN_E,           RUNNING_S,             DefaultFct, none>,
+        Row<READY_S,               RESET_TASK_E,    RESETTING_TASK_S,      DefaultFct, none>,
+
+        Row<RUNNING_S,             STOP_E,          READY_S,               DefaultFct, none>,
+
+        Row<RESETTING_TASK_S,      AUTO_E,          DEVICE_READY_S,        DefaultFct, none>,
+        Row<RESETTING_DEVICE_S,    AUTO_E,          IDLE_S,                DefaultFct, none>,
+
+        Row<OK_S,                  ERROR_FOUND_E,   ERROR_S,               DefaultFct,   none>> {};
+
+    void CallStateChangeCallbacks(const State state) const
+    {
+        if (!fStateChangeSignal.empty()) {
+            fStateChangeSignal(state);
+        }
+    }
+
+    void CallStateHandler(const State state) const
+    {
+        if (!fStateHandleSignal.empty()) {
+            fStateHandleSignal(state);
+        }
+    }
+
+    void CallNewTransitionCallbacks(const Transition transition) const
+    {
+        if (!fNewTransitionSignal.empty()) {
+            fNewTransitionSignal(transition);
+        }
+    }
+
+
+    atomic<State> fState;
+    atomic<State> fNewState;
+    atomic<bool> fLastTransitionResult;
+
+    mutex fStateMtx;
+    atomic<bool> fNewStatePending;
+    condition_variable fNewStatePendingCV;
+
+    boost::signals2::signal<void(const State)> fStateChangeSignal;
+    boost::signals2::signal<void(const State)> fStateHandleSignal;
+    boost::signals2::signal<void(const Transition)> fNewTransitionSignal;
+    unordered_map<string, boost::signals2::connection> fStateChangeSignalsMap;
+    unordered_map<string, boost::signals2::connection> fNewTransitionSignalsMap;
+
+    void ProcessWork()
+    {
+        bool stop = false;
+
+        while (!stop) {
+            {
+                unique_lock<mutex> lock(fStateMtx);
+
+                while (!fNewStatePending) {
+                    fNewStatePendingCV.wait_for(lock, chrono::milliseconds(100));
+                }
+
+                LOG(state) << fState << " ---> " << fNewState;
+                fState = static_cast<State>(fNewState);
+                fNewStatePending = false;
+
+                if (fState == State::Exiting || fState == State::Error) {
+                    stop = true;
+                }
+            }
+
+            CallStateChangeCallbacks(fState);
+            CallStateHandler(fState);
         }
 
-        State lastState;
-
-        if (fNextStates.front() == State::Error)
-        {
-            // advance error FSM
-            lastState = fErrorState;
-            fErrorState = fNextStates.front();
-            fNextStates.pop_front();
-            LOG(error) << "Entering " << fErrorState << " state (orthogonal error state machine)";
+        if (fState == State::Error) {
+            throw StateMachine::ErrorStateException("Device transitioned to error state");
         }
-        else
-        {
-            // advance regular FSM
-            lastState = fState;
-            fState = fNextStates.front();
-            fNextStates.pop_front();
-            LOG(state) << "Entering " << fState << " state";
+    }
+
+    // replaces the default no-transition response.
+    template<typename FSM, typename Transition>
+    void no_transition(Transition const& t, FSM& fsm, int state)
+    {
+        using RecursiveStt = typename recursive_get_transition_table<FSM>::type;
+        using AllStates = typename generate_state_set<RecursiveStt>::type;
+
+        string stateName;
+
+        bmpl::for_each<AllStates, wrap<bmpl::placeholders::_1>>(get_state_name<RecursiveStt>(stateName, state));
+
+        stateName = boost::core::demangle(stateName.c_str());
+        size_t pos = stateName.rfind(":");
+        stateName = stateName.substr(pos + 1);
+        size_t pos2 = stateName.rfind("_");
+        stateName = stateName.substr(0, pos2);
+
+        if (stateName != "OK") {
+            LOG(state) << "No transition from state " << stateName << " on transition " << t.Name();
         }
-        lock.unlock();
-
-        fCallbacks.Emit<StateChange, State>(fState, lastState);
-
-        lock.lock();
-        if (fState == State::Exiting || fErrorState == State::Error) break;
+        fsm.fLastTransitionResult = false;
     }
+}; // Machine_
 
-    LOG(state) << "Exiting FairMQ state machine";
-}
+using FairMQFSM = state_machine<Machine_>;
 
-auto StateMachine::ChangeState(StateTransition transition) -> void
-{
-    State lastState;
+} // namespace fsm
+} // namespace mq
+} // namespace fair
 
-    std::unique_lock<std::mutex> lock{fMutex};
+using namespace fair::mq::fsm;
+using namespace fair::mq;
 
-    if (transition == StateTransition::ErrorFound)
-    {
-        lastState = fErrorState;
+StateMachine::StateMachine() : fFsm(new FairMQFSM) {}
+void StateMachine::Start() { static_pointer_cast<FairMQFSM>(fFsm)->start(); }
+StateMachine::~StateMachine() { static_pointer_cast<FairMQFSM>(fFsm)->stop(); }
+
+bool StateMachine::ChangeState(const Transition transition)
+try {
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    lock_guard<mutex> lock(fsm->fStateMtx);
+    if (!static_cast<bool>(fsm->fNewStatePending) || transition == Transition::ErrorFound) {
+        switch (transition) {
+            case Transition::Auto:
+                fsm->process_event(AUTO_E());
+                return fsm->fLastTransitionResult;
+            case Transition::InitDevice:
+                fsm->process_event(INIT_DEVICE_E());
+                return fsm->fLastTransitionResult;
+            case Transition::CompleteInit:
+                fsm->process_event(COMPLETE_INIT_E());
+                return fsm->fLastTransitionResult;
+            case Transition::Bind:
+                fsm->process_event(BIND_E());
+                return fsm->fLastTransitionResult;
+            case Transition::Connect:
+                fsm->process_event(CONNECT_E());
+                return fsm->fLastTransitionResult;
+            case Transition::InitTask:
+                fsm->process_event(INIT_TASK_E());
+                return fsm->fLastTransitionResult;
+            case Transition::Run:
+                fsm->process_event(RUN_E());
+                return fsm->fLastTransitionResult;
+            case Transition::Stop:
+                fsm->process_event(STOP_E());
+                return fsm->fLastTransitionResult;
+            case Transition::ResetDevice:
+                fsm->process_event(RESET_DEVICE_E());
+                return fsm->fLastTransitionResult;
+            case Transition::ResetTask:
+                fsm->process_event(RESET_TASK_E());
+                return fsm->fLastTransitionResult;
+            case Transition::End:
+                fsm->process_event(END_E());
+                return fsm->fLastTransitionResult;
+            case Transition::ErrorFound:
+                fsm->process_event(ERROR_FOUND_E());
+                return fsm->fLastTransitionResult;
+            default:
+                LOG(error) << "Requested unsupported state transition: " << transition << endl;
+                return false;
+             }
+    } else {
+        LOG(state) << "Transition " << transitionNames.at(static_cast<int>(transition)) << " incoming, but another state transition is already ongoing.";
+        return false;
     }
-    else if (fNextStates.empty())
-    {
-        lastState = fState;
+} catch (exception& e) {
+    LOG(error) << "Exception in StateMachine::ChangeState(): " << e.what();
+    return false;
+}
+
+void StateMachine::SubscribeToStateChange(const string& key, function<void(const State)> callback)
+{
+    static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.insert({key, static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignal.connect(callback)});
+}
+
+void StateMachine::UnsubscribeFromStateChange(const string& key)
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (fsm->fStateChangeSignalsMap.count(key)) {
+        fsm->fStateChangeSignalsMap.at(key).disconnect();
+        fsm->fStateChangeSignalsMap.erase(key);
     }
-    else
-    {
-        lastState = fNextStates.back();
+}
+
+void StateMachine::HandleStates(function<void(const State)> callback)
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (fsm->fStateHandleSignal.empty()) {
+        fsm->fStateHandleSignal.connect(callback);
+    } else {
+        LOG(error) << "state handler is already set";
     }
-
-    const State nextState{Transition(lastState, transition)};
-    fNextStates.push_back(nextState);
-    lock.unlock();
-
-    fCallbacks.Emit<StateQueued, State>(nextState, lastState);
-    fNewState.notify_one();
 }
-    
-auto StateMachine::Transition(const State currentState, const StateTransition transition) -> State
+
+void StateMachine::StopHandlingStates()
 {
-    switch (currentState) {
-        case State::Idle:
-            if (transition == StateTransition::InitDevice ) return State::InitializingDevice;
-            if (transition == StateTransition::End        ) return State::Exiting;
-            break;
-        case State::InitializingDevice:
-            if (transition == StateTransition::Automatic  ) return State::DeviceReady;
-            break;
-        case State::DeviceReady:
-            if (transition == StateTransition::InitTask   ) return State::InitializingTask;
-            if (transition == StateTransition::ResetDevice) return State::ResettingDevice;
-            break;
-        case State::InitializingTask:
-            if (transition == StateTransition::Automatic  ) return State::Ready;
-            break;
-        case State::Ready:
-            if (transition == StateTransition::Run        ) return State::Running;
-            if (transition == StateTransition::ResetTask  ) return State::ResettingTask;
-            break;
-        case State::Running:
-            if (transition == StateTransition::Stop       ) return State::Ready;
-            break;
-        case State::ResettingTask:
-            if (transition == StateTransition::Automatic  ) return State::DeviceReady;
-            break;
-        case State::ResettingDevice:
-            if (transition == StateTransition::Automatic  ) return State::Idle;
-            break;
-        case State::Exiting:
-            break;
-        case State::Ok:
-            if (transition == StateTransition::ErrorFound ) return State::Error;
-            break;
-        case State::Error:
-            break;
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (!fsm->fStateHandleSignal.empty()) {
+        fsm->fStateHandleSignal.disconnect_all_slots();
     }
-    throw IllegalTransition{tools::ToString("No transition ", transition, " from state ", currentState, ".")};
 }
 
-StateMachine::StateMachine()
-: fState{State::Idle}
-, fErrorState{State::Ok}
+void StateMachine::SubscribeToNewTransition(const string& key, function<void(const Transition)> callback)
 {
+    static_pointer_cast<FairMQFSM>(fFsm)->fNewTransitionSignalsMap.insert({key, static_pointer_cast<FairMQFSM>(fFsm)->fNewTransitionSignal.connect(callback)});
 }
 
-auto StateMachine::Reset() -> void
+void StateMachine::UnsubscribeFromNewTransition(const string& key)
 {
-    std::unique_lock<std::mutex> lock{fMutex};
-
-    fState = State::Idle;
-    fErrorState = State::Ok;
-    fNextStates.clear();
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (fsm->fNewTransitionSignalsMap.count(key)) {
+        fsm->fNewTransitionSignalsMap.at(key).disconnect();
+        fsm->fNewTransitionSignalsMap.erase(key);
+    }
 }
 
-auto StateMachine::NextStatePending() -> bool
+State StateMachine::GetCurrentState() const { return static_pointer_cast<FairMQFSM>(fFsm)->fState; }
+string StateMachine::GetCurrentStateName() const { return GetStateName(static_pointer_cast<FairMQFSM>(fFsm)->fState); }
+
+bool StateMachine::NewStatePending() const { return static_cast<bool>(static_pointer_cast<FairMQFSM>(fFsm)->fNewStatePending); }
+void StateMachine::WaitForPendingState() const
 {
-    std::unique_lock<std::mutex> lock{fMutex};
-
-    return fNextStates.size() > 0;
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    unique_lock<mutex> lock(fsm->fStateMtx);
+    fsm->fNewStatePendingCV.wait(lock, [&]{ return static_cast<bool>(fsm->fNewStatePending); });
 }
+bool StateMachine::WaitForPendingStateFor(const int durationInMs) const
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    unique_lock<mutex> lock(fsm->fStateMtx);
+    return fsm->fNewStatePendingCV.wait_for(lock, std::chrono::milliseconds(durationInMs), [&]{ return static_cast<bool>(fsm->fNewStatePending); });
+}
+
+void StateMachine::ProcessWork()
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+
+    try {
+        fsm->CallStateChangeCallbacks(State::Idle);
+        fsm->ProcessWork();
+    } catch(ErrorStateException& ese) {
+        LOG(trace) << "ErrorStateException caught in ProcessWork(), rethrowing";
+        throw;
+    } catch(...) {
+        LOG(debug) << "Exception caught in ProcessWork(), going to Error state and rethrowing";
+        {
+            lock_guard<mutex> lock(fsm->fStateMtx);
+            fsm->fState = State::Error;
+            fsm->CallStateChangeCallbacks(State::Error);
+        }
+        ChangeState(Transition::ErrorFound);
+        throw;
+    }
+}
+
+string StateMachine::GetStateName(const State state) { return stateNames.at(static_cast<int>(state)); }
+string StateMachine::GetTransitionName(const Transition transition) { return transitionNames.at(static_cast<int>(transition)); }
+State StateMachine::GetState(const string& state) { return stateNumbers.at(state); }
+Transition StateMachine::GetTransition(const string& transition) { return transitionNumbers.at(transition); }
+

fairmq/StateMachine.h

@@ -1,132 +1,109 @@
 /********************************************************************************
- *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#ifndef FAIR_MQ_STATEMACHINE_H
-#define FAIR_MQ_STATEMACHINE_H
+#ifndef FAIRMQSTATEMACHINE_H_
+#define FAIRMQSTATEMACHINE_H_
 
-#include <utility>
-#include <FairMQLogger.h>
-#include <fairmq/Tools.h>
-#include <fairmq/EventManager.h>
-#include <deque>
+#include "FairMQLogger.h"
+
+#include <string>
+#include <memory>
 #include <functional>
-#include <mutex>
 #include <condition_variable>
-#include <thread>
-#include <unordered_map>
+#include <ostream>
+#include <queue>
+#include <mutex>
+#include <stdexcept>
 
 namespace fair
 {
 namespace mq
 {
 
-/**
- * @class StateMachine StateMachine.h <fairmq/StateMachine.h>
- * @brief Implements the state machine for FairMQ devices
- *
- * See https://github.com/FairRootGroup/FairRoot/blob/dev/fairmq/docs/Device.md#13-state-machine
- */
+enum class State : int
+{
+    Ok,
+    Error,
+    Idle,
+    InitializingDevice,
+    Initialized,
+    Binding,
+    Bound,
+    Connecting,
+    DeviceReady,
+    InitializingTask,
+    Ready,
+    Running,
+    ResettingTask,
+    ResettingDevice,
+    Exiting
+};
+
+enum class Transition : int
+{
+    Auto,
+    InitDevice,
+    CompleteInit,
+    Bind,
+    Connect,
+    InitTask,
+    Run,
+    Stop,
+    ResetTask,
+    ResetDevice,
+    End,
+    ErrorFound
+};
+
 class StateMachine
 {
   public:
-    enum class State : int
-    {
-        Ok,
-        Error,
-        Idle,
-        InitializingDevice,
-        DeviceReady,
-        InitializingTask,
-        Ready,
-        Running,
-        ResettingTask,
-        ResettingDevice,
-        Exiting
-    };
-
-    enum class StateTransition : int // transition event between States
-    {
-        InitDevice,
-        InitTask,
-        Run,
-        Stop,
-        ResetTask,
-        ResetDevice,
-        End,
-        ErrorFound,
-        Automatic
-    };
-
-    /// @brief Convert string to State
-    /// @param state to convert
-    /// @return State enum entry
-    /// @throw std::out_of_range if a string cannot be resolved to a State
-    static auto ToState(const std::string& state) -> State { return fkStateStrMap.at(state); }
-
-    /// @brief Convert string to StateTransition
-    /// @param transition to convert
-    /// @return StateTransition enum entry
-    /// @throw std::out_of_range if a string cannot be resolved to a StateTransition
-    static auto ToStateTransition(const std::string& transition) -> StateTransition { return fkStateTransitionStrMap.at(transition); }
-
-    /// @brief Convert State to string
-    /// @param state to convert
-    /// @return string representation of State enum entry
-    static auto ToStr(State state) -> std::string { return fkStrStateMap.at(state); }
-
-    /// @brief Convert StateTransition to string
-    /// @param transition to convert
-    /// @return string representation of StateTransition enum entry
-    static auto ToStr(StateTransition transition) -> std::string { return fkStrStateTransitionMap.at(transition); }
-
-    friend auto operator<<(std::ostream& os, const State& state) -> std::ostream& { return os << ToStr(state); }
-    friend auto operator<<(std::ostream& os, const StateTransition& transition) -> std::ostream& { return os << ToStr(transition); }
-
     StateMachine();
+    virtual ~StateMachine();
 
-    struct IllegalTransition : std::runtime_error { using std::runtime_error::runtime_error; };
+    bool ChangeState(const Transition transition);
+    bool ChangeState(const std::string& transition) { return ChangeState(GetTransition(transition)); }
 
-    struct StateChange : Event<State> {};
-    struct StateQueued : Event<State> {};
-    auto SubscribeToStateChange(const std::string& subscriber, std::function<void(typename StateChange::KeyType newState, State lastState)> callback) -> void { fCallbacks.Subscribe<StateChange, State>(subscriber, callback); }
-    auto UnsubscribeFromStateChange(const std::string& subscriber) -> void { fCallbacks.Unsubscribe<StateChange, State>(subscriber); }
-    auto SubscribeToStateQueued(const std::string& subscriber, std::function<void(typename StateQueued::KeyType newState, State lastState)> callback) -> void { fCallbacks.Subscribe<StateQueued, State>(subscriber, callback); }
-    auto UnsubscribeFromStateQueued(const std::string& subscriber) -> void { fCallbacks.Unsubscribe<StateQueued, State>(subscriber); }
+    void SubscribeToStateChange(const std::string& key, std::function<void(const State)> callback);
+    void UnsubscribeFromStateChange(const std::string& key);
 
-    auto GetCurrentState() const -> State { std::lock_guard<std::mutex> lock{fMutex}; return fState; }
-    auto GetCurrentErrorState() const -> State { std::lock_guard<std::mutex> lock{fMutex}; return fErrorState; }
-    auto GetLastQueuedState() const -> State { std::lock_guard<std::mutex> lock{fMutex}; return fNextStates.back(); }
+    void HandleStates(std::function<void(const State)> callback);
+    void StopHandlingStates();
 
-    auto ChangeState(StateTransition transition) -> void;
+    void SubscribeToNewTransition(const std::string& key, std::function<void(const Transition)> callback);
+    void UnsubscribeFromNewTransition(const std::string& key);
 
-    auto Run() -> void;
-    auto Reset() -> void;
+    bool NewStatePending() const;
+    void WaitForPendingState() const;
+    bool WaitForPendingStateFor(const int durationInMs) const;
 
-    auto NextStatePending() -> bool;
+    State GetCurrentState() const;
+    std::string GetCurrentStateName() const;
+
+    void Start();
+
+    void ProcessWork();
+
+    static std::string GetStateName(const State);
+    static std::string GetTransitionName(const Transition);
+    static State GetState(const std::string& state);
+    static Transition GetTransition(const std::string& transition);
+
+    struct ErrorStateException : std::runtime_error { using std::runtime_error::runtime_error; };
 
   private:
-    State fState;
-    State fErrorState;
-    std::deque<State> fNextStates;
-    EventManager fCallbacks;
+    std::shared_ptr<void> fFsm;
+};
 
-    static const std::unordered_map<std::string, State> fkStateStrMap;
-    static const std::unordered_map<State, std::string, tools::HashEnum<State>> fkStrStateMap;
-    static const std::unordered_map<std::string, StateTransition> fkStateTransitionStrMap;
-    static const std::unordered_map<StateTransition, std::string, tools::HashEnum<StateTransition>> fkStrStateTransitionMap;
+inline std::ostream& operator<<(std::ostream& os, const State& state) { return os << StateMachine::GetStateName(state); }
+inline std::ostream& operator<<(std::ostream& os, const Transition& transition) { return os << StateMachine::GetTransitionName(transition); }
 
-    mutable std::mutex fMutex;
-    std::condition_variable fNewState;
+} // namespace mq
+} // namespace fair
 
-    static auto Transition(const State currentState, const StateTransition transition) -> State;
-}; /* class StateMachine */
-
-} /* namespace mq */
-} /* namespace fair */
-
-#endif /* FAIR_MQ_STATEMACHINE_H */
+#endif /* FAIRMQSTATEMACHINE_H_ */

fairmq/Version.h.in

@@ -21,7 +21,7 @@
 #define FAIRMQ_GIT_DATE "@PROJECT_GIT_DATE@"
 #define FAIRMQ_REPO_URL "https://github.com/FairRootGroup/FairMQ"
 #define FAIRMQ_LICENSE "LGPL-3.0"
-#define FAIRMQ_COPYRIGHT "2012-2018 GSI"
+#define FAIRMQ_COPYRIGHT "2012-2019 GSI"
 #define FAIRMQ_BUILD_TYPE "@CMAKE_BUILD_TYPE@"
 
 #endif // FAIR_MQ_VERSION_H

fairmq/apply-clang-format.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-find . -type f \( -iname "*.h" ! -iname "*.pb.h" ! -iname "*LinkDef.h" -o -iname "*.cxx" -o -iname "*.tpl" \) -execdir clang-format -i {} \;

fairmq/devices/FairMQBenchmarkSampler.cxx

@@ -54,7 +54,7 @@ void FairMQBenchmarkSampler::Run()
 
     fair::mq::tools::RateLimiter rateLimiter(fMsgRate);
 
-    while (CheckCurrentState(RUNNING))
+    while (!NewStatePending())
     {
         if (fMultipart)
         {

fairmq/devices/FairMQMerger.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**
@@ -60,7 +60,7 @@ void FairMQMerger::Run()
 
     if (fMultipart)
     {
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             poller->Poll(100);
 
@@ -91,7 +91,7 @@ void FairMQMerger::Run()
     }
     else
     {
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             poller->Poll(100);
 

fairmq/devices/FairMQMerger.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/devices/FairMQMultiplier.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/devices/FairMQMultiplier.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/devices/FairMQProxy.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**
@@ -41,7 +41,7 @@ void FairMQProxy::Run()
 {
     if (fMultipart)
     {
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             FairMQParts payload;
             if (Receive(payload, fInChannelName) >= 0)
@@ -61,7 +61,7 @@ void FairMQProxy::Run()
     }
     else
     {
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             unique_ptr<FairMQMessage> payload(fTransportFactory->CreateMessage());
             if (Receive(payload, fInChannelName) >= 0)

fairmq/devices/FairMQProxy.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/devices/FairMQSink.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**
@@ -57,7 +57,7 @@ class FairMQSink : public FairMQDevice//, public OutputPolicy
         LOG(info) << "Starting the benchmark and expecting to receive " << fMaxIterations << " messages.";
         auto tStart = std::chrono::high_resolution_clock::now();
 
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             if (fMultipart)
             {

fairmq/devices/FairMQSplitter.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/devices/FairMQSplitter.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/nanomsg/FairMQPollerNN.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/nanomsg/FairMQPollerNN.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/nanomsg/FairMQSocketNN.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/nanomsg/FairMQSocketNN.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/ofi/CMakeLists.txt

@@ -1,12 +0,0 @@
-################################################################################
-#    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
-#                                                                              #
-#              This software is distributed under the terms of the             #
-#              GNU Lesser General Public Licence (LGPL) version 3,             #
-#                  copied verbatim in the file "LICENSE"                       #
-################################################################################
-
-protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS Control.proto)
-
-add_library(OfiTransport OBJECT ${PROTO_SRCS} ${PROTO_HDRS})
-target_include_directories(OfiTransport PRIVATE $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_INCLUDE_DIRECTORIES>)

fairmq/ofi/Context.cxx

@@ -10,21 +10,17 @@
 #include <fairmq/Tools.h>
 #include <FairMQLogger.h>
 
+#include <asiofi/version.hpp>
 #include <arpa/inet.h>
 #include <boost/version.hpp>
+#include <cassert>
 #include <cstring>
-#include <google/protobuf/stubs/common.h>
 #include <memory>
 #include <netinet/in.h>
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_errno.h>
 #include <regex>
 #include <string>
 #include <string.h>
 #include <sys/socket.h>
-#include <zmq.h>
 
 namespace fair
 {
@@ -35,20 +31,14 @@ namespace ofi
 
 using namespace std;
 
-Context::Context(int numberIoThreads)
-    : fOfiDomain(nullptr)
-    , fOfiFabric(nullptr)
-    , fOfiInfo(nullptr)
-    , fOfiAddressVector(nullptr)
-    , fOfiEventQueue(nullptr)
-    , fZmqContext(zmq_ctx_new())
-    , fIoWork(fIoContext)
+Context::Context(FairMQTransportFactory& sendFactory,
+                 FairMQTransportFactory& receiveFactory,
+                 int numberIoThreads)
+    : fIoWork(fIoContext)
+    , fReceiveFactory(receiveFactory)
+    , fSendFactory(sendFactory)
+    , fSizeHint(0)
 {
-    if (!fZmqContext)
-        throw ContextError{tools::ToString("Failed creating zmq context, reason: ", zmq_strerror(errno))};
-
-    GOOGLE_PROTOBUF_VERIFY_VERSION;
-
     InitThreadPool(numberIoThreads);
 }
 
@@ -58,257 +48,36 @@ auto Context::InitThreadPool(int numberIoThreads) -> void
 
     for (int i = 1; i <= numberIoThreads; ++i) {
         fThreadPool.emplace_back([&, i, numberIoThreads]{
-            LOG(debug) << "I/O thread #" << i << "/" << numberIoThreads << " started";
-            fIoContext.run();
-            LOG(debug) << "I/O thread #" << i << "/" << numberIoThreads << " stopped";
+            try {
+                LOG(debug) << "OFI transport: I/O thread #" << i << " of " << numberIoThreads << " started";
+                fIoContext.run();
+                LOG(debug) << "OFI transport: I/O thread #" << i << " of " << numberIoThreads << " stopped";
+            } catch (const std::exception& e) {
+                LOG(error) << "OFI transport: Uncaught exception in I/O thread #" << i << ": " << e.what();
+            } catch (...) {
+                LOG(error) << "OFI transport: Uncaught exception in I/O thread #" << i;
+            }
         });
     }
 }
 
+auto Context::Reset() -> void
+{
+    // TODO "Linger", rethink this
+    std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+
+    fIoContext.stop();
+}
+
 Context::~Context()
 {
-    fIoContext.stop();
     for (auto& thread : fThreadPool)
         thread.join();
-
-    if (zmq_ctx_term(fZmqContext) != 0)
-        LOG(error) << "Failed closing zmq context, reason: " << zmq_strerror(errno);
-
-	if (fOfiEventQueue) {
-        auto ret = fi_close(&fOfiEventQueue->fid);
-        if (ret != FI_SUCCESS)
-            LOG(error) << "Failed closing ofi event queue, reason: " << fi_strerror(ret);
-    }
-
-	if (fOfiAddressVector) {
-        auto ret = fi_close(&fOfiAddressVector->fid);
-        if (ret != FI_SUCCESS)
-            LOG(error) << "Failed closing ofi address vector, reason: " << fi_strerror(ret);
-    }
-
-	if (fOfiDomain) {
-        auto ret = fi_close(&fOfiDomain->fid);
-        if (ret != FI_SUCCESS)
-            LOG(error) << "Failed closing ofi domain, reason: " << fi_strerror(ret);
-    }
-
-	if (fOfiFabric) {
-        auto ret = fi_close(&fOfiFabric->fid);
-        if (ret != FI_SUCCESS)
-            LOG(error) << "Failed closing ofi fabric, reason: " << fi_strerror(ret);
-    }
 }
 
-auto Context::GetZmqVersion() const -> string
+auto Context::GetAsiofiVersion() const -> string
 {
-    int major, minor, patch;
-    zmq_version(&major, &minor, &patch);
-    return tools::ToString(major, ".", minor, ".", patch);
-}
-
-auto Context::GetOfiApiVersion() const -> string
-{
-    // Disable for now, does not compile with gcc 4.9.2 debian jessie
-    //auto ofi_version{fi_version()};
-    //return tools::ToString(FI_MAJOR(ofi_version), ".", FI_MINOR(ofi_version));
-    return "unknown";
-}
-
-auto Context::GetPbVersion() const -> string
-{
-    return google::protobuf::internal::VersionString(GOOGLE_PROTOBUF_VERSION);
-}
-
-auto Context::GetBoostVersion() const -> std::string
-{
-    return tools::ToString(BOOST_VERSION / 100000, ".", BOOST_VERSION / 100 % 1000, ".", BOOST_VERSION % 100);
-}
-
-auto Context::InitOfi(ConnectionType type, Address addr) -> void
-{
-    if (!fOfiInfo) {
-        sockaddr_in* sa = static_cast<sockaddr_in*>(malloc(sizeof(sockaddr_in)));
-        addr.Port = 0;
-        auto sa2 = ConvertAddress(addr);
-        memcpy(sa, &sa2, sizeof(sockaddr_in));
-
-        // Prepare fi_getinfo query
-        unique_ptr<fi_info, void(*)(fi_info*)> ofi_hints(fi_allocinfo(), fi_freeinfo);
-        ofi_hints->caps = FI_MSG;
-        //ofi_hints->mode = FI_CONTEXT;
-        ofi_hints->addr_format = FI_SOCKADDR_IN;
-        if (addr.Protocol == "tcp") {
-            ofi_hints->fabric_attr->prov_name = strdup("sockets");
-        } else if (addr.Protocol == "verbs") {
-            ofi_hints->fabric_attr->prov_name = strdup("verbs;ofi_rxm");
-        }
-        ofi_hints->ep_attr->type = FI_EP_RDM;
-        //ofi_hints->domain_attr->mr_mode = FI_MR_BASIC | FI_MR_SCALABLE;
-        ofi_hints->domain_attr->threading = FI_THREAD_SAFE;
-        ofi_hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
-        ofi_hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
-        ofi_hints->tx_attr->op_flags = FI_COMPLETION;
-        ofi_hints->rx_attr->op_flags = FI_COMPLETION;
-        if (type == ConnectionType::Bind) {
-            ofi_hints->src_addr = sa;
-            ofi_hints->src_addrlen = sizeof(sockaddr_in);
-            ofi_hints->dest_addr = nullptr;
-            ofi_hints->dest_addrlen = 0;
-        } else {
-            ofi_hints->src_addr = nullptr;
-            ofi_hints->src_addrlen = 0;
-            ofi_hints->dest_addr = sa;
-            ofi_hints->dest_addrlen = sizeof(sockaddr_in);
-        }
-
-        // Query fi_getinfo for fabric to use
-        auto res = fi_getinfo(FI_VERSION(1, 5), nullptr, nullptr, 0, ofi_hints.get(), &fOfiInfo);
-        if (res != 0) throw ContextError{tools::ToString("Failed querying fi_getinfo, reason: ", fi_strerror(res))};
-        if (!fOfiInfo) throw ContextError{"Could not find any ofi compatible fabric."};
-
-        // for(auto cursor{ofi_info}; cursor->next != nullptr; cursor = cursor->next) {
-            // LOG(debug) << fi_tostr(fOfiInfo, FI_TYPE_INFO);
-        // }
-        //
-    } else {
-        LOG(debug) << "Ofi info already queried. Skipping.";
-    }
-
-    OpenOfiFabric();
-//    OpenOfiEventQueue();
-    OpenOfiDomain();
-    OpenOfiAddressVector();
-}
-
-auto Context::OpenOfiFabric() -> void
-{
-    if (!fOfiFabric) {
-        assert(fOfiInfo);
-        fi_context ctx;
-        auto ret = fi_fabric(fOfiInfo->fabric_attr, &fOfiFabric, &ctx);
-        if (ret != FI_SUCCESS)
-            throw ContextError{tools::ToString("Failed opening ofi fabric, reason: ", fi_strerror(ret))};
-    } else {
-        // TODO Check, if requested fabric matches existing one.
-        // TODO Decide, if we want to support more than one fabric simultaneously.
-        LOG(debug) << "Ofi fabric already opened. Skipping.";
-    }
-}
-
-auto Context::OpenOfiDomain() -> void
-{
-    if (!fOfiDomain) {
-        assert(fOfiInfo);
-        assert(fOfiFabric);
-        fi_context ctx;
-        auto ret = fi_domain(fOfiFabric, fOfiInfo, &fOfiDomain, &ctx);
-        if (ret != FI_SUCCESS)
-            throw ContextError{tools::ToString("Failed opening ofi domain, reason: ", fi_strerror(ret))};
-    } else {
-        LOG(debug) << "Ofi domain already opened. Skipping.";
-    }
-}
-
-auto Context::OpenOfiEventQueue() -> void
-{
-    fi_eq_attr eqAttr = {100, 0, FI_WAIT_UNSPEC, 0, nullptr};
-    // size_t               size;      [> # entries for EQ <]
-    // uint64_t             flags;     [> operation flags <]
-    // enum fi_wait_obj     wait_obj;  [> requested wait object <]
-    // int                  signaling_vector; [> interrupt affinity <]
-    // struct fid_wait     *wait_set;  [> optional wait set <]
-    fi_context ctx;
-    auto ret = fi_eq_open(fOfiFabric, &eqAttr, &fOfiEventQueue, &ctx);
-    if (ret != FI_SUCCESS)
-        throw ContextError{tools::ToString("Failed opening ofi event queue, reason: ", fi_strerror(ret))};
-}
-
-auto Context::OpenOfiAddressVector() -> void
-{
-    if (!fOfiAddressVector) {
-        assert(fOfiDomain);
-        fi_av_attr attr = {fOfiInfo->domain_attr->av_type, 0, 1000, 0, nullptr, nullptr, 0};
-        // enum fi_av_type  type;        [> type of AV <]
-        // int              rx_ctx_bits; [> address bits to identify rx ctx <]
-        // size_t           count;       [> # entries for AV <]
-        // size_t           ep_per_node; [> # endpoints per fabric address <]
-        // const char       *name;       [> system name of AV <]
-        // void             *map_addr;   [> base mmap address <]
-        // uint64_t         flags;       [> operation flags <]
-        fi_context ctx;
-        auto ret = fi_av_open(fOfiDomain, &attr, &fOfiAddressVector, &ctx);
-        if (ret != FI_SUCCESS)
-            throw ContextError{tools::ToString("Failed opening ofi address vector, reason: ", fi_strerror(ret))};
-
-        //assert(fOfiEventQueue);
-        //ret = fi_av_bind(fOfiAddressVector, &fOfiEventQueue->fid, 0);
-        //if (ret != FI_SUCCESS)
-        //    throw ContextError{tools::ToString("Failed binding ofi event queue to address vector, reason: ", fi_strerror(ret))};
-    } else {
-        LOG(debug) << "Ofi address vector already opened. Skipping.";
-    }
-}
-
-auto Context::CreateOfiEndpoint() -> fid_ep*
-{
-    assert(fOfiDomain);
-    assert(fOfiInfo);
-    fid_ep* ep = nullptr;
-    fi_context ctx;
-    auto ret = fi_endpoint(fOfiDomain, fOfiInfo, &ep, &ctx);
-    if (ret != FI_SUCCESS)
-        throw ContextError{tools::ToString("Failed creating ofi endpoint, reason: ", fi_strerror(ret))};
-
-    //assert(fOfiEventQueue);
-    //ret = fi_ep_bind(ep, &fOfiEventQueue->fid, 0);
-    //if (ret != FI_SUCCESS)
-    //    throw ContextError{tools::ToString("Failed binding ofi event queue to ofi endpoint, reason: ", fi_strerror(ret))};
-
-    assert(fOfiAddressVector);
-    ret = fi_ep_bind(ep, &fOfiAddressVector->fid, 0);
-    if (ret != FI_SUCCESS)
-        throw ContextError{tools::ToString("Failed binding ofi address vector to ofi endpoint, reason: ", fi_strerror(ret))};
-
-    return ep;
-}
-
-auto Context::CreateOfiCompletionQueue(Direction dir) -> fid_cq*
-{
-    fid_cq* cq = nullptr;
-    fi_cq_attr attr = {0, 0, FI_CQ_FORMAT_DATA, FI_WAIT_UNSPEC, 0, FI_CQ_COND_NONE, nullptr};
-    if (dir == Direction::Receive) {
-        attr.size = fOfiInfo->rx_attr->size;
-    } else {
-        attr.size = fOfiInfo->tx_attr->size;
-    }
-	// size_t               size;      [> # entries for CQ <]
-	// uint64_t             flags;     [> operation flags <]
-	// enum fi_cq_format    format;    [> completion format <]
-	// enum fi_wait_obj     wait_obj;  [> requested wait object <]
-	// int                  signaling_vector; [> interrupt affinity <]
-	// enum fi_cq_wait_cond wait_cond; [> wait condition format <]
-	// struct fid_wait     *wait_set;  [> optional wait set <]
-    fi_context ctx;
-    auto ret = fi_cq_open(fOfiDomain, &attr, &cq, &ctx);
-    if (ret != FI_SUCCESS)
-        throw ContextError{tools::ToString("Failed creating ofi completion queue, reason: ", fi_strerror(ret))};
-    return cq;
-}
-
-auto Context::InsertAddressVector(sockaddr_in address) -> fi_addr_t
-{
-    fi_addr_t mappedAddress;
-    fi_context ctx;
-    auto ret = fi_av_insert(fOfiAddressVector, &address, 1, &mappedAddress, 0, &ctx);
-    if (ret != 1)
-        throw ContextError{tools::ToString("Failed to insert address into ofi address vector")};
-
-    return mappedAddress;
-}
-
-auto Context::AddressVectorLookup(fi_addr_t address) -> sockaddr_in
-{
-    throw ContextError("Not yet implemented");
+    return ASIOFI_VERSION;
 }
 
 auto Context::ConvertAddress(std::string address) -> Address
@@ -355,6 +124,16 @@ auto Context::VerifyAddress(const std::string& address) -> Address
     return addr;
 }
 
+auto Context::MakeReceiveMessage(size_t size) -> MessagePtr
+{
+    return fReceiveFactory.CreateMessage(size);
+}
+
+auto Context::MakeSendMessage(size_t size) -> MessagePtr
+{
+    return fSendFactory.CreateMessage(size);
+}
+
 } /* namespace ofi */
 } /* namespace mq */
 } /* namespace fair */

fairmq/ofi/Context.h

@@ -9,11 +9,16 @@
 #ifndef FAIR_MQ_OFI_CONTEXT_H
 #define FAIR_MQ_OFI_CONTEXT_H
 
-#include <boost/asio.hpp>
+#include <FairMQLogger.h>
+#include <FairMQTransportFactory.h>
+
+#include <asiofi/domain.hpp>
+#include <asiofi/fabric.hpp>
+#include <asiofi/info.hpp>
+#include <boost/asio/io_context.hpp>
 #include <memory>
 #include <netinet/in.h>
 #include <ostream>
-#include <rdma/fabric.h>
 #include <stdexcept>
 #include <string>
 #include <thread>
@@ -27,7 +32,20 @@ namespace ofi
 {
 
 enum class ConnectionType : bool { Bind, Connect };
-enum class Direction : bool { Receive, Transmit };
+
+struct Address {
+    std::string Protocol;
+    std::string Ip;
+    unsigned int Port;
+    friend auto operator<<(std::ostream& os, const Address& a) -> std::ostream&
+    {
+        return os << a.Protocol << "://" << a.Ip << ":" << a.Port;
+    }
+    friend auto operator==(const Address& lhs, const Address& rhs) -> bool
+    {
+        return (lhs.Protocol == rhs.Protocol) && (lhs.Ip == rhs.Ip) && (lhs.Port == rhs.Port);
+    }
+};
 
 /**
  * @class Context Context.h <fairmq/ofi/Context.h>
@@ -38,46 +56,33 @@ enum class Direction : bool { Receive, Transmit };
 class Context
 {
   public:
-    Context(int numberIoThreads = 2);
+    Context(FairMQTransportFactory& sendFactory,
+            FairMQTransportFactory& receiveFactory,
+            int numberIoThreads = 1);
     ~Context();
 
-    auto CreateOfiEndpoint() -> fid_ep*;
-    auto CreateOfiCompletionQueue(Direction dir) -> fid_cq*;
-    auto GetZmqVersion() const -> std::string;
-    auto GetOfiApiVersion() const -> std::string;
-    auto GetPbVersion() const -> std::string;
-    auto GetBoostVersion() const -> std::string;
-    auto GetZmqContext() const -> void* { return fZmqContext; }
-    auto GetIoContext() -> boost::asio::io_service& { return fIoContext; }
-    auto InsertAddressVector(sockaddr_in address) -> fi_addr_t;
-    auto AddressVectorLookup(fi_addr_t address) -> sockaddr_in;
-    struct Address {
-        std::string Protocol;
-        std::string Ip;
-        unsigned int Port;
-        friend auto operator<<(std::ostream& os, const Address& a) -> std::ostream& { return os << a.Protocol << "://" << a.Ip << ":" << a.Port; }
-    };
-    auto InitOfi(ConnectionType type, Address address) -> void;
+    auto GetAsiofiVersion() const -> std::string;
+    auto GetIoContext() -> boost::asio::io_context& { return fIoContext; }
     static auto ConvertAddress(std::string address) -> Address;
     static auto ConvertAddress(Address address) -> sockaddr_in;
     static auto ConvertAddress(sockaddr_in address) -> Address;
     static auto VerifyAddress(const std::string& address) -> Address;
+    auto Interrupt() -> void { LOG(debug) << "OFI transport: Interrupted (NOOP - not implemented)."; }
+    auto Resume() -> void { LOG(debug) << "OFI transport: Resumed (NOOP - not implemented)."; }
+    auto Reset() -> void;
+    auto MakeReceiveMessage(size_t size) -> MessagePtr;
+    auto MakeSendMessage(size_t size) -> MessagePtr;
+    auto GetSizeHint() -> size_t { return fSizeHint; }
+    auto SetSizeHint(size_t size) -> void { fSizeHint = size; }
 
   private:
-    void* fZmqContext;
-    fi_info* fOfiInfo;
-    fid_fabric* fOfiFabric;
-    fid_domain* fOfiDomain;
-    fid_av* fOfiAddressVector;
-    fid_eq* fOfiEventQueue;
-    boost::asio::io_service fIoContext;
-    boost::asio::io_service::work fIoWork;
+    boost::asio::io_context fIoContext;
+    boost::asio::io_context::work fIoWork;
     std::vector<std::thread> fThreadPool;
+    FairMQTransportFactory& fReceiveFactory;
+    FairMQTransportFactory& fSendFactory;
+    size_t fSizeHint;
 
-    auto OpenOfiFabric() -> void;
-    auto OpenOfiEventQueue() -> void;
-    auto OpenOfiDomain() -> void;
-    auto OpenOfiAddressVector() -> void;
     auto InitThreadPool(int numberIoThreads) -> void;
 }; /* class Context */
 

fairmq/ofi/Control.proto

@@ -1,25 +0,0 @@
-syntax = "proto3";
-option optimize_for = SPEED;
-
-package fair.mq.ofi;
-
-message DataAddressAnnouncement {
-  uint32 ipv4 = 1; // in_addr_t from <netinet/in.h>
-  uint32 port = 2; // in_port_t from <netinet/in.h>
-}
-
-message PostBuffer {
-  uint64 size = 1; // buffer size (size_t)
-}
-
-message PostBufferAcknowledgement {
-  uint64 size = 1; // size_t
-}
-
-message ControlMessage {
-  oneof type {
-    DataAddressAnnouncement data_address_announcement = 1;
-    PostBuffer post_buffer = 2;
-    PostBufferAcknowledgement post_buffer_acknowledgement = 3;
-  }    
-}

fairmq/ofi/ControlMessages.h

@@ -0,0 +1,116 @@
+/********************************************************************************
+ *    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_OFI_CONTROLMESSAGES_H
+#define FAIR_MQ_OFI_CONTROLMESSAGES_H
+
+#include <FairMQLogger.h>
+#include <boost/asio/buffer.hpp>
+#include <boost/container/pmr/memory_resource.hpp>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <type_traits>
+
+namespace boost {
+namespace asio {
+
+template<typename PodType>
+auto buffer(const PodType& obj) -> boost::asio::const_buffer
+{
+    return boost::asio::const_buffer(static_cast<const void*>(&obj), sizeof(PodType));
+}
+
+}   // namespace asio
+}   // namespace boost
+
+namespace fair {
+namespace mq {
+namespace ofi {
+
+enum class ControlMessageType
+{
+    Empty = 1,
+    PostBuffer,
+    PostMultiPartStartBuffer
+};
+
+struct Empty
+{};
+
+struct PostBuffer
+{
+    uint64_t size;   // buffer size (size_t)
+};
+
+struct PostMultiPartStartBuffer
+{
+    uint32_t numParts;   // buffer size (size_t)
+    uint64_t size;       // buffer size (size_t)
+};
+
+union ControlMessageContent
+{
+    PostBuffer postBuffer;
+    PostMultiPartStartBuffer postMultiPartStartBuffer;
+};
+
+struct ControlMessage
+{
+    ControlMessageType type;
+    ControlMessageContent msg;
+};
+
+template<typename T>
+using unique_ptr = std::unique_ptr<T, std::function<void(T*)>>;
+
+template<typename T, typename... Args>
+auto MakeControlMessageWithPmr(boost::container::pmr::memory_resource& pmr, Args&&... args)
+    -> ofi::unique_ptr<ControlMessage>
+{
+    void* mem = pmr.allocate(sizeof(ControlMessage));
+    ControlMessage* ctrl = new (mem) ControlMessage();
+
+    if (std::is_same<T, PostBuffer>::value) {
+        ctrl->type = ControlMessageType::PostBuffer;
+        ctrl->msg.postBuffer = PostBuffer(std::forward<Args>(args)...);
+    } else if (std::is_same<T, PostMultiPartStartBuffer>::value) {
+        ctrl->type = ControlMessageType::PostMultiPartStartBuffer;
+        ctrl->msg.postMultiPartStartBuffer = PostMultiPartStartBuffer(std::forward<Args>(args)...);
+    } else if (std::is_same<T, Empty>::value) {
+        ctrl->type = ControlMessageType::Empty;
+    }
+
+    return ofi::unique_ptr<ControlMessage>(ctrl, [&pmr](ControlMessage* p) {
+        p->~ControlMessage();
+        pmr.deallocate(p, sizeof(T));
+    });
+}
+
+template<typename T, typename... Args>
+auto MakeControlMessage(Args&&... args) -> ControlMessage
+{
+    ControlMessage ctrl;
+
+    if (std::is_same<T, PostBuffer>::value) {
+        ctrl.type = ControlMessageType::PostBuffer;
+    } else if (std::is_same<T, PostMultiPartStartBuffer>::value) {
+        ctrl.type = ControlMessageType::PostMultiPartStartBuffer;
+    } else if (std::is_same<T, Empty>::value) {
+        ctrl.type = ControlMessageType::Empty;
+    }
+    ctrl.msg = T(std::forward<Args>(args)...);
+
+    return ctrl;
+}
+
+}   // namespace ofi
+}   // namespace mq
+}   // namespace fair
+
+#endif /* FAIR_MQ_OFI_CONTROLMESSAGES_H */ 

fairmq/ofi/Message.cxx

@@ -10,6 +10,7 @@
 #include <fairmq/Tools.h>
 #include <FairMQLogger.h>
 
+#include <asiofi.hpp>
 #include <cassert>
 #include <cstdlib>
 #include <zmq.h>
@@ -23,38 +24,48 @@ namespace ofi
 
 using namespace std;
 
-Message::Message()
+Message::Message(boost::container::pmr::memory_resource* pmr)
     : fInitialSize(0)
     , fSize(0)
     , fData(nullptr)
     , fFreeFunction(nullptr)
     , fHint(nullptr)
+    , fPmr(pmr)
 {
 }
 
-Message::Message(const size_t size)
+Message::Message(boost::container::pmr::memory_resource* pmr, const size_t size)
     : fInitialSize(size)
     , fSize(size)
     , fData(nullptr)
     , fFreeFunction(nullptr)
     , fHint(nullptr)
+    , fPmr(pmr)
 {
     if (size) {
-        fData = malloc(size);
+        fData = fPmr->allocate(size);
         assert(fData);
     }
 }
 
-Message::Message(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
+Message::Message(boost::container::pmr::memory_resource* pmr,
+                 void* data,
+                 const size_t size,
+                 fairmq_free_fn* ffn,
+                 void* hint)
     : fInitialSize(size)
     , fSize(size)
     , fData(data)
     , fFreeFunction(ffn)
     , fHint(hint)
-{
-}
+    , fPmr(pmr)
+{}
 
-Message::Message(FairMQUnmanagedRegionPtr& /*region*/, void* /*data*/, const size_t /*size*/, void* /*hint*/)
+Message::Message(boost::container::pmr::memory_resource* /*pmr*/,
+                 FairMQUnmanagedRegionPtr& /*region*/,
+                 void* /*data*/,
+                 const size_t /*size*/,
+                 void* /*hint*/)
 {
     throw MessageError{"Not yet implemented."};
 }
@@ -62,9 +73,11 @@ Message::Message(FairMQUnmanagedRegionPtr& /*region*/, void* /*data*/, const siz
 auto Message::Rebuild() -> void
 {
     if (fFreeFunction) {
-      fFreeFunction(fData, fHint);
+        fFreeFunction(fData, fHint);
     } else {
-      free(fData);
+        if (fData) {
+            fPmr->deallocate(fData, fSize);
+        }
     }
     fData = nullptr;
     fInitialSize = 0;
@@ -78,10 +91,12 @@ auto Message::Rebuild(const size_t size) -> void
     if (fFreeFunction) {
       fFreeFunction(fData, fHint);
     } else {
-      free(fData);
+        if (fData) {
+            fPmr->deallocate(fData, fSize);
+        }
     }
     if (size) {
-        fData = malloc(size);
+        fData = fPmr->allocate(size);
         assert(fData);
     } else {
         fData = nullptr;
@@ -97,10 +112,12 @@ auto Message::Rebuild(void* /*data*/, const size_t size, fairmq_free_fn* ffn, vo
     if (fFreeFunction) {
       fFreeFunction(fData, fHint);
     } else {
-      free(fData);
+        if (fData) {
+            fPmr->deallocate(fData, fSize);
+        }
     }
     if (size) {
-        fData = malloc(size);
+        fData = fPmr->allocate(size);
         assert(fData);
     } else {
         fData = nullptr;
@@ -141,9 +158,11 @@ auto Message::Copy(const fair::mq::Message& /*msg*/) -> void
 Message::~Message()
 {
     if (fFreeFunction) {
-      fFreeFunction(fData, fHint);
+        fFreeFunction(fData, fHint);
     } else {
-      free(fData);
+        if (fData) {
+            fPmr->deallocate(fData, fSize);
+        }
     }
 }
 

fairmq/ofi/Message.h

@@ -12,10 +12,10 @@
 #include <FairMQMessage.h>
 #include <FairMQUnmanagedRegion.h>
 
-#include <zmq.h>
-
-#include <cstddef> // size_t
+#include <asiofi.hpp>
 #include <atomic>
+#include <cstddef> // size_t
+#include <zmq.h>
 
 namespace fair
 {
@@ -33,10 +33,18 @@ namespace ofi
 class Message final : public fair::mq::Message
 {
   public:
-    Message();
-    Message(const size_t size);
-    Message(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr);
-    Message(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0);
+    Message(boost::container::pmr::memory_resource* pmr);
+    Message(boost::container::pmr::memory_resource* pmr, const size_t size);
+    Message(boost::container::pmr::memory_resource* pmr,
+            void* data,
+            const size_t size,
+            fairmq_free_fn* ffn,
+            void* hint = nullptr);
+    Message(boost::container::pmr::memory_resource* pmr,
+            FairMQUnmanagedRegionPtr& region,
+            void* data,
+            const size_t size,
+            void* hint = 0);
 
     Message(const Message&) = delete;
     Message operator=(const Message&) = delete;
@@ -62,6 +70,7 @@ class Message final : public fair::mq::Message
     void* fData;
     fairmq_free_fn* fFreeFunction;
     void* fHint;
+    boost::container::pmr::memory_resource* fPmr;
 }; /* class Message */  
 
 } /* namespace ofi */

fairmq/ofi/Poller.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
@@ -28,13 +28,13 @@ Poller::Poller(const vector<FairMQChannel>& channels)
     fItems = new zmq_pollitem_t[fNumItems];
 
     for (int i = 0; i < fNumItems; ++i) {
-        fItems[i].socket = static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket();
+        fItems[i].socket = static_cast<Socket*>(&(channels.at(i).GetSocket()))->GetSocket();
         fItems[i].fd = 0;
         fItems[i].revents = 0;
 
         int type = 0;
         size_t size = sizeof(type);
-        zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+        zmq_getsockopt(static_cast<Socket*>(&(channels.at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
 
         SetItemEvents(fItems[i], type);
     }
@@ -46,13 +46,13 @@ Poller::Poller(const vector<const FairMQChannel*>& channels)
     fItems = new zmq_pollitem_t[fNumItems];
 
     for (int i = 0; i < fNumItems; ++i) {
-        fItems[i].socket = static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket();
+        fItems[i].socket = static_cast<Socket*>(&(channels.at(i)->GetSocket()))->GetSocket();
         fItems[i].fd = 0;
         fItems[i].revents = 0;
 
         int type = 0;
         size_t size = sizeof(type);
-        zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+        zmq_getsockopt(static_cast<Socket*>(&(channels.at(i)->GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
 
         SetItemEvents(fItems[i], type);
     }
@@ -76,13 +76,13 @@ Poller::Poller(const unordered_map<string, vector<FairMQChannel>>& channelsMap,
             for (unsigned int i = 0; i < channelsMap.at(channel).size(); ++i) {
                 index = fOffsetMap[channel] + i;
 
-                fItems[index].socket = static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
+                fItems[index].socket = static_cast<Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
                 fItems[index].fd = 0;
                 fItems[index].revents = 0;
 
                 int type = 0;
                 size_t size = sizeof(type);
-                zmq_getsockopt(static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+                zmq_getsockopt(static_cast<Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
 
                 SetItemEvents(fItems[index], type);
             }
@@ -124,7 +124,7 @@ auto Poller::CheckOutput(const int index) -> bool
     return fItems[index].revents & ZMQ_POLLOUT;
 }
 
-auto Poller::CheckInput(const string channelKey, const int index) -> bool
+auto Poller::CheckInput(const string& channelKey, const int index) -> bool
 {
     try {
         return fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLIN;
@@ -136,7 +136,7 @@ auto Poller::CheckInput(const string channelKey, const int index) -> bool
     }
 }
 
-auto Poller::CheckOutput(const string channelKey, const int index) -> bool
+auto Poller::CheckOutput(const string& channelKey, const int index) -> bool
 {
     try {
         return fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLOUT;

fairmq/ofi/Poller.h

@@ -51,8 +51,8 @@ class Poller final : public FairMQPoller
     auto Poll(const int timeout) -> void override;
     auto CheckInput(const int index) -> bool override;
     auto CheckOutput(const int index) -> bool override;
-    auto CheckInput(const std::string channelKey, const int index) -> bool override;
-    auto CheckOutput(const std::string channelKey, const int index) -> bool override;
+    auto CheckInput(const std::string& channelKey, const int index) -> bool override;
+    auto CheckOutput(const std::string& channelKey, const int index) -> bool override;
 
     ~Poller() override;
 

fairmq/ofi/Socket.h

@@ -12,13 +12,16 @@
 #include <FairMQSocket.h>
 #include <FairMQMessage.h>
 #include <fairmq/ofi/Context.h>
-#include <fairmq/ofi/Control.pb.h>
+#include <fairmq/ofi/ControlMessages.h>
 
+#include <asiofi/connected_endpoint.hpp>
+#include <asiofi/memory_resources.hpp>
+#include <asiofi/passive_endpoint.hpp>
+#include <asiofi/semaphore.hpp>
 #include <boost/asio.hpp>
 #include <memory> // unique_ptr
-#include <netinet/in.h>
-#include <rdma/fabric.h>
-class FairMQTransportFactory;
+#include <mutex>
+
 
 namespace fair
 {
@@ -36,7 +39,7 @@ namespace ofi
 class Socket final : public fair::mq::Socket
 {
   public:
-    Socket(Context& factory, const std::string& type, const std::string& name, const std::string& id = "", FairMQTransportFactory* fac);
+    Socket(Context& context, const std::string& type, const std::string& name, const std::string& id = "");
     Socket(const Socket&) = delete;
     Socket operator=(const Socket&) = delete;
 
@@ -50,12 +53,7 @@ class Socket final : public fair::mq::Socket
     auto Send(std::vector<MessagePtr>& msgVec, int timeout = 0) -> int64_t override;
     auto Receive(std::vector<MessagePtr>& msgVec, int timeout = 0) -> int64_t override;
 
-    auto TrySend(MessagePtr& msg) -> int override;
-    auto TryReceive(MessagePtr& msg) -> int override;
-    auto TrySend(std::vector<MessagePtr>& msgVec) -> int64_t override;
-    auto TryReceive(std::vector<MessagePtr>& msgVec) -> int64_t override;
-
-    auto GetSocket() const -> void* { return fControlSocket; }
+    auto GetSocket() const -> void* { return nullptr; }
 
     void SetLinger(const int value) override;
     int GetLinger() const override;
@@ -83,43 +81,42 @@ class Socket final : public fair::mq::Socket
     ~Socket() override;
 
   private:
-    void* fControlSocket;
-    void* fMonitorSocket;
-    fid_ep* fDataEndpoint;
-    fid_cq* fDataCompletionQueueTx;
-    fid_cq* fDataCompletionQueueRx;
+    Context& fContext;
+    asiofi::allocated_pool_resource fControlMemPool;
+    std::unique_ptr<asiofi::info> fOfiInfo;
+    std::unique_ptr<asiofi::fabric> fOfiFabric;
+    std::unique_ptr<asiofi::domain> fOfiDomain;
+    std::unique_ptr<asiofi::passive_endpoint> fPassiveEndpoint;
+    std::unique_ptr<asiofi::connected_endpoint> fDataEndpoint, fControlEndpoint;
     std::string fId;
     std::atomic<unsigned long> fBytesTx;
     std::atomic<unsigned long> fBytesRx;
     std::atomic<unsigned long> fMessagesTx;
     std::atomic<unsigned long> fMessagesRx;
-    Context& fContext;
-    fi_addr_t fRemoteDataAddr;
-    sockaddr_in fLocalDataAddr;
-    bool fWaitingForControlPeer;
-    boost::asio::io_service::strand fIoStrand;
-
+    Address fRemoteAddr;
+    Address fLocalAddr;
     int fSndTimeout;
     int fRcvTimeout;
+    std::mutex fSendQueueMutex, fRecvQueueMutex;
+    std::queue<std::vector<MessagePtr>> fSendQueue, fRecvQueue;
+    std::vector<MessagePtr> fInflightMultiPartMessage;
+    int64_t fMultiPartRecvCounter;
+    asiofi::synchronized_semaphore fSendPushSem, fSendPopSem, fRecvPushSem, fRecvPopSem;
+    std::atomic<bool> fNeedOfiMemoryRegistration;
 
-    auto SendImpl(MessagePtr& msg, const int flags, const int timeout) -> int;
-    auto ReceiveImpl(MessagePtr& msg, const int flags, const int timeout) -> int;
-    auto SendImpl(std::vector<MessagePtr>& msgVec, const int flags, const int timeout) -> int64_t;
-    auto ReceiveImpl(std::vector<MessagePtr>& msgVec, const int flags, const int timeout) -> int64_t;
-
-    auto InitDataEndpoint() -> void;
-    auto WaitForControlPeer() -> void;
-    auto AnnounceDataAddress() -> void;
-    auto SendControlMessage(std::unique_ptr<ControlMessage> ctrl) -> void;
-    auto ReceiveControlMessage() -> std::unique_ptr<ControlMessage>;
-    auto ProcessDataAddressAnnouncement(std::unique_ptr<ControlMessage> ctrl) -> void;
-    auto ConnectControlSocket(Context::Address address) -> void;
-    auto BindControlSocket(Context::Address address) -> void;
+    auto InitOfi(Address addr) -> void;
+    auto BindControlEndpoint() -> void;
+    auto BindDataEndpoint() -> void;
+    enum class Band { Control, Data };
+    auto ConnectEndpoint(std::unique_ptr<asiofi::connected_endpoint>& endpoint, Band type) -> void;
+    auto SendQueueReader() -> void;
+    auto SendQueueReaderStatic() -> void;
+    auto RecvControlQueueReader() -> void;
+    auto RecvQueueReaderStatic() -> void;
+    auto OnRecvControl(ofi::unique_ptr<ControlMessage> ctrl) -> void;
+    auto DataMessageReceived(MessagePtr msg) -> void;
 }; /* class Socket */
 
-// helper function to clean up the object holding the data after it is transported.
-void free_string(void* /*data*/, void* hint);
-
 struct SilentSocketError : SocketError { using SocketError::SocketError; };
 
 } /* namespace ofi */

fairmq/ofi/TransportFactory.cxx

@@ -24,59 +24,68 @@ namespace ofi
 using namespace std;
 
 TransportFactory::TransportFactory(const string& id, const FairMQProgOptions* config)
-try : FairMQTransportFactory{id}
-{
-    LOG(debug) << "Transport: Using ZeroMQ (" << fContext.GetZmqVersion() << ") & "
-               << "OFI libfabric (API " << fContext.GetOfiApiVersion() << ") & "
-               << "Google Protobuf (" << fContext.GetPbVersion() << ") & "
-               << "Boost.Asio (" << fContext.GetBoostVersion() << ")";
-}
-catch (ContextError& e)
+try : FairMQTransportFactory(id)
+    , fContext(*this, *this, 1)
 {
+    LOG(debug) << "OFI transport: asiofi (" << fContext.GetAsiofiVersion() << ")";
+
+    if (config) {
+        fContext.SetSizeHint(config->GetValue<size_t>("ofi-size-hint"));
+    }
+} catch (ContextError& e) {
     throw TransportFactoryError{e.what()};
 }
 
-auto TransportFactory::CreateMessage() const -> MessagePtr
+auto TransportFactory::CreateMessage() -> MessagePtr
 {
-    return MessagePtr{new Message()};
+    return MessagePtr{new Message(&fMemoryResource)};
 }
 
-auto TransportFactory::CreateMessage(const size_t size) const -> MessagePtr
+auto TransportFactory::CreateMessage(const size_t size) -> MessagePtr
 {
-    return MessagePtr{new Message(size)};
+    return MessagePtr{new Message(&fMemoryResource, size)};
 }
 
-auto TransportFactory::CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint) const -> MessagePtr
+auto TransportFactory::CreateMessage(void* data,
+                                     const size_t size,
+                                     fairmq_free_fn* ffn,
+                                     void* hint) -> MessagePtr
 {
-    return MessagePtr{new Message(data, size, ffn, hint)};
+    return MessagePtr{new Message(&fMemoryResource, data, size, ffn, hint)};
 }
 
-auto TransportFactory::CreateMessage(UnmanagedRegionPtr& region, void* data, const size_t size, void* hint) const -> MessagePtr
+auto TransportFactory::CreateMessage(UnmanagedRegionPtr& region,
+                                     void* data,
+                                     const size_t size,
+                                     void* hint) -> MessagePtr
 {
-    return MessagePtr{new Message(region, data, size, hint)};
+    return MessagePtr{new Message(&fMemoryResource, region, data, size, hint)};
 }
 
 auto TransportFactory::CreateSocket(const string& type, const string& name) -> SocketPtr
 {
-    return SocketPtr{new Socket(fContext, type, name, GetId(), this)};
+    return SocketPtr{new Socket(fContext, type, name, GetId())};
 }
 
-auto TransportFactory::CreatePoller(const vector<FairMQChannel>& channels) const -> PollerPtr
+auto TransportFactory::CreatePoller(const vector<FairMQChannel>& /*channels*/) const -> PollerPtr
 {
-    return PollerPtr{new Poller(channels)};
+    throw runtime_error{"Not yet implemented (Poller)."};
+    // return PollerPtr{new Poller(channels)};
 }
 
-auto TransportFactory::CreatePoller(const vector<const FairMQChannel*>& channels) const -> PollerPtr
+auto TransportFactory::CreatePoller(const vector<FairMQChannel*>& /*channels*/) const -> PollerPtr
 {
-    return PollerPtr{new Poller(channels)};
+    throw runtime_error{"Not yet implemented (Poller)."};
+    // return PollerPtr{new Poller(channels)};
 }
 
-auto TransportFactory::CreatePoller(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList) const -> PollerPtr
+auto TransportFactory::CreatePoller(const unordered_map<string, vector<FairMQChannel>>& /*channelsMap*/, const vector<string>& /*channelList*/) const -> PollerPtr
 {
-    return PollerPtr{new Poller(channelsMap, channelList)};
+    throw runtime_error{"Not yet implemented (Poller)."};
+    // return PollerPtr{new Poller(channelsMap, channelList)};
 }
 
-auto TransportFactory::CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback) const -> UnmanagedRegionPtr
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, FairMQRegionCallback /*callback*/) const -> UnmanagedRegionPtr
 {
     throw runtime_error{"Not yet implemented UMR."};
 }

fairmq/ofi/TransportFactory.h

@@ -13,6 +13,8 @@
 #include <options/FairMQProgOptions.h>
 #include <fairmq/ofi/Context.h>
 
+#include <asiofi.hpp>
+
 namespace fair
 {
 namespace mq
@@ -22,7 +24,7 @@ namespace ofi
 
 /**
  * @class TransportFactory TransportFactory.h <fairmq/ofi/TransportFactory.h>
- * @brief FairMQ transport factory for the ofi transport (implemented with ZeroMQ + libfabric)
+ * @brief FairMQ transport factory for the ofi transport
  *
  * @todo TODO insert long description
  */
@@ -33,27 +35,28 @@ class TransportFactory final : public FairMQTransportFactory
     TransportFactory(const TransportFactory&) = delete;
     TransportFactory operator=(const TransportFactory&) = delete;
 
-    auto CreateMessage() const -> MessagePtr override;
-    auto CreateMessage(const std::size_t size) const -> MessagePtr override;
-    auto CreateMessage(void* data, const std::size_t size, fairmq_free_fn* ffn, void* hint = nullptr) const -> MessagePtr override;
-    auto CreateMessage(UnmanagedRegionPtr& region, void* data, const std::size_t size, void* hint = nullptr) const -> MessagePtr override;
+    auto CreateMessage() -> MessagePtr override;
+    auto CreateMessage(const std::size_t size) -> MessagePtr override;
+    auto CreateMessage(void* data, const std::size_t size, fairmq_free_fn* ffn, void* hint = nullptr) -> MessagePtr override;
+    auto CreateMessage(UnmanagedRegionPtr& region, void* data, const std::size_t size, void* hint = nullptr) -> MessagePtr override;
 
     auto CreateSocket(const std::string& type, const std::string& name) -> SocketPtr override;
 
     auto CreatePoller(const std::vector<FairMQChannel>& channels) const -> PollerPtr override;
-    auto CreatePoller(const std::vector<const FairMQChannel*>& channels) const -> PollerPtr override;
+    auto CreatePoller(const std::vector<FairMQChannel*>& channels) const -> PollerPtr override;
     auto CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const -> PollerPtr override;
 
     auto CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr) const -> UnmanagedRegionPtr override;
 
     auto GetType() const -> Transport override;
 
-    void Interrupt() override {}
-    void Resume() override {}
-    void Reset() override {}
+    void Interrupt() override { fContext.Interrupt(); }
+    void Resume() override { fContext.Resume(); }
+    void Reset() override { fContext.Reset(); }
 
   private:
     mutable Context fContext;
+    asiofi::allocated_pool_resource fMemoryResource;
 }; /* class TransportFactory */  
 
 } /* namespace ofi */