Update

Improve ctest definitions

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,6 @@
 Describe your proposal.
 
-Mention any issue this PR is resolves or is related to.
+Mention any issue this PR resolves or is related to.
 
 ---
 

CMakeLists.txt

@@ -6,7 +6,7 @@
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
-cmake_minimum_required(VERSION 3.9.4 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
 
 
 # Project ######################################################################
@@ -19,6 +19,12 @@ get_git_version()
 project(FairMQ VERSION ${PROJECT_VERSION} LANGUAGES CXX)
 message(STATUS "${BWhite}${PROJECT_NAME}${CR} ${PROJECT_GIT_VERSION} from ${PROJECT_DATE}")
 
+if(BUILD_OFI_TRANSPORT)
+  set(PROJECT_MIN_CXX_STANDARD 14)
+else()
+  set(PROJECT_MIN_CXX_STANDARD 11)
+endif()
+
 set_fairmq_defaults()
 
 include(CTest)
@@ -32,6 +38,7 @@ cmake_dependent_option(BUILD_TESTING "Build tests." OFF "BUILD_FAIRMQ" OFF)
 cmake_dependent_option(BUILD_NANOMSG_TRANSPORT "Build nanomsg transport." OFF "BUILD_FAIRMQ" OFF)
 cmake_dependent_option(BUILD_OFI_TRANSPORT "Build experimental OFI transport." OFF "BUILD_FAIRMQ" OFF)
 cmake_dependent_option(BUILD_DDS_PLUGIN "Build DDS plugin." OFF "BUILD_FAIRMQ" OFF)
+cmake_dependent_option(BUILD_PMIX_PLUGIN "Build PMIx plugin." OFF "BUILD_FAIRMQ" OFF)
 cmake_dependent_option(BUILD_EXAMPLES "Build FairMQ examples." ON "BUILD_FAIRMQ" OFF)
 option(BUILD_DOCS "Build FairMQ documentation." OFF)
 option(FAST_BUILD "Fast production build. Not recommended for development." OFF)
@@ -47,35 +54,69 @@ set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
 set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_package(Threads REQUIRED)
 
-if(BUILD_FAIRMQ)
-  find_package2(PUBLIC Boost VERSION 1.64 REQUIRED
-    COMPONENTS container program_options thread system filesystem regex date_time signals
-  )
-  find_package2(PUBLIC FairLogger VERSION 1.2.0 REQUIRED)
-  find_package2(PRIVATE ZeroMQ VERSION 4.1.5 REQUIRED)
-endif()
-
 if(BUILD_NANOMSG_TRANSPORT)
   find_package2(PRIVATE nanomsg REQUIRED)
   set(PROJECT_nanomsg_VERSION 1.1.3) # Once upstream releases 1.1.5, we should bump again and use version check
-  find_package2(PRIVATE msgpack VERSION 3.1.0 REQUIRED)
 endif()
 
 if(BUILD_OFI_TRANSPORT)
-  find_package2(PRIVATE OFI VERSION 1.6.0 REQUIRED COMPONENTS fi_sockets fi_verbs)
-  find_package2(PRIVATE Protobuf VERSION 3.4.0 REQUIRED)
+  find_package2(PRIVATE asiofi REQUIRED
+    VERSION 0.2.0
+  )
+  find_package2(PRIVATE OFI REQUIRED
+    VERSION ${asiofi_OFI_VERSION}
+    COMPONENTS ${asiofi_OFI_COMPONENTS}
+  )
+  find_package2(PRIVATE AZMQ REQUIRED)
+  set(PROJECT_AZMQ_VERSION 1.0.2)
+endif()
+
+if(BUILD_NANOMSG_TRANSPORT)
+  find_package2(PRIVATE msgpack REQUIRED
+    VERSION 3.1.0
+  )
+endif()
+
+if(BUILD_FAIRMQ)
+  find_package2(PUBLIC Boost REQUIRED
+    VERSION 1.64 ${asiofi_Boost_VERSION}
+
+    COMPONENTS
+    container
+    program_options
+    filesystem
+    date_time
+    regex
+    ${asiofi_Boost_COMPONENTS}
+  )
+  find_package2(PUBLIC FairLogger REQUIRED
+    VERSION 1.2.0
+  )
+  find_package2(PRIVATE ZeroMQ REQUIRED
+    VERSION 4.1.5
+  )
 endif()
 
 if(BUILD_DDS_PLUGIN)
   find_package2(PRIVATE DDS VERSION 2.2 REQUIRED)
 endif()
 
+if(BUILD_PMIX_PLUGIN)
+  find_package2(PRIVATE PMIx VERSION 2.1.4 REQUIRED)
+endif()
+
 if(BUILD_TESTING)
-  find_package2(PRIVATE GTest VERSION 1.7.0 REQUIRED)
+  find_package2(PRIVATE GTest REQUIRED
+    VERSION 1.7.0
+  )
 endif()
 
 if(BUILD_DOCS)
-  find_package2(PRIVATE Doxygen VERSION 1.8.8 REQUIRED COMPONENTS dot OPTIONAL_COMPONENTS mscgen dia)
+  find_package2(PRIVATE Doxygen REQUIRED
+    VERSION 1.8.8
+    COMPONENTS dot
+    OPTIONAL_COMPONENTS mscgen dia
+  )
 endif()
 ################################################################################
 
@@ -101,7 +142,7 @@ endif()
 if(BUILD_DOCS)
   set(DOXYGEN_OUTPUT_DIRECTORY doxygen)
   set(DOXYGEN_PROJECT_NUMBER ${PROJECT_GIT_VERSION})
-  set(DOXYGEN_PROJECT_BRIEF "C++ Message Passing Framework")
+  set(DOXYGEN_PROJECT_BRIEF "C++ Message Queuing Library and Framework")
   set(DOXYGEN_USE_MDFILE_AS_MAINPAGE README.md)
   set(DOXYGEN_HTML_FOOTER docs/footer.html)
   doxygen_add_docs(doxygen README.md fairmq)
@@ -120,6 +161,9 @@ endif()
 if(BUILD_DDS_PLUGIN)
   list(APPEND PROJECT_PACKAGE_COMPONENTS dds_plugin)
 endif()
+if(BUILD_PMIX_PLUGIN)
+  list(APPEND PROJECT_PACKAGE_COMPONENTS pmix_plugin)
+endif()
 if(BUILD_NANOMSG_TRANSPORT)
   list(APPEND PROJECT_PACKAGE_COMPONENTS nanomsg_transport)
 endif()
@@ -151,7 +195,7 @@ if(BUILD_DDS_PLUGIN)
   )
 endif()
 if(BUILD_OFI_TRANSPORT)
-  install(FILES cmake/FindOFI.cmake
+  install(FILES cmake/FindAZMQ.cmake
     DESTINATION ${PROJECT_INSTALL_CMAKEMODDIR}
   )
 endif()
@@ -166,6 +210,8 @@ install_cmake_package()
 
 
 # Summary ######################################################################
+message(STATUS "  ")
+message(STATUS "  ${Cyan}CXX STANDARD${CR}       ${BGreen}C++${CMAKE_CXX_STANDARD}${CR} (>= C++${PROJECT_MIN_CXX_STANDARD}, change with ${BMagenta}-DCMAKE_CXX_STANDARD=17${CR})")
 if(CMAKE_CXX_FLAGS)
   message(STATUS "  ")
   message(STATUS "  ${Cyan}GLOBAL CXX FLAGS${CR}  ${BGreen}${CMAKE_CXX_FLAGS}${CR}")
@@ -218,6 +264,8 @@ if(PROJECT_PACKAGE_DEPENDENCIES)
     elseif(${dep} STREQUAL msgpack)
       get_target_property(msgpack_include msgpackc-cxx INTERFACE_INCLUDE_DIRECTORIES)
       get_filename_component(prefix ${msgpack_include}/.. ABSOLUTE)
+    elseif(${dep} STREQUAL asiofi)
+      set(prefix ${asiofi_ROOT})
     elseif(${dep} STREQUAL OFI)
       get_filename_component(prefix ${${dep}_INCLUDE_DIRS}/.. ABSOLUTE)
     elseif(${dep} STREQUAL nanomsg)
@@ -257,9 +305,9 @@ else()
 endif()
 message(STATUS "  ${BWhite}nanomsg_transport${CR}  ${nn_summary}")
 if(BUILD_OFI_TRANSPORT)
-  set(ofi_summary "${BGreen}YES${CR}    EXPERIMENTAL (disable with ${BMagenta}-DBUILD_OFI_TRANSPORT=OFF${CR})")
+  set(ofi_summary "${BGreen}YES${CR}    EXPERIMENTAL (requires C++14) (disable with ${BMagenta}-DBUILD_OFI_TRANSPORT=OFF${CR})")
 else()
-  set(ofi_summary "${BRed} NO${CR}    EXPERIMENTAL (default, enable with ${BMagenta}-DBUILD_OFI_TRANSPORT=ON${CR})")
+  set(ofi_summary "${BRed} NO${CR}    EXPERIMENTAL (requires C++14) (default, enable with ${BMagenta}-DBUILD_OFI_TRANSPORT=ON${CR})")
 endif()
 message(STATUS "  ${BWhite}ofi_transport${CR}      ${ofi_summary}")
 if(BUILD_DDS_PLUGIN)
@@ -268,6 +316,12 @@ else()
   set(dds_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_DDS_PLUGIN=ON${CR})")
 endif()
 message(STATUS "  ${BWhite}dds_plugin${CR}         ${dds_summary}")
+if(BUILD_PMIX_PLUGIN)
+  set(pmix_summary "${BGreen}YES${CR}    (disable with ${BMagenta}-DBUILD_PMIX_PLUGIN=OFF${CR})")
+else()
+  set(pmix_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_PMIX_PLUGIN=ON${CR})")
+endif()
+message(STATUS "  ${BWhite}pmix_plugin${CR}        ${pmix_summary}")
 if(BUILD_EXAMPLES)
   set(examples_summary "${BGreen}YES${CR}    (default, disable with ${BMagenta}-DBUILD_EXAMPLES=OFF${CR})")
 else()

FairMQTest.cmake

@@ -27,6 +27,7 @@ Set(configure_options "${configure_options};-DCTEST_USE_LAUNCHERS=${CTEST_USE_LA
 Set(configure_options "${configure_options};-DDISABLE_COLOR=ON")
 Set(configure_options "${configure_options};-DCMAKE_PREFIX_PATH=$ENV{SIMPATH}")
 Set(configure_options "${configure_options};-DBUILD_NANOMSG_TRANSPORT=ON")
+# Set(configure_options "${configure_options};-DBUILD_OFI_TRANSPORT=ON")
 Set(configure_options "${configure_options};-DBUILD_DDS_PLUGIN=ON")
 Set(configure_options "${configure_options};-DFAST_BUILD=ON")
 Set(configure_options "${configure_options};-DCOTIRE_MAXIMUM_NUMBER_OF_UNITY_INCLUDES=-j$ENV{number_of_processors}")
@@ -57,7 +58,8 @@ Ctest_Configure(BUILD "${CTEST_BINARY_DIRECTORY}"
 Ctest_Build(BUILD "${CTEST_BINARY_DIRECTORY}")
 
 Ctest_Test(BUILD "${CTEST_BINARY_DIRECTORY}" 
-           PARALLEL_LEVEL $ENV{number_of_processors}
+           # PARALLEL_LEVEL $ENV{number_of_processors}
+           PARALLEL_LEVEL 1
            RETURN_VALUE _ctest_test_ret_val
           )
 If("$ENV{do_codecov_upload}")

Jenkinsfile

@@ -66,14 +66,15 @@ pipeline{
       steps{
         script {
           def build_jobs = jobMatrix('alfa-ci/build', [
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',          fairsoft: 'fairmq_dev'],
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
             [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
+            [os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             sh './Dart.sh alfa_ci Dart.cfg'
           }
 
           def profile_jobs = jobMatrix('alfa-ci/codecov', [
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',          fairsoft: 'fairmq_dev'],
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             withCredentials([string(credentialsId: 'fairmq_codecov_token', variable: 'CODECOV_TOKEN')]) {
               sh './Dart.sh codecov Dart.cfg'

Jenkinsfile.nightly

@@ -63,8 +63,9 @@ pipeline{
       steps{
         script {
           parallel(buildMatrix([
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',         fairsoft: 'fairmq_dev'],
-            [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0',  fairsoft: 'fairmq_dev'],
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
+            [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
+            [os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             sh './Dart.sh Nightly Dart.cfg'
             sh './Dart.sh Profile Dart.cfg'

README.md

@@ -38,7 +38,7 @@ a simulation, reconstruction and analysis framework.
   * PUBLIC: [**Boost**](https://www.boost.org/), [**FairLogger**](https://github.com/FairRootGroup/FairLogger)
   * BUILD: [CMake](https://cmake.org/), [GTest](https://github.com/google/googletest), [Doxygen](http://www.doxygen.org/)
   * PRIVATE: [ZeroMQ](http://zeromq.org/), [Msgpack](https://msgpack.org/index.html), [nanomsg](http://nanomsg.org/),
-[OFI](https://ofiwg.github.io/libfabric/), [Protobuf](https://developers.google.com/protocol-buffers/), [DDS](http://dds.gsi.de)
+[asiofi](https://github.com/FairRootGroup/asiofi), [DDS](http://dds.gsi.de), [PMIx](https://pmix.org/), [AZMQ](https://github.com/zeromq/azmq), [asiofi](https://github.com/FairRootGroup/asiofi)
 
   Supported platforms: Linux and MacOS.
 
@@ -51,7 +51,7 @@ cmake -DCMAKE_INSTALL_PREFIX=./fairmq_install ../fairmq
 cmake --build . --target install
 ```
 
-If dependencies are not installed in standard system directories, you can hint the installation location via `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...`. `{DEPENDENCY}` can be `GTEST`, `BOOST`, `FAIRLOGGER`, `ZEROMQ`, `MSGPACK`, `NANOMSG`, `OFI`, `PROTOBUF`, or `DDS` (`*_ROOT` variables can also be environment variables).
+If dependencies are not installed in standard system directories, you can hint the installation location via `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...`. `{DEPENDENCY}` can be `GTEST`, `BOOST`, `FAIRLOGGER`, `ZEROMQ`, `MSGPACK`, `NANOMSG`, `OFI`, `PMIX`, `ASIOFI`, `AZMQ` or `DDS` (`*_ROOT` variables can also be environment variables).
 
 ## Usage
 
@@ -104,6 +104,7 @@ On command line:
   * `-DBUILD_NANOMSG_TRANSPORT=ON` enables building of nanomsg transport.
   * `-DBUILD_OFI_TRANSPORT=ON` enables building of the experimental OFI transport.
   * `-DBUILD_DDS_PLUGIN=ON` enables building of the DDS plugin.
+  * `-DBUILD_PMIX_PLUGIN=ON` enables building of the PMIx plugin.
   * `-DBUILD_DOCS=ON` enables building of API docs.
   * You can hint non-system installations for dependent packages, see the #Installation section above
 
@@ -119,7 +120,7 @@ After the `find_package(FairMQ)` call the following CMake variables are defined:
 | `${FairMQ_#COMPONENT#_FOUND}` | `TRUE` if this component was built |
 | `${FairMQ_VERSION}` | the version in format `MAJOR.MINOR.PATCH` |
 | `${FairMQ_GIT_VERSION}` | the version in the format returned by `git describe --tags --dirty --match "v*"` |
-| `${FairMQ_ROOT}` | the actual installation prefix, notice the difference to the hint variable `FAIRMQ_ROOT` |
+| `${FairMQ_PREFIX}` | the actual installation prefix |
 | `${FairMQ_BINDIR}` | the installation bin directory |
 | `${FairMQ_INCDIR}` | the installation include directory |
 | `${FairMQ_LIBDIR}` | the installation lib directory |

cmake/FairMQConfig.cmake.in

@@ -13,7 +13,7 @@ set(@PROJECT_NAME@_VERSION @PROJECT_VERSION@)
 set(@PROJECT_NAME@_GIT_VERSION @PROJECT_GIT_VERSION@)
 set(@PROJECT_NAME@_GIT_DATE @PROJECT_GIT_DATE@)
 
-set_and_check(@PROJECT_NAME@_ROOT @PACKAGE_CMAKE_INSTALL_PREFIX@)
+set_and_check(@PROJECT_NAME@_PREFIX @PACKAGE_CMAKE_INSTALL_PREFIX@)
 set_and_check(@PROJECT_NAME@_BINDIR @PACKAGE_CMAKE_INSTALL_PREFIX@/@PROJECT_INSTALL_BINDIR@)
 set_and_check(@PROJECT_NAME@_INCDIR @PACKAGE_CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@)
 set_and_check(@PROJECT_NAME@_LIBDIR @PACKAGE_CMAKE_INSTALL_PREFIX@/@PROJECT_INSTALL_LIBDIR@)

cmake/FairMQLib.cmake

@@ -40,6 +40,7 @@ macro(set_fairmq_cmake_policies)
     CMP0042 # MACOSX_RPATH is enabled by default.
     CMP0048 # The ``project()`` command manages VERSION variables.
     CMP0054 # Only interpret ``if()`` arguments as variables or keywords when unquoted.
+    CMP0074 # ``find_package()`` uses ``<PackageName>_ROOT`` variables.
   )
     if(POLICY ${policy})
       cmake_policy(SET ${policy} NEW)
@@ -115,11 +116,9 @@ macro(set_fairmq_defaults)
   # Handle C++ standard level
   set(CMAKE_CXX_STANDARD_REQUIRED ON)
   if(NOT CMAKE_CXX_STANDARD)
-    set(CMAKE_CXX_STANDARD 11)
-  elseif(${CMAKE_CXX_STANDARD} LESS 11)
-    message(FATAL_ERROR "A minimum CMAKE_CXX_STANDARD of 11 is required.")
-  elseif(${CMAKE_CXX_STANDARD} GREATER 11)
-    message(WARNING "A CMAKE_CXX_STANDARD of ${CMAKE_CXX_STANDARD} (greater than 11) is not tested. Use on your own risk.")
+    set(CMAKE_CXX_STANDARD ${PROJECT_MIN_CXX_STANDARD})
+  elseif(${CMAKE_CXX_STANDARD} LESS ${PROJECT_MIN_CXX_STANDARD})
+    message(FATAL_ERROR "A minimum CMAKE_CXX_STANDARD of ${PROJECT_MIN_CXX_STANDARD} is required.")
   endif()
   set(CMAKE_CXX_EXTENSIONS OFF)
 
@@ -261,6 +260,8 @@ endfunction()
 
 # Configure/Install CMake package
 macro(install_cmake_package)
+  list(SORT PROJECT_PACKAGE_DEPENDENCIES)
+  list(SORT PROJECT_INTERFACE_PACKAGE_DEPENDENCIES)
   include(CMakePackageConfigHelpers)
   set(PACKAGE_INSTALL_DESTINATION
     ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}-${PROJECT_GIT_VERSION}
@@ -293,35 +294,50 @@ macro(install_cmake_package)
 endmacro()
 
 macro(find_package2 qualifier pkgname)
-  cmake_parse_arguments(ARGS "" "VERSION" "COMPONENTS" ${ARGN})
+  cmake_parse_arguments(ARGS "" "" "VERSION;COMPONENTS" ${ARGN})
 
   string(TOUPPER ${pkgname} pkgname_upper)
   set(old_CPP ${CMAKE_PREFIX_PATH})
   set(CMAKE_PREFIX_PATH ${${pkgname_upper}_ROOT} $ENV{${pkgname_upper}_ROOT} ${CMAKE_PREFIX_PATH})
+  unset(__version__)
+  unset(__components__)
+  if(ARGS_VERSION)
+    list(GET ARGS_VERSION 0 __version__)
+    list(LENGTH ARGS_VERSION __length__)
+    foreach(v IN LISTS ARGS_VERSION)
+      if(${v} VERSION_GREATER ${__version__})
+        set(__version__ ${v})
+      endif()
+    endforeach()
+  endif()
   if(ARGS_COMPONENTS)
-    find_package(${pkgname} ${ARGS_VERSION} QUIET COMPONENTS ${ARGS_COMPONENTS} ${ARGS_UNPARSED_ARGUMENTS})
+    list(REMOVE_DUPLICATES ARGS_COMPONENTS)
+    find_package(${pkgname} ${__version__} QUIET COMPONENTS ${ARGS_COMPONENTS} ${ARGS_UNPARSED_ARGUMENTS})
   else()
-    find_package(${pkgname} ${ARGS_VERSION} QUIET ${ARGS_UNPARSED_ARGUMENTS})
+    find_package(${pkgname} ${__version__} QUIET ${ARGS_UNPARSED_ARGUMENTS})
   endif()
   set(CMAKE_PREFIX_PATH ${old_CPP})
   unset(old_CPP)
 
   if(${pkgname}_FOUND)
     if(${qualifier} STREQUAL PRIVATE)
-      set(PROJECT_${pkgname}_VERSION ${ARGS_VERSION})
-      set(PROJECT_${pkgname}_COMPONENTS ${ARGS_COMPONENTS})
+      set(PROJECT_${pkgname}_VERSION ${__version__})
+      set(PROJECT_${pkgname}_COMPONENTS ${__components__})
       set(PROJECT_PACKAGE_DEPENDENCIES ${PROJECT_PACKAGE_DEPENDENCIES} ${pkgname})
     elseif(${qualifier} STREQUAL PUBLIC)
-      set(PROJECT_${pkgname}_VERSION ${ARGS_VERSION})
-      set(PROJECT_${pkgname}_COMPONENTS ${ARGS_COMPONENTS})
+      set(PROJECT_${pkgname}_VERSION ${__version__})
+      set(PROJECT_${pkgname}_COMPONENTS ${__components__})
       set(PROJECT_PACKAGE_DEPENDENCIES ${PROJECT_PACKAGE_DEPENDENCIES} ${pkgname})
-      set(PROJECT_INTERFACE_${pkgname}_VERSION ${ARGS_VERSION})
-      set(PROJECT_INTERFACE_${pkgname}_COMPONENTS ${ARGS_COMPONENTS})
+      set(PROJECT_INTERFACE_${pkgname}_VERSION ${__version__})
+      set(PROJECT_INTERFACE_${pkgname}_COMPONENTS ${__components__})
       set(PROJECT_INTERFACE_PACKAGE_DEPENDENCIES ${PROJECT_INTERFACE_PACKAGE_DEPENDENCIES} ${pkgname})
     elseif(${qualifier} STREQUAL INTERFACE)
-      set(PROJECT_INTERFACE_${pkgname}_VERSION ${ARGS_VERSION})
-      set(PROJECT_INTERFACE_${pkgname}_COMPONENTS ${ARGS_COMPONENTS})
+      set(PROJECT_INTERFACE_${pkgname}_VERSION ${__version__})
+      set(PROJECT_INTERFACE_${pkgname}_COMPONENTS ${__components__})
       set(PROJECT_INTERFACE_PACKAGE_DEPENDENCIES ${PROJECT_INTERFACE_PACKAGE_DEPENDENCIES} ${pkgname})
     endif()
   endif()
+
+  unset(__components__)
+  unset(__version__)
 endmacro()

cmake/FindAZMQ.cmake

@@ -0,0 +1,57 @@
+################################################################################
+#    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+#
+# ###########################
+# # Locate the AZMQ library #
+# ###########################
+#
+#
+# Usage:
+#
+#   find_package(AZMQ [version] [QUIET] [REQUIRED])
+#
+#
+# Defines the following variables:
+#
+#   AZMQ_FOUND - Found the ZeroMQ library
+#   AZMQ_INCLUDE_DIR (CMake cache) - Include directory
+#
+#
+# Accepts the following variables as hints for installation directories:
+#
+#   AZMQ_ROOT (CMake var, ENV var)
+#
+#
+# If the above variables are not defined, or if ZeroMQ could not be found there,
+# it will look for it in the system directories. Custom ZeroMQ installations
+# will always have priority over system ones.
+#
+
+if(NOT AZMQ_ROOT)
+  set(AZMQ_ROOT $ENV{AZMQ_ROOT})
+endif()
+
+find_path(AZMQ_INCLUDE_DIR
+  NAMES azmq/socket.hpp
+  HINTS ${AZMQ_ROOT} $ENV{AZMQ_ROOT}
+  PATH_SUFFIXES include
+  DOC "AZMQ include directory"
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(AZMQ
+    REQUIRED_VARS AZMQ_INCLUDE_DIR
+)
+
+if(AZMQ_FOUND AND NOT TARGET AZMQ::AZMQ)
+  add_library(AZMQ::AZMQ INTERFACE IMPORTED)
+  set_target_properties(AZMQ::AZMQ PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES ${AZMQ_INCLUDE_DIR}
+    INTERFACE_LINK_LIBRARIES "libzmq;Boost::boost;Boost::container;Boost::system"
+  )
+endif()

cmake/FindFlatBuffers.cmake

@@ -1,67 +0,0 @@
-################################################################################
-# Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
-#                                                                              #
-#              This software is distributed under the terms of the             #
-#              GNU Lesser General Public Licence (LGPL) version 3,             #
-#                  copied verbatim in the file "LICENSE"                       #
-################################################################################
-
-find_path(FLATBUFFERS_INCLUDE_DIR
-  NAMES flatbuffers/flatbuffers.h
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES include
-)
-
-find_path(FLATBUFFERS_LIBRARY_DIR
-  NAMES libflatbuffers.a
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES lib
-)
-
-find_library(FLATBUFFERS_STATIC_LIBRARY
-  NAMES libflatbuffers.a
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES lib
-)
-
-find_path(FLATBUFFERS_BINARY_DIR
-  NAMES flatc
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES bin
-)
-
-find_program(FLATBUFFERS_BINARY_FLATC
-  NAMES flatc
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES bin
-)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(FlatBuffers
-  REQUIRED_VARS
-  FLATBUFFERS_INCLUDE_DIR
-  FLATBUFFERS_LIBRARY_DIR
-  FLATBUFFERS_BINARY_DIR
-)
-
-# idempotently import targets
-if(NOT TARGET FlatBuffers)
-  if(FLATBUFFERS_FOUND)
-    # import target
-    add_library(FlatBuffers STATIC IMPORTED)
-    set_target_properties(FlatBuffers PROPERTIES
-      INTERFACE_INCLUDE_DIRECTORIES ${FLATBUFFERS_INCLUDE_DIR}
-      IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIBRARY}
-    )
-  endif()
-endif()
-
-if(NOT TARGET FlatBuffers::flatc)
-  if(FLATBUFFERS_FOUND)
-  # import target
-  add_executable(FlatBuffers::flatc IMPORTED)
-  set_target_properties(FlatBuffers::flatc PROPERTIES
-    IMPORTED_LOCATION ${FLATBUFFERS_BINARY_FLATC}
-  )
-  endif()
-endif()

cmake/FindOFI.cmake

@@ -1,89 +0,0 @@
-################################################################################
-#    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
-#                                                                              #
-#              This software is distributed under the terms of the             #
-#              GNU Lesser General Public Licence (LGPL) version 3,             #
-#                  copied verbatim in the file "LICENSE"                       #
-################################################################################
-
-# According to the docs the modification of the PKG_CONFIG_PATH environment should
-# not be necessary, but it does not work otherwise.
-if(OFI_ROOT)
-  list(APPEND CMAKE_PREFIX_PATH "${OFI_ROOT}/lib/pkgconfig")
-  set(ENV{PKG_CONFIG_PATH} "${OFI_ROOT}/lib/pkgconfig:" $ENV{PKG_CONFIG_PATH})
-endif()
-
-if(ENV{OFI_ROOT})
-  list(APPEND CMAKE_PREFIX_PATH "$ENV{OFI_ROOT}/lib/pkgconfig")
-  set(ENV{PKG_CONFIG_PATH} "$ENV{OFI_ROOT}/lib/pkgconfig:" $ENV{PKG_CONFIG_PATH})
-endif()
-
-# This should be the default as of CMake 3.1, but it is not set. BUG? Also, it does not work
-set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH 1)
-find_package(PkgConfig QUIET)
-
-if(PKG_CONFIG_FOUND)
-  # Find include dir and dependencies from pkgconfig
-  pkg_check_modules(_OFI libfabric QUIET)
-
-  # Retrieve version from pkgconfig
-  execute_process(
-    COMMAND ${PKG_CONFIG_EXECUTABLE} libfabric --modversion
-    OUTPUT_VARIABLE OFI_VERSION
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
-
-  # The IMPORTED_TARGET option of the pkg_check_modules() function is useless,
-  # so let's build it ourselves
-  find_library(OFI_LIBFABRIC
-    NAMES libfabric.so libfabric.dylib
-    HINTS ${OFI_ROOT} $ENV{OFI_ROOT}
-    PATH_SUFFIXES lib
-  )
-
-  # Just take the include dirs found by the PkgConfig module
-  set(OFI_INCLUDE_DIRS ${_OFI_INCLUDE_DIRS})
-
-  # Find fi_info command to be able to check required features of the OFI installation
-  find_program(OFI_INFO_EXECUTABLE
-    NAMES fi_info
-    HINTS ${OFI_ROOT} $ENV{OFI_ROOT}
-    PATH_SUFFIXES bin
-  )
-
-  # Detect ofi providers, they can be required via the COMPONENTS argument of find_package
-  if(OFI_INFO_EXECUTABLE)
-    execute_process(
-      COMMAND ${OFI_INFO_EXECUTABLE} -l
-      OUTPUT_VARIABLE output
-    )
-    string(REPLACE "\n" ";" lines ${output})
-    foreach(line IN LISTS lines)
-      string(REGEX
-        MATCH "^([a-zA-Z0-9_]+):"
-        found "${line}"
-      )
-      if(found)
-        string(TOLOWER "${CMAKE_MATCH_1}" provider)
-        set(OFI_fi_${provider}_FOUND TRUE)
-      endif()
-    endforeach()
-  endif()
-
-  # Check search result, check version constraints and print status
-  include(FindPackageHandleStandardArgs)
-  find_package_handle_standard_args(OFI
-    REQUIRED_VARS OFI_LIBFABRIC OFI_INCLUDE_DIRS OFI_INFO_EXECUTABLE
-    VERSION_VAR OFI_VERSION
-    HANDLE_COMPONENTS
-  )
-endif()
-
-if(NOT TARGET OFI::libfabric AND OFI_FOUND)
-  # Define an imported target
-  add_library(OFI::libfabric SHARED IMPORTED GLOBAL)
-  set_target_properties(OFI::libfabric PROPERTIES
-    IMPORTED_LOCATION ${OFI_LIBFABRIC}
-    INTERFACE_INCLUDE_DIRECTORIES ${OFI_INCLUDE_DIRS}
-  )
-endif()

cmake/FindPMIx.cmake

@@ -0,0 +1,67 @@
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+find_path(PMIx_INCLUDE_DIR
+  NAMES pmix.h
+  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
+  PATH_SUFFIXES include
+)
+
+find_path(PMIx_LIBRARY_DIR
+  NAMES libpmix.dylib libpmix.so
+  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
+  PATH_SUFFIXES lib lib64
+)
+
+find_library(PMIx_LIBRARY_SHARED
+  NAMES libpmix.dylib libpmix.so
+  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
+  PATH_SUFFIXES lib lib64
+)
+
+find_file(PMIx_VERSION_FILE
+  NAMES pmix_version.h
+  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
+  PATH_SUFFIXES include
+)
+
+file(READ "${PMIx_VERSION_FILE}" __version_raw)
+string(REGEX MATCH "#define PMIX_VERSION_MAJOR ([0-9]?)L?"
+  __version_major_raw "${__version_raw}"
+)
+set(PMIx_VERSION_MAJOR "${CMAKE_MATCH_1}")
+
+string(REGEX MATCH "#define PMIX_VERSION_MINOR ([0-9]?)L?"
+  __version_minor_raw "${__version_raw}"
+)
+set(PMIx_VERSION_MINOR "${CMAKE_MATCH_1}")
+
+string(REGEX MATCH "#define PMIX_VERSION_RELEASE ([0-9]?)L?"
+  __version_patch_raw "${__version_raw}"
+)
+set(PMIx_VERSION_PATCH "${CMAKE_MATCH_1}")
+
+set(PMIx_VERSION "${PMIx_VERSION_MAJOR}.${PMIx_VERSION_MINOR}.${PMIx_VERSION_PATCH}")
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(PMIx
+  REQUIRED_VARS
+  PMIx_INCLUDE_DIR
+  PMIx_LIBRARY_DIR
+  PMIx_LIBRARY_SHARED
+
+  VERSION_VAR PMIx_VERSION
+)
+
+if(NOT TARGET PMIx::libpmix AND PMIx_FOUND)
+  add_library(PMIx::libpmix SHARED IMPORTED)
+  set_target_properties(PMIx::libpmix PROPERTIES
+    IMPORTED_LOCATION ${PMIx_LIBRARY_SHARED}
+    INTERFACE_INCLUDE_DIRECTORIES ${PMIx_INCLUDE_DIR}
+  )
+endif()

cmake/GTestHelper.cmake

@@ -47,6 +47,8 @@
 #
 #
 
+include(GoogleTest)
+
 function(add_testsuite suitename)
     cmake_parse_arguments(testsuite
         ""
@@ -73,13 +75,23 @@ function(add_testsuite suitename)
         target_compile_definitions("${target}" PUBLIC ${testsuite_DEFINITIONS})
     endif()
 
-    add_test(NAME "${suitename}" WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMAND ${target})
+    # add_test(NAME "${suitename}" WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMAND ${target})
     if(testsuite_TIMEOUT)
-        set_tests_properties("${suitename}" PROPERTIES TIMEOUT ${testsuite_TIMEOUT})
+        # set_tests_properties("${suitename}" PROPERTIES TIMEOUT ${testsuite_TIMEOUT})
+    else()
+        set(testsuite_TIMEOUT 10)
     endif()
     if(testsuite_RUN_SERIAL)
-        set_tests_properties("${suitename}" PROPERTIES RUN_SERIAL ${testsuite_RUN_SERIAL})
+        # set_tests_properties("${suitename}" PROPERTIES RUN_SERIAL ${testsuite_RUN_SERIAL})
+    else()
+        set(testsuite_RUN_SERIAL OFF)
     endif()
+    gtest_discover_tests(${target}
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+      TEST_PREFIX ${suitename}.
+      PROPERTIES RUN_SERIAL ${testsuite_RUN_SERIAL}
+                 TIMEOUT ${testsuite_TIMEOUT}
+    )
 
     list(APPEND ALL_TEST_TARGETS ${target})
     set(ALL_TEST_TARGETS ${ALL_TEST_TARGETS} PARENT_SCOPE)

docs/Device.md

@@ -31,14 +31,15 @@ The state machine can be querried and controlled via `GetCurrentStateName()` and
 If the device is running in interactive mode (default), states can be changed via keyboard input:
 
  - `'h'` - help
- - `'p'` - pause
  - `'r'` - run
  - `'s'` - stop
  - `'t'` - reset task
  - `'d'` - reset device
  - `'q'` - end
  - `'j'` - init task
- - `'i'` - init device
+ - `'i'` - initialize
+ - `'b'` - bind
+ - `'x'` - connect
 
 Without the interactive mode, for example for a run in background, two other control mechanisms are available:
 

examples/1-1/CMakeLists.txt

@@ -31,16 +31,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-1-1.sh.in ${CMAKE_CUR
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-1-1.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh)
 
-add_test(NAME Example-1-1-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh zeromq)
-set_tests_properties(Example-1-1-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+add_test(NAME Example.1-1.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh zeromq)
+set_tests_properties(Example.1-1.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-1-1-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh nanomsg)
-  set_tests_properties(Example-1-1-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+  add_test(NAME Example.1-1.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh nanomsg)
+  set_tests_properties(Example.1-1.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 endif()
 
-add_test(NAME Example-1-1-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh shmem)
-set_tests_properties(Example-1-1-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+add_test(NAME Example.1-1.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh shmem)
+set_tests_properties(Example.1-1.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
 # install
 

examples/1-n-1/CMakeLists.txt

@@ -39,16 +39,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-1-n-1.json ${CMAKE_CURRENT_BINARY_
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-1-n-1.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh)
 
-add_test(NAME Example-1-n-1-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh zeromq)
-set_tests_properties(Example-1-n-1-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+add_test(NAME Example.1-n-1.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh zeromq)
+set_tests_properties(Example.1-n-1.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-1-n-1-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh nanomsg)
-  set_tests_properties(Example-1-n-1-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+  add_test(NAME Example.1-n-1.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh nanomsg)
+  set_tests_properties(Example.1-n-1.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 endif()
 
-add_test(NAME Example-1-n-1-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh shmem)
-set_tests_properties(Example-1-n-1-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+add_test(NAME Example.1-n-1.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh shmem)
+set_tests_properties(Example.1-n-1.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
 # install
 

examples/CMakeLists.txt

@@ -16,5 +16,6 @@ add_subdirectory(multiple-channels)
 if(BUILD_NANOMSG_TRANSPORT)
   add_subdirectory(multiple-transports)
 endif()
+add_subdirectory(readout)
 add_subdirectory(region)
 add_subdirectory(req-rep)

examples/builtin-devices/CMakeLists.txt

@@ -14,27 +14,27 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-builtin-devices.sh.in
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-builtin-devices.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh)
 
-add_test(NAME Example-Builtin-Devices-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq)
-set_tests_properties(Example-Builtin-Devices-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+add_test(NAME Example.BuiltinDevices.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq)
+set_tests_properties(Example.BuiltinDevices.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Builtin-Devices-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg)
-  set_tests_properties(Example-Builtin-Devices-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+  add_test(NAME Example.BuiltinDevices.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg)
+  set_tests_properties(Example.BuiltinDevices.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 endif()
 
-add_test(NAME Example-Builtin-Devices-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem)
-set_tests_properties(Example-Builtin-Devices-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+add_test(NAME Example.BuiltinDevices.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem)
+set_tests_properties(Example.BuiltinDevices.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
-add_test(NAME Example-Builtin-Devices-zeromq-multipart COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq true 2)
-set_tests_properties(Example-Builtin-Devices-zeromq-multipart PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+add_test(NAME Example.BuiltinDevices.multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq true 2)
+set_tests_properties(Example.BuiltinDevices.multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Builtin-Devices-nanomsg-multipart COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg true 2)
-  set_tests_properties(Example-Builtin-Devices-nanomsg-multipart PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+  add_test(NAME Example.BuiltinDevices.multipart.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg true 2)
+  set_tests_properties(Example.BuiltinDevices.multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 endif()
 
-add_test(NAME Example-Builtin-Devices-shmem-multipart COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem true 2)
-set_tests_properties(Example-Builtin-Devices-shmem-multipart PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+add_test(NAME Example.BuiltinDevices.multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem true 2)
+set_tests_properties(Example.BuiltinDevices.multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
 # install
 

examples/copypush/CMakeLists.txt

@@ -32,16 +32,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-copypush.sh.in ${CMAK
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-copypush.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh)
 
-add_test(NAME Example-CopyPush-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh zeromq)
-set_tests_properties(Example-CopyPush-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
+add_test(NAME Example.CopyPush.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh zeromq)
+set_tests_properties(Example.CopyPush.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-CopyPush-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh nanomsg)
-  set_tests_properties(Example-CopyPush-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
+  add_test(NAME Example.CopyPush.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh nanomsg)
+  set_tests_properties(Example.CopyPush.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 endif()
 
-add_test(NAME Example-CopyPush-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh shmem)
-set_tests_properties(Example-CopyPush-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
+add_test(NAME Example.CopyPush.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh shmem)
+set_tests_properties(Example.CopyPush.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 
 # install
 

examples/dds/Processor.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

examples/dds/Sampler.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

examples/dds/Sink.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

examples/dds/ex-dds-hosts.cfg

@@ -2,6 +2,6 @@
 # source setup.sh
 @bash_end@
 
-sampler, username@localhost, , /path/to/dds-work-dir/, 1
-processor, username@localhost, , /path/to/dds-work-dir/, 10
-sink, username@localhost, , /path/to/dds-work-dir/, 1
+sampler, username@localhost, , /path/to/dds-work/, 1
+processor, username@localhost, , /path/to/dds-work/, 10
+sink, username@localhost, , /path/to/dds-work/, 1

examples/multipart/CMakeLists.txt

@@ -31,16 +31,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multipart.sh.in ${CMA
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multipart.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh)
 
-add_test(NAME Example-Multipart-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh zeromq)
-set_tests_properties(Example-Multipart-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
+add_test(NAME Example.Multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh zeromq)
+set_tests_properties(Example.Multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Multipart-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh nanomsg)
-  set_tests_properties(Example-Multipart-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
+  add_test(NAME Example.Multipart.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh nanomsg)
+  set_tests_properties(Example.Multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
 endif()
 
-add_test(NAME Example-Multipart-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh shmem)
-set_tests_properties(Example-Multipart-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
+add_test(NAME Example.Multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh shmem)
+set_tests_properties(Example.Multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
 
 # install
 

examples/multiple-channels/CMakeLists.txt

@@ -36,12 +36,12 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multiple-channels.sh.
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multiple-channels.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh)
 
-add_test(NAME Example-Multiple-Channels-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh zeromq)
-set_tests_properties(Example-Multiple-Channels-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
+add_test(NAME Example.MultipleChannels.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh zeromq)
+set_tests_properties(Example.MultipleChannels.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Multiple-Channels-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh nanomsg)
-  set_tests_properties(Example-Multiple-Channels-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
+  add_test(NAME Example.MultipleChannels.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh nanomsg)
+  set_tests_properties(Example.MultipleChannels.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
 endif()
 
 # install

examples/multiple-channels/Sampler.cxx

@@ -41,7 +41,7 @@ void Sampler::Run()
 {
     FairMQPollerPtr poller(NewPoller("data", "broadcast"));
 
-    while (CheckCurrentState(RUNNING))
+    while (!NewStatePending())
     {
         poller->Poll(100);
 

examples/multiple-transports/CMakeLists.txt

@@ -35,8 +35,8 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multiple-transports.s
 # test
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multiple-transports.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-transports.sh)
-add_test(NAME Example-Multiple-Transports COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-transports.sh)
-set_tests_properties(Example-Multiple-Transports PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
+add_test(NAME Example.MultipleTransports COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-transports.sh)
+set_tests_properties(Example.MultipleTransports PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
 
 # install
 

examples/multiple-transports/Sampler1.cxx

@@ -60,7 +60,7 @@ void Sampler1::ListenForAcks()
 {
     uint64_t numAcks = 0;
 
-    while (CheckCurrentState(RUNNING))
+    while (!NewStatePending())
     {
         FairMQMessagePtr ack(NewMessageFor("ack", 0));
         if (Receive(ack, "ack") < 0)

examples/readout/Builder.h

@@ -0,0 +1,40 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIRMQEXAMPLEREGIONBUILDER_H
+#define FAIRMQEXAMPLEREGIONBUILDER_H
+
+#include <atomic>
+
+#include "FairMQDevice.h"
+
+namespace example_region
+{
+
+class Builder : public FairMQDevice
+{
+  public:
+    Builder() {
+        OnData("data1", &Builder::HandleData);
+    }
+    virtual ~Builder() {}
+
+  protected:
+    bool HandleData(FairMQMessagePtr& msg, int /*index*/)
+    {
+        if (Send(msg, "data2") < 0) {
+            return false;
+        }
+
+        return true;
+    }
+};
+
+} // namespace example_region
+
+#endif /* FAIRMQEXAMPLEREGIONBUILDER_H */

examples/readout/CMakeLists.txt

@@ -0,0 +1,54 @@
+ ################################################################################
+ #    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+ #                                                                              #
+ #              This software is distributed under the terms of the             #
+ #              GNU Lesser General Public Licence (LGPL) version 3,             #
+ #                  copied verbatim in the file "LICENSE"                       #
+ ################################################################################
+
+add_library(ExampleReadoutLib STATIC
+    "Sampler.cxx"
+    "Sampler.h"
+    "Builder.h"
+    "Sink.cxx"
+    "Sink.h"
+)
+
+target_link_libraries(ExampleReadoutLib PUBLIC FairMQ)
+
+add_executable(fairmq-ex-readout-sampler runSampler.cxx)
+target_link_libraries(fairmq-ex-readout-sampler PRIVATE ExampleReadoutLib)
+
+add_executable(fairmq-ex-readout-builder runBuilder.cxx)
+target_link_libraries(fairmq-ex-readout-builder PRIVATE ExampleReadoutLib)
+
+add_executable(fairmq-ex-readout-sink runSink.cxx)
+target_link_libraries(fairmq-ex-readout-sink PRIVATE ExampleReadoutLib)
+
+add_custom_target(Examplereadout DEPENDS fairmq-ex-readout-sampler fairmq-ex-readout-sink)
+
+set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
+set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-readout.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout.sh)
+
+# install
+
+install(
+    TARGETS
+    fairmq-ex-readout-sampler
+    fairmq-ex-readout-sink
+
+    LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
+    RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
+)
+
+# configure run script with different executable paths for build and for install directories
+set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
+set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-readout.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout.sh_install)
+
+install(
+    PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout.sh_install
+    DESTINATION ${PROJECT_INSTALL_BINDIR}
+    RENAME fairmq-start-ex-readout.sh
+)

examples/readout/README.md

@@ -0,0 +1,5 @@
+Region example
+==============
+
+This example demonstrates the use of a more advanced feature - UnmanagedRegion, that can be used to create a buffer through one of FairMQ transports. The contents of this buffer are managed by the user, who can also create messages out of sub-buffers of the created buffer. Such feature can be interesting in environments that have special requirements by the hardware that writes the data, to keep the transfer efficient (e.g. shared memory).
+

examples/readout/Sampler.cxx

@@ -0,0 +1,91 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+/**
+ * Sampler.cpp
+ *
+ * @since 2014-10-10
+ * @author A. Rybalchenko
+ */
+
+#include "Sampler.h"
+
+#include <thread>
+
+using namespace std;
+
+namespace example_region
+{
+
+Sampler::Sampler()
+    : fMsgSize(10000)
+    , fMaxIterations(0)
+    , fNumIterations(0)
+    , fRegion(nullptr)
+    , fNumUnackedMsgs(0)
+{
+}
+
+void Sampler::InitTask()
+{
+    fMsgSize = fConfig->GetValue<int>("msg-size");
+    fMaxIterations = fConfig->GetValue<uint64_t>("max-iterations");
+
+    fRegion = FairMQUnmanagedRegionPtr(NewUnmanagedRegionFor("data1",
+                                                             0,
+                                                             10000000,
+                                                             [this](void* /*data*/, size_t /*size*/, void* /*hint*/) { // callback to be called when message buffers no longer needed by transport
+                                                                 --fNumUnackedMsgs;
+                                                                 if (fMaxIterations > 0)
+                                                                 {
+                                                                     LOG(debug) << "Received ack";
+                                                                 }
+                                                             }
+                                                             ));
+}
+
+bool Sampler::ConditionalRun()
+{
+    FairMQMessagePtr msg(NewMessageFor("data1", // channel
+                                        0, // sub-channel
+                                        fRegion, // region
+                                        fRegion->GetData(), // ptr within region
+                                        fMsgSize, // offset from ptr
+                                        nullptr // hint
+                                        ));
+
+    if (Send(msg, "data1", 0) > 0)
+    {
+        ++fNumUnackedMsgs;
+
+        if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations)
+        {
+            LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
+            return false;
+        }
+    }
+
+    return true;
+}
+
+void Sampler::ResetTask()
+{
+    // if not all messages acknowledged, wait for a bit. But only once, since receiver could be already dead.
+    if (fNumUnackedMsgs != 0)
+    {
+        LOG(debug) << "waiting for all acknowledgements... (" << fNumUnackedMsgs << ")";
+        this_thread::sleep_for(chrono::milliseconds(500));
+        LOG(debug) << "done, still unacked: " << fNumUnackedMsgs;
+    }
+    fRegion.reset();
+}
+
+Sampler::~Sampler()
+{
+}
+
+} // namespace example_region

examples/readout/Sampler.h

@@ -0,0 +1,46 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+/**
+ * Sampler.h
+ *
+ * @since 2014-10-10
+ * @author A. Rybalchenko
+ */
+
+#ifndef FAIRMQEXAMPLEREGIONSAMPLER_H
+#define FAIRMQEXAMPLEREGIONSAMPLER_H
+
+#include <atomic>
+
+#include "FairMQDevice.h"
+
+namespace example_region
+{
+
+class Sampler : public FairMQDevice
+{
+  public:
+    Sampler();
+    virtual ~Sampler();
+
+  protected:
+    virtual void InitTask();
+    virtual bool ConditionalRun();
+    virtual void ResetTask();
+
+  private:
+    int fMsgSize;
+    uint64_t fMaxIterations;
+    uint64_t fNumIterations;
+    FairMQUnmanagedRegionPtr fRegion;
+    std::atomic<uint64_t> fNumUnackedMsgs;
+};
+
+} // namespace example_region
+
+#endif /* FAIRMQEXAMPLEREGIONSAMPLER_H */

examples/readout/Sink.cxx

@@ -0,0 +1,56 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+/**
+ * Sink.cxx
+ *
+ * @since 2014-10-10
+ * @author A. Rybalchenko
+ */
+
+#include "Sink.h"
+
+using namespace std;
+
+namespace example_region
+{
+
+Sink::Sink()
+    : fMaxIterations(0)
+    , fNumIterations(0)
+{
+}
+
+void Sink::InitTask()
+{
+    // Get the fMaxIterations value from the command line options (via fConfig)
+    fMaxIterations = fConfig->GetValue<uint64_t>("max-iterations");
+}
+
+void Sink::Run()
+{
+    FairMQChannel& dataInChannel = fChannels.at("data").at(0);
+
+    while (!NewStatePending())
+    {
+        FairMQMessagePtr msg(dataInChannel.Transport()->CreateMessage());
+        dataInChannel.Receive(msg);
+        // void* ptr = msg->GetData();
+
+        if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations)
+        {
+            LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
+            break;
+        }
+    }
+}
+
+Sink::~Sink()
+{
+}
+
+} // namespace example_region

examples/readout/Sink.h

@@ -0,0 +1,42 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+/**
+ * Sink.h
+ *
+ * @since 2014-10-10
+ * @author A. Rybalchenko
+ */
+
+#ifndef FAIRMQEXAMPLEREGIONSINK_H
+#define FAIRMQEXAMPLEREGIONSINK_H
+
+#include <string>
+
+#include "FairMQDevice.h"
+
+namespace example_region
+{
+
+class Sink : public FairMQDevice
+{
+  public:
+    Sink();
+    virtual ~Sink();
+
+  protected:
+    virtual void Run();
+    virtual void InitTask();
+
+  private:
+    uint64_t fMaxIterations;
+    uint64_t fNumIterations;
+};
+
+} // namespace example_region
+
+#endif /* FAIRMQEXAMPLEREGIONSINK_H */

examples/readout/fairmq-start-ex-readout.sh.in

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
+
+msgSize="1000000"
+
+if [[ $1 =~ ^[0-9]+$ ]]; then
+    msgSize=$1
+fi
+
+SAMPLER="fairmq-ex-readout-sampler"
+SAMPLER+=" --id sampler1"
+SAMPLER+=" --severity debug"
+SAMPLER+=" --msg-size $msgSize"
+# SAMPLER+=" --rate 10"
+SAMPLER+=" --channel-config name=data1,type=pair,method=bind,address=tcp://127.0.0.1:7777,transport=shmem"
+xterm -geometry 80x23+0+0 -hold -e @EX_BIN_DIR@/$SAMPLER &
+
+BUILDER="fairmq-ex-readout-builder"
+BUILDER+=" --id builder1"
+BUILDER+=" --severity debug"
+BUILDER+=" --channel-config name=data1,type=pair,method=connect,address=tcp://127.0.0.1:7777,transport=shmem"
+BUILDER+="                  name=data2,type=pair,method=connect,address=tcp://127.0.0.1:7778,transport=ofi"
+xterm -geometry 80x23+500+0 -hold -e @EX_BIN_DIR@/$BUILDER &
+
+SINK="fairmq-ex-readout-sink"
+SINK+=" --id sink1"
+SINK+=" --severity debug"
+SINK+=" --channel-config name=data,type=pair,method=bind,address=tcp://127.0.0.1:7778,transport=ofi"
+xterm -geometry 80x23+1000+0 -hold -e @EX_BIN_DIR@/$SINK &

examples/readout/runBuilder.cxx

@@ -0,0 +1,20 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "Builder.h"
+
+namespace bpo = boost::program_options;
+
+void addCustomOptions(bpo::options_description& /* options */)
+{}
+
+FairMQDevicePtr getDevice(const FairMQProgOptions& /*config*/)
+{
+    return new example_region::Builder();
+}

examples/readout/runSampler.cxx

@@ -0,0 +1,24 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "Sampler.h"
+
+namespace bpo = boost::program_options;
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()
+        ("msg-size", bpo::value<int>()->default_value(1000), "Message size in bytes")
+        ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
+}
+
+FairMQDevicePtr getDevice(const FairMQProgOptions& /*config*/)
+{
+    return new example_region::Sampler();
+}

examples/readout/runSink.cxx

@@ -0,0 +1,23 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "Sink.h"
+
+namespace bpo = boost::program_options;
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()
+        ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
+}
+
+FairMQDevicePtr getDevice(const FairMQProgOptions& /*config*/)
+{
+    return new example_region::Sink();
+}

examples/region/CMakeLists.txt

@@ -18,7 +18,6 @@ target_link_libraries(ExampleRegionLib PUBLIC FairMQ)
 add_executable(fairmq-ex-region-sampler runSampler.cxx)
 target_link_libraries(fairmq-ex-region-sampler PRIVATE ExampleRegionLib)
 
-
 add_executable(fairmq-ex-region-sink runSink.cxx)
 target_link_libraries(fairmq-ex-region-sink PRIVATE ExampleRegionLib)
 
@@ -32,16 +31,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-region.sh.in ${CMAKE_
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-region.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh)
 
-add_test(NAME Example-Region-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh zeromq)
-set_tests_properties(Example-Region-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
+add_test(NAME Example.Region.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh zeromq)
+set_tests_properties(Example.Region.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Region-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh nanomsg)
-  set_tests_properties(Example-Region-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
+  add_test(NAME Example.Region.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh nanomsg)
+  set_tests_properties(Example.Region.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
 endif()
 
-add_test(NAME Example-Region-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh shmem)
-set_tests_properties(Example-Region-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
+add_test(NAME Example.Region.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh shmem)
+set_tests_properties(Example.Region.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
 
 # install
 

examples/region/Sink.cxx

@@ -35,7 +35,7 @@ void Sink::Run()
 {
     FairMQChannel& dataInChannel = fChannels.at("data").at(0);
 
-    while (CheckCurrentState(RUNNING))
+    while (!NewStatePending())
     {
         FairMQMessagePtr msg(dataInChannel.Transport()->CreateMessage());
         dataInChannel.Receive(msg);

examples/req-rep/CMakeLists.txt

@@ -32,16 +32,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-req-rep.sh.in ${CMAKE
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-req-rep.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh)
 
-add_test(NAME Example-ReqRep-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh zeromq)
-set_tests_properties(Example-ReqRep-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
+add_test(NAME Example.ReqRep.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh zeromq)
+set_tests_properties(Example.ReqRep.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-ReqRep-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh nanomsg)
-  set_tests_properties(Example-ReqRep-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
+  add_test(NAME Example.ReqRep.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh nanomsg)
+  set_tests_properties(Example.ReqRep.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 endif()
 
-add_test(NAME Example-ReqRep-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh shmem)
-set_tests_properties(Example-ReqRep-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
+add_test(NAME Example.ReqRep.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh shmem)
+set_tests_properties(Example.ReqRep.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 
 # install
 

fairmq/CMakeLists.txt

@@ -12,24 +12,8 @@
 if(BUILD_DDS_PLUGIN)
   add_subdirectory(plugins/DDS)
 endif()
-
-
-############################
-# preprocessor definitions #
-############################
-if(BUILD_NANOMSG_TRANSPORT)
-  add_definitions(-DBUILD_NANOMSG_TRANSPORT)
-endif()
-if(BUILD_OFI_TRANSPORT)
-  add_definitions(-DBUILD_OFI_TRANSPORT)
-endif()
-
-
-##################
-# subdirectories #
-##################
-if(BUILD_OFI_TRANSPORT)
-  add_subdirectory(ofi)
+if(BUILD_PMIX_PLUGIN)
+  add_subdirectory(plugins/PMIx)
 endif()
 
 
@@ -47,7 +31,7 @@ set(FAIRMQ_PUBLIC_HEADER_FILES
   FairMQPoller.h
   FairMQUnmanagedRegion.h
   FairMQSocket.h
-  FairMQStateMachine.h
+  StateMachine.h
   FairMQTransportFactory.h
   MemoryResources.h
   MemoryResourceTools.h
@@ -80,7 +64,6 @@ set(FAIRMQ_PRIVATE_HEADER_FILES
   options/FairProgOptionsHelper.h
   plugins/Builtin.h
   plugins/Control.h
-  StateMachine.h
   shmem/FairMQMessageSHM.h
   shmem/FairMQPollerSHM.h
   shmem/FairMQUnmanagedRegionSHM.h
@@ -127,7 +110,7 @@ set(FAIRMQ_SOURCE_FILES
   FairMQMessage.cxx
   FairMQPoller.cxx
   FairMQSocket.cxx
-  FairMQStateMachine.cxx
+  StateMachine.cxx
   FairMQTransportFactory.cxx
   devices/FairMQBenchmarkSampler.cxx
   devices/FairMQMerger.cxx
@@ -141,7 +124,6 @@ set(FAIRMQ_SOURCE_FILES
   PluginManager.cxx
   PluginServices.cxx
   plugins/Control.cxx
-  StateMachine.cxx
   shmem/FairMQMessageSHM.cxx
   shmem/FairMQPollerSHM.cxx
   shmem/FairMQUnmanagedRegionSHM.cxx
@@ -207,6 +189,19 @@ if(FAST_BUILD)
   set_target_properties(${_target} PROPERTIES OUTPUT_NAME FairMQ)
 endif()
 
+
+############################
+# preprocessor definitions #
+############################
+target_compile_definitions(${_target} PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
+if(BUILD_NANOMSG_TRANSPORT)
+  target_compile_definitions(${_target} PRIVATE BUILD_NANOMSG_TRANSPORT)
+endif()
+if(BUILD_OFI_TRANSPORT)
+  target_compile_definitions(${_target} PRIVATE BUILD_OFI_TRANSPORT)
+endif()
+
+
 #######################
 # include directories #
 #######################
@@ -226,7 +221,11 @@ if(BUILD_NANOMSG_TRANSPORT)
   set(NANOMSG_DEPS nanomsg msgpackc-cxx)
 endif()
 if(BUILD_OFI_TRANSPORT)
-  set(OFI_DEPS OFI::libfabric protobuf::libprotobuf $<TARGET_OBJECTS:OfiTransport>)
+  set(OFI_DEPS
+    asiofi::asiofi
+    Boost::container
+    AZMQ::AZMQ
+  )
 endif()
 set(optional_deps ${NANOMSG_DEPS} ${OFI_DEPS})
 if(optional_deps)
@@ -243,12 +242,8 @@ target_link_libraries(${_target}
   $<$<PLATFORM_ID:Linux>:rt>
   Boost::boost
   Boost::program_options
-  Boost::thread
-  Boost::system
   Boost::filesystem
   Boost::regex
-  Boost::date_time
-  Boost::signals
   FairLogger::FairLogger
 
   PRIVATE # only libFairMQ links against private dependencies

fairmq/DeviceRunner.cxx

@@ -74,15 +74,14 @@ auto DeviceRunner::Run() -> int
     fDevice->RegisterChannelEndpoints();
     if (fConfig.Count("print-channels")) {
         fDevice->PrintRegisteredChannels();
-        fDevice->ChangeState(FairMQDevice::END);
+        fDevice->ChangeState(fair::mq::Transition::End);
         return 0;
     }
 
     // Handle --version
     if (fConfig.Count("version")) {
         std::cout << "User device version: " << fDevice->GetVersion() << std::endl;
-        std::cout << "FAIRMQ_INTERFACE_VERSION: " << FAIRMQ_INTERFACE_VERSION << std::endl;
-        fDevice->ChangeState(FairMQDevice::END);
+        fDevice->ChangeState(fair::mq::Transition::End);
         return 0;
     }
 

fairmq/FairMQDevice.cxx

@@ -27,6 +27,34 @@
 
 using namespace std;
 
+static map<fair::mq::Transition, fair::mq::State> backwardsCompatibilityWaitForEndOfStateHelper =
+{
+    { fair::mq::Transition::InitDevice,   fair::mq::State::InitializingDevice },
+    { fair::mq::Transition::CompleteInit, fair::mq::State::Initialized },
+    { fair::mq::Transition::Bind,         fair::mq::State::Bound },
+    { fair::mq::Transition::Connect,      fair::mq::State::DeviceReady },
+    { fair::mq::Transition::InitTask,     fair::mq::State::Ready },
+    { fair::mq::Transition::Run,          fair::mq::State::Ready },
+    { fair::mq::Transition::Stop,         fair::mq::State::Ready },
+    { fair::mq::Transition::ResetTask,    fair::mq::State::DeviceReady },
+    { fair::mq::Transition::ResetDevice,  fair::mq::State::Idle }
+};
+
+static map<int, fair::mq::Transition> backwardsCompatibilityChangeStateHelper =
+{
+    { FairMQDevice::Event::INIT_DEVICE, fair::mq::Transition::InitDevice },
+    { FairMQDevice::Event::internal_DEVICE_READY, fair::mq::Transition::Auto },
+    { FairMQDevice::Event::INIT_TASK, fair::mq::Transition::InitTask },
+    { FairMQDevice::Event::internal_READY, fair::mq::Transition::Auto },
+    { FairMQDevice::Event::RUN, fair::mq::Transition::Run },
+    { FairMQDevice::Event::STOP, fair::mq::Transition::Stop },
+    { FairMQDevice::Event::RESET_TASK, fair::mq::Transition::ResetTask },
+    { FairMQDevice::Event::RESET_DEVICE, fair::mq::Transition::ResetDevice },
+    { FairMQDevice::Event::internal_IDLE, fair::mq::Transition::Auto },
+    { FairMQDevice::Event::END, fair::mq::Transition::End },
+    { FairMQDevice::Event::ERROR_FOUND, fair::mq::Transition::ErrorFound }
+};
+
 FairMQDevice::FairMQDevice()
     : FairMQDevice(nullptr, {0, 0, 0})
 {
@@ -54,7 +82,10 @@ FairMQDevice::FairMQDevice(FairMQProgOptions* config, const fair::mq::tools::Ver
     , fInternalConfig(config ? nullptr : fair::mq::tools::make_unique<FairMQProgOptions>())
     , fConfig(config ? config : fInternalConfig.get())
     , fId()
-    , fDefaultTransportType(fair::mq::Transport::DEFAULT)
+    , fDefaultTransportType(fair::mq::Transport::ZMQ)
+    , fStateMachine()
+    , fUninitializedBindingChannels()
+    , fUninitializedConnectingChannels()
     , fDataCallbacks(false)
     , fMsgInputs()
     , fMultipartInputs()
@@ -65,18 +96,107 @@ FairMQDevice::FairMQDevice(FairMQProgOptions* config, const fair::mq::tools::Ver
     , fMultitransportProceed(false)
     , fVersion(version)
     , fRate(0.)
+    , fMaxRunRuntimeInS(0)
     , fRawCmdLineArgs()
-    , fInterrupted(false)
-    , fInterruptedCV()
-    , fInterruptedMtx()
-    , fRateLogging(true)
 {
+    SubscribeToNewTransition("device", [&](fair::mq::Transition transition) {
+        LOG(trace) << "device notified on new transition: " << transition;
+
+        switch (transition) {
+            case fair::mq::Transition::Stop:
+                UnblockTransports();
+                break;
+            default:
+                break;
+        }
+    });
+
+    fStateMachine.HandleStates([&](fair::mq::State state) {
+        LOG(trace) << "device notified on new state: " << state;
+
+        {
+            lock_guard<mutex> lock(fStatesMtx);
+            fStates.push(state);
+        }
+        fStatesCV.notify_one();
+
+        switch (state) {
+            case fair::mq::State::InitializingDevice:
+                InitWrapper();
+                break;
+            case fair::mq::State::Binding:
+                BindWrapper();
+                break;
+            case fair::mq::State::Connecting:
+                ConnectWrapper();
+                break;
+            case fair::mq::State::InitializingTask:
+                InitTaskWrapper();
+                break;
+            case fair::mq::State::Running:
+                RunWrapper();
+                break;
+            case fair::mq::State::ResettingTask:
+                ResetTaskWrapper();
+                break;
+            case fair::mq::State::ResettingDevice:
+                ResetWrapper();
+                break;
+            case fair::mq::State::Exiting:
+                Exit();
+                break;
+            default:
+                LOG(trace) << "device notified on new state without a matching handler: " << state;
+                break;
+        }
+    });
+
+    fStateMachine.Start();
+}
+
+fair::mq::State FairMQDevice::WaitForNextState()
+{
+    unique_lock<mutex> lock(fStatesMtx);
+    while (fStates.empty()) {
+        fStatesCV.wait_for(lock, chrono::milliseconds(50));
+    }
+
+    auto result = fStates.front();
+
+    if (result == fair::mq::State::Error) {
+        throw DeviceStateError("Device transitioned to error state.");
+    }
+
+    fStates.pop();
+
+    return result;
+}
+
+void FairMQDevice::WaitForState(fair::mq::State state)
+{
+    while (WaitForNextState() != state) {}
+}
+
+bool FairMQDevice::ChangeState(const int transition)
+{
+    return ChangeState(backwardsCompatibilityChangeStateHelper.at(transition));
+}
+
+void FairMQDevice::WaitForEndOfState(fair::mq::Transition transition)
+{
+    WaitForState(backwardsCompatibilityWaitForEndOfStateHelper.at(transition));
 }
 
 void FairMQDevice::InitWrapper()
 {
+    fStateMachine.WaitForPendingState();
+
     fId = fConfig->GetValue<string>("id");
+
+    Init();
+
     fRate = fConfig->GetValue<float>("rate");
+    fMaxRunRuntimeInS = fConfig->GetValue<uint64_t>("max-run-time");
 
     try {
         fDefaultTransportType = fair::mq::TransportTypes.at(fConfig->GetValue<string>("transport"));
@@ -96,13 +216,9 @@ void FairMQDevice::InitWrapper()
         }
     }
 
-    LOG(debug) << "Requesting '" << fair::mq::TransportNames.at(fDefaultTransportType) << "' as default transport for the device";
+    LOG(debug) << "Setting '" << fair::mq::TransportNames.at(fDefaultTransportType) << "' as default transport for the device";
     fTransportFactory = AddTransport(fDefaultTransportType);
 
-    // Containers to store the uninitialized channels.
-    vector<FairMQChannel*> uninitializedBindingChannels;
-    vector<FairMQChannel*> uninitializedConnectingChannels;
-
     string networkInterface = fConfig->GetValue<string>("network-interface");
 
     // Fill the uninitialized channel containers
@@ -113,13 +229,8 @@ void FairMQDevice::InitWrapper()
             vi.fName = fair::mq::tools::ToString(mi.first, "[", subChannelIndex, "]");
 
             // set channel transport
-            if (vi.fTransportType == fair::mq::Transport::DEFAULT || vi.fTransportType == fTransportFactory->GetType()) {
-                LOG(debug) << vi.fName << ": using default transport";
-                vi.InitTransport(fTransportFactory);
-            } else {
-                LOG(debug) << vi.fName << ": channel transport (" << fair::mq::TransportNames.at(fDefaultTransportType) << ") overriden to " << fair::mq::TransportNames.at(vi.fTransportType);
-                vi.InitTransport(AddTransport(vi.fTransportType));
-            }
+            LOG(debug) << "Initializing transport for channel " << vi.fName << ": " << fair::mq::TransportNames.at(vi.fTransportType);
+            vi.InitTransport(AddTransport(vi.fTransportType));
 
             if (vi.fMethod == "bind") {
                 // if binding address is not specified, try getting it from the configured network interface
@@ -131,13 +242,13 @@ void FairMQDevice::InitWrapper()
                     vi.fAddress = "tcp://" + fair::mq::tools::getInterfaceIP(networkInterface) + ":1";
                 }
                 // fill the uninitialized list
-                uninitializedBindingChannels.push_back(&vi);
+                fUninitializedBindingChannels.push_back(&vi);
             } else if (vi.fMethod == "connect") {
                 // fill the uninitialized list
-                uninitializedConnectingChannels.push_back(&vi);
+                fUninitializedConnectingChannels.push_back(&vi);
             } else if (vi.fAddress.find_first_of("@+>") != string::npos) {
                 // fill the uninitialized list
-                uninitializedConnectingChannels.push_back(&vi);
+                fUninitializedConnectingChannels.push_back(&vi);
             } else {
                 LOG(error) << "Cannot update configuration. Socket method (bind/connect) for channel '" << vi.fName << "' not specified.";
                 throw runtime_error(fair::mq::tools::ToString("Cannot update configuration. Socket method (bind/connect) for channel ", vi.fName, " not specified."));
@@ -147,17 +258,27 @@ void FairMQDevice::InitWrapper()
         }
     }
 
+    // ChangeState(fair::mq::Transition::Auto);
+}
+
+void FairMQDevice::BindWrapper()
+{
     // Bind channels. Here one run is enough, because bind settings should be available locally
     // If necessary this could be handled in the same way as the connecting channels
-    AttachChannels(uninitializedBindingChannels);
+    AttachChannels(fUninitializedBindingChannels);
 
-    if (!uninitializedBindingChannels.empty()) {
-        LOG(error) << uninitializedBindingChannels.size() << " of the binding channels could not initialize. Initial configuration incomplete.";
-        throw runtime_error(fair::mq::tools::ToString(uninitializedBindingChannels.size(), " of the binding channels could not initialize. Initial configuration incomplete."));
+    if (!fUninitializedBindingChannels.empty()) {
+        LOG(error) << fUninitializedBindingChannels.size() << " of the binding channels could not initialize. Initial configuration incomplete.";
+        throw runtime_error(fair::mq::tools::ToString(fUninitializedBindingChannels.size(), " of the binding channels could not initialize. Initial configuration incomplete."));
     }
 
-    CallStateChangeCallbacks(INITIALIZING_DEVICE);
+    Bind();
 
+    ChangeState(fair::mq::Transition::Auto);
+}
+
+void FairMQDevice::ConnectWrapper()
+{
     int initializationTimeoutInS = fConfig->GetValue<int>("initialization-timeout");
 
     // go over the list of channels until all are initialized (and removed from the uninitialized list)
@@ -165,12 +286,12 @@ void FairMQDevice::InitWrapper()
     auto sleepTimeInMS = 50;
     auto maxAttempts = initializationTimeoutInS * 1000 / sleepTimeInMS;
     // first attempt
-    AttachChannels(uninitializedConnectingChannels);
+    AttachChannels(fUninitializedConnectingChannels);
     // if not all channels could be connected, update their address values from config and retry
-    while (!uninitializedConnectingChannels.empty()) {
+    while (!fUninitializedConnectingChannels.empty()) {
         this_thread::sleep_for(chrono::milliseconds(sleepTimeInMS));
 
-        for (auto& chan : uninitializedConnectingChannels) {
+        for (auto& chan : fUninitializedConnectingChannels) {
             string key{"chans." + chan->GetChannelPrefix() + "." + chan->GetChannelIndex() + ".address"};
             string newAddress = fConfig->GetValue<string>(key);
             if (newAddress != chan->GetAddress()) {
@@ -183,20 +304,16 @@ void FairMQDevice::InitWrapper()
             throw runtime_error(fair::mq::tools::ToString("could not connect all channels after ", initializationTimeoutInS, " attempts"));
         }
 
-        AttachChannels(uninitializedConnectingChannels);
+        AttachChannels(fUninitializedConnectingChannels);
     }
 
-    Init();
-
     if (fChannels.empty()) {
         LOG(warn) << "No channels created after finishing initialization";
     }
 
-    ChangeState(internal_DEVICE_READY);
-}
+    Connect();
 
-void FairMQDevice::Init()
-{
+    ChangeState(fair::mq::Transition::Auto);
 }
 
 void FairMQDevice::AttachChannels(vector<FairMQChannel*>& chans)
@@ -295,15 +412,9 @@ bool FairMQDevice::AttachChannel(FairMQChannel& chan)
 
 void FairMQDevice::InitTaskWrapper()
 {
-    CallStateChangeCallbacks(INITIALIZING_TASK);
-
     InitTask();
 
-    ChangeState(internal_READY);
-}
-
-void FairMQDevice::InitTask()
-{
+    ChangeState(fair::mq::Transition::Auto);
 }
 
 bool FairMQDevice::SortSocketsByAddress(const FairMQChannel &lhs, const FairMQChannel &rhs)
@@ -334,21 +445,13 @@ void FairMQDevice::SortChannel(const string& name, const bool reindex)
 
 void FairMQDevice::RunWrapper()
 {
-    CallStateChangeCallbacks(RUNNING);
-
     LOG(info) << "DEVICE: Running...";
 
     // start the rate logger thread
-    fRateLogging = true;
     future<void> rateLogger = async(launch::async, &FairMQDevice::LogSocketRates, this);
 
     // notify transports to resume transfers
-    {
-        lock_guard<mutex> guard(fInterruptedMtx);
-        fInterrupted = false;
-    }
-    for (auto& t : fTransports)
-    {
+    for (auto& t : fTransports) {
         t.second->Resume();
     }
 
@@ -356,50 +459,43 @@ void FairMQDevice::RunWrapper()
         PreRun();
 
         // process either data callbacks or ConditionalRun/Run
-        if (fDataCallbacks)
-        {
+        if (fDataCallbacks) {
             // if only one input channel, do lightweight handling without additional polling.
-            if (fInputChannelKeys.size() == 1 && fChannels.at(fInputChannelKeys.at(0)).size() == 1)
-            {
+            if (fInputChannelKeys.size() == 1 && fChannels.at(fInputChannelKeys.at(0)).size() == 1) {
                 HandleSingleChannelInput();
-            }
-            else // otherwise do full handling with polling
-            {
+            } else {// otherwise do full handling with polling
                 HandleMultipleChannelInput();
             }
-        }
-        else
-        {
+        } else {
             fair::mq::tools::RateLimiter rateLimiter(fRate);
 
-            while (CheckCurrentState(RUNNING) && ConditionalRun())
-            {
-                if (fRate > 0.001)
-                {
+            while (!NewStatePending() && ConditionalRun()) {
+                if (fRate > 0.001) {
                     rateLimiter.maybe_sleep();
                 }
             }
 
             Run();
         }
+
+        // if Run() exited and the state is still RUNNING, transition to READY.
+        if (!NewStatePending()) {
+            UnblockTransports();
+            ChangeState(fair::mq::Transition::Stop);
+        }
+
+        PostRun();
+
     } catch (const out_of_range& oor) {
         LOG(error) << "out of range: " << oor.what();
         LOG(error) << "incorrect/incomplete channel configuration?";
-        fRateLogging = false;
+        ChangeState(fair::mq::Transition::ErrorFound);
         throw;
     } catch (...) {
-        fRateLogging = false;
+        ChangeState(fair::mq::Transition::ErrorFound);
         throw;
     }
 
-    // if Run() exited and the state is still RUNNING, transition to READY.
-    if (CheckCurrentState(RUNNING))
-    {
-        ChangeState(internal_READY);
-    }
-
-    PostRun();
-
     rateLogger.get();
 }
 
@@ -409,14 +505,14 @@ void FairMQDevice::HandleSingleChannelInput()
 
     if (fMsgInputs.size() > 0)
     {
-        while (CheckCurrentState(RUNNING) && proceed)
+        while (!NewStatePending() && proceed)
         {
             proceed = HandleMsgInput(fInputChannelKeys.at(0), fMsgInputs.begin()->second, 0);
         }
     }
     else if (fMultipartInputs.size() > 0)
     {
-        while (CheckCurrentState(RUNNING) && proceed)
+        while (!NewStatePending() && proceed)
         {
             proceed = HandleMultipartInput(fInputChannelKeys.at(0), fMultipartInputs.begin()->second, 0);
         }
@@ -468,7 +564,7 @@ void FairMQDevice::HandleMultipleChannelInput()
 
         FairMQPollerPtr poller(fChannels.at(fInputChannelKeys.at(0)).at(0).fTransportFactory->CreatePoller(fChannels, fInputChannelKeys));
 
-        while (CheckCurrentState(RUNNING) && proceed)
+        while (!NewStatePending() && proceed)
         {
             poller->Poll(200);
 
@@ -526,7 +622,7 @@ void FairMQDevice::PollForTransport(const FairMQTransportFactory* factory, const
     {
         FairMQPollerPtr poller(factory->CreatePoller(fChannels, channelKeys));
 
-        while (CheckCurrentState(RUNNING) && fMultitransportProceed)
+        while (!NewStatePending() && fMultitransportProceed)
         {
             poller->Poll(500);
 
@@ -600,58 +696,21 @@ bool FairMQDevice::HandleMultipartInput(const string& chName, const InputMultipa
     }
 }
 
-void FairMQDevice::Run()
+shared_ptr<FairMQTransportFactory> FairMQDevice::AddTransport(fair::mq::Transport transport)
 {
-}
-
-void FairMQDevice::PreRun()
-{
-}
-
-bool FairMQDevice::ConditionalRun()
-{
-    return false;
-}
-
-void FairMQDevice::PostRun()
-{
-}
-
-void FairMQDevice::PauseWrapper()
-{
-    CallStateChangeCallbacks(PAUSED);
-
-    Pause();
-}
-
-void FairMQDevice::Pause()
-{
-    while (CheckCurrentState(PAUSED))
-    {
-        this_thread::sleep_for(chrono::milliseconds(500));
-        LOG(debug) << "paused...";
+    if (transport == fair::mq::Transport::DEFAULT) {
+        transport = fDefaultTransportType;
     }
-    LOG(debug) << "Unpausing";
-}
 
-shared_ptr<FairMQTransportFactory> FairMQDevice::AddTransport(const fair::mq::Transport transport)
-{
     auto i = fTransports.find(transport);
 
-    if (i == fTransports.end())
-    {
+    if (i == fTransports.end()) {
+        LOG(debug) << "Adding '" << fair::mq::TransportNames.at(transport) << "' transport";
         auto tr = FairMQTransportFactory::CreateTransportFactory(fair::mq::TransportNames.at(transport), fId, fConfig);
-
-        LOG(debug) << "Adding '" << fair::mq::TransportNames.at(transport) << "' transport to the device.";
-
-        pair<fair::mq::Transport, shared_ptr<FairMQTransportFactory>> trPair(transport, tr);
-        fTransports.insert(trPair);
-
+        fTransports.insert({transport, tr});
         return tr;
-    }
-    else
-    {
-        LOG(debug) << "Reusing existing '" << fair::mq::TransportNames.at(transport) << "' transport.";
+    } else {
+        LOG(debug) << "Reusing existing '" << fair::mq::TransportNames.at(transport) << "' transport";
         return i->second;
     }
 }
@@ -664,9 +723,6 @@ void FairMQDevice::SetConfig(FairMQProgOptions& config)
 
 void FairMQDevice::LogSocketRates()
 {
-    chrono::time_point<chrono::high_resolution_clock> t0;
-    chrono::time_point<chrono::high_resolution_clock> t1;
-
     vector<FairMQSocket*> filteredSockets;
     vector<string> filteredChannelNames;
     vector<int> logIntervals;
@@ -675,13 +731,10 @@ void FairMQDevice::LogSocketRates()
     size_t chanNameLen = 0;
 
     // iterate over the channels map
-    for (const auto& mi : fChannels)
-    {
+    for (const auto& mi : fChannels) {
         // iterate over the channels vector
-        for (auto vi = (mi.second).begin(); vi != (mi.second).end(); ++vi)
-        {
-            if (vi->fRateLogging > 0)
-            {
+        for (auto vi = (mi.second).begin(); vi != (mi.second).end(); ++vi) {
+            if (vi->fRateLogging > 0) {
                 filteredSockets.push_back(vi->fSocket.get());
                 logIntervals.push_back(vi->fRateLogging);
                 intervalCounters.push_back(0);
@@ -691,55 +744,50 @@ void FairMQDevice::LogSocketRates()
         }
     }
 
-    unsigned int numFilteredSockets = filteredSockets.size();
+    vector<unsigned long> bytesIn(filteredSockets.size());
+    vector<unsigned long> msgIn(filteredSockets.size());
+    vector<unsigned long> bytesOut(filteredSockets.size());
+    vector<unsigned long> msgOut(filteredSockets.size());
 
-    if (numFilteredSockets > 0)
-    {
-        vector<unsigned long> bytesIn(numFilteredSockets);
-        vector<unsigned long> msgIn(numFilteredSockets);
-        vector<unsigned long> bytesOut(numFilteredSockets);
-        vector<unsigned long> msgOut(numFilteredSockets);
+    vector<unsigned long> bytesInNew(filteredSockets.size());
+    vector<unsigned long> msgInNew(filteredSockets.size());
+    vector<unsigned long> bytesOutNew(filteredSockets.size());
+    vector<unsigned long> msgOutNew(filteredSockets.size());
 
-        vector<unsigned long> bytesInNew(numFilteredSockets);
-        vector<unsigned long> msgInNew(numFilteredSockets);
-        vector<unsigned long> bytesOutNew(numFilteredSockets);
-        vector<unsigned long> msgOutNew(numFilteredSockets);
+    vector<double> mbPerSecIn(filteredSockets.size());
+    vector<double> msgPerSecIn(filteredSockets.size());
+    vector<double> mbPerSecOut(filteredSockets.size());
+    vector<double> msgPerSecOut(filteredSockets.size());
 
-        vector<double> mbPerSecIn(numFilteredSockets);
-        vector<double> msgPerSecIn(numFilteredSockets);
-        vector<double> mbPerSecOut(numFilteredSockets);
-        vector<double> msgPerSecOut(numFilteredSockets);
+    int i = 0;
+    for (const auto& vi : filteredSockets) {
+        bytesIn.at(i) = vi->GetBytesRx();
+        bytesOut.at(i) = vi->GetBytesTx();
+        msgIn.at(i) = vi->GetMessagesRx();
+        msgOut.at(i) = vi->GetMessagesTx();
+        ++i;
+    }
 
-        int i = 0;
-        for (const auto& vi : filteredSockets)
-        {
-            bytesIn.at(i) = vi->GetBytesRx();
-            bytesOut.at(i) = vi->GetBytesTx();
-            msgIn.at(i) = vi->GetMessagesRx();
-            msgOut.at(i) = vi->GetMessagesTx();
-            ++i;
-        }
+    chrono::time_point<chrono::high_resolution_clock> t0(chrono::high_resolution_clock::now());
+    chrono::time_point<chrono::high_resolution_clock> t1;
+    uint64_t secondsElapsed = 0;
 
-        t0 = chrono::high_resolution_clock::now();
+    while (!NewStatePending()) {
+        WaitFor(chrono::seconds(1));
 
-        LOG(debug) << "<channel>: in: <#msgs> (<MB>) out: <#msgs> (<MB>)";
+        t1 = chrono::high_resolution_clock::now();
 
-        while (fRateLogging)
-        {
-            t1 = chrono::high_resolution_clock::now();
+        uint64_t msSinceLastLog = chrono::duration_cast<chrono::milliseconds>(t1 - t0).count();
 
-            unsigned long long msSinceLastLog = chrono::duration_cast<chrono::milliseconds>(t1 - t0).count();
+        i = 0;
 
-            i = 0;
+        for (const auto& vi : filteredSockets) {
+            intervalCounters.at(i)++;
 
-            for (const auto& vi : filteredSockets)
-            {
-                intervalCounters.at(i)++;
-
-                if (intervalCounters.at(i) == logIntervals.at(i))
-                {
-                    intervalCounters.at(i) = 0;
+            if (intervalCounters.at(i) == logIntervals.at(i)) {
+                intervalCounters.at(i) = 0;
 
+                if (msSinceLastLog > 0) {
                     bytesInNew.at(i) = vi->GetBytesRx();
                     msgInNew.at(i) = vi->GetMessagesRx();
                     bytesOutNew.at(i) = vi->GetBytesTx();
@@ -759,58 +807,42 @@ void FairMQDevice::LogSocketRates()
                               << "in: " << msgPerSecIn.at(i) << " (" << mbPerSecIn.at(i) << " MB) "
                               << "out: " << msgPerSecOut.at(i) << " (" << mbPerSecOut.at(i) << " MB)";
                 }
-
-                ++i;
             }
 
-            t0 = t1;
-            WaitFor(chrono::milliseconds(1000));
+            ++i;
+        }
+
+        t0 = t1;
+        if (fMaxRunRuntimeInS > 0 && ++secondsElapsed >= fMaxRunRuntimeInS) {
+            ChangeState(fair::mq::Transition::Stop);
         }
     }
 }
 
-void FairMQDevice::Unblock()
+void FairMQDevice::UnblockTransports()
 {
-    for (auto& t : fTransports)
-    {
+    for (auto& t : fTransports) {
         t.second->Interrupt();
     }
-    {
-        lock_guard<mutex> guard(fInterruptedMtx);
-        fInterrupted = true;
-        fRateLogging = false;
-    }
-    fInterruptedCV.notify_all();
 }
 
 void FairMQDevice::ResetTaskWrapper()
 {
-    CallStateChangeCallbacks(RESETTING_TASK);
-
     ResetTask();
 
-    ChangeState(internal_DEVICE_READY);
-}
-
-void FairMQDevice::ResetTask()
-{
+    ChangeState(fair::mq::Transition::Auto);
 }
 
 void FairMQDevice::ResetWrapper()
 {
-    CallStateChangeCallbacks(RESETTING_DEVICE);
-
-    for (auto& t : fTransports)
-    {
+    for (auto& t : fTransports) {
         t.second->Reset();
     }
 
     // iterate over the channels map
-    for (auto& mi : fChannels)
-    {
+    for (auto& mi : fChannels) {
         // iterate over the channels vector
-        for (auto& vi : mi.second)
-        {
+        for (auto& vi : mi.second) {
             // vi.fReset = true;
             vi.fSocket.reset(); // destroy FairMQSocket
         }
@@ -818,18 +850,12 @@ void FairMQDevice::ResetWrapper()
 
     Reset();
 
-    ChangeState(internal_IDLE);
-}
-
-void FairMQDevice::Reset()
-{
-}
-
-void FairMQDevice::Exit()
-{
+    ChangeState(fair::mq::Transition::Auto);
 }
 
 FairMQDevice::~FairMQDevice()
 {
-    LOG(debug) << "Destructing device " << fId;
+    UnsubscribeFromNewTransition("device");
+    fStateMachine.StopHandlingStates();
+    LOG(debug) << "Shutting down device " << fId;
 }

fairmq/FairMQDevice.h

@@ -9,7 +9,7 @@
 #ifndef FAIRMQDEVICE_H_
 #define FAIRMQDEVICE_H_
 
-#include <FairMQStateMachine.h>
+#include <StateMachine.h>
 #include <FairMQTransportFactory.h>
 #include <fairmq/Transports.h>
 
@@ -32,6 +32,7 @@
 #include <assert.h> // static_assert
 #include <type_traits> // is_trivially_copyable
 #include <stdexcept>
+#include <queue>
 
 #include <mutex>
 #include <condition_variable>
@@ -43,11 +44,43 @@ using FairMQChannelMap = std::unordered_map<std::string, std::vector<FairMQChann
 using InputMsgCallback = std::function<bool(FairMQMessagePtr&, int)>;
 using InputMultipartCallback = std::function<bool(FairMQParts&, int)>;
 
-class FairMQDevice : public FairMQStateMachine
+class FairMQDevice
 {
     friend class FairMQChannel;
 
   public:
+    // backwards-compatibility enum for old state machine interface, todo: delete this
+    enum Event
+    {
+        INIT_DEVICE,
+        internal_DEVICE_READY,
+        INIT_TASK,
+        internal_READY,
+        RUN,
+        STOP,
+        RESET_TASK,
+        RESET_DEVICE,
+        internal_IDLE,
+        END,
+        ERROR_FOUND
+    };
+
+    // backwards-compatibility enum for old state machine interface, todo: delete this
+    enum State
+    {
+        OK,
+        Error,
+        IDLE,
+        INITIALIZING_DEVICE,
+        DEVICE_READY,
+        INITIALIZING_TASK,
+        READY,
+        RUNNING,
+        RESETTING_TASK,
+        RESETTING_DEVICE,
+        EXITING
+    };
+
     /// Default constructor
     FairMQDevice();
     /// Constructor with external FairMQProgOptions
@@ -70,9 +103,6 @@ class FairMQDevice : public FairMQStateMachine
     /// Default destructor
     virtual ~FairMQDevice();
 
-    /// Catches interrupt signals (SIGINT, SIGTERM)
-    void CatchSignals();
-
     /// Outputs the socket transfer rates
     virtual void LogSocketRates();
 
@@ -330,7 +360,6 @@ class FairMQDevice : public FairMQStateMachine
     } catch (const std::out_of_range& oor) {
         LOG(error) << "out of range: " << oor.what();
         LOG(error) << "requested channel has not been configured? check channel names/configuration.";
-        fRateLogging = false;
         throw;
     }
 
@@ -339,8 +368,7 @@ class FairMQDevice : public FairMQStateMachine
     bool RegisterChannelEndpoint(const std::string& channelName, uint16_t minNumSubChannels = 1, uint16_t maxNumSubChannels = 1)
     {
         bool ok = fChannelRegistry.insert(std::make_pair(channelName, std::make_pair(minNumSubChannels, maxNumSubChannels))).second;
-        if (!ok)
-        {
+        if (!ok) {
             LOG(warn) << "Registering channel: name already registered: \"" << channelName << "\"";
         }
         return ok;
@@ -348,14 +376,10 @@ class FairMQDevice : public FairMQStateMachine
 
     void PrintRegisteredChannels()
     {
-        if (fChannelRegistry.size() < 1)
-        {
+        if (fChannelRegistry.size() < 1) {
             std::cout << "no channels registered." << std::endl;
-        }
-        else
-        {
-            for (const auto& c : fChannelRegistry)
-            {
+        } else {
+            for (const auto& c : fChannelRegistry) {
                 std::cout << c.first << ":" << c.second.first << ":" << c.second.second << std::endl;
             }
         }
@@ -389,8 +413,7 @@ class FairMQDevice : public FairMQStateMachine
 
     void RunStateMachine()
     {
-      CallStateChangeCallbacks(FairMQStateMachine::IDLE);
-      ProcessWork();
+        fStateMachine.ProcessWork();
     };
 
     /// Wait for the supplied amount of time or for interruption.
@@ -399,8 +422,7 @@ class FairMQDevice : public FairMQStateMachine
     template<class Rep, class Period>
     bool WaitFor(std::chrono::duration<Rep, Period> const& duration)
     {
-        std::unique_lock<std::mutex> lock(fInterruptedMtx);
-        return !fInterruptedCV.wait_for(lock, duration, [&] { return fInterrupted.load(); }); // return true if no interruption happened
+        return fStateMachine.WaitForPendingStateFor(std::chrono::duration_cast<std::chrono::milliseconds>(duration).count());
     }
 
   protected:
@@ -421,53 +443,92 @@ class FairMQDevice : public FairMQStateMachine
     std::string fId; ///< Device ID
 
     /// Additional user initialization (can be overloaded in child classes). Prefer to use InitTask().
-    virtual void Init();
+    virtual void Init() {}
+
+    virtual void Bind() {}
+
+    virtual void Connect() {}
 
     /// Task initialization (can be overloaded in child classes)
-    virtual void InitTask();
+    virtual void InitTask() {}
 
     /// Runs the device (to be overloaded in child classes)
-    virtual void Run();
+    virtual void Run() {}
 
     /// Called in the RUNNING state once before executing the Run()/ConditionalRun() method
-    virtual void PreRun();
+    virtual void PreRun() {}
 
     /// Called during RUNNING state repeatedly until it returns false or device state changes
-    virtual bool ConditionalRun();
+    virtual bool ConditionalRun() { return false; }
 
     /// Called in the RUNNING state once after executing the Run()/ConditionalRun() method
-    virtual void PostRun();
+    virtual void PostRun() {}
 
-    /// Handles the PAUSE state
-    virtual void Pause();
+    virtual void Pause() __attribute__((deprecated("PAUSE state is removed. This method is never called. To pause Run, go to READY with STOP transition and back to RUNNING with RUN to resume."))) {}
 
     /// Resets the user task (to be overloaded in child classes)
-    virtual void ResetTask();
+    virtual void ResetTask() {}
 
     /// Resets the device (can be overloaded in child classes)
-    virtual void Reset();
+    virtual void Reset() {}
+
+  public:
+    bool ChangeState(const fair::mq::Transition transition) { return fStateMachine.ChangeState(transition); }
+    bool ChangeState(const std::string& transition) { return fStateMachine.ChangeState(fair::mq::StateMachine::GetTransition(transition)); }
+
+    bool ChangeState(const int transition) __attribute__((deprecated("Use ChangeState(const fair::mq::Transition transition).")));
+
+    void WaitForEndOfState(const fair::mq::Transition transition) __attribute__((deprecated("Use WaitForState(fair::mq::State expectedState).")));
+    void WaitForEndOfState(const std::string& transition) __attribute__((deprecated("Use WaitForState(fair::mq::State expectedState)."))) { WaitForState(transition); }
+
+    fair::mq::State WaitForNextState();
+    void WaitForState(fair::mq::State state);
+    void WaitForState(const std::string& state) { fair::mq::StateMachine::GetState(state); }
+
+    void SubscribeToStateChange(const std::string& key, std::function<void(const fair::mq::State)> callback) { fStateMachine.SubscribeToStateChange(key, callback); }
+    void UnsubscribeFromStateChange(const std::string& key) { fStateMachine.UnsubscribeFromStateChange(key); }
+
+    void SubscribeToNewTransition(const std::string& key, std::function<void(const fair::mq::Transition)> callback) { fStateMachine.SubscribeToNewTransition(key, callback); }
+    void UnsubscribeFromNewTransition(const std::string& key) { fStateMachine.UnsubscribeFromNewTransition(key); }
+
+    bool CheckCurrentState(const int /* state */) const __attribute__((deprecated("Use NewStatePending()."))) { return !fStateMachine.NewStatePending(); }
+    bool CheckCurrentState(const std::string& /* state */) const __attribute__((deprecated("Use NewStatePending()."))) { return !fStateMachine.NewStatePending(); }
+
+    /// Returns true is a new state has been requested, signaling the current handler to stop.
+    bool NewStatePending() const { return fStateMachine.NewStatePending(); }
+
+    fair::mq::State GetCurrentState() const { return fStateMachine.GetCurrentState(); }
+    std::string GetCurrentStateName() const { return fStateMachine.GetCurrentStateName(); }
+
+    static std::string GetStateName(const fair::mq::State state) { return fair::mq::StateMachine::GetStateName(state); }
+    static std::string GetTransitionName(const fair::mq::Transition transition) { return fair::mq::StateMachine::GetTransitionName(transition); }
+
+    struct DeviceStateError : std::runtime_error { using std::runtime_error::runtime_error; };
 
   private:
     fair::mq::Transport fDefaultTransportType; ///< Default transport for the device
+    fair::mq::StateMachine fStateMachine;
 
-    /// Handles the initialization and the Init() method
+    /// Handles the initialization
     void InitWrapper();
+    /// Initializes binding channels
+    void BindWrapper();
+    /// Initializes connecting channels
+    void ConnectWrapper();
     /// Handles the InitTask() method
     void InitTaskWrapper();
     /// Handles the Run() method
     void RunWrapper();
-    /// Handles the Pause() method
-    void PauseWrapper();
     /// Handles the ResetTask() method
     void ResetTaskWrapper();
     /// Handles the Reset() method
     void ResetWrapper();
 
-    /// Unblocks blocking channel send/receive calls
-    void Unblock();
+    /// Notifies transports to cease any blocking activity
+    void UnblockTransports();
 
     /// Shuts down the transports and the device
-    void Exit();
+    void Exit() {}
 
     /// Attach (bind/connect) channels in the list
     void AttachChannels(std::vector<FairMQChannel*>& chans);
@@ -481,7 +542,8 @@ class FairMQDevice : public FairMQStateMachine
     bool HandleMsgInput(const std::string& chName, const InputMsgCallback& callback, int i);
     bool HandleMultipartInput(const std::string& chName, const InputMultipartCallback& callback, int i);
 
-    void CreateOwnConfig();
+    std::vector<FairMQChannel*> fUninitializedBindingChannels;
+    std::vector<FairMQChannel*> fUninitializedConnectingChannels;
 
     bool fDataCallbacks;
     std::unordered_map<std::string, InputMsgCallback> fMsgInputs;
@@ -494,12 +556,12 @@ class FairMQDevice : public FairMQStateMachine
 
     const fair::mq::tools::Version fVersion;
     float fRate; ///< Rate limiting for ConditionalRun
+    uint64_t fMaxRunRuntimeInS; ///< Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).
     std::vector<std::string> fRawCmdLineArgs;
 
-    std::atomic<bool> fInterrupted;
-    std::condition_variable fInterruptedCV;
-    std::mutex fInterruptedMtx;
-    mutable std::atomic<bool> fRateLogging;
+    std::queue<fair::mq::State> fStates;
+    std::mutex fStatesMtx;
+    std::condition_variable fStatesCV;
 };
 
 #endif /* FAIRMQDEVICE_H_ */

fairmq/FairMQMessage.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/FairMQPoller.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/FairMQPoller.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/FairMQSocket.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/FairMQSocket.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/FairMQStateMachine.cxx

@@ -1,672 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQStateMachine.cxx
- *
- * @since 2012-10-25
- * @author D. Klein, A. Rybalchenko
- */
-
-#include "FairMQStateMachine.h"
-#include <fairmq/Tools.h>
-
-// Increase maximum number of boost::msm states (default is 10)
-// This #define has to be before any msm header includes
-#define FUSION_MAX_VECTOR_SIZE 20
-
-#include <boost/mpl/for_each.hpp>
-#include <boost/msm/back/state_machine.hpp>
-#include <boost/msm/back/tools.hpp>
-#include <boost/msm/back/metafunctions.hpp>
-#include <boost/msm/front/state_machine_def.hpp>
-#include <boost/msm/front/functor_row.hpp>
-#include <boost/core/demangle.hpp>
-#include <boost/signals2.hpp> // signal/slot for onStateChange callbacks
-
-#include <atomic>
-#include <condition_variable>
-#include <chrono>
-#include <array>
-#include <unordered_map>
-
-using namespace std;
-using namespace boost::msm::front;
-
-namespace std
-{
-
-template<>
-struct hash<FairMQStateMachine::Event> : fair::mq::tools::HashEnum<FairMQStateMachine::Event> {};
-
-} /* namespace std */
-
-namespace fair
-{
-namespace mq
-{
-namespace fsm
-{
-
-// list of FSM states
-struct OK_FSM_STATE                  : public state<> { static string Name() { return "OK"; }                  static FairMQStateMachine::State Type() { return FairMQStateMachine::State::OK; } };
-struct ERROR_FSM_STATE               : public terminate_state<> { static string Name() { return "ERROR"; }     static FairMQStateMachine::State Type() { return FairMQStateMachine::State::Error; } };
-
-struct IDLE_FSM_STATE                : public state<> { static string Name() { return "IDLE"; }                static FairMQStateMachine::State Type() { return FairMQStateMachine::State::IDLE; } };
-struct INITIALIZING_DEVICE_FSM_STATE : public state<> { static string Name() { return "INITIALIZING_DEVICE"; } static FairMQStateMachine::State Type() { return FairMQStateMachine::State::INITIALIZING_DEVICE; } };
-struct DEVICE_READY_FSM_STATE        : public state<> { static string Name() { return "DEVICE_READY"; }        static FairMQStateMachine::State Type() { return FairMQStateMachine::State::DEVICE_READY; } };
-struct INITIALIZING_TASK_FSM_STATE   : public state<> { static string Name() { return "INITIALIZING_TASK"; }   static FairMQStateMachine::State Type() { return FairMQStateMachine::State::INITIALIZING_TASK; } };
-struct READY_FSM_STATE               : public state<> { static string Name() { return "READY"; }               static FairMQStateMachine::State Type() { return FairMQStateMachine::State::READY; } };
-struct RUNNING_FSM_STATE             : public state<> { static string Name() { return "RUNNING"; }             static FairMQStateMachine::State Type() { return FairMQStateMachine::State::RUNNING; } };
-struct PAUSED_FSM_STATE              : public state<> { static string Name() { return "PAUSED"; }              static FairMQStateMachine::State Type() { return FairMQStateMachine::State::PAUSED; } };
-struct RESETTING_TASK_FSM_STATE      : public state<> { static string Name() { return "RESETTING_TASK"; }      static FairMQStateMachine::State Type() { return FairMQStateMachine::State::RESETTING_TASK; } };
-struct RESETTING_DEVICE_FSM_STATE    : public state<> { static string Name() { return "RESETTING_DEVICE"; }    static FairMQStateMachine::State Type() { return FairMQStateMachine::State::RESETTING_DEVICE; } };
-struct EXITING_FSM_STATE             : public state<> { static string Name() { return "EXITING"; }             static FairMQStateMachine::State Type() { return FairMQStateMachine::State::EXITING; } };
-
-// list of FSM events
-struct INIT_DEVICE_FSM_EVENT           { static string Name() { return "INIT_DEVICE"; }           static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::INIT_DEVICE; } };
-struct internal_DEVICE_READY_FSM_EVENT { static string Name() { return "internal_DEVICE_READY"; } static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::internal_DEVICE_READY; } };
-struct INIT_TASK_FSM_EVENT             { static string Name() { return "INIT_TASK"; }             static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::INIT_TASK; } };
-struct internal_READY_FSM_EVENT        { static string Name() { return "internal_READY"; }        static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::internal_READY; } };
-struct RUN_FSM_EVENT                   { static string Name() { return "RUN"; }                   static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::RUN; } };
-struct PAUSE_FSM_EVENT                 { static string Name() { return "PAUSE"; }                 static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::PAUSE; } };
-struct STOP_FSM_EVENT                  { static string Name() { return "STOP"; }                  static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::STOP; } };
-struct RESET_TASK_FSM_EVENT            { static string Name() { return "RESET_TASK"; }            static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::RESET_TASK; } };
-struct RESET_DEVICE_FSM_EVENT          { static string Name() { return "RESET_DEVICE"; }          static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::RESET_DEVICE; } };
-struct internal_IDLE_FSM_EVENT         { static string Name() { return "internal_IDLE"; }         static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::internal_IDLE; } };
-struct END_FSM_EVENT                   { static string Name() { return "END"; }                   static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::END; } };
-struct ERROR_FOUND_FSM_EVENT           { static string Name() { return "ERROR_FOUND"; }           static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::ERROR_FOUND; } };
-
-static array<string, 12> stateNames =
-{
-    {
-        "OK",
-        "Error",
-        "IDLE",
-        "INITIALIZING_DEVICE",
-        "DEVICE_READY",
-        "INITIALIZING_TASK",
-        "READY",
-        "RUNNING",
-        "PAUSED",
-        "RESETTING_TASK",
-        "RESETTING_DEVICE",
-        "EXITING"
-    }
-};
-
-static array<string, 12> eventNames =
-{
-    {
-        "INIT_DEVICE",
-        "internal_DEVICE_READY",
-        "INIT_TASK",
-        "internal_READY",
-        "RUN",
-        "PAUSE",
-        "STOP",
-        "RESET_TASK",
-        "RESET_DEVICE",
-        "internal_IDLE",
-        "END",
-        "ERROR_FOUND"
-    }
-};
-
-static map<string, int> stateNumbers =
-{
-    { "OK",                  FairMQStateMachine::State::OK },
-    { "Error",               FairMQStateMachine::State::Error },
-    { "IDLE",                FairMQStateMachine::State::IDLE },
-    { "INITIALIZING_DEVICE", FairMQStateMachine::State::INITIALIZING_DEVICE },
-    { "DEVICE_READY",        FairMQStateMachine::State::DEVICE_READY },
-    { "INITIALIZING_TASK",   FairMQStateMachine::State::INITIALIZING_TASK },
-    { "READY",               FairMQStateMachine::State::READY },
-    { "RUNNING",             FairMQStateMachine::State::RUNNING },
-    { "PAUSED",              FairMQStateMachine::State::PAUSED },
-    { "RESETTING_TASK",      FairMQStateMachine::State::RESETTING_TASK },
-    { "RESETTING_DEVICE",    FairMQStateMachine::State::RESETTING_DEVICE },
-    { "EXITING",             FairMQStateMachine::State::EXITING }
-};
-
-static map<string, int> eventNumbers =
-{
-    { "INIT_DEVICE",           FairMQStateMachine::Event::INIT_DEVICE },
-    { "internal_DEVICE_READY", FairMQStateMachine::Event::internal_DEVICE_READY },
-    { "INIT_TASK",             FairMQStateMachine::Event::INIT_TASK },
-    { "internal_READY",        FairMQStateMachine::Event::internal_READY },
-    { "RUN",                   FairMQStateMachine::Event::RUN },
-    { "PAUSE",                 FairMQStateMachine::Event::PAUSE },
-    { "STOP",                  FairMQStateMachine::Event::STOP },
-    { "RESET_TASK",            FairMQStateMachine::Event::RESET_TASK },
-    { "RESET_DEVICE",          FairMQStateMachine::Event::RESET_DEVICE },
-    { "internal_IDLE",         FairMQStateMachine::Event::internal_IDLE },
-    { "END",                   FairMQStateMachine::Event::END },
-    { "ERROR_FOUND",           FairMQStateMachine::Event::ERROR_FOUND }
-};
-
-// defining the boost MSM state machine
-struct Machine_ : public state_machine_def<Machine_>
-{
-  public:
-    Machine_()
-        : fUnblockHandler()
-        , fStateHandlers()
-        , fWork()
-        , fWorkAvailableCondition()
-        , fWorkDoneCondition()
-        , fWorkMutex()
-        , fWorkerTerminated(false)
-        , fWorkActive(false)
-        , fWorkAvailable(false)
-        , fStateChangeSignal()
-        , fStateChangeSignalsMap()
-        , fState()
-    {}
-
-    virtual ~Machine_()
-    {}
-
-    // initial states
-    using initial_state = boost::mpl::vector<IDLE_FSM_STATE, OK_FSM_STATE>;
-
-    template<typename Event, typename FSM>
-    void on_entry(Event const&, FSM& /*fsm*/)
-    {
-        LOG(state) << "Starting FairMQ state machine";
-        fState = FairMQStateMachine::IDLE;
-        LOG(state) << "Entering IDLE state";
-        // fsm.CallStateChangeCallbacks(FairMQStateMachine::IDLE);
-        // we call this for now in FairMQDevice::RunStateMachine()
-    }
-
-    template<typename Event, typename FSM>
-    void on_exit(Event const&, FSM& /*fsm*/)
-    {
-        LOG(state) << "Exiting FairMQ state machine";
-    }
-
-    // actions
-    struct AutomaticFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-            LOG(state) << "Entering " << ts.Name() << " state";
-        }
-    };
-
-    struct DefaultFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const& e, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-
-            unique_lock<mutex> lock(fsm.fWorkMutex);
-            while (fsm.fWorkActive)
-            {
-                fsm.fWorkDoneCondition.wait(lock);
-            }
-            fsm.fWorkAvailable = true;
-            LOG(state) << "Entering " << ts.Name() << " state";
-            fsm.fWork = fsm.fStateHandlers.at(e.Type());
-            fsm.fWorkAvailableCondition.notify_one();
-        }
-    };
-
-    struct PauseFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-
-            fsm.fUnblockHandler();
-            unique_lock<mutex> lock(fsm.fWorkMutex);
-            while (fsm.fWorkActive)
-            {
-                fsm.fWorkDoneCondition.wait(lock);
-            }
-            fsm.fWorkAvailable = true;
-            LOG(state) << "Entering " << ts.Name() << " state";
-            fsm.fWork = fsm.fPauseWrapperHandler;
-            fsm.fWorkAvailableCondition.notify_one();
-        }
-    };
-
-    struct StopFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-
-            fsm.fUnblockHandler();
-            unique_lock<mutex> lock(fsm.fWorkMutex);
-            while (fsm.fWorkActive)
-            {
-                fsm.fWorkDoneCondition.wait(lock);
-            }
-            LOG(state) << "Entering " << ts.Name() << " state";
-        }
-    };
-
-    struct InternalStopFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-            fsm.fUnblockHandler();
-            LOG(state) << "RUNNING state finished without an external event, entering " << ts.Name() << " state";
-        }
-    };
-
-    struct ExitingFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const& e, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            LOG(state) << "Entering " << ts.Name() << " state";
-            fsm.fState = ts.Type();
-            fsm.CallStateChangeCallbacks(FairMQStateMachine::EXITING);
-
-            // Stop ProcessWork()
-            {
-                lock_guard<mutex> lock(fsm.fWorkMutex);
-                fsm.fWorkerTerminated = true;
-                fsm.fWorkAvailableCondition.notify_one();
-            }
-
-            fsm.fStateHandlers.at(e.Type())();
-        }
-    };
-
-    struct ErrorFoundFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-            LOG(state) << "Entering " << ts.Name() << " state";
-            fsm.CallStateChangeCallbacks(FairMQStateMachine::Error);
-        }
-    };
-
-    // Transition table for Machine_
-    struct transition_table : boost::mpl::vector<
-        // Start                           Event                            Next                           Action           Guard
-        Row<IDLE_FSM_STATE,                INIT_DEVICE_FSM_EVENT,           INITIALIZING_DEVICE_FSM_STATE, DefaultFct,      none>,
-        Row<IDLE_FSM_STATE,                END_FSM_EVENT,                   EXITING_FSM_STATE,             ExitingFct,      none>,
-        Row<INITIALIZING_DEVICE_FSM_STATE, internal_DEVICE_READY_FSM_EVENT, DEVICE_READY_FSM_STATE,        AutomaticFct,    none>,
-        Row<DEVICE_READY_FSM_STATE,        INIT_TASK_FSM_EVENT,             INITIALIZING_TASK_FSM_STATE,   DefaultFct,      none>,
-        Row<DEVICE_READY_FSM_STATE,        RESET_DEVICE_FSM_EVENT,          RESETTING_DEVICE_FSM_STATE,    DefaultFct,      none>,
-        Row<INITIALIZING_TASK_FSM_STATE,   internal_READY_FSM_EVENT,        READY_FSM_STATE,               AutomaticFct,    none>,
-        Row<READY_FSM_STATE,               RUN_FSM_EVENT,                   RUNNING_FSM_STATE,             DefaultFct,      none>,
-        Row<READY_FSM_STATE,               RESET_TASK_FSM_EVENT,            RESETTING_TASK_FSM_STATE,      DefaultFct,      none>,
-        Row<RUNNING_FSM_STATE,             PAUSE_FSM_EVENT,                 PAUSED_FSM_STATE,              DefaultFct,      none>,
-        Row<RUNNING_FSM_STATE,             STOP_FSM_EVENT,                  READY_FSM_STATE,               StopFct,         none>,
-        Row<RUNNING_FSM_STATE,             internal_READY_FSM_EVENT,        READY_FSM_STATE,               InternalStopFct, none>,
-        Row<PAUSED_FSM_STATE,              RUN_FSM_EVENT,                   RUNNING_FSM_STATE,             DefaultFct,      none>,
-        Row<RESETTING_TASK_FSM_STATE,      internal_DEVICE_READY_FSM_EVENT, DEVICE_READY_FSM_STATE,        AutomaticFct,    none>,
-        Row<RESETTING_DEVICE_FSM_STATE,    internal_IDLE_FSM_EVENT,         IDLE_FSM_STATE,                AutomaticFct,    none>,
-        Row<OK_FSM_STATE,                  ERROR_FOUND_FSM_EVENT,           ERROR_FSM_STATE,               ErrorFoundFct,   none>>
-    {};
-
-    // replaces the default no-transition response.
-    template<typename FSM, typename Event>
-    void no_transition(Event const& e, FSM&, int state)
-    {
-        using recursive_stt = typename boost::msm::back::recursive_get_transition_table<FSM>::type;
-        using all_states = typename boost::msm::back::generate_state_set<recursive_stt>::type;
-
-        string stateName;
-
-        boost::mpl::for_each<all_states, boost::msm::wrap<boost::mpl::placeholders::_1>>(boost::msm::back::get_state_name<recursive_stt>(stateName, state));
-
-        stateName = boost::core::demangle(stateName.c_str());
-        size_t pos = stateName.rfind(":");
-        if (pos != string::npos)
-        {
-            stateName = stateName.substr(pos + 1);
-            stateName = stateName.substr(0, stateName.size() - 10);
-        }
-
-        if (stateName != "OK")
-        {
-            LOG(state) << "No transition from state " << stateName << " on event " << e.Name();
-        }
-    }
-
-    void CallStateChangeCallbacks(const FairMQStateMachine::State state) const
-    {
-        if (!fStateChangeSignal.empty())
-        {
-            fStateChangeSignal(state);
-        }
-    }
-
-    function<void(void)> fUnblockHandler;
-    unordered_map<FairMQStateMachine::Event, function<void(void)>> fStateHandlers;
-
-    // function to execute user states in a worker thread
-    function<void(void)> fWork;
-    condition_variable fWorkAvailableCondition;
-    condition_variable fWorkDoneCondition;
-    mutex fWorkMutex;
-    bool fWorkerTerminated;
-    bool fWorkActive;
-    bool fWorkAvailable;
-
-    boost::signals2::signal<void(const FairMQStateMachine::State)> fStateChangeSignal;
-    unordered_map<string, boost::signals2::connection> fStateChangeSignalsMap;
-
-    atomic<FairMQStateMachine::State> fState;
-
-    void ProcessWork()
-    {
-        while (true)
-        {
-            {
-                unique_lock<mutex> lock(fWorkMutex);
-                // Wait for work to be done.
-                while (!fWorkAvailable && !fWorkerTerminated)
-                {
-                    fWorkAvailableCondition.wait_for(lock, chrono::milliseconds(100));
-                }
-
-                if (fWorkerTerminated)
-                {
-                    break;
-                }
-
-                fWorkActive = true;
-            }
-
-            fWork();
-
-            {
-                lock_guard<mutex> lock(fWorkMutex);
-                fWorkActive = false;
-                fWorkAvailable = false;
-                fWorkDoneCondition.notify_one();
-            }
-            CallStateChangeCallbacks(fState);
-        }
-    }
-}; // Machine_
-
-using FairMQFSM = boost::msm::back::state_machine<Machine_>;
-
-} // namespace fsm
-} // namespace mq
-} // namespace fair
-
-using namespace fair::mq::fsm;
-
-FairMQStateMachine::FairMQStateMachine()
-    : fChangeStateMutex()
-    , fFsm(new FairMQFSM)
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(INIT_DEVICE, bind(&FairMQStateMachine::InitWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(INIT_TASK, bind(&FairMQStateMachine::InitTaskWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(RUN, bind(&FairMQStateMachine::RunWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(PAUSE, bind(&FairMQStateMachine::PauseWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(RESET_TASK, bind(&FairMQStateMachine::ResetTaskWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(RESET_DEVICE, bind(&FairMQStateMachine::ResetWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(END, bind(&FairMQStateMachine::Exit, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fUnblockHandler = bind(&FairMQStateMachine::Unblock, this);
-
-    static_pointer_cast<FairMQFSM>(fFsm)->start();
-}
-
-FairMQStateMachine::~FairMQStateMachine()
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->stop();
-}
-
-int FairMQStateMachine::GetInterfaceVersion() const
-{
-    return FAIRMQ_INTERFACE_VERSION;
-}
-
-bool FairMQStateMachine::ChangeState(int event)
-{
-    try
-    {
-        switch (event)
-        {
-            case INIT_DEVICE:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(INIT_DEVICE_FSM_EVENT());
-                return true;
-            }
-            case internal_DEVICE_READY:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(internal_DEVICE_READY_FSM_EVENT());
-                return true;
-            }
-            case INIT_TASK:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(INIT_TASK_FSM_EVENT());
-                return true;
-            }
-            case internal_READY:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(internal_READY_FSM_EVENT());
-                return true;
-            }
-            case RUN:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(RUN_FSM_EVENT());
-                return true;
-            }
-            case PAUSE:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(PAUSE_FSM_EVENT());
-                return true;
-            }
-            case STOP:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(STOP_FSM_EVENT());
-                return true;
-            }
-            case RESET_DEVICE:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(RESET_DEVICE_FSM_EVENT());
-                return true;
-            }
-            case RESET_TASK:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(RESET_TASK_FSM_EVENT());
-                return true;
-            }
-            case internal_IDLE:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(internal_IDLE_FSM_EVENT());
-                return true;
-            }
-            case END:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(END_FSM_EVENT());
-                return true;
-            }
-            case ERROR_FOUND:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(ERROR_FOUND_FSM_EVENT());
-                return true;
-            }
-            default:
-            {
-                LOG(error) << "Requested state transition with an unsupported event: " << event << endl
-                           << "Supported are: INIT_DEVICE, INIT_TASK, RUN, PAUSE, STOP, RESET_TASK, RESET_DEVICE, END, ERROR_FOUND";
-                return false;
-            }
-        }
-    }
-    catch (exception& e)
-    {
-        LOG(error) << "Exception in FairMQStateMachine::ChangeState(): " << e.what();
-        exit(EXIT_FAILURE);
-    }
-    return false;
-}
-
-bool FairMQStateMachine::ChangeState(const string& event)
-{
-    return ChangeState(GetEventNumber(event));
-}
-
-void FairMQStateMachine::WaitForEndOfState(int event)
-{
-    try
-    {
-        switch (event)
-        {
-            case INIT_DEVICE:
-            case INIT_TASK:
-            case RUN:
-            case RESET_TASK:
-            case RESET_DEVICE:
-            {
-                unique_lock<mutex> lock(static_pointer_cast<FairMQFSM>(fFsm)->fWorkMutex);
-                while (static_pointer_cast<FairMQFSM>(fFsm)->fWorkActive || static_pointer_cast<FairMQFSM>(fFsm)->fWorkAvailable)
-                {
-                    static_pointer_cast<FairMQFSM>(fFsm)->fWorkDoneCondition.wait_for(lock, chrono::seconds(1));
-                }
-
-                break;
-            }
-            default:
-                LOG(error) << "Requested state is either synchronous or does not exist.";
-                break;
-        }
-    }
-    catch (exception& e)
-    {
-        LOG(error) << "Exception in FairMQStateMachine::WaitForEndOfState(): " << e.what();
-    }
-}
-
-void FairMQStateMachine::WaitForEndOfState(const string& event)
-{
-    return WaitForEndOfState(GetEventNumber(event));
-}
-
-bool FairMQStateMachine::WaitForEndOfStateForMs(int event, int durationInMs)
-{
-    try
-    {
-        switch (event)
-        {
-            case INIT_DEVICE:
-            case INIT_TASK:
-            case RUN:
-            case RESET_TASK:
-            case RESET_DEVICE:
-            {
-                unique_lock<mutex> lock(static_pointer_cast<FairMQFSM>(fFsm)->fWorkMutex);
-                while (static_pointer_cast<FairMQFSM>(fFsm)->fWorkActive || static_pointer_cast<FairMQFSM>(fFsm)->fWorkAvailable)
-                {
-                    static_pointer_cast<FairMQFSM>(fFsm)->fWorkDoneCondition.wait_for(lock, chrono::milliseconds(durationInMs));
-                    if (static_pointer_cast<FairMQFSM>(fFsm)->fWorkActive)
-                    {
-                        return false;
-                    }
-                }
-                return true;
-            }
-            default:
-                LOG(error) << "Requested state is either synchronous or does not exist.";
-                return false;
-        }
-    }
-    catch (exception& e)
-    {
-        LOG(error) << "Exception in FairMQStateMachine::WaitForEndOfStateForMs(): " << e.what();
-    }
-    return false;
-}
-
-bool FairMQStateMachine::WaitForEndOfStateForMs(const string& event, int durationInMs)
-{
-    return WaitForEndOfStateForMs(GetEventNumber(event), durationInMs);
-}
-
-void FairMQStateMachine::SubscribeToStateChange(const string& key, function<void(const State)> callback)
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.insert({key, static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignal.connect(callback)});
-}
-void FairMQStateMachine::UnsubscribeFromStateChange(const string& key)
-{
-    if (static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.count(key))
-    {
-        static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.at(key).disconnect();
-        static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.erase(key);
-    }
-}
-
-void FairMQStateMachine::CallStateChangeCallbacks(const State state) const
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->CallStateChangeCallbacks(state);
-}
-
-string FairMQStateMachine::GetCurrentStateName() const
-{
-    return GetStateName(static_pointer_cast<FairMQFSM>(fFsm)->fState);
-}
-string FairMQStateMachine::GetStateName(const State state)
-{
-    return stateNames.at(state);
-}
-int FairMQStateMachine::GetCurrentState() const
-{
-    return static_pointer_cast<FairMQFSM>(fFsm)->fState;
-}
-bool FairMQStateMachine::CheckCurrentState(int state) const
-{
-    return state == static_pointer_cast<FairMQFSM>(fFsm)->fState;
-}
-bool FairMQStateMachine::CheckCurrentState(const string& state) const
-{
-    return state == GetCurrentStateName();
-}
-
-void FairMQStateMachine::ProcessWork()
-try
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->ProcessWork();
-} catch(...) {
-    {
-        lock_guard<mutex> lock(static_pointer_cast<FairMQFSM>(fFsm)->fWorkMutex);
-        static_pointer_cast<FairMQFSM>(fFsm)->fWorkActive = false;
-        static_pointer_cast<FairMQFSM>(fFsm)->fWorkAvailable = false;
-        static_pointer_cast<FairMQFSM>(fFsm)->fWorkDoneCondition.notify_one();
-    }
-    ChangeState(ERROR_FOUND);
-    throw;
-}
-
-
-int FairMQStateMachine::GetEventNumber(const string& event)
-{
-    return eventNumbers.at(event);
-}

fairmq/FairMQStateMachine.h

@@ -1,107 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQStateMachine.h
- *
- * @since 2012-10-25
- * @author D. Klein, A. Rybalchenko
- */
-
-#ifndef FAIRMQSTATEMACHINE_H_
-#define FAIRMQSTATEMACHINE_H_
-
-#define FAIRMQ_INTERFACE_VERSION 3
-
-#include "FairMQLogger.h"
-
-#include <string>
-#include <memory>
-#include <functional>
-#include <mutex>
-
-class FairMQStateMachine
-{
-  public:
-    enum Event
-    {
-        INIT_DEVICE,
-        internal_DEVICE_READY,
-        INIT_TASK,
-        internal_READY,
-        RUN,
-        PAUSE,
-        STOP,
-        RESET_TASK,
-        RESET_DEVICE,
-        internal_IDLE,
-        END,
-        ERROR_FOUND
-    };
-
-    enum State
-    {
-        OK,
-        Error,
-        IDLE,
-        INITIALIZING_DEVICE,
-        DEVICE_READY,
-        INITIALIZING_TASK,
-        READY,
-        RUNNING,
-        PAUSED,
-        RESETTING_TASK,
-        RESETTING_DEVICE,
-        EXITING
-    };
-
-    FairMQStateMachine();
-    virtual ~FairMQStateMachine();
-
-    int GetInterfaceVersion() const;
-
-    bool ChangeState(int event);
-    bool ChangeState(const std::string& event);
-
-    void WaitForEndOfState(int event);
-    void WaitForEndOfState(const std::string& event);
-
-    bool WaitForEndOfStateForMs(int event, int durationInMs);
-    bool WaitForEndOfStateForMs(const std::string& event, int durationInMs);
-
-    void SubscribeToStateChange(const std::string& key, std::function<void(const State)> callback);
-    void UnsubscribeFromStateChange(const std::string& key);
-
-    void CallStateChangeCallbacks(const State state) const;
-
-    std::string GetCurrentStateName() const;
-    static std::string GetStateName(const State);
-    int GetCurrentState() const;
-    bool CheckCurrentState(int state) const;
-    bool CheckCurrentState(const std::string& state) const;
-
-    // actions to be overwritten by derived classes
-    virtual void InitWrapper() {}
-    virtual void InitTaskWrapper() {}
-    virtual void RunWrapper() {}
-    virtual void PauseWrapper() {}
-    virtual void ResetWrapper() {}
-    virtual void ResetTaskWrapper() {}
-    virtual void Exit() {}
-    virtual void Unblock() {}
-
-    void ProcessWork();
-
-  private:
-    static int GetEventNumber(const std::string& event);
-
-    std::mutex fChangeStateMutex;
-
-    std::shared_ptr<void> fFsm;
-};
-
-#endif /* FAIRMQSTATEMACHINE_H_ */

fairmq/Plugin.h

@@ -78,7 +78,7 @@ class Plugin
     auto TakeDeviceControl() -> void { fPluginServices->TakeDeviceControl(fkName); };
     auto StealDeviceControl() -> void { fPluginServices->StealDeviceControl(fkName); };
     auto ReleaseDeviceControl() -> void { fPluginServices->ReleaseDeviceControl(fkName); };
-    auto ChangeDeviceState(const DeviceStateTransition next) -> void { fPluginServices->ChangeDeviceState(fkName, next); }
+    auto ChangeDeviceState(const DeviceStateTransition next) -> bool { return fPluginServices->ChangeDeviceState(fkName, next); }
     auto SubscribeToDeviceStateChange(std::function<void(DeviceState)> callback) -> void { fPluginServices->SubscribeToDeviceStateChange(fkName, callback); }
     auto UnsubscribeFromDeviceStateChange() -> void { fPluginServices->UnsubscribeFromDeviceStateChange(fkName); }
 

fairmq/PluginManager.cxx

@@ -29,7 +29,7 @@ using boost::optional;
 const std::string fair::mq::PluginManager::fgkLibPrefix = "FairMQPlugin_";
 
 fair::mq::PluginManager::PluginManager()
-    : fSearchPaths{{"."}}
+    : fSearchPaths{}
     , fPluginFactories()
     , fPluginServices()
     , fPlugins()
@@ -39,7 +39,7 @@ fair::mq::PluginManager::PluginManager()
 }
 
 fair::mq::PluginManager::PluginManager(const vector<string> args)
-    : fSearchPaths{{"."}}
+    : fSearchPaths{}
     , fPluginFactories()
     , fPluginServices()
     , fPlugins()
@@ -115,7 +115,8 @@ auto fair::mq::PluginManager::ProgramOptions() -> po::options_description
                                                                             "* Append(>) or prepend(<) to default search path, e.g.\n"
                                                                             "  -S >/lib </home/user/lib\n"
                                                                             "* If you mix the overriding and appending/prepending syntaxes, the overriding paths act as default search path, e.g.\n"
-                                                                            "  -S /usr/lib >/lib </home/user/lib /usr/local/lib results in /home/user/lib,/usr/local/lib,/usr/lib/,/lib")
+                                                                            "  -S /usr/lib >/lib </home/user/lib /usr/local/lib results in /home/user/lib,/usr/local/lib,/usr/lib/,/lib\n"
+                                                                            "If nothing is found, the default dynamic library lookup is performed, see man ld.so(8) for details.")
         ("plugin,P", po::value<vector<string>>(), "List of plugin names to load in order,"
                                                   "e.g. if the file is called 'libFairMQPlugin_example.so', just list 'example' or 'd:example' here."
                                                   "To load a prelinked plugin, list 'p:example' here.");
@@ -170,29 +171,42 @@ auto fair::mq::PluginManager::LoadPluginDynamic(const string& pluginName) -> voi
     if (fPluginFactories.find(pluginName) == fPluginFactories.end())
     {
         auto success = false;
-        for(const auto& searchPath : SearchPaths())
-        {
-            try
-            {
-                LoadSymbols(
-                    pluginName,
-                    searchPath / ToString(LibPrefix(), pluginName),
-                    dll::load_mode::append_decorations
-                );
-
+        for (const auto& searchPath : SearchPaths()) {
+            try {
+                LoadSymbols(pluginName,
+                            searchPath / ToString(LibPrefix(), pluginName),
+                            dll::load_mode::append_decorations | dll::load_mode::rtld_global);
                 fPluginOrder.push_back(pluginName);
                 success = true;
                 break;
-            }
-            catch (boost::system::system_error& e)
-            {
-                if(string{e.what()}.find("No such file or directory") == string::npos)
-                {
-                    throw PluginLoadError(ToString("An error occurred while loading dynamic plugin ", pluginName, ": ", e.what()));
+            } catch (boost::system::system_error& e) {
+                if (string{e.what()}.find("No such file or directory") == string::npos) {
+                    throw PluginLoadError(
+                        ToString("An error occurred while loading dynamic plugin ",
+                                 pluginName, ": ", e.what()));
                 }
             }
         }
-        if(!success) { throw PluginLoadError(ToString("The plugin ", pluginName, " could not be found in the plugin search paths.")); }
+
+        if (!success) {
+            try {
+                // LoadSymbols(pluginName,
+                            // ToString(LibPrefix(), pluginName),
+                            // dll::load_mode::search_system_folders | dll::load_mode::append_decorations);
+                // Not sure, why the above does not work. Workaround for now:
+                LoadSymbols(pluginName,
+                            ToString("lib",
+                                     LibPrefix(),
+                                     pluginName,
+                                     boost::dll::detail::shared_library_impl::suffix().native()),
+                            dll::load_mode::search_system_folders | dll::load_mode::rtld_global);
+                fPluginOrder.push_back(pluginName);
+            } catch (boost::system::system_error& e) {
+                throw PluginLoadError(
+                    ToString("An error occurred while loading dynamic plugin ",
+                             pluginName, ": ", e.what()));
+            }
+        }
     }
 }
 

fairmq/PluginServices.cxx

@@ -16,11 +16,14 @@ const std::unordered_map<std::string, PluginServices::DeviceState> PluginService
     {"ERROR",               DeviceState::Error},
     {"IDLE",                DeviceState::Idle},
     {"INITIALIZING DEVICE", DeviceState::InitializingDevice},
+    {"INITIALIZED",         DeviceState::Initialized},
+    {"BINDING",             DeviceState::Binding},
+    {"BOUND",               DeviceState::Bound},
+    {"CONNECTING",          DeviceState::Connecting},
     {"DEVICE READY",        DeviceState::DeviceReady},
     {"INITIALIZING TASK",   DeviceState::InitializingTask},
     {"READY",               DeviceState::Ready},
     {"RUNNING",             DeviceState::Running},
-    {"PAUSED",              DeviceState::Paused},
     {"RESETTING TASK",      DeviceState::ResettingTask},
     {"RESETTING DEVICE",    DeviceState::ResettingDevice},
     {"EXITING",             DeviceState::Exiting}
@@ -30,83 +33,96 @@ const std::unordered_map<PluginServices::DeviceState, std::string, tools::HashEn
     {DeviceState::Error,              "ERROR"},
     {DeviceState::Idle,               "IDLE"},
     {DeviceState::InitializingDevice, "INITIALIZING DEVICE"},
+    {DeviceState::Initialized,        "INITIALIZED"},
+    {DeviceState::Binding,            "BINDING"},
+    {DeviceState::Bound,              "BOUND"},
+    {DeviceState::Connecting,         "CONNECTING"},
     {DeviceState::DeviceReady,        "DEVICE READY"},
     {DeviceState::InitializingTask,   "INITIALIZING TASK"},
     {DeviceState::Ready,              "READY"},
     {DeviceState::Running,            "RUNNING"},
-    {DeviceState::Paused,             "PAUSED"},
     {DeviceState::ResettingTask,      "RESETTING TASK"},
     {DeviceState::ResettingDevice,    "RESETTING DEVICE"},
     {DeviceState::Exiting,            "EXITING"}
 };
 const std::unordered_map<std::string, PluginServices::DeviceStateTransition> PluginServices::fkDeviceStateTransitionStrMap = {
-    {"INIT DEVICE",  DeviceStateTransition::InitDevice},
-    {"INIT TASK",    DeviceStateTransition::InitTask},
-    {"RUN",          DeviceStateTransition::Run},
-    {"PAUSE",        DeviceStateTransition::Pause},
-    {"RESUME",       DeviceStateTransition::Resume},
-    {"STOP",         DeviceStateTransition::Stop},
-    {"RESET TASK",   DeviceStateTransition::ResetTask},
-    {"RESET DEVICE", DeviceStateTransition::ResetDevice},
-    {"END",          DeviceStateTransition::End},
-    {"ERROR FOUND",  DeviceStateTransition::ErrorFound},
+    {"AUTO",          DeviceStateTransition::Auto},
+    {"INIT DEVICE",   DeviceStateTransition::InitDevice},
+    {"COMPLETE INIT", DeviceStateTransition::CompleteInit},
+    {"BIND",          DeviceStateTransition::Bind},
+    {"CONNECT",       DeviceStateTransition::Connect},
+    {"INIT TASK",     DeviceStateTransition::InitTask},
+    {"RUN",           DeviceStateTransition::Run},
+    {"STOP",          DeviceStateTransition::Stop},
+    {"RESET TASK",    DeviceStateTransition::ResetTask},
+    {"RESET DEVICE",  DeviceStateTransition::ResetDevice},
+    {"END",           DeviceStateTransition::End},
+    {"ERROR FOUND",   DeviceStateTransition::ErrorFound},
 };
 const std::unordered_map<PluginServices::DeviceStateTransition, std::string, tools::HashEnum<PluginServices::DeviceStateTransition>> PluginServices::fkStrDeviceStateTransitionMap = {
-    {DeviceStateTransition::InitDevice,  "INIT DEVICE"},
-    {DeviceStateTransition::InitTask,    "INIT TASK"},
-    {DeviceStateTransition::Run,         "RUN"},
-    {DeviceStateTransition::Pause,       "PAUSE"},
-    {DeviceStateTransition::Resume,      "RESUME"},
-    {DeviceStateTransition::Stop,        "STOP"},
-    {DeviceStateTransition::ResetTask,   "RESET TASK"},
-    {DeviceStateTransition::ResetDevice, "RESET DEVICE"},
-    {DeviceStateTransition::End,         "END"},
-    {DeviceStateTransition::ErrorFound,  "ERROR FOUND"},
+    {DeviceStateTransition::Auto,         "Auto"},
+    {DeviceStateTransition::InitDevice,   "INIT DEVICE"},
+    {DeviceStateTransition::CompleteInit, "COMPLETE INIT"},
+    {DeviceStateTransition::Bind,         "BIND"},
+    {DeviceStateTransition::Connect,      "CONNECT"},
+    {DeviceStateTransition::InitTask,     "INIT TASK"},
+    {DeviceStateTransition::Run,          "RUN"},
+    {DeviceStateTransition::Stop,         "STOP"},
+    {DeviceStateTransition::ResetTask,    "RESET TASK"},
+    {DeviceStateTransition::ResetDevice,  "RESET DEVICE"},
+    {DeviceStateTransition::End,          "END"},
+    {DeviceStateTransition::ErrorFound,   "ERROR FOUND"},
 };
-const std::unordered_map<FairMQDevice::State, PluginServices::DeviceState, fair::mq::tools::HashEnum<FairMQDevice::State>> PluginServices::fkDeviceStateMap = {
-    {FairMQDevice::OK,                  DeviceState::Ok},
-    {FairMQDevice::Error,               DeviceState::Error},
-    {FairMQDevice::IDLE,                DeviceState::Idle},
-    {FairMQDevice::INITIALIZING_DEVICE, DeviceState::InitializingDevice},
-    {FairMQDevice::DEVICE_READY,        DeviceState::DeviceReady},
-    {FairMQDevice::INITIALIZING_TASK,   DeviceState::InitializingTask},
-    {FairMQDevice::READY,               DeviceState::Ready},
-    {FairMQDevice::RUNNING,             DeviceState::Running},
-    {FairMQDevice::PAUSED,              DeviceState::Paused},
-    {FairMQDevice::RESETTING_TASK,      DeviceState::ResettingTask},
-    {FairMQDevice::RESETTING_DEVICE,    DeviceState::ResettingDevice},
-    {FairMQDevice::EXITING,             DeviceState::Exiting}
+const std::unordered_map<fair::mq::State, PluginServices::DeviceState, fair::mq::tools::HashEnum<fair::mq::State>> PluginServices::fkDeviceStateMap = {
+    {fair::mq::State::Ok,                 DeviceState::Ok},
+    {fair::mq::State::Error,              DeviceState::Error},
+    {fair::mq::State::Idle,               DeviceState::Idle},
+    {fair::mq::State::InitializingDevice, DeviceState::InitializingDevice},
+    {fair::mq::State::Initialized,        DeviceState::Initialized},
+    {fair::mq::State::Binding,            DeviceState::Binding},
+    {fair::mq::State::Bound,              DeviceState::Bound},
+    {fair::mq::State::Connecting,         DeviceState::Connecting},
+    {fair::mq::State::DeviceReady,        DeviceState::DeviceReady},
+    {fair::mq::State::InitializingTask,   DeviceState::InitializingTask},
+    {fair::mq::State::Ready,              DeviceState::Ready},
+    {fair::mq::State::Running,            DeviceState::Running},
+    {fair::mq::State::ResettingTask,      DeviceState::ResettingTask},
+    {fair::mq::State::ResettingDevice,    DeviceState::ResettingDevice},
+    {fair::mq::State::Exiting,            DeviceState::Exiting}
 };
-const std::unordered_map<PluginServices::DeviceStateTransition, FairMQDevice::Event, tools::HashEnum<PluginServices::DeviceStateTransition>> PluginServices::fkDeviceStateTransitionMap = {
-    {DeviceStateTransition::InitDevice,  FairMQDevice::INIT_DEVICE},
-    {DeviceStateTransition::InitTask,    FairMQDevice::INIT_TASK},
-    {DeviceStateTransition::Run,         FairMQDevice::RUN},
-    {DeviceStateTransition::Pause,       FairMQDevice::PAUSE},
-    {DeviceStateTransition::Resume,      FairMQDevice::RUN},
-    {DeviceStateTransition::Stop,        FairMQDevice::STOP},
-    {DeviceStateTransition::ResetTask,   FairMQDevice::RESET_TASK},
-    {DeviceStateTransition::ResetDevice, FairMQDevice::RESET_DEVICE},
-    {DeviceStateTransition::End,         FairMQDevice::END},
-    {DeviceStateTransition::ErrorFound,  FairMQDevice::ERROR_FOUND}
+const std::unordered_map<PluginServices::DeviceStateTransition, fair::mq::Transition, tools::HashEnum<PluginServices::DeviceStateTransition>> PluginServices::fkDeviceStateTransitionMap = {
+    {DeviceStateTransition::Auto,         fair::mq::Transition::Auto},
+    {DeviceStateTransition::InitDevice,   fair::mq::Transition::InitDevice},
+    {DeviceStateTransition::CompleteInit, fair::mq::Transition::CompleteInit},
+    {DeviceStateTransition::Bind,         fair::mq::Transition::Bind},
+    {DeviceStateTransition::Connect,      fair::mq::Transition::Connect},
+    {DeviceStateTransition::InitTask,     fair::mq::Transition::InitTask},
+    {DeviceStateTransition::Run,          fair::mq::Transition::Run},
+    {DeviceStateTransition::Stop,         fair::mq::Transition::Stop},
+    {DeviceStateTransition::ResetTask,    fair::mq::Transition::ResetTask},
+    {DeviceStateTransition::ResetDevice,  fair::mq::Transition::ResetDevice},
+    {DeviceStateTransition::End,          fair::mq::Transition::End},
+    {DeviceStateTransition::ErrorFound,   fair::mq::Transition::ErrorFound}
 };
 
-auto PluginServices::ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> void
+auto PluginServices::ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> bool
 {
     lock_guard<mutex> lock{fDeviceControllerMutex};
 
     if (!fDeviceController) fDeviceController = controller;
 
-    if (fDeviceController == controller)
-    {
-        fDevice.ChangeState(fkDeviceStateTransitionMap.at(next));
-    }
-    else
-    {
+    bool result = false;
+
+    if (fDeviceController == controller) {
+        result = fDevice.ChangeState(fkDeviceStateTransitionMap.at(next));
+    } else {
         throw DeviceControlError{tools::ToString(
             "Plugin '", controller, "' is not allowed to change device states. ",
             "Currently, plugin '", *fDeviceController, "' has taken control."
         )};
     }
+
+    return result;
 }
 
 auto PluginServices::TakeDeviceControl(const std::string& controller) -> void

fairmq/PluginServices.h

@@ -63,11 +63,14 @@ class PluginServices
         Error,
         Idle,
         InitializingDevice,
+        Initialized,
+        Binding,
+        Bound,
+        Connecting,
         DeviceReady,
         InitializingTask,
         Ready,
         Running,
-        Paused,
         ResettingTask,
         ResettingDevice,
         Exiting
@@ -75,11 +78,13 @@ class PluginServices
 
     enum class DeviceStateTransition : int // transition event between DeviceStates
     {
+        Auto,
         InitDevice,
+        CompleteInit,
+        Bind,
+        Connect,
         InitTask,
         Run,
-        Pause,
-        Resume,
         Stop,
         ResetTask,
         ResetDevice,
@@ -115,7 +120,7 @@ class PluginServices
     friend auto operator<<(std::ostream& os, const DeviceStateTransition& transition) -> std::ostream& { return os << ToStr(transition); }
 
     /// @return current device state
-    auto GetCurrentDeviceState() const -> DeviceState { return fkDeviceStateMap.at(static_cast<FairMQDevice::State>(fDevice.GetCurrentState())); }
+    auto GetCurrentDeviceState() const -> DeviceState { return fkDeviceStateMap.at(static_cast<fair::mq::State>(fDevice.GetCurrentState())); }
 
     /// @brief Become device controller
     /// @param controller id
@@ -151,7 +156,7 @@ class PluginServices
     /// The state transition may not happen immediately, but when the current state evaluates the
     /// pending transition event and terminates. In other words, the device states are scheduled cooperatively.
     /// If the device control role has not been taken yet, calling this function will take over control implicitely.
-    auto ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> void;
+    auto ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> bool;
 
     /// @brief Subscribe with a callback to device state changes
     /// @param subscriber id
@@ -161,7 +166,7 @@ class PluginServices
     /// the state is running in.
     auto SubscribeToDeviceStateChange(const std::string& subscriber, std::function<void(DeviceState /*newState*/)> callback) -> void
     {
-        fDevice.SubscribeToStateChange(subscriber, [&,callback](FairMQDevice::State newState){
+        fDevice.SubscribeToStateChange(subscriber, [&,callback](fair::mq::State newState){
             callback(fkDeviceStateMap.at(newState));
         });
     }
@@ -187,12 +192,13 @@ class PluginServices
     {
         auto currentState = GetCurrentDeviceState();
         if (   (currentState == DeviceState::InitializingDevice)
-            || ((currentState == DeviceState::Idle) && (key == "channel-config")))
-        {
+            || (currentState == DeviceState::Initialized)
+            || (currentState == DeviceState::Binding)
+            || (currentState == DeviceState::Bound)
+            || (currentState == DeviceState::Connecting)
+            || (currentState == DeviceState::Idle && key == "channel-config")) {
             fConfig.SetValue(key, val);
-        }
-        else
-        {
+        } else {
             throw InvalidStateError{
                 tools::ToString("PluginServices::SetProperty is not supported in device state ", currentState, ". ",
                                 "Supported state is ", DeviceState::InitializingDevice, ".")};
@@ -271,8 +277,8 @@ class PluginServices
     static const std::unordered_map<DeviceState, std::string, tools::HashEnum<DeviceState>> fkStrDeviceStateMap;
     static const std::unordered_map<std::string, DeviceStateTransition> fkDeviceStateTransitionStrMap;
     static const std::unordered_map<DeviceStateTransition, std::string, tools::HashEnum<DeviceStateTransition>> fkStrDeviceStateTransitionMap;
-    static const std::unordered_map<FairMQDevice::State, DeviceState, tools::HashEnum<FairMQDevice::State>> fkDeviceStateMap;
-    static const std::unordered_map<DeviceStateTransition, FairMQDevice::Event, tools::HashEnum<DeviceStateTransition>> fkDeviceStateTransitionMap;
+    static const std::unordered_map<fair::mq::State, DeviceState, tools::HashEnum<fair::mq::State>> fkDeviceStateMap;
+    static const std::unordered_map<DeviceStateTransition, fair::mq::Transition, tools::HashEnum<DeviceStateTransition>> fkDeviceStateTransitionMap;
 
   private:
     FairMQProgOptions& fConfig;

fairmq/StateMachine.cxx

@@ -1,5 +1,5 @@
 /********************************************************************************
- *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -7,189 +7,476 @@
  ********************************************************************************/
 
 #include "StateMachine.h"
+#include <fairmq/Tools.h>
+
+// Increase maximum number of boost::msm states (default is 10)
+// This #define has to be before any msm header includes
+#define FUSION_MAX_VECTOR_SIZE 20
+
+#include <boost/mpl/for_each.hpp>
+#include <boost/msm/back/state_machine.hpp>
+#include <boost/msm/back/tools.hpp>
+#include <boost/msm/back/metafunctions.hpp>
+#include <boost/msm/front/state_machine_def.hpp>
+#include <boost/msm/front/functor_row.hpp>
+#include <boost/core/demangle.hpp>
+#include <boost/signals2.hpp> // signal/slot for onStateChange callbacks
+
+#include <atomic>
+#include <condition_variable>
+#include <chrono>
+#include <array>
+#include <unordered_map>
+#include <mutex>
 
-using namespace fair::mq;
 using namespace std;
+using namespace boost::msm;
+using namespace boost::msm::front;
+using namespace boost::msm::back;
+namespace bmpl = boost::mpl;
 
-const std::unordered_map<std::string, StateMachine::State> StateMachine::fkStateStrMap = {
-    {"OK",                  State::Ok},
-    {"ERROR",               State::Error},
-    {"IDLE",                State::Idle},
-    {"INITIALIZING DEVICE", State::InitializingDevice},
-    {"DEVICE READY",        State::DeviceReady},
-    {"INITIALIZING TASK",   State::InitializingTask},
-    {"READY",               State::Ready},
-    {"RUNNING",             State::Running},
-    {"RESETTING TASK",      State::ResettingTask},
-    {"RESETTING DEVICE",    State::ResettingDevice},
-    {"EXITING",             State::Exiting}
-};
-const std::unordered_map<StateMachine::State, std::string, tools::HashEnum<StateMachine::State>> StateMachine::fkStrStateMap = {
-    {State::Ok,                 "OK"},
-    {State::Error,              "ERROR"},
-    {State::Idle,               "IDLE"},
-    {State::InitializingDevice, "INITIALIZING DEVICE"},
-    {State::DeviceReady,        "DEVICE READY"},
-    {State::InitializingTask,   "INITIALIZING TASK"},
-    {State::Ready,              "READY"},
-    {State::Running,            "RUNNING"},
-    {State::ResettingTask,      "RESETTING TASK"},
-    {State::ResettingDevice,    "RESETTING DEVICE"},
-    {State::Exiting,            "EXITING"}
-};
-const std::unordered_map<std::string, StateMachine::StateTransition> StateMachine::fkStateTransitionStrMap = {
-    {"INIT DEVICE",  StateTransition::InitDevice},
-    {"INIT TASK",    StateTransition::InitTask},
-    {"RUN",          StateTransition::Run},
-    {"STOP",         StateTransition::Stop},
-    {"RESET TASK",   StateTransition::ResetTask},
-    {"RESET DEVICE", StateTransition::ResetDevice},
-    {"END",          StateTransition::End},
-    {"ERROR FOUND",  StateTransition::ErrorFound},
-    {"AUTOMATIC",    StateTransition::Automatic},
-};
-const std::unordered_map<StateMachine::StateTransition, std::string, tools::HashEnum<StateMachine::StateTransition>> StateMachine::fkStrStateTransitionMap = {
-    {StateTransition::InitDevice,  "INIT DEVICE"},
-    {StateTransition::InitTask,    "INIT TASK"},
-    {StateTransition::Run,         "RUN"},
-    {StateTransition::Stop,        "STOP"},
-    {StateTransition::ResetTask,   "RESET TASK"},
-    {StateTransition::ResetDevice, "RESET DEVICE"},
-    {StateTransition::End,         "END"},
-    {StateTransition::ErrorFound,  "ERROR FOUND"},
-    {StateTransition::Automatic,   "AUTOMATIC"},
-};
-
-auto StateMachine::Run() -> void
+namespace std
 {
-    LOG(state) << "Starting FairMQ state machine";
 
-    LOG(debug) << "Entering initial " << fErrorState << " state (orthogonal error state machine)";
-    LOG(state) << "Entering initial " << fState << " state";
+template<>
+struct hash<fair::mq::Transition> : fair::mq::tools::HashEnum<fair::mq::Transition> {};
 
-    std::unique_lock<std::mutex> lock{fMutex};
-    while (true)
+template<>
+struct hash<fair::mq::State> : fair::mq::tools::HashEnum<fair::mq::State> {};
+
+} /* namespace std */
+
+namespace fair
+{
+namespace mq
+{
+namespace fsm
+{
+
+// list of FSM states
+struct OK_S                  : public state<> { static string Name() { return "OK"; }                  static State Type() { return State::Ok; } };
+
+struct IDLE_S                : public state<> { static string Name() { return "IDLE"; }                static State Type() { return State::Idle; } };
+struct INITIALIZING_DEVICE_S : public state<> { static string Name() { return "INITIALIZING_DEVICE"; } static State Type() { return State::InitializingDevice; } };
+struct INITIALIZED_S         : public state<> { static string Name() { return "INITIALIZED"; }         static State Type() { return State::Initialized; } };
+struct BINDING_S             : public state<> { static string Name() { return "BINDING"; }             static State Type() { return State::Binding; } };
+struct BOUND_S               : public state<> { static string Name() { return "BOUND"; }               static State Type() { return State::Bound; } };
+struct CONNECTING_S          : public state<> { static string Name() { return "CONNECTING"; }          static State Type() { return State::Connecting; } };
+struct DEVICE_READY_S        : public state<> { static string Name() { return "DEVICE_READY"; }        static State Type() { return State::DeviceReady; } };
+struct INITIALIZING_TASK_S   : public state<> { static string Name() { return "INITIALIZING_TASK"; }   static State Type() { return State::InitializingTask; } };
+struct READY_S               : public state<> { static string Name() { return "READY"; }               static State Type() { return State::Ready; } };
+struct RUNNING_S             : public state<> { static string Name() { return "RUNNING"; }             static State Type() { return State::Running; } };
+struct RESETTING_TASK_S      : public state<> { static string Name() { return "RESETTING_TASK"; }      static State Type() { return State::ResettingTask; } };
+struct RESETTING_DEVICE_S    : public state<> { static string Name() { return "RESETTING_DEVICE"; }    static State Type() { return State::ResettingDevice; } };
+struct EXITING_S             : public state<> { static string Name() { return "EXITING"; }             static State Type() { return State::Exiting; } };
+
+struct ERROR_S               : public terminate_state<> { static string Name() { return "ERROR"; }     static State Type() { return State::Error; } };
+
+// list of FSM transitions (events)
+struct AUTO_E          { static string Name() { return "AUTO"; }          static Transition Type() { return Transition::Auto; } };
+struct INIT_DEVICE_E   { static string Name() { return "INIT_DEVICE"; }   static Transition Type() { return Transition::InitDevice; } };
+struct COMPLETE_INIT_E { static string Name() { return "COMPLETE_INIT"; } static Transition Type() { return Transition::CompleteInit; } };
+struct BIND_E          { static string Name() { return "BIND"; }          static Transition Type() { return Transition::Bind; } };
+struct CONNECT_E       { static string Name() { return "CONNECT"; }       static Transition Type() { return Transition::Connect; } };
+struct INIT_TASK_E     { static string Name() { return "INIT_TASK"; }     static Transition Type() { return Transition::InitTask; } };
+struct RUN_E           { static string Name() { return "RUN"; }           static Transition Type() { return Transition::Run; } };
+struct STOP_E          { static string Name() { return "STOP"; }          static Transition Type() { return Transition::Stop; } };
+struct RESET_TASK_E    { static string Name() { return "RESET_TASK"; }    static Transition Type() { return Transition::ResetTask; } };
+struct RESET_DEVICE_E  { static string Name() { return "RESET_DEVICE"; }  static Transition Type() { return Transition::ResetDevice; } };
+struct END_E           { static string Name() { return "END"; }           static Transition Type() { return Transition::End; } };
+struct ERROR_FOUND_E   { static string Name() { return "ERROR_FOUND"; }   static Transition Type() { return Transition::ErrorFound; } };
+
+static array<string, 15> stateNames =
+{
     {
-        while (fNextStates.empty())
+        "OK",
+        "Error",
+        "IDLE",
+        "INITIALIZING_DEVICE",
+        "INITIALIZED",
+        "BINDING",
+        "BOUND",
+        "CONNECTING",
+        "DEVICE_READY",
+        "INITIALIZING_TASK",
+        "READY",
+        "RUNNING",
+        "RESETTING_TASK",
+        "RESETTING_DEVICE",
+        "EXITING"
+    }
+};
+
+static array<string, 12> transitionNames =
+{
+    {
+        "AUTO",
+        "INIT_DEVICE",
+        "COMPLETE_INIT",
+        "BIND",
+        "CONNECT",
+        "INIT_TASK",
+        "RUN",
+        "STOP",
+        "RESET_TASK",
+        "RESET_DEVICE",
+        "END",
+        "ERROR_FOUND"
+    }
+};
+
+static map<string, State> stateNumbers =
+{
+    { "OK",                  State::Ok },
+    { "Error",               State::Error },
+    { "IDLE",                State::Idle },
+    { "INITIALIZING_DEVICE", State::InitializingDevice },
+    { "INITIALIZED",         State::Initialized },
+    { "BINDING",             State::Binding },
+    { "BOUND",               State::Bound },
+    { "CONNECTING",          State::Connecting },
+    { "DEVICE_READY",        State::DeviceReady },
+    { "INITIALIZING_TASK",   State::InitializingTask },
+    { "READY",               State::Ready },
+    { "RUNNING",             State::Running },
+    { "RESETTING_TASK",      State::ResettingTask },
+    { "RESETTING_DEVICE",    State::ResettingDevice },
+    { "EXITING",             State::Exiting }
+};
+
+static map<string, Transition> transitionNumbers =
+{
+    { "AUTO",          Transition::Auto },
+    { "INIT_DEVICE",   Transition::InitDevice },
+    { "COMPLETE_INIT", Transition::CompleteInit },
+    { "BIND",          Transition::Bind },
+    { "CONNECT",       Transition::Connect },
+    { "INIT_TASK",     Transition::InitTask },
+    { "RUN",           Transition::Run },
+    { "STOP",          Transition::Stop },
+    { "RESET_TASK",    Transition::ResetTask },
+    { "RESET_DEVICE",  Transition::ResetDevice },
+    { "END",           Transition::End },
+    { "ERROR_FOUND",   Transition::ErrorFound }
+};
+
+// defining the boost MSM state machine
+struct Machine_ : public state_machine_def<Machine_>
+{
+  public:
+    Machine_()
+        : fLastTransitionResult(true)
+        , fNewStatePending(false)
+        , fWorkOngoing(false)
+    {}
+
+    virtual ~Machine_() {}
+
+    // initial states
+    using initial_state = bmpl::vector<IDLE_S, OK_S>;
+
+    template<typename Transition, typename FSM>
+    void on_entry(Transition const&, FSM& /* fsm */)
+    {
+        LOG(state) << "Starting FairMQ state machine --> IDLE";
+        fState = State::Idle;
+    }
+
+    template<typename Transition, typename FSM>
+    void on_exit(Transition const&, FSM& /*fsm*/)
+    {
+        LOG(state) << "Exiting FairMQ state machine";
+    }
+
+    struct DefaultFct
+    {
+        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
+        void operator()(EVT const& e, FSM& fsm, SourceState& /* ss */, TargetState& ts)
         {
-            fNewState.wait(lock);
+            fsm.fNewState = ts.Type();
+            fsm.fLastTransitionResult = true;
+            fsm.CallNewTransitionCallbacks(e.Type());
+            fsm.fNewStatePending = true;
+            fsm.fNewStatePendingCV.notify_all();
         }
+    };
 
-        State lastState;
+    struct transition_table : bmpl::vector<
+        //  Start                  Transition       Next                   Action      Guard
+        Row<IDLE_S,                END_E,           EXITING_S,             DefaultFct, none>,
+        Row<IDLE_S,                INIT_DEVICE_E,   INITIALIZING_DEVICE_S, DefaultFct, none>,
 
-        if (fNextStates.front() == State::Error)
+        Row<INITIALIZING_DEVICE_S, COMPLETE_INIT_E, INITIALIZED_S,         DefaultFct, none>,
+        Row<INITIALIZED_S,         BIND_E,          BINDING_S,             DefaultFct, none>,
+        Row<INITIALIZED_S,         RESET_DEVICE_E,  RESETTING_DEVICE_S,    DefaultFct, none>,
+
+        Row<BINDING_S,             AUTO_E,          BOUND_S,               DefaultFct, none>,
+        Row<BOUND_S,               CONNECT_E,       CONNECTING_S,          DefaultFct, none>,
+        Row<BOUND_S,               RESET_DEVICE_E,  RESETTING_DEVICE_S,    DefaultFct, none>,
+
+        Row<CONNECTING_S,          AUTO_E,          DEVICE_READY_S,        DefaultFct, none>,
+        Row<DEVICE_READY_S,        INIT_TASK_E,     INITIALIZING_TASK_S,   DefaultFct, none>,
+        Row<DEVICE_READY_S,        RESET_DEVICE_E,  RESETTING_DEVICE_S,    DefaultFct, none>,
+
+        Row<INITIALIZING_TASK_S,   AUTO_E,          READY_S,               DefaultFct, none>,
+
+        Row<READY_S,               RUN_E,           RUNNING_S,             DefaultFct, none>,
+        Row<READY_S,               RESET_TASK_E,    RESETTING_TASK_S,      DefaultFct, none>,
+
+        Row<RUNNING_S,             STOP_E,          READY_S,               DefaultFct, none>,
+
+        Row<RESETTING_TASK_S,      AUTO_E,          DEVICE_READY_S,        DefaultFct, none>,
+        Row<RESETTING_DEVICE_S,    AUTO_E,          IDLE_S,                DefaultFct, none>,
+
+        Row<OK_S,                  ERROR_FOUND_E,   ERROR_S,               DefaultFct,   none>> {};
+
+    void CallStateChangeCallbacks(const State state) const
+    {
+        if (!fStateChangeSignal.empty()) {
+            fStateChangeSignal(state);
+        }
+    }
+
+    void CallStateHandler(const State state) const
+    {
+        if (!fStateHandleSignal.empty()) {
+            fStateHandleSignal(state);
+        }
+    }
+
+    void CallNewTransitionCallbacks(const Transition transition) const
+    {
+        if (!fNewTransitionSignal.empty()) {
+            fNewTransitionSignal(transition);
+        }
+    }
+
+
+    atomic<State> fState;
+    atomic<State> fNewState;
+    atomic<bool> fLastTransitionResult;
+
+    mutex fStateMtx;
+    atomic<bool> fNewStatePending;
+    atomic<bool> fWorkOngoing;
+    condition_variable fNewStatePendingCV;
+    condition_variable fWorkDoneCV;
+
+    boost::signals2::signal<void(const State)> fStateChangeSignal;
+    boost::signals2::signal<void(const State)> fStateHandleSignal;
+    boost::signals2::signal<void(const Transition)> fNewTransitionSignal;
+    unordered_map<string, boost::signals2::connection> fStateChangeSignalsMap;
+    unordered_map<string, boost::signals2::connection> fNewTransitionSignalsMap;
+
+    void ProcessWork()
+    {
+        bool stop = false;
+
+        while (!stop) {
+            {
+                unique_lock<mutex> lock(fStateMtx);
+
+                while (!fNewStatePending) {
+                    fNewStatePendingCV.wait_for(lock, chrono::milliseconds(100));
+                }
+
+                LOG(state) << fState << " ---> " << fNewState;
+                fState = static_cast<State>(fNewState);
+                fNewStatePending = false;
+                fWorkOngoing = true;
+
+                if (fState == State::Exiting || fState == State::Error) {
+                    stop = true;
+                }
+            }
+
+            CallStateChangeCallbacks(fState);
+            CallStateHandler(fState);
+
+            {
+                lock_guard<mutex> lock(fStateMtx);
+                fWorkOngoing = false;
+                fWorkDoneCV.notify_one();
+            }
+        }
+    }
+
+    // replaces the default no-transition response.
+    template<typename FSM, typename Transition>
+    void no_transition(Transition const& t, FSM& fsm, int state)
+    {
+        using RecursiveStt = typename recursive_get_transition_table<FSM>::type;
+        using AllStates = typename generate_state_set<RecursiveStt>::type;
+
+        string stateName;
+
+        bmpl::for_each<AllStates, wrap<bmpl::placeholders::_1>>(get_state_name<RecursiveStt>(stateName, state));
+
+        stateName = boost::core::demangle(stateName.c_str());
+        size_t pos = stateName.rfind(":");
+        stateName = stateName.substr(pos + 1);
+        size_t pos2 = stateName.rfind("_");
+        stateName = stateName.substr(0, pos2);
+
+        if (stateName != "OK") {
+            LOG(state) << "No transition from state " << stateName << " on transition " << t.Name();
+        }
+        fsm.fLastTransitionResult = false;
+    }
+}; // Machine_
+
+using FairMQFSM = state_machine<Machine_>;
+
+} // namespace fsm
+} // namespace mq
+} // namespace fair
+
+using namespace fair::mq::fsm;
+using namespace fair::mq;
+
+StateMachine::StateMachine() : fFsm(new FairMQFSM) {}
+void StateMachine::Start() { static_pointer_cast<FairMQFSM>(fFsm)->start(); }
+StateMachine::~StateMachine() { static_pointer_cast<FairMQFSM>(fFsm)->stop(); }
+
+bool StateMachine::ChangeState(const Transition transition)
+try {
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    lock_guard<mutex> lock(fsm->fStateMtx);
+    if (!static_cast<bool>(fsm->fNewStatePending) || transition == Transition::ErrorFound) {
+        switch (transition) {
+            case Transition::Auto:
+                fsm->process_event(AUTO_E());
+                return fsm->fLastTransitionResult;
+            case Transition::InitDevice:
+                fsm->process_event(INIT_DEVICE_E());
+                return fsm->fLastTransitionResult;
+            case Transition::CompleteInit:
+                fsm->process_event(COMPLETE_INIT_E());
+                return fsm->fLastTransitionResult;
+            case Transition::Bind:
+                fsm->process_event(BIND_E());
+                return fsm->fLastTransitionResult;
+            case Transition::Connect:
+                fsm->process_event(CONNECT_E());
+                return fsm->fLastTransitionResult;
+            case Transition::InitTask:
+                fsm->process_event(INIT_TASK_E());
+                return fsm->fLastTransitionResult;
+            case Transition::Run:
+                fsm->process_event(RUN_E());
+                return fsm->fLastTransitionResult;
+            case Transition::Stop:
+                fsm->process_event(STOP_E());
+                return fsm->fLastTransitionResult;
+            case Transition::ResetDevice:
+                fsm->process_event(RESET_DEVICE_E());
+                return fsm->fLastTransitionResult;
+            case Transition::ResetTask:
+                fsm->process_event(RESET_TASK_E());
+                return fsm->fLastTransitionResult;
+            case Transition::End:
+                fsm->process_event(END_E());
+                return fsm->fLastTransitionResult;
+            case Transition::ErrorFound:
+                fsm->process_event(ERROR_FOUND_E());
+                return fsm->fLastTransitionResult;
+            default:
+                LOG(error) << "Requested unsupported state transition: " << transition << endl;
+                return false;
+             }
+    } else {
+        LOG(state) << "Transition " << transitionNames.at(static_cast<int>(transition)) << " incoming, but another state transition is already ongoing.";
+        return false;
+    }
+} catch (exception& e) {
+    LOG(error) << "Exception in StateMachine::ChangeState(): " << e.what();
+    return false;
+}
+
+void StateMachine::SubscribeToStateChange(const string& key, function<void(const State)> callback)
+{
+    static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.insert({key, static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignal.connect(callback)});
+}
+
+void StateMachine::UnsubscribeFromStateChange(const string& key)
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (fsm->fStateChangeSignalsMap.count(key)) {
+        fsm->fStateChangeSignalsMap.at(key).disconnect();
+        fsm->fStateChangeSignalsMap.erase(key);
+    }
+}
+
+void StateMachine::HandleStates(function<void(const State)> callback)
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (fsm->fStateHandleSignal.empty()) {
+        fsm->fStateHandleSignal.connect(callback);
+    } else {
+        LOG(error) << "state handler is already set";
+    }
+}
+
+void StateMachine::StopHandlingStates()
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (!fsm->fStateHandleSignal.empty()) {
+        fsm->fStateHandleSignal.disconnect_all_slots();
+    }
+}
+
+void StateMachine::SubscribeToNewTransition(const string& key, function<void(const Transition)> callback)
+{
+    static_pointer_cast<FairMQFSM>(fFsm)->fNewTransitionSignalsMap.insert({key, static_pointer_cast<FairMQFSM>(fFsm)->fNewTransitionSignal.connect(callback)});
+}
+
+void StateMachine::UnsubscribeFromNewTransition(const string& key)
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (fsm->fNewTransitionSignalsMap.count(key)) {
+        fsm->fNewTransitionSignalsMap.at(key).disconnect();
+        fsm->fNewTransitionSignalsMap.erase(key);
+    }
+}
+
+State StateMachine::GetCurrentState() const { return static_pointer_cast<FairMQFSM>(fFsm)->fState; }
+string StateMachine::GetCurrentStateName() const { return GetStateName(static_pointer_cast<FairMQFSM>(fFsm)->fState); }
+
+bool StateMachine::NewStatePending() const { return static_cast<bool>(static_pointer_cast<FairMQFSM>(fFsm)->fNewStatePending); }
+void StateMachine::WaitForPendingState() const
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    unique_lock<mutex> lock(fsm->fStateMtx);
+    fsm->fNewStatePendingCV.wait(lock, [&]{ return static_cast<bool>(fsm->fNewStatePending); });
+}
+bool StateMachine::WaitForPendingStateFor(const int durationInMs) const
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    unique_lock<mutex> lock(fsm->fStateMtx);
+    return fsm->fNewStatePendingCV.wait_for(lock, std::chrono::milliseconds(durationInMs), [&]{ return static_cast<bool>(fsm->fNewStatePending); });
+}
+
+void StateMachine::ProcessWork()
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+
+    try {
+        fsm->CallStateChangeCallbacks(State::Idle);
+        fsm->ProcessWork();
+    } catch(...) {
         {
-            // advance error FSM
-            lastState = fErrorState;
-            fErrorState = fNextStates.front();
-            fNextStates.pop_front();
-            LOG(error) << "Entering " << fErrorState << " state (orthogonal error state machine)";
+            lock_guard<mutex> lock(fsm->fStateMtx);
+            fsm->fState = State::Error;
+            fsm->CallStateChangeCallbacks(State::Error);
+            fsm->fWorkOngoing = false;
+            fsm->fWorkDoneCV.notify_one();
         }
-        else
-        {
-            // advance regular FSM
-            lastState = fState;
-            fState = fNextStates.front();
-            fNextStates.pop_front();
-            LOG(state) << "Entering " << fState << " state";
-        }
-        lock.unlock();
-
-        fCallbacks.Emit<StateChange, State>(fState, lastState);
-
-        lock.lock();
-        if (fState == State::Exiting || fErrorState == State::Error) break;
+        ChangeState(Transition::ErrorFound);
+        throw;
     }
-
-    LOG(state) << "Exiting FairMQ state machine";
 }
 
-auto StateMachine::ChangeState(StateTransition transition) -> void
-{
-    State lastState;
-
-    std::unique_lock<std::mutex> lock{fMutex};
-
-    if (transition == StateTransition::ErrorFound)
-    {
-        lastState = fErrorState;
-    }
-    else if (fNextStates.empty())
-    {
-        lastState = fState;
-    }
-    else
-    {
-        lastState = fNextStates.back();
-    }
-
-    const State nextState{Transition(lastState, transition)};
-    fNextStates.push_back(nextState);
-    lock.unlock();
-
-    fCallbacks.Emit<StateQueued, State>(nextState, lastState);
-    fNewState.notify_one();
-}
-    
-auto StateMachine::Transition(const State currentState, const StateTransition transition) -> State
-{
-    switch (currentState) {
-        case State::Idle:
-            if (transition == StateTransition::InitDevice ) return State::InitializingDevice;
-            if (transition == StateTransition::End        ) return State::Exiting;
-            break;
-        case State::InitializingDevice:
-            if (transition == StateTransition::Automatic  ) return State::DeviceReady;
-            break;
-        case State::DeviceReady:
-            if (transition == StateTransition::InitTask   ) return State::InitializingTask;
-            if (transition == StateTransition::ResetDevice) return State::ResettingDevice;
-            break;
-        case State::InitializingTask:
-            if (transition == StateTransition::Automatic  ) return State::Ready;
-            break;
-        case State::Ready:
-            if (transition == StateTransition::Run        ) return State::Running;
-            if (transition == StateTransition::ResetTask  ) return State::ResettingTask;
-            break;
-        case State::Running:
-            if (transition == StateTransition::Stop       ) return State::Ready;
-            break;
-        case State::ResettingTask:
-            if (transition == StateTransition::Automatic  ) return State::DeviceReady;
-            break;
-        case State::ResettingDevice:
-            if (transition == StateTransition::Automatic  ) return State::Idle;
-            break;
-        case State::Exiting:
-            break;
-        case State::Ok:
-            if (transition == StateTransition::ErrorFound ) return State::Error;
-            break;
-        case State::Error:
-            break;
-    }
-    throw IllegalTransition{tools::ToString("No transition ", transition, " from state ", currentState, ".")};
-}
-
-StateMachine::StateMachine()
-: fState{State::Idle}
-, fErrorState{State::Ok}
-{
-}
-
-auto StateMachine::Reset() -> void
-{
-    std::unique_lock<std::mutex> lock{fMutex};
-
-    fState = State::Idle;
-    fErrorState = State::Ok;
-    fNextStates.clear();
-}
-
-auto StateMachine::NextStatePending() -> bool
-{
-    std::unique_lock<std::mutex> lock{fMutex};
-
-    return fNextStates.size() > 0;
-}
+string StateMachine::GetStateName(const State state) { return stateNames.at(static_cast<int>(state)); }
+string StateMachine::GetTransitionName(const Transition transition) { return transitionNames.at(static_cast<int>(transition)); }
+State StateMachine::GetState(const string& state) { return stateNumbers.at(state); }
+Transition StateMachine::GetTransition(const string& transition) { return transitionNumbers.at(transition); }

fairmq/StateMachine.h

@@ -1,132 +1,107 @@
 /********************************************************************************
- *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#ifndef FAIR_MQ_STATEMACHINE_H
-#define FAIR_MQ_STATEMACHINE_H
+#ifndef FAIRMQSTATEMACHINE_H_
+#define FAIRMQSTATEMACHINE_H_
 
-#include <utility>
-#include <FairMQLogger.h>
-#include <fairmq/Tools.h>
-#include <fairmq/EventManager.h>
-#include <deque>
+#include "FairMQLogger.h"
+
+#include <string>
+#include <memory>
 #include <functional>
-#include <mutex>
 #include <condition_variable>
-#include <thread>
-#include <unordered_map>
+#include <ostream>
+#include <queue>
+#include <mutex>
+#include <stdexcept>
 
 namespace fair
 {
 namespace mq
 {
 
-/**
- * @class StateMachine StateMachine.h <fairmq/StateMachine.h>
- * @brief Implements the state machine for FairMQ devices
- *
- * See https://github.com/FairRootGroup/FairRoot/blob/dev/fairmq/docs/Device.md#13-state-machine
- */
+enum class State : int
+{
+    Ok,
+    Error,
+    Idle,
+    InitializingDevice,
+    Initialized,
+    Binding,
+    Bound,
+    Connecting,
+    DeviceReady,
+    InitializingTask,
+    Ready,
+    Running,
+    ResettingTask,
+    ResettingDevice,
+    Exiting
+};
+
+enum class Transition : int
+{
+    Auto,
+    InitDevice,
+    CompleteInit,
+    Bind,
+    Connect,
+    InitTask,
+    Run,
+    Stop,
+    ResetTask,
+    ResetDevice,
+    End,
+    ErrorFound
+};
+
 class StateMachine
 {
   public:
-    enum class State : int
-    {
-        Ok,
-        Error,
-        Idle,
-        InitializingDevice,
-        DeviceReady,
-        InitializingTask,
-        Ready,
-        Running,
-        ResettingTask,
-        ResettingDevice,
-        Exiting
-    };
-
-    enum class StateTransition : int // transition event between States
-    {
-        InitDevice,
-        InitTask,
-        Run,
-        Stop,
-        ResetTask,
-        ResetDevice,
-        End,
-        ErrorFound,
-        Automatic
-    };
-
-    /// @brief Convert string to State
-    /// @param state to convert
-    /// @return State enum entry
-    /// @throw std::out_of_range if a string cannot be resolved to a State
-    static auto ToState(const std::string& state) -> State { return fkStateStrMap.at(state); }
-
-    /// @brief Convert string to StateTransition
-    /// @param transition to convert
-    /// @return StateTransition enum entry
-    /// @throw std::out_of_range if a string cannot be resolved to a StateTransition
-    static auto ToStateTransition(const std::string& transition) -> StateTransition { return fkStateTransitionStrMap.at(transition); }
-
-    /// @brief Convert State to string
-    /// @param state to convert
-    /// @return string representation of State enum entry
-    static auto ToStr(State state) -> std::string { return fkStrStateMap.at(state); }
-
-    /// @brief Convert StateTransition to string
-    /// @param transition to convert
-    /// @return string representation of StateTransition enum entry
-    static auto ToStr(StateTransition transition) -> std::string { return fkStrStateTransitionMap.at(transition); }
-
-    friend auto operator<<(std::ostream& os, const State& state) -> std::ostream& { return os << ToStr(state); }
-    friend auto operator<<(std::ostream& os, const StateTransition& transition) -> std::ostream& { return os << ToStr(transition); }
-
     StateMachine();
+    virtual ~StateMachine();
 
-    struct IllegalTransition : std::runtime_error { using std::runtime_error::runtime_error; };
+    bool ChangeState(const Transition transition);
+    bool ChangeState(const std::string& transition) { return ChangeState(GetTransition(transition)); }
 
-    struct StateChange : Event<State> {};
-    struct StateQueued : Event<State> {};
-    auto SubscribeToStateChange(const std::string& subscriber, std::function<void(typename StateChange::KeyType newState, State lastState)> callback) -> void { fCallbacks.Subscribe<StateChange, State>(subscriber, callback); }
-    auto UnsubscribeFromStateChange(const std::string& subscriber) -> void { fCallbacks.Unsubscribe<StateChange, State>(subscriber); }
-    auto SubscribeToStateQueued(const std::string& subscriber, std::function<void(typename StateQueued::KeyType newState, State lastState)> callback) -> void { fCallbacks.Subscribe<StateQueued, State>(subscriber, callback); }
-    auto UnsubscribeFromStateQueued(const std::string& subscriber) -> void { fCallbacks.Unsubscribe<StateQueued, State>(subscriber); }
+    void SubscribeToStateChange(const std::string& key, std::function<void(const State)> callback);
+    void UnsubscribeFromStateChange(const std::string& key);
 
-    auto GetCurrentState() const -> State { std::lock_guard<std::mutex> lock{fMutex}; return fState; }
-    auto GetCurrentErrorState() const -> State { std::lock_guard<std::mutex> lock{fMutex}; return fErrorState; }
-    auto GetLastQueuedState() const -> State { std::lock_guard<std::mutex> lock{fMutex}; return fNextStates.back(); }
+    void HandleStates(std::function<void(const State)> callback);
+    void StopHandlingStates();
 
-    auto ChangeState(StateTransition transition) -> void;
+    void SubscribeToNewTransition(const std::string& key, std::function<void(const Transition)> callback);
+    void UnsubscribeFromNewTransition(const std::string& key);
 
-    auto Run() -> void;
-    auto Reset() -> void;
+    bool NewStatePending() const;
+    void WaitForPendingState() const;
+    bool WaitForPendingStateFor(const int durationInMs) const;
 
-    auto NextStatePending() -> bool;
+    State GetCurrentState() const;
+    std::string GetCurrentStateName() const;
+
+    void Start();
+
+    void ProcessWork();
+
+    static std::string GetStateName(const State);
+    static std::string GetTransitionName(const Transition);
+    static State GetState(const std::string& state);
+    static Transition GetTransition(const std::string& transition);
 
   private:
-    State fState;
-    State fErrorState;
-    std::deque<State> fNextStates;
-    EventManager fCallbacks;
+    std::shared_ptr<void> fFsm;
+};
 
-    static const std::unordered_map<std::string, State> fkStateStrMap;
-    static const std::unordered_map<State, std::string, tools::HashEnum<State>> fkStrStateMap;
-    static const std::unordered_map<std::string, StateTransition> fkStateTransitionStrMap;
-    static const std::unordered_map<StateTransition, std::string, tools::HashEnum<StateTransition>> fkStrStateTransitionMap;
+inline std::ostream& operator<<(std::ostream& os, const State& state) { return os << StateMachine::GetStateName(state); }
+inline std::ostream& operator<<(std::ostream& os, const Transition& transition) { return os << StateMachine::GetTransitionName(transition); }
 
-    mutable std::mutex fMutex;
-    std::condition_variable fNewState;
+} // namespace mq
+} // namespace fair
 
-    static auto Transition(const State currentState, const StateTransition transition) -> State;
-}; /* class StateMachine */
-
-} /* namespace mq */
-} /* namespace fair */
-
-#endif /* FAIR_MQ_STATEMACHINE_H */
+#endif /* FAIRMQSTATEMACHINE_H_ */

fairmq/apply-clang-format.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-find . -type f \( -iname "*.h" ! -iname "*.pb.h" ! -iname "*LinkDef.h" -o -iname "*.cxx" -o -iname "*.tpl" \) -execdir clang-format -i {} \;

fairmq/devices/FairMQBenchmarkSampler.cxx

@@ -54,7 +54,7 @@ void FairMQBenchmarkSampler::Run()
 
     fair::mq::tools::RateLimiter rateLimiter(fMsgRate);
 
-    while (CheckCurrentState(RUNNING))
+    while (!NewStatePending())
     {
         if (fMultipart)
         {

fairmq/devices/FairMQMerger.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**
@@ -60,7 +60,7 @@ void FairMQMerger::Run()
 
     if (fMultipart)
     {
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             poller->Poll(100);
 
@@ -91,7 +91,7 @@ void FairMQMerger::Run()
     }
     else
     {
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             poller->Poll(100);
 

fairmq/devices/FairMQMerger.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/devices/FairMQMultiplier.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/devices/FairMQMultiplier.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/devices/FairMQProxy.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**
@@ -41,7 +41,7 @@ void FairMQProxy::Run()
 {
     if (fMultipart)
     {
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             FairMQParts payload;
             if (Receive(payload, fInChannelName) >= 0)
@@ -61,7 +61,7 @@ void FairMQProxy::Run()
     }
     else
     {
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             unique_ptr<FairMQMessage> payload(fTransportFactory->CreateMessage());
             if (Receive(payload, fInChannelName) >= 0)

fairmq/devices/FairMQProxy.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/devices/FairMQSink.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**
@@ -57,7 +57,7 @@ class FairMQSink : public FairMQDevice//, public OutputPolicy
         LOG(info) << "Starting the benchmark and expecting to receive " << fMaxIterations << " messages.";
         auto tStart = std::chrono::high_resolution_clock::now();
 
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             if (fMultipart)
             {

fairmq/devices/FairMQSplitter.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/devices/FairMQSplitter.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/nanomsg/FairMQPollerNN.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/nanomsg/FairMQPollerNN.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/nanomsg/FairMQSocketNN.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/nanomsg/FairMQSocketNN.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/ofi/CMakeLists.txt

@@ -1,12 +0,0 @@
-################################################################################
-#    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
-#                                                                              #
-#              This software is distributed under the terms of the             #
-#              GNU Lesser General Public Licence (LGPL) version 3,             #
-#                  copied verbatim in the file "LICENSE"                       #
-################################################################################
-
-protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS Control.proto)
-
-add_library(OfiTransport OBJECT ${PROTO_SRCS} ${PROTO_HDRS})
-target_include_directories(OfiTransport PRIVATE $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_INCLUDE_DIRECTORIES>)

fairmq/ofi/Context.cxx

@@ -10,21 +10,17 @@
 #include <fairmq/Tools.h>
 #include <FairMQLogger.h>
 
+#include <asiofi/version.hpp>
 #include <arpa/inet.h>
 #include <boost/version.hpp>
+#include <cassert>
 #include <cstring>
-#include <google/protobuf/stubs/common.h>
 #include <memory>
 #include <netinet/in.h>
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_errno.h>
 #include <regex>
 #include <string>
 #include <string.h>
 #include <sys/socket.h>
-#include <zmq.h>
 
 namespace fair
 {
@@ -35,20 +31,13 @@ namespace ofi
 
 using namespace std;
 
-Context::Context(int numberIoThreads)
-    : fOfiDomain(nullptr)
-    , fOfiFabric(nullptr)
-    , fOfiInfo(nullptr)
-    , fOfiAddressVector(nullptr)
-    , fOfiEventQueue(nullptr)
-    , fZmqContext(zmq_ctx_new())
-    , fIoWork(fIoContext)
+Context::Context(FairMQTransportFactory& sendFactory,
+                 FairMQTransportFactory& receiveFactory,
+                 int numberIoThreads)
+    : fIoWork(fIoContext)
+    , fReceiveFactory(receiveFactory)
+    , fSendFactory(sendFactory)
 {
-    if (!fZmqContext)
-        throw ContextError{tools::ToString("Failed creating zmq context, reason: ", zmq_strerror(errno))};
-
-    GOOGLE_PROTOBUF_VERIFY_VERSION;
-
     InitThreadPool(numberIoThreads);
 }
 
@@ -58,257 +47,27 @@ auto Context::InitThreadPool(int numberIoThreads) -> void
 
     for (int i = 1; i <= numberIoThreads; ++i) {
         fThreadPool.emplace_back([&, i, numberIoThreads]{
-            LOG(debug) << "I/O thread #" << i << "/" << numberIoThreads << " started";
+            LOG(debug) << "OFI transport: I/O thread #" << i << " of " << numberIoThreads << " started";
             fIoContext.run();
-            LOG(debug) << "I/O thread #" << i << "/" << numberIoThreads << " stopped";
+            LOG(debug) << "OFI transport: I/O thread #" << i << " of " << numberIoThreads << " stopped";
         });
     }
 }
 
-Context::~Context()
+auto Context::Reset() -> void
 {
     fIoContext.stop();
+}
+
+Context::~Context()
+{
     for (auto& thread : fThreadPool)
         thread.join();
-
-    if (zmq_ctx_term(fZmqContext) != 0)
-        LOG(error) << "Failed closing zmq context, reason: " << zmq_strerror(errno);
-
-	if (fOfiEventQueue) {
-        auto ret = fi_close(&fOfiEventQueue->fid);
-        if (ret != FI_SUCCESS)
-            LOG(error) << "Failed closing ofi event queue, reason: " << fi_strerror(ret);
-    }
-
-	if (fOfiAddressVector) {
-        auto ret = fi_close(&fOfiAddressVector->fid);
-        if (ret != FI_SUCCESS)
-            LOG(error) << "Failed closing ofi address vector, reason: " << fi_strerror(ret);
-    }
-
-	if (fOfiDomain) {
-        auto ret = fi_close(&fOfiDomain->fid);
-        if (ret != FI_SUCCESS)
-            LOG(error) << "Failed closing ofi domain, reason: " << fi_strerror(ret);
-    }
-
-	if (fOfiFabric) {
-        auto ret = fi_close(&fOfiFabric->fid);
-        if (ret != FI_SUCCESS)
-            LOG(error) << "Failed closing ofi fabric, reason: " << fi_strerror(ret);
-    }
 }
 
-auto Context::GetZmqVersion() const -> string
+auto Context::GetAsiofiVersion() const -> string
 {
-    int major, minor, patch;
-    zmq_version(&major, &minor, &patch);
-    return tools::ToString(major, ".", minor, ".", patch);
-}
-
-auto Context::GetOfiApiVersion() const -> string
-{
-    // Disable for now, does not compile with gcc 4.9.2 debian jessie
-    //auto ofi_version{fi_version()};
-    //return tools::ToString(FI_MAJOR(ofi_version), ".", FI_MINOR(ofi_version));
-    return "unknown";
-}
-
-auto Context::GetPbVersion() const -> string
-{
-    return google::protobuf::internal::VersionString(GOOGLE_PROTOBUF_VERSION);
-}
-
-auto Context::GetBoostVersion() const -> std::string
-{
-    return tools::ToString(BOOST_VERSION / 100000, ".", BOOST_VERSION / 100 % 1000, ".", BOOST_VERSION % 100);
-}
-
-auto Context::InitOfi(ConnectionType type, Address addr) -> void
-{
-    if (!fOfiInfo) {
-        sockaddr_in* sa = static_cast<sockaddr_in*>(malloc(sizeof(sockaddr_in)));
-        addr.Port = 0;
-        auto sa2 = ConvertAddress(addr);
-        memcpy(sa, &sa2, sizeof(sockaddr_in));
-
-        // Prepare fi_getinfo query
-        unique_ptr<fi_info, void(*)(fi_info*)> ofi_hints(fi_allocinfo(), fi_freeinfo);
-        ofi_hints->caps = FI_MSG;
-        //ofi_hints->mode = FI_CONTEXT;
-        ofi_hints->addr_format = FI_SOCKADDR_IN;
-        if (addr.Protocol == "tcp") {
-            ofi_hints->fabric_attr->prov_name = strdup("sockets");
-        } else if (addr.Protocol == "verbs") {
-            ofi_hints->fabric_attr->prov_name = strdup("verbs;ofi_rxm");
-        }
-        ofi_hints->ep_attr->type = FI_EP_RDM;
-        //ofi_hints->domain_attr->mr_mode = FI_MR_BASIC | FI_MR_SCALABLE;
-        ofi_hints->domain_attr->threading = FI_THREAD_SAFE;
-        ofi_hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
-        ofi_hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
-        ofi_hints->tx_attr->op_flags = FI_COMPLETION;
-        ofi_hints->rx_attr->op_flags = FI_COMPLETION;
-        if (type == ConnectionType::Bind) {
-            ofi_hints->src_addr = sa;
-            ofi_hints->src_addrlen = sizeof(sockaddr_in);
-            ofi_hints->dest_addr = nullptr;
-            ofi_hints->dest_addrlen = 0;
-        } else {
-            ofi_hints->src_addr = nullptr;
-            ofi_hints->src_addrlen = 0;
-            ofi_hints->dest_addr = sa;
-            ofi_hints->dest_addrlen = sizeof(sockaddr_in);
-        }
-
-        // Query fi_getinfo for fabric to use
-        auto res = fi_getinfo(FI_VERSION(1, 5), nullptr, nullptr, 0, ofi_hints.get(), &fOfiInfo);
-        if (res != 0) throw ContextError{tools::ToString("Failed querying fi_getinfo, reason: ", fi_strerror(res))};
-        if (!fOfiInfo) throw ContextError{"Could not find any ofi compatible fabric."};
-
-        // for(auto cursor{ofi_info}; cursor->next != nullptr; cursor = cursor->next) {
-            // LOG(debug) << fi_tostr(fOfiInfo, FI_TYPE_INFO);
-        // }
-        //
-    } else {
-        LOG(debug) << "Ofi info already queried. Skipping.";
-    }
-
-    OpenOfiFabric();
-//    OpenOfiEventQueue();
-    OpenOfiDomain();
-    OpenOfiAddressVector();
-}
-
-auto Context::OpenOfiFabric() -> void
-{
-    if (!fOfiFabric) {
-        assert(fOfiInfo);
-        fi_context ctx;
-        auto ret = fi_fabric(fOfiInfo->fabric_attr, &fOfiFabric, &ctx);
-        if (ret != FI_SUCCESS)
-            throw ContextError{tools::ToString("Failed opening ofi fabric, reason: ", fi_strerror(ret))};
-    } else {
-        // TODO Check, if requested fabric matches existing one.
-        // TODO Decide, if we want to support more than one fabric simultaneously.
-        LOG(debug) << "Ofi fabric already opened. Skipping.";
-    }
-}
-
-auto Context::OpenOfiDomain() -> void
-{
-    if (!fOfiDomain) {
-        assert(fOfiInfo);
-        assert(fOfiFabric);
-        fi_context ctx;
-        auto ret = fi_domain(fOfiFabric, fOfiInfo, &fOfiDomain, &ctx);
-        if (ret != FI_SUCCESS)
-            throw ContextError{tools::ToString("Failed opening ofi domain, reason: ", fi_strerror(ret))};
-    } else {
-        LOG(debug) << "Ofi domain already opened. Skipping.";
-    }
-}
-
-auto Context::OpenOfiEventQueue() -> void
-{
-    fi_eq_attr eqAttr = {100, 0, FI_WAIT_UNSPEC, 0, nullptr};
-    // size_t               size;      [> # entries for EQ <]
-    // uint64_t             flags;     [> operation flags <]
-    // enum fi_wait_obj     wait_obj;  [> requested wait object <]
-    // int                  signaling_vector; [> interrupt affinity <]
-    // struct fid_wait     *wait_set;  [> optional wait set <]
-    fi_context ctx;
-    auto ret = fi_eq_open(fOfiFabric, &eqAttr, &fOfiEventQueue, &ctx);
-    if (ret != FI_SUCCESS)
-        throw ContextError{tools::ToString("Failed opening ofi event queue, reason: ", fi_strerror(ret))};
-}
-
-auto Context::OpenOfiAddressVector() -> void
-{
-    if (!fOfiAddressVector) {
-        assert(fOfiDomain);
-        fi_av_attr attr = {fOfiInfo->domain_attr->av_type, 0, 1000, 0, nullptr, nullptr, 0};
-        // enum fi_av_type  type;        [> type of AV <]
-        // int              rx_ctx_bits; [> address bits to identify rx ctx <]
-        // size_t           count;       [> # entries for AV <]
-        // size_t           ep_per_node; [> # endpoints per fabric address <]
-        // const char       *name;       [> system name of AV <]
-        // void             *map_addr;   [> base mmap address <]
-        // uint64_t         flags;       [> operation flags <]
-        fi_context ctx;
-        auto ret = fi_av_open(fOfiDomain, &attr, &fOfiAddressVector, &ctx);
-        if (ret != FI_SUCCESS)
-            throw ContextError{tools::ToString("Failed opening ofi address vector, reason: ", fi_strerror(ret))};
-
-        //assert(fOfiEventQueue);
-        //ret = fi_av_bind(fOfiAddressVector, &fOfiEventQueue->fid, 0);
-        //if (ret != FI_SUCCESS)
-        //    throw ContextError{tools::ToString("Failed binding ofi event queue to address vector, reason: ", fi_strerror(ret))};
-    } else {
-        LOG(debug) << "Ofi address vector already opened. Skipping.";
-    }
-}
-
-auto Context::CreateOfiEndpoint() -> fid_ep*
-{
-    assert(fOfiDomain);
-    assert(fOfiInfo);
-    fid_ep* ep = nullptr;
-    fi_context ctx;
-    auto ret = fi_endpoint(fOfiDomain, fOfiInfo, &ep, &ctx);
-    if (ret != FI_SUCCESS)
-        throw ContextError{tools::ToString("Failed creating ofi endpoint, reason: ", fi_strerror(ret))};
-
-    //assert(fOfiEventQueue);
-    //ret = fi_ep_bind(ep, &fOfiEventQueue->fid, 0);
-    //if (ret != FI_SUCCESS)
-    //    throw ContextError{tools::ToString("Failed binding ofi event queue to ofi endpoint, reason: ", fi_strerror(ret))};
-
-    assert(fOfiAddressVector);
-    ret = fi_ep_bind(ep, &fOfiAddressVector->fid, 0);
-    if (ret != FI_SUCCESS)
-        throw ContextError{tools::ToString("Failed binding ofi address vector to ofi endpoint, reason: ", fi_strerror(ret))};
-
-    return ep;
-}
-
-auto Context::CreateOfiCompletionQueue(Direction dir) -> fid_cq*
-{
-    fid_cq* cq = nullptr;
-    fi_cq_attr attr = {0, 0, FI_CQ_FORMAT_DATA, FI_WAIT_UNSPEC, 0, FI_CQ_COND_NONE, nullptr};
-    if (dir == Direction::Receive) {
-        attr.size = fOfiInfo->rx_attr->size;
-    } else {
-        attr.size = fOfiInfo->tx_attr->size;
-    }
-	// size_t               size;      [> # entries for CQ <]
-	// uint64_t             flags;     [> operation flags <]
-	// enum fi_cq_format    format;    [> completion format <]
-	// enum fi_wait_obj     wait_obj;  [> requested wait object <]
-	// int                  signaling_vector; [> interrupt affinity <]
-	// enum fi_cq_wait_cond wait_cond; [> wait condition format <]
-	// struct fid_wait     *wait_set;  [> optional wait set <]
-    fi_context ctx;
-    auto ret = fi_cq_open(fOfiDomain, &attr, &cq, &ctx);
-    if (ret != FI_SUCCESS)
-        throw ContextError{tools::ToString("Failed creating ofi completion queue, reason: ", fi_strerror(ret))};
-    return cq;
-}
-
-auto Context::InsertAddressVector(sockaddr_in address) -> fi_addr_t
-{
-    fi_addr_t mappedAddress;
-    fi_context ctx;
-    auto ret = fi_av_insert(fOfiAddressVector, &address, 1, &mappedAddress, 0, &ctx);
-    if (ret != 1)
-        throw ContextError{tools::ToString("Failed to insert address into ofi address vector")};
-
-    return mappedAddress;
-}
-
-auto Context::AddressVectorLookup(fi_addr_t address) -> sockaddr_in
-{
-    throw ContextError("Not yet implemented");
+    return ASIOFI_VERSION;
 }
 
 auto Context::ConvertAddress(std::string address) -> Address
@@ -355,6 +114,16 @@ auto Context::VerifyAddress(const std::string& address) -> Address
     return addr;
 }
 
+auto Context::MakeReceiveMessage(size_t size) -> MessagePtr
+{
+    return fReceiveFactory.CreateMessage(size);
+}
+
+auto Context::MakeSendMessage(size_t size) -> MessagePtr
+{
+    return fSendFactory.CreateMessage(size);
+}
+
 } /* namespace ofi */
 } /* namespace mq */
 } /* namespace fair */

fairmq/ofi/Context.h

@@ -9,11 +9,16 @@
 #ifndef FAIR_MQ_OFI_CONTEXT_H
 #define FAIR_MQ_OFI_CONTEXT_H
 
-#include <boost/asio.hpp>
+#include <FairMQLogger.h>
+#include <FairMQTransportFactory.h>
+
+#include <asiofi/domain.hpp>
+#include <asiofi/fabric.hpp>
+#include <asiofi/info.hpp>
+#include <boost/asio/io_context.hpp>
 #include <memory>
 #include <netinet/in.h>
 #include <ostream>
-#include <rdma/fabric.h>
 #include <stdexcept>
 #include <string>
 #include <thread>
@@ -27,7 +32,20 @@ namespace ofi
 {
 
 enum class ConnectionType : bool { Bind, Connect };
-enum class Direction : bool { Receive, Transmit };
+
+struct Address {
+    std::string Protocol;
+    std::string Ip;
+    unsigned int Port;
+    friend auto operator<<(std::ostream& os, const Address& a) -> std::ostream&
+    {
+        return os << a.Protocol << "://" << a.Ip << ":" << a.Port;
+    }
+    friend auto operator==(const Address& lhs, const Address& rhs) -> bool
+    {
+        return (lhs.Protocol == rhs.Protocol) && (lhs.Ip == rhs.Ip) && (lhs.Port == rhs.Port);
+    }
+};
 
 /**
  * @class Context Context.h <fairmq/ofi/Context.h>
@@ -38,46 +56,30 @@ enum class Direction : bool { Receive, Transmit };
 class Context
 {
   public:
-    Context(int numberIoThreads = 2);
+    Context(FairMQTransportFactory& sendFactory,
+            FairMQTransportFactory& receiveFactory,
+            int numberIoThreads = 1);
     ~Context();
 
-    auto CreateOfiEndpoint() -> fid_ep*;
-    auto CreateOfiCompletionQueue(Direction dir) -> fid_cq*;
-    auto GetZmqVersion() const -> std::string;
-    auto GetOfiApiVersion() const -> std::string;
-    auto GetPbVersion() const -> std::string;
-    auto GetBoostVersion() const -> std::string;
-    auto GetZmqContext() const -> void* { return fZmqContext; }
-    auto GetIoContext() -> boost::asio::io_service& { return fIoContext; }
-    auto InsertAddressVector(sockaddr_in address) -> fi_addr_t;
-    auto AddressVectorLookup(fi_addr_t address) -> sockaddr_in;
-    struct Address {
-        std::string Protocol;
-        std::string Ip;
-        unsigned int Port;
-        friend auto operator<<(std::ostream& os, const Address& a) -> std::ostream& { return os << a.Protocol << "://" << a.Ip << ":" << a.Port; }
-    };
-    auto InitOfi(ConnectionType type, Address address) -> void;
+    auto GetAsiofiVersion() const -> std::string;
+    auto GetIoContext() -> boost::asio::io_context& { return fIoContext; }
     static auto ConvertAddress(std::string address) -> Address;
     static auto ConvertAddress(Address address) -> sockaddr_in;
     static auto ConvertAddress(sockaddr_in address) -> Address;
     static auto VerifyAddress(const std::string& address) -> Address;
+    auto Interrupt() -> void { LOG(debug) << "OFI transport: Interrupted (NOOP - not implemented)."; }
+    auto Resume() -> void { LOG(debug) << "OFI transport: Resumed (NOOP - not implemented)."; }
+    auto Reset() -> void;
+    auto MakeReceiveMessage(size_t size) -> MessagePtr;
+    auto MakeSendMessage(size_t size) -> MessagePtr;
 
   private:
-    void* fZmqContext;
-    fi_info* fOfiInfo;
-    fid_fabric* fOfiFabric;
-    fid_domain* fOfiDomain;
-    fid_av* fOfiAddressVector;
-    fid_eq* fOfiEventQueue;
-    boost::asio::io_service fIoContext;
-    boost::asio::io_service::work fIoWork;
+    boost::asio::io_context fIoContext;
+    boost::asio::io_context::work fIoWork;
     std::vector<std::thread> fThreadPool;
+    FairMQTransportFactory& fReceiveFactory;
+    FairMQTransportFactory& fSendFactory;
 
-    auto OpenOfiFabric() -> void;
-    auto OpenOfiEventQueue() -> void;
-    auto OpenOfiDomain() -> void;
-    auto OpenOfiAddressVector() -> void;
     auto InitThreadPool(int numberIoThreads) -> void;
 }; /* class Context */
 

fairmq/ofi/Control.proto

@@ -1,25 +0,0 @@
-syntax = "proto3";
-option optimize_for = SPEED;
-
-package fair.mq.ofi;
-
-message DataAddressAnnouncement {
-  uint32 ipv4 = 1; // in_addr_t from <netinet/in.h>
-  uint32 port = 2; // in_port_t from <netinet/in.h>
-}
-
-message PostBuffer {
-  uint64 size = 1; // buffer size (size_t)
-}
-
-message PostBufferAcknowledgement {
-  uint64 size = 1; // size_t
-}
-
-message ControlMessage {
-  oneof type {
-    DataAddressAnnouncement data_address_announcement = 1;
-    PostBuffer post_buffer = 2;
-    PostBufferAcknowledgement post_buffer_acknowledgement = 3;
-  }    
-}

fairmq/ofi/ControlMessages.h

@@ -0,0 +1,99 @@
+/********************************************************************************
+ *    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_OFI_CONTROLMESSAGES_H
+#define FAIR_MQ_OFI_CONTROLMESSAGES_H
+
+#include <FairMQLogger.h>
+#include <boost/asio/buffer.hpp>
+#include <boost/container/pmr/memory_resource.hpp>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <type_traits>
+
+namespace boost {
+namespace asio {
+
+template<typename PodType>
+auto buffer(const PodType& obj) -> boost::asio::const_buffer
+{
+    return boost::asio::const_buffer(static_cast<const void*>(&obj), sizeof(PodType));
+}
+
+}   // namespace asio
+}   // namespace boost
+
+namespace fair {
+namespace mq {
+namespace ofi {
+
+enum class ControlMessageType
+{
+    DataAddressAnnouncement = 1,
+    PostBuffer,
+    PostBufferAcknowledgement
+};
+
+struct ControlMessage
+{
+    ControlMessageType type;
+};
+
+struct DataAddressAnnouncement : ControlMessage
+{
+    uint32_t ipv4;   // in_addr_t from <netinet/in.h>
+    uint32_t port;   // in_port_t from <netinet/in.h>
+};
+
+struct PostBuffer : ControlMessage
+{
+    uint64_t size;   // buffer size (size_t)
+};
+
+template<typename T>
+using unique_ptr = std::unique_ptr<T, std::function<void(T*)>>;
+
+template<typename T, typename... Args>
+auto MakeControlMessageWithPmr(boost::container::pmr::memory_resource* pmr, Args&&... args)
+    -> ofi::unique_ptr<T>
+{
+    void* mem = pmr->allocate(sizeof(T));
+    T* ctrl = new (mem) T(std::forward<Args>(args)...);
+
+    if (std::is_same<T, DataAddressAnnouncement>::value) {
+        ctrl->type = ControlMessageType::DataAddressAnnouncement;
+    } else if (std::is_same<T, PostBuffer>::value) {
+        ctrl->type = ControlMessageType::PostBuffer;
+    }
+
+    return ofi::unique_ptr<T>(ctrl, [=](T* p) {
+        p->~T();
+        pmr->deallocate(p, sizeof(T));
+    });
+}
+
+template<typename T, typename... Args>
+auto MakeControlMessage(Args&&... args) -> T
+{
+    T ctrl = T(std::forward<Args>(args)...);
+
+    if (std::is_same<T, DataAddressAnnouncement>::value) {
+        ctrl.type = ControlMessageType::DataAddressAnnouncement;
+    } else if (std::is_same<T, PostBuffer>::value) {
+        ctrl.type = ControlMessageType::PostBuffer;
+    }
+
+    return ctrl;
+}
+
+}   // namespace ofi
+}   // namespace mq
+}   // namespace fair
+
+#endif /* FAIR_MQ_OFI_CONTROLMESSAGES_H */ 

fairmq/ofi/Message.cxx

@@ -10,6 +10,7 @@
 #include <fairmq/Tools.h>
 #include <FairMQLogger.h>
 
+#include <asiofi.hpp>
 #include <cassert>
 #include <cstdlib>
 #include <zmq.h>
@@ -23,38 +24,48 @@ namespace ofi
 
 using namespace std;
 
-Message::Message()
+Message::Message(boost::container::pmr::memory_resource* pmr)
     : fInitialSize(0)
     , fSize(0)
     , fData(nullptr)
     , fFreeFunction(nullptr)
     , fHint(nullptr)
+    , fPmr(pmr)
 {
 }
 
-Message::Message(const size_t size)
+Message::Message(boost::container::pmr::memory_resource* pmr, const size_t size)
     : fInitialSize(size)
     , fSize(size)
     , fData(nullptr)
     , fFreeFunction(nullptr)
     , fHint(nullptr)
+    , fPmr(pmr)
 {
     if (size) {
-        fData = malloc(size);
+        fData = fPmr->allocate(size);
         assert(fData);
     }
 }
 
-Message::Message(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
+Message::Message(boost::container::pmr::memory_resource* pmr,
+                 void* data,
+                 const size_t size,
+                 fairmq_free_fn* ffn,
+                 void* hint)
     : fInitialSize(size)
     , fSize(size)
     , fData(data)
     , fFreeFunction(ffn)
     , fHint(hint)
-{
-}
+    , fPmr(pmr)
+{}
 
-Message::Message(FairMQUnmanagedRegionPtr& /*region*/, void* /*data*/, const size_t /*size*/, void* /*hint*/)
+Message::Message(boost::container::pmr::memory_resource* /*pmr*/,
+                 FairMQUnmanagedRegionPtr& /*region*/,
+                 void* /*data*/,
+                 const size_t /*size*/,
+                 void* /*hint*/)
 {
     throw MessageError{"Not yet implemented."};
 }
@@ -62,9 +73,11 @@ Message::Message(FairMQUnmanagedRegionPtr& /*region*/, void* /*data*/, const siz
 auto Message::Rebuild() -> void
 {
     if (fFreeFunction) {
-      fFreeFunction(fData, fHint);
+        fFreeFunction(fData, fHint);
     } else {
-      free(fData);
+        if (fData) {
+            fPmr->deallocate(fData, fSize);
+        }
     }
     fData = nullptr;
     fInitialSize = 0;
@@ -78,10 +91,12 @@ auto Message::Rebuild(const size_t size) -> void
     if (fFreeFunction) {
       fFreeFunction(fData, fHint);
     } else {
-      free(fData);
+        if (fData) {
+            fPmr->deallocate(fData, fSize);
+        }
     }
     if (size) {
-        fData = malloc(size);
+        fData = fPmr->allocate(size);
         assert(fData);
     } else {
         fData = nullptr;
@@ -97,10 +112,12 @@ auto Message::Rebuild(void* /*data*/, const size_t size, fairmq_free_fn* ffn, vo
     if (fFreeFunction) {
       fFreeFunction(fData, fHint);
     } else {
-      free(fData);
+        if (fData) {
+            fPmr->deallocate(fData, fSize);
+        }
     }
     if (size) {
-        fData = malloc(size);
+        fData = fPmr->allocate(size);
         assert(fData);
     } else {
         fData = nullptr;
@@ -141,9 +158,11 @@ auto Message::Copy(const fair::mq::Message& /*msg*/) -> void
 Message::~Message()
 {
     if (fFreeFunction) {
-      fFreeFunction(fData, fHint);
+        fFreeFunction(fData, fHint);
     } else {
-      free(fData);
+        if (fData) {
+            fPmr->deallocate(fData, fSize);
+        }
     }
 }
 

fairmq/ofi/Message.h

@@ -12,10 +12,10 @@
 #include <FairMQMessage.h>
 #include <FairMQUnmanagedRegion.h>
 
-#include <zmq.h>
-
-#include <cstddef> // size_t
+#include <asiofi.hpp>
 #include <atomic>
+#include <cstddef> // size_t
+#include <zmq.h>
 
 namespace fair
 {
@@ -33,10 +33,18 @@ namespace ofi
 class Message final : public fair::mq::Message
 {
   public:
-    Message();
-    Message(const size_t size);
-    Message(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr);
-    Message(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0);
+    Message(boost::container::pmr::memory_resource* pmr);
+    Message(boost::container::pmr::memory_resource* pmr, const size_t size);
+    Message(boost::container::pmr::memory_resource* pmr,
+            void* data,
+            const size_t size,
+            fairmq_free_fn* ffn,
+            void* hint = nullptr);
+    Message(boost::container::pmr::memory_resource* pmr,
+            FairMQUnmanagedRegionPtr& region,
+            void* data,
+            const size_t size,
+            void* hint = 0);
 
     Message(const Message&) = delete;
     Message operator=(const Message&) = delete;
@@ -62,6 +70,7 @@ class Message final : public fair::mq::Message
     void* fData;
     fairmq_free_fn* fFreeFunction;
     void* fHint;
+    boost::container::pmr::memory_resource* fPmr;
 }; /* class Message */  
 
 } /* namespace ofi */

fairmq/ofi/Poller.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
@@ -28,13 +28,13 @@ Poller::Poller(const vector<FairMQChannel>& channels)
     fItems = new zmq_pollitem_t[fNumItems];
 
     for (int i = 0; i < fNumItems; ++i) {
-        fItems[i].socket = static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket();
+        fItems[i].socket = static_cast<Socket*>(&(channels.at(i).GetSocket()))->GetSocket();
         fItems[i].fd = 0;
         fItems[i].revents = 0;
 
         int type = 0;
         size_t size = sizeof(type);
-        zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+        zmq_getsockopt(static_cast<Socket*>(&(channels.at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
 
         SetItemEvents(fItems[i], type);
     }
@@ -46,13 +46,13 @@ Poller::Poller(const vector<const FairMQChannel*>& channels)
     fItems = new zmq_pollitem_t[fNumItems];
 
     for (int i = 0; i < fNumItems; ++i) {
-        fItems[i].socket = static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket();
+        fItems[i].socket = static_cast<Socket*>(&(channels.at(i)->GetSocket()))->GetSocket();
         fItems[i].fd = 0;
         fItems[i].revents = 0;
 
         int type = 0;
         size_t size = sizeof(type);
-        zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+        zmq_getsockopt(static_cast<Socket*>(&(channels.at(i)->GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
 
         SetItemEvents(fItems[i], type);
     }
@@ -76,13 +76,13 @@ Poller::Poller(const unordered_map<string, vector<FairMQChannel>>& channelsMap,
             for (unsigned int i = 0; i < channelsMap.at(channel).size(); ++i) {
                 index = fOffsetMap[channel] + i;
 
-                fItems[index].socket = static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
+                fItems[index].socket = static_cast<Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
                 fItems[index].fd = 0;
                 fItems[index].revents = 0;
 
                 int type = 0;
                 size_t size = sizeof(type);
-                zmq_getsockopt(static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+                zmq_getsockopt(static_cast<Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
 
                 SetItemEvents(fItems[index], type);
             }
@@ -124,7 +124,7 @@ auto Poller::CheckOutput(const int index) -> bool
     return fItems[index].revents & ZMQ_POLLOUT;
 }
 
-auto Poller::CheckInput(const string channelKey, const int index) -> bool
+auto Poller::CheckInput(const string& channelKey, const int index) -> bool
 {
     try {
         return fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLIN;
@@ -136,7 +136,7 @@ auto Poller::CheckInput(const string channelKey, const int index) -> bool
     }
 }
 
-auto Poller::CheckOutput(const string channelKey, const int index) -> bool
+auto Poller::CheckOutput(const string& channelKey, const int index) -> bool
 {
     try {
         return fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLOUT;

fairmq/ofi/Poller.h

@@ -51,8 +51,8 @@ class Poller final : public FairMQPoller
     auto Poll(const int timeout) -> void override;
     auto CheckInput(const int index) -> bool override;
     auto CheckOutput(const int index) -> bool override;
-    auto CheckInput(const std::string channelKey, const int index) -> bool override;
-    auto CheckOutput(const std::string channelKey, const int index) -> bool override;
+    auto CheckInput(const std::string& channelKey, const int index) -> bool override;
+    auto CheckOutput(const std::string& channelKey, const int index) -> bool override;
 
     ~Poller() override;
 

fairmq/ofi/Socket.h

@@ -12,13 +12,16 @@
 #include <FairMQSocket.h>
 #include <FairMQMessage.h>
 #include <fairmq/ofi/Context.h>
-#include <fairmq/ofi/Control.pb.h>
+#include <fairmq/ofi/ControlMessages.h>
 
+#include <asiofi/connected_endpoint.hpp>
+#include <asiofi/memory_resources.hpp>
+#include <asiofi/passive_endpoint.hpp>
+#include <asiofi/semaphore.hpp>
+#include <azmq/socket.hpp>
 #include <boost/asio.hpp>
 #include <memory> // unique_ptr
 #include <netinet/in.h>
-#include <rdma/fabric.h>
-class FairMQTransportFactory;
 
 namespace fair
 {
@@ -36,7 +39,7 @@ namespace ofi
 class Socket final : public fair::mq::Socket
 {
   public:
-    Socket(Context& factory, const std::string& type, const std::string& name, const std::string& id = "", FairMQTransportFactory* fac);
+    Socket(Context& context, const std::string& type, const std::string& name, const std::string& id = "");
     Socket(const Socket&) = delete;
     Socket operator=(const Socket&) = delete;
 
@@ -50,12 +53,7 @@ class Socket final : public fair::mq::Socket
     auto Send(std::vector<MessagePtr>& msgVec, int timeout = 0) -> int64_t override;
     auto Receive(std::vector<MessagePtr>& msgVec, int timeout = 0) -> int64_t override;
 
-    auto TrySend(MessagePtr& msg) -> int override;
-    auto TryReceive(MessagePtr& msg) -> int override;
-    auto TrySend(std::vector<MessagePtr>& msgVec) -> int64_t override;
-    auto TryReceive(std::vector<MessagePtr>& msgVec) -> int64_t override;
-
-    auto GetSocket() const -> void* { return fControlSocket; }
+    auto GetSocket() const -> void* { return nullptr; }
 
     void SetLinger(const int value) override;
     int GetLinger() const override;
@@ -83,43 +81,46 @@ class Socket final : public fair::mq::Socket
     ~Socket() override;
 
   private:
-    void* fControlSocket;
-    void* fMonitorSocket;
-    fid_ep* fDataEndpoint;
-    fid_cq* fDataCompletionQueueTx;
-    fid_cq* fDataCompletionQueueRx;
+    Context& fContext;
+    asiofi::allocated_pool_resource fControlMemPool;
+    std::unique_ptr<asiofi::info> fOfiInfo;
+    std::unique_ptr<asiofi::fabric> fOfiFabric;
+    std::unique_ptr<asiofi::domain> fOfiDomain;
+    std::unique_ptr<asiofi::passive_endpoint> fPassiveEndpoint;
+    std::unique_ptr<asiofi::connected_endpoint> fDataEndpoint, fControlEndpoint;
     std::string fId;
     std::atomic<unsigned long> fBytesTx;
     std::atomic<unsigned long> fBytesRx;
     std::atomic<unsigned long> fMessagesTx;
     std::atomic<unsigned long> fMessagesRx;
-    Context& fContext;
-    fi_addr_t fRemoteDataAddr;
-    sockaddr_in fLocalDataAddr;
-    bool fWaitingForControlPeer;
-    boost::asio::io_service::strand fIoStrand;
-
+    Address fRemoteAddr;
+    Address fLocalAddr;
     int fSndTimeout;
     int fRcvTimeout;
+    azmq::socket fSendQueueWrite, fSendQueueRead;
+    azmq::socket fRecvQueueWrite, fRecvQueueRead;
+    asiofi::semaphore fSendSem, fRecvSem;
+    std::atomic<bool> fNeedOfiMemoryRegistration;
 
-    auto SendImpl(MessagePtr& msg, const int flags, const int timeout) -> int;
+    auto SendQueueReader() -> void;
+    auto OnSend(azmq::message& msg, size_t bytes_transferred) -> void;
+    auto RecvControlQueueReader() -> void;
+    auto OnRecvControl(ofi::unique_ptr<PostBuffer> ctrl) -> void;
+    auto OnReceive() -> void;
     auto ReceiveImpl(MessagePtr& msg, const int flags, const int timeout) -> int;
     auto SendImpl(std::vector<MessagePtr>& msgVec, const int flags, const int timeout) -> int64_t;
     auto ReceiveImpl(std::vector<MessagePtr>& msgVec, const int flags, const int timeout) -> int64_t;
 
-    auto InitDataEndpoint() -> void;
-    auto WaitForControlPeer() -> void;
-    auto AnnounceDataAddress() -> void;
-    auto SendControlMessage(std::unique_ptr<ControlMessage> ctrl) -> void;
-    auto ReceiveControlMessage() -> std::unique_ptr<ControlMessage>;
-    auto ProcessDataAddressAnnouncement(std::unique_ptr<ControlMessage> ctrl) -> void;
-    auto ConnectControlSocket(Context::Address address) -> void;
-    auto BindControlSocket(Context::Address address) -> void;
+    // auto WaitForControlPeer() -> void;
+    // auto AnnounceDataAddress() -> void;
+    auto InitOfi(Address addr) -> void;
+    auto BindControlEndpoint() -> void;
+    auto BindDataEndpoint() -> void;
+    enum class Band { Control, Data };
+    auto ConnectEndpoint(std::unique_ptr<asiofi::connected_endpoint>& endpoint, Band type) -> void;
+    // auto ReceiveDataAddressAnnouncement() -> void;
 }; /* class Socket */
 
-// helper function to clean up the object holding the data after it is transported.
-void free_string(void* /*data*/, void* hint);
-
 struct SilentSocketError : SocketError { using SocketError::SocketError; };
 
 } /* namespace ofi */

fairmq/ofi/TransportFactory.cxx

@@ -23,60 +23,66 @@ namespace ofi
 
 using namespace std;
 
-TransportFactory::TransportFactory(const string& id, const FairMQProgOptions* config)
-try : FairMQTransportFactory{id}
-{
-    LOG(debug) << "Transport: Using ZeroMQ (" << fContext.GetZmqVersion() << ") & "
-               << "OFI libfabric (API " << fContext.GetOfiApiVersion() << ") & "
-               << "Google Protobuf (" << fContext.GetPbVersion() << ") & "
-               << "Boost.Asio (" << fContext.GetBoostVersion() << ")";
-}
-catch (ContextError& e)
+TransportFactory::TransportFactory(const string& id, const FairMQProgOptions* /*config*/)
+try : FairMQTransportFactory(id)
+    , fContext(*this, *this, 1)
 {
+    LOG(debug) << "OFI transport: Using AZMQ & "
+               << "asiofi (" << fContext.GetAsiofiVersion() << ")";
+} catch (ContextError& e) {
     throw TransportFactoryError{e.what()};
 }
 
-auto TransportFactory::CreateMessage() const -> MessagePtr
+auto TransportFactory::CreateMessage() -> MessagePtr
 {
-    return MessagePtr{new Message()};
+    return MessagePtr{new Message(&fMemoryResource)};
 }
 
-auto TransportFactory::CreateMessage(const size_t size) const -> MessagePtr
+auto TransportFactory::CreateMessage(const size_t size) -> MessagePtr
 {
-    return MessagePtr{new Message(size)};
+    return MessagePtr{new Message(&fMemoryResource, size)};
 }
 
-auto TransportFactory::CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint) const -> MessagePtr
+auto TransportFactory::CreateMessage(void* data,
+                                     const size_t size,
+                                     fairmq_free_fn* ffn,
+                                     void* hint) -> MessagePtr
 {
-    return MessagePtr{new Message(data, size, ffn, hint)};
+    return MessagePtr{new Message(&fMemoryResource, data, size, ffn, hint)};
 }
 
-auto TransportFactory::CreateMessage(UnmanagedRegionPtr& region, void* data, const size_t size, void* hint) const -> MessagePtr
+auto TransportFactory::CreateMessage(UnmanagedRegionPtr& region,
+                                     void* data,
+                                     const size_t size,
+                                     void* hint) -> MessagePtr
 {
-    return MessagePtr{new Message(region, data, size, hint)};
+    return MessagePtr{new Message(&fMemoryResource, region, data, size, hint)};
 }
 
 auto TransportFactory::CreateSocket(const string& type, const string& name) -> SocketPtr
 {
-    return SocketPtr{new Socket(fContext, type, name, GetId(), this)};
+    return SocketPtr{new Socket(fContext, type, name, GetId())};
 }
 
-auto TransportFactory::CreatePoller(const vector<FairMQChannel>& channels) const -> PollerPtr
+auto TransportFactory::CreatePoller(const vector<FairMQChannel>& /*channels*/) const -> PollerPtr
 {
-    return PollerPtr{new Poller(channels)};
+    throw runtime_error{"Not yet implemented (Poller)."};
+    // return PollerPtr{new Poller(channels)};
 }
 
-auto TransportFactory::CreatePoller(const vector<const FairMQChannel*>& channels) const -> PollerPtr
+auto TransportFactory::CreatePoller(const vector<FairMQChannel*>& /*channels*/) const -> PollerPtr
 {
-    return PollerPtr{new Poller(channels)};
+    throw runtime_error{"Not yet implemented (Poller)."};
+    // return PollerPtr{new Poller(channels)};
 }
 
-auto TransportFactory::CreatePoller(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList) const -> PollerPtr
+auto TransportFactory::CreatePoller(const unordered_map<string, vector<FairMQChannel>>& /*channelsMap*/, const vector<string>& /*channelList*/) const -> PollerPtr
 {
-    return PollerPtr{new Poller(channelsMap, channelList)};
+    throw runtime_error{"Not yet implemented (Poller)."};
+    // return PollerPtr{new Poller(channelsMap, channelList)};
 }
 
-auto TransportFactory::CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback) const -> UnmanagedRegionPtr
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, FairMQRegionCallback /*callback*/) const -> UnmanagedRegionPtr
 {
     throw runtime_error{"Not yet implemented UMR."};
 }

fairmq/ofi/TransportFactory.h

@@ -13,6 +13,8 @@
 #include <options/FairMQProgOptions.h>
 #include <fairmq/ofi/Context.h>
 
+#include <asiofi.hpp>
+
 namespace fair
 {
 namespace mq
@@ -22,7 +24,7 @@ namespace ofi
 
 /**
  * @class TransportFactory TransportFactory.h <fairmq/ofi/TransportFactory.h>
- * @brief FairMQ transport factory for the ofi transport (implemented with ZeroMQ + libfabric)
+ * @brief FairMQ transport factory for the ofi transport
  *
  * @todo TODO insert long description
  */
@@ -33,27 +35,28 @@ class TransportFactory final : public FairMQTransportFactory
     TransportFactory(const TransportFactory&) = delete;
     TransportFactory operator=(const TransportFactory&) = delete;
 
-    auto CreateMessage() const -> MessagePtr override;
-    auto CreateMessage(const std::size_t size) const -> MessagePtr override;
-    auto CreateMessage(void* data, const std::size_t size, fairmq_free_fn* ffn, void* hint = nullptr) const -> MessagePtr override;
-    auto CreateMessage(UnmanagedRegionPtr& region, void* data, const std::size_t size, void* hint = nullptr) const -> MessagePtr override;
+    auto CreateMessage() -> MessagePtr override;
+    auto CreateMessage(const std::size_t size) -> MessagePtr override;
+    auto CreateMessage(void* data, const std::size_t size, fairmq_free_fn* ffn, void* hint = nullptr) -> MessagePtr override;
+    auto CreateMessage(UnmanagedRegionPtr& region, void* data, const std::size_t size, void* hint = nullptr) -> MessagePtr override;
 
     auto CreateSocket(const std::string& type, const std::string& name) -> SocketPtr override;
 
     auto CreatePoller(const std::vector<FairMQChannel>& channels) const -> PollerPtr override;
-    auto CreatePoller(const std::vector<const FairMQChannel*>& channels) const -> PollerPtr override;
+    auto CreatePoller(const std::vector<FairMQChannel*>& channels) const -> PollerPtr override;
     auto CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const -> PollerPtr override;
 
     auto CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr) const -> UnmanagedRegionPtr override;
 
     auto GetType() const -> Transport override;
 
-    void Interrupt() override {}
-    void Resume() override {}
-    void Reset() override {}
+    void Interrupt() override { fContext.Interrupt(); }
+    void Resume() override { fContext.Resume(); }
+    void Reset() override { fContext.Reset(); }
 
   private:
     mutable Context fContext;
+    asiofi::allocated_pool_resource fMemoryResource;
 }; /* class TransportFactory */  
 
 } /* namespace ofi */

fairmq/options/FairMQProgOptions.cxx

@@ -56,17 +56,18 @@ FairMQProgOptions::FairMQProgOptions()
         ("print-options", po::value<bool  >()->implicit_value(true),    "Print options in machine-readable format (<option>:<computed-value>:<type>:<description>)");
 
     fMQOptions.add_options()
-        ("id",                     po::value<string>(),                            "Device ID (required argument).")
-        ("io-threads",             po::value<int   >()->default_value(1),          "Number of I/O threads.")
-        ("transport",              po::value<string>()->default_value("zeromq"),   "Transport ('zeromq'/'nanomsg'/'shmem')  .")
-        ("network-interface",      po::value<string>()->default_value("default"),  "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).")
-        ("config-key",             po::value<string>(),                            "Use provided value instead of device id for fetching the configuration from the config file.")
-        ("initialization-timeout", po::value<int   >()->default_value(120),        "Timeout for the initialization in seconds (when expecting dynamic initialization).")
-        ("print-channels",         po::value<bool  >()->implicit_value(true),      "Print registered channel endpoints in a machine-readable format (<channel name>:<min num subchannels>:<max num subchannels>)")
-        ("shm-segment-size",       po::value<size_t>()->default_value(2000000000), "Shared memory: size of the shared memory segment (in bytes).")
-        ("shm-monitor",            po::value<bool  >()->default_value(true),       "Shared memory: run monitor daemon.")
-        ("rate",                   po::value<float >()->default_value(0.),         "Rate for conditional run loop (Hz).")
-        ("session",                po::value<string>()->default_value("default"),  "Session name.");
+        ("id",                     po::value<string  >(),                            "Device ID (required argument).")
+        ("io-threads",             po::value<int     >()->default_value(1),          "Number of I/O threads.")
+        ("transport",              po::value<string  >()->default_value("zeromq"),   "Transport ('zeromq'/'nanomsg'/'shmem').")
+        ("network-interface",      po::value<string  >()->default_value("default"),  "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).")
+        ("config-key",             po::value<string  >(),                            "Use provided value instead of device id for fetching the configuration from the config file.")
+        ("initialization-timeout", po::value<int     >()->default_value(120),        "Timeout for the initialization in seconds (when expecting dynamic initialization).")
+        ("max-run-time",           po::value<uint64_t>()->default_value(0),          "Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).")
+        ("print-channels",         po::value<bool    >()->implicit_value(true),      "Print registered channel endpoints in a machine-readable format (<channel name>:<min num subchannels>:<max num subchannels>)")
+        ("shm-segment-size",       po::value<size_t  >()->default_value(2000000000), "Shared memory: size of the shared memory segment (in bytes).")
+        ("shm-monitor",            po::value<bool    >()->default_value(true),       "Shared memory: run monitor daemon.")
+        ("rate",                   po::value<float   >()->default_value(0.),         "Rate for conditional run loop (Hz).")
+        ("session",                po::value<string  >()->default_value("default"),  "Session name.");
 
     fParserOptions.add_options()
         ("mq-config",      po::value<string>(),                                    "JSON input as file.")
@@ -452,7 +453,7 @@ int FairMQProgOptions::PrintOptions()
            << "\n";
     }
 
-    LOG(info) << ss.str();
+    LOG(debug) << ss.str();
 
     return 0;
 }

fairmq/plugins/Control.cxx

@@ -14,6 +14,8 @@
 #include <cstdlib>
 #include <functional>
 #include <atomic>
+#include <thread>
+#include <chrono>
 
 using namespace std;
 
@@ -53,8 +55,8 @@ Control::Control(const string& name, const Plugin::Version version, const string
     , fDeviceHasShutdown(false)
     , fPluginShutdownRequested(false)
 {
-    SubscribeToDeviceStateChange([&](DeviceState newState)
-    {
+    SubscribeToDeviceStateChange([&](DeviceState newState) {
+        LOG(trace) << "control plugin notified on new state: " << newState;
         {
             lock_guard<mutex> lock{fEventsMutex};
             fEvents.push(newState);
@@ -62,47 +64,52 @@ Control::Control(const string& name, const Plugin::Version version, const string
         fNewEvent.notify_one();
     });
 
-    try
-    {
+    try {
         TakeDeviceControl();
 
         auto control = GetProperty<string>("control");
 
-        if (control == "static")
-        {
+        if (control == "static") {
             LOG(debug) << "Running builtin controller: static";
             fControllerThread = thread(&Control::StaticMode, this);
-        }
-        else if (control == "interactive")
-        {
+        } else if (control == "interactive") {
             LOG(debug) << "Running builtin controller: interactive";
             fControllerThread = thread(&Control::InteractiveMode, this);
-        }
-        else
-        {
+        } else {
             LOG(error) << "Unrecognized control mode '" << control << "' requested. " << "Ignoring and falling back to static control mode.";
             fControllerThread = thread(&Control::StaticMode, this);
         }
-    }
-    catch (PluginServices::DeviceControlError& e)
-    {
+    } catch (PluginServices::DeviceControlError& e) {
         // If we are here, it means another plugin has taken control. That's fine, just print the exception message and do nothing else.
         LOG(debug) << e.what();
     }
 
-    LOG(debug) << "catch-signals: " << GetProperty<int>("catch-signals");
-    if (GetProperty<int>("catch-signals") > 0)
-    {
+    if (GetProperty<int>("catch-signals") > 0) {
+        LOG(debug) << "Plugin '" << name << "' is setting up signal handling for SIGINT and SIGTERM";
         fSignalHandlerThread = thread(&Control::SignalHandler, this);
         signal(SIGINT, signal_handler);
         signal(SIGTERM, signal_handler);
-    }
-    else
-    {
+    } else {
         LOG(warn) << "Signal handling (e.g. Ctrl-C) has been deactivated.";
     }
 }
 
+auto Control::RunStartupSequence() -> void
+{
+    ChangeDeviceState(DeviceStateTransition::InitDevice);
+    while (WaitForNextState() != DeviceState::InitializingDevice) {}
+    ChangeDeviceState(DeviceStateTransition::CompleteInit);
+    while (WaitForNextState() != DeviceState::Initialized) {}
+    ChangeDeviceState(DeviceStateTransition::Bind);
+    while (WaitForNextState() != DeviceState::Bound) {}
+    ChangeDeviceState(DeviceStateTransition::Connect);
+    while (WaitForNextState() != DeviceState::DeviceReady) {}
+    ChangeDeviceState(DeviceStateTransition::InitTask);
+    while (WaitForNextState() != DeviceState::Ready) {}
+    ChangeDeviceState(DeviceStateTransition::Run);
+    while (WaitForNextState() != DeviceState::Running) {}
+}
+
 auto ControlPluginProgramOptions() -> Plugin::ProgOptions
 {
     namespace po = boost::program_options;
@@ -135,8 +142,7 @@ struct terminal_config
 };
 
 auto Control::InteractiveMode() -> void
-try
-{
+try {
     RunStartupSequence();
 
     char input; // hold the user console input
@@ -147,129 +153,226 @@ try
 
     terminal_config tconfig;
 
-    PrintInteractiveHelp();
+    bool color = GetProperty<bool>("color");
+
+    if (color) {
+        PrintInteractiveHelpColor();
+    } else {
+        PrintInteractiveHelp();
+    }
 
     bool keepRunning = true;
 
-    while (keepRunning)
-    {
-        if (poll(cinfd, 1, 500))
-        {
-            if (fDeviceShutdownRequested)
-            {
+    while (keepRunning) {
+        if (poll(cinfd, 1, 500)) {
+            if (fDeviceShutdownRequested) {
                 break;
             }
 
             cin >> input;
 
-            switch (input)
-            {
+            switch (input) {
+                case 'c':
+                    cout << "\n --> [i] check current state\n\n" << flush;
+                    LOG(state) << GetCurrentDeviceState();
+                break;
                 case 'i':
-                    LOG(info) << "\n\n --> [i] init device\n";
+                    cout << "\n --> [i] init device\n\n" << flush;
                     ChangeDeviceState(DeviceStateTransition::InitDevice);
+                    while (WaitForNextState() != DeviceState::InitializingDevice) {}
+                    ChangeDeviceState(DeviceStateTransition::CompleteInit);
+                break;
+                case 'b':
+                    cout << "\n --> [b] bind\n\n" << flush;
+                    ChangeDeviceState(DeviceStateTransition::Bind);
+                break;
+                case 'x':
+                    cout << "\n --> [x] connect\n\n" << flush;
+                    ChangeDeviceState(DeviceStateTransition::Connect);
                 break;
                 case 'j':
-                    LOG(info) << "\n\n --> [j] init task\n";
+                    cout << "\n --> [j] init task\n\n" << flush;
                     ChangeDeviceState(DeviceStateTransition::InitTask);
                 break;
-                case 'p':
-                    LOG(info) << "\n\n --> [p] pause\n";
-                    ChangeDeviceState(DeviceStateTransition::Pause);
-                break;
                 case 'r':
-                    LOG(info) << "\n\n --> [r] run\n";
+                    cout << "\n --> [r] run\n\n" << flush;
                     ChangeDeviceState(DeviceStateTransition::Run);
                 break;
                 case 's':
-                    LOG(info) << "\n\n --> [s] stop\n";
+                    cout << "\n --> [s] stop\n\n" << flush;
                     ChangeDeviceState(DeviceStateTransition::Stop);
                 break;
                 case 't':
-                    LOG(info) << "\n\n --> [t] reset task\n";
+                    cout << "\n --> [t] reset task\n\n" << flush;
                     ChangeDeviceState(DeviceStateTransition::ResetTask);
                 break;
                 case 'd':
-                    LOG(info) << "\n\n --> [d] reset device\n";
+                    cout << "\n --> [d] reset device\n\n" << flush;
                     ChangeDeviceState(DeviceStateTransition::ResetDevice);
                 break;
                 case 'k':
-                    LOG(info) << "\n\n --> [k] increase log severity\n";
+                    cout << "\n --> [k] increase log severity\n\n" << flush;
                     CycleLogConsoleSeverityUp();
                 break;
                 case 'l':
-                    LOG(info) << "\n\n --> [l] decrease log severity\n";
+                    cout << "\n --> [l] decrease log severity\n\n" << flush;
                     CycleLogConsoleSeverityDown();
                 break;
                 case 'n':
-                    LOG(info) << "\n\n --> [n] increase log verbosity\n";
+                    cout << "\n --> [n] increase log verbosity\n\n" << flush;
                     CycleLogVerbosityUp();
                 break;
                 case 'm':
-                    LOG(info) << "\n\n --> [m] decrease log verbosity\n";
+                    cout << "\n --> [m] decrease log verbosity\n\n" << flush;
                     CycleLogVerbosityDown();
                 break;
                 case 'h':
-                    LOG(info) << "\n\n --> [h] help\n";
-                    PrintInteractiveHelp();
+                    cout << "\n --> [h] help\n\n" << flush;
+                    if (color) {
+                        PrintInteractiveHelpColor();
+                        PrintStateMachineColor();
+                    } else {
+                        PrintInteractiveHelp();
+                        PrintStateMachine();
+                    }
                 break;
-                // case 'x':
-                //     LOG(info) << "\n\n --> [x] ERROR\n";
-                //     ChangeDeviceState(DeviceStateTransition::ERROR_FOUND);
-                //     break;
                 case 'q':
-                    LOG(info) << "\n\n --> [q] end\n";
+                    cout << "\n --> [q] end\n\n" << flush;
                     keepRunning = false;
                 break;
                 default:
-                    LOG(info) << "Invalid input: [" << input << "]";
+                    cout << "Invalid input: [" << input << "]. " << flush;
                     PrintInteractiveHelp();
                 break;
             }
         }
 
-        if (GetCurrentDeviceState() == DeviceState::Error)
-        {
+        if (GetCurrentDeviceState() == DeviceState::Error) {
             throw DeviceErrorState("Controlled device transitioned to error state.");
         }
 
-        if (fDeviceShutdownRequested)
-        {
+        if (fDeviceShutdownRequested) {
             break;
         }
     }
 
     RunShutdownSequence();
-}
-catch (PluginServices::DeviceControlError& e)
-{
+} catch (PluginServices::DeviceControlError& e) {
     // If we are here, it means another plugin has taken control. That's fine, just print the exception message and do nothing else.
     LOG(debug) << e.what();
+} catch (DeviceErrorState&) {
 }
-catch (DeviceErrorState&)
+
+auto Control::PrintInteractiveHelpColor() -> void
 {
+    stringstream ss;
+    ss << "Following control commands are available:\n\n"
+       << " [\033[01;32mh\033[0m] help, [\033[01;32mc\033[0m] check current device state,\n"
+       << " [\033[01;32mi\033[0m] init device, [\033[01;32mb\033[0m] bind, [\033[01;32mx\033[0m] connect, [\033[01;32mj\033[0m] init task,"
+       << " [\033[01;32mr\033[0m] run, [\033[01;32ms\033[0m] stop,\n"
+       << " [\033[01;32mt\033[0m] reset task, [\033[01;32md\033[0m] reset device, [\033[01;32mq\033[0m] end,\n"
+       << " [\033[01;32mk\033[0m] increase log severity [\033[01;32ml\033[0m] decrease log severity [\033[01;32mn\033[0m] increase log verbosity [\033[01;32mm\033[0m] decrease log verbosity\n\n";
+    cout << ss.str() << flush;
 }
 
 auto Control::PrintInteractiveHelp() -> void
 {
     stringstream ss;
-    ss << "\nFollowing control commands are available:\n\n"
-       << "[h] help, [p] pause, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device\n"
-       << "[k] increase log severity [l] decrease log severity [n] increase log verbosity [m] decrease log verbosity\n\n";
+    ss << "Following control commands are available:\n\n"
+       << " [h] help, [c] check current device state,\n"
+       << " [i] init device, [b] bind, [x] connect, [j] init task,\n"
+       << " [r] run, [s] stop,\n"
+       << " [t] reset task, [d] reset device, [q] end,\n"
+       << " [k] increase log severity, [l] decrease log severity, [n] increase log verbosity, [m] decrease log verbosity.\n\n";
+    cout << ss.str() << flush;
+}
+
+void Control::PrintStateMachineColor()
+{
+    stringstream ss;
+    ss << "               @                                \n"
+       << "   ┌───────────╨─────────────┐  ┌───────────┐   \n"
+       << "   │           \033[01;36mIDLE\033[0m         [\033[01;32mq\033[0m]─▶  \033[01;33mEXITING\033[0m  │   \n"
+       << "   └[\033[01;32mi\033[0m]─────────────────── ▲ ┘  └───────────┘   \n"
+       << " ╔══ ▼ ════════════════╗ ╔═╩══════════════════╗ \n"
+       << " ║ \033[01;33mINITIALIZING DEVICE\033[0m ║ ║  \033[01;33mRESETTING DEVICE\033[0m  ║ \n"
+       << " ╚══════════╦══════════╝ ╚════════ ▲ ═════════╝ \n"
+       << " ┌───────── ▼ ─────────┐           │  ┌────────────────────────────┐ \n"
+       << " │     \033[01;36mINITIALIZED\033[0m     │           │  │ Legend:                    │ \n"
+       << " └─────────[\033[01;32mb\033[0m]─────────┘           │  │----------------------------│ \n"
+       << " ╔═════════ ▼ ═════════╗           │  │ [\033[01;32mk\033[0m] keyboard shortcut for  │ \n"
+       << " ║       \033[01;33mBINDING\033[0m       ║           │  │     interactive controller │ \n"
+       << " ╚══════════╦══════════╝           │  │ ┌────────────────────────┐ │ \n"
+       << " ┌───────── ▼ ─────────┐           │  │ │      \033[01;36mIDLING STATE\033[0m      │ │ \n"
+       << " │        \033[01;36mBOUND\033[0m        │           │  │ └────────────────────────┘ │ \n"
+       << " └─────────[\033[01;32mx\033[0m]─────────┘           │  │ ╔════════════════════════╗ │ \n"
+       << " ╔═════════ ▼ ═════════╗           │  │ ║      \033[01;33mWORKING STATE\033[0m     ║ │ \n"
+       << " ║     \033[01;33mCONNECTING\033[0m      ║           │  │ ╚════════════════════════╝ │ \n"
+       << " ╚══════════╦══════════╝           │  └────────────────────────────┘ \n"
+       << "   ┌─────── ▼ ────────────────────[\033[01;32md\033[0m]───────┐   \n"
+       << "   │              \033[01;36mDEVICE READY\033[0m              │   \n"
+       << "   └───────[\033[01;32mj\033[0m]──────────────────── ▲ ───────┘   \n"
+       << " ╔═════════ ▼ ═════════╗ ╔═════════╩══════════╗ \n"
+       << " ║  \033[01;33mINITIALIZING TASK\033[0m  ║ ║   \033[01;33mRESETTING TASK\033[0m   ║ \n"
+       << " ╚══════════╦══════════╝ ╚════════ ▲ ═════════╝ \n"
+       << "   ┌─────── ▼ ────────────────────[\033[01;32mt\033[0m]───────┐   \n"
+       << "   │                 \033[01;36mREADY\033[0m                  │   \n"
+       << "   └───────[\033[01;32mr\033[0m]──────────────────── ▲ ───────┘   \n"
+       << "    ╔══════ ▼ ════════════════════[\033[01;32ms\033[0m]══════╗    \n"
+       << "    ║               \033[01;33mRUNNING\033[0m                ║    \n"
+       << "    ╚══════════════════════════════════════╝    \n"
+       << "                                                \n";
+    cout << ss.str() << flush;
+}
+
+void Control::PrintStateMachine()
+{
+    stringstream ss;
+    ss << "               @                                                     \n"
+       << "   ┌───────────╨─────────────┐  ┌───────────┐                        \n"
+       << "   │           IDLE         [q]─▶  EXITING  │                        \n"
+       << "   └[i]─────────────────── ▲ ┘  └───────────┘                        \n"
+       << " ╔══ ▼ ════════════════╗ ╔═╩══════════════════╗                      \n"
+       << " ║ INITIALIZING DEVICE ║ ║  RESETTING DEVICE  ║                      \n"
+       << " ╚══════════╦══════════╝ ╚════════ ▲ ═════════╝                      \n"
+       << " ┌───────── ▼ ─────────┐           │  ┌────────────────────────────┐ \n"
+       << " │     INITIALIZED     │           │  │ Legend:                    │ \n"
+       << " └─────────[b]─────────┘           │  │----------------------------│ \n"
+       << " ╔═════════ ▼ ═════════╗           │  │ [k] keyboard shortcut for  │ \n"
+       << " ║       BINDING       ║           │  │     interactive controller │ \n"
+       << " ╚══════════╦══════════╝           │  │ ┌────────────────────────┐ │ \n"
+       << " ┌───────── ▼ ─────────┐           │  │ │      IDLING STATE      │ │ \n"
+       << " │        BOUND        │           │  │ └────────────────────────┘ │ \n"
+       << " └─────────[x]─────────┘           │  │ ╔════════════════════════╗ │ \n"
+       << " ╔═════════ ▼ ═════════╗           │  │ ║      WORKING STATE     ║ │ \n"
+       << " ║     CONNECTING      ║           │  │ ╚════════════════════════╝ │ \n"
+       << " ╚══════════╦══════════╝           │  └────────────────────────────┘ \n"
+       << "   ┌─────── ▼ ────────────────────[d]───────┐                        \n"
+       << "   │              DEVICE READY              │                        \n"
+       << "   └───────[j]──────────────────── ▲ ───────┘                        \n"
+       << " ╔═════════ ▼ ═════════╗ ╔═════════╩══════════╗                      \n"
+       << " ║  INITIALIZING TASK  ║ ║   RESETTING TASK   ║                      \n"
+       << " ╚══════════╦══════════╝ ╚════════ ▲ ═════════╝                      \n"
+       << "   ┌─────── ▼ ────────────────────[t]───────┐                        \n"
+       << "   │                 READY                  │                        \n"
+       << "   └───────[r]──────────────────── ▲ ───────┘                        \n"
+       << "    ╔══════ ▼ ════════════════════[s]══════╗                         \n"
+       << "    ║               RUNNING                ║                         \n"
+       << "    ╚══════════════════════════════════════╝                         \n"
+       << "                                                                     \n";
     cout << ss.str() << flush;
 }
 
 auto Control::WaitForNextState() -> DeviceState
 {
     unique_lock<mutex> lock{fEventsMutex};
-    while (fEvents.empty())
-    {
+    while (fEvents.empty()) {
         fNewEvent.wait_for(lock, chrono::milliseconds(50));
     }
 
     auto result = fEvents.front();
 
-    if (result == DeviceState::Error)
-    {
+    if (result == DeviceState::Error) {
         throw DeviceErrorState("Controlled device transitioned to error state.");
     }
 
@@ -351,14 +454,16 @@ auto Control::SignalHandler() -> void
 auto Control::RunShutdownSequence() -> void
 {
     auto nextState = GetCurrentDeviceState();
-    EmptyEventQueue();
-    while (nextState != DeviceState::Exiting && nextState != DeviceState::Error)
-    {
-        switch (nextState)
-        {
+    if (nextState != DeviceState::Error) {
+        EmptyEventQueue();
+    }
+    while (nextState != DeviceState::Exiting && nextState != DeviceState::Error) {
+        switch (nextState) {
             case DeviceState::Idle:
                 ChangeDeviceState(DeviceStateTransition::End);
                 break;
+            case DeviceState::Initialized:
+            case DeviceState::Bound:
             case DeviceState::DeviceReady:
                 ChangeDeviceState(DeviceStateTransition::ResetDevice);
                 break;
@@ -368,11 +473,8 @@ auto Control::RunShutdownSequence() -> void
             case DeviceState::Running:
                 ChangeDeviceState(DeviceStateTransition::Stop);
                 break;
-            case DeviceState::Paused:
-                ChangeDeviceState(DeviceStateTransition::Resume);
-                break;
             default:
-                LOG(debug) << "Controller ignoring event: " << nextState;
+                // LOG(debug) << "Controller ignoring event: " << nextState;
                 break;
         }
 
@@ -383,16 +485,6 @@ auto Control::RunShutdownSequence() -> void
     ReleaseDeviceControl();
 }
 
-auto Control::RunStartupSequence() -> void
-{
-    ChangeDeviceState(DeviceStateTransition::InitDevice);
-    while (WaitForNextState() != DeviceState::DeviceReady) {}
-    ChangeDeviceState(DeviceStateTransition::InitTask);
-    while (WaitForNextState() != DeviceState::Ready) {}
-    ChangeDeviceState(DeviceStateTransition::Run);
-    while (WaitForNextState() != DeviceState::Running) {}
-}
-
 auto Control::EmptyEventQueue() -> void
 {
     lock_guard<mutex> lock{fEventsMutex};

fairmq/plugins/Control.h

@@ -36,7 +36,10 @@ class Control : public Plugin
 
   private:
     auto InteractiveMode() -> void;
+    static auto PrintInteractiveHelpColor() -> void;
     static auto PrintInteractiveHelp() -> void;
+    static auto PrintStateMachineColor() -> void;
+    static auto PrintStateMachine() -> void;
     auto StaticMode() -> void;
     auto WaitForNextState() -> DeviceState;
     auto SignalHandler() -> void;
@@ -62,9 +65,7 @@ auto ControlPluginProgramOptions() -> Plugin::ProgOptions;
 REGISTER_FAIRMQ_PLUGIN(
     Control,   // Class name
     control,   // Plugin name (string, lower case chars only)
-    (Plugin::Version{FAIRMQ_VERSION_MAJOR,
-                     FAIRMQ_VERSION_MINOR,
-                     FAIRMQ_VERSION_PATCH}),       // Version
+    (Plugin::Version{FAIRMQ_VERSION_MAJOR, FAIRMQ_VERSION_MINOR, FAIRMQ_VERSION_PATCH}), // Version
     "FairRootGroup <fairroot@gsi.de>",             // Maintainer
     "https://github.com/FairRootGroup/FairRoot",   // Homepage
     ControlPluginProgramOptions   // Free function which declares custom program options for the

fairmq/plugins/DDS/DDS.cxx

@@ -40,7 +40,7 @@ DDS::DDS(const string& name, const Plugin::Version version, const string& mainta
     , fConnectingChans()
     , fStopMutex()
     , fStopCondition()
-    , fCommands({ "INIT DEVICE", "INIT TASK", "PAUSE", "RUN", "STOP", "RESET TASK", "RESET DEVICE" })
+    , fCommands({ "BIND", "CONNECT", "INIT TASK", "RUN", "STOP", "RESET TASK", "RESET DEVICE" })
     , fControllerThread()
     , fEvents()
     , fEventsMutex()
@@ -96,6 +96,10 @@ auto DDS::HandleControl() -> void
 
         ChangeDeviceState(DeviceStateTransition::InitDevice);
         while (WaitForNextState() != DeviceState::InitializingDevice) {}
+        ChangeDeviceState(DeviceStateTransition::CompleteInit);
+        while (WaitForNextState() != DeviceState::Initialized) {}
+        ChangeDeviceState(DeviceStateTransition::Bind);
+        while (WaitForNextState() != DeviceState::Bound) {}
 
         // in the Initializing state subscribe to receive addresses of connecting channels from DDS
         // and propagate addresses of bound channels to DDS.
@@ -114,6 +118,7 @@ auto DDS::HandleControl() -> void
         // publish bound addresses via DDS at keys corresponding to the channel prefixes, e.g. 'data' in data[i]
         PublishBoundChannels();
 
+        ChangeDeviceState(DeviceStateTransition::Connect);
         while (WaitForNextState() != DeviceState::DeviceReady) {}
 
         ChangeDeviceState(DeviceStateTransition::InitTask);
@@ -305,13 +310,26 @@ auto DDS::SubscribeForCustomCommands() -> void
 
         if (cmd == "check-state") {
             fDDSCustomCmd.send(id + ": " + ToStr(GetCurrentDeviceState()) + " (pid: " + pid + ")", to_string(senderId));
+        } else if (cmd == "INIT DEVICE") {
+            if (ChangeDeviceState(ToDeviceStateTransition(cmd))) {
+                fDDSCustomCmd.send(id + ": queued " + cmd + " transition", to_string(senderId));
+                while (WaitForNextState() != DeviceState::InitializingDevice) {}
+                ChangeDeviceState(DeviceStateTransition::CompleteInit);
+            } else {
+                fDDSCustomCmd.send(id + ": could not queue " + cmd + " transition", to_string(senderId));
+            }
         } else if (fCommands.find(cmd) != fCommands.end()) {
-            fDDSCustomCmd.send(id + ": attempting to " + cmd, to_string(senderId));
-            ChangeDeviceState(ToDeviceStateTransition(cmd));
+            if (ChangeDeviceState(ToDeviceStateTransition(cmd))) {
+                fDDSCustomCmd.send(id + ": queued " + cmd + " transition", to_string(senderId));
+            } else {
+                fDDSCustomCmd.send(id + ": could not queue " + cmd + " transition", to_string(senderId));
+            }
         } else if (cmd == "END") {
-            fDDSCustomCmd.send(id + ": attempting to " + cmd, to_string(senderId));
-            ChangeDeviceState(ToDeviceStateTransition(cmd));
-            fDDSCustomCmd.send(id + ": " + ToStr(GetCurrentDeviceState()), to_string(senderId));
+            if (ChangeDeviceState(ToDeviceStateTransition(cmd))) {
+                fDDSCustomCmd.send(id + ": queued " + cmd + " transition", to_string(senderId));
+            } else {
+                fDDSCustomCmd.send(id + ": could not queue " + cmd + " transition", to_string(senderId));
+            }
             if (ToStr(GetCurrentDeviceState()) == "EXITING") {
                 unique_lock<mutex> lock(fStopMutex);
                 fStopCondition.notify_one();
@@ -363,7 +381,7 @@ auto DDS::WaitForNextState() -> DeviceState
 {
     unique_lock<mutex> lock{fEventsMutex};
     while (fEvents.empty()) {
-        fNewEvent.wait(lock);
+        fNewEvent.wait_for(lock, chrono::milliseconds(50));
     }
 
     auto result = fEvents.front();

fairmq/plugins/DDS/runDDSCommandUI.cxx

@@ -25,7 +25,7 @@ namespace bpo = boost::program_options;
 void PrintControlsHelp()
 {
     cout << "Use keys to control the devices:" << endl;
-    cout << "[c] check states, [o] dump config, [h] help, [p] pause, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device" << endl;
+    cout << "[c] check states, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device, [b] bind, [x] connect" << endl;
     cout << "To quit press Ctrl+C" << endl;
 }
 
@@ -48,7 +48,7 @@ int main(int argc, char* argv[])
 
         if (vm.count("help")) {
             cout << "FairMQ DDS Command UI" << endl << options << endl;
-            cout << "Commands: [c] check state, [o] dump config, [h] help, [p] pause, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device" << endl;
+            cout << "Commands: [c] check state, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device" << endl;
             return EXIT_SUCCESS;
         }
 
@@ -96,14 +96,18 @@ int main(int argc, char* argv[])
                     cout << " > init devices" << endl;
                     ddsCustomCmd.send("INIT DEVICE", topologyPath);
                     break;
+                case 'b':
+                    cout << " > bind" << endl;
+                    ddsCustomCmd.send("BIND", topologyPath);
+                    break;
+                case 'x':
+                    cout << " > connect" << endl;
+                    ddsCustomCmd.send("CONNECT", topologyPath);
+                    break;
                 case 'j':
                     cout << " > init tasks" << endl;
                     ddsCustomCmd.send("INIT TASK", topologyPath);
                     break;
-                case 'p':
-                    cout << " > pause devices" << endl;
-                    ddsCustomCmd.send("PAUSE", topologyPath);
-                    break;
                 case 'r':
                     cout << " > run tasks" << endl;
                     ddsCustomCmd.send("RUN", topologyPath);

fairmq/plugins/PMIx/CMakeLists.txt

@@ -0,0 +1,27 @@
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+set(plugin FairMQPlugin_pmix)
+add_library(${plugin} SHARED
+  ${CMAKE_CURRENT_SOURCE_DIR}/PMIxPlugin.cxx
+  ${CMAKE_CURRENT_SOURCE_DIR}/PMIxPlugin.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/PMIx.hpp
+)
+target_link_libraries(${plugin} FairMQ PMIx::libpmix)
+target_include_directories(${plugin} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+set_target_properties(${plugin} PROPERTIES
+  CXX_VISIBILITY_PRESET hidden
+  VERSION ${PROJECT_GIT_VERSION}
+  SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
+)
+
+install(TARGETS ${plugin}
+  EXPORT ${PROJECT_EXPORT_SET}
+  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
+)

fairmq/plugins/PMIx/PMIx.hpp

@@ -0,0 +1,219 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef PMIX_HPP
+#define PMIX_HPP
+
+#include <cstring>
+#include <limits>
+#include <memory>
+#include <ostream>
+#include <pmix.h>
+#include <stdexcept>
+#include <string.h>
+#include <type_traits>
+#include <utility>
+#include <vector>
+#include <FairMQLogger.h>
+
+// C++ PMIx v2.1 API
+namespace pmix
+{
+
+struct runtime_error : std::runtime_error
+{
+    using std::runtime_error::runtime_error;
+};
+
+using status = pmix_status_t;
+
+using nspace = pmix_nspace_t;
+
+using key = pmix_key_t;
+
+using data_type = pmix_data_type_t;
+
+struct rank
+{
+    enum named : pmix_rank_t
+    {
+        undef = PMIX_RANK_UNDEF,
+        wildcard = PMIX_RANK_WILDCARD,
+        local_node = PMIX_RANK_LOCAL_NODE
+    };
+
+    explicit rank(pmix_rank_t r)
+        : m_value(r)
+    {}
+
+    operator pmix_rank_t() { return m_value; }
+
+  private:
+    pmix_rank_t m_value;
+};
+
+struct proc : pmix_proc_t
+{
+    proc() { PMIX_PROC_CONSTRUCT(static_cast<pmix_proc_t*>(this)); }
+    ~proc() { PMIX_PROC_DESTRUCT(static_cast<pmix_proc_t*>(this)); }
+
+    proc(pmix::nspace ns, pmix::rank r)
+    {
+        PMIX_PROC_LOAD(static_cast<pmix_proc_t*>(this), ns, static_cast<pmix_rank_t>(r));
+    }
+
+    friend std::ostream& operator<<(std::ostream& os, const proc& p)
+    {
+        return os << "nspace=" << p.nspace << ",rank=" << p.rank;
+    }
+};
+
+struct value : pmix_value_t
+{
+    value() { PMIX_VALUE_CONSTRUCT(static_cast<pmix_value_t*>(this)); }
+    ~value() { PMIX_VALUE_DESTRUCT(static_cast<pmix_value_t*>(this)); }
+
+    value(const value& rhs)
+    {
+      LOG(warn) << "copy ctor";
+        status rc;
+        auto lhs(static_cast<pmix_value_t*>(this));
+        PMIX_VALUE_XFER(rc, lhs, static_cast<pmix_value_t*>(const_cast<value*>(&rhs)));
+
+        if (rc != PMIX_SUCCESS) {
+            throw runtime_error("pmix::value copy ctor failed: rc=" + rc);
+        }
+    }
+
+    // template<typename T>
+    // value(const T* val, data_type dt)
+    // {
+        // PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), const_cast<void*>(val), dt);
+    // }
+
+    template<typename T>
+    explicit value(T)
+    {
+        throw runtime_error("Given value type not supported or not yet implemented.");
+    }
+
+    explicit value(const char* val)
+    {
+        PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), const_cast<char*>(val), PMIX_STRING);
+    }
+
+    explicit value(const std::string& val)
+    {
+        PMIX_VALUE_LOAD(
+            static_cast<pmix_value_t*>(this), const_cast<char*>(val.c_str()), PMIX_STRING);
+    }
+};
+
+struct info : pmix_info_t
+{
+    info() { PMIX_INFO_CONSTRUCT(static_cast<pmix_info_t*>(this)); }
+    ~info() { PMIX_INFO_DESTRUCT(static_cast<pmix_info_t*>(this)); }
+
+    template<typename... Args>
+    info(const std::string& k, Args&&... args)
+    {
+        (void)strncpy(key, k.c_str(), PMIX_MAX_KEYLEN);
+        flags = 0;
+
+        pmix::value rhs(std::forward<Args>(args)...);
+        auto lhs(&value);
+        status rc;
+        PMIX_VALUE_XFER(rc, lhs, static_cast<pmix_value_t*>(&rhs));
+
+        if (rc != PMIX_SUCCESS) {
+            throw runtime_error("pmix::info ctor failed: rc=" + std::to_string(rc));
+        }
+    }
+
+    friend std::ostream& operator<<(std::ostream& os, const info& p)
+    {
+        return os << "key=" << p.key << ",value='" << p.value.data.string << "'";
+    }
+};
+
+struct pdata : pmix_pdata_t
+{
+    pdata() { PMIX_PDATA_CONSTRUCT(static_cast<pmix_pdata_t*>(this)); }
+    ~pdata() { PMIX_PDATA_DESTRUCT(static_cast<pmix_pdata_t*>(this)); }
+    pdata(const pdata& rhs)
+    {
+        PMIX_PDATA_XFER(static_cast<pmix_pdata_t*>(this),
+                        static_cast<pmix_pdata_t*>(const_cast<pdata*>(&rhs)));
+    }
+
+    auto set_key(const std::string& new_key) -> void
+    {
+        (void)strncpy(key, new_key.c_str(), PMIX_MAX_KEYLEN);
+    }
+};
+
+auto init(const std::vector<info>& info = {}) -> proc
+{
+    proc res;
+    status rc;
+
+    rc = PMIx_Init(&res, const_cast<pmix::info*>(info.data()), info.size());
+    if (rc != PMIX_SUCCESS) {
+        throw runtime_error("pmix::init() failed: rc=" + std::to_string(rc));
+    }
+
+    return res;
+}
+
+auto initialized() -> bool { return !!PMIx_Initialized(); }
+
+auto get_version() -> const char* { return PMIx_Get_version(); }
+
+auto finalize(const std::vector<info>& info = {}) -> void
+{
+    status rc;
+
+    rc = PMIx_Finalize(info.data(), info.size());
+    if (rc != PMIX_SUCCESS) {
+        throw runtime_error("pmix::finalize() failed: rc=" + std::to_string(rc));
+    }
+}
+
+auto publish(const std::vector<info>& info) -> void
+{
+    status rc;
+
+    rc = PMIx_Publish(info.data(), info.size());
+    if (rc != PMIX_SUCCESS) {
+        throw runtime_error("pmix::publish() failed: rc=" + std::to_string(rc));
+    }
+}
+
+auto fence(const std::vector<proc>& procs = {}, const std::vector<info>& info = {}) -> void
+{
+    status rc;
+
+    rc = PMIx_Fence(procs.data(), procs.size(), info.data(), info.size());
+    if (rc != PMIX_SUCCESS) {
+        throw runtime_error("pmix::fence() failed: rc=" + std::to_string(rc));
+    }
+}
+
+auto lookup(std::vector<pdata>& pdata, const std::vector<info>& info = {}) -> void
+{
+    status rc;
+
+    rc = PMIx_Lookup(pdata.data(), pdata.size(), info.data(), info.size());
+    if (rc != PMIX_SUCCESS) {
+        throw runtime_error("pmix::lookup() failed: rc=" + std::to_string(rc));
+    }
+}
+
+} /* namespace pmix */
+
+#endif /* PMIX_HPP */

fairmq/plugins/PMIx/PMIxPlugin.cxx

@@ -0,0 +1,146 @@
+/********************************************************************************
+ *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "PMIxPlugin.h"
+
+#include <boost/algorithm/string/join.hpp>
+#include <fairmq/Tools.h>
+#include <stdexcept>
+
+namespace fair
+{
+namespace mq
+{
+namespace plugins
+{
+
+PMIxPlugin::PMIxPlugin(const std::string& name,
+                       const Plugin::Version version,
+                       const std::string& maintainer,
+                       const std::string& homepage,
+                       PluginServices* pluginServices)
+    : Plugin(name, version, maintainer, homepage, pluginServices)
+    , fPid(getpid())
+{
+    SubscribeToDeviceStateChange([&](DeviceState newState) {
+        switch (newState) {
+            case DeviceState::Connecting:
+                Init();
+                Publish();
+                Fence();
+                Lookup();
+                break;
+        case DeviceState::Exiting:
+            UnsubscribeFromDeviceStateChange();
+            break;
+        default:
+            break;
+        }
+    });
+}
+
+PMIxPlugin::~PMIxPlugin()
+{
+    while (pmix::initialized()) {
+        try {
+            pmix::finalize();
+            LOG(debug) << PMIxClient() << " pmix::finalize() OK";
+        } catch (const pmix::runtime_error& e) {
+            LOG(debug) << PMIxClient() << " pmix::finalize() failed: " << e.what();
+        }
+    }
+}
+
+auto PMIxPlugin::PMIxClient() const -> std::string
+{
+    std::stringstream ss;
+    ss << "PMIx client(pid=" << fPid << ")";
+    return ss.str();
+}
+
+auto PMIxPlugin::Init() -> void
+{
+    if (!pmix::initialized()) {
+        fProc = pmix::init();
+        LOG(debug) << PMIxClient() << " pmix::init() OK: " << fProc
+                   << ",version=" << pmix::get_version();
+    }
+}
+
+auto PMIxPlugin::Publish() -> void
+{
+    auto channels(GetChannelInfo());
+    std::vector<pmix::info> info;
+
+    for (const auto& c : channels) {
+        std::string methodKey{"chans." + c.first + "." + std::to_string(c.second - 1) + ".method"};
+        if (GetProperty<std::string>(methodKey) == "bind") {
+            for (int i = 0; i < c.second; ++i) {
+                std::string addressKey{"chans." + c.first + "." + std::to_string(i) + ".address"};
+                info.emplace_back(addressKey, GetProperty<std::string>(addressKey));
+                LOG(debug) << PMIxClient() << info.back();
+            }
+        }
+    }
+
+    if (info.size() > 0) {
+        pmix::publish(info);
+        LOG(debug) << PMIxClient() << " pmix::publish() OK: published "
+                   << info.size() << " binding channels.";
+    }
+}
+
+auto PMIxPlugin::Fence() -> void
+{
+    pmix::proc all(fProc);
+    all.rank = pmix::rank::wildcard;
+
+    pmix::fence({all});
+}
+
+auto PMIxPlugin::Lookup() -> void
+{
+    auto channels(GetChannelInfo());
+    std::vector<pmix::pdata> pdata;
+
+    for (const auto& c : channels) {
+        std::string methodKey{"chans." + c.first + "." + std::to_string(c.second - 1) + ".method"};
+        if (GetProperty<std::string>(methodKey) == "connect") {
+            for (int i = 0; i < c.second; ++i) {
+                std::string addressKey{"chans." + c.first + "." + std::to_string(i) + ".address"};
+                pdata.emplace_back();
+                pdata.back().set_key(addressKey);
+            }
+        }
+    }
+
+    if (pdata.size() > 0) {
+        pmix::lookup(pdata);
+        LOG(debug) << PMIxClient() << " pmix::lookup() OK";
+    }
+
+    LOG(info) << pdata.size();
+
+    for (const auto& p : pdata) {
+        if (p.value.type == PMIX_UNDEF) {
+            LOG(debug) << PMIxClient() << " pmix::lookup() not found: key=" << p.key;
+        } else if (p.value.type == PMIX_STRING) {
+            LOG(debug) << PMIxClient() << " pmix::lookup() found: key=" << p.key << ",value=" << p.value.data.string;
+            SetProperty<std::string>(p.key, p.value.data.string);
+            LOG(info) << GetProperty<std::string>(p.key);
+        } else {
+            LOG(debug) << PMIxClient() << " pmix::lookup() wrong type returned: key=" << p.key << ",type=" << p.value.type;
+        }
+    }
+
+    LOG(info) << pdata.size();
+}
+
+} /* namespace plugins */
+} /* namespace mq */
+} /* namespace fair */

fairmq/plugins/PMIx/PMIxPlugin.h

@@ -0,0 +1,77 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_PLUGINS_PMIX
+#define FAIR_MQ_PLUGINS_PMIX
+
+#include "PMIx.hpp"
+
+#include <fairmq/Plugin.h>
+#include <fairmq/Version.h>
+#include <FairMQLogger.h>
+
+#include <string>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <sys/types.h>
+#include <unistd.h>
+#include <vector>
+
+namespace fair
+{
+namespace mq
+{
+namespace plugins
+{
+
+class PMIxPlugin : public Plugin
+{
+  public:
+    PMIxPlugin(const std::string& name,
+         const Plugin::Version version,
+         const std::string& maintainer,
+         const std::string& homepage,
+         PluginServices* pluginServices);
+    ~PMIxPlugin();
+    auto PMIxClient() const -> std::string;
+
+  private:
+    pmix::proc fProc;
+    pid_t fPid;
+
+    auto Init() -> void;
+    auto Publish() -> void;
+    auto Fence() -> void;
+    auto Lookup() -> void;
+};
+
+Plugin::ProgOptions PMIxProgramOptions()
+{
+    boost::program_options::options_description options{"PMIx Plugin"};
+    options.add_options()(
+        "pmix-dummy", boost::program_options::value<int>()->default_value(0), "Dummy.");
+    return options;
+}
+
+REGISTER_FAIRMQ_PLUGIN(
+    PMIxPlugin,                                  // Class name
+    pmix,                                        // Plugin name (string, lower case chars only)
+    (Plugin::Version{FAIRMQ_VERSION_MAJOR,
+                     FAIRMQ_VERSION_MINOR,
+                     FAIRMQ_VERSION_PATCH}),     // Version
+    "FairRootGroup <fairroot@gsi.de>",           // Maintainer
+    "https://github.com/FairRootGroup/FairMQ",   // Homepage
+    PMIxProgramOptions                           // custom program options for the plugin
+)
+
+} /* namespace plugins */
+} /* namespace mq */
+} /* namespace fair */
+
+#endif /* FAIR_MQ_PLUGINS_PMIX */

fairmq/run/startMQBenchmark.sh.in

@@ -61,7 +61,7 @@ SAMPLER+=" --msg-size $msgSize"
 SAMPLER+=" --num-parts 1"
 # SAMPLER+=" --msg-rate 1000"
 SAMPLER+=" --max-iterations $maxIterations"
-SAMPLER+=" --channel-config name=data,type=push,method=bind,address=tcp://127.0.0.1:5555"
+SAMPLER+=" --channel-config name=data,type=pair,method=bind,address=tcp://127.0.0.1:5555"
 xterm -geometry 90x50+0+0 -hold -e $affinitySamp @CMAKE_CURRENT_BINARY_DIR@/$SAMPLER &
 echo ""
 echo "started: xterm -geometry 90x50+0+0 -hold -e $affinitySamp @CMAKE_CURRENT_BINARY_DIR@/$SAMPLER"
@@ -75,7 +75,7 @@ SINK+=" --transport $transport"
 SINK+=" --severity debug"
 SINK+=" --multipart false"
 SINK+=" --max-iterations $maxIterations"
-SINK+=" --channel-config name=data,type=pull,method=connect,address=tcp://127.0.0.1:5555"
+SINK+=" --channel-config name=data,type=pair,method=connect,address=tcp://127.0.0.1:5555"
 xterm -geometry 90x50+550+0 -hold -e $affinitySink @CMAKE_CURRENT_BINARY_DIR@/$SINK &
 echo ""
 echo "started: xterm -geometry 90x50+550+0 -hold -e $affinitySink @CMAKE_CURRENT_BINARY_DIR@/$SINK"

fairmq/shmem/FairMQMessageSHM.cxx

@@ -350,6 +350,7 @@ void FairMQMessageSHM::CloseMessage()
                 }
                 if (fRegionPtr)
                 {
+                    // LOG(debug) << "sending ack";
                     if (fRegionPtr->fQueue->timed_send(&block, sizeof(RegionBlock), 0, sndTill))
                     {
                         success = true;

fairmq/shmem/FairMQPollerSHM.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**