Mirror/FairMQ
1
0
Fork 0
mirror of https://github.com/FairRootGroup/FairMQ.git synced 2025-10-16 10:01:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding multiple transports support & other fixes:

Browse Source 
- Avoid polling when only one input channel is used.
 - Send only handles for shared memory transport.
 - Avoid waiting in the rate logger thread when nothing to log.
 - Hide warnings from generated files
 - Fix #483
This commit is contained in:
Alexey Rybalchenko
2017-01-13 15:53:25 +01:00
committed by Mohammad Al-Turany
parent e53ad151a7
commit c66fd6fe91
39 changed files with 1840 additions and 1189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fairmq/shmem/FairMQContextSHM.cxx
View File

@@ -7,6 +7,8 @@
********************************************************************************/
#include <zmq.h>
#include <cstdio>
#include <boost/interprocess/managed_shared_memory.hpp>
#include "FairMQLogger.h"
@@ -46,11 +48,11 @@ FairMQContextSHM::~FairMQContextSHM()
if (boost::interprocess::shared_memory_object::remove("FairMQSharedMemory"))
{
LOG(INFO) << "Successfully removed shared memory after the device has stopped.";
printf("Successfully removed shared memory after the device has stopped.\n");
}
else
{
LOG(INFO) << "Did not remove shared memory after the device stopped. Still in use?";
printf("Did not remove shared memory after the device stopped. Already removed?\n");
}
}

201
fairmq/shmem/FairMQMessageSHM.cxx
View File

@@ -14,45 +14,60 @@
using namespace std;
using namespace FairMQ::shmem;
uint64_t FairMQMessageSHM::fMessageID = 0;
string FairMQMessageSHM::fDeviceID = string();
static FairMQ::Transport gTransportType = FairMQ::Transport::SHM;
// uint64_t FairMQMessageSHM::fMessageID = 0;
// string FairMQMessageSHM::fDeviceID = string();
atomic<bool> FairMQMessageSHM::fInterrupted(false);
FairMQMessageSHM::FairMQMessageSHM()
: fMessage()
, fOwner(nullptr)
, fReceiving(false)
// , fOwner(nullptr)
// , fReceiving(false)
, fQueued(false)
, fMetaCreated(false)
, fHandle()
, fChunkSize(0)
, fLocalPtr(nullptr)
{
if (zmq_msg_init(&fMessage) != 0)
{
LOG(ERROR) << "failed initializing message, reason: " << zmq_strerror(errno);
}
fMetaCreated = true;
}
void FairMQMessageSHM::StringDeleter(void* /*data*/, void* str)
{
delete static_cast<string*>(str);
}
// void FairMQMessageSHM::StringDeleter(void* /*data*/, void* str)
// {
// delete static_cast<string*>(str);
// }
FairMQMessageSHM::FairMQMessageSHM(const size_t size)
: fMessage()
, fOwner(nullptr)
, fReceiving(false)
// , fOwner(nullptr)
// , fReceiving(false)
, fQueued(false)
, fMetaCreated(false)
, fHandle()
, fChunkSize(0)
, fLocalPtr(nullptr)
{
InitializeChunk(size);
}
FairMQMessageSHM::FairMQMessageSHM(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
: fMessage()
, fOwner(nullptr)
, fReceiving(false)
// , fOwner(nullptr)
// , fReceiving(false)
, fQueued(false)
, fMetaCreated(false)
, fHandle()
, fChunkSize(0)
, fLocalPtr(nullptr)
{
if (InitializeChunk(size))
{
memcpy(fOwner->fPtr->GetData(), data, size);
memcpy(fLocalPtr, data, size);
if (ffn)
{
ffn(data, hint);
@@ -66,66 +81,76 @@ FairMQMessageSHM::FairMQMessageSHM(void* data, const size_t size, fairmq_free_fn
bool FairMQMessageSHM::InitializeChunk(const size_t size)
{
string chunkID = fDeviceID + "c" + to_string(fMessageID);
string* ownerID = new string(fDeviceID + "o" + to_string(fMessageID));
// string chunkID = fDeviceID + "c" + to_string(fMessageID);
// string* ownerID = new string(fDeviceID + "o" + to_string(fMessageID));
bool success = false;
while (!success)
while (!fHandle)
{
try
{
fOwner = Manager::Instance().Segment()->construct<ShPtrOwner>(ownerID->c_str())(
make_managed_shared_ptr(Manager::Instance().Segment()->construct<Chunk>(chunkID.c_str())(size),
*(Manager::Instance().Segment())));
success = true;
fLocalPtr = Manager::Instance().Segment()->allocate(size);
// fOwner = Manager::Instance().Segment()->construct<ShPtrOwner>(ownerID->c_str())(
// make_managed_shared_ptr(Manager::Instance().Segment()->construct<Chunk>(chunkID.c_str())(size),
// *(Manager::Instance().Segment())));
}
catch (bipc::bad_alloc& ba)
{
LOG(WARN) << "Shared memory full...";
// LOG(WARN) << "Shared memory full...";
this_thread::sleep_for(chrono::milliseconds(50));
if (fInterrupted)
{
break;
return false;
}
else
{
continue;
}
}
fHandle = Manager::Instance().Segment()->get_handle_from_address(fLocalPtr);
}
if (success)
fChunkSize = size;
if (zmq_msg_init_size(&fMessage, sizeof(MetaHeader)) != 0)
{
if (zmq_msg_init_data(&fMessage, const_cast<char*>(ownerID->c_str()), ownerID->length(), StringDeleter, ownerID) != 0)
{
LOG(ERROR) << "failed initializing meta message, reason: " << zmq_strerror(errno);
}
++fMessageID;
LOG(ERROR) << "failed initializing meta message, reason: " << zmq_strerror(errno);
return false;
}
MetaHeader* metaPtr = new(zmq_msg_data(&fMessage)) MetaHeader();
metaPtr->fSize = size;
metaPtr->fHandle = fHandle;
return success;
// if (zmq_msg_init_data(&fMessage, const_cast<char*>(ownerID->c_str()), ownerID->length(), StringDeleter, ownerID) != 0)
// {
// LOG(ERROR) << "failed initializing meta message, reason: " << zmq_strerror(errno);
// }
fMetaCreated = true;
// ++fMessageID;
return true;
}
void FairMQMessageSHM::Rebuild()
{
CloseMessage();
fReceiving = false;
// fReceiving = false;
fQueued = false;
if (zmq_msg_init(&fMessage) != 0)
{
LOG(ERROR) << "failed initializing message, reason: " << zmq_strerror(errno);
}
fMetaCreated = true;
}
void FairMQMessageSHM::Rebuild(const size_t size)
{
CloseMessage();
fReceiving = false;
// fReceiving = false;
fQueued = false;
InitializeChunk(size);
@@ -135,12 +160,12 @@ void FairMQMessageSHM::Rebuild(void* data, const size_t size, fairmq_free_fn* ff
{
CloseMessage();
fReceiving = false;
// fReceiving = false;
fQueued = false;
if (InitializeChunk(size))
{
memcpy(fOwner->fPtr->GetData(), data, size);
memcpy(fLocalPtr, data, size);
if (ffn)
{
ffn(data, hint);
@@ -159,27 +184,42 @@ void* FairMQMessageSHM::GetMessage()
void* FairMQMessageSHM::GetData()
{
if (fOwner)
if (fLocalPtr)
{
return fOwner->fPtr->GetData();
return fLocalPtr;
}
else if (fHandle)
{
return Manager::Instance().Segment()->get_address_from_handle(fHandle);
}
else
{
LOG(ERROR) << "Trying to get data of an empty shared memory message";
exit(EXIT_FAILURE);
// LOG(ERROR) << "Trying to get data of an empty shared memory message";
return nullptr;
}
// if (fOwner)
// {
// return fOwner->fPtr->GetData();
// }
// else
// {
// LOG(ERROR) << "Trying to get data of an empty shared memory message";
// exit(EXIT_FAILURE);
// }
}
size_t FairMQMessageSHM::GetSize()
{
if (fOwner)
{
return fOwner->fPtr->GetSize();
}
else
{
return 0;
}
return fChunkSize;
// if (fOwner)
// {
// return fOwner->fPtr->GetSize();
// }
// else
// {
// return 0;
// }
}
void FairMQMessageSHM::SetMessage(void*, const size_t)
@@ -187,21 +227,26 @@ void FairMQMessageSHM::SetMessage(void*, const size_t)
// dummy method to comply with the interface. functionality not allowed in zeromq.
}
void FairMQMessageSHM::SetDeviceId(const string& deviceId)
void FairMQMessageSHM::SetDeviceId(const string& /*deviceId*/)
{
fDeviceID = deviceId;
// fDeviceID = deviceId;
}
FairMQ::Transport FairMQMessageSHM::GetType() const
{
return gTransportType;
}
void FairMQMessageSHM::Copy(const unique_ptr<FairMQMessage>& msg)
{
if (!fOwner)
if (!fHandle)
{
FairMQ::shmem::ShPtrOwner* otherOwner = static_cast<FairMQMessageSHM*>(msg.get())->fOwner;
if (otherOwner)
bipc::managed_shared_memory::handle_t otherHandle = static_cast<FairMQMessageSHM*>(msg.get())->fHandle;
if (otherHandle)
{
if (InitializeChunk(otherOwner->fPtr->GetSize()))
if (InitializeChunk(msg->GetSize()))
{
memcpy(fOwner->fPtr->GetData(), otherOwner->fPtr->GetData(), otherOwner->fPtr->GetSize());
memcpy(GetData(), msg->GetData(), msg->GetSize());
}
}
else
@@ -266,31 +311,35 @@ void FairMQMessageSHM::Copy(const unique_ptr<FairMQMessage>& msg)
void FairMQMessageSHM::CloseMessage()
{
if (fReceiving)
{
if (fOwner)
// if (fReceiving)
// {
// if (fOwner)
// {
// Manager::Instance().Segment()->destroy_ptr(fOwner);
// fOwner = nullptr;
// }
// else
// {
// LOG(ERROR) << "No shared pointer owner when closing a received message";
// }
// }
// else
// {
if (fHandle && !fQueued)
{
Manager::Instance().Segment()->destroy_ptr(fOwner);
fOwner = nullptr;
// LOG(WARN) << "Destroying unsent message";
// Manager::Instance().Segment()->destroy_ptr(fHandle);
Manager::Instance().Segment()->deallocate(Manager::Instance().Segment()->get_address_from_handle(fHandle));
fHandle = 0;
}
else
{
LOG(ERROR) << "No shared pointer owner when closing a received message";
}
}
else
{
if (fOwner && !fQueued)
{
LOG(WARN) << "Destroying unsent message";
Manager::Instance().Segment()->destroy_ptr(fOwner);
fOwner = nullptr;
}
}
// }
if (zmq_msg_close(&fMessage) != 0)
if (fMetaCreated)
{
LOG(ERROR) << "failed closing message, reason: " << zmq_strerror(errno);
if (zmq_msg_close(&fMessage) != 0)
{
LOG(ERROR) << "failed closing message, reason: " << zmq_strerror(errno);
}
}
}

16
fairmq/shmem/FairMQMessageSHM.h
View File

@@ -42,22 +42,28 @@ class FairMQMessageSHM : public FairMQMessage
virtual void SetDeviceId(const std::string& deviceId);
virtual FairMQ::Transport GetType() const;
virtual void Copy(const std::unique_ptr<FairMQMessage>& msg);
void CloseMessage();
virtual ~FairMQMessageSHM();
static void StringDeleter(void* data, void* str);
// static void StringDeleter(void* data, void* str);
private:
zmq_msg_t fMessage;
FairMQ::shmem::ShPtrOwner* fOwner;
static uint64_t fMessageID;
static std::string fDeviceID;
bool fReceiving;
// FairMQ::shmem::ShPtrOwner* fOwner;
// static uint64_t fMessageID;
// static std::string fDeviceID;
// bool fReceiving;
bool fQueued;
bool fMetaCreated;
static std::atomic<bool> fInterrupted;
bipc::managed_shared_memory::handle_t fHandle;
size_t fChunkSize;
void* fLocalPtr;
};
#endif /* FAIRMQMESSAGESHM_H_ */

23
fairmq/shmem/FairMQPollerSHM.cxx
View File

@@ -51,7 +51,7 @@ FairMQPollerSHM::FairMQPollerSHM(const vector<FairMQChannel>& channels)
}
else
{
LOG(ERROR) << "invalid poller configuration, exiting.";
LOG(ERROR) << "shmem: invalid poller configuration, exiting.";
exit(EXIT_FAILURE);
}
}
@@ -105,7 +105,7 @@ FairMQPollerSHM::FairMQPollerSHM(const unordered_map<string, vector<FairMQChanne
}
else
{
LOG(ERROR) << "invalid poller configuration, exiting.";
LOG(ERROR) << "shmem: invalid poller configuration, exiting.";
exit(EXIT_FAILURE);
}
}
@@ -113,8 +113,8 @@ FairMQPollerSHM::FairMQPollerSHM(const unordered_map<string, vector<FairMQChanne
}
catch (const std::out_of_range& oor)
{
LOG(ERROR) << "At least one of the provided channel keys for poller initialization is invalid";
LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
LOG(ERROR) << "shmem: at least one of the provided channel keys for poller initialization is invalid";
LOG(ERROR) << "shmem: out of range error: " << oor.what() << '\n';
exit(EXIT_FAILURE);
}
}
@@ -153,7 +153,7 @@ FairMQPollerSHM::FairMQPollerSHM(const FairMQSocket& cmdSocket, const FairMQSock
}
else
{
LOG(ERROR) << "invalid poller configuration, exiting.";
LOG(ERROR) << "shmem: invalid poller configuration, exiting.";
exit(EXIT_FAILURE);
}
}
@@ -164,11 +164,12 @@ void FairMQPollerSHM::Poll(const int timeout)
{
if (errno == ETERM)
{
LOG(DEBUG) << "polling exited, reason: " << zmq_strerror(errno);
LOG(DEBUG) << "shmem: polling exited, reason: " << zmq_strerror(errno);
}
else
{
LOG(ERROR) << "polling failed, reason: " << zmq_strerror(errno);
LOG(ERROR) << "shmem: polling failed, reason: " << zmq_strerror(errno);
throw std::runtime_error("shmem: polling failed");
}
}
}
@@ -206,8 +207,8 @@ bool FairMQPollerSHM::CheckInput(const string channelKey, const int index)
}
catch (const std::out_of_range& oor)
{
LOG(ERROR) << "Invalid channel key: \"" << channelKey << "\"";
LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
LOG(ERROR) << "shmem: invalid channel key: \"" << channelKey << "\"";
LOG(ERROR) << "shmem: out of range error: " << oor.what() << '\n';
exit(EXIT_FAILURE);
}
}
@@ -225,8 +226,8 @@ bool FairMQPollerSHM::CheckOutput(const string channelKey, const int index)
}
catch (const std::out_of_range& oor)
{
LOG(ERROR) << "Invalid channel key: \"" << channelKey << "\"";
LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
LOG(ERROR) << "shmem: Invalid channel key: \"" << channelKey << "\"";
LOG(ERROR) << "shmem: out of range error: " << oor.what() << '\n';
exit(EXIT_FAILURE);
}
}

104
fairmq/shmem/FairMQShmManager.h
View File

@@ -121,62 +121,68 @@ class Manager
bipc::managed_shared_memory* fSegment;
};
class Chunk
struct alignas(16) MetaHeader
{
public:
Chunk()
: fHandle()
, fSize(0)
{
}
Chunk(const size_t size)
: fHandle()
, fSize(size)
{
void* ptr = Manager::Instance().Segment()->allocate(size);
fHandle = Manager::Instance().Segment()->get_handle_from_address(ptr);
}
~Chunk()
{
Manager::Instance().Segment()->deallocate(Manager::Instance().Segment()->get_address_from_handle(fHandle));
}
// bipc::managed_shared_memory::handle_t GetHandle() const
// {
// return fHandle;
// }
void* GetData() const
{
return Manager::Instance().Segment()->get_address_from_handle(fHandle);
}
size_t GetSize() const
{
return fSize;
}
private:
uint64_t fSize;
bipc::managed_shared_memory::handle_t fHandle;
size_t fSize;
};
typedef bipc::managed_shared_ptr<Chunk, bipc::managed_shared_memory>::type ShPtrType;
// class Chunk
// {
// public:
// Chunk()
// : fHandle()
// , fSize(0)
// {
// }
struct ShPtrOwner
{
ShPtrOwner(const ShPtrType& other)
: fPtr(other)
{}
// Chunk(const size_t size)
// : fHandle()
// , fSize(size)
// {
// void* ptr = Manager::Instance().Segment()->allocate(size);
// fHandle = Manager::Instance().Segment()->get_handle_from_address(ptr);
// }
ShPtrOwner(const ShPtrOwner& other)
: fPtr(other.fPtr)
{}
// ~Chunk()
// {
// Manager::Instance().Segment()->deallocate(Manager::Instance().Segment()->get_address_from_handle(fHandle));
// }
ShPtrType fPtr;
};
// bipc::managed_shared_memory::handle_t GetHandle() const
// {
// return fHandle;
// }
// void* GetData() const
// {
// return Manager::Instance().Segment()->get_address_from_handle(fHandle);
// }
// size_t GetSize() const
// {
// return fSize;
// }
// private:
// bipc::managed_shared_memory::handle_t fHandle;
// size_t fSize;
// };
// typedef bipc::managed_shared_ptr<Chunk, bipc::managed_shared_memory>::type ShPtrType;
// struct ShPtrOwner
// {
// ShPtrOwner(const ShPtrType& other)
// : fPtr(other)
// {}
// ShPtrOwner(const ShPtrOwner& other)
// : fPtr(other.fPtr)
// {}
// ShPtrType fPtr;
// };
} // namespace shmem

406
fairmq/shmem/FairMQSocketSHM.cxx
View File

@@ -19,6 +19,7 @@ using namespace FairMQ::shmem;
// Context to hold the ZeroMQ sockets
unique_ptr<FairMQContextSHM> FairMQSocketSHM::fContext; // = unique_ptr<FairMQContextSHM>(new FairMQContextSHM(1));
bool FairMQSocketSHM::fContextInitialized = false;
atomic<bool> FairMQSocketSHM::fInterrupted(false);
FairMQSocketSHM::FairMQSocketSHM(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/)
: FairMQSocket(ZMQ_SNDMORE, ZMQ_RCVMORE, ZMQ_DONTWAIT)
@@ -57,22 +58,22 @@ FairMQSocketSHM::FairMQSocketSHM(const string& type, const string& name, const i
// Tell socket to try and send/receive outstanding messages for <linger> milliseconds before terminating.
// Default value for ZeroMQ is -1, which is to wait forever.
int linger = 500;
int linger = 1000;
if (zmq_setsockopt(fSocket, ZMQ_LINGER, &linger, sizeof(linger)) != 0)
{
LOG(ERROR) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
}
int kernelSndSize = 10000;
if (zmq_setsockopt(fSocket, ZMQ_SNDBUF, &kernelSndSize, sizeof(kernelSndSize)) != 0)
int sndTimeout = 700;
if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &sndTimeout, sizeof(sndTimeout)) != 0)
{
LOG(ERROR) << "Failed setting ZMQ_SNDBUF socket option, reason: " << zmq_strerror(errno);
LOG(ERROR) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
}
int kernelRcvSize = 10000;
if (zmq_setsockopt(fSocket, ZMQ_RCVBUF, &kernelRcvSize, sizeof(kernelRcvSize)) != 0)
int rcvTimeout = 700;
if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &rcvTimeout, sizeof(rcvTimeout)) != 0)
{
LOG(ERROR) << "Failed setting ZMQ_RCVBUF socket option, reason: " << zmq_strerror(errno);
LOG(ERROR) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
}
if (type == "sub")
@@ -119,105 +120,183 @@ void FairMQSocketSHM::Connect(const string& address)
}
}
int FairMQSocketSHM::Send(FairMQMessage* msg, const string& flag)
int FairMQSocketSHM::Send(FairMQMessagePtr& msg, const int flags)
{
return Send(msg, GetConstant(flag));
}
int FairMQSocketSHM::Send(FairMQMessage* msg, const int flags)
{
int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
if (nbytes >= 0)
int nbytes = -1;
while (true && !fInterrupted)
{
static_cast<FairMQMessageSHM*>(msg)->fReceiving = false;
static_cast<FairMQMessageSHM*>(msg)->fQueued = true;
size_t size = msg->GetSize();
fBytesTx += size;
++fMessagesTx;
return size;
}
if (zmq_errno() == EAGAIN)
{
return -2;
}
if (zmq_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
return nbytes;
}
int64_t FairMQSocketSHM::Send(const vector<FairMQMessagePtr>& msgVec, const int flags)
{
// Sending vector typicaly handles more then one part
if (msgVec.size() > 1)
{
int64_t totalSize = 0;
for (unsigned int i = 0; i < msgVec.size() - 1; ++i)
nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
if (nbytes == 0)
{
int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec[i]->GetMessage()), fSocket, ZMQ_SNDMORE|flags);
if (nbytes >= 0)
{
static_cast<FairMQMessageSHM*>(msgVec[i].get())->fReceiving = false;
static_cast<FairMQMessageSHM*>(msgVec[i].get())->fQueued = true;
size_t size = msgVec[i]->GetSize();
return nbytes;
}
else if (nbytes > 0)
{
// static_cast<FairMQMessageSHM*>(msg.get())->fReceiving = false;
static_cast<FairMQMessageSHM*>(msg.get())->fQueued = true;
totalSize += size;
fBytesTx += size;
size_t size = msg->GetSize();
fBytesTx += size;
++fMessagesTx;
return size;
}
else if (zmq_errno() == EAGAIN)
{
if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
{
continue;
}
else
{
if (zmq_errno() == EAGAIN)
{
return -2;
}
if (zmq_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
return nbytes;
return -2;
}
}
int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec.back()->GetMessage()), fSocket, flags);
if (nbytes >= 0)
else if (zmq_errno() == ETERM)
{
static_cast<FairMQMessageSHM*>(msgVec.back().get())->fReceiving = false;
static_cast<FairMQMessageSHM*>(msgVec.back().get())->fQueued = true;
size_t size = msgVec.back()->GetSize();
totalSize += size;
fBytesTx += size;
LOG(INFO) << "terminating socket " << fId;
return -1;
}
else
{
if (zmq_errno() == EAGAIN)
{
return -2;
}
if (zmq_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
return nbytes;
}
}
// store statistics on how many messages have been sent (handle all parts as a single message)
++fMessagesTx;
return totalSize;
} // If there's only one part, send it as a regular message
else if (msgVec.size() == 1)
return -1;
}
int FairMQSocketSHM::Receive(FairMQMessagePtr& msg, const int flags)
{
int nbytes = -1;
zmq_msg_t* msgPtr = static_cast<zmq_msg_t*>(msg->GetMessage());
while (true)
{
return Send(msgVec.back().get(), flags);
nbytes = zmq_msg_recv(msgPtr, fSocket, flags);
if (nbytes == 0)
{
++fMessagesRx;
return nbytes;
}
else if (nbytes > 0)
{
// string ownerID(static_cast<char*>(zmq_msg_data(msgPtr)), zmq_msg_size(msgPtr));
// ShPtrOwner* owner = Manager::Instance().Segment()->find<ShPtrOwner>(ownerID.c_str()).first;
MetaHeader* hdr = static_cast<MetaHeader*>(zmq_msg_data(msgPtr));
size_t size = 0;
if (hdr->fHandle)
{
static_cast<FairMQMessageSHM*>(msg.get())->fHandle = hdr->fHandle;
static_cast<FairMQMessageSHM*>(msg.get())->fChunkSize = hdr->fSize;
// static_cast<FairMQMessageSHM*>(msg.get())->fOwner = owner;
// static_cast<FairMQMessageSHM*>(msg.get())->fReceiving = true;
size = msg->GetSize();
fBytesRx += size;
++fMessagesRx;
return size;
}
else
{
LOG(ERROR) << "Received meta data, but could not find corresponding chunk";
return -1;
}
}
else if (zmq_errno() == EAGAIN)
{
if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
{
continue;
}
else
{
return -2;
}
}
else if (zmq_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
else
{
LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
return nbytes;
}
}
}
int64_t FairMQSocketSHM::Send(vector<FairMQMessagePtr>& msgVec, const int flags)
{
const unsigned int vecSize = msgVec.size();
// Sending vector typicaly handles more then one part
if (vecSize > 1)
{
int64_t totalSize = 0;
int nbytes = -1;
bool repeat = false;
while (true && !fInterrupted)
{
for (unsigned int i = 0; i < vecSize; ++i)
{
nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec[i]->GetMessage()),
fSocket,
(i < vecSize - 1) ? ZMQ_SNDMORE|flags : flags);
if (nbytes >= 0)
{
static_cast<FairMQMessageSHM*>(msgVec[i].get())->fQueued = true;
// static_cast<FairMQMessageSHM*>(msgVec[i].get())->fReceiving = false;
// static_cast<FairMQMessageSHM*>(msgVec[i].get())->fQueued = true;
size_t size = msgVec[i]->GetSize();
totalSize += size;
}
else
{
// according to ZMQ docs, this can only occur for the first part
if (zmq_errno() == EAGAIN)
{
if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
{
repeat = true;
break;
}
else
{
return -2;
}
}
if (zmq_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
return nbytes;
}
}
if (repeat)
{
continue;
}
// store statistics on how many messages have been sent (handle all parts as a single message)
++fMessagesTx;
fBytesTx += totalSize;
return totalSize;
}
return -1;
} // If there's only one part, send it as a regular message
else if (vecSize == 1)
{
return Send(msgVec.back(), flags);
}
else // if the vector is empty, something might be wrong
{
@@ -226,112 +305,91 @@ int64_t FairMQSocketSHM::Send(const vector<FairMQMessagePtr>& msgVec, const int
}
}
int FairMQSocketSHM::Receive(FairMQMessage* msg, const string& flag)
{
return Receive(msg, GetConstant(flag));
}
int FairMQSocketSHM::Receive(FairMQMessage* msg, const int flags)
{
zmq_msg_t* msgPtr = static_cast<zmq_msg_t*>(msg->GetMessage());
int nbytes = zmq_msg_recv(msgPtr, fSocket, flags);
if (nbytes == 0)
{
++fMessagesRx;
return nbytes;
}
else if (nbytes > 0)
{
string ownerID(static_cast<char*>(zmq_msg_data(msgPtr)), zmq_msg_size(msgPtr));
ShPtrOwner* owner = Manager::Instance().Segment()->find<ShPtrOwner>(ownerID.c_str()).first;
size_t size = 0;
if (owner)
{
static_cast<FairMQMessageSHM*>(msg)->fOwner = owner;
static_cast<FairMQMessageSHM*>(msg)->fReceiving = true;
size = msg->GetSize();
fBytesRx += size;
++fMessagesRx;
return size;
}
else
{
LOG(ERROR) << "Received meta data, but could not find corresponding chunk";
return -1;
}
}
if (zmq_errno() == EAGAIN)
{
return -2;
}
if (zmq_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
return nbytes;
}
int64_t FairMQSocketSHM::Receive(vector<FairMQMessagePtr>& msgVec, const int flags)
{
// Warn if the vector is filled before Receive() and empty it.
if (msgVec.size() > 0)
{
LOG(WARN) << "Message vector contains elements before Receive(), they will be deleted!";
msgVec.clear();
}
int64_t totalSize = 0;
int64_t more = 0;
bool repeat = false;
do
while (true)
{
FairMQMessagePtr part(new FairMQMessageSHM());
zmq_msg_t* msgPtr = static_cast<zmq_msg_t*>(part->GetMessage());
int nbytes = zmq_msg_recv(msgPtr, fSocket, flags);
if (nbytes == 0)
// Warn if the vector is filled before Receive() and empty it.
if (msgVec.size() > 0)
{
msgVec.push_back(move(part));
LOG(WARN) << "Message vector contains elements before Receive(), they will be deleted!";
msgVec.clear();
}
else if (nbytes > 0)
totalSize = 0;
more = 0;
repeat = false;
do
{
string ownerID(static_cast<char*>(zmq_msg_data(msgPtr)), zmq_msg_size(msgPtr));
ShPtrOwner* owner = Manager::Instance().Segment()->find<ShPtrOwner>(ownerID.c_str()).first;
size_t size = 0;
if (owner)
FairMQMessagePtr part(new FairMQMessageSHM());
zmq_msg_t* msgPtr = static_cast<zmq_msg_t*>(part->GetMessage());
int nbytes = zmq_msg_recv(msgPtr, fSocket, flags);
if (nbytes == 0)
{
static_cast<FairMQMessageSHM*>(part.get())->fOwner = owner;
static_cast<FairMQMessageSHM*>(part.get())->fReceiving = true;
size = part->GetSize();
msgVec.push_back(move(part));
}
else if (nbytes > 0)
{
// string ownerID(static_cast<char*>(zmq_msg_data(msgPtr)), zmq_msg_size(msgPtr));
// ShPtrOwner* owner = Manager::Instance().Segment()->find<ShPtrOwner>(ownerID.c_str()).first;
MetaHeader* hdr = static_cast<MetaHeader*>(zmq_msg_data(msgPtr));
size_t size = 0;
if (hdr->fHandle)
{
static_cast<FairMQMessageSHM*>(part.get())->fHandle = hdr->fHandle;
static_cast<FairMQMessageSHM*>(part.get())->fChunkSize = hdr->fSize;
// static_cast<FairMQMessageSHM*>(msg.get())->fOwner = owner;
// static_cast<FairMQMessageSHM*>(msg.get())->fReceiving = true;
size = part->GetSize();
fBytesRx += size;
totalSize += size;
msgVec.push_back(move(part));
totalSize += size;
}
else
{
LOG(ERROR) << "Received meta data, but could not find corresponding chunk";
return -1;
}
}
else if (zmq_errno() == EAGAIN)
{
if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
{
repeat = true;
break;
}
else
{
return -2;
}
}
else
{
LOG(ERROR) << "Received meta data, but could not find corresponding chunk";
return -1;
return nbytes;
}
size_t more_size = sizeof(more);
zmq_getsockopt(fSocket, ZMQ_RCVMORE, &more, &more_size);
}
else
while (more);
if (repeat)
{
return nbytes;
continue;
}
size_t more_size = sizeof(more);
zmq_getsockopt(fSocket, ZMQ_RCVMORE, &more, &more_size);
// store statistics on how many messages have been received (handle all parts as a single message)
++fMessagesRx;
fBytesRx += totalSize;
return totalSize;
}
while (more);
// store statistics on how many messages have been received (handle all parts as a single message)
++fMessagesRx;
return totalSize;
}
void FairMQSocketSHM::Close()
@@ -362,11 +420,13 @@ void FairMQSocketSHM::Terminate()
void FairMQSocketSHM::Interrupt()
{
FairMQMessageSHM::fInterrupted = true;
fInterrupted = true;
}
void FairMQSocketSHM::Resume()
{
FairMQMessageSHM::fInterrupted = false;
fInterrupted = false;
}
void* FairMQSocketSHM::GetSocket() const

10
fairmq/shmem/FairMQSocketSHM.h
View File

@@ -13,6 +13,7 @@
#include <memory> // unique_ptr
#include "FairMQSocket.h"
#include "FairMQMessage.h"
#include "FairMQContextSHM.h"
#include "FairMQShmManager.h"
@@ -28,12 +29,10 @@ class FairMQSocketSHM : public FairMQSocket
virtual bool Bind(const std::string& address);
virtual void Connect(const std::string& address);
virtual int Send(FairMQMessage* msg, const std::string& flag = "");
virtual int Send(FairMQMessage* msg, const int flags = 0);
virtual int64_t Send(const std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
virtual int Send(FairMQMessagePtr& msg, const int flags = 0);
virtual int Receive(FairMQMessagePtr& msg, const int flags = 0);
virtual int Receive(FairMQMessage* msg, const std::string& flag = "");
virtual int Receive(FairMQMessage* msg, const int flags = 0);
virtual int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
virtual int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
virtual void* GetSocket() const;
@@ -71,6 +70,7 @@ class FairMQSocketSHM : public FairMQSocket
static std::unique_ptr<FairMQContextSHM> fContext;
static bool fContextInitialized;
static std::atomic<bool> fInterrupted;
};
#endif /* FAIRMQSOCKETSHM_H_ */

10
fairmq/shmem/FairMQTransportFactorySHM.cxx
View File

@@ -12,11 +12,13 @@
using namespace std;
static FairMQ::Transport gTransportType = FairMQ::Transport::SHM;
FairMQTransportFactorySHM::FairMQTransportFactorySHM()
{
int major, minor, patch;
zmq_version(&major, &minor, &patch);
LOG(DEBUG) << "Using ZeroMQ (" << major << "." << minor << "." << patch << ") & "
LOG(DEBUG) << "Transport: Using ZeroMQ (" << major << "." << minor << "." << patch << ") & "
<< "boost::interprocess (" << (BOOST_VERSION / 100000) << "." << (BOOST_VERSION / 100 % 1000) << "." << (BOOST_VERSION % 100) << ")";
}
@@ -54,3 +56,9 @@ FairMQPollerPtr FairMQTransportFactorySHM::CreatePoller(const FairMQSocket& cmdS
{
return unique_ptr<FairMQPoller>(new FairMQPollerSHM(cmdSocket, dataSocket));
}
FairMQ::Transport FairMQTransportFactorySHM::GetType() const
{
return gTransportType;
}

3
fairmq/shmem/FairMQTransportFactorySHM.h
View File

@@ -9,6 +9,7 @@
#define FAIRMQTRANSPORTFACTORYSHM_H_
#include <vector>
#include <string>
#include "FairMQTransportFactory.h"
#include "FairMQContextSHM.h"
@@ -31,6 +32,8 @@ class FairMQTransportFactorySHM : public FairMQTransportFactory
virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const;
virtual FairMQPollerPtr CreatePoller(const FairMQSocket& cmdSocket, const FairMQSocket& dataSocket) const;
virtual FairMQ::Transport GetType() const;
virtual ~FairMQTransportFactorySHM() {};
};