Mirror/FairMQ
1
0
Fork 0
mirror of https://github.com/FairRootGroup/FairMQ.git synced 2025-10-15 17:41:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding multiple transports support & other fixes:

Browse Source 
- Avoid polling when only one input channel is used.
 - Send only handles for shared memory transport.
 - Avoid waiting in the rate logger thread when nothing to log.
 - Hide warnings from generated files
 - Fix #483
This commit is contained in:
Alexey Rybalchenko
2017-01-13 15:53:25 +01:00
committed by Mohammad Al-Turany
parent e53ad151a7
commit c66fd6fe91
39 changed files with 1840 additions and 1189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fairmq/zeromq/FairMQMessageZMQ.cxx
View File

@@ -20,6 +20,8 @@
using namespace std;
static FairMQ::Transport gTransportType = FairMQ::Transport::ZMQ;
string FairMQMessageZMQ::fDeviceID = string();
FairMQMessageZMQ::FairMQMessageZMQ()
@@ -101,6 +103,11 @@ void FairMQMessageZMQ::SetDeviceId(const string& deviceId)
fDeviceID = deviceId;
}
FairMQ::Transport FairMQMessageZMQ::GetType() const
{
return gTransportType;
}
void FairMQMessageZMQ::Copy(const unique_ptr<FairMQMessage>& msg)
{
// Shares the message buffer between msg and this fMessage.

2
fairmq/zeromq/FairMQMessageZMQ.h
View File

@@ -41,6 +41,8 @@ class FairMQMessageZMQ : public FairMQMessage
virtual void SetDeviceId(const std::string& deviceId);
virtual FairMQ::Transport GetType() const;
virtual void Copy(const std::unique_ptr<FairMQMessage>& msg);
void CloseMessage();

25
fairmq/zeromq/FairMQPollerZMQ.cxx
View File

@@ -51,7 +51,7 @@ FairMQPollerZMQ::FairMQPollerZMQ(const vector<FairMQChannel>& channels)
}
else
{
LOG(ERROR) << "invalid poller configuration, exiting.";
LOG(ERROR) << "zeromq: invalid poller configuration, exiting.";
exit(EXIT_FAILURE);
}
}
@@ -105,7 +105,7 @@ FairMQPollerZMQ::FairMQPollerZMQ(const unordered_map<string, vector<FairMQChanne
}
else
{
LOG(ERROR) << "invalid poller configuration, exiting.";
LOG(ERROR) << "zeromq: invalid poller configuration, exiting.";
exit(EXIT_FAILURE);
}
}
@@ -113,9 +113,9 @@ FairMQPollerZMQ::FairMQPollerZMQ(const unordered_map<string, vector<FairMQChanne
}
catch (const std::out_of_range& oor)
{
LOG(ERROR) << "At least one of the provided channel keys for poller initialization is invalid";
LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
throw std::out_of_range("Invalid channel during poller initialization");
LOG(ERROR) << "zeromq: at least one of the provided channel keys for poller initialization is invalid";
LOG(ERROR) << "zeromq: out of range error: " << oor.what() << '\n';
throw std::out_of_range("invalid channel during poller initialization");
}
}
@@ -153,7 +153,7 @@ FairMQPollerZMQ::FairMQPollerZMQ(const FairMQSocket& cmdSocket, const FairMQSock
}
else
{
LOG(ERROR) << "invalid poller configuration, exiting.";
LOG(ERROR) << "zeromq: invalid poller configuration, exiting.";
exit(EXIT_FAILURE);
}
}
@@ -164,11 +164,12 @@ void FairMQPollerZMQ::Poll(const int timeout)
{
if (errno == ETERM)
{
LOG(DEBUG) << "polling exited, reason: " << zmq_strerror(errno);
LOG(DEBUG) << "zeromq: polling exited, reason: " << zmq_strerror(errno);
}
else
{
LOG(ERROR) << "polling failed, reason: " << zmq_strerror(errno);
LOG(ERROR) << "zeromq: polling failed, reason: " << zmq_strerror(errno);
throw std::runtime_error("zeromq: polling failed");
}
}
}
@@ -206,8 +207,8 @@ bool FairMQPollerZMQ::CheckInput(const string channelKey, const int index)
}
catch (const std::out_of_range& oor)
{
LOG(ERROR) << "Invalid channel key: \"" << channelKey << "\"";
LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
LOG(ERROR) << "zeromq: invalid channel key: \"" << channelKey << "\"";
LOG(ERROR) << "zeromq: out of range error: " << oor.what() << '\n';
exit(EXIT_FAILURE);
}
}
@@ -225,8 +226,8 @@ bool FairMQPollerZMQ::CheckOutput(const string channelKey, const int index)
}
catch (const std::out_of_range& oor)
{
LOG(ERROR) << "Invalid channel key: \"" << channelKey << "\"";
LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
LOG(ERROR) << "zeromq: invalid channel key: \"" << channelKey << "\"";
LOG(ERROR) << "zeromq: out of range error: " << oor.what() << '\n';
exit(EXIT_FAILURE);
}
}

313
fairmq/zeromq/FairMQSocketZMQ.cxx
View File

@@ -23,7 +23,8 @@
using namespace std;
// Context to hold the ZeroMQ sockets
std::unique_ptr<FairMQContextZMQ> FairMQSocketZMQ::fContext = std::unique_ptr<FairMQContextZMQ>(new FairMQContextZMQ(1));
unique_ptr<FairMQContextZMQ> FairMQSocketZMQ::fContext = unique_ptr<FairMQContextZMQ>(new FairMQContextZMQ(1));
atomic<bool> FairMQSocketZMQ::fInterrupted(false);
FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/)
: FairMQSocket(ZMQ_SNDMORE, ZMQ_RCVMORE, ZMQ_DONTWAIT)
@@ -56,12 +57,24 @@ FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const i
// Tell socket to try and send/receive outstanding messages for <linger> milliseconds before terminating.
// Default value for ZeroMQ is -1, which is to wait forever.
int linger = 500;
int linger = 1000;
if (zmq_setsockopt(fSocket, ZMQ_LINGER, &linger, sizeof(linger)) != 0)
{
LOG(ERROR) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
}
int sndTimeout = 700;
if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &sndTimeout, sizeof(sndTimeout)) != 0)
{
LOG(ERROR) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
}
int rcvTimeout = 700;
if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &rcvTimeout, sizeof(rcvTimeout)) != 0)
{
LOG(ERROR) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
}
if (type == "sub")
{
if (zmq_setsockopt(fSocket, ZMQ_SUBSCRIBE, NULL, 0) != 0)
@@ -106,91 +119,145 @@ void FairMQSocketZMQ::Connect(const string& address)
}
}
int FairMQSocketZMQ::Send(FairMQMessage* msg, const string& flag)
int FairMQSocketZMQ::Send(FairMQMessagePtr& msg, const int flags)
{
return Send(msg, GetConstant(flag));
}
int nbytes = -1;
int FairMQSocketZMQ::Send(FairMQMessage* msg, const int flags)
{
int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
if (nbytes >= 0)
while (true)
{
fBytesTx += nbytes;
++fMessagesTx;
return nbytes;
}
if (zmq_errno() == EAGAIN)
{
return -2;
}
if (zmq_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
return nbytes;
}
int64_t FairMQSocketZMQ::Send(const vector<unique_ptr<FairMQMessage>>& msgVec, const int flags)
{
// Sending vector typicaly handles more then one part
if (msgVec.size() > 1)
{
int64_t totalSize = 0;
for (unsigned int i = 0; i < msgVec.size() - 1; ++i)
nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
if (nbytes >= 0)
{
int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec[i]->GetMessage()), fSocket, ZMQ_SNDMORE|flags);
if (nbytes >= 0)
fBytesTx += nbytes;
++fMessagesTx;
return nbytes;
}
else if (zmq_errno() == EAGAIN)
{
if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
{
totalSize += nbytes;
continue;
}
else
{
if (zmq_errno() == EAGAIN)
{
return -2;
}
if (zmq_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
return nbytes;
return -2;
}
}
int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec.back()->GetMessage()), fSocket, flags);
if (nbytes >= 0)
else if (zmq_errno() == ETERM)
{
totalSize += nbytes;
LOG(INFO) << "terminating socket " << fId;
return -1;
}
else
{
if (zmq_errno() == EAGAIN)
{
return -2;
}
if (zmq_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
return nbytes;
}
}
}
// store statistics on how many messages have been sent (handle all parts as a single message)
++fMessagesTx;
fBytesTx += totalSize;
return totalSize;
} // If there's only one part, send it as a regular message
else if (msgVec.size() == 1)
int FairMQSocketZMQ::Receive(FairMQMessagePtr& msg, const int flags)
{
int nbytes = -1;
while (true)
{
return Send(msgVec.back().get(), flags);
nbytes = zmq_msg_recv(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
if (nbytes >= 0)
{
fBytesRx += nbytes;
++fMessagesRx;
return nbytes;
}
else if (zmq_errno() == EAGAIN)
{
if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
{
continue;
}
else
{
return -2;
}
}
else if (zmq_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
else
{
LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
return nbytes;
}
}
}
int64_t FairMQSocketZMQ::Send(vector<unique_ptr<FairMQMessage>>& msgVec, const int flags)
{
const unsigned int vecSize = msgVec.size();
// Sending vector typicaly handles more then one part
if (vecSize > 1)
{
int64_t totalSize = 0;
int nbytes = -1;
bool repeat = false;
while (true)
{
totalSize = 0;
nbytes = -1;
repeat = false;
for (unsigned int i = 0; i < vecSize; ++i)
{
nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec[i]->GetMessage()),
fSocket,
(i < vecSize - 1) ? ZMQ_SNDMORE|flags : flags);
if (nbytes >= 0)
{
totalSize += nbytes;
}
else
{
// according to ZMQ docs, this can only occur for the first part
if (zmq_errno() == EAGAIN)
{
if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
{
repeat = true;
break;
}
else
{
return -2;
}
}
if (zmq_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
return nbytes;
}
}
if (repeat)
{
continue;
}
// store statistics on how many messages have been sent (handle all parts as a single message)
++fMessagesTx;
fBytesTx += totalSize;
return totalSize;
}
} // If there's only one part, send it as a regular message
else if (vecSize == 1)
{
return Send(msgVec.back(), flags);
}
else // if the vector is empty, something might be wrong
{
@@ -199,69 +266,67 @@ int64_t FairMQSocketZMQ::Send(const vector<unique_ptr<FairMQMessage>>& msgVec, c
}
}
int FairMQSocketZMQ::Receive(FairMQMessage* msg, const string& flag)
{
return Receive(msg, GetConstant(flag));
}
int FairMQSocketZMQ::Receive(FairMQMessage* msg, const int flags)
{
int nbytes = zmq_msg_recv(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
if (nbytes >= 0)
{
fBytesRx += nbytes;
++fMessagesRx;
return nbytes;
}
if (zmq_errno() == EAGAIN)
{
return -2;
}
if (zmq_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
return nbytes;
}
int64_t FairMQSocketZMQ::Receive(vector<unique_ptr<FairMQMessage>>& msgVec, const int flags)
{
// Warn if the vector is filled before Receive() and empty it.
if (msgVec.size() > 0)
{
LOG(WARN) << "Message vector contains elements before Receive(), they will be deleted!";
msgVec.clear();
}
int64_t totalSize = 0;
int64_t more = 0;
bool repeat = false;
do
while (true)
{
unique_ptr<FairMQMessage> part(new FairMQMessageZMQ());
int nbytes = zmq_msg_recv(static_cast<zmq_msg_t*>(part->GetMessage()), fSocket, flags);
if (nbytes >= 0)
// Warn if the vector is filled before Receive() and empty it.
if (msgVec.size() > 0)
{
msgVec.push_back(move(part));
totalSize += nbytes;
}
else
{
return nbytes;
LOG(WARN) << "Message vector contains elements before Receive(), they will be deleted!";
msgVec.clear();
}
size_t more_size = sizeof(more);
zmq_getsockopt(fSocket, ZMQ_RCVMORE, &more, &more_size);
totalSize = 0;
more = 0;
repeat = false;
do
{
unique_ptr<FairMQMessage> part(new FairMQMessageZMQ());
int nbytes = zmq_msg_recv(static_cast<zmq_msg_t*>(part->GetMessage()), fSocket, flags);
if (nbytes >= 0)
{
msgVec.push_back(move(part));
totalSize += nbytes;
}
else if (zmq_errno() == EAGAIN)
{
if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
{
repeat = true;
break;
}
else
{
return -2;
}
}
else
{
return nbytes;
}
size_t more_size = sizeof(more);
zmq_getsockopt(fSocket, ZMQ_RCVMORE, &more, &more_size);
}
while (more);
if (repeat)
{
continue;
}
// store statistics on how many messages have been received (handle all parts as a single message)
++fMessagesRx;
fBytesRx += totalSize;
return totalSize;
}
while (more);
// store statistics on how many messages have been received (handle all parts as a single message)
++fMessagesRx;
fBytesRx += totalSize;
return totalSize;
}
void FairMQSocketZMQ::Close()
@@ -291,10 +356,12 @@ void FairMQSocketZMQ::Terminate()
void FairMQSocketZMQ::Interrupt()
{
fInterrupted = true;
}
void FairMQSocketZMQ::Resume()
{
fInterrupted = false;
}
void* FairMQSocketZMQ::GetSocket() const
@@ -506,10 +573,6 @@ int FairMQSocketZMQ::GetConstant(const string& constant)
if (constant == "linger")
return ZMQ_LINGER;
if (constant == "no-block")
return ZMQ_DONTWAIT;
if (constant == "snd-more no-block")
return ZMQ_DONTWAIT|ZMQ_SNDMORE;
return -1;
}

10
fairmq/zeromq/FairMQSocketZMQ.h
View File

@@ -20,6 +20,7 @@
#include <memory> // unique_ptr
#include "FairMQSocket.h"
#include "FairMQMessage.h"
#include "FairMQContextZMQ.h"
class FairMQSocketZMQ : public FairMQSocket
@@ -34,12 +35,10 @@ class FairMQSocketZMQ : public FairMQSocket
virtual bool Bind(const std::string& address);
virtual void Connect(const std::string& address);
virtual int Send(FairMQMessage* msg, const std::string& flag = "");
virtual int Send(FairMQMessage* msg, const int flags = 0);
virtual int64_t Send(const std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
virtual int Send(FairMQMessagePtr& msg, const int flags = 0);
virtual int Receive(FairMQMessagePtr& msg, const int flags = 0);
virtual int Receive(FairMQMessage* msg, const std::string& flag = "");
virtual int Receive(FairMQMessage* msg, const int flags = 0);
virtual int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
virtual int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
virtual void* GetSocket() const;
@@ -76,6 +75,7 @@ class FairMQSocketZMQ : public FairMQSocket
std::atomic<unsigned long> fMessagesRx;
static std::unique_ptr<FairMQContextZMQ> fContext;
static std::atomic<bool> fInterrupted;
};
#endif /* FAIRMQSOCKETZMQ_H_ */

9
fairmq/zeromq/FairMQTransportFactoryZMQ.cxx
View File

@@ -18,11 +18,13 @@
using namespace std;
static FairMQ::Transport gTransportType = FairMQ::Transport::ZMQ;
FairMQTransportFactoryZMQ::FairMQTransportFactoryZMQ()
{
int major, minor, patch;
zmq_version(&major, &minor, &patch);
LOG(DEBUG) << "Using ZeroMQ library, version: " << major << "." << minor << "." << patch;
LOG(DEBUG) << "Transport: Using ZeroMQ library, version: " << major << "." << minor << "." << patch;
}
FairMQMessagePtr FairMQTransportFactoryZMQ::CreateMessage() const
@@ -59,3 +61,8 @@ FairMQPollerPtr FairMQTransportFactoryZMQ::CreatePoller(const FairMQSocket& cmdS
{
return unique_ptr<FairMQPoller>(new FairMQPollerZMQ(cmdSocket, dataSocket));
}
FairMQ::Transport FairMQTransportFactoryZMQ::GetType() const
{
return gTransportType;
}

3
fairmq/zeromq/FairMQTransportFactoryZMQ.h
View File

@@ -16,6 +16,7 @@
#define FAIRMQTRANSPORTFACTORYZMQ_H_
#include <vector>
#include <string>
#include "FairMQTransportFactory.h"
#include "FairMQContextZMQ.h"
@@ -38,6 +39,8 @@ class FairMQTransportFactoryZMQ : public FairMQTransportFactory
virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const;
virtual FairMQPollerPtr CreatePoller(const FairMQSocket& cmdSocket, const FairMQSocket& dataSocket) const;
virtual FairMQ::Transport GetType() const;
virtual ~FairMQTransportFactoryZMQ() {};
};