mirror of
https://github.com/FairRootGroup/FairMQ.git
synced 2025-10-15 09:31:45 +00:00
Adding multiple transports support & other fixes:
- Avoid polling when only one input channel is used. - Send only handles for shared memory transport. - Avoid waiting in the rate logger thread when nothing to log. - Hide warnings from generated files - Fix #483
This commit is contained in:
Alexey Rybalchenko
committed by
Mohammad Al-Turany
Mohammad Al-Turany
parent
e53ad151a7
commit
c66fd6fe91
@@ -23,7 +23,8 @@
|
||||
using namespace std;
|
||||
|
||||
// Context to hold the ZeroMQ sockets
|
||||
std::unique_ptr<FairMQContextZMQ> FairMQSocketZMQ::fContext = std::unique_ptr<FairMQContextZMQ>(new FairMQContextZMQ(1));
|
||||
unique_ptr<FairMQContextZMQ> FairMQSocketZMQ::fContext = unique_ptr<FairMQContextZMQ>(new FairMQContextZMQ(1));
|
||||
atomic<bool> FairMQSocketZMQ::fInterrupted(false);
|
||||
|
||||
FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/)
|
||||
: FairMQSocket(ZMQ_SNDMORE, ZMQ_RCVMORE, ZMQ_DONTWAIT)
|
||||
@@ -56,12 +57,24 @@ FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const i
|
||||
|
||||
// Tell socket to try and send/receive outstanding messages for <linger> milliseconds before terminating.
|
||||
// Default value for ZeroMQ is -1, which is to wait forever.
|
||||
int linger = 500;
|
||||
int linger = 1000;
|
||||
if (zmq_setsockopt(fSocket, ZMQ_LINGER, &linger, sizeof(linger)) != 0)
|
||||
{
|
||||
LOG(ERROR) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
|
||||
}
|
||||
|
||||
int sndTimeout = 700;
|
||||
if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &sndTimeout, sizeof(sndTimeout)) != 0)
|
||||
{
|
||||
LOG(ERROR) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
|
||||
}
|
||||
|
||||
int rcvTimeout = 700;
|
||||
if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &rcvTimeout, sizeof(rcvTimeout)) != 0)
|
||||
{
|
||||
LOG(ERROR) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
|
||||
}
|
||||
|
||||
if (type == "sub")
|
||||
{
|
||||
if (zmq_setsockopt(fSocket, ZMQ_SUBSCRIBE, NULL, 0) != 0)
|
||||
@@ -106,91 +119,145 @@ void FairMQSocketZMQ::Connect(const string& address)
|
||||
}
|
||||
}
|
||||
|
||||
int FairMQSocketZMQ::Send(FairMQMessage* msg, const string& flag)
|
||||
int FairMQSocketZMQ::Send(FairMQMessagePtr& msg, const int flags)
|
||||
{
|
||||
return Send(msg, GetConstant(flag));
|
||||
}
|
||||
int nbytes = -1;
|
||||
|
||||
int FairMQSocketZMQ::Send(FairMQMessage* msg, const int flags)
|
||||
{
|
||||
int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
|
||||
if (nbytes >= 0)
|
||||
while (true)
|
||||
{
|
||||
fBytesTx += nbytes;
|
||||
++fMessagesTx;
|
||||
return nbytes;
|
||||
}
|
||||
if (zmq_errno() == EAGAIN)
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
if (zmq_errno() == ETERM)
|
||||
{
|
||||
LOG(INFO) << "terminating socket " << fId;
|
||||
return -1;
|
||||
}
|
||||
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
|
||||
return nbytes;
|
||||
}
|
||||
|
||||
int64_t FairMQSocketZMQ::Send(const vector<unique_ptr<FairMQMessage>>& msgVec, const int flags)
|
||||
{
|
||||
// Sending vector typicaly handles more then one part
|
||||
if (msgVec.size() > 1)
|
||||
{
|
||||
int64_t totalSize = 0;
|
||||
|
||||
for (unsigned int i = 0; i < msgVec.size() - 1; ++i)
|
||||
nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
|
||||
if (nbytes >= 0)
|
||||
{
|
||||
int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec[i]->GetMessage()), fSocket, ZMQ_SNDMORE|flags);
|
||||
if (nbytes >= 0)
|
||||
fBytesTx += nbytes;
|
||||
++fMessagesTx;
|
||||
|
||||
return nbytes;
|
||||
}
|
||||
else if (zmq_errno() == EAGAIN)
|
||||
{
|
||||
if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
|
||||
{
|
||||
totalSize += nbytes;
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (zmq_errno() == EAGAIN)
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
if (zmq_errno() == ETERM)
|
||||
{
|
||||
LOG(INFO) << "terminating socket " << fId;
|
||||
return -1;
|
||||
}
|
||||
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
|
||||
return nbytes;
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
|
||||
int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec.back()->GetMessage()), fSocket, flags);
|
||||
if (nbytes >= 0)
|
||||
else if (zmq_errno() == ETERM)
|
||||
{
|
||||
totalSize += nbytes;
|
||||
LOG(INFO) << "terminating socket " << fId;
|
||||
return -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (zmq_errno() == EAGAIN)
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
if (zmq_errno() == ETERM)
|
||||
{
|
||||
LOG(INFO) << "terminating socket " << fId;
|
||||
return -1;
|
||||
}
|
||||
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
|
||||
return nbytes;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// store statistics on how many messages have been sent (handle all parts as a single message)
|
||||
++fMessagesTx;
|
||||
fBytesTx += totalSize;
|
||||
return totalSize;
|
||||
} // If there's only one part, send it as a regular message
|
||||
else if (msgVec.size() == 1)
|
||||
int FairMQSocketZMQ::Receive(FairMQMessagePtr& msg, const int flags)
|
||||
{
|
||||
int nbytes = -1;
|
||||
while (true)
|
||||
{
|
||||
return Send(msgVec.back().get(), flags);
|
||||
nbytes = zmq_msg_recv(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
|
||||
if (nbytes >= 0)
|
||||
{
|
||||
fBytesRx += nbytes;
|
||||
++fMessagesRx;
|
||||
return nbytes;
|
||||
}
|
||||
else if (zmq_errno() == EAGAIN)
|
||||
{
|
||||
if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
else if (zmq_errno() == ETERM)
|
||||
{
|
||||
LOG(INFO) << "terminating socket " << fId;
|
||||
return -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
|
||||
return nbytes;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int64_t FairMQSocketZMQ::Send(vector<unique_ptr<FairMQMessage>>& msgVec, const int flags)
|
||||
{
|
||||
const unsigned int vecSize = msgVec.size();
|
||||
|
||||
// Sending vector typicaly handles more then one part
|
||||
if (vecSize > 1)
|
||||
{
|
||||
int64_t totalSize = 0;
|
||||
int nbytes = -1;
|
||||
bool repeat = false;
|
||||
|
||||
while (true)
|
||||
{
|
||||
totalSize = 0;
|
||||
nbytes = -1;
|
||||
repeat = false;
|
||||
|
||||
for (unsigned int i = 0; i < vecSize; ++i)
|
||||
{
|
||||
nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec[i]->GetMessage()),
|
||||
fSocket,
|
||||
(i < vecSize - 1) ? ZMQ_SNDMORE|flags : flags);
|
||||
if (nbytes >= 0)
|
||||
{
|
||||
totalSize += nbytes;
|
||||
}
|
||||
else
|
||||
{
|
||||
// according to ZMQ docs, this can only occur for the first part
|
||||
if (zmq_errno() == EAGAIN)
|
||||
{
|
||||
if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
|
||||
{
|
||||
repeat = true;
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
if (zmq_errno() == ETERM)
|
||||
{
|
||||
LOG(INFO) << "terminating socket " << fId;
|
||||
return -1;
|
||||
}
|
||||
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
|
||||
return nbytes;
|
||||
}
|
||||
}
|
||||
|
||||
if (repeat)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// store statistics on how many messages have been sent (handle all parts as a single message)
|
||||
++fMessagesTx;
|
||||
fBytesTx += totalSize;
|
||||
return totalSize;
|
||||
}
|
||||
} // If there's only one part, send it as a regular message
|
||||
else if (vecSize == 1)
|
||||
{
|
||||
return Send(msgVec.back(), flags);
|
||||
}
|
||||
else // if the vector is empty, something might be wrong
|
||||
{
|
||||
@@ -199,69 +266,67 @@ int64_t FairMQSocketZMQ::Send(const vector<unique_ptr<FairMQMessage>>& msgVec, c
|
||||
}
|
||||
}
|
||||
|
||||
int FairMQSocketZMQ::Receive(FairMQMessage* msg, const string& flag)
|
||||
{
|
||||
return Receive(msg, GetConstant(flag));
|
||||
}
|
||||
|
||||
int FairMQSocketZMQ::Receive(FairMQMessage* msg, const int flags)
|
||||
{
|
||||
int nbytes = zmq_msg_recv(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
|
||||
if (nbytes >= 0)
|
||||
{
|
||||
fBytesRx += nbytes;
|
||||
++fMessagesRx;
|
||||
return nbytes;
|
||||
}
|
||||
if (zmq_errno() == EAGAIN)
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
if (zmq_errno() == ETERM)
|
||||
{
|
||||
LOG(INFO) << "terminating socket " << fId;
|
||||
return -1;
|
||||
}
|
||||
LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
|
||||
return nbytes;
|
||||
}
|
||||
|
||||
int64_t FairMQSocketZMQ::Receive(vector<unique_ptr<FairMQMessage>>& msgVec, const int flags)
|
||||
{
|
||||
// Warn if the vector is filled before Receive() and empty it.
|
||||
if (msgVec.size() > 0)
|
||||
{
|
||||
LOG(WARN) << "Message vector contains elements before Receive(), they will be deleted!";
|
||||
msgVec.clear();
|
||||
}
|
||||
|
||||
int64_t totalSize = 0;
|
||||
int64_t more = 0;
|
||||
bool repeat = false;
|
||||
|
||||
do
|
||||
while (true)
|
||||
{
|
||||
unique_ptr<FairMQMessage> part(new FairMQMessageZMQ());
|
||||
|
||||
int nbytes = zmq_msg_recv(static_cast<zmq_msg_t*>(part->GetMessage()), fSocket, flags);
|
||||
if (nbytes >= 0)
|
||||
// Warn if the vector is filled before Receive() and empty it.
|
||||
if (msgVec.size() > 0)
|
||||
{
|
||||
msgVec.push_back(move(part));
|
||||
totalSize += nbytes;
|
||||
}
|
||||
else
|
||||
{
|
||||
return nbytes;
|
||||
LOG(WARN) << "Message vector contains elements before Receive(), they will be deleted!";
|
||||
msgVec.clear();
|
||||
}
|
||||
|
||||
size_t more_size = sizeof(more);
|
||||
zmq_getsockopt(fSocket, ZMQ_RCVMORE, &more, &more_size);
|
||||
totalSize = 0;
|
||||
more = 0;
|
||||
repeat = false;
|
||||
|
||||
do
|
||||
{
|
||||
unique_ptr<FairMQMessage> part(new FairMQMessageZMQ());
|
||||
|
||||
int nbytes = zmq_msg_recv(static_cast<zmq_msg_t*>(part->GetMessage()), fSocket, flags);
|
||||
if (nbytes >= 0)
|
||||
{
|
||||
msgVec.push_back(move(part));
|
||||
totalSize += nbytes;
|
||||
}
|
||||
else if (zmq_errno() == EAGAIN)
|
||||
{
|
||||
if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
|
||||
{
|
||||
repeat = true;
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return nbytes;
|
||||
}
|
||||
|
||||
size_t more_size = sizeof(more);
|
||||
zmq_getsockopt(fSocket, ZMQ_RCVMORE, &more, &more_size);
|
||||
}
|
||||
while (more);
|
||||
|
||||
if (repeat)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// store statistics on how many messages have been received (handle all parts as a single message)
|
||||
++fMessagesRx;
|
||||
fBytesRx += totalSize;
|
||||
return totalSize;
|
||||
}
|
||||
while (more);
|
||||
|
||||
// store statistics on how many messages have been received (handle all parts as a single message)
|
||||
++fMessagesRx;
|
||||
fBytesRx += totalSize;
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
void FairMQSocketZMQ::Close()
|
||||
@@ -291,10 +356,12 @@ void FairMQSocketZMQ::Terminate()
|
||||
|
||||
void FairMQSocketZMQ::Interrupt()
|
||||
{
|
||||
fInterrupted = true;
|
||||
}
|
||||
|
||||
void FairMQSocketZMQ::Resume()
|
||||
{
|
||||
fInterrupted = false;
|
||||
}
|
||||
|
||||
void* FairMQSocketZMQ::GetSocket() const
|
||||
@@ -506,10 +573,6 @@ int FairMQSocketZMQ::GetConstant(const string& constant)
|
||||
|
||||
if (constant == "linger")
|
||||
return ZMQ_LINGER;
|
||||
if (constant == "no-block")
|
||||
return ZMQ_DONTWAIT;
|
||||
if (constant == "snd-more no-block")
|
||||
return ZMQ_DONTWAIT|ZMQ_SNDMORE;
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user