Mirror/FairMQ
1
0
Fork 0
mirror of https://github.com/FairRootGroup/FairMQ.git synced 2025-10-15 09:31:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update multi-part features (nanomsg) and various fixes

Browse Source 
- Implement nanomsg multipart with MessagePack.
 - Use the MessagePack from FairSoft and handle not found case.
 - Update splitter, merger and proxy devices to handle multi-part.
 - Let FairMQParts.At() return pointer reference (can be used for moving).
 - Add missing const specifier in the message interface.
 - Add transmit kernel size setting to channels (ZMQ_SNDBUF).
 - Remove FairMQBuffer device.
 - Remove old multi-part methods from Tutorial3 example (to be replaced with Parts API).
 - Make callback mandatory for newMsg(data, size, callback).
 - Add missing <vector> include in FairMQSocket.
This commit is contained in:
Alexey Rybalchenko
2016-03-21 09:59:00 +01:00
parent 4ca66e33da
commit 732373faa2
25 changed files with 436 additions and 381 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
fairmq/nanomsg/FairMQSocketNN.cxx
View File

@@ -12,15 +12,18 @@
* @author A. Rybalchenko
*/
#include <sstream>
#include "FairMQSocketNN.h"
#include "FairMQMessageNN.h"
#include "FairMQLogger.h"
#include <sstream>
#ifdef MSGPACK_FOUND
#include <msgpack.hpp>
#endif /*MSGPACK_FOUND*/
using namespace std;
FairMQSocketNN::FairMQSocketNN(const string& type, const std::string& name, const int numIoThreads, const std::string& id /*= ""*/)
FairMQSocketNN::FairMQSocketNN(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/)
: FairMQSocket(0, 0, NN_DONTWAIT)
, fSocket(-1)
, fId()
@@ -62,7 +65,7 @@ FairMQSocketNN::FairMQSocketNN(const string& type, const std::string& name, cons
}
}
LOG(INFO) << "created socket " << fId;
// LOG(INFO) << "created socket " << fId;
}
string FairMQSocketNN::GetId()
@@ -72,7 +75,7 @@ string FairMQSocketNN::GetId()
bool FairMQSocketNN::Bind(const string& address)
{
LOG(INFO) << "bind socket " << fId << " on " << address;
// LOG(INFO) << "bind socket " << fId << " on " << address;
int eid = nn_bind(fSocket, address.c_str());
if (eid < 0)
@@ -85,7 +88,7 @@ bool FairMQSocketNN::Bind(const string& address)
void FairMQSocketNN::Connect(const string& address)
{
LOG(INFO) << "connect socket " << fId << " to " << address;
// LOG(INFO) << "connect socket " << fId << " to " << address;
int eid = nn_connect(fSocket, address.c_str());
if (eid < 0)
@@ -142,6 +145,46 @@ int FairMQSocketNN::Send(FairMQMessage* msg, const int flags)
return nbytes;
}
int64_t FairMQSocketNN::Send(const vector<unique_ptr<FairMQMessage>>& msgVec)
{
#ifdef MSGPACK_FOUND
// create msgpack simple buffer
msgpack::sbuffer sbuf;
// create msgpack packer
msgpack::packer<msgpack::sbuffer> packer(&sbuf);
// pack all parts into a single msgpack simple buffer
for (int i = 0; i < msgVec.size(); ++i)
{
static_cast<FairMQMessageNN*>(msgVec[i].get())->fReceiving = false;
packer.pack_bin(msgVec[i]->GetSize());
packer.pack_bin_body(static_cast<char*>(msgVec[i]->GetData()), msgVec[i]->GetSize());
}
int64_t nbytes = nn_send(fSocket, sbuf.data(), sbuf.size(), 0);
if (nbytes >= 0)
{
fBytesTx += nbytes;
++fMessagesTx;
return nbytes;
}
if (nn_errno() == EAGAIN)
{
return -2;
}
if (nn_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << nn_strerror(errno);
return nbytes;
#else /*MSGPACK_FOUND*/
LOG(ERROR) << "Cannot use nanomsg multipart because MessagePack was not found.";
exit(EXIT_FAILURE);
#endif /*MSGPACK_FOUND*/
}
int FairMQSocketNN::Receive(FairMQMessage* msg, const string& flag)
{
void* ptr = NULL;
@@ -193,6 +236,67 @@ int FairMQSocketNN::Receive(FairMQMessage* msg, const int flags)
return nbytes;
}
int64_t FairMQSocketNN::Receive(vector<unique_ptr<FairMQMessage>>& msgVec)
{
#ifdef MSGPACK_FOUND
// Warn if the vector is filled before Receive() and empty it.
if (msgVec.size() > 0)
{
LOG(WARN) << "Message vector contains elements before Receive(), they will be deleted!";
msgVec.clear();
}
// pointer to point to received message buffer
char* ptr = NULL;
// receive the message into a buffer allocated by nanomsg and let ptr point to it
int nbytes = nn_recv(fSocket, &ptr, NN_MSG, 0);
if (nbytes >= 0) // if no errors or non-blocking timeouts
{
// store statistics on how many bytes received
fBytesRx += nbytes;
// store statistics on how many messages received (count messages instead of parts)
++fMessagesRx;
// offset to be used by msgpack to handle separate chunks
size_t offset = 0;
while (offset != nbytes) // continue until all parts have been read
{
// vector of chars to hold blob (unlike char*/void* this type can be converted to by msgpack)
std::vector<char> buf;
// unpack and convert chunk
msgpack::unpacked result;
unpack(result, ptr, nbytes, offset);
msgpack::object object(result.get());
object.convert(buf);
// get the single message size
size_t size = buf.size() * sizeof(char);
unique_ptr<FairMQMessage> part(new FairMQMessageNN(size));
static_cast<FairMQMessageNN*>(part.get())->fReceiving = true;
memcpy(part->GetData(), buf.data(), size);
msgVec.push_back(move(part));
}
nn_freemsg(ptr);
return nbytes;
}
if (nn_errno() == EAGAIN)
{
return -2;
}
if (nn_errno() == ETERM)
{
LOG(INFO) << "terminating socket " << fId;
return -1;
}
LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << nn_strerror(errno);
return nbytes;
#else /*MSGPACK_FOUND*/
LOG(ERROR) << "Cannot use nanomsg multipart because MessagePack was not found.";
exit(EXIT_FAILURE);
#endif /*MSGPACK_FOUND*/
}
void FairMQSocketNN::Close()
{
nn_close(fSocket);
@@ -225,7 +329,8 @@ void FairMQSocketNN::SetOption(const string& option, const void* value, size_t v
void FairMQSocketNN::GetOption(const string& option, void* value, size_t* valueSize)
{
int rc = nn_getsockopt(fSocket, NN_SOL_SOCKET, GetConstant(option), value, valueSize);
if (rc < 0) {
if (rc < 0)
{
LOG(ERROR) << "failed getting socket option, reason: " << nn_strerror(errno);
}
}
@@ -329,11 +434,17 @@ int FairMQSocketNN::GetConstant(const string& constant)
return NN_SNDBUF;
if (constant == "rcv-hwm")
return NN_RCVBUF;
if (constant == "snd-more") {
if (constant == "snd-size")
return NN_SNDBUF;
if (constant == "rcv-size")
return NN_RCVBUF;
if (constant == "snd-more")
{
LOG(ERROR) << "Multipart messages functionality currently not supported by nanomsg!";
return -1;
}
if (constant == "rcv-more") {
if (constant == "rcv-more")
{
LOG(ERROR) << "Multipart messages functionality currently not supported by nanomsg!";
return -1;
}