Mirror/FairMQ
1
0
Fork 0
mirror of https://github.com/FairRootGroup/FairMQ.git synced 2025-10-15 17:41:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Configuration and DDS example/tools updates

Browse Source 
- Update DDS example command UI and extract it from example.
 - Unify address handling via DDS properties for dynamic deployment.
 - Update DDS docs with the new approach.
 - Allow `--config-key` to be used to access common config in JSON.
 - Allow common channel properties to be specified for all sockets.
 - Update MQ examples and Tuto3 with new config options.
 - Add start scripts to MQ examples for easier use.
This commit is contained in:
Alexey Rybalchenko
2016-03-31 14:41:05 +02:00
parent 151d3b5de8
commit b9883d3b13
21 changed files with 1082 additions and 701 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
fairmq/run/runMerger.cxx
View File

@@ -14,132 +14,43 @@
#include <iostream>
#include "boost/program_options.hpp"
#include "FairMQLogger.h"
#include "FairMQParser.h"
#include "FairMQProgOptions.h"
#include "FairMQMerger.h"
using namespace std;
typedef struct DeviceOptions
{
DeviceOptions() :
id(), ioThreads(0), transport(), numInputs(0),
inputSocketType(), inputBufSize(), inputMethod(), inputAddress(),
outputSocketType(), outputBufSize(0), outputMethod(), outputAddress() {}
string id;
int ioThreads;
string transport;
int numInputs;
vector<string> inputSocketType;
vector<int> inputBufSize;
vector<string> inputMethod;
vector<string> inputAddress;
string outputSocketType;
int outputBufSize;
string outputMethod;
string outputAddress;
} DeviceOptions_t;
inline bool parse_cmd_line(int _argc, char* _argv[], DeviceOptions* _options)
{
if (_options == NULL)
throw runtime_error("Internal error: options' container is empty.");
namespace bpo = boost::program_options;
bpo::options_description desc("Options");
desc.add_options()
("id", bpo::value<string>()->required(), "Device ID")
("io-threads", bpo::value<int>()->default_value(1), "Number of I/O threads")
("transport", bpo::value<string>()->default_value("zeromq"), "Transport (zeromq/nanomsg)")
("num-inputs", bpo::value<int>()->required(), "Number of Merger input sockets")
("input-socket-type", bpo::value<vector<string>>()->required(), "Input socket type: sub/pull")
("input-buff-size", bpo::value<vector<int>>()->required(), "Input buffer size in number of messages (ZeroMQ)/bytes(nanomsg)")
("input-method", bpo::value<vector<string>>()->required(), "Input method: bind/connect")
("input-address", bpo::value<vector<string>>()->required(), "Input address, e.g.: \"tcp://localhost:5555\"")
("output-socket-type", bpo::value<string>()->required(), "Output socket type: pub/push")
("output-buff-size", bpo::value<int>()->required(), "Output buffer size in number of messages (ZeroMQ)/bytes(nanomsg)")
("output-method", bpo::value<string>()->required(), "Output method: bind/connect")
("output-address", bpo::value<string>()->required(), "Output address, e.g.: \"tcp://localhost:5555\"")
("help", "Print help messages");
bpo::variables_map vm;
bpo::store(bpo::parse_command_line(_argc, _argv, desc), vm);
if (vm.count("help"))
{
LOG(INFO) << "FairMQ Merger" << endl << desc;
return false;
}
bpo::notify(vm);
if (vm.count("id")) { _options->id = vm["id"].as<string>(); }
if (vm.count("io-threads")) { _options->ioThreads = vm["io-threads"].as<int>(); }
if (vm.count("transport")) { _options->transport = vm["transport"].as<string>(); }
if (vm.count("num-inputs")) { _options->numInputs = vm["num-inputs"].as<int>(); }
if (vm.count("input-socket-type")) { _options->inputSocketType = vm["input-socket-type"].as<vector<string>>(); }
if (vm.count("input-buff-size")) { _options->inputBufSize = vm["input-buff-size"].as<vector<int>>(); }
if (vm.count("input-method")) { _options->inputMethod = vm["input-method"].as<vector<string>>(); }
if (vm.count("input-address")) { _options->inputAddress = vm["input-address"].as<vector<string>>(); }
if (vm.count("output-socket-type")) { _options->outputSocketType = vm["output-socket-type"].as<string>(); }
if (vm.count("output-buff-size")) { _options->outputBufSize = vm["output-buff-size"].as<int>(); }
if (vm.count("output-method")) { _options->outputMethod = vm["output-method"].as<string>(); }
if (vm.count("output-address")) { _options->outputAddress = vm["output-address"].as<string>(); }
return true;
}
int main(int argc, char** argv)
{
FairMQMerger merger;
merger.CatchSignals();
DeviceOptions_t options;
FairMQProgOptions config;
try
{
if (!parse_cmd_line(argc, argv, &options))
if (config.ParseAll(argc, argv))
{
return 0;
}
merger.SetConfig(config);
merger.ChangeState("INIT_DEVICE");
merger.WaitForEndOfState("INIT_DEVICE");
merger.ChangeState("INIT_TASK");
merger.WaitForEndOfState("INIT_TASK");
merger.ChangeState("RUN");
merger.InteractiveStateLoop();
}
catch (exception& e)
catch (std::exception& e)
{
LOG(ERROR) << e.what();
LOG(INFO) << "Command line options are the following: ";
config.PrintHelp();
return 1;
}
LOG(INFO) << "PID: " << getpid();
merger.SetTransport(options.transport);
for (unsigned int i = 0; i < options.inputAddress.size(); ++i)
{
FairMQChannel inputChannel(options.inputSocketType.at(i), options.inputMethod.at(i), options.inputAddress.at(i));
inputChannel.UpdateSndBufSize(options.inputBufSize.at(i));
inputChannel.UpdateRcvBufSize(options.inputBufSize.at(i));
inputChannel.UpdateRateLogging(1);
merger.fChannels["data-in"].push_back(inputChannel);
}
FairMQChannel outputChannel(options.outputSocketType, options.outputMethod, options.outputAddress);
outputChannel.UpdateSndBufSize(options.outputBufSize);
outputChannel.UpdateRcvBufSize(options.outputBufSize);
outputChannel.UpdateRateLogging(1);
merger.fChannels["data-out"].push_back(outputChannel);
merger.SetProperty(FairMQMerger::Id, options.id);
merger.SetProperty(FairMQMerger::NumIoThreads, options.ioThreads);
merger.ChangeState("INIT_DEVICE");
merger.WaitForEndOfState("INIT_DEVICE");
merger.ChangeState("INIT_TASK");
merger.WaitForEndOfState("INIT_TASK");
merger.ChangeState("RUN");
merger.InteractiveStateLoop();
return 0;
}

123
fairmq/run/runProxy.cxx
View File

@@ -14,126 +14,45 @@
#include <iostream>
#include "boost/program_options.hpp"
#include "FairMQLogger.h"
#include "FairMQParser.h"
#include "FairMQProgOptions.h"
#include "FairMQProxy.h"
using namespace std;
typedef struct DeviceOptions
{
DeviceOptions() :
id(), ioThreads(0), transport(),
inputSocketType(), inputBufSize(0), inputMethod(), inputAddress(),
outputSocketType(), outputBufSize(0), outputMethod(), outputAddress() {}
string id;
int ioThreads;
string transport;
string inputSocketType;
int inputBufSize;
string inputMethod;
string inputAddress;
string outputSocketType;
int outputBufSize;
string outputMethod;
string outputAddress;
} DeviceOptions_t;
inline bool parse_cmd_line(int _argc, char* _argv[], DeviceOptions* _options)
{
if (_options == NULL)
throw runtime_error("Internal error: options' container is empty.");
namespace bpo = boost::program_options;
bpo::options_description desc("Options");
desc.add_options()
("id", bpo::value<string>()->required(), "Device ID")
("io-threads", bpo::value<int>()->default_value(1), "Number of I/O threads")
("transport", bpo::value<string>()->default_value("zeromq"), "Transport (zeromq/nanomsg)")
("input-socket-type", bpo::value<string>()->required(), "Input socket type: sub/pull")
("input-buff-size", bpo::value<int>()->default_value(1000), "Input buffer size in number of messages (ZeroMQ)/bytes(nanomsg)")
("input-method", bpo::value<string>()->required(), "Input method: bind/connect")
("input-address", bpo::value<string>()->required(), "Input address, e.g.: \"tcp://localhost:5555\"")
("output-socket-type", bpo::value<string>()->required(), "Output socket type: pub/push")
("output-buff-size", bpo::value<int>()->default_value(1000), "Output buffer size in number of messages (ZeroMQ)/bytes(nanomsg)")
("output-method", bpo::value<string>()->required(), "Output method: bind/connect")
("output-address", bpo::value<string>()->required(), "Output address, e.g.: \"tcp://localhost:5555\"")
("help", "Print help messages");
bpo::variables_map vm;
bpo::store(bpo::parse_command_line(_argc, _argv, desc), vm);
if (vm.count("help"))
{
LOG(INFO) << "FairMQ Proxy" << endl << desc;
return false;
}
bpo::notify(vm);
if (vm.count("id")) { _options->id = vm["id"].as<string>(); }
if (vm.count("io-threads")) { _options->ioThreads = vm["io-threads"].as<int>(); }
if (vm.count("transport")) { _options->transport = vm["transport"].as<string>(); }
if (vm.count("input-socket-type")) { _options->inputSocketType = vm["input-socket-type"].as<string>(); }
if (vm.count("input-buff-size")) { _options->inputBufSize = vm["input-buff-size"].as<int>(); }
if (vm.count("input-method")) { _options->inputMethod = vm["input-method"].as<string>(); }
if (vm.count("input-address")) { _options->inputAddress = vm["input-address"].as<string>(); }
if (vm.count("output-socket-type")) { _options->outputSocketType = vm["output-socket-type"].as<string>(); }
if (vm.count("output-buff-size")) { _options->outputBufSize = vm["output-buff-size"].as<int>(); }
if (vm.count("output-method")) { _options->outputMethod = vm["output-method"].as<string>(); }
if (vm.count("output-address")) { _options->outputAddress = vm["output-address"].as<string>(); }
return true;
}
int main(int argc, char** argv)
{
FairMQProxy proxy;
proxy.CatchSignals();
DeviceOptions_t options;
FairMQProgOptions config;
try
{
if (!parse_cmd_line(argc, argv, &options))
if (config.ParseAll(argc, argv))
{
return 0;
}
proxy.SetConfig(config);
proxy.ChangeState("INIT_DEVICE");
proxy.WaitForEndOfState("INIT_DEVICE");
proxy.ChangeState("INIT_TASK");
proxy.WaitForEndOfState("INIT_TASK");
proxy.ChangeState("RUN");
proxy.InteractiveStateLoop();
}
catch (exception& e)
catch (std::exception& e)
{
LOG(ERROR) << e.what();
LOG(INFO) << "Command line options are the following: ";
config.PrintHelp();
return 1;
}
LOG(INFO) << "PID: " << getpid();
proxy.SetTransport(options.transport);
FairMQChannel inputChannel(options.inputSocketType, options.inputMethod, options.inputAddress);
inputChannel.UpdateSndBufSize(options.inputBufSize);
inputChannel.UpdateRcvBufSize(options.inputBufSize);
inputChannel.UpdateRateLogging(1);
proxy.fChannels["data-in"].push_back(inputChannel);
FairMQChannel outputChannel(options.outputSocketType, options.outputMethod, options.outputAddress);
outputChannel.UpdateSndBufSize(options.outputBufSize);
outputChannel.UpdateRcvBufSize(options.outputBufSize);
outputChannel.UpdateRateLogging(1);
proxy.fChannels["data-out"].push_back(outputChannel);
proxy.SetProperty(FairMQProxy::Id, options.id);
proxy.SetProperty(FairMQProxy::NumIoThreads, options.ioThreads);
proxy.ChangeState("INIT_DEVICE");
proxy.WaitForEndOfState("INIT_DEVICE");
proxy.ChangeState("INIT_TASK");
proxy.WaitForEndOfState("INIT_TASK");
proxy.ChangeState("RUN");
proxy.InteractiveStateLoop();
return 0;
}

132
fairmq/run/runSplitter.cxx
View File

@@ -14,133 +14,43 @@
#include <iostream>
#include "boost/program_options.hpp"
#include "FairMQLogger.h"
#include "FairMQParser.h"
#include "FairMQProgOptions.h"
#include "FairMQSplitter.h"
using namespace std;
typedef struct DeviceOptions
{
DeviceOptions() :
id(), ioThreads(0), transport(), numOutputs(0),
inputSocketType(), inputBufSize(0), inputMethod(), inputAddress(),
outputSocketType(), outputBufSize(), outputMethod(), outputAddress()
{}
string id;
int ioThreads;
string transport;
int numOutputs;
string inputSocketType;
int inputBufSize;
string inputMethod;
string inputAddress;
vector<string> outputSocketType;
vector<int> outputBufSize;
vector<string> outputMethod;
vector<string> outputAddress;
} DeviceOptions_t;
inline bool parse_cmd_line(int _argc, char* _argv[], DeviceOptions* _options)
{
if (_options == NULL)
throw runtime_error("Internal error: options' container is empty.");
namespace bpo = boost::program_options;
bpo::options_description desc("Options");
desc.add_options()
("id", bpo::value<string>()->required(), "Device ID")
("io-threads", bpo::value<int>()->default_value(1), "Number of I/O threads")
("transport", bpo::value<string>()->default_value("zeromq"), "Transport (zeromq/nanomsg)")
("num-outputs", bpo::value<int>()->required(), "Number of Splitter output sockets")
("input-socket-type", bpo::value<string>()->required(), "Input socket type: sub/pull")
("input-buff-size", bpo::value<int>(), "Input buffer size in number of messages (ZeroMQ)/bytes(nanomsg)")
("input-method", bpo::value<string>()->required(), "Input method: bind/connect")
("input-address", bpo::value<string>()->required(), "Input address, e.g.: \"tcp://localhost:5555\"")
("output-socket-type", bpo::value<vector<string>>()->required(), "Output socket type: pub/push")
("output-buff-size", bpo::value<vector<int>>(), "Output buffer size in number of messages (ZeroMQ)/bytes(nanomsg)")
("output-method", bpo::value<vector<string>>()->required(), "Output method: bind/connect")
("output-address", bpo::value<vector<string>>()->required(), "Output address, e.g.: \"tcp://localhost:5555\"")
("help", "Print help messages");
bpo::variables_map vm;
bpo::store(bpo::parse_command_line(_argc, _argv, desc), vm);
if (vm.count("help"))
{
LOG(INFO) << "FairMQ Splitter" << endl << desc;
return false;
}
bpo::notify(vm);
if (vm.count("id")) { _options->id = vm["id"].as<string>(); }
if (vm.count("io-threads")) { _options->ioThreads = vm["io-threads"].as<int>(); }
if (vm.count("transport")) { _options->transport = vm["transport"].as<string>(); }
if (vm.count("num-outputs")) { _options->numOutputs = vm["num-outputs"].as<int>(); }
if (vm.count("input-socket-type")) { _options->inputSocketType = vm["input-socket-type"].as<string>(); }
if (vm.count("input-buff-size")) { _options->inputBufSize = vm["input-buff-size"].as<int>(); }
if (vm.count("input-method")) { _options->inputMethod = vm["input-method"].as<string>(); }
if (vm.count("input-address")) { _options->inputAddress = vm["input-address"].as<string>(); }
if (vm.count("output-socket-type")) { _options->outputSocketType = vm["output-socket-type"].as<vector<string>>(); }
if (vm.count("output-buff-size")) { _options->outputBufSize = vm["output-buff-size"].as<vector<int>>(); }
if (vm.count("output-method")) { _options->outputMethod = vm["output-method"].as<vector<string>>(); }
if (vm.count("output-address")) { _options->outputAddress = vm["output-address"].as<vector<string>>(); }
return true;
}
int main(int argc, char** argv)
{
FairMQSplitter splitter;
splitter.CatchSignals();
DeviceOptions_t options;
FairMQProgOptions config;
try
{
if (!parse_cmd_line(argc, argv, &options))
if (config.ParseAll(argc, argv))
{
return 0;
}
splitter.SetConfig(config);
splitter.ChangeState("INIT_DEVICE");
splitter.WaitForEndOfState("INIT_DEVICE");
splitter.ChangeState("INIT_TASK");
splitter.WaitForEndOfState("INIT_TASK");
splitter.ChangeState("RUN");
splitter.InteractiveStateLoop();
}
catch (exception& e)
catch (std::exception& e)
{
LOG(ERROR) << e.what();
LOG(INFO) << "Command line options are the following: ";
config.PrintHelp();
return 1;
}
LOG(INFO) << "PID: " << getpid();
splitter.SetTransport(options.transport);
FairMQChannel inputChannel(options.inputSocketType, options.inputMethod, options.inputAddress);
inputChannel.UpdateSndBufSize(options.inputBufSize);
inputChannel.UpdateRcvBufSize(options.inputBufSize);
inputChannel.UpdateRateLogging(1);
splitter.fChannels["data-in"].push_back(inputChannel);
for (unsigned int i = 0; i < options.outputAddress.size(); ++i)
{
FairMQChannel outputChannel(options.outputSocketType.at(i), options.outputMethod.at(i), options.outputAddress.at(i));
outputChannel.UpdateSndBufSize(options.outputBufSize.at(i));
outputChannel.UpdateRcvBufSize(options.outputBufSize.at(i));
outputChannel.UpdateRateLogging(1);
splitter.fChannels["data-out"].push_back(outputChannel);
}
splitter.SetProperty(FairMQSplitter::Id, options.id);
splitter.SetProperty(FairMQSplitter::NumIoThreads, options.ioThreads);
splitter.ChangeState("INIT_DEVICE");
splitter.WaitForEndOfState("INIT_DEVICE");
splitter.ChangeState("INIT_TASK");
splitter.WaitForEndOfState("INIT_TASK");
splitter.ChangeState("RUN");
splitter.InteractiveStateLoop();
return 0;
}

0
fairmq/run/startBenchmark.sh.in → fairmq/run/startFairMQBenchmark.sh.in
View File