From 0d0c59a98c8fcf926bb3c27cade3929e862bc829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20D=C3=ADaz=20Qu=C3=ADlez?= <33602217+Tempate@users.noreply.github.com> Date: Wed, 8 Nov 2023 08:24:30 +0100 Subject: [PATCH] Manually fix a set of Topic QoS, and implement the max-tx-rate (#86) * Make the DdsPipeConfiguration required Signed-off-by: tempate * Rename max-reception-rate to max-rx-rate Signed-off-by: tempate * Support max-tx-rate in the replayer Signed-off-by: tempate * Support manual topics configuration Signed-off-by: tempate * Move the allowed & builtin topics to the pipe Signed-off-by: tempate * Add Topic QoS to the McapReaderParticipant Signed-off-by: tempate * Apply suggestions Signed-off-by: tempate * Support the new Topic QoS Signed-off-by: tempate * Minor fixes to the documentation Signed-off-by: tempate * Add missing quotes to wildcard entries Signed-off-by: tempate * Fix downsampling test Signed-off-by: tempate * Apply suggestions Signed-off-by: tempate * Apply DdsPipe suggestions Signed-off-by: tempate * Apply suggestions Signed-off-by: tempate * Remove the built-in topics from the DdsReplayer Signed-off-by: tempate * Apply suggestions Signed-off-by: tempate * Apply more suggestions Signed-off-by: tempate * Apply the manual topics to the dynamic writer Signed-off-by: tempate * Fix compilation error Signed-off-by: tempate * Remove Topic Filtering outdated note Signed-off-by: tempate * Fix Windows tag Signed-off-by: tempate * Fix forthcoming notes Signed-off-by: tempate * Uncrustify Signed-off-by: tempate --------- Signed-off-by: tempate --- ddsrecorder/src/cpp/main.cpp | 12 +- ddsrecorder/src/cpp/tool/DdsRecorder.cpp | 17 +- ddsrecorder/src/cpp/tool/DdsRecorder.hpp | 4 +- .../blackbox/mcap/McapFileCreationTest.cpp | 4 +- .../replayer/McapReaderParticipant.hpp | 4 + .../cpp/replayer/McapReaderParticipant.cpp | 5 + .../recorder/YamlReaderConfiguration.hpp | 14 +- .../replayer/YamlReaderConfiguration.hpp | 12 +- .../cpp/recorder/YamlReaderConfiguration.cpp | 57 ++-- .../cpp/replayer/YamlReaderConfiguration.cpp | 39 ++- ddsreplayer/src/cpp/main.cpp | 12 +- ddsreplayer/src/cpp/tool/DdsReplayer.cpp | 50 ++-- ddsreplayer/src/cpp/tool/DdsReplayer.hpp | 6 +- .../blackbox/mcap/resources/config_file.yaml | 6 +- .../resources/config_file_begin_time.yaml | 2 +- .../config_file_begin_time_with_types.yaml | 2 +- .../mcap/resources/config_file_end_time.yaml | 2 +- .../config_file_end_time_with_types.yaml | 3 +- .../mcap/resources/config_file_less_hz.yaml | 2 +- .../mcap/resources/config_file_more_hz.yaml | 2 +- ...config_file_start_replay_time_earlier.yaml | 2 +- ..._start_replay_time_earlier_with_types.yaml | 2 +- docs/rst/notes/forthcoming_version.rst | 8 + .../remote_control/remote_control.rst | 2 +- docs/rst/recording/usage/configuration.rst | 275 +++++++++--------- docs/rst/replaying/usage/configuration.rst | 236 ++++++++------- .../recorder/complete_config.yaml | 19 +- .../replayer/complete_config.yaml | 15 +- 28 files changed, 412 insertions(+), 402 deletions(-) diff --git a/ddsrecorder/src/cpp/main.cpp b/ddsrecorder/src/cpp/main.cpp index d918178ff..0280a97a3 100644 --- a/ddsrecorder/src/cpp/main.cpp +++ b/ddsrecorder/src/cpp/main.cpp @@ -67,11 +67,7 @@ std::unique_ptr create_filewatcher( try { eprosima::ddsrecorder::yaml::RecorderConfiguration new_configuration(file_path); - // Create new allowed topics list - auto new_allowed_topics = std::make_shared( - new_configuration.allowlist, - new_configuration.blocklist); - recorder->reload_allowed_topics(new_allowed_topics); + recorder->reload_configuration(new_configuration); } catch (const std::exception& e) { @@ -102,11 +98,7 @@ std::unique_ptr create_periodic_ha try { eprosima::ddsrecorder::yaml::RecorderConfiguration new_configuration(file_path); - // Create new allowed topics list - auto new_allowed_topics = std::make_shared( - new_configuration.allowlist, - new_configuration.blocklist); - recorder->reload_allowed_topics(new_allowed_topics); + recorder->reload_configuration(new_configuration); } catch (const std::exception& e) { diff --git a/ddsrecorder/src/cpp/tool/DdsRecorder.cpp b/ddsrecorder/src/cpp/tool/DdsRecorder.cpp index 3f318802f..19483e89d 100644 --- a/ddsrecorder/src/cpp/tool/DdsRecorder.cpp +++ b/ddsrecorder/src/cpp/tool/DdsRecorder.cpp @@ -33,11 +33,6 @@ DdsRecorder::DdsRecorder( const DdsRecorderStateCode& init_state, const std::string& file_name) { - // Create allowed topics list - auto allowed_topics = std::make_shared( - configuration.allowlist, - configuration.blocklist); - // Create Discovery Database discovery_database_ = std::make_shared(); @@ -115,19 +110,17 @@ DdsRecorder::DdsRecorder( // Create DDS Pipe pipe_ = std::make_unique( - allowed_topics, + configuration.ddspipe_configuration, discovery_database_, payload_pool_, participants_database_, - thread_pool_, - configuration.builtin_topics, - true); + thread_pool_); } -utils::ReturnCode DdsRecorder::reload_allowed_topics( - const std::shared_ptr& allowed_topics) +utils::ReturnCode DdsRecorder::reload_configuration( + const yaml::RecorderConfiguration& new_configuration) { - return pipe_->reload_allowed_topics(allowed_topics); + return pipe_->reload_configuration(new_configuration.ddspipe_configuration); } void DdsRecorder::start() diff --git a/ddsrecorder/src/cpp/tool/DdsRecorder.hpp b/ddsrecorder/src/cpp/tool/DdsRecorder.hpp index 93b89ad84..1d3d2c58c 100644 --- a/ddsrecorder/src/cpp/tool/DdsRecorder.hpp +++ b/ddsrecorder/src/cpp/tool/DdsRecorder.hpp @@ -77,8 +77,8 @@ class DdsRecorder * @return \c RETCODE_OK if allowed topics list has been updated correctly * @return \c RETCODE_NO_DATA if new allowed topics list is the same as the previous one */ - utils::ReturnCode reload_allowed_topics( - const std::shared_ptr& allowed_topics); + utils::ReturnCode reload_configuration( + const yaml::RecorderConfiguration& new_configuration); //! Start recorder (\c mcap_handler_) void start(); diff --git a/ddsrecorder/test/blackbox/mcap/McapFileCreationTest.cpp b/ddsrecorder/test/blackbox/mcap/McapFileCreationTest.cpp index ef0f413c2..056bc1e79 100644 --- a/ddsrecorder/test/blackbox/mcap/McapFileCreationTest.cpp +++ b/ddsrecorder/test/blackbox/mcap/McapFileCreationTest.cpp @@ -88,10 +88,10 @@ std::unique_ptr create_recorder( YAML::Node yml; eprosima::ddsrecorder::yaml::RecorderConfiguration configuration(yml); - configuration.downsampling = downsampling; + configuration.topic_qos.downsampling = downsampling; // Set default value for downsampling // TODO: Change mechanism setting topic qos' default values from specs - eprosima::ddspipe::core::types::TopicQoS::default_downsampling.store(downsampling); + eprosima::ddspipe::core::types::TopicQoS::default_topic_qos.set_value(configuration.topic_qos); configuration.event_window = event_window; eprosima::ddspipe::core::types::DomainId domainId; domainId.domain_id = test::DOMAIN; diff --git a/ddsrecorder_participants/include/ddsrecorder_participants/replayer/McapReaderParticipant.hpp b/ddsrecorder_participants/include/ddsrecorder_participants/replayer/McapReaderParticipant.hpp index 47ff29da1..bae6e5c46 100644 --- a/ddsrecorder_participants/include/ddsrecorder_participants/replayer/McapReaderParticipant.hpp +++ b/ddsrecorder_participants/include/ddsrecorder_participants/replayer/McapReaderParticipant.hpp @@ -70,6 +70,10 @@ class McapReaderParticipant : public ddspipe::core::IParticipant DDSRECORDER_PARTICIPANTS_DllAPI bool is_rtps_kind() const noexcept override; + //! Override topic_qos() IParticipant method + DDSRECORDER_PARTICIPANTS_DllAPI + ddspipe::core::types::TopicQoS topic_qos() const noexcept override; + //! Override create_writer_() IParticipant method DDSRECORDER_PARTICIPANTS_DllAPI std::shared_ptr create_writer( diff --git a/ddsrecorder_participants/src/cpp/replayer/McapReaderParticipant.cpp b/ddsrecorder_participants/src/cpp/replayer/McapReaderParticipant.cpp index 5eb47871e..217afd64f 100644 --- a/ddsrecorder_participants/src/cpp/replayer/McapReaderParticipant.cpp +++ b/ddsrecorder_participants/src/cpp/replayer/McapReaderParticipant.cpp @@ -68,6 +68,11 @@ bool McapReaderParticipant::is_rtps_kind() const noexcept return false; } +TopicQoS McapReaderParticipant::topic_qos() const noexcept +{ + return configuration_->topic_qos; +} + std::shared_ptr McapReaderParticipant::create_writer( const ITopic& /* topic */) { diff --git a/ddsrecorder_yaml/include/ddsrecorder_yaml/recorder/YamlReaderConfiguration.hpp b/ddsrecorder_yaml/include/ddsrecorder_yaml/recorder/YamlReaderConfiguration.hpp index a9e8a12ce..03414e691 100644 --- a/ddsrecorder_yaml/include/ddsrecorder_yaml/recorder/YamlReaderConfiguration.hpp +++ b/ddsrecorder_yaml/include/ddsrecorder_yaml/recorder/YamlReaderConfiguration.hpp @@ -22,6 +22,8 @@ #include +#include +#include #include #include @@ -52,15 +54,13 @@ class DDSRECORDER_YAML_DllAPI RecorderConfiguration RecorderConfiguration( const std::string& file_path); + // DDS Pipe Configuration + ddspipe::core::DdsPipeConfiguration ddspipe_configuration; + // Participants configurations std::shared_ptr simple_configuration; std::shared_ptr recorder_configuration; - // Topic filtering - std::set> allowlist{}; - std::set> blocklist{}; - std::set> builtin_topics{}; - // Output file params std::string output_filepath = "."; std::string output_filename = "output"; @@ -71,8 +71,6 @@ class DDSRECORDER_YAML_DllAPI RecorderConfiguration unsigned int buffer_size = 100; unsigned int event_window = 20; bool log_publish_time = false; - unsigned int downsampling = 1; - float max_reception_rate = 0; bool only_with_type = false; mcap::McapWriterOptions mcap_writer_options{"ros2"}; bool record_types = true; @@ -86,9 +84,9 @@ class DDSRECORDER_YAML_DllAPI RecorderConfiguration // Specs unsigned int n_threads = 12; - unsigned int max_history_depth = 5000; int max_pending_samples = 5000; // -1 <-> no limit || 0 <-> no pending samples unsigned int cleanup_period; + ddspipe::core::types::TopicQoS topic_qos{}; protected: diff --git a/ddsrecorder_yaml/include/ddsrecorder_yaml/replayer/YamlReaderConfiguration.hpp b/ddsrecorder_yaml/include/ddsrecorder_yaml/replayer/YamlReaderConfiguration.hpp index 32f34db2d..63a0f18e3 100644 --- a/ddsrecorder_yaml/include/ddsrecorder_yaml/replayer/YamlReaderConfiguration.hpp +++ b/ddsrecorder_yaml/include/ddsrecorder_yaml/replayer/YamlReaderConfiguration.hpp @@ -22,6 +22,8 @@ #include #include +#include +#include #include #include @@ -52,15 +54,13 @@ class DDSRECORDER_YAML_DllAPI ReplayerConfiguration ReplayerConfiguration( const std::string& file_path); + // DDS Pipe Configuration + ddspipe::core::DdsPipeConfiguration ddspipe_configuration; + // Participants configurations std::shared_ptr mcap_reader_configuration; std::shared_ptr replayer_configuration; - // Topic filtering - std::set> allowlist{}; - std::set> blocklist{}; - std::set> builtin_topics{}; - // Replay params std::string input_file; utils::Fuzzy begin_time{}; @@ -71,7 +71,7 @@ class DDSRECORDER_YAML_DllAPI ReplayerConfiguration // Specs unsigned int n_threads = 12; - unsigned int max_history_depth = 5000; + ddspipe::core::types::TopicQoS topic_qos{}; protected: diff --git a/ddsrecorder_yaml/src/cpp/recorder/YamlReaderConfiguration.cpp b/ddsrecorder_yaml/src/cpp/recorder/YamlReaderConfiguration.cpp index d79dfdfcf..a8f8be7f9 100644 --- a/ddsrecorder_yaml/src/cpp/recorder/YamlReaderConfiguration.cpp +++ b/ddsrecorder_yaml/src/cpp/recorder/YamlReaderConfiguration.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -115,11 +116,16 @@ void RecorderConfiguration::load_ddsrecorder_configuration_( WildcardDdsFilterTopic rpc_request_topic, rpc_response_topic; rpc_request_topic.topic_name.set_value("rq/*"); rpc_response_topic.topic_name.set_value("rr/*"); - blocklist.insert( + + ddspipe_configuration.blocklist.insert( utils::Heritable::make_heritable(rpc_request_topic)); - blocklist.insert( + + ddspipe_configuration.blocklist.insert( utils::Heritable::make_heritable(rpc_response_topic)); + // The DDS Pipe should be enabled on start up. + ddspipe_configuration.init_enabled = true; + // Initialize controller domain with the same as the one being recorded // WARNING: dds tag must have been parsed beforehand controller_domain = simple_configuration->domain; @@ -198,24 +204,6 @@ void RecorderConfiguration::load_recorder_configuration_( log_publish_time = YamlReader::get(yml, RECORDER_LOG_PUBLISH_TIME_TAG, version); } - ///// - // Get optional downsampling - if (YamlReader::is_tag_present(yml, DOWNSAMPLING_TAG)) - { - downsampling = YamlReader::get_positive_int(yml, DOWNSAMPLING_TAG); - // Set default value for downsampling - TopicQoS::default_downsampling.store(downsampling); - } - - ///// - // Get optional max reception rate - if (YamlReader::is_tag_present(yml, MAX_RECEPTION_RATE_TAG)) - { - // Set default value for max reception rate - TopicQoS::default_max_reception_rate.store(YamlReader::get_nonnegative_float(yml, - MAX_RECEPTION_RATE_TAG)); - } - ///// // Get optional only_with_type if (YamlReader::is_tag_present(yml, RECORDER_ONLY_WITH_TYPE_TAG)) @@ -290,12 +278,12 @@ void RecorderConfiguration::load_specs_configuration_( n_threads = YamlReader::get_positive_int(yml, NUMBER_THREADS_TAG); } - // Get maximum history depth - if (YamlReader::is_tag_present(yml, MAX_HISTORY_DEPTH_TAG)) + ///// + // Get optional Topic QoS + if (YamlReader::is_tag_present(yml, SPECS_QOS_TAG)) { - max_history_depth = YamlReader::get_positive_int(yml, MAX_HISTORY_DEPTH_TAG); - // Set default value for history - TopicQoS::default_history_depth.store(max_history_depth); + YamlReader::fill(topic_qos, YamlReader::get_value_in_tag(yml, SPECS_QOS_TAG), version); + TopicQoS::default_topic_qos.set_value(topic_qos); } // Get max pending samples @@ -363,12 +351,13 @@ void RecorderConfiguration::load_dds_configuration_( // Get optional allowlist if (YamlReader::is_tag_present(yml, ALLOWLIST_TAG)) { - allowlist = YamlReader::get_set>(yml, ALLOWLIST_TAG, version); + ddspipe_configuration.allowlist = YamlReader::get_set>(yml, ALLOWLIST_TAG, + version); // Add to allowlist always the type object topic WildcardDdsFilterTopic internal_topic; internal_topic.topic_name.set_value(TYPE_OBJECT_TOPIC_NAME); - allowlist.insert( + ddspipe_configuration.allowlist.insert( utils::Heritable::make_heritable(internal_topic)); } @@ -376,7 +365,17 @@ void RecorderConfiguration::load_dds_configuration_( // Get optional blocklist if (YamlReader::is_tag_present(yml, BLOCKLIST_TAG)) { - blocklist = YamlReader::get_set>(yml, BLOCKLIST_TAG, version); + ddspipe_configuration.blocklist = YamlReader::get_set>(yml, BLOCKLIST_TAG, + version); + } + + ///// + // Get optional topics + if (YamlReader::is_tag_present(yml, TOPICS_TAG)) + { + const auto& manual_topics = YamlReader::get_list(yml, TOPICS_TAG, version); + ddspipe_configuration.manual_topics = + std::vector(manual_topics.begin(), manual_topics.end()); } ///// @@ -384,7 +383,7 @@ void RecorderConfiguration::load_dds_configuration_( if (YamlReader::is_tag_present(yml, BUILTIN_TAG)) { // WARNING: Parse builtin topics AFTER specs and recorder, as some topic-specific default values are set there - builtin_topics = YamlReader::get_set>(yml, BUILTIN_TAG, + ddspipe_configuration.builtin_topics = YamlReader::get_set>(yml, BUILTIN_TAG, version); } } diff --git a/ddsrecorder_yaml/src/cpp/replayer/YamlReaderConfiguration.cpp b/ddsrecorder_yaml/src/cpp/replayer/YamlReaderConfiguration.cpp index bb15da878..556b3c228 100644 --- a/ddsrecorder_yaml/src/cpp/replayer/YamlReaderConfiguration.cpp +++ b/ddsrecorder_yaml/src/cpp/replayer/YamlReaderConfiguration.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -73,7 +74,6 @@ void ReplayerConfiguration::load_ddsreplayer_configuration_( ///// // Get optional specs configuration - // WARNING: Parse builtin topics (dds tag) AFTER specs, as some topic-specific default values are set there if (YamlReader::is_tag_present(yml, SPECS_TAG)) { auto specs_yml = YamlReader::get_value_in_tag(yml, SPECS_TAG); @@ -120,10 +120,15 @@ void ReplayerConfiguration::load_ddsreplayer_configuration_( WildcardDdsFilterTopic rpc_request_topic, rpc_response_topic; rpc_request_topic.topic_name.set_value("rq/*"); rpc_response_topic.topic_name.set_value("rr/*"); - blocklist.insert( + + ddspipe_configuration.blocklist.insert( utils::Heritable::make_heritable(rpc_request_topic)); - blocklist.insert( + + ddspipe_configuration.blocklist.insert( utils::Heritable::make_heritable(rpc_response_topic)); + + // The DDS Pipe should be enabled on start up. + ddspipe_configuration.init_enabled = true; } catch (const std::exception& e) { @@ -191,12 +196,12 @@ void ReplayerConfiguration::load_specs_configuration_( n_threads = YamlReader::get_positive_int(yml, NUMBER_THREADS_TAG); } - // Get maximum history depth - if (YamlReader::is_tag_present(yml, MAX_HISTORY_DEPTH_TAG)) + ///// + // Get optional Topic QoS + if (YamlReader::is_tag_present(yml, SPECS_QOS_TAG)) { - max_history_depth = YamlReader::get_positive_int(yml, MAX_HISTORY_DEPTH_TAG); - // Set default value for history - TopicQoS::default_history_depth.store(max_history_depth); + YamlReader::fill(topic_qos, YamlReader::get_value_in_tag(yml, SPECS_QOS_TAG), version); + TopicQoS::default_topic_qos.set_value(topic_qos); } // Get wait all acknowledged timeout @@ -253,12 +258,13 @@ void ReplayerConfiguration::load_dds_configuration_( // Get optional allowlist if (YamlReader::is_tag_present(yml, ALLOWLIST_TAG)) { - allowlist = YamlReader::get_set>(yml, ALLOWLIST_TAG, version); + ddspipe_configuration.allowlist = YamlReader::get_set>(yml, ALLOWLIST_TAG, + version); // Add to allowlist always the type object topic WildcardDdsFilterTopic internal_topic; internal_topic.topic_name.set_value(TYPE_OBJECT_TOPIC_NAME); - allowlist.insert( + ddspipe_configuration.allowlist.insert( utils::Heritable::make_heritable(internal_topic)); } @@ -266,16 +272,17 @@ void ReplayerConfiguration::load_dds_configuration_( // Get optional blocklist if (YamlReader::is_tag_present(yml, BLOCKLIST_TAG)) { - blocklist = YamlReader::get_set>(yml, BLOCKLIST_TAG, version); + ddspipe_configuration.blocklist = YamlReader::get_set>(yml, BLOCKLIST_TAG, + version); } ///// - // Get optional builtin topics - if (YamlReader::is_tag_present(yml, BUILTIN_TAG)) + // Get optional topics + if (YamlReader::is_tag_present(yml, TOPICS_TAG)) { - // WARNING: Parse builtin topics AFTER specs, as some topic-specific default values are set there - builtin_topics = YamlReader::get_set>(yml, BUILTIN_TAG, - version); + const auto& manual_topics = YamlReader::get_list(yml, TOPICS_TAG, version); + ddspipe_configuration.manual_topics = + std::vector(manual_topics.begin(), manual_topics.end()); } } diff --git a/ddsreplayer/src/cpp/main.cpp b/ddsreplayer/src/cpp/main.cpp index 0b507fea7..e7c411baa 100644 --- a/ddsreplayer/src/cpp/main.cpp +++ b/ddsreplayer/src/cpp/main.cpp @@ -58,11 +58,7 @@ std::unique_ptr create_filewatcher( try { eprosima::ddsrecorder::yaml::ReplayerConfiguration new_configuration(file_path); - // Create new allowed topics list - auto new_allowed_topics = std::make_shared( - new_configuration.allowlist, - new_configuration.blocklist); - replayer->reload_allowed_topics(new_allowed_topics); + replayer->reload_configuration(new_configuration); } catch (const std::exception& e) { @@ -93,11 +89,7 @@ std::unique_ptr create_periodic_ha try { eprosima::ddsrecorder::yaml::ReplayerConfiguration new_configuration(file_path); - // Create new allowed topics list - auto new_allowed_topics = std::make_shared( - new_configuration.allowlist, - new_configuration.blocklist); - replayer->reload_allowed_topics(new_allowed_topics); + replayer->reload_configuration(new_configuration); } catch (const std::exception& e) { diff --git a/ddsreplayer/src/cpp/tool/DdsReplayer.cpp b/ddsreplayer/src/cpp/tool/DdsReplayer.cpp index e15114d42..373b92f6b 100644 --- a/ddsreplayer/src/cpp/tool/DdsReplayer.cpp +++ b/ddsreplayer/src/cpp/tool/DdsReplayer.cpp @@ -51,16 +51,11 @@ using namespace eprosima::ddsrecorder::participants; using namespace eprosima::utils; DdsReplayer::DdsReplayer( - const yaml::ReplayerConfiguration& configuration, + yaml::ReplayerConfiguration& configuration, std::string& input_file) : dyn_participant_(nullptr) , dyn_publisher_(nullptr) { - // Create allowed topics list - auto allowed_topics = std::make_shared( - configuration.allowlist, - configuration.blocklist); - // Create Discovery Database discovery_database_ = std::make_shared(); @@ -131,17 +126,15 @@ DdsReplayer::DdsReplayer( } // Generate builtin-topics list by combining information from YAML and MCAP files - auto builtin_topics = generate_builtin_topics_(configuration, input_file); + configuration.ddspipe_configuration.builtin_topics = generate_builtin_topics_(configuration, input_file); // Create DDS Pipe pipe_ = std::make_unique( - allowed_topics, + configuration.ddspipe_configuration, discovery_database_, payload_pool_, participants_database_, - thread_pool_, - builtin_topics, - true); + thread_pool_); } DdsReplayer::~DdsReplayer() @@ -170,10 +163,10 @@ DdsReplayer::~DdsReplayer() } } -utils::ReturnCode DdsReplayer::reload_allowed_topics( - const std::shared_ptr& allowed_topics) +utils::ReturnCode DdsReplayer::reload_configuration( + const yaml::ReplayerConfiguration& new_configuration) { - return pipe_->reload_allowed_topics(allowed_topics); + return pipe_->reload_configuration(new_configuration.ddspipe_configuration); } void DdsReplayer::process_mcap() @@ -198,7 +191,7 @@ std::set> DdsReplayer::generate_builtin_topic const yaml::ReplayerConfiguration& configuration, std::string& input_file) { - std::set> builtin_topics = configuration.builtin_topics; + std::set> builtin_topics; mcap::McapReader mcap_reader; @@ -274,6 +267,7 @@ std::set> DdsReplayer::generate_builtin_topic auto channels = mcap_reader.channels(); auto schemas = mcap_reader.schemas(); + for (auto it = channels.begin(); it != channels.end(); it++) { std::string topic_name = it->second->topic; @@ -283,25 +277,31 @@ std::set> DdsReplayer::generate_builtin_topic channel_topic->m_topic_name = topic_name; channel_topic->type_name = type_name; - if (builtin_topics.count(channel_topic) == 1) - { - // Already present in builtin_topics list, using qos provided through configuration - // NOTE: also covers situation where there are channels for same topic with and without (blank) schema - continue; - } - - // Use QoS stored in MCAP file (discovered when recording, or given to recorder's builtin topics list) - channel_topic->topic_qos = deserialize_qos_(it->second->metadata[QOS_SERIALIZATION_QOS]); + // Apply the QoS stored in the MCAP file as if they were the discovered QoS. + channel_topic->topic_qos.set_qos( + deserialize_qos_(it->second->metadata[QOS_SERIALIZATION_QOS]), + utils::FuzzyLevelValues::fuzzy_level_fuzzy); // Insert channel topic in builtin topics list builtin_topics.insert(channel_topic); if (configuration.replay_types && registered_types.count(type_name) != 0) { + // Make a copy of the Topic to customize it according to the Participant's configured QoS. + utils::Heritable topic = channel_topic->copy(); + + // Apply the Manual Topics for this participant. + for (const auto& manual_topic : configuration.ddspipe_configuration.get_manual_topics(*channel_topic)) + { + topic->topic_qos.set_qos(manual_topic.first->topic_qos, utils::FuzzyLevelValues::fuzzy_level_hard); + } + // Create Datawriter in this topic so dynamic type can be shared in EDP - create_dynamic_writer_(channel_topic); + // TODO: Avoid creating the dynamic writer when the topic is not allowed. + create_dynamic_writer_(topic); } } + mcap_reader.close(); return builtin_topics; diff --git a/ddsreplayer/src/cpp/tool/DdsReplayer.hpp b/ddsreplayer/src/cpp/tool/DdsReplayer.hpp index aa3d31c07..229106dc4 100644 --- a/ddsreplayer/src/cpp/tool/DdsReplayer.hpp +++ b/ddsreplayer/src/cpp/tool/DdsReplayer.hpp @@ -61,7 +61,7 @@ class DdsReplayer * @throw utils::InitializationException if failed to create dynamic participant/publisher. */ DdsReplayer( - const yaml::ReplayerConfiguration& configuration, + yaml::ReplayerConfiguration& configuration, std::string& input_file); /** @@ -79,8 +79,8 @@ class DdsReplayer * @return \c RETCODE_OK if allowed topics list has been updated correctly * @return \c RETCODE_NO_DATA if new allowed topics list is the same as the previous one */ - utils::ReturnCode reload_allowed_topics( - const std::shared_ptr& allowed_topics); + utils::ReturnCode reload_configuration( + const yaml::ReplayerConfiguration& new_configuration); //! Process input MCAP file void process_mcap(); diff --git a/ddsreplayer/test/blackbox/mcap/resources/config_file.yaml b/ddsreplayer/test/blackbox/mcap/resources/config_file.yaml index 263abe16d..39ab008a0 100644 --- a/ddsreplayer/test/blackbox/mcap/resources/config_file.yaml +++ b/ddsreplayer/test/blackbox/mcap/resources/config_file.yaml @@ -1,6 +1,6 @@ -builtin-topics: - - name: "/dds/topic" - type: "HelloWorld" +topics: + - name: /dds/topic + type: HelloWorld qos: reliability: true # Use QoS RELIABLE diff --git a/ddsreplayer/test/blackbox/mcap/resources/config_file_begin_time.yaml b/ddsreplayer/test/blackbox/mcap/resources/config_file_begin_time.yaml index 040f5e433..ccebb9e5a 100644 --- a/ddsreplayer/test/blackbox/mcap/resources/config_file_begin_time.yaml +++ b/ddsreplayer/test/blackbox/mcap/resources/config_file_begin_time.yaml @@ -1,4 +1,4 @@ -builtin-topics: +topics: - name: /dds/topic type: HelloWorld qos: diff --git a/ddsreplayer/test/blackbox/mcap/resources/config_file_begin_time_with_types.yaml b/ddsreplayer/test/blackbox/mcap/resources/config_file_begin_time_with_types.yaml index f6de79757..b2785096d 100644 --- a/ddsreplayer/test/blackbox/mcap/resources/config_file_begin_time_with_types.yaml +++ b/ddsreplayer/test/blackbox/mcap/resources/config_file_begin_time_with_types.yaml @@ -1,4 +1,4 @@ -builtin-topics: +topics: - name: /dds/topic type: HelloWorld qos: diff --git a/ddsreplayer/test/blackbox/mcap/resources/config_file_end_time.yaml b/ddsreplayer/test/blackbox/mcap/resources/config_file_end_time.yaml index cce3807d3..e241002ff 100644 --- a/ddsreplayer/test/blackbox/mcap/resources/config_file_end_time.yaml +++ b/ddsreplayer/test/blackbox/mcap/resources/config_file_end_time.yaml @@ -1,4 +1,4 @@ -builtin-topics: +topics: - name: /dds/topic type: HelloWorld qos: diff --git a/ddsreplayer/test/blackbox/mcap/resources/config_file_end_time_with_types.yaml b/ddsreplayer/test/blackbox/mcap/resources/config_file_end_time_with_types.yaml index 25b5a8b66..febc859a6 100644 --- a/ddsreplayer/test/blackbox/mcap/resources/config_file_end_time_with_types.yaml +++ b/ddsreplayer/test/blackbox/mcap/resources/config_file_end_time_with_types.yaml @@ -1,6 +1,5 @@ -builtin-topics: +topics: - name: /dds/topic - type: HelloWorld qos: reliability: true # Use QoS RELIABLE diff --git a/ddsreplayer/test/blackbox/mcap/resources/config_file_less_hz.yaml b/ddsreplayer/test/blackbox/mcap/resources/config_file_less_hz.yaml index 380c92c29..42beefa28 100644 --- a/ddsreplayer/test/blackbox/mcap/resources/config_file_less_hz.yaml +++ b/ddsreplayer/test/blackbox/mcap/resources/config_file_less_hz.yaml @@ -1,4 +1,4 @@ -builtin-topics: +topics: - name: /dds/topic type: HelloWorld qos: diff --git a/ddsreplayer/test/blackbox/mcap/resources/config_file_more_hz.yaml b/ddsreplayer/test/blackbox/mcap/resources/config_file_more_hz.yaml index 4fbd598c1..65045ef01 100644 --- a/ddsreplayer/test/blackbox/mcap/resources/config_file_more_hz.yaml +++ b/ddsreplayer/test/blackbox/mcap/resources/config_file_more_hz.yaml @@ -1,4 +1,4 @@ -builtin-topics: +topics: - name: /dds/topic type: HelloWorld qos: diff --git a/ddsreplayer/test/blackbox/mcap/resources/config_file_start_replay_time_earlier.yaml b/ddsreplayer/test/blackbox/mcap/resources/config_file_start_replay_time_earlier.yaml index c9d2cd7f1..c6e6b3499 100644 --- a/ddsreplayer/test/blackbox/mcap/resources/config_file_start_replay_time_earlier.yaml +++ b/ddsreplayer/test/blackbox/mcap/resources/config_file_start_replay_time_earlier.yaml @@ -1,4 +1,4 @@ -builtin-topics: +topics: - name: /dds/topic type: HelloWorld qos: diff --git a/ddsreplayer/test/blackbox/mcap/resources/config_file_start_replay_time_earlier_with_types.yaml b/ddsreplayer/test/blackbox/mcap/resources/config_file_start_replay_time_earlier_with_types.yaml index c9d2cd7f1..c6e6b3499 100644 --- a/ddsreplayer/test/blackbox/mcap/resources/config_file_start_replay_time_earlier_with_types.yaml +++ b/ddsreplayer/test/blackbox/mcap/resources/config_file_start_replay_time_earlier_with_types.yaml @@ -1,4 +1,4 @@ -builtin-topics: +topics: - name: /dds/topic type: HelloWorld qos: diff --git a/docs/rst/notes/forthcoming_version.rst b/docs/rst/notes/forthcoming_version.rst index a4fc1d58e..9ade025c4 100644 --- a/docs/rst/notes/forthcoming_version.rst +++ b/docs/rst/notes/forthcoming_version.rst @@ -25,3 +25,11 @@ Next release will include the following **DDS Recorder tool configuration featur * Support :ref:`Compression Settings `. * Allow disabling the storage of received types (see :ref:`Record Types `). * New configuration options (``timestamp-format`` and ``local-timestamp``) available for :ref:`output file ` settings. +* New configuration option (``topics``) to configure the :ref:`Manual Topics `. +* Rename ``max-reception-rate`` to ``max-rx-rate``. + +Next release will include the following **DDS Replayer tool configuration features**: + +* New configuration option (``topics``) to configure the :ref:`Manual Topics `. +* New configuration option (``max-tx-rate``) to configure the :ref:`Max transmission rate `. +* Remove the support for `Built-in Topics `_. diff --git a/docs/rst/recording/remote_control/remote_control.rst b/docs/rst/recording/remote_control/remote_control.rst index fbe19dbc5..e63438bd6 100644 --- a/docs/rst/recording/remote_control/remote_control.rst +++ b/docs/rst/recording/remote_control/remote_control.rst @@ -60,7 +60,7 @@ Therefore, any user can create his own application to control the |ddsrecorder| .. note:: Status and command topics are not blocked by default, i.e. messages on this topics will be recorded if listening on the same domain the controller is launched. - If willing to avoid this, include these topics in the :ref:`blocklist `: + If willing to avoid this, include these topics in the :ref:`blocklist `: .. code-block:: yaml diff --git a/docs/rst/recording/usage/configuration.rst b/docs/rst/recording/usage/configuration.rst index e32f8f3e8..37abdf448 100644 --- a/docs/rst/recording/usage/configuration.rst +++ b/docs/rst/recording/usage/configuration.rst @@ -33,112 +33,71 @@ DDS Configuration Configuration related to DDS communication. -.. _recorder_topic_filtering: +.. _recorder_builtin_topics: -Topic Filtering +Built-in Topics ^^^^^^^^^^^^^^^ -|ddsrecorder| includes a mechanism to automatically detect which topics are being used in a DDS network. -By automatically detecting these topics, a |ddsrecorder| creates internal DDS :term:`Readers` for each topic in order to record the data published on each discovered topic. - -.. note:: - - |ddsrecorder| entities are created with the QoS of the first Publisher/Subscriber found in this Topic, unless manually set in the :ref:`built-in topics ` list. +The discovery phase can be accelerated by listing topics under the ``builtin-topics`` tag. +The |ddsrecorder| will create the DataWriters and DataReaders for these topics in the |ddsrecorder| initialization. +The :ref:`Topic QoS ` for these topics can be manually configured with the :ref:`Manual Topic ` and with the :ref:`Specs Topic QoS `; if a :ref:`Topic QoS ` is not configured, it will take its default value. -|ddsrecorder| allows filtering of DDS :term:`Topics`, that is, it allows to define the DDS Topics' data that is going to be recorded by the application. -This way, it is possible to define a set of rules in |ddsrecorder| to filter those data samples the user does not wish to save. +The ``builtin-topics`` must specify a ``name`` and ``type`` without wildcard characters. -It is not mandatory to define such set of rules in the configuration file. -In this case, a |ddsrecorder| will save all the data published under the topics that it automatically discovers within the DDS network to which it connects. - -To define these data filtering rules based on the Topics to which they belong, the following lists are available: - -* Allowed topics list (``allowlist``) -* Block topics list (``blocklist``) - -These lists of topics stated above are defined by a tag in the *YAML* configuration file, which defines a *YAML* vector (``[]``). -This vector contains the list of topics for each filtering rule. -Each Topic is determined by its entries ``name`` and ``type``, with only the first one being mandatory. - -.. list-table:: - :header-rows: 1 - - * - Topic entries - - Data type - - Default value +**Example of usage:** - * - ``name`` - - ``string`` - - \- + .. code-block:: yaml - * - ``type`` - - ``string`` - - ``"*"`` + builtin-topics: + - name: HelloWorldTopic + type: HelloWorld -See :term:`Topic` section for further information about the topic. +.. _recorder_topic_filtering: -.. note:: +Topic Filtering +--------------- - Placing quotation marks around values in a YAML file is generally optional. - However, values containing wildcard characters must be enclosed by single or double quotation marks. +The |ddsrecorder| automatically detects the topics that are being used in a DDS Network. +The |ddsrecorder| then creates internal DDS :term:`Readers` to record the data published on each topic. +The |ddsrecorder| allows filtering DDS :term:`Topics` to allow users to configure the DDS :term:`Topics` that must be recorded. +These data filtering rules can be configured under the ``allowlist`` and ``blocklist`` tags. +If the ``allowlist`` and ``blocklist`` are not configured, the |ddsrecorder| will recorded the data published on every topic it discovers. +If both the ``allowlist`` and ``blocklist`` are configured and a topic appears in both of them, the ``blocklist`` has priority and the topic will be blocked. -Allow topic list -"""""""""""""""" -This is the list of topics that |ddsrecorder| will record, i.e. the data published under the topics matching the expressions in the ``allowlist`` will be saved by |ddsrecorder|. +Topics are determined by the tags ``name`` (required) and ``type``, both of which accept wildcard characters. .. note:: - If no ``allowlist`` is provided, data will be recorded for all topics (unless filtered out in ``blocklist``). - -.. _recorder_topic_filtering_blocklist: - -Block topic list -"""""""""""""""" -This is the list of topics that the |ddsrecorder| will block, that is, all data published under the topics matching the filters specified in the ``blocklist`` will be discarded by the |ddsrecorder| and therefore will not be recorded. - -This list takes precedence over the ``allowlist``. -If a topic matches an expression both in the ``allowlist`` and in the ``blocklist``, the ``blocklist`` takes precedence, causing the data under this topic to be discarded. - -**Example of usage - Allowlist and blocklist collision:** + Placing quotation marks around values in a YAML file is generally optional, but values containing wildcard characters do require single or double quotation marks. - In the following example, the ``HelloWorldTopic`` topic is both in the ``allowlist`` and (implicitly) in the - ``blocklist``, so according to the ``blocklist`` preference rule this topic is blocked. - Moreover, only the topics present in the allowlist are relayed, regardless of whether more topics are dynamically - discovered in the DDS network. - In this case the forwarded topics are ``AllowedTopic1`` with data type ``Allowed`` - and ``AllowedTopic2`` regardless of its data type. +Consider the following example: - .. code-block:: yaml - - allowlist: - - name: AllowedTopic1 - type: Allowed - - name: AllowedTopic2 - type: "*" - - name: HelloWorldTopic - type: HelloWorld +.. code-block:: yaml - blocklist: - - name: "*" - type: HelloWorld + allowlist: + - name: AllowedTopic1 + type: Allowed -.. _recorder_builtin_topics: + - name: AllowedTopic2 + type: "*" -Built-in Topics -^^^^^^^^^^^^^^^ + - name: HelloWorldTopic + type: HelloWorld -Apart from the dynamic DDS topics discovered in the network, the discovery phase can be accelerated by using the builtin topic list (``builtin-topics``). -By defining topics in this list, the |ddsrecorder| will create the DataWriters and DataReaders in recorder initialization. + blocklist: + - name: "*" + type: HelloWorld -The builtin-topics list is defined in the same form as the ``allowlist`` and ``blocklist``. +In this example, the data published in the topic ``AllowedTopic1`` with type ``Allowed`` and in the topic ``AllowedTopic2`` with any type will be recorded by the |ddsrecorder|. +The data published in the topic ``HelloWorldTopic`` with type ``HelloWorld`` will be blocked, since the ``blocklist`` is blocking all topics with any name and with type ``HelloWorld``. -This feature also allows to manually force the QoS of a specific topic, so the entities created in such topic follows the specified QoS and not the one first discovered. +.. _recorder_topic_qos: -Topic Quality of Service -"""""""""""""""""""""""" +Topic QoS +^^^^^^^^^ -For every topic contained in this list, both ``name`` and ``type`` must be specified and contain no wildcard characters. -Apart from these values, the tag ``qos`` under each topic allows to configure the following values: +The following is the set of QoS that are configurable for a topic. +For more information on topics, please read the `Fast DDS Topic `_ section. .. list-table:: :header-rows: 1 @@ -161,11 +120,11 @@ Apart from these values, the tag ``qos`` under each topic allows to configure th - ``false`` - ``TRANSIENT_LOCAL`` / ``VOLATILE`` - * - History Depth - - ``depth`` - - *integer* - - *default value* - - - + * - Ownership + - ``ownership`` + - *bool* + - ``false`` + - ``EXCLUSIVE_OWNERSHIP_QOS`` / ``SHARED_OWNERSHIP_QOS`` * - Partitions - ``partitions`` @@ -173,47 +132,86 @@ Apart from these values, the tag ``qos`` under each topic allows to configure th - ``false`` - Topic with / without partitions - * - Ownership - - ``ownership`` - - *bool* - - ``false`` - - ``EXCLUSIVE_OWNERSHIP_QOS`` / ``SHARED_OWNERSHIP_QOS`` - * - Key - ``keyed`` - *bool* - ``false`` - - Topic with / without key + - Topic with / without `key `_ - * - Downsampling - - ``downsampling`` - - *integer* - - *default value* - - Downsampling factor + * - History Depth + - ``history-depth`` + - *unsigned integer* + - ``5000`` + - :ref:`recorder_history_depth` * - Max Reception Rate - - ``max-reception-rate`` + - ``max-rx-rate`` - *float* - - *default value* - - Maximum sample reception rate [Hz] + - ``0`` (unlimited) + - :ref:`recorder_max_rx_rate` -**Example of usage:** + * - Downsampling + - ``downsampling`` + - *unsigned integer* + - ``1`` + - :ref:`recorder_downsampling` - .. code-block:: yaml +.. warning:: - builtin-topics: - - name: HelloWorldTopic - type: HelloWorld - qos: - reliability: true # Use QoS RELIABLE - durability: true # Use QoS TRANSIENT_LOCAL - depth: 100 # Use History Depth 100 - partitions: true # Topic with partitions - ownership: false # Use QoS SHARED_OWNERSHIP_QOS - keyed: true # Topic with key - downsampling: 4 # Keep 1 of every 4 samples - max-reception-rate: 10 # Discard messages if less than 100ms elapsed since the last sample was processed + Manually configuring ``TRANSIENT_LOCAL`` durability may lead to incompatibility issues when the discovered reliability is ``BEST_EFFORT``. + Please ensure to always configure the ``reliability`` when configuring the ``durability`` to avoid the issue. + +.. _recorder_history_depth: + +History Depth +""""""""""""" + +The ``history-depth`` tag configures the history depth of the Fast DDS internal entities. +By default, the depth of every RTPS History instance is :code:`5000`, which sets a constraint on the maximum number of samples a |ddsrecorder| instance can deliver to late joiner Readers configured with ``TRANSIENT_LOCAL`` `DurabilityQosPolicyKind `_. +Its value should be decreased when the sample size and/or number of created endpoints (increasing with the number of topics) are big enough to cause memory exhaustion issues. +If enough memory is available, however, the ``history-depth`` could be increased to deliver a greater number of samples to late joiners. + +.. _recorder_max_rx_rate: + +Max Reception Rate +"""""""""""""""""" + +The ``max-rx-rate`` tag limits the frequency [Hz] at which samples are processed by discarding messages received before :code:`1/max-rx-rate` seconds have passed since the last processed message. +It only accepts non-negative numbers. +By default it is set to ``0``; it processes samples at an unlimited reception rate. + +.. _recorder_downsampling: + +Downsampling +"""""""""""" +The ``downsampling`` tag reduces the sampling rate of the received data by only keeping *1* out of every *n* samples received (per topic), where *n* is the value specified under the ``downsampling`` tag. +When the ``max-rx-rate`` tag is also set, downsampling only applies to messages that have passed the ``max-rx-rate`` filter. +It only accepts positive integers. +By default it is set to ``1``; it accepts every message. + +.. _recorder_manual_topics: + +Manual Topics +^^^^^^^^^^^^^ + +A subset of :ref:`Topic QoS ` can be manually configured for a specific topic under the tag ``topics``. +The tag ``topics`` has a required ``name`` tag that accepts wildcard characters. +It also has two optional tags: a ``type`` tag that accepts wildcard characters, and a ``qos`` tag with the :ref:`Topic QoS ` that the user wants to manually configure. +If a ``qos`` is not manually configured, it will get its value by discovery. + +.. code-block:: yaml + + topics: + - name: "temperature/*" + type: "temperature/types/*" + qos: + max-rx-rate: 15 + downsampling: 2 + +.. note:: + + The :ref:`Topic QoS ` configured in the Manual Topics take precedence over the :ref:`Specs Topic QoS `. .. _recorder_usage_configuration_domain_id: @@ -371,21 +369,6 @@ By default (``log-publish-time: false``) received messages are stored in the MCA Additionally, the timestamp corresponding to when messages were initially published (``publishTime``) is also included in the information dumped to MCAP files. In some applications, it may be required to use the ``publishTime`` as ``logTime``, which can be achieved by providing the ``log-publish-time: true`` configuration option. -Max Reception Rate -^^^^^^^^^^^^^^^^^^ - -Limits the frequency [Hz] at which samples are processed, by discarding messages received before :code:`1/max-reception-rate` seconds have elapsed since the last processed message was received. -When specified, ``max-reception-rate`` is set for all topics without distinction, but a different value can also set for a particular topic under the ``qos`` configuration tag within the builtin-topics list. -This parameter only accepts non-negative values, and its default value is ``0`` (no limit). - -Downsampling -^^^^^^^^^^^^ - -Reduces the sampling rate of the received data by keeping *1* out of every *n* samples received (per topic), where *n* is the value specified in ``downsampling``. -If ``max-reception-rate`` is also set, downsampling applies to messages that already managed to pass this filter. -When specified, this downsampling factor is set for all topics without distinction, but a different value can also set for a particular topic under the ``qos`` configuration tag within the builtin-topics list. -This parameter only accepts positive integer values, and its default value is ``1`` (no downsampling). - .. _recorder_usage_configuration_onlywithtype: Only With Type @@ -562,6 +545,17 @@ As explained in :ref:`Event Window `, To accomplish this, received samples are stored in memory until the aforementioned event is triggered and, in order to limit memory consumption, outdated (received more than ``event-window`` seconds ago) samples are removed from this buffer every ``cleanup-period`` seconds. By default, its value is equal to twice the ``event-window``. +.. _recorder_specs_topic_qos: + +QoS +^^^ + +``specs`` supports a ``qos`` **optional** tag to configure the default values of the :ref:`Topic QoS `. + +.. note:: + + The :ref:`Topic QoS ` configured in ``specs`` can be overwritten by the :ref:`Manual Topics `. + .. _recorder_usage_configuration_general_example: General Example @@ -589,14 +583,13 @@ A complete example of all the configurations described on this page can be found builtin-topics: - name: "HelloWorldTopic" type: "HelloWorld" + + topics: + - name: "temperature/*" + type: "temperature/types/*" qos: - reliability: true - durability: true - keyed: false - partitions: true - ownership: false - downsampling: 4 - max-reception-rate: 10 + max-rx-rate: 15 + downsampling: 2 ignore-participant-flags: no_filter transport: builtin @@ -613,8 +606,6 @@ A complete example of all the configurations described on this page can be found buffer-size: 50 event-window: 60 log-publish-time: false - downsampling: 3 - max-reception-rate: 20 only-with-type: false compression: algorithm: lz4 @@ -634,6 +625,10 @@ A complete example of all the configurations described on this page can be found max-pending-samples: 10 cleanup-period: 90 + qos: + max-rx-rate: 20 + downsampling: 3 + .. _recorder_usage_fastdds_configuration: diff --git a/docs/rst/replaying/usage/configuration.rst b/docs/rst/replaying/usage/configuration.rst index 19aa6f3e2..799a5f23b 100644 --- a/docs/rst/replaying/usage/configuration.rst +++ b/docs/rst/replaying/usage/configuration.rst @@ -35,104 +35,49 @@ Configuration related to DDS communication. .. _replayer_topic_filtering: Topic Filtering -^^^^^^^^^^^^^^^ - -As seen in :ref:`DDS Recorder topic filtering `, a user can define a set of rules to only record DDS :term:`Topics` of interest. - -In addition to the filters applied to |ddsrecorder| when recording, |ddsreplayer| also allows filtering of DDS :term:`Topics`. -That is, it allows to define the DDS Topics' data that is going to be replayed by the application. -This way, it is possible to define a set of rules in |ddsreplayer| to filter those data samples the user does not wish to replay. - -It is not mandatory to define such set of rules in the configuration file. -In this case, a |ddsreplayer| will publish all data stored in the provided input MCAP file. - -To define these data filtering rules based on the Topics to which they belong, the following lists are available: - -* Allowed topics list (``allowlist``) -* Block topics list (``blocklist``) - -These lists of topics stated above are defined by a tag in the *YAML* configuration file, which defines a *YAML* vector (``[]``). -This vector contains the list of topics for each filtering rule. -Each Topic is determined by its entries ``name`` and ``type``, with only the first one being mandatory. - -.. list-table:: - :header-rows: 1 - - * - Topic entries - - Data type - - Default value - - * - ``name`` - - ``string`` - - \- - - * - ``type`` - - ``string`` - - ``"*"`` - -See :term:`Topic` section for further information about the topic. - -.. note:: +--------------- - Placing quotation marks around values in a YAML file is generally optional. - However, values containing wildcard characters must be enclosed by single or double quotation marks. +The |ddsreplayer| automatically detects the topics that are being used in a DDS Network. +The |ddsreplayer| then creates internal DDS :term:`Writers` to replay the data published on each topic. +The |ddsreplayer| allows filtering DDS :term:`Topics` to allow users to configure the DDS :term:`Topics` that must be replayed. +These data filtering rules can be configured under the ``allowlist`` and ``blocklist`` tags. +If the ``allowlist`` and ``blocklist`` are not configured, the |ddsreplayer| will replayed the data published on every topic it discovers. +If both the ``allowlist`` and ``blocklist`` are configured and a topic appears in both of them, the ``blocklist`` has priority and the topic will be blocked. -Allow topic list -"""""""""""""""" -This is the list of topics that |ddsreplayer| will replay, i.e. only recorded data under the topics matching the expressions in the ``allowlist`` will be published by |ddsreplayer|. +Topics are determined by the tags ``name`` (required) and ``type``, both of which accept wildcard characters. .. note:: - If no ``allowlist`` is provided, data will be replayed for all topics (unless filtered out in ``blocklist``). - -.. _replayer_topic_filtering_blocklist: - -Block topic list -"""""""""""""""" -This is the list of topics that the |ddsreplayer| will block, that is, all input data under the topics matching the filters specified in the ``blocklist`` will be discarded by the |ddsreplayer| and therefore will not be published. - -This list takes precedence over the ``allowlist``. -If a topic matches an expression both in the ``allowlist`` and in the ``blocklist``, the ``blocklist`` takes precedence, causing the data under this topic to be discarded. + Placing quotation marks around values in a YAML file is generally optional, but values containing wildcard characters do require single or double quotation marks. -**Example of usage - Allowlist and blocklist collision:** +Consider the following example: - In the following example, the ``HelloWorldTopic`` topic is both in the ``allowlist`` and (implicitly) in the - ``blocklist``, so according to the ``blocklist`` preference rule this topic is blocked. - Moreover, only the topics present in the allowlist are relayed, regardless of whether more topics are dynamically - discovered in the DDS network. - In this case the forwarded topics are ``AllowedTopic1`` with data type ``Allowed`` - and ``AllowedTopic2`` regardless of its data type. - - .. code-block:: yaml +.. code-block:: yaml - allowlist: - - name: AllowedTopic1 - type: Allowed - - name: AllowedTopic2 - type: "*" - - name: HelloWorldTopic - type: HelloWorld + allowlist: + - name: AllowedTopic1 + type: Allowed - blocklist: - - name: "*" - type: HelloWorld + - name: AllowedTopic2 + type: "*" -Built-in Topics -^^^^^^^^^^^^^^^ + - name: HelloWorldTopic + type: HelloWorld -As seen in :ref:`recorder_topic_filtering`, a |ddsrecorder| uses the QoS of the first Publisher/Subscriber found in every recorded topic, unless manually defined in the :ref:`built-in topics ` list. -This QoS information is stored in the MCAP file along with the user data, and thus a |ddsreplayer| instance is able to publish recorded data preserving the original QoS. + blocklist: + - name: "*" + type: HelloWorld -However, the user has the option to manually set the QoS of any topic to be played back through the replayer's builtin-topics list. -The builtin-topics list is defined in the same form as the ``allowlist`` and ``blocklist``. +In this example, the data published in the topic ``AllowedTopic1`` with type ``Allowed`` and in the topic ``AllowedTopic2`` with any type will be replayed by the |ddsreplayer|. +The data published in the topic ``HelloWorldTopic`` with type ``HelloWorld`` will be blocked, since the ``blocklist`` is blocking all topics with any name and with type ``HelloWorld``. -This feature also allows to manually force the QoS of a specific topic, so the entities created in such topic follows the specified QoS and not the one first discovered. +.. _replayer_topic_qos: -Topic Quality of Service -"""""""""""""""""""""""" +Topic QoS +^^^^^^^^^ -For every topic contained in this list, both ``name`` and ``type`` must be specified and contain no wildcard characters. -Apart from these values, the tag ``qos`` under each topic allows to configure the following values: +The following is the set of QoS that are configurable for a topic. +For more information on topics, please read the `Fast DDS Topic `_ section. .. list-table:: :header-rows: 1 @@ -155,38 +100,87 @@ Apart from these values, the tag ``qos`` under each topic allows to configure th - ``false`` - ``TRANSIENT_LOCAL`` / ``VOLATILE`` - * - Partitions - - ``partitions`` - - *bool* - - ``false`` - - Topic with / without partitions - * - Ownership - ``ownership`` - *bool* - ``false`` - ``EXCLUSIVE_OWNERSHIP_QOS`` / ``SHARED_OWNERSHIP_QOS`` + * - Partitions + - ``partitions`` + - *bool* + - ``false`` + - Topic with / without partitions + * - Key - ``keyed`` - *bool* - ``false`` - - Topic with / without key + - Topic with / without `key `_ + + * - History Depth + - ``history-depth`` + - *unsigned integer* + - ``5000`` + - :ref:`replayer_history_depth` + + * - Max Transmission Rate + - ``max-tx-rate`` + - *float* + - ``0`` (unlimited) + - :ref:`replayer_max_tx_rate` + +.. warning:: + + Manually configuring ``TRANSIENT_LOCAL`` durability may lead to incompatibility issues when the discovered reliability is ``BEST_EFFORT``. + Please ensure to always configure the ``reliability`` when configuring the ``durability`` to avoid the issue. + +.. _replayer_history_depth: + +History Depth +""""""""""""" + +The ``history-depth`` tag configures the history depth of the Fast DDS internal entities. +By default, the depth of every RTPS History instance is :code:`5000`, which sets a constraint on the maximum number of samples a |ddsreplayer| instance can deliver to late joiner Readers configured with ``TRANSIENT_LOCAL`` `DurabilityQosPolicyKind `_. +Its value should be decreased when the sample size and/or number of created endpoints (increasing with the number of topics) are big enough to cause memory exhaustion issues. +If enough memory is available, however, the ``history-depth`` could be increased to deliver a greater number of samples to late joiners. -**Example of usage:** +.. _replayer_max_tx_rate: - .. code-block:: yaml +Max Transmission Rate +""""""""""""""""""""" - builtin-topics: - - name: HelloWorldTopic - type: HelloWorld - qos: - reliability: true # Use QoS RELIABLE - durability: true # Use QoS TRANSIENT_LOCAL - partitions: true # Topic with partitions - ownership: false # Use QoS SHARED_OWNERSHIP_QOS - keyed: true # Topic with key +The ``max-tx-rate`` tag limits the frequency [Hz] at which samples are sent by discarding messages transmitted before :code:`1/max-tx-rate` seconds have passed since the last sent message. +It only accepts non-negative numbers. +By default it is set to ``0``; it sends samples at an unlimited transmission rate. +.. note:: + + The ``max-tx-rate`` tag can be set (in order of precedence) for topics, for participants, and globally in specs. + +.. _replayer_manual_topics: + +Manual Topics +^^^^^^^^^^^^^ + +A subset of QoS can be manually configured for a specific topic under the tag ``topics``. +The tag ``topics`` has a required ``name`` tag that accepts wildcard characters. +It also has two optional tags: a ``type`` tag that accepts wildcard characters, and a ``qos`` tag with the QoS that the user wants to manually configure. +If a ``qos`` is not manually configured, it will get its value by discovery. + +**Example of usage** + +.. code-block:: yaml + + topics: + - name: "temperature/*" + type: "temperature/types/*" + qos: + max-tx-rate: 15 + +.. note:: + + The :ref:`Topic QoS ` configured in the Manual Topics take precedence over the :ref:`Specs Topic QoS `. .. _replayer_usage_configuration_domain_id: @@ -361,6 +355,20 @@ By default, data is replayed at the same rate it was published/received. However, a user might be interested in playing messages back at a rate different than the original one. This can be accomplished through the playback ``rate`` tag, which accepts positive float values (e.g. 0.5 <--> half speed || 2 <--> double speed). +.. _replayer_usage_configuration_max_tx_rate: + +Max Transmission Rate +--------------------- + +The ``max-tx-rate`` tag limits the frequency [Hz] at which samples are sent by discarding messages transmitted before :code:`1/max-tx-rate` seconds have passed since the last sent message. +It only accepts non-negative numbers. +By default it is set to ``0``; it sends samples at an unlimited transmission rate. + +.. note:: + + The ``max-tx-rate`` tag can be set for topics and globally under the ``replayer`` tag. + If both are set, the configuration under topics prevails. + .. _replayer_replay_configuration_replaytypes: Replay Types @@ -393,6 +401,17 @@ Note that this last message might be lost after publication, and if reliable `Re For this purpose, the user can specify the maximum amount of milliseconds (``wait-all-acked-timeout``) to wait on closure until published messages are acknowledged by matched readers. Its value is set to ``0`` by default (no wait). +.. _replayer_specs_topic_qos: + +QoS +^^^ + +``specs`` supports a ``qos`` **optional** tag to configure the default values of the :ref:`Topic QoS `. + +.. note:: + + The :ref:`Topic QoS ` configured in ``specs`` can be overwritten by the :ref:`Manual Topics `. + .. _replayer_usage_configuration_general_example: General Example @@ -417,15 +436,11 @@ A complete example of all the configurations described on this page can be found - name: "topic_name" type: "topic_type" - builtin-topics: - - name: "HelloWorldTopic" - type: "HelloWorld" + topics: + - name: "temperature/*" + type: "temperature/types/*" qos: - reliability: true - durability: true - keyed: false - partitions: true - ownership: false + max-tx-rate: 15 ignore-participant-flags: no_filter transport: builtin @@ -458,3 +473,6 @@ A complete example of all the configurations described on this page can be found specs: threads: 8 wait-all-acked-timeout: 10 + + qos: + max-tx-rate: 20 diff --git a/resources/configurations/recorder/complete_config.yaml b/resources/configurations/recorder/complete_config.yaml index fcd56eb6b..d42b47841 100644 --- a/resources/configurations/recorder/complete_config.yaml +++ b/resources/configurations/recorder/complete_config.yaml @@ -13,14 +13,13 @@ dds: builtin-topics: - name: "HelloWorldTopic" type: "HelloWorld" + + topics: + - name: "temperature/*" + type: "temperature/types/*" qos: - reliability: true - durability: true - keyed: false - partitions: true - ownership: false - downsampling: 4 - max-reception-rate: 10 + max-rx-rate: 15 + downsampling: 2 ignore-participant-flags: filter_same_process transport: shm @@ -37,8 +36,6 @@ recorder: buffer-size: 50 event-window: 60 log-publish-time: false - downsampling: 3 - max-reception-rate: 20 only-with-type: false compression: algorithm: lz4 @@ -57,3 +54,7 @@ specs: threads: 8 max-pending-samples: 10 cleanup-period: 90 + + qos: + max-rx-rate: 20 + downsampling: 3 diff --git a/resources/configurations/replayer/complete_config.yaml b/resources/configurations/replayer/complete_config.yaml index 885717e99..0c9d892bc 100644 --- a/resources/configurations/replayer/complete_config.yaml +++ b/resources/configurations/replayer/complete_config.yaml @@ -10,15 +10,11 @@ dds: - name: "topic_to_block" type: "type_to_block" - builtin-topics: - - name: "HelloWorldTopic" - type: "HelloWorld" + topics: + - name: "temperature/*" + type: "temperature/types/*" qos: - reliability: true - durability: true - keyed: false - partitions: true - ownership: false + max-tx-rate: 15 ignore-participant-flags: filter_same_process transport: shm @@ -51,3 +47,6 @@ replayer: specs: threads: 12 wait-all-acked-timeout: 10 + + qos: + max-tx-rate: 20