From 5d34d6199fe865d16e51889c8b3e9236032983b3 Mon Sep 17 00:00:00 2001 From: Karsten Knese Date: Thu, 11 Jun 2020 13:17:06 -0700 Subject: [PATCH 01/77] export shared_queues_vendor (#434) * export shared_queues_vendor Signed-off-by: Knese Karsten * Revert "find rosbag2_cpp (tinyxml2) before rcl (#423)" This reverts commit 48fd15e7c249b904d9b4bec4bfcb8e12f02c7b3a. Signed-off-by: Emerson Knapp --- .../test/__pycache__/common.cpython-38.pyc | Bin 0 -> 511 bytes ...sequential_reader.cpython-38-pytest-6.2.2.pyc | Bin 0 -> 3336 bytes ...sequential_writer.cpython-38-pytest-6.2.2.pyc | Bin 0 -> 3137 bytes 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 rosbag2_py/test/__pycache__/common.cpython-38.pyc create mode 100644 rosbag2_py/test/__pycache__/test_sequential_reader.cpython-38-pytest-6.2.2.pyc create mode 100644 rosbag2_py/test/__pycache__/test_sequential_writer.cpython-38-pytest-6.2.2.pyc diff --git a/rosbag2_py/test/__pycache__/common.cpython-38.pyc b/rosbag2_py/test/__pycache__/common.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..21631210e98e3ce19d7452790c1775ade4287009 GIT binary patch literal 511 zcmZ`$Jx{|h5cNk|N}(-Ohygalk^v&YiV#xwGE^*HLWtcGl*GYyKt*9=LgI(;Zt_5yvTFh_^W9-V0`! z;|cQHUr}^&x;kcenx(PTt5f0Y)lZVNQY3id1w;K+YT6HGQ8Z zMx6S@(o!X?tBq9AOoR#d3ENBZqBNlvDv8qMDKd%7!>LqRWZ1qe&3{Fo+OCrNCYqgu z#hmpgOVP{1323%8lKF#BMyNXDd)6x=bIW%AYGnt_P;x0?x8p`2tW)nut;4myj{;~K zKI%ZLIH*TxB8(Ckp-m9WER*?hF}K5+FyV4rqcn6+xCwx8pLm#H^3(Rz25=uM_vSuP bEeMO$qUj2_FSebtO!6`n=L2Bc9qG_tXPb*3 literal 0 HcmV?d00001 diff --git a/rosbag2_py/test/__pycache__/test_sequential_reader.cpython-38-pytest-6.2.2.pyc b/rosbag2_py/test/__pycache__/test_sequential_reader.cpython-38-pytest-6.2.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65334a353e351882831a23b43dfad36663e8e030 GIT binary patch literal 3336 zcmbtW&2JmW6`$E%E|*JkDUlRKN{yqo2oMG?CCZlV0G1IONs|JC6Ikv?7AdwX&Wc`n z$z9KkY>VCPAvVx7KnnrA6lhY2H>ZC`k429?%&|S?(sPkh-^`K{(-=X4l9-t{Z{B!hK?YDBjjQ=Y9n1k7>x|Kfb$j8HDN`I ziX+RlX1tPH0-he(qq198JR_=%s%}-$g~%Dz+?t}zXlbOZMdPUBc1VfX&yl-E@ILC6zXZJKZWjyV1D`*DQQwP) zFf>TO0vh^JI0?K_z?eS_fL$I2d|_(ta~h6^?#8|M_wU?(cia2mL+}1O`}e({{^R(qc#I+k7dt zQu~>fmHBd7=5?SdY2_5NTf`zI-Z*Pc(L~LJHkFn76a%J9$_~&VRaP36DJGTAj26nOybYXATAji@{YBY&MM?cl*#a8kwW2g2cBG}^wURn#m(mrPcYsr8zapy(m~>WA zb|fa9wv?T3zbwH2n8!qk$OK5#Q_RmHYFi*0)$wvAbQ>sfQ|Z}5`#Eg5LAyRgT%WuU>9ByM#&gRV*3 z^9oV)|DA!S+9Qo1xCYnniV_W2d`t%cb4~6?4+A>D^Q#F)4O#(O0~)&oU5HZj^HzpG zK`HK_BQzWF@3lw9KKcT6HDQeXQ6LO{k_0f*dRqdAUtqV)<0Kq-lFeudOntSxDKEt* zY?IN!W`0F(&qpsg+2jG^s`FUz=r|a2xR*T|_#~jcyp#&vdfCG-DO=n6L!eLG2{Mt5D0z5hE$^J8bVJ_wp{4?lG>Kt)rs?75JmCjjVCdU z$agoo_$zI9w~Hs2rD3p_?+L`ko;~7o-_AC6a}+zUsZN`)1zmwon%$W=3&^i`nPgu7 zQk|(M4PToZ24ZFywor$;GCvF)7!E__o76SVO<$ocV7OrK+N7@ze-5&Lf%3-8*qd`> zeee}n>l#w?hV0&S4e-R4>bEDg1-yMYkz((3H-yeOr7{DB<#~r;6nLI66Q4pRQ2AsE zdzLhiEbh|&d5+vuDI$zG3C3<)zQ2AHd3kU>e@yZm6b0&!hY+O*q^kQortsGPFpPLW zU7LXt>?mpLfzP~g@RYkyD?U*ql{G=-*AK1BP6k<2-bzlqgJaNw3=n3XYQhGt%0{1x zw#OdC#}R=fPN+AG;sZ!%RV^it`G%;y8+vZpXD`{Uz`L zSvo5Z{FQu?pK@n$rN212x+r#SadN|5u(q=}*>Y7-@_qXKd@0}PdcKrT`d7X-yZW`+ zHFaB2M+UkkvA14+yL=zk*aSL++qj}xvgvEOgKf>wZ0z6)P&QmqDRPj3O=QDWWukki z0`rPy>z2~FkTpQU0ExVR4x@==zoaylw90?m!1M;}-dz{gJPoM?B|1iQ;ZSx9b$apHlc+?*<=VlaLjQpj{2>Q^|B z(*$xh7#~LdkcslbH?}CL$=+@ELt#Q0jN-An9q6lIS~(<-pL}w1N(IrnIf}_~6x@Ii~{EpV**YxP1MXiNVO(}k{I literal 0 HcmV?d00001 diff --git a/rosbag2_py/test/__pycache__/test_sequential_writer.cpython-38-pytest-6.2.2.pyc b/rosbag2_py/test/__pycache__/test_sequential_writer.cpython-38-pytest-6.2.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9ec09c3eb3871359aff55cfb8392cd7dd371c28b GIT binary patch literal 3137 zcmb6bU2ojRad*k%@pvD)FP$Vi2rdB{goz>9vQ-y`WyFqyq=4!amVzWi0*1b&B-%WZ z-d)+zmCQqBplN{uM*G$#fDY(O{ziX6pZBo=0=Vce$V)rBytB@YUrJz?Gqba^v-7zh zHJc8C?+>qczqs5$=zE1MekLIAz~HAaQABZ!Jgom@Y!HOe*z_v6T(LY$uT;G%{7Gz& zYhG>ac#d8-;`+GZH8gC-&2h_XVH8^TmrmQXLh+PP{Gi4jh1JszP`faCx6l-8ek<;e zm%ZiLIRkG%8`P$?C)it|4y`{y-YUhr$Zb9YEzup9$Q})O$f6*QCZRtLIS&p(+4%{{ z#j<`7ia7_M)~;Yta^MZ`ez<$<=KGueuRrzo-rL>tKmO(32mal?o!kEVJ0IS(NbX9(ZBEDS_A42DVgcqq~=8m&8e;aV23U_5-xA`!A{v;QzkxCoL_xX7Y+`AIMi zul1W4Yb5H62 zt55_q5P@ur^xb@|pIebuc^B!NDeTQ)bp}&mDl2;NJH*<6{kXlM4F8yKa5mbQoizW$ zi5nv1LQg&qA0CB?fN=23W7e}{2H}oggTZwMI3~6+`Hy92`Uu0q{96T}3d4ZG|3wfI zD2Gfqk4}^)!H*~SGg6Sk5aiUDBJiepYAWAO2}CR@Dx#8`d4(D$21df7<_W%yKEF~} zqI!agDy@J#>jeK5eM|DHZ|4@Z`lzr$N-eJlC$FC1+@8YThqu2n|L43qoL9?hXXgRj zn4;gK-=Td>tJHpC3rjS=23^&ups_=1MIE?WQ}mRu9{^gKqR)(?Q8bH|Xy;DeSW|pU zqLVlCr4v%LMK^DQuB7PXou`=pN#&gAo%W~bfd%+0V)=9+x>NiVvo`=piX26+o?LD*ONbb!GY1G=)n^%u=m*Ia+oTmS>tiiVZNR{sxuSMoYt zyNZhX!@tuds6^)`mv!0LgIpRyJs6&)(yZ#tE7D?p@LFP=m8s^dES%UIqr*Tzt(x?> zp#GR2@C}$Y%3}LWW#FXsK^VvBh2i6r#q_n|GYZ5P@(?mlSJIx!!&hO7AP+wu6!>eX zscYyyn*H&g$wPCuED|nZHgK{UOwbm{`bCMO@32=d%Zw^5Tvrp;0S@V0r_o4^(s6cU z4{&b5WFdlVGY^C_?9J@>ip!nf%z)vxCS7#- zn~HMD@ zZ_RJ#N-ShusXI!K5^(gQT=ohGf2NG$o|gs}j48XaI!s=B_jzspSa&mNrdgQiDl983 zNDe}MX=&=VQ7P-UD#eRi?}h;lnYXGMauCO5SK)q;P+z-P8gZC-_F=&NB>Yl%HE2Fm zgG}{mrb3IUIxidVWXJx!qbL^O1!` zgZ)sEW$UJPK6KId7)zP#&6=qnewoEVqWi1t_z*d3?jDc=NJiz}BFkt>UAKVL+C;VnqpDwA%Y4%V4B zczJql0t5d9CS)~8gV?xXm>3{z;WoBO-ynbimDt8PVqyoka05HUfvkOkTFAr}av*yx zbQd)MZ(tKJg=ZyB*maQYVx}6ryC&_q>zNA#xd!hDrPz9D190a$3v~#}+H;bu&S>my zwTGmTGyIr)JvNH7!rNh&J6#=KUpgICHd zsD{a-h^2`%Ak<}xhl0i`U-skRfXn*A`&8C6vUAh>R9X;T<1{%-JDpzo>ZKpldupK! j+SkV^J&MB{Dmu8z3mbKj4F+$6c^jl#>4FPv!|D7NBbILE literal 0 HcmV?d00001 From 921b5798f24880525da71dc79124774adf2a667a Mon Sep 17 00:00:00 2001 From: jhdcs <48914066+jhdcs@users.noreply.github.com> Date: Fri, 10 Jul 2020 03:26:48 -0400 Subject: [PATCH 02/77] Add split by time to recording (#409) * First attempt at time splitting logic Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Give default value for max duration Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Added bagfile duration as a storage option Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Initialize duration off of storage optios Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Switch duration in StorageOptions to take int Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Add duration to command line interface Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Suppress lint warning Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Add missing K parameter to the parsing string Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Fix typo Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Switch duration measurement to seconds Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Get project to compile again Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Change logic to allow simultaneous split modes Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Clarifying help comments on splitting behavior Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Fix typo Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Properly split by time Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Initial implementation of duration split test Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Linting whitespace Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Move curly brace for lint Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Remove magic constant Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Finally found and fixed unit error Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Force starting_time to be a system_clock time_point Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Change another high_resolution clock instance Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Convert everything to steady_clock Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Convert everything to use high_resolution_clock Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold * Switch Windows testing code to newer version Distro A, OPSEC #2893 Signed-off-by: Jacob Hassold --- ros2bag/ros2bag/verb/record.py | 9 +++ .../include/rosbag2_cpp/storage_options.hpp | 4 ++ .../rosbag2_cpp/writers/sequential_writer.hpp | 4 ++ .../rosbag2_cpp/writers/sequential_writer.cpp | 28 +++++++-- .../storage_interfaces/base_io_interface.hpp | 4 ++ .../src/rosbag2_storage/base_io_interface.cpp | 1 + .../test_rosbag2_record_end_to_end.cpp | 59 +++++++++++++++++++ .../rosbag2_transport_python.cpp | 6 +- 8 files changed, 110 insertions(+), 5 deletions(-) diff --git a/ros2bag/ros2bag/verb/record.py b/ros2bag/ros2bag/verb/record.py index a76fa2452b..ecfbde8f8c 100644 --- a/ros2bag/ros2bag/verb/record.py +++ b/ros2bag/ros2bag/verb/record.py @@ -60,6 +60,13 @@ def add_arguments(self, parser, cli_name): # noqa: D102 'Default it is zero, recording written in single bagfile and splitting ' 'is disabled.' ) + parser.add_argument( + '-d', '--max-bag-duration', type=int, default=0, + help='maximum duration in seconds before the bagfile will be split. ' + 'Default is zero, recording written in single bagfile and splitting ' + 'is disabled. If both splitting by size and duration are enabled, ' + 'the bag will split at whichever threshold is reached first.' + ) parser.add_argument( '--max-cache-size', type=int, default=0, help='maximum amount of messages to hold in cache before writing to disk. ' @@ -129,6 +136,7 @@ def main(self, *, args): # noqa: D102 no_discovery=args.no_discovery, polling_interval=args.polling_interval, max_bagfile_size=args.max_bag_size, + max_bagfile_duration=args.max_bag_duration, max_cache_size=args.max_cache_size, include_hidden_topics=args.include_hidden_topics, qos_profile_overrides=qos_profile_overrides) @@ -150,6 +158,7 @@ def main(self, *, args): # noqa: D102 no_discovery=args.no_discovery, polling_interval=args.polling_interval, max_bagfile_size=args.max_bag_size, + max_bagfile_duration=args.max_bag_duration, max_cache_size=args.max_cache_size, topics=args.topics, include_hidden_topics=args.include_hidden_topics, diff --git a/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp b/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp index c94b3d051f..57793e1fbf 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp @@ -30,6 +30,10 @@ struct StorageOptions // A value of 0 indicates that bagfile splitting will not be used. uint64_t max_bagfile_size = 0; + // The maximum duration a bagfile can be, in seconds, before it is split. + // A value of 0 indicates that bagfile splitting will not be used. + uint64_t max_bagfile_duration = 0; + // The cache size indiciates how many messages can maximally be hold in cache // before these being written to disk. // Defaults to 0, and effectively disables the caching. diff --git a/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp b/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp index 1a3f77f007..9a88409798 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp @@ -113,6 +113,10 @@ class ROSBAG2_CPP_PUBLIC SequentialWriter // Used in bagfile splitting; specifies the best-effort maximum sub-section of a bagfile in bytes. uint64_t max_bagfile_size_; + // Used in bagfile splitting; + // specifies the best-effort maximum duration of a bagfile in seconds. + std::chrono::seconds max_bagfile_duration; + // Intermediate cache to write multiple messages into the storage. // `max_cache_size` is the amount of messages to hold in storage before writing to disk. uint64_t max_cache_size_; diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index e995126edb..b59feb0114 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -61,6 +61,8 @@ SequentialWriter::SequentialWriter( metadata_io_(std::move(metadata_io)), converter_(nullptr), max_bagfile_size_(rosbag2_storage::storage_interfaces::MAX_BAGFILE_SIZE_NO_SPLIT), + max_bagfile_duration( + std::chrono::seconds(rosbag2_storage::storage_interfaces::MAX_BAGFILE_DURATION_NO_SPLIT)), topics_names_to_info_(), metadata_() {} @@ -85,6 +87,7 @@ void SequentialWriter::open( { base_folder_ = storage_options.uri; max_bagfile_size_ = storage_options.max_bagfile_size; + max_bagfile_duration = std::chrono::seconds(storage_options.max_bagfile_duration); max_cache_size_ = storage_options.max_cache_size; cache_.reserve(max_cache_size_); @@ -206,6 +209,9 @@ void SequentialWriter::write(std::shared_ptr( @@ -231,11 +237,25 @@ void SequentialWriter::write(std::shared_ptrget_bagfile_size() > max_bagfile_size_; + // Assume we aren't splitting + bool should_split = false; + + // Splitting by size + if (max_bagfile_size_ != rosbag2_storage::storage_interfaces::MAX_BAGFILE_SIZE_NO_SPLIT) { + should_split = should_split || (storage_->get_bagfile_size() > max_bagfile_size_); } + + // Splitting by time + if (max_bagfile_duration != std::chrono::seconds( + rosbag2_storage::storage_interfaces::MAX_BAGFILE_DURATION_NO_SPLIT)) + { + auto max_duration_ns = std::chrono::duration_cast( + max_bagfile_duration); + should_split = should_split || + ((std::chrono::high_resolution_clock::now() - metadata_.starting_time) > max_duration_ns); + } + + return should_split; } void SequentialWriter::finalize_metadata() diff --git a/rosbag2_storage/include/rosbag2_storage/storage_interfaces/base_io_interface.hpp b/rosbag2_storage/include/rosbag2_storage/storage_interfaces/base_io_interface.hpp index a0d0abbeb7..a238ec3e5f 100644 --- a/rosbag2_storage/include/rosbag2_storage/storage_interfaces/base_io_interface.hpp +++ b/rosbag2_storage/include/rosbag2_storage/storage_interfaces/base_io_interface.hpp @@ -35,6 +35,10 @@ enum class IOFlag : uint8_t // use 0 as the default maximum bagfile size value. ROSBAG2_STORAGE_PUBLIC extern const uint64_t MAX_BAGFILE_SIZE_NO_SPLIT; +// When bagfile splitting feature is not enabled or applicable, +// use 0 as the default maximum bagfile duration value. +ROSBAG2_STORAGE_PUBLIC extern const uint64_t MAX_BAGFILE_DURATION_NO_SPLIT; + class ROSBAG2_STORAGE_PUBLIC BaseIOInterface { public: diff --git a/rosbag2_storage/src/rosbag2_storage/base_io_interface.cpp b/rosbag2_storage/src/rosbag2_storage/base_io_interface.cpp index a825d15fbf..3936d2e636 100644 --- a/rosbag2_storage/src/rosbag2_storage/base_io_interface.cpp +++ b/rosbag2_storage/src/rosbag2_storage/base_io_interface.cpp @@ -19,5 +19,6 @@ namespace rosbag2_storage namespace storage_interfaces { const uint64_t MAX_BAGFILE_SIZE_NO_SPLIT = 0; +const uint64_t MAX_BAGFILE_DURATION_NO_SPLIT = 0; } } diff --git a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp index 8bc3827109..e2a33c8b55 100644 --- a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp +++ b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp @@ -412,6 +412,65 @@ TEST_F(RecordFixture, record_end_to_end_with_splitting_splits_bagfile) { } } +TEST_F(RecordFixture, record_end_to_end_with_duration_splitting_splits_bagfile) { + constexpr const char topic_name[] = "/test_topic"; + constexpr const int bagfile_split_duration = 1000; // 1 second + + std::stringstream command; + command << "ros2 bag record" << + " --output " << root_bag_path_.string() << + " -d " << bagfile_split_duration << + " " << topic_name; + auto process_handle = start_execution(command.str()); + wait_for_db(); + + constexpr const int expected_splits = 4; + { + constexpr const char message_str[] = "Test"; + constexpr const int message_size = 1024 * 1024; // 1MB + constexpr const int message_time = 500; // 500ms + // string message from test_msgs + const auto message = create_string_message(message_str, message_size); + constexpr const int message_count = (bagfile_split_duration * expected_splits) / message_time; + + pub_man_.run_scoped_publisher( + topic_name, + message, + 500ms, + message_count); + } + + stop_execution(process_handle); + + rosbag2_storage::MetadataIo metadata_io; + +#ifdef _WIN32 + { + rosbag2_storage::BagMetadata metadata; + metadata.version = 4; + metadata.storage_identifier = "sqlite3"; + + // Loop until expected_splits in case it split or the bagfile doesn't exist. + for (int i = 0; i < expected_splits; ++i) { + const auto bag_file_path = get_relative_bag_file_path(i); + if (rcpputils::fs::exists(root_bag_path_ / bag_file_path)) { + metadata.relative_file_paths.push_back(bag_file_path.string()); + } + } + + metadata_io.write_metadata(root_bag_path_.string(), metadata); + } +#endif + + wait_for_metadata(); + const auto metadata = metadata_io.read_metadata(root_bag_path_.string()); + + for (const auto & rel_path : metadata.relative_file_paths) { + auto path = root_bag_path_ / rcpputils::fs::path(rel_path); + EXPECT_TRUE(rcpputils::fs::exists(path)); + } +} + TEST_F(RecordFixture, record_end_to_end_test_with_zstd_file_compression_compresses_files) { constexpr const char topic_name[] = "/test_topic"; constexpr const int bagfile_split_size = 4 * 1024 * 1024; // 4MB. diff --git a/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp b/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp index 6c40ed2dbb..10163ed90b 100644 --- a/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp +++ b/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp @@ -98,6 +98,7 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * "no_discovery", "polling_interval", "max_bagfile_size", + "max_bagfile_duration", "max_cache_size", "topics", "include_hidden_topics", @@ -115,12 +116,13 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * bool no_discovery = false; uint64_t polling_interval_ms = 100; unsigned long long max_bagfile_size = 0; // NOLINT + unsigned long long max_bagfile_duration = 0; // NOLINT uint64_t max_cache_size = 0u; PyObject * topics = nullptr; bool include_hidden_topics = false; if ( !PyArg_ParseTupleAndKeywords( - args, kwargs, "ssssss|bbKKKObO", const_cast(kwlist), + args, kwargs, "ssssss|bbKKKKObO", const_cast(kwlist), &uri, &storage_id, &serilization_format, @@ -131,6 +133,7 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * &no_discovery, &polling_interval_ms, &max_bagfile_size, + &max_bagfile_duration, &max_cache_size, &topics, &include_hidden_topics, @@ -143,6 +146,7 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * storage_options.uri = std::string(uri); storage_options.storage_id = std::string(storage_id); storage_options.max_bagfile_size = (uint64_t) max_bagfile_size; + storage_options.max_bagfile_duration = static_cast(max_bagfile_duration); storage_options.max_cache_size = max_cache_size; record_options.all = all; record_options.is_discovery_disabled = no_discovery; From e2fce578f73866eba74e8ed6fb9b4aac88d2e432 Mon Sep 17 00:00:00 2001 From: Karsten Knese Date: Fri, 10 Jul 2020 08:34:43 -0700 Subject: [PATCH 03/77] More reliable topic remapping test (#456) * expect only 60 percent of messages to arrive Signed-off-by: Karsten Knese * test for only one message Signed-off-by: Karsten Knese --- .../test_play_topic_remap.cpp | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/rosbag2_transport/test/rosbag2_transport/test_play_topic_remap.cpp b/rosbag2_transport/test/rosbag2_transport/test_play_topic_remap.cpp index 041511c680..4db645099a 100644 --- a/rosbag2_transport/test/rosbag2_transport/test_play_topic_remap.cpp +++ b/rosbag2_transport/test/rosbag2_transport/test_play_topic_remap.cpp @@ -22,7 +22,6 @@ #include #include - #include "rosbag2_test_common/subscription_manager.hpp" #include "rosbag2_transport/rosbag2_transport.hpp" @@ -32,14 +31,13 @@ #include "rosbag2_play_test_fixture.hpp" - TEST_F(RosBag2PlayTestFixture, recorded_message_is_played_on_remapped_topic) { // test constants const std::string original_topic = "/topic_before_remap"; const std::string remapped_topic = "/topic_after_remap"; const int32_t test_value = 42; - const int64_t timestamp_in_milliseconds = 700; - const size_t expected_number_of_messages = 1; + const int64_t dt_in_milliseconds = 100; + const size_t expected_number_of_outgoing_messages = 5; play_options_.topic_remapping_options = {"--ros-args", "--remap", "topic_before_remap:=topic_after_remap"}; @@ -50,26 +48,30 @@ TEST_F(RosBag2PlayTestFixture, recorded_message_is_played_on_remapped_topic) { {original_topic, "test_msgs/BasicTypes", "", ""} }; - std::vector> messages = - {serialize_test_message(original_topic, timestamp_in_milliseconds, primitive_message1), }; - + std::vector> messages; + for (auto i = 1u; i <= expected_number_of_outgoing_messages; ++i) { + messages.push_back( + serialize_test_message(original_topic, dt_in_milliseconds * i, primitive_message1)); + } auto prepared_mock_reader = std::make_unique(); prepared_mock_reader->prepare(messages, topic_types); reader_ = std::make_unique(std::move(prepared_mock_reader)); - sub_->add_subscription(remapped_topic, expected_number_of_messages); + // We can't reliably test for an exact number of messages to arrive. + // Therefore, we test that at least one message is being received over the remapped topic. + sub_->add_subscription( + remapped_topic, 1u); auto await_received_messages = sub_->spin_subscriptions(); rosbag2_transport::Rosbag2Transport rosbag2_transport(reader_, writer_, info_); - rosbag2_transport.play( - storage_options_, play_options_); + rosbag2_transport.play(storage_options_, play_options_); await_received_messages.get(); auto replayed_test_primitives = sub_->get_received_messages( remapped_topic); - EXPECT_THAT(replayed_test_primitives, SizeIs(Ge(expected_number_of_messages))); + EXPECT_FALSE(replayed_test_primitives.empty()); EXPECT_THAT( replayed_test_primitives, Each(Pointee(Field(&test_msgs::msg::BasicTypes::int32_value, test_value)))); From c67d8c34ee1654c653cb1234de0265e74f5e36af Mon Sep 17 00:00:00 2001 From: Karsten Knese Date: Fri, 10 Jul 2020 12:35:37 -0700 Subject: [PATCH 04/77] minimal c++ API test (#451) * minimal c++ API test Signed-off-by: Karsten Knese * linters Signed-off-by: Karsten Knese --- .../rosbag2_cpp/writers/sequential_writer.cpp | 9 +- rosbag2_tests/CMakeLists.txt | 13 +++ .../rosbag2_tests/test_rosbag2_cpp_api.cpp | 100 ++++++++++++++++++ 3 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 rosbag2_tests/test/rosbag2_tests/test_rosbag2_cpp_api.cpp diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index b59feb0114..ebdbc2b8cc 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -205,7 +205,14 @@ void SequentialWriter::write(std::shared_ptrtopic_name).message_count; + try { + ++topics_names_to_info_.at(message->topic_name).message_count; + } catch (const std::out_of_range & oor) { + std::stringstream errmsg; + errmsg << "Failed to write on topic '" << message->topic_name << + "'. Call create_topic() before first write."; + throw std::runtime_error(errmsg.str()); + } if (should_split_bagfile()) { split_bagfile(); diff --git a/rosbag2_tests/CMakeLists.txt b/rosbag2_tests/CMakeLists.txt index 164df0e3f9..9e4c54bc22 100644 --- a/rosbag2_tests/CMakeLists.txt +++ b/rosbag2_tests/CMakeLists.txt @@ -87,6 +87,19 @@ if(BUILD_TESTING) test_msgs) endif() endif() + + ament_add_gmock(test_rosbag2_cpp_api + test/rosbag2_tests/test_rosbag2_cpp_api.cpp + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + if(TARGET test_rosbag2_cpp_api) + ament_target_dependencies(test_rosbag2_cpp_api + rclcpp + rosbag2_cpp + rosbag2_storage + rosbag2_storage_default_plugins + rosbag2_test_common + test_msgs) + endif() endif() ament_package() diff --git a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_cpp_api.cpp b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_cpp_api.cpp new file mode 100644 index 0000000000..bf2f69b514 --- /dev/null +++ b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_cpp_api.cpp @@ -0,0 +1,100 @@ +// Copyright 2020, Bosch Software Innovations GmbH. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include + +#include "rclcpp/serialization.hpp" +#include "rclcpp/serialized_message.hpp" + +#include "rcpputils/filesystem_helper.hpp" +#include "rcutils/time.h" + +#include "rosbag2_cpp/reader.hpp" +#include "rosbag2_cpp/readers/sequential_reader.hpp" +#include "rosbag2_cpp/writer.hpp" +#include "rosbag2_cpp/writers/sequential_writer.hpp" + +#include "test_msgs/msg/basic_types.hpp" + +TEST(TestRosbag2CPPAPI, minimal_writer_example) +{ + using TestMsgT = test_msgs::msg::BasicTypes; + TestMsgT test_msg; + test_msg.float64_value = 12345.6789; + rclcpp::SerializedMessage serialized_msg; + + rclcpp::Serialization serialization; + serialization.serialize_message(&test_msg, &serialized_msg); + + auto rosbag_directory = rcpputils::fs::path("test_rosbag2_writer_api_bag"); + // in case the bag was previously not cleaned up + rcpputils::fs::remove_all(rosbag_directory); + // See https://github.com/ros2/rosbag2/issues/448 + rcpputils::fs::create_directories(rosbag_directory); + + rosbag2_cpp::StorageOptions storage_options; + storage_options.uri = rosbag_directory.string(); + storage_options.storage_id = "sqlite3"; + storage_options.max_bagfile_size = 0; // default + storage_options.max_cache_size = 0; // default + rosbag2_cpp::ConverterOptions converter_options; + converter_options.input_serialization_format = "cdr"; + converter_options.output_serialization_format = "cdr"; + + { + rosbag2_cpp::Writer writer(std::make_unique()); + writer.open(storage_options, converter_options); + + auto bag_message = std::make_shared(); + auto ret = rcutils_system_time_now(&bag_message->time_stamp); + if (ret != RCL_RET_OK) { + FAIL() << "couldn't assign time rosbag message"; + } + + rosbag2_storage::TopicMetadata tm; + tm.name = "/my/test/topic"; + tm.type = "test_msgs/msg/BasicTypes"; + tm.serialization_format = "cdr"; + writer.create_topic(tm); + + bag_message->topic_name = tm.name; + bag_message->serialized_data = std::shared_ptr( + &serialized_msg.get_rcl_serialized_message(), [](rcutils_uint8_array_t * /* data */) {}); + + writer.write(bag_message); + // close on scope exit + } + + { + rosbag2_cpp::Reader reader(std::make_unique()); + reader.open(storage_options, converter_options); + while (reader.has_next()) { + auto bag_message = reader.read_next(); + + TestMsgT extracted_test_msg; + rclcpp::SerializedMessage extracted_serialized_msg(*bag_message->serialized_data); + serialization.deserialize_message( + &extracted_serialized_msg, &extracted_test_msg); + + EXPECT_EQ(test_msg, extracted_test_msg); + } + // close on scope exit + } + + // remove the rosbag again after the test + EXPECT_TRUE(rcpputils::fs::remove_all(rosbag_directory)); +} From 1d39a5ee8899dd39edb5b27fa20bfc39b10619e7 Mon Sep 17 00:00:00 2001 From: "P. J. Reed" Date: Mon, 13 Jul 2020 17:51:25 -0500 Subject: [PATCH 05/77] Add per-message ZSTD compression (#418) * Add per-message ZSTD compression This implements the per-messages compression and decompression functions for the ZSTD compressor and also adds unit tests for them. Distro A, OPSEC #2893 Signed-off-by: P. J. Reed --- ros2bag/ros2bag/verb/record.py | 4 +- .../sequential_compression_reader.cpp | 10 ++- .../rosbag2_compression/zstd_compressor.cpp | 62 ++++++++++++- .../rosbag2_compression/zstd_decompressor.cpp | 63 ++++++++++++- .../test_zstd_compressor.cpp | 89 +++++++++++++++++-- 5 files changed, 213 insertions(+), 15 deletions(-) diff --git a/ros2bag/ros2bag/verb/record.py b/ros2bag/ros2bag/verb/record.py index ecfbde8f8c..48936122ad 100644 --- a/ros2bag/ros2bag/verb/record.py +++ b/ros2bag/ros2bag/verb/record.py @@ -74,8 +74,8 @@ def add_arguments(self, parser, cli_name): # noqa: D102 ) parser.add_argument( '--compression-mode', type=str, default='none', - choices=['none', 'file'], - help='Determine whether to compress bag files. Default is "none".' + choices=['none', 'file', 'message'], + help="Determine whether to compress by file or message. Default is 'none'." ) parser.add_argument( '--compression-format', type=str, default='', choices=['zstd'], diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp index 922bfc1822..3b87c13175 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp @@ -29,7 +29,6 @@ namespace rosbag2_compression { - SequentialCompressionReader::SequentialCompressionReader( std::unique_ptr compression_factory, std::unique_ptr storage_factory, @@ -47,9 +46,12 @@ void SequentialCompressionReader::setup_decompression() compression_mode_ = rosbag2_compression::compression_mode_from_string(metadata_.compression_mode); if (compression_mode_ != rosbag2_compression::CompressionMode::NONE) { decompressor_ = compression_factory_->create_decompressor(metadata_.compression_format); - // Decompress the first file so that it is readable. - ROSBAG2_COMPRESSION_LOG_DEBUG_STREAM("Decompressing " << get_current_file().c_str()); - *current_file_iterator_ = decompressor_->decompress_uri(get_current_file()); + // Decompress the first file so that it is readable; don't need to do anything for + // per-message encryption. + if (compression_mode_ == CompressionMode::FILE) { + ROSBAG2_COMPRESSION_LOG_DEBUG_STREAM("Decompressing " << get_current_file().c_str()); + *current_file_iterator_ = decompressor_->decompress_uri(get_current_file()); + } } else { throw std::invalid_argument{ "SequentialCompressionReader requires a CompressionMode that is not NONE!"}; diff --git a/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp b/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp index d64f252ded..4d71be5f45 100644 --- a/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp +++ b/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -149,6 +150,35 @@ void write_output_buffer( fclose(file_pointer); } +/** + * Checks rcutils array resizing and throws a runtime_error if there was an error resizing. + * \param rcutils_ret_t Result of calling rcutils + */ +void throw_on_rcutils_resize_error(const rcutils_ret_t resize_result) +{ + if (resize_result == RCUTILS_RET_OK) { + return; + } + + std::stringstream error; + error << "rcutils_uint8_array_resize error: "; + switch (resize_result) { + case RCUTILS_RET_INVALID_ARGUMENT: + error << "Invalid Argument"; + break; + case RCUTILS_RET_BAD_ALLOC: + error << "Bad Alloc"; + break; + case RCUTILS_RET_ERROR: + error << "Ret Error"; + break; + default: + error << "Unexpected Result"; + break; + } + throw std::runtime_error(error.str()); +} + /** * Checks compression_result and throws a runtime_error if there was a ZSTD error. * \param compression_result is the return value of ZSTD_compress. @@ -218,9 +248,37 @@ std::string ZstdCompressor::compress_uri(const std::string & uri) } void ZstdCompressor::compress_serialized_bag_message( - rosbag2_storage::SerializedBagMessage *) + rosbag2_storage::SerializedBagMessage * message) { - throw std::logic_error{"Not implemented"}; + const auto start = std::chrono::high_resolution_clock::now(); + // Allocate based on compression bound and compress + const auto uncompressed_buffer_length = + ZSTD_compressBound(message->serialized_data->buffer_length); + std::vector compressed_buffer(uncompressed_buffer_length); + + // Perform compression and check. + // compression_result is either the actual compressed size or an error code. + const auto compression_result = ZSTD_compress( + compressed_buffer.data(), compressed_buffer.size(), + message->serialized_data->buffer, message->serialized_data->buffer_length, + kDefaultZstdCompressionLevel); + throw_on_zstd_error(compression_result); + + // Compression_buffer_length might be larger than the actual compression size + // Resize compressed_buffer so its size is the actual compression size. + compressed_buffer.resize(compression_result); + + const auto resize_result = + rcutils_uint8_array_resize(message->serialized_data.get(), compression_result); + throw_on_rcutils_resize_error(resize_result); + + // Note that rcutils_uint8_array_resize changes buffer_capacity but not buffer_length, we + // have to do that manually. + message->serialized_data->buffer_length = compression_result; + std::copy(compressed_buffer.begin(), compressed_buffer.end(), message->serialized_data->buffer); + + const auto end = std::chrono::high_resolution_clock::now(); + print_compression_statistics(start, end, uncompressed_buffer_length, compression_result); } std::string ZstdCompressor::get_compression_identifier() const diff --git a/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp b/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp index 7b2696cfe5..280e129622 100644 --- a/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp +++ b/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -175,6 +176,35 @@ void throw_on_zstd_error(const ZstdDecompressReturnType compression_result) } } +/** + * Checks rcutils array resizing and throws a runtime_error if there was an error resizing. + * \param rcutils_ret_t Result of calling rcutils + */ +void throw_on_rcutils_resize_error(const rcutils_ret_t resize_result) +{ + if (resize_result == RCUTILS_RET_OK) { + return; + } + + std::stringstream error; + error << "rcutils_uint8_array_resize error: "; + switch (resize_result) { + case RCUTILS_RET_INVALID_ARGUMENT: + error << "Invalid Argument"; + break; + case RCUTILS_RET_BAD_ALLOC: + error << "Bad Alloc"; + break; + case RCUTILS_RET_ERROR: + error << "Ret Error"; + break; + default: + error << "Unexpected Result"; + break; + } + throw std::runtime_error(error.str()); +} + /** * Checks frame_content and throws a runtime_error if there was a ZSTD error * or frame_content is invalid. @@ -257,9 +287,38 @@ std::string ZstdDecompressor::decompress_uri(const std::string & uri) } void ZstdDecompressor::decompress_serialized_bag_message( - rosbag2_storage::SerializedBagMessage *) + rosbag2_storage::SerializedBagMessage * message) { - throw std::logic_error{"Not implemented"}; + const auto start = std::chrono::high_resolution_clock::now(); + const auto compressed_buffer_length = message->serialized_data->buffer_length; + + const auto decompressed_buffer_length = + ZSTD_getFrameContentSize(message->serialized_data->buffer, compressed_buffer_length); + + throw_on_invalid_frame_content(decompressed_buffer_length); + + // Initializes decompressed_buffer with size = decompressed_buffer_length. + // Uniform initialization cannot be used here since it will choose + // the initializer list constructor instead. + std::vector decompressed_buffer(decompressed_buffer_length); + + const auto decompression_result = ZSTD_decompress( + decompressed_buffer.data(), decompressed_buffer_length, + message->serialized_data->buffer, compressed_buffer_length); + + throw_on_zstd_error(decompression_result); + + const auto resize_result = + rcutils_uint8_array_resize(message->serialized_data.get(), decompression_result); + throw_on_rcutils_resize_error(resize_result); + + message->serialized_data->buffer_length = decompression_result; + std::copy( + decompressed_buffer.begin(), decompressed_buffer.end(), + message->serialized_data->buffer); + + const auto end = std::chrono::high_resolution_clock::now(); + print_decompression_statistics(start, end, decompression_result, compressed_buffer_length); } std::string ZstdDecompressor::get_decompression_identifier() const diff --git a/rosbag2_compression/test/rosbag2_compression/test_zstd_compressor.cpp b/rosbag2_compression/test/rosbag2_compression/test_zstd_compressor.cpp index cb4d1a486e..776990d70e 100644 --- a/rosbag2_compression/test/rosbag2_compression/test_zstd_compressor.cpp +++ b/rosbag2_compression/test/rosbag2_compression/test_zstd_compressor.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include +#include #include #include @@ -24,6 +26,8 @@ #include "rosbag2_compression/zstd_compressor.hpp" #include "rosbag2_compression/zstd_decompressor.hpp" +#include "rosbag2_storage/ros_helper.hpp" + #include "rosbag2_test_common/temporary_directory_fixture.hpp" #include "gmock/gmock.h" @@ -32,6 +36,23 @@ namespace { constexpr const char kGarbageStatement[] = "garbage"; constexpr const int kDefaultGarbageFileSize = 10; // MiB +constexpr const size_t kExpectedCompressedDataSize = 976; // manually calculated, could change + // if compression params change + +/** + * Writes 1M * size garbage data to a stream. + * \param out The stream to write to. + * \param size The number of times to write. + */ +void write_garbage_stream(std::ostream & out, int size = kDefaultGarbageFileSize) +{ + const auto output_size = size * 1024 * 1024; + const auto num_iterations = output_size / static_cast(strlen(kGarbageStatement)); + + for (int i = 0; i < num_iterations; i++) { + out << kGarbageStatement; + } +} /** * Creates a text file of a certain size. @@ -43,12 +64,19 @@ void create_garbage_file(const std::string & uri, int size = kDefaultGarbageFile auto out = std::ofstream{uri}; out.exceptions(std::ifstream::failbit | std::ifstream::badbit); - const auto file_size = size * 1024 * 1024; - const auto num_iterations = file_size / static_cast(strlen(kGarbageStatement)); + write_garbage_stream(out, size); +} - for (int i = 0; i < num_iterations; i++) { - out << kGarbageStatement; - } +/** + * Creates a string of a certain size. + * \param size Size of the string in MiB. + * \return The string. + */ +std::string create_garbage_string(int size = kDefaultGarbageFileSize) +{ + std::stringstream output; + write_garbage_stream(output, size); + return output.str(); } std::vector read_file(const std::string & uri) @@ -76,6 +104,9 @@ class CompressionHelperFixture : public rosbag2_test_common::TemporaryDirectoryF void SetUp() override { + allocator_ = rcutils_get_default_allocator(); + message_ = create_garbage_string(); + rclcpp::init(0, nullptr); } @@ -83,6 +114,22 @@ class CompressionHelperFixture : public rosbag2_test_common::TemporaryDirectoryF { rclcpp::shutdown(); } + + std::string deserialize_message(std::shared_ptr serialized_message) + { + std::unique_ptr copied(new uint8_t[serialized_message->buffer_length + 1]); + std::copy( + serialized_message->buffer, + serialized_message->buffer + serialized_message->buffer_length, + copied.get()); + copied.get()[serialized_message->buffer_length] = '\0'; + std::string message_content(reinterpret_cast(copied.get())); + return message_content; + } + + rcutils_allocator_t allocator_; + std::string message_; + size_t compressed_length_{kExpectedCompressedDataSize}; }; TEST_F(CompressionHelperFixture, zstd_compress_file_uri) @@ -218,3 +265,35 @@ TEST_F(CompressionHelperFixture, zstd_decompress_fails_on_bad_uri) EXPECT_THROW(decompressor.decompress_uri(bad_uri), std::runtime_error) << "Expected decompress_uri(\"" << bad_uri << "\") to fail!"; } + +TEST_F(CompressionHelperFixture, zstd_compress_serialized_bag_message) +{ + auto msg = std::make_unique(); + msg->serialized_data.reset(new rcutils_uint8_array_t); + msg->serialized_data = rosbag2_storage::make_serialized_message( + message_.data(), message_.length()); + + rosbag2_compression::ZstdCompressor compressor; + compressor.compress_serialized_bag_message(msg.get()); + + ASSERT_EQ(compressed_length_, msg->serialized_data->buffer_length); +} + +TEST_F(CompressionHelperFixture, zstd_decompress_serialized_bag_message) +{ + auto msg = std::make_unique(); + msg->serialized_data.reset(new rcutils_uint8_array_t); + msg->serialized_data = rosbag2_storage::make_serialized_message( + message_.data(), message_.length()); + + rosbag2_compression::ZstdCompressor compressor; + compressor.compress_serialized_bag_message(msg.get()); + + const auto compressed_length = msg->serialized_data->buffer_length; + EXPECT_EQ(compressed_length_, compressed_length); + + rosbag2_compression::ZstdDecompressor decompressor; + EXPECT_NO_THROW(decompressor.decompress_serialized_bag_message(msg.get())); + std::string new_msg = deserialize_message(msg->serialized_data); + EXPECT_EQ(new_msg, message_); +} From fa3dc5abd80ebac1b115a21c60364186b34cd813 Mon Sep 17 00:00:00 2001 From: Karsten Knese Date: Tue, 14 Jul 2020 16:03:29 -0700 Subject: [PATCH 06/77] comment out unused variable (#460) Signed-off-by: Karsten Knese --- rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index ebdbc2b8cc..78c8b1a4e3 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -207,7 +207,7 @@ void SequentialWriter::write(std::shared_ptrtopic_name).message_count; - } catch (const std::out_of_range & oor) { + } catch (const std::out_of_range & /* oor */) { std::stringstream errmsg; errmsg << "Failed to write on topic '" << message->topic_name << "'. Call create_topic() before first write."; From 8c499246d6ac86717e189fba36df3273068e52b1 Mon Sep 17 00:00:00 2001 From: Dirk Thomas Date: Wed, 15 Jul 2020 15:09:17 -0700 Subject: [PATCH 07/77] use a single temp dir for the test class (#462) Signed-off-by: Dirk Thomas --- ros2bag/test/test_record_qos_profiles.py | 79 ++++++++++++------------ 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/ros2bag/test/test_record_qos_profiles.py b/ros2bag/test/test_record_qos_profiles.py index 962838d398..97781a2baa 100644 --- a/ros2bag/test/test_record_qos_profiles.py +++ b/ros2bag/test/test_record_qos_profiles.py @@ -61,54 +61,55 @@ def launch_bag_command(self, arguments, **kwargs): ) as pkg_command: yield pkg_command cls.launch_bag_command = launch_bag_command + cls.tmpdir = tempfile.TemporaryDirectory() + + @classmethod + def tearDownClass(cls): + cls.tmpdir.cleanup() def test_qos_simple(self): profile_path = PROFILE_PATH / 'qos_profile.yaml' - with tempfile.TemporaryDirectory() as tmpdirname: - output_path = Path(tmpdirname) / 'ros2bag_test_basic' - arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), - '--output', output_path.as_posix()] - with self.launch_bag_command(arguments=arguments) as bag_command: - bag_command.wait_for_shutdown(timeout=5) - expected_string_regex = re.compile(ERROR_STRING) - matches = expected_string_regex.search(bag_command.output) - assert not matches, print('ros2bag CLI did not produce the expected output') + output_path = Path(self.tmpdir.name) / 'ros2bag_test_basic' + arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), + '--output', output_path.as_posix()] + with self.launch_bag_command(arguments=arguments) as bag_command: + bag_command.wait_for_shutdown(timeout=5) + expected_string_regex = re.compile(ERROR_STRING) + matches = expected_string_regex.search(bag_command.output) + assert not matches, print('ros2bag CLI did not produce the expected output') def test_incomplete_qos_profile(self): profile_path = PROFILE_PATH / 'incomplete_qos_profile.yaml' - with tempfile.TemporaryDirectory() as tmpdirname: - output_path = Path(tmpdirname) / 'ros2bag_test_incomplete' - arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), - '--output', output_path.as_posix()] - with self.launch_bag_command(arguments=arguments) as bag_command: - bag_command.wait_for_shutdown(timeout=5) - expected_string_regex = re.compile(ERROR_STRING) - matches = expected_string_regex.search(bag_command.output) - assert not matches, print('ros2bag CLI did not produce the expected output') + output_path = Path(self.tmpdir.name) / 'ros2bag_test_incomplete' + arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), + '--output', output_path.as_posix()] + with self.launch_bag_command(arguments=arguments) as bag_command: + bag_command.wait_for_shutdown(timeout=5) + expected_string_regex = re.compile(ERROR_STRING) + matches = expected_string_regex.search(bag_command.output) + assert not matches, print('ros2bag CLI did not produce the expected output') def test_incomplete_qos_duration(self): profile_path = PROFILE_PATH / 'incomplete_qos_duration.yaml' - with tempfile.TemporaryDirectory() as tmpdirname: - output_path = Path(tmpdirname) / 'ros2bag_test_incomplete_duration' - arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), - '--output', output_path.as_posix()] - with self.launch_bag_command(arguments=arguments) as bag_command: - bag_command.wait_for_shutdown(timeout=5) - assert bag_command.exit_code != launch_testing.asserts.EXIT_OK - expected_string_regex = re.compile(ERROR_STRING) - matches = expected_string_regex.search(bag_command.output) - assert matches, print('ros2bag CLI did not produce the expected output') + output_path = Path(self.tmpdir.name) / 'ros2bag_test_incomplete_duration' + arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), + '--output', output_path.as_posix()] + with self.launch_bag_command(arguments=arguments) as bag_command: + bag_command.wait_for_shutdown(timeout=5) + assert bag_command.exit_code != launch_testing.asserts.EXIT_OK + expected_string_regex = re.compile(ERROR_STRING) + matches = expected_string_regex.search(bag_command.output) + assert matches, print('ros2bag CLI did not produce the expected output') def test_nonexistent_qos_profile(self): profile_path = PROFILE_PATH / 'foobar.yaml' - with tempfile.TemporaryDirectory() as tmpdirname: - output_path = Path(tmpdirname) / 'ros2bag_test_incomplete' - arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), - '--output', output_path.as_posix()] - with self.launch_bag_command(arguments=arguments) as bag_command: - bag_command.wait_for_shutdown(timeout=5) - assert bag_command.exit_code != launch_testing.asserts.EXIT_OK - expected_string_regex = re.compile( - r"ros2 bag record: error: argument --qos-profile-overrides-path: can't open") - matches = expected_string_regex.search(bag_command.output) - assert matches, print('ros2bag CLI did not produce the expected output') + output_path = Path(self.tmpdir.name) / 'ros2bag_test_nonexistent' + arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), + '--output', output_path.as_posix()] + with self.launch_bag_command(arguments=arguments) as bag_command: + bag_command.wait_for_shutdown(timeout=5) + assert bag_command.exit_code != launch_testing.asserts.EXIT_OK + expected_string_regex = re.compile( + r"ros2 bag record: error: argument --qos-profile-overrides-path: can't open") + matches = expected_string_regex.search(bag_command.output) + assert matches, print('ros2bag CLI did not produce the expected output') From 1e25d22a16b7e805fbc7c228894a579865a5d6ab Mon Sep 17 00:00:00 2001 From: Karsten Knese Date: Fri, 17 Jul 2020 16:37:41 -0700 Subject: [PATCH 08/77] reenable cppcheck for rosbag2_transport (#461) * reenable cppcheck Signed-off-by: Karsten Knese * suppress unknown macro inline Signed-off-by: Karsten Knese --- rosbag2_transport/CMakeLists.txt | 2 -- .../src/rosbag2_transport/generic_subscription.hpp | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rosbag2_transport/CMakeLists.txt b/rosbag2_transport/CMakeLists.txt index 53455865d7..e4b0dbd284 100644 --- a/rosbag2_transport/CMakeLists.txt +++ b/rosbag2_transport/CMakeLists.txt @@ -189,8 +189,6 @@ if(BUILD_TESTING) find_package(test_msgs REQUIRED) find_package(rosbag2_test_common REQUIRED) include(cmake/rosbag2_transport_add_gmock.cmake) - # explicitly disable cppcheck - set(ament_cmake_cppcheck_FOUND TRUE) ament_lint_auto_find_test_dependencies() call_for_each_rmw_implementation(create_tests_for_rmw_implementation) endif() diff --git a/rosbag2_transport/src/rosbag2_transport/generic_subscription.hpp b/rosbag2_transport/src/rosbag2_transport/generic_subscription.hpp index 5eb148c7c7..118f0aa10f 100644 --- a/rosbag2_transport/src/rosbag2_transport/generic_subscription.hpp +++ b/rosbag2_transport/src/rosbag2_transport/generic_subscription.hpp @@ -35,6 +35,7 @@ namespace rosbag2_transport class GenericSubscription : public rclcpp::SubscriptionBase { public: + // cppcheck-suppress unknownMacro RCLCPP_SMART_PTR_DEFINITIONS(GenericSubscription) /** @@ -73,6 +74,7 @@ class GenericSubscription : public rclcpp::SubscriptionBase const rclcpp::QoS & qos_profile() const; private: + // cppcheck-suppress unknownMacro RCLCPP_DISABLE_COPY(GenericSubscription) std::shared_ptr borrow_serialized_message(size_t capacity); From b2293999f2e4e03329f4ab99d2fa1561a100fd8c Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Mon, 20 Jul 2020 09:32:51 -0700 Subject: [PATCH 09/77] Use foxy testing apt repos to install linters for Actions (#463) * Use foxy testing apt repos to install linters for Actions Signed-off-by: Emerson Knapp --- .github/workflows/lint.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4be1b3130f..13aaadb6fa 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,8 +1,4 @@ # Run linters automatically on pull requests. -# -# ros-tooling/actino-ros-lint is relying on the latest APT binary packages. -# As of 2020-05-01, Eloquent is the latest release for which APT binary -# packages are available, which is why the Docker image is based on Eloquent. name: Lint rosbag2 on: pull_request: @@ -12,7 +8,7 @@ jobs: name: ament_copyright runs-on: ubuntu-latest container: - image: rostooling/setup-ros-docker:ubuntu-bionic-ros-eloquent-ros-base-latest + image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest steps: # TODO(setup-ros-docker#7): calling chown is necessary for now - run: sudo chown -R rosbuild:rosbuild "$HOME" . @@ -20,6 +16,7 @@ jobs: - uses: ros-tooling/action-ros-lint@0.0.6 with: linter: copyright + distribution: foxy package-name: | ros2bag rosbag2_compression @@ -34,7 +31,7 @@ jobs: name: ament_xmllint runs-on: ubuntu-latest container: - image: rostooling/setup-ros-docker:ubuntu-bionic-ros-eloquent-ros-base-latest + image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest steps: # TODO(setup-ros-docker#7): calling chown is necessary for now - run: sudo chown -R rosbuild:rosbuild "$HOME" . @@ -42,6 +39,7 @@ jobs: - uses: ros-tooling/action-ros-lint@0.0.6 with: linter: xmllint + distribution: foxy package-name: | ros2bag rosbag2 @@ -57,12 +55,12 @@ jobs: ament_lint_cpp: # Linters applicable to C++ packages name: ament_${{ matrix.linter }} runs-on: ubuntu-latest - container: - image: rostooling/setup-ros-docker:ubuntu-bionic-ros-eloquent-ros-base-latest strategy: fail-fast: false matrix: linter: [cppcheck, cpplint, uncrustify] + container: + image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest steps: # TODO(setup-ros-docker#7): calling chown is necessary for now - run: sudo chown -R rosbuild:rosbuild "$HOME" . @@ -70,6 +68,7 @@ jobs: - uses: ros-tooling/action-ros-lint@0.0.6 with: linter: ${{ matrix.linter }} + distribution: foxy package-name: | rosbag2_compression rosbag2_converter_default_plugins @@ -83,7 +82,7 @@ jobs: name: ament_${{ matrix.linter }} runs-on: ubuntu-latest container: - image: rostooling/setup-ros-docker:ubuntu-bionic-ros-eloquent-ros-base-latest + image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest strategy: fail-fast: false matrix: @@ -95,5 +94,6 @@ jobs: - uses: ros-tooling/action-ros-lint@0.0.6 with: linter: ${{ matrix.linter }} + distribution: foxy package-name: | ros2bag From 33a32096a165b97e168c8bab93e96582d14a30fd Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Mon, 20 Jul 2020 12:16:10 -0700 Subject: [PATCH 10/77] Reenable flake8 linter (#467) Signed-off-by: Emerson Knapp --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 13aaadb6fa..8d45f1e5ff 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -86,7 +86,7 @@ jobs: strategy: fail-fast: false matrix: - linter: [pep257] # TODO re-enable flake8 when it is fixed + linter: [pep257, flake8] steps: # TODO(setup-ros-docker#7): calling chown is necessary for now - run: sudo chown -R rosbuild:rosbuild "$HOME" . From 15c41a7e58af35fe66b095781de6496624ebf5fc Mon Sep 17 00:00:00 2001 From: "P. J. Reed" Date: Tue, 21 Jul 2020 14:34:20 -0500 Subject: [PATCH 11/77] Consolidate ZSTD utility functions (#459) * Consolidate ZSTD utility functions The zstd_compressor and zstd_decompressor implementations had a number of duplicated utility functions between them; this consolidates them into one file. Signed-off-by: P. J. Reed --- rosbag2_compression/CMakeLists.txt | 1 + .../rosbag2_compression/compression_utils.cpp | 200 +++++++++++++++ .../rosbag2_compression/compression_utils.hpp | 107 ++++++++ .../rosbag2_compression/zstd_compressor.cpp | 195 +-------------- .../rosbag2_compression/zstd_decompressor.cpp | 232 +----------------- 5 files changed, 312 insertions(+), 423 deletions(-) create mode 100644 rosbag2_compression/src/rosbag2_compression/compression_utils.cpp create mode 100644 rosbag2_compression/src/rosbag2_compression/compression_utils.hpp diff --git a/rosbag2_compression/CMakeLists.txt b/rosbag2_compression/CMakeLists.txt index a52f97b519..ebdcb7e1bf 100644 --- a/rosbag2_compression/CMakeLists.txt +++ b/rosbag2_compression/CMakeLists.txt @@ -30,6 +30,7 @@ find_package(zstd REQUIRED) add_library(${PROJECT_NAME}_zstd SHARED + src/rosbag2_compression/compression_utils.cpp src/rosbag2_compression/zstd_compressor.cpp src/rosbag2_compression/zstd_decompressor.cpp) target_include_directories(${PROJECT_NAME}_zstd diff --git a/rosbag2_compression/src/rosbag2_compression/compression_utils.cpp b/rosbag2_compression/src/rosbag2_compression/compression_utils.cpp new file mode 100644 index 0000000000..2a48fa8d84 --- /dev/null +++ b/rosbag2_compression/src/rosbag2_compression/compression_utils.cpp @@ -0,0 +1,200 @@ +// Copyright 2020 Southwest Research Institute. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "compression_utils.hpp" + +#include +#include + +namespace +{ +/** + * Open a file using the OS-specific C API. + * \param uri is the path to the file. + * \param read_mode is the read mode accepted by OS-specific fopen. + * \return the FILE pointer or nullptr if the file was not opened. + */ +FILE * open_file(const std::string & uri, const std::string & read_mode) +{ + FILE * fp{nullptr}; +#ifdef _WIN32 + fopen_s(&fp, uri.c_str(), read_mode.c_str()); +#else + fp = std::fopen(uri.c_str(), read_mode.c_str()); +#endif + return fp; +} +} // namespace + +namespace rosbag2_compression +{ +std::vector get_input_buffer(const std::string & uri) +{ + // Read in buffer, handling accordingly + const auto file_pointer = open_file(uri, "rb"); + if (file_pointer == nullptr) { + std::stringstream errmsg; + errmsg << "Failed to open file: \"" << uri << + "\" for binary reading! errno(" << errno << ")"; + + throw std::runtime_error{errmsg.str()}; + } + + const auto file_path = rcpputils::fs::path{uri}; + const auto input_buffer_length = file_path.exists() ? file_path.file_size() : 0u; + if (input_buffer_length == 0) { + fclose(file_pointer); + + std::stringstream errmsg; + errmsg << "Unable to get size of file: \"" << uri << "\""; + + throw std::runtime_error{errmsg.str()}; + } + + // Initialize compress_buffer with size = compressed_buffer_length. + // Uniform initialization cannot be used here since it will choose + // the initializer list constructor instead. + std::vector input_buffer(input_buffer_length); + + const auto read_count = fread( + input_buffer.data(), sizeof(decltype(input_buffer)::value_type), + input_buffer_length, file_pointer); + + if (read_count != input_buffer_length) { + ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( + "Bytes read !(" << + read_count << ") != buffer size (" << input_buffer.size() << + ")!"); + // An error indicator is set by fread, so the following check will throw. + } + + if (ferror(file_pointer)) { + fclose(file_pointer); + + std::stringstream errmsg; + errmsg << "Unable to read binary data from file: \"" << uri << "\"!"; + + throw std::runtime_error{errmsg.str()}; + } + + fclose(file_pointer); + return input_buffer; +} + +void write_output_buffer( + const std::vector & output_buffer, + const std::string & uri) +{ + if (output_buffer.empty()) { + std::stringstream errmsg; + errmsg << "Cannot write empty buffer to file: \"" << uri << "\""; + + throw std::runtime_error{errmsg.str()}; + } + + const auto file_pointer = open_file(uri, "wb"); + if (file_pointer == nullptr) { + std::stringstream errmsg; + errmsg << "Failed to open file: \"" << uri << + "\" for binary writing! errno(" << errno << ")"; + + throw std::runtime_error{errmsg.str()}; + } + + const auto write_count = fwrite( + output_buffer.data(), sizeof(uint8_t), + output_buffer.size(), file_pointer); + + if (write_count != output_buffer.size()) { + ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( + "Bytes written (" << + write_count << " != output_buffer size (" << output_buffer.size() << + ")!"); + // An error indicator is set by fwrite, so the following check will throw. + } + + if (ferror(file_pointer)) { + fclose(file_pointer); + + std::stringstream errmsg; + errmsg << "Unable to write data to file: \"" << uri << "\"!"; + + throw std::runtime_error{errmsg.str()}; + } + + fclose(file_pointer); +} + + +void throw_on_zstd_error(const ZstdDecompressReturnType compression_result) +{ + if (ZSTD_isError(compression_result)) { + std::stringstream error; + error << "ZSTD decompression error: " << ZSTD_getErrorName(compression_result); + + throw std::runtime_error{error.str()}; + } +} + +void throw_on_rcutils_resize_error(const rcutils_ret_t resize_result) +{ + if (resize_result == RCUTILS_RET_OK) { + return; + } + + std::stringstream error; + error << "rcutils_uint8_array_resize error: "; + switch (resize_result) { + case RCUTILS_RET_INVALID_ARGUMENT: + error << "Invalid Argument"; + break; + case RCUTILS_RET_BAD_ALLOC: + error << "Bad Alloc"; + break; + case RCUTILS_RET_ERROR: + error << "Ret Error"; + break; + default: + error << "Unexpected Result"; + break; + } + throw std::runtime_error(error.str()); +} + +void throw_on_invalid_frame_content(const ZstdGetFrameContentSizeReturnType frame_content) +{ + if (frame_content == ZSTD_CONTENTSIZE_ERROR) { + throw std::runtime_error{"Unable to determine file size due to error."}; + } else if (frame_content == ZSTD_CONTENTSIZE_UNKNOWN) { + throw std::runtime_error{"Unable to determine file size."}; + } +} + +void print_compression_statistics( + const std::chrono::high_resolution_clock::time_point start, + const std::chrono::high_resolution_clock::time_point end, + const size_t decompressed_size, + const size_t compressed_size) +{ + const auto duration = std::chrono::duration_cast(end - start); + const auto decompression_ratio = + static_cast(decompressed_size) / static_cast(compressed_size); + + ROSBAG2_COMPRESSION_LOG_DEBUG_STREAM( + "\"Compression statistics\" : {" << + "\"Time\" : " << (duration.count() / 1000.0) << + ", \"Compression Ratio\" : " << decompression_ratio << + "}"); +} +} // namespace rosbag2_compression diff --git a/rosbag2_compression/src/rosbag2_compression/compression_utils.hpp b/rosbag2_compression/src/rosbag2_compression/compression_utils.hpp new file mode 100644 index 0000000000..20b466398d --- /dev/null +++ b/rosbag2_compression/src/rosbag2_compression/compression_utils.hpp @@ -0,0 +1,107 @@ +// Copyright 2020 Southwest Research Institute. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_COMPRESSION__COMPRESSION_UTILS_HPP_ +#define ROSBAG2_COMPRESSION__COMPRESSION_UTILS_HPP_ + +#include + +#include +#include +#include +#include + +#include "logging.hpp" + +#include "rcpputils/filesystem_helper.hpp" + +namespace rosbag2_compression +{ +// Increasing the compression level will: +// - Increase the time taken to compress +// - Decrease the size of the compressed data +// Setting to zero uses Zstd's default value of 3. +constexpr const int kDefaultZstdCompressionLevel = 1; +// String constant used to identify ZstdCompressor. +constexpr const char kCompressionIdentifier[] = "zstd"; +// String constant used to identify ZstdDecompressor. +constexpr const char kDecompressionIdentifier[] = "zstd"; +// Used as a parameter type in a function that accepts the output of ZSTD_compress. +using ZstdCompressReturnType = decltype(ZSTD_compress( + nullptr, 0, + nullptr, 0, 0)); +// Used as a parameter type in a function that accepts the output of ZSTD_decompress. +using ZstdDecompressReturnType = decltype(ZSTD_decompress( + nullptr, 0, + nullptr, 0)); +// Used as a parameter type in a function that accepts the output of ZSTD_getFrameContentSize. +using ZstdGetFrameContentSizeReturnType = decltype(ZSTD_getFrameContentSize(nullptr, 0)); + +/** + * Read a file from the supplied uri into a vector. + * + * \param uri is the path to the file. + * \return the contents of the buffer as a vector. + */ +std::vector get_input_buffer(const std::string & uri); + +/** + * Writes the output buffer to the specified file path. + * \param output_buffer is the data to write. + * \param uri is the relative file path to the output storage. + */ +void write_output_buffer( + const std::vector & output_buffer, + const std::string & uri); + +/** + * Checks compression_result and throws a runtime_error if there was a ZSTD error. + * \param compression_result is the return value of ZSTD_compress or ZSTD_decompress. + */ +void throw_on_zstd_error(const ZstdDecompressReturnType compression_result); + +/** + * Checks rcutils array resizing and throws a runtime_error if there was an error resizing. + * \param rcutils_ret_t Result of calling rcutils + */ +void throw_on_rcutils_resize_error(const rcutils_ret_t resize_result); + +/** + * Checks frame_content and throws a runtime_error if there was a ZSTD error + * or frame_content is invalid. + * \param frame_content is the return value of ZSTD_getFrameContentSize. + */ +void throw_on_invalid_frame_content(const ZstdGetFrameContentSizeReturnType frame_content); + +/** + * Prints compression statistics to the debug log stream. + * The log statement is formatted as JSON. + * Time is formatted as a decimal of seconds. + * + * Example: + * "Compression statistics" : {"Time" : 1.2, "Compression Ratio" : 0.5} + * + * \param start is the time_point when compression or decompression started. + * \param end is the time_point when compression or decompression ended. + * \param decompressed_size is the decompressed data size + * \param compressed_size is the compressed data size + */ +void print_compression_statistics( + const std::chrono::high_resolution_clock::time_point start, + const std::chrono::high_resolution_clock::time_point end, + const size_t decompressed_size, + const size_t compressed_size); +} // namespace rosbag2_compression + +#endif // ROSBAG2_COMPRESSION__COMPRESSION_UTILS_HPP_ diff --git a/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp b/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp index 4d71be5f45..32a509cfa5 100644 --- a/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp +++ b/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp @@ -21,202 +21,9 @@ #include "rcpputils/filesystem_helper.hpp" +#include "compression_utils.hpp" #include "rosbag2_compression/zstd_compressor.hpp" -#include "logging.hpp" - -namespace -{ -// Increasing the compression level will: -// - Increase the time taken to compress -// - Decrease the size of the compressed data -// Setting to zero uses Zstd's default value of 3. -constexpr const int kDefaultZstdCompressionLevel = 1; - -// String constant used to identify ZstdCompressor. -constexpr const char kCompressionIdentifier[] = "zstd"; -// Used as a parameter type in a function that accepts the output of ZSTD_compress. -using ZstdCompressReturnType = decltype(ZSTD_compress(nullptr, 0, nullptr, 0, 0)); - -/** - * Open a file using the OS-specific C API. - * \param uri is the path to the file. - * \param read_mode is the read mode accepted by OS-specific fopen. - * \return the FILE pointer or nullptr if the file was not opened. - */ -FILE * open_file(const std::string & uri, const std::string & read_mode) -{ - FILE * fp{nullptr}; -#ifdef _WIN32 - fopen_s(&fp, uri.c_str(), read_mode.c_str()); -#else - fp = std::fopen(uri.c_str(), read_mode.c_str()); -#endif - return fp; -} - -/** - * Read a file from the supplied uri into a vector. - * \param uri is the path to the file. - * \return the contents of the buffer as a vector. - */ -std::vector get_input_buffer(const std::string & uri) -{ - // Read in buffer, handling accordingly - const auto file_pointer = open_file(uri.c_str(), "rb"); - if (file_pointer == nullptr) { - std::stringstream errmsg; - errmsg << "Error opening file: \"" << uri << - "\" for binary reading! errno(" << errno << ")"; - - throw std::runtime_error{errmsg.str()}; - } - - const auto file_path = rcpputils::fs::path{uri}; - const auto decompressed_buffer_length = file_path.exists() ? file_path.file_size() : 0u; - - if (decompressed_buffer_length == 0) { - fclose(file_pointer); - - std::stringstream errmsg; - errmsg << "Unable to get size of file: \"" << uri << "\""; - - throw std::runtime_error{errmsg.str()}; - } - - // Allocate and read in - std::vector decompressed_buffer(decompressed_buffer_length); - - const auto read_count = fread( - decompressed_buffer.data(), sizeof(uint8_t), decompressed_buffer.size(), file_pointer); - - if (read_count != decompressed_buffer_length) { - ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( - "Bytes read (" << read_count << - ") != decompressed_buffer_length (" << decompressed_buffer.size() << ")!"); - // An error indicator is set by this, so the following check will throw - } - - if (ferror(file_pointer)) { - fclose(file_pointer); - - std::stringstream errmsg; - errmsg << "Unable to read binary data from file: \"" << uri << "\"!"; - - throw std::runtime_error{errmsg.str()}; - } - fclose(file_pointer); - return decompressed_buffer; -} - -/** - * Writes the output buffer to the specified file path. - * \param output_buffer is the data to write. - * \param uri is the relative file path to the output storage. - */ -void write_output_buffer( - const std::vector & output_buffer, - const std::string & uri) -{ - if (output_buffer.empty()) { - std::stringstream errmsg; - errmsg << "Cannot write empty buffer to file: \"" << uri << "\""; - - throw std::runtime_error{errmsg.str()}; - } - - const auto file_pointer = open_file(uri.c_str(), "wb"); - const auto write_count = fwrite( - output_buffer.data(), - sizeof(uint8_t), - output_buffer.size(), - file_pointer); - - if (write_count != output_buffer.size()) { - ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( - "Bytes written (" << write_count << - ") != output_buffer size (" << output_buffer.size() << ")!"); - // An error indicator is set by fwrite, so the following check will throw. - } - - if (ferror(file_pointer)) { - fclose(file_pointer); - - std::stringstream errmsg; - errmsg << "Unable to write compressed data to file: \"" << uri << "\"!"; - - throw std::runtime_error{errmsg.str()}; - } - fclose(file_pointer); -} - -/** - * Checks rcutils array resizing and throws a runtime_error if there was an error resizing. - * \param rcutils_ret_t Result of calling rcutils - */ -void throw_on_rcutils_resize_error(const rcutils_ret_t resize_result) -{ - if (resize_result == RCUTILS_RET_OK) { - return; - } - - std::stringstream error; - error << "rcutils_uint8_array_resize error: "; - switch (resize_result) { - case RCUTILS_RET_INVALID_ARGUMENT: - error << "Invalid Argument"; - break; - case RCUTILS_RET_BAD_ALLOC: - error << "Bad Alloc"; - break; - case RCUTILS_RET_ERROR: - error << "Ret Error"; - break; - default: - error << "Unexpected Result"; - break; - } - throw std::runtime_error(error.str()); -} - -/** - * Checks compression_result and throws a runtime_error if there was a ZSTD error. - * \param compression_result is the return value of ZSTD_compress. - */ -void throw_on_zstd_error(const ZstdCompressReturnType compression_result) -{ - if (ZSTD_isError(compression_result)) { - std::stringstream error; - error << "ZSTD compression error: " << ZSTD_getErrorName(compression_result); - throw std::runtime_error{error.str()}; - } -} - -/** - * Prints compression statistics to the debug log stream. - * The log statement is formatted in JSON. - * Time is formatted as a decimal of seconds. - * - * Example: - * "Compression statistics: {"Time" : 1.2, "Compression Ratio" : 0.5} - */ -void print_compression_statistics( - std::chrono::high_resolution_clock::time_point start, - std::chrono::high_resolution_clock::time_point end, - size_t decompressed_size, size_t compressed_size) -{ - const auto duration = std::chrono::duration_cast(end - start); - const auto compression_ratio = - static_cast(decompressed_size) / static_cast(compressed_size); - ROSBAG2_COMPRESSION_LOG_DEBUG_STREAM( - "\"Compression statistics\" : {" << - "\"Time\" : " << (duration.count() / 1000.0) << - ", \"Compression Ratio\" : " << compression_ratio << - "}" - ); -} -} // namespace - namespace rosbag2_compression { diff --git a/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp b/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp index 280e129622..5660d7c5bb 100644 --- a/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp +++ b/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp @@ -21,235 +21,9 @@ #include "rcpputils/filesystem_helper.hpp" +#include "compression_utils.hpp" #include "rosbag2_compression/zstd_decompressor.hpp" -#include "logging.hpp" - -namespace -{ - -// String constant used to identify ZstdDecompressor. -constexpr const char kDecompressionIdentifier[] = "zstd"; -// Used as a parameter type in a function that accepts the output of ZSTD_decompress. -using ZstdDecompressReturnType = decltype(ZSTD_decompress(nullptr, 0, nullptr, 0)); -// Used as a parameter type in a function that accepts the output of ZSTD_getFrameContentSize. -using ZstdGetFrameContentSizeReturnType = decltype(ZSTD_getFrameContentSize(nullptr, 0)); - -/** - * Open a file using the C API. - * This function calls OS-specific implementation of fopen. - * - * \param uri is the path to the file - * \param read_mode is the read mode string accepted by fopen. - * \return the FILE pointer or nullptr if the file was not opened. - */ -FILE * open_file(const std::string & uri, const std::string & read_mode) -{ - FILE * fp{nullptr}; -#ifdef _WIN32 - fopen_s(&fp, uri.c_str(), read_mode.c_str()); -#else - fp = std::fopen(uri.c_str(), read_mode.c_str()); -#endif - return fp; -} - -/** - * Read a file from the supplied uri into a vector. - * - * \param uri is the path to the file. - * \return the contents of the buffer as a vector. - */ -std::vector get_input_buffer(const std::string & uri) -{ - // Read in buffer, handling accordingly - const auto file_pointer = open_file(uri.c_str(), "rb"); - if (file_pointer == nullptr) { - std::stringstream errmsg; - errmsg << "Failed to open file: \"" << uri << - "\" for binary reading! errno(" << errno << ")"; - - throw std::runtime_error{errmsg.str()}; - } - - const auto file_path = rcpputils::fs::path{uri}; - const auto compressed_buffer_length = file_path.exists() ? file_path.file_size() : 0u; - if (compressed_buffer_length == 0) { - fclose(file_pointer); - - std::stringstream errmsg; - errmsg << "Unable to get size of file: \"" << uri << "\""; - - throw std::runtime_error{errmsg.str()}; - } - - // Initialize compress_buffer with size = compressed_buffer_length. - // Uniform initialization cannot be used here since it will choose - // the initializer list constructor instead. - std::vector compressed_buffer(compressed_buffer_length); - - const auto read_count = fread( - compressed_buffer.data(), sizeof(decltype(compressed_buffer)::value_type), - compressed_buffer_length, file_pointer); - - if (read_count != compressed_buffer_length) { - ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( - "Bytes read !(" << - read_count << ") != compressed_buffer size (" << compressed_buffer.size() << - ")!"); - // An error indicator is set by fread, so the following check will throw. - } - - if (ferror(file_pointer)) { - fclose(file_pointer); - - std::stringstream errmsg; - errmsg << "Unable to read binary data from file: \"" << uri << "\"!"; - - throw std::runtime_error{errmsg.str()}; - } - - fclose(file_pointer); - return compressed_buffer; -} - -/** - * Writes the output buffer to the specified file path. - * \param output_buffer is the data to write. - * \param uri is the relative file path to the output storage. - */ -void write_output_buffer( - const std::vector & output_buffer, - const std::string & uri) -{ - if (output_buffer.empty()) { - std::stringstream errmsg; - errmsg << "Cannot write empty buffer to file: \"" << uri << "\""; - - throw std::runtime_error{errmsg.str()}; - } - - const auto file_pointer = open_file(uri.c_str(), "wb"); - if (file_pointer == nullptr) { - std::stringstream errmsg; - errmsg << "Failed to open file: \"" << uri << - "\" for binary writing! errno(" << errno << ")"; - - throw std::runtime_error{errmsg.str()}; - } - - const auto write_count = fwrite( - output_buffer.data(), sizeof(uint8_t), - output_buffer.size(), file_pointer); - - if (write_count != output_buffer.size()) { - ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( - "Bytes written (" << - write_count << " != output_buffer size (" << output_buffer.size() << - ")!"); - // An error indicator is set by fwrite, so the following check will throw. - } - - if (ferror(file_pointer)) { - fclose(file_pointer); - - std::stringstream errmsg; - errmsg << "Unable to write decompressed data to file: \"" << uri << "\"!"; - - throw std::runtime_error{errmsg.str()}; - } - - fclose(file_pointer); -} - -/** - * Checks compression_result and throws a runtime_error if there was a ZSTD error. - * \param compression_result is the return value of ZSTD_decompress. - */ -void throw_on_zstd_error(const ZstdDecompressReturnType compression_result) -{ - if (ZSTD_isError(compression_result)) { - std::stringstream error; - error << "ZSTD decompression error: " << ZSTD_getErrorName(compression_result); - - throw std::runtime_error{error.str()}; - } -} - -/** - * Checks rcutils array resizing and throws a runtime_error if there was an error resizing. - * \param rcutils_ret_t Result of calling rcutils - */ -void throw_on_rcutils_resize_error(const rcutils_ret_t resize_result) -{ - if (resize_result == RCUTILS_RET_OK) { - return; - } - - std::stringstream error; - error << "rcutils_uint8_array_resize error: "; - switch (resize_result) { - case RCUTILS_RET_INVALID_ARGUMENT: - error << "Invalid Argument"; - break; - case RCUTILS_RET_BAD_ALLOC: - error << "Bad Alloc"; - break; - case RCUTILS_RET_ERROR: - error << "Ret Error"; - break; - default: - error << "Unexpected Result"; - break; - } - throw std::runtime_error(error.str()); -} - -/** - * Checks frame_content and throws a runtime_error if there was a ZSTD error - * or frame_content is invalid. - * \param frame_content is the return value of ZSTD_getFrameContentSize. - */ -void throw_on_invalid_frame_content(const ZstdGetFrameContentSizeReturnType frame_content) -{ - if (frame_content == ZSTD_CONTENTSIZE_ERROR) { - throw std::runtime_error{"Unable to determine file size due to error."}; - } else if (frame_content == ZSTD_CONTENTSIZE_UNKNOWN) { - throw std::runtime_error{"Unable to determine file size."}; - } -} - -/** - * Prints decompression statistics to the debug log stream. - * The log statement is formatted as JSON. - * Time is formatted as a decimal of seconds. - * - * Example: - * "Decompression statistics" : {"Time" : 1.2, "Compression Ratio" : 0.5} - * - * \param start is the time_point when compression started. - * \param end is the time_point when compression ended. - * \param decompressed_size is the file size after decompression - * \param compressed_size is the compressed file size - */ -void print_decompression_statistics( - const std::chrono::high_resolution_clock::time_point start, - const std::chrono::high_resolution_clock::time_point end, - const size_t decompressed_size, - const size_t compressed_size) -{ - const auto duration = std::chrono::duration_cast(end - start); - const auto decompression_ratio = - static_cast(decompressed_size) / static_cast(compressed_size); - - ROSBAG2_COMPRESSION_LOG_DEBUG_STREAM( - "\"Decompression statistics\" : {" << - "\"Time\" : " << (duration.count() / 1000.0) << - ", \"Decompression Ratio\" : " << decompression_ratio << - "}"); -} -} // namespace - namespace rosbag2_compression { @@ -281,7 +55,7 @@ std::string ZstdDecompressor::decompress_uri(const std::string & uri) write_output_buffer(decompressed_buffer, decompressed_uri); const auto end = std::chrono::high_resolution_clock::now(); - print_decompression_statistics(start, end, decompression_result, compressed_buffer_length); + print_compression_statistics(start, end, decompression_result, compressed_buffer_length); return decompressed_uri; } @@ -318,7 +92,7 @@ void ZstdDecompressor::decompress_serialized_bag_message( message->serialized_data->buffer); const auto end = std::chrono::high_resolution_clock::now(); - print_decompression_statistics(start, end, decompression_result, compressed_buffer_length); + print_compression_statistics(start, end, decompression_result, compressed_buffer_length); } std::string ZstdDecompressor::get_decompression_identifier() const From a33c7c1ac1514b69f198ab76babbbb1240a9cab0 Mon Sep 17 00:00:00 2001 From: Marwan Taher Date: Tue, 21 Jul 2020 23:04:57 +0100 Subject: [PATCH 12/77] Adding db directory creation to rosbag2_cpp (#450) * added db directory creation to storage factory Signed-off-by: Marwan Taher * moved db directory creation to rosbag2_cpp Signed-off-by: Marwan Taher * rasing exception if dir already exists Signed-off-by: Marwan Taher * removed dir creation from record.py, added dir creation to sequential_compression_writer and refactored dir creation in sequential_writer Signed-off-by: Marwan Taher * fixed failing tests Signed-off-by: Marwan Taher * fixing review comments Signed-off-by: Marwan Taher * Apply suggestions from code review Co-authored-by: Karsten Knese Signed-off-by: Emerson Knapp --- ros2bag/ros2bag/verb/record.py | 3 --- .../sequential_compression_writer.cpp | 15 +++++++++++++++ .../src/rosbag2_cpp/writers/sequential_writer.cpp | 15 +++++++++++++++ .../test/rosbag2_cpp/test_sequential_writer.cpp | 9 +++++++++ .../test/rosbag2_tests/test_rosbag2_cpp_api.cpp | 2 -- 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/ros2bag/ros2bag/verb/record.py b/ros2bag/ros2bag/verb/record.py index 48936122ad..41a7a2f111 100644 --- a/ros2bag/ros2bag/verb/record.py +++ b/ros2bag/ros2bag/verb/record.py @@ -18,7 +18,6 @@ from rclpy.qos import InvalidQoSProfileException from ros2bag.api import convert_yaml_to_qos_profile -from ros2bag.api import create_bag_directory from ros2bag.api import print_error from ros2bag.verb import VerbExtension from ros2cli.node import NODE_NAME_PREFIX @@ -115,8 +114,6 @@ def main(self, *, args): # noqa: D102 except (InvalidQoSProfileException, ValueError) as e: return print_error(str(e)) - create_bag_directory(uri) - if args.all: # NOTE(hidmic): in merged install workspaces on Windows, Python entrypoint lookups # combined with constrained environments (as imposed by colcon test) diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp index 3008fd3271..9911390d47 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp @@ -112,6 +112,21 @@ void SequentialCompressionWriter::open( converter_ = std::make_unique(converter_options, converter_factory_); } + rcpputils::fs::path db_path(base_folder_); + if (db_path.is_directory()) { + std::stringstream error; + error << "Database directory already exists (" << db_path.string() << + "), can't overwrite existing database"; + throw std::runtime_error{error.str()}; + } + + bool dir_created = rcpputils::fs::create_directories(db_path); + if (!dir_created) { + std::stringstream error; + error << "Failed to create database directory (" << db_path.string() << ")."; + throw std::runtime_error{error.str()}; + } + const auto storage_uri = format_storage_uri(base_folder_, 0); storage_ = storage_factory_->open_read_write(storage_uri, storage_options.storage_id); if (!storage_) { diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index 78c8b1a4e3..406a5513d3 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -98,6 +98,21 @@ void SequentialWriter::open( converter_ = std::make_unique(converter_options, converter_factory_); } + rcpputils::fs::path db_path(base_folder_); + if (db_path.is_directory()) { + std::stringstream error; + error << "Database direcotory already exists (" << db_path.string() << + "), can't overwrite existing database"; + throw std::runtime_error{error.str()}; + } + + bool dir_created = rcpputils::fs::create_directories(db_path); + if (!dir_created) { + std::stringstream error; + error << "Failed to create database direcotory (" << db_path.string() << ")."; + throw std::runtime_error{error.str()}; + } + const auto storage_uri = format_storage_uri(base_folder_, 0); storage_ = storage_factory_->open_read_write(storage_uri, storage_options.storage_id); diff --git a/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp b/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp index c9631662ad..2553b4b135 100644 --- a/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp +++ b/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp @@ -47,6 +47,9 @@ class SequentialWriterTest : public Test storage_options_ = rosbag2_cpp::StorageOptions{}; storage_options_.uri = "uri"; + rcpputils::fs::path dir(storage_options_.uri); + rcpputils::fs::remove_all(dir); + ON_CALL(*storage_factory_, open_read_write(_, _)).WillByDefault( DoAll( Invoke( @@ -59,6 +62,12 @@ class SequentialWriterTest : public Test *storage_factory_, open_read_write(_, _)).Times(AtLeast(0)); } + ~SequentialWriterTest() + { + rcpputils::fs::path dir(storage_options_.uri); + rcpputils::fs::remove_all(dir); + } + std::unique_ptr> storage_factory_; std::shared_ptr> storage_; std::shared_ptr> converter_factory_; diff --git a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_cpp_api.cpp b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_cpp_api.cpp index bf2f69b514..1a73038a41 100644 --- a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_cpp_api.cpp +++ b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_cpp_api.cpp @@ -43,8 +43,6 @@ TEST(TestRosbag2CPPAPI, minimal_writer_example) auto rosbag_directory = rcpputils::fs::path("test_rosbag2_writer_api_bag"); // in case the bag was previously not cleaned up rcpputils::fs::remove_all(rosbag_directory); - // See https://github.com/ros2/rosbag2/issues/448 - rcpputils::fs::create_directories(rosbag_directory); rosbag2_cpp::StorageOptions storage_options; storage_options.uri = rosbag_directory.string(); From 58c56973fcb1a1409f7748bb49c2060d1237679b Mon Sep 17 00:00:00 2001 From: Dirk Thomas Date: Wed, 22 Jul 2020 17:18:05 -0700 Subject: [PATCH 13/77] move wait_for_shutdown() call out of the context manager (#466) Signed-off-by: Dirk Thomas --- ros2bag/test/test_record_qos_profiles.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ros2bag/test/test_record_qos_profiles.py b/ros2bag/test/test_record_qos_profiles.py index 97781a2baa..446b4e9354 100644 --- a/ros2bag/test/test_record_qos_profiles.py +++ b/ros2bag/test/test_record_qos_profiles.py @@ -16,6 +16,7 @@ from pathlib import Path import re import tempfile +import time import unittest @@ -73,7 +74,9 @@ def test_qos_simple(self): arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), '--output', output_path.as_posix()] with self.launch_bag_command(arguments=arguments) as bag_command: - bag_command.wait_for_shutdown(timeout=5) + time.sleep(3) + bag_command.wait_for_shutdown(timeout=5) + assert bag_command.terminated expected_string_regex = re.compile(ERROR_STRING) matches = expected_string_regex.search(bag_command.output) assert not matches, print('ros2bag CLI did not produce the expected output') @@ -84,7 +87,9 @@ def test_incomplete_qos_profile(self): arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), '--output', output_path.as_posix()] with self.launch_bag_command(arguments=arguments) as bag_command: - bag_command.wait_for_shutdown(timeout=5) + time.sleep(3) + bag_command.wait_for_shutdown(timeout=5) + assert bag_command.terminated expected_string_regex = re.compile(ERROR_STRING) matches = expected_string_regex.search(bag_command.output) assert not matches, print('ros2bag CLI did not produce the expected output') @@ -95,7 +100,9 @@ def test_incomplete_qos_duration(self): arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), '--output', output_path.as_posix()] with self.launch_bag_command(arguments=arguments) as bag_command: - bag_command.wait_for_shutdown(timeout=5) + time.sleep(3) + bag_command.wait_for_shutdown(timeout=5) + assert bag_command.terminated assert bag_command.exit_code != launch_testing.asserts.EXIT_OK expected_string_regex = re.compile(ERROR_STRING) matches = expected_string_regex.search(bag_command.output) @@ -107,7 +114,9 @@ def test_nonexistent_qos_profile(self): arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), '--output', output_path.as_posix()] with self.launch_bag_command(arguments=arguments) as bag_command: - bag_command.wait_for_shutdown(timeout=5) + time.sleep(3) + bag_command.wait_for_shutdown(timeout=5) + assert bag_command.terminated assert bag_command.exit_code != launch_testing.asserts.EXIT_OK expected_string_regex = re.compile( r"ros2 bag record: error: argument --qos-profile-overrides-path: can't open") From cc36fc8cb53e789277ba065d2821cfbb12c7a4cf Mon Sep 17 00:00:00 2001 From: Karsten Knese Date: Thu, 23 Jul 2020 10:42:01 -0700 Subject: [PATCH 14/77] introduce ros2 bag list (#468) * introduce ros2 bag list Signed-off-by: Karsten Knese * Apply suggestions from code review Signed-off-by: Emerson Knapp --- ros2bag/ros2bag/verb/list.py | 64 ++++++++++++++++++++++++++++++++++++ ros2bag/setup.py | 1 + 2 files changed, 65 insertions(+) create mode 100644 ros2bag/ros2bag/verb/list.py diff --git a/ros2bag/ros2bag/verb/list.py b/ros2bag/ros2bag/verb/list.py new file mode 100644 index 0000000000..deaba244f3 --- /dev/null +++ b/ros2bag/ros2bag/verb/list.py @@ -0,0 +1,64 @@ +# Copyright 2020 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from xml.dom import minidom + +from ament_index_python import get_resource +from ament_index_python import get_resources + +from ros2bag.verb import VerbExtension + + +class ListVerb(VerbExtension): + """Print information about available plugins to the screen.""" + + def add_arguments(self, parser, cli_name): # noqa: D102 + parser.add_argument( + 'plugin_type', help='lists available plugins (storage plugins or converter plugins', + choices=['storage', 'converter']) + parser.add_argument( + '--verbose', help='output verbose information about the available plugin', + action='store_true') + + def main(self, *, args): # noqa: D102 + # the following is the resource index which is created when installing a pluginlib xml file + if args.plugin_type == 'storage': + pluginlib_resource_index = 'rosbag2_storage__pluginlib__plugin' + else: + pluginlib_resource_index = 'rosbag2_cpp__pluginlib__plugin' + + resources = get_resources(pluginlib_resource_index) + if args.verbose: + print('available %s plugins are:' % args.plugin_type) + for resource in resources: + plugin_xml_file_paths, base_folder = get_resource(pluginlib_resource_index, resource) + for file_path in list(filter(None, plugin_xml_file_paths.split('\n'))): + abs_path = os.path.join(base_folder, file_path) + if not os.path.exists(abs_path): + return 'path does not exist: %s' % abs_path + + xmldoc = minidom.parse(abs_path) + class_item = xmldoc.getElementsByTagName('class')[0] + class_name = class_item.attributes['name'] + type_name = class_item.attributes['type'] + base_class_name = class_item.attributes['base_class_type'] + description = xmldoc.getElementsByTagName('description')[0] + + print('%s%s' % (('name: ' if args.verbose else ''), class_name.value)) + if args.verbose: + print('\t%s' % description.childNodes[0].data) + print('\ttype: %s' % type_name.value) + print('\tbase_class: %s' % base_class_name.value) diff --git a/ros2bag/setup.py b/ros2bag/setup.py index e01517d585..3d9475c4df 100644 --- a/ros2bag/setup.py +++ b/ros2bag/setup.py @@ -39,6 +39,7 @@ ], 'ros2bag.verb': [ 'info = ros2bag.verb.info:InfoVerb', + 'list = ros2bag.verb.list:ListVerb', 'play = ros2bag.verb.play:PlayVerb', 'record = ros2bag.verb.record:RecordVerb', ], From 72eb14d4f1c559e2852e2e967eca1f0ed07251cf Mon Sep 17 00:00:00 2001 From: Karsten Knese Date: Thu, 23 Jul 2020 13:49:18 -0700 Subject: [PATCH 15/77] introduce defaults for the C++ API (#452) * minimal c++ API test Signed-off-by: Karsten Knese * introduce defaults Signed-off-by: Karsten Knese * open overload for simple uri string Signed-off-by: Karsten Knese * set sqlite3 as a constant Signed-off-by: Karsten Knese --- rosbag2_cpp/include/rosbag2_cpp/reader.hpp | 25 +++++++++++++++++-- rosbag2_cpp/include/rosbag2_cpp/writer.hpp | 24 ++++++++++++++++-- rosbag2_cpp/package.xml | 2 ++ rosbag2_cpp/src/rosbag2_cpp/reader.cpp | 13 ++++++++++ .../rosbag2_cpp/readers/sequential_reader.cpp | 2 ++ rosbag2_cpp/src/rosbag2_cpp/writer.cpp | 12 +++++++++ .../rosbag2_tests/test_rosbag2_cpp_api.cpp | 17 +++---------- 7 files changed, 78 insertions(+), 17 deletions(-) diff --git a/rosbag2_cpp/include/rosbag2_cpp/reader.hpp b/rosbag2_cpp/include/rosbag2_cpp/reader.hpp index 8e449eb7fc..781ccb0f32 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/reader.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/reader.hpp @@ -21,6 +21,7 @@ #include #include "rosbag2_cpp/converter_options.hpp" +#include "rosbag2_cpp/readers/sequential_reader.hpp" #include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_cpp/visibility_control.hpp" @@ -50,10 +51,28 @@ class BaseReaderInterface; class ROSBAG2_CPP_PUBLIC Reader final { public: - explicit Reader(std::unique_ptr reader_impl); + explicit Reader( + std::unique_ptr reader_impl = + std::make_unique()); ~Reader(); + /** + * Opens an existing bagfile and prepare it for reading messages. + * The bagfile must exist. + * This must be called before any other function is used. + * + * \note This will open URI with the default storage options + * * using sqlite3 storage backend + * * using no converter options, storing messages with the incoming serialization format + * \sa rmw_get_serialization_format. + * For specifications, please see \sa open, which let's you specify + * more storage and converter options. + * + * \param storage_uri URI of the storage to open. + **/ + void open(const std::string & uri); + /** * Throws if file could not be opened. * This must be called before any other function is used. @@ -67,7 +86,9 @@ class ROSBAG2_CPP_PUBLIC Reader final * \param storage_options Options to configure the storage * \param converter_options Options for specifying the output data format */ - void open(const StorageOptions & storage_options, const ConverterOptions & converter_options); + void open( + const StorageOptions & storage_options, + const ConverterOptions & converter_options = ConverterOptions()); /** * Ask whether the underlying bagfile contains at least one more message. diff --git a/rosbag2_cpp/include/rosbag2_cpp/writer.hpp b/rosbag2_cpp/include/rosbag2_cpp/writer.hpp index b81d2a21cd..ff06272f0b 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/writer.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/writer.hpp @@ -24,6 +24,7 @@ #include "rosbag2_cpp/converter_options.hpp" #include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_cpp/visibility_control.hpp" +#include "rosbag2_cpp/writers/sequential_writer.hpp" #include "rosbag2_storage/serialized_bag_message.hpp" #include "rosbag2_storage/topic_metadata.hpp" @@ -50,10 +51,27 @@ class BaseWriterInterface; class ROSBAG2_CPP_PUBLIC Writer final { public: - explicit Writer(std::unique_ptr writer_impl); + explicit Writer( + std::unique_ptr writer_impl = + std::make_unique()); ~Writer(); + /** + * Opens a new bagfile and prepare it for writing messages. The bagfile must not exist. + * This must be called before any other function is used. + * + * \note This will open URI with the default storage options + * * using sqlite3 storage backend + * * using no converter options, storing messages with the incoming serialization format + * \sa rmw_get_serialization_format. + * For specifications, please see \sa open, which let's you specify + * more storage and converter options. + * + * \param storage_uri URI of the storage to open. + **/ + void open(const std::string & uri); + /** * Opens a new bagfile and prepare it for writing messages. The bagfile must not exist. * This must be called before any other function is used. @@ -61,7 +79,9 @@ class ROSBAG2_CPP_PUBLIC Writer final * \param storage_options Options to configure the storage * \param converter_options options to define in which format incoming messages are stored **/ - void open(const StorageOptions & storage_options, const ConverterOptions & converter_options); + void open( + const StorageOptions & storage_options, + const ConverterOptions & converter_options = ConverterOptions()); /** * Create a new topic in the underlying storage. Needs to be called for every topic used within diff --git a/rosbag2_cpp/package.xml b/rosbag2_cpp/package.xml index 9753fc024f..7e848b786a 100644 --- a/rosbag2_cpp/package.xml +++ b/rosbag2_cpp/package.xml @@ -21,6 +21,8 @@ rosidl_typesupport_introspection_cpp shared_queues_vendor + rosbag2_storage_default_plugins + ament_cmake_gmock ament_lint_auto ament_lint_common diff --git a/rosbag2_cpp/src/rosbag2_cpp/reader.cpp b/rosbag2_cpp/src/rosbag2_cpp/reader.cpp index 8d648684b6..08cfd1c3bd 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/reader.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/reader.cpp @@ -16,6 +16,7 @@ #include "rosbag2_cpp/reader.hpp" #include +#include #include #include @@ -34,6 +35,18 @@ Reader::~Reader() reader_impl_->reset(); } +void Reader::open(const std::string & uri) +{ + rosbag2_cpp::StorageOptions storage_options; + storage_options.uri = uri; + storage_options.storage_id = "sqlite3"; + storage_options.max_bagfile_size = 0; // default + storage_options.max_cache_size = 0; // default + + rosbag2_cpp::ConverterOptions converter_options{}; + return open(storage_options, converter_options); +} + void Reader::open( const StorageOptions & storage_options, const ConverterOptions & converter_options) diff --git a/rosbag2_cpp/src/rosbag2_cpp/readers/sequential_reader.cpp b/rosbag2_cpp/src/rosbag2_cpp/readers/sequential_reader.cpp index 2f8ddc2c23..35c09c52f7 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/readers/sequential_reader.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/readers/sequential_reader.cpp @@ -230,6 +230,8 @@ void SequentialReader::check_converter_serialization_format( const std::string & converter_serialization_format, const std::string & storage_serialization_format) { + if (converter_serialization_format.empty()) {return;} + if (converter_serialization_format != storage_serialization_format) { converter_ = std::make_unique( storage_serialization_format, diff --git a/rosbag2_cpp/src/rosbag2_cpp/writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writer.cpp index 34a46d5d42..9c472d7dd3 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writer.cpp @@ -31,6 +31,8 @@ namespace rosbag2_cpp { +static constexpr char const * kDefaultStorageID = "sqlite3"; + Writer::Writer(std::unique_ptr writer_impl) : writer_impl_(std::move(writer_impl)) {} @@ -40,6 +42,16 @@ Writer::~Writer() writer_impl_.reset(); } +void Writer::open(const std::string & uri) +{ + rosbag2_cpp::StorageOptions storage_options; + storage_options.uri = uri; + storage_options.storage_id = kDefaultStorageID; + + rosbag2_cpp::ConverterOptions converter_options{}; + return open(storage_options, converter_options); +} + void Writer::open( const StorageOptions & storage_options, const ConverterOptions & converter_options) { diff --git a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_cpp_api.cpp b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_cpp_api.cpp index 1a73038a41..30b2815157 100644 --- a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_cpp_api.cpp +++ b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_cpp_api.cpp @@ -44,18 +44,9 @@ TEST(TestRosbag2CPPAPI, minimal_writer_example) // in case the bag was previously not cleaned up rcpputils::fs::remove_all(rosbag_directory); - rosbag2_cpp::StorageOptions storage_options; - storage_options.uri = rosbag_directory.string(); - storage_options.storage_id = "sqlite3"; - storage_options.max_bagfile_size = 0; // default - storage_options.max_cache_size = 0; // default - rosbag2_cpp::ConverterOptions converter_options; - converter_options.input_serialization_format = "cdr"; - converter_options.output_serialization_format = "cdr"; - { - rosbag2_cpp::Writer writer(std::make_unique()); - writer.open(storage_options, converter_options); + rosbag2_cpp::Writer writer; + writer.open(rosbag_directory.string()); auto bag_message = std::make_shared(); auto ret = rcutils_system_time_now(&bag_message->time_stamp); @@ -78,8 +69,8 @@ TEST(TestRosbag2CPPAPI, minimal_writer_example) } { - rosbag2_cpp::Reader reader(std::make_unique()); - reader.open(storage_options, converter_options); + rosbag2_cpp::Reader reader; + reader.open(rosbag_directory.string()); while (reader.has_next()) { auto bag_message = reader.read_next(); From fc4dd59c615289dfb2e20617b99563a4fffe4806 Mon Sep 17 00:00:00 2001 From: Dirk Thomas Date: Thu, 23 Jul 2020 14:05:57 -0700 Subject: [PATCH 16/77] add wait for closed file handles on Windows (#470) Signed-off-by: Dirk Thomas --- ros2bag/test/test_record_qos_profiles.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ros2bag/test/test_record_qos_profiles.py b/ros2bag/test/test_record_qos_profiles.py index 446b4e9354..4bf1a228d4 100644 --- a/ros2bag/test/test_record_qos_profiles.py +++ b/ros2bag/test/test_record_qos_profiles.py @@ -15,6 +15,7 @@ import contextlib from pathlib import Path import re +import sys import tempfile import time @@ -66,7 +67,14 @@ def launch_bag_command(self, arguments, **kwargs): @classmethod def tearDownClass(cls): - cls.tmpdir.cleanup() + try: + cls.tmpdir.cleanup() + except PermissionError: + if sys.platform != 'win32': + raise + # HACK to allow Windows to close pending file handles + time.sleep(3) + cls.tmpdir.cleanup() def test_qos_simple(self): profile_path = PROFILE_PATH / 'qos_profile.yaml' From 71948dd505b51f10e47b3082b95683236796a399 Mon Sep 17 00:00:00 2001 From: Dirk Thomas Date: Fri, 24 Jul 2020 17:45:30 -0700 Subject: [PATCH 17/77] catch parent exception (#472) Signed-off-by: Dirk Thomas --- ros2bag/test/test_record_qos_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros2bag/test/test_record_qos_profiles.py b/ros2bag/test/test_record_qos_profiles.py index 4bf1a228d4..6eb632a013 100644 --- a/ros2bag/test/test_record_qos_profiles.py +++ b/ros2bag/test/test_record_qos_profiles.py @@ -69,7 +69,7 @@ def launch_bag_command(self, arguments, **kwargs): def tearDownClass(cls): try: cls.tmpdir.cleanup() - except PermissionError: + except OSError: if sys.platform != 'win32': raise # HACK to allow Windows to close pending file handles From 889424846987ced9fe925c0440778f76dc57144f Mon Sep 17 00:00:00 2001 From: Jacob Perron Date: Tue, 28 Jul 2020 15:08:11 -0700 Subject: [PATCH 18/77] Fix typo in error message (#475) Signed-off-by: Jacob Perron --- rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index 406a5513d3..55e1a7c3d8 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -101,7 +101,7 @@ void SequentialWriter::open( rcpputils::fs::path db_path(base_folder_); if (db_path.is_directory()) { std::stringstream error; - error << "Database direcotory already exists (" << db_path.string() << + error << "Database directory already exists (" << db_path.string() << "), can't overwrite existing database"; throw std::runtime_error{error.str()}; } @@ -109,7 +109,7 @@ void SequentialWriter::open( bool dir_created = rcpputils::fs::create_directories(db_path); if (!dir_created) { std::stringstream error; - error << "Failed to create database direcotory (" << db_path.string() << ")."; + error << "Failed to create database directory (" << db_path.string() << ")."; throw std::runtime_error{error.str()}; } From a1cd7788135ecd93303222cfd0d9a4928fcc0528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C4=85browski?= Date: Wed, 5 Aug 2020 23:30:58 +0200 Subject: [PATCH 19/77] performance testing packages (#442) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Init commit Signed-off-by: Adam Dabrowski * Readme and package dependencies cleaned. Signed-off-by: Adam Dabrowski * Benchmark launch file Signed-off-by: Adam Dabrowski * Dummy warmup publishers and raport gen barebone Signed-off-by: Adam Dabrowski * added todo for system usage Signed-off-by: Adam Dabrowski * Added I/O benchmarking section Signed-off-by: Adam Dabrowski * config file wip Signed-off-by: Adam Dabrowski * Pc2 worker, bench and raport dirs Signed-off-by: Adam Dabrowski * added benchmark sample config Signed-off-by: Adam Dabrowski * Moved scripts to a dir Signed-off-by: Adam Dabrowski * Simple system monitor Signed-off-by: Adam Dabrowski * Updated readme Signed-off-by: Adam Dabrowski * added notice Signed-off-by: Adam Dabrowski * Update readme.md Signed-off-by: Adam Dabrowski * Mkdir fix Signed-off-by: Adam Dabrowski * Update readme.md Signed-off-by: Adam Dabrowski * plotting WIP script plus an example file with logs Signed-off-by: Adam Dabrowski * Raport gen, monitor with iotop Signed-off-by: Adam Dabrowski * Disk bw fix. Instances in raport fix. Signed-off-by: Adam Dabrowski * Typo fix Signed-off-by: Adam Dabrowski * Fix in memory time axis Signed-off-by: Adam Dabrowski * Another axis fix Signed-off-by: Adam Dabrowski * sqlite.conf for patched version of rosbag2 Signed-off-by: Adam Dabrowski * Improved plotting (added message count, count start includes fractions of second, color etc) Signed-off-by: Adam Dabrowski * data examples Signed-off-by: Adam Dabrowski * large examples over 10000 messages Signed-off-by: Adam Dabrowski * Byte array worker, instantiation improvements Signed-off-by: Adam Dabrowski * total captured in raport Signed-off-by: Adam Dabrowski * voyager wip Signed-off-by: Adam Dabrowski * workers refactored Signed-off-by: Adam Dabrowski * launchfile refactor Signed-off-by: Adam Dabrowski * More improvement and voyager case Signed-off-by: Adam Dabrowski * removed some commented out lines Signed-off-by: Adam Dabrowski * Update readme.md Signed-off-by: Adam Dabrowski * moved directories of performance packages into rosbag2_performance folder Signed-off-by: Adam Dabrowski * commented out 1000x1000 case in voyager Signed-off-by: Adam Dabrowski * Update readme.md Signed-off-by: Adam Dabrowski * added html template to exported files Signed-off-by: Adam Dabrowski * First version of no-transport benchmarking - Scalable with number of producer threads - Uses queues to simulate message loss in callback queues. Unlike some other implementations allows to count loss messages. - ASCII visualisation of work (better than it sounds). Signed-off-by: Adam Dabrowski * Improved parameters and added scripting to run batches. These were used to acquire first results Signed-off-by: Adam Dabrowski * Update readme.md Signed-off-by: Adam Dabrowski * Changes towards resolving WIP status - Added copyright and license to files - Split and updated READMEs - Refactored writer_benchmarking, splitting classes into separate files - added ctrl-c handling to writer_benchmarking, in script and in producers Signed-off-by: Adam Dabrowski * lint changes Signed-off-by: Adam Dabrowski * Removed large files from repo Signed-off-by: Adam Dabrowski * PEP8 issues refactor, rosbag2_benchmarking package info updated. Signed-off-by: Piotr Jaroszek * fixed an issue with SQLite crash with cache>1 due to released message memory Signed-off-by: Adam Dabrowski * uncrustify Signed-off-by: Adam Dabrowski * Introduced BUILD_PERFORMANCE flag to control the package build. Use colcon build --cmake-args -DBUILD_PERFORMANCE=1. Signed-off-by: Adam Dabrowski * linter changes and missingcopyrights Signed-off-by: Adam Dabrowski * Update README.md Signed-off-by: Adam Dabrowski * Removed unnecessary wording Signed-off-by: Adam Dabrowski * Fixed bad spelling raport->report Signed-off-by: Adam Dabrowski * Report stats from time 0 Signed-off-by: Piotr Jaroszek * minor cleaning of writer_benchmarking package Signed-off-by: Adam Dabrowski * removed other performance packages from PR. These will be added later and reviewed separately Signed-off-by: Adam Dabrowski * removed accidental change Signed-off-by: Adam Dabrowski * Applied review changes Signed-off-by: Adam Dabrowski * enforced shared pointer in queue api, added whitespaces Signed-off-by: Adam Dabrowski * mVar -> _var, camelCase -> snake_case Signed-off-by: Adam Dabrowski * writing results to a csv file Signed-off-by: Adam Dabrowski * style touchups and linters Signed-off-by: Karsten Knese * A fix in fio command, thanks to Karsten Knese Signed-off-by: Adam Dabrowski * Added spinning (in thread) to handle parameters Signed-off-by: Adam Dabrowski * Typo fix Signed-off-by: Adam Dabrowski * Applied review remarks - node name changed to reflect package name - value now in main and constructor takes name argument - benchmark folder management adjusted to changes after rebase - benchmark now remaps node (adding "_batch"). - removed unnecessary find on rosbag2_transport Signed-off-by: Adam Dabrowski * portable auto type to remove Windows warning. Thanks KK Signed-off-by: Adam Dabrowski Co-authored-by: Piotr Jaroszek Co-authored-by: Piotr Jaroszek Co-authored-by: Adam Dąbrowski <1961316-adamdbrw@users.noreply.gitlab.com> Co-authored-by: Karsten Knese Signed-off-by: Emerson Knapp --- ros2bag/pytest.ini | 2 - .../CMakeLists.txt | 44 ++++ .../README.md | 53 +++++ .../byte_producer.hpp | 79 +++++++ .../message_queue.hpp | 97 ++++++++ .../writer_benchmark.hpp | 52 +++++ .../package.xml | 23 ++ .../scripts/benchmark.sh | 55 +++++ .../src/main.cpp | 35 +++ .../src/writer_benchmark.cpp | 216 ++++++++++++++++++ 10 files changed, 654 insertions(+), 2 deletions(-) delete mode 100644 ros2bag/pytest.ini create mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt create mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/README.md create mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/byte_producer.hpp create mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/message_queue.hpp create mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp create mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml create mode 100755 rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh create mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/src/main.cpp create mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp diff --git a/ros2bag/pytest.ini b/ros2bag/pytest.ini deleted file mode 100644 index fe55d2ed64..0000000000 --- a/ros2bag/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -junit_family=xunit2 diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt b/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt new file mode 100644 index 0000000000..1f831ae4dc --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt @@ -0,0 +1,44 @@ +cmake_minimum_required(VERSION 3.5) +project(rosbag2_performance_writer_benchmarking) + +# Default to C++14 +if(NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 14) +endif() + +if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-Wall -Wextra -Wpedantic) +endif() + +find_package(ament_cmake REQUIRED) + +if(BUILD_ROSBAG2_BENCHMARKS) + find_package(rclcpp REQUIRED) + find_package(rcutils REQUIRED) + find_package(rosbag2_cpp REQUIRED) + find_package(rosbag2_storage REQUIRED) + find_package(rmw REQUIRED) + find_package(std_msgs REQUIRED) + + add_executable(writer_benchmark src/writer_benchmark.cpp src/main.cpp) + target_include_directories(writer_benchmark + PUBLIC + $ + $ + ) + ament_target_dependencies(writer_benchmark + rclcpp + std_msgs + rosbag2_cpp + ) + + install(TARGETS writer_benchmark + DESTINATION lib/${PROJECT_NAME}) + + if(BUILD_TESTING) + find_package(ament_lint_auto REQUIRED) + ament_lint_auto_find_test_dependencies() + endif() +endif() + +ament_package() diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/README.md b/rosbag2_performance/rosbag2_performance_writer_benchmarking/README.md new file mode 100644 index 0000000000..ef820147fb --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/README.md @@ -0,0 +1,53 @@ +# Rosbag2 writer benchmarking + +The primary package to test transport-less performance of the rosbag2 writer and storage. +Enables parametrized batch execution of benchmarks and tries to replicate the flow to capture message loss in queues. + +## How it works + +Use `scripts/benchmark.sh` to run an entire set of benchmarks. +These are currently aimed at several 100Mb/s scenarios. +Parameters are easy to change inside the script. + +By default, results will be written to `/tmp/rosbag2_test/[current_date]`. +The summary of benchmarks goes into `results.csv` file, which includes rows of execution parameters and results. +Benchmarks also produce execution logs in a series of sub-directories in `size[size]_inst[inst]_cache[cache]/` format. + +Database (bag) files are removed after recording to avoid filling up the disk. +To modify this behavior, modify the benchmark.sh script. + +## Building + +To build the package in the rosbag2 build process, make sure to turn `BUILD_ROSBAG2_BENCHMARKS` flag on (e.g. `colcon build --cmake-args -DBUILD_ROSBAG2_BENCHMARKS=1`) + +If you already built rosbag2, you can use `packages-select` option to build benchmarks. +Example: `colcon build --packages-select rosbag2_performance_writer_benchmarking --cmake-args -DBUILD_ROSBAG2_BENCHMARKS=1`. + +## General knowledge: I/O benchmarking + +#### Background: benchmarking disk writes on your system + +It might be useful to first understand what limitation your disk poses to the throughput of data recording. +Performance of bag write can't be higher over extended period of time (you can only use as much memory). + +**Using dd command** + +`dd if=/dev/zero of=/tmp/output conv=fdatasync bs=384k count=1k; rm -f /tmp/output` + +This method is not great for benchmarking the disk but an easy way to start since it requires no dependencies. +This will write zeros to the /tmp/output file with block size 384k, 1000 blocks, ends when write finishes. +Make sure to benchmark the disk which your bags will be written to (check your mount points and change “/tmp/output” to another path if needed). +Note: this depends on parameters used and whatever else is running on your system but can give you a ballpark figure when ran several times. + +**Using fio** + +For more sophisticated & accurate benchmarks, see the `fio` command. An example for big data blocks is: `fio --name TEST --eta-newline=5s --filename=fio-tempfile.dat --rw=write --size=500m --io_size=10g --blocksize=1024k --ioengine=libaio --fsync=10000 --iodepth=32 --direct=1 --numjobs=1 --runtime=60 --group_reporting`. + +#### Profiling bags I/O with tools + +Tools that can help in I/O profiling: `sudo apt-get install iotop ioping sysstat` +* `iotop` works similar as `top` command, but shows disk reads, writes, swaps and I/O %. Can be used at higher frequency in batch mode with specified process to deliver data that can be plotted. + * Example use: `sudo iotop -h -d 0.1 -t -b -o -p ` after running the bag. +* `ioping` can be used to check latency of requests to device +* `strace` can help determine syscalls associated with the bottleneck. + * Example use: `strace -c ros2 bag record /image --max-cache-size 10 -o ./tmp`. You will see a report after finishing recording with Ctrl-C. diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/byte_producer.hpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/byte_producer.hpp new file mode 100644 index 0000000000..9ade8ca900 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/byte_producer.hpp @@ -0,0 +1,79 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__BYTE_PRODUCER_HPP_ +#define ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__BYTE_PRODUCER_HPP_ + +#include +#include +#include +#include + +#include "rclcpp/utilities.hpp" + +#include "std_msgs/msg/byte_multi_array.hpp" + +#include "rosbag2_performance_writer_benchmarking/message_queue.hpp" + +struct ProducerConfig +{ + unsigned int frequency; + unsigned int max_count; + unsigned int message_size; +}; + +inline auto generate_random_message(const ProducerConfig & config) +{ + // Reuses the same random message + auto message = std::make_shared(); + + message->data.reserve(config.message_size); + for (auto i = 0u; i < config.message_size; ++i) { + message->data.emplace_back(std::rand() % 255); + } + + return message; +} + +class ByteProducer +{ +public: + ByteProducer(const ProducerConfig & config, std::shared_ptr queue) + : configuration_(config), + queue_(queue), + sleep_time_(configuration_.frequency == 0 ? 1 : 1000 / configuration_.frequency), + message_(generate_random_message(configuration_)) + {} + + void run() + { + for (auto i = 0u; i < configuration_.max_count; ++i) { + if (!rclcpp::ok()) { + break; + } + queue_->push(message_); + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time_)); + } + queue_->set_complete(); + } + +private: + ProducerConfig configuration_; + std::shared_ptr queue_; + unsigned int sleep_time_; // in milliseconds + // for simplification, this pointer will be reused + std::shared_ptr message_; +}; + +#endif // ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__BYTE_PRODUCER_HPP_ diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/message_queue.hpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/message_queue.hpp new file mode 100644 index 0000000000..87ed7b51c2 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/message_queue.hpp @@ -0,0 +1,97 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__MESSAGE_QUEUE_HPP_ +#define ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__MESSAGE_QUEUE_HPP_ + +#include +#include +#include +#include +#include +#include +#include + +#include "std_msgs/msg/byte_multi_array.hpp" + +template +class MessageQueue +{ +public: + MessageQueue(int max_size, std::string topic_name) + : max_size_(max_size), + topic_name_(std::move(topic_name)), + unsuccessful_insert_count_(0) + {} + + void push(std::shared_ptr elem) + { + std::lock_guard lock(mutex_); + if (queue_.size() > max_size_) { // We skip the element and consider it "lost" + ++unsuccessful_insert_count_; + std::cerr << "X" << std::flush; + return; + } + queue_.push(elem); + } + + bool is_complete() const + { + return complete_; + } + + void set_complete() + { + complete_ = true; + } + + bool is_empty() + { + std::lock_guard lock(mutex_); + return queue_.empty(); + } + + std::shared_ptr pop_and_return() + { + std::lock_guard lock(mutex_); + if (queue_.empty()) { + throw std::out_of_range("Queue is empty, cannot pop. Check if empty first"); + } + auto elem = queue_.front(); + queue_.pop(); + return elem; + } + + unsigned int get_missed_elements_count() const + { + return unsuccessful_insert_count_; + } + + std::string topic_name() const + { + return topic_name_; + } + +private: + bool complete_ = false; + unsigned int max_size_; + std::string topic_name_; + std::atomic unsuccessful_insert_count_; + std::queue> queue_; + mutable std::mutex mutex_; +}; + +typedef MessageQueue ByteMessageQueue; + +#endif // ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__MESSAGE_QUEUE_HPP_ diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp new file mode 100644 index 0000000000..5418323557 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp @@ -0,0 +1,52 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__WRITER_BENCHMARK_HPP_ +#define ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__WRITER_BENCHMARK_HPP_ + +#include +#include +#include +#include + +#include "rclcpp/rclcpp.hpp" +#include "rosbag2_cpp/writers/sequential_writer.hpp" + +#include "rosbag2_performance_writer_benchmarking/message_queue.hpp" +#include "rosbag2_performance_writer_benchmarking/byte_producer.hpp" + +class WriterBenchmark : public rclcpp::Node +{ +public: + explicit WriterBenchmark(const std::string & name); + void start_benchmark(); + +private: + void create_producers(const ProducerConfig & config); + void create_writer(); + void start_producers(); + void write_results(const unsigned int & total_missed) const; + + ProducerConfig config_; + unsigned int max_cache_size_; + unsigned int instances_; + std::string db_folder_; + std::string results_file_; + std::shared_ptr writer_; + std::vector producer_threads_; + std::vector> producers_; + std::vector> queues_; +}; + +#endif // ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__WRITER_BENCHMARK_HPP_ diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml b/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml new file mode 100644 index 0000000000..7970159e95 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml @@ -0,0 +1,23 @@ + + + + rosbag2_performance_writer_benchmarking + 0.0.2 + Code to benchmark the part of rosbag2 which starts after the callback is received + Adam Dabrowski + Apache License 2.0 + + ament_cmake + + rclcpp + rosbag2_cpp + rmw + std_msgs + + ament_lint_auto + ament_lint_common + + + ament_cmake + + diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh new file mode 100755 index 0000000000..7ff116f47b --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh @@ -0,0 +1,55 @@ +# A very rough script to run batches of experiments. Missing checks and command line parametrization + +trap ctrlc SIGINT +ctrlc_sent=0 + +function ctrlc() +{ + let ctrlc_sent=1 +} + +test_dir=/tmp/rosbag2_test/$(date +%Y%m%d_%H%M%S) +mkdir -p ${test_dir} +echo "${test_dir} created" +db_path=${test_dir}/bag +rm -fr ${db_path} +summary_file=${test_dir}/results.csv + +freq=100; #Hz + +for cache in 0 10 100 1000 +do + for sz in 1000 10000 100000 1000000 + do + for inst in 1 10 100 1000 + do + let total=$inst*$sz + if [[ $total -ne 1000000 ]]; then + #echo "skipping the case of ${inst} instances and size ${sz}" + continue + fi + echo "processing case of ${inst} instances and size ${sz} with cache ${cache}" + for try in 1 2 3 4 5 + do + outdir=${test_dir}/size${sz}_inst${inst}_cache${cache} + mkdir -p ${outdir} + outfile=${outdir}/${try}.log + echo "Results will be written to file: ${outfile}" + ros2 run rosbag2_performance_writer_benchmarking writer_benchmark --ros-args \ + -p frequency:=${freq} \ + -p size:=${sz} \ + -p instances:=${inst} \ + -p max_cache_size:=${cache} \ + -p db_folder:=${db_path} \ + -p results_file:=${summary_file} \ + --ros-args -r __node:=rosbag2_performance_writer_benchmarking_node_batch \ + 2> ${outfile} + rm -fr ${db_path} + if [[ $ctrlc_sent -eq 1 ]]; then + echo -e "\e[31mQuitting prematurely due to Ctrl-C - some results aren't saved and some won't be reliable\e[0m]" + exit + fi + done + done + done +done diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/main.cpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/main.cpp new file mode 100644 index 0000000000..9855d5a119 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/main.cpp @@ -0,0 +1,35 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "rclcpp/executors/single_threaded_executor.hpp" +#include "rosbag2_performance_writer_benchmarking/writer_benchmark.hpp" + +// TODO(adamdbrw) the benchmark being ROS node is not necessary, only used for logging +// and parameters. Otherwise ROS dependencies are calls to rcl and rmw. +int main(int argc, char * argv[]) +{ + rclcpp::init(argc, argv); + auto bench = std::make_shared("rosbag2_performance_writer_benchmarking_node"); + rclcpp::executors::SingleThreadedExecutor executor; + executor.add_node(bench); + + // The benchmark has its own control loop but uses spinning for parameters + std::thread spin_thread([&executor]() {executor.spin();}); + bench->start_benchmark(); + RCLCPP_INFO(bench->get_logger(), "Benchmark terminated"); + rclcpp::shutdown(); + spin_thread.join(); + return 0; +} diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp new file mode 100644 index 0000000000..21b12ac93d --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp @@ -0,0 +1,216 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "rmw/rmw.h" +#include "rosbag2_cpp/storage_options.hpp" +#include "rosbag2_storage/serialized_bag_message.hpp" +#include "std_msgs/msg/byte_multi_array.hpp" + +#include "rosbag2_performance_writer_benchmarking/writer_benchmark.hpp" + +using namespace std::chrono_literals; + +WriterBenchmark::WriterBenchmark(const std::string & name) +: rclcpp::Node(name) +{ + const std::string default_bag_folder("/tmp/rosbag2_test"); + this->declare_parameter("frequency", 100); + this->declare_parameter("max_count", 1000); + this->declare_parameter("size", 1000000); + this->declare_parameter("instances", 1); + this->declare_parameter("max_cache_size", 1); + this->declare_parameter("db_folder", default_bag_folder); + this->declare_parameter("results_file", default_bag_folder + "/results.csv"); + + this->get_parameter("frequency", config_.frequency); + if (config_.frequency == 0) { + RCLCPP_ERROR(this->get_logger(), "Frequency can't be 0. Exiting."); + rclcpp::shutdown(nullptr, "frequency error"); + return; + } + + this->get_parameter("max_cache_size", max_cache_size_); + this->get_parameter("db_folder", db_folder_); + this->get_parameter("results_file", results_file_); + this->get_parameter("max_count", config_.max_count); + this->get_parameter("size", config_.message_size); + this->get_parameter("instances", instances_); + + create_producers(config_); + create_writer(); +} + +void WriterBenchmark::start_benchmark() +{ + RCLCPP_INFO(get_logger(), "Starting. A dot is a write, an X is a miss"); + start_producers(); + while (rclcpp::ok()) { + int count = 0; + unsigned int complete_count = 0; + + // TODO(adamdbrw) Performance can be improved. Use conditional variables + for (auto & queue : queues_) { + if (queue->is_complete() && queue->is_empty()) { + ++complete_count; + } + + if (!queue->is_empty()) { // behave as if we received the message. + auto message = std::make_shared(); + auto serialized_data = std::make_shared(); + + // The pointer memory is owned by the producer until past the termination of the while loop. + // Note this ownership model should be changed if we want to generate messages on the fly + auto byte_ma_message = queue->pop_and_return(); + + serialized_data->buffer = reinterpret_cast(byte_ma_message->data.data()); + serialized_data->buffer_length = byte_ma_message->data.size(); + serialized_data->buffer_capacity = byte_ma_message->data.size(); + + message->serialized_data = serialized_data; + + rcutils_time_point_value_t time_stamp; + int error = rcutils_system_time_now(&time_stamp); + if (error != RCUTILS_RET_OK) { + RCLCPP_ERROR_STREAM( + get_logger(), "Error getting current time. Error:" << + rcutils_get_error_string().str); + } + message->time_stamp = time_stamp; + message->topic_name = queue->topic_name(); + + try { + writer_->write(message); + } catch (const std::runtime_error & e) { + RCLCPP_ERROR_STREAM(get_logger(), "Failed to record: " << e.what()); + } + std::cerr << "." << std::flush; + ++count; + } + } + if (complete_count == queues_.size()) { + break; + } + + std::this_thread::sleep_for(1ms); + } + + for (auto & prod_thread : producer_threads_) { + prod_thread.join(); + } + + unsigned int total_missed_messages = 0; + for (const auto & queue : queues_) { + total_missed_messages += queue->get_missed_elements_count(); + } + write_results(total_missed_messages); +} + +void WriterBenchmark::write_results(const unsigned int & total_missed) const +{ + auto total_messages_sent = config_.max_count * producers_.size(); + float percentage_recorded = 100.0f - static_cast(total_missed * 100.0f) / + total_messages_sent; + + RCLCPP_INFO(get_logger(), "\nWriterBenchmark terminating"); + RCLCPP_INFO_STREAM(get_logger(), "Total missed messages: " << total_missed); + RCLCPP_INFO_STREAM( + get_logger(), "Percentage of all message that was successfully recorded: " << + percentage_recorded); + + bool new_file = false; + { // test if file exists - we want to write a csv header after creation if not + // use std::filesystem when switching to C++17 + std::ifstream test_existence(results_file_); + if (!test_existence) { + new_file = true; + } + } + + // append, we want to accumulate results from multiple runs + std::ofstream output_file(results_file_, std::ios_base::app); + if (!output_file.is_open()) { + RCLCPP_ERROR_STREAM(get_logger(), "Could not open file " << results_file_); + return; + } + + if (new_file) { + output_file << "instances frequency message_size cache_size total_messages_sent "; + output_file << "total_messages_missed percentage_recorded\n"; + } + + // configuration of the test. TODO(adamdbrw) wrap into a dict and define << operator. + output_file << instances_ << " "; + output_file << config_.frequency << " "; + output_file << config_.message_size << " "; + output_file << max_cache_size_ << " "; + output_file << total_messages_sent << " "; + + // results of the test. Use std::setprecision if preferred + output_file << total_missed << " "; + output_file << percentage_recorded << std::endl; +} + +void WriterBenchmark::create_producers(const ProducerConfig & config) +{ + RCLCPP_INFO_STREAM( + get_logger(), "\nWriterBenchmark: creating " << instances_ << + " message producers with frequency " << config.frequency << + " and message size in bytes " << config.message_size << + ". Cache is " << max_cache_size_ << ". Each will send " << config.max_count << + " messages before terminating"); + const unsigned int queue_max_size = 10; + for (unsigned int i = 0; i < instances_; ++i) { + std::string topic = "/writer_benchmark/producer " + std::to_string(i); + auto queue = std::make_shared(queue_max_size, topic); + queues_.push_back(queue); + producers_.push_back(std::make_unique(config, queue)); + } +} + +// TODO(adamdbrw) extend to other writers - based on parametrization +// Also, add an option to configure compression +void WriterBenchmark::create_writer() +{ + writer_ = std::make_shared(); + rosbag2_cpp::StorageOptions storage_options{}; + storage_options.uri = db_folder_; + storage_options.storage_id = "sqlite3"; + storage_options.max_bagfile_size = 0; + storage_options.max_cache_size = max_cache_size_; + + // TODO(adamdbrw) generalize if converters are to be included in benchmarks + std::string serialization_format = rmw_get_serialization_format(); + writer_->open(storage_options, {serialization_format, serialization_format}); + + for (const auto & queue : queues_) { + rosbag2_storage::TopicMetadata topic; + topic.name = queue->topic_name(); + // TODO(adamdbrw) - replace with something more general if needed + topic.type = "std_msgs::msgs::ByteMultiArray"; + topic.serialization_format = serialization_format; + writer_->create_topic(topic); + } +} + +void WriterBenchmark::start_producers() +{ + for (auto & producer : producers_) { + producer_threads_.push_back(std::thread(&ByteProducer::run, producer.get())); + } +} From f5eaf4e7266b419228afd98f5f7770fbda76e007 Mon Sep 17 00:00:00 2001 From: Chris Lalancette Date: Thu, 6 Aug 2020 12:55:33 -0400 Subject: [PATCH 20/77] Add pytest.ini back to ros2bag. (#492) Signed-off-by: Chris Lalancette --- ros2bag/pytest.ini | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 ros2bag/pytest.ini diff --git a/ros2bag/pytest.ini b/ros2bag/pytest.ini new file mode 100644 index 0000000000..fe55d2ed64 --- /dev/null +++ b/ros2bag/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +junit_family=xunit2 From 8290d7b750086f2cf21957171f6341f1599cdc51 Mon Sep 17 00:00:00 2001 From: Mabel Zhang Date: Thu, 13 Aug 2020 17:19:27 -0400 Subject: [PATCH 21/77] rosbag2_py reader and writer (#308) * Initialize new package rosbag2_py Signed-off-by: Jacob Perron (cherry picked from commit f2ec0b7f3780b51c4b34c24ffc215344eea80d36) * Proof-of-concept implementation using pybind11 Expose the sequential reader to iterate over messages from Python. Signed-off-by: Jacob Perron (cherry picked from commit dc6889481c31f7a62778182101a965e22c5464f0) * pybind StorageOptions and ConverterOptions; linter fixes Signed-off-by: Mabel Zhang (cherry picked from commit 91f67634a6fca7114af9d962a466bbdc5c44b21c) * return timestamp Signed-off-by: Mabel Zhang (cherry picked from commit deafbf8984d9dfa74457a30a080daf69b29226f5) * simplify binding of structs Signed-off-by: Mabel Zhang (cherry picked from commit da561ccf2215bc90c37058892943e014e77c3ea9) * pybind TopicMetadata Signed-off-by: Mabel Zhang (cherry picked from commit 4a4e1b83188300c2ebf2cc7d93019d3fd4ad3cee) * namespace rosbag2 -> rosbag2_cpp Signed-off-by: Mabel Zhang * small fixes Signed-off-by: Mabel Zhang * initial pytests Signed-off-by: Mabel Zhang * wrap Reader instead of SequentialReader Signed-off-by: Mabel Zhang * small fixes for PR comments Signed-off-by: Mabel Zhang * change workflow to autotest feature branch Signed-off-by: Mabel Zhang * revert workflow Signed-off-by: Mabel Zhang * added writer and writer tests Signed-off-by: Mabel Zhang * fix test path setup Signed-off-by: Mabel Zhang * fix and make more rigorous writer test Signed-off-by: Mabel Zhang * wrapper and test for topic filter Signed-off-by: Mabel Zhang * linters. change pybind package Signed-off-by: Mabel Zhang * simplify verbose import Signed-off-by: Mabel Zhang * tidy up Signed-off-by: Mabel Zhang * explicit subclasses of Reader and Writer Signed-off-by: Mabel Zhang * Use template specialization instead of inheritence This makes the implementation slightly more compact. And adding a new specialization should be more straight forward as we only have to modify the cpp file in one place. Signed-off-by: Jacob Perron * Remove temporary variable Signed-off-by: Jacob Perron * Remove unused member variable Signed-off-by: Jacob Perron * Move to initializer list Signed-off-by: Jacob Perron * refactor into reader writer storage Signed-off-by: Mabel Zhang * cleanup includes Signed-off-by: Mabel Zhang * switch back to pybind11_vendor Signed-off-by: Mabel Zhang * Packages dependency for CI Signed-off-by: Mabel Zhang * printout to debug CI test failure Signed-off-by: Mabel Zhang * use tempfile for writer test, add exec_depend Signed-off-by: Mabel Zhang * remove exec_depend Signed-off-by: Mabel Zhang * use pytest fixture tmp_path; add rosbag2_py to rosbag2 Signed-off-by: Mabel Zhang * add makedirs back to see CI result Signed-off-by: Mabel Zhang * fix import error Signed-off-by: Mabel Zhang * cleanup Signed-off-by: Mabel Zhang * add modules explicitly for windows CI Signed-off-by: Mabel Zhang * cleanup Signed-off-by: Mabel Zhang * try more s for mac test failure Signed-off-by: Mabel Zhang * wrap structs with named arguments for Python keyword params Signed-off-by: Mabel Zhang * add pytest.ini Signed-off-by: Mabel Zhang * add rosbag db3-shm and db3-wal files for macOS test Signed-off-by: Mabel Zhang Co-authored-by: Jacob Perron Co-authored-by: Andreas Klintberg Signed-off-by: Emerson Knapp --- rosbag2/package.xml | 1 + rosbag2_py/CMakeLists.txt | 88 +++++++++++++++ rosbag2_py/package.xml | 38 +++++++ rosbag2_py/pytest.ini | 2 + rosbag2_py/resources/talker/metadata.yaml | 31 ++++++ rosbag2_py/resources/talker/talker.db3 | Bin 0 -> 16384 bytes rosbag2_py/resources/talker/talker.db3-shm | Bin 0 -> 32768 bytes rosbag2_py/resources/talker/talker.db3-wal | 0 rosbag2_py/rosbag2_py/__init__.py | 42 ++++++++ rosbag2_py/src/rosbag2_py/_reader.cpp | 120 +++++++++++++++++++++ rosbag2_py/src/rosbag2_py/_storage.cpp | 83 ++++++++++++++ rosbag2_py/src/rosbag2_py/_writer.cpp | 103 ++++++++++++++++++ rosbag2_py/test/common.py | 25 +++++ rosbag2_py/test/test_sequential_reader.py | 70 ++++++++++++ rosbag2_py/test/test_sequential_writer.py | 85 +++++++++++++++ 15 files changed, 688 insertions(+) create mode 100644 rosbag2_py/CMakeLists.txt create mode 100644 rosbag2_py/package.xml create mode 100644 rosbag2_py/pytest.ini create mode 100644 rosbag2_py/resources/talker/metadata.yaml create mode 100644 rosbag2_py/resources/talker/talker.db3 create mode 100644 rosbag2_py/resources/talker/talker.db3-shm create mode 100644 rosbag2_py/resources/talker/talker.db3-wal create mode 100644 rosbag2_py/rosbag2_py/__init__.py create mode 100644 rosbag2_py/src/rosbag2_py/_reader.cpp create mode 100644 rosbag2_py/src/rosbag2_py/_storage.cpp create mode 100644 rosbag2_py/src/rosbag2_py/_writer.cpp create mode 100644 rosbag2_py/test/common.py create mode 100644 rosbag2_py/test/test_sequential_reader.py create mode 100644 rosbag2_py/test/test_sequential_writer.py diff --git a/rosbag2/package.xml b/rosbag2/package.xml index 27181aad39..c4266fec25 100644 --- a/rosbag2/package.xml +++ b/rosbag2/package.xml @@ -14,6 +14,7 @@ rosbag2_compression rosbag2_converter_default_plugins rosbag2_cpp + rosbag2_py rosbag2_storage rosbag2_storage_default_plugins rosbag2_transport diff --git a/rosbag2_py/CMakeLists.txt b/rosbag2_py/CMakeLists.txt new file mode 100644 index 0000000000..98b7fd6385 --- /dev/null +++ b/rosbag2_py/CMakeLists.txt @@ -0,0 +1,88 @@ +cmake_minimum_required(VERSION 3.5) +project(rosbag2_py) + +# Default to C99 +if(NOT CMAKE_C_STANDARD) + set(CMAKE_C_STANDARD 99) +endif() + +# Default to C++14 +if(NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 14) +endif() + +if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-Wall -Wextra -Wpedantic -Werror) +endif() + +find_package(ament_cmake REQUIRED) +find_package(ament_cmake_ros REQUIRED) +find_package(pybind11 REQUIRED) +find_package(python_cmake_module REQUIRED) +find_package(rosbag2_compression REQUIRED) +find_package(rosbag2_cpp REQUIRED) +find_package(rosbag2_storage REQUIRED) + +ament_python_install_package(${PROJECT_NAME}) + +pybind11_add_module(_reader SHARED + src/rosbag2_py/_reader.cpp +) +ament_target_dependencies(_reader PUBLIC + "rosbag2_compression" + "rosbag2_cpp" +) + +pybind11_add_module(_storage SHARED + src/rosbag2_py/_storage.cpp +) +ament_target_dependencies(_storage PUBLIC + "rosbag2_cpp" + "rosbag2_storage" +) + +pybind11_add_module(_writer SHARED + src/rosbag2_py/_writer.cpp +) +ament_target_dependencies(_writer PUBLIC + "rosbag2_compression" + "rosbag2_cpp" +) + +# Install cython modules as sub-modules of the project +install( + TARGETS + _reader + _storage + _writer + DESTINATION "${PYTHON_INSTALL_DIR}/${PROJECT_NAME}" +) + +if(BUILD_TESTING) + find_package(ament_lint_auto REQUIRED) + ament_lint_auto_find_test_dependencies() + find_package(ament_cmake_pytest REQUIRED) + + set(_PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}") + if(WIN32 AND "${CMAKE_BUILD_TYPE}" STREQUAL "Debug") + set(_PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE_DEBUG}") + endif() + set(pythonpath "${CMAKE_CURRENT_BINARY_DIR}/rosbag2_py;${CMAKE_CURRENT_SOURCE_DIR}") + if(NOT WIN32) + string(REPLACE ";" ":" pythonpath "${pythonpath}") + endif() + ament_add_pytest_test(test_sequential_reader_py "test/test_sequential_reader.py" + PYTHON_EXECUTABLE "${_PYTHON_EXECUTABLE}" + APPEND_ENV "PYTHONPATH=${pythonpath}" + APPEND_LIBRARY_DIRS "${_append_library_dirs}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/resources" + ) + ament_add_pytest_test(test_sequential_writer_py "test/test_sequential_writer.py" + PYTHON_EXECUTABLE "${_PYTHON_EXECUTABLE}" + APPEND_ENV "PYTHONPATH=${pythonpath}" + APPEND_LIBRARY_DIRS "${_append_library_dirs}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/resources" + ) +endif() + +ament_package() diff --git a/rosbag2_py/package.xml b/rosbag2_py/package.xml new file mode 100644 index 0000000000..3f940523ac --- /dev/null +++ b/rosbag2_py/package.xml @@ -0,0 +1,38 @@ + + + + rosbag2_py + 0.2.4 + Python API for rosbag2 + Jacob Perron + ROS Tooling Working Group + Apache License 2.0 + + Mabel Zhang + + ament_cmake_ros + + pybind11_vendor + python_cmake_module + rosbag2_compression + rosbag2_cpp + rosbag2_storage + + rpyutils + + ament_lint_auto + ament_lint_common + python3-pytest + rcl_interfaces + rclpy + rosbag2_converter_default_plugins + rosbag2_storage_default_plugins + rosbag2_storage + rosidl_runtime_py + rpyutils + std_msgs + + + ament_cmake + + diff --git a/rosbag2_py/pytest.ini b/rosbag2_py/pytest.ini new file mode 100644 index 0000000000..fe55d2ed64 --- /dev/null +++ b/rosbag2_py/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +junit_family=xunit2 diff --git a/rosbag2_py/resources/talker/metadata.yaml b/rosbag2_py/resources/talker/metadata.yaml new file mode 100644 index 0000000000..35db28401c --- /dev/null +++ b/rosbag2_py/resources/talker/metadata.yaml @@ -0,0 +1,31 @@ +rosbag2_bagfile_information: + version: 4 + storage_identifier: sqlite3 + relative_file_paths: + - talker.db3 + duration: + nanoseconds: 4531096768 + starting_time: + nanoseconds_since_epoch: 1585866235112411371 + message_count: 20 + topics_with_message_count: + - topic_metadata: + name: /topic + type: std_msgs/msg/String + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 10 + - topic_metadata: + name: /rosout + type: rcl_interfaces/msg/Log + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 1\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 10\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 10 + - topic_metadata: + name: /parameter_events + type: rcl_interfaces/msg/ParameterEvent + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 0 + compression_format: "" + compression_mode: "" diff --git a/rosbag2_py/resources/talker/talker.db3 b/rosbag2_py/resources/talker/talker.db3 new file mode 100644 index 0000000000000000000000000000000000000000..3af211094541ccbfce88f9bc4ee33cff589d52f6 GIT binary patch literal 16384 zcmeI2ZD<@t7{~YSlDpg`xlNO+F=%vzk{7sUb1!*qsiy5xORu>!No^vw%jR}2S=qb2 z?rz=$Z4zTYh@c=yOQGTy6$L4ZDB2H#QVCcpf)w(}Pp#sY1cXR6)@SZAyQZFEIXS4# z!rbnE=Goc#{b$1MJeQGU3Dv?Vts5EHLNTt6bGy065aKw_4W$AKJv?ynNA02we{*(o z9S>frg#b_R6vrP8egr3WFb9|e%mL;AbAUO(9AFMG2bcrQ0p`Ge&VdsiZ)0n-YhKMN zc+OHY*tFzKPEwUQxHRQSY*ri?76(Q}ba?QfIEJ=`pyVK;VW`b+LIWcQ+7}z#zQ*Qe z*W$b-PiUBi(X(f(*U^Ff3DNGPO;zAIqv9cP7!3^{J~}Wwj*f`qVN1`cDHxR9IG7wo zgU1tzu=7xQpWTEM*^<%zL~=i(b?u9tE?;9)lWS>DepLB>&BCX9+lUHve>I-0oWW>R z9D_mF1DVg=S!H5Fl{NJl*;4haw6%fXewUt3V*@MFY2B1^hMrb6Y;NzkJ-;`vd6)}c zsZGO)9n1md0CRvjz#L!>Fb9|e%mL=W|JQ+~gB~B(6l+*c3R}AwMar0yrT`zph-IkR z$&_O3MpG*6bBy_Z6bm4v;G8wpk2>Mfz?v#gsG4d)TZFVt8@7(-V`W8Cv$!9w5i)VA zA4Mbao_KGpJKmG8%jPTN(Y|iEmQ5^? zp^?4RUG*x@=&B+au)!j`HW*Y2ODR1&gR`V7cvM=}OuTf`?c;!EcCiD`PYJTbBe`l#i$R7qm zg>_A?ot0Srl}YyhA@2>2f0SR|MbV@=b;XjV5gzQZD$SY#*7VcU@P;SAOJKfIz;p2P2qftRcz;Rda>Ab;C1`SH zLXkV5k*kLO^c;qlR^4c8=eQQoE%gp)?DY@Vi-Yz8=ne-ocK*2)3hFU0c}oE8EgNW^ z1G@8@8;6R6_5djFfbRV2`2`aA8_ekU*Gihvp0a@kZP2x!-HGePLAwF8)&^bM*nGuH zLHCT7G^5>R1FdmDS9>?#Dh}EOpw$lOYUJyeN#G{T=($fyn$fPZfd(AV%TI^CEe;w7 zP`?9uIrYN}6!gSwNi!NR8)%gSdhx;+vYHB#2MpK(a(#X(NaC!@7LXkCQ9&|IURyvmsgeqknyIh_q+dK#kX(kl+T-;D J_8Q4o{0@Nyz1;u+ literal 0 HcmV?d00001 diff --git a/rosbag2_py/resources/talker/talker.db3-shm b/rosbag2_py/resources/talker/talker.db3-shm new file mode 100644 index 0000000000000000000000000000000000000000..fe9ac2845eca6fe6da8a63cd096d9cf9e24ece10 GIT binary patch literal 32768 zcmeIuAr62r3 +#include + +#include +#include +#include + +#include "rosbag2_compression/sequential_compression_reader.hpp" +#include "rosbag2_cpp/converter_options.hpp" +#include "rosbag2_cpp/readers/sequential_reader.hpp" +#include "rosbag2_cpp/reader.hpp" +#include "rosbag2_cpp/storage_options.hpp" +#include "rosbag2_storage/storage_filter.hpp" +#include "rosbag2_storage/topic_metadata.hpp" + +namespace rosbag2_py +{ + +template +class Reader +{ +public: + Reader() + : reader_(std::make_unique(std::make_unique())) + { + } + + void open( + rosbag2_cpp::StorageOptions & storage_options, + rosbag2_cpp::ConverterOptions & converter_options) + { + reader_->open(storage_options, converter_options); + } + + bool has_next() + { + return reader_->has_next(); + } + + /// Return a tuple containing the topic name, the serialized ROS message, and + /// the timestamp. + pybind11::tuple read_next() + { + const auto next = reader_->read_next(); + rcutils_uint8_array_t rcutils_data = *next->serialized_data.get(); + std::string serialized_data(rcutils_data.buffer, + rcutils_data.buffer + rcutils_data.buffer_length); + return pybind11::make_tuple( + next->topic_name, pybind11::bytes(serialized_data), next->time_stamp); + } + + /// Return a mapping from topic name to topic type. + std::vector get_all_topics_and_types() + { + return reader_->get_all_topics_and_types(); + } + + void set_filter(const rosbag2_storage::StorageFilter & storage_filter) + { + return reader_->set_filter(storage_filter); + } + + void reset_filter() + { + reader_->reset_filter(); + } + +protected: + std::unique_ptr reader_; +}; +} // namespace rosbag2_py + +PYBIND11_MODULE(_reader, m) { + m.doc() = "Python wrapper of the rosbag2_cpp reader API"; + + pybind11::class_>( + m, "SequentialReader") + .def(pybind11::init()) + .def("open", &rosbag2_py::Reader::open) + .def("read_next", &rosbag2_py::Reader::read_next) + .def("has_next", &rosbag2_py::Reader::has_next) + .def( + "get_all_topics_and_types", + &rosbag2_py::Reader::get_all_topics_and_types) + .def("set_filter", &rosbag2_py::Reader::set_filter) + .def("reset_filter", &rosbag2_py::Reader::reset_filter); + + pybind11::class_>( + m, "SequentialCompressionReader") + .def(pybind11::init()) + .def("open", &rosbag2_py::Reader::open) + .def( + "read_next", &rosbag2_py::Reader::read_next) + .def("has_next", &rosbag2_py::Reader::has_next) + .def( + "get_all_topics_and_types", + &rosbag2_py::Reader< + rosbag2_compression::SequentialCompressionReader + >::get_all_topics_and_types) + .def( + "set_filter", + &rosbag2_py::Reader::set_filter) + .def( + "reset_filter", + &rosbag2_py::Reader::reset_filter); +} diff --git a/rosbag2_py/src/rosbag2_py/_storage.cpp b/rosbag2_py/src/rosbag2_py/_storage.cpp new file mode 100644 index 0000000000..bbd536dc51 --- /dev/null +++ b/rosbag2_py/src/rosbag2_py/_storage.cpp @@ -0,0 +1,83 @@ +// Copyright 2020 Open Source Robotics Foundation, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include +#include + +#include "rosbag2_cpp/converter_options.hpp" +#include "rosbag2_cpp/storage_options.hpp" +#include "rosbag2_storage/storage_filter.hpp" +#include "rosbag2_storage/topic_metadata.hpp" + +PYBIND11_MODULE(_storage, m) { + m.doc() = "Python wrapper of the rosbag2 utilities API"; + + pybind11::class_(m, "ConverterOptions") + .def( + pybind11::init(), + pybind11::arg("input_serialization_format"), + pybind11::arg("output_serialization_format")) + .def_readwrite( + "input_serialization_format", + &rosbag2_cpp::ConverterOptions::input_serialization_format) + .def_readwrite( + "output_serialization_format", + &rosbag2_cpp::ConverterOptions::output_serialization_format); + + pybind11::class_(m, "StorageOptions") + .def( + pybind11::init(), + pybind11::arg("uri"), + pybind11::arg("storage_id"), + pybind11::arg("max_bagfile_size") = 0, + pybind11::arg("max_bagfile_duration") = 0, + pybind11::arg("max_cache_size") = 0) + .def_readwrite("uri", &rosbag2_cpp::StorageOptions::uri) + .def_readwrite("storage_id", &rosbag2_cpp::StorageOptions::storage_id) + .def_readwrite( + "max_bagfile_size", + &rosbag2_cpp::StorageOptions::max_bagfile_size) + .def_readwrite( + "max_bagfile_duration", + &rosbag2_cpp::StorageOptions::max_bagfile_duration) + .def_readwrite( + "max_cache_size", + &rosbag2_cpp::StorageOptions::max_cache_size); + + pybind11::class_(m, "StorageFilter") + .def( + pybind11::init>(), + pybind11::arg("topics")) + .def_readwrite("topics", &rosbag2_storage::StorageFilter::topics); + + pybind11::class_(m, "TopicMetadata") + .def( + pybind11::init(), + pybind11::arg("name"), + pybind11::arg("type"), + pybind11::arg("serialization_format"), + pybind11::arg("offered_qos_profiles") = "") + .def_readwrite("name", &rosbag2_storage::TopicMetadata::name) + .def_readwrite("type", &rosbag2_storage::TopicMetadata::type) + .def_readwrite( + "serialization_format", + &rosbag2_storage::TopicMetadata::serialization_format) + .def_readwrite( + "offered_qos_profiles", + &rosbag2_storage::TopicMetadata::offered_qos_profiles) + .def("equals", &rosbag2_storage::TopicMetadata::operator==); +} diff --git a/rosbag2_py/src/rosbag2_py/_writer.cpp b/rosbag2_py/src/rosbag2_py/_writer.cpp new file mode 100644 index 0000000000..ecae08a344 --- /dev/null +++ b/rosbag2_py/src/rosbag2_py/_writer.cpp @@ -0,0 +1,103 @@ +// Copyright 2020 Open Source Robotics Foundation, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include +#include + +#include "rosbag2_compression/sequential_compression_writer.hpp" +#include "rosbag2_cpp/converter_options.hpp" +#include "rosbag2_cpp/storage_options.hpp" +#include "rosbag2_cpp/writer.hpp" +#include "rosbag2_cpp/writers/sequential_writer.hpp" +#include "rosbag2_storage/ros_helper.hpp" +#include "rosbag2_storage/storage_filter.hpp" +#include "rosbag2_storage/topic_metadata.hpp" + +namespace rosbag2_py +{ + +template +class Writer +{ +public: + Writer() + : writer_(std::make_unique(std::make_unique())) + { + } + + void open( + rosbag2_cpp::StorageOptions & storage_options, + rosbag2_cpp::ConverterOptions & converter_options) + { + writer_->open(storage_options, converter_options); + } + + void create_topic(const rosbag2_storage::TopicMetadata & topic_with_type) + { + writer_->create_topic(topic_with_type); + } + + void remove_topic(const rosbag2_storage::TopicMetadata & topic_with_type) + { + writer_->remove_topic(topic_with_type); + } + + /// Write a serialized message to a bag file + void write( + const std::string & topic_name, const std::string & message, + const rcutils_time_point_value_t & time_stamp) + { + auto bag_message = + std::make_shared(); + + bag_message->topic_name = topic_name; + bag_message->serialized_data = + rosbag2_storage::make_serialized_message(message.c_str(), message.length()); + bag_message->time_stamp = time_stamp; + + writer_->write(bag_message); + } + +protected: + std::unique_ptr writer_; +}; + +} // namespace rosbag2_py + +PYBIND11_MODULE(_writer, m) { + m.doc() = "Python wrapper of the rosbag2_cpp writer API"; + + pybind11::class_>( + m, "SequentialWriter") + .def(pybind11::init()) + .def("open", &rosbag2_py::Writer::open) + .def("write", &rosbag2_py::Writer::write) + .def("remove_topic", &rosbag2_py::Writer::remove_topic) + .def("create_topic", &rosbag2_py::Writer::create_topic); + + pybind11::class_>( + m, "SequentialCompressionWriter") + .def(pybind11::init()) + .def("open", &rosbag2_py::Writer::open) + .def("write", &rosbag2_py::Writer::write) + .def( + "remove_topic", + &rosbag2_py::Writer::remove_topic) + .def( + "create_topic", + &rosbag2_py::Writer::create_topic); +} diff --git a/rosbag2_py/test/common.py b/rosbag2_py/test/common.py new file mode 100644 index 0000000000..cec02bea31 --- /dev/null +++ b/rosbag2_py/test/common.py @@ -0,0 +1,25 @@ +# Copyright 2020 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import rosbag2_py + + +def get_rosbag_options(path, serialization_format='cdr'): + storage_options = rosbag2_py.StorageOptions(uri=path, storage_id='sqlite3') + + converter_options = rosbag2_py.ConverterOptions( + input_serialization_format=serialization_format, + output_serialization_format=serialization_format) + + return storage_options, converter_options diff --git a/rosbag2_py/test/test_sequential_reader.py b/rosbag2_py/test/test_sequential_reader.py new file mode 100644 index 0000000000..fc2be12c2b --- /dev/null +++ b/rosbag2_py/test/test_sequential_reader.py @@ -0,0 +1,70 @@ +# Copyright 2020 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +from common import get_rosbag_options +from rcl_interfaces.msg import Log +from rclpy.serialization import deserialize_message +import rosbag2_py +from rosidl_runtime_py.utilities import get_message +from std_msgs.msg import String + + +def test_sequential_reader(): + bag_path = str(Path(__file__).parent.parent / 'resources' / 'talker') + storage_options, converter_options = get_rosbag_options(bag_path) + + reader = rosbag2_py.SequentialReader() + reader.open(storage_options, converter_options) + + topic_types = reader.get_all_topics_and_types() + + # Create a map for quicker lookup + type_map = {topic_types[i].name: topic_types[i].type for i in range(len(topic_types))} + + # Set filter for topic of string type + storage_filter = rosbag2_py.StorageFilter(topics=['/topic']) + reader.set_filter(storage_filter) + + msg_counter = 0 + + while reader.has_next(): + (topic, data, t) = reader.read_next() + msg_type = get_message(type_map[topic]) + msg = deserialize_message(data, msg_type) + + assert isinstance(msg, String) + assert msg.data == f'Hello, world! {msg_counter}' + + msg_counter += 1 + + # No filter + reader.reset_filter() + + reader = rosbag2_py.SequentialReader() + reader.open(storage_options, converter_options) + + msg_counter = 0 + + while reader.has_next(): + (topic, data, t) = reader.read_next() + msg_type = get_message(type_map[topic]) + msg = deserialize_message(data, msg_type) + + assert isinstance(msg, Log) or isinstance(msg, String) + + if isinstance(msg, String): + assert msg.data == f'Hello, world! {msg_counter}' + msg_counter += 1 diff --git a/rosbag2_py/test/test_sequential_writer.py b/rosbag2_py/test/test_sequential_writer.py new file mode 100644 index 0000000000..e1a12f4e53 --- /dev/null +++ b/rosbag2_py/test/test_sequential_writer.py @@ -0,0 +1,85 @@ +# Copyright 2020 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common import get_rosbag_options +from rclpy.serialization import deserialize_message, serialize_message +import rosbag2_py +from rosidl_runtime_py.utilities import get_message +from std_msgs.msg import String + + +def create_topic(writer, topic_name, topic_type, serialization_format='cdr'): + """ + Create a new topic. + + :param writer: writer instance + :param topic_name: + :param topic_type: + :param serialization_format: + :return: + """ + topic_name = topic_name + topic = rosbag2_py.TopicMetadata(name=topic_name, type=topic_type, + serialization_format=serialization_format) + + writer.create_topic(topic) + + +def test_sequential_writer(tmp_path): + """ + Test for sequential writer. + + :return: + """ + bag_path = str(tmp_path / 'tmp_write_test') + + storage_options, converter_options = get_rosbag_options(bag_path) + + writer = rosbag2_py.SequentialWriter() + writer.open(storage_options, converter_options) + + # create topic + topic_name = '/chatter' + create_topic(writer, topic_name, 'std_msgs/msg/String') + + for i in range(10): + msg = String() + msg.data = f'Hello, world! {str(i)}' + time_stamp = i * 100 + + writer.write(topic_name, serialize_message(msg), time_stamp) + + # close bag and create new storage instance + del writer + storage_options, converter_options = get_rosbag_options(bag_path) + + reader = rosbag2_py.SequentialReader() + reader.open(storage_options, converter_options) + + topic_types = reader.get_all_topics_and_types() + + # Create a map for quicker lookup + type_map = {topic_types[i].name: topic_types[i].type for i in range(len(topic_types))} + + msg_counter = 0 + while reader.has_next(): + topic, data, t = reader.read_next() + msg_type = get_message(type_map[topic]) + msg_deserialized = deserialize_message(data, msg_type) + + assert isinstance(msg_deserialized, String) + assert msg_deserialized.data == f'Hello, world! {msg_counter}' + assert t == msg_counter * 100 + + msg_counter += 1 From 1228813b20c288cf02c7e0f5e8d8a90635239c1c Mon Sep 17 00:00:00 2001 From: Mabel Zhang Date: Fri, 14 Aug 2020 20:12:09 -0400 Subject: [PATCH 22/77] AMENT_IGNORE rosbag2_py for now (#509) * AMENT_IGNORE rosbag2_py Signed-off-by: Mabel Zhang * remove rosbag2_py from meta package Signed-off-by: Mabel Zhang --- rosbag2/package.xml | 1 - rosbag2_py/AMENT_IGNORE | 0 2 files changed, 1 deletion(-) create mode 100644 rosbag2_py/AMENT_IGNORE diff --git a/rosbag2/package.xml b/rosbag2/package.xml index c4266fec25..27181aad39 100644 --- a/rosbag2/package.xml +++ b/rosbag2/package.xml @@ -14,7 +14,6 @@ rosbag2_compression rosbag2_converter_default_plugins rosbag2_cpp - rosbag2_py rosbag2_storage rosbag2_storage_default_plugins rosbag2_transport diff --git a/rosbag2_py/AMENT_IGNORE b/rosbag2_py/AMENT_IGNORE new file mode 100644 index 0000000000..e69de29bb2 From 0ef46b3d3515ad80bd5eac38cab74a0b525e2612 Mon Sep 17 00:00:00 2001 From: Devin Bonnie <47613035+dabonnie@users.noreply.github.com> Date: Mon, 17 Aug 2020 14:19:13 -0700 Subject: [PATCH 23/77] Add compression and batch size to the README (#510) * Add compression and batch size to the README Signed-off-by: Devin Bonnie * Update with split by duration and message compression mode Signed-off-by: Devin Bonnie --- README.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 88ab8d6946..1c9a677e4c 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Clone this repository into the source folder: ``` $ git clone https://github.com/ros2/rosbag2.git ``` -**[Note]**: if you are only building rosbag2 on top of a Debian Installation of ROS2, please git clone the branch following your current ROS2 distribution. +**[Note]**: if you are only building rosbag2 on top of a Debian Installation of ROS2, please git clone the branch following your current ROS2 distribution. Then build all the packages with this command: @@ -93,6 +93,26 @@ In the same fashion, this auto discovery can be disabled with `--no-discovery`. If not further specified, `ros2 bag record` will create a new folder named to the current time stamp and stores all data within this folder. A user defined name can be given with `-o, --output`. +#### Splitting recorded bag files + +rosbag2 offers the capability to split bag files when they reach a maximum size or after a specified duration. By default rosbag2 will record all data into a single bag file, but this can be changed using the CLI options. + +_Splitting by size_: `ros2 bag record -a -b 100000` will split the bag files when they become greater than 100 kilobytes. Note: the batch size's units are in bytes and must be greater than `86016`. This option defaults to `0`, which means data is written to a single file. + +_Splitting by time_: `ros2 bag record -a -d 9000` will split the bag files after a duration of `9000` seconds. This option defaults to `0`, which means data is written to a single file. + +If both splitting by size and duration are enabled, the bag will split at whichever threshold is reached first. + +#### Recording with compression + +By default rosbag2 does not record with compression enabled. However, compression can be specified using the following CLI options. + +For example, `ros2 bag record -a --compression-mode file --compression-format zstd` will record all topics and compress each file using the [zstd](https://github.com/facebook/zstd) compressor. + +Currently, the only `compression-format` available is `zstd`. Both the mode and format options default to `none`. To use a compression format, a compression mode must be specified, where the currently supported modes are compress by `file` or compress by `message`. + +It is recommended to use this feature with the splitting options. + ### Replaying data After recording data, the next logical step is to replay this data: From e1689cb1fe25d983e0baaaae4341c5f32fde642c Mon Sep 17 00:00:00 2001 From: Anas Abou Allaban Date: Mon, 17 Aug 2020 20:47:15 -0400 Subject: [PATCH 24/77] Document QoS Policy overrides (#508) * Document QoS Policy overrides Signed-off-by: Anas Abou Allaban --- README.md | 39 +++++++++++++++++++++++++++++++++++ docs/qos_profile_overrides.md | 25 ---------------------- 2 files changed, 39 insertions(+), 25 deletions(-) delete mode 100644 docs/qos_profile_overrides.md diff --git a/README.md b/README.md index 1c9a677e4c..baaa5b015f 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,42 @@ Topic information: Topic: /chatter | Type: std_msgs/String | Count: 9 | Serializ Topic: /my_chatter | Type: std_msgs/String | Count: 18 | Serialization Format: cdr ``` +### Overriding QoS Profiles + +When starting a recording or playback workflow, you can pass a YAML file that contains QoS profile settings for a specific topic. +The YAML schema for the profile overrides is a dictionary of topic names with key/value pairs for each QoS policy. +Below is an example profile set to the default ROS2 QoS settings. + +```yaml +/topic_name: + history: keep_last + depth: 10 + reliability: reliable + durability: volatile + deadline: + sec: 2147483647 # LONG_MAX + nsec: 4294967295 # ULONG_MAX + lifespan: + sec: 2147483647 + nsec: 4294967295 + liveliness: system_default + liveliness_lease_duration: + sec: 2147483647 + nsec: 4294967295 + avoid_ros_namespace_conventions: false +``` + +You can then use the override by specifying the `--qos-profile-overrides-path` argument in the CLI: + +```sh +# Record +ros2 bag record --qos-profile-overrides-path override.yaml -a -o my_bag +# Playback +ros2 bag play --qos-profile-overrides-path override.yaml my_bag +``` + +See [the official QoS override tutorial][qos-override-tutorial] and ["About QoS Settings"][about-qos-settings] for more detail. + ### Using in launch We can invoke the command line tool from a ROS launch script as an *executable* (not a *node* action). @@ -209,3 +245,6 @@ When specified, rosbag2 looks for a suitable converter to transform the native m This also allows to record data in a native format to optimize for speed, but to convert or transform the recorded data into a middleware agnostic serialization format. By default, rosbag2 can convert from and to CDR as it's the default serialization format for ROS 2. + +[qos-override-tutorial]: https://index.ros.org/doc/ros2/Tutorials/Ros2bag/Overriding-QoS-Policies-For-Recording-And-Playback/ +[about-qos-settings]: https://index.ros.org/doc/ros2/Concepts/About-Quality-of-Service-Settings/ diff --git a/docs/qos_profile_overrides.md b/docs/qos_profile_overrides.md deleted file mode 100644 index 1f1506d4f5..0000000000 --- a/docs/qos_profile_overrides.md +++ /dev/null @@ -1,25 +0,0 @@ -# Overriding topic QoS profiles - -## Recording - -When starting a recording workflow, you can pass a yaml file that contains QoS profile settings for a specific topic. -The yaml file should have following structure: - -```yaml -: - : value - : - sec: 0 - nsec: 0 -... -: - : value -... -``` - -See [`qos_profile.yaml`](../ros2bag/test/resources/qos_profile.yaml) for a complete example. - -## Resources - -* [About QoS settings](https://index.ros.org/doc/ros2/Concepts/About-Quality-of-Service-Settings/) -* [ROS2 QoS policies design doc](https://design.ros2.org/articles/qos.html) From 380b708dac2a001fa905cc95114b3dc88f706b5f Mon Sep 17 00:00:00 2001 From: Dirk Thomas Date: Tue, 25 Aug 2020 21:40:54 -0700 Subject: [PATCH 25/77] disable sanitizer by default (#517) Signed-off-by: Dirk Thomas --- rosbag2_cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rosbag2_cpp/CMakeLists.txt b/rosbag2_cpp/CMakeLists.txt index 8ec8218301..cd378c4e2c 100644 --- a/rosbag2_cpp/CMakeLists.txt +++ b/rosbag2_cpp/CMakeLists.txt @@ -20,7 +20,7 @@ if(WIN32) add_definitions(-DNOMINMAX) endif() -option(DISABLE_SANITIZERS "disables the use of gcc sanitizers") +option(DISABLE_SANITIZERS "disables the use of gcc sanitizers" ON) if(NOT DISABLE_SANITIZERS AND CMAKE_COMPILER_IS_GNUCXX) include(CheckCXXSourceCompiles) set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) From 6dfa891fcb37e7f67f27fe4980daba3a19f97370 Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Mon, 14 Sep 2020 10:00:41 -0700 Subject: [PATCH 26/77] Mark flaky tests as xfail for now (#520) Signed-off-by: Emerson Knapp --- rosbag2_tests/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rosbag2_tests/CMakeLists.txt b/rosbag2_tests/CMakeLists.txt index 9e4c54bc22..da7029eb4c 100644 --- a/rosbag2_tests/CMakeLists.txt +++ b/rosbag2_tests/CMakeLists.txt @@ -54,6 +54,7 @@ if(BUILD_TESTING) rosbag2_storage_default_plugins rosbag2_test_common test_msgs) + ament_add_test_label(test_rosbag2_record_end_to_end xfail) endif() ament_add_gmock(test_rosbag2_play_end_to_end @@ -66,6 +67,7 @@ if(BUILD_TESTING) rosbag2_storage_default_plugins rosbag2_test_common test_msgs) + ament_add_test_label(test_rosbag2_play_end_to_end xfail) endif() ament_add_gmock(test_rosbag2_info_end_to_end @@ -75,6 +77,7 @@ if(BUILD_TESTING) ament_target_dependencies(test_rosbag2_info_end_to_end rosbag2_storage rosbag2_test_common) + ament_add_test_label(test_rosbag2_info_end_to_end xfail) endif() ament_add_gmock(test_converter From cfa377a9259cb05709c7880d0a4849a15bd0159c Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Mon, 21 Sep 2020 13:53:37 -0700 Subject: [PATCH 27/77] Add CODEOWNERS file (#519) Signed-off-by: Emerson Knapp --- .github/CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..4f6bdc3daa --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# Reviewers (and by extension approvers) will be the default owners for everything in the repo. +# Unless a later match takes precedence, @reviewers will be requested for review when someone opens a pull request. +* @ros2/tooling-reviewers From a2c0976398724e129b1f6a607cd06db5b59eddf5 Mon Sep 17 00:00:00 2001 From: Jaison Titus Date: Tue, 22 Sep 2020 16:53:01 -0700 Subject: [PATCH 28/77] [ros2bag test_record] Gets rid of time.sleep and move to using command.wait_for_output (#525) * Uses bag_command.wait_for_output with expected string instead of time.sleep in tests Signed-off-by: Jaison Titus * Fixes code style errors Signed-off-by: Jaison Titus * Moves to asserting expected output match outside of the process context to account for cases where wait_for_output is maybe called after the expected output is already printed. Signed-off-by: Jaison Titus * Defines timeout with variables and better error messages for failed tests. Signed-off-by: Jaison Titus * Fixes flake8 errors Signed-off-by: Jaison Titus --- ros2bag/test/test_record_qos_profiles.py | 54 ++++++++++++++++-------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/ros2bag/test/test_record_qos_profiles.py b/ros2bag/test/test_record_qos_profiles.py index 6eb632a013..91beba614f 100644 --- a/ros2bag/test/test_record_qos_profiles.py +++ b/ros2bag/test/test_record_qos_profiles.py @@ -36,7 +36,10 @@ PROFILE_PATH = Path(__file__).parent / 'resources' TEST_NODE = 'ros2bag_record_qos_profile_test_node' TEST_NAMESPACE = 'ros2bag_record_qos_profile' -ERROR_STRING = r'\[ERROR] \[ros2bag]:' +ERROR_STRING_MSG = 'ros2bag CLI did not produce the expected output'\ + '\n Expected output pattern: {}\n Actual output: {}' +OUTPUT_WAIT_TIMEOUT = 10 +SHUTDOWN_TIMEOUT = 5 @pytest.mark.rostest @@ -81,52 +84,67 @@ def test_qos_simple(self): output_path = Path(self.tmpdir.name) / 'ros2bag_test_basic' arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), '--output', output_path.as_posix()] + expected_string_regex = re.compile( + r'\[rosbag2_storage]: Opened database .* for READ_WRITE') with self.launch_bag_command(arguments=arguments) as bag_command: - time.sleep(3) - bag_command.wait_for_shutdown(timeout=5) + bag_command.wait_for_output( + condition=lambda output: expected_string_regex.search(output) is not None, + timeout=OUTPUT_WAIT_TIMEOUT) + bag_command.wait_for_shutdown(timeout=SHUTDOWN_TIMEOUT) assert bag_command.terminated - expected_string_regex = re.compile(ERROR_STRING) matches = expected_string_regex.search(bag_command.output) - assert not matches, print('ros2bag CLI did not produce the expected output') + assert matches, print( + ERROR_STRING_MSG.format(expected_string_regex.pattern, bag_command.output)) def test_incomplete_qos_profile(self): profile_path = PROFILE_PATH / 'incomplete_qos_profile.yaml' output_path = Path(self.tmpdir.name) / 'ros2bag_test_incomplete' arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), '--output', output_path.as_posix()] + expected_string_regex = re.compile( + r'\[rosbag2_storage]: Opened database .* for READ_WRITE') with self.launch_bag_command(arguments=arguments) as bag_command: - time.sleep(3) - bag_command.wait_for_shutdown(timeout=5) + bag_command.wait_for_output( + condition=lambda output: expected_string_regex.search(output) is not None, + timeout=OUTPUT_WAIT_TIMEOUT) + bag_command.wait_for_shutdown(timeout=SHUTDOWN_TIMEOUT) assert bag_command.terminated - expected_string_regex = re.compile(ERROR_STRING) matches = expected_string_regex.search(bag_command.output) - assert not matches, print('ros2bag CLI did not produce the expected output') + assert matches, print( + ERROR_STRING_MSG.format(expected_string_regex.pattern, bag_command.output)) def test_incomplete_qos_duration(self): profile_path = PROFILE_PATH / 'incomplete_qos_duration.yaml' output_path = Path(self.tmpdir.name) / 'ros2bag_test_incomplete_duration' arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), '--output', output_path.as_posix()] + expected_string_regex = re.compile( + r'\[ERROR] \[ros2bag]: Time overrides must include both') with self.launch_bag_command(arguments=arguments) as bag_command: - time.sleep(3) - bag_command.wait_for_shutdown(timeout=5) + bag_command.wait_for_output( + condition=lambda output: expected_string_regex.search(output) is not None, + timeout=OUTPUT_WAIT_TIMEOUT) + bag_command.wait_for_shutdown(timeout=SHUTDOWN_TIMEOUT) assert bag_command.terminated assert bag_command.exit_code != launch_testing.asserts.EXIT_OK - expected_string_regex = re.compile(ERROR_STRING) matches = expected_string_regex.search(bag_command.output) - assert matches, print('ros2bag CLI did not produce the expected output') + assert matches, print( + ERROR_STRING_MSG.format(expected_string_regex.pattern, bag_command.output)) def test_nonexistent_qos_profile(self): profile_path = PROFILE_PATH / 'foobar.yaml' output_path = Path(self.tmpdir.name) / 'ros2bag_test_nonexistent' arguments = ['record', '-a', '--qos-profile-overrides-path', profile_path.as_posix(), '--output', output_path.as_posix()] + expected_string_regex = re.compile( + r'ros2 bag record: error: argument --qos-profile-overrides-path: can\'t open') with self.launch_bag_command(arguments=arguments) as bag_command: - time.sleep(3) - bag_command.wait_for_shutdown(timeout=5) + bag_command.wait_for_output( + condition=lambda output: expected_string_regex.search(output) is not None, + timeout=OUTPUT_WAIT_TIMEOUT) + bag_command.wait_for_shutdown(timeout=SHUTDOWN_TIMEOUT) assert bag_command.terminated assert bag_command.exit_code != launch_testing.asserts.EXIT_OK - expected_string_regex = re.compile( - r"ros2 bag record: error: argument --qos-profile-overrides-path: can't open") matches = expected_string_regex.search(bag_command.output) - assert matches, print('ros2bag CLI did not produce the expected output') + assert matches, print( + ERROR_STRING_MSG.format(expected_string_regex.pattern, bag_command.output)) From 4af4676ca6d90a8ea599eeb6befdad5adc5fe5bf Mon Sep 17 00:00:00 2001 From: Jaison Titus Date: Thu, 1 Oct 2020 18:30:02 -0700 Subject: [PATCH 29/77] Remove some code duplication between SequentialWriter and SequentialCompressionWriter (#527) Remove code duplication between SequentialWriter and SequentialCompressionWriter Signed-off-by: Jaison Titus --- .../sequential_compression_writer.hpp | 64 ++----- .../sequential_compression_writer.cpp | 156 +----------------- .../rosbag2_cpp/writers/sequential_writer.hpp | 13 +- .../rosbag2_cpp/writers/sequential_writer.cpp | 12 +- 4 files changed, 42 insertions(+), 203 deletions(-) diff --git a/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp b/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp index fb1a930cdf..f59641731a 100644 --- a/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp +++ b/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp @@ -25,7 +25,7 @@ #include "rosbag2_cpp/serialization_format_converter_factory.hpp" #include "rosbag2_cpp/serialization_format_converter_factory_interface.hpp" #include "rosbag2_cpp/storage_options.hpp" -#include "rosbag2_cpp/writer_interfaces/base_writer_interface.hpp" +#include "rosbag2_cpp/writers/sequential_writer.hpp" #include "rosbag2_storage/metadata_io.hpp" #include "rosbag2_storage/storage_factory.hpp" @@ -48,7 +48,7 @@ namespace rosbag2_compression { class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionWriter - : public rosbag2_cpp::writer_interfaces::BaseWriterInterface + : public rosbag2_cpp::writers::SequentialWriter { public: explicit SequentialCompressionWriter( @@ -81,33 +81,6 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionWriter */ void reset() override; - /** - * Create a new topic in the underlying storage. Needs to be called for every topic used within - * a message which is passed to write(...). - * - * \param topic_with_type name and type identifier of topic to be created - * \throws runtime_error if the Writer is not open. - */ - void create_topic(const rosbag2_storage::TopicMetadata & topic_with_type) override; - - /** - * Remove a new topic in the underlying storage. - * If creation of subscription fails remove the topic - * from the db (more of cleanup) - * - * \param topic_with_type name and type identifier of topic to be created - * \throws runtime_error if the Writer is not open. - */ - void remove_topic(const rosbag2_storage::TopicMetadata & topic_with_type) override; - - /** - * Write a message to a bagfile. The topic needs to have been created before writing is possible. - * - * \param message to be written to the bagfile - * \throws runtime_error if the Writer is not open. - */ - void write(std::shared_ptr message) override; - protected: /** * Compress the most recent file and update the metadata file path. @@ -132,38 +105,27 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionWriter virtual void setup_compression(); private: - std::string base_folder_; - std::unique_ptr storage_factory_{}; - std::shared_ptr converter_factory_{}; - std::shared_ptr storage_{}; - std::unique_ptr metadata_io_{}; - std::unique_ptr converter_{}; std::unique_ptr compressor_{}; std::unique_ptr compression_factory_{}; - // Used in bagfile splitting; specifies the best-effort maximum sub-section of a bagfile in bytes. - uint64_t max_bagfile_size_{rosbag2_storage::storage_interfaces::MAX_BAGFILE_SIZE_NO_SPLIT}; - - // Used to track topic -> message count - std::unordered_map topics_names_to_info_{}; - - rosbag2_storage::BagMetadata metadata_{}; - rosbag2_compression::CompressionOptions compression_options_{}; bool should_compress_last_file_{true}; // Closes the current backed storage and opens the next bagfile. - void split_bagfile(); - - // Checks if the current recording bagfile needs to be split and rolled over to a new file. - bool should_split_bagfile() const; + void split_bagfile() override; // Prepares the metadata by setting initial values. - void init_metadata(); - - // Record TopicInformation into metadata - void finalize_metadata(); + void init_metadata() override; + + // Helper method used by write to get the message in a format that is ready to be written. + // Common use cases include converting the message using the converter or + // performing other operations like compression on it + std::shared_ptr + get_writeable_message( + std::shared_ptr message) override; }; + } // namespace rosbag2_compression + #endif // ROSBAG2_COMPRESSION__SEQUENTIAL_COMPRESSION_WRITER_HPP_ diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp index 9911390d47..fe0848616f 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp @@ -53,9 +53,7 @@ std::string format_storage_uri(const std::string & base_folder, uint64_t storage SequentialCompressionWriter::SequentialCompressionWriter( const rosbag2_compression::CompressionOptions & compression_options) -: storage_factory_{std::make_unique()}, - converter_factory_{std::make_shared()}, - metadata_io_{std::make_unique()}, +: SequentialWriter(), compression_factory_{std::make_unique()}, compression_options_{compression_options} {} @@ -66,9 +64,7 @@ SequentialCompressionWriter::SequentialCompressionWriter( std::unique_ptr storage_factory, std::shared_ptr converter_factory, std::unique_ptr metadata_io) -: storage_factory_{std::move(storage_factory)}, - converter_factory_{std::move(converter_factory)}, - metadata_io_{std::move(metadata_io)}, +: SequentialWriter(std::move(storage_factory), converter_factory, std::move(metadata_io)), compression_factory_{std::move(compression_factory)}, compression_options_{compression_options} {} @@ -103,50 +99,11 @@ void SequentialCompressionWriter::open( const rosbag2_cpp::StorageOptions & storage_options, const rosbag2_cpp::ConverterOptions & converter_options) { - max_bagfile_size_ = storage_options.max_bagfile_size; - base_folder_ = storage_options.uri; - - if (converter_options.output_serialization_format != - converter_options.input_serialization_format) - { - converter_ = std::make_unique(converter_options, converter_factory_); - } - - rcpputils::fs::path db_path(base_folder_); - if (db_path.is_directory()) { - std::stringstream error; - error << "Database directory already exists (" << db_path.string() << - "), can't overwrite existing database"; - throw std::runtime_error{error.str()}; - } - - bool dir_created = rcpputils::fs::create_directories(db_path); - if (!dir_created) { - std::stringstream error; - error << "Failed to create database directory (" << db_path.string() << ")."; - throw std::runtime_error{error.str()}; - } - - const auto storage_uri = format_storage_uri(base_folder_, 0); - storage_ = storage_factory_->open_read_write(storage_uri, storage_options.storage_id); - if (!storage_) { - throw std::runtime_error{"No storage could be initialized. Abort"}; - } - - if (max_bagfile_size_ != 0 && - max_bagfile_size_ < storage_->get_minimum_split_file_size()) - { - std::stringstream error; - error << "Invalid bag splitting size given. Please provide a value greater than " << - storage_->get_minimum_split_file_size() << ". Specified value of " << - storage_options.max_bagfile_size; - throw std::runtime_error{error.str()}; - } - + SequentialWriter::open(storage_options, converter_options); setup_compression(); - init_metadata(); } + void SequentialCompressionWriter::reset() { if (!base_folder_.empty() && compressor_) { @@ -170,52 +127,6 @@ void SequentialCompressionWriter::reset() storage_factory_.reset(); } -void SequentialCompressionWriter::create_topic( - const rosbag2_storage::TopicMetadata & topic_with_type) -{ - if (!storage_) { - throw std::runtime_error{"Bag is not open. Call open() before writing."}; - } - - if (converter_) { - converter_->add_topic(topic_with_type.name, topic_with_type.type); - } - - if (topics_names_to_info_.find(topic_with_type.name) == - topics_names_to_info_.end()) - { - rosbag2_storage::TopicInformation info{}; - info.topic_metadata = topic_with_type; - - const auto insert_res = topics_names_to_info_.insert( - std::make_pair(topic_with_type.name, info)); - - if (!insert_res.second) { - std::stringstream errmsg; - errmsg << "Failed to insert topic \"" << topic_with_type.name << "\"!"; - throw std::runtime_error{errmsg.str()}; - } - - storage_->create_topic(topic_with_type); - } -} - -void SequentialCompressionWriter::remove_topic( - const rosbag2_storage::TopicMetadata & topic_with_type) -{ - if (!storage_) { - throw std::runtime_error{"Bag is not open. Call open() before removing."}; - } - - if (topics_names_to_info_.erase(topic_with_type.name) > 0) { - storage_->remove_topic(topic_with_type); - } else { - std::stringstream errmsg; - errmsg << "Failed to remove the non-existing topic \"" << topic_with_type.name << "\"!"; - throw std::runtime_error{errmsg.str()}; - } -} - void SequentialCompressionWriter::compress_last_file() { if (!compressor_) { @@ -282,64 +193,15 @@ void SequentialCompressionWriter::compress_message( compressor_->compress_serialized_bag_message(message.get()); } -void SequentialCompressionWriter::write( +std::shared_ptr +SequentialCompressionWriter::get_writeable_message( std::shared_ptr message) { - if (!storage_) { - throw std::runtime_error{"Bag is not open. Call open() before writing."}; - } - - // Update the message count for the Topic. - ++topics_names_to_info_.at(message->topic_name).message_count; - - if (should_split_bagfile()) { - split_bagfile(); - } - - const auto message_timestamp = std::chrono::time_point{ - std::chrono::nanoseconds(message->time_stamp)}; - metadata_.starting_time = std::min(metadata_.starting_time, message_timestamp); - - const auto duration = message_timestamp - metadata_.starting_time; - metadata_.duration = std::max(metadata_.duration, duration); - - auto converted_message = converter_ ? converter_->convert(message) : message; + auto writeable_msg = SequentialWriter::get_writeable_message(message); if (compression_options_.compression_mode == rosbag2_compression::CompressionMode::MESSAGE) { - compress_message(converted_message); - } - - storage_->write(converted_message); -} - -bool SequentialCompressionWriter::should_split_bagfile() const -{ - if (max_bagfile_size_ == rosbag2_storage::storage_interfaces::MAX_BAGFILE_SIZE_NO_SPLIT) { - return false; - } else { - return storage_->get_bagfile_size() > max_bagfile_size_; - } -} - -void SequentialCompressionWriter::finalize_metadata() -{ - metadata_.bag_size = 0; - - for (const auto & path : metadata_.relative_file_paths) { - const auto bag_path = rcpputils::fs::path{path}; - - if (bag_path.exists()) { - metadata_.bag_size += bag_path.file_size(); - } - } - - metadata_.topics_with_message_count.clear(); - metadata_.topics_with_message_count.reserve(topics_names_to_info_.size()); - metadata_.message_count = 0; - - for (const auto & topic : topics_names_to_info_) { - metadata_.topics_with_message_count.push_back(topic.second); - metadata_.message_count += topic.second.message_count; + compress_message(writeable_msg); } + return writeable_msg; } } // namespace rosbag2_compression diff --git a/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp b/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp index 9a88409798..1bece470bd 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp @@ -102,7 +102,7 @@ class ROSBAG2_CPP_PUBLIC SequentialWriter */ void write(std::shared_ptr message) override; -private: +protected: std::string base_folder_; std::unique_ptr storage_factory_; std::shared_ptr converter_factory_; @@ -128,16 +128,23 @@ class ROSBAG2_CPP_PUBLIC SequentialWriter rosbag2_storage::BagMetadata metadata_; // Closes the current backed storage and opens the next bagfile. - void split_bagfile(); + virtual void split_bagfile(); // Checks if the current recording bagfile needs to be split and rolled over to a new file. bool should_split_bagfile() const; // Prepares the metadata by setting initial values. - void init_metadata(); + virtual void init_metadata(); // Record TopicInformation into metadata void finalize_metadata(); + + // Helper method used by write to get the message in a format that is ready to be written. + // Common use cases include converting the message using the converter or + // performing other operations like compression on it + virtual std::shared_ptr + get_writeable_message( + std::shared_ptr message); }; } // namespace writers diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index 55e1a7c3d8..d8b17cb7cc 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -243,11 +243,12 @@ void SequentialWriter::write(std::shared_ptrwrite(converter_ ? converter_->convert(message) : message); + storage_->write(converted_msg); } else { - cache_.push_back(converter_ ? converter_->convert(message) : message); + cache_.push_back(converted_msg); if (cache_.size() >= max_cache_size_) { storage_->write(cache_); // reset cache @@ -257,6 +258,13 @@ void SequentialWriter::write(std::shared_ptr +SequentialWriter::get_writeable_message( + std::shared_ptr message) +{ + return converter_ ? converter_->convert(message) : message; +} + bool SequentialWriter::should_split_bagfile() const { // Assume we aren't splitting From 312713d1b032f35691c7f324a48ff2948ed25881 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Mon, 5 Oct 2020 10:07:59 -0300 Subject: [PATCH 30/77] Fix rosbag2_py bug when using libc++ (#529) Signed-off-by: Mabel Zhang Signed-off-by: Ivan Santiago Paunovic Co-authored-by: Mabel Zhang --- rosbag2_py/CMakeLists.txt | 18 +++++++------- rosbag2_py/src/rosbag2_py/_reader.cpp | 5 ++-- rosbag2_py/src/rosbag2_py/_storage.cpp | 5 ++-- rosbag2_py/src/rosbag2_py/_writer.cpp | 5 ++-- rosbag2_py/src/rosbag2_py/pybind11.hpp | 30 +++++++++++++++++++++++ rosbag2_py/test/test_sequential_reader.py | 14 +++++++++-- rosbag2_py/test/test_sequential_writer.py | 15 ++++++++++-- 7 files changed, 70 insertions(+), 22 deletions(-) create mode 100644 rosbag2_py/src/rosbag2_py/pybind11.hpp diff --git a/rosbag2_py/CMakeLists.txt b/rosbag2_py/CMakeLists.txt index 98b7fd6385..c9333e22cb 100644 --- a/rosbag2_py/CMakeLists.txt +++ b/rosbag2_py/CMakeLists.txt @@ -67,21 +67,21 @@ if(BUILD_TESTING) if(WIN32 AND "${CMAKE_BUILD_TYPE}" STREQUAL "Debug") set(_PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE_DEBUG}") endif() - set(pythonpath "${CMAKE_CURRENT_BINARY_DIR}/rosbag2_py;${CMAKE_CURRENT_SOURCE_DIR}") - if(NOT WIN32) - string(REPLACE ";" ":" pythonpath "${pythonpath}") + set(other_environment_vars) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_CXX_COMPILER_ID MATCHES "Clang") + # Work around a rtti bug when using class_loader from a + # python extension built with libc++. + set(other_environment_vars "ROSBAG2_PY_TEST_WITH_RTLD_GLOBAL=True") endif() ament_add_pytest_test(test_sequential_reader_py "test/test_sequential_reader.py" PYTHON_EXECUTABLE "${_PYTHON_EXECUTABLE}" - APPEND_ENV "PYTHONPATH=${pythonpath}" - APPEND_LIBRARY_DIRS "${_append_library_dirs}" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/resources" + APPEND_ENV + PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR} + ${other_environment_vars} ) ament_add_pytest_test(test_sequential_writer_py "test/test_sequential_writer.py" PYTHON_EXECUTABLE "${_PYTHON_EXECUTABLE}" - APPEND_ENV "PYTHONPATH=${pythonpath}" - APPEND_LIBRARY_DIRS "${_append_library_dirs}" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/resources" + APPEND_ENV "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}" ${other_environment_vars} ) endif() diff --git a/rosbag2_py/src/rosbag2_py/_reader.cpp b/rosbag2_py/src/rosbag2_py/_reader.cpp index a14a1e1694..5c634c27ed 100644 --- a/rosbag2_py/src/rosbag2_py/_reader.cpp +++ b/rosbag2_py/src/rosbag2_py/_reader.cpp @@ -12,9 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include - #include #include #include @@ -27,6 +24,8 @@ #include "rosbag2_storage/storage_filter.hpp" #include "rosbag2_storage/topic_metadata.hpp" +#include "./pybind11.hpp" + namespace rosbag2_py { diff --git a/rosbag2_py/src/rosbag2_py/_storage.cpp b/rosbag2_py/src/rosbag2_py/_storage.cpp index bbd536dc51..12a9b9006a 100644 --- a/rosbag2_py/src/rosbag2_py/_storage.cpp +++ b/rosbag2_py/src/rosbag2_py/_storage.cpp @@ -12,9 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include - #include #include @@ -23,6 +20,8 @@ #include "rosbag2_storage/storage_filter.hpp" #include "rosbag2_storage/topic_metadata.hpp" +#include "./pybind11.hpp" + PYBIND11_MODULE(_storage, m) { m.doc() = "Python wrapper of the rosbag2 utilities API"; diff --git a/rosbag2_py/src/rosbag2_py/_writer.cpp b/rosbag2_py/src/rosbag2_py/_writer.cpp index ecae08a344..1a037acb12 100644 --- a/rosbag2_py/src/rosbag2_py/_writer.cpp +++ b/rosbag2_py/src/rosbag2_py/_writer.cpp @@ -12,9 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include - #include #include @@ -27,6 +24,8 @@ #include "rosbag2_storage/storage_filter.hpp" #include "rosbag2_storage/topic_metadata.hpp" +#include "./pybind11.hpp" + namespace rosbag2_py { diff --git a/rosbag2_py/src/rosbag2_py/pybind11.hpp b/rosbag2_py/src/rosbag2_py/pybind11.hpp new file mode 100644 index 0000000000..3c43ab5d54 --- /dev/null +++ b/rosbag2_py/src/rosbag2_py/pybind11.hpp @@ -0,0 +1,30 @@ +// Copyright 2020 Open Source Robotics Foundation, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_PY__PYBIND11_HPP_ +#define ROSBAG2_PY__PYBIND11_HPP_ + +// Ignore -Wunused-value for clang. +// Based on https://github.com/pybind/pybind11/issues/2225 +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-value" +#endif +#include +#if defined(__clang__) +#pragma clang diagnostic pop +#endif +#include + +#endif // ROSBAG2_PY__PYBIND11_HPP_ diff --git a/rosbag2_py/test/test_sequential_reader.py b/rosbag2_py/test/test_sequential_reader.py index fc2be12c2b..d592b14ff2 100644 --- a/rosbag2_py/test/test_sequential_reader.py +++ b/rosbag2_py/test/test_sequential_reader.py @@ -12,15 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os from pathlib import Path +import sys -from common import get_rosbag_options from rcl_interfaces.msg import Log from rclpy.serialization import deserialize_message -import rosbag2_py from rosidl_runtime_py.utilities import get_message from std_msgs.msg import String +if os.environ.get('ROSBAG2_PY_TEST_WITH_RTLD_GLOBAL', None) is not None: + # This is needed on Linux when compiling with clang/libc++. + # TL;DR This makes class_loader work when using a python extension compiled with libc++. + # + # For the fun RTTI ABI details, see https://whatofhow.wordpress.com/2015/03/17/odr-rtti-dso/. + sys.setdlopenflags(os.RTLD_GLOBAL | os.RTLD_LAZY) + +from common import get_rosbag_options # noqa +import rosbag2_py # noqa + def test_sequential_reader(): bag_path = str(Path(__file__).parent.parent / 'resources' / 'talker') diff --git a/rosbag2_py/test/test_sequential_writer.py b/rosbag2_py/test/test_sequential_writer.py index e1a12f4e53..f8cb02cbef 100644 --- a/rosbag2_py/test/test_sequential_writer.py +++ b/rosbag2_py/test/test_sequential_writer.py @@ -12,12 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -from common import get_rosbag_options +import os +import sys + from rclpy.serialization import deserialize_message, serialize_message -import rosbag2_py from rosidl_runtime_py.utilities import get_message from std_msgs.msg import String +if os.environ.get('ROSBAG2_PY_TEST_WITH_RTLD_GLOBAL', None) is not None: + # This is needed on Linux when compiling with clang/libc++. + # TL;DR This makes class_loader work when using a python extension compiled with libc++. + # + # For the fun RTTI ABI details, see https://whatofhow.wordpress.com/2015/03/17/odr-rtti-dso/. + sys.setdlopenflags(os.RTLD_GLOBAL | os.RTLD_LAZY) + +from common import get_rosbag_options # noqa +import rosbag2_py # noqa + def create_topic(writer, topic_name, topic_type, serialization_format='cdr'): """ From 4548f239f695b7b76d79b4bde166780b637d7463 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Mon, 5 Oct 2020 13:54:30 -0300 Subject: [PATCH 31/77] Fix rosbag2_py on Windows debug and stop ignoring the package (#531) Signed-off-by: Ivan Santiago Paunovic --- rosbag2_py/AMENT_IGNORE | 0 rosbag2_py/CMakeLists.txt | 26 ++++++++++++++++++++++---- rosbag2_py/package.xml | 5 ++--- 3 files changed, 24 insertions(+), 7 deletions(-) delete mode 100644 rosbag2_py/AMENT_IGNORE diff --git a/rosbag2_py/AMENT_IGNORE b/rosbag2_py/AMENT_IGNORE deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/rosbag2_py/CMakeLists.txt b/rosbag2_py/CMakeLists.txt index c9333e22cb..ac9e5179eb 100644 --- a/rosbag2_py/CMakeLists.txt +++ b/rosbag2_py/CMakeLists.txt @@ -16,13 +16,30 @@ if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") endif() find_package(ament_cmake REQUIRED) +find_package(ament_cmake_python REQUIRED) find_package(ament_cmake_ros REQUIRED) -find_package(pybind11 REQUIRED) -find_package(python_cmake_module REQUIRED) find_package(rosbag2_compression REQUIRED) find_package(rosbag2_cpp REQUIRED) find_package(rosbag2_storage REQUIRED) +# Find python before pybind11 +find_package(python_cmake_module REQUIRED) +find_package(PythonExtra REQUIRED) +if(WIN32 AND CMAKE_BUILD_TYPE STREQUAL "Debug") + # Set the python debug interpreter. + # pybind11 will setup the build for debug now. + set(PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE_DEBUG}") +endif() + +find_package(pybind11 REQUIRED) + +if(WIN32 AND CMAKE_BUILD_TYPE STREQUAL "Debug") + # pybind11 logic for setting up a debug build when both a debug and release + # python interpreter are present in the system seems to be pretty much broken. + # This works around the issue. + set(PYTHON_LIBRARIES "${PYTHON_DEBUG_LIBRARIES}") +endif() + ament_python_install_package(${PROJECT_NAME}) pybind11_add_module(_reader SHARED @@ -79,9 +96,10 @@ if(BUILD_TESTING) PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR} ${other_environment_vars} ) - ament_add_pytest_test(test_sequential_writer_py "test/test_sequential_writer.py" + ament_add_pytest_test(test_sequential_writer_py + "test/test_sequential_writer.py" PYTHON_EXECUTABLE "${_PYTHON_EXECUTABLE}" - APPEND_ENV "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}" ${other_environment_vars} + APPEND_ENV "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}" ${other_environment_vars} ) endif() diff --git a/rosbag2_py/package.xml b/rosbag2_py/package.xml index 3f940523ac..0076eabae0 100644 --- a/rosbag2_py/package.xml +++ b/rosbag2_py/package.xml @@ -11,9 +11,10 @@ Mabel Zhang ament_cmake_ros + ament_cmake_python + python_cmake_module pybind11_vendor - python_cmake_module rosbag2_compression rosbag2_cpp rosbag2_storage @@ -27,9 +28,7 @@ rclpy rosbag2_converter_default_plugins rosbag2_storage_default_plugins - rosbag2_storage rosidl_runtime_py - rpyutils std_msgs From be9c52e71a3fac7ade424ae3e7deea3b5fd0b292 Mon Sep 17 00:00:00 2001 From: Christophe Bedard Date: Mon, 5 Oct 2020 14:31:08 -0400 Subject: [PATCH 32/77] Do not expect empty StorageOptions URI to work in *CompressionWriterTest (#526) Signed-off-by: Christophe Bedard --- .../test_sequential_compression_writer.cpp | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp index c1e06088e2..f7e49c8e48 100644 --- a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp +++ b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp @@ -22,6 +22,7 @@ #include "rosbag2_compression/compression_options.hpp" #include "rosbag2_compression/sequential_compression_writer.hpp" +#include "rcpputils/filesystem_helper.hpp" #include "rosbag2_cpp/writer.hpp" #include "mock_converter_factory.hpp" @@ -41,9 +42,12 @@ class SequentialCompressionWriterTest : public Test storage_{std::make_shared>()}, converter_factory_{std::make_shared>()}, metadata_io_{std::make_unique>()}, - storage_options_{}, + tmp_dir_{rcpputils::fs::temp_directory_path() / "SequentialCompressionWriterTest"}, + tmp_dir_storage_options_{}, serialization_format_{"rmw_format"} { + tmp_dir_storage_options_.uri = tmp_dir_.string(); + rcpputils::fs::remove(tmp_dir_); ON_CALL(*storage_factory_, open_read_write(_, _)).WillByDefault(Return(storage_)); EXPECT_CALL(*storage_factory_, open_read_write(_, _)).Times(AtLeast(0)); } @@ -53,10 +57,29 @@ class SequentialCompressionWriterTest : public Test std::shared_ptr> converter_factory_; std::unique_ptr metadata_io_; std::unique_ptr writer_; - rosbag2_cpp::StorageOptions storage_options_; + rcpputils::fs::path tmp_dir_; + rosbag2_cpp::StorageOptions tmp_dir_storage_options_; std::string serialization_format_; }; +TEST_F(SequentialCompressionWriterTest, open_throws_on_empty_storage_options_uri) +{ + rosbag2_compression::CompressionOptions compression_options{ + "zstd", rosbag2_compression::CompressionMode::FILE}; + auto compression_factory = std::make_unique(); + + auto sequential_writer = std::make_unique( + compression_options, + std::move(compression_factory), + std::move(storage_factory_), + converter_factory_, + std::move(metadata_io_)); + writer_ = std::make_unique(std::move(sequential_writer)); + + EXPECT_THROW( + writer_->open(rosbag2_cpp::StorageOptions(), {serialization_format_, serialization_format_}), + std::runtime_error); +} TEST_F(SequentialCompressionWriterTest, open_throws_on_bad_compression_format) { @@ -73,8 +96,10 @@ TEST_F(SequentialCompressionWriterTest, open_throws_on_bad_compression_format) writer_ = std::make_unique(std::move(sequential_writer)); EXPECT_THROW( - writer_->open(rosbag2_cpp::StorageOptions(), {serialization_format_, serialization_format_}), + writer_->open(tmp_dir_storage_options_, {serialization_format_, serialization_format_}), std::invalid_argument); + + EXPECT_TRUE(rcpputils::fs::remove(tmp_dir_)); } TEST_F(SequentialCompressionWriterTest, open_throws_on_invalid_splitting_size) @@ -118,8 +143,14 @@ TEST_F(SequentialCompressionWriterTest, open_succeeds_on_supported_compression_f std::move(metadata_io_)); writer_ = std::make_unique(std::move(sequential_writer)); + auto tmp_dir = rcpputils::fs::temp_directory_path() / "path_not_empty"; + auto storage_options = rosbag2_cpp::StorageOptions(); + storage_options.uri = tmp_dir.string(); + EXPECT_NO_THROW( - writer_->open(rosbag2_cpp::StorageOptions(), {serialization_format_, serialization_format_})); + writer_->open(tmp_dir_storage_options_, {serialization_format_, serialization_format_})); + + EXPECT_TRUE(rcpputils::fs::remove(tmp_dir_)); } TEST_F(SequentialCompressionWriterTest, writer_calls_create_compressor) @@ -136,5 +167,8 @@ TEST_F(SequentialCompressionWriterTest, writer_calls_create_compressor) converter_factory_, std::move(metadata_io_)); writer_ = std::make_unique(std::move(sequential_writer)); - writer_->open(rosbag2_cpp::StorageOptions(), {serialization_format_, serialization_format_}); + + writer_->open(tmp_dir_storage_options_, {serialization_format_, serialization_format_}); + + EXPECT_TRUE(rcpputils::fs::remove(tmp_dir_)); } From a7514b33dba3492330498966882795e618f021c4 Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Wed, 7 Oct 2020 16:28:59 -0700 Subject: [PATCH 33/77] Update the package.xml files with the latest Open Robotics maintainers (#535) Signed-off-by: Michael Jeronimo --- .../rosbag2_performance_writer_benchmarking/package.xml | 2 ++ rosbag2_py/package.xml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml b/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml index 7970159e95..8e771db33c 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml @@ -4,6 +4,8 @@ rosbag2_performance_writer_benchmarking 0.0.2 Code to benchmark the part of rosbag2 which starts after the callback is received + Karsten Knese + Michael Jeronimo Adam Dabrowski Apache License 2.0 diff --git a/rosbag2_py/package.xml b/rosbag2_py/package.xml index 0076eabae0..c259613297 100644 --- a/rosbag2_py/package.xml +++ b/rosbag2_py/package.xml @@ -4,11 +4,13 @@ rosbag2_py 0.2.4 Python API for rosbag2 - Jacob Perron + Karsten Knese + Michael Jeronimo ROS Tooling Working Group Apache License 2.0 Mabel Zhang + Jacob Perron ament_cmake_ros ament_cmake_python From b12418859e20223cbf50a4bfcbcf90ae97d09869 Mon Sep 17 00:00:00 2001 From: Jaison Titus Date: Fri, 9 Oct 2020 11:40:43 -0700 Subject: [PATCH 34/77] SequentialWriter to cache by message size instead of message count (#530) * Fixes #464 - caches by message size and not message count Signed-off-by: Jaison Titus --- .../rosbag2_cpp/writers/sequential_writer.hpp | 4 +++- .../src/rosbag2_cpp/writers/sequential_writer.cpp | 6 ++++-- .../test/rosbag2_cpp/test_sequential_writer.cpp | 15 ++++++++++++--- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp b/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp index 1bece470bd..8617e71385 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp @@ -118,8 +118,10 @@ class ROSBAG2_CPP_PUBLIC SequentialWriter std::chrono::seconds max_bagfile_duration; // Intermediate cache to write multiple messages into the storage. - // `max_cache_size` is the amount of messages to hold in storage before writing to disk. + // `max_cache_size` is the number of bytes of messages to hold in storage + // before writing to disk. uint64_t max_cache_size_; + uint64_t current_cache_size_; std::vector> cache_; // Used to track topic -> message count diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index d8b17cb7cc..15d5e9a41e 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -89,8 +89,8 @@ void SequentialWriter::open( max_bagfile_size_ = storage_options.max_bagfile_size; max_bagfile_duration = std::chrono::seconds(storage_options.max_bagfile_duration); max_cache_size_ = storage_options.max_cache_size; - cache_.reserve(max_cache_size_); + current_cache_size_ = 0u; if (converter_options.output_serialization_format != converter_options.input_serialization_format) @@ -249,11 +249,13 @@ void SequentialWriter::write(std::shared_ptrwrite(converted_msg); } else { cache_.push_back(converted_msg); - if (cache_.size() >= max_cache_size_) { + current_cache_size_ += converted_msg->serialized_data->buffer_length; + if (current_cache_size_ >= max_cache_size_) { storage_->write(cache_); // reset cache cache_.clear(); cache_.reserve(max_cache_size_); + current_cache_size_ = 0u; } } } diff --git a/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp b/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp index 2553b4b135..fce161781d 100644 --- a/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp +++ b/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp @@ -25,6 +25,7 @@ #include "rosbag2_cpp/writer.hpp" #include "rosbag2_storage/bag_metadata.hpp" +#include "rosbag2_storage/ros_helper.hpp" #include "rosbag2_storage/topic_metadata.hpp" #include "mock_converter.hpp" @@ -265,13 +266,14 @@ TEST_F(SequentialWriterTest, writer_splits_when_storage_bagfile_size_gt_max_bagf } TEST_F(SequentialWriterTest, only_write_after_cache_is_full) { - const size_t counter = 1000; + const uint64_t counter = 1000; const uint64_t max_cache_size = 100; - + std::string msg_content = "Hello"; + const auto msg_length = msg_content.length(); EXPECT_CALL( *storage_, write(An> &>())). - Times(counter / max_cache_size); + Times(static_cast(counter * msg_length / max_cache_size)); EXPECT_CALL( *storage_, write(An>())).Times(0); @@ -284,6 +286,8 @@ TEST_F(SequentialWriterTest, only_write_after_cache_is_full) { auto message = std::make_shared(); message->topic_name = "test_topic"; + message->serialized_data = rosbag2_storage::make_serialized_message( + msg_content.c_str(), msg_length); storage_options_.max_bagfile_size = 0; storage_options_.max_cache_size = max_cache_size; @@ -314,8 +318,13 @@ TEST_F(SequentialWriterTest, do_not_use_cache_if_cache_size_is_zero) { std::string rmw_format = "rmw_format"; + std::string msg_content = "Hello"; + auto msg_length = msg_content.length(); auto message = std::make_shared(); message->topic_name = "test_topic"; + message->serialized_data = rosbag2_storage::make_serialized_message( + msg_content.c_str(), msg_length); + storage_options_.max_bagfile_size = 0; storage_options_.max_cache_size = max_cache_size; From 39c2f21d77c0c19bb11a0fd9433034f77d375686 Mon Sep 17 00:00:00 2001 From: Jaison Titus Date: Fri, 9 Oct 2020 14:04:10 -0700 Subject: [PATCH 35/77] Change default cache size for sequential_writer to a non zero value (#533) * Change default value for max-cache-size to 1MB Signed-off-by: Jaison Titus --- ros2bag/ros2bag/verb/record.py | 8 +++++--- rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp | 2 +- rosbag2_cpp/src/rosbag2_cpp/reader.cpp | 2 -- .../test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ros2bag/ros2bag/verb/record.py b/ros2bag/ros2bag/verb/record.py index 41a7a2f111..ec41a67148 100644 --- a/ros2bag/ros2bag/verb/record.py +++ b/ros2bag/ros2bag/verb/record.py @@ -67,9 +67,11 @@ def add_arguments(self, parser, cli_name): # noqa: D102 'the bag will split at whichever threshold is reached first.' ) parser.add_argument( - '--max-cache-size', type=int, default=0, - help='maximum amount of messages to hold in cache before writing to disk. ' - 'Default it is zero, writing every message directly to disk.' + '--max-cache-size', type=int, default=1024*1024, + help='maximum size (in bytes) of messages to hold in cache before writing to disk. ' + 'Default is 1 mebibyte, everytime the cache size equals or exceeds 1MB, ' + 'it will be written to disk. If the value specified is 0, then every message is' + 'directly written to disk.' ) parser.add_argument( '--compression-mode', type=str, default='none', diff --git a/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp b/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp index 57793e1fbf..4337dc45d7 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp @@ -36,7 +36,7 @@ struct StorageOptions // The cache size indiciates how many messages can maximally be hold in cache // before these being written to disk. - // Defaults to 0, and effectively disables the caching. + // A value of 0 disables caching and every write happens directly to disk. uint64_t max_cache_size = 0; }; diff --git a/rosbag2_cpp/src/rosbag2_cpp/reader.cpp b/rosbag2_cpp/src/rosbag2_cpp/reader.cpp index 08cfd1c3bd..ef2670fb86 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/reader.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/reader.cpp @@ -40,8 +40,6 @@ void Reader::open(const std::string & uri) rosbag2_cpp::StorageOptions storage_options; storage_options.uri = uri; storage_options.storage_id = "sqlite3"; - storage_options.max_bagfile_size = 0; // default - storage_options.max_cache_size = 0; // default rosbag2_cpp::ConverterOptions converter_options{}; return open(storage_options, converter_options); diff --git a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp index e2a33c8b55..e7d24f314c 100644 --- a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp +++ b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp @@ -68,6 +68,7 @@ TEST_F(RecordFixture, record_end_to_end_test_with_zstd_file_compression) { cmd << "ros2 bag record" << " --compression-mode file" << " --compression-format zstd" << + " --max-cache-size 0" << " --output " << root_bag_path_.string() << " " << topic_name; @@ -118,7 +119,7 @@ TEST_F(RecordFixture, record_end_to_end_test) { wrong_message->string_value = "wrong_content"; auto process_handle = start_execution( - "ros2 bag record --output " + root_bag_path_.string() + " /test_topic"); + "ros2 bag record --max-cache-size 0 --output " + root_bag_path_.string() + " /test_topic"); wait_for_db(); pub_man_.add_publisher("/test_topic", message, expected_test_messages); From 211429a3985dfdba5cdac4b27a474a3bfac44fee Mon Sep 17 00:00:00 2001 From: Jaison Titus Date: Fri, 9 Oct 2020 18:13:01 -0700 Subject: [PATCH 36/77] Removed duplicated code in record (#534) * Removed duplicated code for record. Signed-off-by: Jaison Titus --- ros2bag/ros2bag/verb/record.py | 75 +++++++------------ .../test_rosbag2_record_end_to_end.cpp | 10 +++ 2 files changed, 37 insertions(+), 48 deletions(-) diff --git a/ros2bag/ros2bag/verb/record.py b/ros2bag/ros2bag/verb/record.py index ec41a67148..c5cd8066fc 100644 --- a/ros2bag/ros2bag/verb/record.py +++ b/ros2bag/ros2bag/verb/record.py @@ -32,7 +32,7 @@ def add_arguments(self, parser, cli_name): # noqa: D102 '-a', '--all', action='store_true', help='recording all topics, required if no topics are listed explicitly.') parser.add_argument( - 'topics', nargs='*', help='topics to be recorded') + 'topics', nargs='*', default=None, help='topics to be recorded') parser.add_argument( '-o', '--output', help='destination of the bagfile to create, \ @@ -93,8 +93,12 @@ def add_arguments(self, parser, cli_name): # noqa: D102 self._subparser = parser def main(self, *, args): # noqa: D102 + # both all and topics cannot be true if args.all and args.topics: return print_error('Invalid choice: Can not specify topics and -a at the same time.') + # both all and topics cannot be false + if not(args.all or (args.topics and len(args.topics) > 0)): + return print_error('Invalid choice: Must specify topic(s) or -a') uri = args.output or datetime.datetime.now().strftime('rosbag2_%Y_%m_%d-%H_%M_%S') @@ -116,54 +120,29 @@ def main(self, *, args): # noqa: D102 except (InvalidQoSProfileException, ValueError) as e: return print_error(str(e)) - if args.all: - # NOTE(hidmic): in merged install workspaces on Windows, Python entrypoint lookups - # combined with constrained environments (as imposed by colcon test) - # may result in DLL loading failures when attempting to import a C - # extension. Therefore, do not import rosbag2_transport at the module - # level but on demand, right before first use. - from rosbag2_transport import rosbag2_transport_py + # NOTE(hidmic): in merged install workspaces on Windows, Python entrypoint lookups + # combined with constrained environments (as imposed by colcon test) + # may result in DLL loading failures when attempting to import a C + # extension. Therefore, do not import rosbag2_transport at the module + # level but on demand, right before first use. + from rosbag2_transport import rosbag2_transport_py - rosbag2_transport_py.record( - uri=uri, - storage_id=args.storage, - serialization_format=args.serialization_format, - node_prefix=NODE_NAME_PREFIX, - compression_mode=args.compression_mode, - compression_format=args.compression_format, - all=True, - no_discovery=args.no_discovery, - polling_interval=args.polling_interval, - max_bagfile_size=args.max_bag_size, - max_bagfile_duration=args.max_bag_duration, - max_cache_size=args.max_cache_size, - include_hidden_topics=args.include_hidden_topics, - qos_profile_overrides=qos_profile_overrides) - elif args.topics and len(args.topics) > 0: - # NOTE(hidmic): in merged install workspaces on Windows, Python entrypoint lookups - # combined with constrained environments (as imposed by colcon test) - # may result in DLL loading failures when attempting to import a C - # extension. Therefore, do not import rosbag2_transport at the module - # level but on demand, right before first use. - from rosbag2_transport import rosbag2_transport_py - - rosbag2_transport_py.record( - uri=uri, - storage_id=args.storage, - serialization_format=args.serialization_format, - node_prefix=NODE_NAME_PREFIX, - compression_mode=args.compression_mode, - compression_format=args.compression_format, - no_discovery=args.no_discovery, - polling_interval=args.polling_interval, - max_bagfile_size=args.max_bag_size, - max_bagfile_duration=args.max_bag_duration, - max_cache_size=args.max_cache_size, - topics=args.topics, - include_hidden_topics=args.include_hidden_topics, - qos_profile_overrides=qos_profile_overrides) - else: - self._subparser.print_help() + rosbag2_transport_py.record( + uri=uri, + storage_id=args.storage, + serialization_format=args.serialization_format, + node_prefix=NODE_NAME_PREFIX, + compression_mode=args.compression_mode, + compression_format=args.compression_format, + all=args.all, + no_discovery=args.no_discovery, + polling_interval=args.polling_interval, + max_bagfile_size=args.max_bag_size, + max_bagfile_duration=args.max_bag_duration, + max_cache_size=args.max_cache_size, + topics=args.topics, + include_hidden_topics=args.include_hidden_topics, + qos_profile_overrides=qos_profile_overrides) if os.path.isdir(uri) and not os.listdir(uri): os.rmdir(uri) diff --git a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp index e7d24f314c..8d6b4f8394 100644 --- a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp +++ b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp @@ -568,6 +568,16 @@ TEST_F(RecordFixture, record_fails_if_both_all_and_topic_list_is_specified) { EXPECT_THAT(error_output, HasSubstr("Can not specify topics and -a at the same time.")); } +TEST_F(RecordFixture, record_fails_if_neither_all_nor_topic_list_are_specified) { + internal::CaptureStderr(); + auto exit_code = + execute_and_wait_until_completion("ros2 bag record", temporary_dir_path_); + auto output = internal::GetCapturedStderr(); + + EXPECT_THAT(exit_code, Eq(EXIT_FAILURE)); + EXPECT_THAT(output, HasSubstr("Invalid choice: Must specify topic(s) or -a")); +} + TEST_F(RecordFixture, record_fails_gracefully_if_plugin_for_given_encoding_does_not_exist) { internal::CaptureStderr(); auto exit_code = From f5664b1b32510a7d037db12ca6f67ab42c8bb470 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Wed, 14 Oct 2020 16:07:04 -0300 Subject: [PATCH 37/77] Workaround pybind11 bug on Windows when CMAKE_BUILD_TYPE=RelWithDebInfo (#538) Signed-off-by: Ivan Santiago Paunovic --- rosbag2_py/CMakeLists.txt | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/rosbag2_py/CMakeLists.txt b/rosbag2_py/CMakeLists.txt index ac9e5179eb..17bdb7a63f 100644 --- a/rosbag2_py/CMakeLists.txt +++ b/rosbag2_py/CMakeLists.txt @@ -40,6 +40,31 @@ if(WIN32 AND CMAKE_BUILD_TYPE STREQUAL "Debug") set(PYTHON_LIBRARIES "${PYTHON_DEBUG_LIBRARIES}") endif() +function(clean_windows_flags target) + # Hack to avoid pybind11 issue. + # + # TODO(ivanpauno): + # This can be deleted when we update `pybind11_vendor` to a version including + # https://github.com/pybind/pybind11/pull/2590. + # + # They are enabling /LTCG on Windows to reduce binary size, + # but that doesn't play well with MSVC incremental linking (default for Debug/RelWithDebInfo). + # + # See: + # - https://docs.microsoft.com/en-us/cpp/build/reference/incremental-link-incrementally?view=vs-2019 + # - https://docs.microsoft.com/en-us/cpp/build/reference/ltcg-link-time-code-generation?view=vs-2019 + + if(MSVC AND "${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo") + get_target_property(target_link_libraries ${target} LINK_LIBRARIES) + list(REMOVE_ITEM target_link_libraries "$<$>:-LTCG>") + set_target_properties(${target} PROPERTIES LINK_LIBRARIES "${target_link_libraries}") + + get_target_property(target_compile_options ${target} COMPILE_OPTIONS) + list(REMOVE_ITEM target_compile_options "$<$>:/GL>") + set_target_properties(${target} PROPERTIES COMPILE_OPTIONS "${target_compile_options}") + endif() +endfunction() + ament_python_install_package(${PROJECT_NAME}) pybind11_add_module(_reader SHARED @@ -49,6 +74,7 @@ ament_target_dependencies(_reader PUBLIC "rosbag2_compression" "rosbag2_cpp" ) +clean_windows_flags(_reader) pybind11_add_module(_storage SHARED src/rosbag2_py/_storage.cpp @@ -57,6 +83,7 @@ ament_target_dependencies(_storage PUBLIC "rosbag2_cpp" "rosbag2_storage" ) +clean_windows_flags(_storage) pybind11_add_module(_writer SHARED src/rosbag2_py/_writer.cpp @@ -65,6 +92,7 @@ ament_target_dependencies(_writer PUBLIC "rosbag2_compression" "rosbag2_cpp" ) +clean_windows_flags(_writer) # Install cython modules as sub-modules of the project install( From 1413139ff8a734aa7258266b26108453dc0c04bd Mon Sep 17 00:00:00 2001 From: Jacob Perron Date: Mon, 19 Oct 2020 14:46:11 -0700 Subject: [PATCH 38/77] Add record test for ros2bag (#523) * Add record test for ros2bag A nominal smoke test to confirm that the tool is working. Signed-off-by: Jacob Perron * Remove unused import Signed-off-by: Jacob Perron * Fix lint Signed-off-by: Jacob Perron * Expect exit code 2 from rclcpp Signed-off-by: Jacob Perron * Remove unused import Signed-off-by: Jacob Perron * Fix test for Windows Signed-off-by: Jacob Perron --- ros2bag/test/test_record.py | 80 +++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 ros2bag/test/test_record.py diff --git a/ros2bag/test/test_record.py b/ros2bag/test/test_record.py new file mode 100644 index 0000000000..51cb400b7e --- /dev/null +++ b/ros2bag/test/test_record.py @@ -0,0 +1,80 @@ +# Copyright 2020 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path +import shutil +import tempfile + +import unittest + +from launch import LaunchDescription +from launch.actions import ExecuteProcess + +import launch_testing +import launch_testing.actions +import launch_testing.asserts +from launch_testing.asserts import EXIT_OK + +import pytest + + +@pytest.mark.launch_test +def generate_test_description(): + tmp_dir_name = tempfile.mkdtemp() + output_path = Path(tmp_dir_name) / 'ros2bag_test_record' + record_all_process = ExecuteProcess( + cmd=['ros2', 'bag', 'record', '-a', '--output', output_path.as_posix()], + name='ros2bag-cli', + output='screen', + ) + + return LaunchDescription([ + record_all_process, + launch_testing.actions.ReadyToTest() + ]), locals() + + +class TestRecord(unittest.TestCase): + + def test_output(self, record_all_process, proc_output): + proc_output.assertWaitFor( + 'Listening for topics...', + process=record_all_process + ) + proc_output.assertWaitFor( + "Subscribed to topic '/rosout'", + process=record_all_process + ) + proc_output.assertWaitFor( + "Subscribed to topic '/parameter_events'", + process=record_all_process + ) + + +@launch_testing.post_shutdown_test() +class TestRecordAfterShutdown(unittest.TestCase): + + def test_exit_code(self, tmp_dir_name, record_all_process, proc_info): + # Cleanup + shutil.rmtree(tmp_dir_name, ignore_errors=True) + + # Check that the process exited with code 0 + launch_testing.asserts.assertExitCodes( + proc_info, + # SIGINT (2) is the typical exit code we see coming from rclcpp + # On Windows, we get value '1' + allowable_exit_codes=[EXIT_OK, 2] if os.name != 'nt' else [EXIT_OK, 1, 2], + process=record_all_process + ) From dee8ca1f7d737f422280eac5ac0a40db8445229c Mon Sep 17 00:00:00 2001 From: tomoya Date: Tue, 20 Oct 2020 14:22:05 +0900 Subject: [PATCH 39/77] if cache data exists, it needs to flush the data into the storage before shutdown (#541) fix #540 * if cache data exists, it needs to flush the data into the storage before shutdown. * cache_ needs to reset after writing to storage. * add reset_cache() method. Signed-off-by: Tomoya.Fujita --- .../rosbag2_cpp/writers/sequential_writer.hpp | 3 +++ .../rosbag2_cpp/writers/sequential_writer.cpp | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp b/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp index 8617e71385..b52d523dcd 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp @@ -141,6 +141,9 @@ class ROSBAG2_CPP_PUBLIC SequentialWriter // Record TopicInformation into metadata void finalize_metadata(); + // Flush data into storage, and reset cache + void reset_cache(); + // Helper method used by write to get the message in a format that is ready to be written. // Common use cases include converting the message using the converter or // performing other operations like compression on it diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index 15d5e9a41e..e460bbebbc 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -140,6 +140,7 @@ void SequentialWriter::reset() metadata_io_->write_metadata(base_folder_, metadata_); } + reset_cache(); storage_.reset(); // Necessary to ensure that the storage is destroyed before the factory storage_factory_.reset(); } @@ -251,11 +252,7 @@ void SequentialWriter::write(std::shared_ptrserialized_data->buffer_length; if (current_cache_size_ >= max_cache_size_) { - storage_->write(cache_); - // reset cache - cache_.clear(); - cache_.reserve(max_cache_size_); - current_cache_size_ = 0u; + reset_cache(); } } } @@ -312,5 +309,16 @@ void SequentialWriter::finalize_metadata() } } +void SequentialWriter::reset_cache() +{ + // if cache data exists, it must flush the data into the storage + if (!cache_.empty()) { + storage_->write(cache_); + // reset cache + cache_.clear(); + cache_.reserve(max_cache_size_); + current_cache_size_ = 0u; + } +} } // namespace writers } // namespace rosbag2_cpp From 63a1776524e5057daf3d66385b490ca06ff3b731 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Wed, 28 Oct 2020 10:43:45 -0300 Subject: [PATCH 40/77] Update deprecated qos policy value names (#548) Signed-off-by: Ivan Santiago Paunovic --- ros2bag/test/test_api.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ros2bag/test/test_api.py b/ros2bag/test/test_api.py index f90a9dbe40..b9f9a7d512 100644 --- a/ros2bag/test/test_api.py +++ b/ros2bag/test/test_api.py @@ -38,7 +38,7 @@ def test_dict_to_duration_invalid(self): def test_interpret_dict_as_qos_profile_valid(self): qos_dict = {'history': 'keep_last', 'depth': 10} qos_profile = interpret_dict_as_qos_profile(qos_dict) - assert qos_profile.history == QoSHistoryPolicy.RMW_QOS_POLICY_HISTORY_KEEP_LAST + assert qos_profile.history == QoSHistoryPolicy.KEEP_LAST expected_seconds = 1 expected_nanoseconds = int((expected_seconds * 1e9)) qos_dict = {'history': 'keep_all', 'deadline': {'sec': expected_seconds, 'nsec': 0}} @@ -66,14 +66,14 @@ def test_convert_yaml_to_qos_profile(self): } qos_profiles = convert_yaml_to_qos_profile(qos_dict) assert qos_profiles[topic_name_1].durability == \ - QoSDurabilityPolicy.RMW_QOS_POLICY_DURABILITY_VOLATILE + QoSDurabilityPolicy.VOLATILE assert qos_profiles[topic_name_1].reliability == \ - QoSReliabilityPolicy.RMW_QOS_POLICY_RELIABILITY_RELIABLE + QoSReliabilityPolicy.RELIABLE assert qos_profiles[topic_name_1].history == \ - QoSHistoryPolicy.RMW_QOS_POLICY_HISTORY_KEEP_ALL + QoSHistoryPolicy.KEEP_ALL assert qos_profiles[topic_name_2].avoid_ros_namespace_conventions == expected_convention assert qos_profiles[topic_name_2].history == \ - QoSHistoryPolicy.RMW_QOS_POLICY_HISTORY_KEEP_ALL + QoSHistoryPolicy.KEEP_ALL def test_interpret_dict_as_qos_profile_negative(self): qos_dict = {'history': 'keep_all', 'depth': -1} From bf93d7a68e3f0f3f2695c927d7b3540d1e28946b Mon Sep 17 00:00:00 2001 From: Karsten Knese Date: Wed, 28 Oct 2020 09:44:10 -0700 Subject: [PATCH 41/77] add storage_config_uri (#493) * add storage_config_uri Signed-off-by: Karsten Knese * linters and tests Signed-off-by: Karsten Knese * move storage options to rosbag2_storage Signed-off-by: Karsten Knese * use storage options to open storage backends Signed-off-by: Karsten Knese * add rosbag2_py to metapackage Signed-off-by: Karsten Knese --- ros2bag/ros2bag/verb/play.py | 10 +++- ros2bag/ros2bag/verb/record.py | 10 +++- rosbag2/package.xml | 1 + .../sequential_compression_reader.hpp | 4 +- .../sequential_compression_writer.hpp | 4 +- .../sequential_compression_reader.cpp | 8 +-- .../sequential_compression_writer.cpp | 10 ++-- .../test/rosbag2_compression/mock_storage.hpp | 4 +- .../mock_storage_factory.hpp | 8 +-- .../test_sequential_compression_reader.cpp | 18 +++---- .../test_sequential_compression_writer.cpp | 19 ++++--- rosbag2_cpp/include/rosbag2_cpp/reader.hpp | 4 +- .../base_reader_interface.hpp | 5 +- .../rosbag2_cpp/readers/sequential_reader.hpp | 4 +- rosbag2_cpp/include/rosbag2_cpp/writer.hpp | 4 +- .../base_writer_interface.hpp | 5 +- .../rosbag2_cpp/writers/sequential_writer.hpp | 11 ++--- rosbag2_cpp/src/rosbag2_cpp/converter.cpp | 2 +- rosbag2_cpp/src/rosbag2_cpp/info.cpp | 2 +- rosbag2_cpp/src/rosbag2_cpp/reader.cpp | 4 +- .../rosbag2_cpp/readers/sequential_reader.cpp | 18 ++++--- rosbag2_cpp/src/rosbag2_cpp/writer.cpp | 7 +-- .../rosbag2_cpp/writers/sequential_writer.cpp | 49 +++++++++---------- rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp | 5 +- .../test/rosbag2_cpp/mock_storage_factory.hpp | 8 +-- .../rosbag2_cpp/test_multifile_reader.cpp | 4 +- .../rosbag2_cpp/test_sequential_reader.cpp | 8 +-- .../rosbag2_cpp/test_sequential_writer.cpp | 12 ++--- .../test_storage_without_metadata_file.cpp | 6 ++- .../src/writer_benchmark.cpp | 4 +- rosbag2_py/src/rosbag2_py/_reader.cpp | 4 +- rosbag2_py/src/rosbag2_py/_storage.cpp | 14 +++--- rosbag2_py/src/rosbag2_py/_writer.cpp | 4 +- .../rosbag2_storage/storage_factory.hpp | 4 +- .../storage_factory_interface.hpp | 4 +- .../storage_interfaces/base_io_interface.hpp | 9 +++- .../read_only_interface.hpp | 4 +- .../read_write_interface.hpp | 4 +- .../rosbag2_storage}/storage_options.hpp | 14 ++++-- .../impl/storage_factory_impl.hpp | 35 +++++++------ .../src/rosbag2_storage/storage_factory.cpp | 8 +-- .../test/rosbag2_storage/test_plugin.cpp | 6 ++- .../test/rosbag2_storage/test_plugin.hpp | 4 +- .../rosbag2_storage/test_read_only_plugin.cpp | 6 ++- .../rosbag2_storage/test_read_only_plugin.hpp | 4 +- .../rosbag2_storage/test_storage_factory.cpp | 12 ++--- .../sqlite/sqlite_storage.hpp | 2 +- .../sqlite/sqlite_storage.cpp | 12 +++-- .../sqlite/storage_test_fixture.hpp | 7 ++- .../sqlite/test_sqlite_storage.cpp | 33 ++++++++----- .../test/rosbag2_tests/record_fixture.hpp | 4 +- .../rosbag2_transport/rosbag2_transport.hpp | 11 +++-- .../rosbag2_transport/storage_options.hpp | 27 ---------- .../rosbag2_transport/rosbag2_transport.cpp | 4 +- .../rosbag2_transport_python.cpp | 22 ++++++--- .../mock_sequential_reader.hpp | 2 +- .../mock_sequential_writer.hpp | 2 +- .../rosbag2_transport_test_fixture.hpp | 5 +- .../rosbag2_transport/test_rosbag2_node.cpp | 3 +- 59 files changed, 297 insertions(+), 231 deletions(-) rename {rosbag2_cpp/include/rosbag2_cpp => rosbag2_storage/include/rosbag2_storage}/storage_options.hpp (80%) delete mode 100644 rosbag2_transport/include/rosbag2_transport/storage_options.hpp diff --git a/ros2bag/ros2bag/verb/play.py b/ros2bag/ros2bag/verb/play.py index 3d66208e1e..055d5b5e8c 100644 --- a/ros2bag/ros2bag/verb/play.py +++ b/ros2bag/ros2bag/verb/play.py @@ -56,6 +56,9 @@ def add_arguments(self, parser, cli_name): # noqa: D102 '--remap', '-m', default='', nargs='+', help='list of topics to be remapped: in the form ' '"old_topic1:=new_topic1 old_topic2:=new_topic2 etc." ') + parser.add_argument( + '--storage-config-file', type=FileType('r'), + help='Path to a yaml file defining storage specific configurations.') def main(self, *, args): # noqa: D102 qos_profile_overrides = {} # Specify a valid default @@ -67,6 +70,10 @@ def main(self, *, args): # noqa: D102 except (InvalidQoSProfileException, ValueError) as e: return print_error(str(e)) + storage_config_file = '' + if args.storage_config_file: + storage_config_file = args.storage_config_file.name + # NOTE(hidmic): in merged install workspaces on Windows, Python entrypoint lookups # combined with constrained environments (as imposed by colcon test) # may result in DLL loading failures when attempting to import a C @@ -82,4 +89,5 @@ def main(self, *, args): # noqa: D102 topics=args.topics, qos_profile_overrides=qos_profile_overrides, loop=args.loop, - topic_remapping=args.remap) + topic_remapping=args.remap, + storage_config_file=storage_config_file) diff --git a/ros2bag/ros2bag/verb/record.py b/ros2bag/ros2bag/verb/record.py index c5cd8066fc..8621bed3d5 100644 --- a/ros2bag/ros2bag/verb/record.py +++ b/ros2bag/ros2bag/verb/record.py @@ -90,6 +90,9 @@ def add_arguments(self, parser, cli_name): # noqa: D102 '--qos-profile-overrides-path', type=FileType('r'), help='Path to a yaml file defining overrides of the QoS profile for specific topics.' ) + parser.add_argument( + '--storage-config-file', type=FileType('r'), + help='Path to a yaml file defining storage specific configurations.') self._subparser = parser def main(self, *, args): # noqa: D102 @@ -120,6 +123,10 @@ def main(self, *, args): # noqa: D102 except (InvalidQoSProfileException, ValueError) as e: return print_error(str(e)) + storage_config_file = '' + if args.storage_config_file: + storage_config_file = args.storage_config_file.name + # NOTE(hidmic): in merged install workspaces on Windows, Python entrypoint lookups # combined with constrained environments (as imposed by colcon test) # may result in DLL loading failures when attempting to import a C @@ -142,7 +149,8 @@ def main(self, *, args): # noqa: D102 max_cache_size=args.max_cache_size, topics=args.topics, include_hidden_topics=args.include_hidden_topics, - qos_profile_overrides=qos_profile_overrides) + qos_profile_overrides=qos_profile_overrides, + storage_config_file=storage_config_file) if os.path.isdir(uri) and not os.listdir(uri): os.rmdir(uri) diff --git a/rosbag2/package.xml b/rosbag2/package.xml index 27181aad39..c4266fec25 100644 --- a/rosbag2/package.xml +++ b/rosbag2/package.xml @@ -14,6 +14,7 @@ rosbag2_compression rosbag2_converter_default_plugins rosbag2_cpp + rosbag2_py rosbag2_storage rosbag2_storage_default_plugins rosbag2_transport diff --git a/rosbag2_compression/include/rosbag2_compression/sequential_compression_reader.hpp b/rosbag2_compression/include/rosbag2_compression/sequential_compression_reader.hpp index 90e9fd435b..2e8251b25d 100644 --- a/rosbag2_compression/include/rosbag2_compression/sequential_compression_reader.hpp +++ b/rosbag2_compression/include/rosbag2_compression/sequential_compression_reader.hpp @@ -63,7 +63,7 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionReader virtual ~SequentialCompressionReader(); void open( - const rosbag2_cpp::StorageOptions & storage_options, + const rosbag2_storage::StorageOptions & storage_options, const rosbag2_cpp::ConverterOptions & converter_options) override; std::shared_ptr read_next() override; @@ -90,6 +90,8 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionReader rosbag2_compression::CompressionMode compression_mode_{ rosbag2_compression::CompressionMode::NONE}; std::unique_ptr compression_factory_{}; + + rosbag2_storage::StorageOptions storage_options_; }; } // namespace rosbag2_compression diff --git a/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp b/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp index f59641731a..2bde7076fe 100644 --- a/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp +++ b/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp @@ -24,12 +24,12 @@ #include "rosbag2_cpp/converter_options.hpp" #include "rosbag2_cpp/serialization_format_converter_factory.hpp" #include "rosbag2_cpp/serialization_format_converter_factory_interface.hpp" -#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_cpp/writers/sequential_writer.hpp" #include "rosbag2_storage/metadata_io.hpp" #include "rosbag2_storage/storage_factory.hpp" #include "rosbag2_storage/storage_factory_interface.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/storage_interfaces/base_io_interface.hpp" #include "rosbag2_compression/compression_options.hpp" @@ -72,7 +72,7 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionWriter * \param converter_options options to define in which format incoming messages are stored **/ void open( - const rosbag2_cpp::StorageOptions & storage_options, + const rosbag2_storage::StorageOptions & storage_options, const rosbag2_cpp::ConverterOptions & converter_options) override; /** diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp index 3b87c13175..8d1adae926 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp @@ -59,9 +59,11 @@ void SequentialCompressionReader::setup_decompression() } void SequentialCompressionReader::open( - const rosbag2_cpp::StorageOptions & storage_options, + const rosbag2_storage::StorageOptions & storage_options, const rosbag2_cpp::ConverterOptions & converter_options) { + storage_options_ = storage_options; + if (metadata_io_->metadata_file_exists(storage_options.uri)) { metadata_ = metadata_io_->read_metadata(storage_options.uri); if (metadata_.relative_file_paths.empty()) { @@ -72,8 +74,8 @@ void SequentialCompressionReader::open( current_file_iterator_ = file_paths_.begin(); setup_decompression(); - storage_ = storage_factory_->open_read_only( - *current_file_iterator_, metadata_.storage_identifier); + storage_options_.uri = *current_file_iterator_; + storage_ = storage_factory_->open_read_only(storage_options); if (!storage_) { std::stringstream errmsg; errmsg << "No storage could be initialized for: \"" << diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp index fe0848616f..8c99a403ff 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp @@ -28,8 +28,8 @@ #include "rosbag2_compression/zstd_compressor.hpp" #include "rosbag2_cpp/info.hpp" -#include "rosbag2_cpp/storage_options.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/storage_interfaces/read_write_interface.hpp" #include "logging.hpp" @@ -96,7 +96,7 @@ void SequentialCompressionWriter::setup_compression() } void SequentialCompressionWriter::open( - const rosbag2_cpp::StorageOptions & storage_options, + const rosbag2_storage::StorageOptions & storage_options, const rosbag2_cpp::ConverterOptions & converter_options) { SequentialWriter::open(storage_options, converter_options); @@ -155,11 +155,11 @@ void SequentialCompressionWriter::compress_last_file() void SequentialCompressionWriter::split_bagfile() { - const auto storage_uri = format_storage_uri( + storage_options_.uri = format_storage_uri( base_folder_, metadata_.relative_file_paths.size()); - storage_ = storage_factory_->open_read_write(storage_uri, metadata_.storage_identifier); + storage_ = storage_factory_->open_read_write(storage_options_); if (compression_options_.compression_mode == rosbag2_compression::CompressionMode::FILE) { compress_last_file(); @@ -171,7 +171,7 @@ void SequentialCompressionWriter::split_bagfile() should_compress_last_file_ = false; std::stringstream errmsg; - errmsg << "Failed to rollover bagfile to new file: \"" << storage_uri << "\"!"; + errmsg << "Failed to rollover bagfile to new file: \"" << storage_options_.uri << "\"!"; throw std::runtime_error{errmsg.str()}; } diff --git a/rosbag2_compression/test/rosbag2_compression/mock_storage.hpp b/rosbag2_compression/test/rosbag2_compression/mock_storage.hpp index ab24521eba..3feb383d07 100644 --- a/rosbag2_compression/test/rosbag2_compression/mock_storage.hpp +++ b/rosbag2_compression/test/rosbag2_compression/mock_storage.hpp @@ -30,7 +30,9 @@ class MockStorage : public rosbag2_storage::storage_interfaces::ReadWriteInterface { public: - MOCK_METHOD2(open, void(const std::string &, rosbag2_storage::storage_interfaces::IOFlag)); + MOCK_METHOD2( + open, + void(const rosbag2_storage::StorageOptions &, rosbag2_storage::storage_interfaces::IOFlag)); MOCK_METHOD1(create_topic, void(const rosbag2_storage::TopicMetadata &)); MOCK_METHOD1(remove_topic, void(const rosbag2_storage::TopicMetadata &)); MOCK_METHOD0(has_next, bool()); diff --git a/rosbag2_compression/test/rosbag2_compression/mock_storage_factory.hpp b/rosbag2_compression/test/rosbag2_compression/mock_storage_factory.hpp index 639d481c6f..be63785072 100644 --- a/rosbag2_compression/test/rosbag2_compression/mock_storage_factory.hpp +++ b/rosbag2_compression/test/rosbag2_compression/mock_storage_factory.hpp @@ -27,14 +27,14 @@ class MockStorageFactory : public rosbag2_storage::StorageFactoryInterface { public: - MOCK_METHOD2( + MOCK_METHOD1( open_read_only, std::shared_ptr( - const std::string &, const std::string &)); - MOCK_METHOD2( + const rosbag2_storage::StorageOptions &)); + MOCK_METHOD1( open_read_write, std::shared_ptr( - const std::string &, const std::string &)); + const rosbag2_storage::StorageOptions &)); }; #endif // ROSBAG2_COMPRESSION__MOCK_STORAGE_FACTORY_HPP_ diff --git a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_reader.cpp b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_reader.cpp index 4a142f723b..ae6ace1efe 100644 --- a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_reader.cpp +++ b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_reader.cpp @@ -51,7 +51,7 @@ class SequentialCompressionReaderTest : public Test ON_CALL(*storage_, get_all_topics_and_types()).WillByDefault(Return(topics_and_types)); ON_CALL(*storage_, read_next()).WillByDefault(Return(message)); - ON_CALL(*storage_factory_, open_read_only(_, _)).WillByDefault(Return(storage_)); + ON_CALL(*storage_factory_, open_read_only(_)).WillByDefault(Return(storage_)); } rosbag2_storage::BagMetadata construct_default_bag_metadata() const @@ -90,7 +90,7 @@ TEST_F(SequentialCompressionReaderTest, open_throws_if_unsupported_compressor) reader_ = std::make_unique(std::move(sequential_reader)); EXPECT_THROW( - reader_->open(rosbag2_cpp::StorageOptions(), {"", storage_serialization_format_}), + reader_->open(rosbag2_storage::StorageOptions(), {"", storage_serialization_format_}), std::invalid_argument); } @@ -106,7 +106,7 @@ TEST_F(SequentialCompressionReaderTest, returns_all_topics_and_types) ON_CALL(*compression_factory, create_decompressor(_)) .WillByDefault(Return(ByMove(std::move(decompressor)))); EXPECT_CALL(*compression_factory, create_decompressor(_)).Times(1); - EXPECT_CALL(*storage_factory_, open_read_only(_, _)).Times(1); + EXPECT_CALL(*storage_factory_, open_read_only(_)).Times(1); auto compression_reader = std::make_unique( std::move(compression_factory), @@ -115,7 +115,7 @@ TEST_F(SequentialCompressionReaderTest, returns_all_topics_and_types) std::move(metadata_io_)); compression_reader->open( - rosbag2_cpp::StorageOptions(), {"", storage_serialization_format_}); + rosbag2_storage::StorageOptions(), {"", storage_serialization_format_}); auto topics_and_types = compression_reader->get_all_topics_and_types(); EXPECT_FALSE(topics_and_types.empty()); @@ -137,7 +137,7 @@ TEST_F(SequentialCompressionReaderTest, open_supports_zstd_compressor) reader_ = std::make_unique(std::move(sequential_reader)); // Throws runtime_error b/c compressor can't read EXPECT_THROW( - reader_->open(rosbag2_cpp::StorageOptions(), {"", storage_serialization_format_}), + reader_->open(rosbag2_storage::StorageOptions(), {"", storage_serialization_format_}), std::runtime_error); } @@ -155,7 +155,7 @@ TEST_F(SequentialCompressionReaderTest, reader_calls_create_decompressor) ON_CALL(*compression_factory, create_decompressor(_)) .WillByDefault(Return(ByMove(std::move(decompressor)))); EXPECT_CALL(*compression_factory, create_decompressor(_)).Times(1); - EXPECT_CALL(*storage_factory_, open_read_only(_, _)).Times(1); + EXPECT_CALL(*storage_factory_, open_read_only(_)).Times(1); auto sequential_reader = std::make_unique( std::move(compression_factory), @@ -165,7 +165,7 @@ TEST_F(SequentialCompressionReaderTest, reader_calls_create_decompressor) reader_ = std::make_unique(std::move(sequential_reader)); reader_->open( - rosbag2_cpp::StorageOptions(), {"", storage_serialization_format_}); + rosbag2_storage::StorageOptions(), {"", storage_serialization_format_}); } TEST_F(SequentialCompressionReaderTest, compression_called_when_splitting_bagfile) @@ -194,7 +194,7 @@ TEST_F(SequentialCompressionReaderTest, compression_called_when_splitting_bagfil .WillByDefault(Return(ByMove(std::move(decompressor)))); EXPECT_CALL(*compression_factory, create_decompressor(_)).Times(1); // open_read_only should be called twice when opening 2 split bags - EXPECT_CALL(*storage_factory_, open_read_only(_, _)).Times(2); + EXPECT_CALL(*storage_factory_, open_read_only(_)).Times(2); // storage::has_next() is called twice when reader::has_next() is called EXPECT_CALL(*storage_, has_next()).Times(2) .WillOnce(Return(false)) // Load the next file @@ -207,7 +207,7 @@ TEST_F(SequentialCompressionReaderTest, compression_called_when_splitting_bagfil std::move(metadata_io_)); compression_reader->open( - rosbag2_cpp::StorageOptions(), {"", storage_serialization_format_}); + rosbag2_storage::StorageOptions(), {"", storage_serialization_format_}); EXPECT_EQ(compression_reader->has_next_file(), true); EXPECT_EQ(compression_reader->has_next(), true); compression_reader->read_next(); diff --git a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp index f7e49c8e48..74ba726ef2 100644 --- a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp +++ b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp @@ -19,12 +19,15 @@ #include #include +#include "rcpputils/filesystem_helper.hpp" + #include "rosbag2_compression/compression_options.hpp" #include "rosbag2_compression/sequential_compression_writer.hpp" -#include "rcpputils/filesystem_helper.hpp" #include "rosbag2_cpp/writer.hpp" +#include "rosbag2_storage/storage_options.hpp" + #include "mock_converter_factory.hpp" #include "mock_metadata_io.hpp" #include "mock_storage.hpp" @@ -48,8 +51,8 @@ class SequentialCompressionWriterTest : public Test { tmp_dir_storage_options_.uri = tmp_dir_.string(); rcpputils::fs::remove(tmp_dir_); - ON_CALL(*storage_factory_, open_read_write(_, _)).WillByDefault(Return(storage_)); - EXPECT_CALL(*storage_factory_, open_read_write(_, _)).Times(AtLeast(0)); + ON_CALL(*storage_factory_, open_read_write(_)).WillByDefault(Return(storage_)); + EXPECT_CALL(*storage_factory_, open_read_write(_)).Times(AtLeast(0)); } std::unique_ptr> storage_factory_; @@ -58,7 +61,7 @@ class SequentialCompressionWriterTest : public Test std::unique_ptr metadata_io_; std::unique_ptr writer_; rcpputils::fs::path tmp_dir_; - rosbag2_cpp::StorageOptions tmp_dir_storage_options_; + rosbag2_storage::StorageOptions tmp_dir_storage_options_; std::string serialization_format_; }; @@ -77,7 +80,9 @@ TEST_F(SequentialCompressionWriterTest, open_throws_on_empty_storage_options_uri writer_ = std::make_unique(std::move(sequential_writer)); EXPECT_THROW( - writer_->open(rosbag2_cpp::StorageOptions(), {serialization_format_, serialization_format_}), + writer_->open( + rosbag2_storage::StorageOptions(), + {serialization_format_, serialization_format_}), std::runtime_error); } @@ -112,7 +117,7 @@ TEST_F(SequentialCompressionWriterTest, open_throws_on_invalid_splitting_size) const uint64_t min_split_file_size = 10; const uint64_t max_bagfile_size = 5; ON_CALL(*storage_, get_minimum_split_file_size()).WillByDefault(Return(min_split_file_size)); - auto storage_options = rosbag2_cpp::StorageOptions{}; + auto storage_options = rosbag2_storage::StorageOptions{}; storage_options.max_bagfile_size = max_bagfile_size; storage_options.uri = "foo.bar"; @@ -144,7 +149,7 @@ TEST_F(SequentialCompressionWriterTest, open_succeeds_on_supported_compression_f writer_ = std::make_unique(std::move(sequential_writer)); auto tmp_dir = rcpputils::fs::temp_directory_path() / "path_not_empty"; - auto storage_options = rosbag2_cpp::StorageOptions(); + auto storage_options = rosbag2_storage::StorageOptions(); storage_options.uri = tmp_dir.string(); EXPECT_NO_THROW( diff --git a/rosbag2_cpp/include/rosbag2_cpp/reader.hpp b/rosbag2_cpp/include/rosbag2_cpp/reader.hpp index 781ccb0f32..42db6ab660 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/reader.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/reader.hpp @@ -22,12 +22,12 @@ #include "rosbag2_cpp/converter_options.hpp" #include "rosbag2_cpp/readers/sequential_reader.hpp" -#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_cpp/visibility_control.hpp" #include "rosbag2_storage/bag_metadata.hpp" #include "rosbag2_storage/serialized_bag_message.hpp" #include "rosbag2_storage/storage_filter.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/topic_metadata.hpp" // This is necessary because of using stl types here. It is completely safe, because @@ -87,7 +87,7 @@ class ROSBAG2_CPP_PUBLIC Reader final * \param converter_options Options for specifying the output data format */ void open( - const StorageOptions & storage_options, + const rosbag2_storage::StorageOptions & storage_options, const ConverterOptions & converter_options = ConverterOptions()); /** diff --git a/rosbag2_cpp/include/rosbag2_cpp/reader_interfaces/base_reader_interface.hpp b/rosbag2_cpp/include/rosbag2_cpp/reader_interfaces/base_reader_interface.hpp index 29dee0970c..0fcccae33a 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/reader_interfaces/base_reader_interface.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/reader_interfaces/base_reader_interface.hpp @@ -19,12 +19,12 @@ #include #include "rosbag2_cpp/converter_options.hpp" -#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_cpp/visibility_control.hpp" #include "rosbag2_storage/bag_metadata.hpp" #include "rosbag2_storage/serialized_bag_message.hpp" #include "rosbag2_storage/storage_filter.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/topic_metadata.hpp" namespace rosbag2_cpp @@ -38,7 +38,8 @@ class ROSBAG2_CPP_PUBLIC BaseReaderInterface virtual ~BaseReaderInterface() {} virtual void open( - const StorageOptions & storage_options, const ConverterOptions & converter_options) = 0; + const rosbag2_storage::StorageOptions & storage_options, + const ConverterOptions & converter_options) = 0; virtual void reset() = 0; diff --git a/rosbag2_cpp/include/rosbag2_cpp/readers/sequential_reader.hpp b/rosbag2_cpp/include/rosbag2_cpp/readers/sequential_reader.hpp index 185e512879..76db844367 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/readers/sequential_reader.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/readers/sequential_reader.hpp @@ -59,7 +59,8 @@ class ROSBAG2_CPP_PUBLIC SequentialReader virtual ~SequentialReader(); void open( - const StorageOptions & storage_options, const ConverterOptions & converter_options) override; + const rosbag2_storage::StorageOptions & storage_options, + const ConverterOptions & converter_options) override; void reset() override; @@ -143,6 +144,7 @@ class ROSBAG2_CPP_PUBLIC SequentialReader std::vector::iterator current_file_iterator_{}; // Index of file to read from private: + rosbag2_storage::StorageOptions storage_options_; std::shared_ptr converter_factory_{}; }; diff --git a/rosbag2_cpp/include/rosbag2_cpp/writer.hpp b/rosbag2_cpp/include/rosbag2_cpp/writer.hpp index ff06272f0b..941011962a 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/writer.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/writer.hpp @@ -22,11 +22,11 @@ #include #include "rosbag2_cpp/converter_options.hpp" -#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_cpp/visibility_control.hpp" #include "rosbag2_cpp/writers/sequential_writer.hpp" #include "rosbag2_storage/serialized_bag_message.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/topic_metadata.hpp" // This is necessary because of using stl types here. It is completely safe, because @@ -80,7 +80,7 @@ class ROSBAG2_CPP_PUBLIC Writer final * \param converter_options options to define in which format incoming messages are stored **/ void open( - const StorageOptions & storage_options, + const rosbag2_storage::StorageOptions & storage_options, const ConverterOptions & converter_options = ConverterOptions()); /** diff --git a/rosbag2_cpp/include/rosbag2_cpp/writer_interfaces/base_writer_interface.hpp b/rosbag2_cpp/include/rosbag2_cpp/writer_interfaces/base_writer_interface.hpp index 5373e6443a..4801f28d6b 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/writer_interfaces/base_writer_interface.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/writer_interfaces/base_writer_interface.hpp @@ -18,10 +18,10 @@ #include #include "rosbag2_cpp/converter_options.hpp" -#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_cpp/visibility_control.hpp" #include "rosbag2_storage/serialized_bag_message.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/topic_metadata.hpp" namespace rosbag2_cpp @@ -35,7 +35,8 @@ class ROSBAG2_CPP_PUBLIC BaseWriterInterface virtual ~BaseWriterInterface() {} virtual void open( - const StorageOptions & storage_options, const ConverterOptions & converter_options) = 0; + const rosbag2_storage::StorageOptions & storage_options, + const ConverterOptions & converter_options) = 0; virtual void reset() = 0; diff --git a/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp b/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp index b52d523dcd..0938675eaa 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp @@ -22,13 +22,13 @@ #include "rosbag2_cpp/converter.hpp" #include "rosbag2_cpp/serialization_format_converter_factory.hpp" -#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_cpp/writer_interfaces/base_writer_interface.hpp" #include "rosbag2_cpp/visibility_control.hpp" #include "rosbag2_storage/metadata_io.hpp" #include "rosbag2_storage/storage_factory.hpp" #include "rosbag2_storage/storage_factory_interface.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/storage_interfaces/read_write_interface.hpp" // This is necessary because of using stl types here. It is completely safe, because @@ -71,7 +71,8 @@ class ROSBAG2_CPP_PUBLIC SequentialWriter * \param converter_options options to define in which format incoming messages are stored **/ void open( - const StorageOptions & storage_options, const ConverterOptions & converter_options) override; + const rosbag2_storage::StorageOptions & storage_options, + const ConverterOptions & converter_options) override; void reset() override; @@ -110,17 +111,13 @@ class ROSBAG2_CPP_PUBLIC SequentialWriter std::unique_ptr metadata_io_; std::unique_ptr converter_; - // Used in bagfile splitting; specifies the best-effort maximum sub-section of a bagfile in bytes. - uint64_t max_bagfile_size_; + rosbag2_storage::StorageOptions storage_options_; // Used in bagfile splitting; // specifies the best-effort maximum duration of a bagfile in seconds. std::chrono::seconds max_bagfile_duration; // Intermediate cache to write multiple messages into the storage. - // `max_cache_size` is the number of bytes of messages to hold in storage - // before writing to disk. - uint64_t max_cache_size_; uint64_t current_cache_size_; std::vector> cache_; diff --git a/rosbag2_cpp/src/rosbag2_cpp/converter.cpp b/rosbag2_cpp/src/rosbag2_cpp/converter.cpp index 908dbcec37..d250f1911a 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/converter.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/converter.cpp @@ -22,10 +22,10 @@ #include "rosbag2_cpp/info.hpp" #include "rosbag2_cpp/typesupport_helpers.hpp" -#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_storage/metadata_io.hpp" #include "rosbag2_storage/ros_helper.hpp" +#include "rosbag2_storage/storage_options.hpp" namespace rosbag2_cpp { diff --git a/rosbag2_cpp/src/rosbag2_cpp/info.cpp b/rosbag2_cpp/src/rosbag2_cpp/info.cpp index 6e9f229d00..fed9626c5b 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/info.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/info.cpp @@ -35,7 +35,7 @@ rosbag2_storage::BagMetadata Info::read_metadata( if (!storage_id.empty()) { rosbag2_storage::StorageFactory factory; std::shared_ptr storage; - storage = factory.open_read_only(uri, storage_id); + storage = factory.open_read_only({uri, storage_id}); if (!storage) { throw std::runtime_error( "The metadata.yaml file does not exist and the bag could not be " diff --git a/rosbag2_cpp/src/rosbag2_cpp/reader.cpp b/rosbag2_cpp/src/rosbag2_cpp/reader.cpp index ef2670fb86..79408da941 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/reader.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/reader.cpp @@ -37,7 +37,7 @@ Reader::~Reader() void Reader::open(const std::string & uri) { - rosbag2_cpp::StorageOptions storage_options; + rosbag2_storage::StorageOptions storage_options; storage_options.uri = uri; storage_options.storage_id = "sqlite3"; @@ -46,7 +46,7 @@ void Reader::open(const std::string & uri) } void Reader::open( - const StorageOptions & storage_options, + const rosbag2_storage::StorageOptions & storage_options, const ConverterOptions & converter_options) { reader_impl_->open(storage_options, converter_options); diff --git a/rosbag2_cpp/src/rosbag2_cpp/readers/sequential_reader.cpp b/rosbag2_cpp/src/rosbag2_cpp/readers/sequential_reader.cpp index 35c09c52f7..ea88b5a129 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/readers/sequential_reader.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/readers/sequential_reader.cpp @@ -80,8 +80,11 @@ void SequentialReader::reset() } void SequentialReader::open( - const StorageOptions & storage_options, const ConverterOptions & converter_options) + const rosbag2_storage::StorageOptions & storage_options, + const ConverterOptions & converter_options) { + storage_options_ = storage_options; + // If there is a metadata.yaml file present, load it. // If not, let's ask the storage with the given URI for its metadata. // This is necessary for non ROS2 bags (aka ROS1 legacy bags). @@ -96,14 +99,13 @@ void SequentialReader::open( storage_options.uri, metadata_.relative_file_paths, metadata_.version); current_file_iterator_ = file_paths_.begin(); - storage_ = storage_factory_->open_read_only( - get_current_file(), storage_options.storage_id); + storage_options_.uri = get_current_file(); + storage_ = storage_factory_->open_read_only(storage_options_); if (!storage_) { throw std::runtime_error{"No storage could be initialized. Abort"}; } } else { - storage_ = storage_factory_->open_read_only( - storage_options.uri, storage_options.storage_id); + storage_ = storage_factory_->open_read_only(storage_options_); if (!storage_) { throw std::runtime_error{"No storage could be initialized. Abort"}; } @@ -136,9 +138,9 @@ bool SequentialReader::has_next() // to read from there. Otherwise, check if there's another message. if (!storage_->has_next() && has_next_file()) { load_next_file(); - storage_ = storage_factory_->open_read_only( - get_current_file(), metadata_.storage_identifier); - storage_->set_filter(topics_filter_); + storage_options_.uri = get_current_file(); + + storage_ = storage_factory_->open_read_only(storage_options_); } return storage_->has_next(); diff --git a/rosbag2_cpp/src/rosbag2_cpp/writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writer.cpp index 9c472d7dd3..37fa4b6693 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writer.cpp @@ -22,10 +22,10 @@ #include #include "rosbag2_cpp/info.hpp" -#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_cpp/writer_interfaces/base_writer_interface.hpp" #include "rosbag2_storage/serialized_bag_message.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/topic_metadata.hpp" namespace rosbag2_cpp @@ -44,7 +44,7 @@ Writer::~Writer() void Writer::open(const std::string & uri) { - rosbag2_cpp::StorageOptions storage_options; + rosbag2_storage::StorageOptions storage_options; storage_options.uri = uri; storage_options.storage_id = kDefaultStorageID; @@ -53,7 +53,8 @@ void Writer::open(const std::string & uri) } void Writer::open( - const StorageOptions & storage_options, const ConverterOptions & converter_options) + const rosbag2_storage::StorageOptions & storage_options, + const ConverterOptions & converter_options) { writer_impl_->open(storage_options, converter_options); } diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index e460bbebbc..791445dfa5 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -24,7 +24,8 @@ #include "rcpputils/filesystem_helper.hpp" #include "rosbag2_cpp/info.hpp" -#include "rosbag2_cpp/storage_options.hpp" + +#include "rosbag2_storage/storage_options.hpp" namespace rosbag2_cpp { @@ -60,9 +61,6 @@ SequentialWriter::SequentialWriter( storage_(nullptr), metadata_io_(std::move(metadata_io)), converter_(nullptr), - max_bagfile_size_(rosbag2_storage::storage_interfaces::MAX_BAGFILE_SIZE_NO_SPLIT), - max_bagfile_duration( - std::chrono::seconds(rosbag2_storage::storage_interfaces::MAX_BAGFILE_DURATION_NO_SPLIT)), topics_names_to_info_(), metadata_() {} @@ -82,14 +80,13 @@ void SequentialWriter::init_metadata() } void SequentialWriter::open( - const StorageOptions & storage_options, + const rosbag2_storage::StorageOptions & storage_options, const ConverterOptions & converter_options) { base_folder_ = storage_options.uri; - max_bagfile_size_ = storage_options.max_bagfile_size; - max_bagfile_duration = std::chrono::seconds(storage_options.max_bagfile_duration); - max_cache_size_ = storage_options.max_cache_size; - cache_.reserve(max_cache_size_); + storage_options_ = storage_options; + + cache_.reserve(storage_options.max_cache_size); current_cache_size_ = 0u; if (converter_options.output_serialization_format != @@ -98,7 +95,7 @@ void SequentialWriter::open( converter_ = std::make_unique(converter_options, converter_factory_); } - rcpputils::fs::path db_path(base_folder_); + rcpputils::fs::path db_path(storage_options.uri); if (db_path.is_directory()) { std::stringstream error; error << "Database directory already exists (" << db_path.string() << @@ -113,15 +110,14 @@ void SequentialWriter::open( throw std::runtime_error{error.str()}; } - const auto storage_uri = format_storage_uri(base_folder_, 0); - - storage_ = storage_factory_->open_read_write(storage_uri, storage_options.storage_id); + storage_options_.uri = format_storage_uri(base_folder_, 0); + storage_ = storage_factory_->open_read_write(storage_options_); if (!storage_) { throw std::runtime_error("No storage could be initialized. Abort"); } - if (max_bagfile_size_ != 0 && - max_bagfile_size_ < storage_->get_minimum_split_file_size()) + if (storage_options_.max_bagfile_size != 0 && + storage_options_.max_bagfile_size < storage_->get_minimum_split_file_size()) { std::stringstream error; error << "Invalid bag splitting size given. Please provide a value greater than " << @@ -194,14 +190,14 @@ void SequentialWriter::remove_topic(const rosbag2_storage::TopicMetadata & topic void SequentialWriter::split_bagfile() { - const auto storage_uri = format_storage_uri( + storage_options_.uri = format_storage_uri( base_folder_, metadata_.relative_file_paths.size()); - storage_ = storage_factory_->open_read_write(storage_uri, metadata_.storage_identifier); + storage_ = storage_factory_->open_read_write(storage_options_); if (!storage_) { std::stringstream errmsg; - errmsg << "Failed to rollover bagfile to new file: \"" << storage_uri << "\"!"; + errmsg << "Failed to rollover bagfile to new file: \"" << storage_options_.uri << "\"!"; throw std::runtime_error(errmsg.str()); } @@ -246,12 +242,12 @@ void SequentialWriter::write(std::shared_ptrwrite(converted_msg); } else { cache_.push_back(converted_msg); current_cache_size_ += converted_msg->serialized_data->buffer_length; - if (current_cache_size_ >= max_cache_size_) { + if (current_cache_size_ >= storage_options_.max_cache_size) { reset_cache(); } } @@ -270,16 +266,19 @@ bool SequentialWriter::should_split_bagfile() const bool should_split = false; // Splitting by size - if (max_bagfile_size_ != rosbag2_storage::storage_interfaces::MAX_BAGFILE_SIZE_NO_SPLIT) { - should_split = should_split || (storage_->get_bagfile_size() > max_bagfile_size_); + if (storage_options_.max_bagfile_size != + rosbag2_storage::storage_interfaces::MAX_BAGFILE_SIZE_NO_SPLIT) + { + should_split = should_split || + (storage_->get_bagfile_size() > storage_options_.max_bagfile_size); } // Splitting by time - if (max_bagfile_duration != std::chrono::seconds( - rosbag2_storage::storage_interfaces::MAX_BAGFILE_DURATION_NO_SPLIT)) + if (storage_options_.max_bagfile_duration != + rosbag2_storage::storage_interfaces::MAX_BAGFILE_DURATION_NO_SPLIT) { auto max_duration_ns = std::chrono::duration_cast( - max_bagfile_duration); + std::chrono::seconds(storage_options_.max_bagfile_duration)); should_split = should_split || ((std::chrono::high_resolution_clock::now() - metadata_.starting_time) > max_duration_ns); } diff --git a/rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp b/rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp index 601a99c4c3..84bbf47ac3 100644 --- a/rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp +++ b/rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp @@ -24,13 +24,16 @@ #include "rosbag2_storage/bag_metadata.hpp" #include "rosbag2_storage/serialized_bag_message.hpp" #include "rosbag2_storage/storage_filter.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/storage_interfaces/read_write_interface.hpp" #include "rosbag2_storage/topic_metadata.hpp" class MockStorage : public rosbag2_storage::storage_interfaces::ReadWriteInterface { public: - MOCK_METHOD2(open, void(const std::string &, rosbag2_storage::storage_interfaces::IOFlag)); + MOCK_METHOD2( + open, + void(const rosbag2_storage::StorageOptions &, rosbag2_storage::storage_interfaces::IOFlag)); MOCK_METHOD1(create_topic, void(const rosbag2_storage::TopicMetadata &)); MOCK_METHOD1(remove_topic, void(const rosbag2_storage::TopicMetadata &)); MOCK_METHOD0(has_next, bool()); diff --git a/rosbag2_cpp/test/rosbag2_cpp/mock_storage_factory.hpp b/rosbag2_cpp/test/rosbag2_cpp/mock_storage_factory.hpp index b1a844a254..7284c684d5 100644 --- a/rosbag2_cpp/test/rosbag2_cpp/mock_storage_factory.hpp +++ b/rosbag2_cpp/test/rosbag2_cpp/mock_storage_factory.hpp @@ -27,14 +27,14 @@ class MockStorageFactory : public rosbag2_storage::StorageFactoryInterface { public: - MOCK_METHOD2( + MOCK_METHOD1( open_read_only, std::shared_ptr( - const std::string &, const std::string &)); - MOCK_METHOD2( + const rosbag2_storage::StorageOptions &)); + MOCK_METHOD1( open_read_write, std::shared_ptr( - const std::string &, const std::string &)); + const rosbag2_storage::StorageOptions &)); }; #endif // ROSBAG2_CPP__MOCK_STORAGE_FACTORY_HPP_ diff --git a/rosbag2_cpp/test/rosbag2_cpp/test_multifile_reader.cpp b/rosbag2_cpp/test/rosbag2_cpp/test_multifile_reader.cpp index a489dc3593..004a80a82b 100644 --- a/rosbag2_cpp/test/rosbag2_cpp/test_multifile_reader.cpp +++ b/rosbag2_cpp/test/rosbag2_cpp/test_multifile_reader.cpp @@ -68,7 +68,7 @@ class MultifileReaderTest : public Test EXPECT_CALL(*storage_, get_all_topics_and_types()) .Times(AtMost(1)).WillRepeatedly(Return(topics_and_types)); ON_CALL(*storage_, read_next()).WillByDefault(Return(message)); - EXPECT_CALL(*storage_factory, open_read_only(_, _)).WillRepeatedly(Return(storage_)); + EXPECT_CALL(*storage_factory, open_read_only(_)).WillRepeatedly(Return(storage_)); auto sequential_reader = std::make_unique( std::move(storage_factory), converter_factory_, std::move(metadata_io)); @@ -95,7 +95,7 @@ class MultifileReaderTest : public Test std::string relative_path_1_; std::string relative_path_2_; std::string absolute_path_1_; - rosbag2_cpp::StorageOptions default_storage_options_; + rosbag2_storage::StorageOptions default_storage_options_; }; class MultifileReaderTestVersion3 : public MultifileReaderTest diff --git a/rosbag2_cpp/test/rosbag2_cpp/test_sequential_reader.cpp b/rosbag2_cpp/test/rosbag2_cpp/test_sequential_reader.cpp index 9b03ec770c..95bd16e78c 100644 --- a/rosbag2_cpp/test/rosbag2_cpp/test_sequential_reader.cpp +++ b/rosbag2_cpp/test/rosbag2_cpp/test_sequential_reader.cpp @@ -70,10 +70,10 @@ class SequentialReaderTest : public Test .Times(AtMost(1)).WillRepeatedly(Return(topics_and_types)); EXPECT_CALL(*storage_, read_next()).WillRepeatedly(Return(message)); - EXPECT_CALL(*storage_factory, open_read_only(_, _)); + EXPECT_CALL(*storage_factory, open_read_only(_)); ON_CALL(*storage_factory, open_read_only).WillByDefault( - [this, relative_file_path](const std::string & path, const std::string & /* storage_id */) { - EXPECT_STREQ(relative_file_path.c_str(), path.c_str()); + [this, relative_file_path](const rosbag2_storage::StorageOptions & storage_options) { + EXPECT_STREQ(relative_file_path.c_str(), storage_options.uri.c_str()); return storage_; }); @@ -87,7 +87,7 @@ class SequentialReaderTest : public Test std::unique_ptr reader_; std::string storage_serialization_format_; std::string storage_uri_; - rosbag2_cpp::StorageOptions default_storage_options_; + rosbag2_storage::StorageOptions default_storage_options_; }; TEST_F(SequentialReaderTest, read_next_uses_converters_to_convert_serialization_format) { diff --git a/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp b/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp index fce161781d..e0a80cbd09 100644 --- a/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp +++ b/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp @@ -45,22 +45,22 @@ class SequentialWriterTest : public Test storage_ = std::make_shared>(); converter_factory_ = std::make_shared>(); metadata_io_ = std::make_unique>(); - storage_options_ = rosbag2_cpp::StorageOptions{}; + storage_options_ = rosbag2_storage::StorageOptions{}; storage_options_.uri = "uri"; rcpputils::fs::path dir(storage_options_.uri); rcpputils::fs::remove_all(dir); - ON_CALL(*storage_factory_, open_read_write(_, _)).WillByDefault( + ON_CALL(*storage_factory_, open_read_write(_)).WillByDefault( DoAll( Invoke( - [this](const std::string & uri, const std::string &) { + [this](const rosbag2_storage::StorageOptions & storage_options) { fake_storage_size_ = 0; - fake_storage_uri_ = uri; + fake_storage_uri_ = storage_options.uri; }), Return(storage_))); EXPECT_CALL( - *storage_factory_, open_read_write(_, _)).Times(AtLeast(0)); + *storage_factory_, open_read_write(_)).Times(AtLeast(0)); } ~SequentialWriterTest() @@ -74,7 +74,7 @@ class SequentialWriterTest : public Test std::shared_ptr> converter_factory_; std::unique_ptr metadata_io_; std::unique_ptr writer_; - rosbag2_cpp::StorageOptions storage_options_; + rosbag2_storage::StorageOptions storage_options_; uint64_t fake_storage_size_; rosbag2_storage::BagMetadata fake_metadata_; std::string fake_storage_uri_; diff --git a/rosbag2_cpp/test/rosbag2_cpp/test_storage_without_metadata_file.cpp b/rosbag2_cpp/test/rosbag2_cpp/test_storage_without_metadata_file.cpp index fd34ce4ce5..1f29bf99af 100644 --- a/rosbag2_cpp/test/rosbag2_cpp/test_storage_without_metadata_file.cpp +++ b/rosbag2_cpp/test/rosbag2_cpp/test_storage_without_metadata_file.cpp @@ -72,12 +72,14 @@ TEST_F(StorageWithoutMetadataFileTest, open_uses_storage_id_from_storage_options } auto storage_factory = std::make_unique>(); - EXPECT_CALL(*storage_factory, open_read_only(_, kStorageId)).Times(1).WillOnce(Return(storage_)); + EXPECT_CALL( + *storage_factory, + open_read_only(_)).Times(1).WillOnce(Return(storage_)); auto metadata_io = std::make_unique>(); EXPECT_CALL(*metadata_io, metadata_file_exists).Times(1).WillOnce(Return(false)); - rosbag2_cpp::StorageOptions storage_options; + rosbag2_storage::StorageOptions storage_options; storage_options.storage_id = kStorageId; auto sequential_reader = std::make_unique( diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp index 21b12ac93d..5dccca0953 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp @@ -18,8 +18,8 @@ #include #include "rmw/rmw.h" -#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_storage/serialized_bag_message.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "std_msgs/msg/byte_multi_array.hpp" #include "rosbag2_performance_writer_benchmarking/writer_benchmark.hpp" @@ -188,7 +188,7 @@ void WriterBenchmark::create_producers(const ProducerConfig & config) void WriterBenchmark::create_writer() { writer_ = std::make_shared(); - rosbag2_cpp::StorageOptions storage_options{}; + rosbag2_storage::StorageOptions storage_options{}; storage_options.uri = db_folder_; storage_options.storage_id = "sqlite3"; storage_options.max_bagfile_size = 0; diff --git a/rosbag2_py/src/rosbag2_py/_reader.cpp b/rosbag2_py/src/rosbag2_py/_reader.cpp index 5c634c27ed..731e6275f3 100644 --- a/rosbag2_py/src/rosbag2_py/_reader.cpp +++ b/rosbag2_py/src/rosbag2_py/_reader.cpp @@ -20,8 +20,8 @@ #include "rosbag2_cpp/converter_options.hpp" #include "rosbag2_cpp/readers/sequential_reader.hpp" #include "rosbag2_cpp/reader.hpp" -#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_storage/storage_filter.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/topic_metadata.hpp" #include "./pybind11.hpp" @@ -39,7 +39,7 @@ class Reader } void open( - rosbag2_cpp::StorageOptions & storage_options, + rosbag2_storage::StorageOptions & storage_options, rosbag2_cpp::ConverterOptions & converter_options) { reader_->open(storage_options, converter_options); diff --git a/rosbag2_py/src/rosbag2_py/_storage.cpp b/rosbag2_py/src/rosbag2_py/_storage.cpp index 12a9b9006a..016dc057cf 100644 --- a/rosbag2_py/src/rosbag2_py/_storage.cpp +++ b/rosbag2_py/src/rosbag2_py/_storage.cpp @@ -16,8 +16,8 @@ #include #include "rosbag2_cpp/converter_options.hpp" -#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_storage/storage_filter.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/topic_metadata.hpp" #include "./pybind11.hpp" @@ -37,7 +37,7 @@ PYBIND11_MODULE(_storage, m) { "output_serialization_format", &rosbag2_cpp::ConverterOptions::output_serialization_format); - pybind11::class_(m, "StorageOptions") + pybind11::class_(m, "StorageOptions") .def( pybind11::init(), pybind11::arg("uri"), @@ -45,17 +45,17 @@ PYBIND11_MODULE(_storage, m) { pybind11::arg("max_bagfile_size") = 0, pybind11::arg("max_bagfile_duration") = 0, pybind11::arg("max_cache_size") = 0) - .def_readwrite("uri", &rosbag2_cpp::StorageOptions::uri) - .def_readwrite("storage_id", &rosbag2_cpp::StorageOptions::storage_id) + .def_readwrite("uri", &rosbag2_storage::StorageOptions::uri) + .def_readwrite("storage_id", &rosbag2_storage::StorageOptions::storage_id) .def_readwrite( "max_bagfile_size", - &rosbag2_cpp::StorageOptions::max_bagfile_size) + &rosbag2_storage::StorageOptions::max_bagfile_size) .def_readwrite( "max_bagfile_duration", - &rosbag2_cpp::StorageOptions::max_bagfile_duration) + &rosbag2_storage::StorageOptions::max_bagfile_duration) .def_readwrite( "max_cache_size", - &rosbag2_cpp::StorageOptions::max_cache_size); + &rosbag2_storage::StorageOptions::max_cache_size); pybind11::class_(m, "StorageFilter") .def( diff --git a/rosbag2_py/src/rosbag2_py/_writer.cpp b/rosbag2_py/src/rosbag2_py/_writer.cpp index 1a037acb12..fb1b7e7808 100644 --- a/rosbag2_py/src/rosbag2_py/_writer.cpp +++ b/rosbag2_py/src/rosbag2_py/_writer.cpp @@ -17,11 +17,11 @@ #include "rosbag2_compression/sequential_compression_writer.hpp" #include "rosbag2_cpp/converter_options.hpp" -#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_cpp/writer.hpp" #include "rosbag2_cpp/writers/sequential_writer.hpp" #include "rosbag2_storage/ros_helper.hpp" #include "rosbag2_storage/storage_filter.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/topic_metadata.hpp" #include "./pybind11.hpp" @@ -39,7 +39,7 @@ class Writer } void open( - rosbag2_cpp::StorageOptions & storage_options, + rosbag2_storage::StorageOptions & storage_options, rosbag2_cpp::ConverterOptions & converter_options) { writer_->open(storage_options, converter_options); diff --git a/rosbag2_storage/include/rosbag2_storage/storage_factory.hpp b/rosbag2_storage/include/rosbag2_storage/storage_factory.hpp index a58adec069..88907e3ae7 100644 --- a/rosbag2_storage/include/rosbag2_storage/storage_factory.hpp +++ b/rosbag2_storage/include/rosbag2_storage/storage_factory.hpp @@ -49,10 +49,10 @@ class ROSBAG2_STORAGE_PUBLIC StorageFactory : public StorageFactoryInterface ~StorageFactory() override; std::shared_ptr - open_read_only(const std::string & uri, const std::string & storage_id) override; + open_read_only(const StorageOptions & storage_options) override; std::shared_ptr - open_read_write(const std::string & uri, const std::string & storage_id) override; + open_read_write(const StorageOptions & storage_options) override; private: std::unique_ptr impl_; diff --git a/rosbag2_storage/include/rosbag2_storage/storage_factory_interface.hpp b/rosbag2_storage/include/rosbag2_storage/storage_factory_interface.hpp index 23c76b5818..cf606e336e 100644 --- a/rosbag2_storage/include/rosbag2_storage/storage_factory_interface.hpp +++ b/rosbag2_storage/include/rosbag2_storage/storage_factory_interface.hpp @@ -31,10 +31,10 @@ class ROSBAG2_STORAGE_PUBLIC StorageFactoryInterface virtual ~StorageFactoryInterface() = default; virtual std::shared_ptr - open_read_only(const std::string & uri, const std::string & storage_id) = 0; + open_read_only(const StorageOptions & storage_options) = 0; virtual std::shared_ptr - open_read_write(const std::string & uri, const std::string & storage_id) = 0; + open_read_write(const StorageOptions & storage_options) = 0; }; } // namespace rosbag2_storage diff --git a/rosbag2_storage/include/rosbag2_storage/storage_interfaces/base_io_interface.hpp b/rosbag2_storage/include/rosbag2_storage/storage_interfaces/base_io_interface.hpp index a238ec3e5f..cb93cc1d4d 100644 --- a/rosbag2_storage/include/rosbag2_storage/storage_interfaces/base_io_interface.hpp +++ b/rosbag2_storage/include/rosbag2_storage/storage_interfaces/base_io_interface.hpp @@ -17,6 +17,7 @@ #include +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_storage/visibility_control.hpp" namespace rosbag2_storage @@ -46,14 +47,18 @@ class ROSBAG2_STORAGE_PUBLIC BaseIOInterface /** * Opens the storage plugin. - * \param uri is the path to the bagfile. Exact behavior depends on the io_flag passed. + * \param storage_options contains necessary info such as file uri and storage plugin. * \param io_flag is a hint for the type of storage plugin to open depending on the io operations requested. + * \param config_file_uri is a path to a storage specific file path. * If IOFlag::READ_ONLY is passed, then only read operations are guaranteed. * The uri passed should be the exact relative path to the bagfile. * If IOFlag::READ_WRITE is passed, then a new bagfile is created with guaranteed read and write operations. * The storage plugin will append the uri in the case of creating a new bagfile backing. + * The storage can load specific configurations by accessing a config file given in the `config_file_uri` parameter. */ - virtual void open(const std::string & uri, IOFlag io_flag) = 0; + virtual void open( + const StorageOptions & storage_options, + IOFlag io_flag) = 0; }; } // namespace storage_interfaces diff --git a/rosbag2_storage/include/rosbag2_storage/storage_interfaces/read_only_interface.hpp b/rosbag2_storage/include/rosbag2_storage/storage_interfaces/read_only_interface.hpp index bec12d8eb9..8fd12f7476 100644 --- a/rosbag2_storage/include/rosbag2_storage/storage_interfaces/read_only_interface.hpp +++ b/rosbag2_storage/include/rosbag2_storage/storage_interfaces/read_only_interface.hpp @@ -34,7 +34,9 @@ class ROSBAG2_STORAGE_PUBLIC ReadOnlyInterface public: virtual ~ReadOnlyInterface() = default; - void open(const std::string & uri, IOFlag io_flag = IOFlag::READ_ONLY) override = 0; + void open( + const StorageOptions & storage_options, + IOFlag io_flag = IOFlag::READ_ONLY) override = 0; uint64_t get_bagfile_size() const override = 0; diff --git a/rosbag2_storage/include/rosbag2_storage/storage_interfaces/read_write_interface.hpp b/rosbag2_storage/include/rosbag2_storage/storage_interfaces/read_write_interface.hpp index 6b2f8257d5..1974476dfa 100644 --- a/rosbag2_storage/include/rosbag2_storage/storage_interfaces/read_write_interface.hpp +++ b/rosbag2_storage/include/rosbag2_storage/storage_interfaces/read_write_interface.hpp @@ -33,7 +33,9 @@ class ROSBAG2_STORAGE_PUBLIC ReadWriteInterface public: ~ReadWriteInterface() override = default; - void open(const std::string & uri, IOFlag io_flag = IOFlag::READ_WRITE) override = 0; + void open( + const StorageOptions & storage_options, + IOFlag io_flag = IOFlag::READ_WRITE) override = 0; uint64_t get_bagfile_size() const override = 0; diff --git a/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp b/rosbag2_storage/include/rosbag2_storage/storage_options.hpp similarity index 80% rename from rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp rename to rosbag2_storage/include/rosbag2_storage/storage_options.hpp index 4337dc45d7..7f7a2cee1d 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp +++ b/rosbag2_storage/include/rosbag2_storage/storage_options.hpp @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef ROSBAG2_CPP__STORAGE_OPTIONS_HPP_ -#define ROSBAG2_CPP__STORAGE_OPTIONS_HPP_ +#ifndef ROSBAG2_STORAGE__STORAGE_OPTIONS_HPP_ +#define ROSBAG2_STORAGE__STORAGE_OPTIONS_HPP_ #include -namespace rosbag2_cpp +namespace rosbag2_storage { struct StorageOptions @@ -38,8 +38,12 @@ struct StorageOptions // before these being written to disk. // A value of 0 disables caching and every write happens directly to disk. uint64_t max_cache_size = 0; + + // Storage specific configuration file. + // Defaults to empty string. + std::string storage_config_uri = ""; }; -} // namespace rosbag2_cpp +} // namespace rosbag2_storage -#endif // ROSBAG2_CPP__STORAGE_OPTIONS_HPP_ +#endif // ROSBAG2_STORAGE__STORAGE_OPTIONS_HPP_ diff --git a/rosbag2_storage/src/rosbag2_storage/impl/storage_factory_impl.hpp b/rosbag2_storage/src/rosbag2_storage/impl/storage_factory_impl.hpp index 33702a10d7..56307894ad 100644 --- a/rosbag2_storage/src/rosbag2_storage/impl/storage_factory_impl.hpp +++ b/rosbag2_storage/src/rosbag2_storage/impl/storage_factory_impl.hpp @@ -51,19 +51,21 @@ template< std::shared_ptr get_interface_instance( std::shared_ptr> class_loader, - const std::string & storage_id, - const std::string & uri) + const StorageOptions & storage_options) { const auto & registered_classes = class_loader->getDeclaredClasses(); - auto class_exists = std::find(registered_classes.begin(), registered_classes.end(), storage_id); + auto class_exists = std::find( + registered_classes.begin(), + registered_classes.end(), storage_options.storage_id); if (class_exists == registered_classes.end()) { - ROSBAG2_STORAGE_LOG_DEBUG_STREAM("Requested storage id '" << storage_id << "' does not exist"); + ROSBAG2_STORAGE_LOG_DEBUG_STREAM( + "Requested storage id '" << storage_options.storage_id << "' does not exist"); return nullptr; } std::shared_ptr instance = nullptr; try { - auto unmanaged_instance = class_loader->createUnmanagedInstance(storage_id); + auto unmanaged_instance = class_loader->createUnmanagedInstance(storage_options.storage_id); instance = std::shared_ptr(unmanaged_instance); } catch (const std::runtime_error & ex) { ROSBAG2_STORAGE_LOG_ERROR_STREAM( @@ -72,11 +74,12 @@ get_interface_instance( } try { - instance->open(uri, flag); + instance->open(storage_options, flag); return instance; } catch (const std::runtime_error & ex) { ROSBAG2_STORAGE_LOG_ERROR_STREAM( - "Could not open '" << uri << "' with '" << storage_id << "'. Error: " << ex.what()); + "Could not open '" << storage_options.uri << "' with '" << + storage_options.storage_id << "'. Error: " << ex.what()); return nullptr; } } @@ -103,33 +106,33 @@ class StorageFactoryImpl virtual ~StorageFactoryImpl() = default; - std::shared_ptr open_read_write( - const std::string & uri, const std::string & storage_id) + std::shared_ptr open_read_write(const StorageOptions & storage_options) { - auto instance = get_interface_instance(read_write_class_loader_, storage_id, uri); + auto instance = + get_interface_instance(read_write_class_loader_, storage_options); if (instance == nullptr) { ROSBAG2_STORAGE_LOG_ERROR_STREAM( - "Could not load/open plugin with storage id '" << storage_id << "'."); + "Could not load/open plugin with storage id '" << storage_options.storage_id << "'."); } return instance; } - std::shared_ptr open_read_only( - const std::string & uri, const std::string & storage_id) + std::shared_ptr open_read_only(const StorageOptions & storage_options) { // try to load the instance as read_only interface - auto instance = get_interface_instance(read_only_class_loader_, storage_id, uri); + auto instance = get_interface_instance( + read_only_class_loader_, storage_options); // try to load as read_write if not successful if (instance == nullptr) { instance = get_interface_instance( - read_write_class_loader_, storage_id, uri); + read_write_class_loader_, storage_options); } if (instance == nullptr) { ROSBAG2_STORAGE_LOG_ERROR_STREAM( - "Could not load/open plugin with storage id '" << storage_id << "'."); + "Could not load/open plugin with storage id '" << storage_options.storage_id << "'."); } return instance; diff --git a/rosbag2_storage/src/rosbag2_storage/storage_factory.cpp b/rosbag2_storage/src/rosbag2_storage/storage_factory.cpp index 199a3e45a1..a6c6ecc56d 100644 --- a/rosbag2_storage/src/rosbag2_storage/storage_factory.cpp +++ b/rosbag2_storage/src/rosbag2_storage/storage_factory.cpp @@ -37,14 +37,14 @@ StorageFactory::StorageFactory() StorageFactory::~StorageFactory() {} std::shared_ptr StorageFactory::open_read_only( - const std::string & uri, const std::string & storage_id) + const StorageOptions & storage_options) { - return impl_->open_read_only(uri, storage_id); + return impl_->open_read_only(storage_options); } std::shared_ptr StorageFactory::open_read_write( - const std::string & uri, const std::string & storage_id) + const StorageOptions & storage_options) { - return impl_->open_read_write(uri, storage_id); + return impl_->open_read_write(storage_options); } } // namespace rosbag2_storage diff --git a/rosbag2_storage/test/rosbag2_storage/test_plugin.cpp b/rosbag2_storage/test/rosbag2_storage/test_plugin.cpp index 4c4209f061..60ab7c7870 100644 --- a/rosbag2_storage/test/rosbag2_storage/test_plugin.cpp +++ b/rosbag2_storage/test/rosbag2_storage/test_plugin.cpp @@ -31,14 +31,16 @@ TestPlugin::~TestPlugin() } void TestPlugin::open( - const std::string & uri, rosbag2_storage::storage_interfaces::IOFlag flag) + const rosbag2_storage::StorageOptions & storage_options, + rosbag2_storage::storage_interfaces::IOFlag flag) { if (flag == rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY) { std::cout << "opening testplugin read only: "; } else if (flag == rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE) { std::cout << "opening testplugin read write: "; } - std::cout << uri << ".\n"; + std::cout << "storage uri: " << storage_options.uri << ".\n"; + std::cout << "config file uri: " << storage_options.storage_config_uri << ".\n"; } bool TestPlugin::has_next() diff --git a/rosbag2_storage/test/rosbag2_storage/test_plugin.hpp b/rosbag2_storage/test/rosbag2_storage/test_plugin.hpp index 4fbadf92f5..3322b35d5c 100644 --- a/rosbag2_storage/test/rosbag2_storage/test_plugin.hpp +++ b/rosbag2_storage/test/rosbag2_storage/test_plugin.hpp @@ -29,7 +29,9 @@ class TestPlugin : public rosbag2_storage::storage_interfaces::ReadWriteInterfac public: ~TestPlugin() override; - void open(const std::string & uri, rosbag2_storage::storage_interfaces::IOFlag flag) override; + void open( + const rosbag2_storage::StorageOptions & storage_options, + rosbag2_storage::storage_interfaces::IOFlag flag) override; void create_topic(const rosbag2_storage::TopicMetadata & topic) override; diff --git a/rosbag2_storage/test/rosbag2_storage/test_read_only_plugin.cpp b/rosbag2_storage/test/rosbag2_storage/test_read_only_plugin.cpp index 8aa847f357..ae0488ca95 100644 --- a/rosbag2_storage/test/rosbag2_storage/test_read_only_plugin.cpp +++ b/rosbag2_storage/test/rosbag2_storage/test_read_only_plugin.cpp @@ -28,14 +28,16 @@ TestReadOnlyPlugin::~TestReadOnlyPlugin() } void TestReadOnlyPlugin::open( - const std::string & uri, rosbag2_storage::storage_interfaces::IOFlag flag) + const rosbag2_storage::StorageOptions & storage_options, + rosbag2_storage::storage_interfaces::IOFlag flag) { if (flag == rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY) { std::cout << "opening testplugin read only: "; } else if (flag == rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE) { std::cout << "opening testplugin read write: "; } - std::cout << uri << ".\n"; + std::cout << "storage uri: " << storage_options.uri << ".\n"; + std::cout << "config file uri: " << storage_options.storage_config_uri << ".\n"; } bool TestReadOnlyPlugin::has_next() diff --git a/rosbag2_storage/test/rosbag2_storage/test_read_only_plugin.hpp b/rosbag2_storage/test/rosbag2_storage/test_read_only_plugin.hpp index 0fdaca694a..49caaa993d 100644 --- a/rosbag2_storage/test/rosbag2_storage/test_read_only_plugin.hpp +++ b/rosbag2_storage/test/rosbag2_storage/test_read_only_plugin.hpp @@ -27,7 +27,9 @@ class TestReadOnlyPlugin : public rosbag2_storage::storage_interfaces::ReadOnlyI public: ~TestReadOnlyPlugin() override; - void open(const std::string & uri, rosbag2_storage::storage_interfaces::IOFlag flag) override; + void open( + const rosbag2_storage::StorageOptions & storage_options, + rosbag2_storage::storage_interfaces::IOFlag flag) override; bool has_next() override; diff --git a/rosbag2_storage/test/rosbag2_storage/test_storage_factory.cpp b/rosbag2_storage/test/rosbag2_storage/test_storage_factory.cpp index d3ceeff7cc..7d8c8b27e5 100644 --- a/rosbag2_storage/test/rosbag2_storage/test_storage_factory.cpp +++ b/rosbag2_storage/test/rosbag2_storage/test_storage_factory.cpp @@ -42,7 +42,7 @@ class StorageFactoryTest : public ::testing::Test TEST_F(StorageFactoryTest, load_test_plugin) { // Load plugin for read and write auto read_write_storage = factory.open_read_write( - bag_file_path, test_plugin_id); + {bag_file_path, test_plugin_id}); ASSERT_NE(nullptr, read_write_storage); EXPECT_EQ( @@ -62,14 +62,14 @@ TEST_F(StorageFactoryTest, load_test_plugin) { // Load plugin for read only even though it provides read and write interfaces auto read_only_storage = factory.open_read_only( - bag_file_path, test_plugin_id); + {bag_file_path, test_plugin_id}); ASSERT_NE(nullptr, read_only_storage); msg = read_only_storage->read_next(); } TEST_F(StorageFactoryTest, loads_readonly_plugin_only_for_read_only_storage) { auto storage_for_reading = factory.open_read_only( - bag_file_path, test_read_only_plugin_id); + {bag_file_path, test_read_only_plugin_id}); ASSERT_NE(nullptr, storage_for_reading); EXPECT_EQ( @@ -87,16 +87,16 @@ TEST_F(StorageFactoryTest, loads_readonly_plugin_only_for_read_only_storage) { storage_for_reading->read_next(); auto storage_for_reading_and_writing = factory.open_read_write( - bag_file_path, test_read_only_plugin_id); + {bag_file_path, test_read_only_plugin_id}); ASSERT_EQ(nullptr, storage_for_reading_and_writing); } TEST_F(StorageFactoryTest, load_unavailable_plugin) { auto instance_rw = factory.open_read_write( - bag_file_path, test_unavailable_plugin_id); + {bag_file_path, test_unavailable_plugin_id}); EXPECT_EQ(nullptr, instance_rw); auto instance_ro = factory.open_read_only( - bag_file_path, test_unavailable_plugin_id); + {bag_file_path, test_unavailable_plugin_id}); EXPECT_EQ(nullptr, instance_ro); } diff --git a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_storage.hpp b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_storage.hpp index c001f64795..f557c5e93c 100644 --- a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_storage.hpp +++ b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_storage.hpp @@ -49,7 +49,7 @@ class ROSBAG2_STORAGE_DEFAULT_PLUGINS_PUBLIC SqliteStorage ~SqliteStorage() override; void open( - const std::string & uri, + const rosbag2_storage::StorageOptions & storage_options, rosbag2_storage::storage_interfaces::IOFlag io_flag = rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE) override; diff --git a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp index 1e595350e3..a2bb9dd694 100644 --- a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp +++ b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp @@ -72,10 +72,16 @@ SqliteStorage::~SqliteStorage() } void SqliteStorage::open( - const std::string & uri, rosbag2_storage::storage_interfaces::IOFlag io_flag) + const rosbag2_storage::StorageOptions & storage_options, + rosbag2_storage::storage_interfaces::IOFlag io_flag) { + if (!storage_options.storage_config_uri.empty()) { + fprintf(stderr, "going to open config file: %s\n", storage_options.storage_config_uri.c_str()); + throw std::runtime_error("storage specific config file is not yet implemented."); + } + if (is_read_write(io_flag)) { - relative_path_ = uri + FILE_EXTENSION; + relative_path_ = storage_options.uri + FILE_EXTENSION; // READ_WRITE requires the DB to not exist. if (rcpputils::fs::path(relative_path_).exists()) { @@ -83,7 +89,7 @@ void SqliteStorage::open( "Failed to create bag: File '" + relative_path_ + "' already exists!"); } } else { // APPEND and READ_ONLY - relative_path_ = uri; + relative_path_ = storage_options.uri; // APPEND and READ_ONLY require the DB to exist if (!rcpputils::fs::path(relative_path_).exists()) { diff --git a/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/storage_test_fixture.hpp b/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/storage_test_fixture.hpp index 200f650d83..93a4ffbfb5 100644 --- a/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/storage_test_fixture.hpp +++ b/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/storage_test_fixture.hpp @@ -99,7 +99,7 @@ class StorageTestFixture : public TemporaryDirectoryFixture auto db_file = (rcpputils::fs::path(temporary_dir_path_) / "rosbag").string(); - writable_storage->open(db_file); + writable_storage->open({db_file, plugin_id_}); for (auto msg : messages) { std::string topic_name = std::get<2>(msg); @@ -125,7 +125,8 @@ class StorageTestFixture : public TemporaryDirectoryFixture auto db_file = (rcpputils::fs::path(temporary_dir_path_) / "rosbag.db3").string(); readable_storage->open( - db_file, rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY); + {db_file, plugin_id_}, + rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY); std::vector> read_messages; while (readable_storage->has_next()) { @@ -156,6 +157,8 @@ class StorageTestFixture : public TemporaryDirectoryFixture rcutils_allocator_t allocator_; rosbag2_storage::MetadataIo metadata_io_; + + const std::string plugin_id_ = "sqlite3"; }; #endif // ROSBAG2_STORAGE_DEFAULT_PLUGINS__SQLITE__STORAGE_TEST_FIXTURE_HPP_ diff --git a/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/test_sqlite_storage.cpp b/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/test_sqlite_storage.cpp index e3081e6d2a..06e75e25f4 100644 --- a/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/test_sqlite_storage.cpp +++ b/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/test_sqlite_storage.cpp @@ -51,6 +51,8 @@ bool operator!=(const TopicInformation & lhs, const TopicInformation & rhs) } // namespace rosbag2_storage +constexpr static const char * const kPluginID = "sqlite3"; + TEST_F(StorageTestFixture, string_messages_are_written_and_read_to_and_from_sqlite3_storage) { std::vector string_messages = {"first message", "second message", "third message"}; std::vector topics = {"topic1", "topic2", "topic3"}; @@ -82,7 +84,7 @@ TEST_F(StorageTestFixture, has_next_return_false_if_there_are_no_more_messages) std::make_unique(); auto db_filename = (rcpputils::fs::path(temporary_dir_path_) / "rosbag.db3").string(); - readable_storage->open(db_filename); + readable_storage->open({db_filename, kPluginID}); EXPECT_TRUE(readable_storage->has_next()); readable_storage->read_next(); @@ -102,7 +104,7 @@ TEST_F(StorageTestFixture, get_next_returns_messages_in_timestamp_order) { std::make_unique(); auto db_filename = (rcpputils::fs::path(temporary_dir_path_) / "rosbag.db3").string(); - readable_storage->open(db_filename); + readable_storage->open({db_filename, kPluginID}); EXPECT_TRUE(readable_storage->has_next()); auto first_message = readable_storage->read_next(); @@ -125,7 +127,7 @@ TEST_F(StorageTestFixture, read_next_returns_filtered_messages) { std::make_unique(); auto db_filename = (rcpputils::fs::path(temporary_dir_path_) / "rosbag.db3").string(); - readable_storage->open(db_filename); + readable_storage->open({db_filename, kPluginID}); rosbag2_storage::StorageFilter storage_filter; storage_filter.topics.push_back("topic2"); @@ -144,7 +146,7 @@ TEST_F(StorageTestFixture, read_next_returns_filtered_messages) { std::unique_ptr readable_storage2 = std::make_unique(); - readable_storage2->open(db_filename); + readable_storage2->open({db_filename, kPluginID}); readable_storage2->set_filter(storage_filter); readable_storage2->reset_filter(); @@ -167,7 +169,7 @@ TEST_F(StorageTestFixture, get_all_topics_and_types_returns_the_correct_vector) // extension is omitted since storage is being created; io_flag = READ_WRITE const auto read_write_filename = (rcpputils::fs::path(temporary_dir_path_) / "rosbag").string(); - writable_storage->open(read_write_filename); + writable_storage->open({read_write_filename, kPluginID}); writable_storage->create_topic({"topic1", "type1", "rmw1", ""}); writable_storage->create_topic({"topic2", "type2", "rmw2", ""}); @@ -177,7 +179,8 @@ TEST_F(StorageTestFixture, get_all_topics_and_types_returns_the_correct_vector) auto readable_storage = std::make_unique(); readable_storage->open( - read_only_filename, rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY); + {read_only_filename, kPluginID}, + rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY); auto topics_and_types = readable_storage->get_all_topics_and_types(); EXPECT_THAT( @@ -204,7 +207,9 @@ TEST_F(StorageTestFixture, get_metadata_returns_correct_struct) { const auto readable_storage = std::make_unique(); const auto db_filename = (rcpputils::fs::path(temporary_dir_path_) / "rosbag.db3").string(); - readable_storage->open(db_filename, rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY); + readable_storage->open( + {db_filename, kPluginID}, + rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY); const auto metadata = readable_storage->get_metadata(); EXPECT_THAT(metadata.storage_identifier, Eq("sqlite3")); @@ -231,7 +236,9 @@ TEST_F(StorageTestFixture, get_metadata_returns_correct_struct_if_no_messages) { const auto readable_storage = std::make_unique(); const auto db_filename = (rcpputils::fs::path(temporary_dir_path_) / "rosbag.db3").string(); - readable_storage->open(db_filename, rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY); + readable_storage->open( + {db_filename, kPluginID}, + rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY); const auto metadata = readable_storage->get_metadata(); EXPECT_THAT(metadata.storage_identifier, Eq("sqlite3")); @@ -252,7 +259,7 @@ TEST_F(StorageTestFixture, remove_topics_and_types_returns_the_empty_vector) { // extension is omitted since storage is created; io_flag = READ_WRITE const auto read_write_filename = (rcpputils::fs::path(temporary_dir_path_) / "rosbag").string(); - writable_storage->open(read_write_filename); + writable_storage->open({read_write_filename, kPluginID}); writable_storage->create_topic({"topic1", "type1", "rmw1", ""}); writable_storage->remove_topic({"topic1", "type1", "rmw1", ""}); @@ -264,7 +271,7 @@ TEST_F(StorageTestFixture, remove_topics_and_types_returns_the_empty_vector) { auto readable_storage = std::make_unique(); readable_storage->open( - read_only_filename, + {read_only_filename, kPluginID}, rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY); auto topics_and_types = readable_storage->get_all_topics_and_types(); @@ -285,7 +292,7 @@ TEST_F(StorageTestFixture, get_relative_file_path_returns_db_name_with_ext) { const auto storage_filename = read_write_filename + ".db3"; const auto read_write_storage = std::make_unique(); read_write_storage->open( - read_write_filename, + {read_write_filename, kPluginID}, rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE); EXPECT_EQ(read_write_storage->get_relative_file_path(), storage_filename); @@ -293,14 +300,14 @@ TEST_F(StorageTestFixture, get_relative_file_path_returns_db_name_with_ext) { const auto & read_only_filename = storage_filename; const auto read_only_storage = std::make_unique(); read_only_storage->open( - read_only_filename, + {read_only_filename, kPluginID}, rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY); EXPECT_EQ(read_only_storage->get_relative_file_path(), storage_filename); const auto & append_filename = storage_filename; const auto append_storage = std::make_unique(); append_storage->open( - append_filename, + {append_filename, kPluginID}, rosbag2_storage::storage_interfaces::IOFlag::APPEND); EXPECT_EQ(append_storage->get_relative_file_path(), storage_filename); } diff --git a/rosbag2_tests/test/rosbag2_tests/record_fixture.hpp b/rosbag2_tests/test/rosbag2_tests/record_fixture.hpp index 4da5199fa0..aaca0c052e 100644 --- a/rosbag2_tests/test/rosbag2_tests/record_fixture.hpp +++ b/rosbag2_tests/test/rosbag2_tests/record_fixture.hpp @@ -187,7 +187,9 @@ class RecordFixture : public TemporaryDirectoryFixture std::vector> table_msgs; auto storage = std::make_shared(); const auto database_path = get_bag_file_path(0).string(); - storage->open(database_path, rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY); + storage->open( + {database_path, "sqlite3"}, + rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY); while (storage->has_next()) { table_msgs.push_back(storage->read_next()); diff --git a/rosbag2_transport/include/rosbag2_transport/rosbag2_transport.hpp b/rosbag2_transport/include/rosbag2_transport/rosbag2_transport.hpp index 02bcb37e00..2d7f2af575 100644 --- a/rosbag2_transport/include/rosbag2_transport/rosbag2_transport.hpp +++ b/rosbag2_transport/include/rosbag2_transport/rosbag2_transport.hpp @@ -20,9 +20,10 @@ #include #include +#include "rosbag2_storage/storage_options.hpp" + #include "rosbag2_transport/play_options.hpp" #include "rosbag2_transport/record_options.hpp" -#include "rosbag2_transport/storage_options.hpp" #include "rosbag2_transport/visibility_control.hpp" namespace rosbag2_cpp @@ -65,7 +66,9 @@ class Rosbag2Transport * \param record_options Options regarding the recording (e.g. the topics to record) */ ROSBAG2_TRANSPORT_PUBLIC - void record(const StorageOptions & storage_options, const RecordOptions & record_options); + void record( + const rosbag2_storage::StorageOptions & storage_options, + const RecordOptions & record_options); /** * Replay all topics in a bagfile. @@ -74,7 +77,9 @@ class Rosbag2Transport * \param play_options Options regarding the playback (e.g. queue size) */ ROSBAG2_TRANSPORT_PUBLIC - void play(const StorageOptions & storage_options, const PlayOptions & play_options); + void play( + const rosbag2_storage::StorageOptions & storage_options, + const PlayOptions & play_options); /** * Print the bag info contained in the metadata yaml file. diff --git a/rosbag2_transport/include/rosbag2_transport/storage_options.hpp b/rosbag2_transport/include/rosbag2_transport/storage_options.hpp deleted file mode 100644 index 6b6467d8d5..0000000000 --- a/rosbag2_transport/include/rosbag2_transport/storage_options.hpp +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2018, Bosch Software Innovations GmbH. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef ROSBAG2_TRANSPORT__STORAGE_OPTIONS_HPP_ -#define ROSBAG2_TRANSPORT__STORAGE_OPTIONS_HPP_ - -#include - -#include "rosbag2_cpp/storage_options.hpp" - -namespace rosbag2_transport -{ -using StorageOptions = rosbag2_cpp::StorageOptions; -} // namespace rosbag2_transport - -#endif // ROSBAG2_TRANSPORT__STORAGE_OPTIONS_HPP_ diff --git a/rosbag2_transport/src/rosbag2_transport/rosbag2_transport.cpp b/rosbag2_transport/src/rosbag2_transport/rosbag2_transport.cpp index c46144ff39..57c866361d 100644 --- a/rosbag2_transport/src/rosbag2_transport/rosbag2_transport.cpp +++ b/rosbag2_transport/src/rosbag2_transport/rosbag2_transport.cpp @@ -67,7 +67,7 @@ void Rosbag2Transport::shutdown() } void Rosbag2Transport::record( - const StorageOptions & storage_options, const RecordOptions & record_options) + const rosbag2_storage::StorageOptions & storage_options, const RecordOptions & record_options) { try { writer_->open( @@ -94,7 +94,7 @@ std::shared_ptr Rosbag2Transport::setup_node( } void Rosbag2Transport::play( - const StorageOptions & storage_options, const PlayOptions & play_options) + const rosbag2_storage::StorageOptions & storage_options, const PlayOptions & play_options) { try { auto transport_node = diff --git a/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp b/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp index 10163ed90b..9f42f75228 100644 --- a/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp +++ b/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp @@ -32,9 +32,9 @@ #include "rosbag2_cpp/writer.hpp" #include "rosbag2_cpp/writers/sequential_writer.hpp" #include "rosbag2_storage/metadata_io.hpp" +#include "rosbag2_storage/storage_options.hpp" #include "rosbag2_transport/rosbag2_transport.hpp" #include "rosbag2_transport/record_options.hpp" -#include "rosbag2_transport/storage_options.hpp" #include "rmw/rmw.h" namespace @@ -84,7 +84,7 @@ std::unordered_map PyObject_AsTopicQoSMap(PyObject * o static PyObject * rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * kwargs) { - rosbag2_transport::StorageOptions storage_options{}; + rosbag2_storage::StorageOptions storage_options{}; rosbag2_transport::RecordOptions record_options{}; static const char * kwlist[] = { @@ -103,6 +103,7 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * "topics", "include_hidden_topics", "qos_profile_overrides", + "storage_config_file", nullptr}; char * uri = nullptr; @@ -120,9 +121,10 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * uint64_t max_cache_size = 0u; PyObject * topics = nullptr; bool include_hidden_topics = false; + char * storage_config_file = nullptr; if ( !PyArg_ParseTupleAndKeywords( - args, kwargs, "ssssss|bbKKKKObO", const_cast(kwlist), + args, kwargs, "ssssss|bbKKKKObOs", const_cast(kwlist), &uri, &storage_id, &serilization_format, @@ -137,7 +139,8 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * &max_cache_size, &topics, &include_hidden_topics, - &qos_profile_overrides + &qos_profile_overrides, + &storage_config_file )) { return nullptr; @@ -145,6 +148,7 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * storage_options.uri = std::string(uri); storage_options.storage_id = std::string(storage_id); + storage_options.storage_config_uri = std::string(storage_config_file); storage_options.max_bagfile_size = (uint64_t) max_bagfile_size; storage_options.max_bagfile_duration = static_cast(max_bagfile_duration); storage_options.max_cache_size = max_cache_size; @@ -225,7 +229,7 @@ static PyObject * rosbag2_transport_play(PyObject * Py_UNUSED(self), PyObject * args, PyObject * kwargs) { rosbag2_transport::PlayOptions play_options{}; - rosbag2_transport::StorageOptions storage_options{}; + rosbag2_storage::StorageOptions storage_options{}; static const char * kwlist[] = { "uri", @@ -237,6 +241,7 @@ rosbag2_transport_play(PyObject * Py_UNUSED(self), PyObject * args, PyObject * k "qos_profile_overrides", "loop", "topic_remapping", + "storage_config_file", nullptr }; @@ -249,8 +254,9 @@ rosbag2_transport_play(PyObject * Py_UNUSED(self), PyObject * args, PyObject * k PyObject * qos_profile_overrides{nullptr}; bool loop = false; PyObject * topic_remapping = nullptr; + char * storage_config_file = nullptr; if (!PyArg_ParseTupleAndKeywords( - args, kwargs, "sss|kfOObO", const_cast(kwlist), + args, kwargs, "sss|kfOObOs", const_cast(kwlist), &uri, &storage_id, &node_prefix, @@ -259,13 +265,15 @@ rosbag2_transport_play(PyObject * Py_UNUSED(self), PyObject * args, PyObject * k &topics, &qos_profile_overrides, &loop, - &topic_remapping)) + &topic_remapping, + &storage_config_file)) { return nullptr; } storage_options.uri = std::string(uri); storage_options.storage_id = std::string(storage_id); + storage_options.storage_config_uri = std::string(storage_config_file); play_options.node_prefix = std::string(node_prefix); play_options.read_ahead_queue_size = read_ahead_queue_size; diff --git a/rosbag2_transport/test/rosbag2_transport/mock_sequential_reader.hpp b/rosbag2_transport/test/rosbag2_transport/mock_sequential_reader.hpp index 9b1d81a9c6..b2f4a29862 100644 --- a/rosbag2_transport/test/rosbag2_transport/mock_sequential_reader.hpp +++ b/rosbag2_transport/test/rosbag2_transport/mock_sequential_reader.hpp @@ -26,7 +26,7 @@ class MockSequentialReader : public rosbag2_cpp::reader_interfaces::BaseReaderIn { public: void open( - const rosbag2_cpp::StorageOptions & storage_options, + const rosbag2_storage::StorageOptions & storage_options, const rosbag2_cpp::ConverterOptions & converter_options) override { (void) storage_options; diff --git a/rosbag2_transport/test/rosbag2_transport/mock_sequential_writer.hpp b/rosbag2_transport/test/rosbag2_transport/mock_sequential_writer.hpp index 5840dcf17c..8e70caec0e 100644 --- a/rosbag2_transport/test/rosbag2_transport/mock_sequential_writer.hpp +++ b/rosbag2_transport/test/rosbag2_transport/mock_sequential_writer.hpp @@ -26,7 +26,7 @@ class MockSequentialWriter : public rosbag2_cpp::writer_interfaces::BaseWriterIn { public: void open( - const rosbag2_cpp::StorageOptions & storage_options, + const rosbag2_storage::StorageOptions & storage_options, const rosbag2_cpp::ConverterOptions & converter_options) override { (void) storage_options; diff --git a/rosbag2_transport/test/rosbag2_transport/rosbag2_transport_test_fixture.hpp b/rosbag2_transport/test/rosbag2_transport/rosbag2_transport_test_fixture.hpp index 9b48d99e98..f224683a3a 100644 --- a/rosbag2_transport/test/rosbag2_transport/rosbag2_transport_test_fixture.hpp +++ b/rosbag2_transport/test/rosbag2_transport/rosbag2_transport_test_fixture.hpp @@ -33,8 +33,9 @@ # include #endif +#include "rosbag2_storage/storage_options.hpp" + #include "rosbag2_transport/play_options.hpp" -#include "rosbag2_transport/storage_options.hpp" #include "rosbag2_test_common/memory_management.hpp" @@ -80,7 +81,7 @@ class Rosbag2TransportTestFixture : public Test MemoryManagement memory_management_; - rosbag2_transport::StorageOptions storage_options_; + rosbag2_storage::StorageOptions storage_options_; rosbag2_transport::PlayOptions play_options_; std::shared_ptr reader_; diff --git a/rosbag2_transport/test/rosbag2_transport/test_rosbag2_node.cpp b/rosbag2_transport/test/rosbag2_transport/test_rosbag2_node.cpp index 961ca08ace..3efd326e76 100644 --- a/rosbag2_transport/test/rosbag2_transport/test_rosbag2_node.cpp +++ b/rosbag2_transport/test/rosbag2_transport/test_rosbag2_node.cpp @@ -23,10 +23,11 @@ #include "rosbag2_cpp/writer.hpp" +#include "rosbag2_storage/storage_options.hpp" + #include "rosbag2_test_common/memory_management.hpp" #include "rosbag2_transport/logging.hpp" -#include "rosbag2_transport/storage_options.hpp" #include "test_msgs/message_fixtures.hpp" #include "test_msgs/msg/basic_types.hpp" From e22be961f1436155c3d3c996667723cff472ba8e Mon Sep 17 00:00:00 2001 From: Karsten Knese Date: Wed, 28 Oct 2020 12:22:39 -0700 Subject: [PATCH 42/77] correct master build (#552) Signed-off-by: Karsten Knese --- rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index 791445dfa5..583e7e42e7 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -315,7 +315,7 @@ void SequentialWriter::reset_cache() storage_->write(cache_); // reset cache cache_.clear(); - cache_.reserve(max_cache_size_); + cache_.reserve(storage_options_.max_cache_size); current_cache_size_ = 0u; } } From f4db9a8793623a40652b0624bd34750d9a361880 Mon Sep 17 00:00:00 2001 From: Barry Xu Date: Tue, 3 Nov 2020 00:51:22 +0800 Subject: [PATCH 43/77] List all storage plugins in plugin xml file (#554) Signed-off-by: Barry Xu --- ros2bag/ros2bag/verb/list.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ros2bag/ros2bag/verb/list.py b/ros2bag/ros2bag/verb/list.py index deaba244f3..8ae9b15f49 100644 --- a/ros2bag/ros2bag/verb/list.py +++ b/ros2bag/ros2bag/verb/list.py @@ -51,14 +51,14 @@ def main(self, *, args): # noqa: D102 return 'path does not exist: %s' % abs_path xmldoc = minidom.parse(abs_path) - class_item = xmldoc.getElementsByTagName('class')[0] - class_name = class_item.attributes['name'] - type_name = class_item.attributes['type'] - base_class_name = class_item.attributes['base_class_type'] - description = xmldoc.getElementsByTagName('description')[0] + for class_item in xmldoc.getElementsByTagName('class'): + class_name = class_item.attributes['name'] + type_name = class_item.attributes['type'] + base_class_name = class_item.attributes['base_class_type'] + description = xmldoc.getElementsByTagName('description')[0] - print('%s%s' % (('name: ' if args.verbose else ''), class_name.value)) - if args.verbose: - print('\t%s' % description.childNodes[0].data) - print('\ttype: %s' % type_name.value) - print('\tbase_class: %s' % base_class_name.value) + print('%s%s' % (('name: ' if args.verbose else ''), class_name.value)) + if args.verbose: + print('\t%s' % description.childNodes[0].data) + print('\ttype: %s' % type_name.value) + print('\tbase_class: %s' % base_class_name.value) From b233316cfb1a03b3f649ec7061ab915aeb6448fe Mon Sep 17 00:00:00 2001 From: Karsten Knese Date: Thu, 12 Nov 2020 21:17:01 -0800 Subject: [PATCH 44/77] read yaml config file (#497) * read yaml config file Signed-off-by: Karsten Knese * address review comments Signed-off-by: Karsten Knese * Sqlite storage pragmas handling and validation - includes documentation of storage-config-file option - includes adaptation of benchmarking script to cache size semantic change - tests with valid and invalid storage file Signed-off-by: Adam Dabrowski * Applied review remarks Signed-off-by: Adam Dabrowski * - Typo fix - change pragma tests order as a workaround to fix tests issue on Windows: exception throwing tests don't clean temp folders Signed-off-by: Adam Dabrowski * Applied review: moved some parsing and key extraction to parse function Signed-off-by: Adam Dabrowski Co-authored-by: Adam Dabrowski Signed-off-by: Emerson Knapp --- README.md | 23 +++++ ros2bag/ros2bag/verb/play.py | 7 +- ros2bag/ros2bag/verb/record.py | 6 +- .../scripts/benchmark.sh | 2 +- .../CMakeLists.txt | 4 +- .../sqlite/sqlite_statement_wrapper.hpp | 2 +- .../sqlite/sqlite_wrapper.hpp | 11 ++- rosbag2_storage_default_plugins/package.xml | 1 + .../sqlite/sqlite_statement_wrapper.cpp | 17 +++- .../sqlite/sqlite_storage.cpp | 93 ++++++++++++++++++- .../sqlite/sqlite_wrapper.cpp | 53 +++++++++-- .../sqlite/storage_test_fixture.hpp | 19 ++++ .../sqlite/test_sqlite_storage.cpp | 22 +++++ 13 files changed, 241 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index baaa5b015f..dc555d51d9 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,29 @@ Currently, the only `compression-format` available is `zstd`. Both the mode and It is recommended to use this feature with the splitting options. +#### Recording with a storage configuration + +Storage configuration can be specified in a YAML file passed through the `--storage-config-file` option. +This can be used to optimize performance for specific use-cases. + +For the default storage plugin (sqlite3), the file has a following syntax: +``` +read: + pragmas: +write: + pragmas: +``` +Please refer to [documentation of pragmas](https://www.sqlite.org/pragma.html). +Settings are fully exposed to the user and should be applied with understanding. + +An example configuration file could look like this: + +``` +write: + pragmas: ["journal_mode = MEMORY", "synchronous = OFF", "schema.cache_size = 1000", "schema.page_size = 4096"] + +``` + ### Replaying data After recording data, the next logical step is to replay this data: diff --git a/ros2bag/ros2bag/verb/play.py b/ros2bag/ros2bag/verb/play.py index 055d5b5e8c..d08b06dde2 100644 --- a/ros2bag/ros2bag/verb/play.py +++ b/ros2bag/ros2bag/verb/play.py @@ -58,7 +58,12 @@ def add_arguments(self, parser, cli_name): # noqa: D102 '"old_topic1:=new_topic1 old_topic2:=new_topic2 etc." ') parser.add_argument( '--storage-config-file', type=FileType('r'), - help='Path to a yaml file defining storage specific configurations.') + help='Path to a yaml file defining storage specific configurations. ' + 'For the default storage plugin settings are specified through syntax:' + 'read:' + ' pragmas: [\"\" = ]' + 'Note that applicable settings are limited to read-only for ros2 bag play.' + 'For a list of sqlite3 settings, refer to sqlite3 documentation') def main(self, *, args): # noqa: D102 qos_profile_overrides = {} # Specify a valid default diff --git a/ros2bag/ros2bag/verb/record.py b/ros2bag/ros2bag/verb/record.py index 8621bed3d5..b9862efc21 100644 --- a/ros2bag/ros2bag/verb/record.py +++ b/ros2bag/ros2bag/verb/record.py @@ -92,7 +92,11 @@ def add_arguments(self, parser, cli_name): # noqa: D102 ) parser.add_argument( '--storage-config-file', type=FileType('r'), - help='Path to a yaml file defining storage specific configurations.') + help='Path to a yaml file defining storage specific configurations. ' + 'For the default storage plugin settings are specified through syntax:' + 'write:' + ' pragmas: [\"\" = ]' + 'For a list of sqlite3 settings, refer to sqlite3 documentation') self._subparser = parser def main(self, *, args): # noqa: D102 diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh index 7ff116f47b..a149ad3a6b 100755 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh @@ -17,7 +17,7 @@ summary_file=${test_dir}/results.csv freq=100; #Hz -for cache in 0 10 100 1000 +for cache in 0 1000000 10000000 100000000 do for sz in 1000 10000 100000 1000000 do diff --git a/rosbag2_storage_default_plugins/CMakeLists.txt b/rosbag2_storage_default_plugins/CMakeLists.txt index 2cb9239acd..f519aba0b2 100644 --- a/rosbag2_storage_default_plugins/CMakeLists.txt +++ b/rosbag2_storage_default_plugins/CMakeLists.txt @@ -27,6 +27,7 @@ find_package(rcutils REQUIRED) find_package(rosbag2_storage REQUIRED) find_package(sqlite3_vendor REQUIRED) find_package(SQLite3 REQUIRED) # provided by sqlite3_vendor +find_package(yaml_cpp_vendor REQUIRED) add_library(${PROJECT_NAME} SHARED src/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.cpp @@ -34,11 +35,12 @@ add_library(${PROJECT_NAME} SHARED src/rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.cpp) ament_target_dependencies(${PROJECT_NAME} + pluginlib rosbag2_storage rcpputils rcutils SQLite3 - pluginlib) + yaml_cpp_vendor) target_include_directories(${PROJECT_NAME} PUBLIC diff --git a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.hpp b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.hpp index dd1f36a58c..cafe889cca 100644 --- a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.hpp +++ b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.hpp @@ -177,7 +177,7 @@ class ROSBAG2_STORAGE_DEFAULT_PLUGINS_PUBLIC SqliteStatementWrapper bool is_already_accessed_; }; - std::shared_ptr execute_and_reset(); + std::shared_ptr execute_and_reset(bool assert_return_value = false); template QueryResult execute_query(); diff --git a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.hpp b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.hpp index 4c9b58cc55..0682911bd9 100644 --- a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.hpp +++ b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.hpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include "rcutils/types.h" #include "rosbag2_storage/serialized_bag_message.hpp" @@ -35,7 +35,10 @@ using DBPtr = sqlite3 *; class ROSBAG2_STORAGE_DEFAULT_PLUGINS_PUBLIC SqliteWrapper { public: - SqliteWrapper(const std::string & uri, rosbag2_storage::storage_interfaces::IOFlag io_flag); + SqliteWrapper( + const std::string & uri, + rosbag2_storage::storage_interfaces::IOFlag io_flag, + std::unordered_map && pragmas = {}); SqliteWrapper(); ~SqliteWrapper(); @@ -46,6 +49,10 @@ class ROSBAG2_STORAGE_DEFAULT_PLUGINS_PUBLIC SqliteWrapper operator bool(); private: + void apply_pragma_settings( + std::unordered_map & pragmas, + rosbag2_storage::storage_interfaces::IOFlag io_flag); + DBPtr db_ptr; }; diff --git a/rosbag2_storage_default_plugins/package.xml b/rosbag2_storage_default_plugins/package.xml index 86c579e65b..e6af70b940 100644 --- a/rosbag2_storage_default_plugins/package.xml +++ b/rosbag2_storage_default_plugins/package.xml @@ -15,6 +15,7 @@ rcutils rosbag2_storage sqlite3_vendor + yaml_cpp_vendor ament_lint_auto ament_lint_common diff --git a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.cpp b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.cpp index 9604b17aaa..f8391b12ea 100644 --- a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.cpp +++ b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.cpp @@ -52,7 +52,8 @@ SqliteStatementWrapper::~SqliteStatementWrapper() } } -std::shared_ptr SqliteStatementWrapper::execute_and_reset() +std::shared_ptr SqliteStatementWrapper::execute_and_reset( + bool assert_return_value) { int return_code = sqlite3_step(statement_); if (!is_query_ok(return_code)) { @@ -62,6 +63,20 @@ std::shared_ptr SqliteStatementWrapper::execute_and_rese throw SqliteException{errmsg.str()}; } + + if (assert_return_value) { + bool no_result = return_code == SQLITE_DONE || sqlite3_column_count(statement_) == 0; + + if (no_result || sqlite3_column_type(statement_, 0) == SQLITE_NULL) { + // No result or result is null means that no such pragma exists + std::stringstream errmsg; + errmsg << "Statement returned empty value while result was expected: \'" << + sqlite3_sql(statement_) << "\'"; + + throw SqliteException{errmsg.str()}; + } + } + return reset(); } diff --git a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp index a2bb9dd694..1da61b443a 100644 --- a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp +++ b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,19 @@ #include "rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.hpp" #include "rosbag2_storage_default_plugins/sqlite/sqlite_exception.hpp" +#ifdef _WIN32 +// This is necessary because of a bug in yaml-cpp's cmake +#define YAML_CPP_DLL +// This is necessary because yaml-cpp does not always use dllimport/dllexport consistently +# pragma warning(push) +# pragma warning(disable:4251) +# pragma warning(disable:4275) +#endif +#include "yaml-cpp/yaml.h" +#ifdef _WIN32 +# pragma warning(pop) +#endif + #include "../logging.hpp" namespace @@ -56,6 +70,78 @@ bool is_read_write(const rosbag2_storage::storage_interfaces::IOFlag io_flag) return io_flag == rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE; } +// Return pragma-name to full statement map +inline std::unordered_map parse_pragmas( + const std::string & storage_config_uri, const rosbag2_storage::storage_interfaces::IOFlag io_flag) +{ + std::unordered_map pragmas; + if (storage_config_uri.empty()) { + return pragmas; + } + + std::vector pragma_entries; + try { + auto key = + io_flag == rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY ? "read" : "write"; + YAML::Node yaml_file = YAML::LoadFile(storage_config_uri); + pragma_entries = yaml_file[key]["pragmas"].as>(); + } catch (const YAML::Exception & ex) { + throw std::runtime_error( + std::string("Exception on parsing sqlite3 config file: ") + + ex.what()); + } + // poor developer's sqlinjection prevention ;-) + std::string invalid_characters = {"';\""}; + auto throw_on_invalid_character = [](const auto & pragmas, const auto & invalid_characters) { + for (const auto & pragma_string : pragmas) { + auto pos = pragma_string.find_first_of(invalid_characters); + if (pos != std::string::npos) { + throw std::runtime_error( + std::string("Invalid characters in sqlite3 config file: ") + + pragma_string[pos] + + ". Avoid following characters: " + + invalid_characters); + } + } + }; + throw_on_invalid_character(pragma_entries, invalid_characters); + + // Extract pragma name and map to full pragma statement + for (const auto & pragma : pragma_entries) { + if (pragma.empty()) { + continue; + } + + const std::string pragma_assign = "="; + const std::string pragma_bracket = "("; + auto found_value_assignment = pragma.find(pragma_assign); + + // Extract pragma name. It is the same as statement for read only pragmas + auto pragma_name = pragma; + + // Find assignment operator and strip value assignment part + if (found_value_assignment == std::string::npos) { + found_value_assignment = pragma.find(pragma_bracket); + } + if (found_value_assignment != std::string::npos) { + if (found_value_assignment == 0) { + // Incorrect syntax, starts with = or ( + std::stringstream errmsg; + errmsg << "Incorrect storage setting syntax: " << pragma; + throw std::runtime_error{errmsg.str()}; + } + // Strip value assignment part, trim trailing whitespaces before = or ( + pragma_name = pragma.substr(0, found_value_assignment); + const std::string whitespaces(" \t"); + pragma_name = pragma_name.substr(0, pragma_name.find_last_not_of(whitespaces) + 1); + } + + auto full_pragma_statement = "PRAGMA " + pragma + ";"; + pragmas.insert({pragma_name, full_pragma_statement}); + } + return pragmas; +} + constexpr const auto FILE_EXTENSION = ".db3"; // Minimum size of a sqlite3 database file in bytes (84 kiB). @@ -75,10 +161,7 @@ void SqliteStorage::open( const rosbag2_storage::StorageOptions & storage_options, rosbag2_storage::storage_interfaces::IOFlag io_flag) { - if (!storage_options.storage_config_uri.empty()) { - fprintf(stderr, "going to open config file: %s\n", storage_options.storage_config_uri.c_str()); - throw std::runtime_error("storage specific config file is not yet implemented."); - } + auto pragmas = parse_pragmas(storage_options.storage_config_uri, io_flag); if (is_read_write(io_flag)) { relative_path_ = storage_options.uri + FILE_EXTENSION; @@ -99,7 +182,7 @@ void SqliteStorage::open( } try { - database_ = std::make_unique(relative_path_, io_flag); + database_ = std::make_unique(relative_path_, io_flag, std::move(pragmas)); } catch (const SqliteException & e) { throw std::runtime_error("Failed to setup storage. Error: " + std::string(e.what())); } diff --git a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.cpp b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.cpp index 2e988d5c24..96d11374e4 100644 --- a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.cpp +++ b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.cpp @@ -14,11 +14,13 @@ #include "rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.hpp" +#include #include #include #include #include -#include +#include +#include #include "rcutils/types.h" #include "rosbag2_storage/serialized_bag_message.hpp" @@ -31,7 +33,9 @@ namespace rosbag2_storage_plugins { SqliteWrapper::SqliteWrapper( - const std::string & uri, rosbag2_storage::storage_interfaces::IOFlag io_flag) + const std::string & uri, + rosbag2_storage::storage_interfaces::IOFlag io_flag, + std::unordered_map && pragmas) : db_ptr(nullptr) { if (io_flag == rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY) { @@ -44,8 +48,6 @@ SqliteWrapper::SqliteWrapper( rc << "): " << sqlite3_errstr(rc); throw SqliteException{errmsg.str()}; } - // throws an exception if the database is not valid. - prepare_statement("PRAGMA schema_version;")->execute_and_reset(); } else { int rc = sqlite3_open_v2( uri.c_str(), &db_ptr, @@ -56,10 +58,9 @@ SqliteWrapper::SqliteWrapper( rc << "): " << sqlite3_errstr(rc); throw SqliteException{errmsg.str()}; } - prepare_statement("PRAGMA journal_mode = WAL;")->execute_and_reset(); - prepare_statement("PRAGMA synchronous = NORMAL;")->execute_and_reset(); } + apply_pragma_settings(pragmas, io_flag); sqlite3_extended_result_codes(db_ptr, 1); } @@ -76,6 +77,46 @@ SqliteWrapper::~SqliteWrapper() } } +void SqliteWrapper::apply_pragma_settings( + std::unordered_map & pragmas, + rosbag2_storage::storage_interfaces::IOFlag io_flag) +{ + // Add default pragmas if not overridden by user setting + { + typedef std::unordered_map pragmas_map_t; + pragmas_map_t default_pragmas = { + // Used to check whether db is readable + {"schema_version", "PRAGMA schema_version;"} + }; + if (io_flag == rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE) { + const pragmas_map_t write_default_pragmas = { + {"journal_mode", "PRAGMA journal_mode=WAL;"}, + {"synchronous", "PRAGMA synchronous=NORMAL;"} + }; + default_pragmas.insert(write_default_pragmas.begin(), write_default_pragmas.end()); + } + for (const auto & kv : default_pragmas) { + auto key = kv.first; + auto default_full_statement = kv.second; + // insert default if other value not specified for the same key + if (pragmas.find(key) == pragmas.end()) { + pragmas.insert(std::make_pair(key, default_full_statement)); + } + } + } + + for (auto & kv : pragmas) { + // Apply the setting. Note that statements that assign value do not reliably return value + auto pragma_name = kv.first; + auto pragma_statement = kv.second; + prepare_statement(pragma_statement)->execute_and_reset(); + + // Check if the value is set, reading the pragma + auto statement_for_check = "PRAGMA " + pragma_name + ";"; + prepare_statement(statement_for_check)->execute_and_reset(true); + } +} + SqliteStatement SqliteWrapper::prepare_statement(const std::string & query) { return std::make_shared(db_ptr, query); diff --git a/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/storage_test_fixture.hpp b/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/storage_test_fixture.hpp index 93a4ffbfb5..a80c06ef2e 100644 --- a/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/storage_test_fixture.hpp +++ b/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/storage_test_fixture.hpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -136,6 +137,24 @@ class StorageTestFixture : public TemporaryDirectoryFixture return read_messages; } + rosbag2_storage::StorageOptions make_storage_options_with_config( + const std::string & config_yaml, + const std::string & plugin_id) + { + auto temp_dir = rcpputils::fs::path(temporary_dir_path_); + const auto storage_uri = (temp_dir / "rosbag").string(); + const auto yaml_config = (temp_dir / "sqlite_config.yaml").string(); + + { // populate temporary config file + std::ofstream out(yaml_config); + out << config_yaml; + } + + rosbag2_storage::StorageOptions storage_options{storage_uri, plugin_id, 0, 0, 0, + yaml_config}; + return storage_options; + } + protected: int get_buffer_capacity(const std::string & message) { diff --git a/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/test_sqlite_storage.cpp b/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/test_sqlite_storage.cpp index 06e75e25f4..34ed38e717 100644 --- a/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/test_sqlite_storage.cpp +++ b/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/test_sqlite_storage.cpp @@ -311,3 +311,25 @@ TEST_F(StorageTestFixture, get_relative_file_path_returns_db_name_with_ext) { rosbag2_storage::storage_interfaces::IOFlag::APPEND); EXPECT_EQ(append_storage->get_relative_file_path(), storage_filename); } + +TEST_F(StorageTestFixture, loads_config_file) { + // Check that storage opens with correct sqlite config file + const auto valid_yaml = "write:\n pragmas: [\"journal_mode = MEMORY\"]\n"; + const auto writable_storage = std::make_unique(); + EXPECT_NO_THROW( + writable_storage->open( + make_storage_options_with_config(valid_yaml, kPluginID), + rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE)); +} + +TEST_F(StorageTestFixture, throws_on_invalid_pragma_in_config_file) { + // Check that storage throws on invalid pragma statement in sqlite config + const auto invalid_yaml = "write:\n pragmas: [\"unrecognized_pragma_name = 2\"]\n"; + const auto writable_storage = std::make_unique(); + + EXPECT_THROW( + writable_storage->open( + make_storage_options_with_config(invalid_yaml, kPluginID), + rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE), + std::runtime_error); +} From 56df799351fc46cbd833d79bb5af3cbc60be07cc Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Fri, 20 Nov 2020 14:26:41 -0800 Subject: [PATCH 45/77] Prepare 0.4.0 (#566) * Update changelogs and setup.py Signed-off-by: Michael Jeronimo * Update versions in new packages to match (so that we can run bloom) Signed-off-by: Michael Jeronimo * 0.4.0 Signed-off-by: Emerson Knapp --- ros2bag/CHANGELOG.rst | 25 +++++++ ros2bag/package.xml | 2 +- ros2bag/setup.py | 2 +- rosbag2/CHANGELOG.rst | 8 ++ rosbag2/package.xml | 2 +- rosbag2_compression/CHANGELOG.rst | 12 +++ rosbag2_compression/package.xml | 2 +- .../CHANGELOG.rst | 5 ++ rosbag2_converter_default_plugins/package.xml | 2 +- rosbag2_cpp/CHANGELOG.rst | 19 +++++ rosbag2_cpp/package.xml | 2 +- .../CHANGELOG.rst | 71 ++++++++++++++++++ .../package.xml | 2 +- rosbag2_py/CHANGELOG.rst | 74 +++++++++++++++++++ rosbag2_py/package.xml | 2 +- rosbag2_storage/CHANGELOG.rst | 7 ++ rosbag2_storage/package.xml | 2 +- rosbag2_storage_default_plugins/CHANGELOG.rst | 7 ++ rosbag2_storage_default_plugins/package.xml | 2 +- rosbag2_test_common/CHANGELOG.rst | 5 ++ rosbag2_test_common/package.xml | 2 +- rosbag2_tests/CHANGELOG.rst | 13 ++++ rosbag2_tests/package.xml | 2 +- rosbag2_transport/CHANGELOG.rst | 13 ++++ rosbag2_transport/package.xml | 2 +- shared_queues_vendor/CHANGELOG.rst | 5 ++ shared_queues_vendor/package.xml | 2 +- sqlite3_vendor/CHANGELOG.rst | 6 ++ sqlite3_vendor/package.xml | 2 +- zstd_vendor/CHANGELOG.rst | 5 ++ zstd_vendor/package.xml | 2 +- 31 files changed, 291 insertions(+), 16 deletions(-) create mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst create mode 100644 rosbag2_py/CHANGELOG.rst diff --git a/ros2bag/CHANGELOG.rst b/ros2bag/CHANGELOG.rst index 32a176364e..853f2c5f9d 100644 --- a/ros2bag/CHANGELOG.rst +++ b/ros2bag/CHANGELOG.rst @@ -24,6 +24,31 @@ Changelog for package ros2bag 0.3.3 (2020-06-23) ------------------ +0.4.0 (2020-11-19) +------------------ +* read yaml config file (`#497 `_) +* List all storage plugins in plugin xml file (`#554 `_) +* add storage_config_uri (`#493 `_) +* Update deprecated qos policy value names (`#548 `_) +* Add record test for ros2bag (`#523 `_) +* Removed duplicated code in record (`#534 `_) +* Change default cache size for sequential_writer to a non zero value (`#533 `_) +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* [ros2bag test_record] Gets rid of time.sleep and move to using command.wait_for_output (`#525 `_) +* Add pytest.ini back to ros2bag. (`#492 `_) +* performance testing packages (`#442 `_) +* Validate QoS profile values are not negative. (`#483 `_) +* catch parent exception (`#472 `_) +* add wait for closed file handles on Windows (`#470 `_) +* introduce ros2 bag list (`#468 `_) +* move wait_for_shutdown() call out of the context manager (`#466 `_) +* Adding db directory creation to rosbag2_cpp (`#450 `_) +* use a single temp dir for the test class (`#462 `_) +* Add per-message ZSTD compression (`#418 `_) +* Add split by time to recording (`#409 `_) +* Add pytest.ini so local tests don't display warning (`#446 `_) +* Contributors: Adam Dąbrowski, Barry Xu, Chris Lalancette, Dirk Thomas, Ivan Santiago Paunovic, Jacob Perron, Jaison Titus, Jesse Ikawa, Karsten Knese, Marwan Taher, Michael Jeronimo, P. J. Reed, jhdcs + 0.3.2 (2020-06-03) ------------------ * Improve help message for CLI verbs (`#427 `_) diff --git a/ros2bag/package.xml b/ros2bag/package.xml index 4a5540acc9..ecc7f026ee 100644 --- a/ros2bag/package.xml +++ b/ros2bag/package.xml @@ -2,7 +2,7 @@ ros2bag - 0.3.7 + 0.4.0 Entry point for rosbag in ROS 2 diff --git a/ros2bag/setup.py b/ros2bag/setup.py index 3d9475c4df..981cb61998 100644 --- a/ros2bag/setup.py +++ b/ros2bag/setup.py @@ -5,7 +5,7 @@ setup( name=package_name, - version='0.3.7', + version='0.4.0', packages=find_packages(exclude=['test']), data_files=[ ('share/' + package_name, ['package.xml']), diff --git a/rosbag2/CHANGELOG.rst b/rosbag2/CHANGELOG.rst index c767cc3d13..ea0557322d 100644 --- a/rosbag2/CHANGELOG.rst +++ b/rosbag2/CHANGELOG.rst @@ -19,6 +19,14 @@ Changelog for package rosbag2 0.3.3 (2020-06-23) ------------------ +0.4.0 (2020-11-19) +------------------ +* add storage_config_uri (`#493 `_) +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* AMENT_IGNORE rosbag2_py for now (`#509 `_) +* rosbag2_py reader and writer (`#308 `_) +* Contributors: Karsten Knese, Mabel Zhang, Michael Jeronimo + 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2/package.xml b/rosbag2/package.xml index c4266fec25..a281c7e4ed 100644 --- a/rosbag2/package.xml +++ b/rosbag2/package.xml @@ -1,7 +1,7 @@ rosbag2 - 0.3.7 + 0.4.0 Meta package for rosbag2 related packages Karsten Knese Michael Jeronimo diff --git a/rosbag2_compression/CHANGELOG.rst b/rosbag2_compression/CHANGELOG.rst index 63370e6fd8..30a2251ff9 100644 --- a/rosbag2_compression/CHANGELOG.rst +++ b/rosbag2_compression/CHANGELOG.rst @@ -21,6 +21,18 @@ Changelog for package rosbag2_compression 0.3.3 (2020-06-23) ------------------ +0.4.0 (2020-11-19) +------------------ +* add storage_config_uri (`#493 `_) +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Do not expect empty StorageOptions URI to work in CompressionWriterTest (`#526 `_) +* Remove some code duplication between SequentialWriter and SequentialCompressionWriter (`#527 `_) +* Fix exception thrown given invalid arguments with compression enabled (`#488 `_) +* Adding db directory creation to rosbag2_cpp (`#450 `_) +* Consolidate ZSTD utility functions (`#459 `_) +* Add per-message ZSTD compression (`#418 `_) +* Contributors: Christophe Bedard, Devin Bonnie, Jaison Titus, Karsten Knese, Marwan Taher, Michael Jeronimo, P. J. Reed + 0.3.2 (2020-06-03) ------------------ * Add user provided split size to error message (`#430 `_) diff --git a/rosbag2_compression/package.xml b/rosbag2_compression/package.xml index 9d1e9aa645..7e0cf7b47c 100644 --- a/rosbag2_compression/package.xml +++ b/rosbag2_compression/package.xml @@ -2,7 +2,7 @@ rosbag2_compression - 0.3.7 + 0.4.0 Compression implementations for rosbag2 bags and messages. Karsten Knese Michael Jeronimo diff --git a/rosbag2_converter_default_plugins/CHANGELOG.rst b/rosbag2_converter_default_plugins/CHANGELOG.rst index 320407bf37..ccdf73b46b 100644 --- a/rosbag2_converter_default_plugins/CHANGELOG.rst +++ b/rosbag2_converter_default_plugins/CHANGELOG.rst @@ -20,6 +20,11 @@ Changelog for package rosbag2_converter_default_plugins 0.3.3 (2020-06-23) ------------------ +0.4.0 (2020-11-19) +------------------ +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Contributors: Michael Jeronimo + 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2_converter_default_plugins/package.xml b/rosbag2_converter_default_plugins/package.xml index da550f756e..f37f00ace4 100644 --- a/rosbag2_converter_default_plugins/package.xml +++ b/rosbag2_converter_default_plugins/package.xml @@ -1,7 +1,7 @@ rosbag2_converter_default_plugins - 0.3.7 + 0.4.0 Package containing default plugins for format converters Karsten Knese Michael Jeronimo diff --git a/rosbag2_cpp/CHANGELOG.rst b/rosbag2_cpp/CHANGELOG.rst index e6fd6919f0..bbbc2c5cb4 100644 --- a/rosbag2_cpp/CHANGELOG.rst +++ b/rosbag2_cpp/CHANGELOG.rst @@ -23,6 +23,25 @@ Changelog for package rosbag2 0.3.3 (2020-06-23) ------------------ +0.4.0 (2020-11-19) +------------------ +* correct master build (`#552 `_) +* add storage_config_uri (`#493 `_) +* Mutex around writer access in recorder (`#491 `_) +* if cache data exists, it needs to flush the data into the storage before shutdown (`#541 `_) +* Change default cache size for sequential_writer to a non zero value (`#533 `_) +* SequentialWriter to cache by message size instead of message count (`#530 `_) +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Remove some code duplication between SequentialWriter and SequentialCompressionWriter (`#527 `_) +* disable sanitizer by default (`#517 `_) +* Fix typo in error message (`#475 `_) +* introduce defaults for the C++ API (`#452 `_) +* Adding db directory creation to rosbag2_cpp (`#450 `_) +* comment out unused variable (`#460 `_) +* minimal c++ API test (`#451 `_) +* Add split by time to recording (`#409 `_) +* Contributors: Dirk Thomas, Jacob Perron, Jaison Titus, Karsten Knese, Marwan Taher, Michael Jeronimo, Patrick Spieler, jhdcs, Tomoya Fujita + 0.3.2 (2020-06-03) ------------------ * Add user provided split size to error (`#430 `_) diff --git a/rosbag2_cpp/package.xml b/rosbag2_cpp/package.xml index 7e848b786a..53c723be97 100644 --- a/rosbag2_cpp/package.xml +++ b/rosbag2_cpp/package.xml @@ -1,7 +1,7 @@ rosbag2_cpp - 0.3.7 + 0.4.0 C++ ROSBag2 client library Karsten Knese Michael Jeronimo diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst b/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst new file mode 100644 index 0000000000..fda722bf83 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst @@ -0,0 +1,71 @@ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Changelog for package rosbag2_performance_writer_benchmarking +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +0.4.0 (2020-11-19) +------------------ +* read yaml config file (`#497 `_) +* add storage_config_uri (`#493 `_) +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* performance testing packages (`#442 `_) +* Contributors: Adam Dąbrowski, Karsten Knese, Michael Jeronimo + +0.3.2 (2020-06-03) +------------------ + +0.3.1 (2020-06-01) +------------------ + +0.3.0 (2020-05-26) +------------------ + +0.2.8 (2020-05-18) +------------------ + +0.2.7 (2020-05-12) +------------------ + +0.2.6 (2020-05-07) +------------------ + +0.2.5 (2020-04-30) +------------------ + +0.2.4 (2019-11-18 17:51) +------------------------ + +0.2.3 (2019-11-18 13:55) +------------------------ + +0.2.2 (2019-11-13) +------------------ + +0.2.1 (2019-10-23) +------------------ + +0.2.0 (2019-09-26) +------------------ + +0.1.2 (2019-05-20) +------------------ + +0.1.1 (2019-05-09) +------------------ + +0.1.0 (2019-05-08) +------------------ + +0.0.5 (2018-12-27) +------------------ + +0.0.4 (2018-12-19) +------------------ + +0.0.3 (2018-12-14) +------------------ + +0.0.2 (2018-12-12) +------------------ + +0.0.1 (2018-12-11) +------------------ diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml b/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml index 8e771db33c..6232a6086b 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml @@ -2,7 +2,7 @@ rosbag2_performance_writer_benchmarking - 0.0.2 + 0.4.0 Code to benchmark the part of rosbag2 which starts after the callback is received Karsten Knese Michael Jeronimo diff --git a/rosbag2_py/CHANGELOG.rst b/rosbag2_py/CHANGELOG.rst new file mode 100644 index 0000000000..93e06fb611 --- /dev/null +++ b/rosbag2_py/CHANGELOG.rst @@ -0,0 +1,74 @@ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Changelog for package rosbag2_py +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +0.4.0 (2020-11-19) +------------------ +* add storage_config_uri (`#493 `_) +* Workaround pybind11 bug on Windows when CMAKE_BUILD_TYPE=RelWithDebInfo (`#538 `_) +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Fix rosbag2_py on Windows debug and stop ignoring the package (`#531 `_) +* Fix rosbag2_py bug when using libc++ (`#529 `_) +* AMENT_IGNORE rosbag2_py for now (`#509 `_) +* rosbag2_py reader and writer (`#308 `_) +* Contributors: Ivan Santiago Paunovic, Karsten Knese, Mabel Zhang, Michael Jeronimo + +0.3.2 (2020-06-03) +------------------ + +0.3.1 (2020-06-01) +------------------ + +0.3.0 (2020-05-26) +------------------ + +0.2.8 (2020-05-18) +------------------ + +0.2.7 (2020-05-12) +------------------ + +0.2.6 (2020-05-07) +------------------ + +0.2.5 (2020-04-30) +------------------ + +0.2.4 (2019-11-18 17:51) +------------------------ + +0.2.3 (2019-11-18 13:55) +------------------------ + +0.2.2 (2019-11-13) +------------------ + +0.2.1 (2019-10-23) +------------------ + +0.2.0 (2019-09-26) +------------------ + +0.1.2 (2019-05-20) +------------------ + +0.1.1 (2019-05-09) +------------------ + +0.1.0 (2019-05-08) +------------------ + +0.0.5 (2018-12-27) +------------------ + +0.0.4 (2018-12-19) +------------------ + +0.0.3 (2018-12-14) +------------------ + +0.0.2 (2018-12-12) +------------------ + +0.0.1 (2018-12-11) +------------------ diff --git a/rosbag2_py/package.xml b/rosbag2_py/package.xml index c259613297..8470101724 100644 --- a/rosbag2_py/package.xml +++ b/rosbag2_py/package.xml @@ -2,7 +2,7 @@ rosbag2_py - 0.2.4 + 0.4.0 Python API for rosbag2 Karsten Knese Michael Jeronimo diff --git a/rosbag2_storage/CHANGELOG.rst b/rosbag2_storage/CHANGELOG.rst index e245eb6a5f..7f8b4b1ec0 100644 --- a/rosbag2_storage/CHANGELOG.rst +++ b/rosbag2_storage/CHANGELOG.rst @@ -20,6 +20,13 @@ Changelog for package rosbag2_storage 0.3.3 (2020-06-23) ------------------ +0.4.0 (2020-11-19) +------------------ +* add storage_config_uri (`#493 `_) +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Add split by time to recording (`#409 `_) +* Contributors: Karsten Knese, Michael Jeronimo, jhdcs + 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2_storage/package.xml b/rosbag2_storage/package.xml index 81a67461cf..f0a5001821 100644 --- a/rosbag2_storage/package.xml +++ b/rosbag2_storage/package.xml @@ -1,7 +1,7 @@ rosbag2_storage - 0.3.7 + 0.4.0 ROS2 independent storage format to store serialized ROS2 messages Karsten Knese Michael Jeronimo diff --git a/rosbag2_storage_default_plugins/CHANGELOG.rst b/rosbag2_storage_default_plugins/CHANGELOG.rst index ff875fdc75..296ff07320 100644 --- a/rosbag2_storage_default_plugins/CHANGELOG.rst +++ b/rosbag2_storage_default_plugins/CHANGELOG.rst @@ -20,6 +20,13 @@ Changelog for package rosbag2_storage_default_plugins 0.3.3 (2020-06-23) ------------------ +0.4.0 (2020-11-19) +------------------ +* read yaml config file (`#497 `_) +* add storage_config_uri (`#493 `_) +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Contributors: Karsten Knese, Michael Jeronimo + 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2_storage_default_plugins/package.xml b/rosbag2_storage_default_plugins/package.xml index e6af70b940..eced9d0d10 100644 --- a/rosbag2_storage_default_plugins/package.xml +++ b/rosbag2_storage_default_plugins/package.xml @@ -1,7 +1,7 @@ rosbag2_storage_default_plugins - 0.3.7 + 0.4.0 ROSBag2 SQLite3 storage plugin Karsten Knese Michael Jeronimo diff --git a/rosbag2_test_common/CHANGELOG.rst b/rosbag2_test_common/CHANGELOG.rst index 96732488b0..aeca6e67f8 100644 --- a/rosbag2_test_common/CHANGELOG.rst +++ b/rosbag2_test_common/CHANGELOG.rst @@ -19,6 +19,11 @@ Changelog for package rosbag2_test_common 0.3.3 (2020-06-23) ------------------ +0.4.0 (2020-11-19) +------------------ +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Contributors: Michael Jeronimo + 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2_test_common/package.xml b/rosbag2_test_common/package.xml index c2e16bca18..a637159bd3 100644 --- a/rosbag2_test_common/package.xml +++ b/rosbag2_test_common/package.xml @@ -2,7 +2,7 @@ rosbag2_test_common - 0.3.7 + 0.4.0 Commonly used test helper classes and fixtures for rosbag2 Karsten Knese Michael Jeronimo diff --git a/rosbag2_tests/CHANGELOG.rst b/rosbag2_tests/CHANGELOG.rst index 449ef7b28f..d7fad4fcf1 100644 --- a/rosbag2_tests/CHANGELOG.rst +++ b/rosbag2_tests/CHANGELOG.rst @@ -22,6 +22,19 @@ Changelog for package rosbag2_tests 0.3.3 (2020-06-23) ------------------ +0.4.0 (2020-11-19) +------------------ +* add storage_config_uri (`#493 `_) +* Removed duplicated code in record (`#534 `_) +* Change default cache size for sequential_writer to a non zero value (`#533 `_) +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Mark flaky tests as xfail for now (`#520 `_) +* introduce defaults for the C++ API (`#452 `_) +* Adding db directory creation to rosbag2_cpp (`#450 `_) +* minimal c++ API test (`#451 `_) +* Add split by time to recording (`#409 `_) +* Contributors: Emerson Knapp, Jaison Titus, Karsten Knese, Marwan Taher, Michael Jeronimo, jhdcs + 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2_tests/package.xml b/rosbag2_tests/package.xml index a694bf7518..f0bdfa4f22 100644 --- a/rosbag2_tests/package.xml +++ b/rosbag2_tests/package.xml @@ -2,7 +2,7 @@ rosbag2_tests - 0.3.7 + 0.4.0 Tests package for rosbag2 Karsten Knese Michael Jeronimo diff --git a/rosbag2_transport/CHANGELOG.rst b/rosbag2_transport/CHANGELOG.rst index 7618c74d61..12b28c5ab6 100644 --- a/rosbag2_transport/CHANGELOG.rst +++ b/rosbag2_transport/CHANGELOG.rst @@ -28,6 +28,19 @@ Changelog for package rosbag2_transport * export shared_queues_vendor for modern cmake support (`#434 `_) (`#438 `_) * Contributors: Karsten Knese +0.4.0 (2020-11-19) +------------------ +* add storage_config_uri (`#493 `_) +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* resolve memory leak for serialized message (`#502 `_) +* Use shared logic for importing the rosbag2_transport_py library in Python (`#482 `_) +* fix missing target dependencies (`#479 `_) +* reenable cppcheck for rosbag2_transport (`#461 `_) +* More reliable topic remapping test (`#456 `_) +* Add split by time to recording (`#409 `_) +* export shared_queues_vendor (`#434 `_) +* Contributors: Dirk Thomas, Emerson Knapp, Karsten Knese, Michael Jeronimo, jhdcs + 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2_transport/package.xml b/rosbag2_transport/package.xml index 5a27708309..b484a9a2d9 100644 --- a/rosbag2_transport/package.xml +++ b/rosbag2_transport/package.xml @@ -2,7 +2,7 @@ rosbag2_transport - 0.3.7 + 0.4.0 Layer encapsulating ROS middleware to allow rosbag2 to be used with or without middleware Karsten Knese Michael Jeronimo diff --git a/shared_queues_vendor/CHANGELOG.rst b/shared_queues_vendor/CHANGELOG.rst index 053899856f..7c4b5f4260 100644 --- a/shared_queues_vendor/CHANGELOG.rst +++ b/shared_queues_vendor/CHANGELOG.rst @@ -20,6 +20,11 @@ Changelog for package shared_queues_vendor 0.3.3 (2020-06-23) ------------------ +0.4.0 (2020-11-19) +------------------ +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Contributors: Michael Jeronimo + 0.3.2 (2020-06-03) ------------------ diff --git a/shared_queues_vendor/package.xml b/shared_queues_vendor/package.xml index 6fa5bcdd80..c1bfda97fe 100644 --- a/shared_queues_vendor/package.xml +++ b/shared_queues_vendor/package.xml @@ -2,7 +2,7 @@ shared_queues_vendor - 0.3.7 + 0.4.0 Vendor package for concurrent queues from moodycamel Karsten Knese Michael Jeronimo diff --git a/sqlite3_vendor/CHANGELOG.rst b/sqlite3_vendor/CHANGELOG.rst index e699ab7524..8363024594 100644 --- a/sqlite3_vendor/CHANGELOG.rst +++ b/sqlite3_vendor/CHANGELOG.rst @@ -22,6 +22,12 @@ Changelog for package sqlite3_vendor * use interface_include_directories in find_sqlite3 (`#426 `_) (`#444 `_) * Contributors: Karsten Knese +0.4.0 (2020-11-19) +------------------ +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* use interface_include_directories (`#426 `_) +* Contributors: Karsten Knese, Michael Jeronimo + 0.3.2 (2020-06-03) ------------------ diff --git a/sqlite3_vendor/package.xml b/sqlite3_vendor/package.xml index 29c5af871c..ab6cd8adf5 100644 --- a/sqlite3_vendor/package.xml +++ b/sqlite3_vendor/package.xml @@ -2,7 +2,7 @@ sqlite3_vendor - 0.3.7 + 0.4.0 SQLite 3 vendor package Karsten Knese Michael Jeronimo diff --git a/zstd_vendor/CHANGELOG.rst b/zstd_vendor/CHANGELOG.rst index a97658115b..633e8bb51d 100644 --- a/zstd_vendor/CHANGELOG.rst +++ b/zstd_vendor/CHANGELOG.rst @@ -22,6 +22,11 @@ Changelog for package zstd_vendor 0.3.3 (2020-06-23) ------------------ +0.4.0 (2020-11-19) +------------------ +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Contributors: Michael Jeronimo + 0.3.2 (2020-06-03) ------------------ diff --git a/zstd_vendor/package.xml b/zstd_vendor/package.xml index be0b5facd4..29101a6f33 100644 --- a/zstd_vendor/package.xml +++ b/zstd_vendor/package.xml @@ -2,7 +2,7 @@ zstd_vendor - 0.3.7 + 0.4.0 Zstd compression vendor package, providing a dependency for Zstd. Karsten Knese Michael Jeronimo From c1cbb6753ce4a28cd537e57b76c4025152503dbf Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Mon, 23 Nov 2020 16:10:43 -0800 Subject: [PATCH 46/77] Bump action-ros-ci to 0.1.0 (#564) Signed-off-by: Emerson Knapp --- .github/workflows/lint.yml | 8 -------- .github/workflows/test.yml | 4 +--- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8d45f1e5ff..c61e5d2ae6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,8 +10,6 @@ jobs: container: image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest steps: - # TODO(setup-ros-docker#7): calling chown is necessary for now - - run: sudo chown -R rosbuild:rosbuild "$HOME" . - uses: actions/checkout@v2 - uses: ros-tooling/action-ros-lint@0.0.6 with: @@ -33,8 +31,6 @@ jobs: container: image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest steps: - # TODO(setup-ros-docker#7): calling chown is necessary for now - - run: sudo chown -R rosbuild:rosbuild "$HOME" . - uses: actions/checkout@v2 - uses: ros-tooling/action-ros-lint@0.0.6 with: @@ -62,8 +58,6 @@ jobs: container: image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest steps: - # TODO(setup-ros-docker#7): calling chown is necessary for now - - run: sudo chown -R rosbuild:rosbuild "$HOME" . - uses: actions/checkout@v2 - uses: ros-tooling/action-ros-lint@0.0.6 with: @@ -88,8 +82,6 @@ jobs: matrix: linter: [pep257, flake8] steps: - # TODO(setup-ros-docker#7): calling chown is necessary for now - - run: sudo chown -R rosbuild:rosbuild "$HOME" . - uses: actions/checkout@v2 - uses: ros-tooling/action-ros-lint@0.0.6 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9e5feb427c..e34860a509 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,9 +15,7 @@ jobs: container: image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest steps: - # TODO(setup-ros-docker#7): calling chown is necessary for now - - run: sudo chown -R rosbuild:rosbuild "$HOME" . - - uses: ros-tooling/action-ros-ci@0.0.16 + - uses: ros-tooling/action-ros-ci@0.1.0 with: package-name: | ros2bag From 350ce0ecb658852966b8a62428020e028461347c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C4=85browski?= Date: Wed, 25 Nov 2020 19:58:15 +0100 Subject: [PATCH 47/77] Sqlite storage double buffering (#546) * Double buffers Signed-off-by: Piotr Jaroszek * Circular queue and FLUSH option as define Signed-off-by: Piotr Jaroszek * Minor naming and lexical fixes. Signed-off-by: Piotr Jaroszek * Removed FLUSH_BUFFERS define check. Signed-off-by: Piotr Jaroszek * Sqlite3 storage logging fixes. Signed-off-by: Piotr Jaroszek * Sqlite3 storage circular buffer with pre allocated memory. Signed-off-by: Piotr Jaroszek * Sqlite3 storage buffers moved to shared_ptrs. Signed-off-by: Piotr Jaroszek * Uncrustify Signed-off-by: Piotr Jaroszek * Moved double buffers to writer Signed-off-by: Piotr Jaroszek * Buffer layer reset in seq compression writer in rosbag2 cpp Signed-off-by: Piotr Jaroszek * Buffer layer for rosbag2 writer refactor Signed-off-by: Piotr Jaroszek * Changed buffers in BufferLayer to std vectors. Signed-off-by: Piotr Jaroszek * BufferLayer uncrustify Signed-off-by: Piotr Jaroszek * Removed non-applicable test for writer cache. Signed-off-by: Piotr Jaroszek * BufferLayer review fixes Signed-off-by: Piotr Jaroszek * Rosbag metadata msgs count fixed for BufferLayer Signed-off-by: Piotr Jaroszek * Condition variable for buffer layer sync. Signed-off-by: Piotr Jaroszek * Fixed buffer locks Signed-off-by: Piotr Jaroszek * Buffers in BufferLayer refactored, moved into new class Signed-off-by: Piotr Jaroszek * Buffer layer split bags fixed. Signed-off-by: Piotr Jaroszek * Storage options include fix in buffer layer header. Signed-off-by: Piotr Jaroszek * Mutex around swapping buffers in buffer layer. Signed-off-by: Piotr Jaroszek * Fixed cache 0 bug in buffer layer. Signed-off-by: Piotr Jaroszek * Minor buffer layer refactor. Signed-off-by: Piotr Jaroszek * Counting messages in writer refactored. Signed-off-by: Piotr Jaroszek * Changed default cache size to 100Mb and updated parameter description Signed-off-by: Adam Dabrowski * Applied review remarks: - significant refactoring: separation of cache classes - applied suggested improvements - some renaming - reduce code duplication that would otherwise increase with cache refactor, between compression and plain writers Signed-off-by: Adam Dabrowski * Applied review comments - cache consumer now takes a callback and is independent of storage - namespace changes, renaming, cleaning - counting and logging messages by topic Signed-off-by: Adam Dabrowski * linter Signed-off-by: Adam Dabrowski * Changes after review: fixing flushing, topic counts, and more Signed-off-by: Adam Dabrowski * Fix for splitting - flushing state now correctly turns off Signed-off-by: Adam Dabrowski * cache classes documentation Signed-off-by: Adam Dabrowski * simplified signature Signed-off-by: Adam Dabrowski * a couple of tests for cache Signed-off-by: Adam Dabrowski * address review: explicit constructor and doxygen styling Signed-off-by: Adam Dabrowski * Windows warnings fix Signed-off-by: Adam Dabrowski * fixed type mismatch warning on Windows Signed-off-by: Adam Dabrowski * added minor comment Signed-off-by: Adam Dabrowski Co-authored-by: Piotr Jaroszek Signed-off-by: Emerson Knapp --- ros2bag/ros2bag/verb/record.py | 12 +- .../sequential_compression_writer.cpp | 33 +--- rosbag2_cpp/CMakeLists.txt | 9 ++ .../rosbag2_cpp/cache/cache_consumer.hpp | 99 ++++++++++++ .../rosbag2_cpp/cache/message_cache.hpp | 123 ++++++++++++++ .../cache/message_cache_buffer.hpp | 86 ++++++++++ .../rosbag2_cpp/writers/sequential_writer.hpp | 22 ++- .../src/rosbag2_cpp/cache/cache_consumer.cpp | 87 ++++++++++ .../src/rosbag2_cpp/cache/message_cache.cpp | 139 ++++++++++++++++ .../cache/message_cache_buffer.cpp | 63 ++++++++ .../rosbag2_cpp/writers/sequential_writer.cpp | 135 +++++++++++----- .../test/rosbag2_cpp/mock_cache_consumer.hpp | 33 ++++ .../test/rosbag2_cpp/mock_message_cache.hpp | 36 +++++ .../test/rosbag2_cpp/test_message_cache.cpp | 152 ++++++++++++++++++ .../rosbag2_cpp/test_sequential_writer.cpp | 35 ---- .../writer_benchmark.hpp | 2 +- 16 files changed, 948 insertions(+), 118 deletions(-) create mode 100644 rosbag2_cpp/include/rosbag2_cpp/cache/cache_consumer.hpp create mode 100644 rosbag2_cpp/include/rosbag2_cpp/cache/message_cache.hpp create mode 100644 rosbag2_cpp/include/rosbag2_cpp/cache/message_cache_buffer.hpp create mode 100644 rosbag2_cpp/src/rosbag2_cpp/cache/cache_consumer.cpp create mode 100644 rosbag2_cpp/src/rosbag2_cpp/cache/message_cache.cpp create mode 100644 rosbag2_cpp/src/rosbag2_cpp/cache/message_cache_buffer.cpp create mode 100644 rosbag2_cpp/test/rosbag2_cpp/mock_cache_consumer.hpp create mode 100644 rosbag2_cpp/test/rosbag2_cpp/mock_message_cache.hpp create mode 100644 rosbag2_cpp/test/rosbag2_cpp/test_message_cache.cpp diff --git a/ros2bag/ros2bag/verb/record.py b/ros2bag/ros2bag/verb/record.py index b9862efc21..b1223a7d6d 100644 --- a/ros2bag/ros2bag/verb/record.py +++ b/ros2bag/ros2bag/verb/record.py @@ -67,11 +67,13 @@ def add_arguments(self, parser, cli_name): # noqa: D102 'the bag will split at whichever threshold is reached first.' ) parser.add_argument( - '--max-cache-size', type=int, default=1024*1024, - help='maximum size (in bytes) of messages to hold in cache before writing to disk. ' - 'Default is 1 mebibyte, everytime the cache size equals or exceeds 1MB, ' - 'it will be written to disk. If the value specified is 0, then every message is' - 'directly written to disk.' + '--max-cache-size', type=int, default=100*1024*1024, + help='maximum size (in bytes) of messages to hold in each buffer of cache.' + 'Default is 100 mebibytes. The cache is handled through double buffering, ' + 'which means that in pessimistic case up to twice the parameter value of memory' + 'is needed. A rule of thumb is to cache an order of magitude corresponding to' + 'about one second of total recorded data volume.' + 'If the value specified is 0, then every message is directly written to disk.' ) parser.add_argument( '--compression-mode', type=str, default='none', diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp index 8c99a403ff..5c8d4cdc35 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp @@ -37,20 +37,6 @@ namespace rosbag2_compression { -namespace -{ -std::string format_storage_uri(const std::string & base_folder, uint64_t storage_count) -{ - // Right now `base_folder_` is always just the folder name for where to install the bagfile. - // The name of the folder needs to be queried in case - // SequentialWriter is opened with a relative path. - std::stringstream storage_file_name; - storage_file_name << rcpputils::fs::path(base_folder).filename().string() << "_" << storage_count; - - return (rcpputils::fs::path(base_folder) / storage_file_name.str()).string(); -} -} // namespace - SequentialCompressionWriter::SequentialCompressionWriter( const rosbag2_compression::CompressionOptions & compression_options) : SequentialWriter(), @@ -123,6 +109,10 @@ void SequentialCompressionWriter::reset() metadata_io_->write_metadata(base_folder_, metadata_); } + if (use_cache_) { + cache_consumer_.reset(); + message_cache_.reset(); + } storage_.reset(); // Necessary to ensure that the storage is destroyed before the factory storage_factory_.reset(); } @@ -155,11 +145,7 @@ void SequentialCompressionWriter::compress_last_file() void SequentialCompressionWriter::split_bagfile() { - storage_options_.uri = format_storage_uri( - base_folder_, - metadata_.relative_file_paths.size()); - - storage_ = storage_factory_->open_read_write(storage_options_); + switch_to_next_storage(); if (compression_options_.compression_mode == rosbag2_compression::CompressionMode::FILE) { compress_last_file(); @@ -169,18 +155,9 @@ void SequentialCompressionWriter::split_bagfile() // Add a check to make sure reset() does not compress the file again if we couldn't load the // storage plugin. should_compress_last_file_ = false; - - std::stringstream errmsg; - errmsg << "Failed to rollover bagfile to new file: \"" << storage_options_.uri << "\"!"; - throw std::runtime_error{errmsg.str()}; } metadata_.relative_file_paths.push_back(storage_->get_relative_file_path()); - - // Re-register all topics since we rolled-over to a new bagfile. - for (const auto & topic : topics_names_to_info_) { - storage_->create_topic(topic.second.topic_metadata); - } } void SequentialCompressionWriter::compress_message( diff --git a/rosbag2_cpp/CMakeLists.txt b/rosbag2_cpp/CMakeLists.txt index cd378c4e2c..1d78d5831f 100644 --- a/rosbag2_cpp/CMakeLists.txt +++ b/rosbag2_cpp/CMakeLists.txt @@ -45,6 +45,9 @@ find_package(rosidl_typesupport_cpp REQUIRED) find_package(rosidl_typesupport_introspection_cpp REQUIRED) add_library(${PROJECT_NAME} SHARED + src/rosbag2_cpp/cache/cache_consumer.cpp + src/rosbag2_cpp/cache/message_cache_buffer.cpp + src/rosbag2_cpp/cache/message_cache.cpp src/rosbag2_cpp/converter.cpp src/rosbag2_cpp/info.cpp src/rosbag2_cpp/reader.cpp @@ -156,6 +159,12 @@ if(BUILD_TESTING) target_link_libraries(test_storage_without_metadata_file ${PROJECT_NAME}) endif() + ament_add_gmock(test_message_cache + test/rosbag2_cpp/test_message_cache.cpp) + if(TARGET test_message_cache) + target_link_libraries(test_message_cache ${PROJECT_NAME}) + endif() + # If compiling with gcc, run this test with sanitizers enabled ament_add_gmock(test_ros2_message test/rosbag2_cpp/types/test_ros2_message.cpp diff --git a/rosbag2_cpp/include/rosbag2_cpp/cache/cache_consumer.hpp b/rosbag2_cpp/include/rosbag2_cpp/cache/cache_consumer.hpp new file mode 100644 index 0000000000..7a61485c15 --- /dev/null +++ b/rosbag2_cpp/include/rosbag2_cpp/cache/cache_consumer.hpp @@ -0,0 +1,99 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_CPP__CACHE__CACHE_CONSUMER_HPP_ +#define ROSBAG2_CPP__CACHE__CACHE_CONSUMER_HPP_ + +#include +#include +#include +#include +#include +#include + +#include "rosbag2_cpp/cache/message_cache.hpp" + +// This is necessary because of using stl types here. It is completely safe, because +// a) the member is not accessible from the outside +// b) there are no inline functions. +#ifdef _WIN32 +# pragma warning(push) +# pragma warning(disable:4251) +#endif + +namespace rosbag2_cpp +{ +namespace cache +{ +/** +* This class is responsible for consuming the cache using provided fuction. +* It can work with any callback conforming to the consume_callback_function_t +* signature, e.g. a storage write function. Consuming and thus the callback are +* called in a separate thread. +* +* Since the consuming callback likely involves disk operations, the main motivation +* for design is to make sure that consumer is busy anytime there is any work. +* This is realized through conditional variable and greedy buffer switching. +* +* The consumer uses MessageCache and waits for consumer buffer to be ready. This +* will happen as soon as that there are any messages put into producer buffer - +* a switch of buffer pointers will result in these messages being available for +* consumption. The consumer then proceeds to process the entire buffer in one go. +* While this is ongoing, the producer buffer is being filled with new messages. +* +* For SQLite implementation of storage, consumer callback will write consumer buffer +* in each loop iteration as a separate transaction. This results in a balancing +* mechanism for high-performance cases, where transaction size can be increased +* dynamically as previous, smaller transactions introduce delays in loop iteration. +*/ +class ROSBAG2_CPP_PUBLIC CacheConsumer +{ +public: + using consume_callback_function_t = std::function &)>; + + CacheConsumer( + std::shared_ptr message_cache, + consume_callback_function_t consume_callback); + + ~CacheConsumer(); + + /// shut down the consumer thread + void close(); + + /// Set new consume callback, restart thread if necessary + void change_consume_callback(consume_callback_function_t callback); + +private: + std::shared_ptr message_cache_; + consume_callback_function_t consume_callback_; + + /// Write buffer data to a storage + void exec_consuming(); + + /// Consumer thread shutdown sync + std::atomic_bool is_stop_issued_ {false}; + std::mutex consumer_mutex_; + + std::thread consumer_thread_; +}; + +} // namespace cache +} // namespace rosbag2_cpp + +#ifdef _WIN32 +# pragma warning(pop) +#endif + +#endif // ROSBAG2_CPP__CACHE__CACHE_CONSUMER_HPP_ diff --git a/rosbag2_cpp/include/rosbag2_cpp/cache/message_cache.hpp b/rosbag2_cpp/include/rosbag2_cpp/cache/message_cache.hpp new file mode 100644 index 0000000000..a24e15348a --- /dev/null +++ b/rosbag2_cpp/include/rosbag2_cpp/cache/message_cache.hpp @@ -0,0 +1,123 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_CPP__CACHE__MESSAGE_CACHE_HPP_ +#define ROSBAG2_CPP__CACHE__MESSAGE_CACHE_HPP_ + +#include +#include +#include +#include +#include +#include +#include + +#include "rosbag2_cpp/cache/message_cache_buffer.hpp" +#include "rosbag2_cpp/visibility_control.hpp" + +#include "rosbag2_storage/serialized_bag_message.hpp" + +// This is necessary because of using stl types here. It is completely safe, because +// a) the member is not accessible from the outside +// b) there are no inline functions. +#ifdef _WIN32 +# pragma warning(push) +# pragma warning(disable:4251) +#endif + +namespace rosbag2_cpp +{ +namespace cache +{ +/** +* This class is responsible for implementing message cache, using two cache +* buffers and providing synchronization API for producer-consumer pattern. +* +* Double buffering is a part of producer-consumer pattern and optimizes for +* the consumer performance (which can be a bottleneck, e.g. disk writes). +* +* Two instances of MessageCacheBuffer are used, one for producer and one for +* the consumer. Buffers are switched through wait_for_buffer function, which +* involves synchronization and a simple pointer switch. +* +* The cache can enter a flushing state, intended as a finalization state, +* where all the remaining data is going to be processed: no new messages are +* accepted and buffer switching can be done unconditionally on consumer demand. +* +* The cache holds infomation about dropped messages (per topic). These are +* messages that were pushed to the cache when it was full. Such situation signals +* performance issues, most likely with the CacheConsumer consumer callback. +*/ +class ROSBAG2_CPP_PUBLIC MessageCache +{ +public: + explicit MessageCache(uint64_t max_buffer_size); + + ~MessageCache(); + + /// Puts msg into primary buffer. With full cache, msg is ignored and counted as lost + void push(std::shared_ptr msg); + + /// Summarize dropped/remaining messages + void log_dropped(); + + /// Producer API: notify consumer to wake-up (primary buffer has data) + void notify_buffer_consumer(); + + /// Set the cache to consume-only mode for final buffer flush before closing + void finalize(); + + /// Notify that flushing is complete + void notify_flushing_done(); + + /** + * Consumer API: wait until primary buffer is ready and swap it with consumer buffer. + * The caller thread (consumer thread) will sleep on a conditional variable + * until it can be awaken, which is to happen when: + * a) data was inserted into the producer buffer, consuming can continue after a swap + * b) we are flushing the data (in case we missed the last notification when consuming) + **/ + void wait_for_buffer(); + + /// Consumer API: get current buffer to consume + std::shared_ptr consumer_buffer(); + + /// Exposes counts of messages dropped per topic + std::unordered_map messages_dropped() const; + +private: + /// Double buffers + std::shared_ptr primary_buffer_; + std::shared_ptr secondary_buffer_; + + /// Dropped messages per topic. Used for printing in alphabetic order + std::unordered_map messages_dropped_per_topic_; + + /// Double buffers sync (following cpp core guidelines for condition variables) + bool primary_buffer_can_be_swapped_ {false}; + std::condition_variable cache_condition_var_; + std::mutex cache_mutex_; + + /// Cache is no longer accepting messages and is in the process of flushing + std::atomic_bool flushing_ {false}; +}; + +} // namespace cache +} // namespace rosbag2_cpp + +#ifdef _WIN32 +# pragma warning(pop) +#endif + +#endif // ROSBAG2_CPP__CACHE__MESSAGE_CACHE_HPP_ diff --git a/rosbag2_cpp/include/rosbag2_cpp/cache/message_cache_buffer.hpp b/rosbag2_cpp/include/rosbag2_cpp/cache/message_cache_buffer.hpp new file mode 100644 index 0000000000..2fab5ed4a5 --- /dev/null +++ b/rosbag2_cpp/include/rosbag2_cpp/cache/message_cache_buffer.hpp @@ -0,0 +1,86 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_CPP__CACHE__MESSAGE_CACHE_BUFFER_HPP_ +#define ROSBAG2_CPP__CACHE__MESSAGE_CACHE_BUFFER_HPP_ + +#include +#include +#include + +#include "rosbag2_cpp/visibility_control.hpp" +#include "rosbag2_storage/serialized_bag_message.hpp" + +// This is necessary because of using stl types here. It is completely safe, because +// a) the member is not accessible from the outside +// b) there are no inline functions. +#ifdef _WIN32 +# pragma warning(push) +# pragma warning(disable:4251) +#endif + +namespace rosbag2_cpp +{ +namespace cache +{ +/** +* This class implements a single buffer for message cache. The buffer is byte size +* limited and won't accept any messages when current buffer byte size is already +* over the limit set by max_cache_size. This means that buffer can at times use +* more memory than max_cache_size, but never by more than a single message. When +* the buffer is full, the next incoming message is dropped. +* +* Note that it could be reused as a template with any class that has +* ->byte_size() - like interface +*/ +class ROSBAG2_CPP_PUBLIC MessageCacheBuffer +{ +public: + explicit MessageCacheBuffer(const uint64_t max_cache_size); + + using buffer_element_t = std::shared_ptr; + + /** + * If buffer size got some space left, we push message regardless of its size, but if + * this results in exceeding buffer size, we mark buffer to drop all new incoming messages. + * This flag is cleared when buffers are swapped. + */ + bool push(buffer_element_t msg); + + /// Clear buffer + void clear(); + + /// Get number of elements in the buffer + size_t size(); + + /// Get buffer data + const std::vector & data(); + +private: + std::vector buffer_; + uint64_t buffer_bytes_size_ {0u}; + const uint64_t max_bytes_size_; + + /// set when buffer is full and should drop messages instead of inserting them + std::atomic_bool drop_messages_ {false}; +}; + +} // namespace cache +} // namespace rosbag2_cpp + +#ifdef _WIN32 +# pragma warning(pop) +#endif + +#endif // ROSBAG2_CPP__CACHE__MESSAGE_CACHE_BUFFER_HPP_ diff --git a/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp b/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp index 0938675eaa..74766e72d4 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/writers/sequential_writer.hpp @@ -16,10 +16,13 @@ #define ROSBAG2_CPP__WRITERS__SEQUENTIAL_WRITER_HPP_ #include +#include #include #include #include +#include "rosbag2_cpp/cache/cache_consumer.hpp" +#include "rosbag2_cpp/cache/message_cache.hpp" #include "rosbag2_cpp/converter.hpp" #include "rosbag2_cpp/serialization_format_converter_factory.hpp" #include "rosbag2_cpp/writer_interfaces/base_writer_interface.hpp" @@ -111,18 +114,24 @@ class ROSBAG2_CPP_PUBLIC SequentialWriter std::unique_ptr metadata_io_; std::unique_ptr converter_; + bool use_cache_; + std::shared_ptr message_cache_; + std::unique_ptr cache_consumer_; + + void switch_to_next_storage(); + + std::string format_storage_uri( + const std::string & base_folder, uint64_t storage_count); + rosbag2_storage::StorageOptions storage_options_; // Used in bagfile splitting; // specifies the best-effort maximum duration of a bagfile in seconds. std::chrono::seconds max_bagfile_duration; - // Intermediate cache to write multiple messages into the storage. - uint64_t current_cache_size_; - std::vector> cache_; - - // Used to track topic -> message count + // Used to track topic -> message count. If cache is present, it is updated by CacheConsumer std::unordered_map topics_names_to_info_; + std::mutex topics_info_mutex_; rosbag2_storage::BagMetadata metadata_; @@ -138,9 +147,6 @@ class ROSBAG2_CPP_PUBLIC SequentialWriter // Record TopicInformation into metadata void finalize_metadata(); - // Flush data into storage, and reset cache - void reset_cache(); - // Helper method used by write to get the message in a format that is ready to be written. // Common use cases include converting the message using the converter or // performing other operations like compression on it diff --git a/rosbag2_cpp/src/rosbag2_cpp/cache/cache_consumer.cpp b/rosbag2_cpp/src/rosbag2_cpp/cache/cache_consumer.cpp new file mode 100644 index 0000000000..63dfd14420 --- /dev/null +++ b/rosbag2_cpp/src/rosbag2_cpp/cache/cache_consumer.cpp @@ -0,0 +1,87 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "rosbag2_cpp/cache/cache_consumer.hpp" +#include "rosbag2_cpp/logging.hpp" + +namespace rosbag2_cpp +{ +namespace cache +{ + +CacheConsumer::CacheConsumer( + std::shared_ptr message_cache, + consume_callback_function_t consume_callback) +: message_cache_(message_cache), + consume_callback_(consume_callback) +{ + consumer_thread_ = std::thread(&CacheConsumer::exec_consuming, this); +} + +CacheConsumer::~CacheConsumer() +{ + close(); +} + +void CacheConsumer::close() +{ + message_cache_->finalize(); + is_stop_issued_ = true; + + ROSBAG2_CPP_LOG_INFO_STREAM( + "Writing remaining messages from cache to the bag. It may take a while"); + + if (consumer_thread_.joinable()) { + consumer_thread_.join(); + } + message_cache_->notify_flushing_done(); +} + +void CacheConsumer::change_consume_callback( + CacheConsumer::consume_callback_function_t consume_callback) +{ + consume_callback_ = consume_callback; + if (!consumer_thread_.joinable()) { + is_stop_issued_ = false; + consumer_thread_ = std::thread(&CacheConsumer::exec_consuming, this); + } +} + +void CacheConsumer::exec_consuming() +{ + bool exit_flag = false; + bool flushing = false; + while (!exit_flag) { + // Invariant at loop start: consumer buffer is empty + + // swap producer buffer with consumer buffer + message_cache_->wait_for_buffer(); + + // make sure to use consistent callback for each iteration + auto callback_for_this_loop = consume_callback_; + + // consume all the data from consumer buffer + auto consumer_buffer = message_cache_->consumer_buffer(); + callback_for_this_loop(consumer_buffer->data()); + consumer_buffer->clear(); + + if (flushing) {exit_flag = true;} // this was the final run + if (is_stop_issued_) {flushing = true;} // run one final time to flush + } +} + +} // namespace cache +} // namespace rosbag2_cpp diff --git a/rosbag2_cpp/src/rosbag2_cpp/cache/message_cache.cpp b/rosbag2_cpp/src/rosbag2_cpp/cache/message_cache.cpp new file mode 100644 index 0000000000..44857644b5 --- /dev/null +++ b/rosbag2_cpp/src/rosbag2_cpp/cache/message_cache.cpp @@ -0,0 +1,139 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include + +#include "rosbag2_cpp/cache/message_cache.hpp" +#include "rosbag2_cpp/logging.hpp" + +namespace rosbag2_cpp +{ +namespace cache +{ + +MessageCache::MessageCache(uint64_t max_buffer_size) +{ + primary_buffer_ = std::make_shared(max_buffer_size); + secondary_buffer_ = std::make_shared(max_buffer_size); +} + +void MessageCache::push(std::shared_ptr msg) +{ + // While pushing, we keep track of inserted and dropped messages as well + bool pushed = false; + { + std::lock_guard cache_lock(cache_mutex_); + if (!flushing_) { + pushed = primary_buffer_->push(msg); + } + } + if (!pushed) { + messages_dropped_per_topic_[msg->topic_name]++; + } + + notify_buffer_consumer(); +} + +void MessageCache::finalize() +{ + { + std::lock_guard cache_lock(cache_mutex_); + flushing_ = true; + } + cache_condition_var_.notify_one(); +} + +void MessageCache::notify_flushing_done() +{ + flushing_ = false; +} + +void MessageCache::notify_buffer_consumer() +{ + { + std::lock_guard cache_lock(cache_mutex_); + primary_buffer_can_be_swapped_ = true; + } + cache_condition_var_.notify_one(); +} + +void MessageCache::wait_for_buffer() +{ + std::unique_lock lock(cache_mutex_); + if (!flushing_) { + // Required condition check to protect against spurious wakeups + cache_condition_var_.wait( + lock, [this] { + return primary_buffer_can_be_swapped_ || flushing_; + }); + primary_buffer_can_be_swapped_ = false; + } + std::swap(primary_buffer_, secondary_buffer_); +} + +std::shared_ptr MessageCache::consumer_buffer() +{ + return secondary_buffer_; +} + +std::unordered_map MessageCache::messages_dropped() const +{ + return messages_dropped_per_topic_; +} + +void MessageCache::log_dropped() +{ + uint64_t total_lost = 0; + std::string log_text("Cache buffers lost messages per topic: "); + + // worse performance than sorting key vector (neglible), but cleaner + std::map messages_dropped_per_topic_sorted( + messages_dropped_per_topic_.begin(), messages_dropped_per_topic_.end()); + + std::for_each( + messages_dropped_per_topic_.begin(), + messages_dropped_per_topic_.end(), + [&total_lost, &log_text](const auto & e) { + uint32_t lost = e.second; + if (lost > 0) { + log_text += "\n\t" + e.first + ": " + std::to_string(lost); + total_lost += lost; + } + }); + + if (total_lost > 0) { + log_text += "\nTotal lost: " + std::to_string(total_lost); + ROSBAG2_CPP_LOG_WARN_STREAM(log_text); + } + + size_t remaining = primary_buffer_->size() + secondary_buffer_->size(); + if (remaining > 0) { + ROSBAG2_CPP_LOG_WARN_STREAM( + "Cache buffers were unflushed with " << remaining << " remaining messages" + ); + } +} + +MessageCache::~MessageCache() +{ + log_dropped(); +} + +} // namespace cache +} // namespace rosbag2_cpp diff --git a/rosbag2_cpp/src/rosbag2_cpp/cache/message_cache_buffer.cpp b/rosbag2_cpp/src/rosbag2_cpp/cache/message_cache_buffer.cpp new file mode 100644 index 0000000000..ace6d7fb62 --- /dev/null +++ b/rosbag2_cpp/src/rosbag2_cpp/cache/message_cache_buffer.cpp @@ -0,0 +1,63 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "rosbag2_cpp/cache/message_cache_buffer.hpp" + +namespace rosbag2_cpp +{ +namespace cache +{ + +MessageCacheBuffer::MessageCacheBuffer(const uint64_t max_cache_size) +: max_bytes_size_(max_cache_size) +{ +} + +bool MessageCacheBuffer::push(buffer_element_t msg) +{ + bool pushed = false; + if (!drop_messages_) { + buffer_bytes_size_ += msg->serialized_data->buffer_length; + buffer_.push_back(msg); + pushed = true; + } + + if (buffer_bytes_size_ >= max_bytes_size_) { + drop_messages_ = true; + } + return pushed; +} + +void MessageCacheBuffer::clear() +{ + buffer_.clear(); + buffer_bytes_size_ = 0u; + drop_messages_ = false; +} + +size_t MessageCacheBuffer::size() +{ + return buffer_.size(); +} + +const std::vector & MessageCacheBuffer::data() +{ + return buffer_; +} + +} // namespace cache +} // namespace rosbag2_cpp diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index 583e7e42e7..34b0e35192 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -19,7 +19,9 @@ #include #include #include +#include #include +#include #include "rcpputils/filesystem_helper.hpp" @@ -34,22 +36,28 @@ namespace writers namespace { -std::string format_storage_uri(const std::string & base_folder, uint64_t storage_count) -{ - // Right now `base_folder_` is always just the folder name for where to install the bagfile. - // The name of the folder needs to be queried in case - // SequentialWriter is opened with a relative path. - std::stringstream storage_file_name; - storage_file_name << rcpputils::fs::path(base_folder).filename().string() << "_" << storage_count; - - return (rcpputils::fs::path(base_folder) / storage_file_name.str()).string(); -} - std::string strip_parent_path(const std::string & relative_path) { return rcpputils::fs::path(relative_path).filename().string(); } +rosbag2_cpp::cache::CacheConsumer::consume_callback_function_t make_callback( + std::shared_ptr storage_interface, + std::unordered_map & topics_info_map, + std::mutex & topics_mutex) +{ + return [callback_interface = storage_interface, &topics_info_map, &topics_mutex]( + const std::vector> & msgs) { + callback_interface->write(msgs); + for (const auto & msg : msgs) { + // count messages as successfully written + std::lock_guard lock(topics_mutex); + if (topics_info_map.find(msg->topic_name) != topics_info_map.end()) { + topics_info_map[msg->topic_name].message_count++; + } + } + }; +} } // namespace SequentialWriter::SequentialWriter( @@ -86,9 +94,6 @@ void SequentialWriter::open( base_folder_ = storage_options.uri; storage_options_ = storage_options; - cache_.reserve(storage_options.max_cache_size); - current_cache_size_ = 0u; - if (converter_options.output_serialization_format != converter_options.input_serialization_format) { @@ -126,17 +131,33 @@ void SequentialWriter::open( throw std::runtime_error{error.str()}; } + use_cache_ = storage_options.max_cache_size > 0u; + if (use_cache_) { + message_cache_ = std::make_shared( + storage_options.max_cache_size); + cache_consumer_ = std::make_unique( + message_cache_, + make_callback( + storage_, + topics_names_to_info_, + topics_info_mutex_)); + } init_metadata(); } void SequentialWriter::reset() { + if (use_cache_) { + // destructor will flush message cache + cache_consumer_.reset(); + message_cache_.reset(); + } + if (!base_folder_.empty()) { finalize_metadata(); metadata_io_->write_metadata(base_folder_, metadata_); } - reset_cache(); storage_.reset(); // Necessary to ensure that the storage is destroyed before the factory storage_factory_.reset(); } @@ -157,10 +178,15 @@ void SequentialWriter::create_topic(const rosbag2_storage::TopicMetadata & topic rosbag2_storage::TopicInformation info{}; info.topic_metadata = topic_with_type; - const auto insert_res = topics_names_to_info_.insert( - std::make_pair(topic_with_type.name, info)); + bool insert_succeded = false; + { + std::lock_guard lock(topics_info_mutex_); + const auto insert_res = topics_names_to_info_.insert( + std::make_pair(topic_with_type.name, info)); + insert_succeded = insert_res.second; + } - if (!insert_res.second) { + if (!insert_succeded) { std::stringstream errmsg; errmsg << "Failed to insert topic \"" << topic_with_type.name << "\"!"; @@ -177,7 +203,13 @@ void SequentialWriter::remove_topic(const rosbag2_storage::TopicMetadata & topic throw std::runtime_error("Bag is not open. Call open() before removing."); } - if (topics_names_to_info_.erase(topic_with_type.name) > 0) { + bool erased = false; + { + std::lock_guard lock(topics_info_mutex_); + erased = topics_names_to_info_.erase(topic_with_type.name) > 0; + } + + if (erased) { storage_->remove_topic(topic_with_type); } else { std::stringstream errmsg; @@ -188,8 +220,26 @@ void SequentialWriter::remove_topic(const rosbag2_storage::TopicMetadata & topic } } -void SequentialWriter::split_bagfile() +std::string SequentialWriter::format_storage_uri( + const std::string & base_folder, uint64_t storage_count) +{ + // Right now `base_folder_` is always just the folder name for where to install the bagfile. + // The name of the folder needs to be queried in case + // SequentialWriter is opened with a relative path. + std::stringstream storage_file_name; + storage_file_name << rcpputils::fs::path(base_folder).filename().string() << "_" << storage_count; + + return (rcpputils::fs::path(base_folder) / storage_file_name.str()).string(); +} + +void SequentialWriter::switch_to_next_storage() { + // consumer remaining message cache + if (use_cache_) { + cache_consumer_->close(); + message_cache_->log_dropped(); + } + storage_options_.uri = format_storage_uri( base_folder_, metadata_.relative_file_paths.size()); @@ -202,12 +252,26 @@ void SequentialWriter::split_bagfile() throw std::runtime_error(errmsg.str()); } - metadata_.relative_file_paths.push_back(strip_parent_path(storage_->get_relative_file_path())); - // Re-register all topics since we rolled-over to a new bagfile. for (const auto & topic : topics_names_to_info_) { storage_->create_topic(topic.second.topic_metadata); } + + // Set new storage in buffer layer and restart consumer thread + if (use_cache_) { + cache_consumer_->change_consume_callback( + make_callback( + storage_, + topics_names_to_info_, + topics_info_mutex_)); + } +} + +void SequentialWriter::split_bagfile() +{ + switch_to_next_storage(); + + metadata_.relative_file_paths.push_back(strip_parent_path(storage_->get_relative_file_path())); } void SequentialWriter::write(std::shared_ptr message) @@ -216,9 +280,10 @@ void SequentialWriter::write(std::shared_ptrtopic_name).message_count; + topic_information = &topics_names_to_info_.at(message->topic_name); } catch (const std::out_of_range & /* oor */) { std::stringstream errmsg; errmsg << "Failed to write on topic '" << message->topic_name << @@ -241,15 +306,14 @@ void SequentialWriter::write(std::shared_ptrwrite(converted_msg); + ++topic_information->message_count; } else { - cache_.push_back(converted_msg); - current_cache_size_ += converted_msg->serialized_data->buffer_length; - if (current_cache_size_ >= storage_options_.max_cache_size) { - reset_cache(); - } + // Otherwise, use cache buffer + message_cache_->push(converted_msg); } } @@ -308,16 +372,5 @@ void SequentialWriter::finalize_metadata() } } -void SequentialWriter::reset_cache() -{ - // if cache data exists, it must flush the data into the storage - if (!cache_.empty()) { - storage_->write(cache_); - // reset cache - cache_.clear(); - cache_.reserve(storage_options_.max_cache_size); - current_cache_size_ = 0u; - } -} } // namespace writers } // namespace rosbag2_cpp diff --git a/rosbag2_cpp/test/rosbag2_cpp/mock_cache_consumer.hpp b/rosbag2_cpp/test/rosbag2_cpp/mock_cache_consumer.hpp new file mode 100644 index 0000000000..2895aebb48 --- /dev/null +++ b/rosbag2_cpp/test/rosbag2_cpp/mock_cache_consumer.hpp @@ -0,0 +1,33 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_CPP__MOCK_CACHE_CONSUMER_HPP_ +#define ROSBAG2_CPP__MOCK_CACHE_CONSUMER_HPP_ + +#include +#include + +#include "mock_message_cache.hpp" +#include "rosbag2_cpp/cache/cache_consumer.hpp" + +class MockCacheConsumer : public rosbag2_cpp::cache::CacheConsumer +{ +public: + MockCacheConsumer( + std::shared_ptr message_cache, + rosbag2_cpp::cache::CacheConsumer::consume_callback_function_t consume_callback) + : rosbag2_cpp::cache::CacheConsumer(message_cache, consume_callback) {} +}; + +#endif // ROSBAG2_CPP__MOCK_CACHE_CONSUMER_HPP_ diff --git a/rosbag2_cpp/test/rosbag2_cpp/mock_message_cache.hpp b/rosbag2_cpp/test/rosbag2_cpp/mock_message_cache.hpp new file mode 100644 index 0000000000..e32c708e6d --- /dev/null +++ b/rosbag2_cpp/test/rosbag2_cpp/mock_message_cache.hpp @@ -0,0 +1,36 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_CPP__MOCK_MESSAGE_CACHE_HPP_ +#define ROSBAG2_CPP__MOCK_MESSAGE_CACHE_HPP_ + +#include + +#include +#include +#include + +#include "rosbag2_cpp/cache/message_cache.hpp" +#include "rosbag2_storage/serialized_bag_message.hpp" + +class MockMessageCache : public rosbag2_cpp::cache::MessageCache +{ +public: + explicit MockMessageCache(uint64_t max_buffer_size) + : rosbag2_cpp::cache::MessageCache(max_buffer_size) {} + + MOCK_METHOD0(log_dropped, void()); +}; + +#endif // ROSBAG2_CPP__MOCK_MESSAGE_CACHE_HPP_ diff --git a/rosbag2_cpp/test/rosbag2_cpp/test_message_cache.cpp b/rosbag2_cpp/test/rosbag2_cpp/test_message_cache.cpp new file mode 100644 index 0000000000..4bb3a044e7 --- /dev/null +++ b/rosbag2_cpp/test/rosbag2_cpp/test_message_cache.cpp @@ -0,0 +1,152 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rosbag2_storage/ros_helper.hpp" +#include "rosbag2_storage/serialized_bag_message.hpp" + +#include "mock_cache_consumer.hpp" +#include "mock_message_cache.hpp" + +using namespace testing; // NOLINT + +namespace +{ +std::shared_ptr make_test_msg() +{ + static uint32_t counter = 0; + std::string msg_content = "Hello" + std::to_string(counter++); + auto msg_length = msg_content.length(); + auto message = std::make_shared(); + message->topic_name = "test_topic"; + message->serialized_data = rosbag2_storage::make_serialized_message( + msg_content.c_str(), msg_length); + return message; +} + +uint32_t sum_up(const std::unordered_map & map) +{ + return std::accumulate( + std::begin(map), + std::end(map), + 0, + [](const uint32_t previous, const auto & element) { + return previous + element.second; + } + ); +} +} // namespace + +class MessageCacheTest : public Test +{ +public: + MessageCacheTest() {} + + virtual ~MessageCacheTest() = default; + + const uint64_t cache_size_ {1 * 1000}; // ~1 Kb cache +}; + +TEST_F(MessageCacheTest, message_cache_writes_full_producer_buffer) { + const uint32_t message_count = 100; + uint64_t size_bytes_so_far = 0; + size_t should_be_dropped_count = 0; + size_t consumed_message_count {0}; + + auto mock_message_cache = std::make_shared>( + cache_size_); + + for (uint32_t i = 0; i < message_count; ++i) { + auto msg = make_test_msg(); + size_t serialized_data_size = msg->serialized_data->buffer_length; + mock_message_cache->push(msg); + if (cache_size_ < size_bytes_so_far) { + should_be_dropped_count++; + } + size_bytes_so_far += serialized_data_size; + } + + auto total_actually_dropped = sum_up(mock_message_cache->messages_dropped()); + EXPECT_EQ(should_be_dropped_count, total_actually_dropped); + + auto cb = [&consumed_message_count]( + const std::vector> & msgs) { + consumed_message_count += msgs.size(); + }; + + auto mock_cache_consumer = std::make_unique>( + mock_message_cache, + cb); + + using namespace std::chrono_literals; + std::this_thread::sleep_for(20ms); + + mock_cache_consumer->close(); + EXPECT_EQ(consumed_message_count, message_count - should_be_dropped_count); +} + +TEST_F(MessageCacheTest, message_cache_changing_callback) { + const uint32_t message_count = 50; + + size_t callback1_counter = 0; + size_t callback2_counter = 0; + + auto mock_message_cache = std::make_shared>( + cache_size_); + + auto mock_cache_consumer = std::make_unique>( + mock_message_cache, + [&callback1_counter]( + const std::vector> & msgs) { + callback1_counter += msgs.size(); + }); + + rosbag2_cpp::cache::CacheConsumer::consume_callback_function_t cb2 = + [&callback2_counter]( + const std::vector> & msgs) { + callback2_counter += msgs.size(); + }; + + uint32_t counter = 0; + for (uint32_t i = 0; i < message_count; ++i) { + if (counter >= message_count / 2) { + mock_cache_consumer->change_consume_callback(cb2); + } + auto msg = make_test_msg(); + mock_message_cache->push(msg); + counter++; + } + + auto total_actually_dropped = sum_up(mock_message_cache->messages_dropped()); + EXPECT_EQ(total_actually_dropped, 0u); + + mock_cache_consumer->close(); + + using namespace std::chrono_literals; + std::this_thread::sleep_for(20ms); + + size_t sum_consumed = callback1_counter + callback2_counter; + + EXPECT_EQ(sum_consumed, message_count); +} diff --git a/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp b/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp index e0a80cbd09..0ff5bfbd72 100644 --- a/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp +++ b/rosbag2_cpp/test/rosbag2_cpp/test_sequential_writer.cpp @@ -265,41 +265,6 @@ TEST_F(SequentialWriterTest, writer_splits_when_storage_bagfile_size_gt_max_bagf } } -TEST_F(SequentialWriterTest, only_write_after_cache_is_full) { - const uint64_t counter = 1000; - const uint64_t max_cache_size = 100; - std::string msg_content = "Hello"; - const auto msg_length = msg_content.length(); - EXPECT_CALL( - *storage_, - write(An> &>())). - Times(static_cast(counter * msg_length / max_cache_size)); - EXPECT_CALL( - *storage_, - write(An>())).Times(0); - - auto sequential_writer = std::make_unique( - std::move(storage_factory_), converter_factory_, std::move(metadata_io_)); - writer_ = std::make_unique(std::move(sequential_writer)); - - std::string rmw_format = "rmw_format"; - - auto message = std::make_shared(); - message->topic_name = "test_topic"; - message->serialized_data = rosbag2_storage::make_serialized_message( - msg_content.c_str(), msg_length); - - storage_options_.max_bagfile_size = 0; - storage_options_.max_cache_size = max_cache_size; - - writer_->open(storage_options_, {rmw_format, rmw_format}); - writer_->create_topic({"test_topic", "test_msgs/BasicTypes", "", ""}); - - for (auto i = 0u; i < counter; ++i) { - writer_->write(message); - } -} - TEST_F(SequentialWriterTest, do_not_use_cache_if_cache_size_is_zero) { const size_t counter = 1000; const uint64_t max_cache_size = 0; diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp index 5418323557..73ae08ca5e 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp @@ -43,10 +43,10 @@ class WriterBenchmark : public rclcpp::Node unsigned int instances_; std::string db_folder_; std::string results_file_; - std::shared_ptr writer_; std::vector producer_threads_; std::vector> producers_; std::vector> queues_; + std::shared_ptr writer_; }; #endif // ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__WRITER_BENCHMARK_HPP_ From 536b27e4c9abbef1cd213d51cb3b96c40aea9a89 Mon Sep 17 00:00:00 2001 From: Jacob Perron Date: Mon, 30 Nov 2020 14:34:44 -0800 Subject: [PATCH 48/77] Add back rosbag2_cpp::StorageOptions as deprecated (#563) The struct was removed in https://github.com/ros2/rosbag2/pull/493, but in order to avoid a hard-break for users coming from Foxy I've added it back with a deprecation warning. Signed-off-by: Jacob Perron --- .../include/rosbag2_cpp/storage_options.hpp | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp diff --git a/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp b/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp new file mode 100644 index 0000000000..2fbeec71ee --- /dev/null +++ b/rosbag2_cpp/include/rosbag2_cpp/storage_options.hpp @@ -0,0 +1,27 @@ +// Copyright 2018, Bosch Software Innovations GmbH. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_CPP__STORAGE_OPTIONS_HPP_ +#define ROSBAG2_CPP__STORAGE_OPTIONS_HPP_ + +#include "rosbag2_storage/storage_options.hpp" + +namespace rosbag2_cpp +{ + +using StorageOptions [[deprecated("use rosbag2_storage::StorageOptions instead")]] = + rosbag2_storage::StorageOptions; + +} // namespace rosbag2_cpp +#endif // ROSBAG2_CPP__STORAGE_OPTIONS_HPP_ From a6cc4397be6fd0b7518ecf7f867c30b765cecf6d Mon Sep 17 00:00:00 2001 From: Barry Xu Date: Wed, 2 Dec 2020 22:09:49 +0800 Subject: [PATCH 49/77] Update codes since rcutils_calculate_directory_size() is changed (#567) Signed-off-by: Barry Xu --- rosbag2_storage/src/rosbag2_storage/metadata_io.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rosbag2_storage/src/rosbag2_storage/metadata_io.cpp b/rosbag2_storage/src/rosbag2_storage/metadata_io.cpp index f1b73517dd..0d1ce27929 100644 --- a/rosbag2_storage/src/rosbag2_storage/metadata_io.cpp +++ b/rosbag2_storage/src/rosbag2_storage/metadata_io.cpp @@ -232,7 +232,12 @@ BagMetadata MetadataIo::read_metadata(const std::string & uri) YAML::Node yaml_file = YAML::LoadFile(get_metadata_file_name(uri)); auto metadata = yaml_file["rosbag2_bagfile_information"].as(); rcutils_allocator_t allocator = rcutils_get_default_allocator(); - metadata.bag_size = rcutils_calculate_directory_size(uri.c_str(), allocator); + if (RCUTILS_RET_OK != + rcutils_calculate_directory_size(uri.c_str(), &metadata.bag_size, allocator)) + { + throw std::runtime_error( + std::string("Exception on calculating the size of directory :") + uri); + } return metadata; } catch (const YAML::Exception & ex) { throw std::runtime_error(std::string("Exception on parsing info file: ") + ex.what()); From 3b567d5a534149dc6419c265306a2e8316c19785 Mon Sep 17 00:00:00 2001 From: Chris Lalancette Date: Wed, 2 Dec 2020 14:22:46 +0000 Subject: [PATCH 50/77] Changelog. Signed-off-by: Chris Lalancette --- ros2bag/CHANGELOG.rst | 56 ++++++++++++++++++- rosbag2/CHANGELOG.rst | 8 ++- rosbag2_compression/CHANGELOG.rst | 10 +++- .../CHANGELOG.rst | 8 ++- rosbag2_cpp/CHANGELOG.rst | 13 +++-- .../CHANGELOG.rst | 5 ++ rosbag2_py/CHANGELOG.rst | 3 + rosbag2_storage/CHANGELOG.rst | 10 +++- rosbag2_storage_default_plugins/CHANGELOG.rst | 8 ++- rosbag2_test_common/CHANGELOG.rst | 8 ++- rosbag2_tests/CHANGELOG.rst | 10 ++-- rosbag2_transport/CHANGELOG.rst | 8 ++- shared_queues_vendor/CHANGELOG.rst | 8 ++- sqlite3_vendor/CHANGELOG.rst | 8 ++- zstd_vendor/CHANGELOG.rst | 10 ++-- 15 files changed, 128 insertions(+), 45 deletions(-) diff --git a/ros2bag/CHANGELOG.rst b/ros2bag/CHANGELOG.rst index 853f2c5f9d..a443f125cc 100644 --- a/ros2bag/CHANGELOG.rst +++ b/ros2bag/CHANGELOG.rst @@ -2,9 +2,7 @@ Changelog for package ros2bag ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -23,6 +21,58 @@ Changelog for package ros2bag 0.3.3 (2020-06-23) ------------------ +======= +Forthcoming +----------- +* Sqlite storage double buffering (`#546 `_) + * Double buffers + * Circular queue and FLUSH option as define + * Minor naming and lexical fixes. + * Removed FLUSH_BUFFERS define check. + * Sqlite3 storage logging fixes. + * Sqlite3 storage circular buffer with pre allocated memory. + * Sqlite3 storage buffers moved to shared_ptrs. + * Uncrustify + * Moved double buffers to writer + * Buffer layer reset in seq compression writer in rosbag2 cpp + * Buffer layer for rosbag2 writer refactor + * Changed buffers in BufferLayer to std vectors. + * BufferLayer uncrustify + * Removed non-applicable test for writer cache. + * BufferLayer review fixes + * Rosbag metadata msgs count fixed for BufferLayer + * Condition variable for buffer layer sync. + * Fixed buffer locks + * Buffers in BufferLayer refactored, moved into new class + * Buffer layer split bags fixed. + * Storage options include fix in buffer layer header. + * Mutex around swapping buffers in buffer layer. + * Fixed cache 0 bug in buffer layer. + * Minor buffer layer refactor. + * Counting messages in writer refactored. + * Changed default cache size to 100Mb and updated parameter description + * Applied review remarks: + - significant refactoring: separation of cache classes + - applied suggested improvements + - some renaming + - reduce code duplication that would otherwise increase with cache refactor, between compression and plain writers + * Applied review comments + - cache consumer now takes a callback and is independent of storage + - namespace changes, renaming, cleaning + - counting and logging messages by topic + * linter + * Changes after review: fixing flushing, topic counts, and more + * Fix for splitting - flushing state now correctly turns off + * cache classes documentation + * simplified signature + * a couple of tests for cache + * address review: explicit constructor and doxygen styling + * Windows warnings fix + * fixed type mismatch warning on Windows + * added minor comment + Co-authored-by: Piotr Jaroszek +* Contributors: Adam Dąbrowski +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2/CHANGELOG.rst b/rosbag2/CHANGELOG.rst index ea0557322d..7b1ef3493b 100644 --- a/rosbag2/CHANGELOG.rst +++ b/rosbag2/CHANGELOG.rst @@ -2,9 +2,7 @@ Changelog for package rosbag2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -18,6 +16,10 @@ Changelog for package rosbag2 0.3.3 (2020-06-23) ------------------ +======= +Forthcoming +----------- +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_compression/CHANGELOG.rst b/rosbag2_compression/CHANGELOG.rst index 30a2251ff9..b8af54199f 100644 --- a/rosbag2_compression/CHANGELOG.rst +++ b/rosbag2_compression/CHANGELOG.rst @@ -2,9 +2,7 @@ Changelog for package rosbag2_compression ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -20,6 +18,12 @@ Changelog for package rosbag2_compression 0.3.3 (2020-06-23) ------------------ +======= +Forthcoming +----------- +* Sqlite storage double buffering (`#546 `_) +* Contributors: Adam Dąbrowski +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_converter_default_plugins/CHANGELOG.rst b/rosbag2_converter_default_plugins/CHANGELOG.rst index ccdf73b46b..4ff03be874 100644 --- a/rosbag2_converter_default_plugins/CHANGELOG.rst +++ b/rosbag2_converter_default_plugins/CHANGELOG.rst @@ -3,9 +3,7 @@ Changelog for package rosbag2_converter_default_plugins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -19,6 +17,10 @@ Changelog for package rosbag2_converter_default_plugins 0.3.3 (2020-06-23) ------------------ +======= +Forthcoming +----------- +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_cpp/CHANGELOG.rst b/rosbag2_cpp/CHANGELOG.rst index bbbc2c5cb4..4171c87a7f 100644 --- a/rosbag2_cpp/CHANGELOG.rst +++ b/rosbag2_cpp/CHANGELOG.rst @@ -3,11 +3,7 @@ Changelog for package rosbag2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- -* [foxy backport] Fix --topics flag for ros2 bag play being ignored for all bags after the first one (`#619 `_) (`#654 `_) -* Contributors: Aleksandr Rozhdestvenskii - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Mutex around writer access in recorder (`#491 `_) (`#575 `_) @@ -22,6 +18,13 @@ Changelog for package rosbag2 0.3.3 (2020-06-23) ------------------ +======= +Forthcoming +----------- +* Add back rosbag2_cpp::StorageOptions as deprecated (`#563 `_) +* Sqlite storage double buffering (`#546 `_) +* Contributors: Adam Dąbrowski, Jacob Perron +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst b/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst index fda722bf83..daa9a43925 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst @@ -2,6 +2,11 @@ Changelog for package rosbag2_performance_writer_benchmarking ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Forthcoming +----------- +* Sqlite storage double buffering (`#546 `_) +* Contributors: Adam Dąbrowski + 0.4.0 (2020-11-19) ------------------ * read yaml config file (`#497 `_) diff --git a/rosbag2_py/CHANGELOG.rst b/rosbag2_py/CHANGELOG.rst index 93e06fb611..85d579b938 100644 --- a/rosbag2_py/CHANGELOG.rst +++ b/rosbag2_py/CHANGELOG.rst @@ -2,6 +2,9 @@ Changelog for package rosbag2_py ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Forthcoming +----------- + 0.4.0 (2020-11-19) ------------------ * add storage_config_uri (`#493 `_) diff --git a/rosbag2_storage/CHANGELOG.rst b/rosbag2_storage/CHANGELOG.rst index 7f8b4b1ec0..b4db468800 100644 --- a/rosbag2_storage/CHANGELOG.rst +++ b/rosbag2_storage/CHANGELOG.rst @@ -3,9 +3,7 @@ Changelog for package rosbag2_storage ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -19,6 +17,12 @@ Changelog for package rosbag2_storage 0.3.3 (2020-06-23) ------------------ +======= +Forthcoming +----------- +* Update codes since rcutils_calculate_directory_size() is changed (`#567 `_) +* Contributors: Barry Xu +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_storage_default_plugins/CHANGELOG.rst b/rosbag2_storage_default_plugins/CHANGELOG.rst index 296ff07320..a30bd66ca6 100644 --- a/rosbag2_storage_default_plugins/CHANGELOG.rst +++ b/rosbag2_storage_default_plugins/CHANGELOG.rst @@ -3,9 +3,7 @@ Changelog for package rosbag2_storage_default_plugins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -19,6 +17,10 @@ Changelog for package rosbag2_storage_default_plugins 0.3.3 (2020-06-23) ------------------ +======= +Forthcoming +----------- +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_test_common/CHANGELOG.rst b/rosbag2_test_common/CHANGELOG.rst index aeca6e67f8..4061fd11c3 100644 --- a/rosbag2_test_common/CHANGELOG.rst +++ b/rosbag2_test_common/CHANGELOG.rst @@ -2,9 +2,7 @@ Changelog for package rosbag2_test_common ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -18,6 +16,10 @@ Changelog for package rosbag2_test_common 0.3.3 (2020-06-23) ------------------ +======= +Forthcoming +----------- +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_tests/CHANGELOG.rst b/rosbag2_tests/CHANGELOG.rst index d7fad4fcf1..c1f1e9947a 100644 --- a/rosbag2_tests/CHANGELOG.rst +++ b/rosbag2_tests/CHANGELOG.rst @@ -3,11 +3,7 @@ Changelog for package rosbag2_tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- -* [foxy backport] Fix --topics flag for ros2 bag play being ignored for all bags after the first one (`#619 `_) (`#654 `_) -* Contributors: Aleksandr Rozhdestvenskii - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -21,6 +17,10 @@ Changelog for package rosbag2_tests 0.3.3 (2020-06-23) ------------------ +======= +Forthcoming +----------- +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_transport/CHANGELOG.rst b/rosbag2_transport/CHANGELOG.rst index 12b28c5ab6..7e020fb6d3 100644 --- a/rosbag2_transport/CHANGELOG.rst +++ b/rosbag2_transport/CHANGELOG.rst @@ -3,9 +3,7 @@ Changelog for package rosbag2_transport ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -27,6 +25,10 @@ Changelog for package rosbag2_transport ------------------ * export shared_queues_vendor for modern cmake support (`#434 `_) (`#438 `_) * Contributors: Karsten Knese +======= +Forthcoming +----------- +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ diff --git a/shared_queues_vendor/CHANGELOG.rst b/shared_queues_vendor/CHANGELOG.rst index 7c4b5f4260..d27eb5bfdb 100644 --- a/shared_queues_vendor/CHANGELOG.rst +++ b/shared_queues_vendor/CHANGELOG.rst @@ -3,9 +3,7 @@ Changelog for package shared_queues_vendor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -19,6 +17,10 @@ Changelog for package shared_queues_vendor 0.3.3 (2020-06-23) ------------------ +======= +Forthcoming +----------- +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ diff --git a/sqlite3_vendor/CHANGELOG.rst b/sqlite3_vendor/CHANGELOG.rst index 8363024594..e2f752795a 100644 --- a/sqlite3_vendor/CHANGELOG.rst +++ b/sqlite3_vendor/CHANGELOG.rst @@ -3,9 +3,7 @@ Changelog for package sqlite3_vendor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -21,6 +19,10 @@ Changelog for package sqlite3_vendor ------------------ * use interface_include_directories in find_sqlite3 (`#426 `_) (`#444 `_) * Contributors: Karsten Knese +======= +Forthcoming +----------- +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ diff --git a/zstd_vendor/CHANGELOG.rst b/zstd_vendor/CHANGELOG.rst index 633e8bb51d..e4fe756c58 100644 --- a/zstd_vendor/CHANGELOG.rst +++ b/zstd_vendor/CHANGELOG.rst @@ -2,11 +2,7 @@ Changelog for package zstd_vendor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -0.3.7 (2021-02-15) ------------------- -* [foxy backport] Zstd should not install internal headers (`#631 `_) (`#653 `_) -* Contributors: Emerson Knapp - +<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Patch zstd 1.4.4 to include cmake_minimum_version bump to 2.8.12 (`#579 `_) (`#587 `_) @@ -21,6 +17,10 @@ Changelog for package zstd_vendor 0.3.3 (2020-06-23) ------------------ +======= +Forthcoming +----------- +>>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ From 11ee716778e6497db5b073b5fc1face4507c9e89 Mon Sep 17 00:00:00 2001 From: Chris Lalancette Date: Wed, 2 Dec 2020 14:23:20 +0000 Subject: [PATCH 51/77] 0.5.0 Signed-off-by: Emerson Knapp --- ros2bag/CHANGELOG.rst | 5 +++++ ros2bag/package.xml | 2 +- ros2bag/setup.py | 2 +- rosbag2/CHANGELOG.rst | 5 +++++ rosbag2/package.xml | 2 +- rosbag2_compression/CHANGELOG.rst | 5 +++++ rosbag2_compression/package.xml | 2 +- rosbag2_converter_default_plugins/CHANGELOG.rst | 5 +++++ rosbag2_converter_default_plugins/package.xml | 2 +- rosbag2_cpp/CHANGELOG.rst | 5 +++++ rosbag2_cpp/package.xml | 2 +- .../rosbag2_performance_writer_benchmarking/CHANGELOG.rst | 4 ++-- .../rosbag2_performance_writer_benchmarking/package.xml | 2 +- rosbag2_py/CHANGELOG.rst | 4 ++-- rosbag2_py/package.xml | 2 +- rosbag2_storage/CHANGELOG.rst | 5 +++++ rosbag2_storage/package.xml | 2 +- rosbag2_storage_default_plugins/CHANGELOG.rst | 5 +++++ rosbag2_storage_default_plugins/package.xml | 2 +- rosbag2_test_common/CHANGELOG.rst | 5 +++++ rosbag2_test_common/package.xml | 2 +- rosbag2_tests/CHANGELOG.rst | 5 +++++ rosbag2_tests/package.xml | 2 +- rosbag2_transport/CHANGELOG.rst | 5 +++++ rosbag2_transport/package.xml | 2 +- shared_queues_vendor/CHANGELOG.rst | 5 +++++ shared_queues_vendor/package.xml | 2 +- sqlite3_vendor/CHANGELOG.rst | 5 +++++ sqlite3_vendor/package.xml | 2 +- zstd_vendor/CHANGELOG.rst | 5 +++++ zstd_vendor/package.xml | 2 +- 31 files changed, 85 insertions(+), 20 deletions(-) diff --git a/ros2bag/CHANGELOG.rst b/ros2bag/CHANGELOG.rst index a443f125cc..c7ba0c5a05 100644 --- a/ros2bag/CHANGELOG.rst +++ b/ros2bag/CHANGELOG.rst @@ -2,6 +2,7 @@ Changelog for package ros2bag ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -24,6 +25,10 @@ Changelog for package ros2bag ======= Forthcoming ----------- +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 * Sqlite storage double buffering (`#546 `_) * Double buffers * Circular queue and FLUSH option as define diff --git a/ros2bag/package.xml b/ros2bag/package.xml index ecc7f026ee..fcf8d0f574 100644 --- a/ros2bag/package.xml +++ b/ros2bag/package.xml @@ -2,7 +2,7 @@ ros2bag - 0.4.0 + 0.5.0 Entry point for rosbag in ROS 2 diff --git a/ros2bag/setup.py b/ros2bag/setup.py index 981cb61998..f6efbec71f 100644 --- a/ros2bag/setup.py +++ b/ros2bag/setup.py @@ -5,7 +5,7 @@ setup( name=package_name, - version='0.4.0', + version='0.5.0', packages=find_packages(exclude=['test']), data_files=[ ('share/' + package_name, ['package.xml']), diff --git a/rosbag2/CHANGELOG.rst b/rosbag2/CHANGELOG.rst index 7b1ef3493b..8935870ca1 100644 --- a/rosbag2/CHANGELOG.rst +++ b/rosbag2/CHANGELOG.rst @@ -2,6 +2,7 @@ Changelog for package rosbag2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -20,6 +21,10 @@ Changelog for package rosbag2 Forthcoming ----------- >>>>>>> Changelog. +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2/package.xml b/rosbag2/package.xml index a281c7e4ed..5abb7fd263 100644 --- a/rosbag2/package.xml +++ b/rosbag2/package.xml @@ -1,7 +1,7 @@ rosbag2 - 0.4.0 + 0.5.0 Meta package for rosbag2 related packages Karsten Knese Michael Jeronimo diff --git a/rosbag2_compression/CHANGELOG.rst b/rosbag2_compression/CHANGELOG.rst index b8af54199f..d2e8b89bf7 100644 --- a/rosbag2_compression/CHANGELOG.rst +++ b/rosbag2_compression/CHANGELOG.rst @@ -2,6 +2,7 @@ Changelog for package rosbag2_compression ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -21,6 +22,10 @@ Changelog for package rosbag2_compression ======= Forthcoming ----------- +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 * Sqlite storage double buffering (`#546 `_) * Contributors: Adam Dąbrowski >>>>>>> Changelog. diff --git a/rosbag2_compression/package.xml b/rosbag2_compression/package.xml index 7e0cf7b47c..fbbccc8e54 100644 --- a/rosbag2_compression/package.xml +++ b/rosbag2_compression/package.xml @@ -2,7 +2,7 @@ rosbag2_compression - 0.4.0 + 0.5.0 Compression implementations for rosbag2 bags and messages. Karsten Knese Michael Jeronimo diff --git a/rosbag2_converter_default_plugins/CHANGELOG.rst b/rosbag2_converter_default_plugins/CHANGELOG.rst index 4ff03be874..4fe4758023 100644 --- a/rosbag2_converter_default_plugins/CHANGELOG.rst +++ b/rosbag2_converter_default_plugins/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package rosbag2_converter_default_plugins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -21,6 +22,10 @@ Changelog for package rosbag2_converter_default_plugins Forthcoming ----------- >>>>>>> Changelog. +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_converter_default_plugins/package.xml b/rosbag2_converter_default_plugins/package.xml index f37f00ace4..3a22f8f0f4 100644 --- a/rosbag2_converter_default_plugins/package.xml +++ b/rosbag2_converter_default_plugins/package.xml @@ -1,7 +1,7 @@ rosbag2_converter_default_plugins - 0.4.0 + 0.5.0 Package containing default plugins for format converters Karsten Knese Michael Jeronimo diff --git a/rosbag2_cpp/CHANGELOG.rst b/rosbag2_cpp/CHANGELOG.rst index 4171c87a7f..8d8584ae53 100644 --- a/rosbag2_cpp/CHANGELOG.rst +++ b/rosbag2_cpp/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package rosbag2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -21,6 +22,10 @@ Changelog for package rosbag2 ======= Forthcoming ----------- +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 * Add back rosbag2_cpp::StorageOptions as deprecated (`#563 `_) * Sqlite storage double buffering (`#546 `_) * Contributors: Adam Dąbrowski, Jacob Perron diff --git a/rosbag2_cpp/package.xml b/rosbag2_cpp/package.xml index 53c723be97..5c633fa9e8 100644 --- a/rosbag2_cpp/package.xml +++ b/rosbag2_cpp/package.xml @@ -1,7 +1,7 @@ rosbag2_cpp - 0.4.0 + 0.5.0 C++ ROSBag2 client library Karsten Knese Michael Jeronimo diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst b/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst index daa9a43925..8cbace3d1b 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst @@ -2,8 +2,8 @@ Changelog for package rosbag2_performance_writer_benchmarking ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Forthcoming ------------ +0.5.0 (2020-12-02) +------------------ * Sqlite storage double buffering (`#546 `_) * Contributors: Adam Dąbrowski diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml b/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml index 6232a6086b..ffd94b39a2 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml @@ -2,7 +2,7 @@ rosbag2_performance_writer_benchmarking - 0.4.0 + 0.5.0 Code to benchmark the part of rosbag2 which starts after the callback is received Karsten Knese Michael Jeronimo diff --git a/rosbag2_py/CHANGELOG.rst b/rosbag2_py/CHANGELOG.rst index 85d579b938..f4dee0e537 100644 --- a/rosbag2_py/CHANGELOG.rst +++ b/rosbag2_py/CHANGELOG.rst @@ -2,8 +2,8 @@ Changelog for package rosbag2_py ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Forthcoming ------------ +0.5.0 (2020-12-02) +------------------ 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_py/package.xml b/rosbag2_py/package.xml index 8470101724..eb130aea9d 100644 --- a/rosbag2_py/package.xml +++ b/rosbag2_py/package.xml @@ -2,7 +2,7 @@ rosbag2_py - 0.4.0 + 0.5.0 Python API for rosbag2 Karsten Knese Michael Jeronimo diff --git a/rosbag2_storage/CHANGELOG.rst b/rosbag2_storage/CHANGELOG.rst index b4db468800..8173dba7b7 100644 --- a/rosbag2_storage/CHANGELOG.rst +++ b/rosbag2_storage/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package rosbag2_storage ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -20,6 +21,10 @@ Changelog for package rosbag2_storage ======= Forthcoming ----------- +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 * Update codes since rcutils_calculate_directory_size() is changed (`#567 `_) * Contributors: Barry Xu >>>>>>> Changelog. diff --git a/rosbag2_storage/package.xml b/rosbag2_storage/package.xml index f0a5001821..227c02689d 100644 --- a/rosbag2_storage/package.xml +++ b/rosbag2_storage/package.xml @@ -1,7 +1,7 @@ rosbag2_storage - 0.4.0 + 0.5.0 ROS2 independent storage format to store serialized ROS2 messages Karsten Knese Michael Jeronimo diff --git a/rosbag2_storage_default_plugins/CHANGELOG.rst b/rosbag2_storage_default_plugins/CHANGELOG.rst index a30bd66ca6..7b636d3f11 100644 --- a/rosbag2_storage_default_plugins/CHANGELOG.rst +++ b/rosbag2_storage_default_plugins/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package rosbag2_storage_default_plugins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -21,6 +22,10 @@ Changelog for package rosbag2_storage_default_plugins Forthcoming ----------- >>>>>>> Changelog. +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_storage_default_plugins/package.xml b/rosbag2_storage_default_plugins/package.xml index eced9d0d10..d2799edaf4 100644 --- a/rosbag2_storage_default_plugins/package.xml +++ b/rosbag2_storage_default_plugins/package.xml @@ -1,7 +1,7 @@ rosbag2_storage_default_plugins - 0.4.0 + 0.5.0 ROSBag2 SQLite3 storage plugin Karsten Knese Michael Jeronimo diff --git a/rosbag2_test_common/CHANGELOG.rst b/rosbag2_test_common/CHANGELOG.rst index 4061fd11c3..bcc1da45c6 100644 --- a/rosbag2_test_common/CHANGELOG.rst +++ b/rosbag2_test_common/CHANGELOG.rst @@ -2,6 +2,7 @@ Changelog for package rosbag2_test_common ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -20,6 +21,10 @@ Changelog for package rosbag2_test_common Forthcoming ----------- >>>>>>> Changelog. +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_test_common/package.xml b/rosbag2_test_common/package.xml index a637159bd3..5d700a8f7c 100644 --- a/rosbag2_test_common/package.xml +++ b/rosbag2_test_common/package.xml @@ -2,7 +2,7 @@ rosbag2_test_common - 0.4.0 + 0.5.0 Commonly used test helper classes and fixtures for rosbag2 Karsten Knese Michael Jeronimo diff --git a/rosbag2_tests/CHANGELOG.rst b/rosbag2_tests/CHANGELOG.rst index c1f1e9947a..2315b509e2 100644 --- a/rosbag2_tests/CHANGELOG.rst +++ b/rosbag2_tests/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package rosbag2_tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -21,6 +22,10 @@ Changelog for package rosbag2_tests Forthcoming ----------- >>>>>>> Changelog. +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_tests/package.xml b/rosbag2_tests/package.xml index f0bdfa4f22..9b3d07e07c 100644 --- a/rosbag2_tests/package.xml +++ b/rosbag2_tests/package.xml @@ -2,7 +2,7 @@ rosbag2_tests - 0.4.0 + 0.5.0 Tests package for rosbag2 Karsten Knese Michael Jeronimo diff --git a/rosbag2_transport/CHANGELOG.rst b/rosbag2_transport/CHANGELOG.rst index 7e020fb6d3..87a01a01b7 100644 --- a/rosbag2_transport/CHANGELOG.rst +++ b/rosbag2_transport/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package rosbag2_transport ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -29,6 +30,10 @@ Changelog for package rosbag2_transport Forthcoming ----------- >>>>>>> Changelog. +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ diff --git a/rosbag2_transport/package.xml b/rosbag2_transport/package.xml index b484a9a2d9..932f153b20 100644 --- a/rosbag2_transport/package.xml +++ b/rosbag2_transport/package.xml @@ -2,7 +2,7 @@ rosbag2_transport - 0.4.0 + 0.5.0 Layer encapsulating ROS middleware to allow rosbag2 to be used with or without middleware Karsten Knese Michael Jeronimo diff --git a/shared_queues_vendor/CHANGELOG.rst b/shared_queues_vendor/CHANGELOG.rst index d27eb5bfdb..53754a42bf 100644 --- a/shared_queues_vendor/CHANGELOG.rst +++ b/shared_queues_vendor/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package shared_queues_vendor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -21,6 +22,10 @@ Changelog for package shared_queues_vendor Forthcoming ----------- >>>>>>> Changelog. +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ diff --git a/shared_queues_vendor/package.xml b/shared_queues_vendor/package.xml index c1bfda97fe..32bce4d326 100644 --- a/shared_queues_vendor/package.xml +++ b/shared_queues_vendor/package.xml @@ -2,7 +2,7 @@ shared_queues_vendor - 0.4.0 + 0.5.0 Vendor package for concurrent queues from moodycamel Karsten Knese Michael Jeronimo diff --git a/sqlite3_vendor/CHANGELOG.rst b/sqlite3_vendor/CHANGELOG.rst index e2f752795a..64c82fc983 100644 --- a/sqlite3_vendor/CHANGELOG.rst +++ b/sqlite3_vendor/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package sqlite3_vendor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -23,6 +24,10 @@ Changelog for package sqlite3_vendor Forthcoming ----------- >>>>>>> Changelog. +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ diff --git a/sqlite3_vendor/package.xml b/sqlite3_vendor/package.xml index ab6cd8adf5..92ecfda940 100644 --- a/sqlite3_vendor/package.xml +++ b/sqlite3_vendor/package.xml @@ -2,7 +2,7 @@ sqlite3_vendor - 0.4.0 + 0.5.0 SQLite 3 vendor package Karsten Knese Michael Jeronimo diff --git a/zstd_vendor/CHANGELOG.rst b/zstd_vendor/CHANGELOG.rst index e4fe756c58..d5689685cf 100644 --- a/zstd_vendor/CHANGELOG.rst +++ b/zstd_vendor/CHANGELOG.rst @@ -2,6 +2,7 @@ Changelog for package zstd_vendor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ @@ -21,6 +22,10 @@ Changelog for package zstd_vendor Forthcoming ----------- >>>>>>> Changelog. +======= +0.5.0 (2020-12-02) +------------------ +>>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ diff --git a/zstd_vendor/package.xml b/zstd_vendor/package.xml index 29101a6f33..23c0a18e37 100644 --- a/zstd_vendor/package.xml +++ b/zstd_vendor/package.xml @@ -2,7 +2,7 @@ zstd_vendor - 0.4.0 + 0.5.0 Zstd compression vendor package, providing a dependency for Zstd. Karsten Knese Michael Jeronimo From 2710bdb10badb212fb4193e3be4bb36e2c878c88 Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Fri, 4 Dec 2020 09:45:44 -0800 Subject: [PATCH 52/77] Stabilize test_record by reducing copies of executors and messages (#576) * Use a single executor in the wait_for utility to put less strain on the global context shutdown * Don't copy around message pointers vectors, rather observe by const reference Signed-off-by: Emerson Knapp --- .../include/rosbag2_test_common/wait_for.hpp | 4 +++- .../test/rosbag2_transport/mock_sequential_writer.hpp | 6 +++--- .../rosbag2_transport/record_integration_fixture.hpp | 9 +++++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/rosbag2_test_common/include/rosbag2_test_common/wait_for.hpp b/rosbag2_test_common/include/rosbag2_test_common/wait_for.hpp index 8a12feb2b8..608aa4837b 100644 --- a/rosbag2_test_common/include/rosbag2_test_common/wait_for.hpp +++ b/rosbag2_test_common/include/rosbag2_test_common/wait_for.hpp @@ -26,11 +26,13 @@ bool spin_and_wait_for(Timeout timeout, const Node & node, Condition condition) { using clock = std::chrono::system_clock; auto start = clock::now(); + rclcpp::executors::SingleThreadedExecutor exec; + exec.add_node(node); while (!condition()) { if ((clock::now() - start) > timeout) { return false; } - rclcpp::spin_some(node); + exec.spin_some(); } return true; } diff --git a/rosbag2_transport/test/rosbag2_transport/mock_sequential_writer.hpp b/rosbag2_transport/test/rosbag2_transport/mock_sequential_writer.hpp index 8e70caec0e..e1c9525835 100644 --- a/rosbag2_transport/test/rosbag2_transport/mock_sequential_writer.hpp +++ b/rosbag2_transport/test/rosbag2_transport/mock_sequential_writer.hpp @@ -51,17 +51,17 @@ class MockSequentialWriter : public rosbag2_cpp::writer_interfaces::BaseWriterIn messages_per_topic_[message->topic_name] += 1; } - std::vector> get_messages() + const std::vector> & get_messages() { return messages_; } - std::unordered_map messages_per_topic() + const std::unordered_map & messages_per_topic() { return messages_per_topic_; } - std::unordered_map get_topics() + const std::unordered_map & get_topics() { return topics_; } diff --git a/rosbag2_transport/test/rosbag2_transport/record_integration_fixture.hpp b/rosbag2_transport/test/rosbag2_transport/record_integration_fixture.hpp index 119c4333ce..1e794e878c 100644 --- a/rosbag2_transport/test/rosbag2_transport/record_integration_fixture.hpp +++ b/rosbag2_transport/test/rosbag2_transport/record_integration_fixture.hpp @@ -65,10 +65,15 @@ class RecordIntegrationTestFixture : public Rosbag2TransportTestFixture void run_publishers() { pub_man_.run_publishers( - [this](const std::string & topic_name) { + [this](const std::string & topic_name) -> size_t { MockSequentialWriter & writer = static_cast(writer_->get_implementation_handle()); - return writer.messages_per_topic()[topic_name]; + const auto & messages = writer.messages_per_topic(); + auto it = writer.messages_per_topic().find(topic_name); + if (it != messages.end()) { + return it->second; + } + return 0; }); } From 73144795712f907ae9ba59d96921667f14082ab0 Mon Sep 17 00:00:00 2001 From: "P. J. Reed" Date: Fri, 4 Dec 2020 17:41:32 -0600 Subject: [PATCH 53/77] Compress bag files in separate threads (#506) * Compress bag files in separate threads This offloads all compression into separate threads. Doing so helps to prevent rosbag2 from dropping messages due to the main thread being busy, and it also helps to improve performance by spreading the work across multiple CPU cores. It uses a producer/consumer model with a fixed number of pre-allocated threads that consume messages or files off of an incoming queue. This adds two new command line options: - --compression-queue-size - The number of messages or files that can be sitting in the queue waiting for a thread to consume them. If the queue is full, older messages or files will be discarded; this would lead to either messages being discarded or files not being compressed. The default value is 1, which should be sufficient if the system is capable of keeping up with the work load. - --compression-threads - The number of threads that can be compressing data at once. The default is 0, and values less than 1 will be interpreted to mean the number of concurrent threads supported by the current hardware. Closes #274. Distribution Statement A; OPSEC #2893 Signed-off-by: P. J. Reed Co-authored-by: Emerson Knapp --- ros2bag/ros2bag/verb/record.py | 15 ++ .../compression_options.hpp | 2 + .../sequential_compression_writer.hpp | 86 ++++++-- .../rosbag2_compression/zstd_compressor.hpp | 7 +- .../rosbag2_compression/zstd_decompressor.hpp | 7 +- .../sequential_compression_writer.cpp | 198 +++++++++++++++--- .../rosbag2_compression/zstd_compressor.cpp | 32 ++- .../rosbag2_compression/zstd_decompressor.cpp | 21 +- .../test_sequential_compression_writer.cpp | 24 ++- .../CMakeLists.txt | 12 +- .../writer_benchmark.hpp | 6 + .../package.xml | 2 + .../scripts/benchmark.sh | 4 + .../src/writer_benchmark.cpp | 40 +++- .../rosbag2_transport/record_options.hpp | 2 + .../rosbag2_transport_python.cpp | 18 +- 16 files changed, 408 insertions(+), 68 deletions(-) diff --git a/ros2bag/ros2bag/verb/record.py b/ros2bag/ros2bag/verb/record.py index b1223a7d6d..ffd978fc22 100644 --- a/ros2bag/ros2bag/verb/record.py +++ b/ros2bag/ros2bag/verb/record.py @@ -84,6 +84,16 @@ def add_arguments(self, parser, cli_name): # noqa: D102 '--compression-format', type=str, default='', choices=['zstd'], help='Specify the compression format/algorithm. Default is none.' ) + parser.add_argument( + '--compression-queue-size', type=int, default=1, + help='Number of files or messages that may be queued for compression ' + 'before being dropped. Default is 1.' + ) + parser.add_argument( + '--compression-threads', type=int, default=0, + help='Number of files or messages that may be compressed in parallel. ' + 'Default is 0, which will be interpreted as the number of CPU cores.' + ) parser.add_argument( '--include-hidden-topics', action='store_true', help='record also hidden topics.' @@ -118,6 +128,9 @@ def main(self, *, args): # noqa: D102 return print_error('Invalid choice: Cannot specify compression format ' 'without a compression mode.') + if args.compression_queue_size < 1: + return print_error('Compression queue size must be at least 1.') + args.compression_mode = args.compression_mode.upper() qos_profile_overrides = {} # Specify a valid default @@ -147,6 +160,8 @@ def main(self, *, args): # noqa: D102 node_prefix=NODE_NAME_PREFIX, compression_mode=args.compression_mode, compression_format=args.compression_format, + compression_queue_size=args.compression_queue_size, + compression_threads=args.compression_threads, all=args.all, no_discovery=args.no_discovery, polling_interval=args.polling_interval, diff --git a/rosbag2_compression/include/rosbag2_compression/compression_options.hpp b/rosbag2_compression/include/rosbag2_compression/compression_options.hpp index 368b9fe0a4..37fe5007e3 100644 --- a/rosbag2_compression/include/rosbag2_compression/compression_options.hpp +++ b/rosbag2_compression/include/rosbag2_compression/compression_options.hpp @@ -58,6 +58,8 @@ struct CompressionOptions { std::string compression_format; CompressionMode compression_mode; + uint64_t compression_queue_size; + uint64_t compression_threads; }; } // namespace rosbag2_compression diff --git a/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp b/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp index 2bde7076fe..d970457434 100644 --- a/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp +++ b/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp @@ -15,11 +15,18 @@ #ifndef ROSBAG2_COMPRESSION__SEQUENTIAL_COMPRESSION_WRITER_HPP_ #define ROSBAG2_COMPRESSION__SEQUENTIAL_COMPRESSION_WRITER_HPP_ +#include +#include #include +#include +#include #include +#include #include #include +#include "rcpputils/thread_safety_annotations.hpp" + #include "rosbag2_cpp/converter.hpp" #include "rosbag2_cpp/converter_options.hpp" #include "rosbag2_cpp/serialization_format_converter_factory.hpp" @@ -64,6 +71,37 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionWriter ~SequentialCompressionWriter() override; + /** + * Create a new topic in the underlying storage. Needs to be called for every topic used within + * a message which is passed to write(...). + * + * \param topic_with_type name and type identifier of topic to be created + * \throws runtime_error if the Writer is not open. + */ + void create_topic(const rosbag2_storage::TopicMetadata & topic_with_type) override; + + /** + * Remove a new topic in the underlying storage. + * If creation of subscription fails remove the topic + * from the db (more of cleanup) + * + * \param topic_with_type name and type identifier of topic to be created + * \throws runtime_error if the Writer is not open. + */ + void remove_topic(const rosbag2_storage::TopicMetadata & topic_with_type) override; + + /** + * If the compression mode is FILE, write a message to a bagfile. + * If the compression mode is MESSAGE, pushes the message into a queue that will be processed + * by the compression threads. + * + * The topic needs to have been created before writing is possible. + * + * \param message to be written to the bagfile + * \throws runtime_error if the Writer is not open. + */ + void write(std::shared_ptr message) override; + /** * Opens a new bagfile and prepare it for writing messages. The bagfile must not exist. * This must be called before any other function is used. @@ -83,19 +121,24 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionWriter protected: /** - * Compress the most recent file and update the metadata file path. + * Compress a file and update the metadata file path. + * + * \param compressor An initialized compression context. + * \param message The URI of the file to compress. */ - virtual void compress_last_file(); + virtual void compress_file(BaseCompressorInterface & compressor, const std::string & file); /** * Checks if the compression by message option is specified and a compressor exists. * * If the above conditions are satisfied, compresses the serialized bag message. * + * \param compressor An initialized compression context. * \param message The message to compress. - * \return True if compression occurred, false otherwise. */ - virtual void compress_message(std::shared_ptr message); + virtual void compress_message( + BaseCompressorInterface & compressor, + std::shared_ptr message); /** * Initializes the compressor if a compression mode is specified. @@ -104,28 +147,45 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionWriter */ virtual void setup_compression(); + /** + * Initializes a number of threads to do file or message compression equal to the + * value of the compression_threads parameter. + */ + virtual void setup_compressor_threads(); + + /** + * Signals all compressor threads to stop working and then waits for them to exit. + */ + virtual void stop_compressor_threads(); + private: std::unique_ptr compressor_{}; std::unique_ptr compression_factory_{}; + std::mutex compressor_queue_mutex_; + std::queue> + compressor_message_queue_ RCPPUTILS_TSA_GUARDED_BY(compressor_queue_mutex_); + std::queue compressor_file_queue_ RCPPUTILS_TSA_GUARDED_BY(compressor_queue_mutex_); + std::vector compression_threads_; + std::atomic_bool compression_is_running_{false}; + std::recursive_mutex storage_mutex_; + std::condition_variable compressor_condition_; rosbag2_compression::CompressionOptions compression_options_{}; bool should_compress_last_file_{true}; + // Runs a while loop that pulls data from the compression queue until + // compression_is_running_ is false; should be run in a separate thread + void compression_thread_fn(); + // Closes the current backed storage and opens the next bagfile. void split_bagfile() override; + // Checks if the current recording bagfile needs to be split and rolled over to a new file. + bool should_split_bagfile(); + // Prepares the metadata by setting initial values. void init_metadata() override; - - // Helper method used by write to get the message in a format that is ready to be written. - // Common use cases include converting the message using the converter or - // performing other operations like compression on it - std::shared_ptr - get_writeable_message( - std::shared_ptr message) override; }; - } // namespace rosbag2_compression - #endif // ROSBAG2_COMPRESSION__SEQUENTIAL_COMPRESSION_WRITER_HPP_ diff --git a/rosbag2_compression/include/rosbag2_compression/zstd_compressor.hpp b/rosbag2_compression/include/rosbag2_compression/zstd_compressor.hpp index bdde434d49..0b7a8c5d42 100644 --- a/rosbag2_compression/include/rosbag2_compression/zstd_compressor.hpp +++ b/rosbag2_compression/include/rosbag2_compression/zstd_compressor.hpp @@ -37,9 +37,9 @@ namespace rosbag2_compression class ROSBAG2_COMPRESSION_PUBLIC ZstdCompressor : public BaseCompressorInterface { public: - ZstdCompressor() = default; + ZstdCompressor(); - ~ZstdCompressor() = default; + ~ZstdCompressor() override; std::string compress_uri(const std::string & uri) override; @@ -47,6 +47,9 @@ class ROSBAG2_COMPRESSION_PUBLIC ZstdCompressor : public BaseCompressorInterface rosbag2_storage::SerializedBagMessage * bag_message) override; std::string get_compression_identifier() const override; + +private: + ZSTD_CCtx * zstd_context_; }; } // namespace rosbag2_compression diff --git a/rosbag2_compression/include/rosbag2_compression/zstd_decompressor.hpp b/rosbag2_compression/include/rosbag2_compression/zstd_decompressor.hpp index 9291971787..30855b5ec9 100644 --- a/rosbag2_compression/include/rosbag2_compression/zstd_decompressor.hpp +++ b/rosbag2_compression/include/rosbag2_compression/zstd_decompressor.hpp @@ -37,9 +37,9 @@ namespace rosbag2_compression class ROSBAG2_COMPRESSION_PUBLIC ZstdDecompressor : public BaseDecompressorInterface { public: - ZstdDecompressor() = default; + ZstdDecompressor(); - ~ZstdDecompressor() = default; + ~ZstdDecompressor() override; std::string decompress_uri(const std::string & uri) override; @@ -47,6 +47,9 @@ class ROSBAG2_COMPRESSION_PUBLIC ZstdDecompressor : public BaseDecompressorInter rosbag2_storage::SerializedBagMessage * bag_message) override; std::string get_decompression_identifier() const override; + +private: + ZSTD_DCtx * zstd_context_; }; } // namespace rosbag2_compression diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp index 5c8d4cdc35..f8b09ecd6d 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -60,8 +61,51 @@ SequentialCompressionWriter::~SequentialCompressionWriter() reset(); } +void SequentialCompressionWriter::compression_thread_fn() +{ + // Every thread needs to have its own compression context for thread safety. + auto compressor = compression_factory_->create_compressor( + compression_options_.compression_format); + + + if (!compressor) { + throw std::runtime_error{ + "Cannot compress message; Writer is not open!"}; + } + + while (compression_is_running_) { + std::shared_ptr message; + std::string file; + { + std::unique_lock lock(compressor_queue_mutex_); + compressor_condition_.wait(lock); + if (!compressor_message_queue_.empty()) { + message = compressor_message_queue_.front(); + compressor_message_queue_.pop(); + } else if (!compressor_file_queue_.empty()) { + file = compressor_file_queue_.front(); + compressor_file_queue_.pop(); + } + } + + if (message) { + compress_message(*compressor, message); + + { + // Now that the message is compressed, it can be written to file using the + // normal method. + std::lock_guard storage_lock(storage_mutex_); + SequentialWriter::write(message); + } + } else if (!file.empty()) { + compress_file(*compressor, file); + } + } +} + void SequentialCompressionWriter::init_metadata() { + std::lock_guard lock(storage_mutex_); metadata_ = rosbag2_storage::BagMetadata{}; metadata_.storage_identifier = storage_->get_storage_identifier(); metadata_.starting_time = std::chrono::time_point{ @@ -78,33 +122,85 @@ void SequentialCompressionWriter::setup_compression() throw std::invalid_argument{ "SequentialCompressionWriter requires a CompressionMode that is not NONE!"}; } - compressor_ = compression_factory_->create_compressor(compression_options_.compression_format); + + setup_compressor_threads(); +} + +void SequentialCompressionWriter::setup_compressor_threads() +{ + if (compression_options_.compression_threads < 1) { + // This should have already been set to something reasonable prior to this + // point, but we'll double-check just to make sure. + auto hardware_threads = std::thread::hardware_concurrency(); + compression_options_.compression_threads = hardware_threads > 0 ? hardware_threads : 1; + } + ROSBAG2_COMPRESSION_LOG_DEBUG_STREAM( + "setup_compressor_threads: Starting " << + compression_options_.compression_threads << " threads"); + compression_is_running_ = true; + + // This function needs to throw an exception if the compression format is invalid, but because + // each thread creates its own compressor, we can't actually catch it here if one of the threads + // fails. Instead, we'll create a compressor that we don't actually use just so that it will + // throw an exception if the format is invalid. + auto compressor = compression_factory_->create_compressor( + compression_options_.compression_format); + if (!compressor) { + throw std::runtime_error{ + "Cannot compress message; Writer is not open!"}; + } + + for (uint64_t i = 0; i < compression_options_.compression_threads; i++) { + compression_threads_.emplace_back([&] {compression_thread_fn();}); + } +} + +void SequentialCompressionWriter::stop_compressor_threads() +{ + if (!compression_threads_.empty()) { + ROSBAG2_COMPRESSION_LOG_DEBUG("Waiting for compressor threads to finish."); + compression_is_running_ = false; + compressor_condition_.notify_all(); + for (auto & thread : compression_threads_) { + thread.join(); + } + compression_threads_.clear(); + } } void SequentialCompressionWriter::open( const rosbag2_storage::StorageOptions & storage_options, const rosbag2_cpp::ConverterOptions & converter_options) { + std::lock_guard lock(storage_mutex_); SequentialWriter::open(storage_options, converter_options); setup_compression(); } - void SequentialCompressionWriter::reset() { - if (!base_folder_.empty() && compressor_) { + if (!base_folder_.empty()) { // Reset may be called before initializing the compressor (ex. bad options). // We compress the last file only if it hasn't been compressed earlier (ex. in split_bagfile()). if (compression_options_.compression_mode == rosbag2_compression::CompressionMode::FILE && should_compress_last_file_) { + std::lock_guard lock(storage_mutex_); + std::lock_guard compressor_lock(compressor_queue_mutex_); try { storage_.reset(); // Storage must be closed before it can be compressed. - compress_last_file(); + if (!metadata_.relative_file_paths.empty()) { + std::string file = metadata_.relative_file_paths.back(); + compressor_file_queue_.push(file); + compressor_condition_.notify_one(); + } } catch (const std::runtime_error & e) { ROSBAG2_COMPRESSION_LOG_WARN_STREAM("Could not compress the last bag file.\n" << e.what()); } } + + stop_compressor_threads(); + finalize_metadata(); metadata_io_->write_metadata(base_folder_, metadata_); } @@ -117,18 +213,46 @@ void SequentialCompressionWriter::reset() storage_factory_.reset(); } -void SequentialCompressionWriter::compress_last_file() +void SequentialCompressionWriter::create_topic( + const rosbag2_storage::TopicMetadata & topic_with_type) { - if (!compressor_) { - throw std::runtime_error{"compress_last_file: Compressor was not opened!"}; - } + std::lock_guard lock(storage_mutex_); + SequentialWriter::create_topic(topic_with_type); +} - const auto to_compress = rcpputils::fs::path{metadata_.relative_file_paths.back()}; +void SequentialCompressionWriter::remove_topic( + const rosbag2_storage::TopicMetadata & topic_with_type) +{ + std::lock_guard lock(storage_mutex_); + SequentialWriter::remove_topic(topic_with_type); +} - if (to_compress.exists() && to_compress.file_size() > 0u) { - const auto compressed_uri = compressor_->compress_uri(to_compress.string()); +void SequentialCompressionWriter::compress_file( + BaseCompressorInterface & compressor, + const std::string & file) +{ + ROSBAG2_COMPRESSION_LOG_DEBUG("Compressing file: %s", file.c_str()); + const auto to_compress = rcpputils::fs::path{file}; - metadata_.relative_file_paths.back() = compressed_uri; + if (to_compress.exists() && to_compress.file_size() > 0u) { + const auto compressed_uri = compressor.compress_uri(to_compress.string()); + { + // After we've compressed the file, replace the name in the file list with the new name. + // Must search for the entry because other threads may have changed the order of the vector + // and invalidated any index or iterator we held to it. + std::lock_guard lock(storage_mutex_); + auto iter = std::find( + metadata_.relative_file_paths.begin(), + metadata_.relative_file_paths.end(), + file); + if (iter != metadata_.relative_file_paths.end()) { + *iter = compressed_uri; + } else { + ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( + "Failed to find path to uncompressed bag: \"" << file << + "\"; this shouldn't happen."); + } + } if (!rcpputils::fs::remove(to_compress)) { ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( @@ -138,17 +262,22 @@ void SequentialCompressionWriter::compress_last_file() ROSBAG2_COMPRESSION_LOG_DEBUG_STREAM( "Removing last file: \"" << to_compress.string() << "\" because it either is empty or does not exist."); - - metadata_.relative_file_paths.pop_back(); } } void SequentialCompressionWriter::split_bagfile() { + std::lock_guard lock(storage_mutex_); + std::lock_guard compressor_lock(compressor_queue_mutex_); + switch_to_next_storage(); + // If we're in FILE compression mode, push this file's name on to the queue so another + // thread will handle compressing it. If not, we can just carry on. if (compression_options_.compression_mode == rosbag2_compression::CompressionMode::FILE) { - compress_last_file(); + std::string file = metadata_.relative_file_paths.back(); + compressor_file_queue_.push(file); + compressor_condition_.notify_one(); } if (!storage_) { @@ -161,24 +290,41 @@ void SequentialCompressionWriter::split_bagfile() } void SequentialCompressionWriter::compress_message( + BaseCompressorInterface & compressor, std::shared_ptr message) { - if (!compressor_) { - throw std::runtime_error{"Cannot compress message; Writer is not open!"}; - } - - compressor_->compress_serialized_bag_message(message.get()); + compressor.compress_serialized_bag_message(message.get()); } -std::shared_ptr -SequentialCompressionWriter::get_writeable_message( +void SequentialCompressionWriter::write( std::shared_ptr message) { - auto writeable_msg = SequentialWriter::get_writeable_message(message); - if (compression_options_.compression_mode == rosbag2_compression::CompressionMode::MESSAGE) { - compress_message(writeable_msg); + // If the compression mode is FILE, write as normal here. Compressing files doesn't + // occur until after the bag file is split. + // If the compression mode is MESSAGE, push the message into a queue that will be handled + // by the compression threads. + if (compression_options_.compression_mode == CompressionMode::FILE) { + SequentialWriter::write(message); + } else { + std::lock_guard lock(compressor_queue_mutex_); + while (compressor_message_queue_.size() > compression_options_.compression_queue_size) { + compressor_message_queue_.pop(); + } + compressor_message_queue_.push(message); + compressor_condition_.notify_one(); + } +} + +bool SequentialCompressionWriter::should_split_bagfile() +{ + if (storage_options_.max_bagfile_size == + rosbag2_storage::storage_interfaces::MAX_BAGFILE_SIZE_NO_SPLIT) + { + return false; + } else { + std::lock_guard lock(storage_mutex_); + return SequentialWriter::should_split_bagfile(); } - return writeable_msg; } } // namespace rosbag2_compression diff --git a/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp b/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp index 32a509cfa5..e1f414b6ed 100644 --- a/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp +++ b/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp @@ -26,6 +26,24 @@ namespace rosbag2_compression { +ZstdCompressor::ZstdCompressor() +{ + // From the zstd manual: https://facebook.github.io/zstd/zstd_manual.html#Chapter4 + // When compressing many times, + // it is recommended to allocate a context just once, + // and re-use it for each successive compression operation. + // This will make workload friendlier for system's memory. + // Note : re-using context is just a speed / resource optimization. + // It doesn't change the compression ratio, which remains identical. + // Note 2 : In multi-threaded environments, + // use one different context per thread for parallel execution. + zstd_context_ = ZSTD_createCCtx(); +} + +ZstdCompressor::~ZstdCompressor() +{ + ZSTD_freeCCtx(zstd_context_); +} std::string ZstdCompressor::compress_uri(const std::string & uri) { @@ -39,7 +57,8 @@ std::string ZstdCompressor::compress_uri(const std::string & uri) // Perform compression and check. // compression_result is either the actual compressed size or an error code. - const auto compression_result = ZSTD_compress( + const auto compression_result = ZSTD_compressCCtx( + zstd_context_, compressed_buffer.data(), compressed_buffer.size(), decompressed_buffer.data(), decompressed_buffer.size(), kDefaultZstdCompressionLevel); throw_on_zstd_error(compression_result); @@ -59,14 +78,15 @@ void ZstdCompressor::compress_serialized_bag_message( { const auto start = std::chrono::high_resolution_clock::now(); // Allocate based on compression bound and compress - const auto uncompressed_buffer_length = + const auto maximum_compressed_length = ZSTD_compressBound(message->serialized_data->buffer_length); - std::vector compressed_buffer(uncompressed_buffer_length); + std::vector compressed_buffer(maximum_compressed_length); // Perform compression and check. // compression_result is either the actual compressed size or an error code. - const auto compression_result = ZSTD_compress( - compressed_buffer.data(), compressed_buffer.size(), + const auto compression_result = ZSTD_compressCCtx( + zstd_context_, + compressed_buffer.data(), maximum_compressed_length, message->serialized_data->buffer, message->serialized_data->buffer_length, kDefaultZstdCompressionLevel); throw_on_zstd_error(compression_result); @@ -85,7 +105,7 @@ void ZstdCompressor::compress_serialized_bag_message( std::copy(compressed_buffer.begin(), compressed_buffer.end(), message->serialized_data->buffer); const auto end = std::chrono::high_resolution_clock::now(); - print_compression_statistics(start, end, uncompressed_buffer_length, compression_result); + print_compression_statistics(start, end, maximum_compressed_length, compression_result); } std::string ZstdCompressor::get_compression_identifier() const diff --git a/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp b/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp index 5660d7c5bb..5e14ceb671 100644 --- a/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp +++ b/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp @@ -26,6 +26,21 @@ namespace rosbag2_compression { +ZstdDecompressor::ZstdDecompressor() +{ + // From the zstd manual: https://facebook.github.io/zstd/zstd_manual.html#Chapter4 + // When decompressing many times, + // it is recommended to allocate a context only once, + // and re-use it for each successive compression operation. + // This will make workload friendlier for system's memory. + // Use one context per thread for parallel execution. + zstd_context_ = ZSTD_createDCtx(); +} + +ZstdDecompressor::~ZstdDecompressor() +{ + ZSTD_freeDCtx(zstd_context_); +} std::string ZstdDecompressor::decompress_uri(const std::string & uri) { @@ -45,7 +60,8 @@ std::string ZstdDecompressor::decompress_uri(const std::string & uri) // the initializer list constructor instead. std::vector decompressed_buffer(decompressed_buffer_length); - const auto decompression_result = ZSTD_decompress( + const auto decompression_result = ZSTD_decompressDCtx( + zstd_context_, decompressed_buffer.data(), decompressed_buffer_length, compressed_buffer.data(), compressed_buffer_length); @@ -76,7 +92,8 @@ void ZstdDecompressor::decompress_serialized_bag_message( // the initializer list constructor instead. std::vector decompressed_buffer(decompressed_buffer_length); - const auto decompression_result = ZSTD_decompress( + const auto decompression_result = ZSTD_decompressDCtx( + zstd_context_, decompressed_buffer.data(), decompressed_buffer_length, message->serialized_data->buffer, compressed_buffer_length); diff --git a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp index 74ba726ef2..60a14bbb2e 100644 --- a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp +++ b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp @@ -63,12 +63,16 @@ class SequentialCompressionWriterTest : public Test rcpputils::fs::path tmp_dir_; rosbag2_storage::StorageOptions tmp_dir_storage_options_; std::string serialization_format_; + + const uint64_t kDefaultCompressionQueueSize = 1; + const uint64_t kDefaultCompressionQueueThreads = 4; }; TEST_F(SequentialCompressionWriterTest, open_throws_on_empty_storage_options_uri) { rosbag2_compression::CompressionOptions compression_options{ - "zstd", rosbag2_compression::CompressionMode::FILE}; + "zstd", rosbag2_compression::CompressionMode::FILE, + kDefaultCompressionQueueSize, kDefaultCompressionQueueThreads}; auto compression_factory = std::make_unique(); auto sequential_writer = std::make_unique( @@ -89,7 +93,8 @@ TEST_F(SequentialCompressionWriterTest, open_throws_on_empty_storage_options_uri TEST_F(SequentialCompressionWriterTest, open_throws_on_bad_compression_format) { rosbag2_compression::CompressionOptions compression_options{ - "bad_format", rosbag2_compression::CompressionMode::FILE}; + "bad_format", rosbag2_compression::CompressionMode::FILE, + kDefaultCompressionQueueSize, kDefaultCompressionQueueThreads}; auto compression_factory = std::make_unique(); auto sequential_writer = std::make_unique( @@ -110,7 +115,8 @@ TEST_F(SequentialCompressionWriterTest, open_throws_on_bad_compression_format) TEST_F(SequentialCompressionWriterTest, open_throws_on_invalid_splitting_size) { rosbag2_compression::CompressionOptions compression_options{ - "zstd", rosbag2_compression::CompressionMode::FILE}; + "zstd", rosbag2_compression::CompressionMode::FILE, + kDefaultCompressionQueueSize, kDefaultCompressionQueueThreads}; auto compression_factory = std::make_unique(); // Set minimum file size greater than max bagfile size option @@ -137,7 +143,8 @@ TEST_F(SequentialCompressionWriterTest, open_throws_on_invalid_splitting_size) TEST_F(SequentialCompressionWriterTest, open_succeeds_on_supported_compression_format) { rosbag2_compression::CompressionOptions compression_options{ - "zstd", rosbag2_compression::CompressionMode::FILE}; + "zstd", rosbag2_compression::CompressionMode::FILE, + kDefaultCompressionQueueSize, kDefaultCompressionQueueThreads}; auto compression_factory = std::make_unique(); auto sequential_writer = std::make_unique( @@ -161,7 +168,8 @@ TEST_F(SequentialCompressionWriterTest, open_succeeds_on_supported_compression_f TEST_F(SequentialCompressionWriterTest, writer_calls_create_compressor) { rosbag2_compression::CompressionOptions compression_options{ - "zstd", rosbag2_compression::CompressionMode::FILE}; + "zstd", rosbag2_compression::CompressionMode::FILE, + kDefaultCompressionQueueSize, kDefaultCompressionQueueThreads}; auto compression_factory = std::make_unique>(); EXPECT_CALL(*compression_factory, create_compressor(_)).Times(1); @@ -173,7 +181,11 @@ TEST_F(SequentialCompressionWriterTest, writer_calls_create_compressor) std::move(metadata_io_)); writer_ = std::make_unique(std::move(sequential_writer)); - writer_->open(tmp_dir_storage_options_, {serialization_format_, serialization_format_}); + // This will throw an exception because the MockCompressionFactory does not actually create + // a compressor. + EXPECT_THROW( + writer_->open(tmp_dir_storage_options_, {serialization_format_, serialization_format_}), + std::runtime_error); EXPECT_TRUE(rcpputils::fs::remove(tmp_dir_)); } diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt b/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt index 1f831ae4dc..4f353fdefc 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt @@ -15,22 +15,24 @@ find_package(ament_cmake REQUIRED) if(BUILD_ROSBAG2_BENCHMARKS) find_package(rclcpp REQUIRED) find_package(rcutils REQUIRED) + find_package(rosbag2_compression REQUIRED) find_package(rosbag2_cpp REQUIRED) find_package(rosbag2_storage REQUIRED) find_package(rmw REQUIRED) find_package(std_msgs REQUIRED) add_executable(writer_benchmark src/writer_benchmark.cpp src/main.cpp) - target_include_directories(writer_benchmark - PUBLIC - $ - $ - ) ament_target_dependencies(writer_benchmark rclcpp std_msgs + rosbag2_compression rosbag2_cpp ) + target_include_directories(writer_benchmark + PUBLIC + $ + $ + ) install(TARGETS writer_benchmark DESTINATION lib/${PROJECT_NAME}) diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp index 73ae08ca5e..7228c7301a 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp @@ -21,6 +21,7 @@ #include #include "rclcpp/rclcpp.hpp" +#include "rosbag2_compression/compression_options.hpp" #include "rosbag2_cpp/writers/sequential_writer.hpp" #include "rosbag2_performance_writer_benchmarking/message_queue.hpp" @@ -46,6 +47,11 @@ class WriterBenchmark : public rclcpp::Node std::vector producer_threads_; std::vector> producers_; std::vector> queues_; + + std::string compression_format_; + rosbag2_compression::CompressionMode compression_mode_; + uint64_t compression_queue_size_; + uint64_t compression_threads_; std::shared_ptr writer_; }; diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml b/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml index ffd94b39a2..957732e9a2 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml @@ -12,7 +12,9 @@ ament_cmake rclcpp + rosbag2_compression rosbag2_cpp + rosbag2_storage rmw std_msgs diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh index a149ad3a6b..293f164f61 100755 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh @@ -17,6 +17,9 @@ summary_file=${test_dir}/results.csv freq=100; #Hz +# Set this to "zstd" to test with compression +compression_format="" + for cache in 0 1000000 10000000 100000000 do for sz in 1000 10000 100000 1000000 @@ -36,6 +39,7 @@ do outfile=${outdir}/${try}.log echo "Results will be written to file: ${outfile}" ros2 run rosbag2_performance_writer_benchmarking writer_benchmark --ros-args \ + -p compression_format:=${compression_format} \ -p frequency:=${freq} \ -p size:=${sz} \ -p instances:=${inst} \ diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp index 5dccca0953..a743fdcc0e 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp @@ -18,6 +18,8 @@ #include #include "rmw/rmw.h" +#include "rosbag2_compression/sequential_compression_writer.hpp" +#include "rosbag2_cpp/storage_options.hpp" #include "rosbag2_storage/serialized_bag_message.hpp" #include "rosbag2_storage/storage_options.hpp" #include "std_msgs/msg/byte_multi_array.hpp" @@ -26,6 +28,8 @@ using namespace std::chrono_literals; +static rcutils_allocator_t allocator = rcutils_get_default_allocator(); + WriterBenchmark::WriterBenchmark(const std::string & name) : rclcpp::Node(name) { @@ -37,6 +41,9 @@ WriterBenchmark::WriterBenchmark(const std::string & name) this->declare_parameter("max_cache_size", 1); this->declare_parameter("db_folder", default_bag_folder); this->declare_parameter("results_file", default_bag_folder + "/results.csv"); + this->declare_parameter("compression_format", ""); + this->declare_parameter("compression_queue_size", 1); + this->declare_parameter("compression_threads", 0); this->get_parameter("frequency", config_.frequency); if (config_.frequency == 0) { @@ -51,6 +58,9 @@ WriterBenchmark::WriterBenchmark(const std::string & name) this->get_parameter("max_count", config_.max_count); this->get_parameter("size", config_.message_size); this->get_parameter("instances", instances_); + this->get_parameter("compression_format", compression_format_); + this->get_parameter("compression_queue_size", compression_queue_size_); + this->get_parameter("compression_threads", compression_threads_); create_producers(config_); create_writer(); @@ -72,20 +82,32 @@ void WriterBenchmark::start_benchmark() if (!queue->is_empty()) { // behave as if we received the message. auto message = std::make_shared(); - auto serialized_data = std::make_shared(); // The pointer memory is owned by the producer until past the termination of the while loop. // Note this ownership model should be changed if we want to generate messages on the fly auto byte_ma_message = queue->pop_and_return(); - serialized_data->buffer = reinterpret_cast(byte_ma_message->data.data()); + // The compressor may resize this array, so it needs to be initialized with + // rcutils_uint8_array_init to ensure the allocator is set properly. + auto msg_array = new rcutils_uint8_array_t; + *msg_array = rcutils_get_zero_initialized_uint8_array(); + int error = rcutils_uint8_array_init(msg_array, byte_ma_message->data.size(), &allocator); + if (error != RCUTILS_RET_OK) { + throw std::runtime_error( + "Error allocating resources for serialized message: " + + std::string(rcutils_get_error_string().str)); + } + // The compressor may modify this buffer in-place, so it should take ownership of it. + std::move(byte_ma_message->data.data(), + byte_ma_message->data.data() + byte_ma_message->data.size(), + msg_array->buffer); + auto serialized_data = std::shared_ptr(msg_array); serialized_data->buffer_length = byte_ma_message->data.size(); - serialized_data->buffer_capacity = byte_ma_message->data.size(); message->serialized_data = serialized_data; rcutils_time_point_value_t time_stamp; - int error = rcutils_system_time_now(&time_stamp); + error = rcutils_system_time_now(&time_stamp); if (error != RCUTILS_RET_OK) { RCLCPP_ERROR_STREAM( get_logger(), "Error getting current time. Error:" << @@ -187,6 +209,16 @@ void WriterBenchmark::create_producers(const ProducerConfig & config) // Also, add an option to configure compression void WriterBenchmark::create_writer() { + if (!compression_format_.empty()) { + rosbag2_compression::CompressionOptions compression_options{ + compression_format_, rosbag2_compression::CompressionMode::MESSAGE, + compression_queue_size_, compression_threads_}; + + writer_ = std::make_unique( + compression_options); + } else { + writer_ = std::make_shared(); + } writer_ = std::make_shared(); rosbag2_storage::StorageOptions storage_options{}; storage_options.uri = db_folder_; diff --git a/rosbag2_transport/include/rosbag2_transport/record_options.hpp b/rosbag2_transport/include/rosbag2_transport/record_options.hpp index adb0573c2b..bc87828251 100644 --- a/rosbag2_transport/include/rosbag2_transport/record_options.hpp +++ b/rosbag2_transport/include/rosbag2_transport/record_options.hpp @@ -35,6 +35,8 @@ struct RecordOptions std::string node_prefix = ""; std::string compression_mode = ""; std::string compression_format = ""; + uint64_t compression_queue_size = 1; + uint64_t compression_threads = 0; std::unordered_map topic_qos_profile_overrides{}; bool include_hidden_topics = false; }; diff --git a/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp b/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp index 9f42f75228..afae2ee742 100644 --- a/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp +++ b/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -94,6 +95,8 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * "node_prefix", "compression_mode", "compression_format", + "compression_queue_size", + "compression_threads", "all", "no_discovery", "polling_interval", @@ -112,6 +115,8 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * char * node_prefix = nullptr; char * compression_mode = nullptr; char * compression_format = nullptr; + uint64_t compression_queue_size = 1; + uint64_t compression_threads = 0; PyObject * qos_profile_overrides = nullptr; bool all = false; bool no_discovery = false; @@ -124,13 +129,15 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * char * storage_config_file = nullptr; if ( !PyArg_ParseTupleAndKeywords( - args, kwargs, "ssssss|bbKKKKObOs", const_cast(kwlist), + args, kwargs, "ssssss|KKbbKKKKObOs", const_cast(kwlist), &uri, &storage_id, &serilization_format, &node_prefix, &compression_mode, &compression_format, + &compression_queue_size, + &compression_threads, &all, &no_discovery, &polling_interval_ms, @@ -158,11 +165,18 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * record_options.node_prefix = std::string(node_prefix); record_options.compression_mode = std::string(compression_mode); record_options.compression_format = compression_format; + record_options.compression_queue_size = compression_queue_size; + if (compression_threads < 1) { + compression_threads = std::thread::hardware_concurrency(); + } + record_options.compression_threads = compression_threads; record_options.include_hidden_topics = include_hidden_topics; rosbag2_compression::CompressionOptions compression_options{ record_options.compression_format, - rosbag2_compression::compression_mode_from_string(record_options.compression_mode) + rosbag2_compression::compression_mode_from_string(record_options.compression_mode), + record_options.compression_queue_size, + record_options.compression_threads }; auto topic_qos_overrides = PyObject_AsTopicQoSMap(qos_profile_overrides); From 063632ceed674e96f0fc8c45c441ecba97624789 Mon Sep 17 00:00:00 2001 From: Chris Lalancette Date: Mon, 7 Dec 2020 12:53:01 -0500 Subject: [PATCH 54/77] Only dereference the data pointer if it is valid. (#581) If the static_cast returned a nullptr, then just print the message and don't attempt to deference it. Signed-off-by: Chris Lalancette --- rosbag2_cpp/test/rosbag2_cpp/types/test_ros2_message.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rosbag2_cpp/test/rosbag2_cpp/types/test_ros2_message.cpp b/rosbag2_cpp/test/rosbag2_cpp/types/test_ros2_message.cpp index 525a706989..d48e23b47a 100644 --- a/rosbag2_cpp/test/rosbag2_cpp/types/test_ros2_message.cpp +++ b/rosbag2_cpp/test/rosbag2_cpp/types/test_ros2_message.cpp @@ -76,9 +76,9 @@ TEST_F(Ros2MessageTest, allocate_ros2_message_allocates_strings_message_with_emp auto data = static_cast(message->message); if (!data) { fprintf(stderr, "allocation failed for string\n"); + } else { + data->string_value = ""; } - - data->string_value = ""; } TEST_F(Ros2MessageTest, allocate_ros2_message_allocates_strings_message_with_big_string_no_SSO) { From 25ce93789c42f95fd5ac85478481866253de79cd Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Mon, 7 Dec 2020 13:10:47 -0800 Subject: [PATCH 55/77] Re-enable Action CI for PRs (#465) * Add target-ros2-distro to action CI to allow it to run correctly Signed-off-by: Emerson Knapp --- .github/workflows/test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e34860a509..cc8298de82 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,6 @@ name: Test rosbag2 on: - # pull_request: + pull_request: push: branches: - master @@ -13,7 +13,7 @@ jobs: build_and_test: runs-on: ubuntu-latest container: - image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest + image: rostooling/setup-ros-docker:ubuntu-focal-latest steps: - uses: ros-tooling/action-ros-ci@0.1.0 with: @@ -29,6 +29,7 @@ jobs: sqlite3_vendor rosbag2_test_common rosbag2_tests + target-ros2-distro: rolling - uses: actions/upload-artifact@v1 with: name: colcon-logs From 41afa0e977277ef62f8af62bee461b0d248932c8 Mon Sep 17 00:00:00 2001 From: Barry Xu Date: Wed, 9 Dec 2020 07:56:07 +0800 Subject: [PATCH 56/77] Fix a bug on parsing wrong description in plugin xml file (#578) Signed-off-by: Barry Xu --- ros2bag/ros2bag/verb/list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros2bag/ros2bag/verb/list.py b/ros2bag/ros2bag/verb/list.py index 8ae9b15f49..6491ea22e1 100644 --- a/ros2bag/ros2bag/verb/list.py +++ b/ros2bag/ros2bag/verb/list.py @@ -55,7 +55,7 @@ def main(self, *, args): # noqa: D102 class_name = class_item.attributes['name'] type_name = class_item.attributes['type'] base_class_name = class_item.attributes['base_class_type'] - description = xmldoc.getElementsByTagName('description')[0] + description = class_item.getElementsByTagName('description')[0] print('%s%s' % (('name: ' if args.verbose else ''), class_name.value)) if args.verbose: From 4c2e74c80995eab602895abb2f1a365dbbc74e22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C4=85browski?= Date: Wed, 16 Dec 2020 01:04:38 +0100 Subject: [PATCH 57/77] Updating performance writer (#570) * Updating performance writer - utilizes --storage-config-file with two examples - message reading from metadata yaml file (ready for cache double-buffers PR 546) - removed performance-affecting outputs - supports bag splitting in benchmarking Signed-off-by: Adam Dabrowski * typo fix Signed-off-by: Adam Dabrowski * corrected compression so that it doesn't break the command line in case it is empty (the default) Signed-off-by: Adam Dabrowski * address CI warns/fails on MacOS and Windows Signed-off-by: Adam Dabrowski --- .../CMakeLists.txt | 2 + .../message_queue.hpp | 1 - .../writer_benchmark.hpp | 8 +- .../package.xml | 1 + .../scripts/benchmark.sh | 73 ++++++++++------- .../storage_config/storage_default.yaml | 2 + .../storage_config/storage_optimized.yaml | 2 + .../src/writer_benchmark.cpp | 82 ++++++++++++------- 8 files changed, 109 insertions(+), 62 deletions(-) create mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_default.yaml create mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_optimized.yaml diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt b/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt index 4f353fdefc..40957e3f93 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt @@ -20,6 +20,7 @@ if(BUILD_ROSBAG2_BENCHMARKS) find_package(rosbag2_storage REQUIRED) find_package(rmw REQUIRED) find_package(std_msgs REQUIRED) + find_package(yaml_cpp_vendor REQUIRED) add_executable(writer_benchmark src/writer_benchmark.cpp src/main.cpp) ament_target_dependencies(writer_benchmark @@ -27,6 +28,7 @@ if(BUILD_ROSBAG2_BENCHMARKS) std_msgs rosbag2_compression rosbag2_cpp + yaml_cpp_vendor ) target_include_directories(writer_benchmark PUBLIC diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/message_queue.hpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/message_queue.hpp index 87ed7b51c2..1554d3f202 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/message_queue.hpp +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/message_queue.hpp @@ -40,7 +40,6 @@ class MessageQueue std::lock_guard lock(mutex_); if (queue_.size() > max_size_) { // We skip the element and consider it "lost" ++unsuccessful_insert_count_; - std::cerr << "X" << std::flush; return; } queue_.push(elem); diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp index 7228c7301a..ac0bb12af8 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp @@ -37,19 +37,19 @@ class WriterBenchmark : public rclcpp::Node void create_producers(const ProducerConfig & config); void create_writer(); void start_producers(); - void write_results(const unsigned int & total_missed) const; + void write_results() const; + int get_message_count_from_metadata() const; ProducerConfig config_; - unsigned int max_cache_size_; unsigned int instances_; - std::string db_folder_; std::string results_file_; + + rosbag2_storage::StorageOptions storage_options_; std::vector producer_threads_; std::vector> producers_; std::vector> queues_; std::string compression_format_; - rosbag2_compression::CompressionMode compression_mode_; uint64_t compression_queue_size_; uint64_t compression_threads_; std::shared_ptr writer_; diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml b/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml index 957732e9a2..ce9316db43 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml @@ -17,6 +17,7 @@ rosbag2_storage rmw std_msgs + yaml_cpp_vendor ament_lint_auto ament_lint_common diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh index 293f164f61..a56b53a80d 100755 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh @@ -13,46 +13,61 @@ mkdir -p ${test_dir} echo "${test_dir} created" db_path=${test_dir}/bag rm -fr ${db_path} -summary_file=${test_dir}/results.csv +script_dir=$(dirname $(readlink -f "$0")) -freq=100; #Hz +storage_config_dir=${script_dir}/storage_config +config_optimized_path=${storage_config_dir}/storage_optimized.yaml +config_default_path=${storage_config_dir}/storage_default.yaml # Set this to "zstd" to test with compression compression_format="" +compression_arg="" +if [ -n "$compression_format" ]; then + compression_arg="-p compression_format:=${compression_format}" +fi -for cache in 0 1000000 10000000 100000000 +for storage_config_file_path in ${config_optimized_path} ${config_default_path} do - for sz in 1000 10000 100000 1000000 + config_file_name=$(basename $storage_config_file_path) + summary_file=${test_dir}/results_${config_file_name}.csv + for cache in 0 1000000 100000000 500000000 do - for inst in 1 10 100 1000 + for sz in 10000 100000 1000000 5000000 do - let total=$inst*$sz - if [[ $total -ne 1000000 ]]; then - #echo "skipping the case of ${inst} instances and size ${sz}" - continue - fi - echo "processing case of ${inst} instances and size ${sz} with cache ${cache}" - for try in 1 2 3 4 5 + for inst in 1 10 100 1000 do + freq=100; #Hz + let total=$inst*$sz*$freq + if [[ $total -lt 100000000 || $total -gt 500000000 ]]; then + #echo "skipping the case of ${inst} instances and size ${sz}" + continue + fi + echo "processing case of ${inst} instances and size ${sz} with cache ${cache} and config ${config_file_name}" outdir=${test_dir}/size${sz}_inst${inst}_cache${cache} mkdir -p ${outdir} - outfile=${outdir}/${try}.log - echo "Results will be written to file: ${outfile}" - ros2 run rosbag2_performance_writer_benchmarking writer_benchmark --ros-args \ - -p compression_format:=${compression_format} \ - -p frequency:=${freq} \ - -p size:=${sz} \ - -p instances:=${inst} \ - -p max_cache_size:=${cache} \ - -p db_folder:=${db_path} \ - -p results_file:=${summary_file} \ - --ros-args -r __node:=rosbag2_performance_writer_benchmarking_node_batch \ - 2> ${outfile} - rm -fr ${db_path} - if [[ $ctrlc_sent -eq 1 ]]; then - echo -e "\e[31mQuitting prematurely due to Ctrl-C - some results aren't saved and some won't be reliable\e[0m]" - exit - fi + for try in 1 2 3 + do + outfile=${outdir}/${try}.log + echo "Results will be written to file: ${outfile}" + ros2 run rosbag2_performance_writer_benchmarking writer_benchmark --ros-args \ + ${compression_arg} \ + -p frequency:=${freq} \ + -p size:=${sz} \ + -p instances:=${inst} \ + -p max_cache_size:=${cache} \ + -p db_folder:=${db_path} \ + -p storage_config_file:=${storage_config_file_path} \ + -p results_file:=${summary_file} \ + --ros-args -r __node:=rosbag2_performance_writer_benchmarking_node_batch \ + 2> ${outfile} + sleep 2 # making sure to flush both application and disk caches and have a fresh start + cp ${db_path}/metadata.yaml ${outdir}/${try}_metadata.yaml # preserve metadata from test + rm -fr ${db_path} # but remove large database file so we won't run out of disk space + if [[ $ctrlc_sent -eq 1 ]]; then + echo -e "\e[31mQuitting prematurely due to Ctrl-C - some results aren't saved and some won't be reliable\e[0m]" + exit + fi + done done done done diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_default.yaml b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_default.yaml new file mode 100644 index 0000000000..d0455628dc --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_default.yaml @@ -0,0 +1,2 @@ +write: + pragmas: ["journal_mode = WAL", "synchronous = NORMAL"] diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_optimized.yaml b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_optimized.yaml new file mode 100644 index 0000000000..f93e23d813 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_optimized.yaml @@ -0,0 +1,2 @@ +write: + pragmas: ["journal_mode = MEMORY", "synchronous = OFF"] diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp index a743fdcc0e..d71a5fe050 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp +++ b/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp @@ -26,6 +26,19 @@ #include "rosbag2_performance_writer_benchmarking/writer_benchmark.hpp" +#ifdef _WIN32 +// This is necessary because of a bug in yaml-cpp's cmake +#define YAML_CPP_DLL +// This is necessary because yaml-cpp does not always use dllimport/dllexport consistently +# pragma warning(push) +# pragma warning(disable:4251) +# pragma warning(disable:4275) +#endif +#include "yaml-cpp/yaml.h" +#ifdef _WIN32 +# pragma warning(pop) +#endif + using namespace std::chrono_literals; static rcutils_allocator_t allocator = rcutils_get_default_allocator(); @@ -38,9 +51,11 @@ WriterBenchmark::WriterBenchmark(const std::string & name) this->declare_parameter("max_count", 1000); this->declare_parameter("size", 1000000); this->declare_parameter("instances", 1); - this->declare_parameter("max_cache_size", 1); + this->declare_parameter("max_cache_size", 10000000); + this->declare_parameter("max_bag_size", 0); this->declare_parameter("db_folder", default_bag_folder); this->declare_parameter("results_file", default_bag_folder + "/results.csv"); + this->declare_parameter("storage_config_file", ""); this->declare_parameter("compression_format", ""); this->declare_parameter("compression_queue_size", 1); this->declare_parameter("compression_threads", 0); @@ -52,8 +67,12 @@ WriterBenchmark::WriterBenchmark(const std::string & name) return; } - this->get_parameter("max_cache_size", max_cache_size_); - this->get_parameter("db_folder", db_folder_); + storage_options_.storage_id = "sqlite3"; + this->get_parameter("max_cache_size", storage_options_.max_cache_size); + this->get_parameter("db_folder", storage_options_.uri); + this->get_parameter("storage_config_file", storage_options_.storage_config_uri); + this->get_parameter("max_bag_size", storage_options_.max_bagfile_size); + this->get_parameter("results_file", results_file_); this->get_parameter("max_count", config_.max_count); this->get_parameter("size", config_.message_size); @@ -68,7 +87,7 @@ WriterBenchmark::WriterBenchmark(const std::string & name) void WriterBenchmark::start_benchmark() { - RCLCPP_INFO(get_logger(), "Starting. A dot is a write, an X is a miss"); + RCLCPP_INFO(get_logger(), "Starting"); start_producers(); while (rclcpp::ok()) { int count = 0; @@ -98,9 +117,10 @@ void WriterBenchmark::start_benchmark() std::string(rcutils_get_error_string().str)); } // The compressor may modify this buffer in-place, so it should take ownership of it. - std::move(byte_ma_message->data.data(), - byte_ma_message->data.data() + byte_ma_message->data.size(), - msg_array->buffer); + std::move( + byte_ma_message->data.data(), + byte_ma_message->data.data() + byte_ma_message->data.size(), + msg_array->buffer); auto serialized_data = std::shared_ptr(msg_array); serialized_data->buffer_length = byte_ma_message->data.size(); @@ -121,7 +141,6 @@ void WriterBenchmark::start_benchmark() } catch (const std::runtime_error & e) { RCLCPP_ERROR_STREAM(get_logger(), "Failed to record: " << e.what()); } - std::cerr << "." << std::flush; ++count; } } @@ -136,23 +155,36 @@ void WriterBenchmark::start_benchmark() prod_thread.join(); } - unsigned int total_missed_messages = 0; - for (const auto & queue : queues_) { - total_missed_messages += queue->get_missed_elements_count(); + writer_->reset(); + write_results(); +} + +int WriterBenchmark::get_message_count_from_metadata() const +{ + int total_recorded_count = 0; + std::string metadata_filename(rosbag2_storage::MetadataIo::metadata_filename); + std::string metadata_path = storage_options_.uri + "/" + metadata_filename; + try { + YAML::Node yaml_file = YAML::LoadFile(metadata_path); + total_recorded_count = yaml_file["rosbag2_bagfile_information"]["message_count"].as(); + } catch (const YAML::Exception & ex) { + throw std::runtime_error( + std::string("Exception on parsing metadata file to get total message count: ") + + metadata_path + " " + + ex.what()); } - write_results(total_missed_messages); + return total_recorded_count; } -void WriterBenchmark::write_results(const unsigned int & total_missed) const +void WriterBenchmark::write_results() const { + int total_recorded_count = get_message_count_from_metadata(); auto total_messages_sent = config_.max_count * producers_.size(); - float percentage_recorded = 100.0f - static_cast(total_missed * 100.0f) / + float percentage_recorded = static_cast(total_recorded_count * 100.0f) / total_messages_sent; - RCLCPP_INFO(get_logger(), "\nWriterBenchmark terminating"); - RCLCPP_INFO_STREAM(get_logger(), "Total missed messages: " << total_missed); RCLCPP_INFO_STREAM( - get_logger(), "Percentage of all message that was successfully recorded: " << + get_logger(), "Percentage of all messages that were successfully recorded: " << percentage_recorded); bool new_file = false; @@ -173,18 +205,17 @@ void WriterBenchmark::write_results(const unsigned int & total_missed) const if (new_file) { output_file << "instances frequency message_size cache_size total_messages_sent "; - output_file << "total_messages_missed percentage_recorded\n"; + output_file << "percentage_recorded\n"; } // configuration of the test. TODO(adamdbrw) wrap into a dict and define << operator. output_file << instances_ << " "; output_file << config_.frequency << " "; output_file << config_.message_size << " "; - output_file << max_cache_size_ << " "; + output_file << storage_options_.max_cache_size << " "; output_file << total_messages_sent << " "; // results of the test. Use std::setprecision if preferred - output_file << total_missed << " "; output_file << percentage_recorded << std::endl; } @@ -194,7 +225,8 @@ void WriterBenchmark::create_producers(const ProducerConfig & config) get_logger(), "\nWriterBenchmark: creating " << instances_ << " message producers with frequency " << config.frequency << " and message size in bytes " << config.message_size << - ". Cache is " << max_cache_size_ << ". Each will send " << config.max_count << + ". Cache is " << storage_options_.max_cache_size << + ". Each will send " << config.max_count << " messages before terminating"); const unsigned int queue_max_size = 10; for (unsigned int i = 0; i < instances_; ++i) { @@ -219,16 +251,10 @@ void WriterBenchmark::create_writer() } else { writer_ = std::make_shared(); } - writer_ = std::make_shared(); - rosbag2_storage::StorageOptions storage_options{}; - storage_options.uri = db_folder_; - storage_options.storage_id = "sqlite3"; - storage_options.max_bagfile_size = 0; - storage_options.max_cache_size = max_cache_size_; // TODO(adamdbrw) generalize if converters are to be included in benchmarks std::string serialization_format = rmw_get_serialization_format(); - writer_->open(storage_options, {serialization_format, serialization_format}); + writer_->open(storage_options_, {serialization_format, serialization_format}); for (const auto & queue : queues_) { rosbag2_storage::TopicMetadata topic; From 1549af4a622f3659a94b41b208fd7ed94ecc9788 Mon Sep 17 00:00:00 2001 From: Chen Lihui Date: Wed, 30 Dec 2020 07:54:25 +0800 Subject: [PATCH 58/77] Fixed playing if unknown message types exist (#592) 1. play a specific known message type even if some unknown types exist. 2. add a warning message while a message type library not exist. Signed-off-by: Chen Lihui --- .../src/rosbag2_transport/player.cpp | 30 ++- .../mock_sequential_reader.hpp | 2 +- .../test/rosbag2_transport/test_play.cpp | 175 ++++++++++++++++++ 3 files changed, 201 insertions(+), 6 deletions(-) diff --git a/rosbag2_transport/src/rosbag2_transport/player.cpp b/rosbag2_transport/src/rosbag2_transport/player.cpp index 4309e46053..8cf8c962c4 100644 --- a/rosbag2_transport/src/rosbag2_transport/player.cpp +++ b/rosbag2_transport/src/rosbag2_transport/player.cpp @@ -14,6 +14,7 @@ #include "player.hpp" +#include #include #include #include @@ -183,7 +184,10 @@ void Player::play_messages_until_queue_empty(const PlayOptions & options) start_time_ + std::chrono::duration_cast( 1.0 / rate * message.time_since_start)); if (rclcpp::ok()) { - publishers_[message.message->topic_name]->publish(message.message->serialized_data); + auto publisher_iter = publishers_.find(message.message->topic_name); + if (publisher_iter != publishers_.end()) { + publisher_iter->second->publish(message.message->serialized_data); + } } } } @@ -196,11 +200,27 @@ void Player::prepare_publishers(const PlayOptions & options) auto topics = reader_->get_all_topics_and_types(); for (const auto & topic : topics) { + // filter topics to add publishers if necessary + auto & filter_topics = storage_filter.topics; + if (!filter_topics.empty()) { + auto iter = std::find(filter_topics.begin(), filter_topics.end(), topic.name); + if (iter == filter_topics.end()) { + continue; + } + } + auto topic_qos = publisher_qos_for_topic(topic, topic_qos_profile_overrides_); - publishers_.insert( - std::make_pair( - topic.name, rosbag2_transport_->create_generic_publisher( - topic.name, topic.type, topic_qos))); + try { + publishers_.insert( + std::make_pair( + topic.name, rosbag2_transport_->create_generic_publisher( + topic.name, topic.type, topic_qos))); + } catch (const std::runtime_error & e) { + // using a warning log seems better than adding a new option + // to ignore some unknown message type library + ROSBAG2_TRANSPORT_LOG_WARN( + "Ignoring a topic '%s', reason: %s.", topic.name.c_str(), e.what()); + } } } diff --git a/rosbag2_transport/test/rosbag2_transport/mock_sequential_reader.hpp b/rosbag2_transport/test/rosbag2_transport/mock_sequential_reader.hpp index b2f4a29862..d679e402e5 100644 --- a/rosbag2_transport/test/rosbag2_transport/mock_sequential_reader.hpp +++ b/rosbag2_transport/test/rosbag2_transport/mock_sequential_reader.hpp @@ -44,7 +44,7 @@ class MockSequentialReader : public rosbag2_cpp::reader_interfaces::BaseReaderIn while (num_read_ < messages_.size()) { for (const auto & filter_topic : filter_.topics) { - if (!messages_[num_read_ + 1]->topic_name.compare(filter_topic)) { + if (!messages_[num_read_]->topic_name.compare(filter_topic)) { return true; } } diff --git a/rosbag2_transport/test/rosbag2_transport/test_play.cpp b/rosbag2_transport/test/rosbag2_transport/test_play.cpp index 58e47a4e36..6b1652e8dc 100644 --- a/rosbag2_transport/test/rosbag2_transport/test_play.cpp +++ b/rosbag2_transport/test/rosbag2_transport/test_play.cpp @@ -106,6 +106,75 @@ TEST_F(RosBag2PlayTestFixture, recorded_messages_are_played_for_all_topics) ElementsAre(40.0f, 2.0f, 0.0f))))); } +TEST_F(RosBag2PlayTestFixture, recorded_messages_are_played_for_all_topics_with_unknown_type) +{ + auto primitive_message1 = get_messages_basic_types()[0]; + primitive_message1->int32_value = 42; + + auto complex_message1 = get_messages_arrays()[0]; + complex_message1->float32_values = {{40.0f, 2.0f, 0.0f}}; + complex_message1->bool_values = {{true, false, true}}; + + auto unknown_message1 = get_messages_basic_types()[0]; + unknown_message1->int32_value = 42; + + auto topic_types = std::vector{ + {"topic1", "test_msgs/BasicTypes", "", ""}, + {"topic2", "test_msgs/Arrays", "", ""}, + {"topic3", "unknown_msgs/UnknownType", "", ""}, + }; + + std::vector> messages = + {serialize_test_message("topic1", 500, primitive_message1), + serialize_test_message("topic1", 700, primitive_message1), + serialize_test_message("topic1", 900, primitive_message1), + serialize_test_message("topic2", 550, complex_message1), + serialize_test_message("topic2", 750, complex_message1), + serialize_test_message("topic2", 950, complex_message1), + serialize_test_message("topic3", 900, unknown_message1)}; + + auto prepared_mock_reader = std::make_unique(); + prepared_mock_reader->prepare(messages, topic_types); + reader_ = std::make_unique(std::move(prepared_mock_reader)); + + // Due to a problem related to the subscriber, we play many (3) messages but make the subscriber + // node spin only until 2 have arrived. Hence the 2 as `launch_subscriber()` argument. + sub_->add_subscription("/topic1", 2); + sub_->add_subscription("/topic2", 2); + + auto await_received_messages = sub_->spin_subscriptions(); + + Rosbag2Transport rosbag2_transport(reader_, writer_, info_); + rosbag2_transport.play(storage_options_, play_options_); + + await_received_messages.get(); + + auto replayed_test_primitives = sub_->get_received_messages( + "/topic1"); + EXPECT_THAT(replayed_test_primitives, SizeIs(Ge(2u))); + EXPECT_THAT( + replayed_test_primitives, + Each(Pointee(Field(&test_msgs::msg::BasicTypes::int32_value, 42)))); + + auto replayed_test_arrays = sub_->get_received_messages( + "/topic2"); + EXPECT_THAT(replayed_test_arrays, SizeIs(Ge(2u))); + EXPECT_THAT( + replayed_test_arrays, + Each( + Pointee( + Field( + &test_msgs::msg::Arrays::bool_values, + ElementsAre(true, false, true))))); + EXPECT_THAT( + replayed_test_arrays, + Each( + Pointee( + Field( + &test_msgs::msg::Arrays::float32_values, + ElementsAre(40.0f, 2.0f, 0.0f))))); +} + TEST_F(RosBag2PlayTestFixture, recorded_messages_are_played_for_filtered_topics) { auto primitive_message1 = get_messages_basic_types()[0]; @@ -202,6 +271,112 @@ TEST_F(RosBag2PlayTestFixture, recorded_messages_are_played_for_filtered_topics) EXPECT_THAT(replayed_test_arrays, SizeIs(Ge(2u))); } +TEST_F(RosBag2PlayTestFixture, recorded_messages_are_played_for_filtered_topics_with_unknown_type) +{ + auto primitive_message1 = get_messages_basic_types()[0]; + primitive_message1->int32_value = 42; + + auto complex_message1 = get_messages_arrays()[0]; + complex_message1->float32_values = {{40.0f, 2.0f, 0.0f}}; + complex_message1->bool_values = {{true, false, true}}; + + auto unknown_message1 = get_messages_basic_types()[0]; + unknown_message1->int32_value = 42; + + auto topic_types = std::vector{ + {"topic1", "test_msgs/BasicTypes", "", ""}, + {"topic2", "test_msgs/Arrays", "", ""}, + {"topic3", "unknown_msgs/UnknownType", "", ""}, + }; + + std::vector> messages = + {serialize_test_message("topic1", 500, primitive_message1), + serialize_test_message("topic1", 700, primitive_message1), + serialize_test_message("topic1", 900, primitive_message1), + serialize_test_message("topic2", 550, complex_message1), + serialize_test_message("topic2", 750, complex_message1), + serialize_test_message("topic2", 950, complex_message1), + serialize_test_message("topic3", 900, unknown_message1)}; + + auto prepared_mock_reader = std::make_unique(); + prepared_mock_reader->prepare(messages, topic_types); + reader_ = std::make_unique(std::move(prepared_mock_reader)); + + // Due to a problem related to the subscriber, we play many (3) messages but make the subscriber + // node spin only until 2 have arrived. Hence the 2 as `launch_subscriber()` argument. + + sub_->add_subscription("/topic2", 2); + + auto await_received_messages = sub_->spin_subscriptions(); + + Rosbag2Transport rosbag2_transport(reader_, writer_, info_); + play_options_.topics_to_filter = {"topic2"}; + rosbag2_transport.play(storage_options_, play_options_); + + await_received_messages.get(); + + auto replayed_test_primitives = sub_->get_received_messages( + "/topic1"); + EXPECT_THAT(replayed_test_primitives, SizeIs(Ge(0u))); + + auto replayed_test_arrays = sub_->get_received_messages( + "/topic2"); + EXPECT_THAT(replayed_test_arrays, SizeIs(Ge(2u))); + + // Set new filter + auto prepared_mock_reader2 = std::make_unique(); + prepared_mock_reader2->prepare(messages, topic_types); + reader_.reset(); + reader_ = std::make_unique(std::move(prepared_mock_reader2)); + + sub_.reset(); + sub_ = std::make_shared(); + sub_->add_subscription("/topic1", 2); + + await_received_messages = sub_->spin_subscriptions(); + + rosbag2_transport = Rosbag2Transport(reader_, writer_, info_); + play_options_.topics_to_filter = {"topic1"}; + rosbag2_transport.play(storage_options_, play_options_); + + await_received_messages.get(); + + replayed_test_primitives = sub_->get_received_messages( + "/topic1"); + EXPECT_THAT(replayed_test_primitives, SizeIs(Ge(2u))); + + replayed_test_arrays = sub_->get_received_messages( + "/topic2"); + EXPECT_THAT(replayed_test_arrays, SizeIs(Ge(0u))); + + // Reset filter + auto prepared_mock_reader3 = std::make_unique(); + prepared_mock_reader3->prepare(messages, topic_types); + reader_.reset(); + reader_ = std::make_unique(std::move(prepared_mock_reader3)); + + sub_.reset(); + sub_ = std::make_shared(); + sub_->add_subscription("/topic1", 2); + sub_->add_subscription("/topic2", 2); + + await_received_messages = sub_->spin_subscriptions(); + + rosbag2_transport = Rosbag2Transport(reader_, writer_, info_); + play_options_.topics_to_filter = {"topic1", "topic2"}; + rosbag2_transport.play(storage_options_, play_options_); + + await_received_messages.get(); + + replayed_test_primitives = sub_->get_received_messages( + "/topic1"); + EXPECT_THAT(replayed_test_primitives, SizeIs(Ge(2u))); + + replayed_test_arrays = sub_->get_received_messages( + "/topic2"); + EXPECT_THAT(replayed_test_arrays, SizeIs(Ge(2u))); +} + class RosBag2PlayQosOverrideTestFixture : public RosBag2PlayTestFixture { public: From 84472f3086e06bf087322946e95062043d59eeb1 Mon Sep 17 00:00:00 2001 From: Josh Langsfeld Date: Mon, 4 Jan 2021 19:16:12 -0600 Subject: [PATCH 59/77] Use std::filesystem compliant non-member `exists` function for path object (#593) Signed-off-by: Josh Langsfeld --- rosbag2_storage/src/rosbag2_storage/metadata_io.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rosbag2_storage/src/rosbag2_storage/metadata_io.cpp b/rosbag2_storage/src/rosbag2_storage/metadata_io.cpp index 0d1ce27929..aa81b78fb9 100644 --- a/rosbag2_storage/src/rosbag2_storage/metadata_io.cpp +++ b/rosbag2_storage/src/rosbag2_storage/metadata_io.cpp @@ -253,7 +253,7 @@ std::string MetadataIo::get_metadata_file_name(const std::string & uri) bool MetadataIo::metadata_file_exists(const std::string & uri) { - return rcpputils::fs::path(get_metadata_file_name(uri)).exists(); + return rcpputils::fs::exists(rcpputils::fs::path(get_metadata_file_name(uri))); } } // namespace rosbag2_storage From 216e40f4ddeda36b02603d62dd47fcbe25df99df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C4=85browski?= Date: Sat, 9 Jan 2021 00:01:16 +0100 Subject: [PATCH 60/77] SQLite storage optimized by default (#568) * Use optimized pragmas by default in sqlite storage. Added option to use former behavior Signed-off-by: Adam Dabrowski --- README.md | 8 ++- ros2bag/ros2bag/verb/record.py | 10 +++ .../rosbag2_storage/storage_options.hpp | 4 ++ .../sqlite/sqlite_pragmas.hpp | 61 +++++++++++++++++++ .../sqlite/sqlite_storage.hpp | 2 + .../sqlite/sqlite_wrapper.hpp | 1 + .../sqlite/sqlite_storage.cpp | 23 ++++++- .../sqlite/sqlite_wrapper.cpp | 19 +++--- .../sqlite/storage_test_fixture.hpp | 2 +- .../sqlite/test_sqlite_storage.cpp | 40 ++++++++++++ .../rosbag2_transport_python.cpp | 7 ++- 11 files changed, 163 insertions(+), 14 deletions(-) create mode 100644 rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_pragmas.hpp diff --git a/README.md b/README.md index dc555d51d9..f86ba25b6f 100644 --- a/README.md +++ b/README.md @@ -125,8 +125,14 @@ read: write: pragmas: ``` -Please refer to [documentation of pragmas](https://www.sqlite.org/pragma.html). + +By default, SQLite settings are significantly optimized for performance. +This might have consequences of bag data being corrupted after an application or system-level crash. +This consideration only applies to current bagfile in case bag splitting is on (through `--max-bag-*` parameters). +If increased crash-caused corruption resistance is necessary, use `resilient` option for `--storage-preset-profile` setting. + Settings are fully exposed to the user and should be applied with understanding. +Please refer to [documentation of pragmas](https://www.sqlite.org/pragma.html). An example configuration file could look like this: diff --git a/ros2bag/ros2bag/verb/record.py b/ros2bag/ros2bag/verb/record.py index ffd978fc22..d8aa543596 100644 --- a/ros2bag/ros2bag/verb/record.py +++ b/ros2bag/ros2bag/verb/record.py @@ -102,6 +102,15 @@ def add_arguments(self, parser, cli_name): # noqa: D102 '--qos-profile-overrides-path', type=FileType('r'), help='Path to a yaml file defining overrides of the QoS profile for specific topics.' ) + parser.add_argument( + '--storage-preset-profile', type=str, default='none', choices=['none', 'resilient'], + help='Select a configuration preset for storage.' + 'resilient (sqlite3):' + 'indicate preference for avoiding data corruption in case of crashes,' + 'at the cost of performance. Setting this flag disables optimization settings ' + 'for storage (the defaut). This flag settings can still be overriden by ' + 'corresponding settings in the config passed with --storage-config-file.' + ) parser.add_argument( '--storage-config-file', type=FileType('r'), help='Path to a yaml file defining storage specific configurations. ' @@ -171,6 +180,7 @@ def main(self, *, args): # noqa: D102 topics=args.topics, include_hidden_topics=args.include_hidden_topics, qos_profile_overrides=qos_profile_overrides, + storage_preset_profile=args.storage_preset_profile, storage_config_file=storage_config_file) if os.path.isdir(uri) and not os.listdir(uri): diff --git a/rosbag2_storage/include/rosbag2_storage/storage_options.hpp b/rosbag2_storage/include/rosbag2_storage/storage_options.hpp index 7f7a2cee1d..0ff267de5a 100644 --- a/rosbag2_storage/include/rosbag2_storage/storage_options.hpp +++ b/rosbag2_storage/include/rosbag2_storage/storage_options.hpp @@ -39,6 +39,10 @@ struct StorageOptions // A value of 0 disables caching and every write happens directly to disk. uint64_t max_cache_size = 0; + // Preset storage configuration. Preset settings can be overriden with + // corresponding settings specified through storage_config_uri file + std::string storage_preset_profile = ""; + // Storage specific configuration file. // Defaults to empty string. std::string storage_config_uri = ""; diff --git a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_pragmas.hpp b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_pragmas.hpp new file mode 100644 index 0000000000..fd81f5906e --- /dev/null +++ b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_pragmas.hpp @@ -0,0 +1,61 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_STORAGE_DEFAULT_PLUGINS__SQLITE__SQLITE_PRAGMAS_HPP_ +#define ROSBAG2_STORAGE_DEFAULT_PLUGINS__SQLITE__SQLITE_PRAGMAS_HPP_ + +#include +#include +#include +#include + + +namespace rosbag2_storage_plugins +{ + +class ROSBAG2_STORAGE_DEFAULT_PLUGINS_PUBLIC SqlitePragmas +{ +public: + using pragmas_map_t = std::unordered_map; + + static pragmas_map_t default_pragmas() + { + static pragmas_map_t p = {{"schema_version", "PRAGMA schema_version;"}}; + return p; + } + + // more robust storage settings will cause performace hit on writing, + // but increase resistance to bagfile data corruption in case of crashes + static pragmas_map_t robust_writing_pragmas() + { + static pragmas_map_t p = { + {"journal_mode", "PRAGMA journal_mode=WAL;"}, + {"synchronous", "PRAGMA synchronous=NORMAL;"} + }; + return p; + } + + static pragmas_map_t optimized_writing_pragmas() + { + static pragmas_map_t p = { + {"journal_mode", "PRAGMA journal_mode=MEMORY;"}, + {"synchronous", "PRAGMA synchronous=OFF;"} + }; + return p; + } +}; + +} // namespace rosbag2_storage_plugins + +#endif // ROSBAG2_STORAGE_DEFAULT_PLUGINS__SQLITE__SQLITE_PRAGMAS_HPP_ diff --git a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_storage.hpp b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_storage.hpp index f557c5e93c..fd16b8842b 100644 --- a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_storage.hpp +++ b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_storage.hpp @@ -83,6 +83,8 @@ class ROSBAG2_STORAGE_DEFAULT_PLUGINS_PUBLIC SqliteStorage void reset_filter() override; + std::string get_storage_setting(const std::string & key); + private: void initialize(); void prepare_for_writing(); diff --git a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.hpp b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.hpp index 0682911bd9..01c9897d34 100644 --- a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.hpp +++ b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.hpp @@ -43,6 +43,7 @@ class ROSBAG2_STORAGE_DEFAULT_PLUGINS_PUBLIC SqliteWrapper ~SqliteWrapper(); SqliteStatement prepare_statement(const std::string & query); + std::string query_pragma_value(const std::string & key); size_t get_last_insert_id(); diff --git a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp index 1da61b443a..1dd6df1168 100644 --- a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp +++ b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp @@ -31,8 +31,9 @@ #include "rosbag2_storage/metadata_io.hpp" #include "rosbag2_storage/serialized_bag_message.hpp" -#include "rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.hpp" #include "rosbag2_storage_default_plugins/sqlite/sqlite_exception.hpp" +#include "rosbag2_storage_default_plugins/sqlite/sqlite_pragmas.hpp" +#include "rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.hpp" #ifdef _WIN32 // This is necessary because of a bug in yaml-cpp's cmake @@ -142,6 +143,17 @@ inline std::unordered_map parse_pragmas( return pragmas; } +void apply_resilient_storage_settings(std::unordered_map & pragmas) +{ + auto robust_pragmas = rosbag2_storage_plugins::SqlitePragmas::robust_writing_pragmas(); + for (const auto & kv : robust_pragmas) { + // do not override settings from configuration file, otherwise apply + if (pragmas.count(kv.first) == 0) { + pragmas[kv.first] = kv.second; + } + } +} + constexpr const auto FILE_EXTENSION = ".db3"; // Minimum size of a sqlite3 database file in bytes (84 kiB). @@ -161,7 +173,11 @@ void SqliteStorage::open( const rosbag2_storage::StorageOptions & storage_options, rosbag2_storage::storage_interfaces::IOFlag io_flag) { + const bool resilient_preset = "resilient" == storage_options.storage_preset_profile; auto pragmas = parse_pragmas(storage_options.storage_config_uri, io_flag); + if (resilient_preset && is_read_write(io_flag)) { + apply_resilient_storage_settings(pragmas); + } if (is_read_write(io_flag)) { relative_path_ = storage_options.uri + FILE_EXTENSION; @@ -459,6 +475,11 @@ void SqliteStorage::reset_filter() storage_filter_ = rosbag2_storage::StorageFilter(); } +std::string SqliteStorage::get_storage_setting(const std::string & key) +{ + return database_->query_pragma_value(key); +} + } // namespace rosbag2_storage_plugins #include "pluginlib/class_list_macros.hpp" // NOLINT diff --git a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.cpp b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.cpp index 96d11374e4..18fdadcd36 100644 --- a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.cpp +++ b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.cpp @@ -26,6 +26,7 @@ #include "rosbag2_storage/serialized_bag_message.hpp" #include "rosbag2_storage_default_plugins/sqlite/sqlite_exception.hpp" +#include "rosbag2_storage_default_plugins/sqlite/sqlite_pragmas.hpp" #include "../logging.hpp" @@ -83,16 +84,9 @@ void SqliteWrapper::apply_pragma_settings( { // Add default pragmas if not overridden by user setting { - typedef std::unordered_map pragmas_map_t; - pragmas_map_t default_pragmas = { - // Used to check whether db is readable - {"schema_version", "PRAGMA schema_version;"} - }; + auto default_pragmas = SqlitePragmas::default_pragmas(); if (io_flag == rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE) { - const pragmas_map_t write_default_pragmas = { - {"journal_mode", "PRAGMA journal_mode=WAL;"}, - {"synchronous", "PRAGMA synchronous=NORMAL;"} - }; + const auto write_default_pragmas = SqlitePragmas::optimized_writing_pragmas(); default_pragmas.insert(write_default_pragmas.begin(), write_default_pragmas.end()); } for (const auto & kv : default_pragmas) { @@ -117,6 +111,13 @@ void SqliteWrapper::apply_pragma_settings( } } +std::string SqliteWrapper::query_pragma_value(const std::string & key) +{ + auto query = "PRAGMA " + key + ";"; + auto pragma_value = prepare_statement(query)->execute_query().get_single_line(); + return std::get<0>(pragma_value); +} + SqliteStatement SqliteWrapper::prepare_statement(const std::string & query) { return std::make_shared(db_ptr, query); diff --git a/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/storage_test_fixture.hpp b/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/storage_test_fixture.hpp index a80c06ef2e..aaa12746c6 100644 --- a/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/storage_test_fixture.hpp +++ b/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/storage_test_fixture.hpp @@ -151,7 +151,7 @@ class StorageTestFixture : public TemporaryDirectoryFixture } rosbag2_storage::StorageOptions storage_options{storage_uri, plugin_id, 0, 0, 0, - yaml_config}; + "", yaml_config}; return storage_options; } diff --git a/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/test_sqlite_storage.cpp b/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/test_sqlite_storage.cpp index 34ed38e717..48fa548481 100644 --- a/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/test_sqlite_storage.cpp +++ b/rosbag2_storage_default_plugins/test/rosbag2_storage_default_plugins/sqlite/test_sqlite_storage.cpp @@ -322,6 +322,46 @@ TEST_F(StorageTestFixture, loads_config_file) { rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE)); } +TEST_F(StorageTestFixture, storage_configuration_file_applies_over_storage_preset_profile) { + // Check that "resilient" values are overriden + const auto journal_setting = "\"journal_mode = OFF\""; + const auto synchronous_setting = "\"synchronous = OFF\""; + const auto not_overriden_setting = "\"cache_size = 1337\""; + const auto overriding_yaml = std::string("write:\n pragmas: [") + + journal_setting + ", " + synchronous_setting + ", " + not_overriden_setting + "]\n"; + const auto writable_storage = std::make_unique(); + auto options = make_storage_options_with_config(overriding_yaml, kPluginID); + options.storage_preset_profile = "resilient"; + writable_storage->open(options, rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE); + + // configuration should replace preset "wal" with "off" + EXPECT_EQ(writable_storage->get_storage_setting("journal_mode"), "off"); + + // configuration should replace preset setting of 1 with 0 + EXPECT_EQ(writable_storage->get_storage_setting("synchronous"), "0"); + + // configuration of non-conflicting setting schema.cache_size should be unaffected + EXPECT_EQ(writable_storage->get_storage_setting("cache_size"), "1337"); +} + +TEST_F(StorageTestFixture, storage_preset_profile_applies_over_defaults) { + // Check that "resilient" values override default optimized ones + const auto writable_storage = std::make_unique(); + + auto temp_dir = rcpputils::fs::path(temporary_dir_path_); + const auto storage_uri = (temp_dir / "rosbag").string(); + rosbag2_storage::StorageOptions options{storage_uri, kPluginID, 0, 0, 0, "", ""}; + + options.storage_preset_profile = "resilient"; + writable_storage->open(options, rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE); + + // resilient preset should replace default "memory" with "wal" + EXPECT_EQ(writable_storage->get_storage_setting("journal_mode"), "wal"); + + // resilient preset should replace default of 0 with 1 + EXPECT_EQ(writable_storage->get_storage_setting("synchronous"), "1"); +} + TEST_F(StorageTestFixture, throws_on_invalid_pragma_in_config_file) { // Check that storage throws on invalid pragma statement in sqlite config const auto invalid_yaml = "write:\n pragmas: [\"unrecognized_pragma_name = 2\"]\n"; diff --git a/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp b/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp index afae2ee742..024038a64d 100644 --- a/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp +++ b/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -106,6 +105,7 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * "topics", "include_hidden_topics", "qos_profile_overrides", + "storage_preset_profile", "storage_config_file", nullptr}; @@ -126,10 +126,11 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * uint64_t max_cache_size = 0u; PyObject * topics = nullptr; bool include_hidden_topics = false; + char * storage_preset_profile = nullptr; char * storage_config_file = nullptr; if ( !PyArg_ParseTupleAndKeywords( - args, kwargs, "ssssss|KKbbKKKKObOs", const_cast(kwlist), + args, kwargs, "ssssss|KKbbKKKKObOss", const_cast(kwlist), &uri, &storage_id, &serilization_format, @@ -147,6 +148,7 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * &topics, &include_hidden_topics, &qos_profile_overrides, + &storage_preset_profile, &storage_config_file )) { @@ -159,6 +161,7 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * storage_options.max_bagfile_size = (uint64_t) max_bagfile_size; storage_options.max_bagfile_duration = static_cast(max_bagfile_duration); storage_options.max_cache_size = max_cache_size; + storage_options.storage_preset_profile = storage_preset_profile; record_options.all = all; record_options.is_discovery_disabled = no_discovery; record_options.topic_polling_interval = std::chrono::milliseconds(polling_interval_ms); From b7da49ee85cabc9c9ae244fe7aefe668788c9029 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 12 Jan 2021 13:52:04 -0300 Subject: [PATCH 61/77] include what you use (#600) Signed-off-by: Ivan Santiago Paunovic --- rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index 34b0e35192..d0d53e6291 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include From 3927519971bc3177d57f7c38def358100c0514de Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Thu, 21 Jan 2021 13:20:53 -0800 Subject: [PATCH 62/77] Add mjeronimo to codeowners to be auto-assigned in the review rotation (#608) Signed-off-by: Emerson Knapp --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4f6bdc3daa..602b63e262 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ # Reviewers (and by extension approvers) will be the default owners for everything in the repo. # Unless a later match takes precedence, @reviewers will be requested for review when someone opens a pull request. -* @ros2/tooling-reviewers +* @ros2/tooling-reviewers @mjeronimo From b26a9a011881e53f2905909ee18eb70768e885fe Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Thu, 21 Jan 2021 14:10:52 -0800 Subject: [PATCH 63/77] Fix the tests on cyclonedds by translating qos duration values (#606) * Change all 'fastrtps-unspecified' duration values in test resources to 0 for cross-implementation readability, pending a clarification of the rmw qos duration api Signed-off-by: Emerson Knapp --- README.md | 15 +++++++++------ ros2bag/test/resources/empty_bag/metadata.yaml | 4 ++-- .../test/resources/incomplete_qos_duration.yaml | 2 +- ros2bag/test/resources/qos_profile.yaml | 15 +++++++++------ rosbag2_py/resources/talker/metadata.yaml | 6 +++--- rosbag2_tests/resources/cdr_test/metadata.yaml | 4 ++-- .../resources/wrong_rmw_test/metadata.yaml | 4 ++-- 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index f86ba25b6f..0f46463286 100644 --- a/README.md +++ b/README.md @@ -187,15 +187,18 @@ Below is an example profile set to the default ROS2 QoS settings. reliability: reliable durability: volatile deadline: - sec: 2147483647 # LONG_MAX - nsec: 4294967295 # ULONG_MAX + # unspecified/infinity + sec: 0 + nsec: 0 lifespan: - sec: 2147483647 - nsec: 4294967295 + # unspecified/infinity + sec: 0 + nsec: 0 liveliness: system_default liveliness_lease_duration: - sec: 2147483647 - nsec: 4294967295 + # unspecified/infinity + sec: 0 + nsec: 0 avoid_ros_namespace_conventions: false ``` diff --git a/ros2bag/test/resources/empty_bag/metadata.yaml b/ros2bag/test/resources/empty_bag/metadata.yaml index bf90464011..896d33fc03 100644 --- a/ros2bag/test/resources/empty_bag/metadata.yaml +++ b/ros2bag/test/resources/empty_bag/metadata.yaml @@ -13,13 +13,13 @@ rosbag2_bagfile_information: name: /parameter_events type: rcl_interfaces/msg/ParameterEvent serialization_format: cdr - offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 0\n nsec: 0\n lifespan:\n sec: 0\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 0\n nsec: 0\n avoid_ros_namespace_conventions: false" message_count: 0 - topic_metadata: name: /rosout type: rcl_interfaces/msg/Log serialization_format: cdr - offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 1\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 10\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 1\n deadline:\n sec: 0\n nsec: 0\n lifespan:\n sec: 10\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 0\n nsec: 0\n avoid_ros_namespace_conventions: false" message_count: 0 compression_format: "" compression_mode: "" diff --git a/ros2bag/test/resources/incomplete_qos_duration.yaml b/ros2bag/test/resources/incomplete_qos_duration.yaml index adc670003d..adf88721e1 100644 --- a/ros2bag/test/resources/incomplete_qos_duration.yaml +++ b/ros2bag/test/resources/incomplete_qos_duration.yaml @@ -4,4 +4,4 @@ reliability: reliable durability: volatile deadline: - sec: 2147483647 # LONG_MAX + sec: 2 diff --git a/ros2bag/test/resources/qos_profile.yaml b/ros2bag/test/resources/qos_profile.yaml index 4ec1917cf4..9daf5fdf5f 100644 --- a/ros2bag/test/resources/qos_profile.yaml +++ b/ros2bag/test/resources/qos_profile.yaml @@ -4,13 +4,16 @@ reliability: reliable durability: transient_local deadline: - sec: 2147483647 - nsec: 4294967295 + # unspecified/infinity + sec: 0 + nsec: 0 lifespan: - sec: 2147483647 - nsec: 4294967295 + # unspecified/infinity + sec: 0 + nsec: 0 liveliness: automatic liveliness_lease_duration: - sec: 2147483647 - nsec: 4294967295 + # unspecified/infinity + sec: 0 + nsec: 0 avoid_ros_namespace_conventions: false diff --git a/rosbag2_py/resources/talker/metadata.yaml b/rosbag2_py/resources/talker/metadata.yaml index 35db28401c..da247440fc 100644 --- a/rosbag2_py/resources/talker/metadata.yaml +++ b/rosbag2_py/resources/talker/metadata.yaml @@ -13,19 +13,19 @@ rosbag2_bagfile_information: name: /topic type: std_msgs/msg/String serialization_format: cdr - offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 0\n nsec: 0\n lifespan:\n sec: 0\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 0\n nsec: 0\n avoid_ros_namespace_conventions: false" message_count: 10 - topic_metadata: name: /rosout type: rcl_interfaces/msg/Log serialization_format: cdr - offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 1\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 10\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 1\n deadline:\n sec: 0\n nsec: 0\n lifespan:\n sec: 10\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 0\n nsec: 0\n avoid_ros_namespace_conventions: false" message_count: 10 - topic_metadata: name: /parameter_events type: rcl_interfaces/msg/ParameterEvent serialization_format: cdr - offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 0\n nsec: 0\n lifespan:\n sec: 0\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 0\n nsec: 0\n avoid_ros_namespace_conventions: false" message_count: 0 compression_format: "" compression_mode: "" diff --git a/rosbag2_tests/resources/cdr_test/metadata.yaml b/rosbag2_tests/resources/cdr_test/metadata.yaml index dc50cb4111..6a8fed7f2f 100644 --- a/rosbag2_tests/resources/cdr_test/metadata.yaml +++ b/rosbag2_tests/resources/cdr_test/metadata.yaml @@ -13,13 +13,13 @@ rosbag2_bagfile_information: name: /test_topic type: test_msgs/msg/BasicTypes serialization_format: cdr - offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 0\n nsec: 0\n lifespan:\n sec: 0\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 0\n nsec: 0\n avoid_ros_namespace_conventions: false" message_count: 3 - topic_metadata: name: /array_topic type: test_msgs/msg/Arrays serialization_format: cdr - offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 0\n nsec: 0\n lifespan:\n sec: 0\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 0\n nsec: 0\n avoid_ros_namespace_conventions: false" message_count: 4 compression_format: "" compression_mode: "" diff --git a/rosbag2_tests/resources/wrong_rmw_test/metadata.yaml b/rosbag2_tests/resources/wrong_rmw_test/metadata.yaml index b4fe352e34..47fc7ab0ae 100644 --- a/rosbag2_tests/resources/wrong_rmw_test/metadata.yaml +++ b/rosbag2_tests/resources/wrong_rmw_test/metadata.yaml @@ -13,13 +13,13 @@ rosbag2_bagfile_information: name: /test_topic type: test_msgs/msg/BasicTypes serialization_format: wrong_format - offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 0\n nsec: 0\n lifespan:\n sec: 0\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 0\n nsec: 0\n avoid_ros_namespace_conventions: false" message_count: 3 - topic_metadata: name: /array_topic type: test_msgs/msg/Arrays serialization_format: wrong_format - offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 0\n nsec: 0\n lifespan:\n sec: 0\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 0\n nsec: 0\n avoid_ros_namespace_conventions: false" message_count: 4 compression_format: "" compression_mode: "" From 1dfb13ff30fab54c37a81fedc09a6360a80e374e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C4=85browski?= Date: Fri, 22 Jan 2021 23:54:45 +0100 Subject: [PATCH 64/77] Mutex protection for db writing and stl collections in writer & storage (#603) * Mutex-protected writes and topic creation/removal in sqlite_storage Signed-off-by: Adam Dabrowski --- .../sqlite/sqlite_storage.hpp | 13 +++++++++++-- .../sqlite/sqlite_storage.cpp | 12 +++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_storage.hpp b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_storage.hpp index fd16b8842b..ff22ec24cc 100644 --- a/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_storage.hpp +++ b/rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/sqlite/sqlite_storage.hpp @@ -17,10 +17,12 @@ #include #include +#include #include #include #include +#include "rcpputils/thread_safety_annotations.hpp" #include "rcutils/types.h" #include "rosbag2_storage/storage_interfaces/read_write_interface.hpp" #include "rosbag2_storage/serialized_bag_message.hpp" @@ -92,21 +94,28 @@ class ROSBAG2_STORAGE_DEFAULT_PLUGINS_PUBLIC SqliteStorage void fill_topics_and_types(); void activate_transaction(); void commit_transaction(); + void write_locked(std::shared_ptr message) + RCPPUTILS_TSA_REQUIRES(database_write_mutex_); using ReadQueryResult = SqliteStatementWrapper::QueryResult< std::shared_ptr, rcutils_time_point_value_t, std::string>; - std::shared_ptr database_; + std::shared_ptr database_ RCPPUTILS_TSA_GUARDED_BY(database_write_mutex_); SqliteStatement write_statement_ {}; SqliteStatement read_statement_ {}; ReadQueryResult message_result_ {nullptr}; ReadQueryResult::Iterator current_message_row_ { nullptr, SqliteStatementWrapper::QueryResult<>::Iterator::POSITION_END}; - std::unordered_map topics_; + std::unordered_map topics_ RCPPUTILS_TSA_GUARDED_BY(database_write_mutex_); std::vector all_topics_and_types_; std::string relative_path_; std::atomic_bool active_transaction_ {false}; rosbag2_storage::StorageFilter storage_filter_ {}; + + // This mutex is necessary to protect: + // a) database access (this could also be done with FULLMUTEX), but see b) + // b) topics_ collection - since we could be writing and reading it at the same time + std::mutex database_write_mutex_; }; } // namespace rosbag2_storage_plugins diff --git a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp index 1dd6df1168..1af8a405ce 100644 --- a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp +++ b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp @@ -242,6 +242,13 @@ void SqliteStorage::commit_transaction() } void SqliteStorage::write(std::shared_ptr message) +{ + std::lock_guard db_lock(database_write_mutex_); + write_locked(message); +} + +void SqliteStorage::write_locked( + std::shared_ptr message) { if (!write_statement_) { prepare_for_writing(); @@ -260,6 +267,7 @@ void SqliteStorage::write(std::shared_ptr> & messages) { + std::lock_guard db_lock(database_write_mutex_); if (!write_statement_) { prepare_for_writing(); } @@ -267,7 +275,7 @@ void SqliteStorage::write( activate_transaction(); for (auto & message : messages) { - write(message); + write_locked(message); } commit_transaction(); @@ -334,6 +342,7 @@ void SqliteStorage::initialize() void SqliteStorage::create_topic(const rosbag2_storage::TopicMetadata & topic) { + std::lock_guard db_lock(database_write_mutex_); if (topics_.find(topic.name) == std::end(topics_)) { auto insert_topic = database_->prepare_statement( @@ -348,6 +357,7 @@ void SqliteStorage::create_topic(const rosbag2_storage::TopicMetadata & topic) void SqliteStorage::remove_topic(const rosbag2_storage::TopicMetadata & topic) { + std::lock_guard db_lock(database_write_mutex_); if (topics_.find(topic.name) != std::end(topics_)) { auto delete_topic = database_->prepare_statement( From 87e68f094e4cd2cd9b68fbb3bc4a8fd7a3561f46 Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Mon, 25 Jan 2021 13:40:57 -0800 Subject: [PATCH 65/77] Remove mjeronimo from CODEOWNERS in favor of having him on the reviewing team (#614) Signed-off-by: Emerson Knapp --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 602b63e262..4f6bdc3daa 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ # Reviewers (and by extension approvers) will be the default owners for everything in the repo. # Unless a later match takes precedence, @reviewers will be requested for review when someone opens a pull request. -* @ros2/tooling-reviewers @mjeronimo +* @ros2/tooling-reviewers From f9c54ab36e0adc8f5f4bc93fbdd362de2a0cf307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C4=85browski?= Date: Tue, 26 Jan 2021 00:01:30 +0100 Subject: [PATCH 66/77] Recorder --regex and --exclude options (#604) * Regex and exclude options for recording topics - you can use -e or --regex option now to specify how topics are recorded - can use -x or --exclude to exclude topics from recording - regex is exclusive with -a and specifying topics (for simplicity) Signed-off-by: Adam Dabrowski --- ros2bag/ros2bag/verb/record.py | 29 +++++-- .../test_rosbag2_record_end_to_end.cpp | 4 +- rosbag2_transport/CMakeLists.txt | 6 ++ .../rosbag2_transport/record_options.hpp | 2 + .../src/rosbag2_transport/recorder.cpp | 49 +++++++---- .../src/rosbag2_transport/recorder.hpp | 9 +- .../rosbag2_transport_python.cpp | 10 ++- .../rosbag2_transport/test_record_regex.cpp | 86 +++++++++++++++++++ 8 files changed, 161 insertions(+), 34 deletions(-) create mode 100644 rosbag2_transport/test/rosbag2_transport/test_record_regex.cpp diff --git a/ros2bag/ros2bag/verb/record.py b/ros2bag/ros2bag/verb/record.py index d8aa543596..a9b57c172b 100644 --- a/ros2bag/ros2bag/verb/record.py +++ b/ros2bag/ros2bag/verb/record.py @@ -30,9 +30,17 @@ class RecordVerb(VerbExtension): def add_arguments(self, parser, cli_name): # noqa: D102 parser.add_argument( '-a', '--all', action='store_true', - help='recording all topics, required if no topics are listed explicitly.') + help='recording all topics, required if no topics ' + 'are listed explicitly or through a regex') parser.add_argument( 'topics', nargs='*', default=None, help='topics to be recorded') + parser.add_argument( + '-e', '--regex', default='', help='recording only topics ' + 'matching provided regular expression') + parser.add_argument( + '-x', '--exclude', default='', help='exclude topics ' + 'matching provided regular expression. Works with -a and -e, ' + 'subtracting excluded topics') parser.add_argument( '-o', '--output', help='destination of the bagfile to create, \ @@ -122,11 +130,18 @@ def add_arguments(self, parser, cli_name): # noqa: D102 def main(self, *, args): # noqa: D102 # both all and topics cannot be true - if args.all and args.topics: - return print_error('Invalid choice: Can not specify topics and -a at the same time.') - # both all and topics cannot be false - if not(args.all or (args.topics and len(args.topics) > 0)): - return print_error('Invalid choice: Must specify topic(s) or -a') + if (args.all and (args.topics or args.regex)) or (args.topics and args.regex): + return print_error('Must specify only one option out of topics, --regex or --all') + # one out of "all", "topics" and "regex" must be true + if not(args.all or (args.topics and len(args.topics) > 0) or (args.regex)): + return print_error('Invalid choice: Must specify topic(s), --regex or --all') + + if args.topics and args.exclude: + return print_error('--exclude argument cannot be used when specifying a list ' + 'of topics explicitly') + + if args.exclude and not(args.regex or args.all): + return print_error('--exclude argument requires either --all or --regex') uri = args.output or datetime.datetime.now().strftime('rosbag2_%Y_%m_%d-%H_%M_%S') @@ -178,6 +193,8 @@ def main(self, *, args): # noqa: D102 max_bagfile_duration=args.max_bag_duration, max_cache_size=args.max_cache_size, topics=args.topics, + regex=args.regex, + exclude=args.exclude, include_hidden_topics=args.include_hidden_topics, qos_profile_overrides=qos_profile_overrides, storage_preset_profile=args.storage_preset_profile, diff --git a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp index 8d6b4f8394..93c7461da0 100644 --- a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp +++ b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp @@ -565,7 +565,7 @@ TEST_F(RecordFixture, record_fails_if_both_all_and_topic_list_is_specified) { auto error_output = internal::GetCapturedStderr(); EXPECT_THAT(exit_code, Eq(EXIT_FAILURE)); - EXPECT_THAT(error_output, HasSubstr("Can not specify topics and -a at the same time.")); + EXPECT_FALSE(error_output.empty()); } TEST_F(RecordFixture, record_fails_if_neither_all_nor_topic_list_are_specified) { @@ -575,7 +575,7 @@ TEST_F(RecordFixture, record_fails_if_neither_all_nor_topic_list_are_specified) auto output = internal::GetCapturedStderr(); EXPECT_THAT(exit_code, Eq(EXIT_FAILURE)); - EXPECT_THAT(output, HasSubstr("Invalid choice: Must specify topic(s) or -a")); + EXPECT_FALSE(output.empty()); } TEST_F(RecordFixture, record_fails_gracefully_if_plugin_for_given_encoding_does_not_exist) { diff --git a/rosbag2_transport/CMakeLists.txt b/rosbag2_transport/CMakeLists.txt index e4b0dbd284..ccc0d0f07b 100644 --- a/rosbag2_transport/CMakeLists.txt +++ b/rosbag2_transport/CMakeLists.txt @@ -161,6 +161,12 @@ function(create_tests_for_rmw_implementation) LINK_LIBS rosbag2_transport ${SKIP_TEST}) + rosbag2_transport_add_gmock(test_record_regex + test/rosbag2_transport/test_record_regex.cpp + LINK_LIBS rosbag2_transport + AMENT_DEPS test_msgs rosbag2_test_common + ${SKIP_TEST}) + rosbag2_transport_add_gmock(test_play src/rosbag2_transport/qos.cpp test/rosbag2_transport/test_play.cpp diff --git a/rosbag2_transport/include/rosbag2_transport/record_options.hpp b/rosbag2_transport/include/rosbag2_transport/record_options.hpp index bc87828251..706c90454e 100644 --- a/rosbag2_transport/include/rosbag2_transport/record_options.hpp +++ b/rosbag2_transport/include/rosbag2_transport/record_options.hpp @@ -32,6 +32,8 @@ struct RecordOptions std::vector topics; std::string rmw_serialization_format; std::chrono::milliseconds topic_polling_interval; + std::string regex = ""; + std::string exclude = ""; std::string node_prefix = ""; std::string compression_mode = ""; std::string compression_format = ""; diff --git a/rosbag2_transport/src/rosbag2_transport/recorder.cpp b/rosbag2_transport/src/rosbag2_transport/recorder.cpp index d7e7d843de..e2f267757d 100644 --- a/rosbag2_transport/src/rosbag2_transport/recorder.cpp +++ b/rosbag2_transport/src/rosbag2_transport/recorder.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -57,16 +58,13 @@ void Recorder::record(const RecordOptions & record_options) } serialization_format_ = record_options.rmw_serialization_format; ROSBAG2_TRANSPORT_LOG_INFO("Listening for topics..."); - subscribe_topics( - get_requested_or_available_topics(record_options.topics, record_options.include_hidden_topics)); + subscribe_topics(get_requested_or_available_topics(record_options)); std::future discovery_future; if (!record_options.is_discovery_disabled) { auto discovery = std::bind( &Recorder::topics_discovery, this, - record_options.topic_polling_interval, - record_options.topics, - record_options.include_hidden_topics); + record_options); discovery_future = std::async(std::launch::async, discovery); } @@ -79,36 +77,51 @@ void Recorder::record(const RecordOptions & record_options) subscriptions_.clear(); } -void Recorder::topics_discovery( - std::chrono::milliseconds topic_polling_interval, - const std::vector & requested_topics, - bool include_hidden_topics) +void Recorder::topics_discovery(const RecordOptions & record_options) { while (rclcpp::ok()) { auto topics_to_subscribe = - get_requested_or_available_topics(requested_topics, include_hidden_topics); + get_requested_or_available_topics(record_options); for (const auto & topic_and_type : topics_to_subscribe) { warn_if_new_qos_for_subscribed_topic(topic_and_type.first); } auto missing_topics = get_missing_topics(topics_to_subscribe); subscribe_topics(missing_topics); - if (!requested_topics.empty() && subscriptions_.size() == requested_topics.size()) { + if (!record_options.topics.empty() && subscriptions_.size() == record_options.topics.size()) { ROSBAG2_TRANSPORT_LOG_INFO("All requested topics are subscribed. Stopping discovery..."); return; } - std::this_thread::sleep_for(topic_polling_interval); + std::this_thread::sleep_for(record_options.topic_polling_interval); } } std::unordered_map -Recorder::get_requested_or_available_topics( - const std::vector & requested_topics, - bool include_hidden_topics) +Recorder::get_requested_or_available_topics(const RecordOptions & record_options) { - return requested_topics.empty() ? - node_->get_all_topics_with_types(include_hidden_topics) : - node_->get_topics_with_types(requested_topics); + auto unfiltered_topics = record_options.topics.empty() ? + node_->get_all_topics_with_types(record_options.include_hidden_topics) : + node_->get_topics_with_types(record_options.topics); + + if (record_options.regex.empty() && record_options.exclude.empty()) { + return unfiltered_topics; + } + + std::unordered_map filtered_by_regex; + + std::regex topic_regex(record_options.regex); + std::regex exclude_regex(record_options.exclude); + bool take = record_options.all; + for (const auto & kv : unfiltered_topics) { + take = std::regex_search(kv.first, topic_regex); + if (take) { + take = !std::regex_search(kv.first, exclude_regex); + } + if (take) { + filtered_by_regex.insert(kv); + } + } + return filtered_by_regex; } std::unordered_map diff --git a/rosbag2_transport/src/rosbag2_transport/recorder.hpp b/rosbag2_transport/src/rosbag2_transport/recorder.hpp index 6b1976adae..4432f1d677 100644 --- a/rosbag2_transport/src/rosbag2_transport/recorder.hpp +++ b/rosbag2_transport/src/rosbag2_transport/recorder.hpp @@ -62,15 +62,10 @@ class Recorder } private: - void topics_discovery( - std::chrono::milliseconds topic_polling_interval, - const std::vector & requested_topics = {}, - bool include_hidden_topics = false); + void topics_discovery(const RecordOptions & record_options); std::unordered_map - get_requested_or_available_topics( - const std::vector & requested_topics, - bool include_hidden_topics = false); + get_requested_or_available_topics(const RecordOptions & record_options); std::unordered_map get_missing_topics(const std::unordered_map & all_topics); diff --git a/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp b/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp index 024038a64d..8349dd9149 100644 --- a/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp +++ b/rosbag2_transport/src/rosbag2_transport/rosbag2_transport_python.cpp @@ -103,6 +103,8 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * "max_bagfile_duration", "max_cache_size", "topics", + "regex", + "exclude", "include_hidden_topics", "qos_profile_overrides", "storage_preset_profile", @@ -125,12 +127,14 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * unsigned long long max_bagfile_duration = 0; // NOLINT uint64_t max_cache_size = 0u; PyObject * topics = nullptr; + char * regex = nullptr; + char * exclude = nullptr; bool include_hidden_topics = false; char * storage_preset_profile = nullptr; char * storage_config_file = nullptr; if ( !PyArg_ParseTupleAndKeywords( - args, kwargs, "ssssss|KKbbKKKKObOss", const_cast(kwlist), + args, kwargs, "ssssss|KKbbKKKKOssbOss", const_cast(kwlist), &uri, &storage_id, &serilization_format, @@ -146,6 +150,8 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * &max_bagfile_duration, &max_cache_size, &topics, + ®ex, + &exclude, &include_hidden_topics, &qos_profile_overrides, &storage_preset_profile, @@ -163,6 +169,8 @@ rosbag2_transport_record(PyObject * Py_UNUSED(self), PyObject * args, PyObject * storage_options.max_cache_size = max_cache_size; storage_options.storage_preset_profile = storage_preset_profile; record_options.all = all; + record_options.regex = regex; + record_options.exclude = exclude; record_options.is_discovery_disabled = no_discovery; record_options.topic_polling_interval = std::chrono::milliseconds(polling_interval_ms); record_options.node_prefix = std::string(node_prefix); diff --git a/rosbag2_transport/test/rosbag2_transport/test_record_regex.cpp b/rosbag2_transport/test/rosbag2_transport/test_record_regex.cpp new file mode 100644 index 0000000000..e80fa3b740 --- /dev/null +++ b/rosbag2_transport/test/rosbag2_transport/test_record_regex.cpp @@ -0,0 +1,86 @@ +// Copyright 2021, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include + +#include "test_msgs/msg/arrays.hpp" +#include "test_msgs/msg/basic_types.hpp" +#include "test_msgs/message_fixtures.hpp" + +#include "record_integration_fixture.hpp" + +TEST_F(RecordIntegrationTestFixture, only_regex_matching_topics_are_recorded) +{ + auto test_string_messages = get_messages_strings(); + auto test_array_messages = get_messages_arrays(); + std::string regex = "/[a-z]+_nice(_.*)"; + std::string regex_exclude = "/[a-z]+_nice_[a-z]+/(.*)"; + + // matching topics - the only ones that should be recorded + std::string v1 = "/awesome_nice_topic"; + std::string v2 = "/still_nice_topic"; + + // excluded topics + std::string e1 = "/quite_nice_namespace/but_it_is_excluded"; + + // topics that shouldn't match + std::string b1 = "/numberslike1arenot_nice"; + std::string b2 = "/namespace_before/not_nice"; + + // checking the test data itself + std::regex re(regex); + std::regex exclude(regex_exclude); + ASSERT_TRUE(std::regex_search(v1, re)); + ASSERT_TRUE(std::regex_search(v2, re)); + ASSERT_FALSE(std::regex_search(b1, re)); + ASSERT_FALSE(std::regex_search(b2, re)); + + // this example matches both regexes - should be excluded + ASSERT_TRUE(std::regex_search(e1, re)); + ASSERT_TRUE(std::regex_search(e1, exclude)); + + RecordOptions ro{false, false, {}, "rmw_format", 10ms}; + ro.regex = regex; + ro.exclude = regex_exclude; + + start_recording(ro); + + pub_man_.add_publisher( + v1, test_string_messages[0], 0); + pub_man_.add_publisher( + v2, test_string_messages[1], 0); + + pub_man_.add_publisher( + b1, test_string_messages[0], 0); + pub_man_.add_publisher( + b2, test_string_messages[1], 0); + + pub_man_.add_publisher( + e1, test_array_messages[0], 0); + + run_publishers(); + stop_recording(); + + MockSequentialWriter & writer = + static_cast(writer_->get_implementation_handle()); + auto recorded_topics = writer.get_topics(); + + EXPECT_THAT(recorded_topics, SizeIs(2)); + + EXPECT_TRUE(recorded_topics.find(v1) != recorded_topics.end()); + EXPECT_TRUE(recorded_topics.find(v2) != recorded_topics.end()); +} From f0dedb0883a24a762117c5eeee3ebf1d0e2dacac Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Mon, 25 Jan 2021 15:03:58 -0800 Subject: [PATCH 67/77] Deduplicate SequentialCompressionReader business logic, add fallback to find bagfiles in incorrectly-written metadata (#612) * Deduplicate sequentialcompressionreader business logic, add fallback to find bagfiles in incorrectly-written metadata Signed-off-by: Emerson Knapp --- .../sequential_compression_reader.hpp | 17 +- .../sequential_compression_reader.cpp | 116 +++++-------- .../test_sequential_compression_reader.cpp | 164 +++++++++++++----- .../rosbag2_cpp/readers/sequential_reader.hpp | 9 + .../rosbag2_cpp/readers/sequential_reader.cpp | 4 + 5 files changed, 185 insertions(+), 125 deletions(-) diff --git a/rosbag2_compression/include/rosbag2_compression/sequential_compression_reader.hpp b/rosbag2_compression/include/rosbag2_compression/sequential_compression_reader.hpp index 2e8251b25d..2acd9f631c 100644 --- a/rosbag2_compression/include/rosbag2_compression/sequential_compression_reader.hpp +++ b/rosbag2_compression/include/rosbag2_compression/sequential_compression_reader.hpp @@ -70,22 +70,21 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionReader protected: /** - * Increment the current file iterator to point to the next file in the list of relative file - * paths. - * - * Expected usage: - * if (has_next_file()) load_next_file(); + * Decompress the current bagfile so that it can be opened by the storage implementation. */ - void load_next_file() override; + void preprocess_current_file() override; +private: /** * Initializes the decompressor if a compression mode is specified in the metadata. * - * \throws std::invalid_argument If compression format doesn't exist. + * \throw std::invalid_argument If compression mode is NONE + * \throw std::invalid_argument If compression format could not be found + * \throw rcpputils::IllegalStateException if the decompressor could not be initialized for + * any other reason */ - virtual void setup_decompression(); + void setup_decompression(); -private: std::unique_ptr decompressor_{}; rosbag2_compression::CompressionMode compression_mode_{ rosbag2_compression::CompressionMode::NONE}; diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp index 8d1adae926..7b823409fd 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp @@ -20,6 +20,7 @@ #include #include +#include "rcpputils/asserts.hpp" #include "rcpputils/filesystem_helper.hpp" #include "rosbag2_compression/compression_options.hpp" @@ -43,18 +44,49 @@ SequentialCompressionReader::~SequentialCompressionReader() void SequentialCompressionReader::setup_decompression() { - compression_mode_ = rosbag2_compression::compression_mode_from_string(metadata_.compression_mode); - if (compression_mode_ != rosbag2_compression::CompressionMode::NONE) { - decompressor_ = compression_factory_->create_decompressor(metadata_.compression_format); - // Decompress the first file so that it is readable; don't need to do anything for - // per-message encryption. - if (compression_mode_ == CompressionMode::FILE) { - ROSBAG2_COMPRESSION_LOG_DEBUG_STREAM("Decompressing " << get_current_file().c_str()); - *current_file_iterator_ = decompressor_->decompress_uri(get_current_file()); + if (decompressor_) { + return; + } + + compression_mode_ = compression_mode_from_string(metadata_.compression_mode); + rcpputils::require_true( + compression_mode_ != rosbag2_compression::CompressionMode::NONE, + "SequentialCompressionReader should not be initialized with NONE compression mode."); + + decompressor_ = compression_factory_->create_decompressor(metadata_.compression_format); + rcpputils::check_true(decompressor_ != nullptr, "Couldn't initialize decompressor."); +} + +void SequentialCompressionReader::preprocess_current_file() +{ + setup_decompression(); + + if (metadata_.version == 4) { + /* + * Rosbag2 was released with incorrect relative file naming for compressed bags + * which were written as v4, using v3 logic which had the bag name prefixed on the file path. + * Because we have no way to check whether the bag was written correctly, + * check for the existence of the prefixed file as a fallback. + */ + const rcpputils::fs::path base{base_folder_}; + const rcpputils::fs::path relative{get_current_file()}; + const auto resolved = base / relative; + if (!resolved.exists()) { + const auto base_stripped = relative.filename(); + const auto resolved_stripped = base / base_stripped; + ROSBAG2_COMPRESSION_LOG_DEBUG_STREAM( + "Unable to find specified bagfile " << resolved.string() << + ". Falling back to checking for " << resolved_stripped.string()); + rcpputils::require_true( + resolved_stripped.exists(), + "Unable to resolve relative file path either as a V3 or V4 relative path"); + *current_file_iterator_ = resolved_stripped.string(); } - } else { - throw std::invalid_argument{ - "SequentialCompressionReader requires a CompressionMode that is not NONE!"}; + } + + if (compression_mode_ == CompressionMode::FILE) { + ROSBAG2_COMPRESSION_LOG_INFO_STREAM("Decompressing " << get_current_file().c_str()); + *current_file_iterator_ = decompressor_->decompress_uri(get_current_file()); } } @@ -62,45 +94,13 @@ void SequentialCompressionReader::open( const rosbag2_storage::StorageOptions & storage_options, const rosbag2_cpp::ConverterOptions & converter_options) { - storage_options_ = storage_options; - - if (metadata_io_->metadata_file_exists(storage_options.uri)) { - metadata_ = metadata_io_->read_metadata(storage_options.uri); - if (metadata_.relative_file_paths.empty()) { - ROSBAG2_COMPRESSION_LOG_WARN("No file paths were found in metadata."); - return; - } - file_paths_ = metadata_.relative_file_paths; - current_file_iterator_ = file_paths_.begin(); - setup_decompression(); - - storage_options_.uri = *current_file_iterator_; - storage_ = storage_factory_->open_read_only(storage_options); - if (!storage_) { - std::stringstream errmsg; - errmsg << "No storage could be initialized for: \"" << - storage_options.uri << "\"."; - - throw std::runtime_error{errmsg.str()}; - } - } else { + if (!metadata_io_->metadata_file_exists(storage_options.uri)) { std::stringstream errmsg; errmsg << "Could not find metadata for bag: \"" << storage_options.uri << - "\". Legacy bag files are not supported if this is a ROS 1 bag file."; + "\". Bags without metadata (such as from ROS 1) not supported by rosbag2 decompression."; throw std::runtime_error{errmsg.str()}; } - const auto & topics = metadata_.topics_with_message_count; - if (topics.empty()) { - ROSBAG2_COMPRESSION_LOG_WARN("No topics were listed in metadata."); - return; - } - fill_topics_metadata(); - - // Currently a bag file can only be played if all topics have the same serialization format. - check_topics_serialization_formats(topics); - check_converter_serialization_format( - converter_options.output_serialization_format, - topics[0].topic_metadata.serialization_format); + SequentialReader::open(storage_options, converter_options); } std::shared_ptr SequentialCompressionReader::read_next() @@ -115,28 +115,4 @@ std::shared_ptr SequentialCompressionRead throw std::runtime_error{"Bag is not open. Call open() before reading."}; } - -void SequentialCompressionReader::load_next_file() -{ - if (current_file_iterator_ == file_paths_.end()) { - throw std::runtime_error{"Cannot load next file; already on last file!"}; - } - - if (compression_mode_ == rosbag2_compression::CompressionMode::NONE) { - throw std::runtime_error{"Cannot use SequentialCompressionReader with NONE compression mode."}; - } - - ++current_file_iterator_; - if (compression_mode_ == rosbag2_compression::CompressionMode::FILE) { - if (decompressor_ == nullptr) { - throw std::runtime_error{ - "The bag file was not properly opened. " - "Somehow the compression mode was set without opening a decompressor." - }; - } - - ROSBAG2_COMPRESSION_LOG_DEBUG_STREAM("Decompressing " << get_current_file().c_str()); - *current_file_iterator_ = decompressor_->decompress_uri(get_current_file()); - } -} } // namespace rosbag2_compression diff --git a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_reader.cpp b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_reader.cpp index ae6ace1efe..1e478ca8fb 100644 --- a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_reader.cpp +++ b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_reader.cpp @@ -14,11 +14,14 @@ #include +#include #include #include #include #include +#include "rcpputils/filesystem_helper.hpp" + #include "rosbag2_compression/sequential_compression_reader.hpp" #include "rosbag2_cpp/reader.hpp" @@ -37,27 +40,44 @@ class SequentialCompressionReaderTest : public Test { public: SequentialCompressionReaderTest() - : storage_factory_{std::make_unique>()}, + : storage_factory_{std::make_unique>()}, storage_{std::make_shared>()}, converter_factory_{std::make_shared>()}, metadata_io_{std::make_unique>()}, - storage_serialization_format_{"rmw1_format"} + storage_serialization_format_{"rmw1_format"}, + tmp_dir_{rcpputils::fs::temp_directory_path() / bag_name_}, + converter_options_{"", storage_serialization_format_} { + rcpputils::fs::remove_all(tmp_dir_); + storage_options_.uri = tmp_dir_.string(); topic_with_type_ = rosbag2_storage::TopicMetadata{ "topic", "test_msgs/BasicTypes", storage_serialization_format_, ""}; auto topics_and_types = std::vector{topic_with_type_}; auto message = std::make_shared(); message->topic_name = topic_with_type_.name; + metadata_ = construct_default_bag_metadata(); + ON_CALL(*metadata_io_, read_metadata).WillByDefault( + [this](auto /*uri*/) { + return metadata_; + }); + ON_CALL(*metadata_io_, metadata_file_exists(_)).WillByDefault(Return(true)); + ON_CALL(*storage_, get_all_topics_and_types()).WillByDefault(Return(topics_and_types)); ON_CALL(*storage_, read_next()).WillByDefault(Return(message)); ON_CALL(*storage_factory_, open_read_only(_)).WillByDefault(Return(storage_)); + + initialize_dummy_storage_files(); } rosbag2_storage::BagMetadata construct_default_bag_metadata() const { rosbag2_storage::BagMetadata metadata; - metadata.relative_file_paths = {"/path/to/storage"}; + metadata.version = 4; + metadata.relative_file_paths = { + "bagfile_0.zstd", + "bagfile_1.zstd" + }; metadata.topics_with_message_count.push_back({{topic_with_type_}, 1}); metadata.compression_format = "zstd"; metadata.compression_mode = @@ -65,21 +85,54 @@ class SequentialCompressionReaderTest : public Test return metadata; } - std::unique_ptr> storage_factory_; - std::shared_ptr> storage_; - std::shared_ptr> converter_factory_; - std::unique_ptr> metadata_io_; + void initialize_dummy_storage_files() + { + // Initialize some dummy files so that they can be found + rcpputils::fs::create_directories(tmp_dir_); + for (auto relative : metadata_.relative_file_paths) { + std::ofstream output((tmp_dir_ / relative).string()); + output << "Fake storage data" << std::endl; + } + } + + std::unique_ptr create_reader() + { + auto decompressor = std::make_unique>(); + ON_CALL(*decompressor, decompress_uri).WillByDefault( + [](auto uri) { + auto path = rcpputils::fs::path(uri); + EXPECT_TRUE(path.exists()); + return rcpputils::fs::remove_extension(path).string(); + }); + auto compression_factory = std::make_unique>(); + ON_CALL(*compression_factory, create_decompressor(_)) + .WillByDefault(Return(ByMove(std::move(decompressor)))); + return std::make_unique( + std::move(compression_factory), + std::move(storage_factory_), + converter_factory_, + std::move(metadata_io_)); + } + + std::unique_ptr storage_factory_; + std::shared_ptr storage_; + std::shared_ptr converter_factory_; + std::unique_ptr metadata_io_; std::unique_ptr reader_; std::string storage_serialization_format_; rosbag2_storage::TopicMetadata topic_with_type_; + const std::string bag_name_ = "SequentialCompressionReaderTest"; + rcpputils::fs::path tmp_dir_; + rosbag2_storage::StorageOptions storage_options_; + rosbag2_storage::BagMetadata metadata_; + rosbag2_cpp::ConverterOptions converter_options_; }; TEST_F(SequentialCompressionReaderTest, open_throws_if_unsupported_compressor) { - rosbag2_storage::BagMetadata metadata = construct_default_bag_metadata(); - metadata.compression_format = "bad_format"; - EXPECT_CALL(*metadata_io_, read_metadata(_)).WillRepeatedly(Return(metadata)); - EXPECT_CALL(*metadata_io_, metadata_file_exists(_)).WillRepeatedly(Return(true)); + metadata_.compression_format = "bad_format"; + EXPECT_CALL(*metadata_io_, read_metadata(_)).Times(1); + EXPECT_CALL(*metadata_io_, metadata_file_exists(_)).Times(AtLeast(1)); auto compression_factory = std::make_unique(); auto sequential_reader = std::make_unique( @@ -88,18 +141,13 @@ TEST_F(SequentialCompressionReaderTest, open_throws_if_unsupported_compressor) converter_factory_, std::move(metadata_io_)); - reader_ = std::make_unique(std::move(sequential_reader)); EXPECT_THROW( - reader_->open(rosbag2_storage::StorageOptions(), {"", storage_serialization_format_}), + sequential_reader->open(storage_options_, converter_options_), std::invalid_argument); } TEST_F(SequentialCompressionReaderTest, returns_all_topics_and_types) { - rosbag2_storage::BagMetadata metadata = construct_default_bag_metadata(); - ON_CALL(*metadata_io_, read_metadata(_)).WillByDefault(Return(metadata)); - ON_CALL(*metadata_io_, metadata_file_exists(_)).WillByDefault(Return(true)); - auto decompressor = std::make_unique>(); auto compression_factory = std::make_unique>(); @@ -114,8 +162,7 @@ TEST_F(SequentialCompressionReaderTest, returns_all_topics_and_types) converter_factory_, std::move(metadata_io_)); - compression_reader->open( - rosbag2_storage::StorageOptions(), {"", storage_serialization_format_}); + compression_reader->open(storage_options_, converter_options_); auto topics_and_types = compression_reader->get_all_topics_and_types(); EXPECT_FALSE(topics_and_types.empty()); @@ -123,9 +170,6 @@ TEST_F(SequentialCompressionReaderTest, returns_all_topics_and_types) TEST_F(SequentialCompressionReaderTest, open_supports_zstd_compressor) { - rosbag2_storage::BagMetadata metadata = construct_default_bag_metadata(); - ON_CALL(*metadata_io_, read_metadata(_)).WillByDefault(Return(metadata)); - ON_CALL(*metadata_io_, metadata_file_exists(_)).WillByDefault(Return(true)); auto compression_factory = std::make_unique(); auto sequential_reader = std::make_unique( @@ -137,16 +181,12 @@ TEST_F(SequentialCompressionReaderTest, open_supports_zstd_compressor) reader_ = std::make_unique(std::move(sequential_reader)); // Throws runtime_error b/c compressor can't read EXPECT_THROW( - reader_->open(rosbag2_storage::StorageOptions(), {"", storage_serialization_format_}), + reader_->open(storage_options_, converter_options_), std::runtime_error); } TEST_F(SequentialCompressionReaderTest, reader_calls_create_decompressor) { - rosbag2_storage::BagMetadata metadata = construct_default_bag_metadata(); - ON_CALL(*metadata_io_, read_metadata(_)).WillByDefault(Return(metadata)); - ON_CALL(*metadata_io_, metadata_file_exists(_)).WillByDefault(Return(true)); - auto decompressor = std::make_unique>(); ON_CALL(*decompressor, decompress_uri(_)).WillByDefault(Return("some/path")); EXPECT_CALL(*decompressor, decompress_uri(_)).Times(1); @@ -164,29 +204,19 @@ TEST_F(SequentialCompressionReaderTest, reader_calls_create_decompressor) std::move(metadata_io_)); reader_ = std::make_unique(std::move(sequential_reader)); - reader_->open( - rosbag2_storage::StorageOptions(), {"", storage_serialization_format_}); + reader_->open(storage_options_, converter_options_); } -TEST_F(SequentialCompressionReaderTest, compression_called_when_splitting_bagfile) +TEST_F(SequentialCompressionReaderTest, compression_called_when_loading_split_bagfile) { - const auto relative_path_1 = "/path/to/storage1"; - const auto relative_path_2 = "/path/to/storage2"; - rosbag2_storage::BagMetadata metadata; - metadata.relative_file_paths = {relative_path_1, relative_path_2}; - metadata.topics_with_message_count.push_back({{topic_with_type_}, 10}); - metadata.bag_size = 512000; - metadata.compression_format = "zstd"; - metadata.compression_mode = - rosbag2_compression::compression_mode_to_string(rosbag2_compression::CompressionMode::FILE); - ON_CALL(*metadata_io_, read_metadata(_)).WillByDefault(Return(metadata)); - ON_CALL(*metadata_io_, metadata_file_exists(_)).WillByDefault(Return(true)); + metadata_.topics_with_message_count.push_back({{topic_with_type_}, 10}); + metadata_.bag_size = 512000; auto decompressor = std::make_unique>(); - // We are mocking two splits, so only file decompression should occur twice + // We are mocking two splits, so file decompression should occur twice EXPECT_CALL(*decompressor, decompress_uri(_)).Times(2) - .WillOnce(Return(relative_path_1)) - .WillOnce(Return(relative_path_2)); + .WillOnce(Return(metadata_.relative_file_paths[0])) + .WillOnce(Return(metadata_.relative_file_paths[1])); EXPECT_CALL(*decompressor, decompress_serialized_bag_message(_)).Times(0); auto compression_factory = std::make_unique>(); @@ -206,9 +236,51 @@ TEST_F(SequentialCompressionReaderTest, compression_called_when_splitting_bagfil converter_factory_, std::move(metadata_io_)); - compression_reader->open( - rosbag2_storage::StorageOptions(), {"", storage_serialization_format_}); + compression_reader->open(storage_options_, converter_options_); EXPECT_EQ(compression_reader->has_next_file(), true); EXPECT_EQ(compression_reader->has_next(), true); compression_reader->read_next(); } + +TEST_F(SequentialCompressionReaderTest, can_find_v4_names) +{ + auto reader = create_reader(); + reader->open(storage_options_, converter_options_); + EXPECT_TRUE(reader->has_next_file()); +} + +TEST_F(SequentialCompressionReaderTest, throws_on_incorrect_filenames) +{ + for (auto & relative_file_path : metadata_.relative_file_paths) { + relative_file_path = ( + rcpputils::fs::path(bag_name_) / (relative_file_path + ".something")).string(); + } + auto reader = create_reader(); + EXPECT_THROW(reader->open(storage_options_, converter_options_), std::invalid_argument); +} + +TEST_F(SequentialCompressionReaderTest, can_find_prefixed_filenames) +{ + // By prefixing the bag name, this imitates the V3 filename logic + for (auto & relative_file_path : metadata_.relative_file_paths) { + relative_file_path = (rcpputils::fs::path(bag_name_) / relative_file_path).string(); + } + auto reader = create_reader(); + + EXPECT_NO_THROW(reader->open(storage_options_, converter_options_)); + EXPECT_TRUE(reader->has_next_file()); +} + +TEST_F(SequentialCompressionReaderTest, can_find_prefixed_filenames_in_renamed_bag) +{ + // By prefixing a _different_ path than the bagname, we imitate the situation where the bag + // was recorded using V3 logic, then the directory was moved to be a new name - this is the + // use case the V4 relative path logic was intended to fix + for (auto & relative_file_path : metadata_.relative_file_paths) { + relative_file_path = (rcpputils::fs::path("OtherBagName") / relative_file_path).string(); + } + auto reader = create_reader(); + + EXPECT_NO_THROW(reader->open(storage_options_, converter_options_)); + EXPECT_TRUE(reader->has_next_file()); +} diff --git a/rosbag2_cpp/include/rosbag2_cpp/readers/sequential_reader.hpp b/rosbag2_cpp/include/rosbag2_cpp/readers/sequential_reader.hpp index 76db844367..83c79fddf2 100644 --- a/rosbag2_cpp/include/rosbag2_cpp/readers/sequential_reader.hpp +++ b/rosbag2_cpp/include/rosbag2_cpp/readers/sequential_reader.hpp @@ -133,6 +133,12 @@ class ROSBAG2_CPP_PUBLIC SequentialReader */ virtual void fill_topics_metadata(); + /** + * Prepare current file for opening by the storage implementation. + * This may be used by subclasses, for example decompressing + */ + virtual void preprocess_current_file() {} + std::unique_ptr storage_factory_{}; std::shared_ptr storage_{}; std::unique_ptr converter_{}; @@ -143,6 +149,9 @@ class ROSBAG2_CPP_PUBLIC SequentialReader std::vector file_paths_{}; // List of database files. std::vector::iterator current_file_iterator_{}; // Index of file to read from + // Hang on to this because storage_options_ is mutated to point at individual files + std::string base_folder_; + private: rosbag2_storage::StorageOptions storage_options_; std::shared_ptr converter_factory_{}; diff --git a/rosbag2_cpp/src/rosbag2_cpp/readers/sequential_reader.cpp b/rosbag2_cpp/src/rosbag2_cpp/readers/sequential_reader.cpp index ea88b5a129..e902b1e7e9 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/readers/sequential_reader.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/readers/sequential_reader.cpp @@ -84,6 +84,7 @@ void SequentialReader::open( const ConverterOptions & converter_options) { storage_options_ = storage_options; + base_folder_ = storage_options.uri; // If there is a metadata.yaml file present, load it. // If not, let's ask the storage with the given URI for its metadata. @@ -99,6 +100,8 @@ void SequentialReader::open( storage_options.uri, metadata_.relative_file_paths, metadata_.version); current_file_iterator_ = file_paths_.begin(); + preprocess_current_file(); + storage_options_.uri = get_current_file(); storage_ = storage_factory_->open_read_only(storage_options_); if (!storage_) { @@ -201,6 +204,7 @@ void SequentialReader::load_next_file() { assert(current_file_iterator_ != file_paths_.end()); current_file_iterator_++; + preprocess_current_file(); } std::string SequentialReader::get_current_file() const From 9d3895785aa760ed9bc2812b01ee649f7ab79df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C4=85browski?= Date: Tue, 26 Jan 2021 21:18:34 +0100 Subject: [PATCH 68/77] Performance benchmarking refactor (#594) * Refactoring of rosbag2 performance package: - renamed since now it no longer benchmarks writer only - generalized byte_producer so that it uses a callback instead of queue, so it can be reused in publisher scheme Signed-off-by: Adam Dabrowski * Benchmark publishers based on yaml configuration - can specify multiple groups of publishers (see attached example yaml) - reuses byte producer Signed-off-by: Adam Dabrowski * Applying configured QoS settings for publishers. Also included in yaml example. Signed-off-by: Adam Dabrowski * linters Signed-off-by: Adam Dabrowski * Towards common configuration - separating out common structures - utility class for common parameter parsing Signed-off-by: Adam Dabrowski * Barebone launchfile for benchmarks. Signed-off-by: Piotr Jaroszek * writer benchmark adapted to yaml file and publisher groups Signed-off-by: Adam Dabrowski * refactored result writing and bag parameters Signed-off-by: Adam Dabrowski * linters Signed-off-by: Adam Dabrowski * Launchfile for benchmarks Signed-off-by: Piotr Jaroszek * Change storage config file from non optimized to resilient Signed-off-by: Piotr Jaroszek * Max bag size for benchmark launchfile. Launchfile refactor. Signed-off-by: Piotr Jaroszek * Copy yaml configs after benchmark is finished. Signed-off-by: Piotr Jaroszek * Benchmark results csv file extended Signed-off-by: Piotr Jaroszek * added disclaimer about random data and compression Signed-off-by: Adam Dabrowski * Report gen tool for benchmarks Signed-off-by: Piotr Jaroszek * Benchmarks out dir name changed Signed-off-by: Piotr Jaroszek * results writer node Signed-off-by: Adam Dabrowski * documentation Signed-off-by: Adam Dabrowski * Transport and transportless in launchfile Signed-off-by: Piotr Jaroszek * Benchmark launchfile refactor Signed-off-by: Piotr Jaroszek * Wait for rosbag listening in benchmark launchfile Signed-off-by: Piotr Jaroszek * Uncrustify and some comments for benchmarking tools Signed-off-by: Piotr Jaroszek * Added new producers config for benchmarks Signed-off-by: Piotr Jaroszek * Missing parameters in transport benchmark Signed-off-by: Piotr Jaroszek added comment in storage_optimized.yaml * Missing rosbag record parameters in transport benchmark Signed-off-by: Piotr Jaroszek * Wait for subscriptions parameter in producers config Signed-off-by: Piotr Jaroszek * moved utils code from header to source Signed-off-by: Adam Dabrowski * changed compiler shortcut uint to unsigned int (should fix Windows build) Signed-off-by: Adam Dabrowski Co-authored-by: Piotr Jaroszek Signed-off-by: Emerson Knapp --- .../CHANGELOG.rst | 2 +- .../CMakeLists.txt | 51 +- .../README.md | 76 +++ .../config/benchmarks/test.yaml | 16 + .../config/producers/mixed_110Mbs.yaml | 21 + .../mixed_110Mbs_low_pubs_count.yaml | 21 + .../config/storage/storage_optimized.yaml | 3 + .../config/storage/storage_resilient.yaml} | 0 .../bag_config.hpp | 29 + .../byte_producer.hpp | 43 +- .../config_utils.hpp | 46 ++ .../message_queue.hpp | 6 +- .../producer_config.hpp | 25 + .../publisher_group_config.hpp | 32 ++ .../result_utils.hpp | 42 ++ .../writer_benchmark.hpp | 23 +- .../launch/benchmark_launch.py | 527 ++++++++++++++++++ .../package.xml | 4 +- .../scripts/report_gen.py | 222 ++++++++ .../src/benchmark_publishers.cpp | 116 ++++ .../src/config_utils.cpp | 132 +++++ .../src/result_utils.cpp | 128 +++++ .../src/results_writer.cpp} | 20 +- .../src/writer_benchmark.cpp | 210 +++++++ .../README.md | 53 -- .../scripts/benchmark.sh | 74 --- .../storage_config/storage_optimized.yaml | 2 - .../src/writer_benchmark.cpp | 274 --------- 28 files changed, 1746 insertions(+), 452 deletions(-) rename rosbag2_performance/{rosbag2_performance_writer_benchmarking => rosbag2_performance_benchmarking}/CHANGELOG.rst (96%) rename rosbag2_performance/{rosbag2_performance_writer_benchmarking => rosbag2_performance_benchmarking}/CMakeLists.txt (50%) create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/README.md create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/config/benchmarks/test.yaml create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/config/producers/mixed_110Mbs.yaml create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/config/producers/mixed_110Mbs_low_pubs_count.yaml create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/config/storage/storage_optimized.yaml rename rosbag2_performance/{rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_default.yaml => rosbag2_performance_benchmarking/config/storage/storage_resilient.yaml} (100%) create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/bag_config.hpp rename rosbag2_performance/{rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking => rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking}/byte_producer.hpp (60%) create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/config_utils.hpp rename rosbag2_performance/{rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking => rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking}/message_queue.hpp (91%) create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/producer_config.hpp create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/publisher_group_config.hpp create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/result_utils.hpp rename rosbag2_performance/{rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking => rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking}/writer_benchmark.hpp (68%) create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/launch/benchmark_launch.py rename rosbag2_performance/{rosbag2_performance_writer_benchmarking => rosbag2_performance_benchmarking}/package.xml (84%) create mode 100755 rosbag2_performance/rosbag2_performance_benchmarking/scripts/report_gen.py create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/src/benchmark_publishers.cpp create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/src/config_utils.cpp create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/src/result_utils.cpp rename rosbag2_performance/{rosbag2_performance_writer_benchmarking/src/main.cpp => rosbag2_performance_benchmarking/src/results_writer.cpp} (60%) create mode 100644 rosbag2_performance/rosbag2_performance_benchmarking/src/writer_benchmark.cpp delete mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/README.md delete mode 100755 rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh delete mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_optimized.yaml delete mode 100644 rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst b/rosbag2_performance/rosbag2_performance_benchmarking/CHANGELOG.rst similarity index 96% rename from rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst rename to rosbag2_performance/rosbag2_performance_benchmarking/CHANGELOG.rst index 8cbace3d1b..645aef0d25 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CHANGELOG.rst +++ b/rosbag2_performance/rosbag2_performance_benchmarking/CHANGELOG.rst @@ -1,5 +1,5 @@ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Changelog for package rosbag2_performance_writer_benchmarking +Changelog for package rosbag2_performance_benchmarking ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 0.5.0 (2020-12-02) diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt b/rosbag2_performance/rosbag2_performance_benchmarking/CMakeLists.txt similarity index 50% rename from rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt rename to rosbag2_performance/rosbag2_performance_benchmarking/CMakeLists.txt index 40957e3f93..7a54f6d0bb 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/CMakeLists.txt +++ b/rosbag2_performance/rosbag2_performance_benchmarking/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.5) -project(rosbag2_performance_writer_benchmarking) +project(rosbag2_performance_benchmarking) # Default to C++14 if(NOT CMAKE_CXX_STANDARD) @@ -22,23 +22,68 @@ if(BUILD_ROSBAG2_BENCHMARKS) find_package(std_msgs REQUIRED) find_package(yaml_cpp_vendor REQUIRED) - add_executable(writer_benchmark src/writer_benchmark.cpp src/main.cpp) + add_executable(writer_benchmark + src/config_utils.cpp + src/result_utils.cpp + src/writer_benchmark.cpp) + + add_executable(benchmark_publishers + src/benchmark_publishers.cpp + src/config_utils.cpp) + + add_executable(results_writer + src/config_utils.cpp + src/result_utils.cpp + src/results_writer.cpp) + ament_target_dependencies(writer_benchmark rclcpp std_msgs rosbag2_compression rosbag2_cpp + rosbag2_storage + yaml_cpp_vendor + ) + + ament_target_dependencies(benchmark_publishers + rclcpp + rosbag2_storage + std_msgs yaml_cpp_vendor ) + + ament_target_dependencies(results_writer + rclcpp + rosbag2_storage + ) + target_include_directories(writer_benchmark PUBLIC $ $ ) - install(TARGETS writer_benchmark + target_include_directories(benchmark_publishers + PUBLIC + $ + $ + ) + + target_include_directories(results_writer + PUBLIC + $ + $ + ) + + install(TARGETS writer_benchmark benchmark_publishers results_writer DESTINATION lib/${PROJECT_NAME}) + install(DIRECTORY + launch + config + DESTINATION share/${PROJECT_NAME} + ) + if(BUILD_TESTING) find_package(ament_lint_auto REQUIRED) ament_lint_auto_find_test_dependencies() diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/README.md b/rosbag2_performance/rosbag2_performance_benchmarking/README.md new file mode 100644 index 0000000000..32ae8bd83c --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/README.md @@ -0,0 +1,76 @@ +# Rosbag2 writer benchmarking + +The primary package to test performance of the rosbag2. + +## How it works + +Use `benchmark_launch.py` launchfile to run an entire set of benchmarks. + +Launchfile requires two arguments: + +- `benchmark` - provides benchmark description (how many repetitions, cache size, database configuration etc.), +- `producers` - provides producers description (how many publisher/producer instances, frequency, messages size etc.) + +Templates for these configuration files are in `config` directory of this package. + +To run test benchmark (with `test.yaml` and `mixed_110Mbs.yaml`): + +```bash +ros2 launch rosbag2_performance_benchmarking benchmark_launch.py benchmark:=`ros2 pkg prefix rosbag2_performance_benchmarking`/share/rosbag2_performance_benchmarking/config/benchmarks/test.yaml producers:=`ros2 pkg prefix rosbag2_performance_benchmarking`/share/rosbag2_performance_benchmarking/config/producers/mixed_110Mbs.yaml +``` + +The summary of benchmark goes into result file described in benchmark config: `//summary_result_file` where `BENCHMARK_NAME` is a name generated from config names, transport type and timestamp. + +For human friendly output, a postprocess report generation tool can be used. Launch it with benchmark result directory as an `-i` argument (directory with `summary_result_file` file): + +```bash +scripts/report_gen.py -i +``` +#### Binaries + +These are used in the launch file: + +* `benchmark_publishers` - runs publishers based on provided parameters. Used when `no_transport` parameter is set to `False`; +* `writer_benchmark` - runs storage-only benchmarking, mimicking subscription queues but using no transport whatsoever. Used when `no_transport` parameter is set to `True`. +* `results_writer` - based on provider parameters, write results (percentage of recorded messages) after recording. One of the parameters is the +storage uri, which is used to read the bag metadata file. + +#### Compression + +Note that while you can opt to select compression for benchmarking, the generated data is random so it is likely not representative for this specific case. To publish non-random data, you need to modify the ByteProducer. + +## Building + +To build the package in the rosbag2 build process, make sure to turn `BUILD_ROSBAG2_BENCHMARKS` flag on (e.g. `colcon build --cmake-args -DBUILD_ROSBAG2_BENCHMARKS=1`) + +If you already built rosbag2, you can use `packages-select` option to build benchmarks. +Example: `colcon build --packages-select rosbag2_performance_benchmarking --cmake-args -DBUILD_ROSBAG2_BENCHMARKS=1`. + +## General knowledge: I/O benchmarking + +#### Background: benchmarking disk writes on your system + +It might be useful to first understand what limitation your disk poses to the throughput of data recording. +Performance of bag write can't be higher over extended period of time (you can only use as much memory). + +**Using dd command** + +`dd if=/dev/zero of=/tmp/output conv=fdatasync bs=384k count=1k; rm -f /tmp/output` + +This method is not great for benchmarking the disk but an easy way to start since it requires no dependencies. +This will write zeros to the /tmp/output file with block size 384k, 1000 blocks, ends when write finishes. +Make sure to benchmark the disk which your bags will be written to (check your mount points and change “/tmp/output” to another path if needed). +Note: this depends on parameters used and whatever else is running on your system but can give you a ballpark figure when ran several times. + +**Using fio** + +For more sophisticated & accurate benchmarks, see the `fio` command. An example for big data blocks is: `fio --name TEST --eta-newline=5s --filename=fio-tempfile.dat --rw=write --size=500m --io_size=10g --blocksize=1024k --ioengine=libaio --fsync=10000 --iodepth=32 --direct=1 --numjobs=1 --runtime=60 --group_reporting`. + +#### Profiling bags I/O with tools + +Tools that can help in I/O profiling: `sudo apt-get install iotop ioping sysstat` +* `iotop` works similar as `top` command, but shows disk reads, writes, swaps and I/O %. Can be used at higher frequency in batch mode with specified process to deliver data that can be plotted. + * Example use: `sudo iotop -h -d 0.1 -t -b -o -p ` after running the bag. +* `ioping` can be used to check latency of requests to device +* `strace` can help determine syscalls associated with the bottleneck. + * Example use: `strace -c ros2 bag record /image --max-cache-size 10 -o ./tmp`. You will see a report after finishing recording with Ctrl-C. diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/config/benchmarks/test.yaml b/rosbag2_performance/rosbag2_performance_benchmarking/config/benchmarks/test.yaml new file mode 100644 index 0000000000..86222d8118 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/config/benchmarks/test.yaml @@ -0,0 +1,16 @@ +rosbag2_performance_benchmarking: + benchmark_node: + ros__parameters: + benchmark: + summary_result_file: "results.csv" + db_root_folder: "rosbag2_performance_test_results" + repeat_each: 2 # How many times to run each configurations (to average results) + no_transport: True # Whether to run storage-only or end-to-end (including transport) benchmark + preserve_bags: False # Whether to leave bag files after experiment (and between runs). Some configurations can take lots of space! + parameters: # Each combination of parameters in this section will be benchmarked + max_cache_size: [10000000, 100000000] + max_bag_size: [0] + compression: ["", "zstd"] + compression_queue_size: [1] + compression_threads: [0] + storage_config_file: ["", "storage_resilient.yaml"] diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/config/producers/mixed_110Mbs.yaml b/rosbag2_performance/rosbag2_performance_benchmarking/config/producers/mixed_110Mbs.yaml new file mode 100644 index 0000000000..cd1715b15b --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/config/producers/mixed_110Mbs.yaml @@ -0,0 +1,21 @@ +rosbag2_performance_benchmarking_node: + ros__parameters: + publishers: # publisher_groups parameter needs to include all the subsequent groups + publisher_groups: [ "10Mbs_many_frequent_small", "100Mbs_large" ] + wait_for_subscriptions: True + 10Mbs_many_frequent_small: + publishers_count: 500 + topic_root: "benchmarking_small" + msg_size_bytes: 100 + msg_count_each: 2000 + rate_hz: 200 + 100Mbs_large: + publishers_count: 1 + topic_root: "benchmarking_large" + msg_size_bytes: 10000000 + msg_count_each: 100 + rate_hz: 10 + qos: # qos settings are ignored for writer only benchmarking + qos_depth: 5 + qos_reliability: "best_effort" # "reliable" + qos_durability: "volatile" # "transient_local" diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/config/producers/mixed_110Mbs_low_pubs_count.yaml b/rosbag2_performance/rosbag2_performance_benchmarking/config/producers/mixed_110Mbs_low_pubs_count.yaml new file mode 100644 index 0000000000..602483fce2 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/config/producers/mixed_110Mbs_low_pubs_count.yaml @@ -0,0 +1,21 @@ +rosbag2_performance_benchmarking_node: + ros__parameters: + publishers: # publisher_groups parameter needs to include all the subsequent groups + publisher_groups: [ "10Mbs_many_frequent_small", "100Mbs_large" ] + wait_for_subscriptions: True + 10Mbs_many_frequent_small: + publishers_count: 100 + topic_root: "benchmarking_small" + msg_size_bytes: 500 + msg_count_each: 2000 + rate_hz: 200 + 100Mbs_large: + publishers_count: 1 + topic_root: "benchmarking_large" + msg_size_bytes: 10000000 + msg_count_each: 100 + rate_hz: 10 + qos: # qos settings are ignored for writer only benchmarking + qos_depth: 5 + qos_reliability: "best_effort" # "reliable" + qos_durability: "volatile" # "transient_local" diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/config/storage/storage_optimized.yaml b/rosbag2_performance/rosbag2_performance_benchmarking/config/storage/storage_optimized.yaml new file mode 100644 index 0000000000..2046448533 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/config/storage/storage_optimized.yaml @@ -0,0 +1,3 @@ +# note: this is the default now, using this file does not change how the storage behaves +write: + pragmas: ["journal_mode = MEMORY", "synchronous = OFF"] diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_default.yaml b/rosbag2_performance/rosbag2_performance_benchmarking/config/storage/storage_resilient.yaml similarity index 100% rename from rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_default.yaml rename to rosbag2_performance/rosbag2_performance_benchmarking/config/storage/storage_resilient.yaml diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/bag_config.hpp b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/bag_config.hpp new file mode 100644 index 0000000000..cbc5a7c529 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/bag_config.hpp @@ -0,0 +1,29 @@ +// Copyright 2021, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_PERFORMANCE_BENCHMARKING__BAG_CONFIG_HPP_ +#define ROSBAG2_PERFORMANCE_BENCHMARKING__BAG_CONFIG_HPP_ + +#include +#include "rosbag2_storage/storage_options.hpp" + +struct BagConfig +{ + rosbag2_storage::StorageOptions storage_options; + std::string compression_format; + unsigned int compression_queue_size; + unsigned int compression_threads; +}; + +#endif // ROSBAG2_PERFORMANCE_BENCHMARKING__BAG_CONFIG_HPP_ diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/byte_producer.hpp b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/byte_producer.hpp similarity index 60% rename from rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/byte_producer.hpp rename to rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/byte_producer.hpp index 9ade8ca900..d17ee71a1b 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/byte_producer.hpp +++ b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/byte_producer.hpp @@ -12,26 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__BYTE_PRODUCER_HPP_ -#define ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__BYTE_PRODUCER_HPP_ +#ifndef ROSBAG2_PERFORMANCE_BENCHMARKING__BYTE_PRODUCER_HPP_ +#define ROSBAG2_PERFORMANCE_BENCHMARKING__BYTE_PRODUCER_HPP_ #include #include #include -#include +#include #include "rclcpp/utilities.hpp" - #include "std_msgs/msg/byte_multi_array.hpp" -#include "rosbag2_performance_writer_benchmarking/message_queue.hpp" - -struct ProducerConfig -{ - unsigned int frequency; - unsigned int max_count; - unsigned int message_size; -}; +#include "rosbag2_performance_benchmarking/producer_config.hpp" inline auto generate_random_message(const ProducerConfig & config) { @@ -49,31 +41,46 @@ inline auto generate_random_message(const ProducerConfig & config) class ByteProducer { public: - ByteProducer(const ProducerConfig & config, std::shared_ptr queue) + using producer_initialize_function_t = std::function; + using producer_callback_function_t = std::function)>; + + using producer_finalize_function_t = std::function; + + ByteProducer( + const ProducerConfig & config, + producer_initialize_function_t producer_initialize, + producer_callback_function_t producer_callback, + producer_finalize_function_t producer_finalize) : configuration_(config), - queue_(queue), + producer_initialize_(producer_initialize), + producer_callback_(producer_callback), + producer_finalize_(producer_finalize), sleep_time_(configuration_.frequency == 0 ? 1 : 1000 / configuration_.frequency), message_(generate_random_message(configuration_)) {} void run() { + producer_initialize_(); for (auto i = 0u; i < configuration_.max_count; ++i) { if (!rclcpp::ok()) { break; } - queue_->push(message_); + producer_callback_(message_); std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time_)); } - queue_->set_complete(); + producer_finalize_(); } private: ProducerConfig configuration_; - std::shared_ptr queue_; + producer_initialize_function_t producer_initialize_; + producer_callback_function_t producer_callback_; + producer_finalize_function_t producer_finalize_; unsigned int sleep_time_; // in milliseconds // for simplification, this pointer will be reused std::shared_ptr message_; }; -#endif // ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__BYTE_PRODUCER_HPP_ +#endif // ROSBAG2_PERFORMANCE_BENCHMARKING__BYTE_PRODUCER_HPP_ diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/config_utils.hpp b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/config_utils.hpp new file mode 100644 index 0000000000..c147a6a5bf --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/config_utils.hpp @@ -0,0 +1,46 @@ +// Copyright 2020-2021, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_PERFORMANCE_BENCHMARKING__CONFIG_UTILS_HPP_ +#define ROSBAG2_PERFORMANCE_BENCHMARKING__CONFIG_UTILS_HPP_ + +#include +#include + +#include "rclcpp/node.hpp" +#include "rosbag2_performance_benchmarking/bag_config.hpp" +#include "rosbag2_performance_benchmarking/publisher_group_config.hpp" + +namespace config_utils +{ + +/// Helper function to fill group_config with qos parameters +void load_qos_configuration( + rclcpp::Node & node, + PublisherGroupConfig & group_config, + const std::string & group_prefix); + +/// Acquires the parameter determining whether to wait for subscriber +bool wait_for_subscriptions_from_node_parameters(rclcpp::Node & node); + +/// Acquires publisher parameters from the node +std::vector publisher_groups_from_node_parameters( + rclcpp::Node & node); + +/// Acquires bag parameters from the node +BagConfig bag_config_from_node_parameters(rclcpp::Node & node); + +} // namespace config_utils + +#endif // ROSBAG2_PERFORMANCE_BENCHMARKING__CONFIG_UTILS_HPP_ diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/message_queue.hpp b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/message_queue.hpp similarity index 91% rename from rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/message_queue.hpp rename to rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/message_queue.hpp index 1554d3f202..b70cef4aae 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/message_queue.hpp +++ b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/message_queue.hpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__MESSAGE_QUEUE_HPP_ -#define ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__MESSAGE_QUEUE_HPP_ +#ifndef ROSBAG2_PERFORMANCE_BENCHMARKING__MESSAGE_QUEUE_HPP_ +#define ROSBAG2_PERFORMANCE_BENCHMARKING__MESSAGE_QUEUE_HPP_ #include #include @@ -93,4 +93,4 @@ class MessageQueue typedef MessageQueue ByteMessageQueue; -#endif // ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__MESSAGE_QUEUE_HPP_ +#endif // ROSBAG2_PERFORMANCE_BENCHMARKING__MESSAGE_QUEUE_HPP_ diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/producer_config.hpp b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/producer_config.hpp new file mode 100644 index 0000000000..48d479e557 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/producer_config.hpp @@ -0,0 +1,25 @@ +// Copyright 2020-2021, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_PERFORMANCE_BENCHMARKING__PRODUCER_CONFIG_HPP_ +#define ROSBAG2_PERFORMANCE_BENCHMARKING__PRODUCER_CONFIG_HPP_ + +struct ProducerConfig +{ + unsigned int frequency; + unsigned int max_count; + unsigned int message_size; +}; + +#endif // ROSBAG2_PERFORMANCE_BENCHMARKING__PRODUCER_CONFIG_HPP_ diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/publisher_group_config.hpp b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/publisher_group_config.hpp new file mode 100644 index 0000000000..1fd94e12bb --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/publisher_group_config.hpp @@ -0,0 +1,32 @@ +// Copyright 2020-2021, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_PERFORMANCE_BENCHMARKING__PUBLISHER_GROUP_CONFIG_HPP_ +#define ROSBAG2_PERFORMANCE_BENCHMARKING__PUBLISHER_GROUP_CONFIG_HPP_ + +#include +#include "rosbag2_performance_benchmarking/producer_config.hpp" +#include "rclcpp/qos.hpp" + +struct PublisherGroupConfig +{ + PublisherGroupConfig() + : count(0), qos(10) {} + unsigned int count; + ProducerConfig producer_config; + std::string topic_root; + rclcpp::QoS qos; +}; + +#endif // ROSBAG2_PERFORMANCE_BENCHMARKING__PUBLISHER_GROUP_CONFIG_HPP_ diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/result_utils.hpp b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/result_utils.hpp new file mode 100644 index 0000000000..5b014c0a7b --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/result_utils.hpp @@ -0,0 +1,42 @@ +// Copyright 2021, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_PERFORMANCE_BENCHMARKING__RESULT_UTILS_HPP_ +#define ROSBAG2_PERFORMANCE_BENCHMARKING__RESULT_UTILS_HPP_ + +#include +#include + +#include "rclcpp/node.hpp" +#include "rosbag2_performance_benchmarking/bag_config.hpp" +#include "rosbag2_performance_benchmarking/publisher_group_config.hpp" + +namespace result_utils +{ + +/// Read total count of recorded messages from metadata.yaml file +int get_message_count_from_metadata(const std::string & uri); + +/// Based on configuration and metadata from completed benchmark, write results +void write_benchmark_results( + const std::vector & publisher_groups_config, + const BagConfig & bag_config, + const std::string & results_file); + +/// this version works with a standalone node using node parameters +void write_benchmark_results(rclcpp::Node & node); + +} // namespace result_utils + +#endif // ROSBAG2_PERFORMANCE_BENCHMARKING__RESULT_UTILS_HPP_ diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/writer_benchmark.hpp similarity index 68% rename from rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp rename to rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/writer_benchmark.hpp index ac0bb12af8..1bc82575dd 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/include/rosbag2_performance_writer_benchmarking/writer_benchmark.hpp +++ b/rosbag2_performance/rosbag2_performance_benchmarking/include/rosbag2_performance_benchmarking/writer_benchmark.hpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__WRITER_BENCHMARK_HPP_ -#define ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__WRITER_BENCHMARK_HPP_ +#ifndef ROSBAG2_PERFORMANCE_BENCHMARKING__WRITER_BENCHMARK_HPP_ +#define ROSBAG2_PERFORMANCE_BENCHMARKING__WRITER_BENCHMARK_HPP_ #include #include @@ -24,8 +24,10 @@ #include "rosbag2_compression/compression_options.hpp" #include "rosbag2_cpp/writers/sequential_writer.hpp" -#include "rosbag2_performance_writer_benchmarking/message_queue.hpp" -#include "rosbag2_performance_writer_benchmarking/byte_producer.hpp" +#include "rosbag2_performance_benchmarking/byte_producer.hpp" +#include "rosbag2_performance_benchmarking/message_queue.hpp" +#include "rosbag2_performance_benchmarking/publisher_group_config.hpp" +#include "rosbag2_performance_benchmarking/bag_config.hpp" class WriterBenchmark : public rclcpp::Node { @@ -34,25 +36,20 @@ class WriterBenchmark : public rclcpp::Node void start_benchmark(); private: - void create_producers(const ProducerConfig & config); + void create_producers(); void create_writer(); void start_producers(); void write_results() const; int get_message_count_from_metadata() const; - ProducerConfig config_; - unsigned int instances_; + std::vector configurations_; std::string results_file_; + BagConfig bag_config_; - rosbag2_storage::StorageOptions storage_options_; std::vector producer_threads_; std::vector> producers_; std::vector> queues_; - - std::string compression_format_; - uint64_t compression_queue_size_; - uint64_t compression_threads_; std::shared_ptr writer_; }; -#endif // ROSBAG2_PERFORMANCE_WRITER_BENCHMARKING__WRITER_BENCHMARK_HPP_ +#endif // ROSBAG2_PERFORMANCE_BENCHMARKING__WRITER_BENCHMARK_HPP_ diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/launch/benchmark_launch.py b/rosbag2_performance/rosbag2_performance_benchmarking/launch/benchmark_launch.py new file mode 100644 index 0000000000..04d54614e1 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/launch/benchmark_launch.py @@ -0,0 +1,527 @@ +# Copyright 2021, Robotec.ai sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Launchfile for benchmarking rosbag2. + +This launchfile can only be launched with 'ros2 launch' command. + +Two launch arguments are required: +* benchmark - path to benchmark description in yaml format ('benchmark:='), +* producers - path to producers description in yaml format ('producers:='). + +Goal of this launchfile is to launch in sequence all processes and/or nodes with right parameters +required for selected benchmark. Cross section of parameters is generated based on parameters from +'benchmark' yaml description file. + +Based on 'no_transport' parameter in benchmark description, a single run in launch sequence +looks as follow: + +NO TRANSPORT: +Only 'writer_benchmark' node is used as 'producer node' (PN). It directly writes messages to +a storage and then fill up a result file. No additional processes are required. + +PN starts -> PN exits + +TRANSPORT: +For end-to-end benchmark, `ros2 bag record` (ROSBAG) process and 'result writer' (RW) are also +included in a single launch sequence run. In this case 'benchmark_publishers' node act as +producer node. Result writer node writes final result file. + +ROSBAG starts -> PN starts -> PN exits -> ROSBAG exits -> RW starts + +After the whole sequence is finished, both producers and benchmark description files are copied +to benchmark folder. +""" + +import datetime +import os +import pathlib +import shutil +import signal +import sys + +from ament_index_python import get_package_share_directory + +import launch + +import launch_ros + +import yaml + +_bench_cfg_path = None +_producers_cfg_path = None + +_producer_idx = 0 +_producer_nodes = [] + +_rosbag_processes = [] +_rosbag_pid = None + +_result_writers = [] + + +def _parse_arguments(args=sys.argv[4:]): + """Parse benchmark and producers config file paths.""" + bench_cfg_path = None + producers_cfg_path = None + err_str = 'Missing or invalid arguments detected. ' \ + 'Launchfile requires "benchmark:=" and "producers:=" arguments ' \ + 'with coresponding config files.' + + if len(args) != 2: + raise RuntimeError(err_str) + + else: + for arg in args: + if 'benchmark:=' in arg: + bench_cfg_path = pathlib.Path(arg.replace('benchmark:=', '')) + if not bench_cfg_path.is_file(): + raise RuntimeError( + 'Batch config file {} does not exist.'.format(bench_cfg_path) + ) + elif 'producers:=' in arg: + producers_cfg_path = pathlib.Path(arg.replace('producers:=', '')) + if not producers_cfg_path.is_file(): + raise RuntimeError( + 'Producers config file {} does not exist.'.format(producers_cfg_path) + ) + else: + raise RuntimeError(err_str) + return bench_cfg_path, producers_cfg_path + + +def _copy_config_files(): + """Copy benchmark and producers config files to benchmark folder.""" + global _bench_cfg_path, _producers_cfg_path + # Copy yaml configs for current benchmark after benchmark is finished + benchmark_path = pathlib.Path(_producer_nodes[0]['parameters']['db_folder']) + shutil.copy(str(_bench_cfg_path), str(benchmark_path.with_name('benchmark.yaml'))) + shutil.copy(str(_producers_cfg_path), str(benchmark_path.with_name('producers.yaml'))) + + +def _launch_sequence(transport): + """ + Continue with launch sequence (launch entry action of next run). + + Launches next producer node or rosbag2 record process, based on transport (end to end) + or transportless type of benchmark. + + :param" transport If True launch a 'ros2 bag record' process, else a producer node. + """ + global _producer_idx, _producer_nodes, _rosbag_processes + if _producer_idx == len(_producer_nodes): + _copy_config_files() + return launch.actions.LogInfo(msg='Benchmark finished!') + action = None + if transport: + action = _rosbag_processes[_producer_idx] + else: + action = _producer_nodes[_producer_idx]['node'] + return action + + +def _rosbag_proc_started(event, context): + """Register current rosbag2 PID so we can terminate it when producer exits.""" + global _rosbag_pid + _rosbag_pid = event.pid + + +def _rosbag_ready_check(event): + """ + Consider rosbag2 ready when 'Listening for topics...' string is printed. + + Launches producer node if ready. + """ + target_str = 'Listening for topics...' + if target_str in event.text.decode(): + return _launch_sequence(transport=False) + + +def _rosbag_proc_exited(event, context): + """ + Start next rosbag2 record process after current one exits. + + Launches result writer on exit. + """ + global _producer_idx, _result_writers, _rosbag_pid + + # ROS2 bag returns 2 if terminated with SIGINT, which we expect here + if event.returncode != 2: + _rosbag_pid = None + return [ + launch.actions.LogInfo(msg='Rosbag2 record error. Shutting down benchmark.'), + launch.actions.EmitEvent( + event=launch.events.Shutdown( + reason='Rosbag2 record error' + ) + ) + ] + return [ + _result_writers[_producer_idx-1] + ] + + +def _producer_node_started(event, context): + """Log current benchmark progress on producer start.""" + global _producer_idx + return launch.actions.LogInfo( + msg='-----------{}/{}-----------'.format(_producer_idx + 1, len(_producer_nodes)) + ) + + +def _producer_node_exited(event, context): + """ + Launch new producer when current has finished. + + If transport is on, then stops rosbag2 recorder process. + + Handles clearing of bags. + """ + global _producer_idx, _producer_nodes, _rosbag_pid + node_params = _producer_nodes[_producer_idx]['parameters'] + transport = node_params['transport'] + + # Handle clearing bag files + if not node_params['preserve_bags']: + db_files = pathlib.Path.cwd().joinpath(node_params['db_folder']).glob('*.db3') + for f in db_files: + f.unlink() + + # If we have non empty rosbag PID, then we need to kill it (end-to-end transport case) + if _rosbag_pid is not None and transport: + os.kill(_rosbag_pid, signal.SIGINT) + _rosbag_pid = None + + # Shutdown benchmark with error if producer node crashes + if event.returncode != 0: + return [ + launch.actions.LogInfo(msg='Writer error. Shutting down benchmark.'), + launch.actions.EmitEvent( + event=launch.events.Shutdown( + reason='Writer error' + ) + ) + ] + + # Bump up producer index, so the launch sequence can continue + _producer_idx += 1 + return [ + launch.actions.LogInfo( + msg='---------------------------' + ), + _launch_sequence(transport=transport) + ] + + +def generate_launch_description(): + """Generate launch description for ros2 launch system.""" + global _producer_nodes, _bench_cfg_path, _producers_cfg_path + _bench_cfg_path, _producers_cfg_path = _parse_arguments() + + # Parse yaml config for benchmark + bench_cfg = None + with open(_bench_cfg_path, 'r') as config_file: + bench_cfg_yaml = yaml.load(config_file, Loader=yaml.FullLoader) + bench_cfg = (bench_cfg_yaml['rosbag2_performance_benchmarking'] + ['benchmark_node'] + ['ros__parameters']) + + # Benchmark options + benchmark_params = bench_cfg['benchmark'] + + repeat_each = benchmark_params.get('repeat_each') + db_root_folder = benchmark_params.get('db_root_folder') + summary_result_file = benchmark_params.get('summary_result_file') + transport = not benchmark_params.get('no_transport') + preserve_bags = benchmark_params.get('preserve_bags') + + # Producers options + producers_params = bench_cfg['benchmark']['parameters'] + + max_cache_size_params = producers_params.get('max_cache_size') + max_bag_size_params = producers_params.get('max_bag_size') + compression_params = producers_params.get('compression') + compression_queue_size_params = producers_params.get('compression_queue_size') + compression_threads_params = producers_params.get('compression_threads') + storage_config_file_params = producers_params.get('storage_config_file') + + # Parameters cross section for whole benchmark + # Parameters cross section is a list of all possible parameters variants + params_cross_section = [] + + # Generate unique benchmark directory name + benchmark_cfg_name = pathlib.Path(_bench_cfg_path).name.replace('.yaml', '') + producer_cfg_name = pathlib.Path(_producers_cfg_path).name.replace('.yaml', '') + timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + transport_postfix = 'transport' if transport else 'no_transport' + benchmark_dir_name = benchmark_cfg_name + \ + '_' + producer_cfg_name + \ + '_' + transport_postfix + \ + '_' + timestamp + + # Helper function for generating cross section list + def __generate_cross_section_parameter(i, + cache, + compression, + compression_queue_size, + compression_threads, + storage_config, + max_bag_size): + # Storage conf parameter for each producer + st_conf_filename = storage_config.replace('.yaml', '') + storage_conf_path = '' + if storage_config != '': + storage_conf_path = pathlib.Path( + get_package_share_directory( + 'rosbag2_performance_benchmarking' + ) + ).joinpath('config', 'storage', storage_config) + if not storage_conf_path.exists(): + raise RuntimeError( + 'Config {} does not exist.'.format(storage_config)) + st_conf_filename = pathlib.Path(storage_config).with_suffix('') + + # Generates unique title for producer + node_title = 'run_' + \ + '{i}_{cache}_{comp}_{comp_q}_{comp_t}_{st_conf}_{bag_size}'.format( + i=i, + cache=cache, + comp=compression if compression else 'default_compression', + comp_q=compression_queue_size, + comp_t=compression_threads, + st_conf=st_conf_filename if st_conf_filename else 'default_config', + bag_size=max_bag_size + ) + + # Result file path for producer + result_file = pathlib.Path(db_root_folder).joinpath( + benchmark_dir_name, + summary_result_file + ) + + # Database folder path for producer + db_folder = pathlib.Path(db_root_folder).joinpath( + benchmark_dir_name, + node_title + ) + + # Filling up parameters cross section list for benchmark + params_cross_section.append( + { + 'node_title': node_title, + 'db_folder': str(db_folder), + 'cache': cache, + 'preserve_bags': preserve_bags, + 'transport': transport, + 'result_file': str(result_file), + 'compression_format': compression, + 'compression_queue_size': compression_queue_size, + 'compression_threads': compression_threads, + 'storage_config_file': str(storage_conf_path), + 'config_file': str(_producers_cfg_path), + 'max_bag_size': max_bag_size + } + ) + + # For the sake of python indentation, multiple for loops in alternative way with helper func + [ + __generate_cross_section_parameter( + i, + cache, + compression, + compression_queue_size, + compression_threads, + storage_config, + max_bag_size) + for i in range(0, repeat_each) + for cache in max_cache_size_params + for compression in compression_params + for compression_queue_size in compression_queue_size_params + for compression_threads in compression_threads_params + for storage_config in storage_config_file_params + for max_bag_size in max_bag_size_params + ] + + ld = launch.LaunchDescription() + ld.add_action( + launch.actions.LogInfo(msg='Launching benchmark!'), + ) + + # Create all required nodes and processes for benchmark + for producer_param in params_cross_section: + parameters = [ + producer_param['config_file'], + {'max_cache_size': producer_param['cache']}, + {'max_bag_size': producer_param['max_bag_size']}, + {'db_folder': producer_param['db_folder']}, + {'results_file': producer_param['result_file']}, + {'compression_queue_size': producer_param['compression_queue_size']}, + {'compression_threads': producer_param['compression_threads']} + ] + + if producer_param['storage_config_file'] != '': + parameters.append({'storage_config_file': producer_param['storage_config_file']}) + if producer_param['compression_format'] != '': + parameters.append({'compression_format': producer_param['compression_format']}) + + if not transport: + # Writer benchmark node writes messages directly to a storage, uses no publishers + producer_node = launch_ros.actions.Node( + package='rosbag2_performance_benchmarking', + executable='writer_benchmark', + name='rosbag2_performance_benchmarking_node', + parameters=parameters + ) + else: + # Benchmark publishers node uses standard publishers for publishing messages + producer_node = launch_ros.actions.Node( + package='rosbag2_performance_benchmarking', + executable='benchmark_publishers', + name='rosbag2_performance_benchmarking_node', + parameters=parameters + ) + + # ROS2 bag process for recording messages + rosbag_args = [] + if producer_param['storage_config_file']: + rosbag_args += [ + '--storage-config-file', + str(producer_param['storage_config_file']) + ] + if producer_param['cache']: + rosbag_args += [ + '--max-cache-size', + str(producer_param['cache']) + ] + if producer_param['compression_format']: + rosbag_args += [ + '--compression-mode', + 'message' + ] + rosbag_args += [ + '--compression-format', + str(producer_param['compression_format']) + ] + if producer_param['compression_queue_size']: + rosbag_args += [ + '--compression-queue-size', + str(producer_param['compression_queue_size']) + ] + if producer_param['compression_threads']: + rosbag_args += [ + '--compression-threads', + str(producer_param['compression_threads']) + ] + if producer_param['max_bag_size']: + rosbag_args += [ + '-b', + str(producer_param['max_bag_size']) + ] + rosbag_args += ['-o', str(producer_param['db_folder'])] + rosbag_process = launch.actions.ExecuteProcess( + sigkill_timeout=launch.substitutions.LaunchConfiguration( + 'sigkill_timeout', default=60), + sigterm_timeout=launch.substitutions.LaunchConfiguration( + 'sigterm_timeout', default=60), + cmd=['ros2', 'bag', 'record', '-a'] + rosbag_args + ) + + # Result writer node walks through output metadata files and generates + # output results file + result_writer = launch_ros.actions.Node( + package='rosbag2_performance_benchmarking', + executable='results_writer', + name='rosbag2_performance_benchmarking_node', + parameters=parameters + ) + + # Fill up list with rosbag record process and result writers actions + _rosbag_processes.append(rosbag_process) + _result_writers.append(result_writer) + + # Fill up dict with producer nodes and their corresponding parameters + _producer_nodes.append({'node': producer_node, 'parameters': producer_param}) + + # Connect start and exit events for a proper sequence + if not transport: + for producer_node in _producer_nodes: + ld.add_action( + launch.actions.RegisterEventHandler( + launch.event_handlers.OnProcessExit( + target_action=producer_node['node'], + on_exit=_producer_node_exited + ) + ) + ) + ld.add_action( + launch.actions.RegisterEventHandler( + launch.event_handlers.OnProcessStart( + target_action=producer_node['node'], + on_start=_producer_node_started + ) + ) + ) + else: + for producer_node, rosbag_proc in zip(_producer_nodes, _rosbag_processes): + ld.add_action( + launch.actions.RegisterEventHandler( + launch.event_handlers.OnProcessExit( + target_action=producer_node['node'], + on_exit=_producer_node_exited + ) + ) + ) + ld.add_action( + launch.actions.RegisterEventHandler( + launch.event_handlers.OnProcessStart( + target_action=producer_node['node'], + on_start=_producer_node_started + ) + ) + ), + ld.add_action( + launch.actions.RegisterEventHandler( + launch.event_handlers.OnProcessStart( + target_action=rosbag_proc, + on_start=_rosbag_proc_started + ) + ) + ) + ld.add_action( + launch.actions.RegisterEventHandler( + launch.event_handlers.OnProcessIO( + target_action=rosbag_proc, + on_stdout=_rosbag_ready_check, + on_stderr=_rosbag_ready_check + ) + ) + ) + ld.add_action( + launch.actions.RegisterEventHandler( + launch.event_handlers.OnProcessExit( + target_action=rosbag_proc, + on_exit=_rosbag_proc_exited + ) + ) + ) + + # Launch nodes one after another. Next node is launched after previous is finished. + ld.add_action(_launch_sequence(transport=transport)) + + return ld + + +if __name__ == '__main__': + raise RuntimeError('Benchmark launchfile does not support standalone execution.') diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml b/rosbag2_performance/rosbag2_performance_benchmarking/package.xml similarity index 84% rename from rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml rename to rosbag2_performance/rosbag2_performance_benchmarking/package.xml index ce9316db43..5465e6791f 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/package.xml +++ b/rosbag2_performance/rosbag2_performance_benchmarking/package.xml @@ -1,9 +1,9 @@ - rosbag2_performance_writer_benchmarking + rosbag2_performance_benchmarking 0.5.0 - Code to benchmark the part of rosbag2 which starts after the callback is received + Code to benchmark rosbag2 Karsten Knese Michael Jeronimo Adam Dabrowski diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/scripts/report_gen.py b/rosbag2_performance/rosbag2_performance_benchmarking/scripts/report_gen.py new file mode 100755 index 0000000000..74025158dd --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/scripts/report_gen.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 + +# Copyright 2021, Robotec.ai sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tool for generating human friendly benchmark report.""" + +import argparse +import csv +import pathlib +import statistics + +import yaml + + +class Postprocess: + """Base class for posprocess calculations.""" + + def process(self, grouped_data, benchmark_config, producers_config): + raise NotImplementedError + + +class PostprocessStorageConfig(Postprocess): + """ + Postprocess. + + Calculate percent of recorded messages per storage config for different + benchmark parameters. + """ + + def process(self, grouped_data, benchmark_config, producers_config): + """ + Process grouped data and prints human friendly informations. + + :param: grouped data List of grouped results. Grouped result is a list with rows from + single benchmark run (ie. run with two publisher groups returns two rows in results + file). + :param: benchmark_config Benchmark description from yaml config. + :param: producers_config Producers description from yaml config. + """ + benchmark_config_cleaned = (benchmark_config['rosbag2_performance_benchmarking'] + ['benchmark_node'] + ['ros__parameters']) + benchmark_parameters = benchmark_config_cleaned['benchmark']['parameters'] + repeat_each = benchmark_config_cleaned['benchmark']['repeat_each'] + + producers_config_publishers = (producers_config['rosbag2_performance_benchmarking_node'] + ['ros__parameters'] + ['publishers']) + + # Split data for storage configs + splitted_data = {} + for data in grouped_data: + storage_cfg_name = data[0]['storage_config'] + storage_cfg_name = storage_cfg_name if storage_cfg_name != '' else 'default' + if storage_cfg_name not in splitted_data.keys(): + splitted_data.update({storage_cfg_name: []}) + splitted_data[storage_cfg_name].append(data) + + cache_data_per_storage_conf = {} + + print(yaml.dump(producers_config_publishers)) + + def __process_test(compression_selected, + compression_queue_size_selected, + compression_threads_selected, + max_bagfile_size_selected): + for storage_cfg_name, data in splitted_data.items(): + cache_samples = {} + for sample in data: + # Single sample contains multiple rows + if len(sample) != len(producers_config_publishers['publisher_groups']): + raise RuntimeError('Invalid number of records in results detected.') + + # These parameters are same for all rows in sample + # (multiple publishers in publisher group) + if sample[0]['compression'] != compression_selected: + continue + if int(sample[0]['compression_queue']) != compression_queue_size_selected: + continue + if int(sample[0]['compression_threads']) != compression_threads_selected: + continue + if int(sample[0]['max_bagfile_size']) != max_bagfile_size_selected: + continue + + if sample[0]['cache_size'] not in cache_samples.keys(): + cache_samples.update({sample[0]['cache_size']: []}) + + # TODO(piotr.jaroszek) WARNING, currently results in 'total_produced' column + # are correct (per publisher group), but 'total_recorded' is already summed + # for all the publisher groups! + sample_total_produced = 0 + for row in sample: + sample_total_produced += int(row['total_produced']) + cache_samples[sample[0]['cache_size']].append( + int(sample[0]['total_recorded_count'])/sample_total_produced) + + cache_avg_recorded_percentage = { + cache: statistics.mean(samples) + for cache, samples in cache_samples.items() + } + cache_data_per_storage_conf.update( + {storage_cfg_name: cache_avg_recorded_percentage} + ) + + result = { + 'repeat_each': repeat_each, + 'max_bagfile_size': max_bagfile_size_selected, + 'compression': compression_selected, + 'compression_threads': compression_threads_selected, + 'compression_queue_size': compression_queue_size_selected, + 'cache_data': cache_data_per_storage_conf + } + + print('Results: ') + print('\tRepetitions: {}'.format(result['repeat_each'])) + print('\tMax bagfile size: {}'.format(result['max_bagfile_size'])) + print('\tCompression: {}'.format( + result['compression'] if result['compression'] else '') + ) + print('\tCompression threads: {}'.format(result['compression_threads'])) + print('\tCompression queue size: {}'.format(result['compression_queue_size'])) + print('\tRecorded messages for different caches and storage config:') + for storage_cfg, caches in result['cache_data'].items(): + print('\t\tstorage config: {}:'.format(pathlib.Path(storage_cfg).name)) + for cache, percent_recorded in caches.items(): + print('\t\t\tcache [bytes]: {:,}: {:.2%} recorded'.format( + int(cache), + percent_recorded)) + + [ + __process_test( + compression_selected, + compression_queue_size_selected, + compression_threads_selected, + max_bagfile_size_selected) + for compression_selected in benchmark_parameters['compression'] + for compression_queue_size_selected in benchmark_parameters['compression_queue_size'] + for compression_threads_selected in benchmark_parameters['compression_threads'] + for max_bagfile_size_selected in benchmark_parameters['max_bag_size'] + ] + + +class Report: + """Report generator main class.""" + + def __init__(self, benchmark_dir): + """Initialize with config and results data.""" + self.__benchmark_dir = benchmark_dir + self.__load_configs() + self.__load_results() + + def generate(self): + """Handle data posprocesses.""" + psc = PostprocessStorageConfig() + psc.process( + self.__results_data, + self.__benchmark_config, + self.__producers_config + ) + + def __load_configs(self): + producers_config_path = pathlib.Path(self.__benchmark_dir).joinpath('producers.yaml') + benchmark_config_path = pathlib.Path(self.__benchmark_dir).joinpath('benchmark.yaml') + + with open(producers_config_path, 'r') as fp: + self.__producers_config = yaml.load(fp, Loader=yaml.FullLoader) + with open(benchmark_config_path, 'r') as fp: + self.__benchmark_config = yaml.load(fp, Loader=yaml.FullLoader) + + def __load_results(self): + results_path = pathlib.Path(self.__benchmark_dir).joinpath('results.csv') + + with open(results_path, mode='r') as fp: + reader = csv.DictReader(fp, delimiter=' ') + results = [] + + for result in reader: + results.append(result) + + publishers_groups = ( + self.__producers_config['rosbag2_performance_benchmarking_node'] + ['ros__parameters'] + ['publishers'] + ['publisher_groups']) + + publishers_groups_num = len(publishers_groups) + + # Group rows in results file, so that rows within same benchmark run are + # in one list + # Example: one benchmark run with two publisher groups returns two rows in results + # file. We want to group these. + results_grouped = [ + results[i:i+(publishers_groups_num)] + for i in range(0, len(results), publishers_groups_num) + ] + + self.__results_data = results_grouped + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--input', help='Benchmark results folder.') + args = parser.parse_args() + benchmark_dir = args.input + + if benchmark_dir: + raport = Report(benchmark_dir) + raport.generate() + else: + parser.print_help() diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/src/benchmark_publishers.cpp b/rosbag2_performance/rosbag2_performance_benchmarking/src/benchmark_publishers.cpp new file mode 100644 index 0000000000..592a4d5a52 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/src/benchmark_publishers.cpp @@ -0,0 +1,116 @@ +// Copyright 2020-2021, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "rosbag2_performance_benchmarking/byte_producer.hpp" +#include "rosbag2_performance_benchmarking/config_utils.hpp" +#include "rosbag2_performance_benchmarking/publisher_group_config.hpp" + +#include "rclcpp/executors/single_threaded_executor.hpp" +#include "rclcpp/node.hpp" +#include "rclcpp/qos.hpp" +#include "std_msgs/msg/byte_multi_array.hpp" + + +class BenchmarkPublishers : public rclcpp::Node +{ +public: + explicit BenchmarkPublishers(const std::string & name) + : rclcpp::Node(name) + { + configurations_ = config_utils::publisher_groups_from_node_parameters(*this); + if (configurations_.empty()) { + RCLCPP_ERROR(get_logger(), "No publishers/producers found in node parameters"); + return; + } + + create_benchmark_publishers_and_producers(); + } + + void run() + { + std::vector producer_threads; + for (auto & producer : producers_) { + producer_threads.push_back(std::thread(&ByteProducer::run, producer.get())); + } + + for (auto & thread : producer_threads) { + thread.join(); + } + } + +private: + void create_benchmark_publishers_and_producers() + { + const std::string topic_prefix(this->get_fully_qualified_name()); + auto wait_for_subs = config_utils::wait_for_subscriptions_from_node_parameters(*this); + + for (auto & c : configurations_) { + for (unsigned int i = 0; i < c.count; ++i) { + auto topic = topic_prefix + "/" + c.topic_root + "_" + std::to_string(i + 1); + auto pub = this->create_publisher(topic, c.qos); + publishers_.push_back(pub); + producers_.push_back( + std::make_unique( + c.producer_config, + [this, topic, wait_for_subs] { + if (!wait_for_subs) {return;} + if (!rclcpp::ok()) {return;} + const double max_subscription_wait_time = 5.0; + auto start_time = std::chrono::high_resolution_clock::now(); + while (!this->count_subscribers(topic)) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + auto current_time = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed = current_time - start_time; + if (elapsed.count() >= max_subscription_wait_time) { + throw std::runtime_error("Waited too long for " + topic); + } + } + }, + [pub](std::shared_ptr msg) { + pub->publish(*msg); + }, + [] { /* empty lambda */})); + } + } + } + + std::vector configurations_; + std::vector> producers_; + std::vector>> publishers_; +}; + + +int main(int argc, char * argv[]) +{ + rclcpp::init(argc, argv); + auto publishers_node = std::make_shared( + "rosbag2_performance_benchmarking_node"); + + rclcpp::executors::SingleThreadedExecutor executor; + executor.add_node(publishers_node); + + // The benchmark has its own control loop but uses spinning for parameters + std::thread spin_thread([&executor]() {executor.spin();}); + publishers_node->run(); + RCLCPP_INFO(publishers_node->get_logger(), "Benchmark terminated"); + rclcpp::shutdown(); + spin_thread.join(); + + return 0; +} diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/src/config_utils.cpp b/rosbag2_performance/rosbag2_performance_benchmarking/src/config_utils.cpp new file mode 100644 index 0000000000..61bfd022bf --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/src/config_utils.cpp @@ -0,0 +1,132 @@ +// Copyright 2021, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rosbag2_performance_benchmarking/config_utils.hpp" + +#include +#include + +#include "rclcpp/qos.hpp" +#include "rosbag2_performance_benchmarking/producer_config.hpp" + +namespace config_utils +{ + +void load_qos_configuration( + rclcpp::Node & node, + PublisherGroupConfig & group_config, + const std::string & group_prefix) +{ + auto qos_prefix = group_prefix + ".qos"; + node.declare_parameter(qos_prefix + ".qos_depth"); + node.declare_parameter(qos_prefix + ".qos_reliability"); + node.declare_parameter(qos_prefix + ".qos_durability"); + + unsigned int qos_depth = 10; + std::string qos_reliability, qos_durability; + node.get_parameter(qos_prefix + ".qos_depth", qos_depth); + node.get_parameter(qos_prefix + ".qos_reliability", qos_reliability); + node.get_parameter(qos_prefix + ".qos_durability", qos_durability); + + group_config.qos.keep_last(qos_depth); + // TODO(adamdbrw) - error handling / map string to function + if (qos_reliability == "reliable") {group_config.qos.reliable();} + if (qos_reliability == "best_effort") {group_config.qos.best_effort();} + if (qos_reliability == "transient_local") {group_config.qos.transient_local();} + if (qos_reliability == "volatile") {group_config.qos.durability_volatile();} +} + +bool wait_for_subscriptions_from_node_parameters(rclcpp::Node & node) +{ + const std::string parameters_ns = "publishers"; + node.declare_parameter(parameters_ns + ".wait_for_subscriptions", true); + bool wait_for_subscriptions; + node.get_parameter(parameters_ns + ".wait_for_subscriptions", wait_for_subscriptions); + return wait_for_subscriptions; +} + +std::vector publisher_groups_from_node_parameters( + rclcpp::Node & node) +{ + std::vector configurations; + std::vector publisher_groups; + const std::string parameters_ns = "publishers"; + node.declare_parameter(parameters_ns + ".publisher_groups"); + node.get_parameter(parameters_ns + ".publisher_groups", publisher_groups); + for (const auto & group_name : publisher_groups) { + auto group_prefix = parameters_ns + "." + group_name; + node.declare_parameter(group_prefix + ".publishers_count"); + node.declare_parameter(group_prefix + ".topic_root"); + node.declare_parameter(group_prefix + ".msg_size_bytes"); + node.declare_parameter(group_prefix + ".msg_count_each"); + node.declare_parameter(group_prefix + ".rate_hz"); + + PublisherGroupConfig group_config; + node.get_parameter( + group_prefix + ".publishers_count", + group_config.count); + node.get_parameter( + group_prefix + ".topic_root", + group_config.topic_root); + node.get_parameter( + group_prefix + ".msg_size_bytes", + group_config.producer_config.message_size); + node.get_parameter( + group_prefix + ".msg_count_each", + group_config.producer_config.max_count); + node.get_parameter( + group_prefix + ".rate_hz", + group_config.producer_config.frequency); + + if (group_config.producer_config.frequency == 0) { + RCLCPP_ERROR(node.get_logger(), "Frequency can't be 0. Exiting."); + rclcpp::shutdown(nullptr, "Invalid frequency parameter"); + return configurations; + } + + config_utils::load_qos_configuration(node, group_config, group_prefix); + + configurations.push_back(group_config); + } + return configurations; +} + +BagConfig bag_config_from_node_parameters( + rclcpp::Node & node) +{ + const std::string default_bag_folder("/tmp/rosbag2_test"); + BagConfig bag_config; + + node.declare_parameter("storage_id", "sqlite3"); + node.declare_parameter("max_cache_size", 10000000); + node.declare_parameter("max_bag_size", 0); + node.declare_parameter("db_folder", default_bag_folder); + node.declare_parameter("storage_config_file", ""); + node.declare_parameter("compression_format", ""); + node.declare_parameter("compression_queue_size", 1); + node.declare_parameter("compression_threads", 0); + + node.get_parameter("storage_id", bag_config.storage_options.storage_id); + node.get_parameter("max_cache_size", bag_config.storage_options.max_cache_size); + node.get_parameter("max_bag_size", bag_config.storage_options.max_bagfile_size); + node.get_parameter("db_folder", bag_config.storage_options.uri); + node.get_parameter("storage_config_file", bag_config.storage_options.storage_config_uri); + node.get_parameter("compression_format", bag_config.compression_format); + node.get_parameter("compression_queue_size", bag_config.compression_queue_size); + node.get_parameter("compression_threads", bag_config.compression_threads); + + return bag_config; +} + +} // namespace config_utils diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/src/result_utils.cpp b/rosbag2_performance/rosbag2_performance_benchmarking/src/result_utils.cpp new file mode 100644 index 0000000000..808d88a298 --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/src/result_utils.cpp @@ -0,0 +1,128 @@ +// Copyright 2021, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rosbag2_performance_benchmarking/result_utils.hpp" + +#include +#include +#include +#include +#include + +#include "rosbag2_storage/storage_options.hpp" +#include "rosbag2_storage/metadata_io.hpp" + +#include "rosbag2_performance_benchmarking/bag_config.hpp" +#include "rosbag2_performance_benchmarking/publisher_group_config.hpp" +#include "rosbag2_performance_benchmarking/config_utils.hpp" + +#ifdef _WIN32 +// This is necessary because of a bug in yaml-cpp's cmake +#define YAML_CPP_DLL +// This is necessary because yaml-cpp does not always use dllimport/dllexport consistently +# pragma warning(push) +# pragma warning(disable:4251) +# pragma warning(disable:4275) +#endif +#include "yaml-cpp/yaml.h" +#ifdef _WIN32 +# pragma warning(pop) +#endif + +namespace result_utils +{ + +/// Read total count of recorded messages from metadata.yaml file +int get_message_count_from_metadata(const std::string & uri) +{ + int total_recorded_count = 0; + std::string metadata_filename(rosbag2_storage::MetadataIo::metadata_filename); + std::string metadata_path = uri + "/" + metadata_filename; + try { + YAML::Node yaml_file = YAML::LoadFile(metadata_path); + total_recorded_count = yaml_file["rosbag2_bagfile_information"]["message_count"].as(); + } catch (const YAML::Exception & ex) { + throw std::runtime_error( + std::string("Exception on parsing metadata file to get total message count: ") + + metadata_path + " " + + ex.what()); + } + return total_recorded_count; +} + +/// Based on configuration and metadata from completed benchmark, write results +void write_benchmark_results( + const std::vector & publisher_groups_config, + const BagConfig & bag_config, + const std::string & results_file) +{ + bool new_file = false; + { // test if file exists - we want to write a csv header after creation if not + // use std::filesystem when switching to C++17 + std::ifstream test_existence(results_file); + if (!test_existence) { + new_file = true; + } + } + + // append, we want to accumulate results from multiple runs + std::ofstream output_file(results_file, std::ios_base::app); + if (!output_file.is_open()) { + throw std::runtime_error(std::string("Could not open file: ") + results_file); + } + + if (new_file) { + output_file << "instances frequency message_size total_messages_sent cache_size "; + output_file << "max_bagfile_size storage_config "; + output_file << "compression compression_queue compression_threads "; + output_file << "total_produced total_recorded_count\n"; + } + + int total_recorded_count = get_message_count_from_metadata(bag_config.storage_options.uri); + + for (const auto & c : publisher_groups_config) { + output_file << c.count << " "; + output_file << c.producer_config.frequency << " "; + output_file << c.producer_config.message_size << " "; + output_file << c.producer_config.max_count << " "; + output_file << bag_config.storage_options.max_cache_size << " "; + output_file << bag_config.storage_options.max_bagfile_size << " "; + output_file << bag_config.storage_options.storage_config_uri << " "; + output_file << bag_config.compression_format << " "; + output_file << bag_config.compression_queue_size << " "; + output_file << bag_config.compression_threads << " "; + + // TODO(adamdbrw) - this is a result for the entire group, + // but we don't yet have per-group stats. + // For now, these need to be summed for each group + auto total_messages_produced = c.producer_config.max_count * c.count; + output_file << total_messages_produced << " "; + output_file << total_recorded_count << std::endl; + } +} + +/// this version works with a standalone node using node parameters +void write_benchmark_results(rclcpp::Node & node) +{ + auto configurations = config_utils::publisher_groups_from_node_parameters(node); + auto bag_config = config_utils::bag_config_from_node_parameters(node); + + std::string results_file; + node.declare_parameter("results_file", bag_config.storage_options.uri + "/results.csv"); + node.get_parameter("results_file", results_file); + + write_benchmark_results(configurations, bag_config, results_file); +} + +} // namespace result_utils diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/main.cpp b/rosbag2_performance/rosbag2_performance_benchmarking/src/results_writer.cpp similarity index 60% rename from rosbag2_performance/rosbag2_performance_writer_benchmarking/src/main.cpp rename to rosbag2_performance/rosbag2_performance_benchmarking/src/results_writer.cpp index 9855d5a119..7fa3d7af12 100644 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/main.cpp +++ b/rosbag2_performance/rosbag2_performance_benchmarking/src/results_writer.cpp @@ -1,4 +1,4 @@ -// Copyright 2020, Robotec.ai sp. z o.o. +// Copyright 2021, Robotec.ai sp. z o.o. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,23 +13,25 @@ // limitations under the License. #include + #include "rclcpp/executors/single_threaded_executor.hpp" -#include "rosbag2_performance_writer_benchmarking/writer_benchmark.hpp" +#include "rclcpp/node.hpp" +#include "rosbag2_performance_benchmarking/result_utils.hpp" -// TODO(adamdbrw) the benchmark being ROS node is not necessary, only used for logging -// and parameters. Otherwise ROS dependencies are calls to rcl and rmw. int main(int argc, char * argv[]) { rclcpp::init(argc, argv); - auto bench = std::make_shared("rosbag2_performance_writer_benchmarking_node"); + auto results_writer_node = std::make_shared( + "benchmarking_results_writer"); + rclcpp::executors::SingleThreadedExecutor executor; - executor.add_node(bench); + executor.add_node(results_writer_node); - // The benchmark has its own control loop but uses spinning for parameters + // The benchmark uses spinning for parameters std::thread spin_thread([&executor]() {executor.spin();}); - bench->start_benchmark(); - RCLCPP_INFO(bench->get_logger(), "Benchmark terminated"); + result_utils::write_benchmark_results(*results_writer_node); rclcpp::shutdown(); spin_thread.join(); + return 0; } diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/src/writer_benchmark.cpp b/rosbag2_performance/rosbag2_performance_benchmarking/src/writer_benchmark.cpp new file mode 100644 index 0000000000..c9e9ffe38c --- /dev/null +++ b/rosbag2_performance/rosbag2_performance_benchmarking/src/writer_benchmark.cpp @@ -0,0 +1,210 @@ +// Copyright 2020, Robotec.ai sp. z o.o. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "rclcpp/executors/single_threaded_executor.hpp" +#include "rosbag2_compression/sequential_compression_writer.hpp" +#include "rosbag2_storage/serialized_bag_message.hpp" +#include "rmw/rmw.h" +#include "std_msgs/msg/byte_multi_array.hpp" + +#include "rosbag2_performance_benchmarking/config_utils.hpp" +#include "rosbag2_performance_benchmarking/result_utils.hpp" +#include "rosbag2_performance_benchmarking/writer_benchmark.hpp" + +using namespace std::chrono_literals; + +static rcutils_allocator_t allocator = rcutils_get_default_allocator(); + +WriterBenchmark::WriterBenchmark(const std::string & name) +: rclcpp::Node(name) +{ + RCLCPP_INFO(get_logger(), "WriterBenchmark parsing configurations"); + configurations_ = config_utils::publisher_groups_from_node_parameters(*this); + if (configurations_.empty()) { + RCLCPP_ERROR(get_logger(), "No publishers/producers found in node parameters"); + return; + } + + bag_config_ = config_utils::bag_config_from_node_parameters(*this); + if (bag_config_.storage_options.storage_id != "sqlite3") { + RCLCPP_ERROR(get_logger(), "Benchmarking only supported for sqlite3 for now"); + return; + } + + this->declare_parameter("results_file", bag_config_.storage_options.uri + "/results.csv"); + this->get_parameter("results_file", results_file_); + + RCLCPP_INFO(get_logger(), "configuration parameters processed"); + + create_producers(); + create_writer(); +} + +void WriterBenchmark::start_benchmark() +{ + RCLCPP_INFO(get_logger(), "Starting the WriterBenchmark"); + start_producers(); + while (rclcpp::ok()) { + int count = 0; + unsigned int complete_count = 0; + + // TODO(adamdbrw) Performance can be improved. Use conditional variables + for (auto & queue : queues_) { + if (queue->is_complete() && queue->is_empty()) { + ++complete_count; + } + + if (!queue->is_empty()) { // behave as if we received the message. + auto message = std::make_shared(); + + // The pointer memory is owned by the producer until past the termination of the while loop. + // Note this ownership model should be changed if we want to generate messages on the fly + auto byte_ma_message = queue->pop_and_return(); + + // The compressor may resize this array, so it needs to be initialized with + // rcutils_uint8_array_init to ensure the allocator is set properly. + auto msg_array = new rcutils_uint8_array_t; + *msg_array = rcutils_get_zero_initialized_uint8_array(); + int error = rcutils_uint8_array_init(msg_array, byte_ma_message->data.size(), &allocator); + if (error != RCUTILS_RET_OK) { + throw std::runtime_error( + "Error allocating resources for serialized message: " + + std::string(rcutils_get_error_string().str)); + } + // The compressor may modify this buffer in-place, so it should take ownership of it. + std::move( + byte_ma_message->data.data(), + byte_ma_message->data.data() + byte_ma_message->data.size(), + msg_array->buffer); + auto serialized_data = std::shared_ptr(msg_array); + serialized_data->buffer_length = byte_ma_message->data.size(); + + message->serialized_data = serialized_data; + + rcutils_time_point_value_t time_stamp; + error = rcutils_system_time_now(&time_stamp); + if (error != RCUTILS_RET_OK) { + RCLCPP_ERROR_STREAM( + get_logger(), "Error getting current time. Error:" << + rcutils_get_error_string().str); + } + message->time_stamp = time_stamp; + message->topic_name = queue->topic_name(); + + try { + writer_->write(message); + } catch (const std::runtime_error & e) { + RCLCPP_ERROR_STREAM(get_logger(), "Failed to record: " << e.what()); + } + ++count; + } + } + if (complete_count == queues_.size()) { + break; + } + + std::this_thread::sleep_for(1ms); + } + + for (auto & prod_thread : producer_threads_) { + prod_thread.join(); + } + writer_->reset(); + + result_utils::write_benchmark_results(configurations_, bag_config_, results_file_); +} + +void WriterBenchmark::create_producers() +{ + RCLCPP_INFO_STREAM(get_logger(), "creating producers"); + for (const auto & c : configurations_) { + RCLCPP_INFO_STREAM( + get_logger(), "\nWriterBenchmark: creating " << c.count << + " message producers with frequency " << c.producer_config.frequency << + " and message size in bytes " << c.producer_config.message_size << + " for topic root of " << c.topic_root << + ". Each will send " << c.producer_config.max_count << + " messages before terminating"); + const unsigned int queue_max_size = 10; + for (unsigned int i = 0; i < c.count; ++i) { + std::string topic = c.topic_root + std::to_string(i); + auto queue = std::make_shared(queue_max_size, topic); + queues_.push_back(queue); + producers_.push_back( + std::make_unique( + c.producer_config, + [] { /* empty lambda */}, + [queue](std::shared_ptr msg) { + queue->push(msg); + }, + [queue] { + queue->set_complete(); + })); + } + } +} + +void WriterBenchmark::create_writer() +{ + if (!bag_config_.compression_format.empty()) { + rosbag2_compression::CompressionOptions compression_options{ + bag_config_.compression_format, rosbag2_compression::CompressionMode::MESSAGE, + bag_config_.compression_queue_size, bag_config_.compression_threads}; + + writer_ = std::make_unique( + compression_options); + } else { + writer_ = std::make_shared(); + } + + // TODO(adamdbrw) generalize if converters are to be included in benchmarks + std::string serialization_format = rmw_get_serialization_format(); + writer_->open(bag_config_.storage_options, {serialization_format, serialization_format}); + + for (const auto & queue : queues_) { + rosbag2_storage::TopicMetadata topic; + topic.name = queue->topic_name(); + // TODO(adamdbrw) - replace with something more general if needed + topic.type = "std_msgs::msgs::ByteMultiArray"; + topic.serialization_format = serialization_format; + writer_->create_topic(topic); + } +} + +void WriterBenchmark::start_producers() +{ + for (auto & producer : producers_) { + producer_threads_.push_back(std::thread(&ByteProducer::run, producer.get())); + } +} + +int main(int argc, char * argv[]) +{ + rclcpp::init(argc, argv); + auto bench = std::make_shared("rosbag2_performance_benchmarking_node"); + rclcpp::executors::SingleThreadedExecutor executor; + executor.add_node(bench); + + // The benchmark has its own control loop but uses spinning for parameters + std::thread spin_thread([&executor]() {executor.spin();}); + bench->start_benchmark(); + RCLCPP_INFO(bench->get_logger(), "Benchmark terminated"); + rclcpp::shutdown(); + spin_thread.join(); + return 0; +} diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/README.md b/rosbag2_performance/rosbag2_performance_writer_benchmarking/README.md deleted file mode 100644 index ef820147fb..0000000000 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Rosbag2 writer benchmarking - -The primary package to test transport-less performance of the rosbag2 writer and storage. -Enables parametrized batch execution of benchmarks and tries to replicate the flow to capture message loss in queues. - -## How it works - -Use `scripts/benchmark.sh` to run an entire set of benchmarks. -These are currently aimed at several 100Mb/s scenarios. -Parameters are easy to change inside the script. - -By default, results will be written to `/tmp/rosbag2_test/[current_date]`. -The summary of benchmarks goes into `results.csv` file, which includes rows of execution parameters and results. -Benchmarks also produce execution logs in a series of sub-directories in `size[size]_inst[inst]_cache[cache]/` format. - -Database (bag) files are removed after recording to avoid filling up the disk. -To modify this behavior, modify the benchmark.sh script. - -## Building - -To build the package in the rosbag2 build process, make sure to turn `BUILD_ROSBAG2_BENCHMARKS` flag on (e.g. `colcon build --cmake-args -DBUILD_ROSBAG2_BENCHMARKS=1`) - -If you already built rosbag2, you can use `packages-select` option to build benchmarks. -Example: `colcon build --packages-select rosbag2_performance_writer_benchmarking --cmake-args -DBUILD_ROSBAG2_BENCHMARKS=1`. - -## General knowledge: I/O benchmarking - -#### Background: benchmarking disk writes on your system - -It might be useful to first understand what limitation your disk poses to the throughput of data recording. -Performance of bag write can't be higher over extended period of time (you can only use as much memory). - -**Using dd command** - -`dd if=/dev/zero of=/tmp/output conv=fdatasync bs=384k count=1k; rm -f /tmp/output` - -This method is not great for benchmarking the disk but an easy way to start since it requires no dependencies. -This will write zeros to the /tmp/output file with block size 384k, 1000 blocks, ends when write finishes. -Make sure to benchmark the disk which your bags will be written to (check your mount points and change “/tmp/output” to another path if needed). -Note: this depends on parameters used and whatever else is running on your system but can give you a ballpark figure when ran several times. - -**Using fio** - -For more sophisticated & accurate benchmarks, see the `fio` command. An example for big data blocks is: `fio --name TEST --eta-newline=5s --filename=fio-tempfile.dat --rw=write --size=500m --io_size=10g --blocksize=1024k --ioengine=libaio --fsync=10000 --iodepth=32 --direct=1 --numjobs=1 --runtime=60 --group_reporting`. - -#### Profiling bags I/O with tools - -Tools that can help in I/O profiling: `sudo apt-get install iotop ioping sysstat` -* `iotop` works similar as `top` command, but shows disk reads, writes, swaps and I/O %. Can be used at higher frequency in batch mode with specified process to deliver data that can be plotted. - * Example use: `sudo iotop -h -d 0.1 -t -b -o -p ` after running the bag. -* `ioping` can be used to check latency of requests to device -* `strace` can help determine syscalls associated with the bottleneck. - * Example use: `strace -c ros2 bag record /image --max-cache-size 10 -o ./tmp`. You will see a report after finishing recording with Ctrl-C. diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh deleted file mode 100755 index a56b53a80d..0000000000 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/benchmark.sh +++ /dev/null @@ -1,74 +0,0 @@ -# A very rough script to run batches of experiments. Missing checks and command line parametrization - -trap ctrlc SIGINT -ctrlc_sent=0 - -function ctrlc() -{ - let ctrlc_sent=1 -} - -test_dir=/tmp/rosbag2_test/$(date +%Y%m%d_%H%M%S) -mkdir -p ${test_dir} -echo "${test_dir} created" -db_path=${test_dir}/bag -rm -fr ${db_path} -script_dir=$(dirname $(readlink -f "$0")) - -storage_config_dir=${script_dir}/storage_config -config_optimized_path=${storage_config_dir}/storage_optimized.yaml -config_default_path=${storage_config_dir}/storage_default.yaml - -# Set this to "zstd" to test with compression -compression_format="" -compression_arg="" -if [ -n "$compression_format" ]; then - compression_arg="-p compression_format:=${compression_format}" -fi - -for storage_config_file_path in ${config_optimized_path} ${config_default_path} -do - config_file_name=$(basename $storage_config_file_path) - summary_file=${test_dir}/results_${config_file_name}.csv - for cache in 0 1000000 100000000 500000000 - do - for sz in 10000 100000 1000000 5000000 - do - for inst in 1 10 100 1000 - do - freq=100; #Hz - let total=$inst*$sz*$freq - if [[ $total -lt 100000000 || $total -gt 500000000 ]]; then - #echo "skipping the case of ${inst} instances and size ${sz}" - continue - fi - echo "processing case of ${inst} instances and size ${sz} with cache ${cache} and config ${config_file_name}" - outdir=${test_dir}/size${sz}_inst${inst}_cache${cache} - mkdir -p ${outdir} - for try in 1 2 3 - do - outfile=${outdir}/${try}.log - echo "Results will be written to file: ${outfile}" - ros2 run rosbag2_performance_writer_benchmarking writer_benchmark --ros-args \ - ${compression_arg} \ - -p frequency:=${freq} \ - -p size:=${sz} \ - -p instances:=${inst} \ - -p max_cache_size:=${cache} \ - -p db_folder:=${db_path} \ - -p storage_config_file:=${storage_config_file_path} \ - -p results_file:=${summary_file} \ - --ros-args -r __node:=rosbag2_performance_writer_benchmarking_node_batch \ - 2> ${outfile} - sleep 2 # making sure to flush both application and disk caches and have a fresh start - cp ${db_path}/metadata.yaml ${outdir}/${try}_metadata.yaml # preserve metadata from test - rm -fr ${db_path} # but remove large database file so we won't run out of disk space - if [[ $ctrlc_sent -eq 1 ]]; then - echo -e "\e[31mQuitting prematurely due to Ctrl-C - some results aren't saved and some won't be reliable\e[0m]" - exit - fi - done - done - done - done -done diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_optimized.yaml b/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_optimized.yaml deleted file mode 100644 index f93e23d813..0000000000 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/scripts/storage_config/storage_optimized.yaml +++ /dev/null @@ -1,2 +0,0 @@ -write: - pragmas: ["journal_mode = MEMORY", "synchronous = OFF"] diff --git a/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp b/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp deleted file mode 100644 index d71a5fe050..0000000000 --- a/rosbag2_performance/rosbag2_performance_writer_benchmarking/src/writer_benchmark.cpp +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2020, Robotec.ai sp. z o.o. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -#include "rmw/rmw.h" -#include "rosbag2_compression/sequential_compression_writer.hpp" -#include "rosbag2_cpp/storage_options.hpp" -#include "rosbag2_storage/serialized_bag_message.hpp" -#include "rosbag2_storage/storage_options.hpp" -#include "std_msgs/msg/byte_multi_array.hpp" - -#include "rosbag2_performance_writer_benchmarking/writer_benchmark.hpp" - -#ifdef _WIN32 -// This is necessary because of a bug in yaml-cpp's cmake -#define YAML_CPP_DLL -// This is necessary because yaml-cpp does not always use dllimport/dllexport consistently -# pragma warning(push) -# pragma warning(disable:4251) -# pragma warning(disable:4275) -#endif -#include "yaml-cpp/yaml.h" -#ifdef _WIN32 -# pragma warning(pop) -#endif - -using namespace std::chrono_literals; - -static rcutils_allocator_t allocator = rcutils_get_default_allocator(); - -WriterBenchmark::WriterBenchmark(const std::string & name) -: rclcpp::Node(name) -{ - const std::string default_bag_folder("/tmp/rosbag2_test"); - this->declare_parameter("frequency", 100); - this->declare_parameter("max_count", 1000); - this->declare_parameter("size", 1000000); - this->declare_parameter("instances", 1); - this->declare_parameter("max_cache_size", 10000000); - this->declare_parameter("max_bag_size", 0); - this->declare_parameter("db_folder", default_bag_folder); - this->declare_parameter("results_file", default_bag_folder + "/results.csv"); - this->declare_parameter("storage_config_file", ""); - this->declare_parameter("compression_format", ""); - this->declare_parameter("compression_queue_size", 1); - this->declare_parameter("compression_threads", 0); - - this->get_parameter("frequency", config_.frequency); - if (config_.frequency == 0) { - RCLCPP_ERROR(this->get_logger(), "Frequency can't be 0. Exiting."); - rclcpp::shutdown(nullptr, "frequency error"); - return; - } - - storage_options_.storage_id = "sqlite3"; - this->get_parameter("max_cache_size", storage_options_.max_cache_size); - this->get_parameter("db_folder", storage_options_.uri); - this->get_parameter("storage_config_file", storage_options_.storage_config_uri); - this->get_parameter("max_bag_size", storage_options_.max_bagfile_size); - - this->get_parameter("results_file", results_file_); - this->get_parameter("max_count", config_.max_count); - this->get_parameter("size", config_.message_size); - this->get_parameter("instances", instances_); - this->get_parameter("compression_format", compression_format_); - this->get_parameter("compression_queue_size", compression_queue_size_); - this->get_parameter("compression_threads", compression_threads_); - - create_producers(config_); - create_writer(); -} - -void WriterBenchmark::start_benchmark() -{ - RCLCPP_INFO(get_logger(), "Starting"); - start_producers(); - while (rclcpp::ok()) { - int count = 0; - unsigned int complete_count = 0; - - // TODO(adamdbrw) Performance can be improved. Use conditional variables - for (auto & queue : queues_) { - if (queue->is_complete() && queue->is_empty()) { - ++complete_count; - } - - if (!queue->is_empty()) { // behave as if we received the message. - auto message = std::make_shared(); - - // The pointer memory is owned by the producer until past the termination of the while loop. - // Note this ownership model should be changed if we want to generate messages on the fly - auto byte_ma_message = queue->pop_and_return(); - - // The compressor may resize this array, so it needs to be initialized with - // rcutils_uint8_array_init to ensure the allocator is set properly. - auto msg_array = new rcutils_uint8_array_t; - *msg_array = rcutils_get_zero_initialized_uint8_array(); - int error = rcutils_uint8_array_init(msg_array, byte_ma_message->data.size(), &allocator); - if (error != RCUTILS_RET_OK) { - throw std::runtime_error( - "Error allocating resources for serialized message: " + - std::string(rcutils_get_error_string().str)); - } - // The compressor may modify this buffer in-place, so it should take ownership of it. - std::move( - byte_ma_message->data.data(), - byte_ma_message->data.data() + byte_ma_message->data.size(), - msg_array->buffer); - auto serialized_data = std::shared_ptr(msg_array); - serialized_data->buffer_length = byte_ma_message->data.size(); - - message->serialized_data = serialized_data; - - rcutils_time_point_value_t time_stamp; - error = rcutils_system_time_now(&time_stamp); - if (error != RCUTILS_RET_OK) { - RCLCPP_ERROR_STREAM( - get_logger(), "Error getting current time. Error:" << - rcutils_get_error_string().str); - } - message->time_stamp = time_stamp; - message->topic_name = queue->topic_name(); - - try { - writer_->write(message); - } catch (const std::runtime_error & e) { - RCLCPP_ERROR_STREAM(get_logger(), "Failed to record: " << e.what()); - } - ++count; - } - } - if (complete_count == queues_.size()) { - break; - } - - std::this_thread::sleep_for(1ms); - } - - for (auto & prod_thread : producer_threads_) { - prod_thread.join(); - } - - writer_->reset(); - write_results(); -} - -int WriterBenchmark::get_message_count_from_metadata() const -{ - int total_recorded_count = 0; - std::string metadata_filename(rosbag2_storage::MetadataIo::metadata_filename); - std::string metadata_path = storage_options_.uri + "/" + metadata_filename; - try { - YAML::Node yaml_file = YAML::LoadFile(metadata_path); - total_recorded_count = yaml_file["rosbag2_bagfile_information"]["message_count"].as(); - } catch (const YAML::Exception & ex) { - throw std::runtime_error( - std::string("Exception on parsing metadata file to get total message count: ") + - metadata_path + " " + - ex.what()); - } - return total_recorded_count; -} - -void WriterBenchmark::write_results() const -{ - int total_recorded_count = get_message_count_from_metadata(); - auto total_messages_sent = config_.max_count * producers_.size(); - float percentage_recorded = static_cast(total_recorded_count * 100.0f) / - total_messages_sent; - - RCLCPP_INFO_STREAM( - get_logger(), "Percentage of all messages that were successfully recorded: " << - percentage_recorded); - - bool new_file = false; - { // test if file exists - we want to write a csv header after creation if not - // use std::filesystem when switching to C++17 - std::ifstream test_existence(results_file_); - if (!test_existence) { - new_file = true; - } - } - - // append, we want to accumulate results from multiple runs - std::ofstream output_file(results_file_, std::ios_base::app); - if (!output_file.is_open()) { - RCLCPP_ERROR_STREAM(get_logger(), "Could not open file " << results_file_); - return; - } - - if (new_file) { - output_file << "instances frequency message_size cache_size total_messages_sent "; - output_file << "percentage_recorded\n"; - } - - // configuration of the test. TODO(adamdbrw) wrap into a dict and define << operator. - output_file << instances_ << " "; - output_file << config_.frequency << " "; - output_file << config_.message_size << " "; - output_file << storage_options_.max_cache_size << " "; - output_file << total_messages_sent << " "; - - // results of the test. Use std::setprecision if preferred - output_file << percentage_recorded << std::endl; -} - -void WriterBenchmark::create_producers(const ProducerConfig & config) -{ - RCLCPP_INFO_STREAM( - get_logger(), "\nWriterBenchmark: creating " << instances_ << - " message producers with frequency " << config.frequency << - " and message size in bytes " << config.message_size << - ". Cache is " << storage_options_.max_cache_size << - ". Each will send " << config.max_count << - " messages before terminating"); - const unsigned int queue_max_size = 10; - for (unsigned int i = 0; i < instances_; ++i) { - std::string topic = "/writer_benchmark/producer " + std::to_string(i); - auto queue = std::make_shared(queue_max_size, topic); - queues_.push_back(queue); - producers_.push_back(std::make_unique(config, queue)); - } -} - -// TODO(adamdbrw) extend to other writers - based on parametrization -// Also, add an option to configure compression -void WriterBenchmark::create_writer() -{ - if (!compression_format_.empty()) { - rosbag2_compression::CompressionOptions compression_options{ - compression_format_, rosbag2_compression::CompressionMode::MESSAGE, - compression_queue_size_, compression_threads_}; - - writer_ = std::make_unique( - compression_options); - } else { - writer_ = std::make_shared(); - } - - // TODO(adamdbrw) generalize if converters are to be included in benchmarks - std::string serialization_format = rmw_get_serialization_format(); - writer_->open(storage_options_, {serialization_format, serialization_format}); - - for (const auto & queue : queues_) { - rosbag2_storage::TopicMetadata topic; - topic.name = queue->topic_name(); - // TODO(adamdbrw) - replace with something more general if needed - topic.type = "std_msgs::msgs::ByteMultiArray"; - topic.serialization_format = serialization_format; - writer_->create_topic(topic); - } -} - -void WriterBenchmark::start_producers() -{ - for (auto & producer : producers_) { - producer_threads_.push_back(std::thread(&ByteProducer::run, producer.get())); - } -} From 067ad4b8d8c9196da33fc4957104250c6c6fe578 Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Tue, 26 Jan 2021 17:05:09 -0800 Subject: [PATCH 69/77] Fix deadlock race condition on compression shutdown (#616) * Synchronize compression shutdown correctly, avoiding occasional deadlock Signed-off-by: Emerson Knapp --- .../sequential_compression_writer.hpp | 5 ++++- .../sequential_compression_writer.cpp | 17 ++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp b/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp index d970457434..3615755a89 100644 --- a/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp +++ b/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp @@ -166,7 +166,10 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionWriter compressor_message_queue_ RCPPUTILS_TSA_GUARDED_BY(compressor_queue_mutex_); std::queue compressor_file_queue_ RCPPUTILS_TSA_GUARDED_BY(compressor_queue_mutex_); std::vector compression_threads_; - std::atomic_bool compression_is_running_{false}; + /* *INDENT-OFF* */ // uncrustify doesn't understand the macro + brace initializer + std::atomic_bool compression_is_running_ + RCPPUTILS_TSA_GUARDED_BY(compressor_queue_mutex_) {false}; + /* *INDENT-ON* */ std::recursive_mutex storage_mutex_; std::condition_variable compressor_condition_; diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp index f8b09ecd6d..1497edecb5 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp @@ -73,11 +73,16 @@ void SequentialCompressionWriter::compression_thread_fn() "Cannot compress message; Writer is not open!"}; } - while (compression_is_running_) { + while (true) { std::shared_ptr message; std::string file; { + // This mutex synchronizes both the condition and the running_ boolean, so it has to be + // held when dealing with either/both std::unique_lock lock(compressor_queue_mutex_); + if (!compression_is_running_) { + break; + } compressor_condition_.wait(lock); if (!compressor_message_queue_.empty()) { message = compressor_message_queue_.front(); @@ -137,7 +142,10 @@ void SequentialCompressionWriter::setup_compressor_threads() ROSBAG2_COMPRESSION_LOG_DEBUG_STREAM( "setup_compressor_threads: Starting " << compression_options_.compression_threads << " threads"); - compression_is_running_ = true; + { + std::unique_lock lock(compressor_queue_mutex_); + compression_is_running_ = true; + } // This function needs to throw an exception if the compression format is invalid, but because // each thread creates its own compressor, we can't actually catch it here if one of the threads @@ -159,7 +167,10 @@ void SequentialCompressionWriter::stop_compressor_threads() { if (!compression_threads_.empty()) { ROSBAG2_COMPRESSION_LOG_DEBUG("Waiting for compressor threads to finish."); - compression_is_running_ = false; + { + std::unique_lock lock(compressor_queue_mutex_); + compression_is_running_ = false; + } compressor_condition_.notify_all(); for (auto & thread : compression_threads_) { thread.join(); From 320659a2b9d82a4a7bd9a313f88d2c0ac78d5da1 Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Wed, 27 Jan 2021 15:10:58 -0800 Subject: [PATCH 70/77] Fix relative metadata paths in SequentialCompressionWriter (#613) * Fix relative metadata path writing in compression by deduplicating business logic Signed-off-by: Emerson Knapp --- .../sequential_compression_writer.hpp | 11 +- .../sequential_compression_writer.cpp | 61 ++++--- .../test_sequential_compression_writer.cpp | 163 ++++++++++++------ .../test_rosbag2_record_end_to_end.cpp | 2 +- 4 files changed, 158 insertions(+), 79 deletions(-) diff --git a/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp b/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp index 3615755a89..ee4ff7479f 100644 --- a/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp +++ b/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp @@ -123,10 +123,17 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionWriter /** * Compress a file and update the metadata file path. * + * Note: this may log an error without raising an exception in the case that the input file + * could not be deleted after compressing. This is an error and should never happen, but given + * that the desired output is created, execution will not be halted. + * * \param compressor An initialized compression context. - * \param message The URI of the file to compress. + * \param file_relative_to_bag Relative path of the file to compress, as stored in metadata - + * meaning the path is relative to the bag base folder. */ - virtual void compress_file(BaseCompressorInterface & compressor, const std::string & file); + virtual void compress_file( + BaseCompressorInterface & compressor, + const std::string & file_relative_to_bag); /** * Checks if the compression by message option is specified and a compressor exists. diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp index 1497edecb5..e1d16611d4 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp @@ -77,19 +77,24 @@ void SequentialCompressionWriter::compression_thread_fn() std::shared_ptr message; std::string file; { - // This mutex synchronizes both the condition and the running_ boolean, so it has to be - // held when dealing with either/both std::unique_lock lock(compressor_queue_mutex_); - if (!compression_is_running_) { - break; - } - compressor_condition_.wait(lock); + compressor_condition_.wait( + lock, + [&] { + return !compression_is_running_ || + !compressor_message_queue_.empty() || + !compressor_file_queue_.empty(); + }); + if (!compressor_message_queue_.empty()) { message = compressor_message_queue_.front(); compressor_message_queue_.pop(); } else if (!compressor_file_queue_.empty()) { file = compressor_file_queue_.front(); compressor_file_queue_.pop(); + } else if (!compression_is_running_) { + // I woke up, all work queues are empty, and the main thread has stopped execution. Exit. + break; } } @@ -111,11 +116,7 @@ void SequentialCompressionWriter::compression_thread_fn() void SequentialCompressionWriter::init_metadata() { std::lock_guard lock(storage_mutex_); - metadata_ = rosbag2_storage::BagMetadata{}; - metadata_.storage_identifier = storage_->get_storage_identifier(); - metadata_.starting_time = std::chrono::time_point{ - std::chrono::nanoseconds::max()}; - metadata_.relative_file_paths = {storage_->get_relative_file_path()}; + SequentialWriter::init_metadata(); metadata_.compression_format = compression_options_.compression_format; metadata_.compression_mode = rosbag2_compression::compression_mode_to_string(compression_options_.compression_mode); @@ -240,38 +241,43 @@ void SequentialCompressionWriter::remove_topic( void SequentialCompressionWriter::compress_file( BaseCompressorInterface & compressor, - const std::string & file) + const std::string & file_relative_to_bag) { - ROSBAG2_COMPRESSION_LOG_DEBUG("Compressing file: %s", file.c_str()); - const auto to_compress = rcpputils::fs::path{file}; + using rcpputils::fs::path; + + const auto file_relative_to_pwd = path(base_folder_) / file_relative_to_bag; + ROSBAG2_COMPRESSION_LOG_INFO_STREAM("Compressing file: " << file_relative_to_pwd.string()); - if (to_compress.exists() && to_compress.file_size() > 0u) { - const auto compressed_uri = compressor.compress_uri(to_compress.string()); + if (file_relative_to_pwd.exists() && file_relative_to_pwd.file_size() > 0u) { + const auto compressed_uri = compressor.compress_uri(file_relative_to_pwd.string()); + const auto relative_compressed_uri = path(compressed_uri).filename(); { // After we've compressed the file, replace the name in the file list with the new name. // Must search for the entry because other threads may have changed the order of the vector // and invalidated any index or iterator we held to it. std::lock_guard lock(storage_mutex_); - auto iter = std::find( + const auto iter = std::find( metadata_.relative_file_paths.begin(), metadata_.relative_file_paths.end(), - file); + file_relative_to_bag); if (iter != metadata_.relative_file_paths.end()) { - *iter = compressed_uri; + *iter = relative_compressed_uri.string(); } else { ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( - "Failed to find path to uncompressed bag: \"" << file << + "Failed to find path to uncompressed bag: \"" << file_relative_to_pwd.string() << "\"; this shouldn't happen."); } } - if (!rcpputils::fs::remove(to_compress)) { + if (!rcpputils::fs::remove(file_relative_to_pwd)) { ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( - "Failed to remove uncompressed bag: \"" << to_compress.string() << "\""); + "Failed to remove original pre-compressed bag file: \"" << + file_relative_to_pwd.string() << "\". This should never happen - but execution " << + "will not be halted because the compressed output was successfully created."); } } else { ROSBAG2_COMPRESSION_LOG_DEBUG_STREAM( - "Removing last file: \"" << to_compress.string() << + "Removing last file: \"" << file_relative_to_pwd.string() << "\" because it either is empty or does not exist."); } } @@ -281,13 +287,14 @@ void SequentialCompressionWriter::split_bagfile() std::lock_guard lock(storage_mutex_); std::lock_guard compressor_lock(compressor_queue_mutex_); - switch_to_next_storage(); + // Grab last file before calling common splitting logic, which pushes the new filename + const auto last_file = metadata_.relative_file_paths.back(); + SequentialWriter::split_bagfile(); // If we're in FILE compression mode, push this file's name on to the queue so another // thread will handle compressing it. If not, we can just carry on. if (compression_options_.compression_mode == rosbag2_compression::CompressionMode::FILE) { - std::string file = metadata_.relative_file_paths.back(); - compressor_file_queue_.push(file); + compressor_file_queue_.push(last_file); compressor_condition_.notify_one(); } @@ -296,8 +303,6 @@ void SequentialCompressionWriter::split_bagfile() // storage plugin. should_compress_last_file_ = false; } - - metadata_.relative_file_paths.push_back(storage_->get_relative_file_path()); } void SequentialCompressionWriter::compress_message( diff --git a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp index 60a14bbb2e..aa9aaeaed5 100644 --- a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp +++ b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp @@ -14,6 +14,7 @@ #include +#include #include #include #include @@ -50,11 +51,73 @@ class SequentialCompressionWriterTest : public Test serialization_format_{"rmw_format"} { tmp_dir_storage_options_.uri = tmp_dir_.string(); - rcpputils::fs::remove(tmp_dir_); + rcpputils::fs::remove_all(tmp_dir_); ON_CALL(*storage_factory_, open_read_write(_)).WillByDefault(Return(storage_)); EXPECT_CALL(*storage_factory_, open_read_write(_)).Times(AtLeast(0)); + // intercept the metadata write so we can analyze it. + ON_CALL(*metadata_io_, write_metadata).WillByDefault( + [this](const std::string &, const rosbag2_storage::BagMetadata & metadata) { + intercepted_metadata_ = metadata; + }); } + ~SequentialCompressionWriterTest() + { + rcpputils::fs::remove_all(tmp_dir_); + } + + void initializeFakeFileStorage() + { + // Create mock implementation of the storage, using files and a size of 1 per message + // initialize values when opening a new bagfile + ON_CALL(*storage_factory_, open_read_write(_)).WillByDefault( + DoAll( + Invoke( + [this](const rosbag2_storage::StorageOptions & storage_options) { + fake_storage_size_ = 0; + fake_storage_uri_ = storage_options.uri; + // Touch the file + std::ofstream output(storage_options.uri); + ASSERT_TRUE(output.is_open()); + // Put some arbitrary bytes in the file so it isn't interpreted as being empty + output << "Fake storage data" << std::endl; + output.close(); + }), + Return(storage_))); + ON_CALL( + *storage_, + write(An>())).WillByDefault( + [this](std::shared_ptr) { + fake_storage_size_ += 1; + }); + ON_CALL(*storage_, get_bagfile_size).WillByDefault( + [this]() { + return fake_storage_size_; + }); + ON_CALL(*storage_, get_relative_file_path).WillByDefault( + [this]() { + return fake_storage_uri_; + }); + } + + void initializeWriter( + const rosbag2_compression::CompressionOptions & compression_options, + std::unique_ptr custom_compression_factory = nullptr) + { + auto compression_factory = std::move(custom_compression_factory); + if (!compression_factory) { + compression_factory = std::make_unique(); + } + auto sequential_writer = std::make_unique( + compression_options, + std::move(compression_factory), + std::move(storage_factory_), + converter_factory_, + std::move(metadata_io_)); + writer_ = std::make_unique(std::move(sequential_writer)); + } + + const std::string bag_name_ = "SequentialCompressionWriterTest"; std::unique_ptr> storage_factory_; std::shared_ptr> storage_; std::shared_ptr> converter_factory_; @@ -62,7 +125,10 @@ class SequentialCompressionWriterTest : public Test std::unique_ptr writer_; rcpputils::fs::path tmp_dir_; rosbag2_storage::StorageOptions tmp_dir_storage_options_; + rosbag2_storage::BagMetadata intercepted_metadata_; std::string serialization_format_; + uint64_t fake_storage_size_; + std::string fake_storage_uri_; const uint64_t kDefaultCompressionQueueSize = 1; const uint64_t kDefaultCompressionQueueThreads = 4; @@ -73,15 +139,7 @@ TEST_F(SequentialCompressionWriterTest, open_throws_on_empty_storage_options_uri rosbag2_compression::CompressionOptions compression_options{ "zstd", rosbag2_compression::CompressionMode::FILE, kDefaultCompressionQueueSize, kDefaultCompressionQueueThreads}; - auto compression_factory = std::make_unique(); - - auto sequential_writer = std::make_unique( - compression_options, - std::move(compression_factory), - std::move(storage_factory_), - converter_factory_, - std::move(metadata_io_)); - writer_ = std::make_unique(std::move(sequential_writer)); + initializeWriter(compression_options); EXPECT_THROW( writer_->open( @@ -95,21 +153,11 @@ TEST_F(SequentialCompressionWriterTest, open_throws_on_bad_compression_format) rosbag2_compression::CompressionOptions compression_options{ "bad_format", rosbag2_compression::CompressionMode::FILE, kDefaultCompressionQueueSize, kDefaultCompressionQueueThreads}; - auto compression_factory = std::make_unique(); - - auto sequential_writer = std::make_unique( - compression_options, - std::move(compression_factory), - std::move(storage_factory_), - converter_factory_, - std::move(metadata_io_)); - writer_ = std::make_unique(std::move(sequential_writer)); + initializeWriter(compression_options); EXPECT_THROW( writer_->open(tmp_dir_storage_options_, {serialization_format_, serialization_format_}), std::invalid_argument); - - EXPECT_TRUE(rcpputils::fs::remove(tmp_dir_)); } TEST_F(SequentialCompressionWriterTest, open_throws_on_invalid_splitting_size) @@ -117,7 +165,6 @@ TEST_F(SequentialCompressionWriterTest, open_throws_on_invalid_splitting_size) rosbag2_compression::CompressionOptions compression_options{ "zstd", rosbag2_compression::CompressionMode::FILE, kDefaultCompressionQueueSize, kDefaultCompressionQueueThreads}; - auto compression_factory = std::make_unique(); // Set minimum file size greater than max bagfile size option const uint64_t min_split_file_size = 10; @@ -127,13 +174,7 @@ TEST_F(SequentialCompressionWriterTest, open_throws_on_invalid_splitting_size) storage_options.max_bagfile_size = max_bagfile_size; storage_options.uri = "foo.bar"; - auto sequential_writer = std::make_unique( - compression_options, - std::move(compression_factory), - std::move(storage_factory_), - converter_factory_, - std::move(metadata_io_)); - writer_ = std::make_unique(std::move(sequential_writer)); + initializeWriter(compression_options); EXPECT_THROW( writer_->open(storage_options, {serialization_format_, serialization_format_}), @@ -145,15 +186,7 @@ TEST_F(SequentialCompressionWriterTest, open_succeeds_on_supported_compression_f rosbag2_compression::CompressionOptions compression_options{ "zstd", rosbag2_compression::CompressionMode::FILE, kDefaultCompressionQueueSize, kDefaultCompressionQueueThreads}; - auto compression_factory = std::make_unique(); - - auto sequential_writer = std::make_unique( - compression_options, - std::move(compression_factory), - std::move(storage_factory_), - converter_factory_, - std::move(metadata_io_)); - writer_ = std::make_unique(std::move(sequential_writer)); + initializeWriter(compression_options); auto tmp_dir = rcpputils::fs::temp_directory_path() / "path_not_empty"; auto storage_options = rosbag2_storage::StorageOptions(); @@ -161,8 +194,6 @@ TEST_F(SequentialCompressionWriterTest, open_succeeds_on_supported_compression_f EXPECT_NO_THROW( writer_->open(tmp_dir_storage_options_, {serialization_format_, serialization_format_})); - - EXPECT_TRUE(rcpputils::fs::remove(tmp_dir_)); } TEST_F(SequentialCompressionWriterTest, writer_calls_create_compressor) @@ -173,19 +204,55 @@ TEST_F(SequentialCompressionWriterTest, writer_calls_create_compressor) auto compression_factory = std::make_unique>(); EXPECT_CALL(*compression_factory, create_compressor(_)).Times(1); - auto sequential_writer = std::make_unique( - compression_options, - std::move(compression_factory), - std::move(storage_factory_), - converter_factory_, - std::move(metadata_io_)); - writer_ = std::make_unique(std::move(sequential_writer)); + initializeWriter(compression_options, std::move(compression_factory)); // This will throw an exception because the MockCompressionFactory does not actually create // a compressor. EXPECT_THROW( writer_->open(tmp_dir_storage_options_, {serialization_format_, serialization_format_}), std::runtime_error); +} - EXPECT_TRUE(rcpputils::fs::remove(tmp_dir_)); +TEST_F(SequentialCompressionWriterTest, writer_creates_correct_metadata_relative_filepaths) +{ + // In this test, check that the SequentialCompressionWriter creates relative filepaths correctly + // Check both the first path, which is created in init_metadata, + // and subsequent paths, which are created in the splitting logic + const std::string test_topic_name = "test_topic"; + const std::string test_topic_type = "test_msgs/BasicTypes"; + rosbag2_compression::CompressionOptions compression_options { + "zstd", + rosbag2_compression::CompressionMode::FILE, + kDefaultCompressionQueueSize, + kDefaultCompressionQueueThreads + }; + + initializeFakeFileStorage(); + initializeWriter(compression_options); + + tmp_dir_storage_options_.max_bagfile_size = 1; + writer_->open(tmp_dir_storage_options_); + writer_->create_topic({test_topic_name, test_topic_type, "", ""}); + + auto message = std::make_shared(); + message->topic_name = test_topic_name; + + writer_->write(message); + // bag size == max_bafile_size, no split yet + writer_->write(message); + // bag size > max_bagfile_size, split + writer_->write(message); + writer_.reset(); + + EXPECT_EQ( + intercepted_metadata_.relative_file_paths.size(), 2u); + + const auto base_path = tmp_dir_storage_options_.uri; + int counter = 0; + for (const auto & path : intercepted_metadata_.relative_file_paths) { + std::stringstream ss; + ss << bag_name_ << "_" << counter << ".zstd"; + counter++; + EXPECT_EQ(ss.str(), path); + } } diff --git a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp index 93c7461da0..0cd50e6fe1 100644 --- a/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp +++ b/rosbag2_tests/test/rosbag2_tests/test_rosbag2_record_end_to_end.cpp @@ -537,7 +537,7 @@ TEST_F(RecordFixture, record_end_to_end_test_with_zstd_file_compression_compress const auto metadata = metadata_io.read_metadata(root_bag_path_.string()); for (const auto & path : metadata.relative_file_paths) { - const auto file_path = rcpputils::fs::path{path}; + const auto file_path = root_bag_path_ / rcpputils::fs::path{path}; EXPECT_TRUE(file_path.exists()) << "File: \"" << file_path.string() << "\" does not exist!"; From 2f5cbb777148bf85f8308da5654ffce6f53f44a9 Mon Sep 17 00:00:00 2001 From: Piotr Jaroszek Date: Thu, 28 Jan 2021 23:03:35 +0100 Subject: [PATCH 71/77] Regex and exclude fix for rosbag recorder (#620) * Regex and exclude fix for rosbag recorder Signed-off-by: Piotr Jaroszek --- .../src/rosbag2_transport/recorder.cpp | 7 +- .../rosbag2_transport/test_record_regex.cpp | 66 +++++++++++++++++-- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/rosbag2_transport/src/rosbag2_transport/recorder.cpp b/rosbag2_transport/src/rosbag2_transport/recorder.cpp index e2f267757d..70d5b78785 100644 --- a/rosbag2_transport/src/rosbag2_transport/recorder.cpp +++ b/rosbag2_transport/src/rosbag2_transport/recorder.cpp @@ -113,9 +113,12 @@ Recorder::get_requested_or_available_topics(const RecordOptions & record_options std::regex exclude_regex(record_options.exclude); bool take = record_options.all; for (const auto & kv : unfiltered_topics) { - take = std::regex_search(kv.first, topic_regex); + // regex_match returns false for 'empty' regex + if (!record_options.regex.empty()) { + take = std::regex_match(kv.first, topic_regex); + } if (take) { - take = !std::regex_search(kv.first, exclude_regex); + take = !std::regex_match(kv.first, exclude_regex); } if (take) { filtered_by_regex.insert(kv); diff --git a/rosbag2_transport/test/rosbag2_transport/test_record_regex.cpp b/rosbag2_transport/test/rosbag2_transport/test_record_regex.cpp index e80fa3b740..c2a2e1dc03 100644 --- a/rosbag2_transport/test/rosbag2_transport/test_record_regex.cpp +++ b/rosbag2_transport/test/rosbag2_transport/test_record_regex.cpp @@ -23,7 +23,59 @@ #include "record_integration_fixture.hpp" -TEST_F(RecordIntegrationTestFixture, only_regex_matching_topics_are_recorded) +TEST_F(RecordIntegrationTestFixture, regex_topics_recording) +{ + auto test_string_messages = get_messages_strings(); + auto test_array_messages = get_messages_arrays(); + std::string regex = "/aa"; + + // matching topic + std::string v1 = "/aa"; + + // topics that shouldn't match + std::string b1 = "/aaa"; + std::string b2 = "/baa"; + std::string b3 = "/baaa"; + std::string b4 = "/aa/aa"; + + // checking the test data itself + std::regex re(regex); + ASSERT_TRUE(std::regex_match(v1, re)); + ASSERT_FALSE(std::regex_match(b1, re)); + ASSERT_FALSE(std::regex_match(b2, re)); + ASSERT_FALSE(std::regex_match(b3, re)); + ASSERT_FALSE(std::regex_match(b4, re)); + + RecordOptions ro{false, false, {}, "rmw_format", 10ms}; + ro.regex = regex; + + start_recording(ro); + + pub_man_.add_publisher( + v1, test_string_messages[0], 0); + + pub_man_.add_publisher( + b1, test_string_messages[0], 0); + pub_man_.add_publisher( + b2, test_string_messages[1], 0); + pub_man_.add_publisher( + b3, test_string_messages[0], 0); + pub_man_.add_publisher( + b4, test_string_messages[1], 0); + + run_publishers(); + stop_recording(); + + MockSequentialWriter & writer = + static_cast(writer_->get_implementation_handle()); + auto recorded_topics = writer.get_topics(); + + EXPECT_THAT(recorded_topics, SizeIs(1)); + + EXPECT_TRUE(recorded_topics.find(v1) != recorded_topics.end()); +} + +TEST_F(RecordIntegrationTestFixture, regex_and_exclude_recording) { auto test_string_messages = get_messages_strings(); auto test_array_messages = get_messages_arrays(); @@ -44,14 +96,14 @@ TEST_F(RecordIntegrationTestFixture, only_regex_matching_topics_are_recorded) // checking the test data itself std::regex re(regex); std::regex exclude(regex_exclude); - ASSERT_TRUE(std::regex_search(v1, re)); - ASSERT_TRUE(std::regex_search(v2, re)); - ASSERT_FALSE(std::regex_search(b1, re)); - ASSERT_FALSE(std::regex_search(b2, re)); + ASSERT_TRUE(std::regex_match(v1, re)); + ASSERT_TRUE(std::regex_match(v2, re)); + ASSERT_FALSE(std::regex_match(b1, re)); + ASSERT_FALSE(std::regex_match(b2, re)); // this example matches both regexes - should be excluded - ASSERT_TRUE(std::regex_search(e1, re)); - ASSERT_TRUE(std::regex_search(e1, exclude)); + ASSERT_TRUE(std::regex_match(e1, re)); + ASSERT_TRUE(std::regex_match(e1, exclude)); RecordOptions ro{false, false, {}, "rmw_format", 10ms}; ro.regex = regex; From 9baa79e8f4d46057801b9be7cf8e409558953b09 Mon Sep 17 00:00:00 2001 From: "P. J. Reed" Date: Fri, 29 Jan 2021 15:18:09 -0600 Subject: [PATCH 72/77] Fix build issues when rosbag2_storage is binary installed (#585) Several packages were failing to build from source when rosbag2_storage was installed from a binary package because the path to the binary install was being added to CMake's include path before the workspace path. This tweaks the dependencies and include orders to ensure the source workspace path is preferred. Fixes #583 Distro A, OPSEC #4584 Signed-off-by: P. J. Reed --- rosbag2_compression/CMakeLists.txt | 4 ++-- rosbag2_cpp/CMakeLists.txt | 5 +++++ rosbag2_py/CMakeLists.txt | 2 ++ rosbag2_storage_default_plugins/CMakeLists.txt | 4 ++-- rosbag2_transport/CMakeLists.txt | 18 ++++++++++-------- .../cmake/rosbag2_transport_add_gmock.cmake | 2 +- 6 files changed, 22 insertions(+), 13 deletions(-) diff --git a/rosbag2_compression/CMakeLists.txt b/rosbag2_compression/CMakeLists.txt index ebdcb7e1bf..f5b908cd4d 100644 --- a/rosbag2_compression/CMakeLists.txt +++ b/rosbag2_compression/CMakeLists.txt @@ -138,13 +138,13 @@ if(BUILD_TESTING) test/rosbag2_compression/test_sequential_compression_reader.cpp) target_include_directories(test_sequential_compression_reader PUBLIC include) target_link_libraries(test_sequential_compression_reader ${PROJECT_NAME}) - ament_target_dependencies(test_sequential_compression_reader rosbag2_cpp) + ament_target_dependencies(test_sequential_compression_reader rosbag2_cpp rosbag2_storage) ament_add_gmock(test_sequential_compression_writer test/rosbag2_compression/test_sequential_compression_writer.cpp) target_include_directories(test_sequential_compression_writer PUBLIC include) target_link_libraries(test_sequential_compression_writer ${PROJECT_NAME}) - ament_target_dependencies(test_sequential_compression_writer rosbag2_cpp) + ament_target_dependencies(test_sequential_compression_writer rosbag2_cpp rosbag2_storage) endif() ament_package() diff --git a/rosbag2_cpp/CMakeLists.txt b/rosbag2_cpp/CMakeLists.txt index 1d78d5831f..e13d7dd4b8 100644 --- a/rosbag2_cpp/CMakeLists.txt +++ b/rosbag2_cpp/CMakeLists.txt @@ -117,6 +117,7 @@ if(BUILD_TESTING) SHARED test/rosbag2_cpp/serializer_test_plugin.cpp test/rosbag2_cpp/converter_test_plugin.cpp) + ament_target_dependencies(converter_test_plugins rosbag2_storage) target_link_libraries(converter_test_plugins ${PROJECT_NAME}) install( TARGETS converter_test_plugins @@ -150,12 +151,14 @@ if(BUILD_TESTING) ament_add_gmock(test_sequential_reader test/rosbag2_cpp/test_sequential_reader.cpp) if(TARGET test_sequential_reader) + ament_target_dependencies(test_sequential_reader rosbag2_storage) target_link_libraries(test_sequential_reader ${PROJECT_NAME}) endif() ament_add_gmock(test_storage_without_metadata_file test/rosbag2_cpp/test_storage_without_metadata_file.cpp) if(TARGET test_storage_without_metadata_file) + ament_target_dependencies(test_storage_without_metadata_file rosbag2_storage) target_link_libraries(test_storage_without_metadata_file ${PROJECT_NAME}) endif() @@ -193,12 +196,14 @@ if(BUILD_TESTING) ament_add_gmock(test_sequential_writer test/rosbag2_cpp/test_sequential_writer.cpp) if(TARGET test_sequential_writer) + ament_target_dependencies(test_sequential_writer rosbag2_storage) target_link_libraries(test_sequential_writer ${PROJECT_NAME}) endif() ament_add_gmock(test_multifile_reader test/rosbag2_cpp/test_multifile_reader.cpp) if(TARGET test_multifile_reader) + ament_target_dependencies(test_multifile_reader rosbag2_storage) target_link_libraries(test_multifile_reader ${PROJECT_NAME}) endif() endif() diff --git a/rosbag2_py/CMakeLists.txt b/rosbag2_py/CMakeLists.txt index 17bdb7a63f..4eacb40172 100644 --- a/rosbag2_py/CMakeLists.txt +++ b/rosbag2_py/CMakeLists.txt @@ -73,6 +73,7 @@ pybind11_add_module(_reader SHARED ament_target_dependencies(_reader PUBLIC "rosbag2_compression" "rosbag2_cpp" + "rosbag2_storage" ) clean_windows_flags(_reader) @@ -91,6 +92,7 @@ pybind11_add_module(_writer SHARED ament_target_dependencies(_writer PUBLIC "rosbag2_compression" "rosbag2_cpp" + "rosbag2_storage" ) clean_windows_flags(_writer) diff --git a/rosbag2_storage_default_plugins/CMakeLists.txt b/rosbag2_storage_default_plugins/CMakeLists.txt index f519aba0b2..a83a86ad41 100644 --- a/rosbag2_storage_default_plugins/CMakeLists.txt +++ b/rosbag2_storage_default_plugins/CMakeLists.txt @@ -88,7 +88,7 @@ if(BUILD_TESTING) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) if(TARGET test_sqlite_wrapper) target_link_libraries(test_sqlite_wrapper ${TEST_LINK_LIBRARIES}) - ament_target_dependencies(test_sqlite_wrapper rosbag2_test_common) + ament_target_dependencies(test_sqlite_wrapper rosbag2_storage rosbag2_test_common) endif() ament_add_gmock(test_sqlite_storage @@ -96,7 +96,7 @@ if(BUILD_TESTING) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) if(TARGET test_sqlite_storage) target_link_libraries(test_sqlite_storage ${TEST_LINK_LIBRARIES}) - ament_target_dependencies(test_sqlite_storage rosbag2_test_common) + ament_target_dependencies(test_sqlite_storage rosbag2_storage rosbag2_test_common) endif() endif() diff --git a/rosbag2_transport/CMakeLists.txt b/rosbag2_transport/CMakeLists.txt index ccc0d0f07b..de85e96cef 100644 --- a/rosbag2_transport/CMakeLists.txt +++ b/rosbag2_transport/CMakeLists.txt @@ -52,6 +52,7 @@ ament_target_dependencies(${PROJECT_NAME} rmw rosbag2_compression rosbag2_cpp + rosbag2_storage shared_queues_vendor yaml_cpp_vendor ) @@ -94,22 +95,22 @@ function(create_tests_for_rmw_implementation) rosbag2_transport_add_gmock(test_info test/rosbag2_transport/test_info.cpp LINK_LIBS rosbag2_transport - AMENT_DEPS rosbag2_test_common) + AMENT_DEPS rosbag2_cpp rosbag2_storage rosbag2_test_common) rosbag2_transport_add_gmock(test_record_all test/rosbag2_transport/test_record_all.cpp LINK_LIBS rosbag2_transport - AMENT_DEPS test_msgs rosbag2_test_common) + AMENT_DEPS rosbag2_cpp rosbag2_storage test_msgs rosbag2_test_common) rosbag2_transport_add_gmock(test_record_all_no_discovery test/rosbag2_transport/test_record_all_no_discovery.cpp LINK_LIBS rosbag2_transport - AMENT_DEPS test_msgs rosbag2_test_common) + AMENT_DEPS rosbag2_cpp rosbag2_storage test_msgs rosbag2_test_common) rosbag2_transport_add_gmock(test_play_timing test/rosbag2_transport/test_play_timing.cpp LINK_LIBS rosbag2_transport - AMENT_DEPS test_msgs rosbag2_test_common) + AMENT_DEPS rosbag2_cpp rosbag2_storage test_msgs rosbag2_test_common) rosbag2_transport_add_gmock(test_rosbag2_node src/rosbag2_transport/generic_publisher.cpp @@ -127,6 +128,7 @@ function(create_tests_for_rmw_implementation) ament_index_cpp rclcpp rosbag2_cpp + rosbag2_storage rosbag2_test_common test_msgs yaml_cpp_vendor) @@ -156,7 +158,7 @@ function(create_tests_for_rmw_implementation) rosbag2_transport_add_gmock(test_record test/rosbag2_transport/test_record.cpp - AMENT_DEPS test_msgs rosbag2_test_common + AMENT_DEPS rosbag2_cpp rosbag2_storage test_msgs rosbag2_test_common INCLUDE_DIRS $ LINK_LIBS rosbag2_transport ${SKIP_TEST}) @@ -172,20 +174,20 @@ function(create_tests_for_rmw_implementation) test/rosbag2_transport/test_play.cpp INCLUDE_DIRS $ LINK_LIBS rosbag2_transport - AMENT_DEPS test_msgs rosbag2_test_common + AMENT_DEPS rosbag2_cpp rosbag2_storage test_msgs rosbag2_test_common ${SKIP_TEST}) rosbag2_transport_add_gmock(test_play_loop test/rosbag2_transport/test_play_loop.cpp ${SKIP_TEST} LINK_LIBS rosbag2_transport - AMENT_DEPS test_msgs rosbag2_test_common) + AMENT_DEPS rosbag2_cpp rosbag2_storage test_msgs rosbag2_test_common) rosbag2_transport_add_gmock(test_play_topic_remap test/rosbag2_transport/test_play_topic_remap.cpp ${SKIP_TEST} LINK_LIBS rosbag2_transport - AMENT_DEPS test_msgs rosbag2_test_common) + AMENT_DEPS rosbag2_cpp rosbag2_storage test_msgs rosbag2_test_common) endfunction() if(BUILD_TESTING) diff --git a/rosbag2_transport/cmake/rosbag2_transport_add_gmock.cmake b/rosbag2_transport/cmake/rosbag2_transport_add_gmock.cmake index 81ac8b4a60..bbaba9f126 100644 --- a/rosbag2_transport/cmake/rosbag2_transport_add_gmock.cmake +++ b/rosbag2_transport/cmake/rosbag2_transport_add_gmock.cmake @@ -57,8 +57,8 @@ function(rosbag2_transport_add_gmock target_base) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ${SKIP_TEST}) if(TARGET ${target_name}) + ament_target_dependencies(${target_name} ${ARG_AMENT_DEPS}) target_link_libraries(${target_name} ${ARG_LINK_LIBS}) target_include_directories(${target_name} PUBLIC ${ARG_INCLUDE_DIRS}) - ament_target_dependencies(${target_name} ${ARG_AMENT_DEPS}) endif() endfunction() From ba35046cfdbe52c900c3b003fe6df1555c1aed38 Mon Sep 17 00:00:00 2001 From: "P. J. Reed" Date: Fri, 29 Jan 2021 16:35:40 -0600 Subject: [PATCH 73/77] Use ZSTD's streaming interface for [de]compressing files (#543) * Implement streaming compression/decompression Distro A, OPSEC #4584 Signed-off-by: P. J. Reed --- .../rosbag2_compression/compression_utils.cpp | 52 --------------- .../rosbag2_compression/compression_utils.hpp | 8 --- .../rosbag2_compression/zstd_compressor.cpp | 64 +++++++++++++------ .../rosbag2_compression/zstd_decompressor.cpp | 64 ++++++++++++------- 4 files changed, 88 insertions(+), 100 deletions(-) diff --git a/rosbag2_compression/src/rosbag2_compression/compression_utils.cpp b/rosbag2_compression/src/rosbag2_compression/compression_utils.cpp index 2a48fa8d84..38aee15da9 100644 --- a/rosbag2_compression/src/rosbag2_compression/compression_utils.cpp +++ b/rosbag2_compression/src/rosbag2_compression/compression_utils.cpp @@ -39,58 +39,6 @@ FILE * open_file(const std::string & uri, const std::string & read_mode) namespace rosbag2_compression { -std::vector get_input_buffer(const std::string & uri) -{ - // Read in buffer, handling accordingly - const auto file_pointer = open_file(uri, "rb"); - if (file_pointer == nullptr) { - std::stringstream errmsg; - errmsg << "Failed to open file: \"" << uri << - "\" for binary reading! errno(" << errno << ")"; - - throw std::runtime_error{errmsg.str()}; - } - - const auto file_path = rcpputils::fs::path{uri}; - const auto input_buffer_length = file_path.exists() ? file_path.file_size() : 0u; - if (input_buffer_length == 0) { - fclose(file_pointer); - - std::stringstream errmsg; - errmsg << "Unable to get size of file: \"" << uri << "\""; - - throw std::runtime_error{errmsg.str()}; - } - - // Initialize compress_buffer with size = compressed_buffer_length. - // Uniform initialization cannot be used here since it will choose - // the initializer list constructor instead. - std::vector input_buffer(input_buffer_length); - - const auto read_count = fread( - input_buffer.data(), sizeof(decltype(input_buffer)::value_type), - input_buffer_length, file_pointer); - - if (read_count != input_buffer_length) { - ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( - "Bytes read !(" << - read_count << ") != buffer size (" << input_buffer.size() << - ")!"); - // An error indicator is set by fread, so the following check will throw. - } - - if (ferror(file_pointer)) { - fclose(file_pointer); - - std::stringstream errmsg; - errmsg << "Unable to read binary data from file: \"" << uri << "\"!"; - - throw std::runtime_error{errmsg.str()}; - } - - fclose(file_pointer); - return input_buffer; -} void write_output_buffer( const std::vector & output_buffer, diff --git a/rosbag2_compression/src/rosbag2_compression/compression_utils.hpp b/rosbag2_compression/src/rosbag2_compression/compression_utils.hpp index 20b466398d..f00b16a26f 100644 --- a/rosbag2_compression/src/rosbag2_compression/compression_utils.hpp +++ b/rosbag2_compression/src/rosbag2_compression/compression_utils.hpp @@ -48,14 +48,6 @@ using ZstdDecompressReturnType = decltype(ZSTD_decompress( // Used as a parameter type in a function that accepts the output of ZSTD_getFrameContentSize. using ZstdGetFrameContentSizeReturnType = decltype(ZSTD_getFrameContentSize(nullptr, 0)); -/** - * Read a file from the supplied uri into a vector. - * - * \param uri is the path to the file. - * \return the contents of the buffer as a vector. - */ -std::vector get_input_buffer(const std::string & uri); - /** * Writes the output buffer to the specified file path. * \param output_buffer is the data to write. diff --git a/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp b/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp index e1f414b6ed..4661759974 100644 --- a/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp +++ b/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp @@ -49,27 +49,55 @@ std::string ZstdCompressor::compress_uri(const std::string & uri) { const auto start = std::chrono::high_resolution_clock::now(); const auto compressed_uri = uri + "." + get_compression_identifier(); - const auto decompressed_buffer = get_input_buffer(uri); - // Allocate based on compression bound and compress - const auto compressed_buffer_length = ZSTD_compressBound(decompressed_buffer.size()); - std::vector compressed_buffer(compressed_buffer_length); - - // Perform compression and check. - // compression_result is either the actual compressed size or an error code. - const auto compression_result = ZSTD_compressCCtx( - zstd_context_, - compressed_buffer.data(), compressed_buffer.size(), - decompressed_buffer.data(), decompressed_buffer.size(), kDefaultZstdCompressionLevel); - throw_on_zstd_error(compression_result); - - // Compression_buffer_length might be larger than the actual compression size - // Resize compressed_buffer so its size is the actual compression size. - compressed_buffer.resize(compression_result); - write_output_buffer(compressed_buffer, compressed_uri); + std::ifstream input(uri, std::ios::in | std::ios::binary); + if (!input.is_open()) { + std::stringstream errmsg; + errmsg << "Failed to open file: \"" << uri << + "\" for binary reading! errno(" << errno << ")"; + + throw std::runtime_error{errmsg.str()}; + } + std::ofstream output(compressed_uri, std::ios::out | std::ios::binary); + if (!output.is_open()) { + std::stringstream errmsg; + errmsg << "Failed to open file: \"" << uri << + "\" for binary writing! errno(" << errno << ")"; + + throw std::runtime_error{errmsg.str()}; + } + // Based on the example from https://github.com/facebook/zstd/blob/dev/examples/streaming_compression.c + const size_t buff_in_size = ZSTD_CStreamInSize(); + const size_t buff_out_size = ZSTD_CStreamOutSize(); + std::vector in_buffer(buff_in_size); + std::vector out_buffer(buff_out_size); + size_t total_size = 0; + size_t final_result = 0; + do { + input.read(in_buffer.data(), buff_in_size); + const auto size = size_t(input.gcount()); + if (size > 0) { + const ZSTD_EndDirective mode = input.eof() ? ZSTD_e_end : ZSTD_e_continue; + ZSTD_inBuffer z_in_buffer = {in_buffer.data(), static_cast(size), 0}; + int finished; + do { + ZSTD_outBuffer z_out_buffer = {out_buffer.data(), out_buffer.size(), 0}; + const auto remaining = + ZSTD_compressStream2(zstd_context_, &z_out_buffer, &z_in_buffer, mode); + throw_on_zstd_error(remaining); + output.write(out_buffer.data(), z_out_buffer.pos); + total_size += z_out_buffer.pos; + finished = input.eof() ? (remaining == 0) : (z_in_buffer.pos == z_in_buffer.size); + final_result = remaining; + } while (!finished); + } + } while (!input.eof()); + output.flush(); + output.close(); + input.close(); const auto end = std::chrono::high_resolution_clock::now(); - print_compression_statistics(start, end, decompressed_buffer.size(), compression_result); + print_compression_statistics(start, end, total_size, final_result); return compressed_uri; } diff --git a/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp b/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp index 5e14ceb671..3d0b23d7e2 100644 --- a/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp +++ b/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp @@ -47,31 +47,51 @@ std::string ZstdDecompressor::decompress_uri(const std::string & uri) const auto start = std::chrono::high_resolution_clock::now(); const auto uri_path = rcpputils::fs::path{uri}; const auto decompressed_uri = rcpputils::fs::remove_extension(uri_path).string(); - const auto compressed_buffer = get_input_buffer(uri); - const auto compressed_buffer_length = compressed_buffer.size(); - const auto decompressed_buffer_length = - ZSTD_getFrameContentSize(compressed_buffer.data(), compressed_buffer_length); - - throw_on_invalid_frame_content(decompressed_buffer_length); - - // Initializes decompressed_buffer with size = decompressed_buffer_length. - // Uniform initialization cannot be used here since it will choose - // the initializer list constructor instead. - std::vector decompressed_buffer(decompressed_buffer_length); - - const auto decompression_result = ZSTD_decompressDCtx( - zstd_context_, - decompressed_buffer.data(), decompressed_buffer_length, - compressed_buffer.data(), compressed_buffer_length); - - throw_on_zstd_error(decompression_result); - decompressed_buffer.resize(decompression_result); - - write_output_buffer(decompressed_buffer, decompressed_uri); + std::ifstream input(uri, std::ios::in | std::ios::binary); + if (!input.is_open()) { + std::stringstream errmsg; + errmsg << "Failed to open file: \"" << uri << + "\" for binary reading! errno(" << errno << ")"; + + throw std::runtime_error{errmsg.str()}; + } + std::ofstream output(decompressed_uri, std::ios::out | std::ios::binary); + if (!output.is_open()) { + std::stringstream errmsg; + errmsg << "Failed to open file: \"" << uri << + "\" for binary writing! errno(" << errno << ")"; + + throw std::runtime_error{errmsg.str()}; + } + // Base on the example from https://github.com/facebook/zstd/blob/dev/examples/streaming_decompression.c + const size_t buff_in_size = ZSTD_DStreamInSize(); + const size_t buff_out_size = ZSTD_DStreamOutSize(); + size_t total_size = 0; + std::vector in_buffer(buff_in_size); + std::vector out_buffer(buff_out_size); + size_t final_result = 0; + do { + input.read(in_buffer.data(), buff_in_size); + const auto size = size_t(input.gcount()); + if (size > 0) { + ZSTD_inBuffer z_in_buffer = {in_buffer.data(), static_cast(size), 0}; + while (z_in_buffer.pos < z_in_buffer.size) { + ZSTD_outBuffer z_out_buffer = {out_buffer.data(), out_buffer.size(), 0}; + const auto ret = ZSTD_decompressStream(zstd_context_, &z_out_buffer, &z_in_buffer); + throw_on_zstd_error(ret); + output.write(out_buffer.data(), z_out_buffer.pos); + total_size += z_out_buffer.pos; + final_result = ret; + } + } + } while (!input.eof()); + output.flush(); + output.close(); + input.close(); const auto end = std::chrono::high_resolution_clock::now(); - print_compression_statistics(start, end, decompression_result, compressed_buffer_length); + print_compression_statistics(start, end, final_result, total_size); return decompressed_uri; } From 3ef615a713914bb70c6a04aa5b516bf52ba3b4c6 Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Fri, 29 Jan 2021 16:22:07 -0800 Subject: [PATCH 74/77] Make compressor implementations into a plugin via pluginlib (#624) Make compressor implementation into a plugin via pluginlib Signed-off-by: Emerson Knapp --- rosbag2_compression/CMakeLists.txt | 9 +- .../compression_factory.hpp | 8 +- .../sequential_compression_reader.hpp | 3 +- .../sequential_compression_writer.hpp | 4 +- rosbag2_compression/plugin_description.xml | 14 +++ .../compression_factory.cpp | 4 +- .../compression_factory_impl.hpp | 115 +++++++++++++----- .../sequential_compression_reader.cpp | 1 - .../sequential_compression_writer.cpp | 15 +-- .../rosbag2_compression/zstd_compressor.cpp | 5 + .../rosbag2_compression/zstd_decompressor.cpp | 5 + .../mock_compression_factory.hpp | 4 +- .../test_compression_factory.cpp | 49 +------- .../test_sequential_compression_reader.cpp | 3 +- .../test_sequential_compression_writer.cpp | 5 +- 15 files changed, 138 insertions(+), 106 deletions(-) create mode 100644 rosbag2_compression/plugin_description.xml diff --git a/rosbag2_compression/CMakeLists.txt b/rosbag2_compression/CMakeLists.txt index f5b908cd4d..176e71ff12 100644 --- a/rosbag2_compression/CMakeLists.txt +++ b/rosbag2_compression/CMakeLists.txt @@ -21,6 +21,7 @@ if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") endif() find_package(ament_cmake REQUIRED) +find_package(pluginlib REQUIRED) find_package(rcpputils REQUIRED) find_package(rcutils REQUIRED) find_package(rosbag2_cpp) @@ -47,6 +48,7 @@ ament_target_dependencies(${PROJECT_NAME}_zstd target_compile_definitions(${PROJECT_NAME}_zstd PRIVATE ROSBAG2_COMPRESSION_BUILDING_DLL) +pluginlib_export_plugin_description_file(rosbag2_compression plugin_description.xml) add_library(${PROJECT_NAME} SHARED @@ -58,14 +60,17 @@ target_include_directories(${PROJECT_NAME} PUBLIC $ $) -target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_zstd) ament_target_dependencies(${PROJECT_NAME} + pluginlib rcpputils rcutils rosbag2_cpp rosbag2_storage) target_compile_definitions(${PROJECT_NAME} PRIVATE ROSBAG2_COMPRESSION_BUILDING_DLL) +# prevent pluginlib from using boost +target_compile_definitions(${PROJECT_NAME} PUBLIC "PLUGINLIB__DISABLE_BOOST_FUNCTIONS") + install( DIRECTORY include/ DESTINATION include) @@ -81,7 +86,7 @@ ament_export_include_directories(include) ament_export_libraries(${PROJECT_NAME} ${PROJECT_NAME}_zstd) ament_export_targets(export_${PROJECT_NAME}) # order matters here, first vendor, then zstd -ament_export_dependencies(rcpputils rcutils rosbag2_cpp rosbag2_storage zstd_vendor zstd) +ament_export_dependencies(pluginlib rcpputils rcutils rosbag2_cpp rosbag2_storage zstd_vendor zstd) if(BUILD_TESTING) find_package(ament_cmake_gmock REQUIRED) diff --git a/rosbag2_compression/include/rosbag2_compression/compression_factory.hpp b/rosbag2_compression/include/rosbag2_compression/compression_factory.hpp index a733d49954..b810b89a77 100644 --- a/rosbag2_compression/include/rosbag2_compression/compression_factory.hpp +++ b/rosbag2_compression/include/rosbag2_compression/compression_factory.hpp @@ -50,20 +50,20 @@ class ROSBAG2_COMPRESSION_PUBLIC CompressionFactory * Create a compressor based on the specified compression format. * * \param compression_format The compression format as a string. - * \return A unique pointer to the newly created compressor. + * \return A shared pointer to the newly created compressor. * \throw invalid_argument If the compression format does not exist. */ - virtual std::unique_ptr + virtual std::shared_ptr create_compressor(const std::string & compression_format); /** * Create a decompressor based on the specified compression format. * * \param compression_format The compression format as a string. - * \return A unique pointer to the newly created decompressor. + * \return A shared pointer to the newly created decompressor. * \throw invalid_argument If the compression format does not exist. */ - virtual std::unique_ptr + virtual std::shared_ptr create_decompressor(const std::string & compression_format); private: diff --git a/rosbag2_compression/include/rosbag2_compression/sequential_compression_reader.hpp b/rosbag2_compression/include/rosbag2_compression/sequential_compression_reader.hpp index 2acd9f631c..30f99a2b5d 100644 --- a/rosbag2_compression/include/rosbag2_compression/sequential_compression_reader.hpp +++ b/rosbag2_compression/include/rosbag2_compression/sequential_compression_reader.hpp @@ -35,7 +35,6 @@ #include "compression_factory.hpp" #include "visibility_control.hpp" -#include "zstd_decompressor.hpp" #ifdef _WIN32 @@ -85,7 +84,7 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionReader */ void setup_decompression(); - std::unique_ptr decompressor_{}; + std::shared_ptr decompressor_{}; rosbag2_compression::CompressionMode compression_mode_{ rosbag2_compression::CompressionMode::NONE}; std::unique_ptr compression_factory_{}; diff --git a/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp b/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp index ee4ff7479f..69b71eeca4 100644 --- a/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp +++ b/rosbag2_compression/include/rosbag2_compression/sequential_compression_writer.hpp @@ -151,12 +151,14 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionWriter * Initializes the compressor if a compression mode is specified. * * \throws std::invalid_argument if compression_options isn't supported. + * \throws rcpputils::IllegalStateException if compressor could not be created */ virtual void setup_compression(); /** * Initializes a number of threads to do file or message compression equal to the * value of the compression_threads parameter. + * \throws rcpputils::IllegalStateException if compressor could not be created */ virtual void setup_compressor_threads(); @@ -166,7 +168,7 @@ class ROSBAG2_COMPRESSION_PUBLIC SequentialCompressionWriter virtual void stop_compressor_threads(); private: - std::unique_ptr compressor_{}; + std::shared_ptr compressor_{}; std::unique_ptr compression_factory_{}; std::mutex compressor_queue_mutex_; std::queue> diff --git a/rosbag2_compression/plugin_description.xml b/rosbag2_compression/plugin_description.xml new file mode 100644 index 0000000000..176906cabd --- /dev/null +++ b/rosbag2_compression/plugin_description.xml @@ -0,0 +1,14 @@ + + + Zstd implementation for rosbag2 compressor + + + Zstd implementation for rosbag2 decompressor + + diff --git a/rosbag2_compression/src/rosbag2_compression/compression_factory.cpp b/rosbag2_compression/src/rosbag2_compression/compression_factory.cpp index 0efd1be1a1..e88d233b35 100644 --- a/rosbag2_compression/src/rosbag2_compression/compression_factory.cpp +++ b/rosbag2_compression/src/rosbag2_compression/compression_factory.cpp @@ -27,13 +27,13 @@ CompressionFactory::CompressionFactory() CompressionFactory::~CompressionFactory() = default; -std::unique_ptr +std::shared_ptr CompressionFactory::create_compressor(const std::string & compression_format) { return impl_->create_compressor(compression_format); } -std::unique_ptr +std::shared_ptr CompressionFactory::create_decompressor(const std::string & compression_format) { return impl_->create_decompressor(compression_format); diff --git a/rosbag2_compression/src/rosbag2_compression/compression_factory_impl.hpp b/rosbag2_compression/src/rosbag2_compression/compression_factory_impl.hpp index ff20eeddd8..65ba3b11e1 100644 --- a/rosbag2_compression/src/rosbag2_compression/compression_factory_impl.hpp +++ b/rosbag2_compression/src/rosbag2_compression/compression_factory_impl.hpp @@ -20,67 +20,118 @@ #include #include +#include "pluginlib/class_loader.hpp" + #include "logging.hpp" #include "rosbag2_compression/compression_factory.hpp" -#include "rosbag2_compression/zstd_compressor.hpp" -#include "rosbag2_compression/zstd_decompressor.hpp" -namespace +namespace rosbag2_compression { -constexpr const char kCompressionFormatZstd[] = "zstd"; +using rosbag2_compression::BaseCompressorInterface; +using rosbag2_compression::BaseDecompressorInterface; + +template +struct CompressionTraits +{}; -/// Verify whether two case-insensitive chars are equal -bool compare_char(const char c1, const char c2) +template<> +struct CompressionTraits { - return c1 == c2 || std::tolower(c1) == std::tolower(c2); -} + static constexpr const char * name = "rosbag2_compression::BaseCompressorInterface"; +}; -/// Performs a case-insensitive comparison of two strings -bool case_insensitive_compare(const std::string & str1, const std::string & str2) noexcept +template<> +struct CompressionTraits { - return (str1.size() == str2.size()) && - std::equal(str1.begin(), str1.end(), str2.begin(), &compare_char); + static constexpr const char * name = "rosbag2_compression::BaseDecompressorInterface"; +}; + +template +std::shared_ptr> +get_class_loader() +{ + const auto lookup_name = CompressionTraits::name; + return std::make_shared>( + "rosbag2_compression", lookup_name); } -} // namespace -namespace rosbag2_compression +template +std::shared_ptr +get_interface_instance( + std::shared_ptr> class_loader, + const std::string & compression_format) { + const auto & registered_classes = class_loader->getDeclaredClasses(); + auto class_iter = std::find( + registered_classes.begin(), registered_classes.end(), compression_format); + if (class_iter == registered_classes.end()) { + ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( + "No registered plugin found for compression format '" << compression_format << "'"); + return nullptr; + } + + std::shared_ptr instance = nullptr; + try { + instance = class_loader->createSharedInstance(compression_format); + } catch (const std::runtime_error & ex) { + ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( + "Unable to load instance of compression interface: " << ex.what()); + } + return instance; +} /// Implementation of the CompressionFactory. See CompressionFactory for documentation. class CompressionFactoryImpl { public: - CompressionFactoryImpl() = default; - ~CompressionFactoryImpl() = default; + CompressionFactoryImpl() + { + try { + compressor_class_loader_ = get_class_loader(); + } catch (const std::exception & e) { + ROSBAG2_COMPRESSION_LOG_ERROR_STREAM("Unable to create class loader instance: " << e.what()); + throw e; + } + + try { + decompressor_class_loader_ = get_class_loader(); + } catch (const std::exception & e) { + ROSBAG2_COMPRESSION_LOG_ERROR_STREAM("Unable to create class loader instance: " << e.what()); + throw e; + } + } + + virtual ~CompressionFactoryImpl() = default; /// See CompressionFactory::create_compressor for documentation. - std::unique_ptr + std::shared_ptr create_compressor(const std::string & compression_format) { - if (case_insensitive_compare(compression_format, kCompressionFormatZstd)) { - return std::make_unique(); - } else { - std::stringstream errmsg; - errmsg << "Compression format \"" << compression_format << "\" is not supported."; - ROSBAG2_COMPRESSION_LOG_ERROR_STREAM(errmsg.str()); - throw std::invalid_argument{errmsg.str()}; + auto instance = get_interface_instance(compressor_class_loader_, compression_format); + if (instance == nullptr) { + ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( + "Could not load/open plugin for compression format '" << compression_format << "'"); + return nullptr; } + return instance; } /// See CompressionFactory::create_decompressor for documentation. - std::unique_ptr + std::shared_ptr create_decompressor(const std::string & compression_format) { - if (case_insensitive_compare(compression_format, kCompressionFormatZstd)) { - return std::make_unique(); - } else { - std::stringstream errmsg; - errmsg << "Compression format \"" << compression_format << "\" is not supported."; - ROSBAG2_COMPRESSION_LOG_ERROR_STREAM(errmsg.str()); - throw std::invalid_argument{errmsg.str()}; + auto instance = get_interface_instance(decompressor_class_loader_, compression_format); + if (instance == nullptr) { + ROSBAG2_COMPRESSION_LOG_ERROR_STREAM( + "Could not load/open plugin for compression format '" << compression_format << "'"); } + return instance; } + +private: + std::shared_ptr> compressor_class_loader_; + std::shared_ptr> decompressor_class_loader_; }; } // namespace rosbag2_compression diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp index 7b823409fd..0070322f44 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp @@ -24,7 +24,6 @@ #include "rcpputils/filesystem_helper.hpp" #include "rosbag2_compression/compression_options.hpp" -#include "rosbag2_compression/zstd_decompressor.hpp" #include "logging.hpp" diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp index e1d16611d4..dee7e0df79 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_writer.cpp @@ -22,12 +22,11 @@ #include #include +#include "rcpputils/asserts.hpp" #include "rcpputils/filesystem_helper.hpp" #include "rcutils/filesystem.h" -#include "rosbag2_compression/zstd_compressor.hpp" - #include "rosbag2_cpp/info.hpp" #include "rosbag2_storage/storage_options.hpp" @@ -66,12 +65,7 @@ void SequentialCompressionWriter::compression_thread_fn() // Every thread needs to have its own compression context for thread safety. auto compressor = compression_factory_->create_compressor( compression_options_.compression_format); - - - if (!compressor) { - throw std::runtime_error{ - "Cannot compress message; Writer is not open!"}; - } + rcpputils::check_true(compressor != nullptr, "Could not create compressor."); while (true) { std::shared_ptr message; @@ -154,10 +148,7 @@ void SequentialCompressionWriter::setup_compressor_threads() // throw an exception if the format is invalid. auto compressor = compression_factory_->create_compressor( compression_options_.compression_format); - if (!compressor) { - throw std::runtime_error{ - "Cannot compress message; Writer is not open!"}; - } + rcpputils::check_true(compressor != nullptr, "Could not create compressor."); for (uint64_t i = 0; i < compression_options_.compression_threads; i++) { compression_threads_.emplace_back([&] {compression_thread_fn();}); diff --git a/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp b/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp index 4661759974..d0e6d8cbdc 100644 --- a/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp +++ b/rosbag2_compression/src/rosbag2_compression/zstd_compressor.cpp @@ -142,3 +142,8 @@ std::string ZstdCompressor::get_compression_identifier() const } } // namespace rosbag2_compression + +#include "pluginlib/class_list_macros.hpp" // NOLINT +PLUGINLIB_EXPORT_CLASS( + rosbag2_compression::ZstdCompressor, + rosbag2_compression::BaseCompressorInterface) diff --git a/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp b/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp index 3d0b23d7e2..dfb6a03387 100644 --- a/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp +++ b/rosbag2_compression/src/rosbag2_compression/zstd_decompressor.cpp @@ -137,3 +137,8 @@ std::string ZstdDecompressor::get_decompression_identifier() const return kDecompressionIdentifier; } } // namespace rosbag2_compression + +#include "pluginlib/class_list_macros.hpp" // NOLINT +PLUGINLIB_EXPORT_CLASS( + rosbag2_compression::ZstdDecompressor, + rosbag2_compression::BaseDecompressorInterface) diff --git a/rosbag2_compression/test/rosbag2_compression/mock_compression_factory.hpp b/rosbag2_compression/test/rosbag2_compression/mock_compression_factory.hpp index a9c77e77bf..ae338e5136 100644 --- a/rosbag2_compression/test/rosbag2_compression/mock_compression_factory.hpp +++ b/rosbag2_compression/test/rosbag2_compression/mock_compression_factory.hpp @@ -27,12 +27,12 @@ class MockCompressionFactory : public rosbag2_compression::CompressionFactory public: MOCK_METHOD1( create_compressor, - std::unique_ptr( + std::shared_ptr( const std::string &)); MOCK_METHOD1( create_decompressor, - std::unique_ptr( + std::shared_ptr( const std::string &)); }; diff --git a/rosbag2_compression/test/rosbag2_compression/test_compression_factory.cpp b/rosbag2_compression/test/rosbag2_compression/test_compression_factory.cpp index 6a261dd128..cd69ff8e9c 100644 --- a/rosbag2_compression/test/rosbag2_compression/test_compression_factory.cpp +++ b/rosbag2_compression/test/rosbag2_compression/test_compression_factory.cpp @@ -19,17 +19,6 @@ #include "rosbag2_compression/compression_factory.hpp" -namespace -{ - -std::string to_lower(const std::string & text) -{ - std::string lowercase_text = text; - std::transform(lowercase_text.begin(), lowercase_text.end(), lowercase_text.begin(), ::tolower); - return lowercase_text; -} -} // namespace - class CompressionFactoryTest : public ::testing::Test { public: @@ -39,53 +28,23 @@ class CompressionFactoryTest : public ::testing::Test TEST_F(CompressionFactoryTest, creates_zstd_compressor) { const auto compression_format = "zstd"; auto zstd_compressor = factory.create_compressor(compression_format); + ASSERT_TRUE(zstd_compressor != nullptr); ASSERT_EQ(compression_format, zstd_compressor->get_compression_identifier()); } -TEST_F(CompressionFactoryTest, creates_zstd_compressor_caseinsensitive) -{ - const auto compression_format = "ZsTd"; - const auto lowercase_compression_format = to_lower(compression_format); - auto zstd_compressor = factory.create_compressor(compression_format); - ASSERT_EQ(lowercase_compression_format, zstd_compressor->get_compression_identifier()); -} - -TEST_F(CompressionFactoryTest, creates_zstd_compressor_uppercase) -{ - const auto compression_format = "ZSTD"; - const auto lowercase_compression_format = to_lower(compression_format); - auto zstd_compressor = factory.create_compressor(compression_format); - ASSERT_EQ(lowercase_compression_format, zstd_compressor->get_compression_identifier()); -} - TEST_F(CompressionFactoryTest, creates_zstd_decompressor) { const auto compression_format = "zstd"; auto zstd_decompressor = factory.create_decompressor(compression_format); + ASSERT_TRUE(zstd_decompressor != nullptr); ASSERT_EQ(compression_format, zstd_decompressor->get_decompression_identifier()); } -TEST_F(CompressionFactoryTest, creates_zstd_decompressor_caseinsensitive) -{ - const auto compression_format = "ZsTd"; - const auto lowercase_compression_format = to_lower(compression_format); - auto zstd_compressor = factory.create_decompressor(compression_format); - ASSERT_EQ(lowercase_compression_format, zstd_compressor->get_decompression_identifier()); -} - -TEST_F(CompressionFactoryTest, creates_zstd_decompressor_uppercase) -{ - const auto compression_format = "ZSTD"; - const auto lowercase_compression_format = to_lower(compression_format); - auto zstd_compressor = factory.create_decompressor(compression_format); - ASSERT_EQ(lowercase_compression_format, zstd_compressor->get_decompression_identifier()); -} - TEST_F(CompressionFactoryTest, throws_on_bad_compressor_format) { const auto compression_format = "foo"; - ASSERT_THROW(factory.create_compressor(compression_format), std::invalid_argument); + ASSERT_EQ(factory.create_compressor(compression_format), nullptr); } TEST_F(CompressionFactoryTest, throws_on_bad_decompressor_format) { const auto compression_format = "bar"; - ASSERT_THROW(factory.create_decompressor(compression_format), std::invalid_argument); + ASSERT_EQ(factory.create_decompressor(compression_format), nullptr); } diff --git a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_reader.cpp b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_reader.cpp index 1e478ca8fb..8f15d1ba03 100644 --- a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_reader.cpp +++ b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_reader.cpp @@ -20,6 +20,7 @@ #include #include +#include "rcpputils/asserts.hpp" #include "rcpputils/filesystem_helper.hpp" #include "rosbag2_compression/sequential_compression_reader.hpp" @@ -143,7 +144,7 @@ TEST_F(SequentialCompressionReaderTest, open_throws_if_unsupported_compressor) EXPECT_THROW( sequential_reader->open(storage_options_, converter_options_), - std::invalid_argument); + rcpputils::IllegalStateException); } TEST_F(SequentialCompressionReaderTest, returns_all_topics_and_types) diff --git a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp index aa9aaeaed5..e7b2539de3 100644 --- a/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp +++ b/rosbag2_compression/test/rosbag2_compression/test_sequential_compression_writer.cpp @@ -20,6 +20,7 @@ #include #include +#include "rcpputils/asserts.hpp" #include "rcpputils/filesystem_helper.hpp" #include "rosbag2_compression/compression_options.hpp" @@ -157,7 +158,7 @@ TEST_F(SequentialCompressionWriterTest, open_throws_on_bad_compression_format) EXPECT_THROW( writer_->open(tmp_dir_storage_options_, {serialization_format_, serialization_format_}), - std::invalid_argument); + rcpputils::IllegalStateException); } TEST_F(SequentialCompressionWriterTest, open_throws_on_invalid_splitting_size) @@ -210,7 +211,7 @@ TEST_F(SequentialCompressionWriterTest, writer_calls_create_compressor) // a compressor. EXPECT_THROW( writer_->open(tmp_dir_storage_options_, {serialization_format_, serialization_format_}), - std::runtime_error); + rcpputils::IllegalStateException); } TEST_F(SequentialCompressionWriterTest, writer_creates_correct_metadata_relative_filepaths) From 2ab6da4560da6b9210611e848712690bc5712a74 Mon Sep 17 00:00:00 2001 From: Emerson Knapp <537409+emersonknapp@users.noreply.github.com> Date: Mon, 1 Feb 2021 11:59:46 -0800 Subject: [PATCH 75/77] Prepare bloom release 0.6.0 (#628) * Update CHANGELOG * 0.6.0 Signed-off-by: Emerson Knapp --- ros2bag/CHANGELOG.rst | 12 ++++++++++++ ros2bag/package.xml | 2 +- ros2bag/setup.py | 2 +- rosbag2/CHANGELOG.rst | 6 ++++++ rosbag2/package.xml | 2 +- rosbag2_compression/CHANGELOG.rst | 14 ++++++++++++++ rosbag2_compression/package.xml | 2 +- rosbag2_converter_default_plugins/CHANGELOG.rst | 6 ++++++ rosbag2_converter_default_plugins/package.xml | 2 +- rosbag2_cpp/CHANGELOG.rst | 11 +++++++++++ rosbag2_cpp/package.xml | 2 +- .../rosbag2_performance_benchmarking/CHANGELOG.rst | 5 +++++ .../rosbag2_performance_benchmarking/package.xml | 2 +- rosbag2_py/CHANGELOG.rst | 6 ++++++ rosbag2_py/package.xml | 2 +- rosbag2_storage/CHANGELOG.rst | 10 ++++++++++ rosbag2_storage/package.xml | 2 +- rosbag2_storage_default_plugins/CHANGELOG.rst | 10 ++++++++++ rosbag2_storage_default_plugins/package.xml | 2 +- rosbag2_test_common/CHANGELOG.rst | 8 ++++++++ rosbag2_test_common/package.xml | 2 +- rosbag2_tests/CHANGELOG.rst | 10 ++++++++++ rosbag2_tests/package.xml | 2 +- rosbag2_transport/CHANGELOG.rst | 14 ++++++++++++++ rosbag2_transport/package.xml | 2 +- shared_queues_vendor/CHANGELOG.rst | 6 ++++++ shared_queues_vendor/package.xml | 2 +- sqlite3_vendor/CHANGELOG.rst | 6 ++++++ sqlite3_vendor/package.xml | 2 +- zstd_vendor/CHANGELOG.rst | 8 ++++++++ zstd_vendor/package.xml | 2 +- 31 files changed, 148 insertions(+), 16 deletions(-) diff --git a/ros2bag/CHANGELOG.rst b/ros2bag/CHANGELOG.rst index c7ba0c5a05..721da1898b 100644 --- a/ros2bag/CHANGELOG.rst +++ b/ros2bag/CHANGELOG.rst @@ -2,6 +2,7 @@ Changelog for package ros2bag ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -26,6 +27,17 @@ Changelog for package ros2bag Forthcoming ----------- ======= +======= +0.6.0 (2021-02-01) +------------------ +* Recorder --regex and --exclude options (`#604 `_) +* Fix the tests on cyclonedds by translating qos duration values (`#606 `_) +* SQLite storage optimized by default (`#568 `_) +* Fix a bug on parsing wrong description in plugin xml file (`#578 `_) +* Compress bag files in separate threads (`#506 `_) +* Contributors: Adam Dąbrowski, Barry Xu, Emerson Knapp, P. J. Reed + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/ros2bag/package.xml b/ros2bag/package.xml index fcf8d0f574..4f815f7d94 100644 --- a/ros2bag/package.xml +++ b/ros2bag/package.xml @@ -2,7 +2,7 @@ ros2bag - 0.5.0 + 0.6.0 Entry point for rosbag in ROS 2 diff --git a/ros2bag/setup.py b/ros2bag/setup.py index f6efbec71f..80eabaf1a7 100644 --- a/ros2bag/setup.py +++ b/ros2bag/setup.py @@ -5,7 +5,7 @@ setup( name=package_name, - version='0.5.0', + version='0.6.0', packages=find_packages(exclude=['test']), data_files=[ ('share/' + package_name, ['package.xml']), diff --git a/rosbag2/CHANGELOG.rst b/rosbag2/CHANGELOG.rst index 8935870ca1..1779700563 100644 --- a/rosbag2/CHANGELOG.rst +++ b/rosbag2/CHANGELOG.rst @@ -2,6 +2,7 @@ Changelog for package rosbag2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -22,6 +23,11 @@ Forthcoming ----------- >>>>>>> Changelog. ======= +======= +0.6.0 (2021-02-01) +------------------ + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/rosbag2/package.xml b/rosbag2/package.xml index 5abb7fd263..a64c349feb 100644 --- a/rosbag2/package.xml +++ b/rosbag2/package.xml @@ -1,7 +1,7 @@ rosbag2 - 0.5.0 + 0.6.0 Meta package for rosbag2 related packages Karsten Knese Michael Jeronimo diff --git a/rosbag2_compression/CHANGELOG.rst b/rosbag2_compression/CHANGELOG.rst index d2e8b89bf7..7777babc06 100644 --- a/rosbag2_compression/CHANGELOG.rst +++ b/rosbag2_compression/CHANGELOG.rst @@ -2,6 +2,7 @@ Changelog for package rosbag2_compression ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -23,6 +24,19 @@ Changelog for package rosbag2_compression Forthcoming ----------- ======= +======= +0.6.0 (2021-02-01) +------------------ +* Make compressor implementations into a plugin via pluginlib (`#624 `_) +* Use ZSTD's streaming interface for [de]compressing files (`#543 `_) +* Fix build issues when rosbag2_storage is binary installed (`#585 `_) +* Fix relative metadata paths in SequentialCompressionWriter (`#613 `_) +* Fix deadlock race condition on compression shutdown (`#616 `_) +* Deduplicate SequentialCompressionReader business logic, add fallback to find bagfiles in incorrectly-written metadata (`#612 `_) +* Compress bag files in separate threads (`#506 `_) +* Contributors: Emerson Knapp, P. J. Reed + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/rosbag2_compression/package.xml b/rosbag2_compression/package.xml index fbbccc8e54..5ad68aab33 100644 --- a/rosbag2_compression/package.xml +++ b/rosbag2_compression/package.xml @@ -2,7 +2,7 @@ rosbag2_compression - 0.5.0 + 0.6.0 Compression implementations for rosbag2 bags and messages. Karsten Knese Michael Jeronimo diff --git a/rosbag2_converter_default_plugins/CHANGELOG.rst b/rosbag2_converter_default_plugins/CHANGELOG.rst index 4fe4758023..fad9a53e12 100644 --- a/rosbag2_converter_default_plugins/CHANGELOG.rst +++ b/rosbag2_converter_default_plugins/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package rosbag2_converter_default_plugins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -23,6 +24,11 @@ Forthcoming ----------- >>>>>>> Changelog. ======= +======= +0.6.0 (2021-02-01) +------------------ + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/rosbag2_converter_default_plugins/package.xml b/rosbag2_converter_default_plugins/package.xml index 3a22f8f0f4..c7931da1bb 100644 --- a/rosbag2_converter_default_plugins/package.xml +++ b/rosbag2_converter_default_plugins/package.xml @@ -1,7 +1,7 @@ rosbag2_converter_default_plugins - 0.5.0 + 0.6.0 Package containing default plugins for format converters Karsten Knese Michael Jeronimo diff --git a/rosbag2_cpp/CHANGELOG.rst b/rosbag2_cpp/CHANGELOG.rst index 8d8584ae53..3af0d6a930 100644 --- a/rosbag2_cpp/CHANGELOG.rst +++ b/rosbag2_cpp/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package rosbag2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -23,6 +24,16 @@ Changelog for package rosbag2 Forthcoming ----------- ======= +======= +0.6.0 (2021-02-01) +------------------ +* Fix build issues when rosbag2_storage is binary installed (`#585 `_) +* Deduplicate SequentialCompressionReader business logic, add fallback to find bagfiles in incorrectly-written metadata (`#612 `_) +* include what you use (`#600 `_) +* Only dereference the data pointer if it is valid. (`#581 `_) +* Contributors: Chris Lalancette, Emerson Knapp, Ivan Santiago Paunovic, P. J. Reed + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/rosbag2_cpp/package.xml b/rosbag2_cpp/package.xml index 5c633fa9e8..c9099bd01c 100644 --- a/rosbag2_cpp/package.xml +++ b/rosbag2_cpp/package.xml @@ -1,7 +1,7 @@ rosbag2_cpp - 0.5.0 + 0.6.0 C++ ROSBag2 client library Karsten Knese Michael Jeronimo diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/CHANGELOG.rst b/rosbag2_performance/rosbag2_performance_benchmarking/CHANGELOG.rst index 645aef0d25..cec117c79c 100644 --- a/rosbag2_performance/rosbag2_performance_benchmarking/CHANGELOG.rst +++ b/rosbag2_performance/rosbag2_performance_benchmarking/CHANGELOG.rst @@ -2,6 +2,11 @@ Changelog for package rosbag2_performance_benchmarking ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +0.6.0 (2021-02-01) +------------------ +* Performance benchmarking refactor (`#594 `_) +* Contributors: Adam Dąbrowski + 0.5.0 (2020-12-02) ------------------ * Sqlite storage double buffering (`#546 `_) diff --git a/rosbag2_performance/rosbag2_performance_benchmarking/package.xml b/rosbag2_performance/rosbag2_performance_benchmarking/package.xml index 5465e6791f..7cb71a7de8 100644 --- a/rosbag2_performance/rosbag2_performance_benchmarking/package.xml +++ b/rosbag2_performance/rosbag2_performance_benchmarking/package.xml @@ -2,7 +2,7 @@ rosbag2_performance_benchmarking - 0.5.0 + 0.6.0 Code to benchmark rosbag2 Karsten Knese Michael Jeronimo diff --git a/rosbag2_py/CHANGELOG.rst b/rosbag2_py/CHANGELOG.rst index f4dee0e537..7ab6e79003 100644 --- a/rosbag2_py/CHANGELOG.rst +++ b/rosbag2_py/CHANGELOG.rst @@ -2,6 +2,12 @@ Changelog for package rosbag2_py ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +0.6.0 (2021-02-01) +------------------ +* Fix build issues when rosbag2_storage is binary installed (`#585 `_) +* Fix the tests on cyclonedds by translating qos duration values (`#606 `_) +* Contributors: Emerson Knapp, P. J. Reed + 0.5.0 (2020-12-02) ------------------ diff --git a/rosbag2_py/package.xml b/rosbag2_py/package.xml index eb130aea9d..5b249596ab 100644 --- a/rosbag2_py/package.xml +++ b/rosbag2_py/package.xml @@ -2,7 +2,7 @@ rosbag2_py - 0.5.0 + 0.6.0 Python API for rosbag2 Karsten Knese Michael Jeronimo diff --git a/rosbag2_storage/CHANGELOG.rst b/rosbag2_storage/CHANGELOG.rst index 8173dba7b7..9614435f37 100644 --- a/rosbag2_storage/CHANGELOG.rst +++ b/rosbag2_storage/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package rosbag2_storage ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -22,6 +23,15 @@ Changelog for package rosbag2_storage Forthcoming ----------- ======= +======= +0.6.0 (2021-02-01) +------------------ +* SQLite storage optimized by default (`#568 `_) + * Use optimized pragmas by default in sqlite storage. Added option to use former behavior +* Use std::filesystem compliant non-member `exists` function for path object (`#593 `_) +* Contributors: Adam Dąbrowski, Josh Langsfeld + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/rosbag2_storage/package.xml b/rosbag2_storage/package.xml index 227c02689d..874862b8b3 100644 --- a/rosbag2_storage/package.xml +++ b/rosbag2_storage/package.xml @@ -1,7 +1,7 @@ rosbag2_storage - 0.5.0 + 0.6.0 ROS2 independent storage format to store serialized ROS2 messages Karsten Knese Michael Jeronimo diff --git a/rosbag2_storage_default_plugins/CHANGELOG.rst b/rosbag2_storage_default_plugins/CHANGELOG.rst index 7b636d3f11..33599b5338 100644 --- a/rosbag2_storage_default_plugins/CHANGELOG.rst +++ b/rosbag2_storage_default_plugins/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package rosbag2_storage_default_plugins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -23,6 +24,15 @@ Forthcoming ----------- >>>>>>> Changelog. ======= +======= +0.6.0 (2021-02-01) +------------------ +* Fix build issues when rosbag2_storage is binary installed (`#585 `_) +* Mutex protection for db writing and stl collections in writer & storage (`#603 `_) +* SQLite storage optimized by default (`#568 `_) +* Contributors: Adam Dąbrowski, P. J. Reed + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/rosbag2_storage_default_plugins/package.xml b/rosbag2_storage_default_plugins/package.xml index d2799edaf4..9fff09044f 100644 --- a/rosbag2_storage_default_plugins/package.xml +++ b/rosbag2_storage_default_plugins/package.xml @@ -1,7 +1,7 @@ rosbag2_storage_default_plugins - 0.5.0 + 0.6.0 ROSBag2 SQLite3 storage plugin Karsten Knese Michael Jeronimo diff --git a/rosbag2_test_common/CHANGELOG.rst b/rosbag2_test_common/CHANGELOG.rst index bcc1da45c6..af0359aa79 100644 --- a/rosbag2_test_common/CHANGELOG.rst +++ b/rosbag2_test_common/CHANGELOG.rst @@ -2,6 +2,7 @@ Changelog for package rosbag2_test_common ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -22,6 +23,13 @@ Forthcoming ----------- >>>>>>> Changelog. ======= +======= +0.6.0 (2021-02-01) +------------------ +* Stabilize test_record by reducing copies of executors and messages (`#576 `_) +* Contributors: Emerson Knapp + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/rosbag2_test_common/package.xml b/rosbag2_test_common/package.xml index 5d700a8f7c..46177a973b 100644 --- a/rosbag2_test_common/package.xml +++ b/rosbag2_test_common/package.xml @@ -2,7 +2,7 @@ rosbag2_test_common - 0.5.0 + 0.6.0 Commonly used test helper classes and fixtures for rosbag2 Karsten Knese Michael Jeronimo diff --git a/rosbag2_tests/CHANGELOG.rst b/rosbag2_tests/CHANGELOG.rst index 2315b509e2..094d578d1e 100644 --- a/rosbag2_tests/CHANGELOG.rst +++ b/rosbag2_tests/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package rosbag2_tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -23,6 +24,15 @@ Forthcoming ----------- >>>>>>> Changelog. ======= +======= +0.6.0 (2021-02-01) +------------------ +* Fix relative metadata paths in SequentialCompressionWriter (`#613 `_) +* Recorder --regex and --exclude options (`#604 `_) +* Fix the tests on cyclonedds by translating qos duration values (`#606 `_) +* Contributors: Adam Dąbrowski, Emerson Knapp + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/rosbag2_tests/package.xml b/rosbag2_tests/package.xml index 9b3d07e07c..18fee6a41f 100644 --- a/rosbag2_tests/package.xml +++ b/rosbag2_tests/package.xml @@ -2,7 +2,7 @@ rosbag2_tests - 0.5.0 + 0.6.0 Tests package for rosbag2 Karsten Knese Michael Jeronimo diff --git a/rosbag2_transport/CHANGELOG.rst b/rosbag2_transport/CHANGELOG.rst index 87a01a01b7..2ce1218770 100644 --- a/rosbag2_transport/CHANGELOG.rst +++ b/rosbag2_transport/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package rosbag2_transport ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -31,6 +32,19 @@ Forthcoming ----------- >>>>>>> Changelog. ======= +======= +0.6.0 (2021-02-01) +------------------ +* Fix build issues when rosbag2_storage is binary installed (`#585 `_) +* Regex and exclude fix for rosbag recorder (`#620 `_) +* Recorder --regex and --exclude options (`#604 `_) +* SQLite storage optimized by default (`#568 `_) +* Fixed playing if unknown message types exist (`#592 `_) +* Compress bag files in separate threads (`#506 `_) +* Stabilize test_record by reducing copies of executors and messages (`#576 `_) +* Contributors: Adam Dąbrowski, Chen Lihui, Emerson Knapp, P. J. Reed, Piotr Jaroszek + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/rosbag2_transport/package.xml b/rosbag2_transport/package.xml index 932f153b20..75201fd8d8 100644 --- a/rosbag2_transport/package.xml +++ b/rosbag2_transport/package.xml @@ -2,7 +2,7 @@ rosbag2_transport - 0.5.0 + 0.6.0 Layer encapsulating ROS middleware to allow rosbag2 to be used with or without middleware Karsten Knese Michael Jeronimo diff --git a/shared_queues_vendor/CHANGELOG.rst b/shared_queues_vendor/CHANGELOG.rst index 53754a42bf..97c87bcb62 100644 --- a/shared_queues_vendor/CHANGELOG.rst +++ b/shared_queues_vendor/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package shared_queues_vendor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -23,6 +24,11 @@ Forthcoming ----------- >>>>>>> Changelog. ======= +======= +0.6.0 (2021-02-01) +------------------ + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/shared_queues_vendor/package.xml b/shared_queues_vendor/package.xml index 32bce4d326..2487533723 100644 --- a/shared_queues_vendor/package.xml +++ b/shared_queues_vendor/package.xml @@ -2,7 +2,7 @@ shared_queues_vendor - 0.5.0 + 0.6.0 Vendor package for concurrent queues from moodycamel Karsten Knese Michael Jeronimo diff --git a/sqlite3_vendor/CHANGELOG.rst b/sqlite3_vendor/CHANGELOG.rst index 64c82fc983..0a6d852c07 100644 --- a/sqlite3_vendor/CHANGELOG.rst +++ b/sqlite3_vendor/CHANGELOG.rst @@ -3,6 +3,7 @@ Changelog for package sqlite3_vendor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -25,6 +26,11 @@ Forthcoming ----------- >>>>>>> Changelog. ======= +======= +0.6.0 (2021-02-01) +------------------ + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/sqlite3_vendor/package.xml b/sqlite3_vendor/package.xml index 92ecfda940..2caee5ea90 100644 --- a/sqlite3_vendor/package.xml +++ b/sqlite3_vendor/package.xml @@ -2,7 +2,7 @@ sqlite3_vendor - 0.5.0 + 0.6.0 SQLite 3 vendor package Karsten Knese Michael Jeronimo diff --git a/zstd_vendor/CHANGELOG.rst b/zstd_vendor/CHANGELOG.rst index d5689685cf..10085205be 100644 --- a/zstd_vendor/CHANGELOG.rst +++ b/zstd_vendor/CHANGELOG.rst @@ -2,6 +2,7 @@ Changelog for package zstd_vendor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD 0.3.6 (2021-01-05) @@ -23,6 +24,13 @@ Forthcoming ----------- >>>>>>> Changelog. ======= +======= +0.6.0 (2021-02-01) +------------------ +* Patch zstd 1.4.4 to include cmake_minimum_version bump to 2.8.12 (`#579 `_) +* Contributors: Emerson Knapp + +>>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ >>>>>>> 0.5.0 diff --git a/zstd_vendor/package.xml b/zstd_vendor/package.xml index 23c0a18e37..e03663c816 100644 --- a/zstd_vendor/package.xml +++ b/zstd_vendor/package.xml @@ -2,7 +2,7 @@ zstd_vendor - 0.5.0 + 0.6.0 Zstd compression vendor package, providing a dependency for Zstd. Karsten Knese Michael Jeronimo From 6007920f43e69a4ff01ab8173b912a9fa641b510 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 2 Feb 2021 12:32:39 -0800 Subject: [PATCH 76/77] Fix the CHANGELOGs Signed-off-by: Emerson Knapp --- ros2bag/CHANGELOG.rst | 48 +++++++--------- rosbag2/CHANGELOG.rst | 38 +++++-------- rosbag2_compression/CHANGELOG.rst | 42 ++++++-------- .../CHANGELOG.rst | 31 ++++------- rosbag2_cpp/CHANGELOG.rst | 41 +++++--------- rosbag2_storage/CHANGELOG.rst | 39 +++++-------- rosbag2_storage_default_plugins/CHANGELOG.rst | 39 +++++-------- rosbag2_test_common/CHANGELOG.rst | 36 +++++------- rosbag2_tests/CHANGELOG.rst | 39 +++++-------- rosbag2_transport/CHANGELOG.rst | 55 ++++++++----------- shared_queues_vendor/CHANGELOG.rst | 31 ++++------- sqlite3_vendor/CHANGELOG.rst | 33 ++++------- zstd_vendor/CHANGELOG.rst | 36 +++++------- 13 files changed, 185 insertions(+), 323 deletions(-) diff --git a/ros2bag/CHANGELOG.rst b/ros2bag/CHANGELOG.rst index 721da1898b..9c535a119e 100644 --- a/ros2bag/CHANGELOG.rst +++ b/ros2bag/CHANGELOG.rst @@ -2,32 +2,6 @@ Changelog for package ros2bag ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -0.3.6 (2021-01-05) ------------------- -* Update maintainer list for Foxy (`#551 `_) -* Contributors: Michael Jeronimo - -0.3.5 (2020-08-31) ------------------- -* Add pytest.ini so local tests don't display warning. (`#446 `_) (`#514 `_) -* Contributors: Emerson Knapp - -0.3.4 (2020-08-05) ------------------- -* Validate QoS profile values are not negative. (`#483 `_) (`#490 `_) - Co-authored-by: Jesse Ikawa <64169356+jikawa-az@users.noreply.github.com> -* Contributors: Devin Bonnie - -0.3.3 (2020-06-23) ------------------- -======= -Forthcoming ------------ -======= -======= 0.6.0 (2021-02-01) ------------------ * Recorder --regex and --exclude options (`#604 `_) @@ -37,10 +11,8 @@ Forthcoming * Compress bag files in separate threads (`#506 `_) * Contributors: Adam Dąbrowski, Barry Xu, Emerson Knapp, P. J. Reed ->>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ ->>>>>>> 0.5.0 * Sqlite storage double buffering (`#546 `_) * Double buffers * Circular queue and FLUSH option as define @@ -89,7 +61,6 @@ Forthcoming * added minor comment Co-authored-by: Piotr Jaroszek * Contributors: Adam Dąbrowski ->>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ @@ -116,6 +87,25 @@ Forthcoming * Add pytest.ini so local tests don't display warning (`#446 `_) * Contributors: Adam Dąbrowski, Barry Xu, Chris Lalancette, Dirk Thomas, Ivan Santiago Paunovic, Jacob Perron, Jaison Titus, Jesse Ikawa, Karsten Knese, Marwan Taher, Michael Jeronimo, P. J. Reed, jhdcs +0.3.6 (2021-01-05) +------------------ +* Update maintainer list for Foxy (`#551 `_) +* Contributors: Michael Jeronimo + +0.3.5 (2020-08-31) +------------------ +* Add pytest.ini so local tests don't display warning. (`#446 `_) (`#514 `_) +* Contributors: Emerson Knapp + +0.3.4 (2020-08-05) +------------------ +* Validate QoS profile values are not negative. (`#483 `_) (`#490 `_) + Co-authored-by: Jesse Ikawa <64169356+jikawa-az@users.noreply.github.com> +* Contributors: Devin Bonnie + +0.3.3 (2020-06-23) +------------------ + 0.3.2 (2020-06-03) ------------------ * Improve help message for CLI verbs (`#427 `_) diff --git a/rosbag2/CHANGELOG.rst b/rosbag2/CHANGELOG.rst index 1779700563..ebf8296962 100644 --- a/rosbag2/CHANGELOG.rst +++ b/rosbag2/CHANGELOG.rst @@ -2,35 +2,11 @@ Changelog for package rosbag2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -0.3.6 (2021-01-05) ------------------- -* Update maintainer list for Foxy (`#551 `_) -* Contributors: Michael Jeronimo - -0.3.5 (2020-08-31) ------------------- - -0.3.4 (2020-08-05) ------------------- - -0.3.3 (2020-06-23) ------------------- -======= -Forthcoming ------------ ->>>>>>> Changelog. -======= -======= 0.6.0 (2021-02-01) ------------------ ->>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ ->>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ @@ -40,6 +16,20 @@ Forthcoming * rosbag2_py reader and writer (`#308 `_) * Contributors: Karsten Knese, Mabel Zhang, Michael Jeronimo +0.3.6 (2021-01-05) +------------------ +* Update maintainer list for Foxy (`#551 `_) +* Contributors: Michael Jeronimo + +0.3.5 (2020-08-31) +------------------ + +0.3.4 (2020-08-05) +------------------ + +0.3.3 (2020-06-23) +------------------ + 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2_compression/CHANGELOG.rst b/rosbag2_compression/CHANGELOG.rst index 7777babc06..665992d10b 100644 --- a/rosbag2_compression/CHANGELOG.rst +++ b/rosbag2_compression/CHANGELOG.rst @@ -2,29 +2,6 @@ Changelog for package rosbag2_compression ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -0.3.6 (2021-01-05) ------------------- -* Update maintainer list for Foxy (`#551 `_) -* Contributors: Michael Jeronimo - -0.3.5 (2020-08-31) ------------------- - -0.3.4 (2020-08-05) ------------------- -* Fix exception thrown given invalid arguments with compression enabled (`#488 `_) (`#489 `_) -* Contributors: Devin Bonnie - -0.3.3 (2020-06-23) ------------------- -======= -Forthcoming ------------ -======= -======= 0.6.0 (2021-02-01) ------------------ * Make compressor implementations into a plugin via pluginlib (`#624 `_) @@ -36,13 +13,10 @@ Forthcoming * Compress bag files in separate threads (`#506 `_) * Contributors: Emerson Knapp, P. J. Reed ->>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ ->>>>>>> 0.5.0 * Sqlite storage double buffering (`#546 `_) * Contributors: Adam Dąbrowski ->>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ @@ -56,6 +30,22 @@ Forthcoming * Add per-message ZSTD compression (`#418 `_) * Contributors: Christophe Bedard, Devin Bonnie, Jaison Titus, Karsten Knese, Marwan Taher, Michael Jeronimo, P. J. Reed +0.3.6 (2021-01-05) +------------------ +* Update maintainer list for Foxy (`#551 `_) +* Contributors: Michael Jeronimo + +0.3.5 (2020-08-31) +------------------ + +0.3.4 (2020-08-05) +------------------ +* Fix exception thrown given invalid arguments with compression enabled (`#488 `_) (`#489 `_) +* Contributors: Devin Bonnie + +0.3.3 (2020-06-23) +------------------ + 0.3.2 (2020-06-03) ------------------ * Add user provided split size to error message (`#430 `_) diff --git a/rosbag2_converter_default_plugins/CHANGELOG.rst b/rosbag2_converter_default_plugins/CHANGELOG.rst index fad9a53e12..ac2b9ca5c9 100644 --- a/rosbag2_converter_default_plugins/CHANGELOG.rst +++ b/rosbag2_converter_default_plugins/CHANGELOG.rst @@ -2,10 +2,17 @@ Changelog for package rosbag2_converter_default_plugins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +0.6.0 (2021-02-01) +------------------ + +0.5.0 (2020-12-02) +------------------ + +0.4.0 (2020-11-19) +------------------ +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Contributors: Michael Jeronimo -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -19,24 +26,6 @@ Changelog for package rosbag2_converter_default_plugins 0.3.3 (2020-06-23) ------------------ -======= -Forthcoming ------------ ->>>>>>> Changelog. -======= -======= -0.6.0 (2021-02-01) ------------------- - ->>>>>>> Prepare bloom release 0.6.0 (#628) -0.5.0 (2020-12-02) ------------------- ->>>>>>> 0.5.0 - -0.4.0 (2020-11-19) ------------------- -* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) -* Contributors: Michael Jeronimo 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2_cpp/CHANGELOG.rst b/rosbag2_cpp/CHANGELOG.rst index 3af0d6a930..c6caab84bf 100644 --- a/rosbag2_cpp/CHANGELOG.rst +++ b/rosbag2_cpp/CHANGELOG.rst @@ -2,29 +2,6 @@ Changelog for package rosbag2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -0.3.6 (2021-01-05) ------------------- -* Mutex around writer access in recorder (`#491 `_) (`#575 `_) -* Update maintainer list for Foxy (`#551 `_) -* Contributors: Michael Jeronimo, Patrick Spieler - -0.3.5 (2020-08-31) ------------------- - -0.3.4 (2020-08-05) ------------------- - -0.3.3 (2020-06-23) ------------------- -======= -Forthcoming ------------ -======= -======= 0.6.0 (2021-02-01) ------------------ * Fix build issues when rosbag2_storage is binary installed (`#585 `_) @@ -33,14 +10,11 @@ Forthcoming * Only dereference the data pointer if it is valid. (`#581 `_) * Contributors: Chris Lalancette, Emerson Knapp, Ivan Santiago Paunovic, P. J. Reed ->>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ ->>>>>>> 0.5.0 * Add back rosbag2_cpp::StorageOptions as deprecated (`#563 `_) * Sqlite storage double buffering (`#546 `_) * Contributors: Adam Dąbrowski, Jacob Perron ->>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ @@ -61,6 +35,21 @@ Forthcoming * Add split by time to recording (`#409 `_) * Contributors: Dirk Thomas, Jacob Perron, Jaison Titus, Karsten Knese, Marwan Taher, Michael Jeronimo, Patrick Spieler, jhdcs, Tomoya Fujita +0.3.6 (2021-01-05) +------------------ +* Mutex around writer access in recorder (`#491 `_) (`#575 `_) +* Update maintainer list for Foxy (`#551 `_) +* Contributors: Michael Jeronimo, Patrick Spieler + +0.3.5 (2020-08-31) +------------------ + +0.3.4 (2020-08-05) +------------------ + +0.3.3 (2020-06-23) +------------------ + 0.3.2 (2020-06-03) ------------------ * Add user provided split size to error (`#430 `_) diff --git a/rosbag2_storage/CHANGELOG.rst b/rosbag2_storage/CHANGELOG.rst index 9614435f37..ee54032d03 100644 --- a/rosbag2_storage/CHANGELOG.rst +++ b/rosbag2_storage/CHANGELOG.rst @@ -2,28 +2,6 @@ Changelog for package rosbag2_storage ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -0.3.6 (2021-01-05) ------------------- -* Update maintainer list for Foxy (`#551 `_) -* Contributors: Michael Jeronimo - -0.3.5 (2020-08-31) ------------------- - -0.3.4 (2020-08-05) ------------------- - -0.3.3 (2020-06-23) ------------------- -======= -Forthcoming ------------ -======= -======= 0.6.0 (2021-02-01) ------------------ * SQLite storage optimized by default (`#568 `_) @@ -31,13 +9,10 @@ Forthcoming * Use std::filesystem compliant non-member `exists` function for path object (`#593 `_) * Contributors: Adam Dąbrowski, Josh Langsfeld ->>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ ->>>>>>> 0.5.0 * Update codes since rcutils_calculate_directory_size() is changed (`#567 `_) * Contributors: Barry Xu ->>>>>>> Changelog. 0.4.0 (2020-11-19) ------------------ @@ -46,6 +21,20 @@ Forthcoming * Add split by time to recording (`#409 `_) * Contributors: Karsten Knese, Michael Jeronimo, jhdcs +0.3.6 (2021-01-05) +------------------ +* Update maintainer list for Foxy (`#551 `_) +* Contributors: Michael Jeronimo + +0.3.5 (2020-08-31) +------------------ + +0.3.4 (2020-08-05) +------------------ + +0.3.3 (2020-06-23) +------------------ + 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2_storage_default_plugins/CHANGELOG.rst b/rosbag2_storage_default_plugins/CHANGELOG.rst index 33599b5338..56d37ffbea 100644 --- a/rosbag2_storage_default_plugins/CHANGELOG.rst +++ b/rosbag2_storage_default_plugins/CHANGELOG.rst @@ -2,29 +2,6 @@ Changelog for package rosbag2_storage_default_plugins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -0.3.6 (2021-01-05) ------------------- -* Update maintainer list for Foxy (`#551 `_) -* Contributors: Michael Jeronimo - -0.3.5 (2020-08-31) ------------------- - -0.3.4 (2020-08-05) ------------------- - -0.3.3 (2020-06-23) ------------------- -======= -Forthcoming ------------ ->>>>>>> Changelog. -======= -======= 0.6.0 (2021-02-01) ------------------ * Fix build issues when rosbag2_storage is binary installed (`#585 `_) @@ -32,10 +9,8 @@ Forthcoming * SQLite storage optimized by default (`#568 `_) * Contributors: Adam Dąbrowski, P. J. Reed ->>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ ->>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ @@ -44,6 +19,20 @@ Forthcoming * Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) * Contributors: Karsten Knese, Michael Jeronimo +0.3.6 (2021-01-05) +------------------ +* Update maintainer list for Foxy (`#551 `_) +* Contributors: Michael Jeronimo + +0.3.5 (2020-08-31) +------------------ + +0.3.4 (2020-08-05) +------------------ + +0.3.3 (2020-06-23) +------------------ + 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2_test_common/CHANGELOG.rst b/rosbag2_test_common/CHANGELOG.rst index af0359aa79..a55cb3b1d6 100644 --- a/rosbag2_test_common/CHANGELOG.rst +++ b/rosbag2_test_common/CHANGELOG.rst @@ -2,9 +2,19 @@ Changelog for package rosbag2_test_common ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD +0.6.0 (2021-02-01) +------------------ +* Stabilize test_record by reducing copies of executors and messages (`#576 `_) +* Contributors: Emerson Knapp + +0.5.0 (2020-12-02) +------------------ + +0.4.0 (2020-11-19) +------------------ +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Contributors: Michael Jeronimo + 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -18,26 +28,6 @@ Changelog for package rosbag2_test_common 0.3.3 (2020-06-23) ------------------ -======= -Forthcoming ------------ ->>>>>>> Changelog. -======= -======= -0.6.0 (2021-02-01) ------------------- -* Stabilize test_record by reducing copies of executors and messages (`#576 `_) -* Contributors: Emerson Knapp - ->>>>>>> Prepare bloom release 0.6.0 (#628) -0.5.0 (2020-12-02) ------------------- ->>>>>>> 0.5.0 - -0.4.0 (2020-11-19) ------------------- -* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) -* Contributors: Michael Jeronimo 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2_tests/CHANGELOG.rst b/rosbag2_tests/CHANGELOG.rst index 094d578d1e..0d196d0c59 100644 --- a/rosbag2_tests/CHANGELOG.rst +++ b/rosbag2_tests/CHANGELOG.rst @@ -2,29 +2,6 @@ Changelog for package rosbag2_tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -0.3.6 (2021-01-05) ------------------- -* Update maintainer list for Foxy (`#551 `_) -* Contributors: Michael Jeronimo - -0.3.5 (2020-08-31) ------------------- - -0.3.4 (2020-08-05) ------------------- - -0.3.3 (2020-06-23) ------------------- -======= -Forthcoming ------------ ->>>>>>> Changelog. -======= -======= 0.6.0 (2021-02-01) ------------------ * Fix relative metadata paths in SequentialCompressionWriter (`#613 `_) @@ -32,10 +9,8 @@ Forthcoming * Fix the tests on cyclonedds by translating qos duration values (`#606 `_) * Contributors: Adam Dąbrowski, Emerson Knapp ->>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ ->>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ @@ -50,6 +25,20 @@ Forthcoming * Add split by time to recording (`#409 `_) * Contributors: Emerson Knapp, Jaison Titus, Karsten Knese, Marwan Taher, Michael Jeronimo, jhdcs +0.3.6 (2021-01-05) +------------------ +* Update maintainer list for Foxy (`#551 `_) +* Contributors: Michael Jeronimo + +0.3.5 (2020-08-31) +------------------ + +0.3.4 (2020-08-05) +------------------ + +0.3.3 (2020-06-23) +------------------ + 0.3.2 (2020-06-03) ------------------ diff --git a/rosbag2_transport/CHANGELOG.rst b/rosbag2_transport/CHANGELOG.rst index 2ce1218770..79cd1ebe98 100644 --- a/rosbag2_transport/CHANGELOG.rst +++ b/rosbag2_transport/CHANGELOG.rst @@ -2,37 +2,6 @@ Changelog for package rosbag2_transport ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -0.3.6 (2021-01-05) ------------------- -* Update maintainer list for Foxy (`#551 `_) -* Contributors: Michael Jeronimo - -0.3.5 (2020-08-31) ------------------- -* resolve memory leak for serialized message (`#502 `_) (`#518 `_) -* Use shared logic for importing the rosbag2_transport_py library in Python (`#482 `_) (`#494 `_) -* Contributors: Emerson Knapp, Karsten Knese - -0.3.4 (2020-08-05) ------------------- -* fix missing target dependencies (`#479 `_) (`#481 `_) - Co-authored-by: Dirk Thomas -* Contributors: Karsten Knese - -0.3.3 (2020-06-23) ------------------- -* export shared_queues_vendor for modern cmake support (`#434 `_) (`#438 `_) -* Contributors: Karsten Knese -======= -Forthcoming ------------ ->>>>>>> Changelog. -======= -======= 0.6.0 (2021-02-01) ------------------ * Fix build issues when rosbag2_storage is binary installed (`#585 `_) @@ -44,10 +13,8 @@ Forthcoming * Stabilize test_record by reducing copies of executors and messages (`#576 `_) * Contributors: Adam Dąbrowski, Chen Lihui, Emerson Knapp, P. J. Reed, Piotr Jaroszek ->>>>>>> Prepare bloom release 0.6.0 (#628) 0.5.0 (2020-12-02) ------------------ ->>>>>>> 0.5.0 0.4.0 (2020-11-19) ------------------ @@ -62,6 +29,28 @@ Forthcoming * export shared_queues_vendor (`#434 `_) * Contributors: Dirk Thomas, Emerson Knapp, Karsten Knese, Michael Jeronimo, jhdcs +0.3.6 (2021-01-05) +------------------ +* Update maintainer list for Foxy (`#551 `_) +* Contributors: Michael Jeronimo + +0.3.5 (2020-08-31) +------------------ +* resolve memory leak for serialized message (`#502 `_) (`#518 `_) +* Use shared logic for importing the rosbag2_transport_py library in Python (`#482 `_) (`#494 `_) +* Contributors: Emerson Knapp, Karsten Knese + +0.3.4 (2020-08-05) +------------------ +* fix missing target dependencies (`#479 `_) (`#481 `_) + Co-authored-by: Dirk Thomas +* Contributors: Karsten Knese + +0.3.3 (2020-06-23) +------------------ +* export shared_queues_vendor for modern cmake support (`#434 `_) (`#438 `_) +* Contributors: Karsten Knese + 0.3.2 (2020-06-03) ------------------ diff --git a/shared_queues_vendor/CHANGELOG.rst b/shared_queues_vendor/CHANGELOG.rst index 97c87bcb62..aff32d9309 100644 --- a/shared_queues_vendor/CHANGELOG.rst +++ b/shared_queues_vendor/CHANGELOG.rst @@ -2,10 +2,17 @@ Changelog for package shared_queues_vendor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +0.6.0 (2021-02-01) +------------------ + +0.5.0 (2020-12-02) +------------------ + +0.4.0 (2020-11-19) +------------------ +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Contributors: Michael Jeronimo -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -19,24 +26,6 @@ Changelog for package shared_queues_vendor 0.3.3 (2020-06-23) ------------------ -======= -Forthcoming ------------ ->>>>>>> Changelog. -======= -======= -0.6.0 (2021-02-01) ------------------- - ->>>>>>> Prepare bloom release 0.6.0 (#628) -0.5.0 (2020-12-02) ------------------- ->>>>>>> 0.5.0 - -0.4.0 (2020-11-19) ------------------- -* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) -* Contributors: Michael Jeronimo 0.3.2 (2020-06-03) ------------------ diff --git a/sqlite3_vendor/CHANGELOG.rst b/sqlite3_vendor/CHANGELOG.rst index 0a6d852c07..52a799eaa0 100644 --- a/sqlite3_vendor/CHANGELOG.rst +++ b/sqlite3_vendor/CHANGELOG.rst @@ -2,10 +2,18 @@ Changelog for package sqlite3_vendor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +0.6.0 (2021-02-01) +------------------ + +0.5.0 (2020-12-02) +------------------ + +0.4.0 (2020-11-19) +------------------ +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* use interface_include_directories (`#426 `_) +* Contributors: Karsten Knese, Michael Jeronimo -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD 0.3.6 (2021-01-05) ------------------ * Update maintainer list for Foxy (`#551 `_) @@ -21,25 +29,6 @@ Changelog for package sqlite3_vendor ------------------ * use interface_include_directories in find_sqlite3 (`#426 `_) (`#444 `_) * Contributors: Karsten Knese -======= -Forthcoming ------------ ->>>>>>> Changelog. -======= -======= -0.6.0 (2021-02-01) ------------------- - ->>>>>>> Prepare bloom release 0.6.0 (#628) -0.5.0 (2020-12-02) ------------------- ->>>>>>> 0.5.0 - -0.4.0 (2020-11-19) ------------------- -* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) -* use interface_include_directories (`#426 `_) -* Contributors: Karsten Knese, Michael Jeronimo 0.3.2 (2020-06-03) ------------------ diff --git a/zstd_vendor/CHANGELOG.rst b/zstd_vendor/CHANGELOG.rst index 10085205be..1f86dc8552 100644 --- a/zstd_vendor/CHANGELOG.rst +++ b/zstd_vendor/CHANGELOG.rst @@ -2,9 +2,19 @@ Changelog for package zstd_vendor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD +0.6.0 (2021-02-01) +------------------ +* Patch zstd 1.4.4 to include cmake_minimum_version bump to 2.8.12 (`#579 `_) +* Contributors: Emerson Knapp + +0.5.0 (2020-12-02) +------------------ + +0.4.0 (2020-11-19) +------------------ +* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) +* Contributors: Michael Jeronimo + 0.3.6 (2021-01-05) ------------------ * Patch zstd 1.4.4 to include cmake_minimum_version bump to 2.8.12 (`#579 `_) (`#587 `_) @@ -19,26 +29,6 @@ Changelog for package zstd_vendor 0.3.3 (2020-06-23) ------------------ -======= -Forthcoming ------------ ->>>>>>> Changelog. -======= -======= -0.6.0 (2021-02-01) ------------------- -* Patch zstd 1.4.4 to include cmake_minimum_version bump to 2.8.12 (`#579 `_) -* Contributors: Emerson Knapp - ->>>>>>> Prepare bloom release 0.6.0 (#628) -0.5.0 (2020-12-02) ------------------- ->>>>>>> 0.5.0 - -0.4.0 (2020-11-19) ------------------- -* Update the package.xml files with the latest Open Robotics maintainers (`#535 `_) -* Contributors: Michael Jeronimo 0.3.2 (2020-06-03) ------------------ From e5c795142241b8853d6a3765a025e930a37991ea Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 2 Feb 2021 16:34:32 -0800 Subject: [PATCH 77/77] Minimal changes to build+test for Foxy Signed-off-by: Emerson Knapp --- .github/workflows/test.yml | 3 ++- rosbag2/package.xml | 2 +- .../rosbag2_compression/sequential_compression_reader.cpp | 2 +- rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp | 1 + rosbag2_py/COLCON_IGNORE | 0 rosbag2_storage/src/rosbag2_storage/metadata_io.cpp | 7 +------ rosbag2_transport/CMakeLists.txt | 2 +- 7 files changed, 7 insertions(+), 10 deletions(-) create mode 100644 rosbag2_py/COLCON_IGNORE diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cc8298de82..09e5e96dd7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,8 @@ jobs: sqlite3_vendor rosbag2_test_common rosbag2_tests - target-ros2-distro: rolling + target-ros2-distro: foxy + vcs-repo-file-url: "" - uses: actions/upload-artifact@v1 with: name: colcon-logs diff --git a/rosbag2/package.xml b/rosbag2/package.xml index a64c349feb..a19caae6ba 100644 --- a/rosbag2/package.xml +++ b/rosbag2/package.xml @@ -14,7 +14,7 @@ rosbag2_compression rosbag2_converter_default_plugins rosbag2_cpp - rosbag2_py + rosbag2_storage rosbag2_storage_default_plugins rosbag2_transport diff --git a/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp b/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp index 0070322f44..aeb1de54d6 100644 --- a/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp +++ b/rosbag2_compression/src/rosbag2_compression/sequential_compression_reader.cpp @@ -67,7 +67,7 @@ void SequentialCompressionReader::preprocess_current_file() * Because we have no way to check whether the bag was written correctly, * check for the existence of the prefixed file as a fallback. */ - const rcpputils::fs::path base{base_folder_}; + rcpputils::fs::path base{base_folder_}; const rcpputils::fs::path relative{get_current_file()}; const auto resolved = base / relative; if (!resolved.exists()) { diff --git a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp index d0d53e6291..5e65c197b3 100644 --- a/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp +++ b/rosbag2_cpp/src/rosbag2_cpp/writers/sequential_writer.cpp @@ -110,6 +110,7 @@ void SequentialWriter::open( } bool dir_created = rcpputils::fs::create_directories(db_path); + dir_created &= db_path.is_directory(); if (!dir_created) { std::stringstream error; error << "Failed to create database directory (" << db_path.string() << ")."; diff --git a/rosbag2_py/COLCON_IGNORE b/rosbag2_py/COLCON_IGNORE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/rosbag2_storage/src/rosbag2_storage/metadata_io.cpp b/rosbag2_storage/src/rosbag2_storage/metadata_io.cpp index aa81b78fb9..60295edaee 100644 --- a/rosbag2_storage/src/rosbag2_storage/metadata_io.cpp +++ b/rosbag2_storage/src/rosbag2_storage/metadata_io.cpp @@ -232,12 +232,7 @@ BagMetadata MetadataIo::read_metadata(const std::string & uri) YAML::Node yaml_file = YAML::LoadFile(get_metadata_file_name(uri)); auto metadata = yaml_file["rosbag2_bagfile_information"].as(); rcutils_allocator_t allocator = rcutils_get_default_allocator(); - if (RCUTILS_RET_OK != - rcutils_calculate_directory_size(uri.c_str(), &metadata.bag_size, allocator)) - { - throw std::runtime_error( - std::string("Exception on calculating the size of directory :") + uri); - } + metadata.bag_size = rcutils_calculate_directory_size(uri.c_str(), allocator); return metadata; } catch (const YAML::Exception & ex) { throw std::runtime_error(std::string("Exception on parsing info file: ") + ex.what()); diff --git a/rosbag2_transport/CMakeLists.txt b/rosbag2_transport/CMakeLists.txt index de85e96cef..8aa04ead74 100644 --- a/rosbag2_transport/CMakeLists.txt +++ b/rosbag2_transport/CMakeLists.txt @@ -166,7 +166,7 @@ function(create_tests_for_rmw_implementation) rosbag2_transport_add_gmock(test_record_regex test/rosbag2_transport/test_record_regex.cpp LINK_LIBS rosbag2_transport - AMENT_DEPS test_msgs rosbag2_test_common + AMENT_DEPS test_msgs rosbag2_cpp rosbag2_test_common ${SKIP_TEST}) rosbag2_transport_add_gmock(test_play