[foxy backport] Latest rosbag2

.github/CODEOWNERS

@@ -0,0 +1,3 @@
+# Reviewers (and by extension approvers) will be the default owners for everything in the repo.
+# Unless a later match takes precedence, @reviewers will be requested for review when someone opens a pull request.
+* @ros2/tooling-reviewers

.github/workflows/lint.yml

@@ -1,8 +1,4 @@
 # Run linters automatically on pull requests.
-#
-# ros-tooling/actino-ros-lint is relying on the latest APT binary packages.
-# As of 2020-05-01, Eloquent is the latest release for which APT binary
-# packages are available, which is why the Docker image is based on Eloquent.
 name: Lint rosbag2
 on:
   pull_request:
@@ -12,14 +8,13 @@ jobs:
     name: ament_copyright
     runs-on: ubuntu-latest
     container:
-      image: rostooling/setup-ros-docker:ubuntu-bionic-ros-eloquent-ros-base-latest
+      image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest
     steps:
-    # TODO(setup-ros-docker#7): calling chown is necessary for now
-    - run: sudo chown -R rosbuild:rosbuild "$HOME" .
     - uses: actions/checkout@v2
     - uses: ros-tooling/[email protected]
       with:
         linter: copyright
+        distribution: foxy
         package-name: |
             ros2bag
             rosbag2_compression
@@ -34,14 +29,13 @@ jobs:
     name: ament_xmllint
     runs-on: ubuntu-latest
     container:
-      image: rostooling/setup-ros-docker:ubuntu-bionic-ros-eloquent-ros-base-latest
+      image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest
     steps:
-    # TODO(setup-ros-docker#7): calling chown is necessary for now
-    - run: sudo chown -R rosbuild:rosbuild "$HOME" .
     - uses: actions/checkout@v2
     - uses: ros-tooling/[email protected]
       with:
         linter: xmllint
+        distribution: foxy
         package-name: |
             ros2bag
             rosbag2
@@ -57,19 +51,18 @@ jobs:
   ament_lint_cpp: # Linters applicable to C++ packages
     name: ament_${{ matrix.linter }}
     runs-on: ubuntu-latest
-    container:
-      image: rostooling/setup-ros-docker:ubuntu-bionic-ros-eloquent-ros-base-latest
     strategy:
       fail-fast: false
       matrix:
           linter: [cppcheck, cpplint, uncrustify]
+    container:
+      image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest
     steps:
-    # TODO(setup-ros-docker#7): calling chown is necessary for now
-    - run: sudo chown -R rosbuild:rosbuild "$HOME" .
     - uses: actions/checkout@v2
     - uses: ros-tooling/[email protected]
       with:
         linter: ${{ matrix.linter }}
+        distribution: foxy
         package-name: |
             rosbag2_compression
             rosbag2_converter_default_plugins
@@ -83,17 +76,16 @@ jobs:
     name: ament_${{ matrix.linter }}
     runs-on: ubuntu-latest
     container:
-      image: rostooling/setup-ros-docker:ubuntu-bionic-ros-eloquent-ros-base-latest
+      image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest
     strategy:
       fail-fast: false
       matrix:
-          linter: [pep257] # TODO re-enable flake8 when it is fixed
+          linter: [pep257, flake8]
     steps:
-    # TODO(setup-ros-docker#7): calling chown is necessary for now
-    - run: sudo chown -R rosbuild:rosbuild "$HOME" .
     - uses: actions/checkout@v2
     - uses: ros-tooling/[email protected]
       with:
         linter: ${{ matrix.linter }}
+        distribution: foxy
         package-name: |
             ros2bag

.github/workflows/test.yml

@@ -1,6 +1,6 @@
 name: Test rosbag2
 on:
-  # pull_request:
+  pull_request:
   push:
     branches:
       - master
@@ -13,11 +13,9 @@ jobs:
   build_and_test:
     runs-on: ubuntu-latest
     container:
-      image: rostooling/setup-ros-docker:ubuntu-focal-ros-foxy-ros-base-testing-latest
+      image: rostooling/setup-ros-docker:ubuntu-focal-latest
     steps:
-    # TODO(setup-ros-docker#7): calling chown is necessary for now
-    - run: sudo chown -R rosbuild:rosbuild "$HOME" .
-    - uses: ros-tooling/[email protected]
+    - uses: ros-tooling/[email protected]
       with:
         package-name: |
           ros2bag
@@ -31,6 +29,8 @@ jobs:
           sqlite3_vendor
           rosbag2_test_common
           rosbag2_tests
+        target-ros2-distro: foxy
+        vcs-repo-file-url: ""
     - uses: actions/upload-artifact@v1
       with:
         name: colcon-logs

README.md

@@ -38,7 +38,7 @@ Clone this repository into the source folder:
 ```
 $ git clone https://github.com/ros2/rosbag2.git
 ```
-**[Note]**: if you are only building rosbag2 on top of a Debian Installation of ROS2, please git clone the branch following your current ROS2 distribution. 
+**[Note]**: if you are only building rosbag2 on top of a Debian Installation of ROS2, please git clone the branch following your current ROS2 distribution.
 
 Then build all the packages with this command:
 
@@ -93,6 +93,55 @@ In the same fashion, this auto discovery can be disabled with `--no-discovery`.
 If not further specified, `ros2 bag record` will create a new folder named to the current time stamp and stores all data within this folder.
 A user defined name can be given with `-o, --output`.
 
+#### Splitting recorded bag files
+
+rosbag2 offers the capability to split bag files when they reach a maximum size or after a specified duration. By default rosbag2 will record all data into a single bag file, but this can be changed using the CLI options.
+
+_Splitting by size_: `ros2 bag record -a -b 100000` will split the bag files when they become greater than 100 kilobytes. Note: the batch size's units are in bytes and must be greater than `86016`. This option defaults to `0`, which means data is written to a single file.
+
+_Splitting by time_: `ros2 bag record -a -d 9000` will split the bag files after a duration of `9000` seconds. This option defaults to `0`, which means data is written to a single file.
+
+If both splitting by size and duration are enabled, the bag will split at whichever threshold is reached first.
+
+#### Recording with compression
+
+By default rosbag2 does not record with compression enabled. However, compression can be specified using the following CLI options.
+
+For example, `ros2 bag record -a --compression-mode file --compression-format zstd` will record all topics and compress each file using the [zstd](https://github.com/facebook/zstd) compressor.
+
+Currently, the only `compression-format` available is `zstd`. Both the mode and format options default to `none`. To use a compression format, a compression mode must be specified, where the currently supported modes are compress by `file` or compress by `message`.
+
+It is recommended to use this feature with the splitting options.
+
+#### Recording with a storage configuration
+
+Storage configuration can be specified in a YAML file passed through the `--storage-config-file` option.
+This can be used to optimize performance for specific use-cases.
+
+For the default storage plugin (sqlite3), the file has a following syntax:
+```
+read:
+  pragmas: <list of pragma settings for read-only>
+write:
+  pragmas: <list of pragma settings for read/write>
+```
+
+By default, SQLite settings are significantly optimized for performance.
+This might have consequences of bag data being corrupted after an application or system-level crash.
+This consideration only applies to current bagfile in case bag splitting is on (through `--max-bag-*` parameters).
+If increased crash-caused corruption resistance is necessary, use `resilient` option for `--storage-preset-profile` setting.
+
+Settings are fully exposed to the user and should be applied with understanding.
+Please refer to [documentation of pragmas](https://www.sqlite.org/pragma.html).
+
+An example configuration file could look like this:
+
+```
+write:
+  pragmas: ["journal_mode = MEMORY", "synchronous = OFF", "schema.cache_size = 1000", "schema.page_size = 4096"]
+
+```
+
 ### Replaying data
 
 After recording data, the next logical step is to replay this data:
@@ -125,6 +174,45 @@ Topic information: Topic: /chatter | Type: std_msgs/String | Count: 9 | Serializ
                    Topic: /my_chatter | Type: std_msgs/String | Count: 18 | Serialization Format: cdr
 ```
 
+### Overriding QoS Profiles
+
+When starting a recording or playback workflow, you can pass a YAML file that contains QoS profile settings for a specific topic.
+The YAML schema for the profile overrides is a dictionary of topic names with key/value pairs for each QoS policy.
+Below is an example profile set to the default ROS2 QoS settings.
+
+```yaml
+/topic_name:
+  history: keep_last
+  depth: 10
+  reliability: reliable
+  durability: volatile
+  deadline:
+    # unspecified/infinity
+    sec: 0
+    nsec: 0
+  lifespan:
+    # unspecified/infinity
+    sec: 0
+    nsec: 0
+  liveliness: system_default
+  liveliness_lease_duration:
+    # unspecified/infinity
+    sec: 0
+    nsec: 0
+  avoid_ros_namespace_conventions: false
+```
+
+You can then use the override by specifying the `--qos-profile-overrides-path` argument in the CLI:
+
+```sh
+# Record
+ros2 bag record --qos-profile-overrides-path override.yaml -a -o my_bag
+# Playback
+ros2 bag play --qos-profile-overrides-path override.yaml my_bag
+```
+
+See [the official QoS override tutorial][qos-override-tutorial] and ["About QoS Settings"][about-qos-settings] for more detail.
+
 ### Using in launch
 
 We can invoke the command line tool from a ROS launch script as an *executable* (not a *node* action).
@@ -189,3 +277,6 @@ When specified, rosbag2 looks for a suitable converter to transform the native m
 This also allows to record data in a native format to optimize for speed, but to convert or transform the recorded data into a middleware agnostic serialization format.
 
 By default, rosbag2 can convert from and to CDR as it's the default serialization format for ROS 2.
+
+[qos-override-tutorial]: https://index.ros.org/doc/ros2/Tutorials/Ros2bag/Overriding-QoS-Policies-For-Recording-And-Playback/
+[about-qos-settings]: https://index.ros.org/doc/ros2/Concepts/About-Quality-of-Service-Settings/

ros2bag/CHANGELOG.rst

@@ -2,8 +2,90 @@
 Changelog for package ros2bag
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-0.3.7 (2021-02-15)
-------------------
+0.6.0 (2021-02-01)
+------------------
+* Recorder --regex and --exclude options (`#604 <https://github.com/ros2/rosbag2/issues/604>`_)
+* Fix the tests on cyclonedds by translating qos duration values (`#606 <https://github.com/ros2/rosbag2/issues/606>`_)
+* SQLite storage optimized by default (`#568 <https://github.com/ros2/rosbag2/issues/568>`_)
+* Fix a bug on parsing wrong description in plugin xml file (`#578 <https://github.com/ros2/rosbag2/issues/578>`_)
+* Compress bag files in separate threads (`#506 <https://github.com/ros2/rosbag2/issues/506>`_)
+* Contributors: Adam Dąbrowski, Barry Xu, Emerson Knapp, P. J. Reed
+
+0.5.0 (2020-12-02)
+------------------
+* Sqlite storage double buffering (`#546 <https://github.com/ros2/rosbag2/issues/546>`_)
+  * Double buffers
+  * Circular queue and FLUSH option as define
+  * Minor naming and lexical fixes.
+  * Removed FLUSH_BUFFERS define check.
+  * Sqlite3 storage logging fixes.
+  * Sqlite3 storage circular buffer with pre allocated memory.
+  * Sqlite3 storage buffers moved to shared_ptrs.
+  * Uncrustify
+  * Moved double buffers to writer
+  * Buffer layer reset in seq compression writer in rosbag2 cpp
+  * Buffer layer for rosbag2 writer refactor
+  * Changed buffers in BufferLayer to std vectors.
+  * BufferLayer uncrustify
+  * Removed non-applicable test for writer cache.
+  * BufferLayer review fixes
+  * Rosbag metadata msgs count fixed for BufferLayer
+  * Condition variable for buffer layer sync.
+  * Fixed buffer locks
+  * Buffers in BufferLayer refactored, moved into new class
+  * Buffer layer split bags fixed.
+  * Storage options include fix in buffer layer header.
+  * Mutex around swapping buffers in buffer layer.
+  * Fixed cache 0 bug in buffer layer.
+  * Minor buffer layer refactor.
+  * Counting messages in writer refactored.
+  * Changed default cache size to 100Mb and updated parameter description
+  * Applied review remarks:
+  - significant refactoring: separation of cache classes
+  - applied suggested improvements
+  - some renaming
+  - reduce code duplication that would otherwise increase with cache refactor, between compression and plain writers
+  * Applied review comments
+  - cache consumer now takes a callback and is independent of storage
+  - namespace changes, renaming, cleaning
+  - counting and logging messages by topic
+  * linter
+  * Changes after review: fixing flushing, topic counts, and more
+  * Fix for splitting - flushing state now correctly turns off
+  * cache classes documentation
+  * simplified signature
+  * a couple of tests for cache
+  * address review: explicit constructor and doxygen styling
+  * Windows warnings fix
+  * fixed type mismatch warning on Windows
+  * added minor comment
+  Co-authored-by: Piotr Jaroszek <[email protected]>
+* Contributors: Adam Dąbrowski
+
+0.4.0 (2020-11-19)
+------------------
+* read yaml config file (`#497 <https://github.com/ros2/rosbag2/issues/497>`_)
+* List all storage plugins in plugin xml file (`#554 <https://github.com/ros2/rosbag2/issues/554>`_)
+* add storage_config_uri (`#493 <https://github.com/ros2/rosbag2/issues/493>`_)
+* Update deprecated qos policy value names (`#548 <https://github.com/ros2/rosbag2/issues/548>`_)
+* Add record test for ros2bag (`#523 <https://github.com/ros2/rosbag2/issues/523>`_)
+* Removed duplicated code in record (`#534 <https://github.com/ros2/rosbag2/issues/534>`_)
+* Change default cache size for sequential_writer to a non zero value (`#533 <https://github.com/ros2/rosbag2/issues/533>`_)
+* Update the package.xml files with the latest Open Robotics maintainers (`#535 <https://github.com/ros2/rosbag2/issues/535>`_)
+* [ros2bag test_record] Gets rid of time.sleep and move to using command.wait_for_output (`#525 <https://github.com/ros2/rosbag2/issues/525>`_)
+* Add pytest.ini back to ros2bag. (`#492 <https://github.com/ros2/rosbag2/issues/492>`_)
+* performance testing packages (`#442 <https://github.com/ros2/rosbag2/issues/442>`_)
+* Validate QoS profile values are not negative. (`#483 <https://github.com/ros2/rosbag2/issues/483>`_)
+* catch parent exception (`#472 <https://github.com/ros2/rosbag2/issues/472>`_)
+* add wait for closed file handles on Windows (`#470 <https://github.com/ros2/rosbag2/issues/470>`_)
+* introduce ros2 bag list <plugins> (`#468 <https://github.com/ros2/rosbag2/issues/468>`_)
+* move wait_for_shutdown() call out of the context manager (`#466 <https://github.com/ros2/rosbag2/issues/466>`_)
+* Adding db directory creation to rosbag2_cpp (`#450 <https://github.com/ros2/rosbag2/issues/450>`_)
+* use a single temp dir for the test class (`#462 <https://github.com/ros2/rosbag2/issues/462>`_)
+* Add per-message ZSTD compression (`#418 <https://github.com/ros2/rosbag2/issues/418>`_)
+* Add split by time to recording (`#409 <https://github.com/ros2/rosbag2/issues/409>`_)
+* Add pytest.ini so local tests don't display warning (`#446 <https://github.com/ros2/rosbag2/issues/446>`_)
+* Contributors: Adam Dąbrowski, Barry Xu, Chris Lalancette, Dirk Thomas, Ivan Santiago Paunovic, Jacob Perron, Jaison Titus, Jesse Ikawa, Karsten Knese, Marwan Taher, Michael Jeronimo, P. J. Reed, jhdcs
 
 0.3.6 (2021-01-05)
 ------------------

ros2bag/package.xml

@@ -2,7 +2,7 @@
 <?xml-model href="http://download.ros.org/schema/package_format2.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
 <package format="2">
   <name>ros2bag</name>
-  <version>0.3.7</version>
+  <version>0.6.0</version>
   <description>
     Entry point for rosbag in ROS 2
   </description>