Introducing Reindexer CLI (#699)

* CLI interface to the rosbag2_cpp Reindexer `ros2 bag reindex`
Distro A, OPSEC #4584

Signed-off-by: Jacob Hassold <[email protected]>

ros2bag/ros2bag/verb/reindex.py

@@ -0,0 +1,53 @@
+# Copyright 2021 DCS Corporation, All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# DISTRIBUTION A. Approved for public release; distribution unlimited.
+# OPSEC #4584.
+#
+# Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS
+# Part 252.227-7013 or 7014 (Feb 2014).
+#
+# This notice must appear in all copies of this file and its derivatives.
+
+import os
+
+from ros2bag.api import check_path_exists
+from ros2bag.api import print_error
+from ros2bag.verb import VerbExtension
+from rosbag2_py import get_registered_readers, Reindexer, StorageOptions
+
+
+class ReindexVerb(VerbExtension):
+    """Reconstruct metadata file for a bag."""
+
+    def add_arguments(self, parser, cli_name):
+        storage_choices = get_registered_readers()
+        default_storage = 'sqlite3' if 'sqlite3' in storage_choices else storage_choices[0]
+        parser.add_argument(
+            'bag_directory', type=check_path_exists, help='bag to reindex')
+        parser.add_argument(
+            'storage_id', default=default_storage, choices=storage_choices,
+            help=f"storage identifier to be used, defaults to '{default_storage}'")
+
+    def main(self, *, args):
+        if not os.path.isdir(args.bag_directory):
+            return print_error('Must specify a bag directory')
+
+        storage_options = StorageOptions(
+            uri=args.bag_directory,
+            storage_id=args.storage_id,
+        )
+
+        reindexer = Reindexer()
+        reindexer.reindex(storage_options)

ros2bag/setup.py

@@ -42,6 +42,7 @@
             'list = ros2bag.verb.list:ListVerb',
             'play = ros2bag.verb.play:PlayVerb',
             'record = ros2bag.verb.record:RecordVerb',
+            'reindex = ros2bag.verb.reindex:ReindexVerb'
         ],
     }
 )

rosbag2_py/CMakeLists.txt

@@ -118,6 +118,15 @@ ament_target_dependencies(_transport PUBLIC
 )
 clean_windows_flags(_transport)
 
+pybind11_add_module(_reindexer SHARED
+  src/rosbag2_py/_reindexer.cpp
+)
+ament_target_dependencies(_reindexer PUBLIC
+  "rosbag2_cpp"
+  "rosbag2_storage"
+)
+clean_windows_flags(_reindexer)
+
 # Install cython modules as sub-modules of the project
 install(
   TARGETS
@@ -126,6 +135,7 @@ install(
     _writer
     _info
     _transport
+    _reindexer
   DESTINATION "${PYTHON_INSTALL_DIR}/${PROJECT_NAME}"
 )
 
@@ -160,6 +170,11 @@ if(BUILD_TESTING)
     PYTHON_EXECUTABLE "${_PYTHON_EXECUTABLE}"
     APPEND_ENV "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}" ${other_environment_vars}
   )
+  ament_add_pytest_test(test_reindexer_py
+    "test/test_reindexer.py"
+    PYTHON_EXECUTABLE "${_PYTHON_EXECUTABLE}"
+    APPEND_ENV "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}" ${other_environment_vars}
+  )
 endif()
 
 ament_package()

rosbag2_py/rosbag2_py/__init__.py

@@ -45,11 +45,15 @@
         Recorder,
         RecordOptions,
     )
+    from rosbag2_py._reindexer import (
+        Reindexer
+    )
 
 __all__ = [
     'ConverterOptions',
     'get_registered_readers',
     'get_registered_writers',
+    'Reindexer',
     'SequentialCompressionReader',
     'SequentialCompressionWriter',
     'SequentialReader',

rosbag2_py/src/rosbag2_py/_reindexer.cpp

@@ -0,0 +1,60 @@
+// Copyright 2021 DCS Corporation, All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// DISTRIBUTION A. Approved for public release; distribution unlimited.
+// OPSEC #4584.
+//
+// Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS
+// Part 252.227-7013 or 7014 (Feb 2014).
+//
+// This notice must appear in all copies of this file and its derivatives.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "rosbag2_cpp/reindexer.hpp"
+#include "rosbag2_storage/storage_options.hpp"
+
+#include "./pybind11.hpp"
+
+namespace rosbag2_py
+{
+
+class Reindexer
+{
+public:
+  Reindexer()
+  : reindexer_(std::make_unique<rosbag2_cpp::Reindexer>())
+  {
+  }
+
+  void reindex(const rosbag2_storage::StorageOptions & storage_options)
+  {
+    reindexer_->reindex(storage_options);
+  }
+
+protected:
+  std::unique_ptr<rosbag2_cpp::Reindexer> reindexer_;
+};
+}  // namespace rosbag2_py
+
+PYBIND11_MODULE(_reindexer, m) {
+  m.doc() = "Python wrapper of the rosbag2_cpp reindexer API";
+
+  pybind11::class_<rosbag2_py::Reindexer>(
+    m, "Reindexer")
+  .def(pybind11::init())
+  .def("reindex", &rosbag2_py::Reindexer::reindex);
+}

rosbag2_py/test/test_reindexer.py

@@ -0,0 +1,48 @@
+# Copyright 2021 DCS Corporation, All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# DISTRIBUTION A. Approved for public release; distribution unlimited.
+# OPSEC #4584.
+#
+# Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS
+# Part 252.227-7013 or 7014 (Feb 2014).
+#
+# This notice must appear in all copies of this file and its derivatives.
+
+import os
+from pathlib import Path
+import sys
+
+if os.environ.get('ROSBAG2_PY_TEST_WITH_RTLD_GLOBAL', None) is not None:
+    # This is needed on Linux when compiling with clang/libc++.
+    # TL;DR This makes class_loader work when using a python extension compiled with libc++.
+    #
+    # For the fun RTTI ABI details, see https://whatofhow.wordpress.com/2015/03/17/odr-rtti-dso/.
+    sys.setdlopenflags(os.RTLD_GLOBAL | os.RTLD_LAZY)
+
+from common import get_rosbag_options  # noqa
+import rosbag2_py  # noqa
+
+
+def test_reindexer_multiple_files():
+    bag_path = Path(__file__).parent.parent / 'resources' / 'reindex_test_bags' / 'multiple_files'
+    result_path = bag_path / 'metadata.yaml'
+
+    storage_options, converter_options = get_rosbag_options(str(bag_path))
+    reindexer = rosbag2_py.Reindexer()
+    reindexer.reindex(storage_options)
+
+    assert(result_path.exists())
+
+    result_path.unlink(missing_ok=True)