Create a standalone script for performing pulp repo syncs

ros_buildfarm/pulp.py

@@ -130,7 +130,7 @@ def enumerate_pkgs_in_repo_ver(self, repo_ver_href):
 
     def import_and_invalidate(
             self, distribution_name, packages_to_add,
-            invalidate_expression, invalidate_downstream):
+            invalidate_expression, invalidate_downstream, package_cache={}, dry_run=False):
         distribution = self._rpm_distributions_api.list(
             name=distribution_name).results[0]
         old_publication = self._rpm_publications_api.read(distribution.publication)
@@ -143,7 +143,8 @@ def import_and_invalidate(
         # Get the packages we're adding
         new_pkgs = {
             pkg.pulp_href: pkg for pkg in
-            [current_pkgs.get(pkg_href) or self._rpm_packages_api.read(pkg_href)
+            [package_cache.get(pkg_href) or current_pkgs.get(pkg_href) or
+                self._rpm_packages_api.read(pkg_href)
                 for pkg_href in packages_to_add]}
 
         # Invalidate packages
@@ -174,6 +175,9 @@ def import_and_invalidate(
         for href_in_current in current_pkgs.keys():
             new_pkgs.pop(href_in_current, None)
 
+        if dry_run:
+            return (new_pkgs.values(), pkgs_to_remove.values())
+
         # Commit the changes
         mod_data = pulp_rpm.RepositoryAddRemoveContent(
             add_content_units=list(new_pkgs.keys()),

ros_buildfarm/templates/release/rpm/import_package_job.xml.em

@@ -33,7 +33,12 @@
         {
             'type': 'string',
             'name': 'INVALIDATE_EXPRESSION',
-        }
+        },
+        {
+            'type': 'boolean',
+            'name': 'DRY_RUN',
+            'description': 'Skip the commit operation but show what would change',
+        },
     ],
 ))@
   </properties>
@@ -58,14 +63,16 @@
     script='\n'.join([
         'echo "# BEGIN SECTION: import RPM package"',
         'if [ "$INVALIDATE_DOWNSTREAM" = "true" ]; then INVALIDATE_ARG=--invalidate; fi',
+        'if [ "$DRY_RUN" = "true" ]; then DRY_RUN_ARG="--dry-run"; fi',
         'if [ "$INVALIDATE_EXPRESSION" != "" ]; then INVALIDATE_EXPRESSION_ARG="--invalidate-expression $INVALIDATE_EXPRESSION"; fi',
         'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/import_package.py' +
         ' --pulp-base-url http://repo:24817' +
         ' --pulp-distribution-name $DISTRIBUTION_NAME' +
         ' $PULP_RESOURCES' +
         ' $INVALIDATE_ARG' +
-        ' $INVALIDATE_EXPRESSION_ARG',
+        ' $INVALIDATE_EXPRESSION_ARG' +
+        ' $DRY_RUN_ARG',
         'echo "# END SECTION"',
     ]),
 ))@

ros_buildfarm/templates/release/rpm/sync_packages_to_testing_job.xml.em

@@ -96,62 +96,13 @@
 @(SNIPPET(
     'builder_shell',
     script='\n'.join([
-        'echo "# BEGIN SECTION: determine source packages to sync"',
+        'echo "# BEGIN SECTION: sync packages to testing repos"',
         'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
-        'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/list_packages.py' +
+        'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/sync_repo.py' +
         ' --pulp-base-url http://repo:24817' +
-        ' --pulp-distribution-name ros-building-%s-%s-SRPMS' % (os_name, os_code_name) +
-        ' --package-name-expression "^ros-%s-.*"' % rosdistro_name +
-        ' --pulp-resource-record $WORKSPACE/ros-building-source.txt',
-        'echo "# END SECTION"',
-        '',
-        'echo "# BEGIN SECTION: determine binary packages to sync"',
-        'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
-        'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/list_packages.py' +
-        ' --pulp-base-url http://repo:24817' +
-        ' --pulp-distribution-name ros-building-%s-%s-%s' % (os_name, os_code_name, arch) +
-        ' --package-name-expression "^ros-%s-.*"' % rosdistro_name +
-        ' --pulp-resource-record $WORKSPACE/ros-building-binary.txt',
-        'echo "# END SECTION"',
-        '',
-        'echo "# BEGIN SECTION: determine binary debug packages to sync"',
-        'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
-        'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/list_packages.py' +
-        ' --pulp-base-url http://repo:24817' +
-        ' --pulp-distribution-name ros-building-%s-%s-%s-debug' % (os_name, os_code_name, arch) +
-        ' --package-name-expression "^ros-%s-.*"' % rosdistro_name +
-        ' --pulp-resource-record $WORKSPACE/ros-building-binary-debug.txt',
-        'echo "# END SECTION"',
-    ]),
-))@
-@(SNIPPET(
-    'builder_shell',
-    script='\n'.join([
-        'echo "# BEGIN SECTION: sync source packages to testing repo"',
-        'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
-        'sed "s/^PULP_RESOURCES=//" $WORKSPACE/ros-building-source.txt | xargs' +
-        ' python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/import_package.py' +
-        ' --pulp-base-url http://repo:24817' +
-        ' --pulp-distribution-name ros-testing-%s-%s-SRPMS' % (os_name, os_code_name) +
-        ' --invalidate-expression "^ros-%s-.*"' % rosdistro_name,
-        'echo "# END SECTION"',
-        '',
-        'echo "# BEGIN SECTION: sync binary packages to testing repo"',
-        'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
-        'sed "s/^PULP_RESOURCES=//" $WORKSPACE/ros-building-binary.txt | xargs' +
-        ' python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/import_package.py' +
-        ' --pulp-base-url http://repo:24817' +
-        ' --pulp-distribution-name ros-testing-%s-%s-%s' % (os_name, os_code_name, arch) +
-        ' --invalidate-expression "^ros-%s-.*"' % rosdistro_name,
-        'echo "# END SECTION"',
-        '',
-        'echo "# BEGIN SECTION: sync binary debug packages to testing repo"',
-        'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
-        'sed "s/^PULP_RESOURCES=//" $WORKSPACE/ros-building-binary-debug.txt | xargs' +
-        ' python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/import_package.py' +
-        ' --pulp-base-url http://repo:24817' +
-        ' --pulp-distribution-name ros-testing-%s-%s-%s-debug' % (os_name, os_code_name, arch) +
-        ' --invalidate-expression "^ros-%s-.*"' % rosdistro_name,
+        ' --distribution-source-expression "^ros-building-%s-%s-(SRPMS|%s(-debug)?)$"' % (os_name, os_code_name, arch) +
+        ' --distribution-dest-expression "ros-testing-%s-%s-\\1"' % (os_name, os_code_name) +
+        ' --package-name-expression "^ros-%s-.*"' % rosdistro_name,
         'echo "# END SECTION"',
     ]),
 ))@

scripts/release/rpm/import_package.py

@@ -17,6 +17,7 @@
 import argparse
 import sys
 
+from ros_buildfarm.argument import add_argument_dry_run
 from ros_buildfarm.argument import add_argument_invalidate
 from ros_buildfarm.argument import add_argument_pulp_base_url
 from ros_buildfarm.argument import add_argument_pulp_distribution_name
@@ -35,6 +36,7 @@ def main(argv=sys.argv[1:]):
         'package_resources',
         nargs='*', metavar="PULP_HREF",
         help='Identifiers for packages which should be imported')
+    add_argument_dry_run(parser)
     add_argument_invalidate(parser)
     parser.add_argument(
         '--invalidate-expression',
@@ -54,11 +56,11 @@ def main(argv=sys.argv[1:]):
     with Scope('SUBSECTION', 'performing repository transaction'):
         pkgs_added, pkgs_removed = pulp_client.import_and_invalidate(
             args.pulp_distribution_name, args.package_resources,
-            args.invalidate_expression, args.invalidate)
+            args.invalidate_expression, args.invalidate, dry_run=args.dry_run)
 
     with Scope('SUBSECTION', 'enumerating results'):
         if not pkgs_added:
-            print('Not importing any new packages)')
+            print('Not importing any new packages')
         for pkg in pkgs_added:
             print('Importing package: %s-%s.%s' % (pkg.name, format_pkg_ver(pkg), pkg.arch))
 

scripts/release/rpm/sync_repo.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+# Copyright 2020 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from collections import OrderedDict
+import re
+import sys
+
+from ros_buildfarm.argument import add_argument_dry_run
+from ros_buildfarm.argument import add_argument_pulp_base_url
+from ros_buildfarm.argument import add_argument_pulp_password
+from ros_buildfarm.argument import add_argument_pulp_task_timeout
+from ros_buildfarm.argument import add_argument_pulp_username
+from ros_buildfarm.common import Scope
+from ros_buildfarm.pulp import format_pkg_ver
+from ros_buildfarm.pulp import PulpRpmClient
+
+
+def main(argv=sys.argv[1:]):
+    parser = argparse.ArgumentParser(
+        description='Sync packages between pulp distributions')
+    add_argument_dry_run(parser)
+    add_argument_pulp_base_url(parser)
+    add_argument_pulp_password(parser)
+    add_argument_pulp_task_timeout(parser)
+    add_argument_pulp_username(parser)
+    parser.add_argument(
+        '--package-name-expression', required=True,
+        help='Expression to match against packages in the repositories')
+    parser.add_argument(
+        '--distribution-source-expression', required=True,
+        help='Expression to match for source distribution names')
+    parser.add_argument(
+        '--distribution-dest-expression', required=True,
+        help='Expression to transform matching source distribution names into destination names')
+    args = parser.parse_args(argv)
+
+    pulp_client = PulpRpmClient(
+        args.pulp_base_url, args.pulp_username, args.pulp_password,
+        task_timeout=args.pulp_task_timeout)
+
+    dists_to_sync = OrderedDict()
+    with Scope('SUBSECTION', 'enumerating distributions to sync'):
+        dist_expression = re.compile(args.distribution_source_expression)
+        distributions = {dist.name for dist in pulp_client.enumerate_distributions()}
+        for dist_source in sorted(distributions):
+            (dist_dest, matched) = dist_expression.subn(
+                args.distribution_dest_expression, dist_source)
+            if matched:
+                dists_to_sync[dist_source] = dist_dest
+
+        print('Syncing %d distributions:' % len(dists_to_sync))
+        for dist_source_dest in dists_to_sync.items():
+            print('- %s => %s' % dist_source_dest)
+
+    packages = {}
+    with Scope('SUBSECTION', 'enumerating packages to sync'):
+        package_expression = re.compile(args.package_name_expression)
+        for dist_source in dists_to_sync.keys():
+            packages[dist_source] = {
+                pkg.pulp_href: pkg
+                for pkg in pulp_client.enumerate_pkgs_in_distribution_name(dist_source)
+                if package_expression.match(pkg.name)}
+
+        print('Matched %d packages from source distributions:' % (
+            sum([len(pkgs) for pkgs in packages.values()])))
+        for dist_source in dists_to_sync.keys():
+            print('- %s: %d matching packages' % (dist_source, len(packages[dist_source])))
+
+    with Scope('SUBSECTION', 'invalidation and committing changes'):
+        for dist_source, dist_dest in dists_to_sync.items():
+            packages_to_sync = packages[dist_source]
+            if not packages_to_sync:
+                print('Skipping sync from %s to %s' % (dist_source, dist_dest))
+                continue
+            print('Syncing %d packages from %s to %s...%s' % (
+                len(packages_to_sync), dist_source, dist_dest,
+                ' (dry run)' if args.dry_run else ''))
+            new_pkgs, pkgs_removed = pulp_client.import_and_invalidate(
+                dist_dest, packages_to_sync, args.package_name_expression,
+                False, package_cache=packages_to_sync, dry_run=args.dry_run)
+            print('- Added %d packages%s' % (
+                len(new_pkgs), ' (dry run)' if args.dry_run else ''))
+            for pkg in sorted(new_pkgs, key=lambda pkg: pkg.name):
+                print('  - %s-%s' % (pkg.name, format_pkg_ver(pkg)))
+            print('- Removed %d packages%s' % (
+                len(pkgs_removed), ' (dry run)' if args.dry_run else ''))
+            for pkg in sorted(pkgs_removed, key=lambda pkg: pkg.name):
+                print('  - %s-%s' % (pkg.name, format_pkg_ver(pkg)))
+
+
+if __name__ == '__main__':
+    sys.exit(main())