Merge pull request #1903 from ceph/wip-package-queries

suite: Improve package query caching

teuthology/orchestra/opsys.py

@@ -20,6 +20,9 @@
         "7": "wheezy",
         "8": "jessie",
         "9": "stretch",
+        "10": "buster",
+        "11": "bullseye",
+        "12": "bookworm",
     },
     "rhel": {
         "9": "plow",

teuthology/repo_utils.py

@@ -1,3 +1,4 @@
+import functools
 import logging
 import os
 import re
@@ -56,6 +57,7 @@ def build_git_url(project, project_owner='ceph'):
     return url_templ.format(project_owner=project_owner, project=project)
 
 
+@functools.lru_cache()
 def ls_remote(url, ref):
     """
     Return the current sha1 for a given repository and ref
@@ -119,8 +121,6 @@ def enforce_repo_state(repo_url, dest_path, branch, commit=None, remove_on_error
             set_remote(dest_path, repo_url)
             fetch_branch(dest_path, branch)
             touch_file(sentinel)
-        else:
-            log.info("%s was just updated or references a specific commit; assuming it is current", dest_path)
 
         if commit and os.path.exists(repo_reset):
             return
@@ -275,7 +275,7 @@ def fetch_branch(repo_path, branch, shallow=True):
                       GitError for other errors
     """
     validate_branch(branch)
-    log.info("Fetching %s from origin", branch)
+    log.info("Fetching %s from origin", repo_path.split("/")[-1])
     args = ['git', 'fetch']
     if shallow:
         args.extend(['--depth', '1'])
@@ -312,7 +312,7 @@ def reset_repo(repo_url, dest_path, branch, commit=None):
     else:
         reset_branch = 'origin/%s' % branch
     reset_ref = commit or reset_branch
-    log.info('Resetting repo at %s to %s', dest_path, reset_ref)
+    log.debug('Resetting repo at %s to %s', dest_path, reset_ref)
     # This try/except block will notice if the requested branch doesn't
     # exist, whether it was cloned or fetched.
     try:

teuthology/suite/run.py

@@ -15,7 +15,6 @@
 from teuthology.config import config, JobConfig
 from teuthology.exceptions import (
     BranchMismatchError, BranchNotFoundError, CommitNotFoundError,
-    VersionNotFoundError
 )
 from teuthology.misc import deep_merge, get_results_url
 from teuthology.orchestra.opsys import OS
@@ -34,8 +33,7 @@ class Run(object):
     WAIT_PAUSE = 5 * 60
     __slots__ = (
         'args', 'name', 'base_config', 'suite_repo_path', 'base_yaml_paths',
-        'base_args', 'package_versions', 'kernel_dict', 'config_input',
-        'timestamp', 'user',
+        'base_args', 'kernel_dict', 'config_input', 'timestamp', 'user', 'os',
     )
 
     def __init__(self, args):
@@ -56,8 +54,6 @@ def __init__(self, args):
             config.ceph_qa_suite_git_url = self.args.suite_repo
 
         self.base_config = self.create_initial_config()
-        # caches package versions to minimize requests to gbs
-        self.package_versions = dict()
 
         # Interpret any relative paths as being relative to ceph-qa-suite
         # (absolute paths are unchanged by this)
@@ -90,6 +86,7 @@ def create_initial_config(self):
 
         :returns: A JobConfig object
         """
+        self.os = self.choose_os()
         self.kernel_dict = self.choose_kernel()
         ceph_hash = self.choose_ceph_hash()
         # We don't store ceph_version because we don't use it yet outside of
@@ -118,8 +115,8 @@ def create_initial_config(self):
             teuthology_branch=teuthology_branch,
             teuthology_sha1=teuthology_sha1,
             machine_type=self.args.machine_type,
-            distro=self.args.distro,
-            distro_version=self.args.distro_version,
+            distro=self.os.name,
+            distro_version=self.os.version,
             archive_upload=config.archive_upload,
             archive_upload_key=config.archive_upload_key,
             suite_repo=config.get_ceph_qa_suite_git_url(),
@@ -128,6 +125,16 @@ def create_initial_config(self):
         )
         return self.build_base_config()
 
+    def choose_os(self):
+        os_type = self.args.distro
+        os_version = self.args.distro_version
+        if not (os_type and os_version):
+            os_ = util.get_distro_defaults(
+                self.args.distro, self.args.machine_type)[2]
+        else:
+            os_ = OS(os_type, os_version)
+        return os_
+
     def choose_kernel(self):
         # Put together a stanza specifying the kernel hash
         if self.args.kernel_branch == 'distro':
@@ -165,6 +172,7 @@ def choose_ceph_hash(self):
         """
         repo_name = self.ceph_repo_name
 
+        ceph_hash = None
         if self.args.ceph_sha1:
             ceph_hash = self.args.ceph_sha1
             if self.args.validate_sha1:
@@ -194,13 +202,14 @@ def choose_ceph_version(self, ceph_hash):
         if config.suite_verify_ceph_hash and not self.args.newest:
             # don't bother if newest; we'll search for an older one
             # Get the ceph package version
-            try:
-                ceph_version = util.package_version_for_hash(
-                    ceph_hash, self.args.flavor, self.args.distro,
-                    self.args.distro_version, self.args.machine_type,
-                )
-            except Exception as exc:
-                util.schedule_fail(str(exc), self.name, dry_run=self.args.dry_run)
+            ceph_version = util.package_version_for_hash(
+                ceph_hash, self.args.flavor, self.os.name,
+                self.os.version, self.args.machine_type,
+            )
+            if not ceph_version:
+                msg = f"Packages for os_type '{self.os.name}', flavor " \
+                    f"{self.args.flavor} and ceph hash '{ceph_hash}' not found"
+                util.schedule_fail(msg, self.name, dry_run=self.args.dry_run)
             log.info("ceph version: {ver}".format(ver=ceph_version))
             return ceph_version
         else:
@@ -476,31 +485,16 @@ def collect_jobs(self, arch, configs, newest=False, limit=0):
                 full_job_config = copy.deepcopy(self.base_config.to_dict())
                 deep_merge(full_job_config, parsed_yaml)
                 flavor = util.get_install_task_flavor(full_job_config)
-                # Get package versions for this sha1, os_type and flavor. If
-                # we've already retrieved them in a previous loop, they'll be
-                # present in package_versions and gitbuilder will not be asked
-                # again for them.
-                try:
-                    self.package_versions = util.get_package_versions(
-                        sha1,
-                        os_type,
-                        os_version,
-                        flavor,
-                        self.package_versions
-                    )
-                except VersionNotFoundError:
-                    pass
-                if not util.has_packages_for_distro(
-                    sha1, os_type, os_version, flavor, self.package_versions
-                ):
-                    m = "Packages for os_type '{os}', flavor {flavor} and " + \
-                        "ceph hash '{ver}' not found"
-                    log.error(m.format(os=os_type, flavor=flavor, ver=sha1))
+                version = util.package_version_for_hash(sha1, flavor, os_type,
+                    os_version, self.args.machine_type)
+                if not version:
                     jobs_missing_packages.append(job)
+                    log.error(f"Packages for os_type '{os_type}', flavor {flavor} and "
+                         f"ceph hash '{sha1}' not found")
                     # optimization: one missing package causes backtrack in newest mode;
                     # no point in continuing the search
                     if newest:
-                        return jobs_missing_packages, None
+                        return jobs_missing_packages, []
 
             jobs_to_schedule.append(job)
         return jobs_missing_packages, jobs_to_schedule
@@ -648,6 +642,8 @@ def schedule_suite(self):
         backtrack = 0
         limit = self.args.newest
         sha1s = []
+        jobs_to_schedule = []
+        jobs_missing_packages = []
         while backtrack <= limit:
             jobs_missing_packages, jobs_to_schedule = \
                 self.collect_jobs(arch, configs, self.args.newest, job_limit)
@@ -672,9 +668,6 @@ def schedule_suite(self):
                     dry_run=self.args.dry_run,
                 )
 
-        if self.args.dry_run:
-            log.debug("Base job config:\n%s" % self.base_config)
-
         with open(base_yaml_path, 'w+b') as base_yaml:
             base_yaml.write(str(self.base_config).encode())