Fedora CI scratch builds (#2650)

Fedora CI scratch builds

Related to #2602
RELEASE NOTES BEGIN
N/A
RELEASE NOTES END

Reviewed-by: Maja Massarini
Reviewed-by: Laura Barcziová
Reviewed-by: Matej Focko
Reviewed-by: Nikola Forró

packit_service/config.py

@@ -113,6 +113,7 @@ def __init__(
         package_config_path_override: Optional[str] = None,
         command_handler_storage_class: Optional[str] = None,
         appcode: Optional[str] = None,
+        enabled_projects_for_fedora_ci: Optional[Union[set[str], list[str]]] = None,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -157,6 +158,10 @@ def __init__(
             enabled_projects_for_internal_tf or [],
         )
 
+        # e.g.:
+        #  - https://src.fedoraproject.org/rpms/packit
+        self.enabled_projects_for_fedora_ci: set[str] = set(enabled_projects_for_fedora_ci or [])
+
         self.projects_to_sync = projects_to_sync or []
 
         # Full URL to the dashboard, e.g. https://dashboard.packit.dev
@@ -211,7 +216,8 @@ def hide(token: str) -> str:
             f"enabled_projects_for_srpm_in_copr= '{self.enabled_projects_for_srpm_in_copr}', "
             f"comment_command_prefix='{self.comment_command_prefix}', "
             f"redhat_api_refresh_token='{hide(self.redhat_api_refresh_token)}', "
-            f"package_config_path_override='{self.package_config_path_override}')"
+            f"package_config_path_override='{self.package_config_path_override}', "
+            f"enabled_projects_for_fedora_ci='{self.enabled_projects_for_fedora_ci}')"
         )
 
     @classmethod

packit_service/schema.py

@@ -83,6 +83,7 @@ class ServiceConfigSchema(UserConfigSchema):
     package_config_path_override = fields.String()
     command_handler_storage_class = fields.String(missing="gp2")
     appcode = fields.String()
+    enabled_projects_for_fedora_ci = fields.List(fields.String())
 
     @post_load
     def make_instance(self, data, **kwargs):

packit_service/utils.py

@@ -217,7 +217,7 @@ def get_packit_commands_from_comment(
     return []
 
 
-def get_koji_task_id_and_url_from_stdout(stdout: str):
+def get_koji_task_id_and_url_from_stdout(stdout: str) -> tuple[Optional[int], Optional[str]]:
     task_id, task_url = None, None
 
     task_id_match = search(pattern=r"Created task: (\d+)", string=stdout)

packit_service/worker/checker/koji.py

@@ -3,6 +3,8 @@
 
 import logging
 
+from ogr.services.pagure import PagureProject
+
 from packit_service.constants import (
     KOJI_PRODUCTION_BUILDS_ISSUE,
     PERMISSIONS_ERROR_WRITE_OR_ADMIN,
@@ -25,6 +27,11 @@ def pre_check(self) -> bool:
         return self.koji_build_helper.is_job_config_trigger_matching(self.job_config)
 
 
+class IsUpstreamKojiScratchBuild(Checker, GetKojiBuildJobHelperMixin):
+    def pre_check(self) -> bool:
+        return not isinstance(self.koji_build_helper.project, PagureProject)
+
+
 class PermissionOnKoji(Checker, GetKojiBuildJobHelperMixin):
     def pre_check(self) -> bool:
         if (

packit_service/worker/events/pagure.py

@@ -199,6 +199,7 @@ def __init__(
         project_url: str,
         commit_sha: str,
         user_login: str,
+        target_branch: str,
     ):
         super().__init__(project_url=project_url, pr_id=pr_id)
         self.action = action
@@ -212,6 +213,7 @@ def __init__(
         self.identifier = str(pr_id)
         self.git_ref = None  # pr_id will be used for checkout
         self.project_url = project_url
+        self.target_branch = target_branch
 
     def get_dict(self, default_dict: Optional[dict] = None) -> dict:
         result = super().get_dict()

packit_service/worker/handlers/abstract.py

@@ -43,6 +43,9 @@
     set,
 )
 SUPPORTED_EVENTS_FOR_HANDLER: dict[type["JobHandler"], set[type["Event"]]] = defaultdict(set)
+SUPPORTED_EVENTS_FOR_HANDLER_FEDORA_CI: dict[type["JobHandler"], set[type["Event"]]] = defaultdict(
+    set
+)
 MAP_COMMENT_TO_HANDLER: dict[str, set[type["JobHandler"]]] = defaultdict(set)
 MAP_CHECK_PREFIX_TO_HANDLER: dict[str, set[type["JobHandler"]]] = defaultdict(set)
 
@@ -84,7 +87,7 @@ def reacts_to(event: type["Event"]):
     """
     [class decorator]
     Specify an event for which we want to use this handler.
-    Matching is done via isinstance so you can use some abstract class as well.
+    Matching is done via `isinstance` so you can use some abstract class as well.
 
     Multiple decorators are allowed.
 
@@ -104,6 +107,30 @@ def _add_to_mapping(kls: type["JobHandler"]):
     return _add_to_mapping
 
 
+def reacts_to_as_fedora_ci(event: type["Event"]):
+    """
+    [class decorator]
+    Specify an event for which we want to use this handler as a Fedora CI.
+    Matching is done via `isinstance` so you can use some abstract class as well.
+
+    Multiple decorators are allowed.
+
+    Example:
+    ```
+    @reacts_to(ReleaseEvent)
+    @reacts_to(PullRequestGithubEvent)
+    @reacts_to(PushGitHubEvent)
+    class CoprBuildHandler(JobHandler):
+    ```
+    """
+
+    def _add_to_mapping(kls: type["JobHandler"]):
+        SUPPORTED_EVENTS_FOR_HANDLER_FEDORA_CI[kls].add(event)
+        return kls
+
+    return _add_to_mapping
+
+
 def run_for_comment(command: str):
     """
     [class decorator]
@@ -190,6 +217,8 @@ class TaskName(str, enum.Enum):
     tag_into_sidetag = "task.tag_into_sidetag"
     openscanhub_task_finished = "task.openscanhub_task_finished"
     openscanhub_task_started = "task.openscanhub_task_started"
+    downstream_koji_scratch_build = "task.run_downstream_koji_scratch_build_handler"
+    downstream_koji_scratch_build_report = "task.run_downstream_koji_scratch_build_report_handler"
 
 
 class Handler(PackitAPIProtocol, Config):

packit_service/worker/handlers/distgit.py

@@ -25,6 +25,7 @@
     PackitException,
     ReleaseSkippedPackitException,
 )
+from packit.utils import commands
 from packit.utils.koji_helper import KojiHelper
 
 from packit_service import sentry_integration
@@ -53,6 +54,7 @@
     SyncReleaseTargetStatus,
 )
 from packit_service.service.urls import (
+    get_koji_build_info_url,
     get_propose_downstream_info_url,
     get_pull_from_upstream_info_url,
 )
@@ -79,6 +81,7 @@
     IssueCommentGitlabEvent,
     KojiTaskEvent,
     PullRequestCommentPagureEvent,
+    PullRequestPagureEvent,
     PushPagureEvent,
     ReleaseEvent,
     ReleaseGitlabEvent,
@@ -91,10 +94,12 @@
     TaskName,
     configured_as,
     reacts_to,
+    reacts_to_as_fedora_ci,
     run_for_check_rerun,
     run_for_comment,
 )
 from packit_service.worker.handlers.mixin import GetProjectToSyncMixin
+from packit_service.worker.helpers.fedora_ci import FedoraCIHelper
 from packit_service.worker.helpers.sidetag import SidetagHelper
 from packit_service.worker.helpers.sync_release.propose_downstream import (
     ProposeDownstreamJobHelper,
@@ -740,6 +745,146 @@ def run(self) -> TaskResults:
             return super().run()
 
 
+@reacts_to_as_fedora_ci(event=PullRequestPagureEvent)
+class DownstreamKojiScratchBuildHandler(
+    RetriableJobHandler, ConfigFromUrlMixin, LocalProjectMixin, PackitAPIWithDownstreamMixin
+):
+    """
+    This handler can submit a scratch build in Koji from a dist-git (Fedora CI).
+    """
+
+    task_name = TaskName.downstream_koji_scratch_build
+
+    def __init__(
+        self,
+        package_config: PackageConfig,
+        job_config: JobConfig,
+        event: dict,
+        celery_task: Task,
+        koji_group_model_id: Optional[int] = None,
+    ):
+        super().__init__(
+            package_config=package_config,
+            job_config=job_config,
+            event=event,
+            celery_task=celery_task,
+        )
+        self._project_url = self.data.project_url
+        self._packit_api = None
+        self._koji_group_model_id = koji_group_model_id
+        self._ci_helper: Optional[FedoraCIHelper] = None
+
+    @property
+    def ci_helper(self) -> FedoraCIHelper:
+        if not self._ci_helper:
+            self._ci_helper = FedoraCIHelper(
+                project=self.project,
+                metadata=self.data,
+            )
+        return self._ci_helper
+
+    @property
+    def dist_git_branch(self) -> str:
+        return self.data.event_dict.get("target_branch")
+
+    @property
+    def repo_url(self) -> str:
+        event_dict = self.data.event_dict
+        full_repo_name = (
+            f"forks/{event_dict.get('base_repo_owner')}/"
+            f"{event_dict.get('base_repo_namespace')}/{event_dict.get('base_repo_name')}"
+        )
+        return f"git+https://src.fedoraproject.org/{full_repo_name}.git#{self.data.commit_sha}"
+
+    def run(self) -> TaskResults:
+        try:
+            self.packit_api.init_kerberos_ticket()
+        except PackitCommandFailedError as ex:
+            msg = f"Kerberos authentication error: {ex.stderr_output}"
+            logger.error(msg)
+            self.ci_helper.report(
+                state=BaseCommitStatus.error,
+                description=msg,
+                url=None,
+            )
+            return TaskResults(success=False, details={"msg": msg})
+
+        build_group = KojiBuildGroupModel.create(
+            run_model=PipelineModel.create(
+                project_event=self.data.db_project_event,
+            )
+        )
+
+        koji_build = KojiBuildTargetModel.create(
+            task_id=None,
+            web_url=None,
+            target=self.dist_git_branch,
+            status="pending",
+            scratch=True,
+            koji_build_group=build_group,
+        )
+        try:
+            stdout = self.run_koji_build()
+            if stdout:
+                task_id, web_url = get_koji_task_id_and_url_from_stdout(stdout)
+                koji_build.set_task_id(str(task_id))
+                koji_build.set_web_url(web_url)
+                koji_build.set_build_submission_stdout(stdout)
+            url = get_koji_build_info_url(koji_build.id)
+            self.ci_helper.report(
+                state=BaseCommitStatus.running,
+                description="RPM build was submitted ...",
+                url=url,
+            )
+        except Exception as ex:
+            sentry_integration.send_to_sentry(ex)
+            self.ci_helper.report(
+                state=BaseCommitStatus.error,
+                description=f"Submit of the build failed: {ex}",
+                url=None,
+            )
+            if isinstance(ex, PackitCommandFailedError):
+                error = f"{ex!s}\n{ex.stderr_output}"
+                koji_build.set_build_submission_stdout(ex.stdout_output)
+                koji_build.set_data({"error": error})
+
+            koji_build.set_status("error")
+            return TaskResults(
+                success=False,
+                details={
+                    "msg": "Koji scratch build submit was not successful.",
+                    "error": str(ex),
+                },
+            )
+
+        return TaskResults(success=True, details={})
+
+    def run_koji_build(
+        self,
+    ):
+        """
+        Perform a `koji build` from SCM.
+
+        Returns:
+            str output
+        """
+        cmd = [
+            "koji",
+            "build",
+            "--scratch",
+            "--nowait",
+            self.dist_git_branch,
+            self.repo_url,
+        ]
+        logger.info("Starting a Koji scratch build.")
+        return commands.run_command_remote(
+            cmd=cmd,
+            cwd=self.local_project.working_dir,
+            output=True,
+            print_live=True,
+        ).stdout
+
+
 class AbstractDownstreamKojiBuildHandler(
     abc.ABC,
     RetriableJobHandler,