From 8e86eceaddbbafdb9c0a2e9951fdf1c49c636f96 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Fri, 1 Nov 2024 09:41:42 -0400 Subject: [PATCH 01/17] Extract get_downloader_for_repo_configuration() utility function Extract the code for creating a "downloader" (prepared session.get() method) from the tree_info file - we'll use this when querying and downloading Flatpak repositories from remote sources as well. --- pyanaconda/modules/payloads/base/utils.py | 57 ++++++++++++++++++- .../modules/payloads/payload/dnf/tree_info.py | 57 +------------------ 2 files changed, 59 insertions(+), 55 deletions(-) diff --git a/pyanaconda/modules/payloads/base/utils.py b/pyanaconda/modules/payloads/base/utils.py index 9be5602bbf6..a72fdcfbe66 100644 --- a/pyanaconda/modules/payloads/base/utils.py +++ b/pyanaconda/modules/payloads/base/utils.py @@ -17,8 +17,13 @@ # License and may only be used or replicated with the express permission of # Red Hat, Inc. # +from functools import partial + from pyanaconda.anaconda_loggers import get_module_logger -from pyanaconda.core.payload import rpm_version_key +from pyanaconda.core.configuration.anaconda import conf +from pyanaconda.core.constants import NETWORK_CONNECTION_TIMEOUT, USER_AGENT +from pyanaconda.core.payload import ProxyString, ProxyStringError, rpm_version_key +from pyanaconda.modules.common.structures.payload import RepoConfigurationData log = get_module_logger(__name__) @@ -26,3 +31,53 @@ def sort_kernel_version_list(kernel_version_list): """Sort the given kernel version list.""" kernel_version_list.sort(key=rpm_version_key) + + +def get_downloader_for_repo_configuration(session, data: RepoConfigurationData): + """Get a configured session.get method. + + :return: a partial function + """ + # Prepare the SSL configuration. + ssl_enabled = conf.payload.verify_ssl and data.ssl_verification_enabled + + # ssl_verify can be: + # - the path to a cert file + # - True, to use the system's certificates + # - False, to not verify + ssl_verify = data.ssl_configuration.ca_cert_path or ssl_enabled + + # ssl_cert can be: + # - a tuple of paths to a client cert file and a client key file + # - None + ssl_client_cert = data.ssl_configuration.client_cert_path or None + ssl_client_key = data.ssl_configuration.client_key_path or None + ssl_cert = (ssl_client_cert, ssl_client_key) if ssl_client_cert else None + + # Prepare the proxy configuration. + proxy_url = data.proxy or None + proxies = {} + + if proxy_url: + try: + proxy = ProxyString(proxy_url) + proxies = { + "http": proxy.url, + "https": proxy.url, + "ftp": proxy.url + } + except ProxyStringError as e: + log.debug("Failed to parse the proxy '%s': %s", proxy_url, e) + + # Prepare headers. + headers = {"user-agent": USER_AGENT} + + # Return a partial function. + return partial( + session.get, + headers=headers, + proxies=proxies, + verify=ssl_verify, + cert=ssl_cert, + timeout=NETWORK_CONNECTION_TIMEOUT + ) diff --git a/pyanaconda/modules/payloads/payload/dnf/tree_info.py b/pyanaconda/modules/payloads/payload/dnf/tree_info.py index edba8c8c506..917261f81cc 100644 --- a/pyanaconda/modules/payloads/payload/dnf/tree_info.py +++ b/pyanaconda/modules/payloads/payload/dnf/tree_info.py @@ -20,7 +20,6 @@ import os import time from collections import namedtuple -from functools import partial from productmd.treeinfo import TreeInfo from requests import RequestException @@ -29,16 +28,15 @@ from pyanaconda.core.configuration.anaconda import conf from pyanaconda.core.constants import ( DEFAULT_REPOS, - NETWORK_CONNECTION_TIMEOUT, REPO_ORIGIN_TREEINFO, URL_TYPE_BASEURL, - USER_AGENT, ) from pyanaconda.core.path import join_paths -from pyanaconda.core.payload import ProxyString, ProxyStringError, split_protocol +from pyanaconda.core.payload import split_protocol from pyanaconda.core.util import requests_session, xprogressive_delay from pyanaconda.modules.common.structures.payload import RepoConfigurationData from pyanaconda.modules.common.task import Task +from pyanaconda.modules.payloads.base.utils import get_downloader_for_repo_configuration log = get_module_logger(__name__) @@ -240,7 +238,7 @@ def load_data(self, data: RepoConfigurationData): log.debug("Load treeinfo metadata for '%s'.", data.url) with requests_session() as session: - downloader = self._get_downloader(session, data) + downloader = get_downloader_for_repo_configuration(session, data) content = self._download_metadata(downloader, data.url) # Process the metadata. @@ -249,55 +247,6 @@ def load_data(self, data: RepoConfigurationData): file_content=content ) - def _get_downloader(self, session, data): - """Get a configured session.get method. - - :return: a partial function - """ - # Prepare the SSL configuration. - ssl_enabled = conf.payload.verify_ssl and data.ssl_verification_enabled - - # ssl_verify can be: - # - the path to a cert file - # - True, to use the system's certificates - # - False, to not verify - ssl_verify = data.ssl_configuration.ca_cert_path or ssl_enabled - - # ssl_cert can be: - # - a tuple of paths to a client cert file and a client key file - # - None - ssl_client_cert = data.ssl_configuration.client_cert_path or None - ssl_client_key = data.ssl_configuration.client_key_path or None - ssl_cert = (ssl_client_cert, ssl_client_key) if ssl_client_cert else None - - # Prepare the proxy configuration. - proxy_url = data.proxy or None - proxies = {} - - if proxy_url: - try: - proxy = ProxyString(proxy_url) - proxies = { - "http": proxy.url, - "https": proxy.url, - "ftp": proxy.url - } - except ProxyStringError as e: - log.debug("Failed to parse the proxy '%s': %s", proxy_url, e) - - # Prepare headers. - headers = {"user-agent": USER_AGENT} - - # Return a partial function. - return partial( - session.get, - headers=headers, - proxies=proxies, - verify=ssl_verify, - cert=ssl_cert, - timeout=NETWORK_CONNECTION_TIMEOUT - ) - def _download_metadata(self, downloader, url): """Download metadata from the given URL.""" # How many times should we try the download? From 6915ff7fce5f64d68ebf24659caa08420f2e55c7 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Tue, 17 Dec 2024 13:11:41 -0500 Subject: [PATCH 02/17] Extract pick_download_location(), calculate_required_space() utilities We'll need the functionality of picking a download location and calculating required space for Flatpak installation as well as for DNF installation, so move the utility functions into payloads/base/utils.py. --- pyanaconda/modules/payloads/base/utils.py | 191 +++++++++++++++ .../modules/payloads/payload/dnf/dnf.py | 5 +- .../payloads/payload/dnf/installation.py | 8 +- .../modules/payloads/payload/dnf/utils.py | 192 +-------------- .../payload/test_module_payload_dnf_utils.py | 230 ------------------ .../test_module_payload_base_utils.py | 227 ++++++++++++++++- 6 files changed, 427 insertions(+), 426 deletions(-) diff --git a/pyanaconda/modules/payloads/base/utils.py b/pyanaconda/modules/payloads/base/utils.py index a72fdcfbe66..3127bb564eb 100644 --- a/pyanaconda/modules/payloads/base/utils.py +++ b/pyanaconda/modules/payloads/base/utils.py @@ -17,12 +17,19 @@ # License and may only be used or replicated with the express permission of # Red Hat, Inc. # +import os from functools import partial +from blivet.size import Size + from pyanaconda.anaconda_loggers import get_module_logger from pyanaconda.core.configuration.anaconda import conf from pyanaconda.core.constants import NETWORK_CONNECTION_TIMEOUT, USER_AGENT +from pyanaconda.core.path import join_paths from pyanaconda.core.payload import ProxyString, ProxyStringError, rpm_version_key +from pyanaconda.core.util import execWithCapture +from pyanaconda.modules.common.constants.objects import DEVICE_TREE +from pyanaconda.modules.common.constants.services import STORAGE from pyanaconda.modules.common.structures.payload import RepoConfigurationData log = get_module_logger(__name__) @@ -81,3 +88,187 @@ def get_downloader_for_repo_configuration(session, data: RepoConfigurationData): cert=ssl_cert, timeout=NETWORK_CONNECTION_TIMEOUT ) + + +def get_free_space_map(current=True, scheduled=False): + """Get the available file system disk space. + + :param bool current: use information about current mount points + :param bool scheduled: use information about scheduled mount points + :return: a dictionary of mount points and their available space + """ + mount_points = {} + + if scheduled: + mount_points.update(_get_scheduled_free_space_map()) + + if current: + mount_points.update(_get_current_free_space_map()) + + return mount_points + + +def _get_current_free_space_map(): + """Get the available file system disk space of the current system. + + :return: a dictionary of mount points and their available space + """ + mapping = {} + + # Generate the dictionary of mount points and sizes. + output = execWithCapture('df', ['--output=target,avail']) + lines = output.rstrip().splitlines() + + for line in lines: + key, val = line.rsplit(maxsplit=1) + + if not key.startswith('/'): + continue + + mapping[key] = Size(int(val) * 1024) + + # Add /var/tmp/ if this is a directory or image installation. + if not conf.target.is_hardware: + var_tmp = os.statvfs("/var/tmp") + mapping["/var/tmp"] = Size(var_tmp.f_frsize * var_tmp.f_bfree) + + return mapping + + +def _get_scheduled_free_space_map(): + """Get the available file system disk space of the scheduled system. + + :return: a dictionary of mount points and their available space + """ + device_tree = STORAGE.get_proxy(DEVICE_TREE) + mount_points = {} + + for mount_point in device_tree.GetMountPoints(): + # we can ignore swap + if not mount_point.startswith('/'): + continue + + free_space = Size( + device_tree.GetFileSystemFreeSpace([mount_point]) + ) + mount_point = os.path.normpath( + conf.target.system_root + mount_point + ) + mount_points[mount_point] = free_space + + return mount_points + + +def _pick_mount_points(mount_points, download_size, install_size): + """Pick mount points for the package installation. + + :return: a set of sufficient mount points + """ + suitable = { + '/var/tmp', + conf.target.system_root, + join_paths(conf.target.system_root, 'home'), + join_paths(conf.target.system_root, 'tmp'), + join_paths(conf.target.system_root, 'var'), + } + + sufficient = set() + + for mount_point, size in mount_points.items(): + # Ignore mount points that are not suitable. + if mount_point not in suitable: + continue + + if size >= (download_size + install_size): + log.debug("Considering %s (%s) for download and install.", mount_point, size) + sufficient.add(mount_point) + + elif size >= download_size and not mount_point.startswith(conf.target.system_root): + log.debug("Considering %s (%s) for download.", mount_point, size) + sufficient.add(mount_point) + + return sufficient + + +def _get_biggest_mount_point(mount_points, sufficient): + """Get the biggest sufficient mount point. + + :return: a mount point or None + """ + return max(sufficient, default=None, key=mount_points.get) + + +def pick_download_location(download_size, install_size, cache_dir_suffix): + """Pick a download location + + :param dnf_manager: the DNF manager + :return: a path to the download location + """ + mount_points = get_free_space_map() + + # Try to find mount points that are sufficient for download and install. + sufficient = _pick_mount_points( + mount_points, + download_size, + install_size + ) + + # Or find mount points that are sufficient only for download. + if not sufficient: + sufficient = _pick_mount_points( + mount_points, + download_size, + install_size=0 + ) + + # Ignore the system root if there are other mount points. + if len(sufficient) > 1: + sufficient.discard(conf.target.system_root) + + if not sufficient: + raise RuntimeError( + "Not enough disk space to download the " + "packages; size {}.".format(download_size) + ) + + # Choose the biggest sufficient mount point. + mount_point = _get_biggest_mount_point(mount_points, sufficient) + + log.info("Mount point %s picked as download location", mount_point) + location = join_paths(mount_point, cache_dir_suffix) + + return location + + +def calculate_required_space(download_size, installation_size): + """Calculate the space required for the installation. + + This takes into account whether the download location is part of the installed + system or not. + + :param Size download_size: the download size + :param Size installation_size: the installation size + :return Size: the required space + """ + mount_points = get_free_space_map(scheduled=True) + + # Find sufficient mount points. + sufficient = _pick_mount_points( + mount_points, + download_size, + installation_size + ) + + # Choose the biggest sufficient mount point. + mount_point = _get_biggest_mount_point(mount_points, sufficient) + + if not mount_point or mount_point.startswith(conf.target.system_root): + log.debug("The install and download space is required.") + required_space = installation_size + download_size + else: + log.debug("Use the %s mount point for the %s download.", mount_point, download_size) + log.debug("Only the install space is required.") + required_space = installation_size + + log.debug("The package installation requires %s.", required_space) + return required_space diff --git a/pyanaconda/modules/payloads/payload/dnf/dnf.py b/pyanaconda/modules/payloads/payload/dnf/dnf.py index 6f6ee29d436..9d6246202b2 100644 --- a/pyanaconda/modules/payloads/payload/dnf/dnf.py +++ b/pyanaconda/modules/payloads/payload/dnf/dnf.py @@ -24,6 +24,7 @@ PackagesConfigurationData, PackagesSelectionData, ) +from pyanaconda.modules.payloads.base.utils import calculate_required_space from pyanaconda.modules.payloads.constants import PayloadType, SourceType from pyanaconda.modules.payloads.kickstart import ( convert_ks_data_to_packages_configuration, @@ -54,7 +55,6 @@ ) from pyanaconda.modules.payloads.payload.dnf.tear_down import ResetDNFManagerTask from pyanaconda.modules.payloads.payload.dnf.utils import ( - calculate_required_space, collect_installation_devices, protect_installation_devices, ) @@ -388,7 +388,8 @@ def calculate_required_space(self): :return: required size in bytes :rtype: int """ - required_space = calculate_required_space(self.dnf_manager) + required_space = calculate_required_space(self._dnf_manager.get_download_size(), + self._dnf_manager.get_installation_size()) return required_space.get_bytes() def get_repo_configurations(self): diff --git a/pyanaconda/modules/payloads/payload/dnf/installation.py b/pyanaconda/modules/payloads/payload/dnf/installation.py index af126003f02..7df3b91f0b5 100644 --- a/pyanaconda/modules/payloads/payload/dnf/installation.py +++ b/pyanaconda/modules/payloads/payload/dnf/installation.py @@ -36,6 +36,7 @@ ) from pyanaconda.modules.common.structures.packages import PackagesConfigurationData from pyanaconda.modules.common.task import Task +from pyanaconda.modules.payloads.base.utils import pick_download_location from pyanaconda.modules.payloads.payload.dnf.requirements import ( apply_requirements, collect_dnf_requirements, @@ -46,7 +47,6 @@ ) from pyanaconda.modules.payloads.payload.dnf.utils import ( get_kernel_version_list, - pick_download_location, ) from pyanaconda.modules.payloads.payload.dnf.validation import ( CheckPackagesSelectionTask, @@ -54,6 +54,8 @@ log = get_module_logger(__name__) +DNF_PACKAGE_CACHE_DIR_SUFFIX = 'dnf.package.cache' + class SetRPMMacrosTask(Task): """Installation task to set RPM macros.""" @@ -191,7 +193,9 @@ def run(self): :return: a path of the download location """ - path = pick_download_location(self._dnf_manager) + path = pick_download_location(self._dnf_manager.get_download_size(), + self._dnf_manager.get_installation_size(), + DNF_PACKAGE_CACHE_DIR_SUFFIX) if os.path.exists(path): log.info("Removing existing package download location: %s", path) diff --git a/pyanaconda/modules/payloads/payload/dnf/utils.py b/pyanaconda/modules/payloads/payload/dnf/utils.py index f0b765de57b..e9fd165c842 100644 --- a/pyanaconda/modules/payloads/payload/dnf/utils.py +++ b/pyanaconda/modules/payloads/payload/dnf/utils.py @@ -17,21 +17,17 @@ # import fnmatch import hashlib -import os import rpm -from blivet.size import Size from libdnf.transaction import TransactionItemState_ERROR from pyanaconda.anaconda_loggers import get_module_logger from pyanaconda.core.configuration.anaconda import conf from pyanaconda.core.hw import is_lpae_available -from pyanaconda.core.path import join_paths from pyanaconda.core.payload import parse_hdd_url from pyanaconda.core.product import get_product_name, get_product_version from pyanaconda.core.regexes import VERSION_DIGITS -from pyanaconda.core.util import execWithCapture -from pyanaconda.modules.common.constants.objects import DEVICE_TREE, DISK_SELECTION +from pyanaconda.modules.common.constants.objects import DISK_SELECTION from pyanaconda.modules.common.constants.services import STORAGE from pyanaconda.modules.common.structures.packages import PackagesSelectionData from pyanaconda.modules.payloads.base.utils import sort_kernel_version_list @@ -39,8 +35,6 @@ log = get_module_logger(__name__) -DNF_PACKAGE_CACHE_DIR_SUFFIX = 'dnf.package.cache' - def calculate_hash(data): """Calculate hash from the given data. @@ -183,190 +177,6 @@ def get_kernel_version_list(): return files -def get_free_space_map(current=True, scheduled=False): - """Get the available file system disk space. - - :param bool current: use information about current mount points - :param bool scheduled: use information about scheduled mount points - :return: a dictionary of mount points and their available space - """ - mount_points = {} - - if scheduled: - mount_points.update(_get_scheduled_free_space_map()) - - if current: - mount_points.update(_get_current_free_space_map()) - - return mount_points - - -def _get_current_free_space_map(): - """Get the available file system disk space of the current system. - - :return: a dictionary of mount points and their available space - """ - mapping = {} - - # Generate the dictionary of mount points and sizes. - output = execWithCapture('df', ['--output=target,avail']) - lines = output.rstrip().splitlines() - - for line in lines: - key, val = line.rsplit(maxsplit=1) - - if not key.startswith('/'): - continue - - mapping[key] = Size(int(val) * 1024) - - # Add /var/tmp/ if this is a directory or image installation. - if not conf.target.is_hardware: - var_tmp = os.statvfs("/var/tmp") - mapping["/var/tmp"] = Size(var_tmp.f_frsize * var_tmp.f_bfree) - - return mapping - - -def _get_scheduled_free_space_map(): - """Get the available file system disk space of the scheduled system. - - :return: a dictionary of mount points and their available space - """ - device_tree = STORAGE.get_proxy(DEVICE_TREE) - mount_points = {} - - for mount_point in device_tree.GetMountPoints(): - # we can ignore swap - if not mount_point.startswith('/'): - continue - - free_space = Size( - device_tree.GetFileSystemFreeSpace([mount_point]) - ) - mount_point = os.path.normpath( - conf.target.system_root + mount_point - ) - mount_points[mount_point] = free_space - - return mount_points - - -def _pick_mount_points(mount_points, download_size, install_size): - """Pick mount points for the package installation. - - :return: a set of sufficient mount points - """ - suitable = { - '/var/tmp', - conf.target.system_root, - join_paths(conf.target.system_root, 'home'), - join_paths(conf.target.system_root, 'tmp'), - join_paths(conf.target.system_root, 'var'), - } - - sufficient = set() - - for mount_point, size in mount_points.items(): - # Ignore mount points that are not suitable. - if mount_point not in suitable: - continue - - if size >= (download_size + install_size): - log.debug("Considering %s (%s) for download and install.", mount_point, size) - sufficient.add(mount_point) - - elif size >= download_size and not mount_point.startswith(conf.target.system_root): - log.debug("Considering %s (%s) for download.", mount_point, size) - sufficient.add(mount_point) - - return sufficient - - -def _get_biggest_mount_point(mount_points, sufficient): - """Get the biggest sufficient mount point. - - :return: a mount point or None - """ - return max(sufficient, default=None, key=mount_points.get) - - -def pick_download_location(dnf_manager): - """Pick the download location. - - :param dnf_manager: the DNF manager - :return: a path to the download location - """ - download_size = dnf_manager.get_download_size() - install_size = dnf_manager.get_installation_size() - mount_points = get_free_space_map() - - # Try to find mount points that are sufficient for download and install. - sufficient = _pick_mount_points( - mount_points, - download_size, - install_size - ) - - # Or find mount points that are sufficient only for download. - if not sufficient: - sufficient = _pick_mount_points( - mount_points, - download_size, - install_size=0 - ) - - # Ignore the system root if there are other mount points. - if len(sufficient) > 1: - sufficient.discard(conf.target.system_root) - - if not sufficient: - raise RuntimeError( - "Not enough disk space to download the " - "packages; size {}.".format(download_size) - ) - - # Choose the biggest sufficient mount point. - mount_point = _get_biggest_mount_point(mount_points, sufficient) - - log.info("Mount point %s picked as download location", mount_point) - location = join_paths(mount_point, DNF_PACKAGE_CACHE_DIR_SUFFIX) - - return location - - -def calculate_required_space(dnf_manager): - """Calculate the space required for the installation. - - :param DNFManager dnf_manager: the DNF manager - :return Size: the required space - """ - installation_size = dnf_manager.get_installation_size() - download_size = dnf_manager.get_download_size() - mount_points = get_free_space_map(scheduled=True) - - # Find sufficient mount points. - sufficient = _pick_mount_points( - mount_points, - download_size, - installation_size - ) - - # Choose the biggest sufficient mount point. - mount_point = _get_biggest_mount_point(mount_points, sufficient) - - if not mount_point or mount_point.startswith(conf.target.system_root): - log.debug("The install and download space is required.") - required_space = installation_size + download_size - else: - log.debug("Use the %s mount point for the %s download.", mount_point, download_size) - log.debug("Only the install space is required.") - required_space = installation_size - - log.debug("The package installation requires %s.", required_space) - return required_space - - def collect_installation_devices(sources, repositories): """Collect devices of installation sources. diff --git a/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_dnf_utils.py b/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_dnf_utils.py index c9d39d6fd75..618fdddbbd2 100644 --- a/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_dnf_utils.py +++ b/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_dnf_utils.py @@ -16,36 +16,25 @@ # Red Hat, Inc. # import unittest -from textwrap import dedent from unittest.mock import Mock, patch -import pytest -from blivet.size import Size - from pyanaconda.core.constants import ( GROUP_PACKAGE_TYPES_ALL, GROUP_PACKAGE_TYPES_REQUIRED, ) -from pyanaconda.modules.common.constants.objects import DEVICE_TREE -from pyanaconda.modules.common.constants.services import STORAGE from pyanaconda.modules.common.structures.packages import PackagesSelectionData from pyanaconda.modules.common.structures.payload import RepoConfigurationData from pyanaconda.modules.payloads.payload.dnf.dnf_manager import DNFManager from pyanaconda.modules.payloads.payload.dnf.utils import ( - _pick_mount_points, - calculate_required_space, collect_installation_devices, - get_free_space_map, get_installation_specs, get_kernel_package, get_kernel_version_list, get_product_release_version, - pick_download_location, ) from pyanaconda.modules.payloads.source.cdrom.cdrom import CdromSourceModule from pyanaconda.modules.payloads.source.harddrive.harddrive import HardDriveSourceModule from pyanaconda.modules.payloads.source.url.url import URLSourceModule -from tests.unit_tests.pyanaconda_tests import patch_dbus_get_proxy_with_cache class DNFUtilsPackagesTestCase(unittest.TestCase): @@ -205,225 +194,6 @@ def test_get_kernel_version_list(self, mock_rpm): '8.8.18-200.fc32.x86_64', ] - @patch("pyanaconda.modules.payloads.payload.dnf.utils.execWithCapture") - def test_get_free_space(self, exec_mock): - """Test the get_free_space function.""" - output = """ - Mounted on Avail - /dev 100 - /dev/shm 200 - /run 300 - / 400 - /tmp 500 - /boot 600 - /home 700 - /boot/efi 800 - """ - exec_mock.return_value = dedent(output).strip() - - assert get_free_space_map() == { - '/dev': Size("100 KiB"), - '/dev/shm': Size("200 KiB"), - '/run': Size("300 KiB"), - '/': Size("400 KiB"), - '/tmp': Size("500 KiB"), - '/boot': Size("600 KiB"), - '/home': Size("700 KiB"), - '/boot/efi': Size("800 KiB"), - } - - @patch("os.statvfs") - @patch("pyanaconda.modules.payloads.payload.dnf.utils.conf") - @patch("pyanaconda.modules.payloads.payload.dnf.utils.execWithCapture") - def test_get_free_space_image(self, exec_mock, conf_mock, statvfs_mock): - """Test the get_free_space function.""" - output = """ - Mounted on Avail - / 100 - /boot 200 - """ - exec_mock.return_value = dedent(output).strip() - conf_mock.target.is_hardware = False - statvfs_mock.return_value = Mock(f_frsize=1024, f_bfree=300) - - assert get_free_space_map() == { - '/': Size("100 KiB"), - '/boot': Size("200 KiB"), - '/var/tmp': Size("300 KiB"), - } - - def test_pick_mount_points(self): - """Test the _pick_mount_points function.""" - mount_points = { - "/": Size("1 G"), - "/home": Size("1 G"), - "/var/tmp": Size("1 G"), - "/mnt/sysroot": Size("1 G"), - "/mnt/sysroot/home": Size("1 G"), - "/mnt/sysroot/tmp": Size("1 G"), - "/mnt/sysroot/var": Size("1 G"), - "/mnt/sysroot/usr": Size("1 G"), - - } - - # All mount points are big enough. - # Choose all suitable mount points. - sufficient = _pick_mount_points( - mount_points, - download_size=Size("0.5 G"), - install_size=Size("0.5 G") - ) - assert sufficient == { - "/var/tmp", - "/mnt/sysroot", - "/mnt/sysroot/home", - "/mnt/sysroot/tmp", - "/mnt/sysroot/var" - } - - # No mount point is big enough for installation. - # Choose non-sysroot mount points for download. - sufficient = _pick_mount_points( - mount_points, - download_size=Size("0.5 G"), - install_size=Size("1.5 G") - ) - assert sufficient == { - "/var/tmp", - } - - # No mount point is big enough for installation or download. - sufficient = _pick_mount_points( - mount_points, - download_size=Size("1.5 G"), - install_size=Size("1.5 G") - ) - assert sufficient == set() - - @patch("pyanaconda.modules.payloads.payload.dnf.utils.get_free_space_map") - def test_pick_download_location(self, free_space_getter): - """Test the pick_download_location function.""" - download_size = Size(100) - installation_size = Size(200) - total_size = Size(300) - - dnf_manager = Mock() - dnf_manager.get_download_size.return_value = download_size - dnf_manager.get_installation_size.return_value = installation_size - - # Found mount points for download and install. - # Don't use /mnt/sysroot if possible. - free_space_getter.return_value = { - "/var/tmp": download_size, - "/mnt/sysroot": total_size, - } - - path = pick_download_location(dnf_manager) - assert path == "/var/tmp/dnf.package.cache" - - # Found mount points only for download. - # Use the biggest mount point. - free_space_getter.return_value = { - "/mnt/sysroot/tmp": download_size + 1, - "/mnt/sysroot/home": download_size, - } - - path = pick_download_location(dnf_manager) - assert path == "/mnt/sysroot/tmp/dnf.package.cache" - - # No mount point to use. - # Fail with an exception. - free_space_getter.return_value = {} - - with pytest.raises(RuntimeError) as cm: - pick_download_location(dnf_manager) - - msg = "Not enough disk space to download the packages; size 100 B." - assert str(cm.value) == msg - - @patch("pyanaconda.modules.payloads.payload.dnf.utils.execWithCapture") - @patch_dbus_get_proxy_with_cache - def test_get_combined_free_space(self, proxy_getter, exec_mock): - """Test the get_free_space function with the combined options.""" - output = """ - Mounted on Avail - / 100 - /tmp 200 - """ - exec_mock.return_value = dedent(output).strip() - - mount_points = { - '/': Size("300 KiB"), - '/boot': Size("400 KiB"), - } - - def get_mount_points(): - return list(mount_points.keys()) - - def get_free_space(paths): - return sum(map(mount_points.get, paths)) - - device_tree = STORAGE.get_proxy(DEVICE_TREE) - device_tree.GetMountPoints.side_effect = get_mount_points - device_tree.GetFileSystemFreeSpace.side_effect = get_free_space - - assert get_free_space_map(current=True, scheduled=False) == { - '/': Size("100 KiB"), - '/tmp': Size("200 KiB"), - } - - assert get_free_space_map(current=False, scheduled=True) == { - '/mnt/sysroot': Size("300 KiB"), - '/mnt/sysroot/boot': Size("400 KiB"), - } - - assert get_free_space_map(current=True, scheduled=True) == { - '/': Size("100 KiB"), - '/tmp': Size("200 KiB"), - '/mnt/sysroot': Size("300 KiB"), - '/mnt/sysroot/boot': Size("400 KiB"), - } - - assert get_free_space_map(current=False, scheduled=False) == {} - - @patch("pyanaconda.modules.payloads.payload.dnf.utils.get_free_space_map") - def test_calculate_required_space(self, free_space_getter): - """Test the calculate_required_space function.""" - download_size = Size(100) - installation_size = Size(200) - total_size = Size(300) - - dnf_manager = Mock() - dnf_manager.get_download_size.return_value = download_size - dnf_manager.get_installation_size.return_value = installation_size - - # No mount point to use. - # The total size is required. - free_space_getter.return_value = {} - assert calculate_required_space(dnf_manager) == total_size - - # Found a mount point for download and install. - # The total size is required. - free_space_getter.return_value = { - "/mnt/sysroot/home": total_size - } - assert calculate_required_space(dnf_manager) == total_size - - # Found a mount point for download. - # The installation size is required. - free_space_getter.return_value = { - "/var/tmp": download_size - } - assert calculate_required_space(dnf_manager) == installation_size - - # The biggest mount point can be used for download and install. - # The total size is required. - free_space_getter.return_value = { - "/var/tmp": download_size, - "/mnt/sysroot": total_size - } - assert calculate_required_space(dnf_manager) == total_size - def test_collect_installation_devices(self): """Test the collect_installation_devices function.""" devices = collect_installation_devices([], []) diff --git a/tests/unit_tests/pyanaconda_tests/modules/payloads/test_module_payload_base_utils.py b/tests/unit_tests/pyanaconda_tests/modules/payloads/test_module_payload_base_utils.py index 0bb5cf3a7f1..240160b9ac4 100644 --- a/tests/unit_tests/pyanaconda_tests/modules/payloads/test_module_payload_base_utils.py +++ b/tests/unit_tests/pyanaconda_tests/modules/payloads/test_module_payload_base_utils.py @@ -15,9 +15,23 @@ # License and may only be used or replicated with the express permission of # Red Hat, Inc. # +from textwrap import dedent from unittest.case import TestCase +from unittest.mock import Mock, patch -from pyanaconda.modules.payloads.base.utils import sort_kernel_version_list +import pytest +from blivet.size import Size + +from pyanaconda.modules.common.constants.objects import DEVICE_TREE +from pyanaconda.modules.common.constants.services import STORAGE +from pyanaconda.modules.payloads.base.utils import ( + _pick_mount_points, + calculate_required_space, + get_free_space_map, + pick_download_location, + sort_kernel_version_list, +) +from tests.unit_tests.pyanaconda_tests import patch_dbus_get_proxy_with_cache class PayloadBaseUtilsTest(TestCase): @@ -72,3 +86,214 @@ def test_sort_kernel_version_list(self): '5.9.8-200.fc33.x86_64', '5.10.0-0.rc4.78.fc34.x86_64' ] + + @patch("pyanaconda.modules.payloads.base.utils.get_free_space_map") + def test_pick_download_location(self, free_space_getter): + """Test the pick_download_location function.""" + download_size = Size(100) + installation_size = Size(200) + total_size = Size(300) + + # Found mount points for download and install. + # Don't use /mnt/sysroot if possible. + free_space_getter.return_value = { + "/var/tmp": download_size, + "/mnt/sysroot": total_size, + } + + path = pick_download_location(download_size, installation_size, "TEST_SUFFIX") + assert path == "/var/tmp/TEST_SUFFIX" + + # Found mount points only for download. + # Use the biggest mount point. + free_space_getter.return_value = { + "/mnt/sysroot/tmp": download_size + 1, + "/mnt/sysroot/home": download_size, + } + + path = pick_download_location(download_size, installation_size, "TEST_SUFFIX") + assert path == "/mnt/sysroot/tmp/TEST_SUFFIX" + + # No mount point to use. + # Fail with an exception. + free_space_getter.return_value = {} + + with pytest.raises(RuntimeError) as cm: + pick_download_location(download_size, installation_size, "TEST_SUFFIX") + + msg = "Not enough disk space to download the packages; size 100 B." + assert str(cm.value) == msg + + @patch("pyanaconda.modules.payloads.base.utils.execWithCapture") + def test_get_free_space(self, exec_mock): + """Test the get_free_space function.""" + output = """ + Mounted on Avail + /dev 100 + /dev/shm 200 + /run 300 + / 400 + /tmp 500 + /boot 600 + /home 700 + /boot/efi 800 + """ + exec_mock.return_value = dedent(output).strip() + + assert get_free_space_map() == { + '/dev': Size("100 KiB"), + '/dev/shm': Size("200 KiB"), + '/run': Size("300 KiB"), + '/': Size("400 KiB"), + '/tmp': Size("500 KiB"), + '/boot': Size("600 KiB"), + '/home': Size("700 KiB"), + '/boot/efi': Size("800 KiB"), + } + + @patch("os.statvfs") + @patch("pyanaconda.modules.payloads.base.utils.conf") + @patch("pyanaconda.modules.payloads.base.utils.execWithCapture") + def test_get_free_space_image(self, exec_mock, conf_mock, statvfs_mock): + """Test the get_free_space function.""" + output = """ + Mounted on Avail + / 100 + /boot 200 + """ + exec_mock.return_value = dedent(output).strip() + conf_mock.target.is_hardware = False + statvfs_mock.return_value = Mock(f_frsize=1024, f_bfree=300) + + assert get_free_space_map() == { + '/': Size("100 KiB"), + '/boot': Size("200 KiB"), + '/var/tmp': Size("300 KiB"), + } + + def test_pick_mount_points(self): + """Test the _pick_mount_points function.""" + mount_points = { + "/": Size("1 G"), + "/home": Size("1 G"), + "/var/tmp": Size("1 G"), + "/mnt/sysroot": Size("1 G"), + "/mnt/sysroot/home": Size("1 G"), + "/mnt/sysroot/tmp": Size("1 G"), + "/mnt/sysroot/var": Size("1 G"), + "/mnt/sysroot/usr": Size("1 G"), + + } + + # All mount points are big enough. + # Choose all suitable mount points. + sufficient = _pick_mount_points( + mount_points, + download_size=Size("0.5 G"), + install_size=Size("0.5 G") + ) + assert sufficient == { + "/var/tmp", + "/mnt/sysroot", + "/mnt/sysroot/home", + "/mnt/sysroot/tmp", + "/mnt/sysroot/var" + } + + # No mount point is big enough for installation. + # Choose non-sysroot mount points for download. + sufficient = _pick_mount_points( + mount_points, + download_size=Size("0.5 G"), + install_size=Size("1.5 G") + ) + assert sufficient == { + "/var/tmp", + } + + # No mount point is big enough for installation or download. + sufficient = _pick_mount_points( + mount_points, + download_size=Size("1.5 G"), + install_size=Size("1.5 G") + ) + assert sufficient == set() + + @patch("pyanaconda.modules.payloads.base.utils.execWithCapture") + @patch_dbus_get_proxy_with_cache + def test_get_combined_free_space(self, proxy_getter, exec_mock): + """Test the get_free_space function with the combined options.""" + output = """ + Mounted on Avail + / 100 + /tmp 200 + """ + exec_mock.return_value = dedent(output).strip() + + mount_points = { + '/': Size("300 KiB"), + '/boot': Size("400 KiB"), + } + + def get_mount_points(): + return list(mount_points.keys()) + + def get_free_space(paths): + return sum(map(mount_points.get, paths)) + + device_tree = STORAGE.get_proxy(DEVICE_TREE) + device_tree.GetMountPoints.side_effect = get_mount_points + device_tree.GetFileSystemFreeSpace.side_effect = get_free_space + + assert get_free_space_map(current=True, scheduled=False) == { + '/': Size("100 KiB"), + '/tmp': Size("200 KiB"), + } + + assert get_free_space_map(current=False, scheduled=True) == { + '/mnt/sysroot': Size("300 KiB"), + '/mnt/sysroot/boot': Size("400 KiB"), + } + + assert get_free_space_map(current=True, scheduled=True) == { + '/': Size("100 KiB"), + '/tmp': Size("200 KiB"), + '/mnt/sysroot': Size("300 KiB"), + '/mnt/sysroot/boot': Size("400 KiB"), + } + + assert get_free_space_map(current=False, scheduled=False) == {} + + @patch("pyanaconda.modules.payloads.base.utils.get_free_space_map") + def test_calculate_required_space(self, free_space_getter): + """Test the calculate_required_space function.""" + download_size = Size(100) + installation_size = Size(200) + total_size = Size(300) + + # No mount point to use. + # The total size is required. + free_space_getter.return_value = {} + assert calculate_required_space(download_size, installation_size) == total_size + + # Found a mount point for download and install. + # The total size is required. + free_space_getter.return_value = { + "/mnt/sysroot/home": total_size + } + assert calculate_required_space(download_size, installation_size) == total_size + + # Found a mount point for download. + # The installation size is required. + free_space_getter.return_value = { + "/var/tmp": download_size + } + assert calculate_required_space(download_size, installation_size) == installation_size + + # The biggest mount point can be used for download and install. + # The total size is required. + free_space_getter.return_value = { + "/var/tmp": download_size, + "/mnt/sysroot": total_size + } + assert calculate_required_space(download_size, installation_size) == total_size From dd2fdc8e397a10a8aa4eb17c54ce66d66e1d2035 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Thu, 7 Nov 2024 07:04:45 -0500 Subject: [PATCH 03/17] anaconda.spec: Add Flatpak client to Requires This is needed for the flatpak-oci-authenticator daemon, which is needed for all installs from OCI remotes, whether we are authenticating to them or not. --- anaconda.spec.in | 1 + 1 file changed, 1 insertion(+) diff --git a/anaconda.spec.in b/anaconda.spec.in index 18f99333363..5571487b6d7 100644 --- a/anaconda.spec.in +++ b/anaconda.spec.in @@ -113,6 +113,7 @@ Requires: python3-systemd Requires: python3-productmd Requires: python3-dasbus >= %{dasbusver} Requires: python3-xkbregistry +Requires: flatpak Requires: flatpak-libs %if %{defined rhel} && %{undefined centos} Requires: subscription-manager >= %{subscriptionmanagerver} From 4138b6a8d34fe34163ed9b0b4fc788cc53a56575 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Wed, 6 Nov 2024 16:27:55 -0500 Subject: [PATCH 04/17] Support Flatpak preinstallation as part of a DNF install Add new functionality for running Flatpak preinstallation after installing the base system. It's initially implemented only for the DNF payload. Flatpak installation is done as an extra "side" payload that the payloads service calls after the base payload to install additional content. The base payload provides the list of Flatpak refs that should be installed from the side payload and the Flatpaks are installed from a location that is determined from the base payload's primary source - if the source is a local or remote install tree we look for a OCI image layout there, if the base payload is the network (CDN, closest mirror), we install directly from the configured Flatpak remote. For http/ftp payloads we mirror the OCI image layout locally before runing the installation. --- configure.ac | 1 + pyanaconda/core/constants.py | 1 + .../modules/common/constants/interfaces.py | 5 + pyanaconda/modules/payloads/constants.py | 2 + .../modules/payloads/payload/Makefile.am | 2 +- .../modules/payloads/payload/dnf/dnf.py | 10 + .../payloads/payload/dnf/dnf_manager.py | 15 + .../modules/payloads/payload/factory.py | 4 + .../payloads/payload/flatpak/Makefile.am | 21 + .../payloads/payload/flatpak/__init__.py | 0 .../payloads/payload/flatpak/flatpak.py | 116 +++++ .../payload/flatpak/flatpak_interface.py | 28 ++ .../payload/flatpak/flatpak_manager.py | 271 +++++++++++ .../payloads/payload/flatpak/installation.py | 144 ++++++ .../payloads/payload/flatpak/source.py | 434 ++++++++++++++++++ .../modules/payloads/payload/payload_base.py | 14 + pyanaconda/modules/payloads/payloads.py | 27 ++ .../payloads/source/flatpak/flatpak.py | 17 + .../payload/test_module_payload_dnf.py | 8 +- .../modules/payloads/test_module_payloads.py | 5 +- 20 files changed, 1121 insertions(+), 4 deletions(-) create mode 100644 pyanaconda/modules/payloads/payload/flatpak/Makefile.am create mode 100644 pyanaconda/modules/payloads/payload/flatpak/__init__.py create mode 100644 pyanaconda/modules/payloads/payload/flatpak/flatpak.py create mode 100644 pyanaconda/modules/payloads/payload/flatpak/flatpak_interface.py create mode 100644 pyanaconda/modules/payloads/payload/flatpak/flatpak_manager.py create mode 100644 pyanaconda/modules/payloads/payload/flatpak/installation.py create mode 100644 pyanaconda/modules/payloads/payload/flatpak/source.py diff --git a/configure.ac b/configure.ac index 070342d393d..1c279384ab7 100644 --- a/configure.ac +++ b/configure.ac @@ -165,6 +165,7 @@ AC_CONFIG_FILES([Makefile pyanaconda/modules/payloads/Makefile pyanaconda/modules/payloads/payload/Makefile pyanaconda/modules/payloads/payload/dnf/Makefile + pyanaconda/modules/payloads/payload/flatpak/Makefile pyanaconda/modules/payloads/payload/live_os/Makefile pyanaconda/modules/payloads/payload/live_image/Makefile pyanaconda/modules/payloads/payload/rpm_ostree/Makefile diff --git a/pyanaconda/core/constants.py b/pyanaconda/core/constants.py index fbc81a78d2b..0c1d955d460 100644 --- a/pyanaconda/core/constants.py +++ b/pyanaconda/core/constants.py @@ -394,6 +394,7 @@ class DisplayModes(Enum): # Types of the payload. PAYLOAD_TYPE_DNF = "DNF" +PAYLOAD_TYPE_FLATPAK = "FLATPAK" PAYLOAD_TYPE_LIVE_OS = "LIVE_OS" PAYLOAD_TYPE_LIVE_IMAGE = "LIVE_IMAGE" PAYLOAD_TYPE_RPM_OSTREE = "RPM_OSTREE" diff --git a/pyanaconda/modules/common/constants/interfaces.py b/pyanaconda/modules/common/constants/interfaces.py index f8f25eb3a00..6989e3e98b7 100644 --- a/pyanaconda/modules/common/constants/interfaces.py +++ b/pyanaconda/modules/common/constants/interfaces.py @@ -74,6 +74,11 @@ basename="DNF" ) +PAYLOAD_FLATPAK = DBusInterfaceIdentifier( + namespace=PAYLOAD_NAMESPACE, + basename="FLATPAK" +) + PAYLOAD_LIVE_IMAGE = DBusInterfaceIdentifier( namespace=PAYLOAD_NAMESPACE, basename="LiveImage" diff --git a/pyanaconda/modules/payloads/constants.py b/pyanaconda/modules/payloads/constants.py index eaf84cd0d1b..6b21942c879 100644 --- a/pyanaconda/modules/payloads/constants.py +++ b/pyanaconda/modules/payloads/constants.py @@ -20,6 +20,7 @@ from pyanaconda.core.constants import ( PAYLOAD_TYPE_DNF, + PAYLOAD_TYPE_FLATPAK, PAYLOAD_TYPE_LIVE_IMAGE, PAYLOAD_TYPE_LIVE_OS, PAYLOAD_TYPE_RPM_OSTREE, @@ -51,6 +52,7 @@ class PayloadType(Enum): """Type of the payload.""" DNF = PAYLOAD_TYPE_DNF + FLATPAK = PAYLOAD_TYPE_FLATPAK LIVE_OS = PAYLOAD_TYPE_LIVE_OS LIVE_IMAGE = PAYLOAD_TYPE_LIVE_IMAGE RPM_OSTREE = PAYLOAD_TYPE_RPM_OSTREE diff --git a/pyanaconda/modules/payloads/payload/Makefile.am b/pyanaconda/modules/payloads/payload/Makefile.am index 5bc389bfacf..26906157cda 100644 --- a/pyanaconda/modules/payloads/payload/Makefile.am +++ b/pyanaconda/modules/payloads/payload/Makefile.am @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -SUBDIRS = dnf live_os live_image rpm_ostree +SUBDIRS = dnf flatpak live_os live_image rpm_ostree pkgpyexecdir = $(pyexecdir)/py$(PACKAGE_NAME) dnf_moduledir = $(pkgpyexecdir)/modules/payloads/payload diff --git a/pyanaconda/modules/payloads/payload/dnf/dnf.py b/pyanaconda/modules/payloads/payload/dnf/dnf.py index 9d6246202b2..b84de3535cb 100644 --- a/pyanaconda/modules/payloads/payload/dnf/dnf.py +++ b/pyanaconda/modules/payloads/payload/dnf/dnf.py @@ -392,6 +392,16 @@ def calculate_required_space(self): self._dnf_manager.get_installation_size()) return required_space.get_bytes() + def needs_flatpak_side_payload(self): + return True + + def get_flatpak_refs(self): + """Get the list of Flatpak refs to install. + + :return: list of Flatpak refs + """ + return self._dnf_manager.get_flatpak_refs() + def get_repo_configurations(self): """Get RepoConfiguration structures for all sources. diff --git a/pyanaconda/modules/payloads/payload/dnf/dnf_manager.py b/pyanaconda/modules/payloads/payload/dnf/dnf_manager.py index 44860bfe534..5567d9434ce 100644 --- a/pyanaconda/modules/payloads/payload/dnf/dnf_manager.py +++ b/pyanaconda/modules/payloads/payload/dnf/dnf_manager.py @@ -18,6 +18,7 @@ # Red Hat, Inc. # import multiprocessing +import re import shutil import threading import traceback @@ -598,6 +599,20 @@ def resolve_selection(self): log.info("The software selection has been resolved (%d packages selected).", len(self._base.transaction)) + def get_flatpak_refs(self): + """Determine what Flatpaks need to be preinstalled based on resolved transaction""" + if self._base.transaction is None: + return [] + + refs = [] + for tsi in self._base.transaction: + for provide in tsi.pkg.provides: + m = re.match(r"^flatpak-preinstall\((.*)\)$", str(provide)) + if m: + refs.append(m.group(1)) + + return refs + def clear_selection(self): """Clear the software selection.""" self._base.reset(goal=True) diff --git a/pyanaconda/modules/payloads/payload/factory.py b/pyanaconda/modules/payloads/payload/factory.py index dc85be577ad..d1ff99e40ba 100644 --- a/pyanaconda/modules/payloads/payload/factory.py +++ b/pyanaconda/modules/payloads/payload/factory.py @@ -52,6 +52,10 @@ def create_payload(payload_type: PayloadType): ) return RPMOSTreeModule() + if payload_type == PayloadType.FLATPAK: + from pyanaconda.modules.payloads.payload.flatpak.flatpak import FlatpakModule + return FlatpakModule() + raise ValueError("Unknown payload type: {}".format(payload_type)) @classmethod diff --git a/pyanaconda/modules/payloads/payload/flatpak/Makefile.am b/pyanaconda/modules/payloads/payload/flatpak/Makefile.am new file mode 100644 index 00000000000..06a09ec2715 --- /dev/null +++ b/pyanaconda/modules/payloads/payload/flatpak/Makefile.am @@ -0,0 +1,21 @@ +# +# Copyright (C) 2019 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +pkgpyexecdir = $(pyexecdir)/py$(PACKAGE_NAME) +flatpak_moduledir = $(pkgpyexecdir)/modules/payloads/payload/flatpak +dist_flatpak_module_DATA = $(wildcard $(srcdir)/*.py) + +MAINTAINERCLEANFILES = Makefile.in diff --git a/pyanaconda/modules/payloads/payload/flatpak/__init__.py b/pyanaconda/modules/payloads/payload/flatpak/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyanaconda/modules/payloads/payload/flatpak/flatpak.py b/pyanaconda/modules/payloads/payload/flatpak/flatpak.py new file mode 100644 index 00000000000..ea1c4717d90 --- /dev/null +++ b/pyanaconda/modules/payloads/payload/flatpak/flatpak.py @@ -0,0 +1,116 @@ +# +# Payload module for preinstalling Flatpaks +# +# Copyright (C) 2024 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# +from pyanaconda.anaconda_loggers import get_module_logger +from pyanaconda.modules.payloads.base.utils import calculate_required_space +from pyanaconda.modules.payloads.constants import PayloadType, SourceType +from pyanaconda.modules.payloads.payload.flatpak.flatpak_interface import FlatpakInterface +from pyanaconda.modules.payloads.payload.flatpak.flatpak_manager import FlatpakManager +from pyanaconda.modules.payloads.payload.flatpak.installation import ( + CalculateFlatpaksSizeTask, + CleanUpDownloadLocationTask, + DownloadFlatpaksTask, + InstallFlatpaksTask, + PrepareDownloadLocationTask, +) +from pyanaconda.modules.payloads.payload.payload_base import PayloadBase + +log = get_module_logger(__name__) + + +class FlatpakModule(PayloadBase): + """The Flatpak payload module.""" + + def __init__(self): + super().__init__() + self._flatpak_manager = FlatpakManager() + + def for_publication(self): + """Get the interface used to publish this source.""" + return FlatpakInterface(self) + + @property + def type(self): + """Type of this payload.""" + return PayloadType.FLATPAK + + @property + def default_source_type(self): + """Type of the default source.""" + return None + + @property + def supported_source_types(self): + """List of supported source types.""" + # Include all the types of SourceType. + # FIXME: Flatpak needs it's own source because this way it needs to understand + # all existing and future ones + return list(SourceType) + + def set_sources(self, sources): + """Set a new list of sources to this payload. + + This overrides the base implementation since the sources we set here + are the sources from the main payload, and can already be initialized. + + :param sources: set a new sources + :type sources: instance of pyanaconda.modules.payloads.source.source_base.PayloadSourceBase + """ + self._sources = sources + self._flatpak_manager.set_sources(sources) + self.sources_changed.emit() + + def set_flatpak_refs(self, refs): + """Set the flatpak refs. + + :param refs: a list of flatpak refs + """ + self._flatpak_manager.set_flatpak_refs(refs) + + def calculate_required_space(self): + """Calculate space required for the installation. + + :return: required size in bytes + :rtype: int + """ + return calculate_required_space(self._flatpak_manager.download_size, + self._flatpak_manager.install_size) + + def install_with_tasks(self): + """Install the payload with tasks.""" + + tasks = [ + CalculateFlatpaksSizeTask( + flatpak_manager=self._flatpak_manager, + ), + PrepareDownloadLocationTask( + flatpak_manager=self._flatpak_manager, + ), + DownloadFlatpaksTask( + flatpak_manager=self._flatpak_manager, + ), + InstallFlatpaksTask( + flatpak_manager=self._flatpak_manager, + ), + CleanUpDownloadLocationTask( + flatpak_manager=self._flatpak_manager, + ), + ] + + return tasks diff --git a/pyanaconda/modules/payloads/payload/flatpak/flatpak_interface.py b/pyanaconda/modules/payloads/payload/flatpak/flatpak_interface.py new file mode 100644 index 00000000000..2904b829a1e --- /dev/null +++ b/pyanaconda/modules/payloads/payload/flatpak/flatpak_interface.py @@ -0,0 +1,28 @@ +# +# DBus interface for Flatpak payload. +# +# Copyright (C) 2024 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# +from dasbus.server.interface import dbus_interface + +from pyanaconda.modules.common.constants.interfaces import PAYLOAD_FLATPAK +from pyanaconda.modules.payloads.payload.payload_base_interface import PayloadBaseInterface + + +@dbus_interface(PAYLOAD_FLATPAK.interface_name) +class FlatpakInterface(PayloadBaseInterface): + """DBus interface for Flatpak payload module.""" diff --git a/pyanaconda/modules/payloads/payload/flatpak/flatpak_manager.py b/pyanaconda/modules/payloads/payload/flatpak/flatpak_manager.py new file mode 100644 index 00000000000..3e368e355ae --- /dev/null +++ b/pyanaconda/modules/payloads/payload/flatpak/flatpak_manager.py @@ -0,0 +1,271 @@ +# +# Root object for handling Flatpak pre-installation +# +# Copyright (C) 2024 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# + + +from typing import List, Optional + +import gi + +from pyanaconda.anaconda_loggers import get_module_logger +from pyanaconda.core.configuration.anaconda import conf +from pyanaconda.core.glib import GError +from pyanaconda.core.i18n import _ +from pyanaconda.modules.common.errors.installation import PayloadInstallationError +from pyanaconda.modules.common.task.progress import ProgressReporter +from pyanaconda.modules.payloads.constants import SourceType +from pyanaconda.modules.payloads.payload.flatpak.source import ( + FlatpakRegistrySource, + FlatpakStaticSource, + NoSourceError, +) +from pyanaconda.modules.payloads.source.source_base import PayloadSourceBase, RepositorySourceMixin + +gi.require_version("Flatpak", "1.0") +gi.require_version("Gio", "2.0") + +from gi.repository.Flatpak import Installation, Transaction, TransactionOperationType + +log = get_module_logger(__name__) + +__all__ = ["FlatpakManager"] + + +class FlatpakManager: + """Root object for handling Flatpak pre-installation""" + + def __init__(self): + """Create and initialize this class. + + :param function callback: a progress reporting callback + """ + self._flatpak_refs = [] + self._source_repository = None + self._source = None + self._skip_installation = False + self._collection_location = None + self._progress: Optional[ProgressReporter] = None + self._transaction = None + self._download_location = None + self._download_size = 0 + self._install_size = 0 + + def set_sources(self, sources: List[PayloadSourceBase]): + """Set the source object we use to download Flatpak content. + + If unset, pre-installation will install directly from the configured + Flatpak remote (see flatpak_remote in the anaconda configuration). + + :param str url: URL pointing to the Flatpak content + """ + + if not sources: + return + + source = sources[0] + + if isinstance(source, RepositorySourceMixin): + if self._source and isinstance(self._source, FlatpakStaticSource) \ + and self._source.repository_config == source.repository: + return + self._source = FlatpakStaticSource(source.repository, relative_path="Flatpaks") + elif source.type in (SourceType.CDN, SourceType.CLOSEST_MIRROR): + if self._source and isinstance(self._source, FlatpakRegistrySource): + return + _, remote_url = conf.payload.flatpak_remote + log.debug("Using Flatpak registry source: %s", remote_url) + self._source = FlatpakRegistrySource(remote_url) + else: + self._source = None + + def set_flatpak_refs(self, refs: Optional[List[str]]): + """Set the Flatpak refs to be installed. + + :param refs: List of Flatpak refs to be installed, None to use + all Flatpak refs from the source. Each ref should be in the form + [:](app|runtime)//[]/ + """ + self._skip_installation = False + self._flatpak_refs = refs if refs is not None else [] + + def set_download_location(self, path: str): + """Sets a location that can be used for temporary download of Flatpak content. + + :param path: parent directory to store downloaded Flatpak content + (the download should be to a subdirectory of this path) + """ + self._download_location = path + + @property + def download_location(self) -> str: + """Get the download location.""" + return self._download_location + + def _get_source(self): + if self._source is None: + if self._source_repository: + log.debug("Using Flatpak source repository at: %s/Flatpaks", + self._source_repository.url) + self._source = FlatpakStaticSource(self._source_repository, + relative_path="Flatpaks") + else: + _, remote_url = conf.payload.flatpak_remote + log.debug("Using Flatpak registry source: %s", remote_url) + self._source = FlatpakRegistrySource(remote_url) + + return self._source + + def calculate_size(self): + """Calculate the download and install size of the Flatpak content. + + :param progress: used to report progress of the operation + + The result is available from the download_size and install_size properties. + """ + if self._skip_installation or len(self._flatpak_refs) == 0: + return + + try: + self._download_size, self._install_size = \ + self._get_source().calculate_size(self._flatpak_refs) + except NoSourceError as e: + log.error("Flatpak source not available, skipping installing %s: %s", + ", ".join(self._flatpak_refs), e) + self._skip_installation = True + + @property + def download_size(self): + """Space needed to to temporarily download Flatpak content before installation""" + return self._download_size + + @property + def install_size(self): + """Space used after installation in the target system""" + return self._install_size + + def download(self, progress: ProgressReporter): + """Download Flatpak content to a temporary location. + + :param progress: used to report progress of the operation + + This is only needed if Flatpak can't install the content directly. + """ + if self._skip_installation or len(self._flatpak_refs) == 0: + return + + try: + self._collection_location = self._get_source().download(self._flatpak_refs, + self._download_location, + progress) + except NoSourceError as e: + log.error("Flatpak source not available, skipping installing %s: %s", + ", ".join(self._flatpak_refs), e) + self._skip_installation = True + + def install(self, progress: ProgressReporter): + """Install the Flatpak content to the target system. + + :param progress: used to report progress of the operation + """ + if self._skip_installation or len(self._flatpak_refs) == 0: + return + + installation = self._create_flatpak_installation() + self._transaction = self._create_flatpak_transaction(installation) + + if self._collection_location: + self._transaction.add_sideload_image_collection(self._collection_location, None) + + self._transaction.add_sync_preinstalled() + + try: + self._progress = progress + self._transaction.run() + except GError as e: + raise PayloadInstallationError("Failed to install flatpaks: {}".format(e)) from e + finally: + self._transaction.run_dispose() + self._transaction = None + self._progress = None + + def _create_flatpak_installation(self): + return Installation.new_system(None) + + def _create_flatpak_transaction(self, installation): + transaction = Transaction.new_for_installation(installation) + transaction.connect("new_operation", self._operation_started_callback) + transaction.connect("operation_done", self._operation_stopped_callback) + transaction.connect("operation_error", self._operation_error_callback) + + return transaction + + def _operation_started_callback(self, transaction, operation, progress): + """Start of the new operation. + + :param transaction: the main transaction object + :type transaction: Flatpak.Transaction instance + :param operation: object describing the operation + :type operation: Flatpak.TransactionOperation instance + :param progress: object providing progress of the operation + :type progress: Flatpak.TransactionProgress instance + """ + self._log_operation(operation, "started") + self._report_progress(_("Installing {}").format(operation.get_ref())) + + def _operation_stopped_callback(self, transaction, operation, _commit, result): + """Existing operation ended. + + :param transaction: the main transaction object + :type transaction: Flatpak.Transaction instance + :param operation: object describing the operation + :type operation: Flatpak.TransactionOperation instance + :param str _commit: operation was committed this is a commit id + :param result: object containing details about the result of the operation + :type result: Flatpak.TransactionResult instance + """ + self._log_operation(operation, "stopped") + + def _operation_error_callback(self, transaction, operation, error, details): + """Process error raised by the flatpak operation. + + :param transaction: the main transaction object + :type transaction: Flatpak.Transaction instance + :param operation: object describing the operation + :type operation: Flatpak.TransactionOperation instance + :param error: object containing error description + :type error: GLib.Error instance + :param details: information if the error was fatal + :type details: int value of Flatpak.TransactionErrorDetails + """ + self._log_operation(operation, "failed") + log.error("Flatpak operation has failed with a message: '%s'", error.message) + + def _report_progress(self, message): + """Report a progress message.""" + if not self._progress: + return + + self._progress.report_progress(message) + + @staticmethod + def _log_operation(operation, state): + """Log a Flatpak operation.""" + operation_type_str = TransactionOperationType.to_string(operation.get_operation_type()) + log.debug("Flatpak operation: %s of ref %s state %s", + operation_type_str, operation.get_ref(), state) diff --git a/pyanaconda/modules/payloads/payload/flatpak/installation.py b/pyanaconda/modules/payloads/payload/flatpak/installation.py new file mode 100644 index 00000000000..7c20cd6d044 --- /dev/null +++ b/pyanaconda/modules/payloads/payload/flatpak/installation.py @@ -0,0 +1,144 @@ +# +# Copyright (C) 2024 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# + +import os +import shutil + +from pyanaconda.anaconda_loggers import get_module_logger +from pyanaconda.modules.common.task import Task +from pyanaconda.modules.payloads.base.utils import pick_download_location +from pyanaconda.modules.payloads.payload.flatpak.flatpak_manager import FlatpakManager + +log = get_module_logger(__name__) + +FLATPAK_MIRROR_DIR_SUFFIX = 'flatpak.mirror' + + +class CalculateFlatpaksSizeTask(Task): + """Task to determine space needed for Flatpaks""" + + def __init__(self, flatpak_manager: FlatpakManager): + """Create a new task.""" + super().__init__() + self._flatpak_manager = flatpak_manager + + @property + def name(self): + """Name of the task.""" + return "Calculate needed space for Flatpaks" + + def run(self): + self._flatpak_manager.calculate_size() + + +class PrepareDownloadLocationTask(Task): + """The installation task for setting up the download location.""" + + def __init__(self, flatpak_manager: FlatpakManager): + """Create a new task. + + :param dnf_manager: a DNF manager + """ + super().__init__() + self._flatpak_manager = flatpak_manager + + @property + def name(self): + return "Prepare the package download" + + def run(self): + """Run the task. + + :return: a path of the download location + """ + + self._flatpak_manager.calculate_size() + + path = pick_download_location(self._flatpak_manager.download_size, + self._flatpak_manager.install_size, + FLATPAK_MIRROR_DIR_SUFFIX) + + if os.path.exists(path): + log.info("Removing existing package download location: %s", path) + shutil.rmtree(path) + + self._flatpak_manager.set_download_location(path) + return path + + +class CleanUpDownloadLocationTask(Task): + """The installation task for cleaning up the download location.""" + + def __init__(self, flatpak_manager): + """Create a new task. + + :param flatpak_manager: a Flatpak manager + """ + super().__init__() + self._flatpak_manager = flatpak_manager + + @property + def name(self): + return "Remove downloaded Flatpaks" + + def run(self): + """Run the task.""" + path = self._flatpak_manager.download_location + + if not os.path.exists(path): + # If nothing was downloaded, there is nothing to clean up. + return + + log.info("Removing downloaded packages from %s.", path) + shutil.rmtree(path) + + +class DownloadFlatpaksTask(Task): + """Task to download remote Flatpaks""" + + def __init__(self, flatpak_manager): + """Create a new task.""" + super().__init__() + self._flatpak_manager = flatpak_manager + + @property + def name(self): + """Name of the task.""" + return "Download remote Flatpaks" + + def run(self): + """Run the task.""" + self._flatpak_manager.download(self) + + +class InstallFlatpaksTask(Task): + """Task to install flatpaks""" + + def __init__(self, flatpak_manager): + """Create a new task.""" + super().__init__() + self._flatpak_manager = flatpak_manager + + @property + def name(self): + """Name of the task.""" + return "Install Flatpaks" + + def run(self): + """Run the task.""" + self._flatpak_manager.install(self) diff --git a/pyanaconda/modules/payloads/payload/flatpak/source.py b/pyanaconda/modules/payloads/payload/flatpak/source.py new file mode 100644 index 00000000000..454abefca28 --- /dev/null +++ b/pyanaconda/modules/payloads/payload/flatpak/source.py @@ -0,0 +1,434 @@ +# +# Query and download sources of Flatpak content +# +# Copyright (C) 2024 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# + +import json +import os +from abc import ABC, abstractmethod +from configparser import ConfigParser, NoSectionError +from contextlib import contextmanager +from functools import cached_property +from typing import Dict, List, Optional, Tuple +from urllib.parse import urljoin, urlparse + +from blivet.arch import get_arch + +from pyanaconda.anaconda_loggers import get_module_logger +from pyanaconda.core.i18n import _ +from pyanaconda.core.util import requests_session +from pyanaconda.modules.common.structures.payload import RepoConfigurationData +from pyanaconda.modules.common.task.progress import ProgressReporter +from pyanaconda.modules.payloads.base.utils import get_downloader_for_repo_configuration + +log = get_module_logger(__name__) + +__all__ = ["FlatpakRegistrySource", "FlatpakSource", "FlatpakStaticSource", "NoSourceError"] + + +_CONTAINER_ARCH_MAP = { + "x86_64": "amd64", + "aarch64": "arm64" +} + + +def _get_container_arch(): + """Architecture name as used by docker/podman""" + arch = get_arch() + return _CONTAINER_ARCH_MAP.get(arch, arch) + + +def _canonicalize_flatpak_ref(ref) -> Tuple[Optional[str], str]: + """Split off a collection ID, and add architecture if unspecified + + Turn "org.fedoraproject.Stable:app/org.example.Foo//stable" into + ("org.fedoraproject.Stable", "app/org.example.Foo/amd64/stable") + """ + + collection_parts = ref.split(":", 1) + if len(collection_parts) == 2: + collection = collection_parts[0] + ref = collection_parts[1] + else: + collection = None + + parts = ref.split("/") + if len(parts) != 4: + raise RuntimeError("Can't parse reference") + if parts[2] == "": + parts[2] = get_arch() + + return collection, "/".join(parts) + + +class NoSourceError(Exception): + """Source not found.""" + + +class SourceImage(ABC): + """Representation of a single image of a FlatpakSource.""" + + @property + @abstractmethod + def labels(self) -> Dict[str, str]: + """The labels of the image.""" + + @property + def ref(self) -> Optional[str]: + """Flatpak reference for the image, or None if not a Flatpak""" + return self.labels.get("org.flatpak.ref") + + @property + def download_size(self) -> int: + """Download size, in bytes""" + return int(self.labels["org.flatpak.download-size"]) + + @property + def installed_size(self) -> int: + """Installed size, in bytes""" + return int(self.labels["org.flatpak.installed-size"]) + + +class FlatpakSource(ABC): + """Base class for places where Flatpak images can be downloaded from.""" + + @abstractmethod + def calculate_size(self, refs: List[str]) -> Tuple[int, int]: + """Calculate the total download and installed size of the images in refs and + their dependencies. + + :param refs: list of Flatpak references + :returns: download size, installed size + """ + + @abstractmethod + def download(self, refs: List[str], download_location: str, + progress: Optional[ProgressReporter] = None) -> Optional[str]: + """Downloads the images referenced by refs and any dependencies. + + If they are already local, or they can be installed + directly from the remote location, nothing will be downloaded. + + Whether or not anything as been downloaded, returns + the specification of a sideload repository that can be used to install from + this source, or None if none is needed. + + :param refs: list of Flatpak references + :param download_location: path to location for temporary downloads + :param progress: used to report progress of the download + :returns sideload location, including the transport (e.g. oci:), or None + """ + + @property + @abstractmethod + def _images(self) -> List[SourceImage]: + """All images in the source, filtered for the current architecture.""" + + def _expand_refs(self, refs: List[str]) -> List[str]: + """Expand the list of refs to be in full form and include any dependencies.""" + result = [] + for ref in refs: + # We don't do anything with the collection ID for now + _, ref = _canonicalize_flatpak_ref(ref) + result.append(ref) + + for image in self._images: + if image.ref not in result: + continue + + metadata = image.labels.get("org.flatpak.metadata") + if metadata is None: + continue + + cp = ConfigParser(interpolation=None) + cp.read_string(metadata) + try: + runtime = cp.get('Application', 'Runtime') + if runtime: + runtime_ref = f"runtime/{runtime}" + if runtime_ref not in result: + result.append(runtime_ref) + except (NoSectionError, KeyError): + pass + + return result + + +class StaticSourceImage(SourceImage): + """One image of a FlatpakStaticSource.""" + + def __init__(self, digest, manifest_json, config_json): + self.digest = digest + self.manifest_json = manifest_json + self.config_json = config_json + + @property + def labels(self): + return self.config_json["config"]["Labels"] + + @property + def download_size(self): + # This is more accurate than using the org.flatpak.download-size label, + # because further processing of the image might have recompressed + # the layer using different settings. + return sum(int(layer["size"]) for layer in self.manifest_json["layers"]) + + +class FlatpakStaticSource(FlatpakSource): + """Flatpak images stored in a OCI image layout, either locally or remotely + + https://github.com/opencontainers/image-spec/blob/main/image-layout.md + """ + + def __init__(self, repository_config: RepoConfigurationData, relative_path: str = "Flatpaks"): + """Create a new source. + + :param repository_config: URL of the repository, or a local path + :param relative_path: path of an OCI layout, relative to the repository root + """ + self.repository_config = repository_config + self._url = urljoin(repository_config.url + "/", relative_path) + self._is_local = self._url.startswith("file://") + self._cached_blobs = {} + + @contextmanager + def _downloader(self): + """Prepare a requests.Session.get method appropriately for the repository. + + :returns: a function that acts like requests.Session.get() + """ + with requests_session() as session: + downloader = get_downloader_for_repo_configuration(session, self.repository_config) + yield downloader + + def calculate_size(self, refs): + """Calculate the total download and installed size of the images in refs and + their dependencies. + + :param refs: list of Flatpak references + :returns: download size, installed size + """ + log.debug("Calculating size of: %s", refs) + + download_size = 0 + installed_size = 0 + expanded = self._expand_refs(refs) + + for image in self._images: + if image.ref not in expanded: + continue + + log.debug("%s: download %d%s, installed %d", + image.ref, + " (skipped)" if self._is_local else "", + image.download_size, image.installed_size) + download_size += 0 if self._is_local else image.download_size + installed_size += image.installed_size + + log.debug("Total: download %d, installed %d", download_size, installed_size) + return download_size, installed_size + + def download(self, refs, download_location, progress=None): + if self._is_local: + return "oci:" + self._url.removeprefix("file://") + + collection_location = os.path.join(download_location, "Flatpaks") + expanded_refs = self._expand_refs(refs) + + index_json = { + "schemaVersion": 2, + "manifests": [] + } + + with self._downloader() as downloader: + for image in self._images: + if image.ref in expanded_refs: + log.debug("Downloading %s, %s bytes", image.ref, image.download_size) + if progress: + progress.report_progress(_("Downloading {}").format(image.ref)) + + manifest_len = self._download_blob(downloader, download_location, image.digest) + self._download_blob(downloader, + download_location, image.manifest_json["config"]["digest"]) + index_json["manifests"].append({ + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": image.digest, + "size": manifest_len + }) + + for layer in image.manifest_json["layers"]: + self._download_blob(downloader, + download_location, layer["digest"], + stream=True) + + os.makedirs(collection_location, exist_ok=True) + with open(os.path.join(collection_location, "index.json"), "w") as f: + json.dump(index_json, f) + + with open(os.path.join(collection_location, "oci-layout"), "w") as f: + json.dump({ + "imageLayoutVersion": "1.0.0" + }, f) + + return "oci:" + collection_location + + @cached_property + def _images(self) -> List[StaticSourceImage]: + result = [] + + with self._downloader() as downloader: + url = self._url + "/index.json" + response = downloader(url) + if response.status_code == 404: + raise NoSourceError("No Flatpak source found at {}".format(url)) + response.raise_for_status() + index_json = response.json() + + for manifest in index_json.get("manifests", ()): + if manifest.get("mediaType") == "application/vnd.oci.image.manifest.v1+json": + digest = manifest["digest"] + manifest_json = self._get_json(downloader, manifest["digest"]) + config_json = self._get_json(downloader, manifest_json["config"]["digest"]) + result.append(StaticSourceImage(digest, manifest_json, config_json)) + + return result + + def _blob_url(self, digest): + if not digest.startswith("sha256:"): + raise RuntimeError("Only SHA-256 digests are supported") + return self._url + "/blobs/sha256/" + digest[7:] + + def _get_blob(self, downloader, digest) -> bytes: + result = self._cached_blobs.get(digest) + if result: + return result + + response = downloader(self._blob_url(digest)) + response.raise_for_status() + + self._cached_blobs[digest] = result = response.content + return result + + def _download_blob(self, downloader, download_location, digest, stream=False): + if not digest.startswith("sha256:"): + raise RuntimeError("Only SHA-256 digests are supported") + + blobs_dir = os.path.join(download_location, "blobs/sha256/") + os.makedirs(blobs_dir, exist_ok=True) + + path = os.path.join(blobs_dir, digest[7:]) + with open(path, "wb") as f: + if stream: + response = downloader(self._blob_url(digest), stream=True) + response.raise_for_status() + size = 0 + while True: + chunk = response.raw.read(64*1024) + if not chunk: + break + size += len(chunk) + f.write(chunk) + return size + else: + blob = self._get_blob(downloader, digest) + f.write(blob) + return len(blob) + + def _get_json(self, session, digest): + return json.loads(self._get_blob(session, digest)) + + +class RegistrySourceImage(SourceImage): + def __init__(self, labels): + self._labels = labels + + @property + def labels(self): + return self._labels + + +class FlatpakRegistrySource(FlatpakSource): + """Flatpak images indexed by a remote JSON file, and stored in a registry. + + https://github.com/flatpak/flatpak-oci-specs/blob/main/registry-index.md + """ + + def __init__(self, url): + self._index = None + self._url = url + + def calculate_size(self, refs): + # For registry sources, we don't download the images in advance; + # instead they are downloaded into the /var/tmp of the target + # system and installed one-by-one. So the downloads don't count + # towards the space in the temporary download location, but we + # need space for the largest download in the target system. + # (That space will also be needed for upgrades after installation.) + + log.debug("Calculating size of: %s", refs) + + max_download_size = 0 + installed_size = 0 + expanded = self._expand_refs(refs) + + for image in self._images: + if image.ref not in expanded: + continue + + log.debug("%s: download %d, installed %d", + image.ref, image.download_size, image.installed_size) + + max_download_size = max(max_download_size, image.download_size) + installed_size += image.installed_size + + log.debug("Total: max download %d, installed %d", max_download_size, installed_size) + return 0, installed_size + max_download_size + + @cached_property + def _images(self): + arch = _get_container_arch() + + base_url = self._url.removeprefix("oci+") + parsed = urlparse(base_url) + if parsed.fragment: + tag = parsed.fragment + base_url = parsed._replace(fragment=None, query=None).geturl() + else: + tag = "latest" + + url_pattern = "{}/index/static?label:org.flatpak.ref:exists=1&architecture={}&tag={}" + full_url = url_pattern.format(base_url, arch, tag) + with requests_session() as session: + response = session.get(full_url) + response.raise_for_status() + index = response.json() + + result = [] + + arch = _get_container_arch() + for repository in index["Results"]: + for image in repository["Images"]: + if image['Architecture'] != arch: + continue + + result.append(RegistrySourceImage(image["Labels"])) + + return result + + def download(self, refs, download_location, progress=None): + return None diff --git a/pyanaconda/modules/payloads/payload/payload_base.py b/pyanaconda/modules/payloads/payload/payload_base.py index a6066c5408a..0de3357ce3a 100644 --- a/pyanaconda/modules/payloads/payload/payload_base.py +++ b/pyanaconda/modules/payloads/payload/payload_base.py @@ -185,6 +185,20 @@ def set_kernel_version_list(self, kernels): self._kernel_version_list = kernels log.debug("The kernel version list is set to: %s", kernels) + def needs_flatpak_side_payload(self): + """Does this payload need an extra payload for Flatpak installation + + :return: True or False + """ + return False + + def get_flatpak_refs(self): + """Get the list of Flatpak refs to install. + + :return: list of Flatpak refs + """ + return [] + @abstractmethod def install_with_tasks(self): """Install the payload. diff --git a/pyanaconda/modules/payloads/payloads.py b/pyanaconda/modules/payloads/payloads.py index 71f73fb19f8..ef203bfdd45 100644 --- a/pyanaconda/modules/payloads/payloads.py +++ b/pyanaconda/modules/payloads/payloads.py @@ -24,6 +24,7 @@ from pyanaconda.modules.common.base import KickstartService from pyanaconda.modules.common.constants.services import PAYLOADS from pyanaconda.modules.common.containers import TaskContainer +from pyanaconda.modules.payloads.constants import PayloadType from pyanaconda.modules.payloads.installation import ( CopyDriverDisksFilesTask, PrepareSystemForInstallationTask, @@ -49,6 +50,8 @@ def __init__(self): self._active_payload = None self.active_payload_changed = Signal() + self._flatpak_side_payload = None + def publish(self): """Publish the module.""" TaskContainer.set_namespace(PAYLOADS.namespace) @@ -89,6 +92,13 @@ def active_payload(self): def activate_payload(self, payload): """Activate the payload.""" self._active_payload = payload + + if self._active_payload.needs_flatpak_side_payload(): + payload = self.create_payload(PayloadType.FLATPAK) + self._flatpak_side_payload = payload + else: + self._flatpak_side_payload = None + self.active_payload_changed.emit() log.debug("Activated the payload %s.", payload.type) @@ -142,6 +152,10 @@ def calculate_required_space(self): if self.active_payload: total += self.active_payload.calculate_required_space() + if self._flatpak_side_payload: + self._flatpak_side_payload.set_sources(self.active_payload.sources) + self._flatpak_side_payload.set_flatpak_refs(self.active_payload.get_flatpak_refs()) + total += self._flatpak_side_payload.calculate_required_space() return total @@ -176,6 +190,12 @@ def install_with_tasks(self): ] tasks += self.active_payload.install_with_tasks() + + if self._flatpak_side_payload: + self._flatpak_side_payload.set_sources(self.active_payload.sources) + self._flatpak_side_payload.set_flatpak_refs(self.active_payload.get_flatpak_refs()) + tasks += self._flatpak_side_payload.install_with_tasks() + return tasks def post_install_with_tasks(self): @@ -193,6 +213,10 @@ def post_install_with_tasks(self): ] tasks += self.active_payload.post_install_with_tasks() + + if self._flatpak_side_payload: + tasks += self._flatpak_side_payload.post_install_with_tasks() + return tasks def teardown_with_tasks(self): @@ -205,4 +229,7 @@ def teardown_with_tasks(self): if self.active_payload: tasks += self.active_payload.tear_down_with_tasks() + if self._flatpak_side_payload: + tasks += self._flatpak_side_payload.tear_down_with_tasks() + return tasks diff --git a/pyanaconda/modules/payloads/source/flatpak/flatpak.py b/pyanaconda/modules/payloads/source/flatpak/flatpak.py index affb8bde534..a9bb150b818 100644 --- a/pyanaconda/modules/payloads/source/flatpak/flatpak.py +++ b/pyanaconda/modules/payloads/source/flatpak/flatpak.py @@ -17,6 +17,8 @@ # License and may only be used or replicated with the express permission of # Red Hat, Inc. # +import os + from pyanaconda.anaconda_loggers import get_module_logger from pyanaconda.core.configuration.anaconda import conf from pyanaconda.core.i18n import _ @@ -32,6 +34,21 @@ ) from pyanaconda.modules.payloads.source.source_base import PayloadSourceBase +# We need Flatpak to read configuration files from the target and write +# to the target system installation. Since we use the Flatpak API +# in process, we need to do this by modifying the environment before +# we start any threads. Setting these variables will be harmless if +# we aren't actually using Flatpak. + +# pylint: disable=environment-modify +os.environ["FLATPAK_DOWNLOAD_TMPDIR"] = os.path.join(conf.target.system_root, "var/tmp") +# pylint: disable=environment-modify +os.environ["FLATPAK_CONFIG_DIR"] = os.path.join(conf.target.system_root, "etc/flatpak") +# pylint: disable=environment-modify +os.environ["FLATPAK_OS_CONFIG_DIR"] = os.path.join(conf.target.system_root, "usr/share/flatpak") +# pylint: disable=environment-modify +os.environ["FLATPAK_SYSTEM_DIR"] = os.path.join(conf.target.system_root, "var/lib/flatpak") + log = get_module_logger(__name__) __all__ = ["FlatpakSourceModule"] diff --git a/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_dnf.py b/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_dnf.py index 6002f53a8d7..1914e4043ec 100644 --- a/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_dnf.py +++ b/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_dnf.py @@ -141,7 +141,13 @@ def _check_properties(self, expected_source_type): assert sources[0].type.value == expected_source_type def _test_kickstart(self, ks_in, ks_out, *args, **kwargs): - self.shared_ks_tests.check_kickstart(ks_in, ks_out, *args, **kwargs) + # DNF module also spawns Flatpak module as side payload by default + if "expected_publish_calls" not in kwargs: + kwargs["expected_publish_calls"] = 2 + + self.shared_ks_tests.check_kickstart( + ks_in, ks_out, *args, **kwargs + ) def test_cdrom_kickstart(self): ks_in = """ diff --git a/tests/unit_tests/pyanaconda_tests/modules/payloads/test_module_payloads.py b/tests/unit_tests/pyanaconda_tests/modules/payloads/test_module_payloads.py index 4c3ea3612c4..fd14ad01d98 100644 --- a/tests/unit_tests/pyanaconda_tests/modules/payloads/test_module_payloads.py +++ b/tests/unit_tests/pyanaconda_tests/modules/payloads/test_module_payloads.py @@ -119,7 +119,8 @@ def test_create_dnf_payload(self, publisher): assert self.payload_interface.ActivePayload == payload_path assert isinstance(PayloadContainer.from_object_path(payload_path), DNFModule) - publisher.assert_called_once() + # DNF payload creates also Flatpak side payload so there are two publish calls + assert publisher.call_count == 2 @patch_dbus_publish_object def test_create_live_os_payload(self, publisher): @@ -154,7 +155,7 @@ def test_create_invalid_payload(self, publisher): @patch_dbus_publish_object def test_create_multiple_payloads(self, publisher): """Test creating two payloads.""" - path_1 = self.payload_interface.CreatePayload(PayloadType.DNF.value) + path_1 = self.payload_interface.CreatePayload(PayloadType.RPM_OSTREE.value) assert self.payload_interface.CreatedPayloads == [path_1] assert self.payload_interface.ActivePayload == "" From bcd9bd529a2ac578484159dad1029c821cd6be02 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Mon, 3 Feb 2025 17:03:02 +0100 Subject: [PATCH 05/17] Improve logging of the Flatpak module To make the debugging easier in case something will went wrong. --- pyanaconda/modules/payloads/payloads.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pyanaconda/modules/payloads/payloads.py b/pyanaconda/modules/payloads/payloads.py index ef203bfdd45..4e5e70fed54 100644 --- a/pyanaconda/modules/payloads/payloads.py +++ b/pyanaconda/modules/payloads/payloads.py @@ -148,16 +148,24 @@ def calculate_required_space(self): :return: required size in bytes :rtype: int """ - total = 0 + main_size = 0 + side_size = 0 if self.active_payload: - total += self.active_payload.calculate_required_space() + main_size = self.active_payload.calculate_required_space() if self._flatpak_side_payload: self._flatpak_side_payload.set_sources(self.active_payload.sources) self._flatpak_side_payload.set_flatpak_refs(self.active_payload.get_flatpak_refs()) - total += self._flatpak_side_payload.calculate_required_space() + side_size = self._flatpak_side_payload.calculate_required_space() - return total + log.debug( + "Main payload size: %s, side payload size: %s, total: %s", + main_size, + side_size, + main_size + side_size, + ) + + return main_size + side_size def get_kernel_version_list(self): """Get the kernel versions list. From c43b887b662311fefa289c7447bf66560eae672f Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Tue, 4 Feb 2025 16:22:10 +0100 Subject: [PATCH 06/17] Call calculate_size of the Flatpak manager Currently, the method to calculate required size based on the installation source is not present. Let's add the missing call. TODO: This calculation should be done as separate task during the setup of the sources, because it is downloading metadata and could potentially block UI. To fix this the current sources in flatpak should be promoted to installation source on DBus with all the parts around. --- pyanaconda/modules/payloads/payload/flatpak/flatpak.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyanaconda/modules/payloads/payload/flatpak/flatpak.py b/pyanaconda/modules/payloads/payload/flatpak/flatpak.py index ea1c4717d90..88a8e7722da 100644 --- a/pyanaconda/modules/payloads/payload/flatpak/flatpak.py +++ b/pyanaconda/modules/payloads/payload/flatpak/flatpak.py @@ -89,8 +89,13 @@ def calculate_required_space(self): :return: required size in bytes :rtype: int """ - return calculate_required_space(self._flatpak_manager.download_size, - self._flatpak_manager.install_size) + self._flatpak_manager.calculate_size() + download_size = self._flatpak_manager.download_size + install_size = self._flatpak_manager.install_size + size = calculate_required_space(download_size, install_size) + log.debug("Flatpak size required to download: %s to install: %s required: %s", + download_size, install_size, size) + return size def install_with_tasks(self): """Install the payload with tasks.""" From d63a5d1e968df26036f133401d6cbf999836a5c3 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Tue, 4 Feb 2025 16:26:18 +0100 Subject: [PATCH 07/17] Add more logs for easier Flatpak module debugging To make our life easier :). --- .../modules/payloads/payload/flatpak/flatpak_manager.py | 7 ++++++- pyanaconda/modules/payloads/payload/flatpak/source.py | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pyanaconda/modules/payloads/payload/flatpak/flatpak_manager.py b/pyanaconda/modules/payloads/payload/flatpak/flatpak_manager.py index 3e368e355ae..1a1de5555bc 100644 --- a/pyanaconda/modules/payloads/payload/flatpak/flatpak_manager.py +++ b/pyanaconda/modules/payloads/payload/flatpak/flatpak_manager.py @@ -138,7 +138,12 @@ def calculate_size(self): The result is available from the download_size and install_size properties. """ - if self._skip_installation or len(self._flatpak_refs) == 0: + if self._skip_installation: + log.debug("Flatpak installation is going to be skipped.") + return + + if len(self._flatpak_refs) == 0: + log.debug("No flatpaks are marked for installation.") return try: diff --git a/pyanaconda/modules/payloads/payload/flatpak/source.py b/pyanaconda/modules/payloads/payload/flatpak/source.py index 454abefca28..56cdf1da2a3 100644 --- a/pyanaconda/modules/payloads/payload/flatpak/source.py +++ b/pyanaconda/modules/payloads/payload/flatpak/source.py @@ -149,10 +149,12 @@ def _expand_refs(self, refs: List[str]) -> List[str]: for image in self._images: if image.ref not in result: + log.debug("Skipping source image %s: not requested", image.ref) continue metadata = image.labels.get("org.flatpak.metadata") if metadata is None: + log.debug("Skipping source image %s: missing metadata", image.ref) continue cp = ConfigParser(interpolation=None) @@ -166,6 +168,7 @@ def _expand_refs(self, refs: List[str]) -> List[str]: except (NoSectionError, KeyError): pass + log.debug("Refs: %s are expanded to: %s", refs, result) return result @@ -231,6 +234,8 @@ def calculate_size(self, refs): for image in self._images: if image.ref not in expanded: + log.debug("Skipping Flatpak %s size calculation as it is not in expanded list: %s", + image.ref, expanded) continue log.debug("%s: download %d%s, installed %d", @@ -388,6 +393,7 @@ def calculate_size(self, refs): for image in self._images: if image.ref not in expanded: + log.debug("Image %s is not in expanded refs: %s", image.ref, expanded) continue log.debug("%s: download %d, installed %d", From 9cb3a5f8d44bf0b2dd7aaf38390f80f05d01e8ba Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Fri, 7 Feb 2025 16:33:36 +0100 Subject: [PATCH 08/17] Add side_payload to PayloadBase object Side payload will allow us to link dependent payloads and resolve our long standing issue of missing support for multiple payloads. This was main payload can control it's side_payload. Also side payload can have another side payload but that is not supported currently. --- .../modules/payloads/payload/payload_base.py | 25 +++++++++++++++++++ .../payload/payload_base_interface.py | 20 ++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/pyanaconda/modules/payloads/payload/payload_base.py b/pyanaconda/modules/payloads/payload/payload_base.py index 0de3357ce3a..dc2fd48164c 100644 --- a/pyanaconda/modules/payloads/payload/payload_base.py +++ b/pyanaconda/modules/payloads/payload/payload_base.py @@ -50,6 +50,10 @@ def __init__(self): self.sources_changed = Signal() self._kernel_version_list = None + # side payload is attached to the payload so it stays even when active payload is changed + # side payload standard methods and tasks are included in Payloads base class + self._side_payload = None + @property @abstractmethod def type(self): @@ -59,6 +63,27 @@ def type(self): """ return None + @property + def side_payload(self): + """Get side payload attached to this payload. + + The side payload is a payload attached to this payload. It can be configured by this + payload. + + :return: PayloadBase based class or None + :rtype: PayloadBase based class or None + """ + return self._side_payload + + @side_payload.setter + def side_payload(self, side_payload): + """Set side payload attached to this payload. + + :param side_payload: side payload to be attached to this payload + :type side_payload: PayloadBase based class or None + """ + self._side_payload = side_payload + @property @abstractmethod def default_source_type(self): diff --git a/pyanaconda/modules/payloads/payload/payload_base_interface.py b/pyanaconda/modules/payloads/payload/payload_base_interface.py index bf45b996ae4..615dd0da0d1 100644 --- a/pyanaconda/modules/payloads/payload/payload_base_interface.py +++ b/pyanaconda/modules/payloads/payload/payload_base_interface.py @@ -25,7 +25,11 @@ from pyanaconda.modules.common.base.base_template import ModuleInterfaceTemplate from pyanaconda.modules.common.constants.interfaces import PAYLOAD -from pyanaconda.modules.common.containers import PayloadSourceContainer, TaskContainer +from pyanaconda.modules.common.containers import ( + PayloadContainer, + PayloadSourceContainer, + TaskContainer, +) @dbus_interface(PAYLOAD.interface_name) @@ -50,6 +54,20 @@ def Type(self) -> Str: """ return self.implementation.type.value + @property + def SidePayload(self) -> ObjPath: + """Side payload attached to this payload. + + The side payload is a payload attached to this payload. It can be configured by this + payload. + """ + side_payload = self.implementation.side_payload + + if not side_payload: + return "" + + return PayloadContainer.to_object_path(side_payload) + @property def DefaultSourceType(self) -> Str: """Type of the default source. From 6027366506112541b3c66b3c260a5aff4206e2d2 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Fri, 7 Feb 2025 16:38:34 +0100 Subject: [PATCH 09/17] Change flatpak side payload to common side payload With a new side_payload solution in PayloadBase we can move from a flatpak specific solution in Payloads to the new Payload specific side_payload solution. Extend Payloads to add side payload to main payload for Payloads module API calls. This will improve a way how to work with side payloads in general and make flatpak payload better integrated. Also this will allow us to resolve correct reaction of the Flatpak on the DNF payload change after the package selection change. Which will allow us to move space calculation to Task. That solves issue with potential block of the UI because Flatpak payload is downloading metadata. --- .../modules/payloads/payload/dnf/dnf.py | 10 +++- .../modules/payloads/payload/payload_base.py | 2 + .../payload/payload_base_interface.py | 2 + pyanaconda/modules/payloads/payloads.py | 51 ++++++++++--------- .../modules/payloads/payloads_interface.py | 24 +++++++++ 5 files changed, 64 insertions(+), 25 deletions(-) diff --git a/pyanaconda/modules/payloads/payload/dnf/dnf.py b/pyanaconda/modules/payloads/payload/dnf/dnf.py index b84de3535cb..912edb821af 100644 --- a/pyanaconda/modules/payloads/payload/dnf/dnf.py +++ b/pyanaconda/modules/payloads/payload/dnf/dnf.py @@ -368,6 +368,12 @@ def verify_repomd_hashes_with_task(self): """ return VerifyRepomdHashesTask(self.dnf_manager) + def _refresh_side_payload_selection(self): + """Set new resolved software selection to side payload.""" + if self.side_payload and self.side_payload.type == PayloadType.FLATPAK: + self.side_payload.set_sources(self.sources) + self.side_payload.set_flatpak_refs(self.get_flatpak_refs()) + def validate_packages_selection_with_task(self, data): """Validate the specified packages selection. @@ -377,10 +383,12 @@ def validate_packages_selection_with_task(self, data): :param PackagesSelectionData data: a packages selection :return: a task """ - return CheckPackagesSelectionTask( + task = CheckPackagesSelectionTask( dnf_manager=self.dnf_manager, selection=data, ) + task.succeeded_signal.connect(self._refresh_side_payload_selection) + return task def calculate_required_space(self): """Calculate space required for the installation. diff --git a/pyanaconda/modules/payloads/payload/payload_base.py b/pyanaconda/modules/payloads/payload/payload_base.py index dc2fd48164c..ac3bba077bd 100644 --- a/pyanaconda/modules/payloads/payload/payload_base.py +++ b/pyanaconda/modules/payloads/payload/payload_base.py @@ -70,6 +70,8 @@ def side_payload(self): The side payload is a payload attached to this payload. It can be configured by this payload. + This side payload calls will be automatically queued for some DBus API in Payloads module. + :return: PayloadBase based class or None :rtype: PayloadBase based class or None """ diff --git a/pyanaconda/modules/payloads/payload/payload_base_interface.py b/pyanaconda/modules/payloads/payload/payload_base_interface.py index 615dd0da0d1..affce3025b8 100644 --- a/pyanaconda/modules/payloads/payload/payload_base_interface.py +++ b/pyanaconda/modules/payloads/payload/payload_base_interface.py @@ -60,6 +60,8 @@ def SidePayload(self) -> ObjPath: The side payload is a payload attached to this payload. It can be configured by this payload. + + This side payload calls will be automatically queued for some DBus API in Payloads module. """ side_payload = self.implementation.side_payload diff --git a/pyanaconda/modules/payloads/payloads.py b/pyanaconda/modules/payloads/payloads.py index 4e5e70fed54..0009a3723b6 100644 --- a/pyanaconda/modules/payloads/payloads.py +++ b/pyanaconda/modules/payloads/payloads.py @@ -50,8 +50,6 @@ def __init__(self): self._active_payload = None self.active_payload_changed = Signal() - self._flatpak_side_payload = None - def publish(self): """Publish the module.""" TaskContainer.set_namespace(PAYLOADS.namespace) @@ -94,10 +92,9 @@ def activate_payload(self, payload): self._active_payload = payload if self._active_payload.needs_flatpak_side_payload(): - payload = self.create_payload(PayloadType.FLATPAK) - self._flatpak_side_payload = payload - else: - self._flatpak_side_payload = None + side_payload = self.create_payload(PayloadType.FLATPAK) + self._active_payload.side_payload = side_payload + log.debug("Created side payload %s.", side_payload.type) self.active_payload_changed.emit() log.debug("Activated the payload %s.", payload.type) @@ -145,18 +142,20 @@ def is_network_required(self): def calculate_required_space(self): """Calculate space required for the installation. + Calculate required space for the main payload and the side payload if exists. + :return: required size in bytes :rtype: int """ - main_size = 0 + if not self.active_payload: + return 0 + + main_size = self.active_payload.calculate_required_space() side_size = 0 - if self.active_payload: - main_size = self.active_payload.calculate_required_space() - if self._flatpak_side_payload: - self._flatpak_side_payload.set_sources(self.active_payload.sources) - self._flatpak_side_payload.set_flatpak_refs(self.active_payload.get_flatpak_refs()) - side_size = self._flatpak_side_payload.calculate_required_space() + + if self.active_payload.side_payload: + side_size = self.active_payload.side_payload.calculate_required_space() log.debug( "Main payload size: %s, side payload size: %s, total: %s", @@ -186,6 +185,8 @@ def get_kernel_version_list(self): def install_with_tasks(self): """Return a list of installation tasks. + Concatenate tasks of the main payload together with side payload of that payload. + :return: list of tasks """ if not self.active_payload: @@ -199,16 +200,16 @@ def install_with_tasks(self): tasks += self.active_payload.install_with_tasks() - if self._flatpak_side_payload: - self._flatpak_side_payload.set_sources(self.active_payload.sources) - self._flatpak_side_payload.set_flatpak_refs(self.active_payload.get_flatpak_refs()) - tasks += self._flatpak_side_payload.install_with_tasks() + if self.active_payload.side_payload: + tasks += self.active_payload.side_payload.install_with_tasks() return tasks def post_install_with_tasks(self): """Return a list of post-installation tasks. + Concatenate tasks of the main payload together with side payload of that payload. + :return: a list of tasks """ if not self.active_payload: @@ -222,22 +223,24 @@ def post_install_with_tasks(self): tasks += self.active_payload.post_install_with_tasks() - if self._flatpak_side_payload: - tasks += self._flatpak_side_payload.post_install_with_tasks() + if self.active_payload.side_payload: + tasks += self.active_payload.side_payload.post_install_with_tasks() return tasks def teardown_with_tasks(self): """Returns teardown tasks for this module. + Concatenate tasks of the main payload together with side payload of that payload. + :return: a list of tasks """ - tasks = [] + if not self.active_payload: + return [] - if self.active_payload: - tasks += self.active_payload.tear_down_with_tasks() + tasks = self.active_payload.tear_down_with_tasks() - if self._flatpak_side_payload: - tasks += self._flatpak_side_payload.tear_down_with_tasks() + if self.active_payload.side_payload: + tasks += self.active_payload.side_payload.tear_down_with_tasks() return tasks diff --git a/pyanaconda/modules/payloads/payloads_interface.py b/pyanaconda/modules/payloads/payloads_interface.py index 638bdd3a7a5..1c4394a0fca 100644 --- a/pyanaconda/modules/payloads/payloads_interface.py +++ b/pyanaconda/modules/payloads/payloads_interface.py @@ -107,6 +107,8 @@ def IsNetworkRequired(self) -> Bool: def CalculateRequiredSpace(self) -> UInt64: """Calculate space required for the installation. + Calculate required space for the main payload and the side payload if exists. + :return: required size in bytes :rtype: int """ @@ -123,11 +125,33 @@ def GetKernelVersionList(self) -> List[Str]: """ return self.implementation.get_kernel_version_list() + # Update documentation of this method from parent. + def InstallWithTasks(self) -> List[ObjPath]: # pylint: disable=useless-parent-delegation + """Returns installation tasks of this module. + + Concatenate tasks of the main payload together with side payload of that payload. + + :returns: list of object paths of installation tasks + """ + return super().InstallWithTasks() + def PostInstallWithTasks(self) -> List[ObjPath]: """Return a list of post-installation tasks. + Concatenate tasks of the main payload together with side payload of that payload. + :return: a list of object paths of installation tasks """ return TaskContainer.to_object_path_list( self.implementation.post_install_with_tasks() ) + + # Update documentation of this method from parent. + def TeardownWithTasks(self) -> List[ObjPath]: # pylint: disable=useless-parent-delegation + """Returns teardown tasks for this module. + + Concatenate tasks of the main payload together with side payload of that payload. + + :returns: list of object paths of installation tasks + """ + return super().TeardownWithTasks() From 9e810b5c23a071a6074ea336ecc85017de5c30a5 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Fri, 7 Feb 2025 17:29:39 +0100 Subject: [PATCH 10/17] Add new API for Flatpak to CalculateSizeWithTask This API is used from UI code just after new package selection is resolved. This will allow us to react on package selection in Flatpak immediately. Also move CalculateFlatpaksSizeTask to initialization because this task is not used for installation. --- .../payloads/payload/flatpak/flatpak.py | 10 ++--- .../payload/flatpak/flatpak_interface.py | 9 +++++ .../payload/flatpak/initialization.py | 37 +++++++++++++++++++ .../payloads/payload/flatpak/installation.py | 17 --------- pyanaconda/payload/dnf/payload.py | 8 ++++ 5 files changed, 59 insertions(+), 22 deletions(-) create mode 100644 pyanaconda/modules/payloads/payload/flatpak/initialization.py diff --git a/pyanaconda/modules/payloads/payload/flatpak/flatpak.py b/pyanaconda/modules/payloads/payload/flatpak/flatpak.py index 88a8e7722da..fcbfe548bc7 100644 --- a/pyanaconda/modules/payloads/payload/flatpak/flatpak.py +++ b/pyanaconda/modules/payloads/payload/flatpak/flatpak.py @@ -22,8 +22,8 @@ from pyanaconda.modules.payloads.constants import PayloadType, SourceType from pyanaconda.modules.payloads.payload.flatpak.flatpak_interface import FlatpakInterface from pyanaconda.modules.payloads.payload.flatpak.flatpak_manager import FlatpakManager +from pyanaconda.modules.payloads.payload.flatpak.initialization import CalculateFlatpaksSizeTask from pyanaconda.modules.payloads.payload.flatpak.installation import ( - CalculateFlatpaksSizeTask, CleanUpDownloadLocationTask, DownloadFlatpaksTask, InstallFlatpaksTask, @@ -89,7 +89,6 @@ def calculate_required_space(self): :return: required size in bytes :rtype: int """ - self._flatpak_manager.calculate_size() download_size = self._flatpak_manager.download_size install_size = self._flatpak_manager.install_size size = calculate_required_space(download_size, install_size) @@ -97,13 +96,14 @@ def calculate_required_space(self): download_size, install_size, size) return size + def calculate_size_with_task(self): + """Refresh size requirement with task.""" + return CalculateFlatpaksSizeTask(flatpak_manager=self._flatpak_manager) + def install_with_tasks(self): """Install the payload with tasks.""" tasks = [ - CalculateFlatpaksSizeTask( - flatpak_manager=self._flatpak_manager, - ), PrepareDownloadLocationTask( flatpak_manager=self._flatpak_manager, ), diff --git a/pyanaconda/modules/payloads/payload/flatpak/flatpak_interface.py b/pyanaconda/modules/payloads/payload/flatpak/flatpak_interface.py index 2904b829a1e..46734accfea 100644 --- a/pyanaconda/modules/payloads/payload/flatpak/flatpak_interface.py +++ b/pyanaconda/modules/payloads/payload/flatpak/flatpak_interface.py @@ -18,11 +18,20 @@ # Red Hat, Inc. # from dasbus.server.interface import dbus_interface +from dasbus.typing import * # pylint: disable=wildcard-import from pyanaconda.modules.common.constants.interfaces import PAYLOAD_FLATPAK +from pyanaconda.modules.common.containers import TaskContainer from pyanaconda.modules.payloads.payload.payload_base_interface import PayloadBaseInterface @dbus_interface(PAYLOAD_FLATPAK.interface_name) class FlatpakInterface(PayloadBaseInterface): """DBus interface for Flatpak payload module.""" + + + def CalculateSizeWithTask(self) -> ObjPath: + """Calculate required size based on the software selection with task.""" + return TaskContainer.to_object_path( + self.implementation.calculate_size_with_task() + ) diff --git a/pyanaconda/modules/payloads/payload/flatpak/initialization.py b/pyanaconda/modules/payloads/payload/flatpak/initialization.py new file mode 100644 index 00000000000..b3063bdf7c5 --- /dev/null +++ b/pyanaconda/modules/payloads/payload/flatpak/initialization.py @@ -0,0 +1,37 @@ +# +# Copyright (C) 2025 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# + +from pyanaconda.modules.common.task import Task +from pyanaconda.modules.payloads.payload.flatpak.flatpak_manager import FlatpakManager + + +class CalculateFlatpaksSizeTask(Task): + """Task to determine space needed for Flatpaks""" + + def __init__(self, flatpak_manager: FlatpakManager): + """Create a new task.""" + super().__init__() + self._flatpak_manager = flatpak_manager + + @property + def name(self): + """Name of the task.""" + return "Calculate needed space for Flatpaks" + + def run(self): + self._flatpak_manager.calculate_size() diff --git a/pyanaconda/modules/payloads/payload/flatpak/installation.py b/pyanaconda/modules/payloads/payload/flatpak/installation.py index 7c20cd6d044..491e586dc19 100644 --- a/pyanaconda/modules/payloads/payload/flatpak/installation.py +++ b/pyanaconda/modules/payloads/payload/flatpak/installation.py @@ -29,23 +29,6 @@ FLATPAK_MIRROR_DIR_SUFFIX = 'flatpak.mirror' -class CalculateFlatpaksSizeTask(Task): - """Task to determine space needed for Flatpaks""" - - def __init__(self, flatpak_manager: FlatpakManager): - """Create a new task.""" - super().__init__() - self._flatpak_manager = flatpak_manager - - @property - def name(self): - """Name of the task.""" - return "Calculate needed space for Flatpaks" - - def run(self): - self._flatpak_manager.calculate_size() - - class PrepareDownloadLocationTask(Task): """The installation task for setting up the download location.""" diff --git a/pyanaconda/payload/dnf/payload.py b/pyanaconda/payload/dnf/payload.py index 6063404f99f..559ece4e6a9 100644 --- a/pyanaconda/payload/dnf/payload.py +++ b/pyanaconda/payload/dnf/payload.py @@ -350,6 +350,14 @@ def check_software_selection(self, selection): result = unwrap_variant(task_proxy.GetResult()) report = ValidationReport.from_structure(result) + # Start side payload processing if report is valid + if report.is_valid(): + side_payload_path = self.proxy.SidePayload + if side_payload_path: + side_payload = PAYLOADS.get_proxy(side_payload_path) + side_task_proxy = PAYLOADS.get_proxy(side_payload.CalculateSizeWithTask()) + sync_run_task(side_task_proxy) + # This validation is no longer required. self._software_validation_required = False From 0d70d0d959b390ce40933d47c6f22143f6e66b94 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Fri, 14 Feb 2025 17:02:34 +0100 Subject: [PATCH 11/17] Support payload with no default source It shouldn't happen but for current unfinished Flatpak solution and possible other similar cases in the future let's allow that. --- pyanaconda/modules/payloads/payload/payload_base_interface.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyanaconda/modules/payloads/payload/payload_base_interface.py b/pyanaconda/modules/payloads/payload/payload_base_interface.py index affce3025b8..5057928a147 100644 --- a/pyanaconda/modules/payloads/payload/payload_base_interface.py +++ b/pyanaconda/modules/payloads/payload/payload_base_interface.py @@ -78,6 +78,9 @@ def DefaultSourceType(self) -> Str: :return: a string representation of a source type """ + if self.implementation.default_source_type is None: + return "" + return self.implementation.default_source_type.value @property From 2e276d49e0a1767c2dee4f67efaa23ff199caa52 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Fri, 14 Feb 2025 17:05:07 +0100 Subject: [PATCH 12/17] Ignore not-ready sources in FlatpakManager Unready sources might not be usable and could cause crash. --- .../modules/payloads/payload/flatpak/flatpak_manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyanaconda/modules/payloads/payload/flatpak/flatpak_manager.py b/pyanaconda/modules/payloads/payload/flatpak/flatpak_manager.py index 1a1de5555bc..c46bd3504db 100644 --- a/pyanaconda/modules/payloads/payload/flatpak/flatpak_manager.py +++ b/pyanaconda/modules/payloads/payload/flatpak/flatpak_manager.py @@ -29,7 +29,7 @@ from pyanaconda.core.i18n import _ from pyanaconda.modules.common.errors.installation import PayloadInstallationError from pyanaconda.modules.common.task.progress import ProgressReporter -from pyanaconda.modules.payloads.constants import SourceType +from pyanaconda.modules.payloads.constants import SourceState, SourceType from pyanaconda.modules.payloads.payload.flatpak.source import ( FlatpakRegistrySource, FlatpakStaticSource, @@ -80,6 +80,9 @@ def set_sources(self, sources: List[PayloadSourceBase]): source = sources[0] + if source.get_state != SourceState.READY: + return + if isinstance(source, RepositorySourceMixin): if self._source and isinstance(self._source, FlatpakStaticSource) \ and self._source.repository_config == source.repository: From a04be9c7e100b35b7b00424733ef1371ddc852c4 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Fri, 14 Feb 2025 17:06:35 +0100 Subject: [PATCH 13/17] Do not allow any sources for Flatpak payload Currently it was set as it works with all the sources because it is using other payload sources instead. This is wrong for multiple reasons, mainly these two: - the Flatpak module can use every source - not true, it could work with just a few - set up and tear down methods will be called on stored sources - these would be double set up and tear down as these sources are already resolved in main payload To resolve this issue, change the logic that Flatpak module don't support any source type and the set_sources method will only pass these sources to the FlatpakManager but not store them in module. It is still wrong but it shouldn't break anything. Future solution should implement proper sources for the Flatpak module. However, let's not focus on that right now. --- .../modules/payloads/payload/flatpak/flatpak.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyanaconda/modules/payloads/payload/flatpak/flatpak.py b/pyanaconda/modules/payloads/payload/flatpak/flatpak.py index fcbfe548bc7..de27187dba6 100644 --- a/pyanaconda/modules/payloads/payload/flatpak/flatpak.py +++ b/pyanaconda/modules/payloads/payload/flatpak/flatpak.py @@ -58,23 +58,24 @@ def default_source_type(self): @property def supported_source_types(self): """List of supported source types.""" - # Include all the types of SourceType. + # Flatpak doesn't own any source. # FIXME: Flatpak needs it's own source because this way it needs to understand # all existing and future ones - return list(SourceType) + return [] def set_sources(self, sources): - """Set a new list of sources to this payload. + """Set a new list of sources to a flatpak manager. This overrides the base implementation since the sources we set here are the sources from the main payload, and can already be initialized. + TODO: This DBus API will not work until we have proper handling of the sources. + It will only work as redirect to flatpak_manager but no sources are stored here. + :param sources: set a new sources :type sources: instance of pyanaconda.modules.payloads.source.source_base.PayloadSourceBase """ - self._sources = sources self._flatpak_manager.set_sources(sources) - self.sources_changed.emit() def set_flatpak_refs(self, refs): """Set the flatpak refs. From 0287b5540faab71b240a7612989a92a00d7894d8 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Fri, 14 Feb 2025 17:17:12 +0100 Subject: [PATCH 14/17] Add Flatpak tests for the Flatpak module Tests cover: - Flatpak interface - Flatpak module - Flatpak tasks --- .../payload/test_module_payload_flatpak.py | 180 ++++++++++++++++++ .../test_module_payload_flatpak_tasks.py | 172 +++++++++++++++++ 2 files changed, 352 insertions(+) create mode 100644 tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_flatpak.py create mode 100644 tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_flatpak_tasks.py diff --git a/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_flatpak.py b/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_flatpak.py new file mode 100644 index 00000000000..8f526907e0a --- /dev/null +++ b/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_flatpak.py @@ -0,0 +1,180 @@ +# +# Copyright (C) 2019 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# +# Red Hat Author(s): Jiri Konecny +# +import unittest +from unittest.mock import Mock, patch + +import pytest +from blivet.size import Size + +from pyanaconda.core.constants import PAYLOAD_TYPE_FLATPAK +from pyanaconda.modules.common.constants.interfaces import PAYLOAD_FLATPAK +from pyanaconda.modules.common.errors.general import UnavailableValueError +from pyanaconda.modules.payloads.constants import SourceType +from pyanaconda.modules.payloads.payload.flatpak.flatpak import FlatpakModule +from pyanaconda.modules.payloads.payload.flatpak.flatpak_interface import FlatpakInterface +from pyanaconda.modules.payloads.payload.flatpak.initialization import CalculateFlatpaksSizeTask +from pyanaconda.modules.payloads.payload.flatpak.installation import ( + CleanUpDownloadLocationTask, + DownloadFlatpaksTask, + InstallFlatpaksTask, + PrepareDownloadLocationTask, +) +from pyanaconda.modules.payloads.payload.payload_base import SetUpSourcesTask, TearDownSourcesTask +from pyanaconda.modules.payloads.source.factory import SourceFactory +from tests.unit_tests.pyanaconda_tests import ( + check_dbus_property, + check_instances, + check_task_creation, + patch_dbus_publish_object, +) +from tests.unit_tests.pyanaconda_tests.modules.payloads.payload.module_payload_shared import ( + PayloadSharedTest, +) + + +class FlatpakInterfaceTestCase(unittest.TestCase): + """Test the DBus interface of the Flatpak module.""" + + def setUp(self): + self.module = FlatpakModule() + self.interface = FlatpakInterface(self.module) + self.shared_tests = PayloadSharedTest( + payload=self.module, + payload_intf=self.interface + ) + + def test_type(self): + """Test the Type property.""" + assert self.interface.Type == PAYLOAD_TYPE_FLATPAK + + def test_default_source_type(self): + """Test the DefaultSourceType property.""" + assert self.interface.DefaultSourceType == "" + + def test_supported_sources(self): + """Test DNF supported sources API.""" + assert self.interface.SupportedSourceTypes == [] + + @patch.object(FlatpakModule, "supported_source_types", [SourceType.URL]) + @patch_dbus_publish_object + def test_set_source(self, publisher): + """Test if set source API Flatpak payload.""" + sources = [self.shared_tests.prepare_source(SourceType.URL)] + + # TODO: this API is disabled on purpose, the behavior is currently wrong + # sources stored in the payload are not really stored anywhere just passed + # to flatpak_manager + self.shared_tests.set_sources(sources) + assert self.interface.Sources == [] + + @patch_dbus_publish_object + def test_set_up_sources_with_task(self, publisher): + """Test Flatpak SetUpSourcesWithTask.""" + source = SourceFactory.create_source(SourceType.CDROM) + self.module.add_source(source) + + task_path = self.interface.SetUpSourcesWithTask() + obj = check_task_creation(task_path, publisher, SetUpSourcesTask) + assert obj.implementation._sources == [] + + @patch_dbus_publish_object + def test_tear_down_sources_with_task(self, publisher): + """Test TearDownSourcesWithTask.""" + s1 = SourceFactory.create_source(SourceType.CDROM) + self.module.add_source(s1) + + task_path = self.interface.TearDownSourcesWithTask() + obj = check_task_creation(task_path, publisher, TearDownSourcesTask) + assert obj.implementation._sources == [] + + @patch_dbus_publish_object + def test_calculate_size_with_task(self, publisher): + """Test CalculateSizeWithTask API.""" + task_path = self.interface.CalculateSizeWithTask() + + check_task_creation(task_path, publisher, CalculateFlatpaksSizeTask) + + def _check_dbus_property(self, *args, **kwargs): + check_dbus_property( + PAYLOAD_FLATPAK, + self.interface, + *args, **kwargs + ) + + +class DNFModuleTestCase(unittest.TestCase): + """Test the DNF module.""" + + def setUp(self): + """Set up the test.""" + self.module = FlatpakModule() + + def test_is_network_required(self): + """Test the Flatpak is_network_required function.""" + assert self.module.is_network_required() is False + + def test_set_sources(self): + """Test set_sources method.""" + # FIXME: This method is hacked right now. As it will set sources only to Flatpak manager + # and not the module. + flatpak_manager = Mock() + self.module._flatpak_manager = flatpak_manager + s1 = SourceFactory.create_source(SourceType.URL) + + self.module.set_sources([s1]) + + flatpak_manager.set_sources.assert_called_once_with([s1]) + + def test_flatpak_refs(self): + flatpak_manager = Mock() + self.module._flatpak_manager = flatpak_manager + + refs = ["org.example.App", + "org.example.App2"] + + self.module.set_flatpak_refs(refs) + + flatpak_manager.set_flatpak_refs.assert_called_once_with(refs) + + @patch("pyanaconda.modules.payloads.payload.flatpak.flatpak.calculate_required_space") + def test_calculate_required_space(self, space_getter): + """Test the Flatpak calculate_required_space method.""" + space_getter.return_value = Size("1 MiB") + assert self.module.calculate_required_space() == 1048576 + + def test_get_kernel_version_list(self): + """Test the Flatpak get_kernel_version_list method.""" + with pytest.raises(UnavailableValueError): + self.module.get_kernel_version_list() + + def test_install_with_tasks(self): + """Test the Flatpak install_with_tasks method.""" + tasks = self.module.install_with_tasks() + check_instances(tasks, [ + PrepareDownloadLocationTask, + DownloadFlatpaksTask, + InstallFlatpaksTask, + CleanUpDownloadLocationTask, + ]) + + def test_post_install_with_tasks(self): + """Test the Flatpak post_install_with_tasks method.""" + tasks = self.module.post_install_with_tasks() + check_instances(tasks, []) diff --git a/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_flatpak_tasks.py b/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_flatpak_tasks.py new file mode 100644 index 00000000000..deaff74c48b --- /dev/null +++ b/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_flatpak_tasks.py @@ -0,0 +1,172 @@ +# +# Copyright (C) 2019 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# +# Red Hat Author(s): Jiri Konecny +# +import unittest +from unittest.mock import Mock, patch + +from pyanaconda.modules.payloads.payload.flatpak.initialization import CalculateFlatpaksSizeTask +from pyanaconda.modules.payloads.payload.flatpak.installation import ( + CleanUpDownloadLocationTask, + DownloadFlatpaksTask, + InstallFlatpaksTask, + PrepareDownloadLocationTask, +) + + +class CalculateFlatpaksSizeTaskTestCase(unittest.TestCase): + """Test the CalculateFlatpaksSizeTask task.""" + + def test_calculate_flatpaks_size_task_name(self): + """Test CalculateFlatpaksSizeTask name.""" + flatpak_manager = Mock() + task = CalculateFlatpaksSizeTask(flatpak_manager) + + assert task.name == "Calculate needed space for Flatpaks" + + def test_calculate_flatpaks_size_task_run(self): + """Test CalculateFlatpaksSizeTask run.""" + flatpak_manager = Mock() + + task = CalculateFlatpaksSizeTask(flatpak_manager) + task.run() + + flatpak_manager.calculate_size.assert_called_once() + + +class PrepareDownloadLocationTaskTestCase(unittest.TestCase): + """Test PrepareDownloadLocationTask task.""" + + def test_prepare_download_location_task_name(self): + """Test PrepareDownloadLocationTask name.""" + flatpak_manager = Mock() + task = PrepareDownloadLocationTask(flatpak_manager) + assert task.name == "Prepare the package download" + + @patch("pyanaconda.modules.payloads.payload.flatpak.installation.shutil") + @patch("pyanaconda.modules.payloads.payload.flatpak.installation.os") + @patch("pyanaconda.modules.payloads.payload.flatpak.installation.pick_download_location") + def test_prepare_download_location_task_run(self, + pick_download_location, + os_mock, + shutil_mock): + """Test PrepareDownloadLocationTask run.""" + flatpak_manager = Mock() + flatpak_manager.download_size = 10 + flatpak_manager.install_size = 20 + pick_download_location.return_value = "/result/path" + + # test path exists + os_mock.path.exists.return_value = True + task = PrepareDownloadLocationTask(flatpak_manager) + path = task.run() + + assert path == "/result/path" + flatpak_manager.calculate_size.assert_called_once() + pick_download_location.assert_called_once_with(10, 20, 'flatpak.mirror') + shutil_mock.rmtree.assert_called_once_with("/result/path") + flatpak_manager.set_download_location.assert_called_once_with("/result/path") + + # test path doesn't exists + os_mock.path.exists.return_value = False + flatpak_manager.calculate_size.reset_mock() + pick_download_location.reset_mock() + shutil_mock.rmtree.reset_mock() + flatpak_manager.set_download_location.reset_mock() + + task = PrepareDownloadLocationTask(flatpak_manager) + path = task.run() + + assert path == "/result/path" + pick_download_location.assert_called_once_with(10, 20, 'flatpak.mirror') + shutil_mock.rmtree.assert_not_called() + flatpak_manager.set_download_location.assert_called_once_with("/result/path") + + +class CleanUpDownloadLocationTaskTestCase(unittest.TestCase): + """Test the CleanUpDownloadLocationTask task.""" + + def test_clean_up_download_location_task_name(self): + """Test CleanUpDownloadLocationTask name.""" + flatpak_manager = Mock() + task = CleanUpDownloadLocationTask(flatpak_manager) + + assert task.name == "Remove downloaded Flatpaks" + + @patch("pyanaconda.modules.payloads.payload.flatpak.installation.shutil") + @patch("pyanaconda.modules.payloads.payload.flatpak.installation.os") + def test_clean_up_download_location_task_run(self, os_mock, shutil_mock): + """Test CleanUpDownloadLocationTask run.""" + flatpak_manager = Mock() + flatpak_manager.download_location = "/result/path" + + # test path exists - flatpaks were downloaded + os_mock.path.exists.return_value = True + task = CleanUpDownloadLocationTask(flatpak_manager) + task.run() + + shutil_mock.rmtree.assert_called_once_with("/result/path") + + # test path doesn't exists - flatpaks were not downloaded + shutil_mock.rmtree.reset_mock() + + os_mock.path.exists.return_value = False + task = CleanUpDownloadLocationTask(flatpak_manager) + task.run() + + shutil_mock.rmtree.assert_not_called() + + +class DownloadFlatpaksTaskTestCase(unittest.TestCase): + """Test the DownloadFlatpaksTask task.""" + + def test_clean_up_download_location_task_name(self): + """Test DownloadFlatpaksTask name.""" + flatpak_manager = Mock() + task = DownloadFlatpaksTask(flatpak_manager) + + assert task.name == "Download remote Flatpaks" + + def test_clean_up_download_location_task_run(self): + """Test DownloadFlatpaksTask run.""" + flatpak_manager = Mock() + + task = DownloadFlatpaksTask(flatpak_manager) + task.run() + + flatpak_manager.download.assert_called_once() + + +class InstallFlatpaksTaskTestCase(unittest.TestCase): + """Test the InstallFlatpaksTask task.""" + + def test_clean_up_download_location_task_name(self): + """Test InstallFlatpaksTask name.""" + flatpak_manager = Mock() + task = InstallFlatpaksTask(flatpak_manager) + + assert task.name == "Install Flatpaks" + + def test_clean_up_download_location_task_run(self): + """Test InstallFlatpaksTask run.""" + flatpak_manager = Mock() + + task = InstallFlatpaksTask(flatpak_manager) + task.run() + + flatpak_manager.install.assert_called_once() From 2f448c7389b2f529fb67222eb2fb4c3f77c78a37 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Fri, 14 Feb 2025 17:47:08 +0100 Subject: [PATCH 15/17] Extract flatpak.source utils to a separate module These utilities will be easier to test as separate entity. --- .../payloads/payload/flatpak/source.py | 47 +++----------- .../modules/payloads/payload/flatpak/utils.py | 63 +++++++++++++++++++ 2 files changed, 70 insertions(+), 40 deletions(-) create mode 100644 pyanaconda/modules/payloads/payload/flatpak/utils.py diff --git a/pyanaconda/modules/payloads/payload/flatpak/source.py b/pyanaconda/modules/payloads/payload/flatpak/source.py index 56cdf1da2a3..c6952abdddd 100644 --- a/pyanaconda/modules/payloads/payload/flatpak/source.py +++ b/pyanaconda/modules/payloads/payload/flatpak/source.py @@ -27,55 +27,22 @@ from typing import Dict, List, Optional, Tuple from urllib.parse import urljoin, urlparse -from blivet.arch import get_arch - from pyanaconda.anaconda_loggers import get_module_logger from pyanaconda.core.i18n import _ from pyanaconda.core.util import requests_session from pyanaconda.modules.common.structures.payload import RepoConfigurationData from pyanaconda.modules.common.task.progress import ProgressReporter from pyanaconda.modules.payloads.base.utils import get_downloader_for_repo_configuration +from pyanaconda.modules.payloads.payload.flatpak.utils import ( + canonicalize_flatpak_ref, + get_container_arch, +) log = get_module_logger(__name__) __all__ = ["FlatpakRegistrySource", "FlatpakSource", "FlatpakStaticSource", "NoSourceError"] -_CONTAINER_ARCH_MAP = { - "x86_64": "amd64", - "aarch64": "arm64" -} - - -def _get_container_arch(): - """Architecture name as used by docker/podman""" - arch = get_arch() - return _CONTAINER_ARCH_MAP.get(arch, arch) - - -def _canonicalize_flatpak_ref(ref) -> Tuple[Optional[str], str]: - """Split off a collection ID, and add architecture if unspecified - - Turn "org.fedoraproject.Stable:app/org.example.Foo//stable" into - ("org.fedoraproject.Stable", "app/org.example.Foo/amd64/stable") - """ - - collection_parts = ref.split(":", 1) - if len(collection_parts) == 2: - collection = collection_parts[0] - ref = collection_parts[1] - else: - collection = None - - parts = ref.split("/") - if len(parts) != 4: - raise RuntimeError("Can't parse reference") - if parts[2] == "": - parts[2] = get_arch() - - return collection, "/".join(parts) - - class NoSourceError(Exception): """Source not found.""" @@ -144,7 +111,7 @@ def _expand_refs(self, refs: List[str]) -> List[str]: result = [] for ref in refs: # We don't do anything with the collection ID for now - _, ref = _canonicalize_flatpak_ref(ref) + _, ref = canonicalize_flatpak_ref(ref) result.append(ref) for image in self._images: @@ -407,7 +374,7 @@ def calculate_size(self, refs): @cached_property def _images(self): - arch = _get_container_arch() + arch = get_container_arch() base_url = self._url.removeprefix("oci+") parsed = urlparse(base_url) @@ -426,7 +393,7 @@ def _images(self): result = [] - arch = _get_container_arch() + arch = get_container_arch() for repository in index["Results"]: for image in repository["Images"]: if image['Architecture'] != arch: diff --git a/pyanaconda/modules/payloads/payload/flatpak/utils.py b/pyanaconda/modules/payloads/payload/flatpak/utils.py new file mode 100644 index 00000000000..f34a5efa456 --- /dev/null +++ b/pyanaconda/modules/payloads/payload/flatpak/utils.py @@ -0,0 +1,63 @@ +# +# Query and download sources of Flatpak content +# +# Copyright (C) 2024 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# + +from typing import Optional, Tuple + +from blivet.arch import get_arch + +from pyanaconda.anaconda_loggers import get_module_logger + +log = get_module_logger(__name__) + +__all__ = ["canonicalize_flatpak_ref", "get_container_arch"] + + +_CONTAINER_ARCH_MAP = { + "x86_64": "amd64", + "aarch64": "arm64" +} + + +def get_container_arch(): + """Architecture name as used by docker/podman""" + arch = get_arch() + return _CONTAINER_ARCH_MAP.get(arch, arch) + + +def canonicalize_flatpak_ref(ref) -> Tuple[Optional[str], str]: + """Split off a collection ID, and add architecture if unspecified + + Turn "org.fedoraproject.Stable:app/org.example.Foo//stable" into + ("org.fedoraproject.Stable", "app/org.example.Foo/amd64/stable") + """ + collection_parts = ref.split(":", 1) + if len(collection_parts) == 2: + collection = collection_parts[0] + ref = collection_parts[1] + else: + collection = None + + parts = ref.split("/") + if len(parts) != 4: + raise RuntimeError("Can't parse reference") + if parts[2] == "": + parts[2] = get_arch() + + return collection, "/".join(parts) From 1fbff2ae21c33026dacb42d94edebb68b0b19771 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Fri, 14 Feb 2025 17:47:59 +0100 Subject: [PATCH 16/17] Fix wrong arch is taken for flatpak ref resolution We should be taking podman/container architectures not blivet ones. --- pyanaconda/modules/payloads/payload/flatpak/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyanaconda/modules/payloads/payload/flatpak/utils.py b/pyanaconda/modules/payloads/payload/flatpak/utils.py index f34a5efa456..a5d6ff034cf 100644 --- a/pyanaconda/modules/payloads/payload/flatpak/utils.py +++ b/pyanaconda/modules/payloads/payload/flatpak/utils.py @@ -58,6 +58,6 @@ def canonicalize_flatpak_ref(ref) -> Tuple[Optional[str], str]: if len(parts) != 4: raise RuntimeError("Can't parse reference") if parts[2] == "": - parts[2] = get_arch() + parts[2] = get_container_arch() return collection, "/".join(parts) From 455759b7ffefffec9fdfc78ff76666496a0f27d6 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Fri, 14 Feb 2025 17:49:43 +0100 Subject: [PATCH 17/17] Add tests for Flatpak payload utils --- .../test_module_payload_flatpak_utils.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_flatpak_utils.py diff --git a/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_flatpak_utils.py b/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_flatpak_utils.py new file mode 100644 index 00000000000..0bfb1b38e8a --- /dev/null +++ b/tests/unit_tests/pyanaconda_tests/modules/payloads/payload/test_module_payload_flatpak_utils.py @@ -0,0 +1,67 @@ +# +# Copyright (C) 2025 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# +# Red Hat Author(s): Jiri Konecny +# +import unittest +from unittest.mock import patch + +import pytest + +from pyanaconda.modules.payloads.payload.flatpak.utils import ( + canonicalize_flatpak_ref, + get_container_arch, +) + + +class FlatpakUtilsTestCase(unittest.TestCase): + """Test flatpak module utility functions.""" + + @patch("pyanaconda.modules.payloads.payload.flatpak.utils.get_arch") + def test_get_container_arch(self, get_arch): + """Test get_container_arch function.""" + + get_arch.return_value = "x86_64" + assert get_container_arch() == "amd64" + + get_arch.return_value = "aarch64" + assert get_container_arch() == "arm64" + + # Assisted by watsonx Code Assistant + @patch("pyanaconda.modules.payloads.payload.flatpak.utils.get_container_arch") + def test_canonicalize_flatpak_ref(self, get_container_arch_mock): + """Test canonicalize_flatpak_ref function.""" + get_container_arch_mock.return_value = "amd64" + + ref = "org.fedoraproject.Stable:app/org.example.Foo//stable" + collection, ref = canonicalize_flatpak_ref(ref) + assert collection == "org.fedoraproject.Stable" + assert ref == "app/org.example.Foo/amd64/stable" + + ref = "app/org.example.Foo//stable" + collection, ref = canonicalize_flatpak_ref(ref) + assert collection is None + assert ref == "app/org.example.Foo/amd64/stable" + + ref = "app/org.example.Foo/arm64/stable" + collection, ref = canonicalize_flatpak_ref(ref) + assert collection is None + assert ref == "app/org.example.Foo/arm64/stable" + + ref = "org.example.Foo//stable" + with pytest.raises(RuntimeError): + canonicalize_flatpak_ref(ref)