salt: Add a check about conflicting packages for MetalK8s

If some package are installed on the host where we want to deploy
MetalK8s the installation does not work (e.g: containerd.io)
Add a new function to check that those package are not installed on the
host before deploying all the MetalK8s components.
NOTE: We do not automatically uninstall the package from the host since
those packages may have been installed for good reason, so just ask the
user to remove those packages

Fixes: #2992

CHANGELOG.md

@@ -27,6 +27,10 @@
   reducing disk space used by the ISO and in image caches
   (PR [#3047](https://github.com/scality/metalk8s/pull/3047))
 
+- [#2992](https://github.com/scality/metalk8s/issues/2992) - Check for conflicting
+  packages already installed on the machine before doing all the installation
+  (PR [#3050](https://github.com/scality/metalk8s/pull/3050))
+
 ### Bug fixes
 - [#3022](https://github.com/scality/metalk8s/issues/3022) - Ensure salt-master
   container can start at reboot even if local salt-minion is down

buildchain/buildchain/iso.py

@@ -64,9 +64,16 @@
     helper.FileTree(
         basename='_iso_add_tree',
         files=(
-            Path('common.sh'),
             Path('iso-manager.sh'),
             Path('solutions.sh'),
+            helper.TemplateFile(
+                task_name='common.sh',
+                source=constants.ROOT/'scripts'/'common.sh.in',
+                destination=constants.ISO_ROOT/'common.sh',
+                context={'VERSION': versions.VERSION},
+                file_dep=[versions.VERSION_FILE],
+                task_dep=['_iso_mkdir_root'],
+            ),
             helper.TemplateFile(
                 task_name='downgrade.sh',
                 source=constants.ROOT/'scripts'/'downgrade.sh.in',

salt/_modules/metalk8s_checks.py

@@ -12,6 +12,81 @@ def __virtual__():
     return __virtualname__
 
 
+def node(raises=True, **kwargs):
+    """Check if the current Salt-minion match some requirements so that it can
+    be used as a MetalK8s node.
+
+    Arguments:
+        raises (bool): the method will raise if there is any problem.
+    """
+    errors = []
+
+    # Run `packages` check
+    pkg_ret = __salt__['metalk8s_checks.packages'](raises=False, **kwargs)
+    if pkg_ret is not True:
+        errors.append(pkg_ret)
+
+    # Compute return of the function
+    if errors:
+        error_msg = 'Node {}: {}'.format(__grains__['id'], '\n'.join(errors))
+        if raises:
+            raise CheckError(error_msg)
+        return error_msg
+
+    return True
+
+
+def packages(conflicting_packages=None, raises=True, **kwargs):
+    """Check if some conflicting package are installed on the machine,
+    return a string (or raise if `raises` is set to `True`) with the list of
+    conflicting packages.
+
+    Arguments:
+        conflicting_packages (dict): override the list of package that conflict
+            with MetalK8s installation.
+        raises (bool): the method will raise if there is any conflicting
+            package.
+    """
+    if conflicting_packages is None:
+        conflicting_packages = __salt__['metalk8s.get_from_map'](
+            'repo', saltenv=kwargs.get('saltenv')
+        )['conflicting_packages']
+
+    if isinstance(conflicting_packages, str):
+        conflicting_packages = {conflicting_packages: None}
+    elif isinstance(conflicting_packages, list):
+        conflicting_packages = {
+            package: None
+            for package in conflicting_packages
+        }
+    errors = []
+
+    installed_packages = __salt__['pkg.list_pkgs'](attr="version")
+    for package, version in conflicting_packages.items():
+        if isinstance(version, str):
+            version = [version]
+        if package in installed_packages:
+            conflicting_versions = set(
+                pkg_info['version']
+                for pkg_info in installed_packages[package]
+            )
+            if version:
+                conflicting_versions &= set(version)
+
+            if conflicting_versions:
+                for ver in sorted(conflicting_versions):
+                    errors.append(
+                        "Package {}-{} conflict with MetalK8s installation, "
+                        "please remove it.".format(package, ver)
+                    )
+
+    error_msg = '\n'.join(errors)
+    if error_msg and raises:
+        raise CheckError(error_msg)
+
+    return error_msg or True
+
+
 def sysctl(params, raises=True):
     """Check if the given sysctl key-values match the ones in memory and
     return a string (or raise if `raises` is set to `True`) with the list

salt/metalk8s/defaults.yaml

@@ -37,6 +37,14 @@ kubeadm_preflight:
       - coreutils             # provides touch
 
 repo:
+  conflicting_packages:
+    # List of package that conflict with MetalK8s installation
+    # <package_name>: [<version1>, <version2>]
+    # Is version list is None then we consider all versions as conflicting
+    docker: null
+    docker-ce: null
+    containerd.io: null
+
   config:
     directory: '/var/lib/metalk8s/repositories/conf.d'
     default: 'default.conf'

salt/metalk8s/orchestrate/deploy_node.sls

@@ -1,3 +1,5 @@
+{%- from "metalk8s/map.jinja" import repo with context %}
+
 {%- set node_name = pillar.orchestrate.node_name %}
 {%- set run_drain = not pillar.orchestrate.get('skip_draining', False) %}
 {%- set version = pillar.metalk8s.nodes[node_name].version %}
@@ -17,6 +19,23 @@ Install python36:
     - raw_shell: true
     - roster: kubernetes
 
+Check node:
+  metalk8s.saltutil_cmd:
+    - name: metalk8s_checks.node
+    - tgt: {{ node_name }}
+    - ssh: true
+    - roster: kubernetes
+    - kwarg:
+        # NOTE: We need to use the `conflicting_packages` from the salt
+        # master since in salt-ssh when running an execution module we cannot
+        # embbed additional files (especially `map.jinja` in this case)
+        # Sees: https://github.com/saltstack/salt/issues/59314
+        conflicting_packages: >-
+          {{ repo.conflicting_packages | tojson }}
+    - failhard: true
+    - require:
+      - metalk8s: Install python36
+
 Deploy salt-minion on a new node:
   salt.state:
     - ssh: true
@@ -25,6 +44,8 @@ Deploy salt-minion on a new node:
     - saltenv: metalk8s-{{ version }}
     - sls:
       - metalk8s.roles.minion
+    - require:
+      - metalk8s: Check node
 
 Accept key:
   module.run:

salt/tests/unit/modules/files/test_metalk8s_checks.yaml

@@ -1,3 +1,160 @@
+node:
+  - packages_ret: True
+    result: True
+  - packages_ret: "Package abcd got an error because of banana"
+    expect_raise: True
+    result: "Node my_node_1: Package abcd got an error because of banana"
+  - packages_ret: "Package toto got an error :)"
+    raises: False
+    result: "Node my_node_1: Package toto got an error :)"
+
+packages:
+  # 1. Success: No conflicting packages to check
+  - conflicting_packages: {}
+    result: True
+
+  # 2. Success: Conflicting packages not installed
+  - conflicting_packages:
+      my-not-installed-package: null
+      my-not-installed-package2: null
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+    result: True
+  # 2. bis
+  - conflicting_packages: my-not-installed-package
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+    result: True
+  # 2. ter
+  - conflicting_packages:
+     - my-not-installed-package
+     - my-not-installed-package2
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+    result: True
+
+  # 3. Success: Conflicting packages (from map) not installed
+  - get_map_ret:
+      conflicting_packages:
+        my-not-installed-package: null
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+    result: True
+
+  # 4. Success: Conflicting packages in specific version not installed
+  - conflicting_packages:
+      my-installed-package:
+        - 1.2.4
+        - 1.2.5
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+    result: True
+
+  # 5. Error: Conflicting packages installed
+  - conflicting_packages:
+      my-installed-package: null
+      my-not-installed-package: null
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+    expect_raise: True
+    result: |-
+      Package my-installed-package-1.2.3 conflict with MetalK8s installation, please remove it.
+  # 5. bis
+  - conflicting_packages: my-installed-package
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+    expect_raise: True
+    result: |-
+      Package my-installed-package-1.2.3 conflict with MetalK8s installation, please remove it.
+  # 5. ter
+  - conflicting_packages:
+      - my-not-installed-package
+      - my-installed-package
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+    expect_raise: True
+    result: |-
+      Package my-installed-package-1.2.3 conflict with MetalK8s installation, please remove it.
+
+  # 6. Error: Conflicting packages in specific version installed
+  - conflicting_packages:
+      my-installed-package: 1.2.3
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+    expect_raise: True
+    result: |-
+      Package my-installed-package-1.2.3 conflict with MetalK8s installation, please remove it.
+  # 6. bis
+  - conflicting_packages:
+      my-installed-package:
+        - 0.1.3
+        - 1.2.4
+        - 1.2.3
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+    expect_raise: True
+    result: |-
+      Package my-installed-package-1.2.3 conflict with MetalK8s installation, please remove it.
+  # 6. ter (no raise)
+  - conflicting_packages:
+      my-installed-package:
+        - 0.1.3
+        - 1.2.4
+        - 1.2.3
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+    raises: False
+    result: |-
+      Package my-installed-package-1.2.3 conflict with MetalK8s installation, please remove it.
+
+  # 7. Error: Multiple conflicting package installed
+  - conflicting_packages:
+      my-installed-package1: null
+      my-installed-package2:
+        - 5.6.7
+    list_pkgs_ret:
+      my-installed-package1:
+        - version: 1.2.3
+      my-installed-package2:
+        - version: 5.6.7
+    expect_raise: True
+    result: |-
+      Package my-installed-package1-1.2.3 conflict with MetalK8s installation, please remove it.
+      Package my-installed-package2-5.6.7 conflict with MetalK8s installation, please remove it.
+
+  # 8. Error: Multiple package of same version installed
+  - conflicting_packages:
+      my-installed-package: null
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+        - version: 4.5.6
+    expect_raise: True
+    result: |-
+      Package my-installed-package-1.2.3 conflict with MetalK8s installation, please remove it.
+      Package my-installed-package-4.5.6 conflict with MetalK8s installation, please remove it.
+  # 8. bis
+  - conflicting_packages:
+      my-installed-package: 4.5.6
+    list_pkgs_ret:
+      my-installed-package:
+        - version: 1.2.3
+        - version: 4.5.6
+    expect_raise: True
+    result: |-
+      Package my-installed-package-4.5.6 conflict with MetalK8s installation, please remove it.
+
 sysctl:
   - params:
       net.ipv4.ip_forward: 1

salt/tests/unit/modules/test_metalk8s_checks.py

@@ -32,6 +32,60 @@ def test_virtual(self):
         """
         self.assertEqual(metalk8s_checks.__virtual__(), 'metalk8s_checks')
 
+    @utils.parameterized_from_cases(YAML_TESTS_CASES["node"])
+    def test_node(self, packages_ret, result, expect_raise=False, **kwargs):
+        """
+        Tests the return of `node` function
+        """
+        packages_mock = MagicMock(return_value=packages_ret)
+
+        salt_dict = {
+            'metalk8s_checks.packages': packages_mock
+        }
+
+        with patch.dict(metalk8s_checks.__grains__, {'id': 'my_node_1'}), \
+                patch.dict(metalk8s_checks.__salt__, salt_dict):
+            if expect_raise:
+                self.assertRaisesRegex(
+                    CheckError,
+                    result,
+                    metalk8s_checks.node,
+                    **kwargs
+                )
+            else:
+                self.assertEqual(
+                    metalk8s_checks.node(**kwargs),
+                    result
+                )
+
+    @utils.parameterized_from_cases(YAML_TESTS_CASES["packages"])
+    def test_packages(self, result, get_map_ret=None, list_pkgs_ret=None,
+                      expect_raise=False, **kwargs):
+        """
+        Tests the return of `packages` function
+        """
+        get_map_mock = MagicMock(return_value=get_map_ret)
+        list_pkgs_mock = MagicMock(return_value=list_pkgs_ret or {})
+
+        salt_dict = {
+            'metalk8s.get_from_map': get_map_mock,
+            'pkg.list_pkgs': list_pkgs_mock
+        }
+
+        with patch.dict(metalk8s_checks.__salt__, salt_dict):
+            if expect_raise:
+                self.assertRaisesRegex(
+                    CheckError,
+                    result,
+                    metalk8s_checks.packages,
+                    **kwargs
+                )
+            else:
+                self.assertEqual(
+                    metalk8s_checks.packages(**kwargs),
+                    result
+                )
+
     @utils.parameterized_from_cases(YAML_TESTS_CASES["sysctl"])
     def test_sysctl(self, params, data, result, raises=False):
         """

scripts/bootstrap.sh.in

@@ -182,6 +182,7 @@ main() {
     run "Installing mandatory packages" install_packages "${PACKAGES[@]}"
     run "Configuring Salt minion to run in local mode" configure_salt_minion_local_mode
     run "Ensure archive is available" ensure_archives_mounted
+    run "Checking local node" check_local_node
 
     orchestrate_bootstrap