Migration to Python3

CHANGELOG.md

@@ -9,6 +9,9 @@
 
 - Bump `coredns` version to 1.6.7 (PR [#2816](https://github.com/scality/metalk8s/pull/2816))
 
+- [#2203](https://github.com/scality/metalk8s/issues/2203) - Migrate Salt to Python3
+  and bump to version 3002.2 (PR [#2839](https://github.com/scality/metalk8s/pull/2839))
+
 ### Bug fixes
 
 - [#2908](https://github.com/scality/metalk8s/issues/2908) - Make upgrade script

buildchain/buildchain/salt_tree.py

@@ -563,6 +563,7 @@ def _get_parts(self) -> Iterator[str]:
     Path('salt/metalk8s/orchestrate/apiserver.sls'),
     Path('salt/metalk8s/orchestrate/deploy_node.sls'),
     Path('salt/metalk8s/orchestrate/etcd.sls'),
+    Path('salt/metalk8s/orchestrate/migrate_salt.sls'),
     Path('salt/metalk8s/orchestrate/register_etcd.sls'),
 
     Path('salt/metalk8s/orchestrate/bootstrap/init.sls'),
@@ -635,6 +636,7 @@ def _get_parts(self) -> Iterator[str]:
     Path('salt/metalk8s/salt/master/certs/salt-api.sls'),
 
     Path('salt/metalk8s/salt/minion/configured.sls'),
+    Path('salt/metalk8s/salt/minion/dependencies.sls'),
     Path('salt/metalk8s/salt/minion/files/minion-99-metalk8s.conf.j2'),
     Path('salt/metalk8s/salt/minion/init.sls'),
     Path('salt/metalk8s/salt/minion/installed.sls'),

buildchain/buildchain/versions.py

@@ -19,7 +19,7 @@
 
 CALICO_VERSION     : str = '3.16.1'
 K8S_VERSION        : str = '1.18.12'
-SALT_VERSION       : str = '3000.5'
+SALT_VERSION       : str = '3002.2'
 CONTAINERD_VERSION : str = '1.4.1'
 CONTAINERD_RELEASE : str = '1.el7'
 
@@ -353,6 +353,7 @@ def deb_full_name(self) -> str:
             version=SHORT_VERSION,
             release='1.el7'
         ),
+        PackageVersion(name='python36-rpm'),
         PackageVersion(name='yum-plugin-versionlock'),
         PackageVersion(name='yum-utils'),
     ),

docs/installation/expansion.rst

@@ -237,11 +237,26 @@ Open a terminal in the Salt Master container using
 #. Check that SSH access from the Salt Master to the new node is properly
    configured (see :ref:`Bootstrap SSH Provisioning`).
 
+   .. note::
+
+     Salt SSH requires Python 3 to be installed on the remote host to run Salt
+     functions. It will be installed automatically when deploying the node,
+     though you can send raw shell commands before (using ``--raw-shell``) if
+     needed.
+
    .. code-block:: shell
 
-      root@salt-master-bootstrap $ salt-ssh --roster kubernetes <node-name> test.ping
-      <node-name>:
-          True
+      root@salt-master-bootstrap $ salt-ssh --roster=kubernetes <node_name> --raw-shell 'echo OK'
+      <node_name>:
+          ----------
+          retcode:
+              0
+          stderr:
+              Warning: Permanently added '<ip>' (ECDSA) to the list of known hosts.
+          stdout:
+              OK
+
+
 
 #. Start the node deployment.
 

images/metalk8s-utils/configure-repos.sh

@@ -19,8 +19,8 @@ EOF
 cat > /etc/yum.repos.d/saltstack.repo << EOF
 [saltstack]
 name=SaltStack repo for RHEL/CentOS \$releasever
-baseurl=https://repo.saltstack.com/yum/redhat/\$releasever/\$basearch/archive/$SALT_VERSION
+baseurl=https://repo.saltstack.com/py3/redhat/\$releasever/\$basearch/archive/$SALT_VERSION
 enabled=1
 gpgcheck=1
-gpgkey=https://repo.saltstack.com/yum/redhat/\$releasever/\$basearch/archive/$SALT_VERSION/SALTSTACK-GPG-KEY.pub
+gpgkey=https://repo.saltstack.com/py3/redhat/\$releasever/\$basearch/archive/$SALT_VERSION/SALTSTACK-GPG-KEY.pub
 EOF

images/salt-master/Dockerfile

@@ -3,25 +3,25 @@ FROM centos:7.6.1810
 MAINTAINER moonshot-platform <[email protected]>
 
 # Versions to use
-ARG SALT_VERSION=3000.5
+ARG SALT_VERSION=3002.2
 
 # Install saltstack
 RUN printf "[saltstack-repo]\n\
 name=SaltStack repo for RHEL/CentOS \$releasever\n\
-baseurl=https://repo.saltstack.com/yum/redhat/\$releasever/\$basearch/archive/%s\n\
+baseurl=https://repo.saltstack.com/py3/redhat/\$releasever/\$basearch/archive/%s\n\
 enabled=1\n\
 gpgcheck=1\n\
-gpgkey=https://repo.saltstack.com/yum/redhat/\$releasever/\$basearch/archive/%s/SALTSTACK-GPG-KEY.pub\n" ${SALT_VERSION} ${SALT_VERSION} >/etc/yum.repos.d/saltstack.repo \
- && rpm --import https://repo.saltstack.com/yum/redhat/7/x86_64/archive/${SALT_VERSION}/SALTSTACK-GPG-KEY.pub \
+gpgkey=https://repo.saltstack.com/py3/redhat/\$releasever/\$basearch/archive/%s/SALTSTACK-GPG-KEY.pub\n" ${SALT_VERSION} ${SALT_VERSION} >/etc/yum.repos.d/saltstack.repo \
+ && rpm --import https://repo.saltstack.com/py3/redhat/7/x86_64/archive/${SALT_VERSION}/SALTSTACK-GPG-KEY.pub \
  && yum clean expire-cache \
  && yum install -y epel-release \
- && yum install -y python-pip \
- && pip install pip==20.1 \
- && pip install "etcd3 != 0.11.0" \
+ && yum install -y python3-pip \
+ && pip3 install pip==20.1 \
+ && pip3 install "etcd3 != 0.11.0" \
  && yum install -y git \
- && pip install "git+https://github.com/kubernetes-client/python.git@cef5e9bd10a6d5ca4d9c83da46ccfe2114cdaaf8#egg=kubernetes" \
+ && pip3 install "git+https://github.com/kubernetes-client/python.git@cef5e9bd10a6d5ca4d9c83da46ccfe2114cdaaf8#egg=kubernetes" \
  && yum remove -y git \
- && pip uninstall -y \
+ && pip3 uninstall -y \
      requests \
      urllib3 \
  && yum install -y salt-master salt-api salt-ssh openssh-clients \

packages/debian/download_packages.py

@@ -110,12 +110,12 @@ def add_external_repositories(salt_version: str) -> None:
         }, {
             'name': 'saltstack',
             'key': urllib.parse.urljoin(
-                'https://repo.saltstack.com/apt/ubuntu/',
+                'https://repo.saltstack.com/py3/ubuntu/',
                 '18.04/amd64/archive/{}/SALTSTACK-GPG-KEY.pub'.format(
                     salt_version
                 )
             ),
-            'source': 'deb http://repo.saltstack.com/apt/ubuntu/'\
+            'source': 'deb http://repo.saltstack.com/py3/ubuntu/'\
                       '18.04/amd64/archive/{} bionic main'.format(salt_version)
         }
     ]

packages/redhat/yum_repositories/saltstack.repo

@@ -1,6 +1,6 @@
 [saltstack]
 name=SaltStack repo for RHEL/CentOS $releasever
-baseurl=https://repo.saltstack.com/yum/redhat/$releasever/$basearch/archive/@SALT_VERSION@
+baseurl=https://repo.saltstack.com/py3/redhat/$releasever/$basearch/archive/@SALT_VERSION@
 enabled=1
 gpgcheck=1
-gpgkey=https://repo.saltstack.com/yum/redhat/$releasever/$basearch/archive/@SALT_VERSION@/SALTSTACK-GPG-KEY.pub
+gpgkey=https://repo.saltstack.com/py3/redhat/$releasever/$basearch/archive/@SALT_VERSION@/SALTSTACK-GPG-KEY.pub

salt/_modules/metalk8s.py

@@ -171,7 +171,7 @@ def archive_info_from_iso(path):
 
     cmd = ' '.join([
         'isoinfo',
-        '-x', '/PRODUCT.TXT\;1',
+        '-x', r'/PRODUCT.TXT\;1',
         '-i', '"{}"'.format(path),
     ])
     result = __salt__['cmd.run_all'](cmd=cmd)

salt/_modules/metalk8s_solutions.py

@@ -302,7 +302,7 @@ def manifest_from_iso(path):
 
     cmd = ' '.join([
         'isoinfo',
-        '-x', '/{}\;1'.format(SOLUTION_MANIFEST.upper()),
+        '-x', r'/{}\;1'.format(SOLUTION_MANIFEST.upper()),
         '-i', '"{}"'.format(path),
     ])
     result = __salt__['cmd.run_all'](cmd=cmd)

salt/_states/metalk8s.py

@@ -103,3 +103,65 @@ def module_run(name, attemps=1, sleep_time=10, **kwargs):
             time.sleep(sleep_time)
 
     return ret
+
+
+def saltutil_cmd(name, **kwargs):
+    """Simple `saltutil.cmd` state as `salt.function` do not support roster and
+    raw ssh, https://github.com/saltstack/salt/issues/58662"""
+    ret = {
+        'name': name,
+        'changes': {},
+        'result': True,
+        'comment': ''
+    }
+
+    try:
+        cmd_ret = __salt__['saltutil.cmd'](fun=name, **kwargs)
+    except Exception as exc:  # pylint: disable=broad-except
+        ret['result'] = False
+        ret['comment'] = str(exc)
+        return ret
+
+    try:
+        ret['__jid__'] = cmd_ret[next(iter(cmd_ret))]['jid']
+    except (StopIteration, KeyError):
+        pass
+
+    fail = set()
+
+    for minion, mdata in cmd_ret.items():
+        m_ret = False
+        if mdata.get('retcode'):
+            ret['result'] = False
+            fail.add(minion)
+        if mdata.get('failed', False):
+            fail.add(minion)
+        else:
+            if 'return' in mdata and 'ret' not in mdata:
+                mdata['ret'] = mdata.pop('return')
+            if 'ret' in mdata:
+                m_ret = mdata['ret']
+            if 'stderr' in mdata or 'stdout' in mdata:
+                m_ret = {
+                    'retcode': mdata.get('retcode'),
+                    'stderr': mdata.get('stderr'),
+                    'stdout': mdata.get('stdout')
+                }
+            if m_ret is False:
+                fail.add(minion)
+
+        ret['changes'][minion] = m_ret
+
+    if not cmd_ret:
+        ret['result'] = False
+        ret['comment'] = 'No minions responded'
+    else:
+        if fail:
+            ret['result'] = False
+            ret['comment'] = 'Running function {} failed on minions: {}'.format(
+                name, ', '.join(fail)
+            )
+        else:
+            ret['comment'] = 'Function ran successfully'
+
+    return ret

salt/metalk8s/orchestrate/deploy_node.sls

@@ -6,6 +6,16 @@
 {%- set roles = pillar.get('metalk8s', {}).get('nodes', {}).get(node_name, {}).get('roles', []) %}
 
 {%- if node_name not in salt.saltutil.runner('manage.up') %}
+# Salt-ssh need python3 to be installed on the destination host, so install it
+# manually using raw ssh
+Install python36:
+  metalk8s.saltutil_cmd:
+    - name: '[ "$EUID" -eq 0 ] && yum install -y python3 || sudo yum install -y python3'
+    - tgt: {{ node_name }}
+    - ssh: true
+    - raw_shell: true
+    - roster: kubernetes
+
 Deploy salt-minion on a new node:
   salt.state:
     - ssh: true
@@ -140,6 +150,7 @@ Check pillar before etcd deployment:
         attempts: 5
     - require:
       - salt: Sync module on the node
+      - salt: Wait minion available
 
 Install etcd node:
   salt.state:
@@ -243,4 +254,3 @@ Kill kube-controller-manager on all master nodes:
         pattern: kube-controller-manager
     - require:
       - salt: Run the highstate
-

salt/metalk8s/orchestrate/downgrade/init.sls

@@ -39,7 +39,11 @@ Wait for API server to be available on {{ node }}:
   - verify_ssl: false
   - require:
     - salt: Execute the downgrade prechecks
-  {%- if previous_node is defined %}
+  {%- if loop.previtem is defined %}
+    - salt: Deploy node {{ loop.previtem }}
+  {%- endif %}
+  {#- NOTE: This can be removed in `development/2.8` #}
+  {%- if salt.pkg.version_cmp(dest_version, '2.7.0') == -1 and previous_node is defined %}
     - salt: Deploy node {{ previous_node }}
   {%- endif %}
 
@@ -55,6 +59,24 @@ Set node {{ node }} version to {{ dest_version }}:
     - require:
       - http: Wait for API server to be available on {{ node }}
 
+{%- if salt.pkg.version_cmp(dest_version, '2.7.0') == -1 %}
+# We need a new step to upgrade salt-minion as if we downgrade to 2.6.x
+# we have to migrate salt from Python3 to Python2
+# NOTE: This can be removed in `development/2.8`
+Upgrade salt-minion on {{ node }}:
+  salt.runner:
+    - name: state.orchestrate
+    - mods:
+      - metalk8s.orchestrate.migrate_salt
+    - pillar:
+        orchestrate:
+          node_name: {{ node }}
+    - require:
+      - metalk8s_kubernetes: Set node {{ node }} version to {{ dest_version }}
+    - require_in:
+      - salt: Deploy node {{ node }}
+{%- endif %}
+
 Deploy node {{ node }}:
   salt.runner:
     - name: state.orchestrate
@@ -79,8 +101,11 @@ Deploy node {{ node }}:
     - require_in:
       - salt: Downgrade etcd cluster
 
-    {#- Ugly but needed since we have jinja2.7 (`loop.previtem` added in 2.10) #}
-    {%- set previous_node = node %}
+    {#- NOTE: This can be removed in `development/2.8` #}
+    {%- if salt.pkg.version_cmp(dest_version, '2.7.0') == -1 %}
+      {#- Ugly but needed since we have jinja2.7 (`loop.previtem` added in 2.10) #}
+      {%- set previous_node = node %}
+    {%- endif %}
 
   {%- endif %}
 

salt/metalk8s/orchestrate/downgrade/precheck.sls

@@ -4,8 +4,7 @@
 
 {#- When downgrading saltenv should be the newest version #}
 {%- set nodes_versions = pillar.metalk8s.nodes.values() | map(attribute='version') | list %}
-{%- do nodes_versions.sort(cmp=salt.pkg.version_cmp, reverse=True) %}
-{%- set expected = nodes_versions | first %}
+{%- set expected = salt.metalk8s.cmp_sorted(nodes_versions, cmp=salt.pkg.version_cmp, reverse=True) | first %}
 {%- if salt.pkg.version_cmp(saltenv | replace('metalk8s-', ''), expected) < 0 %}
 
 Invalid saltenv "{{ saltenv }}" consider using "metalk8s-{{ expected }}":

salt/metalk8s/orchestrate/migrate_salt.sls

@@ -0,0 +1,24 @@
+# This state is used for Salt migration:
+#  - for upgrade from 2.6.x to 2.7.x (migration from Python2 to Python3)
+#  - for downgrade from 2.7.x to 2.6.x (migration from Python3 to Python2)
+#
+# Salt migration cannot be done automatically by a salt state because of the
+# change between Python2 and Python3
+#
+# NOTE: This orchestrate can be removed in `developement/2.8`
+
+{%- set node_name = pillar.orchestrate.node_name %}
+{%- set version = pillar.metalk8s.nodes[node_name].version %}
+
+# Just retry on the salt minion state since this state only manage
+# salt-minion package it's "ok" to just retry here
+Install Salt-minion:
+  salt.state:
+    - tgt: {{ node_name }}
+    - saltenv: metalk8s-{{ version }}
+    - sls:
+      - metalk8s.salt.minion.installed
+    - timeout: 200
+    - retry:
+        attempts: 3
+        interval: 60

salt/metalk8s/orchestrate/solutions/files/operator/configmap.yaml

@@ -5,11 +5,11 @@ metadata:
   namespace: {{ namespace }}
   labels:
     app: {{ solution }}-operator
-    app.kubernetes.io/name: {{ solution }}-operator
-    app.kubernetes.io/instance: {{ solution }}-operator
+    app.kubernetes.io/name: {{ solution }}-operator
+    app.kubernetes.io/instance: {{ solution }}-operator
     app.kubernetes.io/version: {{ version }}
     app.kubernetes.io/component: operator
-    app.kubernetes.io/part-of: {{ solution }}
+    app.kubernetes.io/part-of: {{ solution }}
 data:
   operator.yaml: |
     apiVersion: solutions.metalk8s.scality.com/v1alpha1

salt/metalk8s/orchestrate/solutions/files/operator/deployment.yaml

@@ -5,11 +5,11 @@ metadata:
   namespace: {{ namespace }}
   labels:
     app: {{ solution }}-operator
-    app.kubernetes.io/name: {{ solution }}-operator
-    app.kubernetes.io/instance: {{ solution }}-operator
+    app.kubernetes.io/name: {{ solution }}-operator
+    app.kubernetes.io/instance: {{ solution }}-operator
     app.kubernetes.io/version: {{ version }}
     app.kubernetes.io/component: operator
-    app.kubernetes.io/part-of: {{ solution }}
+    app.kubernetes.io/part-of: {{ solution }}
 spec:
   replicas: 1
   selector:
@@ -19,11 +19,11 @@ spec:
     metadata:
       labels:
         app: {{ solution }}-operator
-        app.kubernetes.io/name: {{ solution }}-operator
-        app.kubernetes.io/instance: {{ solution }}-operator
+        app.kubernetes.io/name: {{ solution }}-operator
+        app.kubernetes.io/instance: {{ solution }}-operator
         app.kubernetes.io/version: {{ version }}
         app.kubernetes.io/component: operator
-        app.kubernetes.io/part-of: {{ solution }}
+        app.kubernetes.io/part-of: {{ solution }}
         app.kubernetes.io/managed-by: salt
         heritage: salt
     spec: