Fix a bug where salt-minion process does not restart during upgrade

CHANGELOG.md

@@ -12,6 +12,11 @@
   - Add bootstrap and solutions configuration files
   (PR [#3222](https://github.com/scality/metalk8s/pull/3222))
 
+### Bug fixes
+- [#3247](https://github.com/scality/metalk8s/issues/3247) - Fix a bug where
+  Salt minion process may fail to restart during upgrade or downgrade process
+  (PR [#3281](https://github.com/scality/metalk8s/pull/3281))
+
 ## Release 2.5.2
 
 ### Enhancements

buildchain/buildchain/salt_tree.py

@@ -561,7 +561,7 @@ def _get_parts(self) -> Iterator[str]:
     Path('salt/metalk8s/salt/minion/init.sls'),
     Path('salt/metalk8s/salt/minion/installed.sls'),
     Path('salt/metalk8s/salt/minion/local.sls'),
-    Path('salt/metalk8s/salt/minion/running.sls'),
+    Path('salt/metalk8s/salt/minion/restart.sls'),
 
     Path('salt/metalk8s/solutions/available.sls'),
     Path('salt/metalk8s/solutions/init.sls'),

salt/metalk8s/orchestrate/deploy_node.sls

@@ -23,7 +23,7 @@ Accept key:
     - require:
       - salt: Deploy salt-minion on a new node
 
-Wait minion available:
+Wait minion available ssh:
   salt.runner:
     - name: metalk8s_saltutil.wait_minions
     - tgt: {{ node_name }}
@@ -73,8 +73,6 @@ Sync module on the node:
     - kwarg:
         saltenv: {{ saltenv }}
 
-{%- if node_name in salt.saltutil.runner('manage.up') %}
-
 Check pillar before salt-minion configuration:
   salt.function:
     - name: metalk8s.check_pillar_keys
@@ -109,16 +107,20 @@ Reconfigure salt-minion:
       - salt: Check pillar before salt-minion configuration
 
 Wait minion available:
+  test.configurable_test_state:
+    - changes: False
+    - result: __slot__:salt:test.sleep(10)
+    - comment: Wait a bit for 'salt-minion' to restart before checking status
+    - onchanges:
+      - salt: Reconfigure salt-minion
   salt.runner:
     - name: metalk8s_saltutil.wait_minions
     - tgt: {{ node_name }}
     - require:
-      - salt: Reconfigure salt-minion
+      - test: Wait minion available
     - require_in:
       - http: Wait for API server to be available before highstate
 
-{%- endif %}
-
 {%- if 'etcd' in roles and 'etcd' not in skip_roles %}
 
 Check pillar before etcd deployment:

salt/metalk8s/salt/master/files/master-99-metalk8s.conf.j2

@@ -2,6 +2,10 @@ interface: {{ salt_ip }}
 
 log_level: {{ 'debug' if debug else 'info' }}
 
+timeout: 20
+sock_pool_size: 15
+worker_threads: 10
+
 peer:
   .*:
     - x509.sign_remote_certificate

salt/metalk8s/salt/minion/configured.sls

@@ -2,7 +2,7 @@
 
 include:
   - .installed
-  - .running
+  - .restart
 
 Configure salt minion:
   file.managed:

salt/metalk8s/salt/minion/init.sls

@@ -8,9 +8,12 @@
 # Available states
 # ================
 #
-# * running   -> Ensure salt minion running
+# * local       -> Configure salt minion so that it can run in local mode
+#                  (need to have access on the ISO, so only run it on the bootstrap)
+# * installed   -> Install/Upgrade and enable salt minion
+# * configured  -> Configure salt minion process
+# * restart     -> Restart salt Minion if required `watch_in` set
 #
 
 include:
-  - .installed
-  - .running
+  - .configured

salt/metalk8s/salt/minion/installed.sls

@@ -2,7 +2,7 @@
 
 include :
   - metalk8s.repo
-  - .running
+  - .restart
 
 Install salt-minion:
   {{ pkg_installed('salt-minion') }}
@@ -11,5 +11,11 @@ Install salt-minion:
     - order: last
     - require:
       - test: Repositories configured
-    - require_in:
+    - watch_in:
       - cmd: Restart salt-minion
+
+Enable Salt minion:
+  service.enabled:
+    - name: salt-minion
+    - require:
+      - metalk8s_package_manager: Install salt-minion

salt/metalk8s/salt/minion/restart.sls

@@ -0,0 +1,4 @@
+Restart salt-minion:
+  cmd.wait:
+    - name: 'salt-call --local service.restart salt-minion > /dev/null'
+    - bg: True