check api resources before install proxy (#1210)

* Find root cause:check api resources before install proxy * Solution: ensure daemonset registerd before start kube-proxy
microsoft · Aug 31, 2018 · 4da827a · 4da827a
1 parent 21ca648
commit 4da827a
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 1 deletion.
diff --git a/pai-management/k8sPaiLibrary/maintainlib/common.py b/pai-management/k8sPaiLibrary/maintainlib/common.py
@@ -66,7 +66,7 @@ def execute_shell_retry(shell_cmd, error_msg, retry_count):
         except subprocess.CalledProcessError:
             count += 1
             logger.error(error_msg)
-            logger.info("%s error, retrying %d", shell_cmd, count)
+            logger.info("run command \" %s \" exception, retrying %d", shell_cmd, count)
             if count == retry_count:
                 sys.exit(1)
             time.sleep(5)

diff --git a/pai-management/k8sPaiLibrary/maintainlib/deploy.py b/pai-management/k8sPaiLibrary/maintainlib/deploy.py
@@ -203,6 +203,12 @@ def run(self):
         kubectl_install_instance = kubectl_install.kubectl_install(self.cluster_config)
         kubectl_install_instance.run()
 
+        # check the registerd api resources
+        common.execute_shell_retry("kubectl api-resources", "kubectl command failed!", 5)
+
+        # create kube-proxy until daemonset resource is registerd
+        common.execute_shell_retry("kubectl api-resources | grep -q daemonsets", "Controller manager hasn't create daemonset object!", 5)
+
         self.create_kube_proxy()
         self.create_k8s_dashboard()