From 6621f34fdffa004ebcd221abf5901326c45faac8 Mon Sep 17 00:00:00 2001
From: Wilson Wang <3913185+wilsonwang371@users.noreply.github.com>
Date: Tue, 5 Jul 2022 16:34:46 -0700
Subject: [PATCH] fix wrong kuberay image used by compatibility test (#327)

---
 tests/compatibility-test.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/tests/compatibility-test.py b/tests/compatibility-test.py
index 980995a61c..a09cd64079 100755
--- a/tests/compatibility-test.py
+++ b/tests/compatibility-test.py
@@ -18,12 +18,13 @@
 raycluster_service_file = 'tests/config/raycluster-service.yaml'
 
 logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
 
 kuberay_sha = 'nightly'
 
 
 def shell_run(cmd):
-    logger.info(cmd)
+    logger.info('executing cmd: {}'.format(cmd))
     return os.system(cmd)
 
 
@@ -45,8 +46,12 @@ def apply_kuberay_resources():
     shell_assert_success('kind load docker-image kuberay/apiserver:{}'.format(kuberay_sha))
     shell_assert_success(
         'kubectl create -k manifests/cluster-scope-resources')
+    # use kustomize to build the yaml, then change the image to the one we want to testing.
     shell_assert_success(
-        'kubectl create -k manifests/base')
+        ('rm -f kustomization.yaml && kustomize create --resources manifests/base && ' +
+         'kustomize edit set image ' +
+         'kuberay/operator:nightly=kuberay/operator:{0} kuberay/apiserver:nightly=kuberay/apiserver:{0} && ' +
+         'kubectl apply -k .').format(kuberay_sha))
 
 
 def create_kuberay_cluster():
@@ -128,11 +133,11 @@ def f(x):
         container.stop()
 
         if stdout_str != b'[0, 1, 4, 9]\n':
-            print(output, file=sys.stderr)
+            logger.info(output)
             raise Exception('invalid result.')
         if rtn_code != 0:
             msg = 'invalid return code {}'.format(rtn_code)
-            print(msg, file=sys.stderr)
+            logger.info(msg)
             raise Exception(msg)
 
         client.close()
@@ -160,11 +165,11 @@ def test_cluster_info(self):
         container.stop()
 
         if stdout_str != b'2\n':
-            print(output, file=sys.stderr)
+            logger.info(output)
             raise Exception('invalid result.')
         if rtn_code != 0:
             msg = 'invalid return code {}'.format(rtn_code)
-            print(msg, file=sys.stderr)
+            logger.info(msg)
             raise Exception(msg)
 
         client.close()
@@ -178,11 +183,11 @@ def parse_environment():
     global ray_version, ray_image, kuberay_sha
     for k, v in os.environ.items():
         if k == 'RAY_VERSION':
-            print('Setting Ray image to: {}'.format(v), file=sys.stderr)
+            logger.info('Setting Ray image to: {}'.format(v))
             ray_version = v
             ray_image = 'rayproject/ray:{}'.format(ray_version)
         if k == 'KUBERAY_IMG_SHA':
-            print('Using KubeRay docker build SHA: {}'.format(v))
+            logger.info('Using KubeRay docker build SHA: {}'.format(v))
             kuberay_sha = v