Fix CI

mars-project · Sep 16, 2021 · 29a7db2 · 29a7db2
1 parent f313be0
commit 29a7db2
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 5 deletions.
diff --git a/mars/deploy/kubernetes/client.py b/mars/deploy/kubernetes/client.py
@@ -74,7 +74,7 @@ class KubernetesCluster:
     _default_web_port = 7104
 
     def __init__(self, kube_api_client=None, image=None, namespace=None,
-                 supervisor_num=1, supervisor_cpu=None, supervisor_mem=None,
+                 supervisor_num=1, supervisor_cpu=1, supervisor_mem='4G',
                  supervisor_mem_limit_ratio=None,
                  worker_num=1, worker_cpu=None, worker_mem=None,
                  worker_spill_paths=None, worker_cache_mem=None, min_worker_num=None,

diff --git a/mars/deploy/kubernetes/tests/test_kubernetes.py b/mars/deploy/kubernetes/tests/test_kubernetes.py
@@ -160,7 +160,8 @@ def _start_kube_cluster(use_test_docker_file=True, **kwargs):
 @pytest.mark.skipif(not kube_available, reason='Cannot run without kubernetes')
 def test_run_in_kubernetes(use_test_docker_file):
     with _start_kube_cluster(
-            worker_mem='1G', worker_cache_mem='128m',
+            supervisor_cpu=0.5, supervisor_mem='1G',
+            worker_cpu=0.5, worker_mem='1G', worker_cache_mem='64m',
             extra_labels={'mars-test/group': 'test-label-name'},
             extra_env={'MARS_K8S_GROUP_LABELS': 'mars-test/group'},
             use_test_docker_file=use_test_docker_file):
@@ -186,6 +187,8 @@ def test_create_timeout():
         extra_vol_config = HostPathVolumeConfig('mars-src-path', '/mnt/mars', MARS_ROOT)
         with pytest.raises(TimeoutError):
             cluster = new_cluster(api_client, image='pseudo_image',
+                                  supervisor_cpu=0.5, supervisor_mem='1G',
+                                  worker_cpu=0.5, worker_mem='1G',
                                   extra_volumes=[extra_vol_config], timeout=1)
     finally:
         if cluster:

diff --git a/mars/deploy/yarn/tests/test_yarn.py b/mars/deploy/yarn/tests/test_yarn.py
@@ -74,7 +74,8 @@ def _run_yarn_test_with_env(env_path, timeout):
         cmd_tmpl = '"{executable}" -m coverage run --source=%s/mars --rcfile=%s/setup.cfg' \
             % (MARS_ROOT, MARS_ROOT)
         extra_env = {'COVERAGE_FILE': coverage_result, 'COVERAGE_PROCESS_START': f'{MARS_ROOT}/setup.cfg'}
-        cluster = new_cluster(env_path, timeout=timeout, extra_env=extra_env, log_config=log_config_file,
+        cluster = new_cluster(env_path, timeout=timeout, worker_cpu=1, worker_mem='1G',
+                              extra_env=extra_env, log_config=log_config_file,
                               extra_args=f'--config-file {MARS_ROOT}/mars/deploy/yarn/tests/test_yarn_config.yml',
                               log_when_fail=True, cmd_tmpl=cmd_tmpl)
         assert cluster.endpoint is not None
@@ -139,7 +140,9 @@ def test_create_timeout():
         log_config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'yarn-logging.conf')
 
         with pytest.raises(TimeoutError):
-            cluster = new_cluster(env_path, log_config=log_config_file, worker_cache_mem='64m',
+            cluster = new_cluster(env_path, log_config=log_config_file,
+                                  worker_cpu=1, worker_mem='1G',
+                                  worker_cache_mem='64m',
                                   log_when_fail=True, timeout=1)
     finally:
         if cluster is not None:

diff --git a/mars/resource.py b/mars/resource.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import logging
+import math
 import os
 import subprocess  # nosec
 import sys
@@ -48,7 +49,7 @@
     _cpu_use_cgroup_stat = _mem_use_cgroup_stat = _use_cgroup_stat
 
 if 'MARS_CPU_TOTAL' in os.environ:
-    _cpu_total = int(os.environ['MARS_CPU_TOTAL'])
+    _cpu_total = int(math.ceil(float(os.environ['MARS_CPU_TOTAL'])))
 else:
     _cpu_total = psutil.cpu_count(logical=True)