Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Bug] Fix flakiness of RayService e2e tests #1385

Merged
merged 1 commit into from
Sep 1, 2023
Merged

[Bug] Fix flakiness of RayService e2e tests #1385

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions ray-operator/config/samples/ray-service.autoscaler.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,13 @@ spec:
deployments:
- name: Blocked
autoscaling_config:
metrics_interval_s: 0.1
metrics_interval_s: 0.2
min_replicas: 1
max_replicas: 14
look_back_period_s: 0.2
downscale_delay_s: 0
upscale_delay_s: 0
graceful_shutdown_timeout_s: 1
look_back_period_s: 2
downscale_delay_s: 5
upscale_delay_s: 2
graceful_shutdown_timeout_s: 5
max_concurrent_queries: 1000
ray_actor_options:
num_cpus: 0.5
Expand Down
2 changes: 1 addition & 1 deletion tests/framework/prototype.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ def assert_rule(self, custom_resource, cr_namespace):
# If curl pod doesn't exist, create one
if get_pod("default", "run=curl") is None:
start_curl_pod("curl", cr_namespace, timeout_s=30)

for query in self.queries:
cmd = self.CURL_CMD_FMT.format(
name=custom_resource["metadata"]["name"],
Expand Down
8 changes: 4 additions & 4 deletions tests/test_sample_rayservice_yamls.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,11 @@ def wait(self):
namespace = self.namespace, label_selector='ray.io/node-type=head')
workerpods = k8s_v1_api.list_namespaced_pod(
namespace = self.namespace, label_selector='ray.io/node-type=worker')
head_services = k8s_v1_api.list_namespaced_service(
serve_services = k8s_v1_api.list_namespaced_service(
namespace = self.namespace, label_selector =
f"ray.io/serve={self.custom_resource_object['metadata']['name']}-serve")

if (len(head_services.items) == 1 and len(headpods.items) == expected_head_pods
if (len(serve_services.items) == 1 and len(headpods.items) == expected_head_pods
and len(workerpods.items) == expected_worker_pods
and check_pod_running(headpods.items) and check_pod_running(workerpods.items)):
logger.info("--- RayServiceAddCREvent %s seconds ---", time.time() - start_time)
Expand Down Expand Up @@ -287,14 +287,14 @@ def test_zero_downtime_rollout(self, set_up_cluster):
custom_resource_object=self.cr,
rulesets=[RuleSet([EasyJobRule(), CurlServiceRule(queries=self.default_queries)])],
filepath=self.sample_path
),
),
RayServiceUpdateCREvent(
custom_resource_object=self.cr,
rulesets=[RuleSet([CurlServiceRule(queries=updated_queries)])],
filepath=yaml_copy.name,
switch_cluster=True,
query_while_updating=allowed_queries_during_update,
),
),
RayServiceDeleteCREvent(custom_resource_object=self.cr, filepath=self.sample_path),
]

Expand Down