From 7d2651bbefeb573723de4c76a9a03bebbc1fd771 Mon Sep 17 00:00:00 2001 From: "Wang, Fei" Date: Sun, 22 Dec 2024 23:31:50 -0800 Subject: [PATCH] [CELEBORN-1783][FOLLOWUP][0.4] Fix UT --- .../client/LifecycleManagerCommitFilesSuite.scala | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/client/LifecycleManagerCommitFilesSuite.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/client/LifecycleManagerCommitFilesSuite.scala index 7ccf6c74a47..d39d244826f 100644 --- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/client/LifecycleManagerCommitFilesSuite.scala +++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/client/LifecycleManagerCommitFilesSuite.scala @@ -155,14 +155,18 @@ class LifecycleManagerCommitFilesSuite extends WithShuffleClientSuite with MiniC } celebornConf .set(CelebornConf.CLIENT_PUSH_REPLICATE_ENABLED.key, "false") + val masterConf = Map( + "celeborn.master.host" -> "localhost", + "celeborn.master.port" -> masterPort.toString, + "celeborn.master.endpoints" -> s"localhost:$masterPort") val workerConf0 = Map( s"${CelebornConf.WORKER_SHUFFLE_COMMIT_TIMEOUT.key}" -> "100", s"${CelebornConf.WORKER_COMMIT_THREADS.key}" -> "1", s"${CelebornConf.TEST_MOCK_COMMIT_FILES_FAILURE.key}" -> "true") - val (master, _) = setupMiniClusterWithRandomPorts(workerConf = workerConf0) + val (master, workers) = setUpMiniCluster(masterConf = masterConf, workerConf = workerConf0) celebornConf.set( CelebornConf.MASTER_ENDPOINTS.key, - master.conf.get(CelebornConf.MASTER_ENDPOINTS.key)) + workers.head.conf.get(CelebornConf.MASTER_ENDPOINTS.key)) val shuffleId = nextShuffleId val conf = celebornConf.clone @@ -176,7 +180,7 @@ class LifecycleManagerCommitFilesSuite extends WithShuffleClientSuite with MiniC assert(res.status == StatusCode.SUCCESS) lifecycleManager.setupEndpoints( - res.workerResource.keySet(), + res.workerResource, shuffleId, new ShuffleFailedWorkers()) @@ -186,7 +190,7 @@ class LifecycleManagerCommitFilesSuite extends WithShuffleClientSuite with MiniC res.workerResource, updateEpoch = false) - lifecycleManager.commitManager.registerShuffle(shuffleId, 1, false) + lifecycleManager.commitManager.registerShuffle(shuffleId, 1) 0 until 1000 foreach { partitionId => lifecycleManager.commitManager.finishMapperAttempt(shuffleId, 0, 0, 1, partitionId) }