From 653e726daab17412e3553c6707bc232f0e7e278e Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Fri, 5 May 2023 09:18:34 -0400 Subject: [PATCH 1/9] Double the number of Centaur workflows that can run simultaneously --- centaur/test_cromwell.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centaur/test_cromwell.sh b/centaur/test_cromwell.sh index 2f330f1feb0..b88b3899517 100755 --- a/centaur/test_cromwell.sh +++ b/centaur/test_cromwell.sh @@ -37,7 +37,7 @@ Arguments: INITIAL_DIR=$(pwd) RUN_DIR=$(pwd) LOG_DIR="${RUN_DIR}"/logs -TEST_THREAD_COUNT=16 +TEST_THREAD_COUNT=32 CENTAUR_SBT_COVERAGE=false CROMWELL_TIMEOUT=10s SUITE="" From 4bd556da2ad79b4c6799f1aa7a10ff1b7b2f12d3 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 8 May 2023 17:07:39 -0400 Subject: [PATCH 2/9] Diagnostic logging --- .../backend/impl/tes/TesAsyncBackendJobExecutionActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala index f4259de90d8..dde9a644d31 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala @@ -144,7 +144,7 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn jobShell, mode) }) - + jobLogger.info(s"Submitting task, will poll starting each ${pollBackOff.googleBackoff.getInitialIntervalMillis} millis") tesTask.map(TesTask.makeTask) } From ace3a203456cb5ef8b556acac2fab57360e07058 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Tue, 9 May 2023 10:40:27 -0400 Subject: [PATCH 3/9] Remove diagnostic logging --- .../backend/impl/tes/TesAsyncBackendJobExecutionActor.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala index dde9a644d31..4c35d5747a3 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala @@ -144,7 +144,6 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn jobShell, mode) }) - jobLogger.info(s"Submitting task, will poll starting each ${pollBackOff.googleBackoff.getInitialIntervalMillis} millis") tesTask.map(TesTask.makeTask) } From 6d6752ccd22c9ffa9c052c4bae21144597bd06d0 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Tue, 9 May 2023 10:55:29 -0400 Subject: [PATCH 4/9] What happens if this thread count is even higher? --- centaur/test_cromwell.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centaur/test_cromwell.sh b/centaur/test_cromwell.sh index b88b3899517..ef07151211f 100755 --- a/centaur/test_cromwell.sh +++ b/centaur/test_cromwell.sh @@ -37,7 +37,7 @@ Arguments: INITIAL_DIR=$(pwd) RUN_DIR=$(pwd) LOG_DIR="${RUN_DIR}"/logs -TEST_THREAD_COUNT=32 +TEST_THREAD_COUNT=64 CENTAUR_SBT_COVERAGE=false CROMWELL_TIMEOUT=10s SUITE="" From b3147e92be81b97bcc1fcc7d8f956c2663a78bba Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Tue, 9 May 2023 13:28:23 -0400 Subject: [PATCH 5/9] Revert redoubling --- centaur/test_cromwell.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/centaur/test_cromwell.sh b/centaur/test_cromwell.sh index ef07151211f..ac7d544d7c0 100755 --- a/centaur/test_cromwell.sh +++ b/centaur/test_cromwell.sh @@ -30,14 +30,14 @@ Arguments: -i If supplied, will include tests with this tag -e If supplied, will exclude tests with this tag -s If supplied, will run only the specified suite - -p If supplied, number of tests to be run in parallel. 16 is the default + -p If supplied, number of tests to be run in parallel. 32 is the default -d If supplied, will run the tests in this directory instead of the standard tests " INITIAL_DIR=$(pwd) RUN_DIR=$(pwd) LOG_DIR="${RUN_DIR}"/logs -TEST_THREAD_COUNT=64 +TEST_THREAD_COUNT=32 CENTAUR_SBT_COVERAGE=false CROMWELL_TIMEOUT=10s SUITE="" From 8cd4aef5427f5e972c9f898beaa917163a0fcb3f Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Tue, 9 May 2023 14:40:28 -0400 Subject: [PATCH 6/9] Change parallelism in the place that actually matters --- centaur/test_cromwell.sh | 4 ++-- src/ci/bin/testCentaurTes.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/centaur/test_cromwell.sh b/centaur/test_cromwell.sh index ac7d544d7c0..2f330f1feb0 100755 --- a/centaur/test_cromwell.sh +++ b/centaur/test_cromwell.sh @@ -30,14 +30,14 @@ Arguments: -i If supplied, will include tests with this tag -e If supplied, will exclude tests with this tag -s If supplied, will run only the specified suite - -p If supplied, number of tests to be run in parallel. 32 is the default + -p If supplied, number of tests to be run in parallel. 16 is the default -d If supplied, will run the tests in this directory instead of the standard tests " INITIAL_DIR=$(pwd) RUN_DIR=$(pwd) LOG_DIR="${RUN_DIR}"/logs -TEST_THREAD_COUNT=32 +TEST_THREAD_COUNT=16 CENTAUR_SBT_COVERAGE=false CROMWELL_TIMEOUT=10s SUITE="" diff --git a/src/ci/bin/testCentaurTes.sh b/src/ci/bin/testCentaurTes.sh index 8822a727b24..361654d2960 100755 --- a/src/ci/bin/testCentaurTes.sh +++ b/src/ci/bin/testCentaurTes.sh @@ -60,7 +60,7 @@ startup_funnel # Limiting Centaur to 4 threads here in order to try to prevent exceeding the OS's max open files limit: BA-6153 cromwell::build::run_centaur \ - -p 2 \ + -p 8 \ -e docker_hash_dockerhub \ -e call_cache_capoeira_local \ -e draft3_call_cache_capoeira_local \ From 9a79e2c6f224a60daf70a984c9aaeac3eb9980ec Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Tue, 9 May 2023 14:45:12 -0400 Subject: [PATCH 7/9] Revert inadvertent whitespace change --- .../backend/impl/tes/TesAsyncBackendJobExecutionActor.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala index 4c35d5747a3..c38a4d31879 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala @@ -144,6 +144,7 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn jobShell, mode) }) + tesTask.map(TesTask.makeTask) } From 10e41e6e133e7025a2703270bde6e076c99d552b Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Wed, 10 May 2023 09:14:35 -0400 Subject: [PATCH 8/9] Comments --- centaur/test_cromwell.sh | 2 +- src/ci/bin/testCentaurTes.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/centaur/test_cromwell.sh b/centaur/test_cromwell.sh index 2f330f1feb0..9161f2da5ad 100755 --- a/centaur/test_cromwell.sh +++ b/centaur/test_cromwell.sh @@ -37,7 +37,7 @@ Arguments: INITIAL_DIR=$(pwd) RUN_DIR=$(pwd) LOG_DIR="${RUN_DIR}"/logs -TEST_THREAD_COUNT=16 +TEST_THREAD_COUNT=16 # Note that most users of this script override this value CENTAUR_SBT_COVERAGE=false CROMWELL_TIMEOUT=10s SUITE="" diff --git a/src/ci/bin/testCentaurTes.sh b/src/ci/bin/testCentaurTes.sh index 361654d2960..7cad5b26832 100755 --- a/src/ci/bin/testCentaurTes.sh +++ b/src/ci/bin/testCentaurTes.sh @@ -58,7 +58,8 @@ startup_funnel # read_file_limits: Fail only in Travis for unknown reason (Note that the draft 3 version does not fail) # docker_hash_dockerhub: Prone to request rate limiting by Dockerhub -# Limiting Centaur to 4 threads here in order to try to prevent exceeding the OS's max open files limit: BA-6153 +# Setting threads to 8 upon moving from Travis to Github Actions. If we run into problems with +# exceeding the OS's max open files limit, reduce. See BA-6153. cromwell::build::run_centaur \ -p 8 \ -e docker_hash_dockerhub \ From 19e00b5f8d8df0da99a96025641089ce6436f3c2 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Wed, 10 May 2023 09:15:50 -0400 Subject: [PATCH 9/9] Whitespace --- .../backend/impl/tes/TesAsyncBackendJobExecutionActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala index c38a4d31879..f4259de90d8 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala @@ -144,7 +144,7 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn jobShell, mode) }) - + tesTask.map(TesTask.makeTask) }