more

DataBiosphere · Apr 24, 2020 · 705e070 · 705e070
1 parent 32c26e5
commit 705e070
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 9 deletions.
diff --git a/.../org/broadinstitute/dsde/workbench/leonardo/notebooks/NotebookClusterMonitoringSpec.scala b/.../org/broadinstitute/dsde/workbench/leonardo/notebooks/NotebookClusterMonitoringSpec.scala
@@ -12,7 +12,6 @@ import org.broadinstitute.dsde.workbench.model.google.{parseGcsPath, EmailGcsEnt
 import org.broadinstitute.dsde.workbench.service.Sam
 
 import org.scalatest.tagobjects.Retryable
-import org.scalatest.time.{Seconds, Span}
 import org.scalatest.{DoNotDiscover, ParallelTestExecution}
 
 import scala.concurrent.duration._

diff --git a/...src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/GceRuntimeMonitor.scala b/...src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/GceRuntimeMonitor.scala
@@ -41,7 +41,7 @@ object GceMonitorState {
 }
 
 final case class MonitorContext(start: Instant, runtimeId: Long, traceId: TraceId) {
-  override def toString: String = s"runtimeId(${runtimeId}), traceId(${traceId.asString}))"
+  override def toString: String = s"${runtimeId}/${traceId.asString}"
 }
 
 final case class GceMonitorConfig(initialDelay: FiniteDuration,

diff --git a/...in/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/GceRuntimeMonitorInterp.scala b/...in/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/GceRuntimeMonitorInterp.scala
@@ -459,14 +459,14 @@ class GceRuntimeMonitorInterp[F[_]: Timer: Parallel](
             case Some(x) =>
               logger
                 .info(
-                  s"Runtime ${runtimeAndRuntimeConfig.runtime.projectNameString}'s tools(${toolsToCheck}) is not ready yet and has taken ${timeElapsed.toSeconds} seconds so far. Checking again in ${config.pollingInterval}. ${message
+                  s"${monitorContext} | Runtime ${runtimeAndRuntimeConfig.runtime.projectNameString}'s tools(${toolsToCheck}) is not ready yet and has taken ${timeElapsed.toSeconds} seconds so far. Checking again in ${config.pollingInterval}. ${message
                     .getOrElse("")}"
                 )
                 .as(((), Some(CheckTools(x, runtimeAndRuntimeConfig, toolsToCheck))))
             case None =>
               logger
                 .info(
-                  s"Runtime ${runtimeAndRuntimeConfig.runtime.projectNameString} is not in final state yet and has taken ${timeElapsed.toSeconds} seconds so far. Checking again in ${config.pollingInterval}. ${message
+                  s"${monitorContext} | Runtime ${runtimeAndRuntimeConfig.runtime.projectNameString} is not in final state yet and has taken ${timeElapsed.toSeconds} seconds so far. Checking again in ${config.pollingInterval}. ${message
                     .getOrElse("")}"
                 )
                 .as(((), Some(Check(runtimeAndRuntimeConfig))))

diff --git a/...scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala b/...scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala
@@ -17,8 +17,8 @@ import org.broadinstitute.dsde.workbench.leonardo.monitor.LeoPubsubMessage._
 import org.broadinstitute.dsde.workbench.leonardo.util._
 import org.broadinstitute.dsde.workbench.model.google.{GcsObjectName, GcsPath}
 import org.broadinstitute.dsde.workbench.model.{ErrorReport, TraceId, WorkbenchException}
-
 import cats.effect.implicits._
+
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
 import scala.util.control.NoStackTrace
@@ -219,12 +219,14 @@ class LeoPubsubMessageSubscriber[F[_]: Timer: ContextShift](
           _ <- logger.error(e)(s"Failed to create runtime ${msg.runtimeProjectAndName} in Google")
           errorMessage = e match {
             case leoEx: LeoException =>
-              ErrorReport.loggableString(leoEx.toErrorReport)
+              Some(ErrorReport.loggableString(leoEx.toErrorReport))
+            case ee: com.google.api.gax.rpc.AbortedException if ee.getStatusCode().getCode == 409 && ee.getMessage().contains("already exists") =>
+              None //this could happen when pubsub redelivers an event unexpectedly
             case _ =>
-              s"Failed to create cluster ${msg.runtimeProjectAndName} due to ${e.getMessage}"
+              Some(s"Failed to create cluster ${msg.runtimeProjectAndName} due to ${e.getMessage}")
           }
-          _ <- (clusterErrorQuery.save(msg.runtimeId, RuntimeError(errorMessage, -1, now)) >>
-            clusterQuery.updateClusterStatus(msg.runtimeId, RuntimeStatus.Error, now)).transaction[F]
+          _ <- errorMessage.traverse(m => (clusterErrorQuery.save(msg.runtimeId, RuntimeError(m, -1, now)) >>
+            clusterQuery.updateClusterStatus(msg.runtimeId, RuntimeStatus.Error, now)).transaction[F])
         } yield ()
     }
   }