From 7077bd0cbdc9346bd667dffc5e89e04f7ff17489 Mon Sep 17 00:00:00 2001 From: jiafu zhang Date: Tue, 28 Mar 2023 05:37:05 +0000 Subject: [PATCH 1/2] [raydp-321] Prepend ":job_id:" to java-worker--.log to avoid too many executor logs flushed to shell console Signed-off-by: jiafu zhang --- .../src/main/java/org/apache/spark/raydp/Agent.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/agent/src/main/java/org/apache/spark/raydp/Agent.java b/core/agent/src/main/java/org/apache/spark/raydp/Agent.java index 7c35edaf..1bbfaec0 100644 --- a/core/agent/src/main/java/org/apache/spark/raydp/Agent.java +++ b/core/agent/src/main/java/org/apache/spark/raydp/Agent.java @@ -17,9 +17,11 @@ package org.apache.spark.raydp; +import org.apache.commons.lang3.StringUtils; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.io.PrintStream; import java.lang.instrument.Instrumentation; @@ -68,5 +70,12 @@ public static void premain(String agentArgs, Instrumentation inst) System.setErr(DEFAULT_ERR_PS); System.setOut(DEFAULT_OUT_PS); } + String jobId = System.getenv("RAY_JOB_ID"); + String rayAddress = System.getProperty("ray.address"); + if (jobId != null && rayAddress != null) { + try (FileWriter writer = new FileWriter(logDir + "/java-worker-" + jobId + "-" + pid + ".log")) { + writer.write(":job_id:" + jobId + "\n"); + } + } } } From 868f66d0b4018226188f9359d04cbf9ecce43a73 Mon Sep 17 00:00:00 2001 From: jiafu zhang Date: Wed, 29 Mar 2023 01:53:05 +0000 Subject: [PATCH 2/2] [raydp-321] Prepend ":job_id:" to java-worker--.log to avoid too many executor logs flushed to shell console Signed-off-by: jiafu zhang --- core/agent/src/main/java/org/apache/spark/raydp/Agent.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/agent/src/main/java/org/apache/spark/raydp/Agent.java b/core/agent/src/main/java/org/apache/spark/raydp/Agent.java index 1bbfaec0..50bdf8cd 100644 --- a/core/agent/src/main/java/org/apache/spark/raydp/Agent.java +++ b/core/agent/src/main/java/org/apache/spark/raydp/Agent.java @@ -17,7 +17,6 @@ package org.apache.spark.raydp; -import org.apache.commons.lang3.StringUtils; import org.slf4j.LoggerFactory; import java.io.File;