Merge branch 'master' into ci/update-docker-build-image-20250124

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 8b05076e897fe62206d7704f2e8e650ed83ebd1f
+default_system_tests_commit: &default_system_tests_commit c706e333ef06800b866ac300e4b6cdb7566cc5e5
 
 parameters:
   nightly:

.gitlab-ci.yml

@@ -20,24 +20,34 @@ stages:
 variables:
   REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com
   BUILD_JOB_NAME: "build"
+  DEPENDENCY_CACHE_POLICY: pull
+  BUILD_CACHE_POLICY: pull
+  GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties
 
-.common: &common
-  tags: [ "runner:main", "size:large" ]
+default:
+  tags: [ "arch:amd64" ]
 
 .gradle_build: &gradle_build
-  <<: *common
   image: ghcr.io/datadog/dd-trace-java-docker-build:v24.10-base
   variables:
     GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M'"
-    GRADLE_ARGS: " -PskipTests --build-cache --stacktrace --no-daemon --parallel --max-workers=8"
+    GRADLE_ARGS: " -PskipTests --build-cache --stacktrace --no-daemon --parallel --max-workers=2"
     KUBERNETES_CPU_REQUEST: 8
-    KUBERNETES_MEMORY_REQUEST: 4Gi
-  cache: &default_cache
-    key: '$CI_SERVER_VERSION' # Reset the cache every time gitlab is upgraded.  ~Every couple months
-    paths:
-      - .gradle/wrapper
-      - .gradle/caches
-    policy: pull
+    KUBERNETES_MEMORY_REQUEST: 6Gi
+  cache:
+    - key: '$CI_SERVER_VERSION-v2' # Dependencies cache. Reset the cache every time gitlab is upgraded.  ~Every couple months
+      paths:
+        # Cached dependencies and wrappers for gradle
+        - .gradle/wrapper
+        - .gradle/caches
+        - .gradle/notifications
+      policy: $DEPENDENCY_CACHE_POLICY
+    - key: $CI_PIPELINE_ID # Incremental build cache. Shared by all jobs in the pipeline
+      paths:
+        - .gradle/caches/$GRADLE_VERSION
+        - .gradle/$GRADLE_VERSION/executionHistory
+        - workspace
+      policy: $BUILD_CACHE_POLICY
   before_script:
     - export GRADLE_USER_HOME=`pwd`/.gradle
     # for weird reasons, gradle will always "chmod 700" the .gradle folder
@@ -49,9 +59,12 @@ variables:
     - mv .gradle-copy .gradle
     - ls -la
 
-build: &build
-  <<: *gradle_build
+build:
+  extends: .gradle_build
   stage: build
+  variables:
+    BUILD_CACHE_POLICY: push
+    DEPENDENCY_CACHE_POLICY: pull
   script:
     - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar $GRADLE_ARGS
     - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env
@@ -66,16 +79,16 @@ build: &build
     reports:
       dotenv: build.env
 
-build_with_cache:
-  <<: *build
+build_and_populate_dep_cache:
+  extends: build
+  variables:
+    BUILD_CACHE_POLICY: push
+    DEPENDENCY_CACHE_POLICY: push
   rules:
     - if: '$POPULATE_CACHE'
       when: on_success
     - when: manual
       allow_failure: true
-  cache:
-    <<: *default_cache
-    policy: push
 
 deploy_to_profiling_backend:
   stage: publish
@@ -149,7 +162,7 @@ deploy_to_di_backend:manual:
     UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA
 
 deploy_to_sonatype:
-  <<: *gradle_build
+  extends: .gradle_build
   stage: publish
   needs: [ build ]
   rules:
@@ -177,7 +190,6 @@ deploy_to_sonatype:
 deploy_artifacts_to_github:
   stage: publish
   image: registry.ddbuild.io/github-cli:v27480869-eafb11d-2.43.0
-  tags: [ "arch:amd64" ]
   rules:
     - if: '$POPULATE_CACHE'
       when: never
@@ -234,7 +246,6 @@ create_key:
   stage: generate-signing-key
   when: manual
   needs: [ ]
-  tags: [ "arch:amd64", "size:large" ]
   variables:
     PROJECT_NAME: "dd-trace-java"
     EXPORT_TO_KEYSERVER: "true"

communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java

@@ -11,6 +11,7 @@
 import datadog.remoteconfig.DefaultConfigurationPoller;
 import datadog.trace.api.Config;
 import datadog.trace.util.AgentTaskScheduler;
+import java.security.Security;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -59,6 +60,7 @@ public void createRemaining(Config config) {
     }
   }
 
+  /** Registers a callback to be called when remote communications resume. */
   public void whenReady(Runnable callback) {
     if (paused) {
       synchronized (pausedComponents) {
@@ -71,8 +73,15 @@ public void whenReady(Runnable callback) {
     callback.run(); // not paused, run immediately
   }
 
+  /** Resumes remote communications including any paused callbacks. */
   public void resume() {
     paused = false;
+    // attempt discovery first to avoid potential race condition on IBM Java8
+    if (null != featuresDiscovery) {
+      featuresDiscovery.discoverIfOutdated();
+    } else {
+      Security.getProviders(); // fallback to preloading provider extensions
+    }
     synchronized (pausedComponents) {
       for (Runnable callback : pausedComponents) {
         try {

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -1,7 +1,6 @@
 package datadog.trace.bootstrap;
 
 import static datadog.trace.api.ConfigDefaults.DEFAULT_STARTUP_LOGS_ENABLED;
-import static datadog.trace.api.Platform.getRuntimeVendor;
 import static datadog.trace.api.Platform.isJavaVersionAtLeast;
 import static datadog.trace.api.Platform.isOracleJDK8;
 import static datadog.trace.bootstrap.Library.WILDFLY;
@@ -329,7 +328,7 @@ public void run() {
       if (appUsingCustomJMXBuilder) {
         log.debug("Custom JMX builder detected. Delaying JMXFetch initialization.");
         registerMBeanServerBuilderCallback(new StartJmxCallback(jmxStartDelay));
-        // one minute fail-safe in case nothing touches JMX and and callback isn't triggered
+        // one minute fail-safe in case nothing touches JMX and callback isn't triggered
         scheduleJmxStart(60 + jmxStartDelay);
       } else if (appUsingCustomLogManager) {
         log.debug("Custom logger detected. Delaying JMXFetch initialization.");
@@ -339,20 +338,31 @@ public void run() {
       }
     }
 
-    boolean delayOkHttp = appUsingCustomLogManager && okHttpMayIndirectlyLoadJUL();
-
     /*
      * Similar thing happens with DatadogTracer on (at least) zulu-8 because it uses OkHttp which indirectly loads JFR
      * events which in turn loads LogManager. This is not a problem on newer JDKs because there JFR uses different
      * logging facility. Likewise on IBM JDKs OkHttp may indirectly load 'IBMSASL' which in turn loads LogManager.
      */
+    boolean delayOkHttp = !ciVisibilityEnabled && okHttpMayIndirectlyLoadJUL();
+    boolean waitForJUL = appUsingCustomLogManager && delayOkHttp;
+    int okHttpDelayMillis;
+    if (waitForJUL) {
+      okHttpDelayMillis = 1_000;
+    } else if (delayOkHttp) {
+      okHttpDelayMillis = 100;
+    } else {
+      okHttpDelayMillis = 0;
+    }
+
     InstallDatadogTracerCallback installDatadogTracerCallback =
-        new InstallDatadogTracerCallback(initTelemetry, inst, delayOkHttp);
-    if (delayOkHttp) {
+        new InstallDatadogTracerCallback(initTelemetry, inst, okHttpDelayMillis);
+    if (waitForJUL) {
       log.debug("Custom logger detected. Delaying Datadog Tracer initialization.");
       registerLogManagerCallback(installDatadogTracerCallback);
+    } else if (okHttpDelayMillis > 0) {
+      installDatadogTracerCallback.run(); // complete on different thread (after premain)
     } else {
-      installDatadogTracerCallback.execute();
+      installDatadogTracerCallback.execute(); // complete on primordial thread in premain
     }
 
     /*
@@ -362,7 +372,7 @@ public void run() {
     if (profilingEnabled && !isOracleJDK8()) {
       StaticEventLogger.begin("Profiling");
 
-      if (delayOkHttp) {
+      if (waitForJUL) {
         log.debug("Custom logger detected. Delaying Profiling initialization.");
         registerLogManagerCallback(new StartProfilingAgentCallback(inst));
       } else {
@@ -499,18 +509,18 @@ protected static class InstallDatadogTracerCallback extends ClassLoadCallBack {
     private final Instrumentation instrumentation;
     private final Object sco;
     private final Class<?> scoClass;
-    private final boolean delayOkHttp;
+    private final int okHttpDelayMillis;
 
     public InstallDatadogTracerCallback(
         InitializationTelemetry initTelemetry,
         Instrumentation instrumentation,
-        boolean delayOkHttp) {
-      this.delayOkHttp = delayOkHttp;
+        int okHttpDelayMillis) {
+      this.okHttpDelayMillis = okHttpDelayMillis;
       this.instrumentation = instrumentation;
       try {
         scoClass =
             AGENT_CLASSLOADER.loadClass("datadog.communication.ddagent.SharedCommunicationObjects");
-        sco = scoClass.getConstructor(boolean.class).newInstance(delayOkHttp);
+        sco = scoClass.getConstructor(boolean.class).newInstance(okHttpDelayMillis > 0);
       } catch (ClassNotFoundException
           | NoSuchMethodException
           | InstantiationException
@@ -521,6 +531,7 @@ public InstallDatadogTracerCallback(
 
       installDatadogTracer(initTelemetry, scoClass, sco);
       maybeInstallLogsIntake(scoClass, sco);
+      maybeStartIast(instrumentation);
     }
 
     @Override
@@ -530,12 +541,11 @@ public AgentThread agentThread() {
 
     @Override
     public void execute() {
-      if (delayOkHttp) {
+      if (okHttpDelayMillis > 0) {
         resumeRemoteComponents();
       }
 
       maybeStartAppSec(scoClass, sco);
-      maybeStartIast(instrumentation, scoClass, sco);
       maybeStartCiVisibility(instrumentation, scoClass, sco);
       // start debugger before remote config to subscribe to it before starting to poll
       maybeStartDebugger(instrumentation, scoClass, sco);
@@ -550,7 +560,7 @@ private void resumeRemoteComponents() {
       try {
         // remote components were paused for custom log-manager/jmx-builder
         // add small delay before resuming remote I/O to help stabilization
-        Thread.sleep(1_000);
+        Thread.sleep(okHttpDelayMillis);
         scoClass.getMethod("resume").invoke(sco);
       } catch (InterruptedException ignore) {
       } catch (Throwable e) {
@@ -837,14 +847,14 @@ private static boolean isSupportedAppSecArch() {
     return true;
   }
 
-  private static void maybeStartIast(Instrumentation instrumentation, Class<?> scoClass, Object o) {
+  private static void maybeStartIast(Instrumentation instrumentation) {
     if (iastEnabled || !iastFullyDisabled) {
 
       StaticEventLogger.begin("IAST");
 
       try {
         SubscriptionService ss = AgentTracer.get().getSubscriptionService(RequestContextSlot.IAST);
-        startIast(instrumentation, ss, scoClass, o);
+        startIast(instrumentation, ss);
       } catch (Exception e) {
         log.error("Error starting IAST subsystem", e);
       }
@@ -853,8 +863,7 @@ private static void maybeStartIast(Instrumentation instrumentation, Class<?> sco
     }
   }
 
-  private static void startIast(
-      Instrumentation instrumentation, SubscriptionService ss, Class<?> scoClass, Object sco) {
+  private static void startIast(Instrumentation instrumentation, SubscriptionService ss) {
     try {
       final Class<?> appSecSysClass = AGENT_CLASSLOADER.loadClass("com.datadog.iast.IastSystem");
       final Method iastInstallerMethod =
@@ -1339,15 +1348,24 @@ private static String ddGetEnv(final String sysProp) {
   }
 
   private static boolean okHttpMayIndirectlyLoadJUL() {
-    if ("IBM Corporation".equals(getRuntimeVendor())) {
-      return true; // IBM JDKs ship with 'IBMSASL' which will load JUL when OkHttp accesses TLS
+    if (isCustomSecurityProviderInstalled() || isIBMSASLInstalled()) {
+      return true; // custom security providers may load JUL when OkHttp accesses TLS
     }
     if (isJavaVersionAtLeast(9)) {
       return false; // JDKs since 9 have reworked JFR to use a different logging facility, not JUL
     }
     return isJFRSupported(); // assume OkHttp will indirectly load JUL via its JFR events
   }
 
+  private static boolean isCustomSecurityProviderInstalled() {
+    return ClassLoader.getSystemResource("META-INF/services/java.security.Provider") != null;
+  }
+
+  private static boolean isIBMSASLInstalled() {
+    // need explicit check as this is installed without using the service-loader mechanism
+    return ClassLoader.getSystemResource("com/ibm/security/sasl/IBMSASL.class") != null;
+  }
+
   private static boolean isJFRSupported() {
     // FIXME: this is quite a hack because there maybe jfr classes on classpath somehow that have
     // nothing to do with JDK - but this should be safe because only thing this does is to delay

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/QueueTimerHelper.java

@@ -30,23 +30,31 @@ private static final class RateLimiterHolder {
   }
 
   public static <T> void startQueuingTimer(
-      ContextStore<T, State> taskContextStore, Class<?> schedulerClass, T task) {
+      ContextStore<T, State> taskContextStore,
+      Class<?> schedulerClass,
+      Class<?> queueClass,
+      int queueLength,
+      T task) {
     State state = taskContextStore.get(task);
-    startQueuingTimer(state, schedulerClass, task);
+    startQueuingTimer(state, schedulerClass, queueClass, queueLength, task);
   }
 
-  public static void startQueuingTimer(State state, Class<?> schedulerClass, Object task) {
+  public static void startQueuingTimer(
+      State state, Class<?> schedulerClass, Class<?> queueClass, int queueLength, Object task) {
     if (Platform.isNativeImage()) {
       // explicitly not supported for Graal native image
       return;
     }
+    // TODO consider queue length based sampling here to reduce overhead
     // avoid calling this before JFR is initialised because it will lead to reading the wrong
     // TSC frequency before JFR has set it up properly
     if (task != null && state != null && InstrumentationBasedProfiling.isJFRReady()) {
       QueueTiming timing =
           (QueueTiming) AgentTracer.get().getProfilingContext().start(Timer.TimerType.QUEUEING);
       timing.setTask(task);
       timing.setScheduler(schedulerClass);
+      timing.setQueue(queueClass);
+      timing.setQueueLength(queueLength);
       state.setTiming(timing);
     }
   }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java

@@ -31,21 +31,12 @@ TestSuiteImpl testSuiteStart(
   boolean isModified(TestSourceData testSourceData);
 
   /**
-   * Checks if a given test should be skipped with Intelligent Test Runner or not
+   * Checks if a given test can be skipped with Intelligent Test Runner or not.
    *
    * @param test Test to be checked
    * @return {@code true} if the test can be skipped, {@code false} otherwise
    */
-  boolean shouldBeSkipped(TestIdentifier test);
-
-  /**
-   * Checks if a given test can be skipped with Intelligent Test Runner or not. If the test is
-   * considered skippable, the count of skippable tests is incremented.
-   *
-   * @param test Test to be checked
-   * @return {@code true} if the test can be skipped, {@code false} otherwise
-   */
-  boolean skip(TestIdentifier test);
+  boolean isSkippable(TestIdentifier test);
 
   @Nonnull
   TestRetryPolicy retryPolicy(TestIdentifier test, TestSourceData testSource);