.github/workflows/gradle.yml

@@ -1,17 +1,21 @@
 name: Java CI
 
-on: [push]
+on: [pull_request, push]
 
 jobs:
   build:
 
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        java-version: [8, 11]
+
     steps:
     - uses: actions/checkout@v1
-    - name: Set up JDK 1.8
+    - name: Set up JDK ${{ matrix.java-version }}
       uses: actions/setup-java@v1
       with:
-        java-version: 1.8
+        java-version: ${{ matrix.java-version }}
     - name: Build with Gradle
       run: ./gradlew build --info --stacktrace --continue

CHANGELOG.md

@@ -1,5 +1,55 @@
 # Change Log
 
+## Version 0.13.0 - 03/09/2023
+
+### Major new features
+
+* Introduced parallel instrumentation preprocessing to lower time spent preprocessing
+* Introduced instrumentation preprocessing artifact caching to enable configurable artifact caching strategies to prevent redundant preprocessing
+* Added support for JDK-17 by enabling building patched copy of ByteBuddy dependency
+
+### Minor new features
+
+* Deprecated HeaderRetrievable in favor of Activity/Downstream x Request/Response events and added corresponding Apache HttpClient downstream Request/Response header events
+* Implemented Killswitch file mechanism
+* Added additional logging to indicate when Signed Jars are preprocessed
+* Extended ServiceEvent API to support Event IDs
+* Implemented AgentConfig override via properties file
+
+### Bug fixes
+
+* Fixed mvn mysql-connector-java coordinates
+* Fixed (Scheduled)ThreadPoolExecutor task removal
+* Removed thread id comparison on thread enter to allow TX propagation during async workflows
+
+## Version 0.12.0 - 04/25/2022
+
+### Major new features
+
+* Introduced instrumentation preprocess feature under disco-java-agent-instrumentation-preprocess. Instrumentation preprocess addresses issues caused by runtime instrumentation overhead such as longer startup time by instrumenting the JDK and all the dependencies at build-time instead.
+* Added `PluginClassLoader`. This is the new default class loader for loading Disco plugins.
+* Added a new plugin to support transaction context (TX) propagation for Kotlin coroutines.
+
+### Minor new features
+
+* Implemented `TrieNameMatcher` to match class name using Trie data structure.
+* Generated transaction ID using ThreadLocalRandom. The implementation is adapted from X-Ray SDK FastIdGenerator.
+* Provided timing metric to measure how long it take for Disco agent to start.
+* Disposed of Thread, ForkJoinPool, and ForkJoinTask interceptors after they are applied once. This is an optimization to bypass class matching for these named classes.
+
+### Bug fixes
+
+* Added check to prevent Disco agent to be loaded more than once.
+* Cleaned up ThreadLocal transaction context when the thread ends or is pushed back into the pool.
+* Fixed NPE when Thread is instantiated without a target.
+* Fixed ClassCircularityError when running with security manager. This is a known problem with ByteBuddy, since the [OpenTelemetry](https://github.com/open-telemetry/opentelemetry-java-instrumentation/pull/4557) also encountered it.
+* Handled exceptions in Disco EventBus when listening to incoming events.
+* Upgraded to the latest `net.bytebuddy:byte-buddy-dep-1.12.6` and `org.ow2.asm:asm-9.2`.
+
+## Version 0.11.0 - 04/07/2021
+
+* Added additional SQL interception support for prepared statements and calls [PR #16](https://github.com/awslabs/disco/pull/16)
+
 ## Version 0.10.0 - 08/25/2020
 
 * Added SQL interception package [PR #10](https://github.com/awslabs/disco/pull/10)
@@ -23,4 +73,4 @@
 
 ## Version 0.9.1 - 12/2/2019
 
-* Initial commit of DiSCo Toolkit
+* Initial commit of DiSCo Toolkit

README.md

@@ -88,8 +88,8 @@ is passed from thread to thread at the time of thread handoff (e.g. when calling
 or someExecutor.submit(), or when using a Java 8 parallel stream), by the 'forking' thread
 giving the 'forked' thread access to its Transaction Context data store.
 
-By default, upon creation, the Transaction Context always contains a randomly 
-allocated UUID as a Transaction ID. This can be overridden by plugins or 
+By default, upon creation, the Transaction Context always contains a 96 bit random number
+formatted as a hexadecimal string as a Transaction ID. This can be overridden by plugins or
 client code if desirable, and any other arbitrary data may also be added at
 any point in the lifetime of the service activity. Once the data is placed
 in the Transaction Context, it becomes available across the activity's family
@@ -180,7 +180,7 @@ Disco packages easier.
         <dependency>
             <groupId>software.amazon.disco</groupId>
             <artifactId>disco-toolkit-bom</artifactId>
-            <version>0.10.0</version>
+            <version>0.13.0</version>
             <type>pom</type>
             <scope>import</scope>
         </dependency>
@@ -197,14 +197,14 @@ Disco packages easier.
 
 #### Using Gradle's default DSL
 ```groovy
-implementation platform('software.amazon.disco:disco-toolkit-bom:0.10.0')
+implementation platform('software.amazon.disco:disco-toolkit-bom:0.13.0')
 implementation 'software.amazon.disco:disco-java-agent-api'
 // Other disco dependencies
 ```
 
 #### Using Gradle's Kotlin DSL
 ```kotlin
-implementation(platform("software.amazon.disco:disco-toolkit-bom:0.10.0"))
+implementation(platform("software.amazon.disco:disco-toolkit-bom:0.13.0"))
 implementation("software.amazon.disco:disco-java-agent-api")
 // Other disco dependencies
 ```

build.gradle.kts

@@ -21,24 +21,51 @@ plugins {
     id("com.github.johnrengelman.shadow") version "5.2.0" apply false
 }
 
+val standardOutputLoggerFactoryFQN by extra("software.amazon.disco.agent.reflect.logging.StandardOutputLoggerFactory")
+
 subprojects {
-    version = "0.10.0"
+    version = "0.13.0"
 
     repositories {
         mavenCentral()
     }
 
+    // Common build logic for plugins that need to relocate certain classes under /resources of the build artifact jar.
+    afterEvaluate {
+        if (ext.has("classesToMove")) {
+            tasks.named<Jar>("jar") {
+                includeEmptyDirs = false
+
+                (ext.get("classesToMove") as Array<String>).forEach {
+                    val name = it.replace('.', '/')
+
+                    // copy these compiled classes to destination dir while maintaining their namespaces.
+                    from("build/classes/java/main/$name.class") {
+                        into("resources/${name.substringBeforeLast('/')}")
+                    }
+
+                    // exclude the original ones from this jar
+                    exclude("$name.class")
+                }
+            }
+        }
+    }
+
     // Set up creation of shaded Jars
     pluginManager.withPlugin("com.github.johnrengelman.shadow") {
         tasks {
             named<ShadowJar>("shadowJar") {
+                //manually exclude module-info.class. This class is only relevant to the shadowed lib (bytebuddy-agent, asm-commons) and of no use to the resulting jar
+                //this is an open bug: https://github.com/johnrengelman/shadow/issues/729
+                exclude("**/module-info.class")
 
                 //suppress the "-all" suffix on the jar name, simply replace the default built jar instead (disco-java-agent-web-plugin-x.y.z.jar)
                 archiveClassifier.set(null as String?)
 
-                //Must relocate both of these inner dependencies of the Disco agent, to avoid conflicts in your customer's application
+                //Must relocate three of these inner dependencies of the Disco agent, to avoid conflicts in your customer's application
                 relocate("org.objectweb.asm", "software.amazon.disco.agent.jar.asm")
                 relocate("net.bytebuddy", "software.amazon.disco.agent.jar.bytebuddy")
+                relocate("com.sun.jna", "software.amazon.disco.agent.jar.jna")
             }
 
             //once gradle has made its default jar, follow up by producing the shadow/uber jar
@@ -52,11 +79,16 @@ subprojects {
     }
 
     plugins.withId("java-library") {
+        plugins.apply("jacoco")
+
         dependencies {
             add("testImplementation", "junit:junit:4.12")
             add("testImplementation", "org.mockito:mockito-core:3.+")
         }
 
+        // Catch Javadoc errors in the "build" task of each subproject
+        tasks["build"].dependsOn(tasks["javadoc"])
+
         configure<JavaPluginConvention> {
             sourceCompatibility = JavaVersion.VERSION_1_8
         }
@@ -83,6 +115,9 @@ subprojects {
                 add("testImplementation", "org.mockito:mockito-core:1.+")
             }
 
+            val agentJarPath = project(":disco-java-agent:disco-java-agent").buildDir.absolutePath + "/libs/disco-java-agent-$ver.jar"
+            val pluginsDir = project.buildDir.absolutePath + "/libs"
+
             // Configure integ tests, which need a loaded agent, and the loaded plugin
             tasks.named<Test>("test") {
                 // explicitly remove the runtime classpath from the tests since they are integ tests, and may not access the
@@ -92,13 +127,87 @@ subprojects {
                 classpath = classpath.minus(configurations.named<Configuration>("runtimeClasspath").get())
 
                 //load the agent for the tests, and have it discover the plugin
-                jvmArgs("-javaagent:../../disco-java-agent/disco-java-agent/build/libs/disco-java-agent-$ver.jar=pluginPath=./build/libs:extraverbose")
+                jvmArgs("-agentlib:jdwp=transport=dt_socket,address=localhost:1337,server=y,suspend=n", "-javaagent:${agentJarPath}=pluginPath=${pluginsDir}:extraverbose")
 
                 //we do not take any normal compile/runtime dependency on this, but it must be built first since the above jvmArg
                 //refers to its built artifact.
                 dependsOn(":disco-java-agent:disco-java-agent:build")
                 dependsOn("$libraryName:${project.name}:assemble")
             }
+
+            val originalTestDir = file(project.buildDir.absolutePath + "/classes/java/test")
+
+            // task to statically instrument all required sources to run plugin integ tests with preprocessing enabled.
+            val preprocess = tasks.register<JavaExec>("preprocess") {
+                onlyIf {
+                    originalTestDir.exists() && originalTestDir.isDirectory && originalTestDir.listFiles().isNotEmpty()
+                }
+
+                val preprocessProjName = ":disco-java-agent-instrumentation-preprocess";
+                val preprocessorJarPath = project(preprocessProjName).buildDir.absolutePath + "/libs/disco-java-agent-instrumentation-preprocess-$ver.jar"
+
+                val testJarDependencies = (tasks["test"] as Test).classpath.minus(configurations.named<Configuration>("runtimeClasspath").get()).files.filter { it.isFile };
+                val outputDir = project.buildDir.absolutePath + "/static-instrumentation"
+
+                main = "software.amazon.disco.instrumentation.preprocess.cli.Driver";
+                classpath = files(preprocessorJarPath, testJarDependencies, originalTestDir)
+                args = listOf(
+                    "-ap", agentJarPath,
+                    "-jdks", System.getProperty("java.home"),
+                    "-sps", testJarDependencies.joinToString(":") + "@testJars",
+                    "-sps", "${originalTestDir}@test",
+                    "-out", outputDir,
+                    "-arg", "verbose:loggerfactory=${standardOutputLoggerFactoryFQN}:pluginPath=${pluginsDir}",
+                )
+
+                dependsOn("$preprocessProjName:build")
+                dependsOn(tasks["test"])
+            }
+
+            // Configure integ tests to run with preprocessing enabled
+            val preprocessingTest = tasks.register<Test>("preprocessing_test") {
+                onlyIf {
+                    originalTestDir.exists() && originalTestDir.isDirectory && originalTestDir.listFiles().isNotEmpty()
+                }
+
+                val outputDir = project.buildDir.absolutePath + "/static-instrumentation"
+
+                // update test CP
+                doFirst {
+                    val instrumentedTestDir = file("$outputDir/test")
+                    val instrumentedJars = file("$outputDir/testJars")
+
+                    // copy the content of the original test dir to a temp folder without overriding any content to keep the original test dir clean.
+                    // the content of the temp folder will then be added to the runtime classpath.
+                    originalTestDir.copyRecursively(instrumentedTestDir, false, onError = { file, exception -> OnErrorAction.SKIP })
+
+                    val map = instrumentedJars.listFiles().associateBy { it.name }
+                    classpath = layout.files(classpath.files.filter { !map.containsKey(it.name) })
+                        .plus(layout.files(instrumentedJars.listFiles()))
+                        .plus(layout.files(instrumentedTestDir))
+                        .minus(layout.files(originalTestDir))
+                }
+
+                jvmArgs("-agentlib:jdwp=transport=dt_socket,address=localhost:1337,server=y,suspend=n")
+
+                // attach the Disco agent in 'runtimeonly' mode
+                jvmArgs("-javaagent:${agentJarPath}=pluginPath=${pluginsDir}:runtimeonly:verbose:loggerfactory=${standardOutputLoggerFactoryFQN}")
+
+                if (JavaVersion.current().isJava9Compatible) {
+                    // configure module patching and create read link from java.base to all unnamed modules
+                    jvmArgs("--patch-module=java.base=${outputDir}/jdk/InstrumentedJDK.jar")
+                    jvmArgs("--add-reads=java.base=ALL-UNNAMED")
+                    jvmArgs("--add-exports=java.base/software.amazon.disco.agent.concurrent.preprocess=ALL-UNNAMED")
+                } else {
+                    // prepend the instrumented JDK artifact to the bootstrap class path
+                    jvmArgs("-Xbootclasspath/p:${outputDir}/jdk/InstrumentedJDK.jar")
+                }
+
+                dependsOn(tasks["test"])
+                dependsOn(preprocess)
+            }
+
+            tasks["build"].dependsOn(preprocessingTest)
         }
     }
 
@@ -197,6 +306,11 @@ subprojects {
                                 name.set("Ben Smithers")
                                 email.set("besmithe@amazon.co.uk")
                             }
+                            developer {
+                                id.set("oelmohan")
+                                name.set("Omar Elmohandes")
+                                email.set("oelmohan@amazon.com")
+                            }
                         }
                         scm {
                             connection.set("scm:git:git://github.com/awslabs/disco.git")
@@ -223,4 +337,37 @@ subprojects {
             sign(the<PublishingExtension>().publications["maven"])
         }
     }
+
+    plugins.withId("jacoco") {
+        tasks.withType<Test> {
+            finalizedBy("jacocoTestCoverageVerification")
+        }
+
+        tasks.withType<JacocoReport> {
+            dependsOn("test")
+            // Enable both XML and HTML outputs
+            reports {
+                // The default XML path will be at build/reports/jacoco/test/jacocoTestReport.xml
+                xml.required.set(true)
+
+                // The default HTML path will be at build/reports/jacoco/test/html/index.html
+                html.required.set(true)
+            }
+        }
+
+        tasks.withType<JacocoCoverageVerification> {
+            dependsOn("jacocoTestReport")
+            violationRules {
+                // Unless we increase the coverage for all projects, setting this to true will fail some. So
+                // this will only be a warning for now
+                isFailOnViolation = false
+                rule {
+                    limit {
+                        // Set the default to 90% coverage over all instructions
+                        minimum = "0.9".toBigDecimal()
+                    }
+                }
+            }
+        }
+    }
 }

disco-java-agent-aws/build.gradle.kts

@@ -19,7 +19,8 @@ plugins {
 }
 
 dependencies {
-    // Compile against AWS SDK V2, but we do not take a runtime dependency on it
+    // Compile against AWS SDKs, but we do not take a runtime dependency on them
+    compileOnly("com.amazonaws", "aws-java-sdk-core", "1.11.840")
     compileOnly("software.amazon.awssdk", "sdk-core", "2.13.76")
 
     implementation(project(":disco-java-agent-aws:disco-java-agent-aws-api"))
@@ -30,3 +31,12 @@ dependencies {
     testImplementation("com.amazonaws", "aws-java-sdk-sqs", "1.11.840")
     testImplementation("software.amazon.awssdk", "dynamodb", "2.13.76")
 }
+
+// For classes which need to be accessed in the context of the application code's classloader, they need to be injected/forced
+// into that classloader. They cannot be placed in the bootstrap classloader, nor any isolated/orphaned classloader, since they
+// either inherit from, or use, classes from the AWS SDK, which are assumed not to be present on the bootstrap classloader
+ext.set("classesToMove", arrayOf(
+        "software.amazon.disco.agent.awsv2.DiscoExecutionInterceptor",
+        "software.amazon.disco.agent.event.AwsServiceDownstreamRequestEventImpl",
+        "software.amazon.disco.agent.event.AwsServiceDownstreamResponseEventImpl"
+))

disco-java-agent-aws/disco-java-agent-aws-api/build.gradle.kts

@@ -18,5 +18,5 @@ plugins {
 }
 
 dependencies {
-    implementation(project(":disco-java-agent:disco-java-agent-api"))
+    compileOnly(project(":disco-java-agent:disco-java-agent-api"))
 }

disco-java-agent-aws/disco-java-agent-aws-api/src/main/java/software/amazon/disco/agent/event/AwsServiceDownstreamEvent.java

@@ -44,6 +44,9 @@ enum DataKey {
     /**
      * Obtain the header map that the particular event holds.
      * @return A key-value pairing map of the headers.
+     *
+     * @deprecated deprecated in favour of {@link HeaderRetrievable} which should be used wherever possible instead.
      */
+    @Deprecated
     Map<String, List<String>> getHeaderMap();
 }