From b56cbba8e525a4624991b1464776fd0cc3c58d3c Mon Sep 17 00:00:00 2001 From: Alexey Loubyansky Date: Fri, 8 Dec 2023 15:56:33 +0100 Subject: [PATCH 01/10] Dump all Quarkus app dependencies along with their checksums as part of track-config-changes goal --- .../quarkus/maven/TrackConfigChangesMojo.java | 134 ++++++++++++++---- docs/src/main/asciidoc/config-reference.adoc | 5 + 2 files changed, 110 insertions(+), 29 deletions(-) diff --git a/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java b/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java index 13f4379079981..47b75a6446c3d 100644 --- a/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java +++ b/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java @@ -1,12 +1,19 @@ package io.quarkus.maven; import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.File; import java.io.IOException; import java.lang.reflect.Method; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Properties; +import java.util.zip.Adler32; +import java.util.zip.Checksum; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -18,6 +25,7 @@ import io.quarkus.bootstrap.app.CuratedApplication; import io.quarkus.bootstrap.classloading.QuarkusClassLoader; import io.quarkus.bootstrap.model.ApplicationModel; +import io.quarkus.maven.dependency.DependencyFlags; import io.quarkus.runtime.LaunchMode; /** @@ -58,6 +66,18 @@ public class TrackConfigChangesMojo extends QuarkusBootstrapMojo { @Parameter(defaultValue = "false", property = "quarkus.track-config-changes.dump-current-when-recorded-unavailable") boolean dumpCurrentWhenRecordedUnavailable; + /** + * Whether to dump Quarkus application dependencies along with their checksums + */ + @Parameter(defaultValue = "true", property = "quarkus.track-config-changes.dump-dependencies") + boolean dumpDependencies; + + /** + * Dependency dump file + */ + @Parameter(property = "quarkus.track-config-changes.dependenciesFile") + File dependenciesFile; + @Override protected boolean beforeExecute() throws MojoExecutionException, MojoFailureException { if (skip) { @@ -82,16 +102,6 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException { getLog().debug("Bootstrapping Quarkus application in mode " + launchMode); } - Path targetFile; - if (outputFile == null) { - targetFile = outputDirectory.toPath() - .resolve("quarkus-" + launchMode.getDefaultProfile() + "-config-check"); - } else if (outputFile.isAbsolute()) { - targetFile = outputFile.toPath(); - } else { - targetFile = outputDirectory.toPath().resolve(outputFile.toPath()); - } - Path compareFile; if (this.recordedBuildConfigFile == null) { compareFile = recordedBuildConfigDirectory.toPath() @@ -102,34 +112,60 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException { compareFile = recordedBuildConfigDirectory.toPath().resolve(this.recordedBuildConfigFile.toPath()); } - final Properties compareProps = new Properties(); - if (Files.exists(compareFile)) { - try (BufferedReader reader = Files.newBufferedReader(compareFile)) { - compareProps.load(reader); - } catch (IOException e) { - throw new RuntimeException("Failed to read " + compareFile, e); - } - } else if (!dumpCurrentWhenRecordedUnavailable) { - getLog().info(compareFile + " not found"); + final boolean prevConfigExists = Files.exists(compareFile); + if (!prevConfigExists && !dumpCurrentWhenRecordedUnavailable && !dumpDependencies) { + getLog().info("Config dump from the previous build does not exist at " + compareFile); return; } CuratedApplication curatedApplication = null; QuarkusClassLoader deploymentClassLoader = null; final ClassLoader originalCl = Thread.currentThread().getContextClassLoader(); - Properties actualProps; final boolean clearPackageTypeSystemProperty = setPackageTypeSystemPropertyIfNativeProfileEnabled(); try { curatedApplication = bootstrapApplication(launchMode); - deploymentClassLoader = curatedApplication.createDeploymentClassLoader(); - Thread.currentThread().setContextClassLoader(deploymentClassLoader); - - final Class codeGenerator = deploymentClassLoader.loadClass("io.quarkus.deployment.CodeGenerator"); - final Method dumpConfig = codeGenerator.getMethod("dumpCurrentConfigValues", ApplicationModel.class, String.class, - Properties.class, QuarkusClassLoader.class, Properties.class, Path.class); - dumpConfig.invoke(null, curatedApplication.getApplicationModel(), - launchMode.name(), getBuildSystemProperties(true), - deploymentClassLoader, compareProps, targetFile); + if (prevConfigExists || dumpCurrentWhenRecordedUnavailable) { + final Path targetFile = getOutputFile(outputFile, launchMode.getDefaultProfile(), "-config-check"); + Properties compareProps = null; + if (prevConfigExists) { + compareProps = new Properties(); + try (BufferedReader reader = Files.newBufferedReader(compareFile)) { + compareProps.load(reader); + } catch (IOException e) { + throw new RuntimeException("Failed to read " + compareFile, e); + } + } + + deploymentClassLoader = curatedApplication.createDeploymentClassLoader(); + Thread.currentThread().setContextClassLoader(deploymentClassLoader); + + final Class codeGenerator = deploymentClassLoader.loadClass("io.quarkus.deployment.CodeGenerator"); + final Method dumpConfig = codeGenerator.getMethod("dumpCurrentConfigValues", ApplicationModel.class, + String.class, + Properties.class, QuarkusClassLoader.class, Properties.class, Path.class); + dumpConfig.invoke(null, curatedApplication.getApplicationModel(), + launchMode.name(), getBuildSystemProperties(true), + deploymentClassLoader, compareProps, targetFile); + } + + if (dumpDependencies) { + final List deps = new ArrayList<>(); + for (var d : curatedApplication.getApplicationModel().getDependencies(DependencyFlags.DEPLOYMENT_CP)) { + var adler32 = new Adler32(); + updateChecksum(adler32, d.getResolvedPaths()); + deps.add(d.toGACTVString() + " " + adler32.getValue()); + } + Collections.sort(deps); + final Path targetFile = getOutputFile(dependenciesFile, launchMode.getDefaultProfile(), + "-dependency-checksums.txt"); + Files.createDirectories(targetFile.getParent()); + try (BufferedWriter writer = Files.newBufferedWriter(targetFile)) { + for (var s : deps) { + writer.write(s); + writer.newLine(); + } + } + } } catch (Exception any) { throw new MojoExecutionException("Failed to bootstrap Quarkus application", any); } finally { @@ -142,4 +178,44 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException { } } } + + private Path getOutputFile(File outputFile, String profile, String fileNameSuffix) { + if (outputFile == null) { + return outputDirectory.toPath().resolve("quarkus-" + profile + fileNameSuffix); + } + if (outputFile.isAbsolute()) { + return outputFile.toPath(); + } + return outputDirectory.toPath().resolve(outputFile.toPath()); + } + + private static void updateChecksum(Checksum checksum, Iterable pc) throws IOException { + for (var path : sort(pc)) { + if (Files.isDirectory(path)) { + try (DirectoryStream stream = Files.newDirectoryStream(path)) { + updateChecksum(checksum, stream); + } + } else { + checksum.update(Files.readAllBytes(path)); + } + } + } + + private static Iterable sort(Iterable original) { + var i = original.iterator(); + if (!i.hasNext()) { + return List.of(); + } + var o = i.next(); + if (!i.hasNext()) { + return List.of(o); + } + final List sorted = new ArrayList<>(); + sorted.add(o); + while (i.hasNext()) { + sorted.add(i.next()); + } + Collections.sort(sorted); + return sorted; + } } diff --git a/docs/src/main/asciidoc/config-reference.adoc b/docs/src/main/asciidoc/config-reference.adoc index d46012b9d3700..bfd359c43ae38 100644 --- a/docs/src/main/asciidoc/config-reference.adoc +++ b/docs/src/main/asciidoc/config-reference.adoc @@ -738,6 +738,11 @@ Maven projects could add the following goal to their `quarkus-maven-plugin` conf The `track-config-changes` goal looks for `${project.basedir}/.quarkus/quarkus-prod-config-dump` (file name and directory are configurable) and, if the file already exists, checks whether the values stored in the config dump have changed. It will log the changed options and save the current values of each of the options present in `${project.basedir}/.quarkus/quarkus-prod-config-dump` in `${project.basedir}/target/quarkus-prod-config.check` (the target file name and location can be configured). If the build time configuration has not changed since the last build both `${project.basedir}/.quarkus/quarkus-prod-config-dump` and `${project.basedir}/.quarkus/quarkus-prod-config-dump` will be identical. +==== Dump Quarkus application dependencies + +In addition to dumping configuration values, `track-config-changes` goal also dumps all the Quarkus application dependencies, including Quarkus build time dependencies, along with their checksums (Adler32). This file could be used to check whether Quarkus build classpath has changed since the previous run. +By default, the dependency checksums will be stored under `target/quarkus-prod-dependency-checksums.txt` file. A different location could be configured using plugin parameters. + ==== Dump current build configuration when the recorded configuration isn't found By default, `track-config-changes` looks for the configuration recorded during previous build and does nothing if it's not found. Enabling `dumpCurrentWhenRecordedUnavailable` parameter will make it dump the current build configuration From ae18537c359fb03ab3d7910304a7326bc12c1502 Mon Sep 17 00:00:00 2001 From: Alexey Loubyansky Date: Fri, 8 Dec 2023 15:56:49 +0100 Subject: [PATCH 02/10] Execute quarkus:track-config-changes for all the IT modules to dump quarkus application dependency checksums --- integration-tests/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index e0b6a8a90dd0f..718888e33f771 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -70,6 +70,16 @@ io.quarkus quarkus-maven-plugin ${project.version} + + + + track-config-changes + process-resources + + track-config-changes + + + true ${quarkus.build.skip} From f6faadb4f069d212bc5e7732b72b7682c1f4130d Mon Sep 17 00:00:00 2001 From: Guillaume Smet Date: Thu, 1 Feb 2024 17:37:42 +0100 Subject: [PATCH 03/10] Only compute checksum if snapshot dependency Non snapshot dependencies are supposed to be immutable so we can skip computing the checksum. --- .../java/io/quarkus/maven/TrackConfigChangesMojo.java | 11 ++++++++--- .../io/quarkus/maven/dependency/ArtifactCoords.java | 4 ++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java b/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java index 47b75a6446c3d..5064f9f79f424 100644 --- a/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java +++ b/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java @@ -151,9 +151,14 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException { if (dumpDependencies) { final List deps = new ArrayList<>(); for (var d : curatedApplication.getApplicationModel().getDependencies(DependencyFlags.DEPLOYMENT_CP)) { - var adler32 = new Adler32(); - updateChecksum(adler32, d.getResolvedPaths()); - deps.add(d.toGACTVString() + " " + adler32.getValue()); + StringBuilder entry = new StringBuilder(d.toGACTVString()); + if (d.isSnapshot()) { + var adler32 = new Adler32(); + updateChecksum(adler32, d.getResolvedPaths()); + entry.append(" ").append(adler32.getValue()); + } + + deps.add(entry.toString()); } Collections.sort(deps); final Path targetFile = getOutputFile(dependenciesFile, launchMode.getDefaultProfile(), diff --git a/independent-projects/bootstrap/app-model/src/main/java/io/quarkus/maven/dependency/ArtifactCoords.java b/independent-projects/bootstrap/app-model/src/main/java/io/quarkus/maven/dependency/ArtifactCoords.java index a0b099e1e9277..946a7cb785bda 100644 --- a/independent-projects/bootstrap/app-model/src/main/java/io/quarkus/maven/dependency/ArtifactCoords.java +++ b/independent-projects/bootstrap/app-model/src/main/java/io/quarkus/maven/dependency/ArtifactCoords.java @@ -42,6 +42,10 @@ default boolean isJar() { return TYPE_JAR.equals(getType()); } + default boolean isSnapshot() { + return getVersion() != null && getVersion().endsWith("-SNAPSHOT"); + } + default String toGACTVString() { return getGroupId() + ":" + getArtifactId() + ":" + getClassifier() + ":" + getType() + ":" + getVersion(); } From fe16620abf9f7398d79ca5525f6f754de1c017a6 Mon Sep 17 00:00:00 2001 From: Alexey Loubyansky Date: Fri, 25 Aug 2023 16:31:09 +0200 Subject: [PATCH 04/10] Enable caching of quarkus:build outcome for the main integration test --- .github/workflows/ci-actions-incremental.yml | 5 +++++ .mvn/extensions.xml | 5 +++++ integration-tests/main/pom.xml | 13 +++++++++++++ 3 files changed, 23 insertions(+) diff --git a/.github/workflows/ci-actions-incremental.yml b/.github/workflows/ci-actions-incremental.yml index 11c9a43357c29..f23cbc7c23ac1 100644 --- a/.github/workflows/ci-actions-incremental.yml +++ b/.github/workflows/ci-actions-incremental.yml @@ -1072,6 +1072,11 @@ jobs: build-scan-capture-strategy: ON_DEMAND job-name: "Native Tests - ${{matrix.category}}" wrapper-init: true + - name: Cache Quarkus metadata + uses: actions/cache@v3 + with: + path: '**/.quarkus/quarkus-prod-config-dump' + key: ${{ runner.os }}-quarkus-metadata - name: Build env: TEST_MODULES: ${{matrix.test-modules}} diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 279f59a0f501c..a48dbdff4d4f1 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -9,4 +9,9 @@ common-custom-user-data-maven-extension 1.12.5 + + com.gradle + quarkus-build-caching-extension + 0.10 + diff --git a/integration-tests/main/pom.xml b/integration-tests/main/pom.xml index 5f9e1a8821a7c..7bad969b10ce5 100644 --- a/integration-tests/main/pom.xml +++ b/integration-tests/main/pom.xml @@ -23,6 +23,9 @@ 1.13.0 + + true + true @@ -507,6 +510,16 @@ io.quarkus quarkus-maven-plugin + + track-prod-config-changes + process-resources + + track-config-changes + + + true + + build From 8f7dec895d02730819fb5b5d7400669492b6af99 Mon Sep 17 00:00:00 2001 From: Guillaume Smet Date: Thu, 1 Feb 2024 18:42:28 +0100 Subject: [PATCH 05/10] Fix name of config property for consistency --- .../src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java b/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java index 5064f9f79f424..07de86749ea52 100644 --- a/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java +++ b/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java @@ -75,7 +75,7 @@ public class TrackConfigChangesMojo extends QuarkusBootstrapMojo { /** * Dependency dump file */ - @Parameter(property = "quarkus.track-config-changes.dependenciesFile") + @Parameter(property = "quarkus.track-config-changes.dependencies-file") File dependenciesFile; @Override From 057ef3305e5bfa9031ac5b5c93c884a328fc4ee0 Mon Sep 17 00:00:00 2001 From: Guillaume Smet Date: Thu, 1 Feb 2024 18:42:42 +0100 Subject: [PATCH 06/10] Enable config tracking for all ITs --- integration-tests/main/pom.xml | 13 ------------- integration-tests/pom.xml | 23 +++++++++++++++-------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/integration-tests/main/pom.xml b/integration-tests/main/pom.xml index 7bad969b10ce5..5f9e1a8821a7c 100644 --- a/integration-tests/main/pom.xml +++ b/integration-tests/main/pom.xml @@ -23,9 +23,6 @@ 1.13.0 - - true - true @@ -510,16 +507,6 @@ io.quarkus quarkus-maven-plugin - - track-prod-config-changes - process-resources - - track-config-changes - - - true - - build diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 718888e33f771..ead6ae822d9bf 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -22,6 +22,9 @@ ${skipTests} ${skipTests} true + + true + true @@ -71,14 +74,18 @@ quarkus-maven-plugin ${project.version} - - - track-config-changes - process-resources - - track-config-changes - - + + + track-config-changes + process-resources + + track-config-changes + + + true + + true From 970c62f146a1ca853c7f046da4b3e566ba89a84d Mon Sep 17 00:00:00 2001 From: Guillaume Smet Date: Thu, 1 Feb 2024 18:54:22 +0100 Subject: [PATCH 07/10] Avoid NPE in CodeGenerator.dumpCurrentConfigValues When we don't have any previous recorded config properties, we should push an empty Properties. At least, that's what CodeGenerator.dumpCurrentConfigValues expects. --- .../src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java b/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java index 07de86749ea52..b023983db78b2 100644 --- a/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java +++ b/devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java @@ -126,9 +126,8 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException { curatedApplication = bootstrapApplication(launchMode); if (prevConfigExists || dumpCurrentWhenRecordedUnavailable) { final Path targetFile = getOutputFile(outputFile, launchMode.getDefaultProfile(), "-config-check"); - Properties compareProps = null; + Properties compareProps = new Properties(); if (prevConfigExists) { - compareProps = new Properties(); try (BufferedReader reader = Files.newBufferedReader(compareFile)) { compareProps.load(reader); } catch (IOException e) { From c76e348069be38ce6601d062f5575a681ddab551 Mon Sep 17 00:00:00 2001 From: Guillaume Smet Date: Thu, 1 Feb 2024 19:03:13 +0100 Subject: [PATCH 08/10] Configure additional input for cached plugins Add the dumped dependencies as an input for Surefire, Failsafe and the Quarkus Maven plugin. --- independent-projects/parent/pom.xml | 57 +++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/independent-projects/parent/pom.xml b/independent-projects/parent/pom.xml index 2df6e460321f0..7aebd7d6a21b8 100644 --- a/independent-projects/parent/pom.xml +++ b/independent-projects/parent/pom.xml @@ -424,6 +424,63 @@ + + maven-surefire-plugin + + + + dependency-checksums + + ${project.build.directory} + + + quarkus-*-dependency-checksums.txt + + RELATIVE_PATH + + + + + + maven-failsafe-plugin + + + + dependency-checksums + + ${project.build.directory} + + + quarkus-*-dependency-checksums.txt + + RELATIVE_PATH + + + + + + io.quarkus + quarkus-maven-plugin + + + default + + + + dependency-checksums + + ${project.build.directory} + + + quarkus-*-dependency-checksums.txt + + RELATIVE_PATH + + + + + + org.jetbrains.kotlin kotlin-maven-plugin From 471c4fc5b9fc4a2952629c614107848979ebfe04 Mon Sep 17 00:00:00 2001 From: Guillaume Smet Date: Thu, 15 Feb 2024 15:08:30 +0100 Subject: [PATCH 09/10] Revert "Enable config tracking for all ITs" This reverts commit b3eeb7c0c049a5c9c75db538446a52b71a45e8e5. In the end, it is not a very good idea to enable this for all of them. --- integration-tests/main/pom.xml | 13 +++++++++++++ integration-tests/pom.xml | 23 ++++++++--------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/integration-tests/main/pom.xml b/integration-tests/main/pom.xml index 5f9e1a8821a7c..7bad969b10ce5 100644 --- a/integration-tests/main/pom.xml +++ b/integration-tests/main/pom.xml @@ -23,6 +23,9 @@ 1.13.0 + + true + true @@ -507,6 +510,16 @@ io.quarkus quarkus-maven-plugin + + track-prod-config-changes + process-resources + + track-config-changes + + + true + + build diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index ead6ae822d9bf..718888e33f771 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -22,9 +22,6 @@ ${skipTests} ${skipTests} true - - true - true @@ -74,18 +71,14 @@ quarkus-maven-plugin ${project.version} - - - track-config-changes - process-resources - - track-config-changes - - - true - - + + + track-config-changes + process-resources + + track-config-changes + + true From a17175448deda96670fe6bb7cd35b8b39f77f7c9 Mon Sep 17 00:00:00 2001 From: Guillaume Smet Date: Thu, 15 Feb 2024 15:17:40 +0100 Subject: [PATCH 10/10] Some refinements to make sure the execution only runs once --- integration-tests/main/pom.xml | 2 +- integration-tests/pom.xml | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/integration-tests/main/pom.xml b/integration-tests/main/pom.xml index 7bad969b10ce5..152e4f371975d 100644 --- a/integration-tests/main/pom.xml +++ b/integration-tests/main/pom.xml @@ -511,7 +511,7 @@ quarkus-maven-plugin - track-prod-config-changes + track-config-changes process-resources track-config-changes diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 718888e33f771..f2ab116a3cbd2 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -71,14 +71,15 @@ quarkus-maven-plugin ${project.version} - - - track-config-changes - process-resources - - track-config-changes - - + + + track-config-changes + process-resources + + track-config-changes + + true @@ -314,7 +315,7 @@ amazon-lambda-rest-resteasy-reactive amazon-lambda-rest-reactive-routes amazon-lambda-rest-funqy - amazon-lambda-rest-servlet + amazon-lambda-rest-servlet amazon-lambda-http-resteasy amazon-lambda-http-resteasy-reactive container-image