Merge pull request #37622 from aloubyansky/dump-deps-from-track-confi…

…g-changes

Enable caching for quarkus:build and declare dependencies as inputs

.github/workflows/ci-actions-incremental.yml

@@ -1072,6 +1072,11 @@ jobs:
           build-scan-capture-strategy: ON_DEMAND
           job-name: "Native Tests - ${{matrix.category}}"
           wrapper-init: true
+      - name: Cache Quarkus metadata
+        uses: actions/cache@v3
+        with:
+          path: '**/.quarkus/quarkus-prod-config-dump'
+          key: ${{ runner.os }}-quarkus-metadata
       - name: Build
         env:
           TEST_MODULES: ${{matrix.test-modules}}

.mvn/extensions.xml

@@ -9,4 +9,9 @@
         <artifactId>common-custom-user-data-maven-extension</artifactId>
         <version>1.12.5</version>
     </extension>
+    <extension>
+        <groupId>com.gradle</groupId>
+        <artifactId>quarkus-build-caching-extension</artifactId>
+        <version>0.10</version>
+    </extension>
 </extensions>

devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java

@@ -1,12 +1,19 @@
 package io.quarkus.maven;
 
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Method;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.Properties;
+import java.util.zip.Adler32;
+import java.util.zip.Checksum;
 
 import org.apache.maven.plugin.MojoExecutionException;
 import org.apache.maven.plugin.MojoFailureException;
@@ -18,6 +25,7 @@
 import io.quarkus.bootstrap.app.CuratedApplication;
 import io.quarkus.bootstrap.classloading.QuarkusClassLoader;
 import io.quarkus.bootstrap.model.ApplicationModel;
+import io.quarkus.maven.dependency.DependencyFlags;
 import io.quarkus.runtime.LaunchMode;
 
 /**
@@ -58,6 +66,18 @@ public class TrackConfigChangesMojo extends QuarkusBootstrapMojo {
     @Parameter(defaultValue = "false", property = "quarkus.track-config-changes.dump-current-when-recorded-unavailable")
     boolean dumpCurrentWhenRecordedUnavailable;
 
+    /**
+     * Whether to dump Quarkus application dependencies along with their checksums
+     */
+    @Parameter(defaultValue = "true", property = "quarkus.track-config-changes.dump-dependencies")
+    boolean dumpDependencies;
+
+    /**
+     * Dependency dump file
+     */
+    @Parameter(property = "quarkus.track-config-changes.dependencies-file")
+    File dependenciesFile;
+
     @Override
     protected boolean beforeExecute() throws MojoExecutionException, MojoFailureException {
         if (skip) {
@@ -82,16 +102,6 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException {
             getLog().debug("Bootstrapping Quarkus application in mode " + launchMode);
         }
 
-        Path targetFile;
-        if (outputFile == null) {
-            targetFile = outputDirectory.toPath()
-                    .resolve("quarkus-" + launchMode.getDefaultProfile() + "-config-check");
-        } else if (outputFile.isAbsolute()) {
-            targetFile = outputFile.toPath();
-        } else {
-            targetFile = outputDirectory.toPath().resolve(outputFile.toPath());
-        }
-
         Path compareFile;
         if (this.recordedBuildConfigFile == null) {
             compareFile = recordedBuildConfigDirectory.toPath()
@@ -102,34 +112,64 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException {
             compareFile = recordedBuildConfigDirectory.toPath().resolve(this.recordedBuildConfigFile.toPath());
         }
 
-        final Properties compareProps = new Properties();
-        if (Files.exists(compareFile)) {
-            try (BufferedReader reader = Files.newBufferedReader(compareFile)) {
-                compareProps.load(reader);
-            } catch (IOException e) {
-                throw new RuntimeException("Failed to read " + compareFile, e);
-            }
-        } else if (!dumpCurrentWhenRecordedUnavailable) {
-            getLog().info(compareFile + " not found");
+        final boolean prevConfigExists = Files.exists(compareFile);
+        if (!prevConfigExists && !dumpCurrentWhenRecordedUnavailable && !dumpDependencies) {
+            getLog().info("Config dump from the previous build does not exist at " + compareFile);
             return;
         }
 
         CuratedApplication curatedApplication = null;
         QuarkusClassLoader deploymentClassLoader = null;
         final ClassLoader originalCl = Thread.currentThread().getContextClassLoader();
-        Properties actualProps;
         final boolean clearPackageTypeSystemProperty = setPackageTypeSystemPropertyIfNativeProfileEnabled();
         try {
             curatedApplication = bootstrapApplication(launchMode);
-            deploymentClassLoader = curatedApplication.createDeploymentClassLoader();
-            Thread.currentThread().setContextClassLoader(deploymentClassLoader);
-
-            final Class<?> codeGenerator = deploymentClassLoader.loadClass("io.quarkus.deployment.CodeGenerator");
-            final Method dumpConfig = codeGenerator.getMethod("dumpCurrentConfigValues", ApplicationModel.class, String.class,
-                    Properties.class, QuarkusClassLoader.class, Properties.class, Path.class);
-            dumpConfig.invoke(null, curatedApplication.getApplicationModel(),
-                    launchMode.name(), getBuildSystemProperties(true),
-                    deploymentClassLoader, compareProps, targetFile);
+            if (prevConfigExists || dumpCurrentWhenRecordedUnavailable) {
+                final Path targetFile = getOutputFile(outputFile, launchMode.getDefaultProfile(), "-config-check");
+                Properties compareProps = new Properties();
+                if (prevConfigExists) {
+                    try (BufferedReader reader = Files.newBufferedReader(compareFile)) {
+                        compareProps.load(reader);
+                    } catch (IOException e) {
+                        throw new RuntimeException("Failed to read " + compareFile, e);
+                    }
+                }
+
+                deploymentClassLoader = curatedApplication.createDeploymentClassLoader();
+                Thread.currentThread().setContextClassLoader(deploymentClassLoader);
+
+                final Class<?> codeGenerator = deploymentClassLoader.loadClass("io.quarkus.deployment.CodeGenerator");
+                final Method dumpConfig = codeGenerator.getMethod("dumpCurrentConfigValues", ApplicationModel.class,
+                        String.class,
+                        Properties.class, QuarkusClassLoader.class, Properties.class, Path.class);
+                dumpConfig.invoke(null, curatedApplication.getApplicationModel(),
+                        launchMode.name(), getBuildSystemProperties(true),
+                        deploymentClassLoader, compareProps, targetFile);
+            }
+
+            if (dumpDependencies) {
+                final List<String> deps = new ArrayList<>();
+                for (var d : curatedApplication.getApplicationModel().getDependencies(DependencyFlags.DEPLOYMENT_CP)) {
+                    StringBuilder entry = new StringBuilder(d.toGACTVString());
+                    if (d.isSnapshot()) {
+                        var adler32 = new Adler32();
+                        updateChecksum(adler32, d.getResolvedPaths());
+                        entry.append(" ").append(adler32.getValue());
+                    }
+
+                    deps.add(entry.toString());
+                }
+                Collections.sort(deps);
+                final Path targetFile = getOutputFile(dependenciesFile, launchMode.getDefaultProfile(),
+                        "-dependency-checksums.txt");
+                Files.createDirectories(targetFile.getParent());
+                try (BufferedWriter writer = Files.newBufferedWriter(targetFile)) {
+                    for (var s : deps) {
+                        writer.write(s);
+                        writer.newLine();
+                    }
+                }
+            }
         } catch (Exception any) {
             throw new MojoExecutionException("Failed to bootstrap Quarkus application", any);
         } finally {
@@ -142,4 +182,44 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException {
             }
         }
     }
+
+    private Path getOutputFile(File outputFile, String profile, String fileNameSuffix) {
+        if (outputFile == null) {
+            return outputDirectory.toPath().resolve("quarkus-" + profile + fileNameSuffix);
+        }
+        if (outputFile.isAbsolute()) {
+            return outputFile.toPath();
+        }
+        return outputDirectory.toPath().resolve(outputFile.toPath());
+    }
+
+    private static void updateChecksum(Checksum checksum, Iterable<Path> pc) throws IOException {
+        for (var path : sort(pc)) {
+            if (Files.isDirectory(path)) {
+                try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
+                    updateChecksum(checksum, stream);
+                }
+            } else {
+                checksum.update(Files.readAllBytes(path));
+            }
+        }
+    }
+
+    private static Iterable<Path> sort(Iterable<Path> original) {
+        var i = original.iterator();
+        if (!i.hasNext()) {
+            return List.of();
+        }
+        var o = i.next();
+        if (!i.hasNext()) {
+            return List.of(o);
+        }
+        final List<Path> sorted = new ArrayList<>();
+        sorted.add(o);
+        while (i.hasNext()) {
+            sorted.add(i.next());
+        }
+        Collections.sort(sorted);
+        return sorted;
+    }
 }

docs/src/main/asciidoc/config-reference.adoc

@@ -738,6 +738,11 @@ Maven projects could add the following goal to their `quarkus-maven-plugin` conf
 The `track-config-changes` goal looks for `${project.basedir}/.quarkus/quarkus-prod-config-dump` (file name and directory are configurable) and, if the file already exists, checks whether the values stored in the config dump have changed.
 It will log the changed options and save the current values of each of the options present in `${project.basedir}/.quarkus/quarkus-prod-config-dump` in `${project.basedir}/target/quarkus-prod-config.check` (the target file name and location can be configured). If the build time configuration has not changed since the last build both `${project.basedir}/.quarkus/quarkus-prod-config-dump` and `${project.basedir}/.quarkus/quarkus-prod-config-dump` will be identical.
 
+==== Dump Quarkus application dependencies
+
+In addition to dumping configuration values, `track-config-changes` goal also dumps all the Quarkus application dependencies, including Quarkus build time dependencies, along with their checksums (Adler32). This file could be used to check whether Quarkus build classpath has changed since the previous run.
+By default, the dependency checksums will be stored under `target/quarkus-prod-dependency-checksums.txt` file. A different location could be configured using plugin parameters.
+
 ==== Dump current build configuration when the recorded configuration isn't found
 
 By default, `track-config-changes` looks for the configuration recorded during previous build and does nothing if it's not found. Enabling `dumpCurrentWhenRecordedUnavailable` parameter will make it dump the current build configuration

independent-projects/bootstrap/app-model/src/main/java/io/quarkus/maven/dependency/ArtifactCoords.java

@@ -42,6 +42,10 @@ default boolean isJar() {
         return TYPE_JAR.equals(getType());
     }
 
+    default boolean isSnapshot() {
+        return getVersion() != null && getVersion().endsWith("-SNAPSHOT");
+    }
+
     default String toGACTVString() {
         return getGroupId() + ":" + getArtifactId() + ":" + getClassifier() + ":" + getType() + ":" + getVersion();
     }

independent-projects/parent/pom.xml

@@ -424,6 +424,63 @@
                                 </runtimeClassPath>
                             </normalization>
                             <plugins>
+                                <plugin>
+                                    <artifactId>maven-surefire-plugin</artifactId>
+                                    <inputs>
+                                        <fileSets>
+                                            <fileSet>
+                                                <name>dependency-checksums</name>
+                                                <paths>
+                                                    <path>${project.build.directory}</path>
+                                                </paths>
+                                                <includes>
+                                                    <include>quarkus-*-dependency-checksums.txt</include>
+                                                </includes>
+                                                <normalization>RELATIVE_PATH</normalization>
+                                            </fileSet>
+                                        </fileSets>
+                                    </inputs>
+                                </plugin>
+                                <plugin>
+                                    <artifactId>maven-failsafe-plugin</artifactId>
+                                    <inputs>
+                                        <fileSets>
+                                            <fileSet>
+                                                <name>dependency-checksums</name>
+                                                <paths>
+                                                    <path>${project.build.directory}</path>
+                                                </paths>
+                                                <includes>
+                                                    <include>quarkus-*-dependency-checksums.txt</include>
+                                                </includes>
+                                                <normalization>RELATIVE_PATH</normalization>
+                                            </fileSet>
+                                        </fileSets>
+                                    </inputs>
+                                </plugin>
+                                <plugin>
+                                    <groupId>io.quarkus</groupId>
+                                    <artifactId>quarkus-maven-plugin</artifactId>
+                                    <executions>
+                                        <execution>
+                                            <id>default</id>
+                                            <inputs>
+                                                <fileSets>
+                                                    <fileSet>
+                                                        <name>dependency-checksums</name>
+                                                        <paths>
+                                                            <path>${project.build.directory}</path>
+                                                        </paths>
+                                                        <includes>
+                                                            <include>quarkus-*-dependency-checksums.txt</include>
+                                                        </includes>
+                                                        <normalization>RELATIVE_PATH</normalization>
+                                                    </fileSet>
+                                                </fileSets>
+                                            </inputs>
+                                        </execution>
+                                    </executions>
+                                </plugin>
                                 <plugin>
                                     <groupId>org.jetbrains.kotlin</groupId>
                                     <artifactId>kotlin-maven-plugin</artifactId>

integration-tests/main/pom.xml

@@ -23,6 +23,9 @@
 
         <!-- do not update this dependency, it is only used for testing -->
         <webjar.jquery-ui.version>1.13.0</webjar.jquery-ui.version>
+
+        <quarkus.config-tracking.enabled>true</quarkus.config-tracking.enabled>
+        <quarkus.native.container-build>true</quarkus.native.container-build>
     </properties>
 
     <dependencies>
@@ -507,6 +510,16 @@
                 <groupId>io.quarkus</groupId>
                 <artifactId>quarkus-maven-plugin</artifactId>
                 <executions>
+                    <execution>
+                        <id>track-config-changes</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>track-config-changes</goal>
+                        </goals>
+                        <configuration>
+                            <dumpCurrentWhenRecordedUnavailable>true</dumpCurrentWhenRecordedUnavailable>
+                        </configuration>
+                    </execution>
                     <execution>
                         <goals>
                             <goal>build</goal>

integration-tests/pom.xml

@@ -70,6 +70,17 @@
                     <groupId>io.quarkus</groupId>
                     <artifactId>quarkus-maven-plugin</artifactId>
                     <version>${project.version}</version>
+                    <executions>
+                        <execution>
+                            <!-- for now this execution will only dump Quarkus application
+                            dependency checksums that could indicate build time classpath changes -->
+                            <id>track-config-changes</id>
+                            <phase>process-resources</phase>
+                            <goals>
+                                <goal>track-config-changes</goal>
+                            </goals>
+                        </execution>
+                    </executions>
                     <configuration>
                         <noDeps>true</noDeps>
                         <skip>${quarkus.build.skip}</skip>
@@ -304,7 +315,7 @@
                 <module>amazon-lambda-rest-resteasy-reactive</module>
                 <module>amazon-lambda-rest-reactive-routes</module>
                 <module>amazon-lambda-rest-funqy</module>
-                <module>amazon-lambda-rest-servlet</module>		
+                <module>amazon-lambda-rest-servlet</module>
                 <module>amazon-lambda-http-resteasy</module>
                 <module>amazon-lambda-http-resteasy-reactive</module>
                 <module>container-image</module>