Skip to content

Commit

Permalink
Merge pull request #22547 from gsmet/fix-module-info
Browse files Browse the repository at this point in the history 
Clean up how entries are ignored in jar building
  • Loading branch information
@gsmet
gsmet authored Jan 1, 2022
2 parents bdad107 + f0e667f commit 5c120b1
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 59 deletions.
5 changes: 5 additions & 0 deletions core/deployment/src/main/java/io/quarkus/deployment/mutability/DevModeTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,11 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx
Files.createDirectories(target);
} else {
if (!Files.exists(target)) {
// make sure the parent directories are created first
// META-INF/MANIFEST.MF is often written first,
// even before META-INF is written probably due to
// https://bugs.openjdk.java.net/browse/JDK-8031748
Files.createDirectories(target.getParent());
try (OutputStream out = Files.newOutputStream(target)) {
IoUtils.copy(out, fs);
}
Expand Down
154 changes: 96 additions & 58 deletions core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,36 +103,9 @@
*/
public class JarResultBuildStep {

private static final Collection<String> IGNORED_ENTRIES = Arrays.asList(
"META-INF/INDEX.LIST",
"META-INF/MANIFEST.MF",
"module-info.class",
"META-INF/LICENSE",
"META-INF/LICENSE.txt",
"META-INF/LICENSE.md",
"META-INF/LGPL-3.0.txt",
"META-INF/ASL-2.0.txt",
"META-INF/NOTICE",
"META-INF/NOTICE.txt",
"META-INF/NOTICE.md",
"META-INF/README",
"META-INF/README.txt",
"META-INF/README.md",
"META-INF/DEPENDENCIES",
"META-INF/DEPENDENCIES.txt",
"META-INF/beans.xml",
"META-INF/quarkus-config-roots.list",
"META-INF/quarkus-javadoc.properties",
"META-INF/quarkus-extension.properties",
"META-INF/quarkus-extension.json",
"META-INF/quarkus-extension.yaml",
"META-INF/quarkus-deployment-dependency.graph",
"META-INF/jandex.idx",
"META-INF/panache-archive.marker",
"META-INF/build.metadata", // present in the Red Hat Build of Quarkus
"LICENSE");

private static final Predicate<String> CONCATENATED_ENTRIES_PREDICATE = new Predicate<>() {
private static final Predicate<String> UBER_JAR_IGNORED_ENTRIES_PREDICATE = new IsEntryIgnoredForUberJarPredicate();

private static final Predicate<String> UBER_JAR_CONCATENATED_ENTRIES_PREDICATE = new Predicate<>() {
@Override
public boolean test(String path) {
return "META-INF/io.netty.versions.properties".equals(path) ||
Expand Down Expand Up @@ -334,7 +307,7 @@ private void buildUberJar0(CurateOutcomeBuildItem curateOutcomeBuildItem,
Path runnerJar) throws Exception {
try (FileSystem runnerZipFs = ZipUtils.newZip(runnerJar)) {

log.info("Building fat jar: " + runnerJar);
log.info("Building uber jar: " + runnerJar);

final Map<String, String> seen = new HashMap<>();
final Map<String, Set<Dependency>> duplicateCatcher = new HashMap<>();
Expand All @@ -343,11 +316,19 @@ private void buildUberJar0(CurateOutcomeBuildItem curateOutcomeBuildItem,
.map(UberJarMergedResourceBuildItem::getPath)
.collect(Collectors.toSet());
final Set<ArtifactKey> removed = getRemovedKeys(classLoadingConfig);
Set<String> finalIgnoredEntries = new HashSet<>(IGNORED_ENTRIES);
packageConfig.userConfiguredIgnoredEntries.ifPresent(finalIgnoredEntries::addAll);

Set<String> ignoredEntries = new HashSet<>();
packageConfig.userConfiguredIgnoredEntries.ifPresent(ignoredEntries::addAll);
ignoredResources.stream()
.map(UberJarIgnoredResourceBuildItem::getPath)
.forEach(finalIgnoredEntries::add);
.forEach(ignoredEntries::add);
Predicate<String> allIgnoredEntriesPredicate = new Predicate<String>() {
@Override
public boolean test(String path) {
return UBER_JAR_IGNORED_ENTRIES_PREDICATE.test(path)
|| ignoredEntries.contains(path);
}
};

final Collection<ResolvedDependency> appDeps = curateOutcomeBuildItem.getApplicationModel()
.getRuntimeDependencies();
Expand Down Expand Up @@ -380,12 +361,12 @@ private void buildUberJar0(CurateOutcomeBuildItem curateOutcomeBuildItem,
try (FileSystem artifactFs = ZipUtils.newFileSystem(resolvedDep)) {
for (final Path root : artifactFs.getRootDirectories()) {
walkFileDependencyForDependency(root, runnerZipFs, seen, duplicateCatcher, concatenatedEntries,
finalIgnoredEntries, appDep, existingEntries, mergeResourcePaths);
allIgnoredEntriesPredicate, appDep, existingEntries, mergeResourcePaths);
}
}
} else {
walkFileDependencyForDependency(resolvedDep, runnerZipFs, seen, duplicateCatcher,
concatenatedEntries, finalIgnoredEntries, appDep, existingEntries,
concatenatedEntries, allIgnoredEntriesPredicate, appDep, existingEntries,
mergeResourcePaths);
}
}
Expand All @@ -394,16 +375,14 @@ private void buildUberJar0(CurateOutcomeBuildItem curateOutcomeBuildItem,
for (Map.Entry<String, Set<Dependency>> entry : duplicateCatcher.entrySet()) {
if (entry.getValue().size() > 1) {
if (explained.add(entry.getValue())) {
if (!"module-info.class".endsWith(entry.getKey())) {
log.warn("Dependencies with duplicate files detected. The dependencies " + entry.getValue()
+ " contain duplicate files, e.g. " + entry.getKey());
}
log.warn("Dependencies with duplicate files detected. The dependencies " + entry.getValue()
+ " contain duplicate files, e.g. " + entry.getKey());
}
}
}
copyCommonContent(runnerZipFs, concatenatedEntries, applicationArchivesBuildItem, transformedClasses,
generatedClasses,
generatedResources, seen, finalIgnoredEntries);
generatedResources, seen, allIgnoredEntriesPredicate);
// now that all entries have been added, check if there's a META-INF/versions/ entry. If present,
// mark this jar as multi-release jar. Strictly speaking, the jar spec expects META-INF/versions/N
// directory where N is an integer greater than 8, but we don't do that level of checks here but that
Expand Down Expand Up @@ -458,7 +437,7 @@ private static boolean includeAppDep(ResolvedDependency appDep, Optional<Set<Art

private void walkFileDependencyForDependency(Path root, FileSystem runnerZipFs, Map<String, String> seen,
Map<String, Set<Dependency>> duplicateCatcher, Map<String, List<byte[]>> concatenatedEntries,
Set<String> finalIgnoredEntries, Dependency appDep, Set<String> existingEntries,
Predicate<String> ignoredEntriesPredicate, Dependency appDep, Set<String> existingEntries,
Set<String> mergeResourcePaths) throws IOException {
final Path metaInfDir = root.resolve("META-INF");
Files.walkFileTree(root, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE,
Expand Down Expand Up @@ -489,12 +468,12 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
return FileVisitResult.CONTINUE;
}
if (!existingEntries.contains(relativePath)) {
if (CONCATENATED_ENTRIES_PREDICATE.test(relativePath)
if (UBER_JAR_CONCATENATED_ENTRIES_PREDICATE.test(relativePath)
|| mergeResourcePaths.contains(relativePath)) {
concatenatedEntries.computeIfAbsent(relativePath, (u) -> new ArrayList<>())
.add(Files.readAllBytes(file));
return FileVisitResult.CONTINUE;
} else if (!finalIgnoredEntries.contains(relativePath)) {
} else if (!ignoredEntriesPredicate.test(relativePath)) {
duplicateCatcher.computeIfAbsent(relativePath, (a) -> new HashSet<>())
.add(appDep);
if (!seen.containsKey(relativePath)) {
Expand Down Expand Up @@ -673,11 +652,11 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,
jars.add(runnerJar);

if (!rebuild) {
Set<String> finalIgnoredEntries = new HashSet<>(IGNORED_ENTRIES);
packageConfig.userConfiguredIgnoredEntries.ifPresent(finalIgnoredEntries::addAll);
Predicate<String> ignoredEntriesPredicate = getThinJarIgnoredEntriesPredicate(packageConfig);

try (FileSystem runnerZipFs = ZipUtils.newZip(runnerJar)) {
for (Path root : applicationArchivesBuildItem.getRootArchive().getRootDirectories()) {
copyFiles(root, runnerZipFs, null, finalIgnoredEntries);
copyFiles(root, runnerZipFs, null, ignoredEntriesPredicate);
}
}
}
Expand Down Expand Up @@ -1141,12 +1120,12 @@ private void doLegacyThinJarGeneration(CurateOutcomeBuildItem curateOutcomeBuild

final Collection<ResolvedDependency> appDeps = curateOutcomeBuildItem.getApplicationModel()
.getRuntimeDependencies();
final Set<String> finalIgnoredEntries = new HashSet<>(IGNORED_ENTRIES);
packageConfig.userConfiguredIgnoredEntries.ifPresent(finalIgnoredEntries::addAll);

Predicate<String> ignoredEntriesPredicate = getThinJarIgnoredEntriesPredicate(packageConfig);

final Set<ArtifactKey> removed = getRemovedKeys(classLoadingConfig);
copyLibraryJars(runnerZipFs, outputTargetBuildItem, transformedClasses, libDir, classPath, appDeps, services,
finalIgnoredEntries, removed);
ignoredEntriesPredicate, removed);

ResolvedDependency appArtifact = curateOutcomeBuildItem.getApplicationModel().getAppArtifact();
// the manifest needs to be the first entry in the jar, otherwise JarInputStream does not work properly
Expand All @@ -1155,13 +1134,13 @@ private void doLegacyThinJarGeneration(CurateOutcomeBuildItem curateOutcomeBuild
applicationInfo);

copyCommonContent(runnerZipFs, services, applicationArchivesBuildItem, transformedClasses, allClasses,
generatedResources, seen, finalIgnoredEntries);
generatedResources, seen, ignoredEntriesPredicate);
}

private void copyLibraryJars(FileSystem runnerZipFs, OutputTargetBuildItem outputTargetBuildItem,
TransformedClassesBuildItem transformedClasses, Path libDir,
StringBuilder classPath, Collection<ResolvedDependency> appDeps, Map<String, List<byte[]>> services,
Set<String> ignoredEntries, Set<ArtifactKey> removedDependencies) throws IOException {
Predicate<String> ignoredEntriesPredicate, Set<ArtifactKey> removedDependencies) throws IOException {

for (ResolvedDependency appDep : appDeps) {

Expand Down Expand Up @@ -1198,7 +1177,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
final Path relativePath = resolvedDep.relativize(file);
final String relativeUri = toUri(relativePath);
if (ignoredEntries.contains(relativeUri)) {
if (ignoredEntriesPredicate.test(relativeUri)) {
return FileVisitResult.CONTINUE;
}
if (relativeUri.startsWith("META-INF/services/") && relativeUri.length() > 18) {
Expand All @@ -1223,7 +1202,7 @@ private void copyCommonContent(FileSystem runnerZipFs, Map<String, List<byte[]>>
ApplicationArchivesBuildItem appArchives, TransformedClassesBuildItem transformedClassesBuildItem,
List<GeneratedClassBuildItem> generatedClasses,
List<GeneratedResourceBuildItem> generatedResources, Map<String, String> seen,
Set<String> ignoredEntries)
Predicate<String> ignoredEntriesPredicate)
throws IOException {

//TODO: this is probably broken in gradle
Expand Down Expand Up @@ -1257,7 +1236,7 @@ private void copyCommonContent(FileSystem runnerZipFs, Map<String, List<byte[]>>
}

for (GeneratedResourceBuildItem i : generatedResources) {
if (ignoredEntries.contains(i.getName())) {
if (ignoredEntriesPredicate.test(i.getName())) {
continue;
}
Path target = runnerZipFs.getPath(i.getName());
Expand All @@ -1275,7 +1254,7 @@ private void copyCommonContent(FileSystem runnerZipFs, Map<String, List<byte[]>>
}

for (Path root : appArchives.getRootArchive().getRootDirectories()) {
copyFiles(root, runnerZipFs, concatenatedEntries, ignoredEntries);
copyFiles(root, runnerZipFs, concatenatedEntries, ignoredEntriesPredicate);
}

for (Map.Entry<String, List<byte[]>> entry : concatenatedEntries.entrySet()) {
Expand Down Expand Up @@ -1400,15 +1379,16 @@ private void generateManifest(FileSystem runnerZipFs, final String classPath, Pa
* @param services the services map
* @throws IOException if an error occurs
*/
private void copyFiles(Path dir, FileSystem fs, Map<String, List<byte[]>> services, Set<String> ignoredEntries)
private void copyFiles(Path dir, FileSystem fs, Map<String, List<byte[]>> services,
Predicate<String> ignoredEntriesPredicate)
throws IOException {
try (Stream<Path> fileTreeElements = Files.walk(dir)) {
fileTreeElements.forEach(new Consumer<Path>() {
@Override
public void accept(Path path) {
final Path file = dir.relativize(path);
final String relativePath = toUri(file);
if (relativePath.isEmpty() || ignoredEntries.contains(relativePath)) {
if (relativePath.isEmpty() || ignoredEntriesPredicate.test(relativePath)) {
return;
}
try {
Expand Down Expand Up @@ -1504,6 +1484,64 @@ static boolean isBlockOrSF(final String s) {
|| s.endsWith(".EC");
}

private Predicate<String> getThinJarIgnoredEntriesPredicate(PackageConfig packageConfig) {
if (packageConfig.userConfiguredIgnoredEntries.isEmpty()) {
return new Predicate<String>() {
@Override
public boolean test(String path) {
return false;
}
};
}

Set<String> ignoredEntries = new HashSet<>(packageConfig.userConfiguredIgnoredEntries.get());
Predicate<String> ignoredEntriesPredicate = new Predicate<String>() {
@Override
public boolean test(String path) {
return ignoredEntries.contains(path);
}
};
return ignoredEntriesPredicate;
}

private static class IsEntryIgnoredForUberJarPredicate implements Predicate<String> {

private static final Set<String> UBER_JAR_IGNORED_ENTRIES = Set.of(
"META-INF/INDEX.LIST",
"META-INF/MANIFEST.MF",
"module-info.class",
"META-INF/LICENSE",
"META-INF/LICENSE.txt",
"META-INF/LICENSE.md",
"META-INF/LGPL-3.0.txt",
"META-INF/ASL-2.0.txt",
"META-INF/NOTICE",
"META-INF/NOTICE.txt",
"META-INF/NOTICE.md",
"META-INF/README",
"META-INF/README.txt",
"META-INF/README.md",
"META-INF/DEPENDENCIES",
"META-INF/DEPENDENCIES.txt",
"META-INF/beans.xml",
"META-INF/quarkus-config-roots.list",
"META-INF/quarkus-javadoc.properties",
"META-INF/quarkus-extension.properties",
"META-INF/quarkus-extension.json",
"META-INF/quarkus-extension.yaml",
"META-INF/quarkus-deployment-dependency.graph",
"META-INF/jandex.idx",
"META-INF/panache-archive.marker",
"META-INF/build.metadata", // present in the Red Hat Build of Quarkus
"LICENSE");

@Override
public boolean test(String path) {
return UBER_JAR_IGNORED_ENTRIES.contains(path)
|| path.endsWith("module-info.class");
}
}

private static class IsJsonFilePredicate implements BiPredicate<Path, BasicFileAttributes> {

@Override
Expand Down
9 changes: 8 additions & 1 deletion ...rojects/bootstrap/runner/src/main/java/io/quarkus/bootstrap/runner/QuarkusEntryPoint.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.quarkus.bootstrap.runner;

import io.quarkus.bootstrap.forkjoin.QuarkusForkJoinWorkerThread;
import io.quarkus.bootstrap.logging.InitialConfigurator;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
Expand All @@ -25,7 +26,13 @@ public static void main(String... args) throws Throwable {
System.setProperty("java.util.concurrent.ForkJoinPool.common.threadFactory",
"io.quarkus.bootstrap.forkjoin.QuarkusForkJoinWorkerThreadFactory");
Timing.staticInitStarted(false);
doRun(args);

try {
doRun(args);
} catch (Exception e) {
InitialConfigurator.DELAYED_HANDLER.close();
throw e;
}
}

private static void doRun(Object args) throws IOException, ClassNotFoundException, IllegalAccessException,
Expand Down

0 comments on commit 5c120b1

Please sign in to comment.