diff --git a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/Constants.java b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/Constants.java index e92993defe167..0383fc1c39d85 100644 --- a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/Constants.java +++ b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/Constants.java @@ -16,4 +16,6 @@ public interface Constants { String PLATFORM_PROPERTIES_ARTIFACT_ID_SUFFIX = "-quarkus-platform-properties"; String JSON = "json"; + + String LAST_UPDATED = "last-updated"; } diff --git a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/ExtensionCatalogResolver.java b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/ExtensionCatalogResolver.java index 20fb5943d338b..fdaaa7e0586f8 100644 --- a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/ExtensionCatalogResolver.java +++ b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/ExtensionCatalogResolver.java @@ -260,13 +260,30 @@ public PlatformCatalog resolvePlatformCatalog(String quarkusVersion) throws Regi final JsonPlatformCatalog result = new JsonPlatformCatalog(); final List collectedPlatforms = new ArrayList<>(); - result.setPlatforms(collectedPlatforms); final Set collectedPlatformKeys = new HashSet<>(); + String lastUpdated = null; + boolean sawUnknownLastUpdate = false; for (PlatformCatalog c : catalogs) { collectPlatforms(c, collectedPlatforms, collectedPlatformKeys); + if (!sawUnknownLastUpdate) { + final Object catalogLastUpdated = c.getMetadata().get(Constants.LAST_UPDATED); + if (catalogLastUpdated == null) { + // if for one of the catalogs it's unknown, it's going to be unknown for the merged catalog + lastUpdated = null; + sawUnknownLastUpdate = true; + } else if (lastUpdated == null) { + lastUpdated = catalogLastUpdated.toString(); + } else if (lastUpdated.compareTo(catalogLastUpdated.toString()) < 0) { + lastUpdated = catalogLastUpdated.toString(); + } + } } + if (lastUpdated != null) { + result.getMetadata().put(Constants.LAST_UPDATED, lastUpdated); + } + result.setPlatforms(collectedPlatforms); return result; } diff --git a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/catalog/json/JsonEntityWithAnySupport.java b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/catalog/json/JsonEntityWithAnySupport.java index d43f773818aff..85b8b7bcb221c 100644 --- a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/catalog/json/JsonEntityWithAnySupport.java +++ b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/catalog/json/JsonEntityWithAnySupport.java @@ -2,17 +2,16 @@ import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; -import java.util.Collections; import java.util.HashMap; import java.util.Map; abstract class JsonEntityWithAnySupport { - private Map metadata; + private Map metadata = new HashMap<>(0); @JsonAnyGetter public Map getMetadata() { - return metadata == null ? Collections.emptyMap() : metadata; + return metadata; } public void setMetadata(Map metadata) { @@ -21,9 +20,6 @@ public void setMetadata(Map metadata) { @JsonAnySetter public void setAny(String name, Object value) { - if (metadata == null) { - metadata = new HashMap<>(); - } metadata.put(name, value); } } diff --git a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenPlatformsResolver.java b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenPlatformsResolver.java index 585e96d78c8f8..c2c261f0a8517 100644 --- a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenPlatformsResolver.java +++ b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenPlatformsResolver.java @@ -2,6 +2,7 @@ import io.quarkus.devtools.messagewriter.MessageWriter; import io.quarkus.maven.ArtifactCoords; +import io.quarkus.registry.Constants; import io.quarkus.registry.RegistryResolutionException; import io.quarkus.registry.catalog.PlatformCatalog; import io.quarkus.registry.catalog.json.JsonCatalogMapperHelper; @@ -11,8 +12,10 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Objects; +import org.apache.maven.artifact.repository.metadata.Metadata; import org.eclipse.aether.artifact.Artifact; import org.eclipse.aether.artifact.DefaultArtifact; +import org.eclipse.aether.resolution.ArtifactResult; public class MavenPlatformsResolver implements RegistryPlatformsResolver { @@ -33,18 +36,41 @@ public PlatformCatalog resolvePlatforms(String quarkusVersion) throws RegistryRe final Artifact catalogArtifact = new DefaultArtifact(baseCoords.getGroupId(), baseCoords.getArtifactId(), quarkusVersion, baseCoords.getType(), baseCoords.getVersion()); log.debug("Resolving platform catalog %s", catalogArtifact); - final Path jsonFile; + final ArtifactResult artifactResult; try { - jsonFile = artifactResolver.resolve(catalogArtifact); + artifactResult = artifactResolver.resolveArtifact(catalogArtifact); } catch (Exception e) { log.debug("Failed to resolve platform catalog %s", catalogArtifact); return null; } + final Path jsonFile = artifactResult.getArtifact().getFile().toPath(); + final JsonPlatformCatalog catalog; try { - return JsonCatalogMapperHelper.deserialize(jsonFile, JsonPlatformCatalog.class); + catalog = JsonCatalogMapperHelper.deserialize(jsonFile, JsonPlatformCatalog.class); } catch (IOException e) { throw new RegistryResolutionException( "Failed to load platform catalog from " + jsonFile, e); } + + try { + final Metadata mavenMetadata = artifactResolver.resolveMetadata(artifactResult); + if (mavenMetadata != null) { + final String lastUpdated = mavenMetadata.getVersioning() == null ? null + : mavenMetadata.getVersioning().getLastUpdated(); + if (lastUpdated != null) { + /* + * This is how it can be parsed + * java.util.TimeZone timezone = java.util.TimeZone.getTimeZone("UTC"); + * java.text.DateFormat fmt = new java.text.SimpleDateFormat("yyyyMMddHHmmss"); + * fmt.setTimeZone(timezone); + * final Date date = fmt.parse(lastUpdated); + */ + catalog.getMetadata().put(Constants.LAST_UPDATED, lastUpdated); + } + } + } catch (Exception e) { + log.debug("Failed to resolve Maven metadata for %s", catalogArtifact); + } + return catalog; } } diff --git a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryArtifactResolver.java b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryArtifactResolver.java index 66fa80dcb4f40..0634d3e719e2c 100644 --- a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryArtifactResolver.java +++ b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryArtifactResolver.java @@ -2,13 +2,21 @@ import io.quarkus.bootstrap.resolver.maven.BootstrapMavenException; import java.nio.file.Path; +import org.apache.maven.artifact.repository.metadata.Metadata; import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.resolution.ArtifactResult; public interface MavenRegistryArtifactResolver { - Path resolve(Artifact artifact) throws BootstrapMavenException; + default Path resolve(Artifact artifact) throws BootstrapMavenException { + return resolveArtifact(artifact).getArtifact().getFile().toPath(); + } + + ArtifactResult resolveArtifact(Artifact artifact) throws BootstrapMavenException; Path findArtifactDirectory(Artifact artifact) throws BootstrapMavenException; String getLatestVersionFromRange(Artifact artifact, String versionRange) throws BootstrapMavenException; + + Metadata resolveMetadata(ArtifactResult result) throws BootstrapMavenException; } diff --git a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryArtifactResolverWithCleanup.java b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryArtifactResolverWithCleanup.java index 59136692abb8b..c0ea6eca30a8c 100644 --- a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryArtifactResolverWithCleanup.java +++ b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryArtifactResolverWithCleanup.java @@ -2,15 +2,23 @@ import io.quarkus.bootstrap.resolver.maven.BootstrapMavenException; import io.quarkus.bootstrap.resolver.maven.MavenArtifactResolver; +import java.io.BufferedReader; import java.io.File; import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; +import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader; import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.metadata.DefaultMetadata; +import org.eclipse.aether.metadata.Metadata; +import org.eclipse.aether.metadata.Metadata.Nature; import org.eclipse.aether.repository.LocalRepositoryManager; +import org.eclipse.aether.repository.RemoteRepository; import org.eclipse.aether.resolution.ArtifactResult; +import org.eclipse.aether.resolution.MetadataRequest; +import org.eclipse.aether.resolution.MetadataResult; public class MavenRegistryArtifactResolverWithCleanup implements MavenRegistryArtifactResolver { @@ -23,9 +31,8 @@ public MavenRegistryArtifactResolverWithCleanup(MavenArtifactResolver resolver, } @Override - public Path resolve(Artifact artifact) throws BootstrapMavenException { - return resolveAndCleanupOldTimestampedVersions(resolver, artifact, cleanupTimestampedVersions).getArtifact().getFile() - .toPath(); + public ArtifactResult resolveArtifact(Artifact artifact) throws BootstrapMavenException { + return resolveAndCleanupOldTimestampedVersions(resolver, artifact, cleanupTimestampedVersions); } @Override @@ -78,6 +85,40 @@ protected static ArtifactResult resolveAndCleanupOldTimestampedVersions(MavenArt } } } + return result; } + + @Override + public org.apache.maven.artifact.repository.metadata.Metadata resolveMetadata(ArtifactResult result) + throws BootstrapMavenException { + final Artifact artifact = result.getArtifact(); + Metadata md = new DefaultMetadata(artifact.getGroupId(), artifact.getArtifactId(), + artifact.isSnapshot() ? artifact.getBaseVersion() : artifact.getVersion(), + "maven-metadata.xml", artifact.isSnapshot() ? Nature.SNAPSHOT : Nature.RELEASE); + + final MetadataRequest mdr = new MetadataRequest().setMetadata(md); + final String repoId = result.getRepository().getId(); + if (repoId != null && !repoId.equals("local")) { + for (RemoteRepository r : resolver.getRepositories()) { + if (r.getId().equals(repoId)) { + mdr.setRepository(r); + break; + } + } + } + + final List mdResults = resolver.getSystem().resolveMetadata(resolver.getSession(), Arrays.asList(mdr)); + if (!mdResults.isEmpty()) { + md = mdResults.get(0).getMetadata(); + if (md != null && md.getFile() != null && md.getFile().exists()) { + try (BufferedReader reader = new BufferedReader(new java.io.FileReader(md.getFile()))) { + return new MetadataXpp3Reader().read(reader); + } catch (Exception e) { + // ignore for now + } + } + } + return null; + } } diff --git a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryCache.java b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryCache.java index 5723aadc31cc8..e6dd61eb72fc7 100644 --- a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryCache.java +++ b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryCache.java @@ -9,8 +9,9 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Objects; import org.eclipse.aether.artifact.DefaultArtifact; @@ -24,8 +25,15 @@ public class MavenRegistryCache implements RegistryCache { public MavenRegistryCache(RegistryConfig config, MavenRegistryArtifactResolver resolver, MessageWriter log) { this.config = config; - this.artifacts = Arrays.asList(config.getDescriptor().getArtifact(), - config.getNonPlatformExtensions().getArtifact(), config.getPlatforms().getArtifact()); + final List artifacts = new ArrayList<>(3); + artifacts.add(config.getDescriptor().getArtifact()); + if (config.getNonPlatformExtensions() != null) { + artifacts.add(config.getNonPlatformExtensions().getArtifact()); + } + if (config.getPlatforms() != null) { + artifacts.add(config.getPlatforms().getArtifact()); + } + this.artifacts = artifacts; this.resolver = Objects.requireNonNull(resolver); this.log = Objects.requireNonNull(log); } diff --git a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryClientFactory.java b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryClientFactory.java index 53f65dc4e1d57..4cbf8d8a058f8 100644 --- a/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryClientFactory.java +++ b/independent-projects/tools/registry-client/src/main/java/io/quarkus/registry/client/maven/MavenRegistryClientFactory.java @@ -161,7 +161,7 @@ public RegistryClient buildRegistryClient(RegistryConfig config) throws Registry } return new RegistryClientDispatcher(config, platformsResolver, - Boolean.TRUE.equals(config.getPlatforms().getExtensionCatalogsIncluded()) + Boolean.TRUE.equals(platformsConfig == null ? Boolean.FALSE : platformsConfig.getExtensionCatalogsIncluded()) ? new MavenPlatformExtensionsResolver(defaultResolver, log) : new MavenPlatformExtensionsResolver(defaultResolver(originalResolver, cleanupTimestampedArtifacts), log), diff --git a/independent-projects/tools/registry-client/src/test/java/io/quarkus/registry/catalog/platform/PlatformCatalogLastUpdatedTest.java b/independent-projects/tools/registry-client/src/test/java/io/quarkus/registry/catalog/platform/PlatformCatalogLastUpdatedTest.java new file mode 100644 index 0000000000000..74f4f2820fada --- /dev/null +++ b/independent-projects/tools/registry-client/src/test/java/io/quarkus/registry/catalog/platform/PlatformCatalogLastUpdatedTest.java @@ -0,0 +1,170 @@ +package io.quarkus.registry.catalog.platform; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.quarkus.bootstrap.resolver.maven.MavenArtifactResolver; +import io.quarkus.maven.ArtifactCoords; +import io.quarkus.registry.Constants; +import io.quarkus.registry.ExtensionCatalogResolver; +import io.quarkus.registry.catalog.Platform; +import io.quarkus.registry.catalog.PlatformCatalog; +import io.quarkus.registry.catalog.json.JsonCatalogMapperHelper; +import io.quarkus.registry.catalog.json.JsonPlatform; +import io.quarkus.registry.catalog.json.JsonPlatformCatalog; +import io.quarkus.registry.catalog.json.JsonPlatformRelease; +import io.quarkus.registry.catalog.json.JsonPlatformReleaseVersion; +import io.quarkus.registry.catalog.json.JsonPlatformStream; +import io.quarkus.registry.config.json.JsonRegistriesConfig; +import io.quarkus.registry.config.json.JsonRegistryConfig; +import io.quarkus.registry.config.json.JsonRegistryDescriptorConfig; +import io.quarkus.registry.config.json.JsonRegistryPlatformsConfig; +import io.quarkus.registry.config.json.RegistriesConfigMapperHelper; +import java.io.Reader; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Iterator; +import org.apache.maven.artifact.repository.metadata.Metadata; +import org.apache.maven.artifact.repository.metadata.SnapshotVersion; +import org.apache.maven.artifact.repository.metadata.Versioning; +import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader; +import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Writer; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.artifact.DefaultArtifact; +import org.eclipse.aether.metadata.DefaultMetadata; +import org.eclipse.aether.metadata.Metadata.Nature; +import org.junit.jupiter.api.Test; + +public class PlatformCatalogLastUpdatedTest { + + @Test + void testLastUpdated() throws Exception { + + final Path registryWorkDir = Paths.get("target").resolve("test-registry").normalize().toAbsolutePath(); + Files.createDirectories(registryWorkDir); + + final Path registryRepoDir = registryWorkDir.resolve("repo"); + Files.createDirectories(registryRepoDir); + + final MavenArtifactResolver mvn = MavenArtifactResolver.builder().setWorkspaceDiscovery(false) + .setLocalRepository(registryRepoDir.toString()).build(); + + final JsonRegistriesConfig config = new JsonRegistriesConfig(); + + configureRegistry("foo", config, registryWorkDir, mvn); + final String fooTimestamp = "20210101010101"; + setLastUpdated("foo", fooTimestamp, registryRepoDir, mvn); + + PlatformCatalog platformCatalog = newCatalogResolver(config, mvn).resolvePlatformCatalog(); + assertThat(platformCatalog).isNotNull(); + assertThat(platformCatalog.getPlatforms()).hasSize(1); + assertThat(platformCatalog.getPlatforms().iterator().next().getPlatformKey()).isEqualTo(toPlatformKey("foo")); + assertThat(platformCatalog.getMetadata().get(Constants.LAST_UPDATED)).isEqualTo(fooTimestamp); + + configureRegistry("bar", config, registryWorkDir, mvn); + final String barTimestamp = "20210101010102"; + setLastUpdated("bar", barTimestamp, registryRepoDir, mvn); + + platformCatalog = newCatalogResolver(config, mvn).resolvePlatformCatalog(); + assertThat(platformCatalog).isNotNull(); + assertThat(platformCatalog.getPlatforms()).hasSize(2); + final Iterator platforms = platformCatalog.getPlatforms().iterator(); + assertThat(platforms.next().getPlatformKey()).isEqualTo(toPlatformKey("foo")); + assertThat(platforms.next().getPlatformKey()).isEqualTo(toPlatformKey("bar")); + assertThat(platformCatalog.getMetadata().get(Constants.LAST_UPDATED)).isEqualTo(barTimestamp); + } + + private ExtensionCatalogResolver newCatalogResolver(JsonRegistriesConfig config, MavenArtifactResolver mvn) + throws Exception { + return ExtensionCatalogResolver.builder() + .config(config) + .artifactResolver(mvn) + .build(); + } + + private static void configureRegistry(String shortName, final JsonRegistriesConfig config, + final Path registryWorkDir, final MavenArtifactResolver mvn) throws Exception { + final JsonRegistryConfig registry = new JsonRegistryConfig(); + config.addRegistry(registry); + registry.setId("registry." + shortName + ".org"); + + final String groupId = toRegistryGroupId(shortName); + final JsonRegistryDescriptorConfig descriptorConfig = new JsonRegistryDescriptorConfig(); + registry.setDescriptor(descriptorConfig); + final ArtifactCoords descriptorCoords = ArtifactCoords + .fromString(groupId + ":quarkus-registry-descriptor::json:1.0-SNAPSHOT"); + descriptorConfig.setArtifact(descriptorCoords); + + final JsonRegistryPlatformsConfig platformsConfig = new JsonRegistryPlatformsConfig(); + registry.setPlatforms(platformsConfig); + ArtifactCoords platformsCoords = ArtifactCoords + .fromString(groupId + ":quarkus-registry-platforms::json:1.0-SNAPSHOT"); + platformsConfig.setArtifact(platformsCoords); + + Path json = registryWorkDir.resolve(shortName + "-quarkus-registry-descriptor.json"); + RegistriesConfigMapperHelper.serialize(registry, json); + Artifact a = new DefaultArtifact(descriptorCoords.getGroupId(), descriptorCoords.getArtifactId(), + descriptorCoords.getClassifier(), descriptorCoords.getType(), + descriptorCoords.getVersion()); + a = a.setFile(json.toFile()); + mvn.install(a); + + JsonPlatformCatalog platforms = new JsonPlatformCatalog(); + JsonPlatform platform = new JsonPlatform(); + platforms.addPlatform(platform); + final String platformKey = toPlatformKey(shortName); + platform.setPlatformKey(platformKey); + JsonPlatformStream stream = new JsonPlatformStream(); + platform.addStream(stream); + stream.setId("1.0"); + JsonPlatformRelease release = new JsonPlatformRelease(); + stream.addRelease(release); + release.setVersion(JsonPlatformReleaseVersion.fromString("1.0.0")); + release.setQuarkusCoreVersion("1.2.3"); + release.setMemberBoms(Collections + .singletonList(ArtifactCoords.fromString(platformKey + ":" + shortName + "-quarkus-bom::pom:1.0.0"))); + + json = registryWorkDir.resolve(shortName + "-quarkus-platforms.json"); + JsonCatalogMapperHelper.serialize(platforms, json); + a = new DefaultArtifact(platformsCoords.getGroupId(), platformsCoords.getArtifactId(), platformsCoords.getClassifier(), + platformsCoords.getType(), + platformsCoords.getVersion()); + a = a.setFile(json.toFile()); + mvn.install(a); + } + + private static void setLastUpdated(final String shortName, final String timestamp, final Path registryRepoDir, + final MavenArtifactResolver mvn) throws Exception { + final Path mdXml = registryRepoDir.resolve(mvn.getSession().getLocalRepositoryManager() + .getPathForLocalMetadata(new DefaultMetadata(toRegistryGroupId(shortName), "quarkus-registry-platforms", + "1.0-SNAPSHOT", "maven-metadata.xml", Nature.SNAPSHOT))); + if (!Files.exists(mdXml)) { + assertThat(mdXml).exists(); + } + final MetadataXpp3Reader mdReader = new MetadataXpp3Reader(); + final Metadata md; + try (Reader reader = Files.newBufferedReader(mdXml)) { + md = mdReader.read(reader); + final Versioning versioning = md.getVersioning(); + assertThat(versioning).isNotNull(); + versioning.setLastUpdated(timestamp); + for (SnapshotVersion sv : versioning.getSnapshotVersions()) { + sv.setUpdated(timestamp); + } + } + final MetadataXpp3Writer mdWriter = new MetadataXpp3Writer(); + try (Writer writer = Files.newBufferedWriter(mdXml)) { + mdWriter.write(writer, md); + } + } + + private static String toPlatformKey(String shortName) { + return "org." + shortName + ".platform"; + } + + private static String toRegistryGroupId(String shortName) { + return "org." + shortName + ".registry"; + } +}