Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a version filter while parsing guides #352

Merged
merged 2 commits into from
Nov 6, 2024
Merged

Add a version filter while parsing guides #352

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9e1b09f
Refresh samples and adjust corresponding tests
marko-bekhta Nov 4, 2024
17438b7
Add version filtering based on _data/versions.yaml
marko-bekhta Nov 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 125 additions & 11 deletions src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,14 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand Down Expand Up @@ -79,11 +82,15 @@ public static Path yamlMetadataPath(String version) {
return Path.of("_data", "versioned", version.replace('.', '-'), "index", "quarkus.yaml");
}

public static Path yamlVersionMetadataPath() {
return Path.of("_data", "versions.yaml");
}

public static Path yamlQuarkiverseMetadataPath(String version) {
return Path.of("_data", "versioned", version.replace('.', '-'), "index", "quarkiverse.yaml");
}

private final Map<Language, GitCloneDirectory> allSites;
private final Map<Language, QuarkusIOCloneDirectory> allSites;
private final Map<Language, URI> siteUris;
private final CloseableDirectory prefetchedGuides = CloseableDirectory.temp("quarkiverse-guides-");
private final FailureCollector failureCollector;
Expand All @@ -96,8 +103,11 @@ public QuarkusIO(QuarkusIOConfig config, GitCloneDirectory mainRepository,
this.siteUris = Collections.unmodifiableMap(languageUriMap);
this.failureCollector = failureCollector;

Map<Language, GitCloneDirectory> all = new HashMap<>(localizedSites);
all.put(Language.ENGLISH, mainRepository);
Map<Language, QuarkusIOCloneDirectory> all = new HashMap<>();
all.put(Language.ENGLISH, new QuarkusIOCloneDirectory(failureCollector, mainRepository));
for (var entry : localizedSites.entrySet()) {
all.put(entry.getKey(), new QuarkusIOCloneDirectory(failureCollector, entry.getValue()));
}
this.allSites = Collections.unmodifiableMap(all);

validateRepositories(mainRepository, localizedSites, failureCollector);
Expand Down Expand Up @@ -142,9 +152,12 @@ private static void validateRepositories(

@Override
public void close() throws IOException {
for (QuarkusIOCloneDirectory directory : allSites.values()) {
directory.unprocessed().ifPresent(m -> failureCollector.warning(FailureCollector.Stage.PARSING, m));
}
try (var closer = new Closer<IOException>()) {
closer.push(CloseableDirectory::close, prefetchedGuides);
closer.pushAll(GitCloneDirectory::close, allSites.values());
closer.pushAll(QuarkusIOCloneDirectory::close, allSites.values());
}
}

Expand All @@ -159,12 +172,14 @@ private Stream<Guide> versionedGuides() {
return allSites.entrySet().stream()
.flatMap(entry -> {
Language language = entry.getKey();
GitCloneDirectory cloneDirectory = entry.getValue();
GitCloneDirectory cloneDirectory = entry.getValue().cloneDirectory();
VersionFilter versionFilter = entry.getValue().versionFilter();
RevTree translationSourcesTree = cloneDirectory.sourcesTranslationTree();
Repository repository = cloneDirectory.git().getRepository();

return cloneDirectory.sourcesFileStream("_data/versioned", path -> path.endsWith("quarkus.yaml"))
.map(QuarkusIO::extractVersion)
.filter(versionFilter)
.flatMap(quarkusVersion -> {
String quarkus = quarkusVersion.path();

Expand All @@ -190,12 +205,14 @@ private Stream<Guide> legacyGuides() {
return allSites.entrySet().stream()
.flatMap(entry -> {
Language language = entry.getKey();
GitCloneDirectory cloneDirectory = entry.getValue();
GitCloneDirectory cloneDirectory = entry.getValue().cloneDirectory();
VersionFilter versionFilter = entry.getValue().versionFilter();
RevTree translationSourcesTree = cloneDirectory.sourcesTranslationTree();
Repository repository = cloneDirectory.git().getRepository();

return cloneDirectory.sourcesFileStream("_data", path -> path.matches("_data/guides-\\d+-\\d+\\.yaml"))
.map(QuarkusIO::extractLegacyVersion)
.filter(versionFilter)
.flatMap(quarkusVersion -> {
String quarkus = quarkusVersion.path();

Expand All @@ -204,7 +221,7 @@ private Stream<Guide> legacyGuides() {
resolveLegacyTranslationPath(quarkusVersion.versionDirectory(), language));

try (InputStream file = cloneDirectory.sourcesFile(quarkus)) {
return parseYamlLegacyMetadata(entry.getValue(), file, quarkusVersion.version(), language,
return parseYamlLegacyMetadata(cloneDirectory, file, quarkusVersion.version(), language,
translations);
} catch (IOException e) {
throw new IllegalStateException(
Expand All @@ -217,10 +234,13 @@ private Stream<Guide> legacyGuides() {

private Stream<Guide> quarkiverseGuides() {
Language language = Language.ENGLISH;
GitCloneDirectory cloneDirectory = allSites.get(language);
QuarkusIOCloneDirectory quarkusIOCloneDirectory = allSites.get(language);
GitCloneDirectory cloneDirectory = quarkusIOCloneDirectory.cloneDirectory();
VersionFilter versionFilter = quarkusIOCloneDirectory.versionFilter();

return cloneDirectory.sourcesFileStream("_data/versioned", path -> path.endsWith("quarkiverse.yaml"))
.map(QuarkusIO::extractQuarkiverseVersion)
.filter(versionFilter)
.flatMap(quarkusVersion -> {
String quarkus = quarkusVersion.path();

Expand All @@ -239,10 +259,13 @@ private Stream<Guide> quarkiverseGuides() {

private Stream<Guide> legacyQuarkiverseGuides() {
Language language = Language.ENGLISH;
GitCloneDirectory cloneDirectory = allSites.get(language);
QuarkusIOCloneDirectory quarkusIOCloneDirectory = allSites.get(language);
GitCloneDirectory cloneDirectory = quarkusIOCloneDirectory.cloneDirectory();
VersionFilter versionFilter = quarkusIOCloneDirectory.versionFilter();

return cloneDirectory.sourcesFileStream("_data", path -> path.matches("_data/guides-\\d+-\\d+\\.yaml"))
.map(QuarkusIO::extractLegacyVersion)
.filter(versionFilter)
.flatMap(quarkusVersion -> {
String quarkus = quarkusVersion.path();

Expand All @@ -261,10 +284,11 @@ private Stream<Guide> legacyQuarkiverseGuides() {

private Map<Language, Catalog> createTranslations(Function<Language, String> pathCreator) {
Map<Language, Catalog> translations = new HashMap<>();
for (Map.Entry<Language, GitCloneDirectory> entry : allSites.entrySet()) {
for (Map.Entry<Language, QuarkusIOCloneDirectory> entry : allSites.entrySet()) {
Language lang = entry.getKey();
GitCloneDirectory cloneDirectory = entry.getValue().cloneDirectory();
translations.put(lang, translations(
entry.getValue().git().getRepository(), entry.getValue().sourcesTranslationTree(),
cloneDirectory.git().getRepository(), cloneDirectory.sourcesTranslationTree(),
pathCreator.apply(lang)));
}
return translations;
Expand Down Expand Up @@ -485,6 +509,17 @@ private static Stream<Guide> parse(InputStream inputStream,
return parser.apply(quarkusYaml);
}

@SuppressWarnings("unchecked")
private static Set<String> parseVersions(InputStream inputStream) {
Map<String, Object> versionsYaml;
Yaml yaml = new Yaml();
versionsYaml = yaml.load(inputStream);

Collection<String> versions = (Collection<String>) versionsYaml.get("documentation");

return new LinkedHashSet<>(versions);
}

private Guide createGuide(GitCloneDirectory cloneDirectory, String quarkusVersion, String type,
Map<String, Object> parsedGuide,
String summaryKey, Language language, Catalog messages) {
Expand Down Expand Up @@ -582,4 +617,83 @@ private static Set<String> toSet(String value) {
record VersionAndPaths(String version, String versionDirectory, String path) {
}

record QuarkusIOCloneDirectory(VersionFilter versionFilter, GitCloneDirectory cloneDirectory) implements Closeable {
public QuarkusIOCloneDirectory(FailureCollector failureCollector, GitCloneDirectory cloneDirectory) {
this(VersionFilter.filter(cloneDirectory, failureCollector), cloneDirectory);
}

@Override
public void close() throws IOException {
try (var closer = new Closer<IOException>()) {
closer.push(GitCloneDirectory::close, cloneDirectory);
}
}

public Optional<String> unprocessed() {
Collection<String> unprocessedVersions = versionFilter.unprocessedVersions();
if (!unprocessedVersions.isEmpty()) {
return Optional.of(
"Not all expected versions were discovered while parsing %s. Missing versions are: %s"
.formatted(cloneDirectory.details(), unprocessedVersions));
}
return Optional.empty();
}
}

private static abstract class VersionFilter implements Predicate<VersionAndPaths> {

private static VersionFilter filter(GitCloneDirectory cloneDirectory, FailureCollector failureCollector) {
try (InputStream inputStream = cloneDirectory.sourcesFile("_data/versions.yaml")) {
Set<String> versions = parseVersions(inputStream);
return new CollectionFilter(versions);
} catch (Exception e) {
failureCollector.warning(FailureCollector.Stage.PARSING,
"Unable to find versions file with explicit list of versions to index within %s, resulting in including all discovered versions."
.formatted(cloneDirectory.toString()));
return AcceptAllFilter.INSTANCE;

}
}

public abstract Collection<String> unprocessedVersions();

private static class CollectionFilter extends VersionFilter {

private final Map<String, Boolean> versions;

public CollectionFilter(Collection<String> versions) {
this.versions = new HashMap<>(versions.size());
for (String version : versions) {
this.versions.put(version, false);
}
}

@Override
public Collection<String> unprocessedVersions() {
return versions.entrySet().stream()
.filter(Predicate.not(Map.Entry::getValue))
.map(Map.Entry::getKey)
.toList();
}

@Override
public boolean test(VersionAndPaths version) {
return Boolean.TRUE.equals(versions.compute(version.version(), (key, value) -> value == null ? null : true));
}
}

private static class AcceptAllFilter extends VersionFilter {
public static final VersionFilter INSTANCE = new AcceptAllFilter();

@Override
public boolean test(VersionAndPaths s) {
return true;
}

@Override
public Collection<String> unprocessedVersions() {
return List.of();
}
}
}
}
14 changes: 7 additions & 7 deletions src/test/java/io/quarkus/search/app/SearchServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,7 +406,7 @@ void language() {
.extract().body().as(SEARCH_RESULT_SEARCH_HITS);
assertThat(result.hits()).extracting(GuideSearchHit::title)
.contains("Stork リファレンス<span class=\"highlighted\">ガイド</span>",
"Use Hibernate Search with Hibernate ORM and Elasticsearch/OpenSearch");
"Hibernate ORMとElasticsearch/OpenSearchでHibernate Searchを使用");
}

@Test
Expand Down Expand Up @@ -455,9 +455,9 @@ void searchForPhrase() {
@Test
void findEnvVariable() {
var result = given()
// the variable that we are "planning" to find is actually QUARKUS_DATASOURCE_JDBC_TRACING_IGNORE_FOR_TRACING
// the variable that we are "planning" to find is actually QUARKUS_DATASOURCE_JDBC_URL
// But we'll be looking only for a part of it.
.queryParam("q", "QUARKUS_DATASOURCE_JDBC_TRACING_")
.queryParam("q", "QUARKUS_DATASOURCE_JDBC_U")
.when().get(GUIDES_SEARCH)
.then()
.statusCode(200)
Expand All @@ -470,14 +470,14 @@ void findEnvVariable() {
@Test
void findConfigProperty() {
var result = given()
.queryParam("q", "quarkus.websocket.max-frame-size")
.queryParam("q", "quarkus.vertx.eventbus.tcp-keep-alive")
.when().get(GUIDES_SEARCH)
.then()
.statusCode(200)
.extract().body().as(SEARCH_RESULT_SEARCH_HITS);
assertThat(result.hits()).extracting(GuideSearchHit::content)
.containsOnly(
Set.of("…Default <span class=\"highlighted\">quarkus.websocket.max</span>-<span class=\"highlighted\">frame</span>-<span class=\"highlighted\">size</span> The maximum amount of data that can be sent in a single <span class=\"highlighted\">frame</span>. Messages…"));
Set.of("…false <span class=\"highlighted\">quarkus.vertx.eventbus.tcp</span>-<span class=\"highlighted\">keep</span>-<span class=\"highlighted\">alive</span> Whether to <span class=\"highlighted\">keep</span> the TCP connection opened (<span class=\"highlighted\">keep</span>-<span class=\"highlighted\">alive</span>). Environment…"));
}

@Test
Expand All @@ -490,7 +490,7 @@ void findFQCN() {
.extract().body().as(SEARCH_RESULT_SEARCH_HITS);
assertThat(result.hits()).extracting(GuideSearchHit::content)
.containsOnly(Set.of(
"…allow No Javadoc found <span class=\"highlighted\">io.quarkus.deployment.pkg.builditem.NativeImageBuildItem</span> No Javadoc found Path…"));
"…allow No Javadoc found <span class=\"highlighted\">io.quarkus.deployment.pkg.builditem.NativeImageBuildItem</span> No Javadoc found Show…"));
}

@Test
Expand All @@ -503,7 +503,7 @@ void findBuildItem() {
.extract().body().as(SEARCH_RESULT_SEARCH_HITS);
assertThat(result.hits()).extracting(GuideSearchHit::content)
.containsOnly(Set.of(
"…allow No Javadoc found <span class=\"highlighted\">io.quarkus.deployment.pkg.builditem.NativeImageBuildItem</span> No Javadoc found Path…"));
"…allow No Javadoc found <span class=\"highlighted\">io.quarkus.deployment.pkg.builditem.NativeImageBuildItem</span> No Javadoc found Show…"));
}

@Test
Expand Down
22 changes: 22 additions & 0 deletions src/test/java/io/quarkus/search/app/testsupport/QuarkusIOSample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
Expand Down Expand Up @@ -291,6 +292,19 @@ private static void yamlQuarkiverseEditor(String version, Path fileToEdit) {
});
}

private static void yamlVersionEditor(Path fileToEdit, Collection<String> versions) {
yamlQuarkusEditor(fileToEdit, quarkusYaml -> {
Map<String, Object> filtered = new HashMap<>();
filtered.put("quarkus", quarkusYaml.get("quarkus"));
filtered.put("graalvm", quarkusYaml.get("graalvm"));
filtered.put("jdk", quarkusYaml.get("jdk"));
filtered.put("maven", quarkusYaml.get("maven"));

filtered.put("documentation", versions);
return filtered;
});
}

private static void yamlQuarkusEditor(Path fileToEdit, Function<Map<String, Object>, Map<String, Object>> editor) {
Map<String, Object> filtered;
try (InputStream inputStream = Files.newInputStream(fileToEdit)) {
Expand Down Expand Up @@ -384,6 +398,7 @@ protected AbstractGuideRefSetFilterDefinition(String name, GuideRef... guides) {

@Override
public void define(FilterDefinitionCollector c) {
c.addVersionMetadata(SAMPLED_VERSIONS);
for (String version : SAMPLED_VERSIONS) {
c.addMetadata(version, guides);
if (QuarkusVersions.MAIN.equals(version)) {
Expand Down Expand Up @@ -437,6 +452,13 @@ public FilterDefinitionCollector addGuide(GuideRef ref, String version) {
return this;
}

public FilterDefinitionCollector addVersionMetadata(Collection<String> versions) {
String metadataPath = QuarkusIO.yamlVersionMetadataPath().toString();
addOnSourceBranch(metadataPath, metadataPath);
addMetadataToFilter(metadataPath, path -> yamlVersionEditor(path, versions));
return this;
}

public FilterDefinitionCollector addMetadata(String version, GuideRef[] guides) {
String metadataPath = QuarkusIO.yamlMetadataPath(version).toString();
addOnSourceBranch(metadataPath, metadataPath);
Expand Down
Binary file modified src/test/resources/quarkusio-sample-cn.zip
View file
Open in desktop
Binary file not shown.
Binary file modified src/test/resources/quarkusio-sample-es.zip
View file
Open in desktop
Binary file not shown.
Binary file modified src/test/resources/quarkusio-sample-ja.zip
View file
Open in desktop
Binary file not shown.
Binary file modified src/test/resources/quarkusio-sample-pt.zip
View file
Open in desktop
Binary file not shown.
Binary file modified src/test/resources/quarkusio-sample.zip
View file
Open in desktop
Binary file not shown.
Loading