diff --git a/.buildkite/scripts/gradle-configuration-cache-validation.sh b/.buildkite/scripts/gradle-configuration-cache-validation.sh index 8249155c5ffc5..55a4b18a1e887 100755 --- a/.buildkite/scripts/gradle-configuration-cache-validation.sh +++ b/.buildkite/scripts/gradle-configuration-cache-validation.sh @@ -2,18 +2,17 @@ set -euo pipefail -# TODO/ FIXIT without a full resolved gradle home, we see issues configuration cache reuse -./gradlew --max-workers=8 --parallel --scan --no-daemon precommit +# This is a workaround for https://github.com/gradle/gradle/issues/28159 +.ci/scripts/run-gradle.sh --no-daemon precommit -./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 +.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 # Create a temporary file tmpOutputFile=$(mktemp) trap "rm $tmpOutputFile" EXIT echo "2nd run" -# TODO run-gradle.sh script causes issues because of init script handling -./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile +.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile # Check if the command was successful if grep -q "Configuration cache entry reused." $tmpOutputFile; then diff --git a/TESTING.asciidoc b/TESTING.asciidoc index 3a066be2f8ea9..6f2dc3c64febe 100644 --- a/TESTING.asciidoc +++ b/TESTING.asciidoc @@ -472,7 +472,7 @@ You can run a group of YAML test by using wildcards: --tests "org.elasticsearch.test.rest.ClientYamlTestSuiteIT.test {yaml=index/*/*}" --------------------------------------------------------------------------- -or +or --------------------------------------------------------------------------- ./gradlew :rest-api-spec:yamlRestTest \ @@ -564,8 +564,8 @@ Sometimes a backward compatibility change spans two versions. A common case is a new functionality that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x). Another use case, since the introduction of serverless, is to test BWC against main in addition to the other released branches. To do so, specify the `bwc.refspec` remote and branch to use for the BWC build as `origin/main`. -To test against main, you will also need to create a new version in link:./server/src/main/java/org/elasticsearch/Version.java[Version.java], -increment `elasticsearch` in link:./build-tools-internal/version.properties[version.properties], and hard-code the `project.version` for ml-cpp +To test against main, you will also need to create a new version in link:./server/src/main/java/org/elasticsearch/Version.java[Version.java], +increment `elasticsearch` in link:./build-tools-internal/version.properties[version.properties], and hard-code the `project.version` for ml-cpp in link:./x-pack/plugin/ml/build.gradle[ml/build.gradle]. In general, to test the changes, you can instruct Gradle to build the BWC version from another remote/branch combination instead of pulling the release branch from GitHub. @@ -625,7 +625,7 @@ For specific YAML rest tests one can use For disabling entire types of tests for subprojects, one can use for example: ------------------------------------------------ -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm) { // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC tasks.named("javaRestTest").configure{enabled = false } } diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index f3ced9f16d327..25cfae6c9803a 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -1,4 +1,3 @@ -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.TestUtil /* @@ -78,7 +77,7 @@ tasks.register("copyPainless", Copy) { } tasks.named("run").configure { - executable = "${BuildParams.runtimeJavaHome}/bin/java" + executable = "${buildParams.runtimeJavaHome.get()}/bin/java" args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index" dependsOn "copyExpression", "copyPainless", configurations.nativeLib systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString()) diff --git a/build-conventions/build.gradle b/build-conventions/build.gradle index d8c211c0f02f9..b0eda5a34065a 100644 --- a/build-conventions/build.gradle +++ b/build-conventions/build.gradle @@ -12,9 +12,6 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder buildscript { repositories { - maven { - url 'https://jitpack.io' - } mavenCentral() } } @@ -70,10 +67,6 @@ gradlePlugin { } repositories { - maven { - url 'https://jitpack.io' - } - mavenCentral() gradlePluginPortal() } diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GUtils.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GUtils.java index 9a35aa41ba1e5..0b04496866ca9 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GUtils.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GUtils.java @@ -16,4 +16,12 @@ public abstract class GUtils { public static String capitalize(String s) { return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1); } + + public static T elvis(T given, T fallback) { + if (given == null) { + return fallback; + } else { + return given; + } + } } diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java index cd13743ee0746..c3124812e5089 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java @@ -9,12 +9,14 @@ package org.elasticsearch.gradle.internal.conventions; -import org.elasticsearch.gradle.internal.conventions.precommit.PomValidationPrecommitPlugin; +import groovy.util.Node; + import com.github.jengelman.gradle.plugins.shadow.ShadowExtension; import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin; -import groovy.util.Node; -import org.elasticsearch.gradle.internal.conventions.util.Util; + import org.elasticsearch.gradle.internal.conventions.info.GitInfo; +import org.elasticsearch.gradle.internal.conventions.precommit.PomValidationPrecommitPlugin; +import org.elasticsearch.gradle.internal.conventions.util.Util; import org.gradle.api.NamedDomainObjectSet; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -35,11 +37,12 @@ import org.gradle.api.tasks.bundling.Jar; import org.gradle.initialization.layout.BuildLayout; import org.gradle.language.base.plugins.LifecycleBasePlugin; +import org.w3c.dom.Element; -import javax.inject.Inject; import java.io.File; import java.util.Map; import java.util.concurrent.Callable; +import javax.inject.Inject; public class PublishPlugin implements Plugin { @@ -64,6 +67,7 @@ public void apply(Project project) { configureSourcesJar(project); configurePomGeneration(project); configurePublications(project); + formatGeneratedPom(project); } private void configurePublications(Project project) { @@ -113,29 +117,32 @@ private void configurePomGeneration(Project project) { var archivesBaseName = providerFactory.provider(() -> getArchivesBaseName(extensions)); var projectVersion = providerFactory.provider(() -> project.getVersion()); var generateMavenPoms = project.getTasks().withType(GenerateMavenPom.class); - generateMavenPoms.configureEach( - pomTask -> pomTask.setDestination( + generateMavenPoms.configureEach(pomTask -> { + pomTask.setDestination( (Callable) () -> String.format( "%s/distributions/%s-%s.pom", projectLayout.getBuildDirectory().get().getAsFile().getPath(), archivesBaseName.get(), projectVersion.get() ) - ) - ); + ); + }); + var publishing = extensions.getByType(PublishingExtension.class); final var mavenPublications = publishing.getPublications().withType(MavenPublication.class); - addNameAndDescriptiontoPom(project, mavenPublications); + addNameAndDescriptionToPom(project, mavenPublications); mavenPublications.configureEach(publication -> { - // Add git origin info to generated POM files for internal builds - publication.getPom().withXml(xml -> addScmInfo(xml, gitInfo.get())); + publication.getPom().withXml(xml -> { + // Add git origin info to generated POM files for internal builds + addScmInfo(xml, gitInfo.get()); + }); // have to defer this until archivesBaseName is set project.afterEvaluate(p -> publication.setArtifactId(archivesBaseName.get())); generatePomTask.configure(t -> t.dependsOn(generateMavenPoms)); }); } - private void addNameAndDescriptiontoPom(Project project, NamedDomainObjectSet mavenPublications) { + private void addNameAndDescriptionToPom(Project project, NamedDomainObjectSet mavenPublications) { var name = project.getName(); var description = providerFactory.provider(() -> project.getDescription() != null ? project.getDescription() : ""); mavenPublications.configureEach(p -> p.getPom().withXml(xml -> { @@ -186,4 +193,32 @@ static void configureSourcesJar(Project project) { project.getTasks().named(BasePlugin.ASSEMBLE_TASK_NAME).configure(t -> t.dependsOn(sourcesJarTask)); }); } + + + /** + * Format the generated pom files to be in a sort of reproducible order. + */ + private void formatGeneratedPom(Project project) { + var publishing = project.getExtensions().getByType(PublishingExtension.class); + final var mavenPublications = publishing.getPublications().withType(MavenPublication.class); + mavenPublications.configureEach(publication -> { + publication.getPom().withXml(xml -> { + // Add some pom formatting + formatDependencies(xml); + }); + }); + } + + /** + * just ensure we put dependencies to the end. more a cosmetic thing than anything else + * */ + private void formatDependencies(XmlProvider xml) { + Element rootElement = xml.asElement(); + var dependencies = rootElement.getElementsByTagName("dependencies"); + if (dependencies.getLength() == 1 && dependencies.item(0) != null) { + org.w3c.dom.Node item = dependencies.item(0); + rootElement.removeChild(item); + rootElement.appendChild(item); + } + } } diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 38d3c0cd326f9..f2a02645f8c09 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -258,9 +258,6 @@ tasks.named('licenseHeaders').configure { *****************************************************************************/ repositories { - maven { - url 'https://jitpack.io' - } mavenCentral() gradlePluginPortal() } @@ -386,10 +383,13 @@ tasks.named("jar") { spotless { java { - // IDEs can sometimes run annotation processors that leave files in - // here, causing Spotless to complain. Even though this path ought not - // to exist, exclude it anyway in order to avoid spurious failures. - toggleOffOn() + + // workaround for https://github.com/diffplug/spotless/issues/2317 + //toggleOffOn() + target project.fileTree("src/main/java") { + include '**/*.java' + exclude '**/DockerBase.java' + } } } diff --git a/build-tools-internal/settings.gradle b/build-tools-internal/settings.gradle index 1b4fb1215a59d..8c88d36046768 100644 --- a/build-tools-internal/settings.gradle +++ b/build-tools-internal/settings.gradle @@ -1,8 +1,5 @@ pluginManagement { repositories { - maven { - url 'https://jitpack.io' - } mavenCentral() gradlePluginPortal() } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy index b853fdef6a13f..34fa73ce502ac 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy @@ -73,7 +73,7 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest { buildFile << """ plugins { id 'elasticsearch.java-doc' - id 'com.github.johnrengelman.shadow' version '7.1.2' + id 'com.gradleup.shadow' id 'java' } group = 'org.acme.depending' diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy index 87f4bbee05780..6d080e1c80763 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy @@ -9,16 +9,10 @@ package org.elasticsearch.gradle.internal -import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.fixtures.AbstractGitAwareGradleFuncTest import org.gradle.testkit.runner.TaskOutcome -import spock.lang.IgnoreIf import spock.lang.Unroll -/* - * Test is ignored on ARM since this test case tests the ability to build certain older BWC branches that we don't support on ARM - */ -@IgnoreIf({ Architecture.current() == Architecture.AARCH64 }) class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest { def setup() { diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy index 6e403c85a23f4..c7e11ba96c7dd 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy @@ -96,7 +96,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { plugins { id 'elasticsearch.java' id 'elasticsearch.publish' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } repositories { @@ -117,7 +117,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { } version = "1.0" group = 'org.acme' - description = 'some description' + description = 'shadowed project' """ when: @@ -137,7 +137,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { hello-world 1.0 hello-world - some description + shadowed project unknown unknown @@ -186,7 +186,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { plugins { id 'elasticsearch.java' id 'elasticsearch.publish' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } dependencies { @@ -206,7 +206,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { group = 'org.acme' } - description = 'some description' + description = 'with shadowed dependencies' """ when: @@ -226,7 +226,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { hello-world 1.0 hello-world - some description + with shadowed dependencies unknown unknown @@ -277,13 +277,13 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { plugins { id 'elasticsearch.internal-es-plugin' id 'elasticsearch.publish' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } esplugin { name = 'hello-world-plugin' classname 'org.acme.HelloWorldPlugin' - description = "custom project description" + description = "shadowed es plugin" } publishing { @@ -324,7 +324,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { hello-world-plugin 1.0 hello-world - custom project description + shadowed es plugin unknown unknown @@ -353,7 +353,6 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { https://www.elastic.co - """ ) } @@ -440,8 +439,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { // scm info only added for internal builds internalBuild() buildFile << """ - BuildParams.init { it.setGitOrigin("https://some-repo.com/repo.git") } - + buildParams.getGitOriginProperty().set("https://some-repo.com/repo.git") apply plugin:'elasticsearch.java' apply plugin:'elasticsearch.publish' diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy index 354100a9b82c5..725f117d17e64 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy @@ -161,7 +161,7 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleInterna }, "target": { "remoteUrl": "http://acme.org", - "branch": "unknown" + "branch": "$version" }, "targetReference": "$version", "projectAttributes": { diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy index 97f03d9821117..ce5c1519fe11f 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy @@ -10,6 +10,7 @@ package org.elasticsearch.gradle.internal.test.rest import spock.lang.IgnoreIf +import spock.lang.IgnoreRest import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest @@ -20,16 +21,16 @@ import org.gradle.testkit.runner.TaskOutcome class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest { def setup() { + configurationCacheCompatible = true buildApiRestrictionsDisabled = true } def "yamlRestTest does nothing when there are no tests"() { given: + internalBuild() buildFile << """ - plugins { - id 'elasticsearch.legacy-yaml-rest-test' - } + apply plugin: 'elasticsearch.legacy-yaml-rest-test' """ when: @@ -136,7 +137,7 @@ class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest { """ when: - def result = gradleRunner("yamlRestTest", "--console", 'plain', '--stacktrace').buildAndFail() + def result = gradleRunner("yamlRestTest", "--console", 'plain').buildAndFail() then: result.task(":distribution:archives:integ-test-zip:buildExpanded").outcome == TaskOutcome.SUCCESS diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle index 592e6af41ab00..847eda7a355c0 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle @@ -12,11 +12,14 @@ import java.time.LocalDateTime; import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.OS -import org.elasticsearch.gradle.internal.info.BuildParams +import static org.elasticsearch.gradle.internal.util.CiUtils.safeName import java.lang.management.ManagementFactory import java.time.LocalDateTime +// Resolving this early to avoid issues with the build scan plugin in combination with the configuration cache usage +def taskNames = gradle.startParameter.taskNames.join(' ') + develocity { buildScan { @@ -34,12 +37,15 @@ develocity { publishing.onlyIf { false } } + def fips = buildParams.inFipsJvm + def gitRevision = buildParams.gitRevision + background { tag OS.current().name() tag Architecture.current().name() // Tag if this build is run in FIPS mode - if (BuildParams.inFipsJvm) { + if (fips) { tag 'FIPS' } @@ -92,8 +98,8 @@ develocity { link 'Source', "${prBaseUrl}/tree/${System.getenv('BUILDKITE_COMMIT')}" link 'Pull Request', "https://github.com/${repository}/pull/${prId}" } else { - value 'Git Commit ID', BuildParams.gitRevision - link 'Source', "https://github.com/${repository}/tree/${BuildParams.gitRevision}" + value 'Git Commit ID', gitRevision + link 'Source', "https://github.com/${repository}/tree/${gitRevision}" } buildFinished { result -> @@ -108,7 +114,7 @@ develocity { // Add a build annotation // See: https://buildkite.com/docs/agent/v3/cli-annotate - def body = """
${System.getenv('BUILDKITE_LABEL')} :gradle: ${result.failures ? 'failed' : 'successful'} build: gradle ${gradle.startParameter.taskNames.join(' ')}
""" + def body = """
${System.getenv('BUILDKITE_LABEL')} :gradle: ${result.failures ? 'failed' : 'successful'} build: gradle ${taskNames}
""" def process = [ 'buildkite-agent', 'annotate', @@ -129,7 +135,3 @@ develocity { } } } - -static def safeName(String string) { - return string.replaceAll(/[^a-zA-Z0-9_\-\.]+/, ' ').trim().replaceAll(' ', '_').toLowerCase() -} diff --git a/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle b/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle index 9a988292b5b8c..77e509ea97870 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle @@ -9,7 +9,6 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask @@ -19,7 +18,7 @@ ext.bwcTaskName = { Version version -> def bwcTestSnapshots = tasks.register("bwcTestSnapshots") { if (project.bwc_tests_enabled) { - dependsOn tasks.matching { task -> BuildParams.bwcVersions.unreleased.any { version -> bwcTaskName(version) == task.name } } + dependsOn tasks.matching { task -> buildParams.bwcVersions.unreleased.any { version -> bwcTaskName(version) == task.name } } } } diff --git a/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle b/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle index 3bff30d9511fb..567812c740817 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle @@ -15,11 +15,12 @@ import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask import org.elasticsearch.gradle.testclusters.TestClustersAware import org.elasticsearch.gradle.testclusters.TestDistribution -// Common config when running with a FIPS-140 runtime JVM -if (BuildParams.inFipsJvm) { +//apply plugin: org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin +// Common config when running with a FIPS-140 runtime JVM +if (buildParams.inFipsJvm) { allprojects { - String javaSecurityFilename = BuildParams.runtimeJavaDetails.toLowerCase().contains('oracle') ? 'fips_java_oracle.security' : 'fips_java.security' + String javaSecurityFilename = buildParams.runtimeJavaDetails.toLowerCase().contains('oracle') ? 'fips_java_oracle.security' : 'fips_java.security' File fipsResourcesDir = new File(project.buildDir, 'fips-resources') File fipsSecurity = new File(fipsResourcesDir, javaSecurityFilename) File fipsPolicy = new File(fipsResourcesDir, 'fips_java.policy') diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle index 63a3cb6d86d68..5640409e0ff44 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle @@ -171,7 +171,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { idea { project { vcs = 'Git' - jdkName = BuildParams.minimumCompilerVersion.majorVersion + jdkName = buildParams.minimumCompilerVersion.majorVersion settings { delegateActions { diff --git a/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle b/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle index aacc86e764d51..224e6bd4c50d1 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle @@ -10,7 +10,6 @@ import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.OS import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.precommit.ThirdPartyAuditPrecommitPlugin import org.elasticsearch.gradle.internal.precommit.ThirdPartyAuditTask import org.elasticsearch.gradle.internal.test.rest.RestTestBasePlugin @@ -27,8 +26,8 @@ configure(allprojects) { JvmVendorSpec.matching(VersionProperties.bundledJdkVendor) } project.tasks.withType(Test).configureEach { Test test -> - if (BuildParams.getIsRuntimeJavaHomeSet()) { - test.executable = "${BuildParams.runtimeJavaHome}/bin/java" + + if (buildParams.getIsRuntimeJavaHomeSet()) { + test.executable = "${buildParams.runtimeJavaHome.get()}/bin/java" + (OS.current() == OS.WINDOWS ? '.exe' : '') } else { test.javaLauncher = javaToolchains.launcherFor { @@ -41,7 +40,7 @@ configure(allprojects) { } project.plugins.withId("elasticsearch.testclusters") { testClustersPlugin -> project.plugins.withId("elasticsearch.internal-testclusters") { internalPlugin -> - if (BuildParams.getIsRuntimeJavaHomeSet() == false) { + if (buildParams.getIsRuntimeJavaHomeSet() == false) { // If no runtime java home is set, use the bundled JDK for test clusters testClustersPlugin.setRuntimeJava(launcher.map { it.metadata.installationPath.asFile }) } diff --git a/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle b/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle index 1fab4d035177a..27b490329f8cb 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle @@ -17,11 +17,11 @@ dependencies { newJar project(":libs:${project.name}") } -BuildParams.bwcVersions.withIndexCompatible({ it.onOrAfter(Version.fromString(ext.stableApiSince)) +buildParams.bwcVersions.withIndexCompatible({ it.onOrAfter(Version.fromString(ext.stableApiSince)) && it != VersionProperties.elasticsearchVersion }) { bwcVersion, baseName -> - BwcVersions.UnreleasedVersionInfo unreleasedVersion = BuildParams.bwcVersions.unreleasedInfo(bwcVersion) + BwcVersions.UnreleasedVersionInfo unreleasedVersion = buildParams.bwcVersions.unreleasedInfo(bwcVersion) configurations { "oldJar${baseName}" { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java index 04f031d4a5169..49887dac5b6fd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java @@ -12,7 +12,7 @@ import groovy.lang.Closure; import org.elasticsearch.gradle.internal.conventions.util.Util; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin; import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin; import org.elasticsearch.gradle.plugin.PluginBuildPlugin; @@ -39,6 +39,7 @@ public void apply(Project project) { project.getPluginManager().apply(JarHellPrecommitPlugin.class); project.getPluginManager().apply(ElasticsearchJavaPlugin.class); project.getPluginManager().apply(HistoricalFeaturesMetadataPlugin.class); + boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).isCi(); // Clear default dependencies added by public PluginBuildPlugin as we add our // own project dependencies for internal builds // TODO remove once we removed default dependencies from PluginBuildPlugin @@ -54,7 +55,7 @@ public void apply(Project project) { .set("addQaCheckDependencies", new Closure(BaseInternalPluginBuildPlugin.this, BaseInternalPluginBuildPlugin.this) { public void doCall(Project proj) { // This is only a convenience for local developers so make this a noop when running in CI - if (BuildParams.isCi() == false) { + if (isCi == false) { proj.afterEvaluate(project1 -> { // let check depend on check tasks of qa sub-projects final var checkTaskProvider = project1.getTasks().named("check"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index 40d16bafbb26b..d7bf839817e12 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -13,7 +13,6 @@ import org.elasticsearch.gradle.LoggedExec; import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.Version; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.Action; import org.gradle.api.GradleException; import org.gradle.api.Project; @@ -47,6 +46,7 @@ public class BwcSetupExtension { private final ProviderFactory providerFactory; private final JavaToolchainService toolChainService; private final Provider unreleasedVersionInfo; + private final Boolean isCi; private Provider checkoutDir; @@ -56,7 +56,8 @@ public BwcSetupExtension( ProviderFactory providerFactory, JavaToolchainService toolChainService, Provider unreleasedVersionInfo, - Provider checkoutDir + Provider checkoutDir, + Boolean isCi ) { this.project = project; this.objectFactory = objectFactory; @@ -64,6 +65,7 @@ public BwcSetupExtension( this.toolChainService = toolChainService; this.unreleasedVersionInfo = unreleasedVersionInfo; this.checkoutDir = checkoutDir; + this.isCi = isCi; } TaskProvider bwcTask(String name, Action configuration) { @@ -80,7 +82,8 @@ TaskProvider bwcTask(String name, Action configuration, toolChainService, name, configuration, - useUniqueUserHome + useUniqueUserHome, + isCi ); } @@ -93,7 +96,8 @@ private static TaskProvider createRunBwcGradleTask( JavaToolchainService toolChainService, String name, Action configAction, - boolean useUniqueUserHome + boolean useUniqueUserHome, + boolean isCi ) { return project.getTasks().register(name, LoggedExec.class, loggedExec -> { loggedExec.dependsOn("checkoutBwcBranch"); @@ -104,7 +108,7 @@ private static TaskProvider createRunBwcGradleTask( spec.getParameters().getCheckoutDir().set(checkoutDir); }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s)))); - if (BuildParams.isCi() && OS.current() != OS.WINDOWS) { + if (isCi && OS.current() != OS.WINDOWS) { // TODO: Disabled for now until we can figure out why files are getting corrupted // loggedExec.getEnvironment().put("GRADLE_RO_DEP_CACHE", System.getProperty("user.home") + "/gradle_ro_cache"); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java index f467a204c0348..93c2623a23d31 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java @@ -11,6 +11,7 @@ import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; +import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -60,7 +61,8 @@ * We are then able to map the unreleased version to branches in git and Gradle projects that are capable of checking * out and building them, so we can include these in the testing plan as well. */ -public class BwcVersions { + +public class BwcVersions implements Serializable { private static final Pattern LINE_PATTERN = Pattern.compile( "\\W+public static final Version V_(\\d+)_(\\d+)_(\\d+)(_alpha\\d+|_beta\\d+|_rc\\d+)?.*\\);" @@ -68,7 +70,7 @@ public class BwcVersions { private static final String GLIBC_VERSION_ENV_VAR = "GLIBC_VERSION"; private final Version currentVersion; - private final List versions; + private final transient List versions; private final Map unreleased; public BwcVersions(List versionLines) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java index 05b7af83aa8e4..c897b142da2fb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java @@ -11,7 +11,7 @@ import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.internal.test.MutedTestPlugin; import org.elasticsearch.gradle.internal.test.TestUtil; @@ -49,6 +49,7 @@ public class ElasticsearchJavaBasePlugin implements Plugin { private final JavaToolchainService javaToolchains; + private BuildParameterExtension buildParams; @Inject ElasticsearchJavaBasePlugin(JavaToolchainService javaToolchains) { @@ -57,8 +58,10 @@ public class ElasticsearchJavaBasePlugin implements Plugin { @Override public void apply(Project project) { + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); // make sure the global build info plugin is applied to the root project project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); + buildParams = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class); project.getPluginManager().apply(JavaBasePlugin.class); // common repositories setup project.getPluginManager().apply(RepositoriesSetupPlugin.class); @@ -129,14 +132,14 @@ private static void disableTransitiveDependenciesForSourceSet(Project project, S public void configureCompile(Project project) { project.getExtensions().getExtraProperties().set("compactProfile", "full"); JavaPluginExtension java = project.getExtensions().getByType(JavaPluginExtension.class); - if (BuildParams.getJavaToolChainSpec().isPresent()) { - java.toolchain(BuildParams.getJavaToolChainSpec().get()); + if (buildParams.getJavaToolChainSpec().isPresent()) { + java.toolchain(buildParams.getJavaToolChainSpec().get()); } - java.setSourceCompatibility(BuildParams.getMinimumRuntimeVersion()); - java.setTargetCompatibility(BuildParams.getMinimumRuntimeVersion()); + java.setSourceCompatibility(buildParams.getMinimumRuntimeVersion()); + java.setTargetCompatibility(buildParams.getMinimumRuntimeVersion()); project.getTasks().withType(JavaCompile.class).configureEach(compileTask -> { compileTask.getJavaCompiler().set(javaToolchains.compilerFor(spec -> { - spec.getLanguageVersion().set(JavaLanguageVersion.of(BuildParams.getMinimumRuntimeVersion().getMajorVersion())); + spec.getLanguageVersion().set(JavaLanguageVersion.of(buildParams.getMinimumRuntimeVersion().getMajorVersion())); })); CompileOptions compileOptions = compileTask.getOptions(); @@ -159,7 +162,7 @@ public void configureCompile(Project project) { compileTask.getConventionMapping().map("sourceCompatibility", () -> java.getSourceCompatibility().toString()); compileTask.getConventionMapping().map("targetCompatibility", () -> java.getTargetCompatibility().toString()); compileOptions.getRelease().set(releaseVersionProviderFromCompileTask(project, compileTask)); - compileOptions.setIncremental(BuildParams.isCi() == false); + compileOptions.setIncremental(buildParams.isCi() == false); }); // also apply release flag to groovy, which is used in build-tools project.getTasks().withType(GroovyCompile.class).configureEach(compileTask -> { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java index d064c70c72819..e62c26c7fbc01 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java @@ -15,8 +15,10 @@ import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.internal.conventions.util.Util; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.Action; +import org.gradle.api.JavaVersion; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; @@ -24,6 +26,7 @@ import org.gradle.api.plugins.BasePlugin; import org.gradle.api.plugins.JavaLibraryPlugin; import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.provider.Property; import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.bundling.Jar; import org.gradle.api.tasks.javadoc.Javadoc; @@ -34,6 +37,7 @@ import java.util.Map; import static org.elasticsearch.gradle.internal.conventions.util.Util.toStringable; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; /** * A wrapper around Gradle's Java plugin that applies our @@ -42,13 +46,15 @@ public class ElasticsearchJavaPlugin implements Plugin { @Override public void apply(Project project) { + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + Property buildParams = loadBuildParams(project); project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class); project.getPluginManager().apply(JavaLibraryPlugin.class); project.getPluginManager().apply(ElasticsearchJavaModulePathPlugin.class); // configureConfigurations(project); - configureJars(project); - configureJarManifest(project); + configureJars(project, buildParams.get()); + configureJarManifest(project, buildParams.get()); configureJavadoc(project); testCompileOnlyDeps(project); } @@ -63,7 +69,9 @@ private static void testCompileOnlyDeps(Project project) { /** * Adds additional manifest info to jars */ - static void configureJars(Project project) { + static void configureJars(Project project, BuildParameterExtension buildParams) { + String buildDate = buildParams.getBuildDate().toString(); + JavaVersion gradleJavaVersion = buildParams.getGradleJavaVersion(); project.getTasks().withType(Jar.class).configureEach(jarTask -> { // we put all our distributable files under distributions jarTask.getDestinationDirectory().set(new File(project.getBuildDir(), "distributions")); @@ -75,14 +83,11 @@ static void configureJars(Project project) { public void execute(Task task) { // this doFirst is added before the info plugin, therefore it will run // after the doFirst added by the info plugin, and we can override attributes - jarTask.getManifest() - .attributes( - Map.of("Build-Date", BuildParams.getBuildDate(), "Build-Java-Version", BuildParams.getGradleJavaVersion()) - ); + jarTask.getManifest().attributes(Map.of("Build-Date", buildDate, "Build-Java-Version", gradleJavaVersion)); } }); }); - project.getPluginManager().withPlugin("com.github.johnrengelman.shadow", p -> { + project.getPluginManager().withPlugin("com.gradleup.shadow", p -> { project.getTasks().withType(ShadowJar.class).configureEach(shadowJar -> { /* * Replace the default "-all" classifier with null @@ -102,10 +107,13 @@ public void execute(Task task) { }); } - private static void configureJarManifest(Project project) { + private static void configureJarManifest(Project project, BuildParameterExtension buildParams) { + String gitOrigin = buildParams.getGitOrigin(); + String gitRevision = buildParams.getGitRevision(); + project.getPlugins().withType(InfoBrokerPlugin.class).whenPluginAdded(manifestPlugin -> { - manifestPlugin.add("Module-Origin", toStringable(BuildParams::getGitOrigin)); - manifestPlugin.add("Change", toStringable(BuildParams::getGitRevision)); + manifestPlugin.add("Module-Origin", toStringable(() -> gitOrigin)); + manifestPlugin.add("Change", toStringable(() -> gitRevision)); manifestPlugin.add("X-Compile-Elasticsearch-Version", toStringable(VersionProperties::getElasticsearch)); manifestPlugin.add("X-Compile-Lucene-Version", toStringable(VersionProperties::getLucene)); manifestPlugin.add( diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java index 7a831fbcc1464..4446952fec2bb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java @@ -13,7 +13,7 @@ import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.internal.conventions.util.Util; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.internal.test.ErrorReportingTestListener; import org.elasticsearch.gradle.internal.test.SimpleCommandLineArgumentProvider; @@ -26,6 +26,7 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.FileCollection; import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.provider.Property; import org.gradle.api.provider.ProviderFactory; import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.SourceSetContainer; @@ -37,6 +38,7 @@ import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; import static org.elasticsearch.gradle.util.FileUtils.mkdirs; import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure; @@ -52,6 +54,9 @@ public abstract class ElasticsearchTestBasePlugin implements Plugin { @Override public void apply(Project project) { + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + Property buildParams = loadBuildParams(project); + project.getPluginManager().apply(GradleTestPolicySetupPlugin.class); // for fips mode check project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); @@ -100,7 +105,7 @@ public void execute(Task t) { test.getExtensions().add("nonInputProperties", nonInputProperties); test.setWorkingDir(project.file(project.getBuildDir() + "/testrun/" + test.getName().replace("#", "_"))); - test.setMaxParallelForks(Integer.parseInt(System.getProperty("tests.jvms", BuildParams.getDefaultParallel().toString()))); + test.setMaxParallelForks(Integer.parseInt(System.getProperty("tests.jvms", buildParams.get().getDefaultParallel().toString()))); test.exclude("**/*$*.class"); @@ -146,9 +151,9 @@ public void execute(Task t) { // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability experimentation if (System.getProperty("ignore.tests.seed") != null) { - nonInputProperties.systemProperty("tests.seed", BuildParams.getTestSeed()); + nonInputProperties.systemProperty("tests.seed", buildParams.get().getTestSeed()); } else { - test.systemProperty("tests.seed", BuildParams.getTestSeed()); + test.systemProperty("tests.seed", buildParams.get().getTestSeed()); } // don't track these as inputs since they contain absolute paths and break cache relocatability @@ -193,7 +198,7 @@ public void execute(Task t) { * If this project builds a shadow JAR than any unit tests should test against that artifact instead of * compiled class output and dependency jars. This better emulates the runtime environment of consumers. */ - project.getPluginManager().withPlugin("com.github.johnrengelman.shadow", p -> { + project.getPluginManager().withPlugin("com.gradleup.shadow", p -> { if (test.getName().equals(JavaPlugin.TEST_TASK_NAME)) { // Remove output class files and any other dependencies from the test classpath, since the shadow JAR includes these SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java index fcf286ed471dd..80fd6db59cf9f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java @@ -10,7 +10,7 @@ package org.elasticsearch.gradle.internal; import org.elasticsearch.gradle.Version; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.Action; import org.gradle.api.InvalidUserDataException; @@ -39,6 +39,7 @@ import static java.util.Arrays.asList; import static java.util.Arrays.stream; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; /** * We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots. @@ -64,23 +65,29 @@ public void apply(Project project) { project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); project.getPlugins().apply(JvmToolchainsPlugin.class); toolChainService = project.getExtensions().getByType(JavaToolchainService.class); - BuildParams.getBwcVersions().forPreviousUnreleased((BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> { + BuildParameterExtension buildParams = loadBuildParams(project).get(); + Boolean isCi = buildParams.isCi(); + buildParams.getBwcVersions().forPreviousUnreleased((BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> { configureBwcProject( project.project(unreleasedVersion.gradleProjectPath()), + buildParams, unreleasedVersion, providerFactory, objectFactory, - toolChainService + toolChainService, + isCi ); }); } private static void configureBwcProject( Project project, + BuildParameterExtension buildParams, BwcVersions.UnreleasedVersionInfo versionInfo, ProviderFactory providerFactory, ObjectFactory objectFactory, - JavaToolchainService toolChainService + JavaToolchainService toolChainService, + Boolean isCi ) { ProjectLayout layout = project.getLayout(); Provider versionInfoProvider = providerFactory.provider(() -> versionInfo); @@ -96,7 +103,8 @@ private static void configureBwcProject( providerFactory, toolChainService, versionInfoProvider, - checkoutDir + checkoutDir, + isCi ); BwcGitExtension gitExtension = project.getPlugins().apply(InternalBwcGitPlugin.class).getGitExtension(); Provider bwcVersion = versionInfoProvider.map(info -> info.version()); @@ -122,6 +130,7 @@ private static void configureBwcProject( for (DistributionProject distributionProject : distributionProjects) { createBuildBwcTask( bwcSetupExtension, + buildParams, project, bwcVersion, distributionProject.name, @@ -144,6 +153,7 @@ private static void configureBwcProject( createBuildBwcTask( bwcSetupExtension, + buildParams, project, bwcVersion, "jdbc", @@ -177,6 +187,7 @@ private static void configureBwcProject( createBuildBwcTask( bwcSetupExtension, + buildParams, project, bwcVersion, stableApiProject.getName(), @@ -296,6 +307,7 @@ public static String buildBwcTaskName(String projectName) { static void createBuildBwcTask( BwcSetupExtension bwcSetupExtension, + BuildParameterExtension buildParams, Project project, Provider bwcVersion, String projectName, @@ -316,7 +328,7 @@ static void createBuildBwcTask( } else { c.getOutputs().files(expectedOutputFile); } - c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> BuildParams.isCi() == false); + c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> buildParams.isCi() == false); c.getArgs().add("-p"); c.getArgs().add(projectPath); c.getArgs().add(assembleTaskName); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java index 0bf4bcb33c23b..60699522cdc3f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java @@ -20,7 +20,7 @@ import org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes; import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin; import org.elasticsearch.gradle.internal.docker.DockerSupportService; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.GradleException; @@ -35,6 +35,8 @@ import java.util.Map; import java.util.function.Function; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + /** * An internal elasticsearch build plugin that registers additional * distribution resolution strategies to the 'elasticsearch.download-distribution' plugin @@ -47,6 +49,8 @@ public void apply(Project project) { // this is needed for isInternal project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class); + BuildParameterExtension buildParams = loadBuildParams(project).get(); + DistributionDownloadPlugin distributionDownloadPlugin = project.getPlugins().apply(DistributionDownloadPlugin.class); Provider dockerSupport = GradleUtils.getBuildService( project.getGradle().getSharedServices(), @@ -55,7 +59,10 @@ public void apply(Project project) { distributionDownloadPlugin.setDockerAvailability( dockerSupport.map(dockerSupportService -> dockerSupportService.getDockerAvailability().isAvailable()) ); - registerInternalDistributionResolutions(DistributionDownloadPlugin.getRegistrationsContainer(project)); + registerInternalDistributionResolutions( + DistributionDownloadPlugin.getRegistrationsContainer(project), + buildParams.getBwcVersionsProperty() + ); } /** @@ -66,7 +73,7 @@ public void apply(Project project) { *

* BWC versions are resolved as project to projects under `:distribution:bwc`. */ - private void registerInternalDistributionResolutions(List resolutions) { + private void registerInternalDistributionResolutions(List resolutions, Provider bwcVersions) { resolutions.add(new DistributionResolution("local-build", (project, distribution) -> { if (isCurrentVersion(distribution)) { // non-external project, so depend on local build @@ -78,7 +85,7 @@ private void registerInternalDistributionResolutions(List { - BwcVersions.UnreleasedVersionInfo unreleasedInfo = BuildParams.getBwcVersions() + BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.get() .unreleasedInfo(Version.fromString(distribution.getVersion())); if (unreleasedInfo != null) { if (distribution.getBundledJdk() == false) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java index c7ab83ff7829a..7e7ffad12a9a5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java @@ -10,34 +10,29 @@ package org.elasticsearch.gradle.internal; import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; -import org.gradle.api.provider.ProviderFactory; -import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; public class InternalTestClustersPlugin implements Plugin { - private ProviderFactory providerFactory; - - @Inject - public InternalTestClustersPlugin(ProviderFactory providerFactory) { - this.providerFactory = providerFactory; - } - @Override public void apply(Project project) { project.getPlugins().apply(InternalDistributionDownloadPlugin.class); + project.getRootProject().getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + BuildParameterExtension buildParams = loadBuildParams(project).get(); project.getRootProject().getPluginManager().apply(InternalReaperPlugin.class); TestClustersPlugin testClustersPlugin = project.getPlugins().apply(TestClustersPlugin.class); - testClustersPlugin.setRuntimeJava(providerFactory.provider(() -> BuildParams.getRuntimeJavaHome())); + testClustersPlugin.setRuntimeJava(buildParams.getRuntimeJavaHome()); testClustersPlugin.setIsReleasedVersion( - version -> (version.equals(VersionProperties.getElasticsearchVersion()) && BuildParams.isSnapshotBuild() == false) - || BuildParams.getBwcVersions().unreleasedInfo(version) == null + version -> (version.equals(VersionProperties.getElasticsearchVersion()) && buildParams.isSnapshotBuild() == false) + || buildParams.getBwcVersions().unreleasedInfo(version) == null ); if (shouldConfigureTestClustersWithOneProcessor()) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java index f988208ab4fec..d1585120b0803 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java @@ -9,7 +9,8 @@ package org.elasticsearch.gradle.internal; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.JavaVersion; @@ -47,6 +48,7 @@ import javax.inject.Inject; import static de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin.FORBIDDEN_APIS_TASK_NAME; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; import static org.objectweb.asm.Opcodes.V_PREVIEW; public class MrjarPlugin implements Plugin { @@ -64,6 +66,8 @@ public class MrjarPlugin implements Plugin { @Override public void apply(Project project) { project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class); + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); var javaExtension = project.getExtensions().getByType(JavaPluginExtension.class); var isIdeaSync = System.getProperty("idea.sync.active", "false").equals("true"); var ideaSourceSetsEnabled = project.hasProperty(MRJAR_IDEA_ENABLED) && project.property(MRJAR_IDEA_ENABLED).equals("true"); @@ -89,7 +93,7 @@ public void apply(Project project) { String testSourceSetName = SourceSet.TEST_SOURCE_SET_NAME + javaVersion; SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion); testSourceSets.add(testSourceSetName); - createTestTask(project, testSourceSet, javaVersion, mainSourceSets); + createTestTask(project, buildParams, testSourceSet, javaVersion, mainSourceSets); } } @@ -163,7 +167,13 @@ private void configureSourceSetInJar(Project project, SourceSet sourceSet, int j jarTask.configure(task -> task.into("META-INF/versions/" + javaVersion, copySpec -> copySpec.from(sourceSet.getOutput()))); } - private void createTestTask(Project project, SourceSet sourceSet, int javaVersion, List mainSourceSets) { + private void createTestTask( + Project project, + BuildParameterExtension buildParams, + SourceSet sourceSet, + int javaVersion, + List mainSourceSets + ) { var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME); var testTaskProvider = project.getTasks().register(JavaPlugin.TEST_TASK_NAME + javaVersion, Test.class); testTaskProvider.configure(testTask -> { @@ -180,9 +190,9 @@ private void createTestTask(Project project, SourceSet sourceSet, int javaVersio // only set the jdk if runtime java isn't set because setting the toolchain is incompatible with // runtime java setting the executable directly - if (BuildParams.getIsRuntimeJavaHomeSet()) { + if (buildParams.getIsRuntimeJavaHomeSet()) { testTask.onlyIf("runtime java must support java " + javaVersion, t -> { - JavaVersion runtimeJavaVersion = BuildParams.getRuntimeJavaVersion(); + JavaVersion runtimeJavaVersion = buildParams.getRuntimeJavaVersion().get(); return runtimeJavaVersion.isCompatibleWith(JavaVersion.toVersion(javaVersion)); }); } else { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java index 84a6432041ed1..7348181c4199c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java @@ -8,6 +8,7 @@ */ package org.elasticsearch.gradle.internal.docker; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; @@ -17,6 +18,8 @@ import java.util.List; import java.util.stream.Collectors; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + /** * Plugin providing {@link DockerSupportService} for detecting Docker installations and determining requirements for Docker-based * Elasticsearch build tasks. @@ -30,11 +33,14 @@ public void apply(Project project) { if (project != project.getRootProject()) { throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project."); } + project.getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); Provider dockerSupportServiceProvider = project.getGradle() .getSharedServices() .registerIfAbsent(DOCKER_SUPPORT_SERVICE_NAME, DockerSupportService.class, spec -> spec.parameters(params -> { params.setExclusionsFile(new File(project.getRootDir(), DOCKER_ON_LINUX_EXCLUSIONS_FILE)); + params.getIsCI().set(buildParams.isCi()); })); // Ensure that if we are trying to run any DockerBuildTask tasks, we assert an available Docker installation exists diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java index 344a477e74ef9..f40f5d932b701 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java @@ -13,10 +13,10 @@ import org.elasticsearch.gradle.Architecture; import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.Version; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.GradleException; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Property; import org.gradle.api.provider.ProviderFactory; import org.gradle.api.services.BuildService; import org.gradle.api.services.BuildServiceParameters; @@ -59,7 +59,6 @@ public abstract class DockerSupportService implements BuildService serviceInfos; private Map> tcpPorts; private Map> udpPorts; @@ -228,7 +227,7 @@ private boolean isExcludedOs() { // We don't attempt to check the current flavor and version of Linux unless we're // running in CI, because we don't want to stop people running the Docker tests in // their own environments if they really want to. - if (BuildParams.isCi() == false) { + if (getParameters().getIsCI().get().booleanValue() == false) { return false; } @@ -356,10 +355,6 @@ public Map> getUdpPorts() { return udpPorts; } - public void setServiceInfos(Map serviceInfos) { - this.serviceInfos = serviceInfos; - } - /** * An immutable class that represents the results of a Docker search from {@link #getDockerAvailability()}}. */ @@ -402,5 +397,7 @@ interface Parameters extends BuildServiceParameters { File getExclusionsFile(); void setExclusionsFile(File exclusionsFile); + + Property getIsCI(); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java new file mode 100644 index 0000000000000..5531194e0abde --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.info; + +import org.elasticsearch.gradle.internal.BwcVersions; +import org.gradle.api.Action; +import org.gradle.api.JavaVersion; +import org.gradle.api.Task; +import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; +import org.gradle.api.provider.ProviderFactory; +import org.gradle.jvm.toolchain.JavaToolchainSpec; + +import java.io.File; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicReference; + +public abstract class BuildParameterExtension { + private final Provider inFipsJvm; + private final Provider runtimeJavaHome; + private final Boolean isRuntimeJavaHomeSet; + private final List javaVersions; + private final JavaVersion minimumCompilerVersion; + private final JavaVersion minimumRuntimeVersion; + private final JavaVersion gradleJavaVersion; + private final Provider runtimeJavaVersion; + private final Provider> javaToolChainSpec; + private final Provider runtimeJavaDetails; + private final String gitRevision; + private transient AtomicReference buildDate = new AtomicReference<>(); + private final String testSeed; + private final Boolean isCi; + private final Integer defaultParallel; + private final Boolean isSnapshotBuild; + + public BuildParameterExtension( + ProviderFactory providers, + Provider runtimeJavaHome, + Provider> javaToolChainSpec, + Provider runtimeJavaVersion, + boolean isRuntimeJavaHomeSet, + Provider runtimeJavaDetails, + List javaVersions, + JavaVersion minimumCompilerVersion, + JavaVersion minimumRuntimeVersion, + JavaVersion gradleJavaVersion, + String gitRevision, + String gitOrigin, + ZonedDateTime buildDate, + String testSeed, + boolean isCi, + int defaultParallel, + final boolean isSnapshotBuild, + Provider bwcVersions + ) { + this.inFipsJvm = providers.systemProperty("tests.fips.enabled").map(BuildParameterExtension::parseBoolean); + this.runtimeJavaHome = runtimeJavaHome; + this.javaToolChainSpec = javaToolChainSpec; + this.runtimeJavaVersion = runtimeJavaVersion; + this.isRuntimeJavaHomeSet = isRuntimeJavaHomeSet; + this.runtimeJavaDetails = runtimeJavaDetails; + this.javaVersions = javaVersions; + this.minimumCompilerVersion = minimumCompilerVersion; + this.minimumRuntimeVersion = minimumRuntimeVersion; + this.gradleJavaVersion = gradleJavaVersion; + this.gitRevision = gitRevision; + this.testSeed = testSeed; + this.isCi = isCi; + this.defaultParallel = defaultParallel; + this.isSnapshotBuild = isSnapshotBuild; + this.getBwcVersionsProperty().set(bwcVersions); + this.getGitOriginProperty().set(gitOrigin); + } + + private static boolean parseBoolean(String s) { + if (s == null) { + return false; + } + return Boolean.parseBoolean(s); + } + + public boolean getInFipsJvm() { + return inFipsJvm.getOrElse(false); + } + + public Provider getRuntimeJavaHome() { + return runtimeJavaHome; + } + + public void withFipsEnabledOnly(Task task) { + task.onlyIf("FIPS mode disabled", task1 -> getInFipsJvm() == false); + } + + public Boolean getIsRuntimeJavaHomeSet() { + return isRuntimeJavaHomeSet; + } + + public List getJavaVersions() { + return javaVersions; + } + + public JavaVersion getMinimumCompilerVersion() { + return minimumCompilerVersion; + } + + public JavaVersion getMinimumRuntimeVersion() { + return minimumRuntimeVersion; + } + + public JavaVersion getGradleJavaVersion() { + return gradleJavaVersion; + } + + public Provider getRuntimeJavaVersion() { + return runtimeJavaVersion; + } + + public Provider> getJavaToolChainSpec() { + return javaToolChainSpec; + } + + public Provider getRuntimeJavaDetails() { + return runtimeJavaDetails; + } + + public String getGitRevision() { + return gitRevision; + } + + public String getGitOrigin() { + return getGitOriginProperty().get(); + } + + public ZonedDateTime getBuildDate() { + ZonedDateTime value = buildDate.get(); + if (value == null) { + value = ZonedDateTime.now(ZoneOffset.UTC); + if (buildDate.compareAndSet(null, value) == false) { + // If another thread initialized it first, return the initialized value + value = buildDate.get(); + } + } + return value; + } + + public String getTestSeed() { + return testSeed; + } + + public Boolean isCi() { + return isCi; + } + + public Integer getDefaultParallel() { + return defaultParallel; + } + + public Boolean isSnapshotBuild() { + return isSnapshotBuild; + } + + public BwcVersions getBwcVersions() { + return getBwcVersionsProperty().get(); + } + + public abstract Property getBwcVersionsProperty(); + + public abstract Property getGitOriginProperty(); + + public Random getRandom() { + return new Random(Long.parseUnsignedLong(testSeed.split(":")[0], 16)); + } + + public Boolean isGraalVmRuntime() { + return runtimeJavaDetails.get().toLowerCase().contains("graalvm"); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterService.java new file mode 100644 index 0000000000000..ec1bc4aec1324 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterService.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.info; + +import org.gradle.api.provider.Property; +import org.gradle.api.services.BuildService; +import org.gradle.api.services.BuildServiceParameters; + +public abstract class BuildParameterService implements BuildService, AutoCloseable { + public interface Params extends BuildServiceParameters { + Property getBuildParams(); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java index d3afeed9f8578..ea8aeda8fc099 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java @@ -8,43 +8,13 @@ */ package org.elasticsearch.gradle.internal.info; -import org.elasticsearch.gradle.internal.BwcVersions; -import org.gradle.api.Action; -import org.gradle.api.JavaVersion; -import org.gradle.api.Task; -import org.gradle.api.provider.Provider; -import org.gradle.jvm.toolchain.JavaToolchainSpec; - -import java.io.File; -import java.io.IOException; import java.lang.reflect.Modifier; -import java.time.ZonedDateTime; import java.util.Arrays; -import java.util.List; -import java.util.Random; import java.util.function.Consumer; -import static java.util.Objects.requireNonNull; - +@Deprecated public class BuildParams { - private static Provider runtimeJavaHome; - private static Boolean isRuntimeJavaHomeSet; - private static List javaVersions; - private static JavaVersion minimumCompilerVersion; - private static JavaVersion minimumRuntimeVersion; - private static JavaVersion gradleJavaVersion; - private static Provider runtimeJavaVersion; - private static Provider> javaToolChainSpec; - private static Provider runtimeJavaDetails; - private static Boolean inFipsJvm; - private static String gitRevision; - private static String gitOrigin; - private static ZonedDateTime buildDate; - private static String testSeed; private static Boolean isCi; - private static Integer defaultParallel; - private static Boolean isSnapshotBuild; - private static Provider bwcVersions; /** * Initialize global build parameters. This method accepts and a initialization function which in turn accepts a @@ -58,90 +28,10 @@ public static void init(Consumer initializer) { initializer.accept(MutableBuildParams.INSTANCE); } - public static File getRuntimeJavaHome() { - return value(runtimeJavaHome).get(); - } - - public static Boolean getIsRuntimeJavaHomeSet() { - return value(isRuntimeJavaHomeSet); - } - - public static List getJavaVersions() { - return value(javaVersions); - } - - public static JavaVersion getMinimumCompilerVersion() { - return value(minimumCompilerVersion); - } - - public static JavaVersion getMinimumRuntimeVersion() { - return value(minimumRuntimeVersion); - } - - public static JavaVersion getGradleJavaVersion() { - return value(gradleJavaVersion); - } - - public static JavaVersion getRuntimeJavaVersion() { - return value(runtimeJavaVersion.get()); - } - - public static String getRuntimeJavaDetails() { - return value(runtimeJavaDetails.get()); - } - - public static Boolean isInFipsJvm() { - return value(inFipsJvm); - } - - public static void withFipsEnabledOnly(Task task) { - task.onlyIf("FIPS mode disabled", task1 -> isInFipsJvm() == false); - } - - public static String getGitRevision() { - return value(gitRevision); - } - - public static String getGitOrigin() { - return value(gitOrigin); - } - - public static ZonedDateTime getBuildDate() { - return value(buildDate); - } - - public static BwcVersions getBwcVersions() { - return value(bwcVersions).get(); - } - - public static String getTestSeed() { - return value(testSeed); - } - - public static Random getRandom() { - return new Random(Long.parseUnsignedLong(testSeed.split(":")[0], 16)); - } - public static Boolean isCi() { return value(isCi); } - public static Boolean isGraalVmRuntime() { - return value(runtimeJavaDetails.get().toLowerCase().contains("graalvm")); - } - - public static Integer getDefaultParallel() { - return value(defaultParallel); - } - - public static boolean isSnapshotBuild() { - return value(BuildParams.isSnapshotBuild); - } - - public static Provider> getJavaToolChainSpec() { - return javaToolChainSpec; - } - private static T value(T object) { if (object == null) { String callingMethod = Thread.currentThread().getStackTrace()[2].getMethodName(); @@ -183,82 +73,8 @@ public void reset() { }); } - public void setRuntimeJavaHome(Provider runtimeJavaHome) { - BuildParams.runtimeJavaHome = runtimeJavaHome.map(javaHome -> { - try { - return javaHome.getCanonicalFile(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - - public void setIsRuntimeJavaHomeSet(boolean isRuntimeJavaHomeSet) { - BuildParams.isRuntimeJavaHomeSet = isRuntimeJavaHomeSet; - } - - public void setJavaVersions(List javaVersions) { - BuildParams.javaVersions = requireNonNull(javaVersions); - } - - public void setMinimumCompilerVersion(JavaVersion minimumCompilerVersion) { - BuildParams.minimumCompilerVersion = requireNonNull(minimumCompilerVersion); - } - - public void setMinimumRuntimeVersion(JavaVersion minimumRuntimeVersion) { - BuildParams.minimumRuntimeVersion = requireNonNull(minimumRuntimeVersion); - } - - public void setGradleJavaVersion(JavaVersion gradleJavaVersion) { - BuildParams.gradleJavaVersion = requireNonNull(gradleJavaVersion); - } - - public void setRuntimeJavaVersion(Provider runtimeJavaVersion) { - BuildParams.runtimeJavaVersion = requireNonNull(runtimeJavaVersion); - } - - public void setRuntimeJavaDetails(Provider runtimeJavaDetails) { - BuildParams.runtimeJavaDetails = runtimeJavaDetails; - } - - public void setInFipsJvm(boolean inFipsJvm) { - BuildParams.inFipsJvm = inFipsJvm; - } - - public void setGitRevision(String gitRevision) { - BuildParams.gitRevision = requireNonNull(gitRevision); - } - - public void setGitOrigin(String gitOrigin) { - BuildParams.gitOrigin = requireNonNull(gitOrigin); - } - - public void setBuildDate(ZonedDateTime buildDate) { - BuildParams.buildDate = requireNonNull(buildDate); - } - - public void setTestSeed(String testSeed) { - BuildParams.testSeed = requireNonNull(testSeed); - } - public void setIsCi(boolean isCi) { BuildParams.isCi = isCi; } - - public void setDefaultParallel(int defaultParallel) { - BuildParams.defaultParallel = defaultParallel; - } - - public void setIsSnapshotBuild(final boolean isSnapshotBuild) { - BuildParams.isSnapshotBuild = isSnapshotBuild; - } - - public void setBwcVersions(Provider bwcVersions) { - BuildParams.bwcVersions = requireNonNull(bwcVersions); - } - - public void setJavaToolChainSpec(Provider> javaToolChain) { - BuildParams.javaToolChainSpec = javaToolChain; - } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java index 2d5a28bdd6af9..761b0601a1c24 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java @@ -58,6 +58,8 @@ import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.conventions.GUtils.elvis; + public class GlobalBuildInfoPlugin implements Plugin { private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class); private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "server/src/main/java/org/elasticsearch/Version.java"; @@ -67,6 +69,7 @@ public class GlobalBuildInfoPlugin implements Plugin { private final JvmMetadataDetector metadataDetector; private final ProviderFactory providers; private JavaToolchainService toolChainService; + private Project project; @Inject public GlobalBuildInfoPlugin( @@ -87,6 +90,7 @@ public void apply(Project project) { if (project != project.getRootProject()) { throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project."); } + this.project = project; project.getPlugins().apply(JvmToolchainsPlugin.class); toolChainService = project.getExtensions().getByType(JavaToolchainService.class); GradleVersion minimumGradleVersion = GradleVersion.version(getResourceContents("/minimumGradleVersion")); @@ -98,55 +102,61 @@ public void apply(Project project) { JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion")); Provider explicitRuntimeJavaHome = findRuntimeJavaHome(); - boolean isExplicitRuntimeJavaHomeSet = explicitRuntimeJavaHome.isPresent(); - Provider actualRuntimeJavaHome = isExplicitRuntimeJavaHomeSet + boolean isRuntimeJavaHomeExplicitlySet = explicitRuntimeJavaHome.isPresent(); + Provider actualRuntimeJavaHome = isRuntimeJavaHomeExplicitlySet ? explicitRuntimeJavaHome : resolveJavaHomeFromToolChainService(VersionProperties.getBundledJdkMajorVersion()); GitInfo gitInfo = GitInfo.gitInfo(project.getRootDir()); - BuildParams.init(params -> { - params.reset(); - params.setRuntimeJavaHome(actualRuntimeJavaHome); - params.setJavaToolChainSpec(resolveToolchainSpecFromEnv()); - Provider runtimeJdkMetaData = actualRuntimeJavaHome.map( - runtimeJavaHome -> metadataDetector.getMetadata(getJavaInstallation(runtimeJavaHome)) - ); - params.setRuntimeJavaVersion( + Provider runtimeJdkMetaData = actualRuntimeJavaHome.map( + runtimeJavaHome -> metadataDetector.getMetadata(getJavaInstallation(runtimeJavaHome)) + ); + AtomicReference cache = new AtomicReference<>(); + Provider bwcVersionsProvider = providers.provider( + () -> cache.updateAndGet(val -> val == null ? resolveBwcVersions() : val) + ); + BuildParameterExtension buildParams = project.getExtensions() + .create( + "buildParams", + BuildParameterExtension.class, + actualRuntimeJavaHome, + resolveToolchainSpecFromEnv(), actualRuntimeJavaHome.map( javaHome -> determineJavaVersion( "runtime java.home", javaHome, - isExplicitRuntimeJavaHomeSet + isRuntimeJavaHomeExplicitlySet ? minimumRuntimeVersion : JavaVersion.toVersion(VersionProperties.getBundledJdkMajorVersion()) ) - ) + ), + isRuntimeJavaHomeExplicitlySet, + runtimeJdkMetaData.map(m -> formatJavaVendorDetails(m)), + getAvailableJavaVersions(), + minimumCompilerVersion, + minimumRuntimeVersion, + Jvm.current().getJavaVersion(), + gitInfo.getRevision(), + gitInfo.getOrigin(), + ZonedDateTime.now(ZoneOffset.UTC), + getTestSeed(), + System.getenv("JENKINS_URL") != null || System.getenv("BUILDKITE_BUILD_URL") != null || System.getProperty("isCI") != null, + ParallelDetector.findDefaultParallel(project), + Util.getBooleanProperty("build.snapshot", true), + bwcVersionsProvider ); - params.setIsRuntimeJavaHomeSet(isExplicitRuntimeJavaHomeSet); - params.setRuntimeJavaDetails(runtimeJdkMetaData.map(m -> formatJavaVendorDetails(m))); - params.setJavaVersions(getAvailableJavaVersions()); - params.setMinimumCompilerVersion(minimumCompilerVersion); - params.setMinimumRuntimeVersion(minimumRuntimeVersion); - params.setGradleJavaVersion(Jvm.current().getJavaVersion()); - params.setGitRevision(gitInfo.getRevision()); - params.setGitOrigin(gitInfo.getOrigin()); - params.setBuildDate(ZonedDateTime.now(ZoneOffset.UTC)); - params.setTestSeed(getTestSeed()); + + project.getGradle().getSharedServices().registerIfAbsent("buildParams", BuildParameterService.class, spec -> { + // Provide some parameters + spec.getParameters().getBuildParams().set(buildParams); + }); + + BuildParams.init(params -> { + params.reset(); params.setIsCi( System.getenv("JENKINS_URL") != null || System.getenv("BUILDKITE_BUILD_URL") != null || System.getProperty("isCI") != null ); - params.setDefaultParallel(ParallelDetector.findDefaultParallel(project)); - params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false)); - params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true)); - AtomicReference cache = new AtomicReference<>(); - params.setBwcVersions( - providers.provider( - () -> cache.updateAndGet( - val -> val == null ? resolveBwcVersions(Util.locateElasticsearchWorkspace(project.getGradle())) : val - ) - ) - ); }); // Enforce the minimum compiler version @@ -155,7 +165,7 @@ public void apply(Project project) { // Print global build info header just before task execution // Only do this if we are the root build of a composite if (GradleUtils.isIncludedBuild(project) == false) { - project.getGradle().getTaskGraph().whenReady(graph -> logGlobalBuildInfo()); + project.getGradle().getTaskGraph().whenReady(graph -> logGlobalBuildInfo(buildParams)); } } @@ -180,9 +190,12 @@ private String formatJavaVendorDetails(JvmInstallationMetadata runtimeJdkMetaDat /* Introspect all versions of ES that may be tested against for backwards * compatibility. It is *super* important that this logic is the same as the * logic in VersionUtils.java. */ - private static BwcVersions resolveBwcVersions(File root) { - File versionsFile = new File(root, DEFAULT_VERSION_JAVA_FILE_PATH); - try (var is = new FileInputStream(versionsFile)) { + private BwcVersions resolveBwcVersions() { + String versionsFilePath = elvis( + System.getProperty("BWC_VERSION_SOURCE"), + new File(Util.locateElasticsearchWorkspace(project.getGradle()), DEFAULT_VERSION_JAVA_FILE_PATH).getPath() + ); + try (var is = new FileInputStream(versionsFilePath)) { List versionLines = IOUtils.readLines(is, "UTF-8"); return new BwcVersions(versionLines); } catch (IOException e) { @@ -190,7 +203,7 @@ private static BwcVersions resolveBwcVersions(File root) { } } - private void logGlobalBuildInfo() { + private void logGlobalBuildInfo(BuildParameterExtension buildParams) { final String osName = System.getProperty("os.name"); final String osVersion = System.getProperty("os.version"); final String osArch = System.getProperty("os.arch"); @@ -202,14 +215,14 @@ private void logGlobalBuildInfo() { LOGGER.quiet("Elasticsearch Build Hamster says Hello!"); LOGGER.quiet(" Gradle Version : " + GradleVersion.current().getVersion()); LOGGER.quiet(" OS Info : " + osName + " " + osVersion + " (" + osArch + ")"); - if (BuildParams.getIsRuntimeJavaHomeSet()) { - JvmInstallationMetadata runtimeJvm = metadataDetector.getMetadata(getJavaInstallation(BuildParams.getRuntimeJavaHome())); + if (buildParams.getIsRuntimeJavaHomeSet()) { + JvmInstallationMetadata runtimeJvm = metadataDetector.getMetadata(getJavaInstallation(buildParams.getRuntimeJavaHome().get())); final String runtimeJvmVendorDetails = runtimeJvm.getVendor().getDisplayName(); final String runtimeJvmImplementationVersion = runtimeJvm.getJvmVersion(); final String runtimeVersion = runtimeJvm.getRuntimeVersion(); final String runtimeExtraDetails = runtimeJvmVendorDetails + ", " + runtimeVersion; LOGGER.quiet(" Runtime JDK Version : " + runtimeJvmImplementationVersion + " (" + runtimeExtraDetails + ")"); - LOGGER.quiet(" Runtime java.home : " + BuildParams.getRuntimeJavaHome()); + LOGGER.quiet(" Runtime java.home : " + buildParams.getRuntimeJavaHome().get()); LOGGER.quiet(" Gradle JDK Version : " + gradleJvmImplementationVersion + " (" + gradleJvmVendorDetails + ")"); LOGGER.quiet(" Gradle java.home : " + gradleJvm.getJavaHome()); } else { @@ -220,8 +233,8 @@ private void logGlobalBuildInfo() { if (javaToolchainHome != null) { LOGGER.quiet(" JAVA_TOOLCHAIN_HOME : " + javaToolchainHome); } - LOGGER.quiet(" Random Testing Seed : " + BuildParams.getTestSeed()); - LOGGER.quiet(" In FIPS 140 mode : " + BuildParams.isInFipsJvm()); + LOGGER.quiet(" Random Testing Seed : " + buildParams.getTestSeed()); + LOGGER.quiet(" In FIPS 140 mode : " + buildParams.getInFipsJvm()); LOGGER.quiet("======================================="); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenApisPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenApisPrecommitPlugin.java index f71c86b19a140..f1ec236efe646 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenApisPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenApisPrecommitPlugin.java @@ -11,7 +11,7 @@ import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask; import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.plugins.JavaBasePlugin; @@ -30,7 +30,7 @@ public class ForbiddenApisPrecommitPlugin extends PrecommitPlugin { @Override public TaskProvider createTask(Project project) { project.getPluginManager().apply(JavaBasePlugin.class); - + var buildParams = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class); // Create a convenience task for all checks (this does not conflict with extension, as it has higher priority in DSL): var forbiddenTask = project.getTasks() .register(FORBIDDEN_APIS_TASK_NAME, task -> { task.setDescription("Runs forbidden-apis checks."); }); @@ -57,7 +57,7 @@ public TaskProvider createTask(Project project) { t.setClassesDirs(sourceSet.getOutput().getClassesDirs()); t.dependsOn(resourcesTask); t.setClasspath(sourceSet.getRuntimeClasspath().plus(sourceSet.getCompileClasspath())); - t.setTargetCompatibility(BuildParams.getMinimumRuntimeVersion().getMajorVersion()); + t.setTargetCompatibility(buildParams.getMinimumRuntimeVersion().getMajorVersion()); t.getBundledSignatures().set(BUNDLED_SIGNATURE_DEFAULTS); t.setSignaturesFiles( project.files( diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java index 80cece6074ab7..f70e25a57e331 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java @@ -12,17 +12,19 @@ import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin; import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask; import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.component.ModuleComponentIdentifier; +import org.gradle.api.provider.Property; import org.gradle.api.tasks.TaskProvider; import java.io.File; import java.nio.file.Path; import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { @@ -31,10 +33,14 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { @Override public TaskProvider createTask(Project project) { + project.getRootProject().getPlugins().apply(CompileOnlyResolvePlugin.class); + Property buildParams = loadBuildParams(project); + project.getPlugins().apply(CompileOnlyResolvePlugin.class); project.getConfigurations().create("forbiddenApisCliJar"); project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.6"); Configuration jdkJarHellConfig = project.getConfigurations().create(JDK_JAR_HELL_CONFIG_NAME); + if (project.getPath().equals(LIBS_ELASTICSEARCH_CORE_PROJECT_PATH) == false) { // Internal projects are not all plugins, so make sure the check is available // we are not doing this for this project itself to avoid jar hell with itself @@ -66,9 +72,12 @@ public TaskProvider createTask(Project project) { && ((ModuleComponentIdentifier) identifier).getGroup().startsWith("org.elasticsearch") == false ) ); + if (buildParams.get().getIsRuntimeJavaHomeSet()) { + t.getRuntimeJavaVersion().set(buildParams.get().getRuntimeJavaVersion()); + } t.dependsOn(resourcesTask); - t.getTargetCompatibility().set(project.provider(BuildParams::getRuntimeJavaVersion)); - t.getJavaHome().set(project.provider(BuildParams::getRuntimeJavaHome).map(File::getPath)); + t.getTargetCompatibility().set(buildParams.flatMap(params -> params.getRuntimeJavaVersion())); + t.getJavaHome().set(buildParams.flatMap(params -> params.getRuntimeJavaHome()).map(File::getPath)); t.setSignatureFile(resourcesDir.resolve("forbidden/third-party-audit.txt").toFile()); t.getJdkJarHellClasspath().from(jdkJarHellConfig); t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnly)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java index 7afee8acdd4d2..442797775de2f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java @@ -13,7 +13,6 @@ import org.apache.commons.io.output.NullOutputStream; import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.DefaultTask; import org.gradle.api.JavaVersion; import org.gradle.api.file.ArchiveOperations; @@ -194,6 +193,10 @@ public Set getMissingClassExcludes() { @SkipWhenEmpty public abstract ConfigurableFileCollection getJarsToScan(); + @Input + @Optional + public abstract Property getRuntimeJavaVersion(); + @Classpath public FileCollection getClasspath() { return classpath; @@ -371,14 +374,10 @@ private String runForbiddenAPIsCli() throws IOException { /** Returns true iff the build Java version is the same as the given version. */ private boolean isJavaVersion(JavaVersion version) { - if (BuildParams.getIsRuntimeJavaHomeSet()) { - if (version.equals(BuildParams.getRuntimeJavaVersion())) { - return true; - } - } else if (version.getMajorVersion().equals(VersionProperties.getBundledJdkMajorVersion())) { - return true; + if (getRuntimeJavaVersion().isPresent()) { + return getRuntimeJavaVersion().get().equals(version); } - return false; + return version.getMajorVersion().equals(VersionProperties.getBundledJdkMajorVersion()); } private Set runJdkJarHellCheck() throws IOException { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java index 31c6b503d7328..b19c1207d56fb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java @@ -11,7 +11,6 @@ import groovy.json.JsonOutput; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; import org.gradle.api.artifacts.Configuration; @@ -118,7 +117,7 @@ private Map> projectAttributesData() { } private Object buildTargetData() { - return Map.of("remoteUrl", remoteUrl.get(), "branch", BuildParams.getGitRevision()); + return Map.of("remoteUrl", remoteUrl.get(), "branch", getGitRevision().get()); } @InputFiles @@ -160,4 +159,9 @@ public Property getRemoteUrl() { public Property getTargetReference() { return targetReference; } + + @Input + public Property getGitRevision() { + return targetReference; + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java index b3e3d7f7c004e..fa10daf8dfaaa 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java @@ -10,17 +10,22 @@ package org.elasticsearch.gradle.internal.snyk; import org.elasticsearch.gradle.internal.conventions.info.GitInfo; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.ProjectLayout; import org.gradle.api.plugins.JavaPlugin; import org.gradle.api.plugins.JavaPluginExtension; +import org.gradle.api.provider.Property; import org.gradle.api.provider.ProviderFactory; import org.gradle.api.tasks.SourceSet; import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + public class SnykDependencyMonitoringGradlePlugin implements Plugin { public static final String UPLOAD_TASK_NAME = "uploadSnykDependencyGraph"; @@ -35,10 +40,14 @@ public SnykDependencyMonitoringGradlePlugin(ProjectLayout projectLayout, Provide @Override public void apply(Project project) { + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + Property buildParams = loadBuildParams(project); + var generateTaskProvider = project.getTasks() .register("generateSnykDependencyGraph", GenerateSnykDependencyGraph.class, generateSnykDependencyGraph -> { generateSnykDependencyGraph.getProjectPath().set(project.getPath()); generateSnykDependencyGraph.getProjectName().set(project.getName()); + generateSnykDependencyGraph.getGitRevision().set(buildParams.get().getGitRevision()); String projectVersion = project.getVersion().toString(); generateSnykDependencyGraph.getVersion().set(projectVersion); generateSnykDependencyGraph.getGradleVersion().set(project.getGradle().getGradleVersion()); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java index 8e7884888b63b..e8d2bbd93ff20 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java @@ -16,11 +16,11 @@ import org.elasticsearch.gradle.ElasticsearchDistributionType; import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.internal.BwcVersions; import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin; import org.elasticsearch.gradle.internal.JdkDownloadPlugin; import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin; import org.elasticsearch.gradle.internal.docker.DockerSupportService; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Action; @@ -54,6 +54,7 @@ import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; /** * This class defines gradle tasks for testing our various distribution artifacts. @@ -72,6 +73,7 @@ public void apply(Project project) { project.getPlugins().apply(InternalDistributionDownloadPlugin.class); project.getPlugins().apply(JdkDownloadPlugin.class); project.getPluginManager().apply("elasticsearch.java"); + var buildParams = loadBuildParams(project).get(); Provider dockerSupport = GradleUtils.getBuildService( project.getGradle().getSharedServices(), @@ -84,7 +86,7 @@ public void apply(Project project) { List testDistributions = configureDistributions(project); Map> lifecycleTasks = lifecycleTasks(project, "destructiveDistroTest"); - Map> versionTasks = versionTasks(project, "destructiveDistroUpgradeTest"); + Map> versionTasks = versionTasks(project, "destructiveDistroUpgradeTest", buildParams.getBwcVersions()); TaskProvider destructiveDistroTest = project.getTasks().register("destructiveDistroTest"); Configuration examplePlugin = configureExamplePlugin(project); @@ -115,7 +117,7 @@ public void apply(Project project) { lifecycleTask.configure(t -> t.dependsOn(destructiveTask)); if ((type == DEB || type == RPM) && distribution.getBundledJdk()) { - for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) { + for (Version version : buildParams.getBwcVersions().getIndexCompatible()) { final ElasticsearchDistribution bwcDistro; if (version.equals(Version.fromString(distribution.getVersion()))) { // this is the same as the distribution we are testing @@ -156,10 +158,10 @@ private static Map> lifecycleTask return lifecyleTasks; } - private static Map> versionTasks(Project project, String taskPrefix) { + private static Map> versionTasks(Project project, String taskPrefix, BwcVersions bwcVersions) { Map> versionTasks = new HashMap<>(); - for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) { + for (Version version : bwcVersions.getIndexCompatible()) { versionTasks.put(version.toString(), project.getTasks().register(taskPrefix + ".v" + version)); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/InternalClusterTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/InternalClusterTestPlugin.java index 3619c9c1ec76d..e13c2544ae9cf 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/InternalClusterTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/InternalClusterTestPlugin.java @@ -9,7 +9,7 @@ package org.elasticsearch.gradle.internal.test; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.JavaVersion; import org.gradle.api.Plugin; @@ -18,16 +18,21 @@ import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.testing.Test; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + public class InternalClusterTestPlugin implements Plugin { public static final String SOURCE_SET_NAME = "internalClusterTest"; @Override public void apply(Project project) { + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); + TaskProvider internalClusterTest = GradleUtils.addTestSourceSet(project, SOURCE_SET_NAME); internalClusterTest.configure(task -> { // Set GC options to mirror defaults in jvm.options - if (BuildParams.getRuntimeJavaVersion().compareTo(JavaVersion.VERSION_14) < 0) { + if (buildParams.getRuntimeJavaVersion().get().compareTo(JavaVersion.VERSION_14) < 0) { task.jvmArgs("-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75", "-XX:+UseCMSInitiatingOccupancyOnly"); } else { task.jvmArgs("-XX:+UseG1GC"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java index fddddbd14d3ab..c13a5f0e4d30d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java @@ -9,7 +9,7 @@ package org.elasticsearch.gradle.internal.test; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.file.RegularFile; @@ -19,6 +19,8 @@ import java.util.Arrays; import java.util.List; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + public class MutedTestPlugin implements Plugin { private static final String ADDITIONAL_FILES_PROPERTY = "org.elasticsearch.additional.muted.tests"; @@ -32,6 +34,9 @@ public void apply(Project project) { .map(p -> project.getRootProject().getLayout().getProjectDirectory().file(p)) .toList(); + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); + Provider mutedTestsProvider = project.getGradle() .getSharedServices() .registerIfAbsent("mutedTests", MutedTestsBuildService.class, spec -> { @@ -46,7 +51,7 @@ public void apply(Project project) { } // Don't fail when all tests are ignored when running in CI - filter.setFailOnNoMatchingTests(BuildParams.isCi() == false); + filter.setFailOnNoMatchingTests(buildParams.isCi() == false); }); }); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java index 564465fbb2554..68711881b02f4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java @@ -11,7 +11,7 @@ import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask; import org.elasticsearch.gradle.internal.conventions.util.Util; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin; import org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin; import org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsTask; @@ -35,6 +35,7 @@ public class TestWithSslPlugin implements Plugin { @Override public void apply(Project project) { File keyStoreDir = new File(project.getBuildDir(), "keystore"); + BuildParameterExtension buildParams = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class); TaskProvider exportKeyStore = project.getTasks() .register("copyTestCertificates", ExportElasticsearchBuildResourcesTask.class, (t) -> { t.copy("test/ssl/test-client.crt"); @@ -87,7 +88,7 @@ public void apply(Project project) { .getExtensions() .getByName(TestClustersPlugin.EXTENSION_NAME); clusters.configureEach(c -> { - if (BuildParams.isInFipsJvm()) { + if (buildParams.getInFipsJvm()) { c.setting("xpack.security.transport.ssl.key", "test-node.key"); c.keystore("xpack.security.transport.ssl.secure_key_passphrase", "test-node-key-password"); c.setting("xpack.security.transport.ssl.certificate", "test-node.crt"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java index 777a6d931e50e..548791b9496c2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java @@ -20,7 +20,6 @@ import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes; import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin; import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.internal.test.ErrorReportingTestListener; import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin; import org.elasticsearch.gradle.plugin.BasePluginBuildPlugin; @@ -58,6 +57,8 @@ import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + /** * Base plugin used for wiring up build tasks to REST testing tasks using new JUnit rule-based test clusters framework. */ @@ -92,6 +93,7 @@ public RestTestBasePlugin(ProviderFactory providerFactory) { public void apply(Project project) { project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class); project.getPluginManager().apply(InternalDistributionDownloadPlugin.class); + var bwcVersions = loadBuildParams(project).get().getBwcVersions(); // Register integ-test and default distributions ElasticsearchDistribution defaultDistro = createDistribution( @@ -176,7 +178,7 @@ public void apply(Project project) { task.systemProperty("tests.system_call_filter", "false"); // Pass minimum wire compatible version which is used by upgrade tests - task.systemProperty(MINIMUM_WIRE_COMPATIBLE_VERSION_SYSPROP, BuildParams.getBwcVersions().getMinimumWireCompatibleVersion()); + task.systemProperty(MINIMUM_WIRE_COMPATIBLE_VERSION_SYSPROP, bwcVersions.getMinimumWireCompatibleVersion()); // Register plugins and modules as task inputs and pass paths as system properties to tests var modulePath = project.getObjects().fileCollection().from(modulesConfiguration); @@ -223,7 +225,7 @@ public Void call(Object... args) { } Version version = (Version) args[0]; - boolean isReleased = BuildParams.getBwcVersions().unreleasedInfo(version) == null; + boolean isReleased = bwcVersions.unreleasedInfo(version) == null; String versionString = version.toString(); ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString); @@ -235,9 +237,9 @@ public Void call(Object... args) { providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath()) ); - if (version.getMajor() > 0 && version.before(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion())) { + if (version.getMajor() > 0 && version.before(bwcVersions.getMinimumWireCompatibleVersion())) { // If we are upgrade testing older versions we also need to upgrade to 7.last - this.call(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion()); + this.call(bwcVersions.getMinimumWireCompatibleVersion()); } return null; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java index fe305b8b46cf7..61dea47eb15c1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java @@ -11,7 +11,8 @@ import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.internal.test.rest.CopyRestApiTask; import org.elasticsearch.gradle.internal.test.rest.CopyRestTestsTask; import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin; @@ -47,6 +48,7 @@ import javax.inject.Inject; import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.setupYamlRestTestDependenciesDefaults; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; /** * Apply this plugin to run the YAML based REST tests from a prior major version against this version's cluster. @@ -74,6 +76,8 @@ public AbstractYamlRestCompatTestPlugin(ProjectLayout projectLayout, FileOperati @Override public void apply(Project project) { + project.getRootProject().getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + BuildParameterExtension buildParams = loadBuildParams(project).get(); final Path compatRestResourcesDir = Path.of("restResources").resolve("compat"); final Path compatSpecsDir = compatRestResourcesDir.resolve("yamlSpecs"); @@ -91,14 +95,14 @@ public void apply(Project project) { GradleUtils.extendSourceSet(project, YamlRestTestPlugin.YAML_REST_TEST, SOURCE_SET_NAME); // determine the previous rest compatibility version and BWC project path - int currentMajor = BuildParams.getBwcVersions().getCurrentVersion().getMajor(); - Version lastMinor = BuildParams.getBwcVersions() + int currentMajor = buildParams.getBwcVersions().getCurrentVersion().getMajor(); + Version lastMinor = buildParams.getBwcVersions() .getUnreleased() .stream() .filter(v -> v.getMajor() == currentMajor - 1) .min(Comparator.reverseOrder()) .get(); - String lastMinorProjectPath = BuildParams.getBwcVersions().unreleasedInfo(lastMinor).gradleProjectPath(); + String lastMinorProjectPath = buildParams.getBwcVersions().unreleasedInfo(lastMinor).gradleProjectPath(); // copy compatible rest specs Configuration bwcMinorConfig = project.getConfigurations().create(BWC_MINOR_CONFIG_NAME); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java index 53fb4c61e151c..a934164d11af6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java @@ -12,7 +12,7 @@ import org.apache.commons.lang.StringUtils; import org.elasticsearch.gradle.Architecture; import org.elasticsearch.gradle.internal.docker.DockerBuildTask; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -20,6 +20,8 @@ import java.util.Arrays; import java.util.List; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + public class TestFixturesDeployPlugin implements Plugin { public static final String DEPLOY_FIXTURE_TASK_NAME = "deployFixtureDockerImages"; @@ -27,13 +29,19 @@ public class TestFixturesDeployPlugin implements Plugin { @Override public void apply(Project project) { + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); NamedDomainObjectContainer fixtures = project.container(TestFixtureDeployment.class); project.getExtensions().add("dockerFixtures", fixtures); - registerDeployTaskPerFixture(project, fixtures); + registerDeployTaskPerFixture(project, fixtures, buildParams.isCi()); project.getTasks().register(DEPLOY_FIXTURE_TASK_NAME, task -> task.dependsOn(project.getTasks().withType(DockerBuildTask.class))); } - private static void registerDeployTaskPerFixture(Project project, NamedDomainObjectContainer fixtures) { + private static void registerDeployTaskPerFixture( + Project project, + NamedDomainObjectContainer fixtures, + boolean isCi + ) { fixtures.all( fixture -> project.getTasks() .register("deploy" + StringUtils.capitalize(fixture.getName()) + "DockerImage", DockerBuildTask.class, task -> { @@ -42,12 +50,12 @@ private static void registerDeployTaskPerFixture(Project project, NamedDomainObj if (baseImages.isEmpty() == false) { task.setBaseImages(baseImages.toArray(new String[baseImages.size()])); } - task.setNoCache(BuildParams.isCi()); + task.setNoCache(isCi); task.setTags( new String[] { resolveTargetDockerRegistry(fixture) + "/" + fixture.getName() + "-fixture:" + fixture.getVersion().get() } ); - task.getPush().set(BuildParams.isCi()); + task.getPush().set(isCi); task.getPlatforms().addAll(Arrays.stream(Architecture.values()).map(a -> a.dockerPlatform).toList()); task.setGroup("Deploy TestFixtures"); task.setDescription("Deploys the " + fixture.getName() + " test fixture"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java index 504b081fd505d..ab28a66d93065 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java @@ -17,7 +17,7 @@ import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin; import org.elasticsearch.gradle.internal.docker.DockerSupportService; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Action; @@ -47,6 +47,8 @@ import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + public class TestFixturesPlugin implements Plugin { private static final Logger LOGGER = Logging.getLogger(TestFixturesPlugin.class); @@ -68,6 +70,8 @@ protected FileSystemOperations getFileSystemOperations() { @Override public void apply(Project project) { project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class); + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); TaskContainer tasks = project.getTasks(); Provider dockerComposeThrottle = project.getGradle() @@ -127,7 +131,7 @@ public void apply(Project project) { tasks.withType(ComposeUp.class).named("composeUp").configure(t -> { // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions - if (BuildParams.isCi()) { + if (buildParams.isCi()) { t.usesService(dockerComposeThrottle); t.usesService(dockerSupport); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java new file mode 100644 index 0000000000000..1b019a6cbd3e6 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.util; + +public class CiUtils { + + static String safeName(String input) { + return input.replaceAll("[^a-zA-Z0-9_\\-\\.]+", " ").trim().replaceAll(" ", "_").toLowerCase(); + } + +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ParamsUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ParamsUtils.java new file mode 100644 index 0000000000000..0afe654bc5fbc --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ParamsUtils.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.util; + +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; +import org.elasticsearch.gradle.internal.info.BuildParameterService; +import org.gradle.api.Project; +import org.gradle.api.provider.Property; +import org.gradle.api.services.BuildServiceRegistration; + +public class ParamsUtils { + + public static Property loadBuildParams(Project project) { + BuildServiceRegistration buildParamsRegistrations = (BuildServiceRegistration< + BuildParameterService, + BuildParameterService.Params>) project.getGradle().getSharedServices().getRegistrations().getByName("buildParams"); + Property buildParams = buildParamsRegistrations.getParameters().getBuildParams(); + return buildParams; + } + +} diff --git a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt index 5388f942be8d7..a9da7995c2b36 100644 --- a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt +++ b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt @@ -119,7 +119,7 @@ java.time.zone.ZoneRules#getStandardOffset(java.time.Instant) java.time.zone.ZoneRules#getDaylightSavings(java.time.Instant) java.time.zone.ZoneRules#isDaylightSavings(java.time.Instant) -@defaultMessage Use logger methods with non-Object parameter +@defaultMessage The first parameter to a log4j log statement should be a String, a log4j Supplier (not java.util.function.Supplier), or another object that log4j supports. org.apache.logging.log4j.Logger#trace(java.lang.Object) org.apache.logging.log4j.Logger#trace(java.lang.Object, java.lang.Throwable) org.apache.logging.log4j.Logger#debug(java.lang.Object) diff --git a/build-tools/build.gradle b/build-tools/build.gradle index 7fd01f0c3d4f7..e457999fedfee 100644 --- a/build-tools/build.gradle +++ b/build-tools/build.gradle @@ -9,9 +9,6 @@ buildscript { repositories { - maven { - url 'https://jitpack.io' - } mavenCentral() } } @@ -117,9 +114,6 @@ configurations { } repositories { - maven { - url 'https://jitpack.io' - } mavenCentral() gradlePluginPortal() } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java index e3adfe8d28148..b3a792b418384 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java @@ -167,7 +167,7 @@ private static CopySpec createBundleSpec( copySpec.exclude("plugin-security.codebases"); }); bundleSpec.from( - (Callable>) () -> project.getPluginManager().hasPlugin("com.github.johnrengelman.shadow") + (Callable>) () -> project.getPluginManager().hasPlugin("com.gradleup.shadow") ? project.getTasks().named("shadowJar") : project.getTasks().named("jar") ); diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index d3d06b2de3575..f3f8e4703eba2 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -168,7 +168,6 @@ abstract class AbstractGradleFuncTest extends Specification { ${extraPlugins.collect { p -> "id '$p'" }.join('\n')} } import org.elasticsearch.gradle.Architecture - import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.BwcVersions import org.elasticsearch.gradle.Version @@ -182,7 +181,7 @@ abstract class AbstractGradleFuncTest extends Specification { ] BwcVersions versions = new BwcVersions(currentVersion, versionList) - BuildParams.init { it.setBwcVersions(provider(() -> versions)) } + buildParams.getBwcVersionsProperty().set(versions) """ } diff --git a/build.gradle b/build.gradle index 71386a37cbb0d..715614c1beea4 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,6 @@ import org.elasticsearch.gradle.DistributionDownloadPlugin import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.BaseInternalPluginBuildPlugin import org.elasticsearch.gradle.internal.ResolveAllDependencies -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.util.GradleUtils import org.gradle.plugins.ide.eclipse.model.AccessRule @@ -28,10 +27,6 @@ import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure buildscript { repositories { - maven { - url 'https://jitpack.io' - } - mavenCentral() } } @@ -143,23 +138,23 @@ tasks.register("updateCIBwcVersions") { } doLast { - writeVersions(file(".ci/bwcVersions"), filterIntermediatePatches(BuildParams.bwcVersions.indexCompatible)) - writeVersions(file(".ci/snapshotBwcVersions"), filterIntermediatePatches(BuildParams.bwcVersions.unreleasedIndexCompatible)) + writeVersions(file(".ci/bwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.indexCompatible)) + writeVersions(file(".ci/snapshotBwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible)) expandBwcList( ".buildkite/pipelines/intake.yml", ".buildkite/pipelines/intake.template.yml", - filterIntermediatePatches(BuildParams.bwcVersions.unreleasedIndexCompatible) + filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible) ) writeBuildkitePipeline( ".buildkite/pipelines/periodic.yml", ".buildkite/pipelines/periodic.template.yml", [ - new ListExpansion(versions: filterIntermediatePatches(BuildParams.bwcVersions.unreleasedIndexCompatible), variable: "BWC_LIST"), + new ListExpansion(versions: filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible), variable: "BWC_LIST"), ], [ new StepExpansion( templatePath: ".buildkite/pipelines/periodic.bwc.template.yml", - versions: filterIntermediatePatches(BuildParams.bwcVersions.indexCompatible), + versions: filterIntermediatePatches(buildParams.bwcVersions.indexCompatible), variable: "BWC_STEPS" ), ] @@ -169,7 +164,7 @@ tasks.register("updateCIBwcVersions") { ".buildkite/pipelines/periodic-packaging.yml", ".buildkite/pipelines/periodic-packaging.template.yml", ".buildkite/pipelines/periodic-packaging.bwc.template.yml", - filterIntermediatePatches(BuildParams.bwcVersions.indexCompatible) + filterIntermediatePatches(buildParams.bwcVersions.indexCompatible) ) } } @@ -191,19 +186,19 @@ tasks.register("verifyVersions") { // Fetch the metadata and parse the xml into Version instances because it's more straight forward here // rather than bwcVersion ( VersionCollection ). new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s -> - BuildParams.bwcVersions.compareToAuthoritative( + buildParams.bwcVersions.compareToAuthoritative( new XmlParser().parse(s) .versioning.versions.version .collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ } .collect { Version.fromString(it) } ) } - verifyCiYaml(file(".ci/bwcVersions"), filterIntermediatePatches(BuildParams.bwcVersions.indexCompatible)) - verifyCiYaml(file(".ci/snapshotBwcVersions"), BuildParams.bwcVersions.unreleasedIndexCompatible) + verifyCiYaml(file(".ci/bwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.indexCompatible)) + verifyCiYaml(file(".ci/snapshotBwcVersions"), buildParams.bwcVersions.unreleasedIndexCompatible) // Make sure backport bot config file is up to date JsonNode backportConfig = new ObjectMapper().readTree(file(".backportrc.json")) - BuildParams.bwcVersions.forPreviousUnreleased { unreleasedVersion -> + buildParams.bwcVersions.forPreviousUnreleased { unreleasedVersion -> boolean valid = backportConfig.get("targetBranchChoices").elements().any { branchChoice -> if (branchChoice.isObject()) { return branchChoice.get("name").textValue() == unreleasedVersion.branch diff --git a/distribution/build.gradle b/distribution/build.gradle index 5b865b36f9e4d..e3481706ef230 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -14,7 +14,6 @@ import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.internal.ConcatFilesTask import org.elasticsearch.gradle.internal.DependenciesInfoPlugin import org.elasticsearch.gradle.internal.NoticeTask -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin import java.nio.file.Files @@ -208,7 +207,7 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each { distro.copyModule(processDefaultOutputsTaskProvider, module) dependencies.add('featuresMetadata', module) - if (module.name.startsWith('transport-') || (BuildParams.snapshotBuild == false && module.name == 'apm')) { + if (module.name.startsWith('transport-') || (buildParams.snapshotBuild == false && module.name == 'apm')) { distro.copyModule(processIntegTestOutputsTaskProvider, module) } @@ -378,7 +377,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { exclude "**/platform/${excludePlatform}/**" } } - if (BuildParams.isSnapshotBuild()) { + if (buildParams.isSnapshotBuild()) { from(buildExternalTestModulesTaskProvider) } if (project.path.startsWith(':distribution:packages')) { diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 788e836f8f045..d73f9c395f15c 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -8,7 +8,6 @@ import org.elasticsearch.gradle.internal.docker.DockerSupportService import org.elasticsearch.gradle.internal.docker.ShellRetry import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter import org.elasticsearch.gradle.internal.docker.* -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.util.GradleUtils import org.elasticsearch.gradle.Architecture import java.nio.file.Path @@ -120,7 +119,7 @@ ext.expansions = { Architecture architecture, DockerBase base -> // the image. When developing the Docker images, it's very tedious to completely rebuild // an image for every single change. Therefore, outside of CI, we fix the // build time to midnight so that the Docker build cache is usable. - def buildDate = BuildParams.isCi() ? BuildParams.buildDate : BuildParams.buildDate.truncatedTo(ChronoUnit.DAYS).toString() + def buildDate = buildParams.isCi() ? buildParams.buildDate : buildParams.buildDate.truncatedTo(ChronoUnit.DAYS).toString() return [ 'arch' : architecture.classifier, @@ -128,7 +127,7 @@ ext.expansions = { Architecture architecture, DockerBase base -> 'bin_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'bin', 'build_date' : buildDate, 'config_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'config', - 'git_revision' : BuildParams.gitRevision, + 'git_revision' : buildParams.gitRevision, 'license' : base == DockerBase.IRON_BANK ? 'Elastic License 2.0' : 'Elastic-License-2.0', 'package_manager' : base.packageManager, 'docker_base' : base.name().toLowerCase(), @@ -390,7 +389,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) { dockerContext.fileProvider(transformTask.map { Sync task -> task.getDestinationDir() }) - noCache = BuildParams.isCi() + noCache = buildParams.isCi() tags = generateTags(base, architecture) platforms.add(architecture.dockerPlatform) @@ -485,7 +484,7 @@ void addBuildEssDockerImageTask(Architecture architecture) { dockerContext.fileProvider(buildContextTask.map { it.getDestinationDir() }) - noCache = BuildParams.isCi() + noCache = buildParams.isCi() baseImages = [] tags = generateTags(dockerBase, architecture) platforms.add(architecture.dockerPlatform) diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index e08f16c14ab88..918980fea616a 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -301,7 +301,7 @@ ospackage { url 'https://www.elastic.co/' // signing setup - if (project.hasProperty('signing.password') && BuildParams.isSnapshotBuild() == false) { + if (project.hasProperty('signing.password') && buildParams.isSnapshotBuild() == false) { signingKeyId = project.hasProperty('signing.keyId') ? project.property('signing.keyId') : 'D88E42B4' signingKeyPassphrase = project.property('signing.password') signingKeyRingFile = project.hasProperty('signing.secretKeyRingFile') ? diff --git a/docs/build.gradle b/docs/build.gradle index e495ecacce27b..dec0de8ffa844 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -1,5 +1,4 @@ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.doc.DocSnippetTask import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT @@ -29,7 +28,7 @@ ext.docsFileTree = fileTree(projectDir) { // These files simply don't pass yet. We should figure out how to fix them. exclude 'reference/watcher/reference/actions.asciidoc' exclude 'reference/rest-api/security/ssl.asciidoc' - if (BuildParams.inFipsJvm) { + if (buildParams.inFipsJvm) { // We don't support this component in FIPS 140 exclude 'reference/ingest/processors/attachment.asciidoc' // We can't conditionally control output, this would be missing the ingest-attachment component @@ -38,7 +37,7 @@ ext.docsFileTree = fileTree(projectDir) { } tasks.named("yamlRestTest") { - if (BuildParams.isSnapshotBuild() == false) { + if (buildParams.isSnapshotBuild() == false) { // LOOKUP is not available in snapshots systemProperty 'tests.rest.blacklist', [ "reference/esql/processing-commands/lookup/esql-lookup-example" @@ -83,7 +82,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' keystorePassword 'keystore-password' - if (BuildParams.isSnapshotBuild() == false) { + if (buildParams.isSnapshotBuild() == false) { requiresFeature 'es.failure_store_feature_flag_enabled', new Version(8, 12, 0) } } @@ -170,7 +169,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach { return } // Do not install ingest-attachment in a FIPS 140 JVM as this is not supported - if (subproj.path.startsWith(':modules:ingest-attachment') && BuildParams.inFipsJvm) { + if (subproj.path.startsWith(':modules:ingest-attachment') && buildParams.inFipsJvm) { return } plugin subproj.path diff --git a/docs/changelog/115142.yaml b/docs/changelog/115142.yaml new file mode 100644 index 0000000000000..2af968ae156da --- /dev/null +++ b/docs/changelog/115142.yaml @@ -0,0 +1,6 @@ +pr: 115142 +summary: Attempt to clean up index before remote transfer +area: Recovery +type: enhancement +issues: + - 104473 diff --git a/docs/changelog/115678.yaml b/docs/changelog/115678.yaml new file mode 100644 index 0000000000000..31240eae1ebb4 --- /dev/null +++ b/docs/changelog/115678.yaml @@ -0,0 +1,5 @@ +pr: 115678 +summary: "ESQL: extract common filter from aggs" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/115930.yaml b/docs/changelog/115930.yaml new file mode 100644 index 0000000000000..788a01b5cac96 --- /dev/null +++ b/docs/changelog/115930.yaml @@ -0,0 +1,5 @@ +pr: 115930 +summary: Inconsistency in the _analyzer api when the index is not included +area: Search +type: bug +issues: [] diff --git a/docs/changelog/116339.yaml b/docs/changelog/116339.yaml new file mode 100644 index 0000000000000..1767183271812 --- /dev/null +++ b/docs/changelog/116339.yaml @@ -0,0 +1,5 @@ +pr: 116339 +summary: "Index stats enhancement: creation date and `tier_preference`" +area: Stats +type: feature +issues: [] diff --git a/docs/reference/connector/docs/sync-rules.asciidoc b/docs/reference/connector/docs/sync-rules.asciidoc index 9b2a77be7db03..3ab72093666b8 100644 --- a/docs/reference/connector/docs/sync-rules.asciidoc +++ b/docs/reference/connector/docs/sync-rules.asciidoc @@ -116,6 +116,12 @@ A "match" is determined based on a condition defined by a combination of "field" The `Field` column should be used to define which field on a given document should be considered. +[NOTE] +==== +Only top-level fields are supported. +Nested/object fields cannot be referenced with "dot notation". +==== + The following rules are available in the `Rule` column: * `equals` - The field value is equal to the specified value. diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 44f90eded8632..4c16f260c13e7 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -117,12 +117,10 @@ that sacrifices result accuracy for improved speed. The `dense_vector` type supports quantization to reduce the memory footprint required when <> `float` vectors. The three following quantization strategies are supported: -+ --- -`int8` - Quantizes each dimension of the vector to 1-byte integers. This reduces the memory footprint by 75% (or 4x) at the cost of some accuracy. -`int4` - Quantizes each dimension of the vector to half-byte integers. This reduces the memory footprint by 87% (or 8x) at the cost of accuracy. -`bbq` - experimental:[] Better binary quantization which reduces each dimension to a single bit precision. This reduces the memory footprint by 96% (or 32x) at a larger cost of accuracy. Generally, oversampling during query time and reranking can help mitigate the accuracy loss. --- +* `int8` - Quantizes each dimension of the vector to 1-byte integers. This reduces the memory footprint by 75% (or 4x) at the cost of some accuracy. +* `int4` - Quantizes each dimension of the vector to half-byte integers. This reduces the memory footprint by 87% (or 8x) at the cost of accuracy. +* `bbq` - experimental:[] Better binary quantization which reduces each dimension to a single bit precision. This reduces the memory footprint by 96% (or 32x) at a larger cost of accuracy. Generally, oversampling during query time and reranking can help mitigate the accuracy loss. + When using a quantized format, you may want to oversample and rescore the results to improve accuracy. See <> for more information. @@ -245,12 +243,11 @@ their vector field's similarity to the query vector. The `_score` of each document will be derived from the similarity, in a way that ensures scores are positive and that a larger score corresponds to a higher ranking. Defaults to `l2_norm` when `element_type: bit` otherwise defaults to `cosine`. - -NOTE: `bit` vectors only support `l2_norm` as their similarity metric. - + ^*^ This parameter can only be specified when `index` is `true`. + +NOTE: `bit` vectors only support `l2_norm` as their similarity metric. + .Valid values for `similarity` [%collapsible%open] ==== diff --git a/docs/reference/searchable-snapshots/index.asciidoc b/docs/reference/searchable-snapshots/index.asciidoc index a38971a0bae6a..8b0b3dc57686e 100644 --- a/docs/reference/searchable-snapshots/index.asciidoc +++ b/docs/reference/searchable-snapshots/index.asciidoc @@ -176,10 +176,10 @@ nodes that have a shared cache. ==== Manually mounting snapshots captured by an Index Lifecycle Management ({ilm-init}) policy can interfere with {ilm-init}'s automatic management. This may lead to issues such as data loss -or complications with snapshot handling. +or complications with snapshot handling. For optimal results, allow {ilm-init} to manage -snapshots automatically. +snapshots automatically. <>. ==== @@ -293,6 +293,14 @@ repository. If you wish to search data across multiple regions, configure multiple clusters and use <> or <> instead of {search-snaps}. +It's worth noting that if a searchable snapshot index has no replicas, then when the node +hosting it is shut down, allocation will immediately try to relocate the index to a new node +in order to maximize availability. For fully mounted indices this will result in the new node +downloading the entire index snapshot from the cloud repository. Under a rolling cluster restart, +this may happen multiple times for each searchable snapshot index. Temporarily +disabling allocation during planned node restart will prevent this, as described in +the <>. + [discrete] [[back-up-restore-searchable-snapshots]] === Back up and restore {search-snaps} diff --git a/docs/reference/snapshot-restore/repository-azure.asciidoc b/docs/reference/snapshot-restore/repository-azure.asciidoc index 0e6e1478cfc55..50dc42ac9163d 100644 --- a/docs/reference/snapshot-restore/repository-azure.asciidoc +++ b/docs/reference/snapshot-restore/repository-azure.asciidoc @@ -181,7 +181,7 @@ is running. When running {es} in https://azure.microsoft.com/en-gb/products/kubernetes-service[Azure Kubernetes -Service], for instance using {eck-ref}[{eck}], you should use +Service], for instance using {eck-ref}/k8s-snapshots.html#k8s-azure-workload-identity[{eck}], you should use https://azure.github.io/azure-workload-identity/docs/introduction.html[Azure Workload Identity] to provide credentials to {es}. To use Azure Workload Identity, mount the `azure-identity-token` volume as a subdirectory of the diff --git a/gradle.properties b/gradle.properties index 745fb4f9e51ae..aa38a61ab0057 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,7 +2,7 @@ org.gradle.welcome=never org.gradle.warning.mode=none org.gradle.parallel=true # We need to declare --add-exports to make spotless working seamlessly with jdk16 -org.gradle.jvmargs=-XX:+HeapDumpOnOutOfMemoryError -Xss2m --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED +org.gradle.jvmargs=-XX:+HeapDumpOnOutOfMemoryError -Xss2m --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-opens java.base/java.time=ALL-UNNAMED # Enforce the build to fail on deprecated gradle api usage systemProp.org.gradle.warning.mode=fail diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml index d11c4b7fd9c91..e3148c6f3ef2e 100644 --- a/gradle/build.versions.toml +++ b/gradle/build.versions.toml @@ -39,7 +39,7 @@ maven-model = "org.apache.maven:maven-model:3.6.2" mockito-core = "org.mockito:mockito-core:1.9.5" nebula-info = "com.netflix.nebula:gradle-info-plugin:11.3.3" reflections = "org.reflections:reflections:0.9.12" -shadow-plugin = "com.github.breskeby:shadow:3b035f2" +shadow-plugin = "com.gradleup.shadow:shadow-gradle-plugin:8.3.5" snakeyaml = { group = "org.yaml", name = "snakeyaml", version = { strictly = "2.0" } } spock-core = { group = "org.spockframework", name="spock-core", version.ref="spock" } spock-junit4 = { group = "org.spockframework", name="spock-junit4", version.ref="spock" } diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 2b8f1b2a09ad9..5e874b52fc4c6 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -219,6 +219,11 @@ + + + + + @@ -234,16 +239,31 @@ + + + + + + + + + + + + + + + @@ -414,21 +434,16 @@ + + + + + - - - - - - - - - - @@ -614,6 +629,11 @@ + + + + + @@ -799,6 +819,11 @@ + + + + + @@ -1196,6 +1221,11 @@ + + + + + @@ -1236,6 +1266,11 @@ + + + + + @@ -1311,6 +1346,11 @@ + + + + + @@ -1922,6 +1962,11 @@ + + + + + @@ -1937,6 +1982,11 @@ + + + + + @@ -2282,6 +2332,11 @@ + + + + + @@ -2810,6 +2865,11 @@ + + + + + @@ -2830,6 +2890,11 @@ + + + + + @@ -2965,6 +3030,16 @@ + + + + + + + + + + @@ -3020,6 +3095,11 @@ + + + + + @@ -3393,6 +3473,16 @@ + + + + + + + + + + @@ -3423,11 +3513,21 @@ + + + + + + + + + + @@ -3533,6 +3633,11 @@ + + + + + @@ -3548,66 +3653,131 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -3618,11 +3788,21 @@ + + + + + + + + + + @@ -3653,6 +3833,11 @@ + + + + + @@ -4153,6 +4338,11 @@ + + + + + @@ -4198,6 +4388,11 @@ + + + + + @@ -4238,6 +4433,11 @@ + + + + + @@ -4423,6 +4623,11 @@ + + + + + diff --git a/libs/simdvec/build.gradle b/libs/simdvec/build.gradle index 02f960130e690..ffc50ecb1f6ff 100644 --- a/libs/simdvec/build.gradle +++ b/libs/simdvec/build.gradle @@ -33,7 +33,7 @@ tasks.matching { it.name == "compileMain21Java" }.configureEach { } tasks.named('test').configure { - if (BuildParams.getRuntimeJavaVersion().majorVersion.toInteger() >= 21) { + if (buildParams.getRuntimeJavaVersion().map{ it.majorVersion.toInteger() }.get() >= 21) { jvmArgs '--add-modules=jdk.incubator.vector' } } diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle index a1ab6363166cb..5df0a890af753 100644 --- a/modules/aggregations/build.gradle +++ b/modules/aggregations/build.gradle @@ -28,7 +28,7 @@ restResources { } } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index d5ce1bfc8d93d..b6fc1e3722ccd 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -28,14 +28,14 @@ tasks.withType(StandaloneRestIntegTestTask).configureEach { usesDefaultDistribution() } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // These fail in CI but only when run as part of checkPart2 and not individually. // Tracked in : tasks.named("javaRestTest").configure{enabled = false } tasks.named("yamlRestTest").configure{enabled = false } } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.withType(Test).configureEach { systemProperty 'es.failure_store_feature_flag_enabled', 'true' } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/FailureStoreMetricsWithIncrementalBulkIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/FailureStoreMetricsWithIncrementalBulkIT.java new file mode 100644 index 0000000000000..2c9b7417b2832 --- /dev/null +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/FailureStoreMetricsWithIncrementalBulkIT.java @@ -0,0 +1,251 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.bulk.FailureStoreMetrics; +import org.elasticsearch.action.bulk.IncrementalBulkService; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.datastreams.GetDataStreamAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.AbstractRefCounted; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Strings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexingPressure; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.TestTelemetryPlugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +public class FailureStoreMetricsWithIncrementalBulkIT extends ESIntegTestCase { + + private static final List METRICS = List.of( + FailureStoreMetrics.METRIC_TOTAL, + FailureStoreMetrics.METRIC_FAILURE_STORE, + FailureStoreMetrics.METRIC_REJECTED + ); + + private static final String DATA_STREAM_NAME = "data-stream-incremental"; + + @Override + protected Collection> nodePlugins() { + return List.of(DataStreamsPlugin.class, TestTelemetryPlugin.class, MapperExtrasPlugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK.getKey(), "512B") + .put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK_SIZE.getKey(), "2048B") + .put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK.getKey(), "2KB") + .put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK_SIZE.getKey(), "1024B") + .build(); + } + + public void testShortCircuitFailure() throws Exception { + createDataStreamWithFailureStore(); + + String coordinatingOnlyNode = internalCluster().startCoordinatingOnlyNode(Settings.EMPTY); + + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, coordinatingOnlyNode); + try (IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest()) { + + AtomicBoolean nextRequested = new AtomicBoolean(true); + int successfullyStored = 0; + while (nextRequested.get()) { + nextRequested.set(false); + refCounted.incRef(); + handler.addItems(List.of(indexRequest(DATA_STREAM_NAME)), refCounted::decRef, () -> nextRequested.set(true)); + successfullyStored++; + } + assertBusy(() -> assertTrue(nextRequested.get())); + var metrics = collectTelemetry(); + assertDataStreamMetric(metrics, FailureStoreMetrics.METRIC_TOTAL, DATA_STREAM_NAME, successfullyStored); + assertDataStreamMetric(metrics, FailureStoreMetrics.METRIC_FAILURE_STORE, DATA_STREAM_NAME, 0); + assertDataStreamMetric(metrics, FailureStoreMetrics.METRIC_REJECTED, DATA_STREAM_NAME, 0); + + // Introduce artificial pressure that will reject the following requests + String node = findNodeOfPrimaryShard(DATA_STREAM_NAME); + IndexingPressure primaryPressure = internalCluster().getInstance(IndexingPressure.class, node); + long memoryLimit = primaryPressure.stats().getMemoryLimit(); + long primaryRejections = primaryPressure.stats().getPrimaryRejections(); + try (Releasable ignored = primaryPressure.markPrimaryOperationStarted(10, memoryLimit, false)) { + while (primaryPressure.stats().getPrimaryRejections() == primaryRejections) { + while (nextRequested.get()) { + nextRequested.set(false); + refCounted.incRef(); + List> requests = new ArrayList<>(); + for (int i = 0; i < 20; ++i) { + requests.add(indexRequest(DATA_STREAM_NAME)); + } + handler.addItems(requests, refCounted::decRef, () -> nextRequested.set(true)); + } + assertBusy(() -> assertTrue(nextRequested.get())); + } + } + + while (nextRequested.get()) { + nextRequested.set(false); + refCounted.incRef(); + handler.addItems(List.of(indexRequest(DATA_STREAM_NAME)), refCounted::decRef, () -> nextRequested.set(true)); + } + + assertBusy(() -> assertTrue(nextRequested.get())); + + PlainActionFuture future = new PlainActionFuture<>(); + handler.lastItems(List.of(indexRequest(DATA_STREAM_NAME)), refCounted::decRef, future); + + BulkResponse bulkResponse = safeGet(future); + + for (int i = 0; i < bulkResponse.getItems().length; ++i) { + // the first requests were successful + boolean hasFailed = i >= successfullyStored; + assertThat(bulkResponse.getItems()[i].isFailed(), is(hasFailed)); + assertThat(bulkResponse.getItems()[i].getFailureStoreStatus(), is(IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN)); + } + + metrics = collectTelemetry(); + assertDataStreamMetric(metrics, FailureStoreMetrics.METRIC_TOTAL, DATA_STREAM_NAME, bulkResponse.getItems().length); + assertDataStreamMetric( + metrics, + FailureStoreMetrics.METRIC_REJECTED, + DATA_STREAM_NAME, + bulkResponse.getItems().length - successfullyStored + ); + assertDataStreamMetric(metrics, FailureStoreMetrics.METRIC_FAILURE_STORE, DATA_STREAM_NAME, 0); + } + } + + private void createDataStreamWithFailureStore() throws IOException { + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request( + "template-incremental" + ); + request.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(DATA_STREAM_NAME + "*")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, true)) + .template(new Template(null, new CompressedXContent(""" + { + "dynamic": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "count": { + "type": "long" + } + } + }"""), null)) + .build() + ); + assertAcked(safeGet(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request))); + + final var createDataStreamRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + DATA_STREAM_NAME + ); + assertAcked(safeGet(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest))); + } + + private static Map> collectTelemetry() { + Map> measurements = new HashMap<>(); + for (PluginsService pluginsService : internalCluster().getInstances(PluginsService.class)) { + final TestTelemetryPlugin telemetryPlugin = pluginsService.filterPlugins(TestTelemetryPlugin.class).findFirst().orElseThrow(); + + telemetryPlugin.collect(); + + for (String metricName : METRICS) { + measurements.put(metricName, telemetryPlugin.getLongCounterMeasurement(metricName)); + } + } + return measurements; + } + + private void assertDataStreamMetric(Map> metrics, String metric, String dataStreamName, int expectedValue) { + List measurements = metrics.get(metric); + assertThat(measurements, notNullValue()); + long totalValue = measurements.stream() + .filter(m -> m.attributes().get("data_stream").equals(dataStreamName)) + .mapToLong(Measurement::getLong) + .sum(); + assertThat(totalValue, equalTo((long) expectedValue)); + } + + private static IndexRequest indexRequest(String dataStreamName) { + String time = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); + String value = "1"; + return new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE) + .source(Strings.format("{\"%s\":\"%s\", \"count\": %s}", DEFAULT_TIMESTAMP_FIELD, time, value), XContentType.JSON); + } + + protected static String findNodeOfPrimaryShard(String dataStreamName) { + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + new String[] { dataStreamName } + ); + GetDataStreamAction.Response getDataStreamResponse = safeGet(client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)); + assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); + DataStream dataStream = getDataStreamResponse.getDataStreams().getFirst().getDataStream(); + assertThat(dataStream.getName(), equalTo(DATA_STREAM_NAME)); + assertThat(dataStream.getIndices().size(), equalTo(1)); + String backingIndex = dataStream.getIndices().getFirst().getName(); + assertThat(backingIndex, backingIndexEqualTo(DATA_STREAM_NAME, 1)); + + Index index = resolveIndex(backingIndex); + int shardId = 0; + for (String node : internalCluster().getNodeNames()) { + var indicesService = internalCluster().getInstance(IndicesService.class, node); + IndexService indexService = indicesService.indexService(index); + if (indexService != null) { + IndexShard shard = indexService.getShardOrNull(shardId); + if (shard != null && shard.isActive() && shard.routingEntry().primary()) { + return node; + } + } + } + throw new AssertionError("IndexShard instance not found for shard " + new ShardId(index, shardId)); + } +} diff --git a/modules/ingest-attachment/build.gradle b/modules/ingest-attachment/build.gradle index 92e843fa31a63..821de8f834a44 100644 --- a/modules/ingest-attachment/build.gradle +++ b/modules/ingest-attachment/build.gradle @@ -143,7 +143,7 @@ tasks.named("thirdPartyAudit").configure { ignoreMissingClasses() } -if (BuildParams.inFipsJvm) { +if (buildParams.inFipsJvm) { tasks.named("test").configure { enabled = false } tasks.named("yamlRestTest").configure { enabled = false }; tasks.named("yamlRestCompatTest").configure { enabled = false }; diff --git a/modules/ingest-geoip/qa/full-cluster-restart/build.gradle b/modules/ingest-geoip/qa/full-cluster-restart/build.gradle index b51fa497c8492..8e7d20108a869 100644 --- a/modules/ingest-geoip/qa/full-cluster-restart/build.gradle +++ b/modules/ingest-geoip/qa/full-cluster-restart/build.gradle @@ -24,7 +24,7 @@ dependencies { // once we are ready to test migrations from 8.x to 9.x, we can set the compatible version to 8.0.0 // see https://github.com/elastic/elasticsearch/pull/93666 -BuildParams.bwcVersions.withWireCompatible(v -> v.before("9.0.0")) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(v -> v.before("9.0.0")) { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt index a739635e85a9c..875b9a1dac3e8 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt @@ -132,6 +132,21 @@ class org.elasticsearch.script.field.SeqNoDocValuesField @dynamic_type { class org.elasticsearch.script.field.VersionDocValuesField @dynamic_type { } +class org.elasticsearch.script.field.vectors.MultiDenseVector { + MultiDenseVector EMPTY + float[] getMagnitudes() + + Iterator getVectors() + boolean isEmpty() + int getDims() + int size() +} + +class org.elasticsearch.script.field.vectors.MultiDenseVectorDocValuesField { + MultiDenseVector get() + MultiDenseVector get(MultiDenseVector) +} + class org.elasticsearch.script.field.vectors.DenseVector { DenseVector EMPTY float getMagnitude() diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt index 7ab9eb32852b6..b2db0d1006d40 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt @@ -123,6 +123,11 @@ class org.elasticsearch.index.mapper.vectors.DenseVectorScriptDocValues { float getMagnitude() } +class org.elasticsearch.index.mapper.vectors.MultiDenseVectorScriptDocValues { + Iterator getVectorValues() + float[] getMagnitudes() +} + class org.apache.lucene.util.BytesRef { byte[] bytes int offset diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_multi_dense_vector_dv_fields_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_multi_dense_vector_dv_fields_api.yml new file mode 100644 index 0000000000000..66cb3f3c46fcc --- /dev/null +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_multi_dense_vector_dv_fields_api.yml @@ -0,0 +1,178 @@ +setup: + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ multi_dense_vector_script_access ] + test_runner_features: capabilities + reason: "Support for multi dense vector field script access capability required" + - skip: + features: headers + + - do: + indices.create: + index: test-index + body: + settings: + number_of_shards: 1 + mappings: + properties: + vector: + type: multi_dense_vector + dims: 5 + byte_vector: + type: multi_dense_vector + dims: 5 + element_type: byte + bit_vector: + type: multi_dense_vector + dims: 40 + element_type: bit + - do: + index: + index: test-index + id: "1" + body: + vector: [[230.0, 300.33, -34.8988, 15.555, -200.0], [-0.5, 100.0, -13, 14.8, -156.0]] + byte_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]] + bit_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]] + + - do: + index: + index: test-index + id: "3" + body: + vector: [[0.5, 111.3, -13.0, 14.8, -156.0]] + byte_vector: [[2, 18, -5, 0, -124]] + bit_vector: [[2, 18, -5, 0, -124]] + + - do: + indices.refresh: {} +--- +"Test vector magnitude equality": + - skip: + features: close_to + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "doc['vector'].magnitudes[0]" + + - match: {hits.total: 2} + + - match: {hits.hits.0._id: "1"} + - close_to: {hits.hits.0._score: {value: 429.6021, error: 0.01}} + + - match: {hits.hits.1._id: "3"} + - close_to: {hits.hits.1._score: {value: 192.6447, error: 0.01}} + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "doc['byte_vector'].magnitudes[0]" + + - match: {hits.total: 2} + + - match: {hits.hits.0._id: "3"} + - close_to: {hits.hits.0._score: {value: 125.41531, error: 0.01}} + + - match: {hits.hits.1._id: "1"} + - close_to: {hits.hits.1._score: {value: 19.07878, error: 0.01}} + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "doc['bit_vector'].magnitudes[0]" + + - match: {hits.total: 2} + + - match: {hits.hits.0._id: "1"} + - close_to: {hits.hits.0._score: {value: 3.872983, error: 0.01}} + + - match: {hits.hits.1._id: "3"} + - close_to: {hits.hits.1._score: {value: 3.464101, error: 0.01}} +--- +"Test vector value scoring": + - skip: + features: close_to + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "doc['vector'].vectorValues.next()[0];" + + - match: {hits.total: 2} + + - match: {hits.hits.0._id: "1"} + - close_to: {hits.hits.0._score: {value: 230, error: 0.01}} + + - match: {hits.hits.1._id: "3"} + - close_to: {hits.hits.1._score: {value: 0.5, error: 0.01}} + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "doc['byte_vector'].vectorValues.next()[0];" + + - match: {hits.total: 2} + + - match: {hits.hits.0._id: "1"} + - close_to: {hits.hits.0._score: {value: 8, error: 0.01}} + + - match: {hits.hits.1._id: "3"} + - close_to: {hits.hits.1._score: {value: 2, error: 0.01}} + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "doc['bit_vector'].vectorValues.next()[0];" + + - match: {hits.total: 2} + + - match: {hits.hits.0._id: "1"} + - close_to: {hits.hits.0._score: {value: 8, error: 0.01}} + + - match: {hits.hits.1._id: "3"} + - close_to: {hits.hits.1._score: {value: 2, error: 0.01}} diff --git a/modules/legacy-geo/build.gradle b/modules/legacy-geo/build.gradle index d936276362340..1b4fd9d52bbaf 100644 --- a/modules/legacy-geo/build.gradle +++ b/modules/legacy-geo/build.gradle @@ -26,7 +26,7 @@ dependencies { testImplementation project(":test:framework") } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/modules/mapper-extras/build.gradle b/modules/mapper-extras/build.gradle index 35842ad27643f..a7bdc11e15550 100644 --- a/modules/mapper-extras/build.gradle +++ b/modules/mapper-extras/build.gradle @@ -24,7 +24,7 @@ restResources { } } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 14a6b1e3f5b82..bb1500ba55664 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -132,7 +132,7 @@ if (OS.current() == OS.WINDOWS) { oldEsDependency.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); TaskProvider fixture = tasks.register("oldEs${version}Fixture", AntFixture) { dependsOn project.configurations.oldesFixture, jdks.legacy, oldEsDependency - executable = "${BuildParams.runtimeJavaHome}/bin/java" + executable = "${buildParams.runtimeJavaHome.get()}/bin/java" env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}" // old versions of Elasticsearch need JAVA_HOME env 'JAVA_HOME', jdks.legacy.javaHomePath diff --git a/modules/repository-azure/build.gradle b/modules/repository-azure/build.gradle index 86776e743685e..4babac68f1e71 100644 --- a/modules/repository-azure/build.gradle +++ b/modules/repository-azure/build.gradle @@ -321,7 +321,7 @@ tasks.register("workloadIdentityYamlRestTest", RestIntegTestTask) { // omitting key and sas_token so that we use a bearer token from workload identity } -if (BuildParams.inFipsJvm) { +if (buildParams.inFipsJvm) { // Cannot override the trust store in FIPS mode, and these tasks require a HTTPS fixture tasks.named("managedIdentityYamlRestTest").configure { enabled = false } tasks.named("workloadIdentityYamlRestTest").configure { enabled = false } diff --git a/modules/repository-gcs/build.gradle b/modules/repository-gcs/build.gradle index 246611e4803a2..605d886a71056 100644 --- a/modules/repository-gcs/build.gradle +++ b/modules/repository-gcs/build.gradle @@ -178,7 +178,7 @@ tasks.named("thirdPartyAudit").configure { ) - if(BuildParams.graalVmRuntime == false) { + if(buildParams.graalVmRuntime == false) { ignoreMissingClasses( 'org.graalvm.nativeimage.hosted.Feature', 'org.graalvm.nativeimage.hosted.Feature$BeforeAnalysisAccess', @@ -240,7 +240,7 @@ def gcsThirdPartyTest = tasks.register("gcsThirdPartyUnitTest", Test) { systemProperty 'tests.security.manager', false systemProperty 'test.google.bucket', gcsBucket systemProperty 'test.google.fixture', Boolean.toString(useFixture) - nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + buildParams.testSeed if (useFixture == false) { nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}" } diff --git a/modules/repository-s3/build.gradle b/modules/repository-s3/build.gradle index 59dfa6b9aace2..c1cd1a13719a7 100644 --- a/modules/repository-s3/build.gradle +++ b/modules/repository-s3/build.gradle @@ -115,7 +115,7 @@ String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs") String s3STSBucket = System.getenv("amazon_s3_bucket_sts") String s3STSBasePath = System.getenv("amazon_s3_base_path_sts") -boolean s3DisableChunkedEncoding = BuildParams.random.nextBoolean() +boolean s3DisableChunkedEncoding = buildParams.random.nextBoolean() // If all these variables are missing then we are testing against the internal fixture instead, which has the following // credentials hard-coded in. @@ -203,7 +203,7 @@ tasks.register("s3ThirdPartyTest", Test) { systemProperty 'test.s3.account', s3PermanentAccessKey systemProperty 'test.s3.key', s3PermanentSecretKey systemProperty 'test.s3.bucket', s3PermanentBucket - nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + buildParams.testSeed } tasks.named("thirdPartyAudit").configure { diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java index 1e84f65cdd842..3095139ca4685 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java @@ -174,7 +174,7 @@ public void testClientConnectionCloseMidStream() throws Exception { // await stream handler is ready and request full content var handler = ctx.awaitRestChannelAccepted(opaqueId); - assertBusy(() -> assertNotNull(handler.stream.buf())); + assertBusy(() -> assertNotEquals(0, handler.stream.bufSize())); assertFalse(handler.streamClosed); @@ -187,7 +187,7 @@ public void testClientConnectionCloseMidStream() throws Exception { // wait for resources to be released assertBusy(() -> { - assertNull(handler.stream.buf()); + assertEquals(0, handler.stream.bufSize()); assertTrue(handler.streamClosed); }); } @@ -204,15 +204,13 @@ public void testServerCloseConnectionMidStream() throws Exception { // await stream handler is ready and request full content var handler = ctx.awaitRestChannelAccepted(opaqueId); - assertBusy(() -> assertNotNull(handler.stream.buf())); + assertBusy(() -> assertNotEquals(0, handler.stream.bufSize())); assertFalse(handler.streamClosed); // terminate connection on server and wait resources are released handler.channel.request().getHttpChannel().close(); assertBusy(() -> { - // Cannot be simplified to assertNull. - // assertNull requires object to not fail on toString() method, but closing buffer can - assertTrue(handler.stream.buf() == null); + assertEquals(0, handler.stream.bufSize()); assertTrue(handler.streamClosed); }); } @@ -228,14 +226,14 @@ public void testServerExceptionMidStream() throws Exception { // await stream handler is ready and request full content var handler = ctx.awaitRestChannelAccepted(opaqueId); - assertBusy(() -> assertNotNull(handler.stream.buf())); + assertBusy(() -> assertNotEquals(0, handler.stream.bufSize())); assertFalse(handler.streamClosed); handler.shouldThrowInsideHandleChunk = true; handler.stream.next(); assertBusy(() -> { - assertNull(handler.stream.buf()); + assertEquals(0, handler.stream.bufSize()); assertTrue(handler.streamClosed); }); } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java index 238faa7a9237e..ac3e3aecf97b9 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java @@ -37,12 +37,15 @@ public class Netty4HttpRequestBodyStream implements HttpBody.Stream { private final List tracingHandlers = new ArrayList<>(4); private final ThreadContext threadContext; private ByteBuf buf; - private boolean hasLast = false; private boolean requested = false; private boolean closing = false; private HttpBody.ChunkHandler handler; private ThreadContext.StoredContext requestContext; + // used in tests + private volatile int bufSize = 0; + private volatile boolean hasLast = false; + public Netty4HttpRequestBodyStream(Channel channel, ThreadContext threadContext) { this.channel = channel; this.threadContext = threadContext; @@ -112,11 +115,12 @@ private void addChunk(ByteBuf chunk) { comp.addComponent(true, chunk); buf = comp; } + bufSize = buf.readableBytes(); } // visible for test - ByteBuf buf() { - return buf; + int bufSize() { + return bufSize; } // visible for test @@ -130,6 +134,7 @@ private void send() { var bytesRef = Netty4Utils.toReleasableBytesReference(buf); requested = false; buf = null; + bufSize = 0; try (var ignored = threadContext.restoreExistingContext(requestContext)) { for (var tracer : tracingHandlers) { tracer.onNext(bytesRef, hasLast); @@ -164,6 +169,7 @@ private void doClose() { if (buf != null) { buf.release(); buf = null; + bufSize = 0; } channel.config().setAutoRead(true); } diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java index 5ff5a27e2d551..d456bbecfbd20 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java @@ -67,7 +67,7 @@ public void testEnqueueChunksBeforeRequest() { for (int i = 0; i < totalChunks; i++) { channel.writeInbound(randomContent(1024)); } - assertEquals(totalChunks * 1024, stream.buf().readableBytes()); + assertEquals(totalChunks * 1024, stream.bufSize()); } // ensures all received chunks can be flushed downstream @@ -119,7 +119,7 @@ public void testReadFromChannel() { channel.writeInbound(randomLastContent(chunkSize)); for (int i = 0; i < totalChunks; i++) { - assertNull("should not enqueue chunks", stream.buf()); + assertEquals("should not enqueue chunks", 0, stream.bufSize()); stream.next(); channel.runPendingTasks(); assertEquals("each next() should produce single chunk", i + 1, gotChunks.size()); diff --git a/muted-tests.yml b/muted-tests.yml index a9dda9ac80782..625813642eb60 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -11,9 +11,6 @@ tests: - class: org.elasticsearch.xpack.restart.FullClusterRestartIT method: testDataStreams {cluster=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/111448 -- class: org.elasticsearch.upgrades.FullClusterRestartIT - method: testSnapshotRestore {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/111798 - class: org.elasticsearch.smoketest.WatcherYamlRestIT method: test {p0=watcher/usage/10_basic/Test watcher usage stats output} issue: https://github.com/elastic/elasticsearch/issues/112189 @@ -147,9 +144,6 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=cat.shards/10_basic/Help} issue: https://github.com/elastic/elasticsearch/issues/116110 -- class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT - method: testSnapshotRestore {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/111799 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT method: testLookbackWithIndicesOptions issue: https://github.com/elastic/elasticsearch/issues/116127 @@ -229,9 +223,31 @@ tests: - class: org.elasticsearch.repositories.s3.RepositoryS3RestIT method: testReloadCredentialsFromKeystore issue: https://github.com/elastic/elasticsearch/issues/116811 -- class: org.elasticsearch.http.netty4.Netty4IncrementalRequestHandlingIT - method: testClientConnectionCloseMidStream - issue: https://github.com/elastic/elasticsearch/issues/116815 +- class: org.elasticsearch.xpack.searchablesnapshots.hdfs.SecureHdfsSearchableSnapshotsIT + issue: https://github.com/elastic/elasticsearch/issues/116851 +- class: org.elasticsearch.xpack.esql.analysis.VerifierTests + method: testCategorizeWithinAggregations + issue: https://github.com/elastic/elasticsearch/issues/116856 +- class: org.elasticsearch.xpack.esql.analysis.VerifierTests + method: testCategorizeSingleGrouping + issue: https://github.com/elastic/elasticsearch/issues/116857 +- class: org.elasticsearch.xpack.esql.analysis.VerifierTests + method: testCategorizeNestedGrouping + issue: https://github.com/elastic/elasticsearch/issues/116858 +- class: org.elasticsearch.search.basic.SearchWithRandomIOExceptionsIT + method: testRandomDirectoryIOExceptions + issue: https://github.com/elastic/elasticsearch/issues/114824 +- class: org.elasticsearch.index.mapper.vectors.MultiDenseVectorScriptDocValuesTests + method: testFloatGetVectorValueAndGetMagnitude + issue: https://github.com/elastic/elasticsearch/issues/116863 +- class: org.elasticsearch.xpack.inference.InferenceRestIT + method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint} + issue: https://github.com/elastic/elasticsearch/issues/116542 +- class: org.elasticsearch.compute.lucene.LuceneQueryExpressionEvaluatorTests + method: testTermQuery + issue: https://github.com/elastic/elasticsearch/issues/116879 +- class: org.elasticsearch.xpack.inference.InferenceRestIT + issue: https://github.com/elastic/elasticsearch/issues/116899 # Examples: # diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle index 40b12c46c0bfe..f9245ed32c325 100644 --- a/plugins/analysis-icu/build.gradle +++ b/plugins/analysis-icu/build.gradle @@ -28,7 +28,7 @@ dependencies { api "com.ibm.icu:icu4j:${versions.icu4j}" } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle index 6eb5b574b88f9..16786c6c31074 100644 --- a/plugins/discovery-azure-classic/build.gradle +++ b/plugins/discovery-azure-classic/build.gradle @@ -63,7 +63,7 @@ TaskProvider createKey = tasks.register("createKey", LoggedExec) { keystore.parentFile.mkdirs() } outputs.file(keystore).withPropertyName('keystoreFile') - executable = "${BuildParams.runtimeJavaHome}/bin/keytool" + executable = "${buildParams.runtimeJavaHome.get()}/bin/keytool" getStandardInput().set('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n') args '-genkey', '-alias', 'test-node', diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index d9e86315d9468..a166a89ad4026 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -57,7 +57,7 @@ tasks.register("writeTestJavaPolicy") { throw new GradleException("failed to create temporary directory [${tmp}]") } final File javaPolicy = file("${tmp}/java.policy") - if (BuildParams.inFipsJvm) { + if (buildParams.inFipsJvm) { javaPolicy.write( [ "grant {", @@ -98,7 +98,7 @@ tasks.named("test").configure { // this is needed to manipulate com.amazonaws.sdk.ec2MetadataServiceEndpointOverride system property // it is better rather disable security manager at all with `systemProperty 'tests.security.manager', 'false'` - if (BuildParams.inFipsJvm){ + if (buildParams.inFipsJvm){ nonInputProperties.systemProperty 'java.security.policy', "=file://${buildDir}/tmp/java.policy" } else { nonInputProperties.systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy" diff --git a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle index 5cdcdc59cafe9..aad59be376262 100644 --- a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle +++ b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle @@ -56,7 +56,7 @@ tasks.named("yamlRestTest").configure { enabled = false } TaskProvider fixture = tasks.register("ec2Fixture${action}", AntFixture) { dependsOn project.sourceSets.yamlRestTest.runtimeClasspath env 'CLASSPATH', "${-> project.sourceSets.yamlRestTest.runtimeClasspath.asPath}" - executable = "${BuildParams.runtimeJavaHome}/bin/java" + executable = "${buildParams.runtimeJavaHome.get()}/bin/java" args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, "${buildDir}/testclusters/yamlRestTest${action}-1/config/unicast_hosts.txt" } diff --git a/plugins/discovery-gce/qa/gce/build.gradle b/plugins/discovery-gce/qa/gce/build.gradle index 14a904e107188..a22678b9a67dc 100644 --- a/plugins/discovery-gce/qa/gce/build.gradle +++ b/plugins/discovery-gce/qa/gce/build.gradle @@ -32,7 +32,7 @@ restResources { def gceFixtureProvider = tasks.register("gceFixture", AntFixture) { dependsOn project.sourceSets.yamlRestTest.runtimeClasspath env 'CLASSPATH', "${-> project.sourceSets.yamlRestTest.runtimeClasspath.asPath}" - executable = "${BuildParams.runtimeJavaHome}/bin/java" + executable = "${buildParams.runtimeJavaHome.get()}/bin/java" args 'org.elasticsearch.cloud.gce.GCEFixture', baseDir, "${buildDir}/testclusters/yamlRestTest-1/config/unicast_hosts.txt" } diff --git a/plugins/mapper-annotated-text/build.gradle b/plugins/mapper-annotated-text/build.gradle index d0b1163970616..545dfe49bfcf3 100644 --- a/plugins/mapper-annotated-text/build.gradle +++ b/plugins/mapper-annotated-text/build.gradle @@ -16,7 +16,7 @@ esplugin { classname 'org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextPlugin' } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/plugins/mapper-murmur3/build.gradle b/plugins/mapper-murmur3/build.gradle index 0fa710c130a29..e5108814154a3 100644 --- a/plugins/mapper-murmur3/build.gradle +++ b/plugins/mapper-murmur3/build.gradle @@ -22,7 +22,7 @@ dependencies { testImplementation project(':modules:lang-painless') } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 741542477e446..b7f7816a3a0e1 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -84,7 +84,7 @@ tasks.named("dependencyLicenses").configure { tasks.withType(RestIntegTestTask).configureEach { usesDefaultDistribution() - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) jvmArgs '--add-exports', 'java.security.jgss/sun.security.krb5=ALL-UNNAMED' } diff --git a/plugins/repository-hdfs/hadoop-client-api/build.gradle b/plugins/repository-hdfs/hadoop-client-api/build.gradle index 4ac6f79530fcb..24e4213780fe2 100644 --- a/plugins/repository-hdfs/hadoop-client-api/build.gradle +++ b/plugins/repository-hdfs/hadoop-client-api/build.gradle @@ -1,5 +1,5 @@ apply plugin: 'elasticsearch.build' -apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'com.gradleup.shadow' dependencies { implementation "org.apache.hadoop:hadoop-client-api:${project.parent.versions.hadoop}" diff --git a/qa/ccs-rolling-upgrade-remote-cluster/build.gradle b/qa/ccs-rolling-upgrade-remote-cluster/build.gradle index 585124f223c9c..ce5b840e6dc91 100644 --- a/qa/ccs-rolling-upgrade-remote-cluster/build.gradle +++ b/qa/ccs-rolling-upgrade-remote-cluster/build.gradle @@ -16,7 +16,7 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.bwc-test' apply plugin: 'elasticsearch.rest-resources' -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> /** * We execute tests 3 times. @@ -52,7 +52,7 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> nonInputProperties.systemProperty('tests.rest.remote_cluster', remoteCluster.map(c -> c.allHttpSocketURI.join(","))) } - onlyIf("FIPS mode disabled") { BuildParams.inFipsJvm == false } + onlyIf("FIPS mode disabled") { buildParams.inFipsJvm == false } } tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle index 8d950eea616d6..5e68c4d1ad26b 100644 --- a/qa/full-cluster-restart/build.gradle +++ b/qa/full-cluster-restart/build.gradle @@ -7,14 +7,13 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.bwc-test' -BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartArchivedSettingsIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartArchivedSettingsIT.java new file mode 100644 index 0000000000000..caa57f1e605a2 --- /dev/null +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartArchivedSettingsIT.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.upgrades; + +import io.netty.handler.codec.http.HttpMethod; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.WarningsHandler; +import org.elasticsearch.core.UpdateForV10; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ObjectPath; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; + +import static org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator.THRESHOLD_SETTING; + +/** + * Tests to run before and after a full cluster restart. This is run twice, + * one with {@code tests.is_old_cluster} set to {@code true} against a cluster + * of an older version. The cluster is shutdown and a cluster of the new + * version is started with the same data directories and then this is rerun + * with {@code tests.is_old_cluster} set to {@code false}. + */ +public class FullClusterRestartArchivedSettingsIT extends ParameterizedFullClusterRestartTestCase { + + private static TemporaryFolder repoDirectory = new TemporaryFolder(); + + protected static LocalClusterConfigProvider clusterConfig = c -> {}; + + private static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(2) + .setting("path.repo", () -> repoDirectory.getRoot().getPath()) + .setting("xpack.security.enabled", "false") + // some tests rely on the translog not being flushed + .setting("indices.memory.shard_inactive_time", "60m") + .apply(() -> clusterConfig) + .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.FAILURE_STORE_ENABLED) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster); + + public FullClusterRestartArchivedSettingsIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } + + @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION) // this test is just about v8->v9 upgrades, remove it in v10 + public void testBalancedShardsAllocatorThreshold() throws Exception { + assumeTrue("test only applies for v8->v9 upgrades", getOldClusterTestVersion().getMajor() == 8); + + final var chosenValue = randomFrom("0", "0.1", "0.5", "0.999"); + + if (isRunningAgainstOldCluster()) { + final var request = newXContentRequest( + HttpMethod.PUT, + "/_cluster/settings", + (builder, params) -> builder.startObject("persistent").field(THRESHOLD_SETTING.getKey(), chosenValue).endObject() + ); + request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); + assertOK(client().performRequest(request)); + } + + final var clusterSettingsResponse = ObjectPath.createFromResponse( + client().performRequest(new Request("GET", "/_cluster/settings")) + ); + + final var settingsPath = "persistent." + THRESHOLD_SETTING.getKey(); + final var settingValue = clusterSettingsResponse.evaluate(settingsPath); + + if (isRunningAgainstOldCluster()) { + assertEquals(chosenValue, settingValue); + } else { + assertNull(settingValue); + assertNotNull(clusterSettingsResponse.evaluate("persistent.archived." + THRESHOLD_SETTING.getKey())); + } + } +} diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index daadf936ae841..26e4f3146da2f 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -16,11 +16,9 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.Build; import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.common.Strings; @@ -29,7 +27,6 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.CheckedFunction; -import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -75,7 +72,6 @@ import static java.util.stream.Collectors.toList; import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ENFORCEMENT_INDEX_VERSION; import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING; -import static org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator.THRESHOLD_SETTING; import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; @@ -1959,35 +1955,4 @@ public static void assertNumHits(String index, int numHits, int totalShards) thr assertThat(XContentMapValues.extractValue("_shards.successful", resp), equalTo(totalShards)); assertThat(extractTotalHits(resp), equalTo(numHits)); } - - @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION) // this test is just about v8->v9 upgrades, remove it in v10 - public void testBalancedShardsAllocatorThreshold() throws Exception { - assumeTrue("test only applies for v8->v9 upgrades", getOldClusterTestVersion().getMajor() == 8); - - final var chosenValue = randomFrom("0", "0.1", "0.5", "0.999"); - - if (isRunningAgainstOldCluster()) { - final var request = newXContentRequest( - HttpMethod.PUT, - "/_cluster/settings", - (builder, params) -> builder.startObject("persistent").field(THRESHOLD_SETTING.getKey(), chosenValue).endObject() - ); - request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); - assertOK(client().performRequest(request)); - } - - final var clusterSettingsResponse = ObjectPath.createFromResponse( - client().performRequest(new Request("GET", "/_cluster/settings")) - ); - - final var settingsPath = "persistent." + THRESHOLD_SETTING.getKey(); - final var settingValue = clusterSettingsResponse.evaluate(settingsPath); - - if (isRunningAgainstOldCluster()) { - assertEquals(chosenValue, settingValue); - } else { - assertNull(settingValue); - assertNotNull(clusterSettingsResponse.evaluate("persistent.archived." + THRESHOLD_SETTING.getKey())); - } - } } diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index f3fd57f3fc8ae..f6549a2d83fe6 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -10,7 +10,6 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.internal-testclusters' @@ -64,8 +63,7 @@ excludeList.add('indices.resolve_index/20_resolve_system_index/*') // Excluded because the error has changed excludeList.add('aggregations/percentiles_hdr_metric/Negative values test') -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> - +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> if (bwcVersion != VersionProperties.getElasticsearchVersion()) { /* This project runs the core REST tests against a 4 node cluster where two of the nodes has a different minor. */ diff --git a/qa/multi-cluster-search/build.gradle b/qa/multi-cluster-search/build.gradle index 146acedd164b2..906a49134bb51 100644 --- a/qa/multi-cluster-search/build.gradle +++ b/qa/multi-cluster-search/build.gradle @@ -35,7 +35,7 @@ def ccsSupportedVersion = bwcVersion -> { return currentVersion.minor == 0 || (currentVersion.major == bwcVersion.major && currentVersion.minor - bwcVersion.minor <= 1) } -BuildParams.bwcVersions.withWireCompatible(ccsSupportedVersion) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(ccsSupportedVersion) { bwcVersion, baseName -> def remoteCluster = testClusters.register("${baseName}-remote") { numberOfNodes = 2 diff --git a/qa/repository-multi-version/build.gradle b/qa/repository-multi-version/build.gradle index 17888efaa2b49..79a8be4c1be24 100644 --- a/qa/repository-multi-version/build.gradle +++ b/qa/repository-multi-version/build.gradle @@ -16,7 +16,7 @@ apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.bwc-test' -BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> String oldClusterName = "${baseName}-old" String newClusterName = "${baseName}-new" diff --git a/qa/rolling-upgrade-legacy/build.gradle b/qa/rolling-upgrade-legacy/build.gradle index 4ebb3888e9f20..e1c31fd50c0d4 100644 --- a/qa/rolling-upgrade-legacy/build.gradle +++ b/qa/rolling-upgrade-legacy/build.gradle @@ -10,7 +10,6 @@ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.internal-testclusters' @@ -18,7 +17,7 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.bwc-test' apply plugin: 'elasticsearch.rest-resources' -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> /* * NOTE: This module is for the tests that were problematic when converting :qa:rolling-upgrade to the junit-based bwc test definition * Over time, these should be migrated into the :qa:rolling-upgrade module and fixed properly diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index ef31f6421c187..2f717f201f248 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -18,7 +18,7 @@ testArtifacts { registerTestArtifactFromSourceSet(sourceSets.javaRestTest) } -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle index af4e55a709a64..c707c2b5e8c80 100644 --- a/qa/smoke-test-plugins/build.gradle +++ b/qa/smoke-test-plugins/build.gradle @@ -8,7 +8,6 @@ */ import org.apache.tools.ant.filters.ReplaceTokens -import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.legacy-yaml-rest-test' diff --git a/qa/verify-version-constants/build.gradle b/qa/verify-version-constants/build.gradle index f74ee7c59b26b..ee29da53dc51b 100644 --- a/qa/verify-version-constants/build.gradle +++ b/qa/verify-version-constants/build.gradle @@ -19,7 +19,7 @@ dependencies { testImplementation project(':modules:rest-root') } -BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> def baseCluster = testClusters.register(baseName) { version = bwcVersion.toString() setting 'xpack.security.enabled', 'true' diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.delete.json index 745136848786c..cb4eee007a246 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.delete.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.delete.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-inference-api.html", "description": "Delete an inference endpoint" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.get.json index 7b7aa0f56fcbc..14e7519c3796e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.get.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.get.json @@ -4,7 +4,7 @@ "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/get-inference-api.html", "description":"Get an inference endpoint" }, - "stability":"experimental", + "stability":"stable", "visibility":"public", "headers":{ "accept": [ "application/json"] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.inference.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.inference.json index 3195476ce1e9e..eb4c1268c28ca 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.inference.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.inference.json @@ -4,7 +4,7 @@ "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/post-inference-api.html", "description":"Perform inference" }, - "stability":"experimental", + "stability":"stable", "visibility":"public", "headers":{ "accept": [ "application/json"], diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put.json index 9ff5ff4b80c58..411392fe39908 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put.json @@ -4,7 +4,7 @@ "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/put-inference-api.html", "description":"Configure an inference endpoint for use in the Inference API" }, - "stability":"experimental", + "stability":"stable", "visibility":"public", "headers":{ "accept": [ "application/json"], diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json index 32b4b2f311837..493306e10d5c7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json @@ -4,7 +4,7 @@ "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/post-stream-inference-api.html", "description":"Perform streaming inference" }, - "stability":"experimental", + "stability":"stable", "visibility":"public", "headers":{ "accept": [ "text/event-stream"], diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/16_creation_date_tier_preference.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/16_creation_date_tier_preference.yml new file mode 100644 index 0000000000000..6ecd9c3e9c2ce --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/16_creation_date_tier_preference.yml @@ -0,0 +1,14 @@ +--- +"Ensure index creation date and tier preference are exposed": + - requires: + cluster_features: ["stats.tier_creation_date"] + reason: index creation date and tier preference added to stats in 8.17 + + - do: + indices.create: + index: myindex + - do: + indices.stats: {} + + - is_true: indices.myindex.creation_date + - is_true: indices.myindex.tier_preference diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml index db718959919da..54b2bf59c8ddc 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml @@ -129,7 +129,7 @@ noop update: {} --- -update: +regular update: - requires: cluster_features: ["gte_v8.2.0"] reason: tsdb indexing changed in 8.2.0 diff --git a/server/build.gradle b/server/build.gradle index ef64b0746dfc4..bc8decfa8babc 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -133,7 +133,7 @@ def generatePluginsList = tasks.register("generatePluginsList") { sourceSets.main.output.dir(generatedResourcesDir) sourceSets.main.compiledBy(generateModulesList, generatePluginsList) -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' systemProperty 'es.failure_store_feature_flag_enabled', 'true' diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerMetricsIT.java index 9a71bf86388a4..b3ec4a5331180 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerMetricsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerMetricsIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.routing.allocation.allocator; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterInfoServiceUtils; import org.elasticsearch.cluster.InternalClusterInfoService; @@ -68,6 +69,7 @@ public void testDesiredBalanceMetrics() { final var infoService = (InternalClusterInfoService) internalCluster().getCurrentMasterNodeInstance(ClusterInfoService.class); ClusterInfoServiceUtils.setUpdateFrequency(infoService, TimeValue.timeValueMillis(200)); assertNotNull("info should not be null", ClusterInfoServiceUtils.refresh(infoService)); + ClusterRerouteUtils.reroute(client()); // ensure we leverage the latest cluster info final var telemetryPlugin = getTelemetryPlugin(internalCluster().getMasterName()); telemetryPlugin.collect(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java index 039a596f53b38..38eef4f720623 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java @@ -19,14 +19,19 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.PeerRecoveryTargetService; import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest; +import org.elasticsearch.indices.recovery.RecoveryFilesInfoRequest; import org.elasticsearch.node.RecoverySettingsChunkSizePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -34,6 +39,7 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; import static org.elasticsearch.node.RecoverySettingsChunkSizePlugin.CHUNK_SIZE_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -72,16 +78,14 @@ public void testCancelRecoveryAndResume() throws Exception { // we use 2 nodes a lucky and unlucky one // the lucky one holds the primary // the unlucky one gets the replica and the truncated leftovers - NodeStats primariesNode = dataNodeStats.get(0); - NodeStats unluckyNode = dataNodeStats.get(1); + String primariesNode = dataNodeStats.get(0).getNode().getName(); + String unluckyNode = dataNodeStats.get(1).getNode().getName(); // create the index and prevent allocation on any other nodes than the lucky one // we have no replicas so far and make sure that we allocate the primary on the lucky node assertAcked( prepareCreate("test").setMapping("field1", "type=text", "the_id", "type=text") - .setSettings( - indexSettings(numberOfShards(), 0).put("index.routing.allocation.include._name", primariesNode.getNode().getName()) - ) + .setSettings(indexSettings(numberOfShards(), 0).put("index.routing.allocation.include._name", primariesNode)) ); // only allocate on the lucky node // index some docs and check if they are coming back @@ -102,20 +106,54 @@ public void testCancelRecoveryAndResume() throws Exception { indicesAdmin().prepareFlush().setForce(true).get(); // double flush to create safe commit in case of async durability indicesAdmin().prepareForceMerge().setMaxNumSegments(1).setFlush(true).get(); + // We write some garbage into the shard directory so that we can verify that it is cleaned up before we resend. + // Cleanup helps prevent recovery from failing due to lack of space from garbage left over from a previous + // recovery that crashed during file transmission. #104473 + // We can't look for the presence of the recovery temp files themselves because they are automatically + // cleaned up on clean shutdown by MultiFileWriter. + final String GARBAGE_PREFIX = "recovery.garbage."; + final CountDownLatch latch = new CountDownLatch(1); final AtomicBoolean truncate = new AtomicBoolean(true); + + IndicesService unluckyIndices = internalCluster().getInstance(IndicesService.class, unluckyNode); + Function getUnluckyIndexPath = (shardId) -> unluckyIndices.indexService(shardId.getIndex()) + .getShard(shardId.getId()) + .shardPath() + .resolveIndex(); + for (NodeStats dataNode : dataNodeStats) { MockTransportService.getInstance(dataNode.getNode().getName()) .addSendBehavior( - internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), + internalCluster().getInstance(TransportService.class, unluckyNode), (connection, requestId, action, request, options) -> { if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) { RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request; logger.info("file chunk [{}] lastChunk: {}", req, req.lastChunk()); + // During the first recovery attempt (when truncate is set), write an extra garbage file once for each + // file transmitted. We get multiple chunks per file but only one is the last. + if (truncate.get() && req.lastChunk()) { + final var shardPath = getUnluckyIndexPath.apply(req.shardId()); + final var garbagePath = Files.createTempFile(shardPath, GARBAGE_PREFIX, null); + logger.info("writing garbage at: {}", garbagePath); + } if ((req.name().endsWith("cfs") || req.name().endsWith("fdt")) && req.lastChunk() && truncate.get()) { latch.countDown(); throw new RuntimeException("Caused some truncated files for fun and profit"); } + } else if (action.equals(PeerRecoveryTargetService.Actions.FILES_INFO)) { + // verify there are no garbage files present at the FILES_INFO stage of recovery. This precedes FILES_CHUNKS + // and so will run before garbage has been introduced on the first attempt, and before post-transfer cleanup + // has been performed on the second. + final var shardPath = getUnluckyIndexPath.apply(((RecoveryFilesInfoRequest) request).shardId()); + try (var list = Files.list(shardPath).filter(path -> path.getFileName().startsWith(GARBAGE_PREFIX))) { + final var garbageFiles = list.toArray(); + assertArrayEquals( + "garbage files should have been cleaned before file transmission", + new Path[0], + garbageFiles + ); + } } connection.sendRequest(requestId, action, request, options); } @@ -128,14 +166,14 @@ public void testCancelRecoveryAndResume() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) .put( "index.routing.allocation.include._name", // now allow allocation on all nodes - primariesNode.getNode().getName() + "," + unluckyNode.getNode().getName() + primariesNode + "," + unluckyNode ), "test" ); latch.await(); - // at this point we got some truncated left overs on the replica on the unlucky node + // at this point we got some truncated leftovers on the replica on the unlucky node // now we are allowing the recovery to allocate again and finish to see if we wipe the truncated files truncate.compareAndSet(true, false); ensureGreen("test"); diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 3b3b06a1c6924..29c869a9f8d77 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -419,6 +419,7 @@ provides org.elasticsearch.features.FeatureSpecification with + org.elasticsearch.action.admin.indices.stats.IndicesStatsFeatures, org.elasticsearch.action.bulk.BulkFeatures, org.elasticsearch.features.FeatureInfrastructureFeatures, org.elasticsearch.health.HealthFeatures, diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index b7da6115a1a48..5b5d12d738194 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -104,6 +104,7 @@ static TransportVersion def(int id) { public static final TransportVersion V_8_14_0 = def(8_636_00_1); public static final TransportVersion V_8_15_0 = def(8_702_00_2); public static final TransportVersion V_8_15_2 = def(8_702_00_3); + public static final TransportVersion QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_15 = def(8_702_00_4); public static final TransportVersion ML_INFERENCE_DONT_DELETE_WHEN_SEMANTIC_TEXT_EXISTS = def(8_703_00_0); public static final TransportVersion INFERENCE_ADAPTIVE_ALLOCATIONS = def(8_704_00_0); public static final TransportVersion INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN = def(8_705_00_0); @@ -177,6 +178,7 @@ static TransportVersion def(int id) { public static final TransportVersion INFERENCE_DONT_PERSIST_ON_READ_BACKPORT_8_16 = def(8_772_00_1); public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO_BACKPORT_8_16 = def(8_772_00_2); public static final TransportVersion SKIP_INNER_HITS_SEARCH_SOURCE_BACKPORT_8_16 = def(8_772_00_3); + public static final TransportVersion QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16 = def(8_772_00_4); public static final TransportVersion REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_773_00_0); public static final TransportVersion REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_774_00_0); public static final TransportVersion ESQL_FIELD_ATTRIBUTE_PARENT_SIMPLIFIED = def(8_775_00_0); @@ -197,6 +199,7 @@ static TransportVersion def(int id) { public static final TransportVersion VERTEX_AI_INPUT_TYPE_ADDED = def(8_790_00_0); public static final TransportVersion SKIP_INNER_HITS_SEARCH_SOURCE = def(8_791_00_0); public static final TransportVersion QUERY_RULES_LIST_INCLUDES_TYPES = def(8_792_00_0); + public static final TransportVersion INDEX_STATS_ADDITIONAL_FIELDS = def(8_793_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java index 6106e620521f7..7cefc086e17dc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.core.Nullable; +import org.elasticsearch.features.NodeFeature; import java.util.ArrayList; import java.util.HashMap; @@ -21,6 +22,8 @@ public class IndexStats implements Iterable { + public static final NodeFeature TIER_CREATION_DATE = new NodeFeature("stats.tier_creation_date"); + private final String index; private final String uuid; @@ -29,6 +32,10 @@ public class IndexStats implements Iterable { private final IndexMetadata.State state; + private final List tierPreference; + + private final Long creationDate; + private final ShardStats shards[]; public IndexStats( @@ -36,12 +43,16 @@ public IndexStats( String uuid, @Nullable ClusterHealthStatus health, @Nullable IndexMetadata.State state, + @Nullable List tierPreference, + @Nullable Long creationDate, ShardStats[] shards ) { this.index = index; this.uuid = uuid; this.health = health; this.state = state; + this.tierPreference = tierPreference; + this.creationDate = creationDate; this.shards = shards; } @@ -61,6 +72,14 @@ public IndexMetadata.State getState() { return state; } + public List getTierPreference() { + return tierPreference; + } + + public Long getCreationDate() { + return creationDate; + } + public ShardStats[] getShards() { return this.shards; } @@ -129,13 +148,24 @@ public static class IndexStatsBuilder { private final String uuid; private final ClusterHealthStatus health; private final IndexMetadata.State state; + private final List tierPreference; + private final Long creationDate; private final List shards = new ArrayList<>(); - public IndexStatsBuilder(String indexName, String uuid, @Nullable ClusterHealthStatus health, @Nullable IndexMetadata.State state) { + public IndexStatsBuilder( + String indexName, + String uuid, + @Nullable ClusterHealthStatus health, + @Nullable IndexMetadata.State state, + @Nullable List tierPreference, + @Nullable Long creationDate + ) { this.indexName = indexName; this.uuid = uuid; this.health = health; this.state = state; + this.tierPreference = tierPreference; + this.creationDate = creationDate; } public IndexStatsBuilder add(ShardStats shardStats) { @@ -144,7 +174,15 @@ public IndexStatsBuilder add(ShardStats shardStats) { } public IndexStats build() { - return new IndexStats(indexName, uuid, health, state, shards.toArray(new ShardStats[shards.size()])); + return new IndexStats( + indexName, + uuid, + health, + state, + tierPreference, + creationDate, + shards.toArray(new ShardStats[shards.size()]) + ); } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsFeatures.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsFeatures.java new file mode 100644 index 0000000000000..2b67885273d05 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsFeatures.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.indices.stats; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +public class IndicesStatsFeatures implements FeatureSpecification { + + @Override + public Set getFeatures() { + return Set.of(IndexStats.TIER_CREATION_DATE); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java index 65f7e1969ea69..205f1cbc04e8b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java @@ -47,6 +47,10 @@ public class IndicesStatsResponse extends ChunkedBroadcastResponse { private final Map indexStateMap; + private final Map> indexTierPreferenceMap; + + private final Map indexCreationDateMap; + private final ShardStats[] shards; private Map shardStatsMap; @@ -54,12 +58,23 @@ public class IndicesStatsResponse extends ChunkedBroadcastResponse { IndicesStatsResponse(StreamInput in) throws IOException { super(in); shards = in.readArray(ShardStats::new, ShardStats[]::new); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_1_0)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_STATS_ADDITIONAL_FIELDS)) { indexHealthMap = in.readMap(ClusterHealthStatus::readFrom); indexStateMap = in.readMap(IndexMetadata.State::readFrom); + indexTierPreferenceMap = in.readMap(StreamInput::readStringCollectionAsList); + indexCreationDateMap = in.readMap(StreamInput::readLong); + } else if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_1_0)) { + // Between 8.1 and INDEX_STATS_ADDITIONAL_FIELDS, we had a different format for the response + // where we only had health and state available. + indexHealthMap = in.readMap(ClusterHealthStatus::readFrom); + indexStateMap = in.readMap(IndexMetadata.State::readFrom); + indexTierPreferenceMap = Map.of(); + indexCreationDateMap = Map.of(); } else { indexHealthMap = Map.of(); indexStateMap = Map.of(); + indexTierPreferenceMap = Map.of(); + indexCreationDateMap = Map.of(); } } @@ -79,6 +94,8 @@ public class IndicesStatsResponse extends ChunkedBroadcastResponse { Objects.requireNonNull(shards); Map indexHealthModifiableMap = new HashMap<>(); Map indexStateModifiableMap = new HashMap<>(); + Map> indexTierPreferenceModifiableMap = new HashMap<>(); + Map indexCreationDateModifiableMap = new HashMap<>(); for (ShardStats shard : shards) { Index index = shard.getShardRouting().index(); IndexMetadata indexMetadata = metadata.index(index); @@ -88,10 +105,14 @@ public class IndicesStatsResponse extends ChunkedBroadcastResponse { ignored -> new ClusterIndexHealth(indexMetadata, routingTable.index(index)).getStatus() ); indexStateModifiableMap.computeIfAbsent(index.getName(), ignored -> indexMetadata.getState()); + indexTierPreferenceModifiableMap.computeIfAbsent(index.getName(), ignored -> indexMetadata.getTierPreference()); + indexCreationDateModifiableMap.computeIfAbsent(index.getName(), ignored -> indexMetadata.getCreationDate()); } } indexHealthMap = unmodifiableMap(indexHealthModifiableMap); indexStateMap = unmodifiableMap(indexStateModifiableMap); + indexTierPreferenceMap = unmodifiableMap(indexTierPreferenceModifiableMap); + indexCreationDateMap = unmodifiableMap(indexCreationDateModifiableMap); } public Map asMap() { @@ -129,7 +150,14 @@ public Map getIndices() { Index index = shard.getShardRouting().index(); IndexStatsBuilder indexStatsBuilder = indexToIndexStatsBuilder.computeIfAbsent( index.getName(), - k -> new IndexStatsBuilder(k, index.getUUID(), indexHealthMap.get(index.getName()), indexStateMap.get(index.getName())) + k -> new IndexStatsBuilder( + k, + index.getUUID(), + indexHealthMap.get(index.getName()), + indexStateMap.get(index.getName()), + indexTierPreferenceMap.get(index.getName()), + indexCreationDateMap.get(index.getName()) + ) ); indexStatsBuilder.add(shard); } @@ -174,7 +202,13 @@ public CommonStats getPrimaries() { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeArray(shards); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_1_0)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_STATS_ADDITIONAL_FIELDS)) { + out.writeMap(indexHealthMap, StreamOutput::writeWriteable); + out.writeMap(indexStateMap, StreamOutput::writeWriteable); + out.writeMap(indexTierPreferenceMap, StreamOutput::writeStringCollection); + out.writeMap(indexCreationDateMap, StreamOutput::writeLong); + + } else if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_1_0)) { out.writeMap(indexHealthMap, StreamOutput::writeWriteable); out.writeMap(indexStateMap, StreamOutput::writeWriteable); } @@ -203,6 +237,12 @@ protected Iterator customXContentChunks(ToXContent.Params params) { if (indexStats.getState() != null) { builder.field("status", indexStats.getState().toString().toLowerCase(Locale.ROOT)); } + if (indexStats.getTierPreference() != null) { + builder.field("tier_preference", indexStats.getTierPreference()); + } + if (indexStats.getCreationDate() != null) { + builder.field("creation_date", indexStats.getCreationDate()); + } builder.startObject("primaries"); indexStats.getPrimaries().toXContent(builder, p); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java index ce3e189149451..ad1fda2534fab 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java @@ -42,6 +42,7 @@ import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; @@ -543,7 +544,8 @@ private IndexDocFailureStoreStatus processFailure(BulkItemRequest bulkItemReques var isFailureStoreRequest = isFailureStoreRequest(docWriteRequest); if (isFailureStoreRequest == false && failureStoreCandidate.isFailureStoreEnabled() - && error instanceof VersionConflictEngineException == false) { + && error instanceof VersionConflictEngineException == false + && error instanceof EsRejectedExecutionException == false) { // Prepare the data stream failure store if necessary maybeMarkFailureStoreForRollover(failureStoreCandidate); @@ -563,8 +565,8 @@ private IndexDocFailureStoreStatus processFailure(BulkItemRequest bulkItemReques } } else { // If we can't redirect to a failure store (because either the data stream doesn't have the failure store enabled - // or this request was already targeting a failure store), or this was a version conflict we increment the - // rejected counter. + // or this request was already targeting a failure store), or this was an error that is not eligible for the failure store + // such as a version conflict or a load rejection we increment the rejected counter. failureStoreMetrics.incrementRejected( bulkItemRequest.index(), errorType, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java index 6841c6e49e3f1..62d3f5e5866bc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java @@ -467,7 +467,7 @@ public void handleCommit(ApplyCommitRequest applyCommit) { logger.debug( "handleCommit: ignored commit request due to term mismatch " + "(expected: [term {} version {}], actual: [term {} version {}])", - getLastAcceptedTerm(), + getCurrentTerm(), getLastAcceptedVersion(), applyCommit.getTerm(), applyCommit.getVersion() diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index 2c1175648c219..91c4b780db0bd 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -189,7 +189,11 @@ public Analyzer getAnalyzer(String analyzer) throws IOException { } }); } - return analyzerProvider.get(environment, analyzer).get(); + + return overridePositionIncrementGap( + (NamedAnalyzer) analyzerProvider.get(environment, analyzer).get(), + TextFieldMapper.Defaults.POSITION_INCREMENT_GAP + ); } @Override @@ -720,13 +724,8 @@ private static NamedAnalyzer produceAnalyzer( throw new IllegalArgumentException("analyzer [" + analyzerFactory.name() + "] created null analyzer"); } NamedAnalyzer analyzer; - if (analyzerF instanceof NamedAnalyzer) { - // if we got a named analyzer back, use it... - analyzer = (NamedAnalyzer) analyzerF; - if (overridePositionIncrementGap >= 0 && analyzer.getPositionIncrementGap(analyzer.name()) != overridePositionIncrementGap) { - // unless the positionIncrementGap needs to be overridden - analyzer = new NamedAnalyzer(analyzer, overridePositionIncrementGap); - } + if (analyzerF instanceof NamedAnalyzer namedAnalyzer) { + analyzer = overridePositionIncrementGap(namedAnalyzer, overridePositionIncrementGap); } else { analyzer = new NamedAnalyzer(name, analyzerFactory.scope(), analyzerF, overridePositionIncrementGap); } @@ -734,6 +733,13 @@ private static NamedAnalyzer produceAnalyzer( return analyzer; } + private static NamedAnalyzer overridePositionIncrementGap(NamedAnalyzer analyzer, int overridePositionIncrementGap) { + if (overridePositionIncrementGap >= 0 && analyzer.getPositionIncrementGap(analyzer.name()) != overridePositionIncrementGap) { + analyzer = new NamedAnalyzer(analyzer, overridePositionIncrementGap); + } + return analyzer; + } + private static void processNormalizerFactory( String name, AnalyzerProvider normalizerFactory, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValues.java new file mode 100644 index 0000000000000..a91960832239f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValues.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.vectors; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.field.vectors.MultiDenseVector; + +import java.util.Iterator; + +public class MultiDenseVectorScriptDocValues extends ScriptDocValues { + + public static final String MISSING_VECTOR_FIELD_MESSAGE = "A document doesn't have a value for a multi-vector field!"; + + private final int dims; + protected final MultiDenseVectorSupplier dvSupplier; + + public MultiDenseVectorScriptDocValues(MultiDenseVectorSupplier supplier, int dims) { + super(supplier); + this.dvSupplier = supplier; + this.dims = dims; + } + + public int dims() { + return dims; + } + + private MultiDenseVector getCheckedVector() { + MultiDenseVector vector = dvSupplier.getInternal(); + if (vector == null) { + throw new IllegalArgumentException(MISSING_VECTOR_FIELD_MESSAGE); + } + return vector; + } + + /** + * Get multi-dense vector's value as an array of floats + */ + public Iterator getVectorValues() { + return getCheckedVector().getVectors(); + } + + /** + * Get dense vector's magnitude + */ + public float[] getMagnitudes() { + return getCheckedVector().getMagnitudes(); + } + + @Override + public BytesRef get(int index) { + throw new UnsupportedOperationException( + "accessing a multi-vector field's value through 'get' or 'value' is not supported, use 'vectorValues' or 'magnitudes' instead." + ); + } + + @Override + public int size() { + MultiDenseVector mdv = dvSupplier.getInternal(); + if (mdv != null) { + return mdv.size(); + } + return 0; + } + + public interface MultiDenseVectorSupplier extends Supplier { + @Override + default BytesRef getInternal(int index) { + throw new UnsupportedOperationException(); + } + + MultiDenseVector getInternal(); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorDVLeafFieldData.java index cc6fb38274451..b9716d315f33a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorDVLeafFieldData.java @@ -9,37 +9,44 @@ package org.elasticsearch.index.mapper.vectors; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReader; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.vectors.BitMultiDenseVectorDocValuesField; +import org.elasticsearch.script.field.vectors.ByteMultiDenseVectorDocValuesField; +import org.elasticsearch.script.field.vectors.FloatMultiDenseVectorDocValuesField; + +import java.io.IOException; final class MultiVectorDVLeafFieldData implements LeafFieldData { private final LeafReader reader; private final String field; - private final IndexVersion indexVersion; private final DenseVectorFieldMapper.ElementType elementType; private final int dims; - MultiVectorDVLeafFieldData( - LeafReader reader, - String field, - IndexVersion indexVersion, - DenseVectorFieldMapper.ElementType elementType, - int dims - ) { + MultiVectorDVLeafFieldData(LeafReader reader, String field, DenseVectorFieldMapper.ElementType elementType, int dims) { this.reader = reader; this.field = field; - this.indexVersion = indexVersion; this.elementType = elementType; this.dims = dims; } @Override public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { - // TODO - return null; + try { + BinaryDocValues values = DocValues.getBinary(reader, field); + BinaryDocValues magnitudeValues = DocValues.getBinary(reader, field + MultiDenseVectorFieldMapper.VECTOR_MAGNITUDES_SUFFIX); + return switch (elementType) { + case BYTE -> new ByteMultiDenseVectorDocValuesField(values, magnitudeValues, name, elementType, dims); + case FLOAT -> new FloatMultiDenseVectorDocValuesField(values, magnitudeValues, name, elementType, dims); + case BIT -> new BitMultiDenseVectorDocValuesField(values, magnitudeValues, name, elementType, dims); + }; + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values for multi-vector field!", e); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorIndexFieldData.java index 65ef492ce052b..44a666e25a611 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorIndexFieldData.java @@ -55,7 +55,7 @@ public ValuesSourceType getValuesSourceType() { @Override public MultiVectorDVLeafFieldData load(LeafReaderContext context) { - return new MultiVectorDVLeafFieldData(context.reader(), fieldName, indexVersion, elementType, dims); + return new MultiVectorDVLeafFieldData(context.reader(), fieldName, elementType, dims); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorEncoderDecoder.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorEncoderDecoder.java index 9d09a7493d605..3db2d164846bd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorEncoderDecoder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorEncoderDecoder.java @@ -84,4 +84,24 @@ public static void decodeDenseVector(IndexVersion indexVersion, BytesRef vectorB } } + public static float[] getMultiMagnitudes(BytesRef magnitudes) { + assert magnitudes.length % Float.BYTES == 0; + float[] multiMagnitudes = new float[magnitudes.length / Float.BYTES]; + ByteBuffer byteBuffer = ByteBuffer.wrap(magnitudes.bytes, magnitudes.offset, magnitudes.length).order(ByteOrder.LITTLE_ENDIAN); + for (int i = 0; i < magnitudes.length / Float.BYTES; i++) { + multiMagnitudes[i] = byteBuffer.getFloat(); + } + return multiMagnitudes; + } + + public static void decodeMultiDenseVector(BytesRef vectorBR, int numVectors, float[][] multiVectorValue) { + if (vectorBR == null) { + throw new IllegalArgumentException(MultiDenseVectorScriptDocValues.MISSING_VECTOR_FIELD_MESSAGE); + } + FloatBuffer fb = ByteBuffer.wrap(vectorBR.bytes, vectorBR.offset, vectorBR.length).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer(); + for (int i = 0; i < numVectors; i++) { + fb.get(multiVectorValue[i]); + } + } + } diff --git a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java index 626875c75a5fe..83bca7d27aeeb 100644 --- a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java @@ -112,6 +112,13 @@ public QueryBuilder query() { return query; } + /** + * Returns path to the searched nested object. + */ + public String path() { + return path; + } + /** * Returns inner hit definition in the scope of this query and reusing the defined type and query. */ diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java b/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java index 70ba9950f7689..d8bd460f6f819 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java @@ -245,7 +245,11 @@ public ShardSnapshotResult getShardSnapshotResult() { } public void ensureNotAborted() { - switch (stage.get()) { + ensureNotAborted(stage.get()); + } + + public static void ensureNotAborted(Stage shardSnapshotStage) { + switch (shardSnapshotStage) { case ABORTED -> throw new AbortedSnapshotException(); case PAUSING -> throw new PausedSnapshotException(); } diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java index 308f1894b78db..c8d31d2060caf 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java @@ -397,6 +397,36 @@ record StartRecoveryRequestToSend(StartRecoveryRequest startRecoveryRequest, Str } indexShard.recoverLocallyUpToGlobalCheckpoint(ActionListener.assertOnce(l)); }) + // peer recovery can consume a lot of disk space, so it's worth cleaning up locally ahead of the attempt + // operation runs only if the previous operation succeeded, and returns the previous operation's result. + // Failures at this stage aren't fatal, we can attempt to recover and then clean up again at the end. #104473 + .andThenApply(startingSeqNo -> { + Store.MetadataSnapshot snapshot; + try { + snapshot = indexShard.snapshotStoreMetadata(); + } catch (IOException e) { + // We give up on the contents for any checked exception thrown by snapshotStoreMetadata. We don't want to + // allow those to bubble up and interrupt recovery because the subsequent recovery attempt is expected + // to fix up these problems for us if it completes successfully. + if (e instanceof org.apache.lucene.index.IndexNotFoundException) { + // this is the expected case on first recovery, so don't spam the logs with exceptions + logger.debug(() -> format("no snapshot found for shard %s, treating as empty", indexShard.shardId())); + } else { + logger.warn(() -> format("unable to load snapshot for shard %s, treating as empty", indexShard.shardId()), e); + } + snapshot = Store.MetadataSnapshot.EMPTY; + } + + Store store = indexShard.store(); + store.incRef(); + try { + logger.debug(() -> format("cleaning up index directory for %s before recovery", indexShard.shardId())); + store.cleanupAndVerify("cleanup before peer recovery", snapshot); + } finally { + store.decRef(); + } + return startingSeqNo; + }) // now construct the start-recovery request .andThenApply(startingSeqNo -> { assert startingSeqNo == UNASSIGNED_SEQ_NO || recoveryTarget.state().getStage() == RecoveryState.Stage.TRANSLOG diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index b43fe05a541f6..8c847da344fe5 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -191,6 +191,11 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent implements Repository { private static final Logger logger = LogManager.getLogger(BlobStoreRepository.class); + private class ShutdownLogger { + // Creating a separate logger so that the log-level can be manipulated separately from the parent class. + private static final Logger shutdownLogger = LogManager.getLogger(ShutdownLogger.class); + } + protected volatile RepositoryMetadata metadata; protected final ThreadPool threadPool; @@ -3467,10 +3472,37 @@ private void doSnapshotShard(SnapshotShardContext context) { } private static void ensureNotAborted(ShardId shardId, SnapshotId snapshotId, IndexShardSnapshotStatus snapshotStatus, String fileName) { + var shardSnapshotStage = snapshotStatus.getStage(); try { - snapshotStatus.ensureNotAborted(); + IndexShardSnapshotStatus.ensureNotAborted(shardSnapshotStage); + + if (shardSnapshotStage != IndexShardSnapshotStatus.Stage.INIT && shardSnapshotStage != IndexShardSnapshotStatus.Stage.STARTED) { + // A normally running shard snapshot should be in stage INIT or STARTED. And we know it's not in PAUSING or ABORTED because + // the ensureNotAborted() call above did not throw. The remaining options don't make sense, if they ever happen. + logger.error( + () -> Strings.format( + "Shard snapshot found an unexpected state. ShardId [{}], SnapshotID [{}], Stage [{}]", + shardId, + snapshotId, + shardSnapshotStage + ) + ); + assert false; + } } catch (Exception e) { - logger.debug("[{}] [{}] {} on the file [{}], exiting", shardId, snapshotId, e.getMessage(), fileName); + // We want to see when a shard snapshot operation checks for and finds an interrupt signal during shutdown. A + // PausedSnapshotException indicates we're in shutdown because that's the only case when shard snapshots are signaled to pause. + // An AbortedSnapshotException may also occur during shutdown if an uncommon error occurs. + ShutdownLogger.shutdownLogger.debug( + () -> Strings.format( + "Shard snapshot operation is aborting. ShardId [%s], SnapshotID [%s], File [%s], Stage [%s]", + shardId, + snapshotId, + fileName, + shardSnapshotStage + ), + e + ); assert e instanceof AbortedSnapshotException || e instanceof PausedSnapshotException : e; throw e; } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index 3bc1c467323a3..7b57481ad5716 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -38,6 +38,8 @@ private SearchCapabilities() {} private static final String MULTI_DENSE_VECTOR_FIELD_MAPPER = "multi_dense_vector_field_mapper"; /** Support propagating nested retrievers' inner_hits to top-level compound retrievers . */ private static final String NESTED_RETRIEVER_INNER_HITS_SUPPORT = "nested_retriever_inner_hits_support"; + /** Support multi-dense-vector script field access. */ + private static final String MULTI_DENSE_VECTOR_SCRIPT_ACCESS = "multi_dense_vector_script_access"; public static final Set CAPABILITIES; static { @@ -50,6 +52,7 @@ private SearchCapabilities() {} capabilities.add(NESTED_RETRIEVER_INNER_HITS_SUPPORT); if (MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()) { capabilities.add(MULTI_DENSE_VECTOR_FIELD_MAPPER); + capabilities.add(MULTI_DENSE_VECTOR_SCRIPT_ACCESS); } if (Build.current().isSnapshot()) { capabilities.add(KQL_QUERY_SUPPORTED); diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/BitMultiDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/BitMultiDenseVector.java new file mode 100644 index 0000000000000..24e19a803ff38 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/BitMultiDenseVector.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.script.field.vectors; + +import org.apache.lucene.util.BytesRef; + +import java.util.Iterator; + +public class BitMultiDenseVector extends ByteMultiDenseVector { + public BitMultiDenseVector(Iterator vectorValues, BytesRef magnitudesBytes, int numVecs, int dims) { + super(vectorValues, magnitudesBytes, numVecs, dims); + } + + @Override + public void checkDimensions(int qvDims) { + if (qvDims != dims) { + throw new IllegalArgumentException( + "The query vector has a different number of dimensions [" + + qvDims * Byte.SIZE + + "] than the document vectors [" + + dims * Byte.SIZE + + "]." + ); + } + } + + @Override + public int getDims() { + return dims * Byte.SIZE; + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/BitMultiDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/BitMultiDenseVectorDocValuesField.java new file mode 100644 index 0000000000000..35a43eabb8f0c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/BitMultiDenseVectorDocValuesField.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.script.field.vectors; + +import org.apache.lucene.index.BinaryDocValues; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; + +public class BitMultiDenseVectorDocValuesField extends ByteMultiDenseVectorDocValuesField { + + public BitMultiDenseVectorDocValuesField( + BinaryDocValues input, + BinaryDocValues magnitudes, + String name, + ElementType elementType, + int dims + ) { + super(input, magnitudes, name, elementType, dims / 8); + } + + @Override + protected MultiDenseVector getVector() { + return new BitMultiDenseVector(vectorValue, magnitudesValue, numVecs, dims); + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteMultiDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteMultiDenseVector.java new file mode 100644 index 0000000000000..e610d10146b2f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteMultiDenseVector.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.script.field.vectors; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.vectors.VectorEncoderDecoder; + +import java.util.Iterator; + +public class ByteMultiDenseVector implements MultiDenseVector { + + protected final Iterator vectorValues; + protected final int numVecs; + protected final int dims; + + private Iterator floatDocVectors; + private float[] magnitudes; + private final BytesRef magnitudesBytes; + + public ByteMultiDenseVector(Iterator vectorValues, BytesRef magnitudesBytes, int numVecs, int dims) { + assert magnitudesBytes.length == numVecs * Float.BYTES; + this.vectorValues = vectorValues; + this.numVecs = numVecs; + this.dims = dims; + this.magnitudesBytes = magnitudesBytes; + } + + @Override + public Iterator getVectors() { + if (floatDocVectors == null) { + floatDocVectors = new ByteToFloatIteratorWrapper(vectorValues, dims); + } + return floatDocVectors; + } + + @Override + public float[] getMagnitudes() { + if (magnitudes == null) { + magnitudes = VectorEncoderDecoder.getMultiMagnitudes(magnitudesBytes); + } + return magnitudes; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public int getDims() { + return dims; + } + + @Override + public int size() { + return numVecs; + } + + static class ByteToFloatIteratorWrapper implements Iterator { + private final Iterator byteIterator; + private final float[] buffer; + private final int dims; + + ByteToFloatIteratorWrapper(Iterator byteIterator, int dims) { + this.byteIterator = byteIterator; + this.buffer = new float[dims]; + this.dims = dims; + } + + @Override + public boolean hasNext() { + return byteIterator.hasNext(); + } + + @Override + public float[] next() { + byte[] next = byteIterator.next(); + for (int i = 0; i < dims; i++) { + buffer[i] = next[i]; + } + return buffer; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteMultiDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteMultiDenseVectorDocValuesField.java new file mode 100644 index 0000000000000..d1e062e0a3dee --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteMultiDenseVectorDocValuesField.java @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.script.field.vectors; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; +import org.elasticsearch.index.mapper.vectors.MultiDenseVectorScriptDocValues; + +import java.io.IOException; +import java.util.Iterator; + +public class ByteMultiDenseVectorDocValuesField extends MultiDenseVectorDocValuesField { + + protected final BinaryDocValues input; + private final BinaryDocValues magnitudes; + protected final int dims; + protected int numVecs; + protected Iterator vectorValue; + protected boolean decoded; + protected BytesRef value; + protected BytesRef magnitudesValue; + private byte[] buffer; + + public ByteMultiDenseVectorDocValuesField( + BinaryDocValues input, + BinaryDocValues magnitudes, + String name, + ElementType elementType, + int dims + ) { + super(name, elementType); + this.input = input; + this.dims = dims; + this.buffer = new byte[dims]; + this.magnitudes = magnitudes; + } + + @Override + public void setNextDocId(int docId) throws IOException { + decoded = false; + if (input.advanceExact(docId)) { + boolean magnitudesFound = magnitudes.advanceExact(docId); + assert magnitudesFound; + value = input.binaryValue(); + assert value.length % dims == 0; + numVecs = value.length / dims; + magnitudesValue = magnitudes.binaryValue(); + assert magnitudesValue.length == (numVecs * Float.BYTES); + } else { + value = null; + magnitudesValue = null; + vectorValue = null; + numVecs = 0; + } + } + + @Override + public MultiDenseVectorScriptDocValues toScriptDocValues() { + return new MultiDenseVectorScriptDocValues(this, dims); + } + + protected MultiDenseVector getVector() { + return new ByteMultiDenseVector(vectorValue, magnitudesValue, numVecs, dims); + } + + @Override + public MultiDenseVector get() { + if (isEmpty()) { + return MultiDenseVector.EMPTY; + } + decodeVectorIfNecessary(); + return getVector(); + } + + @Override + public MultiDenseVector get(MultiDenseVector defaultValue) { + if (isEmpty()) { + return defaultValue; + } + decodeVectorIfNecessary(); + return getVector(); + } + + @Override + public MultiDenseVector getInternal() { + return get(null); + } + + private void decodeVectorIfNecessary() { + if (decoded == false && value != null) { + vectorValue = new ByteVectorIterator(value, buffer, numVecs); + decoded = true; + } + } + + @Override + public int size() { + return value == null ? 0 : value.length / dims; + } + + @Override + public boolean isEmpty() { + return value == null; + } + + static class ByteVectorIterator implements Iterator { + private final byte[] buffer; + private final BytesRef vectorValues; + private final int size; + private int idx = 0; + + ByteVectorIterator(BytesRef vectorValues, byte[] buffer, int size) { + assert vectorValues.length == (buffer.length * size); + this.vectorValues = vectorValues; + this.size = size; + this.buffer = buffer; + } + + @Override + public boolean hasNext() { + return idx < size; + } + + @Override + public byte[] next() { + if (hasNext() == false) { + throw new IllegalArgumentException("No more elements in the iterator"); + } + System.arraycopy(vectorValues.bytes, vectorValues.offset + idx * buffer.length, buffer, 0, buffer.length); + idx++; + return buffer; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatMultiDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatMultiDenseVector.java new file mode 100644 index 0000000000000..9ffe8b3b970c4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatMultiDenseVector.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.script.field.vectors; + +import org.apache.lucene.util.BytesRef; + +import java.util.Iterator; + +import static org.elasticsearch.index.mapper.vectors.VectorEncoderDecoder.getMultiMagnitudes; + +public class FloatMultiDenseVector implements MultiDenseVector { + + private final BytesRef magnitudes; + private float[] magnitudesArray = null; + private final int dims; + private final int numVectors; + private final Iterator decodedDocVector; + + public FloatMultiDenseVector(Iterator decodedDocVector, BytesRef magnitudes, int numVectors, int dims) { + assert magnitudes.length == numVectors * Float.BYTES; + this.decodedDocVector = decodedDocVector; + this.magnitudes = magnitudes; + this.numVectors = numVectors; + this.dims = dims; + } + + @Override + public Iterator getVectors() { + return decodedDocVector; + } + + @Override + public float[] getMagnitudes() { + if (magnitudesArray == null) { + magnitudesArray = getMultiMagnitudes(magnitudes); + } + return magnitudesArray; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public int getDims() { + return dims; + } + + @Override + public int size() { + return numVectors; + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatMultiDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatMultiDenseVectorDocValuesField.java new file mode 100644 index 0000000000000..356db58d989c5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatMultiDenseVectorDocValuesField.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.script.field.vectors; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; +import org.elasticsearch.index.mapper.vectors.MultiDenseVectorScriptDocValues; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; +import java.util.Iterator; + +public class FloatMultiDenseVectorDocValuesField extends MultiDenseVectorDocValuesField { + + private final BinaryDocValues input; + private final BinaryDocValues magnitudes; + private boolean decoded; + private final int dims; + private BytesRef value; + private BytesRef magnitudesValue; + private FloatVectorIterator vectorValues; + private int numVectors; + private float[] buffer; + + public FloatMultiDenseVectorDocValuesField( + BinaryDocValues input, + BinaryDocValues magnitudes, + String name, + ElementType elementType, + int dims + ) { + super(name, elementType); + this.input = input; + this.magnitudes = magnitudes; + this.dims = dims; + this.buffer = new float[dims]; + } + + @Override + public void setNextDocId(int docId) throws IOException { + decoded = false; + if (input.advanceExact(docId)) { + boolean magnitudesFound = magnitudes.advanceExact(docId); + assert magnitudesFound; + + value = input.binaryValue(); + assert value.length % (Float.BYTES * dims) == 0; + numVectors = value.length / (Float.BYTES * dims); + magnitudesValue = magnitudes.binaryValue(); + assert magnitudesValue.length == (Float.BYTES * numVectors); + } else { + value = null; + magnitudesValue = null; + numVectors = 0; + } + } + + @Override + public MultiDenseVectorScriptDocValues toScriptDocValues() { + return new MultiDenseVectorScriptDocValues(this, dims); + } + + @Override + public boolean isEmpty() { + return value == null; + } + + @Override + public MultiDenseVector get() { + if (isEmpty()) { + return MultiDenseVector.EMPTY; + } + decodeVectorIfNecessary(); + return new FloatMultiDenseVector(vectorValues, magnitudesValue, numVectors, dims); + } + + @Override + public MultiDenseVector get(MultiDenseVector defaultValue) { + if (isEmpty()) { + return defaultValue; + } + decodeVectorIfNecessary(); + return new FloatMultiDenseVector(vectorValues, magnitudesValue, numVectors, dims); + } + + @Override + public MultiDenseVector getInternal() { + return get(null); + } + + @Override + public int size() { + return value == null ? 0 : value.length / (Float.BYTES * dims); + } + + private void decodeVectorIfNecessary() { + if (decoded == false && value != null) { + vectorValues = new FloatVectorIterator(value, buffer, numVectors); + decoded = true; + } + } + + static class FloatVectorIterator implements Iterator { + private final float[] buffer; + private final FloatBuffer vectorValues; + private final int size; + private int idx = 0; + + FloatVectorIterator(BytesRef vectorValues, float[] buffer, int size) { + assert vectorValues.length == (buffer.length * Float.BYTES * size); + this.vectorValues = ByteBuffer.wrap(vectorValues.bytes, vectorValues.offset, vectorValues.length) + .order(ByteOrder.LITTLE_ENDIAN) + .asFloatBuffer(); + this.size = size; + this.buffer = buffer; + } + + @Override + public boolean hasNext() { + return idx < size; + } + + @Override + public float[] next() { + if (hasNext() == false) { + throw new IllegalArgumentException("No more elements in the iterator"); + } + vectorValues.get(buffer); + idx++; + return buffer; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/MultiDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/MultiDenseVector.java new file mode 100644 index 0000000000000..85c851dbe545c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/MultiDenseVector.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.script.field.vectors; + +import java.util.Iterator; + +public interface MultiDenseVector { + + default void checkDimensions(int qvDims) { + checkDimensions(getDims(), qvDims); + } + + Iterator getVectors(); + + float[] getMagnitudes(); + + boolean isEmpty(); + + int getDims(); + + int size(); + + static void checkDimensions(int dvDims, int qvDims) { + if (dvDims != qvDims) { + throw new IllegalArgumentException( + "The query vector has a different number of dimensions [" + qvDims + "] than the document vectors [" + dvDims + "]." + ); + } + } + + private static String badQueryVectorType(Object queryVector) { + return "Cannot use vector [" + queryVector + "] with class [" + queryVector.getClass().getName() + "] as query vector"; + } + + MultiDenseVector EMPTY = new MultiDenseVector() { + public static final String MISSING_VECTOR_FIELD_MESSAGE = "Multi Dense vector value missing for a field," + + " use isEmpty() to check for a missing vector value"; + + @Override + public Iterator getVectors() { + throw new IllegalArgumentException(MISSING_VECTOR_FIELD_MESSAGE); + } + + @Override + public float[] getMagnitudes() { + throw new IllegalArgumentException(MISSING_VECTOR_FIELD_MESSAGE); + } + + @Override + public boolean isEmpty() { + return true; + } + + @Override + public int getDims() { + throw new IllegalArgumentException(MISSING_VECTOR_FIELD_MESSAGE); + } + + @Override + public int size() { + return 0; + } + }; +} diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/MultiDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/MultiDenseVectorDocValuesField.java new file mode 100644 index 0000000000000..61ae4304683c8 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/MultiDenseVectorDocValuesField.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.script.field.vectors; + +import org.elasticsearch.index.mapper.vectors.MultiDenseVectorScriptDocValues; +import org.elasticsearch.script.field.AbstractScriptFieldFactory; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.Field; + +import java.util.Iterator; + +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; + +public abstract class MultiDenseVectorDocValuesField extends AbstractScriptFieldFactory + implements + Field, + DocValuesScriptFieldFactory, + MultiDenseVectorScriptDocValues.MultiDenseVectorSupplier { + protected final String name; + protected final ElementType elementType; + + public MultiDenseVectorDocValuesField(String name, ElementType elementType) { + this.name = name; + this.elementType = elementType; + } + + @Override + public String getName() { + return name; + } + + public ElementType getElementType() { + return elementType; + } + + /** + * Get the DenseVector for a document if one exists, DenseVector.EMPTY otherwise + */ + public abstract MultiDenseVector get(); + + public abstract MultiDenseVector get(MultiDenseVector defaultValue); + + public abstract MultiDenseVectorScriptDocValues toScriptDocValues(); + + // DenseVector fields are single valued, so Iterable does not make sense. + @Override + public Iterator iterator() { + throw new UnsupportedOperationException("Cannot iterate over single valued multi_dense_vector field, use get() instead"); + } +} diff --git a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification index 5cd8935f72403..37b9b5836ca5f 100644 --- a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -7,6 +7,7 @@ # License v3.0 only", or the "Server Side Public License, v 1". # +org.elasticsearch.action.admin.indices.stats.IndicesStatsFeatures org.elasticsearch.action.bulk.BulkFeatures org.elasticsearch.features.FeatureInfrastructureFeatures org.elasticsearch.health.HealthFeatures diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java index 0b9cba837583d..5cf7b438b41ab 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java @@ -42,6 +42,7 @@ import java.io.IOException; import java.io.Reader; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -250,6 +251,32 @@ public void testFillsAttributes() throws IOException { assertEquals("", tokens.get(3).getType()); } + public void testAnalyzerWithTwoTextsAndNoIndexName() throws IOException { + AnalyzeAction.Request request = new AnalyzeAction.Request(); + + for (String analyzer : Arrays.asList("standard", "simple", "stop", "keyword", "whitespace", "classic")) { + request.analyzer(analyzer); + request.text("a a", "b b"); + + AnalyzeAction.Response analyzeIndex = TransportAnalyzeAction.analyze(request, registry, mockIndexService(), maxTokenCount); + List tokensIndex = analyzeIndex.getTokens(); + + AnalyzeAction.Response analyzeNoIndex = TransportAnalyzeAction.analyze(request, registry, null, maxTokenCount); + List tokensNoIndex = analyzeNoIndex.getTokens(); + + assertEquals(tokensIndex.size(), tokensNoIndex.size()); + for (int i = 0; i < tokensIndex.size(); i++) { + AnalyzeAction.AnalyzeToken withIndex = tokensIndex.get(i); + AnalyzeAction.AnalyzeToken withNoIndex = tokensNoIndex.get(i); + + assertEquals(withIndex.getStartOffset(), withNoIndex.getStartOffset()); + assertEquals(withIndex.getEndOffset(), withNoIndex.getEndOffset()); + assertEquals(withIndex.getPosition(), withNoIndex.getPosition()); + assertEquals(withIndex.getType(), withNoIndex.getType()); + } + } + } + public void testWithIndexAnalyzers() throws IOException { AnalyzeAction.Request request = new AnalyzeAction.Request(); request.text("the quick brown fox"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValuesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValuesTests.java new file mode 100644 index 0000000000000..ef316c5addefa --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValuesTests.java @@ -0,0 +1,374 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.vectors; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; +import org.elasticsearch.script.field.vectors.ByteMultiDenseVectorDocValuesField; +import org.elasticsearch.script.field.vectors.FloatMultiDenseVectorDocValuesField; +import org.elasticsearch.script.field.vectors.MultiDenseVector; +import org.elasticsearch.script.field.vectors.MultiDenseVectorDocValuesField; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.index.IndexVersionUtils; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Iterator; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; + +public class MultiDenseVectorScriptDocValuesTests extends ESTestCase { + + public void testFloatGetVectorValueAndGetMagnitude() throws IOException { + int dims = 3; + float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; + float[][] expectedMagnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; + + for (IndexVersion indexVersion : List.of(IndexVersionUtils.randomCompatibleVersion(random()), IndexVersion.current())) { + BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT, indexVersion); + BinaryDocValues magnitudeValues = wrap(expectedMagnitudes); + MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( + docValues, + magnitudeValues, + "test", + ElementType.FLOAT, + dims + ); + MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + for (int i = 0; i < vectors.length; i++) { + field.setNextDocId(i); + assertEquals(vectors[i].length, field.size()); + assertEquals(dims, scriptDocValues.dims()); + Iterator iterator = scriptDocValues.getVectorValues(); + float[] magnitudes = scriptDocValues.getMagnitudes(); + assertEquals(expectedMagnitudes[i].length, magnitudes.length); + for (int j = 0; j < vectors[i].length; j++) { + assertTrue(iterator.hasNext()); + assertArrayEquals(vectors[i][j], iterator.next(), 0.0001f); + assertEquals(expectedMagnitudes[i][j], magnitudes[j], 0.0001f); + } + } + } + } + + public void testByteGetVectorValueAndGetMagnitude() throws IOException { + int dims = 3; + float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; + float[][] expectedMagnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; + + BinaryDocValues docValues = wrap(vectors, ElementType.BYTE, IndexVersion.current()); + BinaryDocValues magnitudeValues = wrap(expectedMagnitudes); + MultiDenseVectorDocValuesField field = new ByteMultiDenseVectorDocValuesField( + docValues, + magnitudeValues, + "test", + ElementType.BYTE, + dims + ); + MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + for (int i = 0; i < vectors.length; i++) { + field.setNextDocId(i); + assertEquals(vectors[i].length, field.size()); + assertEquals(dims, scriptDocValues.dims()); + Iterator iterator = scriptDocValues.getVectorValues(); + float[] magnitudes = scriptDocValues.getMagnitudes(); + assertEquals(expectedMagnitudes[i].length, magnitudes.length); + for (int j = 0; j < vectors[i].length; j++) { + assertTrue(iterator.hasNext()); + assertArrayEquals(vectors[i][j], iterator.next(), 0.0001f); + assertEquals(expectedMagnitudes[i][j], magnitudes[j], 0.0001f); + } + } + } + + public void testFloatMetadataAndIterator() throws IOException { + int dims = 3; + IndexVersion indexVersion = IndexVersion.current(); + float[][][] vectors = new float[][][] { fill(new float[3][dims], ElementType.FLOAT), fill(new float[2][dims], ElementType.FLOAT) }; + float[][] magnitudes = new float[][] { new float[3], new float[2] }; + BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT, indexVersion); + BinaryDocValues magnitudeValues = wrap(magnitudes); + + MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( + docValues, + magnitudeValues, + "test", + ElementType.FLOAT, + dims + ); + for (int i = 0; i < vectors.length; i++) { + field.setNextDocId(i); + MultiDenseVector dv = field.get(); + assertEquals(vectors[i].length, dv.size()); + assertFalse(dv.isEmpty()); + assertEquals(dims, dv.getDims()); + UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, field::iterator); + assertEquals("Cannot iterate over single valued multi_dense_vector field, use get() instead", e.getMessage()); + } + field.setNextDocId(vectors.length); + MultiDenseVector dv = field.get(); + assertEquals(dv, MultiDenseVector.EMPTY); + } + + public void testByteMetadataAndIterator() throws IOException { + int dims = 3; + IndexVersion indexVersion = IndexVersion.current(); + float[][][] vectors = new float[][][] { fill(new float[3][dims], ElementType.BYTE), fill(new float[2][dims], ElementType.BYTE) }; + float[][] magnitudes = new float[][] { new float[3], new float[2] }; + BinaryDocValues docValues = wrap(vectors, ElementType.BYTE, indexVersion); + BinaryDocValues magnitudeValues = wrap(magnitudes); + MultiDenseVectorDocValuesField field = new ByteMultiDenseVectorDocValuesField( + docValues, + magnitudeValues, + "test", + ElementType.BYTE, + dims + ); + for (int i = 0; i < vectors.length; i++) { + field.setNextDocId(i); + MultiDenseVector dv = field.get(); + assertEquals(vectors[i].length, dv.size()); + assertFalse(dv.isEmpty()); + assertEquals(dims, dv.getDims()); + UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, field::iterator); + assertEquals("Cannot iterate over single valued multi_dense_vector field, use get() instead", e.getMessage()); + } + field.setNextDocId(vectors.length); + MultiDenseVector dv = field.get(); + assertEquals(dv, MultiDenseVector.EMPTY); + } + + protected float[][] fill(float[][] vectors, ElementType elementType) { + for (float[] vector : vectors) { + for (int i = 0; i < vector.length; i++) { + vector[i] = elementType == ElementType.FLOAT ? randomFloat() : randomByte(); + } + } + return vectors; + } + + public void testFloatMissingValues() throws IOException { + int dims = 3; + float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; + float[][] magnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; + BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT, IndexVersion.current()); + BinaryDocValues magnitudeValues = wrap(magnitudes); + MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( + docValues, + magnitudeValues, + "test", + ElementType.FLOAT, + dims + ); + MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + + field.setNextDocId(3); + assertEquals(0, field.size()); + Exception e = expectThrows(IllegalArgumentException.class, scriptDocValues::getVectorValues); + assertEquals("A document doesn't have a value for a multi-vector field!", e.getMessage()); + + e = expectThrows(IllegalArgumentException.class, scriptDocValues::getMagnitudes); + assertEquals("A document doesn't have a value for a multi-vector field!", e.getMessage()); + } + + public void testByteMissingValues() throws IOException { + int dims = 3; + float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; + float[][] magnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; + BinaryDocValues docValues = wrap(vectors, ElementType.BYTE, IndexVersion.current()); + BinaryDocValues magnitudeValues = wrap(magnitudes); + MultiDenseVectorDocValuesField field = new ByteMultiDenseVectorDocValuesField( + docValues, + magnitudeValues, + "test", + ElementType.BYTE, + dims + ); + MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + + field.setNextDocId(3); + assertEquals(0, field.size()); + Exception e = expectThrows(IllegalArgumentException.class, scriptDocValues::getVectorValues); + assertEquals("A document doesn't have a value for a multi-vector field!", e.getMessage()); + + e = expectThrows(IllegalArgumentException.class, scriptDocValues::getMagnitudes); + assertEquals("A document doesn't have a value for a multi-vector field!", e.getMessage()); + } + + public void testFloatGetFunctionIsNotAccessible() throws IOException { + int dims = 3; + float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; + float[][] magnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; + BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT, IndexVersion.current()); + BinaryDocValues magnitudeValues = wrap(magnitudes); + MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( + docValues, + magnitudeValues, + "test", + ElementType.FLOAT, + dims + ); + MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + + field.setNextDocId(0); + Exception e = expectThrows(UnsupportedOperationException.class, () -> scriptDocValues.get(0)); + assertThat( + e.getMessage(), + containsString( + "accessing a multi-vector field's value through 'get' or 'value' is not supported," + + " use 'vectorValues' or 'magnitudes' instead." + ) + ); + } + + public void testByteGetFunctionIsNotAccessible() throws IOException { + int dims = 3; + float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; + float[][] magnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; + BinaryDocValues docValues = wrap(vectors, ElementType.BYTE, IndexVersion.current()); + BinaryDocValues magnitudeValues = wrap(magnitudes); + MultiDenseVectorDocValuesField field = new ByteMultiDenseVectorDocValuesField( + docValues, + magnitudeValues, + "test", + ElementType.BYTE, + dims + ); + MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + + field.setNextDocId(0); + Exception e = expectThrows(UnsupportedOperationException.class, () -> scriptDocValues.get(0)); + assertThat( + e.getMessage(), + containsString( + "accessing a multi-vector field's value through 'get' or 'value' is not supported," + + " use 'vectorValues' or 'magnitudes' instead." + ) + ); + } + + public static BinaryDocValues wrap(float[][] magnitudes) { + return new BinaryDocValues() { + int idx = -1; + int maxIdx = magnitudes.length; + + @Override + public BytesRef binaryValue() { + if (idx >= maxIdx) { + throw new IllegalStateException("max index exceeded"); + } + ByteBuffer magnitudeBuffer = ByteBuffer.allocate(magnitudes[idx].length * Float.BYTES).order(ByteOrder.LITTLE_ENDIAN); + for (float magnitude : magnitudes[idx]) { + magnitudeBuffer.putFloat(magnitude); + } + return new BytesRef(magnitudeBuffer.array()); + } + + @Override + public boolean advanceExact(int target) { + idx = target; + if (target < maxIdx) { + return true; + } + return false; + } + + @Override + public int docID() { + return idx; + } + + @Override + public int nextDoc() { + return idx++; + } + + @Override + public int advance(int target) { + throw new IllegalArgumentException("not defined!"); + } + + @Override + public long cost() { + throw new IllegalArgumentException("not defined!"); + } + }; + } + + public static BinaryDocValues wrap(float[][][] vectors, ElementType elementType, IndexVersion indexVersion) { + return new BinaryDocValues() { + int idx = -1; + int maxIdx = vectors.length; + + @Override + public BytesRef binaryValue() { + if (idx >= maxIdx) { + throw new IllegalStateException("max index exceeded"); + } + return mockEncodeDenseVector(vectors[idx], elementType, indexVersion); + } + + @Override + public boolean advanceExact(int target) { + idx = target; + if (target < maxIdx) { + return true; + } + return false; + } + + @Override + public int docID() { + return idx; + } + + @Override + public int nextDoc() { + return idx++; + } + + @Override + public int advance(int target) { + throw new IllegalArgumentException("not defined!"); + } + + @Override + public long cost() { + throw new IllegalArgumentException("not defined!"); + } + }; + } + + public static BytesRef mockEncodeDenseVector(float[][] values, ElementType elementType, IndexVersion indexVersion) { + int dims = values[0].length; + if (elementType == ElementType.BIT) { + dims *= Byte.SIZE; + } + int numBytes = elementType.getNumBytes(dims); + ByteBuffer byteBuffer = elementType.createByteBuffer(indexVersion, numBytes * values.length); + for (float[] vector : values) { + for (float value : vector) { + if (elementType == ElementType.FLOAT) { + byteBuffer.putFloat(value); + } else if (elementType == ElementType.BYTE || elementType == ElementType.BIT) { + byteBuffer.put((byte) value); + } else { + throw new IllegalStateException("unknown element_type [" + elementType + "]"); + } + } + } + return new BytesRef(byteBuffer.array()); + } + +} diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesModuleTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesModuleTests.java index aa47f3c066f57..7f6885e7a977f 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesModuleTests.java @@ -65,14 +65,26 @@ public void setUp() throws Exception { } public void testCanRegisterTwoRepositoriesWithDifferentTypes() { - when(plugin1.getRepositories(eq(environment), eq(contentRegistry), eq(clusterService), - eq(MockBigArrays.NON_RECYCLING_INSTANCE), eq(recoverySettings), - any(RepositoriesMetrics.class))) - .thenReturn(Collections.singletonMap("type1", factory)); - when(plugin2.getRepositories(eq(environment), eq(contentRegistry), eq(clusterService), - eq(MockBigArrays.NON_RECYCLING_INSTANCE), eq(recoverySettings), - any(RepositoriesMetrics.class))) - .thenReturn(Collections.singletonMap("type2", factory)); + when( + plugin1.getRepositories( + eq(environment), + eq(contentRegistry), + eq(clusterService), + eq(MockBigArrays.NON_RECYCLING_INSTANCE), + eq(recoverySettings), + any(RepositoriesMetrics.class) + ) + ).thenReturn(Collections.singletonMap("type1", factory)); + when( + plugin2.getRepositories( + eq(environment), + eq(contentRegistry), + eq(clusterService), + eq(MockBigArrays.NON_RECYCLING_INSTANCE), + eq(recoverySettings), + any(RepositoriesMetrics.class) + ) + ).thenReturn(Collections.singletonMap("type2", factory)); // Would throw new RepositoriesModule( @@ -83,18 +95,32 @@ public void testCanRegisterTwoRepositoriesWithDifferentTypes() { mock(ClusterService.class), MockBigArrays.NON_RECYCLING_INSTANCE, contentRegistry, - recoverySettings, TelemetryProvider.NOOP); + recoverySettings, + TelemetryProvider.NOOP + ); } public void testCannotRegisterTwoRepositoriesWithSameTypes() { - when(plugin1.getRepositories(eq(environment), eq(contentRegistry), eq(clusterService), - eq(MockBigArrays.NON_RECYCLING_INSTANCE), eq(recoverySettings), - any(RepositoriesMetrics.class))) - .thenReturn(Collections.singletonMap("type1", factory)); - when(plugin2.getRepositories(eq(environment), eq(contentRegistry), eq(clusterService), - eq(MockBigArrays.NON_RECYCLING_INSTANCE), eq(recoverySettings), - any(RepositoriesMetrics.class))) - .thenReturn(Collections.singletonMap("type1", factory)); + when( + plugin1.getRepositories( + eq(environment), + eq(contentRegistry), + eq(clusterService), + eq(MockBigArrays.NON_RECYCLING_INSTANCE), + eq(recoverySettings), + any(RepositoriesMetrics.class) + ) + ).thenReturn(Collections.singletonMap("type1", factory)); + when( + plugin2.getRepositories( + eq(environment), + eq(contentRegistry), + eq(clusterService), + eq(MockBigArrays.NON_RECYCLING_INSTANCE), + eq(recoverySettings), + any(RepositoriesMetrics.class) + ) + ).thenReturn(Collections.singletonMap("type1", factory)); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, @@ -106,7 +132,9 @@ public void testCannotRegisterTwoRepositoriesWithSameTypes() { clusterService, MockBigArrays.NON_RECYCLING_INSTANCE, contentRegistry, - recoverySettings, TelemetryProvider.NOOP) + recoverySettings, + TelemetryProvider.NOOP + ) ); assertEquals("Repository type [type1] is already registered", ex.getMessage()); @@ -130,17 +158,25 @@ public void testCannotRegisterTwoInternalRepositoriesWithSameTypes() { clusterService, MockBigArrays.NON_RECYCLING_INSTANCE, contentRegistry, - recoverySettings, TelemetryProvider.NOOP) + recoverySettings, + TelemetryProvider.NOOP + ) ); assertEquals("Internal repository type [type1] is already registered", ex.getMessage()); } public void testCannotRegisterNormalAndInternalRepositoriesWithSameTypes() { - when(plugin1.getRepositories(eq(environment), eq(contentRegistry), eq(clusterService), - eq(MockBigArrays.NON_RECYCLING_INSTANCE), eq(recoverySettings), - any(RepositoriesMetrics.class))) - .thenReturn(Collections.singletonMap("type1", factory)); + when( + plugin1.getRepositories( + eq(environment), + eq(contentRegistry), + eq(clusterService), + eq(MockBigArrays.NON_RECYCLING_INSTANCE), + eq(recoverySettings), + any(RepositoriesMetrics.class) + ) + ).thenReturn(Collections.singletonMap("type1", factory)); when(plugin2.getInternalRepositories(environment, contentRegistry, clusterService, recoverySettings)).thenReturn( Collections.singletonMap("type1", factory) ); @@ -155,7 +191,9 @@ public void testCannotRegisterNormalAndInternalRepositoriesWithSameTypes() { clusterService, MockBigArrays.NON_RECYCLING_INSTANCE, contentRegistry, - recoverySettings, TelemetryProvider.NOOP) + recoverySettings, + TelemetryProvider.NOOP + ) ); assertEquals("Internal repository type [type1] is already registered as a non-internal repository", ex.getMessage()); diff --git a/settings.gradle b/settings.gradle index 54a9514490db0..d04d45bffc3ad 100644 --- a/settings.gradle +++ b/settings.gradle @@ -4,9 +4,6 @@ import org.elasticsearch.gradle.internal.toolchain.AdoptiumJdkToolchainResolver pluginManagement { repositories { - maven { - url 'https://jitpack.io' - } mavenCentral() gradlePluginPortal() } diff --git a/test/external-modules/apm-integration/build.gradle b/test/external-modules/apm-integration/build.gradle index d0f5f889e9b30..91e01d363749c 100644 --- a/test/external-modules/apm-integration/build.gradle +++ b/test/external-modules/apm-integration/build.gradle @@ -17,7 +17,7 @@ tasks.named("test").configure { } tasks.named('javaRestTest').configure { - it.onlyIf("snapshot build") { BuildParams.isSnapshotBuild() } + it.onlyIf("snapshot build") { buildParams.isSnapshotBuild() } } dependencies { diff --git a/test/external-modules/delayed-aggs/build.gradle b/test/external-modules/delayed-aggs/build.gradle index c17fa64f4d127..f57bd37d65171 100644 --- a/test/external-modules/delayed-aggs/build.gradle +++ b/test/external-modules/delayed-aggs/build.gradle @@ -11,7 +11,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.legacy-yaml-rest-test' tasks.named('yamlRestTest').configure { - it.onlyIf("snapshot build") { BuildParams.isSnapshotBuild() } + it.onlyIf("snapshot build") { buildParams.isSnapshotBuild() } } esplugin { diff --git a/test/external-modules/die-with-dignity/build.gradle b/test/external-modules/die-with-dignity/build.gradle index 34a9a71533d3c..1f98e43052589 100644 --- a/test/external-modules/die-with-dignity/build.gradle +++ b/test/external-modules/die-with-dignity/build.gradle @@ -17,5 +17,5 @@ tasks.named("test").configure { } tasks.named('javaRestTest').configure { - it.onlyIf("snapshot build") { BuildParams.isSnapshotBuild() } + it.onlyIf("snapshot build") { buildParams.isSnapshotBuild() } } diff --git a/test/external-modules/error-query/build.gradle b/test/external-modules/error-query/build.gradle index de9101364ca02..ff4783552ebf5 100644 --- a/test/external-modules/error-query/build.gradle +++ b/test/external-modules/error-query/build.gradle @@ -11,7 +11,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.legacy-yaml-rest-test' tasks.named('yamlRestTest').configure { - it.onlyIf("snapshot build") { BuildParams.isSnapshotBuild() } + it.onlyIf("snapshot build") { buildParams.isSnapshotBuild() } } esplugin { diff --git a/test/external-modules/esql-heap-attack/build.gradle b/test/external-modules/esql-heap-attack/build.gradle index 2276766d52df7..3d6291f6d011a 100644 --- a/test/external-modules/esql-heap-attack/build.gradle +++ b/test/external-modules/esql-heap-attack/build.gradle @@ -21,5 +21,5 @@ esplugin { tasks.named('javaRestTest') { usesDefaultDistribution() - it.onlyIf("snapshot build") { BuildParams.isSnapshotBuild() } + it.onlyIf("snapshot build") { buildParams.isSnapshotBuild() } } diff --git a/test/external-modules/jvm-crash/build.gradle b/test/external-modules/jvm-crash/build.gradle index c1344d570a33b..73ad8b851a220 100644 --- a/test/external-modules/jvm-crash/build.gradle +++ b/test/external-modules/jvm-crash/build.gradle @@ -21,5 +21,5 @@ esplugin { tasks.named('javaRestTest') { usesDefaultDistribution() - it.onlyIf("snapshot build") { BuildParams.isSnapshotBuild() } + it.onlyIf("snapshot build") { buildParams.isSnapshotBuild() } } diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 8911bdf286a0f..9dc0263f49aee 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -8,7 +8,7 @@ */ apply plugin: 'elasticsearch.java' -apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'com.gradleup.shadow' import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar diff --git a/test/framework/build.gradle b/test/framework/build.gradle index f130ecf131848..c61a3b1851ea9 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -85,9 +85,9 @@ tasks.named("thirdPartyAudit").configure { } tasks.named("test").configure { - systemProperty 'tests.gradle_index_compat_versions', BuildParams.bwcVersions.indexCompatible.join(',') - systemProperty 'tests.gradle_wire_compat_versions', BuildParams.bwcVersions.wireCompatible.join(',') - systemProperty 'tests.gradle_unreleased_versions', BuildParams.bwcVersions.unreleased.join(',') + systemProperty 'tests.gradle_index_compat_versions', buildParams.bwcVersions.indexCompatible.join(',') + systemProperty 'tests.gradle_wire_compat_versions', buildParams.bwcVersions.wireCompatible.join(',') + systemProperty 'tests.gradle_unreleased_versions', buildParams.bwcVersions.unreleased.join(',') } tasks.register("integTest", Test) { diff --git a/test/immutable-collections-patch/build.gradle b/test/immutable-collections-patch/build.gradle index 28aad96754629..381c0cd6dd044 100644 --- a/test/immutable-collections-patch/build.gradle +++ b/test/immutable-collections-patch/build.gradle @@ -26,14 +26,14 @@ def outputDir = layout.buildDirectory.dir("jdk-patches") def generatePatch = tasks.register("generatePatch", JavaExec) generatePatch.configure { dependsOn tasks.named("compileJava") - inputs.property("java-home-set", BuildParams.getIsRuntimeJavaHomeSet()) - inputs.property("java-version", BuildParams.runtimeJavaVersion) + inputs.property("java-home-set", buildParams.getIsRuntimeJavaHomeSet()) + inputs.property("java-version", buildParams.runtimeJavaVersion) outputs.dir(outputDir) classpath = sourceSets.main.runtimeClasspath mainClass = 'org.elasticsearch.jdk.patch.ImmutableCollectionsPatcher' - if (BuildParams.getIsRuntimeJavaHomeSet()) { - executable = "${BuildParams.runtimeJavaHome}/bin/java" + (OS.current() == OS.WINDOWS ? '.exe' : '') + if (buildParams.getIsRuntimeJavaHomeSet()) { + executable = "${buildParams.runtimeJavaHome.get()}/bin/java" + (OS.current() == OS.WINDOWS ? '.exe' : '') } else { javaLauncher = javaToolchains.launcherFor { languageVersion = JavaLanguageVersion.of(VersionProperties.bundledJdkMajorVersion) diff --git a/x-pack/libs/es-opensaml-security-api/build.gradle b/x-pack/libs/es-opensaml-security-api/build.gradle index b36d0bfa7b37d..3b4434ec5d9e5 100644 --- a/x-pack/libs/es-opensaml-security-api/build.gradle +++ b/x-pack/libs/es-opensaml-security-api/build.gradle @@ -7,7 +7,7 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' -apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'com.gradleup.shadow' dependencies { implementation "org.opensaml:opensaml-security-api:${versions.opensaml}" diff --git a/x-pack/plugin/analytics/build.gradle b/x-pack/plugin/analytics/build.gradle index c451df58b9fab..ddc075cc9adcc 100644 --- a/x-pack/plugin/analytics/build.gradle +++ b/x-pack/plugin/analytics/build.gradle @@ -21,7 +21,7 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/x-pack/plugin/async-search/qa/rest/build.gradle b/x-pack/plugin/async-search/qa/rest/build.gradle index 4fc557a5b6048..c950646930779 100644 --- a/x-pack/plugin/async-search/qa/rest/build.gradle +++ b/x-pack/plugin/async-search/qa/rest/build.gradle @@ -28,5 +28,5 @@ testClusters.configureEach { // Test clusters run with security disabled tasks.named("yamlRestTest") { - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) } diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 193a82436f26a..e25d7fb359acb 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -37,7 +37,7 @@ artifacts { def restTestBlacklist = [] // TODO: fix this rest test to not depend on a hardcoded port! restTestBlacklist.addAll(['getting_started/10_monitor_cluster_health/*']) -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { // these tests attempt to install basic/internal licenses signed against the dev/public.key // Since there is no infrastructure in place (anytime soon) to generate licenses using the production // private key, these tests are blacklisted in non-snapshot test runs diff --git a/x-pack/plugin/ccr/qa/build.gradle b/x-pack/plugin/ccr/qa/build.gradle index 583ad5d8c3df3..4be504e616920 100644 --- a/x-pack/plugin/ccr/qa/build.gradle +++ b/x-pack/plugin/ccr/qa/build.gradle @@ -10,6 +10,6 @@ subprojects { tasks.withType(Test).configureEach { // These fail in CI but only when run as part of checkPart2 and not individually. // Tracked in : https://github.com/elastic/elasticsearch/issues/66661 - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) } } diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle index fb4acb0055a8c..b4f17cb436df5 100644 --- a/x-pack/plugin/core/build.gradle +++ b/x-pack/plugin/core/build.gradle @@ -94,7 +94,7 @@ tasks.named("processResources").configure { String licenseKey = providers.systemProperty("license.key").getOrNull() if (licenseKey != null) { println "Using provided license key from ${licenseKey}" - } else if (BuildParams.isSnapshotBuild()) { + } else if (buildParams.isSnapshotBuild()) { licenseKey = Paths.get(project.projectDir.path, 'snapshot.key') } else { throw new IllegalArgumentException('Property license.key must be set for release build') @@ -155,13 +155,13 @@ testClusters.configureEach { requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.15.0") } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.withType(Test).configureEach { systemProperty 'es.failure_store_feature_flag_enabled', 'true' } } -if (BuildParams.inFipsJvm) { +if (buildParams.inFipsJvm) { // Test clusters run with security disabled tasks.named("javaRestTest").configure { enabled = false } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForNoFollowersStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForNoFollowersStepTests.java index 01a12fb795316..21e3155501995 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForNoFollowersStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForNoFollowersStepTests.java @@ -189,7 +189,10 @@ public void testNoShardStats() { shardStats[0] = sStats; mockXPackInfo(true, true); - mockIndexStatsCall(indexName, new IndexStats(indexName, "uuid", ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN, shardStats)); + mockIndexStatsCall( + indexName, + new IndexStats(indexName, "uuid", ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN, null, null, shardStats) + ); final SetOnce conditionMetHolder = new SetOnce<>(); final SetOnce stepInfoHolder = new SetOnce<>(); @@ -289,7 +292,7 @@ private IndexStats randomIndexStats(boolean isLeaderIndex, int numOfShards) { for (int i = 0; i < numOfShards; i++) { shardStats[i] = randomShardStats(isLeaderIndex); } - return new IndexStats(randomAlphaOfLength(5), randomAlphaOfLength(10), null, null, shardStats); + return new IndexStats(randomAlphaOfLength(5), randomAlphaOfLength(10), null, null, null, null, shardStats); } private ShardStats randomShardStats(boolean isLeaderIndex) { diff --git a/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle b/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle index 2d8859bdcea3d..a9580f4e14d6b 100644 --- a/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle +++ b/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle @@ -37,6 +37,6 @@ testClusters.configureEach { // Test clusters run with security disabled tasks.named("javaRestTest") { - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) } diff --git a/x-pack/plugin/deprecation/qa/rest/build.gradle b/x-pack/plugin/deprecation/qa/rest/build.gradle index 70c0cadbce375..9a8b228763fe0 100644 --- a/x-pack/plugin/deprecation/qa/rest/build.gradle +++ b/x-pack/plugin/deprecation/qa/rest/build.gradle @@ -34,5 +34,5 @@ testClusters.configureEach { // Test clusters run with security disabled tasks.named("javaRestTest") { - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) } diff --git a/x-pack/plugin/downsample/qa/mixed-cluster/build.gradle b/x-pack/plugin/downsample/qa/mixed-cluster/build.gradle index 6b1c7e42c0fde..c4f2a239d48e2 100644 --- a/x-pack/plugin/downsample/qa/mixed-cluster/build.gradle +++ b/x-pack/plugin/downsample/qa/mixed-cluster/build.gradle @@ -29,7 +29,7 @@ def supportedVersion = bwcVersion -> { return bwcVersion.onOrAfter("8.10.0") && bwcVersion != VersionProperties.elasticsearchVersion } -BuildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName -> def yamlRestTest = tasks.register("v${bwcVersion}#yamlRestTest", StandaloneRestIntegTestTask) { usesDefaultDistribution() diff --git a/x-pack/plugin/downsample/qa/rest/build.gradle b/x-pack/plugin/downsample/qa/rest/build.gradle index ba5ac7b0c7317..c5cfbea000ebe 100644 --- a/x-pack/plugin/downsample/qa/rest/build.gradle +++ b/x-pack/plugin/downsample/qa/rest/build.gradle @@ -32,7 +32,7 @@ tasks.named('yamlRestTest') { tasks.named('yamlRestCompatTest') { usesDefaultDistribution() } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC tasks.named("yamlRestTest").configure{enabled = false } } diff --git a/x-pack/plugin/downsample/qa/with-security/build.gradle b/x-pack/plugin/downsample/qa/with-security/build.gradle index 5eed735950187..849c242f372bd 100644 --- a/x-pack/plugin/downsample/qa/with-security/build.gradle +++ b/x-pack/plugin/downsample/qa/with-security/build.gradle @@ -28,7 +28,7 @@ testClusters.configureEach { user username: 'elastic_admin', password: 'admin-password' } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC tasks.named("yamlRestTest").configure{enabled = false } } diff --git a/x-pack/plugin/enrich/qa/rest-with-security/build.gradle b/x-pack/plugin/enrich/qa/rest-with-security/build.gradle index 69fec4ad32c74..844cfcc61adff 100644 --- a/x-pack/plugin/enrich/qa/rest-with-security/build.gradle +++ b/x-pack/plugin/enrich/qa/rest-with-security/build.gradle @@ -6,7 +6,7 @@ dependencies { javaRestTestImplementation project(path: xpackModule('core')) javaRestTestImplementation project(path: xpackModule('enrich:qa:common')) } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC tasks.named("javaRestTest").configure{enabled = false } } diff --git a/x-pack/plugin/enrich/qa/rest/build.gradle b/x-pack/plugin/enrich/qa/rest/build.gradle index f96eff5f933c4..637ab21a98fd7 100644 --- a/x-pack/plugin/enrich/qa/rest/build.gradle +++ b/x-pack/plugin/enrich/qa/rest/build.gradle @@ -19,7 +19,7 @@ dependencies { javaRestTestImplementation project(path: xpackModule('enrich:qa:common')) } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC tasks.named("javaRestTest").configure{enabled = false } tasks.named("yamlRestTest").configure{enabled = false } diff --git a/x-pack/plugin/ent-search/qa/full-cluster-restart/build.gradle b/x-pack/plugin/ent-search/qa/full-cluster-restart/build.gradle index e84adf0c0325d..47a1ffaa37fa4 100644 --- a/x-pack/plugin/ent-search/qa/full-cluster-restart/build.gradle +++ b/x-pack/plugin/ent-search/qa/full-cluster-restart/build.gradle @@ -17,7 +17,7 @@ dependencies { javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart"), "javaRestTest")) } -BuildParams.bwcVersions.withWireCompatible(v -> v.after("8.8.0")) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(v -> v.after("8.8.0")) { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java index a5e2d3f79da0e..3a61c848d3813 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java @@ -67,7 +67,10 @@ public QueryRulesetListItem(StreamInput in) throws IOException { } else { this.criteriaTypeToCountMap = Map.of(); } - if (in.getTransportVersion().onOrAfter(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES)) { + TransportVersion streamTransportVersion = in.getTransportVersion(); + if (streamTransportVersion.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_15) + || streamTransportVersion.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16) + || streamTransportVersion.onOrAfter(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES)) { this.ruleTypeToCountMap = in.readMap(m -> in.readEnum(QueryRule.QueryRuleType.class), StreamInput::readInt); } else { this.ruleTypeToCountMap = Map.of(); @@ -100,7 +103,10 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(EXPANDED_RULESET_COUNT_TRANSPORT_VERSION)) { out.writeMap(criteriaTypeToCountMap, StreamOutput::writeEnum, StreamOutput::writeInt); } - if (out.getTransportVersion().onOrAfter(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES)) { + TransportVersion streamTransportVersion = out.getTransportVersion(); + if (streamTransportVersion.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_15) + || streamTransportVersion.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16) + || streamTransportVersion.onOrAfter(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES)) { out.writeMap(ruleTypeToCountMap, StreamOutput::writeEnum, StreamOutput::writeInt); } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java index 27ac214558f89..27d5e240534b2 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java @@ -59,7 +59,9 @@ protected ListQueryRulesetsAction.Response mutateInstanceForVersion( ListQueryRulesetsAction.Response instance, TransportVersion version ) { - if (version.onOrAfter(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES)) { + if (version.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_15) + || version.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16) + || version.onOrAfter(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES)) { return instance; } else if (version.onOrAfter(QueryRulesetListItem.EXPANDED_RULESET_COUNT_TRANSPORT_VERSION)) { List updatedResults = new ArrayList<>(); diff --git a/x-pack/plugin/eql/build.gradle b/x-pack/plugin/eql/build.gradle index cda236c3d02ae..b0b5fefa37fcd 100644 --- a/x-pack/plugin/eql/build.gradle +++ b/x-pack/plugin/eql/build.gradle @@ -32,7 +32,7 @@ dependencies { * Enable QA/rest integration tests for snapshot builds only * * TODO: Enable for all builds upon this feature release * ****************************************************************/ -if (BuildParams.isSnapshotBuild()) { +if (buildParams.isSnapshotBuild()) { addQaCheckDependencies(project) } diff --git a/x-pack/plugin/eql/qa/ccs-rolling-upgrade/build.gradle b/x-pack/plugin/eql/qa/ccs-rolling-upgrade/build.gradle index a16c24c852377..cbea0896264d5 100644 --- a/x-pack/plugin/eql/qa/ccs-rolling-upgrade/build.gradle +++ b/x-pack/plugin/eql/qa/ccs-rolling-upgrade/build.gradle @@ -15,7 +15,7 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.bwc-test' apply plugin: 'elasticsearch.rest-resources' -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> /** * We execute tests 3 times. diff --git a/x-pack/plugin/eql/qa/correctness/build.gradle b/x-pack/plugin/eql/qa/correctness/build.gradle index d245dc444f0b8..a791356499f5c 100644 --- a/x-pack/plugin/eql/qa/correctness/build.gradle +++ b/x-pack/plugin/eql/qa/correctness/build.gradle @@ -41,7 +41,7 @@ def runTaskCluster = testClusters.register('runTask') { tasks.named('javaRestTest').configure { onlyIf("FIPS mode disabled and service accoutn file available") { - serviceAccountFile && BuildParams.inFipsJvm == false + serviceAccountFile && buildParams.inFipsJvm == false } testLogging { diff --git a/x-pack/plugin/eql/qa/mixed-node/build.gradle b/x-pack/plugin/eql/qa/mixed-node/build.gradle index 8b9e082215fc4..d3aa227c7ef88 100644 --- a/x-pack/plugin/eql/qa/mixed-node/build.gradle +++ b/x-pack/plugin/eql/qa/mixed-node/build.gradle @@ -13,7 +13,7 @@ dependencies { tasks.named("javaRestTest").configure { enabled = false } -BuildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.10.0") && +buildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.10.0") && v != VersionProperties.getElasticsearchVersion()) { bwcVersion, baseName -> def cluster = testClusters.register(baseName) { versions = [bwcVersion.toString(), project.version] diff --git a/x-pack/plugin/eql/qa/rest/build.gradle b/x-pack/plugin/eql/qa/rest/build.gradle index d5b0cc42091f3..00f196d863f2e 100644 --- a/x-pack/plugin/eql/qa/rest/build.gradle +++ b/x-pack/plugin/eql/qa/rest/build.gradle @@ -30,7 +30,7 @@ tasks.named('yamlRestCompatTest') { usesDefaultDistribution() } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC tasks.named("javaRestTest").configure{enabled = false } tasks.named("yamlRestTest").configure{enabled = false } diff --git a/x-pack/plugin/eql/qa/security/build.gradle b/x-pack/plugin/eql/qa/security/build.gradle index 0641c47273f0e..1f0f949cab706 100644 --- a/x-pack/plugin/eql/qa/security/build.gradle +++ b/x-pack/plugin/eql/qa/security/build.gradle @@ -10,7 +10,7 @@ tasks.named('javaRestTest') { usesDefaultDistribution() } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC tasks.named("javaRestTest").configure{enabled = false } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Predicates.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Predicates.java index 28bbf956fd71e..e63cc1fcf25fe 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Predicates.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Predicates.java @@ -6,7 +6,9 @@ */ package org.elasticsearch.xpack.esql.core.expression.predicate; +import org.elasticsearch.core.Tuple; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; @@ -113,4 +115,43 @@ public static List subtract(List from, List } return diff.isEmpty() ? emptyList() : diff; } + + /** + * Given a list of expressions of predicates, extract a new expression of + * all the common ones and return it, along the original list with the + * common ones removed. + *

+ * Example: for ['field1 > 0 AND field2 > 0', 'field1 > 0 AND field3 > 0', + * 'field1 > 0'], the function will return 'field1 > 0' as the common + * predicate expression and ['field2 > 0', 'field3 > 0', Literal.TRUE] as + * the left predicates list. + * + * @param expressions list of expressions to extract common predicates from. + * @return a tuple having as the first element an expression of the common + * predicates and as the second element the list of expressions with the + * common predicates removed. If there are no common predicates, `null` will + * be returned as the first element and the original list as the second. If + * for one of the expressions in the input list, nothing is left after + * trimming the common predicates, it will be replaced with Literal.TRUE. + */ + public static Tuple> extractCommon(List expressions) { + List common = null; + List> splitAnds = new ArrayList<>(expressions.size()); + for (var expression : expressions) { + var split = splitAnd(expression); + common = common == null ? split : inCommon(split, common); + if (common.isEmpty()) { + return Tuple.tuple(null, expressions); + } + splitAnds.add(split); + } + + List trimmed = new ArrayList<>(expressions.size()); + final List finalCommon = common; + splitAnds.forEach(split -> { + var subtracted = subtract(split, finalCommon); + trimmed.add(subtracted.isEmpty() ? Literal.TRUE : combineAnd(subtracted)); + }); + return Tuple.tuple(combineAnd(common), trimmed); + } } diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 1cf39f06f77c8..6541fcd84afef 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -74,7 +74,7 @@ interface Injected { } tasks.named("test").configure { - if (BuildParams.isCi() == false) { + if (buildParams.isCi() == false) { systemProperty 'generateDocs', true def injected = project.objects.newInstance(Injected) doFirst { @@ -145,7 +145,7 @@ tasks.named("test").configure { * Enable QA/rest integration tests for snapshot builds only * * TODO: Enable for all builds upon this feature release * ****************************************************************/ -if (BuildParams.isSnapshotBuild()) { +if (buildParams.isSnapshotBuild()) { addQaCheckDependencies(project) } diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle index fb47255e8d52e..68c0e8e30f814 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle @@ -31,7 +31,7 @@ def supportedVersion = bwcVersion -> { return bwcVersion.onOrAfter(Version.fromString("8.11.0")) && bwcVersion != VersionProperties.elasticsearchVersion } -BuildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName -> def javaRestTest = tasks.register("v${bwcVersion}#javaRestTest", StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle b/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle index 77497597a18c6..2c432eb94ebf1 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle +++ b/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle @@ -23,7 +23,7 @@ def supportedVersion = bwcVersion -> { return bwcVersion.onOrAfter(Version.fromString("8.13.0")); } -BuildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 96aa779ad38c3..ad9de4674f8e1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -2641,6 +2641,57 @@ c2:l |c2_f:l |m2:i |m2_f:i |c:l 1 |1 |5 |5 |21 ; +commonFilterExtractionWithAliasing +required_capability: per_agg_filtering +from employees +| eval eno = emp_no +| drop emp_no +| stats min_sal = min(salary) where eno <= 10010, + min_hei = min(height) where eno <= 10010 +; + +min_sal:integer |min_hei:double +36174 |1.56 +; + +commonFilterExtractionWithAliasAndOriginal +required_capability: per_agg_filtering +from employees +| eval eno = emp_no +| stats min_sal = min(salary) where eno <= 10010, + min_hei = min(height) where emp_no <= 10010 +; + +// same results as above in commonFilterExtractionWithAliasing +min_sal:integer |min_hei:double +36174 |1.56 +; + +commonFilterExtractionWithAliasAndOriginalNeedingNormalization +required_capability: per_agg_filtering +from employees +| eval eno = emp_no +| stats min_sal = min(salary) where eno <= 10010, + min_hei = min(height) where emp_no <= 10010, + max_hei = max(height) where 10010 >= emp_no +; + +min_sal:integer |min_hei:double |max_hei:double +36174 |1.56 |2.1 +; + +commonFilterExtractionWithAliasAndOriginalNeedingNormalizationAndSimplification +required_capability: per_agg_filtering +from employees +| eval eno = emp_no +| stats min_sal = min(salary) where eno <= 10010, + min_hei = min(height) where not (emp_no > 10010), + max_hei = max(height) where 10010 >= emp_no +; + +min_sal:integer |min_hei:double |max_hei:double +36174 |1.56 |2.1 +; statsByConstantExpressionNoAggs required_capability: fix_stats_by_foldable_expression diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index a0e257d1a8953..5007b011092f0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.optimizer.rules.logical.CombineProjections; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ConstantFolding; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ConvertStringToByteRef; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.ExtractAggregateCommonFilter; import org.elasticsearch.xpack.esql.optimizer.rules.logical.FoldNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.LiteralsOnTheRight; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PartiallyFoldCase; @@ -124,8 +125,9 @@ protected static Batch substitutions() { "Substitutions", Limiter.ONCE, new SubstituteSurrogatePlans(), - // translate filtered expressions into aggregate with filters - can't use surrogate expressions because it was - // retrofitted for constant folding - this needs to be fixed + // Translate filtered expressions into aggregate with filters - can't use surrogate expressions because it was + // retrofitted for constant folding - this needs to be fixed. + // Needs to occur before ReplaceAggregateAggExpressionWithEval, which will update the functions, losing the filter. new SubstituteFilteredExpression(), new RemoveStatsOverride(), // first extract nested expressions inside aggs @@ -170,8 +172,10 @@ protected static Batch operators() { new BooleanFunctionEqualsElimination(), new CombineBinaryComparisons(), new CombineDisjunctions(), + // TODO: bifunction can now (since we now have just one data types set) be pushed into the rule new SimplifyComparisonsArithmetics(DataType::areCompatible), new ReplaceStatsFilteredAggWithEval(), + new ExtractAggregateCommonFilter(), // prune/elimination new PruneFilters(), new PruneColumns(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ExtractAggregateCommonFilter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ExtractAggregateCommonFilter.java new file mode 100644 index 0000000000000..f00a8103f913e --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ExtractAggregateCommonFilter.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules.logical; + +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Filter; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.extractCommon; + +/** + * Extract a per-function expression filter applied to all the aggs as a query {@link Filter}, when no groups are provided. + *

+ * Example: + *

+ *         ... | STATS MIN(a) WHERE b > 0, MIN(c) WHERE b > 0 | ...
+ *         =>
+ *         ... | WHERE b > 0 | STATS MIN(a), MIN(c) | ...
+ *     
+ */ +public final class ExtractAggregateCommonFilter extends OptimizerRules.OptimizerRule { + public ExtractAggregateCommonFilter() { + super(OptimizerRules.TransformDirection.UP); + } + + @Override + protected LogicalPlan rule(Aggregate aggregate) { + if (aggregate.groupings().isEmpty() == false) { + return aggregate; // no optimization for grouped stats + } + + // collect all filters from the agg functions + List filters = new ArrayList<>(aggregate.aggregates().size()); + for (NamedExpression ne : aggregate.aggregates()) { + if (ne instanceof Alias alias && alias.child() instanceof AggregateFunction aggFunction && aggFunction.hasFilter()) { + filters.add(aggFunction.filter()); + } else { + return aggregate; // (at least one) agg function has no filter -- skip optimization + } + } + + // extract common filters + var common = extractCommon(filters); + if (common.v1() == null) { // no common filter + return aggregate; + } + + // replace agg functions' filters with trimmed ones + var newFilters = common.v2(); + List newAggs = new ArrayList<>(aggregate.aggregates().size()); + for (int i = 0; i < aggregate.aggregates().size(); i++) { + var alias = (Alias) aggregate.aggregates().get(i); + var newChild = ((AggregateFunction) alias.child()).withFilter(newFilters.get(i)); + newAggs.add(alias.replaceChild(newChild)); + } + + // build the new agg on top of extracted filter + return new Aggregate( + aggregate.source(), + new Filter(aggregate.source(), aggregate.child(), common.v1()), + aggregate.aggregateType(), + aggregate.groupings(), + newAggs + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index d9a0f9ad57fb1..c29f111488f96 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -840,6 +840,265 @@ public void testReplaceStatsFilteredAggWithEvalSingleAggWithGroup() { var source = as(aggregate.child(), EsRelation.class); } + public void testExtractStatsCommonFilter() { + var plan = plan(""" + from test + | stats m = min(salary) where emp_no > 1, + max(salary) where emp_no > 1 + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates().size(), is(2)); + + var alias = as(agg.aggregates().get(0), Alias.class); + var aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), is(Literal.TRUE)); + + alias = as(agg.aggregates().get(1), Alias.class); + aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), is(Literal.TRUE)); + + var filter = as(agg.child(), Filter.class); + assertThat(Expressions.name(filter.condition()), is("emp_no > 1")); + + var source = as(filter.child(), EsRelation.class); + } + + public void testExtractStatsCommonFilterUsingAliases() { + var plan = plan(""" + from test + | eval eno = emp_no + | drop emp_no + | stats min(salary) where eno > 1, + max(salary) where eno > 1 + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates().size(), is(2)); + + var alias = as(agg.aggregates().get(0), Alias.class); + var aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), is(Literal.TRUE)); + + alias = as(agg.aggregates().get(1), Alias.class); + aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), is(Literal.TRUE)); + + var filter = as(agg.child(), Filter.class); + assertThat(Expressions.name(filter.condition()), is("eno > 1")); + + var source = as(filter.child(), EsRelation.class); + } + + public void testExtractStatsCommonFilterUsingJustOneAlias() { + var plan = plan(""" + from test + | eval eno = emp_no + | stats min(salary) where emp_no > 1, + max(salary) where eno > 1 + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates().size(), is(2)); + + var alias = as(agg.aggregates().get(0), Alias.class); + var aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), is(Literal.TRUE)); + + alias = as(agg.aggregates().get(1), Alias.class); + aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), is(Literal.TRUE)); + + var filter = as(agg.child(), Filter.class); + var gt = as(filter.condition(), GreaterThan.class); + assertThat(Expressions.name(gt.left()), is("emp_no")); + assertTrue(gt.right().foldable()); + assertThat(gt.right().fold(), is(1)); + + var source = as(filter.child(), EsRelation.class); + } + + public void testExtractStatsCommonFilterSkippedNotSameFilter() { + var plan = plan(""" + from test + | stats min(salary) where emp_no > 1, + max(salary) where emp_no > 2 + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates().size(), is(2)); + + var alias = as(agg.aggregates().get(0), Alias.class); + var aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), instanceOf(BinaryComparison.class)); + + alias = as(agg.aggregates().get(1), Alias.class); + aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), instanceOf(BinaryComparison.class)); + + var source = as(agg.child(), EsRelation.class); + } + + public void testExtractStatsCommonFilterSkippedOnLackingFilter() { + var plan = plan(""" + from test + | stats min(salary), + max(salary) where emp_no > 2 + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates().size(), is(2)); + + var alias = as(agg.aggregates().get(0), Alias.class); + var aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), is(Literal.TRUE)); + + alias = as(agg.aggregates().get(1), Alias.class); + aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), instanceOf(BinaryComparison.class)); + + var source = as(agg.child(), EsRelation.class); + } + + public void testExtractStatsCommonFilterSkippedWithGroups() { + var plan = plan(""" + from test + | stats min(salary) where emp_no > 2, + max(salary) where emp_no > 2 by first_name + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates().size(), is(3)); + + var alias = as(agg.aggregates().get(0), Alias.class); + var aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), instanceOf(BinaryComparison.class)); + + alias = as(agg.aggregates().get(1), Alias.class); + aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), instanceOf(BinaryComparison.class)); + + var source = as(agg.child(), EsRelation.class); + } + + public void testExtractStatsCommonFilterNormalizeAndCombineWithExistingFilter() { + var plan = plan(""" + from test + | where emp_no > 3 + | stats min(salary) where emp_no > 2, + max(salary) where 2 < emp_no + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates().size(), is(2)); + + var alias = as(agg.aggregates().get(0), Alias.class); + var aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), is(Literal.TRUE)); + + alias = as(agg.aggregates().get(1), Alias.class); + aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), is(Literal.TRUE)); + + var filter = as(agg.child(), Filter.class); + assertThat(Expressions.name(filter.condition()), is("emp_no > 3")); + + var source = as(filter.child(), EsRelation.class); + } + + public void testExtractStatsCommonFilterInConjunction() { + var plan = plan(""" + from test + | stats min(salary) where emp_no > 2 and first_name == "John", + max(salary) where emp_no > 1 + 1 and length(last_name) < 19 + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates().size(), is(2)); + + var alias = as(agg.aggregates().get(0), Alias.class); + var aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(Expressions.name(aggFunc.filter()), is("first_name == \"John\"")); + + alias = as(agg.aggregates().get(1), Alias.class); + aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(Expressions.name(aggFunc.filter()), is("length(last_name) < 19")); + + var filter = as(agg.child(), Filter.class); + var gt = as(filter.condition(), GreaterThan.class); // name is "emp_no > 1 + 1" + assertThat(Expressions.name(gt.left()), is("emp_no")); + assertTrue(gt.right().foldable()); + assertThat(gt.right().fold(), is(2)); + + var source = as(filter.child(), EsRelation.class); + } + + public void testExtractStatsCommonFilterInConjunctionWithMultipleCommonConjunctions() { + var plan = plan(""" + from test + | stats min(salary) where emp_no < 10 and first_name == "John" and last_name == "Doe", + max(salary) where emp_no - 1 < 2 + 7 and length(last_name) < 19 and last_name == "Doe" + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates().size(), is(2)); + + var alias = as(agg.aggregates().get(0), Alias.class); + var aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(Expressions.name(aggFunc.filter()), is("first_name == \"John\"")); + + alias = as(agg.aggregates().get(1), Alias.class); + aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(Expressions.name(aggFunc.filter()), is("length(last_name) < 19")); + + var filter = as(agg.child(), Filter.class); + var and = as(filter.condition(), And.class); + + var lt = as(and.left(), LessThan.class); + assertThat(Expressions.name(lt.left()), is("emp_no")); + assertTrue(lt.right().foldable()); + assertThat(lt.right().fold(), is(10)); + + var equals = as(and.right(), Equals.class); + assertThat(Expressions.name(equals.left()), is("last_name")); + assertTrue(equals.right().foldable()); + assertThat(equals.right().fold(), is(BytesRefs.toBytesRef("Doe"))); + + var source = as(filter.child(), EsRelation.class); + } + + public void testExtractStatsCommonFilterSkippedDueToDisjunction() { + // same query as in testExtractStatsCommonFilterInConjunction, except for the OR in the filter + var plan = plan(""" + from test + | stats min(salary) where emp_no > 2 OR first_name == "John", + max(salary) where emp_no > 1 + 1 and length(last_name) < 19 + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates().size(), is(2)); + + var alias = as(agg.aggregates().get(0), Alias.class); + var aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), instanceOf(Or.class)); + + alias = as(agg.aggregates().get(1), Alias.class); + aggFunc = as(alias.child(), AggregateFunction.class); + assertThat(aggFunc.filter(), instanceOf(And.class)); + + var source = as(agg.child(), EsRelation.class); + } + public void testQlComparisonOptimizationsApply() { var plan = plan(""" from test diff --git a/x-pack/plugin/fleet/qa/rest/build.gradle b/x-pack/plugin/fleet/qa/rest/build.gradle index dec624bc3cc56..0959e883997d3 100644 --- a/x-pack/plugin/fleet/qa/rest/build.gradle +++ b/x-pack/plugin/fleet/qa/rest/build.gradle @@ -5,8 +5,6 @@ * 2.0. */ -import org.elasticsearch.gradle.internal.info.BuildParams - apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-test-artifact' @@ -27,7 +25,7 @@ tasks.named('yamlRestTest') { tasks.named('yamlRestCompatTest') { usesDefaultDistribution() } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC tasks.named("yamlRestTest").configure{enabled = false } } diff --git a/x-pack/plugin/identity-provider/build.gradle b/x-pack/plugin/identity-provider/build.gradle index dd085e62efa48..f3b0def7eee97 100644 --- a/x-pack/plugin/identity-provider/build.gradle +++ b/x-pack/plugin/identity-provider/build.gradle @@ -281,7 +281,7 @@ tasks.named("thirdPartyAudit").configure { addQaCheckDependencies(project) -if (BuildParams.inFipsJvm) { +if (buildParams.inFipsJvm) { // We don't support the IDP in FIPS-140 mode, so no need to run tests tasks.named("test").configure { enabled = false } } diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle b/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle index 46e705ce27244..b109c01181729 100644 --- a/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle +++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle @@ -48,6 +48,6 @@ testClusters.configureEach { // We don't support the IDP in FIPS-140 mode, so no need to run java rest tests tasks.named("javaRestTest").configure { - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) } diff --git a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle index 111496669afe3..256225c5ef3bf 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle @@ -59,5 +59,5 @@ testClusters.matching{ it.name == 'follow-cluster' }.configureEach { tasks.named("check").configure { dependsOn 'follow-cluster' } // Security is explicitly disabled for follow-cluster and leader-cluster, do not run these in FIPS mode tasks.withType(Test).configureEach { - enabled = BuildParams.inFipsJvm == false + enabled = buildParams.inFipsJvm == false } diff --git a/x-pack/plugin/ilm/qa/multi-node/build.gradle b/x-pack/plugin/ilm/qa/multi-node/build.gradle index 8712af84ac245..d420ac9effdde 100644 --- a/x-pack/plugin/ilm/qa/multi-node/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-node/build.gradle @@ -40,7 +40,7 @@ testClusters.configureEach { setting 'time_series.poll_interval', '10m' } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // Test clusters run with security disabled tasks.named("javaRestTest").configure{enabled = false } } diff --git a/x-pack/plugin/inference/qa/mixed-cluster/build.gradle b/x-pack/plugin/inference/qa/mixed-cluster/build.gradle index 64edb196397a0..c05e71fa1cd55 100644 --- a/x-pack/plugin/inference/qa/mixed-cluster/build.gradle +++ b/x-pack/plugin/inference/qa/mixed-cluster/build.gradle @@ -20,7 +20,7 @@ def supportedVersion = bwcVersion -> { return bwcVersion.onOrAfter(Version.fromString("8.11.0")) && bwcVersion != VersionProperties.elasticsearchVersion } -BuildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName -> def javaRestTest = tasks.register("v${bwcVersion}#javaRestTest", StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle b/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle index 5d72fc96d98d8..bfaff7c84d9ad 100644 --- a/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle +++ b/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle @@ -20,7 +20,7 @@ dependencies { } // Inference API added in 8.11 -BuildParams.bwcVersions.withWireCompatible(v -> v.after("8.11.0")) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(v -> v.after("8.11.0")) { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java index 767dd4d64a7d3..22ef35c3a46d3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java @@ -71,8 +71,10 @@ public void testValidate_ServiceThrowsException() { any() ); - assertThrows(ElasticsearchStatusException.class, () -> { - underTest.validate(mockInferenceService, mockModel, mockActionListener);}); + assertThrows( + ElasticsearchStatusException.class, + () -> { underTest.validate(mockInferenceService, mockModel, mockActionListener); } + ); verifyCallToService(false); } diff --git a/x-pack/plugin/kql/build.gradle b/x-pack/plugin/kql/build.gradle index 7e4df5654f225..054011a458fe0 100644 --- a/x-pack/plugin/kql/build.gradle +++ b/x-pack/plugin/kql/build.gradle @@ -26,14 +26,14 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) } -tasks.named('yamlRestTest') { +tasks.named('yamlRestTest').configure { usesDefaultDistribution() -}.configure { + /**************************************************************** * Enable QA/rest integration tests for snapshot builds only * * TODO: Enable for all builds upon this feature release * ****************************************************************/ - enabled = BuildParams.isSnapshotBuild() + enabled = buildParams.isSnapshotBuild() } /********************************** diff --git a/x-pack/plugin/kql/src/main/antlr/KqlBase.g4 b/x-pack/plugin/kql/src/main/antlr/KqlBase.g4 index da015b699cb15..739fa5eb0c6eb 100644 --- a/x-pack/plugin/kql/src/main/antlr/KqlBase.g4 +++ b/x-pack/plugin/kql/src/main/antlr/KqlBase.g4 @@ -46,9 +46,26 @@ notQuery: ; nestedQuery - : fieldName COLON LEFT_CURLY_BRACKET query RIGHT_CURLY_BRACKET + : fieldName COLON LEFT_CURLY_BRACKET nestedSubQuery RIGHT_CURLY_BRACKET ; +nestedSubQuery + : nestedSubQuery operator=(AND|OR) nestedSubQuery #booleanNestedQuery + | nestedSimpleSubQuery #defaultNestedQuery + ; + +nestedSimpleSubQuery + : notQuery + | nestedQuery + | matchAllQuery + | nestedParenthesizedQuery + | existsQuery + | rangeQuery + | fieldQuery; + +nestedParenthesizedQuery + : LEFT_PARENTHESIS nestedSubQuery RIGHT_PARENTHESIS; + matchAllQuery : (WILDCARD COLON)? WILDCARD ; diff --git a/x-pack/plugin/kql/src/main/java/module-info.java b/x-pack/plugin/kql/src/main/java/module-info.java index 41e51033b9c70..e3bb6fb99bbd3 100644 --- a/x-pack/plugin/kql/src/main/java/module-info.java +++ b/x-pack/plugin/kql/src/main/java/module-info.java @@ -13,6 +13,7 @@ requires org.apache.lucene.queryparser; requires org.elasticsearch.logging; requires org.apache.lucene.core; + requires org.apache.lucene.join; exports org.elasticsearch.xpack.kql; exports org.elasticsearch.xpack.kql.parser; diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java index 5fe3a61c0a761..2d810a33190ca 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java @@ -9,6 +9,7 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -20,6 +21,7 @@ import org.elasticsearch.index.query.QueryStringQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; +import java.util.List; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.BiFunction; @@ -56,15 +58,15 @@ public QueryBuilder toQueryBuilder(ParserRuleContext ctx) { @Override public QueryBuilder visitBooleanQuery(KqlBaseParser.BooleanQueryContext ctx) { assert ctx.operator != null; - return isAndQuery(ctx) ? visitAndBooleanQuery(ctx) : visitOrBooleanQuery(ctx); + return isAndQuery(ctx) ? visitAndBooleanQuery(ctx.query()) : visitOrBooleanQuery(ctx.query()); } - public QueryBuilder visitAndBooleanQuery(KqlBaseParser.BooleanQueryContext ctx) { + public QueryBuilder visitAndBooleanQuery(List clauses) { BoolQueryBuilder builder = QueryBuilders.boolQuery(); // TODO: KQLContext has an option to wrap the clauses into a filter instead of a must clause. Do we need it? - for (ParserRuleContext subQueryCtx : ctx.query()) { - if (subQueryCtx instanceof KqlBaseParser.BooleanQueryContext booleanSubQueryCtx && isAndQuery(booleanSubQueryCtx)) { + for (ParserRuleContext subQueryCtx : clauses) { + if (isAndQuery(subQueryCtx)) { typedParsing(this, subQueryCtx, BoolQueryBuilder.class).must().forEach(builder::must); } else { builder.must(typedParsing(this, subQueryCtx, QueryBuilder.class)); @@ -74,11 +76,11 @@ public QueryBuilder visitAndBooleanQuery(KqlBaseParser.BooleanQueryContext ctx) return rewriteConjunctionQuery(builder); } - public QueryBuilder visitOrBooleanQuery(KqlBaseParser.BooleanQueryContext ctx) { + public QueryBuilder visitOrBooleanQuery(List clauses) { BoolQueryBuilder builder = QueryBuilders.boolQuery().minimumShouldMatch(1); - for (ParserRuleContext subQueryCtx : ctx.query()) { - if (subQueryCtx instanceof KqlBaseParser.BooleanQueryContext booleanSubQueryCtx && isOrQuery(booleanSubQueryCtx)) { + for (ParserRuleContext subQueryCtx : clauses) { + if (isOrQuery(subQueryCtx)) { typedParsing(this, subQueryCtx, BoolQueryBuilder.class).should().forEach(builder::should); } else { builder.should(typedParsing(this, subQueryCtx, QueryBuilder.class)); @@ -100,8 +102,40 @@ public QueryBuilder visitParenthesizedQuery(KqlBaseParser.ParenthesizedQueryCont @Override public QueryBuilder visitNestedQuery(KqlBaseParser.NestedQueryContext ctx) { - // TODO: implementation - return new MatchNoneQueryBuilder(); + String nestedFieldName = extractText(ctx.fieldName()); + + if (kqlParsingContext.isNestedField(nestedFieldName) == false) { + throw new KqlParsingException( + "[{}] is not a valid nested field name.", + ctx.start.getLine(), + ctx.start.getCharPositionInLine(), + nestedFieldName + ); + } + QueryBuilder subQuery = kqlParsingContext.withNestedPath( + nestedFieldName, + () -> typedParsing(this, ctx.nestedSubQuery(), QueryBuilder.class) + ); + + if (subQuery instanceof MatchNoneQueryBuilder) { + return subQuery; + } + + return wrapWithNestedQuery( + nestedFieldName, + QueryBuilders.nestedQuery(kqlParsingContext.fullFieldName(nestedFieldName), subQuery, ScoreMode.None) + ); + } + + @Override + public QueryBuilder visitBooleanNestedQuery(KqlBaseParser.BooleanNestedQueryContext ctx) { + assert ctx.operator != null; + return isAndQuery(ctx) ? visitAndBooleanQuery(ctx.nestedSubQuery()) : visitOrBooleanQuery(ctx.nestedSubQuery()); + } + + @Override + public QueryBuilder visitNestedParenthesizedQuery(KqlBaseParser.NestedParenthesizedQueryContext ctx) { + return typedParsing(this, ctx.nestedSubQuery(), QueryBuilder.class); } @Override @@ -116,7 +150,7 @@ public QueryBuilder visitExistsQuery(KqlBaseParser.ExistsQueryContext ctx) { BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().minimumShouldMatch(1); withFields(ctx.fieldName(), (fieldName, mappedFieldType) -> { if (isRuntimeField(mappedFieldType) == false) { - boolQueryBuilder.should(QueryBuilders.existsQuery(fieldName)); + boolQueryBuilder.should(wrapWithNestedQuery(fieldName, QueryBuilders.existsQuery(fieldName))); } }); @@ -137,7 +171,7 @@ public QueryBuilder visitRangeQuery(KqlBaseParser.RangeQueryContext ctx) { rangeQuery.timeZone(kqlParsingContext.timeZone().getId()); } - boolQueryBuilder.should(rangeQuery); + boolQueryBuilder.should(wrapWithNestedQuery(fieldName, rangeQuery)); }); return rewriteDisjunctionQuery(boolQueryBuilder); @@ -200,24 +234,33 @@ public QueryBuilder visitFieldQuery(KqlBaseParser.FieldQueryContext ctx) { } if (fieldQuery != null) { - boolQueryBuilder.should(fieldQuery); + boolQueryBuilder.should(wrapWithNestedQuery(fieldName, fieldQuery)); } }); return rewriteDisjunctionQuery(boolQueryBuilder); } - private static boolean isAndQuery(KqlBaseParser.BooleanQueryContext ctx) { - return ctx.operator.getType() == KqlBaseParser.AND; + private static boolean isAndQuery(ParserRuleContext ctx) { + return switch (ctx) { + case KqlBaseParser.BooleanQueryContext booleanQueryCtx -> booleanQueryCtx.operator.getType() == KqlBaseParser.AND; + case KqlBaseParser.BooleanNestedQueryContext booleanNestedCtx -> booleanNestedCtx.operator.getType() == KqlBaseParser.AND; + default -> false; + }; } - private static boolean isOrQuery(KqlBaseParser.BooleanQueryContext ctx) { - return ctx.operator.getType() == KqlBaseParser.OR; + private static boolean isOrQuery(ParserRuleContext ctx) { + return switch (ctx) { + case KqlBaseParser.BooleanQueryContext booleanQueryCtx -> booleanQueryCtx.operator.getType() == KqlBaseParser.OR; + case KqlBaseParser.BooleanNestedQueryContext booleanNestedCtx -> booleanNestedCtx.operator.getType() == KqlBaseParser.OR; + default -> false; + }; } private void withFields(KqlBaseParser.FieldNameContext ctx, BiConsumer fieldConsummer) { assert ctx != null : "Field ctx cannot be null"; String fieldNamePattern = extractText(ctx); + Set fieldNames = kqlParsingContext.resolveFieldNames(fieldNamePattern); if (ctx.value.getType() == KqlBaseParser.QUOTED_STRING && Regex.isSimpleMatchPattern(fieldNamePattern)) { @@ -267,4 +310,14 @@ private BiFunction rangeOperation( default -> throw new IllegalArgumentException(format(null, "Invalid range operator {}\"", operator.getText())); }; } + + private QueryBuilder wrapWithNestedQuery(String fieldName, QueryBuilder query) { + String nestedPath = kqlParsingContext.nestedPath(fieldName); + + if (nestedPath == null || nestedPath.equals(kqlParsingContext.currentNestedPath())) { + return query; + } + + return wrapWithNestedQuery(nestedPath, QueryBuilders.nestedQuery(nestedPath, query, ScoreMode.None)); + } } diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase.interp b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase.interp index 7af37d7e3c3b5..fbfe52afa4cd5 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase.interp +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase.interp @@ -42,6 +42,9 @@ query simpleQuery notQuery nestedQuery +nestedSubQuery +nestedSimpleSubQuery +nestedParenthesizedQuery matchAllQuery parenthesizedQuery rangeQuery @@ -54,4 +57,4 @@ fieldName atn: -[4, 1, 16, 136, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 1, 0, 3, 0, 30, 8, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 40, 8, 1, 10, 1, 12, 1, 43, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 53, 8, 2, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 3, 5, 66, 8, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 4, 8, 79, 8, 8, 11, 8, 12, 8, 80, 1, 8, 3, 8, 84, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 100, 8, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 3, 11, 107, 8, 11, 1, 12, 3, 12, 110, 8, 12, 1, 12, 4, 12, 113, 8, 12, 11, 12, 12, 12, 114, 1, 12, 3, 12, 118, 8, 12, 1, 12, 1, 12, 3, 12, 122, 8, 12, 1, 12, 1, 12, 3, 12, 126, 8, 12, 1, 12, 3, 12, 129, 8, 12, 1, 13, 1, 13, 1, 13, 3, 13, 134, 8, 13, 1, 13, 0, 1, 2, 14, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 0, 4, 1, 0, 2, 3, 1, 0, 6, 9, 2, 0, 14, 14, 16, 16, 1, 0, 2, 4, 145, 0, 29, 1, 0, 0, 0, 2, 33, 1, 0, 0, 0, 4, 52, 1, 0, 0, 0, 6, 54, 1, 0, 0, 0, 8, 57, 1, 0, 0, 0, 10, 65, 1, 0, 0, 0, 12, 69, 1, 0, 0, 0, 14, 73, 1, 0, 0, 0, 16, 83, 1, 0, 0, 0, 18, 85, 1, 0, 0, 0, 20, 99, 1, 0, 0, 0, 22, 106, 1, 0, 0, 0, 24, 128, 1, 0, 0, 0, 26, 133, 1, 0, 0, 0, 28, 30, 3, 2, 1, 0, 29, 28, 1, 0, 0, 0, 29, 30, 1, 0, 0, 0, 30, 31, 1, 0, 0, 0, 31, 32, 5, 0, 0, 1, 32, 1, 1, 0, 0, 0, 33, 34, 6, 1, -1, 0, 34, 35, 3, 4, 2, 0, 35, 41, 1, 0, 0, 0, 36, 37, 10, 2, 0, 0, 37, 38, 7, 0, 0, 0, 38, 40, 3, 2, 1, 2, 39, 36, 1, 0, 0, 0, 40, 43, 1, 0, 0, 0, 41, 39, 1, 0, 0, 0, 41, 42, 1, 0, 0, 0, 42, 3, 1, 0, 0, 0, 43, 41, 1, 0, 0, 0, 44, 53, 3, 6, 3, 0, 45, 53, 3, 8, 4, 0, 46, 53, 3, 12, 6, 0, 47, 53, 3, 10, 5, 0, 48, 53, 3, 18, 9, 0, 49, 53, 3, 14, 7, 0, 50, 53, 3, 20, 10, 0, 51, 53, 3, 22, 11, 0, 52, 44, 1, 0, 0, 0, 52, 45, 1, 0, 0, 0, 52, 46, 1, 0, 0, 0, 52, 47, 1, 0, 0, 0, 52, 48, 1, 0, 0, 0, 52, 49, 1, 0, 0, 0, 52, 50, 1, 0, 0, 0, 52, 51, 1, 0, 0, 0, 53, 5, 1, 0, 0, 0, 54, 55, 5, 4, 0, 0, 55, 56, 3, 4, 2, 0, 56, 7, 1, 0, 0, 0, 57, 58, 3, 26, 13, 0, 58, 59, 5, 5, 0, 0, 59, 60, 5, 12, 0, 0, 60, 61, 3, 2, 1, 0, 61, 62, 5, 13, 0, 0, 62, 9, 1, 0, 0, 0, 63, 64, 5, 16, 0, 0, 64, 66, 5, 5, 0, 0, 65, 63, 1, 0, 0, 0, 65, 66, 1, 0, 0, 0, 66, 67, 1, 0, 0, 0, 67, 68, 5, 16, 0, 0, 68, 11, 1, 0, 0, 0, 69, 70, 5, 10, 0, 0, 70, 71, 3, 2, 1, 0, 71, 72, 5, 11, 0, 0, 72, 13, 1, 0, 0, 0, 73, 74, 3, 26, 13, 0, 74, 75, 7, 1, 0, 0, 75, 76, 3, 16, 8, 0, 76, 15, 1, 0, 0, 0, 77, 79, 7, 2, 0, 0, 78, 77, 1, 0, 0, 0, 79, 80, 1, 0, 0, 0, 80, 78, 1, 0, 0, 0, 80, 81, 1, 0, 0, 0, 81, 84, 1, 0, 0, 0, 82, 84, 5, 15, 0, 0, 83, 78, 1, 0, 0, 0, 83, 82, 1, 0, 0, 0, 84, 17, 1, 0, 0, 0, 85, 86, 3, 26, 13, 0, 86, 87, 5, 5, 0, 0, 87, 88, 5, 16, 0, 0, 88, 19, 1, 0, 0, 0, 89, 90, 3, 26, 13, 0, 90, 91, 5, 5, 0, 0, 91, 92, 3, 24, 12, 0, 92, 100, 1, 0, 0, 0, 93, 94, 3, 26, 13, 0, 94, 95, 5, 5, 0, 0, 95, 96, 5, 10, 0, 0, 96, 97, 3, 24, 12, 0, 97, 98, 5, 11, 0, 0, 98, 100, 1, 0, 0, 0, 99, 89, 1, 0, 0, 0, 99, 93, 1, 0, 0, 0, 100, 21, 1, 0, 0, 0, 101, 107, 3, 24, 12, 0, 102, 103, 5, 10, 0, 0, 103, 104, 3, 24, 12, 0, 104, 105, 5, 11, 0, 0, 105, 107, 1, 0, 0, 0, 106, 101, 1, 0, 0, 0, 106, 102, 1, 0, 0, 0, 107, 23, 1, 0, 0, 0, 108, 110, 7, 3, 0, 0, 109, 108, 1, 0, 0, 0, 109, 110, 1, 0, 0, 0, 110, 112, 1, 0, 0, 0, 111, 113, 7, 2, 0, 0, 112, 111, 1, 0, 0, 0, 113, 114, 1, 0, 0, 0, 114, 112, 1, 0, 0, 0, 114, 115, 1, 0, 0, 0, 115, 117, 1, 0, 0, 0, 116, 118, 7, 3, 0, 0, 117, 116, 1, 0, 0, 0, 117, 118, 1, 0, 0, 0, 118, 129, 1, 0, 0, 0, 119, 121, 7, 0, 0, 0, 120, 122, 7, 3, 0, 0, 121, 120, 1, 0, 0, 0, 121, 122, 1, 0, 0, 0, 122, 129, 1, 0, 0, 0, 123, 125, 5, 4, 0, 0, 124, 126, 7, 0, 0, 0, 125, 124, 1, 0, 0, 0, 125, 126, 1, 0, 0, 0, 126, 129, 1, 0, 0, 0, 127, 129, 5, 15, 0, 0, 128, 109, 1, 0, 0, 0, 128, 119, 1, 0, 0, 0, 128, 123, 1, 0, 0, 0, 128, 127, 1, 0, 0, 0, 129, 25, 1, 0, 0, 0, 130, 134, 5, 14, 0, 0, 131, 134, 5, 15, 0, 0, 132, 134, 5, 16, 0, 0, 133, 130, 1, 0, 0, 0, 133, 131, 1, 0, 0, 0, 133, 132, 1, 0, 0, 0, 134, 27, 1, 0, 0, 0, 15, 29, 41, 52, 65, 80, 83, 99, 106, 109, 114, 117, 121, 125, 128, 133] \ No newline at end of file +[4, 1, 16, 165, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 1, 0, 3, 0, 36, 8, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 46, 8, 1, 10, 1, 12, 1, 49, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 59, 8, 2, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 76, 8, 5, 10, 5, 12, 5, 79, 9, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 87, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 3, 8, 95, 8, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 4, 11, 108, 8, 11, 11, 11, 12, 11, 109, 1, 11, 3, 11, 113, 8, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 3, 13, 129, 8, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 3, 14, 136, 8, 14, 1, 15, 3, 15, 139, 8, 15, 1, 15, 4, 15, 142, 8, 15, 11, 15, 12, 15, 143, 1, 15, 3, 15, 147, 8, 15, 1, 15, 1, 15, 3, 15, 151, 8, 15, 1, 15, 1, 15, 3, 15, 155, 8, 15, 1, 15, 3, 15, 158, 8, 15, 1, 16, 1, 16, 1, 16, 3, 16, 163, 8, 16, 1, 16, 0, 2, 2, 10, 17, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 0, 4, 1, 0, 2, 3, 1, 0, 6, 9, 2, 0, 14, 14, 16, 16, 1, 0, 2, 4, 177, 0, 35, 1, 0, 0, 0, 2, 39, 1, 0, 0, 0, 4, 58, 1, 0, 0, 0, 6, 60, 1, 0, 0, 0, 8, 63, 1, 0, 0, 0, 10, 69, 1, 0, 0, 0, 12, 86, 1, 0, 0, 0, 14, 88, 1, 0, 0, 0, 16, 94, 1, 0, 0, 0, 18, 98, 1, 0, 0, 0, 20, 102, 1, 0, 0, 0, 22, 112, 1, 0, 0, 0, 24, 114, 1, 0, 0, 0, 26, 128, 1, 0, 0, 0, 28, 135, 1, 0, 0, 0, 30, 157, 1, 0, 0, 0, 32, 162, 1, 0, 0, 0, 34, 36, 3, 2, 1, 0, 35, 34, 1, 0, 0, 0, 35, 36, 1, 0, 0, 0, 36, 37, 1, 0, 0, 0, 37, 38, 5, 0, 0, 1, 38, 1, 1, 0, 0, 0, 39, 40, 6, 1, -1, 0, 40, 41, 3, 4, 2, 0, 41, 47, 1, 0, 0, 0, 42, 43, 10, 2, 0, 0, 43, 44, 7, 0, 0, 0, 44, 46, 3, 2, 1, 2, 45, 42, 1, 0, 0, 0, 46, 49, 1, 0, 0, 0, 47, 45, 1, 0, 0, 0, 47, 48, 1, 0, 0, 0, 48, 3, 1, 0, 0, 0, 49, 47, 1, 0, 0, 0, 50, 59, 3, 6, 3, 0, 51, 59, 3, 8, 4, 0, 52, 59, 3, 18, 9, 0, 53, 59, 3, 16, 8, 0, 54, 59, 3, 24, 12, 0, 55, 59, 3, 20, 10, 0, 56, 59, 3, 26, 13, 0, 57, 59, 3, 28, 14, 0, 58, 50, 1, 0, 0, 0, 58, 51, 1, 0, 0, 0, 58, 52, 1, 0, 0, 0, 58, 53, 1, 0, 0, 0, 58, 54, 1, 0, 0, 0, 58, 55, 1, 0, 0, 0, 58, 56, 1, 0, 0, 0, 58, 57, 1, 0, 0, 0, 59, 5, 1, 0, 0, 0, 60, 61, 5, 4, 0, 0, 61, 62, 3, 4, 2, 0, 62, 7, 1, 0, 0, 0, 63, 64, 3, 32, 16, 0, 64, 65, 5, 5, 0, 0, 65, 66, 5, 12, 0, 0, 66, 67, 3, 10, 5, 0, 67, 68, 5, 13, 0, 0, 68, 9, 1, 0, 0, 0, 69, 70, 6, 5, -1, 0, 70, 71, 3, 12, 6, 0, 71, 77, 1, 0, 0, 0, 72, 73, 10, 2, 0, 0, 73, 74, 7, 0, 0, 0, 74, 76, 3, 10, 5, 2, 75, 72, 1, 0, 0, 0, 76, 79, 1, 0, 0, 0, 77, 75, 1, 0, 0, 0, 77, 78, 1, 0, 0, 0, 78, 11, 1, 0, 0, 0, 79, 77, 1, 0, 0, 0, 80, 87, 3, 6, 3, 0, 81, 87, 3, 8, 4, 0, 82, 87, 3, 14, 7, 0, 83, 87, 3, 24, 12, 0, 84, 87, 3, 20, 10, 0, 85, 87, 3, 26, 13, 0, 86, 80, 1, 0, 0, 0, 86, 81, 1, 0, 0, 0, 86, 82, 1, 0, 0, 0, 86, 83, 1, 0, 0, 0, 86, 84, 1, 0, 0, 0, 86, 85, 1, 0, 0, 0, 87, 13, 1, 0, 0, 0, 88, 89, 5, 10, 0, 0, 89, 90, 3, 10, 5, 0, 90, 91, 5, 11, 0, 0, 91, 15, 1, 0, 0, 0, 92, 93, 5, 16, 0, 0, 93, 95, 5, 5, 0, 0, 94, 92, 1, 0, 0, 0, 94, 95, 1, 0, 0, 0, 95, 96, 1, 0, 0, 0, 96, 97, 5, 16, 0, 0, 97, 17, 1, 0, 0, 0, 98, 99, 5, 10, 0, 0, 99, 100, 3, 2, 1, 0, 100, 101, 5, 11, 0, 0, 101, 19, 1, 0, 0, 0, 102, 103, 3, 32, 16, 0, 103, 104, 7, 1, 0, 0, 104, 105, 3, 22, 11, 0, 105, 21, 1, 0, 0, 0, 106, 108, 7, 2, 0, 0, 107, 106, 1, 0, 0, 0, 108, 109, 1, 0, 0, 0, 109, 107, 1, 0, 0, 0, 109, 110, 1, 0, 0, 0, 110, 113, 1, 0, 0, 0, 111, 113, 5, 15, 0, 0, 112, 107, 1, 0, 0, 0, 112, 111, 1, 0, 0, 0, 113, 23, 1, 0, 0, 0, 114, 115, 3, 32, 16, 0, 115, 116, 5, 5, 0, 0, 116, 117, 5, 16, 0, 0, 117, 25, 1, 0, 0, 0, 118, 119, 3, 32, 16, 0, 119, 120, 5, 5, 0, 0, 120, 121, 3, 30, 15, 0, 121, 129, 1, 0, 0, 0, 122, 123, 3, 32, 16, 0, 123, 124, 5, 5, 0, 0, 124, 125, 5, 10, 0, 0, 125, 126, 3, 30, 15, 0, 126, 127, 5, 11, 0, 0, 127, 129, 1, 0, 0, 0, 128, 118, 1, 0, 0, 0, 128, 122, 1, 0, 0, 0, 129, 27, 1, 0, 0, 0, 130, 136, 3, 30, 15, 0, 131, 132, 5, 10, 0, 0, 132, 133, 3, 30, 15, 0, 133, 134, 5, 11, 0, 0, 134, 136, 1, 0, 0, 0, 135, 130, 1, 0, 0, 0, 135, 131, 1, 0, 0, 0, 136, 29, 1, 0, 0, 0, 137, 139, 7, 3, 0, 0, 138, 137, 1, 0, 0, 0, 138, 139, 1, 0, 0, 0, 139, 141, 1, 0, 0, 0, 140, 142, 7, 2, 0, 0, 141, 140, 1, 0, 0, 0, 142, 143, 1, 0, 0, 0, 143, 141, 1, 0, 0, 0, 143, 144, 1, 0, 0, 0, 144, 146, 1, 0, 0, 0, 145, 147, 7, 3, 0, 0, 146, 145, 1, 0, 0, 0, 146, 147, 1, 0, 0, 0, 147, 158, 1, 0, 0, 0, 148, 150, 7, 0, 0, 0, 149, 151, 7, 3, 0, 0, 150, 149, 1, 0, 0, 0, 150, 151, 1, 0, 0, 0, 151, 158, 1, 0, 0, 0, 152, 154, 5, 4, 0, 0, 153, 155, 7, 0, 0, 0, 154, 153, 1, 0, 0, 0, 154, 155, 1, 0, 0, 0, 155, 158, 1, 0, 0, 0, 156, 158, 5, 15, 0, 0, 157, 138, 1, 0, 0, 0, 157, 148, 1, 0, 0, 0, 157, 152, 1, 0, 0, 0, 157, 156, 1, 0, 0, 0, 158, 31, 1, 0, 0, 0, 159, 163, 5, 14, 0, 0, 160, 163, 5, 15, 0, 0, 161, 163, 5, 16, 0, 0, 162, 159, 1, 0, 0, 0, 162, 160, 1, 0, 0, 0, 162, 161, 1, 0, 0, 0, 163, 33, 1, 0, 0, 0, 17, 35, 47, 58, 77, 86, 94, 109, 112, 128, 135, 138, 143, 146, 150, 154, 157, 162] \ No newline at end of file diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseListener.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseListener.java index e1015edcd4931..c3fc1281b6fd9 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseListener.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseListener.java @@ -92,6 +92,54 @@ class KqlBaseBaseListener implements KqlBaseListener { *

The default implementation does nothing.

*/ @Override public void exitNestedQuery(KqlBaseParser.NestedQueryContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterBooleanNestedQuery(KqlBaseParser.BooleanNestedQueryContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitBooleanNestedQuery(KqlBaseParser.BooleanNestedQueryContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterDefaultNestedQuery(KqlBaseParser.DefaultNestedQueryContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitDefaultNestedQuery(KqlBaseParser.DefaultNestedQueryContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterNestedSimpleSubQuery(KqlBaseParser.NestedSimpleSubQueryContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitNestedSimpleSubQuery(KqlBaseParser.NestedSimpleSubQueryContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterNestedParenthesizedQuery(KqlBaseParser.NestedParenthesizedQueryContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitNestedParenthesizedQuery(KqlBaseParser.NestedParenthesizedQueryContext ctx) { } /** * {@inheritDoc} * diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseVisitor.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseVisitor.java index 3973a647c8cd8..84c882c2e2bcf 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseVisitor.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseVisitor.java @@ -62,6 +62,34 @@ class KqlBaseBaseVisitor extends AbstractParseTreeVisitor implements KqlBa * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitNestedQuery(KqlBaseParser.NestedQueryContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitBooleanNestedQuery(KqlBaseParser.BooleanNestedQueryContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitDefaultNestedQuery(KqlBaseParser.DefaultNestedQueryContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitNestedSimpleSubQuery(KqlBaseParser.NestedSimpleSubQueryContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitNestedParenthesizedQuery(KqlBaseParser.NestedParenthesizedQueryContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseListener.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseListener.java index 49f2031208642..a44ecf1ecad23 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseListener.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseListener.java @@ -79,6 +79,50 @@ interface KqlBaseListener extends ParseTreeListener { * @param ctx the parse tree */ void exitNestedQuery(KqlBaseParser.NestedQueryContext ctx); + /** + * Enter a parse tree produced by the {@code booleanNestedQuery} + * labeled alternative in {@link KqlBaseParser#nestedSubQuery}. + * @param ctx the parse tree + */ + void enterBooleanNestedQuery(KqlBaseParser.BooleanNestedQueryContext ctx); + /** + * Exit a parse tree produced by the {@code booleanNestedQuery} + * labeled alternative in {@link KqlBaseParser#nestedSubQuery}. + * @param ctx the parse tree + */ + void exitBooleanNestedQuery(KqlBaseParser.BooleanNestedQueryContext ctx); + /** + * Enter a parse tree produced by the {@code defaultNestedQuery} + * labeled alternative in {@link KqlBaseParser#nestedSubQuery}. + * @param ctx the parse tree + */ + void enterDefaultNestedQuery(KqlBaseParser.DefaultNestedQueryContext ctx); + /** + * Exit a parse tree produced by the {@code defaultNestedQuery} + * labeled alternative in {@link KqlBaseParser#nestedSubQuery}. + * @param ctx the parse tree + */ + void exitDefaultNestedQuery(KqlBaseParser.DefaultNestedQueryContext ctx); + /** + * Enter a parse tree produced by {@link KqlBaseParser#nestedSimpleSubQuery}. + * @param ctx the parse tree + */ + void enterNestedSimpleSubQuery(KqlBaseParser.NestedSimpleSubQueryContext ctx); + /** + * Exit a parse tree produced by {@link KqlBaseParser#nestedSimpleSubQuery}. + * @param ctx the parse tree + */ + void exitNestedSimpleSubQuery(KqlBaseParser.NestedSimpleSubQueryContext ctx); + /** + * Enter a parse tree produced by {@link KqlBaseParser#nestedParenthesizedQuery}. + * @param ctx the parse tree + */ + void enterNestedParenthesizedQuery(KqlBaseParser.NestedParenthesizedQueryContext ctx); + /** + * Exit a parse tree produced by {@link KqlBaseParser#nestedParenthesizedQuery}. + * @param ctx the parse tree + */ + void exitNestedParenthesizedQuery(KqlBaseParser.NestedParenthesizedQueryContext ctx); /** * Enter a parse tree produced by {@link KqlBaseParser#matchAllQuery}. * @param ctx the parse tree diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseParser.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseParser.java index 118ac32aadd61..7e797b9edbb93 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseParser.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseParser.java @@ -30,12 +30,15 @@ class KqlBaseParser extends Parser { RIGHT_CURLY_BRACKET=13, UNQUOTED_LITERAL=14, QUOTED_STRING=15, WILDCARD=16; public static final int RULE_topLevelQuery = 0, RULE_query = 1, RULE_simpleQuery = 2, RULE_notQuery = 3, - RULE_nestedQuery = 4, RULE_matchAllQuery = 5, RULE_parenthesizedQuery = 6, - RULE_rangeQuery = 7, RULE_rangeQueryValue = 8, RULE_existsQuery = 9, RULE_fieldQuery = 10, - RULE_fieldLessQuery = 11, RULE_fieldQueryValue = 12, RULE_fieldName = 13; + RULE_nestedQuery = 4, RULE_nestedSubQuery = 5, RULE_nestedSimpleSubQuery = 6, + RULE_nestedParenthesizedQuery = 7, RULE_matchAllQuery = 8, RULE_parenthesizedQuery = 9, + RULE_rangeQuery = 10, RULE_rangeQueryValue = 11, RULE_existsQuery = 12, + RULE_fieldQuery = 13, RULE_fieldLessQuery = 14, RULE_fieldQueryValue = 15, + RULE_fieldName = 16; private static String[] makeRuleNames() { return new String[] { - "topLevelQuery", "query", "simpleQuery", "notQuery", "nestedQuery", "matchAllQuery", + "topLevelQuery", "query", "simpleQuery", "notQuery", "nestedQuery", "nestedSubQuery", + "nestedSimpleSubQuery", "nestedParenthesizedQuery", "matchAllQuery", "parenthesizedQuery", "rangeQuery", "rangeQueryValue", "existsQuery", "fieldQuery", "fieldLessQuery", "fieldQueryValue", "fieldName" }; @@ -139,17 +142,17 @@ public final TopLevelQueryContext topLevelQuery() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(29); + setState(35); _errHandler.sync(this); _la = _input.LA(1); if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 115740L) != 0)) { { - setState(28); + setState(34); query(0); } } - setState(31); + setState(37); match(EOF); } } @@ -244,11 +247,11 @@ private QueryContext query(int _p) throws RecognitionException { _ctx = _localctx; _prevctx = _localctx; - setState(34); + setState(40); simpleQuery(); } _ctx.stop = _input.LT(-1); - setState(41); + setState(47); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,1,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -259,9 +262,9 @@ private QueryContext query(int _p) throws RecognitionException { { _localctx = new BooleanQueryContext(new QueryContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_query); - setState(36); + setState(42); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(37); + setState(43); ((BooleanQueryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==AND || _la==OR) ) { @@ -272,12 +275,12 @@ private QueryContext query(int _p) throws RecognitionException { _errHandler.reportMatch(this); consume(); } - setState(38); + setState(44); query(2); } } } - setState(43); + setState(49); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,1,_ctx); } @@ -343,62 +346,62 @@ public final SimpleQueryContext simpleQuery() throws RecognitionException { SimpleQueryContext _localctx = new SimpleQueryContext(_ctx, getState()); enterRule(_localctx, 4, RULE_simpleQuery); try { - setState(52); + setState(58); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(44); + setState(50); notQuery(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(45); + setState(51); nestedQuery(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(46); + setState(52); parenthesizedQuery(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(47); + setState(53); matchAllQuery(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(48); + setState(54); existsQuery(); } break; case 6: enterOuterAlt(_localctx, 6); { - setState(49); + setState(55); rangeQuery(); } break; case 7: enterOuterAlt(_localctx, 7); { - setState(50); + setState(56); fieldQuery(); } break; case 8: enterOuterAlt(_localctx, 8); { - setState(51); + setState(57); fieldLessQuery(); } break; @@ -447,9 +450,9 @@ public final NotQueryContext notQuery() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(54); + setState(60); match(NOT); - setState(55); + setState(61); ((NotQueryContext)_localctx).subQuery = simpleQuery(); } } @@ -471,8 +474,8 @@ public FieldNameContext fieldName() { } public TerminalNode COLON() { return getToken(KqlBaseParser.COLON, 0); } public TerminalNode LEFT_CURLY_BRACKET() { return getToken(KqlBaseParser.LEFT_CURLY_BRACKET, 0); } - public QueryContext query() { - return getRuleContext(QueryContext.class,0); + public NestedSubQueryContext nestedSubQuery() { + return getRuleContext(NestedSubQueryContext.class,0); } public TerminalNode RIGHT_CURLY_BRACKET() { return getToken(KqlBaseParser.RIGHT_CURLY_BRACKET, 0); } public NestedQueryContext(ParserRuleContext parent, int invokingState) { @@ -500,15 +503,15 @@ public final NestedQueryContext nestedQuery() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(57); + setState(63); fieldName(); - setState(58); + setState(64); match(COLON); - setState(59); + setState(65); match(LEFT_CURLY_BRACKET); - setState(60); - query(0); - setState(61); + setState(66); + nestedSubQuery(0); + setState(67); match(RIGHT_CURLY_BRACKET); } } @@ -523,6 +526,288 @@ public final NestedQueryContext nestedQuery() throws RecognitionException { return _localctx; } + @SuppressWarnings("CheckReturnValue") + public static class NestedSubQueryContext extends ParserRuleContext { + public NestedSubQueryContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_nestedSubQuery; } + + public NestedSubQueryContext() { } + public void copyFrom(NestedSubQueryContext ctx) { + super.copyFrom(ctx); + } + } + @SuppressWarnings("CheckReturnValue") + public static class BooleanNestedQueryContext extends NestedSubQueryContext { + public Token operator; + public List nestedSubQuery() { + return getRuleContexts(NestedSubQueryContext.class); + } + public NestedSubQueryContext nestedSubQuery(int i) { + return getRuleContext(NestedSubQueryContext.class,i); + } + public TerminalNode AND() { return getToken(KqlBaseParser.AND, 0); } + public TerminalNode OR() { return getToken(KqlBaseParser.OR, 0); } + public BooleanNestedQueryContext(NestedSubQueryContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterBooleanNestedQuery(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitBooleanNestedQuery(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor)visitor).visitBooleanNestedQuery(this); + else return visitor.visitChildren(this); + } + } + @SuppressWarnings("CheckReturnValue") + public static class DefaultNestedQueryContext extends NestedSubQueryContext { + public NestedSimpleSubQueryContext nestedSimpleSubQuery() { + return getRuleContext(NestedSimpleSubQueryContext.class,0); + } + public DefaultNestedQueryContext(NestedSubQueryContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterDefaultNestedQuery(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitDefaultNestedQuery(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor)visitor).visitDefaultNestedQuery(this); + else return visitor.visitChildren(this); + } + } + + public final NestedSubQueryContext nestedSubQuery() throws RecognitionException { + return nestedSubQuery(0); + } + + private NestedSubQueryContext nestedSubQuery(int _p) throws RecognitionException { + ParserRuleContext _parentctx = _ctx; + int _parentState = getState(); + NestedSubQueryContext _localctx = new NestedSubQueryContext(_ctx, _parentState); + NestedSubQueryContext _prevctx = _localctx; + int _startState = 10; + enterRecursionRule(_localctx, 10, RULE_nestedSubQuery, _p); + int _la; + try { + int _alt; + enterOuterAlt(_localctx, 1); + { + { + _localctx = new DefaultNestedQueryContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + + setState(70); + nestedSimpleSubQuery(); + } + _ctx.stop = _input.LT(-1); + setState(77); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,3,_ctx); + while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { + if ( _alt==1 ) { + if ( _parseListeners!=null ) triggerExitRuleEvent(); + _prevctx = _localctx; + { + { + _localctx = new BooleanNestedQueryContext(new NestedSubQueryContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_nestedSubQuery); + setState(72); + if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); + setState(73); + ((BooleanNestedQueryContext)_localctx).operator = _input.LT(1); + _la = _input.LA(1); + if ( !(_la==AND || _la==OR) ) { + ((BooleanNestedQueryContext)_localctx).operator = (Token)_errHandler.recoverInline(this); + } + else { + if ( _input.LA(1)==Token.EOF ) matchedEOF = true; + _errHandler.reportMatch(this); + consume(); + } + setState(74); + nestedSubQuery(2); + } + } + } + setState(79); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,3,_ctx); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + unrollRecursionContexts(_parentctx); + } + return _localctx; + } + + @SuppressWarnings("CheckReturnValue") + public static class NestedSimpleSubQueryContext extends ParserRuleContext { + public NotQueryContext notQuery() { + return getRuleContext(NotQueryContext.class,0); + } + public NestedQueryContext nestedQuery() { + return getRuleContext(NestedQueryContext.class,0); + } + public NestedParenthesizedQueryContext nestedParenthesizedQuery() { + return getRuleContext(NestedParenthesizedQueryContext.class,0); + } + public ExistsQueryContext existsQuery() { + return getRuleContext(ExistsQueryContext.class,0); + } + public RangeQueryContext rangeQuery() { + return getRuleContext(RangeQueryContext.class,0); + } + public FieldQueryContext fieldQuery() { + return getRuleContext(FieldQueryContext.class,0); + } + public NestedSimpleSubQueryContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_nestedSimpleSubQuery; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterNestedSimpleSubQuery(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitNestedSimpleSubQuery(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor)visitor).visitNestedSimpleSubQuery(this); + else return visitor.visitChildren(this); + } + } + + public final NestedSimpleSubQueryContext nestedSimpleSubQuery() throws RecognitionException { + NestedSimpleSubQueryContext _localctx = new NestedSimpleSubQueryContext(_ctx, getState()); + enterRule(_localctx, 12, RULE_nestedSimpleSubQuery); + try { + setState(86); + _errHandler.sync(this); + switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) { + case 1: + enterOuterAlt(_localctx, 1); + { + setState(80); + notQuery(); + } + break; + case 2: + enterOuterAlt(_localctx, 2); + { + setState(81); + nestedQuery(); + } + break; + case 3: + enterOuterAlt(_localctx, 3); + { + setState(82); + nestedParenthesizedQuery(); + } + break; + case 4: + enterOuterAlt(_localctx, 4); + { + setState(83); + existsQuery(); + } + break; + case 5: + enterOuterAlt(_localctx, 5); + { + setState(84); + rangeQuery(); + } + break; + case 6: + enterOuterAlt(_localctx, 6); + { + setState(85); + fieldQuery(); + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + @SuppressWarnings("CheckReturnValue") + public static class NestedParenthesizedQueryContext extends ParserRuleContext { + public TerminalNode LEFT_PARENTHESIS() { return getToken(KqlBaseParser.LEFT_PARENTHESIS, 0); } + public NestedSubQueryContext nestedSubQuery() { + return getRuleContext(NestedSubQueryContext.class,0); + } + public TerminalNode RIGHT_PARENTHESIS() { return getToken(KqlBaseParser.RIGHT_PARENTHESIS, 0); } + public NestedParenthesizedQueryContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_nestedParenthesizedQuery; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterNestedParenthesizedQuery(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitNestedParenthesizedQuery(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor)visitor).visitNestedParenthesizedQuery(this); + else return visitor.visitChildren(this); + } + } + + public final NestedParenthesizedQueryContext nestedParenthesizedQuery() throws RecognitionException { + NestedParenthesizedQueryContext _localctx = new NestedParenthesizedQueryContext(_ctx, getState()); + enterRule(_localctx, 14, RULE_nestedParenthesizedQuery); + try { + enterOuterAlt(_localctx, 1); + { + setState(88); + match(LEFT_PARENTHESIS); + setState(89); + nestedSubQuery(0); + setState(90); + match(RIGHT_PARENTHESIS); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + @SuppressWarnings("CheckReturnValue") public static class MatchAllQueryContext extends ParserRuleContext { public List WILDCARD() { return getTokens(KqlBaseParser.WILDCARD); } @@ -551,23 +836,23 @@ public T accept(ParseTreeVisitor visitor) { public final MatchAllQueryContext matchAllQuery() throws RecognitionException { MatchAllQueryContext _localctx = new MatchAllQueryContext(_ctx, getState()); - enterRule(_localctx, 10, RULE_matchAllQuery); + enterRule(_localctx, 16, RULE_matchAllQuery); try { enterOuterAlt(_localctx, 1); { - setState(65); + setState(94); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,3,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) { case 1: { - setState(63); + setState(92); match(WILDCARD); - setState(64); + setState(93); match(COLON); } break; } - setState(67); + setState(96); match(WILDCARD); } } @@ -610,15 +895,15 @@ public T accept(ParseTreeVisitor visitor) { public final ParenthesizedQueryContext parenthesizedQuery() throws RecognitionException { ParenthesizedQueryContext _localctx = new ParenthesizedQueryContext(_ctx, getState()); - enterRule(_localctx, 12, RULE_parenthesizedQuery); + enterRule(_localctx, 18, RULE_parenthesizedQuery); try { enterOuterAlt(_localctx, 1); { - setState(69); + setState(98); match(LEFT_PARENTHESIS); - setState(70); + setState(99); query(0); - setState(71); + setState(100); match(RIGHT_PARENTHESIS); } } @@ -667,14 +952,14 @@ public T accept(ParseTreeVisitor visitor) { public final RangeQueryContext rangeQuery() throws RecognitionException { RangeQueryContext _localctx = new RangeQueryContext(_ctx, getState()); - enterRule(_localctx, 14, RULE_rangeQuery); + enterRule(_localctx, 20, RULE_rangeQuery); int _la; try { enterOuterAlt(_localctx, 1); { - setState(73); + setState(102); fieldName(); - setState(74); + setState(103); ((RangeQueryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 960L) != 0)) ) { @@ -685,7 +970,7 @@ public final RangeQueryContext rangeQuery() throws RecognitionException { _errHandler.reportMatch(this); consume(); } - setState(75); + setState(104); rangeQueryValue(); } } @@ -732,18 +1017,18 @@ public T accept(ParseTreeVisitor visitor) { public final RangeQueryValueContext rangeQueryValue() throws RecognitionException { RangeQueryValueContext _localctx = new RangeQueryValueContext(_ctx, getState()); - enterRule(_localctx, 16, RULE_rangeQueryValue); + enterRule(_localctx, 22, RULE_rangeQueryValue); int _la; try { int _alt; - setState(83); + setState(112); _errHandler.sync(this); switch (_input.LA(1)) { case UNQUOTED_LITERAL: case WILDCARD: enterOuterAlt(_localctx, 1); { - setState(78); + setState(107); _errHandler.sync(this); _alt = 1; do { @@ -751,7 +1036,7 @@ public final RangeQueryValueContext rangeQueryValue() throws RecognitionExceptio case 1: { { - setState(77); + setState(106); _la = _input.LA(1); if ( !(_la==UNQUOTED_LITERAL || _la==WILDCARD) ) { _errHandler.recoverInline(this); @@ -767,16 +1052,16 @@ public final RangeQueryValueContext rangeQueryValue() throws RecognitionExceptio default: throw new NoViableAltException(this); } - setState(80); + setState(109); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,4,_ctx); + _alt = getInterpreter().adaptivePredict(_input,6,_ctx); } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ); } break; case QUOTED_STRING: enterOuterAlt(_localctx, 2); { - setState(82); + setState(111); match(QUOTED_STRING); } break; @@ -823,15 +1108,15 @@ public T accept(ParseTreeVisitor visitor) { public final ExistsQueryContext existsQuery() throws RecognitionException { ExistsQueryContext _localctx = new ExistsQueryContext(_ctx, getState()); - enterRule(_localctx, 18, RULE_existsQuery); + enterRule(_localctx, 24, RULE_existsQuery); try { enterOuterAlt(_localctx, 1); { - setState(85); + setState(114); fieldName(); - setState(86); + setState(115); match(COLON); - setState(87); + setState(116); match(WILDCARD); } } @@ -878,34 +1163,34 @@ public T accept(ParseTreeVisitor visitor) { public final FieldQueryContext fieldQuery() throws RecognitionException { FieldQueryContext _localctx = new FieldQueryContext(_ctx, getState()); - enterRule(_localctx, 20, RULE_fieldQuery); + enterRule(_localctx, 26, RULE_fieldQuery); try { - setState(99); + setState(128); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(89); + setState(118); fieldName(); - setState(90); + setState(119); match(COLON); - setState(91); + setState(120); fieldQueryValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(93); + setState(122); fieldName(); - setState(94); + setState(123); match(COLON); - setState(95); + setState(124); match(LEFT_PARENTHESIS); - setState(96); + setState(125); fieldQueryValue(); - setState(97); + setState(126); match(RIGHT_PARENTHESIS); } break; @@ -950,9 +1235,9 @@ public T accept(ParseTreeVisitor visitor) { public final FieldLessQueryContext fieldLessQuery() throws RecognitionException { FieldLessQueryContext _localctx = new FieldLessQueryContext(_ctx, getState()); - enterRule(_localctx, 22, RULE_fieldLessQuery); + enterRule(_localctx, 28, RULE_fieldLessQuery); try { - setState(106); + setState(135); _errHandler.sync(this); switch (_input.LA(1)) { case AND: @@ -963,18 +1248,18 @@ public final FieldLessQueryContext fieldLessQuery() throws RecognitionException case WILDCARD: enterOuterAlt(_localctx, 1); { - setState(101); + setState(130); fieldQueryValue(); } break; case LEFT_PARENTHESIS: enterOuterAlt(_localctx, 2); { - setState(102); + setState(131); match(LEFT_PARENTHESIS); - setState(103); + setState(132); fieldQueryValue(); - setState(104); + setState(133); match(RIGHT_PARENTHESIS); } break; @@ -1037,22 +1322,22 @@ public T accept(ParseTreeVisitor visitor) { public final FieldQueryValueContext fieldQueryValue() throws RecognitionException { FieldQueryValueContext _localctx = new FieldQueryValueContext(_ctx, getState()); - enterRule(_localctx, 24, RULE_fieldQueryValue); + enterRule(_localctx, 30, RULE_fieldQueryValue); int _la; try { int _alt; - setState(128); + setState(157); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(109); + setState(138); _errHandler.sync(this); _la = _input.LA(1); if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 28L) != 0)) { { - setState(108); + setState(137); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 28L) != 0)) ) { _errHandler.recoverInline(this); @@ -1065,7 +1350,7 @@ public final FieldQueryValueContext fieldQueryValue() throws RecognitionExceptio } } - setState(112); + setState(141); _errHandler.sync(this); _alt = 1; do { @@ -1073,7 +1358,7 @@ public final FieldQueryValueContext fieldQueryValue() throws RecognitionExceptio case 1: { { - setState(111); + setState(140); _la = _input.LA(1); if ( !(_la==UNQUOTED_LITERAL || _la==WILDCARD) ) { _errHandler.recoverInline(this); @@ -1089,16 +1374,16 @@ public final FieldQueryValueContext fieldQueryValue() throws RecognitionExceptio default: throw new NoViableAltException(this); } - setState(114); + setState(143); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,9,_ctx); + _alt = getInterpreter().adaptivePredict(_input,11,_ctx); } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ); - setState(117); + setState(146); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,10,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) { case 1: { - setState(116); + setState(145); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 28L) != 0)) ) { _errHandler.recoverInline(this); @@ -1116,7 +1401,7 @@ public final FieldQueryValueContext fieldQueryValue() throws RecognitionExceptio case 2: enterOuterAlt(_localctx, 2); { - setState(119); + setState(148); _la = _input.LA(1); if ( !(_la==AND || _la==OR) ) { _errHandler.recoverInline(this); @@ -1126,12 +1411,12 @@ public final FieldQueryValueContext fieldQueryValue() throws RecognitionExceptio _errHandler.reportMatch(this); consume(); } - setState(121); + setState(150); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: { - setState(120); + setState(149); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 28L) != 0)) ) { _errHandler.recoverInline(this); @@ -1149,14 +1434,14 @@ public final FieldQueryValueContext fieldQueryValue() throws RecognitionExceptio case 3: enterOuterAlt(_localctx, 3); { - setState(123); + setState(152); match(NOT); - setState(125); + setState(154); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: { - setState(124); + setState(153); _la = _input.LA(1); if ( !(_la==AND || _la==OR) ) { _errHandler.recoverInline(this); @@ -1174,7 +1459,7 @@ public final FieldQueryValueContext fieldQueryValue() throws RecognitionExceptio case 4: enterOuterAlt(_localctx, 4); { - setState(127); + setState(156); match(QUOTED_STRING); } break; @@ -1218,29 +1503,29 @@ public T accept(ParseTreeVisitor visitor) { public final FieldNameContext fieldName() throws RecognitionException { FieldNameContext _localctx = new FieldNameContext(_ctx, getState()); - enterRule(_localctx, 26, RULE_fieldName); + enterRule(_localctx, 32, RULE_fieldName); try { - setState(133); + setState(162); _errHandler.sync(this); switch (_input.LA(1)) { case UNQUOTED_LITERAL: enterOuterAlt(_localctx, 1); { - setState(130); + setState(159); ((FieldNameContext)_localctx).value = match(UNQUOTED_LITERAL); } break; case QUOTED_STRING: enterOuterAlt(_localctx, 2); { - setState(131); + setState(160); ((FieldNameContext)_localctx).value = match(QUOTED_STRING); } break; case WILDCARD: enterOuterAlt(_localctx, 3); { - setState(132); + setState(161); ((FieldNameContext)_localctx).value = match(WILDCARD); } break; @@ -1263,6 +1548,8 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { switch (ruleIndex) { case 1: return query_sempred((QueryContext)_localctx, predIndex); + case 5: + return nestedSubQuery_sempred((NestedSubQueryContext)_localctx, predIndex); } return true; } @@ -1273,87 +1560,117 @@ private boolean query_sempred(QueryContext _localctx, int predIndex) { } return true; } + private boolean nestedSubQuery_sempred(NestedSubQueryContext _localctx, int predIndex) { + switch (predIndex) { + case 1: + return precpred(_ctx, 2); + } + return true; + } public static final String _serializedATN = - "\u0004\u0001\u0010\u0088\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ + "\u0004\u0001\u0010\u00a5\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ "\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004"+ "\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007"+ "\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b"+ - "\u0002\f\u0007\f\u0002\r\u0007\r\u0001\u0000\u0003\u0000\u001e\b\u0000"+ - "\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ - "\u0001\u0001\u0001\u0001\u0005\u0001(\b\u0001\n\u0001\f\u0001+\t\u0001"+ - "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"+ - "\u0001\u0002\u0001\u0002\u0003\u00025\b\u0002\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004"+ - "\u0001\u0004\u0001\u0005\u0001\u0005\u0003\u0005B\b\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0007"+ - "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0004\bO\b\b\u000b\b\f\b"+ - "P\u0001\b\u0003\bT\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001\n"+ - "\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0003"+ - "\nd\b\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0003"+ - "\u000bk\b\u000b\u0001\f\u0003\fn\b\f\u0001\f\u0004\fq\b\f\u000b\f\f\f"+ - "r\u0001\f\u0003\fv\b\f\u0001\f\u0001\f\u0003\fz\b\f\u0001\f\u0001\f\u0003"+ - "\f~\b\f\u0001\f\u0003\f\u0081\b\f\u0001\r\u0001\r\u0001\r\u0003\r\u0086"+ - "\b\r\u0001\r\u0000\u0001\u0002\u000e\u0000\u0002\u0004\u0006\b\n\f\u000e"+ - "\u0010\u0012\u0014\u0016\u0018\u001a\u0000\u0004\u0001\u0000\u0002\u0003"+ - "\u0001\u0000\u0006\t\u0002\u0000\u000e\u000e\u0010\u0010\u0001\u0000\u0002"+ - "\u0004\u0091\u0000\u001d\u0001\u0000\u0000\u0000\u0002!\u0001\u0000\u0000"+ - "\u0000\u00044\u0001\u0000\u0000\u0000\u00066\u0001\u0000\u0000\u0000\b"+ - "9\u0001\u0000\u0000\u0000\nA\u0001\u0000\u0000\u0000\fE\u0001\u0000\u0000"+ - "\u0000\u000eI\u0001\u0000\u0000\u0000\u0010S\u0001\u0000\u0000\u0000\u0012"+ - "U\u0001\u0000\u0000\u0000\u0014c\u0001\u0000\u0000\u0000\u0016j\u0001"+ - "\u0000\u0000\u0000\u0018\u0080\u0001\u0000\u0000\u0000\u001a\u0085\u0001"+ - "\u0000\u0000\u0000\u001c\u001e\u0003\u0002\u0001\u0000\u001d\u001c\u0001"+ - "\u0000\u0000\u0000\u001d\u001e\u0001\u0000\u0000\u0000\u001e\u001f\u0001"+ - "\u0000\u0000\u0000\u001f \u0005\u0000\u0000\u0001 \u0001\u0001\u0000\u0000"+ - "\u0000!\"\u0006\u0001\uffff\uffff\u0000\"#\u0003\u0004\u0002\u0000#)\u0001"+ - "\u0000\u0000\u0000$%\n\u0002\u0000\u0000%&\u0007\u0000\u0000\u0000&(\u0003"+ - "\u0002\u0001\u0002\'$\u0001\u0000\u0000\u0000(+\u0001\u0000\u0000\u0000"+ - ")\'\u0001\u0000\u0000\u0000)*\u0001\u0000\u0000\u0000*\u0003\u0001\u0000"+ - "\u0000\u0000+)\u0001\u0000\u0000\u0000,5\u0003\u0006\u0003\u0000-5\u0003"+ - "\b\u0004\u0000.5\u0003\f\u0006\u0000/5\u0003\n\u0005\u000005\u0003\u0012"+ - "\t\u000015\u0003\u000e\u0007\u000025\u0003\u0014\n\u000035\u0003\u0016"+ - "\u000b\u00004,\u0001\u0000\u0000\u00004-\u0001\u0000\u0000\u00004.\u0001"+ - "\u0000\u0000\u00004/\u0001\u0000\u0000\u000040\u0001\u0000\u0000\u0000"+ - "41\u0001\u0000\u0000\u000042\u0001\u0000\u0000\u000043\u0001\u0000\u0000"+ - "\u00005\u0005\u0001\u0000\u0000\u000067\u0005\u0004\u0000\u000078\u0003"+ - "\u0004\u0002\u00008\u0007\u0001\u0000\u0000\u00009:\u0003\u001a\r\u0000"+ - ":;\u0005\u0005\u0000\u0000;<\u0005\f\u0000\u0000<=\u0003\u0002\u0001\u0000"+ - "=>\u0005\r\u0000\u0000>\t\u0001\u0000\u0000\u0000?@\u0005\u0010\u0000"+ - "\u0000@B\u0005\u0005\u0000\u0000A?\u0001\u0000\u0000\u0000AB\u0001\u0000"+ - "\u0000\u0000BC\u0001\u0000\u0000\u0000CD\u0005\u0010\u0000\u0000D\u000b"+ - "\u0001\u0000\u0000\u0000EF\u0005\n\u0000\u0000FG\u0003\u0002\u0001\u0000"+ - "GH\u0005\u000b\u0000\u0000H\r\u0001\u0000\u0000\u0000IJ\u0003\u001a\r"+ - "\u0000JK\u0007\u0001\u0000\u0000KL\u0003\u0010\b\u0000L\u000f\u0001\u0000"+ - "\u0000\u0000MO\u0007\u0002\u0000\u0000NM\u0001\u0000\u0000\u0000OP\u0001"+ - "\u0000\u0000\u0000PN\u0001\u0000\u0000\u0000PQ\u0001\u0000\u0000\u0000"+ - "QT\u0001\u0000\u0000\u0000RT\u0005\u000f\u0000\u0000SN\u0001\u0000\u0000"+ - "\u0000SR\u0001\u0000\u0000\u0000T\u0011\u0001\u0000\u0000\u0000UV\u0003"+ - "\u001a\r\u0000VW\u0005\u0005\u0000\u0000WX\u0005\u0010\u0000\u0000X\u0013"+ - "\u0001\u0000\u0000\u0000YZ\u0003\u001a\r\u0000Z[\u0005\u0005\u0000\u0000"+ - "[\\\u0003\u0018\f\u0000\\d\u0001\u0000\u0000\u0000]^\u0003\u001a\r\u0000"+ - "^_\u0005\u0005\u0000\u0000_`\u0005\n\u0000\u0000`a\u0003\u0018\f\u0000"+ - "ab\u0005\u000b\u0000\u0000bd\u0001\u0000\u0000\u0000cY\u0001\u0000\u0000"+ - "\u0000c]\u0001\u0000\u0000\u0000d\u0015\u0001\u0000\u0000\u0000ek\u0003"+ - "\u0018\f\u0000fg\u0005\n\u0000\u0000gh\u0003\u0018\f\u0000hi\u0005\u000b"+ - "\u0000\u0000ik\u0001\u0000\u0000\u0000je\u0001\u0000\u0000\u0000jf\u0001"+ - "\u0000\u0000\u0000k\u0017\u0001\u0000\u0000\u0000ln\u0007\u0003\u0000"+ - "\u0000ml\u0001\u0000\u0000\u0000mn\u0001\u0000\u0000\u0000np\u0001\u0000"+ - "\u0000\u0000oq\u0007\u0002\u0000\u0000po\u0001\u0000\u0000\u0000qr\u0001"+ - "\u0000\u0000\u0000rp\u0001\u0000\u0000\u0000rs\u0001\u0000\u0000\u0000"+ - "su\u0001\u0000\u0000\u0000tv\u0007\u0003\u0000\u0000ut\u0001\u0000\u0000"+ - "\u0000uv\u0001\u0000\u0000\u0000v\u0081\u0001\u0000\u0000\u0000wy\u0007"+ - "\u0000\u0000\u0000xz\u0007\u0003\u0000\u0000yx\u0001\u0000\u0000\u0000"+ - "yz\u0001\u0000\u0000\u0000z\u0081\u0001\u0000\u0000\u0000{}\u0005\u0004"+ - "\u0000\u0000|~\u0007\u0000\u0000\u0000}|\u0001\u0000\u0000\u0000}~\u0001"+ - "\u0000\u0000\u0000~\u0081\u0001\u0000\u0000\u0000\u007f\u0081\u0005\u000f"+ - "\u0000\u0000\u0080m\u0001\u0000\u0000\u0000\u0080w\u0001\u0000\u0000\u0000"+ - "\u0080{\u0001\u0000\u0000\u0000\u0080\u007f\u0001\u0000\u0000\u0000\u0081"+ - "\u0019\u0001\u0000\u0000\u0000\u0082\u0086\u0005\u000e\u0000\u0000\u0083"+ - "\u0086\u0005\u000f\u0000\u0000\u0084\u0086\u0005\u0010\u0000\u0000\u0085"+ - "\u0082\u0001\u0000\u0000\u0000\u0085\u0083\u0001\u0000\u0000\u0000\u0085"+ - "\u0084\u0001\u0000\u0000\u0000\u0086\u001b\u0001\u0000\u0000\u0000\u000f"+ - "\u001d)4APScjmruy}\u0080\u0085"; + "\u0002\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e\u0002\u000f\u0007"+ + "\u000f\u0002\u0010\u0007\u0010\u0001\u0000\u0003\u0000$\b\u0000\u0001"+ + "\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0001\u0005\u0001.\b\u0001\n\u0001\f\u00011\t\u0001\u0001"+ + "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ + "\u0002\u0001\u0002\u0003\u0002;\b\u0002\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ + "\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0005\u0005L\b\u0005\n\u0005\f\u0005O\t\u0005\u0001\u0006\u0001"+ + "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006W\b"+ + "\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b"+ + "\u0003\b_\b\b\u0001\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n"+ + "\u0001\n\u0001\n\u0001\n\u0001\u000b\u0004\u000bl\b\u000b\u000b\u000b"+ + "\f\u000bm\u0001\u000b\u0003\u000bq\b\u000b\u0001\f\u0001\f\u0001\f\u0001"+ + "\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001"+ + "\r\u0001\r\u0003\r\u0081\b\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001"+ + "\u000e\u0001\u000e\u0003\u000e\u0088\b\u000e\u0001\u000f\u0003\u000f\u008b"+ + "\b\u000f\u0001\u000f\u0004\u000f\u008e\b\u000f\u000b\u000f\f\u000f\u008f"+ + "\u0001\u000f\u0003\u000f\u0093\b\u000f\u0001\u000f\u0001\u000f\u0003\u000f"+ + "\u0097\b\u000f\u0001\u000f\u0001\u000f\u0003\u000f\u009b\b\u000f\u0001"+ + "\u000f\u0003\u000f\u009e\b\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0003"+ + "\u0010\u00a3\b\u0010\u0001\u0010\u0000\u0002\u0002\n\u0011\u0000\u0002"+ + "\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a\u001c\u001e"+ + " \u0000\u0004\u0001\u0000\u0002\u0003\u0001\u0000\u0006\t\u0002\u0000"+ + "\u000e\u000e\u0010\u0010\u0001\u0000\u0002\u0004\u00b1\u0000#\u0001\u0000"+ + "\u0000\u0000\u0002\'\u0001\u0000\u0000\u0000\u0004:\u0001\u0000\u0000"+ + "\u0000\u0006<\u0001\u0000\u0000\u0000\b?\u0001\u0000\u0000\u0000\nE\u0001"+ + "\u0000\u0000\u0000\fV\u0001\u0000\u0000\u0000\u000eX\u0001\u0000\u0000"+ + "\u0000\u0010^\u0001\u0000\u0000\u0000\u0012b\u0001\u0000\u0000\u0000\u0014"+ + "f\u0001\u0000\u0000\u0000\u0016p\u0001\u0000\u0000\u0000\u0018r\u0001"+ + "\u0000\u0000\u0000\u001a\u0080\u0001\u0000\u0000\u0000\u001c\u0087\u0001"+ + "\u0000\u0000\u0000\u001e\u009d\u0001\u0000\u0000\u0000 \u00a2\u0001\u0000"+ + "\u0000\u0000\"$\u0003\u0002\u0001\u0000#\"\u0001\u0000\u0000\u0000#$\u0001"+ + "\u0000\u0000\u0000$%\u0001\u0000\u0000\u0000%&\u0005\u0000\u0000\u0001"+ + "&\u0001\u0001\u0000\u0000\u0000\'(\u0006\u0001\uffff\uffff\u0000()\u0003"+ + "\u0004\u0002\u0000)/\u0001\u0000\u0000\u0000*+\n\u0002\u0000\u0000+,\u0007"+ + "\u0000\u0000\u0000,.\u0003\u0002\u0001\u0002-*\u0001\u0000\u0000\u0000"+ + ".1\u0001\u0000\u0000\u0000/-\u0001\u0000\u0000\u0000/0\u0001\u0000\u0000"+ + "\u00000\u0003\u0001\u0000\u0000\u00001/\u0001\u0000\u0000\u00002;\u0003"+ + "\u0006\u0003\u00003;\u0003\b\u0004\u00004;\u0003\u0012\t\u00005;\u0003"+ + "\u0010\b\u00006;\u0003\u0018\f\u00007;\u0003\u0014\n\u00008;\u0003\u001a"+ + "\r\u00009;\u0003\u001c\u000e\u0000:2\u0001\u0000\u0000\u0000:3\u0001\u0000"+ + "\u0000\u0000:4\u0001\u0000\u0000\u0000:5\u0001\u0000\u0000\u0000:6\u0001"+ + "\u0000\u0000\u0000:7\u0001\u0000\u0000\u0000:8\u0001\u0000\u0000\u0000"+ + ":9\u0001\u0000\u0000\u0000;\u0005\u0001\u0000\u0000\u0000<=\u0005\u0004"+ + "\u0000\u0000=>\u0003\u0004\u0002\u0000>\u0007\u0001\u0000\u0000\u0000"+ + "?@\u0003 \u0010\u0000@A\u0005\u0005\u0000\u0000AB\u0005\f\u0000\u0000"+ + "BC\u0003\n\u0005\u0000CD\u0005\r\u0000\u0000D\t\u0001\u0000\u0000\u0000"+ + "EF\u0006\u0005\uffff\uffff\u0000FG\u0003\f\u0006\u0000GM\u0001\u0000\u0000"+ + "\u0000HI\n\u0002\u0000\u0000IJ\u0007\u0000\u0000\u0000JL\u0003\n\u0005"+ + "\u0002KH\u0001\u0000\u0000\u0000LO\u0001\u0000\u0000\u0000MK\u0001\u0000"+ + "\u0000\u0000MN\u0001\u0000\u0000\u0000N\u000b\u0001\u0000\u0000\u0000"+ + "OM\u0001\u0000\u0000\u0000PW\u0003\u0006\u0003\u0000QW\u0003\b\u0004\u0000"+ + "RW\u0003\u000e\u0007\u0000SW\u0003\u0018\f\u0000TW\u0003\u0014\n\u0000"+ + "UW\u0003\u001a\r\u0000VP\u0001\u0000\u0000\u0000VQ\u0001\u0000\u0000\u0000"+ + "VR\u0001\u0000\u0000\u0000VS\u0001\u0000\u0000\u0000VT\u0001\u0000\u0000"+ + "\u0000VU\u0001\u0000\u0000\u0000W\r\u0001\u0000\u0000\u0000XY\u0005\n"+ + "\u0000\u0000YZ\u0003\n\u0005\u0000Z[\u0005\u000b\u0000\u0000[\u000f\u0001"+ + "\u0000\u0000\u0000\\]\u0005\u0010\u0000\u0000]_\u0005\u0005\u0000\u0000"+ + "^\\\u0001\u0000\u0000\u0000^_\u0001\u0000\u0000\u0000_`\u0001\u0000\u0000"+ + "\u0000`a\u0005\u0010\u0000\u0000a\u0011\u0001\u0000\u0000\u0000bc\u0005"+ + "\n\u0000\u0000cd\u0003\u0002\u0001\u0000de\u0005\u000b\u0000\u0000e\u0013"+ + "\u0001\u0000\u0000\u0000fg\u0003 \u0010\u0000gh\u0007\u0001\u0000\u0000"+ + "hi\u0003\u0016\u000b\u0000i\u0015\u0001\u0000\u0000\u0000jl\u0007\u0002"+ + "\u0000\u0000kj\u0001\u0000\u0000\u0000lm\u0001\u0000\u0000\u0000mk\u0001"+ + "\u0000\u0000\u0000mn\u0001\u0000\u0000\u0000nq\u0001\u0000\u0000\u0000"+ + "oq\u0005\u000f\u0000\u0000pk\u0001\u0000\u0000\u0000po\u0001\u0000\u0000"+ + "\u0000q\u0017\u0001\u0000\u0000\u0000rs\u0003 \u0010\u0000st\u0005\u0005"+ + "\u0000\u0000tu\u0005\u0010\u0000\u0000u\u0019\u0001\u0000\u0000\u0000"+ + "vw\u0003 \u0010\u0000wx\u0005\u0005\u0000\u0000xy\u0003\u001e\u000f\u0000"+ + "y\u0081\u0001\u0000\u0000\u0000z{\u0003 \u0010\u0000{|\u0005\u0005\u0000"+ + "\u0000|}\u0005\n\u0000\u0000}~\u0003\u001e\u000f\u0000~\u007f\u0005\u000b"+ + "\u0000\u0000\u007f\u0081\u0001\u0000\u0000\u0000\u0080v\u0001\u0000\u0000"+ + "\u0000\u0080z\u0001\u0000\u0000\u0000\u0081\u001b\u0001\u0000\u0000\u0000"+ + "\u0082\u0088\u0003\u001e\u000f\u0000\u0083\u0084\u0005\n\u0000\u0000\u0084"+ + "\u0085\u0003\u001e\u000f\u0000\u0085\u0086\u0005\u000b\u0000\u0000\u0086"+ + "\u0088\u0001\u0000\u0000\u0000\u0087\u0082\u0001\u0000\u0000\u0000\u0087"+ + "\u0083\u0001\u0000\u0000\u0000\u0088\u001d\u0001\u0000\u0000\u0000\u0089"+ + "\u008b\u0007\u0003\u0000\u0000\u008a\u0089\u0001\u0000\u0000\u0000\u008a"+ + "\u008b\u0001\u0000\u0000\u0000\u008b\u008d\u0001\u0000\u0000\u0000\u008c"+ + "\u008e\u0007\u0002\u0000\u0000\u008d\u008c\u0001\u0000\u0000\u0000\u008e"+ + "\u008f\u0001\u0000\u0000\u0000\u008f\u008d\u0001\u0000\u0000\u0000\u008f"+ + "\u0090\u0001\u0000\u0000\u0000\u0090\u0092\u0001\u0000\u0000\u0000\u0091"+ + "\u0093\u0007\u0003\u0000\u0000\u0092\u0091\u0001\u0000\u0000\u0000\u0092"+ + "\u0093\u0001\u0000\u0000\u0000\u0093\u009e\u0001\u0000\u0000\u0000\u0094"+ + "\u0096\u0007\u0000\u0000\u0000\u0095\u0097\u0007\u0003\u0000\u0000\u0096"+ + "\u0095\u0001\u0000\u0000\u0000\u0096\u0097\u0001\u0000\u0000\u0000\u0097"+ + "\u009e\u0001\u0000\u0000\u0000\u0098\u009a\u0005\u0004\u0000\u0000\u0099"+ + "\u009b\u0007\u0000\u0000\u0000\u009a\u0099\u0001\u0000\u0000\u0000\u009a"+ + "\u009b\u0001\u0000\u0000\u0000\u009b\u009e\u0001\u0000\u0000\u0000\u009c"+ + "\u009e\u0005\u000f\u0000\u0000\u009d\u008a\u0001\u0000\u0000\u0000\u009d"+ + "\u0094\u0001\u0000\u0000\u0000\u009d\u0098\u0001\u0000\u0000\u0000\u009d"+ + "\u009c\u0001\u0000\u0000\u0000\u009e\u001f\u0001\u0000\u0000\u0000\u009f"+ + "\u00a3\u0005\u000e\u0000\u0000\u00a0\u00a3\u0005\u000f\u0000\u0000\u00a1"+ + "\u00a3\u0005\u0010\u0000\u0000\u00a2\u009f\u0001\u0000\u0000\u0000\u00a2"+ + "\u00a0\u0001\u0000\u0000\u0000\u00a2\u00a1\u0001\u0000\u0000\u0000\u00a3"+ + "!\u0001\u0000\u0000\u0000\u0011#/:MV^mp\u0080\u0087\u008a\u008f\u0092"+ + "\u0096\u009a\u009d\u00a2"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseVisitor.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseVisitor.java index 18ef8f389195b..8200bfe0da25d 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseVisitor.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseVisitor.java @@ -56,6 +56,32 @@ interface KqlBaseVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitNestedQuery(KqlBaseParser.NestedQueryContext ctx); + /** + * Visit a parse tree produced by the {@code booleanNestedQuery} + * labeled alternative in {@link KqlBaseParser#nestedSubQuery}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitBooleanNestedQuery(KqlBaseParser.BooleanNestedQueryContext ctx); + /** + * Visit a parse tree produced by the {@code defaultNestedQuery} + * labeled alternative in {@link KqlBaseParser#nestedSubQuery}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitDefaultNestedQuery(KqlBaseParser.DefaultNestedQueryContext ctx); + /** + * Visit a parse tree produced by {@link KqlBaseParser#nestedSimpleSubQuery}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitNestedSimpleSubQuery(KqlBaseParser.NestedSimpleSubQueryContext ctx); + /** + * Visit a parse tree produced by {@link KqlBaseParser#nestedParenthesizedQuery}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitNestedParenthesizedQuery(KqlBaseParser.NestedParenthesizedQueryContext ctx); /** * Visit a parse tree produced by {@link KqlBaseParser#matchAllQuery}. * @param ctx the parse tree diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java index 5f88080fb3ed4..30740833ee40e 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java @@ -11,11 +11,18 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NestedLookup; +import org.elasticsearch.index.mapper.NestedObjectMapper; import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.support.NestedScope; import java.time.ZoneId; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.function.Supplier; + +import static org.elasticsearch.common.Strings.format; public class KqlParsingContext { @@ -32,10 +39,11 @@ public static Builder builder(QueryRewriteContext queryRewriteContext) { return new Builder(queryRewriteContext); } - private QueryRewriteContext queryRewriteContext; + private final QueryRewriteContext queryRewriteContext; private final boolean caseInsensitive; private final ZoneId timeZone; private final String defaultField; + private final NestedScope nestedScope = new NestedScope(); public KqlParsingContext(QueryRewriteContext queryRewriteContext, boolean caseInsensitive, ZoneId timeZone, String defaultField) { this.queryRewriteContext = queryRewriteContext; @@ -56,9 +64,17 @@ public String defaultField() { return defaultField; } + public String nestedPath(String fieldName) { + return nestedLookup().getNestedParent(fieldName); + } + + public boolean isNestedField(String fieldName) { + return nestedMappers().containsKey(fullFieldName(fieldName)); + } + public Set resolveFieldNames(String fieldNamePattern) { assert fieldNamePattern != null && fieldNamePattern.isEmpty() == false : "fieldNamePattern cannot be null or empty"; - return queryRewriteContext.getMatchingFieldNames(fieldNamePattern); + return queryRewriteContext.getMatchingFieldNames(fullFieldName(fieldNamePattern)); } public Set resolveDefaultFieldNames() { @@ -89,6 +105,38 @@ public boolean isSearchableField(String fieldName) { return isSearchableField(fieldName, fieldType(fieldName)); } + public NestedScope nestedScope() { + return nestedScope; + } + + public T withNestedPath(String nestedFieldName, Supplier supplier) { + assert isNestedField(nestedFieldName); + nestedScope.nextLevel(nestedMappers().get(fullFieldName(nestedFieldName))); + T result = supplier.get(); + nestedScope.previousLevel(); + return result; + } + + public String currentNestedPath() { + return nestedScope().getObjectMapper() != null ? nestedScope().getObjectMapper().fullPath() : null; + } + + public String fullFieldName(String fieldName) { + if (nestedScope.getObjectMapper() == null) { + return fieldName; + } + + return format("%s.%s", nestedScope.getObjectMapper().fullPath(), fieldName); + } + + private NestedLookup nestedLookup() { + return queryRewriteContext.getMappingLookup().nestedLookup(); + } + + private Map nestedMappers() { + return nestedLookup().getNestedMappers(); + } + public static class Builder { private final QueryRewriteContext queryRewriteContext; private boolean caseInsensitive = true; diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java index 588e60bd4dd75..e6e4e20cfd3ca 100644 --- a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java +++ b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java @@ -46,11 +46,9 @@ import static org.hamcrest.Matchers.equalTo; public abstract class AbstractKqlParserTestCase extends AbstractBuilderTestCase { - protected static final String SUPPORTED_QUERY_FILE_PATH = "/supported-queries"; protected static final String UNSUPPORTED_QUERY_FILE_PATH = "/unsupported-queries"; protected static final Predicate BOOLEAN_QUERY_FILTER = (q) -> q.matches("(?i)[^{]*[^\\\\]*(NOT|AND|OR)[^}]*"); - protected static final String NESTED_FIELD_NAME = "mapped_nested"; @Override diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlNestedFieldQueryTests.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlNestedFieldQueryTests.java new file mode 100644 index 0000000000000..5660945fa0db3 --- /dev/null +++ b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlNestedFieldQueryTests.java @@ -0,0 +1,297 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.kql.parser; + +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.common.Strings.format; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + +public class KqlNestedFieldQueryTests extends AbstractKqlParserTestCase { + public void testInvalidNestedFieldName() { + for (String invalidFieldName : List.of(OBJECT_FIELD_NAME, TEXT_FIELD_NAME, "not_a_field", "mapped_nest*")) { + KqlParsingException e = assertThrows( + KqlParsingException.class, + () -> parseKqlQuery(format("%s : { %s: foo AND %s < 10 } ", invalidFieldName, TEXT_FIELD_NAME, INT_FIELD_NAME)) + ); + assertThat(e.getMessage(), Matchers.containsString(invalidFieldName)); + assertThat(e.getMessage(), Matchers.containsString("is not a valid nested field name")); + } + } + + public void testInlineNestedFieldMatchTextQuery() { + for (String fieldName : List.of(TEXT_FIELD_NAME, INT_FIELD_NAME)) { + { + // Querying a nested text subfield. + String nestedFieldName = format("%s.%s", NESTED_FIELD_NAME, fieldName); + String searchTerms = randomSearchTerms(); + String kqlQueryString = format("%s: %s", nestedFieldName, searchTerms); + + NestedQueryBuilder nestedQuery = asInstanceOf(NestedQueryBuilder.class, parseKqlQuery(kqlQueryString)); + + assertThat(nestedQuery.path(), equalTo(NESTED_FIELD_NAME)); + assertMatchQueryBuilder(nestedQuery.query(), nestedFieldName, searchTerms); + } + + { + // Several levels of nested fields. + String nestedFieldName = format("%s.%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, fieldName); + String searchTerms = randomSearchTerms(); + String kqlQueryString = format("%s: %s", nestedFieldName, searchTerms); + + NestedQueryBuilder nestedQuery = asInstanceOf(NestedQueryBuilder.class, parseKqlQuery(kqlQueryString)); + assertThat(nestedQuery.path(), equalTo(NESTED_FIELD_NAME)); + + NestedQueryBuilder nestedSubQuery = asInstanceOf(NestedQueryBuilder.class, nestedQuery.query()); + assertThat(nestedSubQuery.path(), equalTo(format("%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME))); + + assertMatchQueryBuilder(nestedSubQuery.query(), nestedFieldName, searchTerms); + } + } + } + + public void testInlineNestedFieldMatchKeywordFieldQuery() { + { + // Querying a nested text subfield. + String nestedFieldName = format("%s.%s", NESTED_FIELD_NAME, KEYWORD_FIELD_NAME); + String searchTerms = randomSearchTerms(); + String kqlQueryString = format("%s: %s", nestedFieldName, searchTerms); + + NestedQueryBuilder nestedQuery = asInstanceOf(NestedQueryBuilder.class, parseKqlQuery(kqlQueryString)); + + assertThat(nestedQuery.path(), equalTo(NESTED_FIELD_NAME)); + assertTermQueryBuilder(nestedQuery.query(), nestedFieldName, searchTerms); + } + + { + // Several levels of nested fields. + String nestedFieldName = format("%s.%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, KEYWORD_FIELD_NAME); + String searchTerms = randomSearchTerms(); + String kqlQueryString = format("%s: %s", nestedFieldName, searchTerms); + + NestedQueryBuilder nestedQuery = asInstanceOf(NestedQueryBuilder.class, parseKqlQuery(kqlQueryString)); + assertThat(nestedQuery.path(), equalTo(NESTED_FIELD_NAME)); + + NestedQueryBuilder nestedSubQuery = asInstanceOf(NestedQueryBuilder.class, nestedQuery.query()); + assertThat(nestedSubQuery.path(), equalTo(format("%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME))); + + assertTermQueryBuilder(nestedSubQuery.query(), nestedFieldName, searchTerms); + } + } + + public void testInlineNestedFieldRangeQuery() { + { + // Querying a nested text subfield. + String nestedFieldName = format("%s.%s", NESTED_FIELD_NAME, INT_FIELD_NAME); + String operator = randomFrom(">", ">=", "<", "<="); + String kqlQueryString = format("%s %s %s", nestedFieldName, operator, randomDouble()); + + NestedQueryBuilder nestedQuery = asInstanceOf(NestedQueryBuilder.class, parseKqlQuery(kqlQueryString)); + + assertThat(nestedQuery.path(), equalTo(NESTED_FIELD_NAME)); + assertRangeQueryBuilder(nestedQuery.query(), nestedFieldName, rangeQueryBuilder -> {}); + } + + { + // Several levels of nested fields. + String nestedFieldName = format("%s.%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, INT_FIELD_NAME); + String operator = randomFrom(">", ">=", "<", "<="); + String kqlQueryString = format("%s %s %s", nestedFieldName, operator, randomDouble()); + + NestedQueryBuilder nestedQuery = asInstanceOf(NestedQueryBuilder.class, parseKqlQuery(kqlQueryString)); + assertThat(nestedQuery.path(), equalTo(NESTED_FIELD_NAME)); + + NestedQueryBuilder nestedSubQuery = asInstanceOf(NestedQueryBuilder.class, nestedQuery.query()); + assertThat(nestedSubQuery.path(), equalTo(format("%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME))); + + assertRangeQueryBuilder(nestedSubQuery.query(), nestedFieldName, rangeQueryBuilder -> {}); + } + } + + public void testNestedQuerySyntax() { + // Single word - Keyword & text field + List.of(KEYWORD_FIELD_NAME, TEXT_FIELD_NAME) + .forEach( + fieldName -> assertThat( + parseKqlQuery(format("%s : { %s : %s }", NESTED_FIELD_NAME, fieldName, "foo")), + equalTo(parseKqlQuery(format("%s.%s : %s", NESTED_FIELD_NAME, fieldName, "foo"))) + ) + ); + + // Multiple words - Keyword & text field + List.of(KEYWORD_FIELD_NAME, TEXT_FIELD_NAME) + .forEach( + fieldName -> assertThat( + parseKqlQuery(format("%s : { %s : %s }", NESTED_FIELD_NAME, fieldName, "foo bar")), + equalTo(parseKqlQuery(format("%s.%s : %s", NESTED_FIELD_NAME, fieldName, "foo bar"))) + ) + ); + + // Range syntax + { + String operator = randomFrom("<", "<=", ">", ">="); + double rangeValue = randomDouble(); + assertThat( + parseKqlQuery(format("%s : { %s %s %s }", NESTED_FIELD_NAME, INT_FIELD_NAME, operator, rangeValue)), + equalTo(parseKqlQuery(format("%s.%s %s %s", NESTED_FIELD_NAME, INT_FIELD_NAME, operator, rangeValue))) + ); + } + + // Several level of nesting + { + QueryBuilder inlineQuery = parseKqlQuery( + format("%s.%s.%s : %s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, TEXT_FIELD_NAME, "foo bar") + ); + + assertThat( + parseKqlQuery(format("%s : { %s : { %s : %s } }", NESTED_FIELD_NAME, NESTED_FIELD_NAME, TEXT_FIELD_NAME, "foo bar")), + equalTo(inlineQuery) + ); + + assertThat( + parseKqlQuery(format("%s.%s : { %s : %s }", NESTED_FIELD_NAME, NESTED_FIELD_NAME, TEXT_FIELD_NAME, "foo bar")), + equalTo(inlineQuery) + ); + + assertThat( + parseKqlQuery(format("%s : { %s.%s : %s }", NESTED_FIELD_NAME, NESTED_FIELD_NAME, TEXT_FIELD_NAME, "foo bar")), + equalTo(inlineQuery) + ); + } + } + + public void testBooleanAndNestedQuerySyntax() { + NestedQueryBuilder nestedQuery = asInstanceOf( + NestedQueryBuilder.class, + parseKqlQuery( + format("%s: { %s : foo AND %s: bar AND %s > 3}", NESTED_FIELD_NAME, TEXT_FIELD_NAME, KEYWORD_FIELD_NAME, INT_FIELD_NAME) + ) + ); + assertThat(nestedQuery.path(), equalTo(NESTED_FIELD_NAME)); + + BoolQueryBuilder subQuery = asInstanceOf(BoolQueryBuilder.class, nestedQuery.query()); + assertThat(subQuery.should(), empty()); + assertThat(subQuery.filter(), empty()); + assertThat(subQuery.mustNot(), empty()); + assertThat(subQuery.must(), hasSize(3)); + assertMatchQueryBuilder( + subQuery.must().stream().filter(q -> q instanceof MatchQueryBuilder).findFirst().get(), + format("%s.%s", NESTED_FIELD_NAME, TEXT_FIELD_NAME), + "foo" + ); + assertTermQueryBuilder( + subQuery.must().stream().filter(q -> q instanceof TermQueryBuilder).findFirst().get(), + format("%s.%s", NESTED_FIELD_NAME, KEYWORD_FIELD_NAME), + "bar" + ); + assertRangeQueryBuilder( + subQuery.must().stream().filter(q -> q instanceof RangeQueryBuilder).findAny().get(), + format("%s.%s", NESTED_FIELD_NAME, INT_FIELD_NAME), + q -> {} + ); + } + + public void testBooleanOrNestedQuerySyntax() { + NestedQueryBuilder nestedQuery = asInstanceOf( + NestedQueryBuilder.class, + parseKqlQuery( + format("%s: { %s : foo OR %s: bar OR %s > 3 }", NESTED_FIELD_NAME, TEXT_FIELD_NAME, KEYWORD_FIELD_NAME, INT_FIELD_NAME) + ) + ); + + assertThat(nestedQuery.path(), equalTo(NESTED_FIELD_NAME)); + + BoolQueryBuilder subQuery = asInstanceOf(BoolQueryBuilder.class, nestedQuery.query()); + assertThat(subQuery.must(), empty()); + assertThat(subQuery.filter(), empty()); + assertThat(subQuery.mustNot(), empty()); + assertThat(subQuery.should(), hasSize(3)); + assertMatchQueryBuilder( + subQuery.should().stream().filter(q -> q instanceof MatchQueryBuilder).findFirst().get(), + format("%s.%s", NESTED_FIELD_NAME, TEXT_FIELD_NAME), + "foo" + ); + assertTermQueryBuilder( + subQuery.should().stream().filter(q -> q instanceof TermQueryBuilder).findFirst().get(), + format("%s.%s", NESTED_FIELD_NAME, KEYWORD_FIELD_NAME), + "bar" + ); + assertRangeQueryBuilder( + subQuery.should().stream().filter(q -> q instanceof RangeQueryBuilder).findAny().get(), + format("%s.%s", NESTED_FIELD_NAME, INT_FIELD_NAME), + q -> {} + ); + } + + public void testBooleanNotNestedQuerySyntax() { + { + NestedQueryBuilder nestedQuery = asInstanceOf( + NestedQueryBuilder.class, + parseKqlQuery(format("%s: { NOT %s : foo }", NESTED_FIELD_NAME, TEXT_FIELD_NAME)) + ); + + assertThat(nestedQuery.path(), equalTo(NESTED_FIELD_NAME)); + + BoolQueryBuilder subQuery = asInstanceOf(BoolQueryBuilder.class, nestedQuery.query()); + assertThat(subQuery.must(), empty()); + assertThat(subQuery.filter(), empty()); + assertThat(subQuery.should(), empty()); + assertThat(subQuery.mustNot(), hasSize(1)); + assertMatchQueryBuilder(subQuery.mustNot().get(0), format("%s.%s", NESTED_FIELD_NAME, TEXT_FIELD_NAME), "foo"); + } + + { + NestedQueryBuilder nestedQuery = asInstanceOf( + NestedQueryBuilder.class, + parseKqlQuery(format("%s: { NOT %s : foo }", NESTED_FIELD_NAME, KEYWORD_FIELD_NAME)) + ); + + assertThat(nestedQuery.path(), equalTo(NESTED_FIELD_NAME)); + + BoolQueryBuilder subQuery = asInstanceOf(BoolQueryBuilder.class, nestedQuery.query()); + assertThat(subQuery.must(), empty()); + assertThat(subQuery.filter(), empty()); + assertThat(subQuery.should(), empty()); + assertThat(subQuery.mustNot(), hasSize(1)); + assertTermQueryBuilder(subQuery.mustNot().get(0), format("%s.%s", NESTED_FIELD_NAME, KEYWORD_FIELD_NAME), "foo"); + } + + { + NestedQueryBuilder nestedQuery = asInstanceOf( + NestedQueryBuilder.class, + parseKqlQuery(format("%s: { NOT %s < 3 }", NESTED_FIELD_NAME, INT_FIELD_NAME)) + ); + + assertThat(nestedQuery.path(), equalTo(NESTED_FIELD_NAME)); + + BoolQueryBuilder subQuery = asInstanceOf(BoolQueryBuilder.class, nestedQuery.query()); + assertThat(subQuery.must(), empty()); + assertThat(subQuery.filter(), empty()); + assertThat(subQuery.should(), empty()); + assertThat(subQuery.mustNot(), hasSize(1)); + assertRangeQueryBuilder(subQuery.mustNot().get(0), format("%s.%s", NESTED_FIELD_NAME, INT_FIELD_NAME), q -> {}); + } + } + + private static String randomSearchTerms() { + return Stream.generate(ESTestCase::randomIdentifier).limit(randomIntBetween(1, 10)).collect(Collectors.joining(" ")); + } +} diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserExistsQueryTests.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserExistsQueryTests.java index 45dd3312bbc03..6415cdb94ada7 100644 --- a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserExistsQueryTests.java +++ b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserExistsQueryTests.java @@ -10,7 +10,10 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.NestedQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; + +import java.util.regex.Pattern; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; @@ -35,11 +38,18 @@ public void testParseExistsQueryWithNoMatchingFields() { public void testParseExistsQueryWithASingleField() { for (String fieldName : searchableFields()) { - ExistsQueryBuilder parsedQuery = asInstanceOf(ExistsQueryBuilder.class, parseKqlQuery(kqlExistsQuery(fieldName))); - assertThat(parsedQuery.fieldName(), equalTo(fieldName)); + QueryBuilder parsedQuery = parseKqlQuery(kqlExistsQuery(fieldName)); // Using quotes to wrap the field name does not change the result. assertThat(parseKqlQuery(kqlExistsQuery("\"" + fieldName + "\"")), equalTo(parsedQuery)); + + long nestingLevel = Pattern.compile("[.]").splitAsStream(fieldName).takeWhile(s -> s.equals(NESTED_FIELD_NAME)).count(); + for (int i = 0; i < nestingLevel; i++) { + parsedQuery = asInstanceOf(NestedQueryBuilder.class, parsedQuery).query(); + } + + ExistsQueryBuilder existsQuery = asInstanceOf(ExistsQueryBuilder.class, parsedQuery); + assertThat(existsQuery.fieldName(), equalTo(fieldName)); } } @@ -53,7 +63,9 @@ public void testParseExistsQueryUsingWildcardFieldName() { assertThat( parsedQuery.should(), - containsInAnyOrder(searchableFields(fieldNamePattern).stream().map(QueryBuilders::existsQuery).toArray()) + containsInAnyOrder( + searchableFields(fieldNamePattern).stream().map(fieldName -> parseKqlQuery(kqlExistsQuery(fieldName))).toArray() + ) ); } diff --git a/x-pack/plugin/kql/src/test/resources/supported-queries b/x-pack/plugin/kql/src/test/resources/supported-queries index b659b1ae5b1db..f54a1d32fe3be 100644 --- a/x-pack/plugin/kql/src/test/resources/supported-queries +++ b/x-pack/plugin/kql/src/test/resources/supported-queries @@ -91,13 +91,6 @@ mapped_nested: { NOT(mapped_string:foo AND mapped_string_2:foo bar) } mapped_nested: { NOT mapped_string:foo AND NOT mapped_string_2:foo bar } mapped_nested: { (NOT mapped_string:foo) AND (NOT mapped_string_2:foo bar) } mapped_nested: { NOT(mapped_string:foo) AND NOT(mapped_string_2:foo bar) } -mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar AND foo bar } -mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar OR foo bar } -mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar OR foo bar } -mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar AND foo bar } -mapped_nested: { mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) } -mapped_nested: { mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) } -mapped_nested: { mapped_string:foo OR (mapped_string_2:foo bar OR foo bar) } mapped_nested: { mapped_str*:foo } mapped_nested: { mapped_nested : { mapped_string:foo AND mapped_int < 3 } AND mapped_string_2:foo bar } mapped_nested: { mapped_nested.mapped_string:foo AND mapped_string_2:foo bar } diff --git a/x-pack/plugin/kql/src/test/resources/unsupported-queries b/x-pack/plugin/kql/src/test/resources/unsupported-queries index 149bcf5bd2b5a..526ae94d6ac88 100644 --- a/x-pack/plugin/kql/src/test/resources/unsupported-queries +++ b/x-pack/plugin/kql/src/test/resources/unsupported-queries @@ -25,6 +25,20 @@ mapped_string:(foo (bar)) // Bad syntax for nested fields: mapped_nested { mapped_string: bar } +// Unknown nested field or not a nested field +not_nested : { mapped_string: bar } +mapped_string: { mapped_string: bar } + +// Nested query can not use fieldless subqueries +mapped_nested: { foo } +mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar AND foo bar } +mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar OR foo bar } +mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar OR foo bar } +mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar AND foo bar } +mapped_nested: { mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) } +mapped_nested: { mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) } +mapped_nested: { mapped_string:foo OR (mapped_string_2:foo bar OR foo bar) } + // Missing escape sequences: mapped_string: foo:bar mapped_string: (foo and bar) diff --git a/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/50_kql_nested_fields_query.yml b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/50_kql_nested_fields_query.yml new file mode 100644 index 0000000000000..4ce6688e5222d --- /dev/null +++ b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/50_kql_nested_fields_query.yml @@ -0,0 +1,218 @@ +setup: + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ kql_query ] + test_runner_features: [ capabilities, contains ] + reason: KQL query is not available + + - requires: + "test_runner_features": "contains" + + - do: + indices.create: + index: test-index + body: + mappings: + properties: + department: + type: keyword + staff: + type: integer + courses: + type: nested + properties: + name: + type: text + credits: + type: integer + sessions: + type: nested + properties: + semester: + type: keyword + students: + type: integer + + - do: + bulk: + index: test-index + refresh: true + body: | + { "index" : { "_id": "doc-1" } } + { "department": "compsci", "staff": 12, "courses": [ { "name": "Object Oriented Programming", "credits": 3, "sessions": [ { "semester": "spr2021", "students": 37 }, { "semester": "fall2020", "students": 45} ] }, { "name": "Theory of Computation", "credits": 4, "sessions": [ { "semester": "spr2021", "students": 19 }, { "semester": "fall2020", "students": 14 } ] } ] } + { "index" : { "_id": "doc-42" } } + { "department": "math", "staff": 20, "courses": [ { "name": "Precalculus", "credits": 1, "sessions": [ { "semester": "spr2021", "students": 100 }, { "semester": "fall2020", "students": 134 } ] }, { "name": "Linear Algebra", "credits": 3, "sessions": [ { "semester": "spr2021", "students": 29 }, { "semester": "fall2020", "students": 23 } ] } ] } + +--- +"Inline syntax": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { + "kql": { + "query": "courses.name: object oriented programming" + } + } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { + "kql": { + "query": "courses.name: object oriented programming AND courses.credits > 3" + } + } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { + "kql": { + "query": "courses.name: object oriented programming OR courses.credits > 3" + } + } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + +--- +"Nested field syntax": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { + "kql": { + "query": "courses : { name: object oriented programming }" + } + } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { + "kql": { + "query": "courses: { name: object oriented programming AND credits > 3 }" + } + } + } + - match: { hits.total: 0 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { + "kql": { + "query": "courses: { name: object oriented programming AND credits >= 3 }" + } + } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { + "kql": { + "query": "courses: { name: object oriented programming OR credits > 3 }" + } + } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { + "kql": { + "query": "courses: { NOT name: object oriented programming AND credits < 4 }" + } + } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-42" } + + +--- +"Several level of nesting field syntax": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { + "kql": { + "query": "courses: { name: object oriented programming AND sessions.semester: spr2021 }" + } + } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { + "kql": { + "query": "courses: { sessions : { semester: spr2021 AND students < 20 } }" + } + } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { + "kql": { + "query": "courses: { name: computation AND sessions : { semester: spr2021 AND students < 20 } }" + } + } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } diff --git a/x-pack/plugin/logsdb/qa/with-basic/build.gradle b/x-pack/plugin/logsdb/qa/with-basic/build.gradle index 2fdeed338e1c1..44ebd83bf4f4c 100644 --- a/x-pack/plugin/logsdb/qa/with-basic/build.gradle +++ b/x-pack/plugin/logsdb/qa/with-basic/build.gradle @@ -15,7 +15,7 @@ dependencies { tasks.named("javaRestTest").configure { // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) usesDefaultDistribution() } diff --git a/x-pack/plugin/mapper-constant-keyword/build.gradle b/x-pack/plugin/mapper-constant-keyword/build.gradle index ad9d3c2f86637..3b11d951fe37a 100644 --- a/x-pack/plugin/mapper-constant-keyword/build.gradle +++ b/x-pack/plugin/mapper-constant-keyword/build.gradle @@ -18,7 +18,7 @@ dependencies { compileOnly project(path: xpackModule('core')) } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/x-pack/plugin/mapper-unsigned-long/build.gradle b/x-pack/plugin/mapper-unsigned-long/build.gradle index e011723da6230..faad1db822560 100644 --- a/x-pack/plugin/mapper-unsigned-long/build.gradle +++ b/x-pack/plugin/mapper-unsigned-long/build.gradle @@ -37,7 +37,7 @@ restResources { } } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/x-pack/plugin/mapper-version/build.gradle b/x-pack/plugin/mapper-version/build.gradle index 69622762b9d5b..fb760b3446dfd 100644 --- a/x-pack/plugin/mapper-version/build.gradle +++ b/x-pack/plugin/mapper-version/build.gradle @@ -25,7 +25,7 @@ dependencies { testImplementation project(path: xpackModule('analytics')) } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle index e79a771293392..67c26c78a6741 100644 --- a/x-pack/plugin/ml/build.gradle +++ b/x-pack/plugin/ml/build.gradle @@ -94,7 +94,7 @@ dependencies { } def mlCppVersion(){ - return (project.gradle.parent != null && BuildParams.isSnapshotBuild() == false) ? + return (project.gradle.parent != null && buildParams.isSnapshotBuild() == false) ? (project.version + "-SNAPSHOT") : project.version; } diff --git a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle index 64970d18b5c82..3854c70b0f389 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle +++ b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle @@ -18,7 +18,7 @@ testClusters.configureEach { setting 'slm.history_index_enabled', 'false' } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // Test clusters run with security disabled tasks.named("javaRestTest").configure{enabled = false } } diff --git a/x-pack/plugin/ml/qa/disabled/build.gradle b/x-pack/plugin/ml/qa/disabled/build.gradle index 232700d5f84aa..0d1d8d6484afc 100644 --- a/x-pack/plugin/ml/qa/disabled/build.gradle +++ b/x-pack/plugin/ml/qa/disabled/build.gradle @@ -12,7 +12,7 @@ testClusters.configureEach { setting 'xpack.ml.enabled', 'false' } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // Test clusters run with security disabled tasks.named("javaRestTest").configure{enabled = false } } diff --git a/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle index bc22552d0d734..b43132c2daf50 100644 --- a/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle @@ -12,7 +12,7 @@ dependencies { testImplementation project(':x-pack:qa') } -Version ccsCompatVersion = BuildParams.bwcVersions.minimumWireCompatibleVersion +Version ccsCompatVersion = buildParams.bwcVersions.minimumWireCompatibleVersion restResources { restApi { diff --git a/x-pack/plugin/ml/qa/single-node-tests/build.gradle b/x-pack/plugin/ml/qa/single-node-tests/build.gradle index 6979ec4dcbd31..5ed1c5179716f 100644 --- a/x-pack/plugin/ml/qa/single-node-tests/build.gradle +++ b/x-pack/plugin/ml/qa/single-node-tests/build.gradle @@ -12,7 +12,7 @@ testClusters.configureEach { setting 'xpack.license.self_generated.type', 'trial' } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // Test clusters run with security disabled tasks.named("javaRestTest").configure{enabled = false } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java index 56cdcc88df91b..15fb2b2b81f30 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java @@ -138,16 +138,20 @@ public void testRunJob_TaskIsStopping() { when(task.isStopping()).thenReturn(true); when(task.getParams()).thenReturn(new StartDataFrameAnalyticsAction.TaskParams("data_frame_id", MlConfigVersion.CURRENT, false)); - processManager.runJob(task, dataFrameAnalyticsConfig, dataExtractorFactory, + processManager.runJob( + task, + dataFrameAnalyticsConfig, + dataExtractorFactory, ActionTestUtils.assertNoFailureListener(stepResponse -> { - assertThat(processManager.getProcessContextCount(), equalTo(0)); - assertThat(stepResponse.isTaskComplete(), is(true)); + assertThat(processManager.getProcessContextCount(), equalTo(0)); + assertThat(stepResponse.isTaskComplete(), is(true)); - InOrder inOrder = inOrder(task); - inOrder.verify(task).isStopping(); - inOrder.verify(task).getParams(); - verifyNoMoreInteractions(task); - })); + InOrder inOrder = inOrder(task); + inOrder.verify(task).isStopping(); + inOrder.verify(task).getParams(); + verifyNoMoreInteractions(task); + }) + ); } public void testRunJob_ProcessContextAlreadyExists() { diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java index 6822f54633bdc..b1d4f3ff7045f 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java @@ -53,6 +53,8 @@ public void setUp() throws Exception { "dcvO5uZATE-EhIKc3tk9Bg", null, null, + null, + null, new ShardStats[] { // Primaries new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null, null, false, 0), diff --git a/x-pack/plugin/repositories-metering-api/qa/azure/build.gradle b/x-pack/plugin/repositories-metering-api/qa/azure/build.gradle index 01264d7849680..4683c13f1fc0c 100644 --- a/x-pack/plugin/repositories-metering-api/qa/azure/build.gradle +++ b/x-pack/plugin/repositories-metering-api/qa/azure/build.gradle @@ -38,7 +38,7 @@ tasks.named("javaRestTest") { systemProperty 'test.azure.container', azureContainer systemProperty 'test.azure.key', azureKey systemProperty 'test.azure.sas_token', azureSasToken - nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repositories_metering_tests_" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repositories_metering_tests_" + buildParams.testSeed } tasks.register("azureThirdPartyTest") { diff --git a/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle b/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle index b8c345c99b895..62fe47c08f5f5 100644 --- a/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle +++ b/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle @@ -35,7 +35,7 @@ if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { tasks.named("javaRestTest").configure { systemProperty 'test.google.fixture', Boolean.toString(useFixture) systemProperty 'test.gcs.bucket', gcsBucket - nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_repositories_metering" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_repositories_metering" + buildParams.testSeed if (useFixture == false) { systemProperty 'test.google.account', serviceAccountFile } diff --git a/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle b/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle index 5f2bf66f31b21..3c58e6a06af69 100644 --- a/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle +++ b/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle @@ -38,7 +38,7 @@ tasks.named("javaRestTest").configure { systemProperty 'test.s3.bucket', s3Bucket systemProperty("s3AccessKey", s3AccessKey) systemProperty("s3SecretKey", s3SecretKey) - nonInputProperties.systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_repositories_metering" + BuildParams.testSeed : 'base_path_integration_tests' + nonInputProperties.systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_repositories_metering" + buildParams.testSeed : 'base_path_integration_tests' } tasks.register("s3ThirdPartyTest").configure { dependsOn "javaRestTest" diff --git a/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle b/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle index 6774ef920f280..e2f77fae89225 100644 --- a/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle @@ -33,7 +33,7 @@ tasks.named("javaRestTest").configure { systemProperty 'test.azure.container', azureContainer systemProperty 'test.azure.key', azureKey systemProperty 'test.azure.sas_token', azureSasToken - nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_searchable_snapshots_tests_" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_searchable_snapshots_tests_" + buildParams.testSeed } tasks.register("azureThirdPartyTest") { diff --git a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle index 3099f0787998e..c0a420aff313a 100644 --- a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle @@ -29,7 +29,7 @@ if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { tasks.named("javaRestTest").configure { systemProperty 'test.google.fixture', Boolean.toString(useFixture) systemProperty 'test.gcs.bucket', gcsBucket - nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_searchable_snapshots_tests" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_searchable_snapshots_tests" + buildParams.testSeed if (useFixture == false) { systemProperty 'test.google.account', serviceAccountFile diff --git a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle index 52ea873ae53bf..e8d97da9a9e37 100644 --- a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle @@ -28,6 +28,6 @@ restResources { tasks.named("javaRestTest").configure { usesDefaultDistribution() - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) jvmArgs '--add-exports', 'java.security.jgss/sun.security.krb5=ALL-UNNAMED' } diff --git a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle index 8919ddc6d29fd..430df2a7e8122 100644 --- a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle @@ -40,7 +40,7 @@ tasks.named("javaRestTest").configure { systemProperty("s3AccessKey", s3AccessKey) systemProperty("s3SecretKey", s3SecretKey) - nonInputProperties.systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_searchable_snapshots_tests" + BuildParams.testSeed : 'base_path_integration_tests' + nonInputProperties.systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_searchable_snapshots_tests" + buildParams.testSeed : 'base_path_integration_tests' } tasks.register("s3ThirdPartyTest") { diff --git a/x-pack/plugin/security/cli/build.gradle b/x-pack/plugin/security/cli/build.gradle index dcf3c7305dbc7..8fd3dd29f87a4 100644 --- a/x-pack/plugin/security/cli/build.gradle +++ b/x-pack/plugin/security/cli/build.gradle @@ -36,7 +36,7 @@ tasks.named("test").configure { systemProperty 'tests.security.manager', 'false' // the main code under test runs without the SecurityManager } -if (BuildParams.inFipsJvm) { +if (buildParams.inFipsJvm) { tasks.named("test").configure { enabled = false } diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle index f751fcd0a655d..f53ff7027f126 100644 --- a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle +++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle @@ -6,7 +6,7 @@ */ apply plugin: 'elasticsearch.build' -apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'com.gradleup.shadow' // See the build.gradle file in the parent directory for an explanation of this unusual build diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part2/build.gradle b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part2/build.gradle index c4c0f2ebd2fe1..d24299a3847da 100644 --- a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part2/build.gradle +++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part2/build.gradle @@ -6,7 +6,7 @@ */ apply plugin: 'elasticsearch.build' -apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'com.gradleup.shadow' // See the build.gradle file in the parent directory for an explanation of this unusual build diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle index 580ca45055219..4418bd32e64cf 100644 --- a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle +++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle @@ -6,7 +6,7 @@ */ apply plugin: 'elasticsearch.build' -apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'com.gradleup.shadow' // See the build.gradle file in the parent directory for an explanation of this unusual build diff --git a/x-pack/plugin/security/qa/basic-enable-security/build.gradle b/x-pack/plugin/security/qa/basic-enable-security/build.gradle index 5957216a3e12d..a6930d38d41e5 100644 --- a/x-pack/plugin/security/qa/basic-enable-security/build.gradle +++ b/x-pack/plugin/security/qa/basic-enable-security/build.gradle @@ -16,7 +16,7 @@ dependencies { tasks.named("javaRestTest").configure { // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) usesDefaultDistribution() } diff --git a/x-pack/plugin/security/qa/multi-cluster/build.gradle b/x-pack/plugin/security/qa/multi-cluster/build.gradle index b8eccb14819a4..8ee449d39dcce 100644 --- a/x-pack/plugin/security/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/security/qa/multi-cluster/build.gradle @@ -35,7 +35,7 @@ tasks.named("javaRestTest") { exclude '**/RemoteClusterSecurityBWCToRCS2ClusterRestIT.class' } -BuildParams.bwcVersions.withWireCompatible() { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible() { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/x-pack/plugin/security/qa/profile/build.gradle b/x-pack/plugin/security/qa/profile/build.gradle index ac821e670fde0..7465ef9917258 100644 --- a/x-pack/plugin/security/qa/profile/build.gradle +++ b/x-pack/plugin/security/qa/profile/build.gradle @@ -7,7 +7,7 @@ dependencies { javaRestTestImplementation project(':x-pack:plugin:security') } -boolean literalUsername = BuildParams.random.nextBoolean() +boolean literalUsername = buildParams.random.nextBoolean() tasks.named("javaRestTest").configure { usesDefaultDistribution() diff --git a/x-pack/plugin/security/qa/security-basic/build.gradle b/x-pack/plugin/security/qa/security-basic/build.gradle index 7684d879671ab..30751705bd75f 100644 --- a/x-pack/plugin/security/qa/security-basic/build.gradle +++ b/x-pack/plugin/security/qa/security-basic/build.gradle @@ -13,7 +13,7 @@ tasks.named('javaRestTest') { } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC tasks.named("javaRestTest").configure{enabled = false } } diff --git a/x-pack/plugin/security/qa/security-disabled/build.gradle b/x-pack/plugin/security/qa/security-disabled/build.gradle index eba70753c9f28..0a05eae479d33 100644 --- a/x-pack/plugin/security/qa/security-disabled/build.gradle +++ b/x-pack/plugin/security/qa/security-disabled/build.gradle @@ -17,5 +17,5 @@ dependencies { tasks.named("javaRestTest").configure { usesDefaultDistribution() // Test clusters run with security disabled - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) } diff --git a/x-pack/plugin/security/qa/tls-basic/build.gradle b/x-pack/plugin/security/qa/tls-basic/build.gradle index fbe91009011e3..e3b51bde45cc8 100644 --- a/x-pack/plugin/security/qa/tls-basic/build.gradle +++ b/x-pack/plugin/security/qa/tls-basic/build.gradle @@ -7,7 +7,7 @@ dependencies { javaRestTestImplementation(testArtifact(project(xpackModule('core')))) } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC tasks.named("javaRestTest").configure{enabled = false } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java index 6b9594c1c68ea..87651a96d75a6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java @@ -82,8 +82,9 @@ public void testInvalidateTokensWhenIndexUnavailable() throws Exception { when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(false); when(securityIndex.indexExists()).thenReturn(true); when(securityIndex.defensiveCopy()).thenReturn(securityIndex); - when(securityIndex.getUnavailableReason(SecurityIndexManager.Availability.PRIMARY_SHARDS)) - .thenReturn(new ElasticsearchException("simulated")); + when(securityIndex.getUnavailableReason(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn( + new ElasticsearchException("simulated") + ); final TokenService tokenService = new TokenService( SETTINGS, Clock.systemUTC(), @@ -102,8 +103,11 @@ public void testInvalidateTokensWhenIndexUnavailable() throws Exception { Tuple newTokenBytes = tokenService.getRandomTokenBytes(true); InvalidateTokenRequest request = new InvalidateTokenRequest( - tokenService.prependVersionAndEncodeAccessToken(TransportVersion.current(), newTokenBytes.v1()), - ACCESS_TOKEN.getValue(), null, null); + tokenService.prependVersionAndEncodeAccessToken(TransportVersion.current(), newTokenBytes.v1()), + ACCESS_TOKEN.getValue(), + null, + null + ); PlainActionFuture accessTokenfuture = new PlainActionFuture<>(); action.doExecute(null, request, accessTokenfuture); ElasticsearchSecurityException ese = expectThrows(ElasticsearchSecurityException.class, accessTokenfuture::actionGet); @@ -148,8 +152,11 @@ public void testInvalidateTokensWhenIndexClosed() throws Exception { Tuple newTokenBytes = tokenService.getRandomTokenBytes(true); InvalidateTokenRequest request = new InvalidateTokenRequest( - tokenService.prependVersionAndEncodeAccessToken(TransportVersion.current(), newTokenBytes.v1()), - ACCESS_TOKEN.getValue(), null, null); + tokenService.prependVersionAndEncodeAccessToken(TransportVersion.current(), newTokenBytes.v1()), + ACCESS_TOKEN.getValue(), + null, + null + ); PlainActionFuture accessTokenfuture = new PlainActionFuture<>(); action.doExecute(null, request, accessTokenfuture); ElasticsearchSecurityException ese = expectThrows(ElasticsearchSecurityException.class, accessTokenfuture::actionGet); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index cd6c88cf525af..7b66a95609b05 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -2041,8 +2041,14 @@ public void testExpiredToken() throws Exception { .user(new User("creator")) .realmRef(new RealmRef("test", "test", "test")) .build(false); - tokenService.createOAuth2Tokens(newTokenBytes.v1(), newTokenBytes.v2(), expected, originatingAuth, Collections.emptyMap(), - tokenFuture); + tokenService.createOAuth2Tokens( + newTokenBytes.v1(), + newTokenBytes.v2(), + expected, + originatingAuth, + Collections.emptyMap(), + tokenFuture + ); } String token = tokenFuture.get().getAccessToken(); mockGetTokenFromAccessTokenBytes(tokenService, newTokenBytes.v1(), expected, Map.of(), true, null, client); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java index 7219561dcf9df..aed39b24f217d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java @@ -76,11 +76,13 @@ public void init() throws Exception { } public void testAuthenticateThrowsOnUnsupportedMinVersions() throws IOException { - when(clusterService.state().getMinTransportVersion()).thenReturn(TransportVersionUtils.randomVersionBetween( + when(clusterService.state().getMinTransportVersion()).thenReturn( + TransportVersionUtils.randomVersionBetween( random(), TransportVersions.MINIMUM_COMPATIBLE, TransportVersionUtils.getPreviousVersion(TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY) - )); + ) + ); final var authcContext = mock(Authenticator.Context.class, Mockito.RETURNS_DEEP_STUBS); when(authcContext.getThreadContext()).thenReturn(threadContext); final var crossClusterAccessHeaders = new CrossClusterAccessHeaders( @@ -93,12 +95,14 @@ public void testAuthenticateThrowsOnUnsupportedMinVersions() throws IOException when(auditableRequest.exceptionProcessingRequest(any(), any())).thenAnswer( i -> new ElasticsearchSecurityException("potato", (Exception) i.getArguments()[0]) ); - doAnswer(invocationOnMock -> new Authenticator.Context( + doAnswer( + invocationOnMock -> new Authenticator.Context( threadContext, auditableRequest, mock(Realms.class), (AuthenticationToken) invocationOnMock.getArguments()[2] - )).when(authenticationService).newContext(anyString(), any(), any()); + ) + ).when(authenticationService).newContext(anyString(), any(), any()); final PlainActionFuture future = new PlainActionFuture<>(); crossClusterAccessAuthenticationService.authenticate("action", mock(TransportRequest.class), future); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsAuthenticatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsAuthenticatorTests.java index b35a2f8ccc4d3..02f397c23d3bd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsAuthenticatorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsAuthenticatorTests.java @@ -326,7 +326,7 @@ public void testRecordingFailedAuthenticationMetric() { @SuppressWarnings("unchecked") final ActionListener> listener = (ActionListener>) invocationOnMock .getArguments()[1]; - listener.onResponse(AuthenticationResult.unsuccessful("unsuccessful realms authentication", null)); + listener.onResponse(AuthenticationResult.unsuccessful("unsuccessful realms authentication", null)); return null; }).when(unsuccessfulRealm).authenticate(eq(authenticationToken), any()); @@ -337,7 +337,7 @@ public void testRecordingFailedAuthenticationMetric() { final PlainActionFuture> future = new PlainActionFuture<>(); realmsAuthenticator.authenticate(context, future); - var e = expectThrows(ElasticsearchSecurityException.class, () -> future.actionGet()); + var e = expectThrows(ElasticsearchSecurityException.class, () -> future.actionGet()); assertThat(e, sameInstance(exception)); assertSingleFailedAuthMetric( diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index 73a45dc20ac42..ed3949450cb9f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -363,9 +363,7 @@ public void testGetPrivilegesWillOnlyWaitOnUnavailableShardException() { public void testGetPrivilegesFailsAfterWaitOnUnavailableShardException() { when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(false).thenReturn(false); - when(securityIndex.getUnavailableReason(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn( - unavailableShardsException() - ); + when(securityIndex.getUnavailableReason(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(unavailableShardsException()); doAnswer(invocation -> { @SuppressWarnings("unchecked") final var listener = (ActionListener) invocation.getArguments()[0]; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateCrossClusterApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateCrossClusterApiKeyActionTests.java index 879e1ac8ad157..6c71f30243eaf 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateCrossClusterApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateCrossClusterApiKeyActionTests.java @@ -89,13 +89,10 @@ public void testLicenseEnforcement() throws Exception { // Disallow by license when(licenseState.isAllowed(Security.ADVANCED_REMOTE_CLUSTER_SECURITY_FEATURE)).thenReturn(false); - final FakeRestRequest restRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withContent( - new BytesArray(""" - { - "metadata": {} - }"""), - XContentType.JSON - ).withParams(Map.of("id", randomAlphaOfLength(10))).build(); + final FakeRestRequest restRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withContent(new BytesArray(""" + { + "metadata": {} + }"""), XContentType.JSON).withParams(Map.of("id", randomAlphaOfLength(10))).build(); final SetOnce responseSetOnce = new SetOnce<>(); final RestChannel restChannel = new AbstractRestChannel(restRequest, true) { @Override diff --git a/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle b/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle index 45bca88600495..515ffca4a59bf 100644 --- a/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle +++ b/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle @@ -13,7 +13,7 @@ dependencies { javaRestTestImplementation project(':x-pack:qa') } -BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/x-pack/plugin/shutdown/qa/rolling-upgrade/build.gradle b/x-pack/plugin/shutdown/qa/rolling-upgrade/build.gradle index 32cab39f665d3..4c98276abe154 100644 --- a/x-pack/plugin/shutdown/qa/rolling-upgrade/build.gradle +++ b/x-pack/plugin/shutdown/qa/rolling-upgrade/build.gradle @@ -36,7 +36,7 @@ tasks.register("copyTestNodeKeyMaterial", Copy) { into outputDir } -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> String oldVersion = bwcVersion.toString() // SearchableSnapshotsRollingUpgradeIT uses a specific repository to not interfere with other tests diff --git a/x-pack/plugin/slm/qa/multi-node/build.gradle b/x-pack/plugin/slm/qa/multi-node/build.gradle index 1f4b0c3b10c30..d6b1fe8a1e219 100644 --- a/x-pack/plugin/slm/qa/multi-node/build.gradle +++ b/x-pack/plugin/slm/qa/multi-node/build.gradle @@ -31,7 +31,7 @@ testClusters.configureEach { setting 'logger.org.elasticsearch.xpack.slm', 'TRACE' } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // Test clusters run with security disabled tasks.named("javaRestTest").configure{enabled = false } } diff --git a/x-pack/plugin/snapshot-based-recoveries/qa/azure/build.gradle b/x-pack/plugin/snapshot-based-recoveries/qa/azure/build.gradle index 03426bdddce62..cb2831f0cf273 100644 --- a/x-pack/plugin/snapshot-based-recoveries/qa/azure/build.gradle +++ b/x-pack/plugin/snapshot-based-recoveries/qa/azure/build.gradle @@ -39,7 +39,7 @@ tasks.named("javaRestTest").configure { systemProperty 'test.azure.container', azureContainer systemProperty 'test.azure.key', azureKey systemProperty 'test.azure.sas_token', azureSasToken - nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_snapshot_based_recoveries_tests_" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_snapshot_based_recoveries_tests_" + buildParams.testSeed } tasks.register("azureThirdPartyTest") { diff --git a/x-pack/plugin/snapshot-based-recoveries/qa/gcs/build.gradle b/x-pack/plugin/snapshot-based-recoveries/qa/gcs/build.gradle index 267ed84aa45d4..7550ab8585e13 100644 --- a/x-pack/plugin/snapshot-based-recoveries/qa/gcs/build.gradle +++ b/x-pack/plugin/snapshot-based-recoveries/qa/gcs/build.gradle @@ -28,7 +28,7 @@ if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { tasks.named("javaRestTest").configure { systemProperty 'test.google.fixture', Boolean.toString(useFixture) systemProperty 'test.gcs.bucket', gcsBucket - nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_snapshot_based_recoveries_tests" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_snapshot_based_recoveries_tests" + buildParams.testSeed if (useFixture == false) { systemProperty 'test.google.account', serviceAccountFile diff --git a/x-pack/plugin/snapshot-based-recoveries/qa/s3/build.gradle b/x-pack/plugin/snapshot-based-recoveries/qa/s3/build.gradle index b669641363bd1..e676e1f1f2162 100644 --- a/x-pack/plugin/snapshot-based-recoveries/qa/s3/build.gradle +++ b/x-pack/plugin/snapshot-based-recoveries/qa/s3/build.gradle @@ -47,7 +47,7 @@ tasks.named("javaRestTest").configure { systemProperty("s3AccessKey", s3AccessKey) systemProperty("s3SecretKey", s3SecretKey) nonInputProperties.systemProperty 'test.s3.base_path', - s3BasePath ? s3BasePath + "_snapshot_based_recoveries_tests" + BuildParams.testSeed : 'base_path_integration_tests' + s3BasePath ? s3BasePath + "_snapshot_based_recoveries_tests" + buildParams.testSeed : 'base_path_integration_tests' } tasks.register("s3ThirdPartyTest") { diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle index e304b2ff5c263..af4ed719a9c2f 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle @@ -46,7 +46,7 @@ tasks.named("javaRestTest") { systemProperty 'test.azure.sas_token', azureSasToken systemProperty 'test.azure.tenant_id', azureTenantId systemProperty 'test.azure.client_id', azureClientId - nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repository_test_kit_tests_" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repository_test_kit_tests_" + buildParams.testSeed } tasks.register("azureThirdPartyTest") { @@ -61,7 +61,7 @@ tasks.register("managedIdentityJavaRestTest", RestIntegTestTask) { systemProperty 'test.azure.container', azureContainer // omitting key and sas_token so that we use a bearer token from the metadata service // omitting client_id and tenant_id so that we use a bearer token from the metadata service, not from workload identity - nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repository_test_kit_tests_" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repository_test_kit_tests_" + buildParams.testSeed } tasks.register("workloadIdentityJavaRestTest", RestIntegTestTask) { @@ -74,10 +74,10 @@ tasks.register("workloadIdentityJavaRestTest", RestIntegTestTask) { systemProperty 'test.azure.tenant_id', azureTenantId ?: "583d4f71-148a-4163-bad5-2311e13c60dc" systemProperty 'test.azure.client_id', azureClientId ?: "86dd1b33-96c1-4a2e-92ac-b844404fc691" // omitting key and sas_token so that we use a bearer token from workload identity - nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repository_test_kit_tests_" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repository_test_kit_tests_" + buildParams.testSeed } -if (BuildParams.inFipsJvm) { +if (buildParams.inFipsJvm) { // Cannot override the trust store in FIPS mode, and these tasks require a HTTPS fixture tasks.named("managedIdentityJavaRestTest").configure { enabled = false } tasks.named("workloadIdentityJavaRestTest").configure { enabled = false } diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/build.gradle index 4f0a1c4faf0af..b7e1036ab3e26 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/build.gradle +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/build.gradle @@ -36,7 +36,7 @@ if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { tasks.named("javaRestTest").configure { systemProperty 'test.google.fixture', Boolean.toString(useFixture) systemProperty 'test.gcs.bucket', gcsBucket - nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_repository_test_kit_tests" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_repository_test_kit_tests" + buildParams.testSeed if (useFixture == false) { systemProperty 'test.google.account', serviceAccountFile diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle index 3fbb55ca4eb3a..14e2b05bc140e 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle @@ -34,7 +34,7 @@ dependencies { tasks.named("javaRestTest").configure { usesDefaultDistribution() description = "Runs rest tests against an elasticsearch cluster with HDFS." - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) // required for krb5kdc-fixture to work jvmArgs '--add-exports', 'java.security.jgss/sun.security.krb5=ALL-UNNAMED' } diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle index 21cf952f05bf1..313a11f8ce431 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle @@ -45,7 +45,7 @@ tasks.named("javaRestTest").configure { systemProperty("s3AccessKey", s3AccessKey) systemProperty("s3SecretKey", s3SecretKey) nonInputProperties.systemProperty 'test.s3.base_path', - s3BasePath ? s3BasePath + "_repo_test_kit_tests" + BuildParams.testSeed : 'base_path_integration_tests' + s3BasePath ? s3BasePath + "_repo_test_kit_tests" + buildParams.testSeed : 'base_path_integration_tests' } tasks.register("s3ThirdPartyTest") { diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index 5bcec68c227ce..4304bae5b9991 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -28,7 +28,7 @@ testClusters.configureEach { setting 'xpack.security.enabled', 'false' } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/x-pack/plugin/sql/build.gradle b/x-pack/plugin/sql/build.gradle index 85d778f9ec87f..d1dcbc3adbd95 100644 --- a/x-pack/plugin/sql/build.gradle +++ b/x-pack/plugin/sql/build.gradle @@ -137,7 +137,7 @@ allprojects { } } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // Test clusters run with security disabled tasks.named("internalClusterTest").configure{enabled = false } } diff --git a/x-pack/plugin/sql/jdbc/build.gradle b/x-pack/plugin/sql/jdbc/build.gradle index 138f3e63af462..d1b179f09e403 100644 --- a/x-pack/plugin/sql/jdbc/build.gradle +++ b/x-pack/plugin/sql/jdbc/build.gradle @@ -1,6 +1,6 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' -apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'com.gradleup.shadow' description = 'JDBC driver for Elasticsearch' diff --git a/x-pack/plugin/sql/qa/jdbc/build.gradle b/x-pack/plugin/sql/qa/jdbc/build.gradle index 022306fe9b306..a444399ed28ce 100644 --- a/x-pack/plugin/sql/qa/jdbc/build.gradle +++ b/x-pack/plugin/sql/qa/jdbc/build.gradle @@ -72,11 +72,11 @@ subprojects { // Configure compatibility testing tasks // Compatibility testing for JDBC driver started with version 7.9.0 - BuildParams.bwcVersions.indexCompatible.findAll({ it.onOrAfter(Version.fromString("7.9.0")) && it != VersionProperties.elasticsearchVersion }).each { bwcVersion -> + buildParams.bwcVersions.indexCompatible.findAll({ it.onOrAfter(Version.fromString("7.9.0")) && it != VersionProperties.elasticsearchVersion }).each { bwcVersion -> def baseName = "v${bwcVersion}" def cluster = testClusters.register(baseName) - UnreleasedVersionInfo unreleasedVersion = BuildParams.bwcVersions.unreleasedInfo(bwcVersion) + UnreleasedVersionInfo unreleasedVersion = buildParams.bwcVersions.unreleasedInfo(bwcVersion) Configuration driverConfiguration = configurations.create("jdbcDriver${baseName}") { // TODO: Temporary workaround for https://github.com/elastic/elasticsearch/issues/73433 transitive = false diff --git a/x-pack/plugin/sql/qa/jdbc/security/with-ssl/build.gradle b/x-pack/plugin/sql/qa/jdbc/security/with-ssl/build.gradle index ec88fcffa941c..971c7bf319244 100644 --- a/x-pack/plugin/sql/qa/jdbc/security/with-ssl/build.gradle +++ b/x-pack/plugin/sql/qa/jdbc/security/with-ssl/build.gradle @@ -11,5 +11,5 @@ testClusters.configureEach { // JDBC client can only be configured for SSL with keystores, but we can't use JKS/PKCS12 keystores in FIPS 140-2 mode. tasks.withType(Test).configureEach { - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) } diff --git a/x-pack/plugin/sql/qa/mixed-node/build.gradle b/x-pack/plugin/sql/qa/mixed-node/build.gradle index 412dec62f81f8..06e3b61d5b303 100644 --- a/x-pack/plugin/sql/qa/mixed-node/build.gradle +++ b/x-pack/plugin/sql/qa/mixed-node/build.gradle @@ -19,7 +19,7 @@ testClusters.configureEach { tasks.named("javaRestTest").configure{ enabled = false} // A bug (https://github.com/elastic/elasticsearch/issues/68439) limits us to perform tests with versions from 7.10.3 onwards -BuildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.10.3") && +buildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.10.3") && v != VersionProperties.getElasticsearchVersion()) { bwcVersion, baseName -> def baseCluster = testClusters.register(baseName) { diff --git a/x-pack/plugin/sql/qa/server/security/with-ssl/build.gradle b/x-pack/plugin/sql/qa/server/security/with-ssl/build.gradle index 907d72e606bda..51a3f83a909af 100644 --- a/x-pack/plugin/sql/qa/server/security/with-ssl/build.gradle +++ b/x-pack/plugin/sql/qa/server/security/with-ssl/build.gradle @@ -6,7 +6,7 @@ tasks.named("javaRestTest").configure { // Do not attempt to form a cluster in a FIPS JVM, as doing so with a JKS keystore will fail. // TODO Revisit this when SQL CLI client can handle key/certificate instead of only Keystores. // https://github.com/elastic/elasticsearch/issues/32306 - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) } testClusters.matching { it.name == "javaRestTest" }.configureEach { diff --git a/x-pack/plugin/sql/sql-cli/build.gradle b/x-pack/plugin/sql/sql-cli/build.gradle index b9713bcb8e7a3..cd24dcc15c863 100644 --- a/x-pack/plugin/sql/sql-cli/build.gradle +++ b/x-pack/plugin/sql/sql-cli/build.gradle @@ -8,7 +8,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams */ apply plugin: 'elasticsearch.build' -apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'com.gradleup.shadow' /* We don't use the 'application' plugin because it builds a zip and tgz which * we don't want. */ @@ -55,7 +55,7 @@ tasks.register("runcli") { description = 'Run the CLI and connect to elasticsearch running on 9200' dependsOn "shadowJar" doLast { - List command = ["${BuildParams.runtimeJavaHome}/bin/java"] + List command = ["${buildParams.runtimeJavaHome.get()}/bin/java"] if ('true'.equals(providers.systemProperty('debug').orElse('false').get())) { command += '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000' } diff --git a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle index b429e123bb631..eb0551a4d10e1 100644 --- a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle +++ b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle @@ -12,7 +12,7 @@ dependencies { testImplementation project(':x-pack:qa') } -Version ccsCompatVersion = BuildParams.bwcVersions.minimumWireCompatibleVersion +Version ccsCompatVersion = buildParams.bwcVersions.minimumWireCompatibleVersion restResources { restApi { diff --git a/x-pack/plugin/watcher/qa/rest/build.gradle b/x-pack/plugin/watcher/qa/rest/build.gradle index a911c022212b2..8382a71092720 100644 --- a/x-pack/plugin/watcher/qa/rest/build.gradle +++ b/x-pack/plugin/watcher/qa/rest/build.gradle @@ -29,7 +29,7 @@ testClusters.configureEach { setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG' } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // Test clusters run with security disabled tasks.named("javaRestTest").configure{enabled = false } tasks.named("yamlRestTest").configure{enabled = false } diff --git a/x-pack/plugin/wildcard/build.gradle b/x-pack/plugin/wildcard/build.gradle index 2bcf0db057aa5..b582f3fcea903 100644 --- a/x-pack/plugin/wildcard/build.gradle +++ b/x-pack/plugin/wildcard/build.gradle @@ -20,7 +20,7 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle index 0b8e459ed231b..8a67a2c1dde0d 100644 --- a/x-pack/qa/core-rest-tests-with-security/build.gradle +++ b/x-pack/qa/core-rest-tests-with-security/build.gradle @@ -27,7 +27,7 @@ tasks.named("yamlRestTest").configure { 'index/10_with_id/Index with ID', 'indices.get_alias/10_basic/Get alias against closed indices' ]; - if (BuildParams.isSnapshotBuild() == false) { + if (buildParams.isSnapshotBuild() == false) { blacklist += [ 'synonyms_privileges/10_synonyms_with_privileges/*', 'synonyms_privileges/20_synonyms_no_privileges/*' diff --git a/x-pack/qa/full-cluster-restart/build.gradle b/x-pack/qa/full-cluster-restart/build.gradle index 7248d1b0a6bfb..d6b05242f613b 100644 --- a/x-pack/qa/full-cluster-restart/build.gradle +++ b/x-pack/qa/full-cluster-restart/build.gradle @@ -15,7 +15,7 @@ dependencies { } -BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/x-pack/qa/mixed-tier-cluster/build.gradle b/x-pack/qa/mixed-tier-cluster/build.gradle index bf05be45e18a0..79e7d6a655993 100644 --- a/x-pack/qa/mixed-tier-cluster/build.gradle +++ b/x-pack/qa/mixed-tier-cluster/build.gradle @@ -10,7 +10,7 @@ dependencies { } // Only run tests for 7.9+, since the node.roles setting was introduced in 7.9.0 -BuildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.9.0") && +buildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.9.0") && v != VersionProperties.getElasticsearchVersion()) { bwcVersion, baseName -> def baseCluster = testClusters.register(baseName) { @@ -54,5 +54,5 @@ tasks.withType(Test).configureEach { classpath = sourceSets.javaRestTest.runtimeClasspath testClassesDirs = sourceSets.javaRestTest.output.classesDirs // Security is explicitly disabled, do not run tests in FIPS mode - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) } diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle index 6d41c4eddf31c..9c0648abca21b 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle @@ -13,7 +13,7 @@ restResources { } // randomise between sniff and proxy modes -boolean proxyMode = BuildParams.random.nextBoolean() +boolean proxyMode = buildParams.random.nextBoolean() def fulfillingCluster = testClusters.register('fulfilling-cluster') { setting 'xpack.security.enabled', 'true' diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle b/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle index 69c0e8b20c2c4..ca79bb7ec3825 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle @@ -13,7 +13,7 @@ restResources { } // randomise between sniff and proxy modes -boolean proxyMode = BuildParams.random.nextBoolean() +boolean proxyMode = buildParams.random.nextBoolean() def fulfillingCluster = testClusters.register('fulfilling-cluster') { setting 'xpack.security.enabled', 'true' diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle b/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle index 1164aa240ee22..b9f8369763476 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle @@ -23,7 +23,7 @@ tasks.register("copyCerts", Sync) { } // randomise between sniff and proxy modes -boolean proxyMode = BuildParams.random.nextBoolean() +boolean proxyMode = buildParams.random.nextBoolean() def fulfillingCluster = testClusters.register('fulfilling-cluster') { setting 'xpack.security.enabled', 'true' diff --git a/x-pack/qa/oidc-op-tests/build.gradle b/x-pack/qa/oidc-op-tests/build.gradle index 8f46613d5d9f0..b53539b224861 100644 --- a/x-pack/qa/oidc-op-tests/build.gradle +++ b/x-pack/qa/oidc-op-tests/build.gradle @@ -11,5 +11,5 @@ dependencies { tasks.named('javaRestTest') { usesDefaultDistribution() // test suite uses jks which is not supported in fips mode - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) } diff --git a/x-pack/qa/repository-old-versions/build.gradle b/x-pack/qa/repository-old-versions/build.gradle index 1abf6662a1b8b..78cfc0f688e4a 100644 --- a/x-pack/qa/repository-old-versions/build.gradle +++ b/x-pack/qa/repository-old-versions/build.gradle @@ -98,7 +98,7 @@ if (OS.current() == OS.WINDOWS) { TaskProvider fixture = tasks.register("oldES${versionNoDots}Fixture", AntFixture) { dependsOn project.configurations.oldesFixture, jdks.legacy, config - executable = "${BuildParams.runtimeJavaHome}/bin/java" + executable = "${buildParams.runtimeJavaHome.get()}/bin/java" env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}" // old versions of Elasticsearch need JAVA_HOME env 'JAVA_HOME', jdks.legacy.javaHomePath diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle index a7ea1695c477a..09b3b7db7c917 100644 --- a/x-pack/qa/rolling-upgrade-basic/build.gradle +++ b/x-pack/qa/rolling-upgrade-basic/build.gradle @@ -9,7 +9,7 @@ dependencies { testImplementation project(':x-pack:qa') } -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> def baseCluster = testClusters.register(baseName) { testDistribution = "DEFAULT" versions = [bwcVersion.toString(), project.version] @@ -78,5 +78,5 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> // Security is explicitly disabled, do not run tests in FIPS mode tasks.withType(Test).configureEach { - enabled = BuildParams.inFipsJvm == false -} \ No newline at end of file + enabled = buildParams.inFipsJvm == false +} diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle index 969ba23e19254..0d1cfbd5ff022 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle +++ b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle @@ -9,7 +9,7 @@ dependencies { testImplementation project(':x-pack:qa') } -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> def baseLeaderCluster = testClusters.register("${baseName}-leader") { numberOfNodes = 3 @@ -92,5 +92,5 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> // Security is explicitly disabled, do not run tests in FIPS mode tasks.withType(Test).configureEach { - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) } diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 271aadfe4b388..60fb55e9a2593 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -31,7 +31,7 @@ tasks.register("copyTestNodeKeyMaterial", Copy) { into outputDir } -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> String oldVersion = bwcVersion.toString() // SearchableSnapshotsRollingUpgradeIT uses a specific repository to not interfere with other tests diff --git a/x-pack/qa/third-party/jira/build.gradle b/x-pack/qa/third-party/jira/build.gradle index b7268af807535..626693a8f295f 100644 --- a/x-pack/qa/third-party/jira/build.gradle +++ b/x-pack/qa/third-party/jira/build.gradle @@ -55,7 +55,7 @@ if (!jiraUrl && !jiraUser && !jiraPassword && !jiraProject) { tasks.named("yamlRestTest")configure { finalizedBy "cleanJira" } } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // Test clusters run with security disabled tasks.named("yamlRestTest").configure{ enabled = false } } diff --git a/x-pack/qa/third-party/pagerduty/build.gradle b/x-pack/qa/third-party/pagerduty/build.gradle index 4b5a0bbeeeb4a..86ed67ccbb2d6 100644 --- a/x-pack/qa/third-party/pagerduty/build.gradle +++ b/x-pack/qa/third-party/pagerduty/build.gradle @@ -28,7 +28,7 @@ if (!pagerDutyServiceKey) { } } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // Test clusters run with security disabled tasks.named("yamlRestTest").configure{enabled = false } } diff --git a/x-pack/qa/third-party/slack/build.gradle b/x-pack/qa/third-party/slack/build.gradle index b2b0478da0471..ff501a7c99c9b 100644 --- a/x-pack/qa/third-party/slack/build.gradle +++ b/x-pack/qa/third-party/slack/build.gradle @@ -28,7 +28,7 @@ if (!slackUrl) { } } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // Test clusters run with security disabled tasks.named("yamlRestTest").configure{enabled = false } }