HSEARCH-5164 Add Elasticsearch 8.14.0 compatibility / HSEARCH-5165 Upgrade to Elasticsearch client 8.14.0

Jenkinsfile

@@ -267,7 +267,8 @@ stage('Configure') {
 					new LocalElasticsearchBuildEnvironment(version: '8.10.4', condition: TestCondition.ON_DEMAND),
 					new LocalElasticsearchBuildEnvironment(version: '8.11.4', condition: TestCondition.ON_DEMAND),
 					new LocalElasticsearchBuildEnvironment(version: '8.12.2', condition: TestCondition.ON_DEMAND),
-					new LocalElasticsearchBuildEnvironment(version: '8.13.4', condition: TestCondition.BEFORE_MERGE, isDefault: true),
+					new LocalElasticsearchBuildEnvironment(version: '8.13.4', condition: TestCondition.ON_DEMAND),
+					new LocalElasticsearchBuildEnvironment(version: '8.14.0', condition: TestCondition.BEFORE_MERGE, isDefault: true),
 					// IMPORTANT: Make sure to update the documentation for any newly supported Elasticsearch versions
 					//            See version.org.elasticsearch.compatible.expected.text
 					//            and version.org.elasticsearch.compatible.regularly-tested.text in POMs.

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/dialect/impl/ElasticsearchDialectFactory.java

@@ -13,6 +13,7 @@
 import org.hibernate.search.backend.elasticsearch.ElasticsearchVersion;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch7ModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch812ModelDialect;
+import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch814ModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch8ModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.ElasticsearchModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.OpenSearch1ModelDialect;
@@ -104,7 +105,11 @@ else if ( major == 7 ) {
 			if ( major == 8 && ( minorOptional.isEmpty() || minorOptional.getAsInt() < 12 ) ) {
 				return new Elasticsearch8ModelDialect();
 			}
-			return new Elasticsearch812ModelDialect();
+			if ( major == 8 && minorOptional.getAsInt() < 14 ) {
+				return new Elasticsearch812ModelDialect();
+			}
+
+			return new Elasticsearch814ModelDialect();
 		}
 	}
 
@@ -190,7 +195,7 @@ private ElasticsearchProtocolDialect createProtocolDialectElasticV7(Elasticsearc
 	}
 
 	private ElasticsearchProtocolDialect createProtocolDialectElasticV8(ElasticsearchVersion version, int minor) {
-		if ( minor > 13 ) {
+		if ( minor > 14 ) {
 			log.unknownElasticsearchVersion( version );
 		}
 		else if ( minor == 0 ) {

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/dialect/model/impl/Elasticsearch814ModelDialect.java

@@ -0,0 +1,30 @@
+/*
+ * Hibernate Search, full-text search for your domain model
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later
+ * See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
+ */
+package org.hibernate.search.backend.elasticsearch.dialect.model.impl;
+
+import org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl.Elasticsearch814IndexFieldTypeFactoryProvider;
+import org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl.ElasticsearchIndexFieldTypeFactoryProvider;
+import org.hibernate.search.backend.elasticsearch.validation.impl.Elasticsearch814PropertyMappingValidatorProvider;
+import org.hibernate.search.backend.elasticsearch.validation.impl.ElasticsearchPropertyMappingValidatorProvider;
+
+import com.google.gson.Gson;
+
+/**
+ * The model dialect for Elasticsearch 8.14+.
+ */
+public class Elasticsearch814ModelDialect implements ElasticsearchModelDialect {
+
+	@Override
+	public ElasticsearchIndexFieldTypeFactoryProvider createIndexTypeFieldFactoryProvider(Gson userFacingGson) {
+		return new Elasticsearch814IndexFieldTypeFactoryProvider( userFacingGson );
+	}
+
+	@Override
+	public ElasticsearchPropertyMappingValidatorProvider createElasticsearchPropertyMappingValidatorProvider() {
+		return new Elasticsearch814PropertyMappingValidatorProvider();
+	}
+}

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/dsl/provider/impl/Elasticsearch812IndexFieldTypeFactoryProvider.java

@@ -12,7 +12,7 @@
 import com.google.gson.Gson;
 
 /**
- * The index field type factory provider for ES8.12+.
+ * The index field type factory provider for ES8.12-8.13.
  */
 public class Elasticsearch812IndexFieldTypeFactoryProvider extends AbstractIndexFieldTypeFactoryProvider {
 

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/dsl/provider/impl/Elasticsearch814IndexFieldTypeFactoryProvider.java

@@ -0,0 +1,30 @@
+/*
+ * Hibernate Search, full-text search for your domain model
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later
+ * See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
+ */
+package org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl;
+
+import org.hibernate.search.backend.elasticsearch.types.mapping.impl.Elasticsearch814VectorFieldTypeMappingContributor;
+import org.hibernate.search.backend.elasticsearch.types.mapping.impl.ElasticsearchVectorFieldTypeMappingContributor;
+
+import com.google.gson.Gson;
+
+/**
+ * The index field type factory provider for ES8.14+.
+ */
+public class Elasticsearch814IndexFieldTypeFactoryProvider extends AbstractIndexFieldTypeFactoryProvider {
+
+	private final Elasticsearch814VectorFieldTypeMappingContributor vectorFieldTypeMappingContributor =
+			new Elasticsearch814VectorFieldTypeMappingContributor();
+
+	public Elasticsearch814IndexFieldTypeFactoryProvider(Gson userFacingGson) {
+		super( userFacingGson );
+	}
+
+	@Override
+	protected ElasticsearchVectorFieldTypeMappingContributor vectorFieldTypeMappingContributor() {
+		return vectorFieldTypeMappingContributor;
+	}
+}

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/mapping/impl/Elasticsearch812VectorFieldTypeMappingContributor.java

@@ -26,7 +26,7 @@ public void contribute(PropertyMapping mapping, Context context) {
 		if ( resolvedVectorSimilarity != null ) {
 			mapping.setSimilarity( resolvedVectorSimilarity );
 		}
-		if ( context.m() != null || context.efConstruction() != null ) {
+		if ( indexOptionAddCondition( context ) ) {
 			ElasticsearchDenseVectorIndexOptions indexOptions = new ElasticsearchDenseVectorIndexOptions();
 			indexOptions.setType( "hnsw" );
 			if ( context.m() != null ) {
@@ -39,6 +39,10 @@ public void contribute(PropertyMapping mapping, Context context) {
 		}
 	}
 
+	protected boolean indexOptionAddCondition(Context context) {
+		return context.m() != null || context.efConstruction() != null;
+	}
+
 	@Override
 	public <F> void contribute(ElasticsearchIndexValueFieldType.Builder<F> builder, Context context) {
 		builder.queryElementFactory( PredicateTypeKeys.KNN,

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/mapping/impl/Elasticsearch814VectorFieldTypeMappingContributor.java

@@ -0,0 +1,16 @@
+/*
+ * Hibernate Search, full-text search for your domain model
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later
+ * See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
+ */
+package org.hibernate.search.backend.elasticsearch.types.mapping.impl;
+
+public class Elasticsearch814VectorFieldTypeMappingContributor extends Elasticsearch812VectorFieldTypeMappingContributor {
+
+	@Override
+	protected boolean indexOptionAddCondition(Context context) {
+		// we want to always add index options and in particular include `hnsw` type.
+		return context.searchable();
+	}
+}

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/validation/impl/Elasticsearch814PropertyMappingValidatorProvider.java

@@ -0,0 +1,16 @@
+/*
+ * Hibernate Search, full-text search for your domain model
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later
+ * See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
+ */
+package org.hibernate.search.backend.elasticsearch.validation.impl;
+
+import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.PropertyMapping;
+
+public class Elasticsearch814PropertyMappingValidatorProvider implements ElasticsearchPropertyMappingValidatorProvider {
+	@Override
+	public Validator<PropertyMapping> create() {
+		return new PropertyMappingValidator.Elasticsearch814PropertyMappingValidator();
+	}
+}

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/validation/impl/PropertyMappingValidator.java

@@ -136,7 +136,24 @@ protected void validateVectorMapping(ValidationErrorCollector errorCollector, Pr
 		}
 	}
 
-	static class Elasticsearch812PropertyMappingValidator extends PropertyMappingValidator {
+	static class Elasticsearch812PropertyMappingValidator extends Elasticsearch8xPropertyMappingValidator {
+	}
+
+	static class Elasticsearch814PropertyMappingValidator extends Elasticsearch8xPropertyMappingValidator {
+		@Override
+		protected boolean indexOptionsRequireValidation(ElasticsearchDenseVectorIndexOptions expected,
+				ElasticsearchDenseVectorIndexOptions actual) {
+			if ( expected != null
+					&& actual == null
+					&& expected.getEfConstruction() == null && expected.getM() == null && expected.getType() != null ) {
+				// if we set type only then ES will not return the index option block ... so we skip
+				return false;
+			}
+			return super.indexOptionsRequireValidation( expected, actual );
+		}
+	}
+
+	static class Elasticsearch8xPropertyMappingValidator extends PropertyMappingValidator {
 
 		private final ElasticsearchDenseVectorIndexOptionsValidator indexOptionsValidator =
 				new ElasticsearchDenseVectorIndexOptionsValidator();
@@ -160,10 +177,16 @@ protected void validateVectorMapping(ValidationErrorCollector errorCollector, Pr
 			);
 
 			ElasticsearchDenseVectorIndexOptions indexOptions = expectedMapping.getIndexOptions();
-			if ( indexOptions != null ) {
-				indexOptionsValidator.validate( errorCollector, indexOptions, actualMapping.getIndexOptions() );
+			ElasticsearchDenseVectorIndexOptions actual = actualMapping.getIndexOptions();
+			if ( indexOptionsRequireValidation( indexOptions, actual ) ) {
+				indexOptionsValidator.validate( errorCollector, indexOptions, actual );
 			}
 		}
+
+		protected boolean indexOptionsRequireValidation(ElasticsearchDenseVectorIndexOptions expected,
+				ElasticsearchDenseVectorIndexOptions actual) {
+			return expected != null;
+		}
 	}
 
 	static class OpenSearch1PropertyMappingValidator extends PropertyMappingValidator {

backend/elasticsearch/src/test/java/org/hibernate/search/backend/elasticsearch/dialect/impl/ElasticsearchDialectFactoryTest.java

@@ -16,6 +16,7 @@
 import org.hibernate.search.backend.elasticsearch.ElasticsearchVersion;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch7ModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch812ModelDialect;
+import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch814ModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch8ModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.ElasticsearchModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.OpenSearch1ModelDialect;
@@ -257,17 +258,25 @@ public static List<? extends Arguments> params() {
 						ElasticsearchDistributionName.ELASTIC, "8.13.0", "8.13.0",
 						Elasticsearch812ModelDialect.class, Elasticsearch81ProtocolDialect.class
 				),
-				successWithWarning(
+				success(
 						ElasticsearchDistributionName.ELASTIC, "8.14", "8.14.0",
-						Elasticsearch812ModelDialect.class, Elasticsearch81ProtocolDialect.class
+						Elasticsearch814ModelDialect.class, Elasticsearch81ProtocolDialect.class
 				),
-				successWithWarning(
+				success(
 						ElasticsearchDistributionName.ELASTIC, "8.14.0", "8.14.0",
-						Elasticsearch812ModelDialect.class, Elasticsearch81ProtocolDialect.class
+						Elasticsearch814ModelDialect.class, Elasticsearch81ProtocolDialect.class
+				),
+				successWithWarning(
+						ElasticsearchDistributionName.ELASTIC, "8.15", "8.15.0",
+						Elasticsearch814ModelDialect.class, Elasticsearch81ProtocolDialect.class
+				),
+				successWithWarning(
+						ElasticsearchDistributionName.ELASTIC, "8.15.0", "8.15.0",
+						Elasticsearch814ModelDialect.class, Elasticsearch81ProtocolDialect.class
 				),
 				successWithWarning(
 						ElasticsearchDistributionName.ELASTIC, "9.0.0", "9.0.0",
-						Elasticsearch812ModelDialect.class, Elasticsearch81ProtocolDialect.class
+						Elasticsearch814ModelDialect.class, Elasticsearch81ProtocolDialect.class
 				),
 				success(
 						ElasticsearchDistributionName.OPENSEARCH, "1", "1.3.1",

build/container/search-backend/elastic.Dockerfile

@@ -4,4 +4,4 @@
 # IMPORTANT! When updating the version of Elasticsearch in this Dockerfile,
 # make sure to update `version.org.elasticsearch.latest` property in a POM file.
 #
-FROM docker.io/elastic/elasticsearch:8.13.4
+FROM docker.io/elastic/elasticsearch:8.14.0

build/parents/build/pom.xml

@@ -52,15 +52,15 @@
         <!-- >>> Elasticsearch -->
         <!-- The version of the Elasticsearch client used by Hibernate Search, independently of the version of the remote cluster -->
         <!-- Use the latest open-source version here. Currently, low-level clients are open-source even in 8.5+ -->
-        <version.org.elasticsearch.client>8.13.4</version.org.elasticsearch.client>
+        <version.org.elasticsearch.client>8.14.0</version.org.elasticsearch.client>
         <!-- The main compatible version of Elasticsearch, advertised by default. Used in documentation links. -->
         <version.org.elasticsearch.compatible.main>${version.org.elasticsearch.latest}</version.org.elasticsearch.compatible.main>
         <documentation.org.elasticsearch.url>https://www.elastic.co/guide/en/elasticsearch/reference/${parsed-version.org.elasticsearch.compatible.main.majorVersion}.${parsed-version.org.elasticsearch.compatible.main.minorVersion}</documentation.org.elasticsearch.url>
 
         <!-- The versions of Elasticsearch advertised as compatible with Hibernate Search -->
         <!-- Make sure to only mention tested versions here -->
         <!-- Make sure that 7.10 stays explicitly mentioned here, because that's the last open-source version -->
-        <version.org.elasticsearch.compatible.regularly-tested.text>7.10, 7.17 or 8.13</version.org.elasticsearch.compatible.regularly-tested.text>
+        <version.org.elasticsearch.compatible.regularly-tested.text>7.10, 7.17 or 8.14</version.org.elasticsearch.compatible.regularly-tested.text>
         <!-- These are the versions same as above, but pointing only to the major part (used in compatibility section of ES backend documentation
           as versions that Hibernate Search is compatible with. -->
         <!-- NOTE: Adding new major versions would require to update the compatibility table in `backend-elasticsearch-compatibility` section of `backend-elasticsearch.asciidoc`. -->

integrationtest/backend/tck/src/main/java/org/hibernate/search/integrationtest/backend/tck/search/predicate/AbstractPredicateSingleFieldIT.java

@@ -10,6 +10,7 @@
 
 import java.util.Collection;
 import java.util.Map;
+import java.util.function.Consumer;
 
 import org.hibernate.search.engine.backend.document.DocumentElement;
 import org.hibernate.search.engine.backend.document.model.dsl.IndexSchemaElement;
@@ -75,8 +76,16 @@ private String fieldPath(SimpleMappedIndex<IndexBinding> index, DataSet<?, V> da
 	public static final class IndexBinding {
 		private final SimpleFieldModelsByType field;
 
-		public IndexBinding(IndexSchemaElement root, Collection<
-				? extends FieldTypeDescriptor<?, ? extends SearchableProjectableIndexFieldTypeOptionsStep<?, ?>>> fieldTypes) {
+		public IndexBinding(IndexSchemaElement root,
+				Collection<? extends FieldTypeDescriptor<?,
+						? extends SearchableProjectableIndexFieldTypeOptionsStep<?, ?>>> fieldTypes,
+				Consumer<? super SearchableProjectableIndexFieldTypeOptionsStep<?, ?>> additionalConfiguration) {
+			field = SimpleFieldModelsByType.mapAll( fieldTypes, root, "field0_", additionalConfiguration );
+		}
+
+		public IndexBinding(IndexSchemaElement root,
+				Collection<? extends FieldTypeDescriptor<?,
+						? extends SearchableProjectableIndexFieldTypeOptionsStep<?, ?>>> fieldTypes) {
 			field = SimpleFieldModelsByType.mapAll( fieldTypes, root, "field0_" );
 		}
 	}

integrationtest/backend/tck/src/main/java/org/hibernate/search/integrationtest/backend/tck/search/predicate/KnnPredicateBaseIT.java

@@ -12,6 +12,8 @@
 import java.util.List;
 import java.util.Map;
 
+import org.hibernate.search.engine.backend.types.VectorSimilarity;
+import org.hibernate.search.engine.backend.types.dsl.VectorFieldTypeOptionsStep;
 import org.hibernate.search.engine.search.predicate.dsl.KnnPredicateOptionsStep;
 import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep;
 import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory;
@@ -80,7 +82,9 @@ class SingleFieldIT<F> extends SingleFieldConfigured<F> {
 
 	abstract static class SingleFieldConfigured<F> extends AbstractPredicateSingleFieldIT<KnnPredicateTestValues<F>> {
 		private static final SimpleMappedIndex<IndexBinding> index =
-				SimpleMappedIndex.of( root -> new IndexBinding( root, supportedFieldTypes ) )
+				SimpleMappedIndex.of(
+						root -> new IndexBinding( root, supportedFieldTypes,
+								c -> ( (VectorFieldTypeOptionsStep<?, ?>) c ).vectorSimilarity( VectorSimilarity.L2 ) ) )
 						.name( "singleField" );
 
 		private static final List<DataSet<?, ?>> dataSets = new ArrayList<>();