Add ignore_keywords flag to word delimiter graph (#5121) (#5128)

This introduces a new property for the word delimiter graph token filter to configure ignoring of keywords. It relates to this change elastic/elasticsearch#59563 Co-authored-by: Steve Gordon <[email protected]>
elastic · Nov 30, 2020 · 8f2c52b · 8f2c52b
1 parent 49de084
commit 8f2c52b
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 1 deletion.
diff --git a/src/Nest/Analysis/TokenFilters/WordDelimiterGraph/WordDelimiterGraphTokenFilter.cs b/src/Nest/Analysis/TokenFilters/WordDelimiterGraph/WordDelimiterGraphTokenFilter.cs
@@ -56,6 +56,13 @@ public interface IWordDelimiterGraphTokenFilter : ITokenFilter
 		[JsonFormatter(typeof(NullableStringBooleanFormatter))]
 		bool? GenerateWordParts { get; set; }
 
+		/// <summary>
+		/// If true, the filter skips tokens with a keyword attribute of true. Defaults to false.
+		/// </summary>
+		[DataMember(Name = "ignore_keywords")]
+		[JsonFormatter(typeof(NullableStringBooleanFormatter))]
+		bool? IgnoreKeywords { get; set; }
+
 		/// <summary>
 		/// If true includes original words in subwords: "500-42" ⇒ "500-42" "500" "42". Defaults to false.
 		/// </summary>
@@ -133,6 +140,9 @@ public WordDelimiterGraphTokenFilter() : base("word_delimiter_graph") { }
 		/// <inheritdoc />
 		public bool? GenerateWordParts { get; set; }
 
+		/// <inheritdoc />
+		public bool? IgnoreKeywords { get; set; }
+
 		/// <inheritdoc />
 		public bool? PreserveOriginal { get; set; }
 
@@ -169,8 +179,8 @@ public class WordDelimiterGraphTokenFilterDescriptor
 		bool? IWordDelimiterGraphTokenFilter.CatenateWords { get; set; }
 		bool? IWordDelimiterGraphTokenFilter.GenerateNumberParts { get; set; }
 		bool? IWordDelimiterGraphTokenFilter.GenerateWordParts { get; set; }
+		bool? IWordDelimiterGraphTokenFilter.IgnoreKeywords { get; set; }
 		bool? IWordDelimiterGraphTokenFilter.PreserveOriginal { get; set; }
-
 		IEnumerable<string> IWordDelimiterGraphTokenFilter.ProtectedWords { get; set; }
 		string IWordDelimiterGraphTokenFilter.ProtectedWordsPath { get; set; }
 		bool? IWordDelimiterGraphTokenFilter.SplitOnCaseChange { get; set; }
@@ -187,6 +197,14 @@ public WordDelimiterGraphTokenFilterDescriptor GenerateWordParts(bool? generateW
 		public WordDelimiterGraphTokenFilterDescriptor GenerateNumberParts(bool? generateNumberParts = true) =>
 			Assign(generateNumberParts, (a, v) => a.GenerateNumberParts = v);
 
+		/// <summary>
+		/// <para>Configure whether the filter will skip tokens with a keyword attribute of true.</para>
+		/// <para>(Optional) When not configured, this defaults to false in Elasticsearch.</para>
+		/// </summary>
+		/// <param name="ignoreKeywords">If true, the filter skips tokens with a keyword attribute of true.</param>
+		public WordDelimiterGraphTokenFilterDescriptor IgnoreKeywords(bool? ignoreKeywords = true) =>
+			Assign(ignoreKeywords, (a, v) => a.IgnoreKeywords = v);
+
 		/// <inheritdoc />
 		public WordDelimiterGraphTokenFilterDescriptor CatenateWords(bool? catenateWords = true) => Assign(catenateWords, (a, v) => a.CatenateWords = v);
 

diff --git a/tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs b/tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs
@@ -923,6 +923,7 @@ public class WordDelimiterGraphTests : TokenFilterAssertionBase<WordDelimiterGra
 					.CatenateWords()
 					.GenerateNumberParts()
 					.GenerateWordParts()
+					.IgnoreKeywords()
 					.PreserveOriginal()
 					.ProtectedWords("x", "y", "z")
 					.SplitOnCaseChange()
@@ -939,6 +940,7 @@ public class WordDelimiterGraphTests : TokenFilterAssertionBase<WordDelimiterGra
 					CatenateWords = true,
 					GenerateNumberParts = true,
 					GenerateWordParts = true,
+					IgnoreKeywords = true,
 					PreserveOriginal = true,
 					ProtectedWords = new[] { "x", "y", "z" },
 					SplitOnCaseChange = true,
@@ -952,6 +954,7 @@ public class WordDelimiterGraphTests : TokenFilterAssertionBase<WordDelimiterGra
 				adjust_offsets = true,
 				generate_word_parts = true,
 				generate_number_parts = true,
+				ignore_keywords = true,
 				catenate_words = true,
 				catenate_numbers = true,
 				catenate_all = true,

diff --git a/tests/Tests/Analysis/TokenFilters/TokenFilterUsageTests.cs b/tests/Tests/Analysis/TokenFilters/TokenFilterUsageTests.cs
@@ -141,6 +141,7 @@ public class TokenFilterUsageTests : PromiseUsageTestBase<IIndexSettings, IndexS
 						.CatenateWords()
 						.GenerateNumberParts()
 						.GenerateWordParts()
+						.IgnoreKeywords()
 						.PreserveOriginal()
 						.ProtectedWords("x", "y", "z")
 						.SplitOnCaseChange()
@@ -301,6 +302,7 @@ public class TokenFilterUsageTests : PromiseUsageTestBase<IIndexSettings, IndexS
 								CatenateWords = true,
 								GenerateNumberParts = true,
 								GenerateWordParts = true,
+								IgnoreKeywords = true,
 								PreserveOriginal = true,
 								ProtectedWords = new[] { "x", "y", "z" },
 								SplitOnCaseChange = true,
@@ -624,6 +626,7 @@ public class TokenFilterUsageTests : PromiseUsageTestBase<IIndexSettings, IndexS
 						type = "word_delimiter_graph",
 						generate_word_parts = true,
 						generate_number_parts = true,
+						ignore_keywords = true,
 						catenate_words = true,
 						catenate_numbers = true,
 						catenate_all = true,