fix(token_filters): fix mandatory token filter test and refactor sche…

…ma to enforce original intent
pelias · Nov 7, 2019 · 83e0cf2 · 83e0cf2
1 parent dbba70f
commit 83e0cf2
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 14 deletions.
diff --git a/settings.js b/settings.js
@@ -45,6 +45,7 @@ function generate(){
             "trim",
             "word_delimiter",
             "custom_admin",
+            "unique_only_same_position",
             "notnull"
           ]
         },
@@ -144,7 +145,10 @@ function generate(){
           "char_filter" : ["alphanumeric"],
           "filter": [
             "lowercase",
-            "trim"
+            "icu_folding",
+            "trim",
+            "unique_only_same_position",
+            "notnull"
           ]
         },
         "peliasUnit": {
@@ -153,7 +157,10 @@ function generate(){
           "char_filter" : ["alphanumeric"],
           "filter": [
             "lowercase",
-            "trim"
+            "icu_folding",
+            "trim",
+            "unique_only_same_position",
+            "notnull"
           ]
         },
         "peliasHousenumber": {
@@ -174,7 +181,9 @@ function generate(){
             "directionals",
             "icu_folding",
             "remove_ordinals",
-            "trim"
+            "trim",
+            "unique_only_same_position",
+            "notnull"
           ]
         }
       },

diff --git a/test/fixtures/expected.json b/test/fixtures/expected.json
@@ -25,6 +25,7 @@
             "trim",
             "word_delimiter",
             "custom_admin",
+            "unique_only_same_position",
             "notnull"
           ]
         },
@@ -141,7 +142,10 @@
           ],
           "filter": [
             "lowercase",
-            "trim"
+            "icu_folding",
+            "trim",
+            "unique_only_same_position",
+            "notnull"
           ]
         },
         "peliasUnit": {
@@ -152,7 +156,10 @@
           ],
           "filter": [
             "lowercase",
-            "trim"
+            "icu_folding",
+            "trim",
+            "unique_only_same_position",
+            "notnull"
           ]
         },
         "peliasHousenumber": {
@@ -178,7 +185,9 @@
             "directionals",
             "icu_folding",
             "remove_ordinals",
-            "trim"
+            "trim",
+            "unique_only_same_position",
+            "notnull"
           ]
         }
       },

diff --git a/test/settings.js b/test/settings.js
@@ -227,7 +227,10 @@ module.exports.tests.peliasZipAnalyzer = function(test, common) {
     var analyzer = settings().analysis.analyzer.peliasZip;
     t.deepEqual( analyzer.filter, [
       "lowercase",
-      "trim"
+      "icu_folding",
+      "trim",
+      "unique_only_same_position",
+      "notnull"
     ]);
     t.end();
   });
@@ -248,7 +251,10 @@ module.exports.tests.peliasUnitAnalyzer = function(test, common) {
     var analyzer = settings().analysis.analyzer.peliasUnit;
     t.deepEqual( analyzer.filter, [
       "lowercase",
-      "trim"
+      "icu_folding",
+      "trim",
+      "unique_only_same_position",
+      "notnull"
     ]);
     t.end();
   });
@@ -289,13 +295,15 @@ module.exports.tests.peliasStreetAnalyzer = function(test, common) {
       "directionals",
       "icu_folding",
       "remove_ordinals",
-      "trim"
+      "trim",
+      "unique_only_same_position",
+      "notnull"
     ]);
     t.end();
   });
 };
 
-// cycle through all analyzers and ensure the corrsponding token filters are defined
+// cycle through all analyzers and ensure the corrsponding token filters are globally defined
 module.exports.tests.allTokenFiltersPresent = function(test, common) {
   var ES_INBUILT_FILTERS = [
     'lowercase', 'icu_folding', 'trim', 'word_delimiter', 'unique'
@@ -306,10 +314,10 @@ module.exports.tests.allTokenFiltersPresent = function(test, common) {
       var analyzer = s.analysis.analyzer[analyzerName];
       if( Array.isArray( analyzer.filter ) ){
         analyzer.filter.forEach( function( tokenFilterName ){
-          var filterExists = s.analysis.filter.hasOwnProperty( tokenFilterName );
-          if( !filterExists && -1 < ES_INBUILT_FILTERS.indexOf( tokenFilterName ) ){
-            filterExists = true;
-          }
+          var filterExists = (
+            s.analysis.filter.hasOwnProperty( tokenFilterName ) ||
+            ES_INBUILT_FILTERS.includes( tokenFilterName )
+          );
           t.true( filterExists, 'token filter exists: ' + tokenFilterName );
         });
       }
@@ -318,6 +326,28 @@ module.exports.tests.allTokenFiltersPresent = function(test, common) {
   });
 };
 
+// cycle through all analyzers and ensure the mandatory token filters are defined on each
+module.exports.tests.mandatoryTokenFiltersPresent = function (test, common) {
+  var MANDATORY_FILTERS = [
+    'lowercase', 'icu_folding', 'trim', 'unique_only_same_position', 'notnull'
+  ];
+  test('mandatory filters present', function (t) {
+    var s = settings();
+    for (var analyzerName in s.analysis.analyzer) {
+      var analyzer = s.analysis.analyzer[analyzerName];
+      if (Array.isArray(analyzer.filter)) {
+        MANDATORY_FILTERS.forEach(function (filterName) {
+          t.true(
+            analyzer.filter.includes(filterName),
+            `mandatory token filter ${filterName} not defined on ${analyzerName}`
+          );
+        });
+      }
+    }
+    t.end();
+  });
+};
+
 // cycle through all analyzers and ensure the corrsponding character filters are defined
 module.exports.tests.allCharacterFiltersPresent = function(test, common) {
   var ES_INBUILT_FILTERS = [];