feat(flatten_graph): add flatten_graph filter to ensure multi-word to…

…kens dont produce errors during indexing
pelias · Nov 15, 2019 · 39afc56 · 39afc56
1 parent 2afa1e3
commit 39afc56
Show file tree

Hide file tree

Showing 5 changed files with 74 additions and 13 deletions.
diff --git a/integration/multi_token_synonyms.js b/integration/multi_token_synonyms.js
@@ -0,0 +1,49 @@
+// validate analyzer is behaving as expected
+
+const elastictest = require('elastictest');
+const schema = require('../schema');
+
+module.exports.tests = {};
+
+// simple test to cover the issue noted in:
+// https://github.com/pelias/schema/issues/381#issuecomment-548305594
+module.exports.tests.functional = function (test, common) {
+  test('functional', function (t) {
+
+    var suite = new elastictest.Suite(common.clientOpts, { schema: schema });
+    suite.action(function (done) { setTimeout(done, 500); }); // wait for es to bring some shards up
+
+    // index a document with all admin values
+    // note: this will return an error if multi-token synonyms are
+    // not supported on ES6+
+    suite.action(function (done) {
+      suite.client.index({
+        index: suite.props.index, type: 'doc',
+        id: '1', body: {
+          name: { default: 'set' },
+          phrase: { default: 'set' },
+          address_parts: {
+            name: 'set',
+            street: 'set'
+          },
+          parent: {
+            country: 'set'
+          }
+        }
+      }, done);
+    });
+
+    suite.run(t.end);
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('multi token synonyms: ' + name, testFunction);
+  }
+
+  for (var testCase in module.exports.tests) {
+    module.exports.tests[testCase](test, common);
+  }
+};
diff --git a/integration/run.js b/integration/run.js
@@ -84,7 +84,8 @@ var tests = [
   require('./bounding_box.js'),
   require('./autocomplete_street_synonym_expansion.js'),
   require('./autocomplete_directional_synonym_expansion.js'),
-  require('./autocomplete_abbreviated_street_names.js')
+  require('./autocomplete_abbreviated_street_names.js'),
+  require('./multi_token_synonyms.js')
 ];
 
 tests.map(function(t) {

diff --git a/settings.js b/settings.js
@@ -46,7 +46,8 @@ function generate(){
             "word_delimiter",
             "custom_admin",
             "unique_only_same_position",
-            "notnull"
+            "notnull",
+            "flatten_graph"
           ]
         },
         "peliasIndexOneEdgeGram" : {
@@ -65,7 +66,8 @@ function generate(){
             "removeAllZeroNumericPrefix",
             "peliasOneEdgeGramFilter",
             "unique_only_same_position",
-            "notnull"
+            "notnull",
+            "flatten_graph"
           ]
         },
         "peliasQuery": {
@@ -130,7 +132,8 @@ function generate(){
             "icu_folding",
             "remove_ordinals",
             "unique_only_same_position",
-            "notnull"
+            "notnull",
+            "flatten_graph"
           ]
         },
         "peliasZip": {
@@ -177,7 +180,8 @@ function generate(){
             "remove_ordinals",
             "trim",
             "unique_only_same_position",
-            "notnull"
+            "notnull",
+            "flatten_graph"
           ]
         }
       },

diff --git a/test/fixtures/expected.json b/test/fixtures/expected.json
@@ -26,7 +26,8 @@
             "word_delimiter",
             "custom_admin",
             "unique_only_same_position",
-            "notnull"
+            "notnull",
+            "flatten_graph"
           ]
         },
         "peliasIndexOneEdgeGram": {
@@ -48,7 +49,8 @@
             "removeAllZeroNumericPrefix",
             "peliasOneEdgeGramFilter",
             "unique_only_same_position",
-            "notnull"
+            "notnull",
+            "flatten_graph"
           ]
         },
         "peliasQuery": {
@@ -125,7 +127,8 @@
             "icu_folding",
             "remove_ordinals",
             "unique_only_same_position",
-            "notnull"
+            "notnull",
+            "flatten_graph"
           ]
         },
         "peliasZip": {
@@ -181,7 +184,8 @@
             "remove_ordinals",
             "trim",
             "unique_only_same_position",
-            "notnull"
+            "notnull",
+            "flatten_graph"
           ]
         }
       },

diff --git a/test/settings.js b/test/settings.js
@@ -87,7 +87,8 @@ module.exports.tests.peliasIndexOneEdgeGramAnalyzer = function(test, common) {
       "removeAllZeroNumericPrefix",
       "peliasOneEdgeGramFilter",
       "unique_only_same_position",
-      "notnull"
+      "notnull",
+      "flatten_graph"
     ]);
     t.end();
   });
@@ -200,7 +201,8 @@ module.exports.tests.peliasPhraseAnalyzer = function(test, common) {
       "icu_folding",
       "remove_ordinals",
       "unique_only_same_position",
-      "notnull"
+      "notnull",
+      "flatten_graph"
     ]);
     t.end();
   });
@@ -291,7 +293,8 @@ module.exports.tests.peliasStreetAnalyzer = function(test, common) {
       "remove_ordinals",
       "trim",
       "unique_only_same_position",
-      "notnull"
+      "notnull",
+      "flatten_graph"
     ]);
     t.end();
   });
@@ -300,7 +303,7 @@ module.exports.tests.peliasStreetAnalyzer = function(test, common) {
 // cycle through all analyzers and ensure the corrsponding token filters are globally defined
 module.exports.tests.allTokenFiltersPresent = function(test, common) {
   var ES_INBUILT_FILTERS = [
-    'lowercase', 'icu_folding', 'trim', 'word_delimiter', 'unique'
+    'lowercase', 'icu_folding', 'trim', 'word_delimiter', 'unique', 'flatten_graph'
   ];
   test('all token filters present', function(t) {
     var s = settings();