From 0784e14cfd81b2ece7c6824da444cc1e589922fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 19 Nov 2020 18:44:44 +0100 Subject: [PATCH 1/7] fix --- libbeat/template/template.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 0fbfa30ccab8..d1f64fa7b98a 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -304,8 +304,10 @@ func (t *Template) generateComponent(properties common.MapStr) common.MapStr { } func (t *Template) generateIndex(properties common.MapStr) common.MapStr { - tmpl := t.generateLegacy(properties) + tmpl := t.generateComponent(properties) tmpl["priority"] = t.priority + keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern()) + tmpl[keyPattern] = patterns delete(tmpl, "order") return tmpl } From ab470aa40d84b4627de786fcde550a312ce49b47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 20 Nov 2020 11:25:32 +0100 Subject: [PATCH 2/7] add e2e test --- libbeat/tests/system/test_template.py | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 67a344574843..cb4c9ffd8b84 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -322,6 +322,34 @@ def test_template_created_on_ilm_policy_created(self): index = resp[self.custom_alias]["settings"]["index"] assert index["number_of_shards"] == "2", index["number_of_shards"] + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @pytest.mark.tag('integration') + def test_setup_template_index(self): + """ + Test template setup with config options + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.setupCmd, + "-E", "setup.template.type=index"]) + + assert exit_code == 0 + self.idxmgmt.assert_index_template_loaded(self.index_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @pytest.mark.tag('integration') + def test_setup_template_component(self): + """ + Test template setup with config options + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.setupCmd, + "-E", "setup.template.type=component"]) + + assert exit_code == 0 + self.idxmgmt.assert_index_template_loaded(self.index_name) + class TestCommandExportTemplate(BaseTest): """ From 14cc16d0e0d788f5b2d3116753bb95958de0b4fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 20 Nov 2020 11:32:33 +0100 Subject: [PATCH 3/7] add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 129ab8bc45ba..0a6fcbd96d3d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -215,6 +215,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fixed documentation for commands in beats dev guide {pull}22194[22194] - Fix parsing of expired licences. {issue}21112[21112] {pull}22180[22180] - Fix duplicated pod events in kubernetes autodiscover for pods with init or ephemeral containers. {pull}22438[22438] +- Fix index template loading when the new index format is selected. {issue}22482[22482] {pull}22682[22682] *Auditbeat* From f2ceb0ac6e35db5e6025b2bb4081fa8f45371647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 20 Nov 2020 11:35:29 +0100 Subject: [PATCH 4/7] more fixes --- libbeat/template/template.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libbeat/template/template.go b/libbeat/template/template.go index d1f64fa7b98a..f6366ddc6761 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -235,9 +235,8 @@ func (t *Template) loadMinimalComponent() common.MapStr { } func (t *Template) loadMinimalIndex() common.MapStr { - m := t.loadMinimalLegacy() + m := t.loadMinimalComponent() m["priority"] = t.priority - delete(m, "order") return m } @@ -308,7 +307,6 @@ func (t *Template) generateIndex(properties common.MapStr) common.MapStr { tmpl["priority"] = t.priority keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern()) tmpl[keyPattern] = patterns - delete(tmpl, "order") return tmpl } From 8b24b6c04a8cf0eb21a06460e6a64203f09d84f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 20 Nov 2020 11:48:08 +0100 Subject: [PATCH 5/7] fix test comments --- libbeat/tests/system/test_template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index cb4c9ffd8b84..2e58c7753146 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -326,7 +326,7 @@ def test_template_created_on_ilm_policy_created(self): @pytest.mark.tag('integration') def test_setup_template_index(self): """ - Test template setup with config options + Test template setup of new index templates """ self.render_config() exit_code = self.run_beat(logging_args=["-v", "-d", "*"], @@ -340,7 +340,7 @@ def test_setup_template_index(self): @pytest.mark.tag('integration') def test_setup_template_component(self): """ - Test template setup with config options + Test template setup of component index templates """ self.render_config() exit_code = self.run_beat(logging_args=["-v", "-d", "*"], From 6ce09daa4d25036bfdd7cf21a8831eae8b5836a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 23 Nov 2020 16:08:54 +0100 Subject: [PATCH 6/7] adjust E2E tests --- libbeat/tests/system/idxmgmt.py | 10 +++++++++- .../tests/system/test_cmd_setup_index_management.py | 8 ++++---- libbeat/tests/system/test_ilm.py | 4 ++-- libbeat/tests/system/test_template.py | 4 ++-- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py index 864d134a9c98..4ec01a8cd5bf 100644 --- a/libbeat/tests/system/idxmgmt.py +++ b/libbeat/tests/system/idxmgmt.py @@ -56,11 +56,19 @@ def assert_index_template_not_loaded(self, template): with pytest.raises(NotFoundError): self._client.transport.perform_request('GET', '/_template/' + template) - def assert_index_template_loaded(self, template): + def assert_legacy_index_template_loaded(self, template): resp = self._client.transport.perform_request('GET', '/_template/' + template) assert template in resp assert "lifecycle" not in resp[template]["settings"]["index"] + def assert_index_template_loaded(self, template): + resp = self._client.transport.perform_request('GET', '/_index_template/' + template) + found = False + for index_template in resp['index_templates']: + if index_template['name'] == template: + found = True + assert found + def assert_ilm_template_loaded(self, template, policy, alias): resp = self._client.transport.perform_request('GET', '/_template/' + template) assert resp[template]["settings"]["index"]["lifecycle"]["name"] == policy diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index 234f4e76a0b8..cd6d19eefbe4 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -110,7 +110,7 @@ def test_setup_ilm_disabled(self): "-E", "setup.ilm.enabled=false"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) self.idxmgmt.assert_alias_not_created(self.alias_name) self.idxmgmt.assert_policy_not_created(self.policy_name) @@ -242,7 +242,7 @@ def test_setup_template_name_and_pattern_on_ilm_disabled(self): "-E", "setup.template.pattern=" + self.custom_template + "*"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.custom_template) + self.idxmgmt.assert_legacy_index_template_loaded(self.custom_template) self.idxmgmt.assert_index_template_index_pattern(self.custom_template, [self.custom_template + "*"]) self.idxmgmt.assert_alias_not_created(self.alias_name) self.idxmgmt.assert_policy_not_created(self.policy_name) @@ -261,7 +261,7 @@ def test_setup_template_with_opts(self): "-E", "setup.template.settings.index.number_of_shards=2"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) # check that settings are overwritten resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name) @@ -284,7 +284,7 @@ def test_setup_overwrite_template_on_ilm_policy_created(self): "-E", "setup.template.name=" + self.custom_alias, "-E", "setup.template.pattern=" + self.custom_alias + "*"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.custom_alias) + self.idxmgmt.assert_legacy_index_template_loaded(self.custom_alias) self.idxmgmt.assert_policy_not_created(self.policy_name) # ensure ilm policy is created, triggering overwriting existing template diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index 3d37125f6e4a..2913893e11c2 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -68,7 +68,7 @@ def test_ilm_disabled(self): self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) proc.check_kill_and_wait() - self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) self.idxmgmt.assert_alias_not_created(self.alias_name) self.idxmgmt.assert_policy_not_created(self.policy_name) @@ -234,7 +234,7 @@ def test_setup_ilm_disabled(self): "-E", "setup.ilm.enabled=false"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) self.idxmgmt.assert_alias_not_created(self.alias_name) self.idxmgmt.assert_policy_not_created(self.policy_name) diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 2e58c7753146..28f41688cbed 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -267,7 +267,7 @@ def test_setup_template_with_opts(self): "-E", "setup.template.settings.index.number_of_shards=2"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) # check that settings are overwritten resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name) @@ -305,7 +305,7 @@ def test_template_created_on_ilm_policy_created(self): "-E", "setup.template.name=" + self.custom_alias, "-E", "setup.template.pattern=" + self.custom_alias + "*"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.custom_alias) + self.idxmgmt.assert_legacy_index_template_loaded(self.custom_alias) self.idxmgmt.assert_policy_not_created(self.policy_name) # ensure ilm policy is created, triggering overwriting existing template From 9cda065ee85402ccfb6141008f3b5207f8e4e71d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 23 Nov 2020 20:22:14 +0100 Subject: [PATCH 7/7] add separate assertion for component templates --- libbeat/tests/system/idxmgmt.py | 9 +++++++++ libbeat/tests/system/test_template.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py index 4ec01a8cd5bf..f789b919bc26 100644 --- a/libbeat/tests/system/idxmgmt.py +++ b/libbeat/tests/system/idxmgmt.py @@ -69,6 +69,15 @@ def assert_index_template_loaded(self, template): found = True assert found + def assert_component_template_loaded(self, template): + resp = self._client.transport.perform_request('GET', '/_component_template/' + template) + found = False + print(resp) + for index_template in resp['component_templates']: + if index_template['name'] == template: + found = True + assert found + def assert_ilm_template_loaded(self, template, policy, alias): resp = self._client.transport.perform_request('GET', '/_template/' + template) assert resp[template]["settings"]["index"]["lifecycle"]["name"] == policy diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 28f41688cbed..35dc8915a082 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -348,7 +348,7 @@ def test_setup_template_component(self): "-E", "setup.template.type=component"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_component_template_loaded(self.index_name) class TestCommandExportTemplate(BaseTest):