From 18bbdaf2554e5c9f925146ef5b666d09061a2a3d Mon Sep 17 00:00:00 2001 From: Adrian Serrano <adrisr83@gmail.com> Date: Thu, 16 Apr 2020 11:12:23 +0200 Subject: [PATCH 1/4] Fix setup.dashboards.index not working Due to type casting nightmare, the setting of `setup.dashboards.index` configuration option to replace the index name in use for dashboards and index pattern wasn't being honored. --- CHANGELOG.next.asciidoc | 1 + libbeat/dashboards/kibana_loader.go | 14 ++++++-- libbeat/dashboards/modify_json.go | 50 +++++++++++++++++++---------- 3 files changed, 46 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 71aae9155955..8076c533689e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -76,6 +76,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix building on FreeBSD by removing build flags from `add_cloudfoundry_metadata` processor. {pull}17486[17486] - Do not rotate log files on startup when interval is configured and rotateonstartup is disabled. {pull}17613[17613] - Fix goroutine leak and Elasticsearch output file descriptor leak when output reloading is in use. {issue}10491[10491] {pull}17381[17381] +- Fix `setup.dashboards.index` setting not working. {pull}17749[17749] *Auditbeat* diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index 93dd0e5dc0e3..1733f94750ce 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -25,6 +25,9 @@ import ( "net/url" "time" + "github.com/joeshaw/multierror" + "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/kibana" "github.com/elastic/beats/v7/libbeat/logp" @@ -101,11 +104,18 @@ func (loader KibanaLoader) ImportIndexFile(file string) error { // ImportIndex imports the passed index pattern to Kibana func (loader KibanaLoader) ImportIndex(pattern common.MapStr) error { + var errs multierror.Errors + params := url.Values{} params.Set("force", "true") //overwrite the existing dashboards - indexContent := ReplaceIndexInIndexPattern(loader.config.Index, pattern) - return loader.client.ImportJSON(importAPI, params, indexContent) + if err := ReplaceIndexInIndexPattern(loader.config.Index, pattern); err != nil { + errs = append(errs, errors.Wrapf(err, "error setting index '%s' in index pattern", loader.config.Index)) + } + if err := loader.client.ImportJSON(importAPI, params, pattern); err != nil { + errs = append(errs, errors.Wrap(err, "error loading index pattern")) + } + return errs.Err() } // ImportDashboard imports the dashboard file diff --git a/libbeat/dashboards/modify_json.go b/libbeat/dashboards/modify_json.go index c8b1c79da6b0..2e80a1626f4e 100644 --- a/libbeat/dashboards/modify_json.go +++ b/libbeat/dashboards/modify_json.go @@ -22,6 +22,8 @@ import ( "encoding/json" "fmt" + "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -41,32 +43,46 @@ type JSONFormat struct { Objects []JSONObject `json:"objects"` } -func ReplaceIndexInIndexPattern(index string, content common.MapStr) common.MapStr { +func ReplaceIndexInIndexPattern(index string, content common.MapStr) error { if index == "" { - return content + return nil } - objects, ok := content["objects"].([]interface{}) + list, ok := content["objects"] if !ok { - return content + return errors.New("empty index pattern") } - // change index pattern name - for i, object := range objects { - objectMap, ok := object.(map[string]interface{}) - if !ok { - continue + repl := common.MapStr{ + "id": index, + "attributes": common.MapStr{ + "title": index, + }, + } + switch v := list.(type) { + case []interface{}: + for _, objIf := range v { + switch obj := objIf.(type) { + case common.MapStr: + obj.DeepUpdate(repl) + case map[string]interface{}: + common.MapStr(obj).DeepUpdate(repl) + default: + return errors.Errorf("index pattern object has unexpected type %T", v) + } } - - objectMap["id"] = index - if attributes, ok := objectMap["attributes"].(map[string]interface{}); ok { - attributes["title"] = index + case []map[string]interface{}: + for _, obj := range v { + common.MapStr(obj).DeepUpdate(repl) } - objects[i] = objectMap + case []common.MapStr: + for _, obj := range v { + obj.DeepUpdate(repl) + } + default: + return errors.Errorf("index pattern objects have unexpected type %T", v) } - content["objects"] = objects - - return content + return nil } func replaceIndexInSearchObject(index string, savedObject string) (string, error) { From 6d0e71c484e520eea80d18bc375d762644c33a05 Mon Sep 17 00:00:00 2001 From: Adrian Serrano <adrisr83@gmail.com> Date: Thu, 16 Apr 2020 12:58:38 +0200 Subject: [PATCH 2/4] Type-safe update --- libbeat/dashboards/modify_json.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/libbeat/dashboards/modify_json.go b/libbeat/dashboards/modify_json.go index 2e80a1626f4e..a4a51027a74f 100644 --- a/libbeat/dashboards/modify_json.go +++ b/libbeat/dashboards/modify_json.go @@ -43,7 +43,7 @@ type JSONFormat struct { Objects []JSONObject `json:"objects"` } -func ReplaceIndexInIndexPattern(index string, content common.MapStr) error { +func ReplaceIndexInIndexPattern(index string, content common.MapStr) (err error) { if index == "" { return nil } @@ -53,31 +53,32 @@ func ReplaceIndexInIndexPattern(index string, content common.MapStr) error { return errors.New("empty index pattern") } - repl := common.MapStr{ - "id": index, - "attributes": common.MapStr{ - "title": index, - }, + updateObject := func(obj common.MapStr) { + // This uses Put instead of DeepUpdate to avoid modifying types for + // inner objects. (DeepUpdate will replace maps with MapStr). + obj.Put("id", index) + obj.Put("attributes.title", index) } + switch v := list.(type) { case []interface{}: for _, objIf := range v { switch obj := objIf.(type) { case common.MapStr: - obj.DeepUpdate(repl) + updateObject(obj) case map[string]interface{}: - common.MapStr(obj).DeepUpdate(repl) + updateObject(obj) default: return errors.Errorf("index pattern object has unexpected type %T", v) } } case []map[string]interface{}: for _, obj := range v { - common.MapStr(obj).DeepUpdate(repl) + updateObject(obj) } case []common.MapStr: for _, obj := range v { - obj.DeepUpdate(repl) + updateObject(obj) } default: return errors.Errorf("index pattern objects have unexpected type %T", v) From ea029b6cf107494140f9b0eda126857515d9cd4b Mon Sep 17 00:00:00 2001 From: Adrian Serrano <adrisr83@gmail.com> Date: Thu, 16 Apr 2020 12:59:00 +0200 Subject: [PATCH 3/4] Add test --- libbeat/dashboards/modify_json_test.go | 103 +++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/libbeat/dashboards/modify_json_test.go b/libbeat/dashboards/modify_json_test.go index 08fedac4df33..e562e560d06a 100644 --- a/libbeat/dashboards/modify_json_test.go +++ b/libbeat/dashboards/modify_json_test.go @@ -111,3 +111,106 @@ func TestReplaceIndexInDashboardObject(t *testing.T) { assert.Equal(t, test.expected, result) } } + +func TestReplaceIndexInIndexPattern(t *testing.T) { + // Test that replacing of index name in index pattern works no matter + // what the inner types are (MapStr, map[string]interface{} or interface{}). + // Also ensures that the inner types are not modified after replacement. + tests := []struct { + input common.MapStr + index string + expected common.MapStr + }{ + { + input: common.MapStr{"objects": []interface{}{map[string]interface{}{ + "id": "phonybeat-*", + "type": "index-pattern", + "attributes": map[string]interface{}{ + "title": "phonybeat-*", + "timeFieldName": "@timestamp", + }}}}, + index: "otherindex-*", + expected: common.MapStr{"objects": []interface{}{map[string]interface{}{ + "id": "otherindex-*", + "type": "index-pattern", + "attributes": map[string]interface{}{ + "title": "otherindex-*", + "timeFieldName": "@timestamp", + }}}}, + }, + { + input: common.MapStr{"objects": []interface{}{map[string]interface{}{ + "id": "phonybeat-*", + "type": "index-pattern", + "attributes": common.MapStr{ + "title": "phonybeat-*", + "timeFieldName": "@timestamp", + }}}}, + index: "otherindex-*", + expected: common.MapStr{"objects": []interface{}{map[string]interface{}{ + "id": "otherindex-*", + "type": "index-pattern", + "attributes": common.MapStr{ + "title": "otherindex-*", + "timeFieldName": "@timestamp", + }}}}, + }, + { + input: common.MapStr{"objects": []map[string]interface{}{{ + "id": "phonybeat-*", + "type": "index-pattern", + "attributes": common.MapStr{ + "title": "phonybeat-*", + "timeFieldName": "@timestamp", + }}}}, + index: "otherindex-*", + expected: common.MapStr{"objects": []map[string]interface{}{{ + "id": "otherindex-*", + "type": "index-pattern", + "attributes": common.MapStr{ + "title": "otherindex-*", + "timeFieldName": "@timestamp", + }}}}, + }, + { + input: common.MapStr{"objects": []common.MapStr{{ + "id": "phonybeat-*", + "type": "index-pattern", + "attributes": common.MapStr{ + "title": "phonybeat-*", + "timeFieldName": "@timestamp", + }}}}, + index: "otherindex-*", + expected: common.MapStr{"objects": []common.MapStr{{ + "id": "otherindex-*", + "type": "index-pattern", + "attributes": common.MapStr{ + "title": "otherindex-*", + "timeFieldName": "@timestamp", + }}}}, + }, + { + input: common.MapStr{"objects": []common.MapStr{{ + "id": "phonybeat-*", + "type": "index-pattern", + "attributes": interface{}(common.MapStr{ + "title": "phonybeat-*", + "timeFieldName": "@timestamp", + })}}}, + index: "otherindex-*", + expected: common.MapStr{"objects": []common.MapStr{{ + "id": "otherindex-*", + "type": "index-pattern", + "attributes": interface{}(common.MapStr{ + "title": "otherindex-*", + "timeFieldName": "@timestamp", + })}}}, + }, + } + + for _, test := range tests { + err := ReplaceIndexInIndexPattern(test.index, test.input) + assert.NoError(t, err) + assert.Equal(t, test.expected, test.input) + } +} From e3642de8275d52b1b84a91a1aef9f97bbf071f8e Mon Sep 17 00:00:00 2001 From: Adrian Serrano <adrisr83@gmail.com> Date: Thu, 16 Apr 2020 13:18:16 +0200 Subject: [PATCH 4/4] Do not create attributes.title if it didn't exist previously --- libbeat/dashboards/modify_json.go | 5 +++- libbeat/dashboards/modify_json_test.go | 37 +++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/libbeat/dashboards/modify_json.go b/libbeat/dashboards/modify_json.go index a4a51027a74f..2e0c48e38b05 100644 --- a/libbeat/dashboards/modify_json.go +++ b/libbeat/dashboards/modify_json.go @@ -57,7 +57,10 @@ func ReplaceIndexInIndexPattern(index string, content common.MapStr) (err error) // This uses Put instead of DeepUpdate to avoid modifying types for // inner objects. (DeepUpdate will replace maps with MapStr). obj.Put("id", index) - obj.Put("attributes.title", index) + // Only overwrite title if it exists. + if _, err := obj.GetValue("attributes.title"); err == nil { + obj.Put("attributes.title", index) + } } switch v := list.(type) { diff --git a/libbeat/dashboards/modify_json_test.go b/libbeat/dashboards/modify_json_test.go index e562e560d06a..a7424414b53a 100644 --- a/libbeat/dashboards/modify_json_test.go +++ b/libbeat/dashboards/modify_json_test.go @@ -117,11 +117,13 @@ func TestReplaceIndexInIndexPattern(t *testing.T) { // what the inner types are (MapStr, map[string]interface{} or interface{}). // Also ensures that the inner types are not modified after replacement. tests := []struct { + title string input common.MapStr index string expected common.MapStr }{ { + title: "Replace in []interface(map).map", input: common.MapStr{"objects": []interface{}{map[string]interface{}{ "id": "phonybeat-*", "type": "index-pattern", @@ -139,6 +141,7 @@ func TestReplaceIndexInIndexPattern(t *testing.T) { }}}}, }, { + title: "Replace in []interface(map).mapstr", input: common.MapStr{"objects": []interface{}{map[string]interface{}{ "id": "phonybeat-*", "type": "index-pattern", @@ -156,6 +159,7 @@ func TestReplaceIndexInIndexPattern(t *testing.T) { }}}}, }, { + title: "Replace in []map.mapstr", input: common.MapStr{"objects": []map[string]interface{}{{ "id": "phonybeat-*", "type": "index-pattern", @@ -173,6 +177,7 @@ func TestReplaceIndexInIndexPattern(t *testing.T) { }}}}, }, { + title: "Replace in []mapstr.mapstr", input: common.MapStr{"objects": []common.MapStr{{ "id": "phonybeat-*", "type": "index-pattern", @@ -190,6 +195,7 @@ func TestReplaceIndexInIndexPattern(t *testing.T) { }}}}, }, { + title: "Replace in []mapstr.interface(mapstr)", input: common.MapStr{"objects": []common.MapStr{{ "id": "phonybeat-*", "type": "index-pattern", @@ -206,11 +212,36 @@ func TestReplaceIndexInIndexPattern(t *testing.T) { "timeFieldName": "@timestamp", })}}}, }, + { + title: "Do not create missing attributes", + input: common.MapStr{"objects": []common.MapStr{{ + "id": "phonybeat-*", + "type": "index-pattern", + }}}, + index: "otherindex-*", + expected: common.MapStr{"objects": []common.MapStr{{ + "id": "otherindex-*", + "type": "index-pattern", + }}}, + }, + { + title: "Create missing id", + input: common.MapStr{"objects": []common.MapStr{{ + "type": "index-pattern", + }}}, + index: "otherindex-*", + expected: common.MapStr{"objects": []common.MapStr{{ + "id": "otherindex-*", + "type": "index-pattern", + }}}, + }, } for _, test := range tests { - err := ReplaceIndexInIndexPattern(test.index, test.input) - assert.NoError(t, err) - assert.Equal(t, test.expected, test.input) + t.Run(test.title, func(t *testing.T) { + err := ReplaceIndexInIndexPattern(test.index, test.input) + assert.NoError(t, err) + assert.Equal(t, test.expected, test.input) + }) } }