From 0485704727f0d97899ff958a6f763edb33f0cc66 Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Tue, 16 May 2023 16:14:53 +0300 Subject: [PATCH 01/11] Add replicaset metaGenerator based on watcher Signed-off-by: ChrsMark --- kubernetes/metadata/metadata.go | 8 ++- kubernetes/metadata/pod.go | 23 +++++--- kubernetes/metadata/replicaset.go | 95 +++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 10 deletions(-) create mode 100644 kubernetes/metadata/replicaset.go diff --git a/kubernetes/metadata/metadata.go b/kubernetes/metadata/metadata.go index 64409d23bc..770cb132c4 100644 --- a/kubernetes/metadata/metadata.go +++ b/kubernetes/metadata/metadata.go @@ -93,16 +93,20 @@ func GetPodMetaGen( podWatcher kubernetes.Watcher, nodeWatcher kubernetes.Watcher, namespaceWatcher kubernetes.Watcher, + replicasetWatcher kubernetes.Watcher, metaConf *AddResourceMetadataConfig) MetaGen { - var nodeMetaGen, namespaceMetaGen MetaGen + var nodeMetaGen, namespaceMetaGen, rsMetaGen MetaGen if nodeWatcher != nil && metaConf.Node.Enabled() { nodeMetaGen = NewNodeMetadataGenerator(metaConf.Node, nodeWatcher.Store(), nodeWatcher.Client()) } if namespaceWatcher != nil && metaConf.Namespace.Enabled() { namespaceMetaGen = NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store(), namespaceWatcher.Client()) } - metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), podWatcher.Client(), nodeMetaGen, namespaceMetaGen, metaConf) + if replicasetWatcher != nil && metaConf.Deployment { + rsMetaGen = NewReplicasetMetadataGenerator(cfg, replicasetWatcher.Store(), replicasetWatcher.Client()) + } + metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), podWatcher.Client(), nodeMetaGen, namespaceMetaGen, rsMetaGen, metaConf) return metaGen } diff --git a/kubernetes/metadata/pod.go b/kubernetes/metadata/pod.go index 02009b2943..7a80aa35d8 100644 --- a/kubernetes/metadata/pod.go +++ b/kubernetes/metadata/pod.go @@ -33,6 +33,7 @@ type pod struct { store cache.Store client k8s.Interface node MetaGen + replicaset MetaGen resource *Resource addResourceMetadata *AddResourceMetadataConfig } @@ -44,12 +45,14 @@ func NewPodMetadataGenerator( client k8s.Interface, node MetaGen, namespace MetaGen, + replicaset MetaGen, addResourceMetadata *AddResourceMetadataConfig) MetaGen { return &pod{ resource: NewNamespaceAwareResourceMetadataGenerator(cfg, client, namespace), store: pods, node: node, + replicaset: replicaset, client: client, addResourceMetadata: addResourceMetadata, } @@ -88,19 +91,23 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr. out := p.resource.GenerateK8s("pod", obj, opts...) - // check if Pod is handled by a ReplicaSet which is controlled by a Deployment - // same happens with CronJob vs Job. The hierarchy there is CronJob->Job->Pod + // check if Pod is handled by a ReplicaSet which is controlled by a Deployment. + // The hierarchy there is Deployment->ReplicaSet->Pod. if p.addResourceMetadata.Deployment { - rsName, _ := out.GetValue("replicaset.name") - if rsName, ok := rsName.(string); ok { - dep := p.getRSDeployment(rsName, po.GetNamespace()) - if dep != "" { - _, _ = out.Put("deployment.name", dep) + if p.replicaset != nil { + rsName, _ := out.GetValue("replicaset.name") + if rsName, ok := rsName.(string); ok { + meta := p.replicaset.GenerateFromName(rsName) + deploymentName, _ := meta.GetValue("deployment.name") + if deploymentName != "" { + _, _ = out.Put("deployment.name", deploymentName) + } } } } - // check if Pod is handled by a Job which is controlled by a CronJob + // check if Pod is handled by a Job which is controlled by a CronJob. + // The hierarchy there is CronJob->Job->Pod if p.addResourceMetadata.CronJob { jobName, _ := out.GetValue("job.name") if jobName, ok := jobName.(string); ok { diff --git a/kubernetes/metadata/replicaset.go b/kubernetes/metadata/replicaset.go new file mode 100644 index 0000000000..f79c2657c1 --- /dev/null +++ b/kubernetes/metadata/replicaset.go @@ -0,0 +1,95 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +const resourceType = "replicaset" + +type replicaset struct { + store cache.Store + resource *Resource +} + +// NewReplicasetMetadataGenerator creates a metagen for namespace resources +func NewReplicasetMetadataGenerator(cfg *config.C, replicasets cache.Store, client k8s.Interface) MetaGen { + return &replicaset{ + resource: NewResourceMetadataGenerator(cfg, client), + store: replicasets, + } +} + +// Generate generates pod metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (rs *replicaset) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := rs.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": rs.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates namespace ECS metadata from a resource object +func (rs *replicaset) GenerateECS(obj kubernetes.Resource) mapstr.M { + return rs.resource.GenerateECS(obj) +} + +// GenerateK8s generates namespace metadata from a resource object +func (rs *replicaset) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + _, ok := obj.(*kubernetes.ReplicaSet) + if !ok { + return nil + } + + meta := rs.resource.GenerateK8s(resourceType, obj, opts...) + return meta +} + +// GenerateFromName generates pod metadata from a namespace name +func (rs *replicaset) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if rs.store == nil { + return nil + } + + if obj, ok, _ := rs.store.GetByKey(name); ok { + replicaSet, ok := obj.(*kubernetes.ReplicaSet) + if !ok { + return nil + } + + return rs.GenerateK8s(replicaSet, opts...) + } + + return nil +} From b07be241595d941d483b9214905f44137a959fa4 Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Tue, 16 May 2023 16:56:45 +0300 Subject: [PATCH 02/11] Add job metaGenerator based on watcher Signed-off-by: ChrsMark --- kubernetes/metadata/job.go | 93 +++++++++++++++++++++++++++++++++ kubernetes/metadata/metadata.go | 14 ++++- kubernetes/metadata/pod.go | 61 ++++----------------- 3 files changed, 117 insertions(+), 51 deletions(-) create mode 100644 kubernetes/metadata/job.go diff --git a/kubernetes/metadata/job.go b/kubernetes/metadata/job.go new file mode 100644 index 0000000000..ee9dc8df48 --- /dev/null +++ b/kubernetes/metadata/job.go @@ -0,0 +1,93 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type job struct { + store cache.Store + resource *Resource +} + +// NewJobMetadataGenerator creates a metagen for namespace resources +func NewJobMetadataGenerator(cfg *config.C, jobs cache.Store, client k8s.Interface) MetaGen { + return &replicaset{ + resource: NewResourceMetadataGenerator(cfg, client), + store: jobs, + } +} + +// Generate generates pod metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (rs *job) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := rs.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": rs.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates namespace ECS metadata from a resource object +func (rs *job) GenerateECS(obj kubernetes.Resource) mapstr.M { + return rs.resource.GenerateECS(obj) +} + +// GenerateK8s generates namespace metadata from a resource object +func (rs *job) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + _, ok := obj.(*kubernetes.Job) + if !ok { + return nil + } + + meta := rs.resource.GenerateK8s("job", obj, opts...) + return meta +} + +// GenerateFromName generates pod metadata from a namespace name +func (rs *job) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if rs.store == nil { + return nil + } + + if obj, ok, _ := rs.store.GetByKey(name); ok { + job, ok := obj.(*kubernetes.Job) + if !ok { + return nil + } + + return rs.GenerateK8s(job, opts...) + } + + return nil +} diff --git a/kubernetes/metadata/metadata.go b/kubernetes/metadata/metadata.go index 770cb132c4..6a5083b4c0 100644 --- a/kubernetes/metadata/metadata.go +++ b/kubernetes/metadata/metadata.go @@ -94,6 +94,7 @@ func GetPodMetaGen( nodeWatcher kubernetes.Watcher, namespaceWatcher kubernetes.Watcher, replicasetWatcher kubernetes.Watcher, + jobWatcher kubernetes.Watcher, metaConf *AddResourceMetadataConfig) MetaGen { var nodeMetaGen, namespaceMetaGen, rsMetaGen MetaGen @@ -106,7 +107,18 @@ func GetPodMetaGen( if replicasetWatcher != nil && metaConf.Deployment { rsMetaGen = NewReplicasetMetadataGenerator(cfg, replicasetWatcher.Store(), replicasetWatcher.Client()) } - metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), podWatcher.Client(), nodeMetaGen, namespaceMetaGen, rsMetaGen, metaConf) + if jobWatcher != nil && metaConf.CronJob { + rsMetaGen = NewReplicasetMetadataGenerator(cfg, replicasetWatcher.Store(), replicasetWatcher.Client()) + } + metaGen := NewPodMetadataGenerator( + cfg, + podWatcher.Store(), + podWatcher.Client(), + nodeMetaGen, + namespaceMetaGen, + rsMetaGen, + jobWatcher, + metaConf) return metaGen } diff --git a/kubernetes/metadata/pod.go b/kubernetes/metadata/pod.go index 7a80aa35d8..36525940c4 100644 --- a/kubernetes/metadata/pod.go +++ b/kubernetes/metadata/pod.go @@ -18,9 +18,6 @@ package metadata import ( - "context" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8s "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" @@ -34,6 +31,7 @@ type pod struct { client k8s.Interface node MetaGen replicaset MetaGen + job MetaGen resource *Resource addResourceMetadata *AddResourceMetadataConfig } @@ -46,6 +44,7 @@ func NewPodMetadataGenerator( node MetaGen, namespace MetaGen, replicaset MetaGen, + job MetaGen, addResourceMetadata *AddResourceMetadataConfig) MetaGen { return &pod{ @@ -53,6 +52,7 @@ func NewPodMetadataGenerator( store: pods, node: node, replicaset: replicaset, + job: job, client: client, addResourceMetadata: addResourceMetadata, } @@ -109,11 +109,14 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr. // check if Pod is handled by a Job which is controlled by a CronJob. // The hierarchy there is CronJob->Job->Pod if p.addResourceMetadata.CronJob { - jobName, _ := out.GetValue("job.name") - if jobName, ok := jobName.(string); ok { - dep := p.getCronjobOfJob(jobName, po.GetNamespace()) - if dep != "" { - _, _ = out.Put("cronjob.name", dep) + if p.job != nil { + jobName, _ := out.GetValue("job.name") + if jobName, ok := jobName.(string); ok { + meta := p.replicaset.GenerateFromName(jobName) + cronjobName, _ := meta.GetValue("cronjob.name") + if cronjobName != "" { + _, _ = out.Put("cronjob.name", cronjobName) + } } } } @@ -153,45 +156,3 @@ func (p *pod) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { return nil } - -// getRSDeployment return the name of the Deployment object that -// owns the ReplicaSet with the given name under the given Namespace -func (p *pod) getRSDeployment(rsName string, ns string) string { - if p.client == nil { - return "" - } - rs, err := p.client.AppsV1().ReplicaSets(ns).Get(context.TODO(), rsName, metav1.GetOptions{}) - if err != nil { - return "" - } - for _, ref := range rs.GetOwnerReferences() { - if ref.Controller != nil && *ref.Controller { - switch ref.Kind { - case "Deployment": - return ref.Name - } - } - } - return "" -} - -// getCronjobOfJob return the name of the Cronjob object that -// owns the Job with the given name under the given Namespace -func (p *pod) getCronjobOfJob(jobName string, ns string) string { - if p.client == nil { - return "" - } - cronjob, err := p.client.BatchV1().Jobs(ns).Get(context.TODO(), jobName, metav1.GetOptions{}) - if err != nil { - return "" - } - for _, ref := range cronjob.GetOwnerReferences() { - if ref.Controller != nil && *ref.Controller { - switch ref.Kind { - case "CronJob": - return ref.Name - } - } - } - return "" -} From cb6bfeaf20ef8b5df9a02b39873bb9f30acc5f3e Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Tue, 16 May 2023 17:08:52 +0300 Subject: [PATCH 03/11] fix types Signed-off-by: ChrsMark --- kubernetes/metadata/metadata.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/metadata/metadata.go b/kubernetes/metadata/metadata.go index 6a5083b4c0..7a445c3b76 100644 --- a/kubernetes/metadata/metadata.go +++ b/kubernetes/metadata/metadata.go @@ -97,7 +97,7 @@ func GetPodMetaGen( jobWatcher kubernetes.Watcher, metaConf *AddResourceMetadataConfig) MetaGen { - var nodeMetaGen, namespaceMetaGen, rsMetaGen MetaGen + var nodeMetaGen, namespaceMetaGen, rsMetaGen, jobMetaGen MetaGen if nodeWatcher != nil && metaConf.Node.Enabled() { nodeMetaGen = NewNodeMetadataGenerator(metaConf.Node, nodeWatcher.Store(), nodeWatcher.Client()) } @@ -108,7 +108,7 @@ func GetPodMetaGen( rsMetaGen = NewReplicasetMetadataGenerator(cfg, replicasetWatcher.Store(), replicasetWatcher.Client()) } if jobWatcher != nil && metaConf.CronJob { - rsMetaGen = NewReplicasetMetadataGenerator(cfg, replicasetWatcher.Store(), replicasetWatcher.Client()) + jobMetaGen = NewJobMetadataGenerator(cfg, replicasetWatcher.Store(), replicasetWatcher.Client()) } metaGen := NewPodMetadataGenerator( cfg, @@ -117,7 +117,7 @@ func GetPodMetaGen( nodeMetaGen, namespaceMetaGen, rsMetaGen, - jobWatcher, + jobMetaGen, metaConf) return metaGen } From 891423bfff4083376797d8935295e2bb89bc5c29 Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Wed, 17 May 2023 13:13:28 +0300 Subject: [PATCH 04/11] Fix references of objects Signed-off-by: ChrsMark --- kubernetes/metadata/job.go | 27 +++++++++++++-------------- kubernetes/metadata/metadata.go | 2 +- kubernetes/metadata/pod.go | 1 + kubernetes/metadata/replicaset.go | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/kubernetes/metadata/job.go b/kubernetes/metadata/job.go index ee9dc8df48..f4619542c4 100644 --- a/kubernetes/metadata/job.go +++ b/kubernetes/metadata/job.go @@ -33,7 +33,7 @@ type job struct { // NewJobMetadataGenerator creates a metagen for namespace resources func NewJobMetadataGenerator(cfg *config.C, jobs cache.Store, client k8s.Interface) MetaGen { - return &replicaset{ + return &job{ resource: NewResourceMetadataGenerator(cfg, client), store: jobs, } @@ -49,44 +49,43 @@ func NewJobMetadataGenerator(cfg *config.C, jobs cache.Store, client k8s.Interfa // // All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by // GenerateK8s method while fields that are part of ECS are generated by GenerateECS method -func (rs *job) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { - ecsFields := rs.GenerateECS(obj) +func (jb *job) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := jb.GenerateECS(obj) meta := mapstr.M{ - "kubernetes": rs.GenerateK8s(obj, opts...), + "kubernetes": jb.GenerateK8s(obj, opts...), } meta.DeepUpdate(ecsFields) return meta } // GenerateECS generates namespace ECS metadata from a resource object -func (rs *job) GenerateECS(obj kubernetes.Resource) mapstr.M { - return rs.resource.GenerateECS(obj) +func (jb *job) GenerateECS(obj kubernetes.Resource) mapstr.M { + return jb.resource.GenerateECS(obj) } // GenerateK8s generates namespace metadata from a resource object -func (rs *job) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { +func (jb *job) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { _, ok := obj.(*kubernetes.Job) if !ok { return nil } - meta := rs.resource.GenerateK8s("job", obj, opts...) + meta := jb.resource.GenerateK8s("job", obj, opts...) return meta } // GenerateFromName generates pod metadata from a namespace name -func (rs *job) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { - if rs.store == nil { +func (jb *job) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if jb.store == nil { return nil } - - if obj, ok, _ := rs.store.GetByKey(name); ok { - job, ok := obj.(*kubernetes.Job) + if obj, ok, _ := jb.store.GetByKey(name); ok { + jobObj, ok := obj.(*kubernetes.Job) if !ok { return nil } - return rs.GenerateK8s(job, opts...) + return jb.GenerateK8s(jobObj, opts...) } return nil diff --git a/kubernetes/metadata/metadata.go b/kubernetes/metadata/metadata.go index 7a445c3b76..dc5406790a 100644 --- a/kubernetes/metadata/metadata.go +++ b/kubernetes/metadata/metadata.go @@ -108,7 +108,7 @@ func GetPodMetaGen( rsMetaGen = NewReplicasetMetadataGenerator(cfg, replicasetWatcher.Store(), replicasetWatcher.Client()) } if jobWatcher != nil && metaConf.CronJob { - jobMetaGen = NewJobMetadataGenerator(cfg, replicasetWatcher.Store(), replicasetWatcher.Client()) + jobMetaGen = NewJobMetadataGenerator(cfg, jobWatcher.Store(), jobWatcher.Client()) } metaGen := NewPodMetadataGenerator( cfg, diff --git a/kubernetes/metadata/pod.go b/kubernetes/metadata/pod.go index 36525940c4..605e611efe 100644 --- a/kubernetes/metadata/pod.go +++ b/kubernetes/metadata/pod.go @@ -113,6 +113,7 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr. jobName, _ := out.GetValue("job.name") if jobName, ok := jobName.(string); ok { meta := p.replicaset.GenerateFromName(jobName) + cronjobName, _ := meta.GetValue("cronjob.name") if cronjobName != "" { _, _ = out.Put("cronjob.name", cronjobName) diff --git a/kubernetes/metadata/replicaset.go b/kubernetes/metadata/replicaset.go index f79c2657c1..891de680c7 100644 --- a/kubernetes/metadata/replicaset.go +++ b/kubernetes/metadata/replicaset.go @@ -81,7 +81,7 @@ func (rs *replicaset) GenerateFromName(name string, opts ...FieldOptions) mapstr if rs.store == nil { return nil } - + if obj, ok, _ := rs.store.GetByKey(name); ok { replicaSet, ok := obj.(*kubernetes.ReplicaSet) if !ok { From 9e8aaeec3839c3f4355cfd0f2b36635643092c98 Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Wed, 17 May 2023 15:03:30 +0300 Subject: [PATCH 05/11] Fix resource name format Signed-off-by: ChrsMark --- kubernetes/metadata/job.go | 1 + kubernetes/metadata/pod.go | 5 ++--- kubernetes/metadata/replicaset.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes/metadata/job.go b/kubernetes/metadata/job.go index f4619542c4..3015b8c471 100644 --- a/kubernetes/metadata/job.go +++ b/kubernetes/metadata/job.go @@ -79,6 +79,7 @@ func (jb *job) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { if jb.store == nil { return nil } + if obj, ok, _ := jb.store.GetByKey(name); ok { jobObj, ok := obj.(*kubernetes.Job) if !ok { diff --git a/kubernetes/metadata/pod.go b/kubernetes/metadata/pod.go index 605e611efe..3a5f3d539e 100644 --- a/kubernetes/metadata/pod.go +++ b/kubernetes/metadata/pod.go @@ -97,7 +97,7 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr. if p.replicaset != nil { rsName, _ := out.GetValue("replicaset.name") if rsName, ok := rsName.(string); ok { - meta := p.replicaset.GenerateFromName(rsName) + meta := p.replicaset.GenerateFromName(po.Namespace + "/" + rsName) deploymentName, _ := meta.GetValue("deployment.name") if deploymentName != "" { _, _ = out.Put("deployment.name", deploymentName) @@ -112,8 +112,7 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr. if p.job != nil { jobName, _ := out.GetValue("job.name") if jobName, ok := jobName.(string); ok { - meta := p.replicaset.GenerateFromName(jobName) - + meta := p.job.GenerateFromName(po.Namespace + "/" + jobName) cronjobName, _ := meta.GetValue("cronjob.name") if cronjobName != "" { _, _ = out.Put("cronjob.name", cronjobName) diff --git a/kubernetes/metadata/replicaset.go b/kubernetes/metadata/replicaset.go index 891de680c7..f79c2657c1 100644 --- a/kubernetes/metadata/replicaset.go +++ b/kubernetes/metadata/replicaset.go @@ -81,7 +81,7 @@ func (rs *replicaset) GenerateFromName(name string, opts ...FieldOptions) mapstr if rs.store == nil { return nil } - + if obj, ok, _ := rs.store.GetByKey(name); ok { replicaSet, ok := obj.(*kubernetes.ReplicaSet) if !ok { From bbc944a200774ac7dd5782f7215e0a098c3d05a9 Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Wed, 17 May 2023 18:11:23 +0300 Subject: [PATCH 06/11] Fix tests Signed-off-by: ChrsMark --- kubernetes/metadata/pod_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes/metadata/pod_test.go b/kubernetes/metadata/pod_test.go index 79920ce853..4ed3dd3171 100644 --- a/kubernetes/metadata/pod_test.go +++ b/kubernetes/metadata/pod_test.go @@ -429,7 +429,7 @@ func TestPod_Generate(t *testing.T) { }) assert.NoError(t, err) - metagen := NewPodMetadataGenerator(config, nil, client, nil, nil, addResourceMetadata) + metagen := NewPodMetadataGenerator(config, nil, client, nil, nil, nil, nil, addResourceMetadata) for _, test := range tests { t.Run(test.name, func(t *testing.T) { assert.Equal(t, test.output, metagen.Generate(test.input)) @@ -549,7 +549,7 @@ func TestPod_GenerateFromName(t *testing.T) { pods := cache.NewStore(cache.MetaNamespaceKeyFunc) err = pods.Add(test.input) require.NoError(t, err) - metagen := NewPodMetadataGenerator(config, pods, client, nil, nil, addResourceMetadata) + metagen := NewPodMetadataGenerator(config, pods, client, nil, nil, nil, nil, addResourceMetadata) accessor, err := meta.Accessor(test.input) require.NoError(t, err) @@ -671,7 +671,7 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) { require.NoError(t, err) nsMeta := NewNamespaceMetadataGenerator(config, namespaces, client) - metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, addResourceMetadata) + metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, nil, nil, addResourceMetadata) t.Run(test.name, func(t *testing.T) { assert.Equal(t, test.output, metagen.Generate(test.input)) }) @@ -832,7 +832,7 @@ func TestPod_GenerateWithNodeNamespaceWithAddResourceConfig(t *testing.T) { require.NoError(t, err) nsMeta := NewNamespaceMetadataGenerator(namespaceConfig, namespaces, client) - metagen := NewPodMetadataGenerator(c, pods, client, nodeMeta, nsMeta, &metaConfig) + metagen := NewPodMetadataGenerator(c, pods, client, nodeMeta, nsMeta, nil, nil, &metaConfig) t.Run(test.name, func(t *testing.T) { assert.Equal(t, test.output, metagen.Generate(test.input)) }) From 004a321b1609c3c715fa5e26b280e3be9791cd9b Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Thu, 18 May 2023 10:34:24 +0300 Subject: [PATCH 07/11] Adjust tests Signed-off-by: ChrsMark --- kubernetes/metadata/pod_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kubernetes/metadata/pod_test.go b/kubernetes/metadata/pod_test.go index 4ed3dd3171..f5ac59b199 100644 --- a/kubernetes/metadata/pod_test.go +++ b/kubernetes/metadata/pod_test.go @@ -429,7 +429,11 @@ func TestPod_Generate(t *testing.T) { }) assert.NoError(t, err) - metagen := NewPodMetadataGenerator(config, nil, client, nil, nil, nil, nil, addResourceMetadata) + replicaSets := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = replicaSets.Add(rs) + require.NoError(t, err) + rsMeta := NewReplicasetMetadataGenerator(config, replicaSets, client) + metagen := NewPodMetadataGenerator(config, nil, client, nil, nil, rsMeta, nil, addResourceMetadata) for _, test := range tests { t.Run(test.name, func(t *testing.T) { assert.Equal(t, test.output, metagen.Generate(test.input)) From e39accd8ea666fe282761d6588633f8d095b90b6 Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Thu, 18 May 2023 12:20:06 +0300 Subject: [PATCH 08/11] fix docstrings Signed-off-by: ChrsMark --- kubernetes/metadata/job.go | 10 +++++----- kubernetes/metadata/replicaset.go | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/kubernetes/metadata/job.go b/kubernetes/metadata/job.go index 3015b8c471..ba13e826b2 100644 --- a/kubernetes/metadata/job.go +++ b/kubernetes/metadata/job.go @@ -31,7 +31,7 @@ type job struct { resource *Resource } -// NewJobMetadataGenerator creates a metagen for namespace resources +// NewJobMetadataGenerator creates a metagen for job resources func NewJobMetadataGenerator(cfg *config.C, jobs cache.Store, client k8s.Interface) MetaGen { return &job{ resource: NewResourceMetadataGenerator(cfg, client), @@ -39,7 +39,7 @@ func NewJobMetadataGenerator(cfg *config.C, jobs cache.Store, client k8s.Interfa } } -// Generate generates pod metadata from a resource object +// Generate generates job metadata from a resource object // Metadata map is in the following form: // // { @@ -58,12 +58,12 @@ func (jb *job) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M return meta } -// GenerateECS generates namespace ECS metadata from a resource object +// GenerateECS generates job ECS metadata from a resource object func (jb *job) GenerateECS(obj kubernetes.Resource) mapstr.M { return jb.resource.GenerateECS(obj) } -// GenerateK8s generates namespace metadata from a resource object +// GenerateK8s generates job metadata from a resource object func (jb *job) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { _, ok := obj.(*kubernetes.Job) if !ok { @@ -74,7 +74,7 @@ func (jb *job) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr return meta } -// GenerateFromName generates pod metadata from a namespace name +// GenerateFromName generates job metadata from a namespace name func (jb *job) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { if jb.store == nil { return nil diff --git a/kubernetes/metadata/replicaset.go b/kubernetes/metadata/replicaset.go index f79c2657c1..3fccc802df 100644 --- a/kubernetes/metadata/replicaset.go +++ b/kubernetes/metadata/replicaset.go @@ -33,7 +33,7 @@ type replicaset struct { resource *Resource } -// NewReplicasetMetadataGenerator creates a metagen for namespace resources +// NewReplicasetMetadataGenerator creates a metagen for replicaset resources func NewReplicasetMetadataGenerator(cfg *config.C, replicasets cache.Store, client k8s.Interface) MetaGen { return &replicaset{ resource: NewResourceMetadataGenerator(cfg, client), @@ -41,7 +41,7 @@ func NewReplicasetMetadataGenerator(cfg *config.C, replicasets cache.Store, clie } } -// Generate generates pod metadata from a resource object +// Generate generates replicaset metadata from a resource object // Metadata map is in the following form: // // { @@ -60,12 +60,12 @@ func (rs *replicaset) Generate(obj kubernetes.Resource, opts ...FieldOptions) ma return meta } -// GenerateECS generates namespace ECS metadata from a resource object +// GenerateECS generates replicaset ECS metadata from a resource object func (rs *replicaset) GenerateECS(obj kubernetes.Resource) mapstr.M { return rs.resource.GenerateECS(obj) } -// GenerateK8s generates namespace metadata from a resource object +// GenerateK8s generates replicaset metadata from a resource object func (rs *replicaset) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { _, ok := obj.(*kubernetes.ReplicaSet) if !ok { @@ -76,7 +76,7 @@ func (rs *replicaset) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) return meta } -// GenerateFromName generates pod metadata from a namespace name +// GenerateFromName generates replicaset metadata from a replicaset name func (rs *replicaset) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { if rs.store == nil { return nil From bc27148114802fe3f7ee6b4bb7e38ca88c258da7 Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Thu, 18 May 2023 13:40:15 +0300 Subject: [PATCH 09/11] Add Jobs tests Signed-off-by: ChrsMark --- kubernetes/metadata/job_test.go | 163 ++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 kubernetes/metadata/job_test.go diff --git a/kubernetes/metadata/job_test.go b/kubernetes/metadata/job_test.go new file mode 100644 index 0000000000..1f2e54bde6 --- /dev/null +++ b/kubernetes/metadata/job_test.go @@ -0,0 +1,163 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestJob_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object with owner", + input: &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "CronJob", + Name: "nginx-job", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "job": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "cronjob": mapstr.M{ + "name": "nginx-job", + }, + "namespace": defaultNs, + }, + }, + }, + } + + cfg := config.NewConfig() + metagen := NewJobMetadataGenerator(cfg, nil, client) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} + +func TestJob_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object with owner", + input: &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "CronJob", + Name: "nginx-job", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "job": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "cronjob": mapstr.M{ + "name": "nginx-job", + }, + "namespace": defaultNs, + }, + }, + } + + for _, test := range tests { + cfg := config.NewConfig() + jobs := cache.NewStore(cache.MetaNamespaceKeyFunc) + err := jobs.Add(test.input) + require.NoError(t, err) + metagen := NewJobMetadataGenerator(cfg, jobs, client) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetNamespace(), "/", accessor.GetName()))) + }) + } +} From 79acb607a721ffbc5549dade8a74d51851dfeeab Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Thu, 18 May 2023 13:50:16 +0300 Subject: [PATCH 10/11] Add rs tests Signed-off-by: ChrsMark --- kubernetes/metadata/replicaset_test.go | 202 +++++++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 kubernetes/metadata/replicaset_test.go diff --git a/kubernetes/metadata/replicaset_test.go b/kubernetes/metadata/replicaset_test.go new file mode 100644 index 0000000000..a42bfb6a47 --- /dev/null +++ b/kubernetes/metadata/replicaset_test.go @@ -0,0 +1,202 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestReplicaset_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object with owner", + input: &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + UID: uid, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "demo", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "demo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx:1.12", + Ports: []v1.ContainerPort{ + { + Name: "http", + Protocol: v1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "replicaset": mapstr.M{ + "name": "nginx-rs", + "uid": uid, + }, + "deployment": mapstr.M{ + "name": "nginx-deployment", + }, + "namespace": defaultNs, + }, + }, + }, + } + + cfg := config.NewConfig() + metagen := NewReplicasetMetadataGenerator(cfg, nil, client) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} + +func TestReplicase_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object with owner", + input: &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + UID: uid, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "demo", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "demo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx:1.12", + Ports: []v1.ContainerPort{ + { + Name: "http", + Protocol: v1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "replicaset": mapstr.M{ + "name": "nginx-rs", + "uid": uid, + }, + "deployment": mapstr.M{ + "name": "nginx-deployment", + }, + "namespace": defaultNs, + }, + }, + }, + } + + for _, test := range tests { + cfg := config.NewConfig() + replicasets := cache.NewStore(cache.MetaNamespaceKeyFunc) + err := replicasets.Add(test.input) + require.NoError(t, err) + metagen := NewReplicasetMetadataGenerator(cfg, replicasets, client) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetNamespace(), "/", accessor.GetName()))) + }) + } +} From ccd7825dc46722ad75984f3fba5e3247b96635b6 Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Thu, 18 May 2023 14:04:35 +0300 Subject: [PATCH 11/11] Fix test Signed-off-by: ChrsMark --- kubernetes/metadata/replicaset_test.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/kubernetes/metadata/replicaset_test.go b/kubernetes/metadata/replicaset_test.go index a42bfb6a47..1a19d1f8af 100644 --- a/kubernetes/metadata/replicaset_test.go +++ b/kubernetes/metadata/replicaset_test.go @@ -171,16 +171,14 @@ func TestReplicase_GenerateFromName(t *testing.T) { }, }, output: mapstr.M{ - "kubernetes": mapstr.M{ - "replicaset": mapstr.M{ - "name": "nginx-rs", - "uid": uid, - }, - "deployment": mapstr.M{ - "name": "nginx-deployment", - }, - "namespace": defaultNs, + "replicaset": mapstr.M{ + "name": "nginx-rs", + "uid": uid, + }, + "deployment": mapstr.M{ + "name": "nginx-deployment", }, + "namespace": defaultNs, }, }, }