Skip to content

Commit

Permalink
[AWS] [EC2] enrich events with EC2 tags with add_cloud_metadata proce…
Browse files Browse the repository at this point in the history
…ssor (#41477)

* add support to extract ec2 tags from IMDS endpoint

Signed-off-by: Kavindu Dodanduwa <[email protected]>

* add dedicated tests for tag extractor

Signed-off-by: Kavindu Dodanduwa <[email protected]>

* expand test case and add documentation

Signed-off-by: Kavindu Dodanduwa <[email protected]>

* add changelog entry

Signed-off-by: Kavindu Dodanduwa <[email protected]>

* handle empty tags, add tests and close underlying body

Signed-off-by: Kavindu Dodanduwa <[email protected]>

* review change - use aws.tags as tag prefix

Signed-off-by: Kavindu Dodanduwa <[email protected]>

---------

Signed-off-by: Kavindu Dodanduwa <[email protected]>
  • Loading branch information
Kavindu-Dodan authored Nov 12, 2024
1 parent 734ae05 commit c878397
Showing 5 changed files with 367 additions and 121 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
@@ -340,6 +340,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]

*Libbeat*

- enrich events with EC2 tags in add_cloud_metadata processor {pull}41477[41477]


*Heartbeat*
Original file line number Diff line number Diff line change
@@ -83,6 +83,8 @@ examples for each of the supported providers.

_AWS_

Metadata given below are extracted from https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-identity-documents.html[instance identity document],

[source,json]
-------------------------------------------------------------------------------
{
@@ -98,6 +100,22 @@ _AWS_
}
-------------------------------------------------------------------------------

If the EC2 instance has IMDS enabled and if tags are allowed through IMDS endpoint, the processor will further append tags in metadata.
Please refer official documentation on https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html[IMDS endpoint] for further details.

[source,json]
-------------------------------------------------------------------------------
{
"aws": {
"tags": {
"org" : "myOrg",
"owner": "userID"
}
}
}
-------------------------------------------------------------------------------


_Digital Ocean_

[source,json]
137 changes: 114 additions & 23 deletions libbeat/processors/add_cloud_metadata/provider_aws_ec2.go
Original file line number Diff line number Diff line change
@@ -20,12 +20,15 @@ package add_cloud_metadata
import (
"context"
"fmt"
"io"
"net/http"
"strings"

"github.com/elastic/elastic-agent-libs/logp"

awssdk "github.com/aws/aws-sdk-go-v2/aws"
awscfg "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds"
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"
@@ -35,7 +38,14 @@ import (
conf "github.com/elastic/elastic-agent-libs/config"
)

const (
eksClusterNameTagKey = "eks:cluster-name"
tagsCategory = "tags/instance"
tagPrefix = "aws.tags"
)

type IMDSClient interface {
ec2rolecreds.GetMetadataAPIClient
GetInstanceIdentityDocument(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error)
}

@@ -90,30 +100,17 @@ func fetchRawProviderMetadata(
result.err = fmt.Errorf("failed loading AWS default configuration: %w", err)
return
}
awsClient := NewIMDSClient(awsConfig)

instanceIdentity, err := awsClient.GetInstanceIdentityDocument(context.TODO(), &imds.GetInstanceIdentityDocumentInput{})
imdsClient := NewIMDSClient(awsConfig)
instanceIdentity, err := imdsClient.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
if err != nil {
result.err = fmt.Errorf("failed fetching EC2 Identity Document: %w", err)
return
}

// AWS Region must be set to be able to get EC2 Tags
awsRegion := instanceIdentity.InstanceIdentityDocument.Region
awsConfig.Region = awsRegion
accountID := instanceIdentity.InstanceIdentityDocument.AccountID

clusterName, err := fetchEC2ClusterNameTag(awsConfig, instanceIdentity.InstanceIdentityDocument.InstanceID)
if err != nil {
logger.Warnf("error fetching cluster name metadata: %s.", err)
} else if clusterName != "" {
// for AWS cluster ID is used cluster ARN: arn:partition:service:region:account-id:resource-type/resource-id, example:
// arn:aws:eks:us-east-2:627286350134:cluster/cluster-name
clusterARN := fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%v", awsRegion, accountID, clusterName)

_, _ = result.metadata.Put("orchestrator.cluster.id", clusterARN)
_, _ = result.metadata.Put("orchestrator.cluster.name", clusterName)
}
instanceID := instanceIdentity.InstanceIdentityDocument.InstanceID

_, _ = result.metadata.Put("cloud.instance.id", instanceIdentity.InstanceIdentityDocument.InstanceID)
_, _ = result.metadata.Put("cloud.machine.type", instanceIdentity.InstanceIdentityDocument.InstanceType)
@@ -122,10 +119,106 @@ func fetchRawProviderMetadata(
_, _ = result.metadata.Put("cloud.account.id", accountID)
_, _ = result.metadata.Put("cloud.image.id", instanceIdentity.InstanceIdentityDocument.ImageID)

// AWS Region must be set to be able to get EC2 Tags
awsConfig.Region = awsRegion
tags := getTags(ctx, imdsClient, NewEC2Client(awsConfig), instanceID, logger)

if tags[eksClusterNameTagKey] != "" {
// for AWS cluster ID is used cluster ARN: arn:partition:service:region:account-id:resource-type/resource-id, example:
// arn:aws:eks:us-east-2:627286350134:cluster/cluster-name
clusterARN := fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%v", awsRegion, accountID, tags[eksClusterNameTagKey])

_, _ = result.metadata.Put("orchestrator.cluster.id", clusterARN)
_, _ = result.metadata.Put("orchestrator.cluster.name", tags[eksClusterNameTagKey])
}

if len(tags) == 0 {
return
}

logger.Infof("Adding retrieved tags with key: %s", tagPrefix)
for k, v := range tags {
_, _ = result.metadata.Put(fmt.Sprintf("%s.%s", tagPrefix, k), v)
}
}

// getTags is a helper to extract EC2 tags. Internally it utilize multiple extraction methods.
func getTags(ctx context.Context, imdsClient IMDSClient, ec2Client EC2Client, instanceId string, logger *logp.Logger) map[string]string {
logger.Info("Extracting EC2 tags from IMDS endpoint")
tags, ok := getTagsFromIMDS(ctx, imdsClient, logger)
if ok {
return tags
}

logger.Info("Tag extraction from IMDS failed, fallback to DescribeTags API to obtain EKS cluster name.")
clusterName, err := clusterNameFromDescribeTag(ctx, ec2Client, instanceId)
if err != nil {
logger.Warnf("error obtaining cluster name: %v.", err)
return tags
}

if clusterName != "" {
tags[eksClusterNameTagKey] = clusterName
}
return tags
}

// getTagsFromIMDS is a helper to extract EC2 tags using instance metadata service.
// Note that this call could get throttled and currently does not implement a retry mechanism.
// See - https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html#instancedata-throttling
func getTagsFromIMDS(ctx context.Context, client IMDSClient, logger *logp.Logger) (tags map[string]string, ok bool) {
tags = make(map[string]string)

b, err := getMetadataHelper(ctx, client, tagsCategory, logger)
if err != nil {
logger.Warnf("error obtaining tags category: %v", err)
return tags, false
}

for _, tag := range strings.Split(string(b), "\n") {
tagPath := fmt.Sprintf("%s/%s", tagsCategory, tag)
b, err := getMetadataHelper(ctx, client, tagPath, logger)
if err != nil {
logger.Warnf("error extracting tag value of %s: %v", tag, err)
return tags, false
}

tagValue := string(b)
if tagValue == "" {
logger.Infof("Ignoring tag key %s as value is empty", tag)
continue
}

tags[tag] = tagValue
}

return tags, true
}

// getMetadataHelper performs the IMDS call for the given path and returns the response content after closing the underlying content reader.
func getMetadataHelper(ctx context.Context, client IMDSClient, path string, logger *logp.Logger) (content []byte, err error) {
metadata, err := client.GetMetadata(ctx, &imds.GetMetadataInput{Path: path})
if err != nil {
return nil, fmt.Errorf("error from IMDS metadata request: %w", err)
}

defer func(Content io.ReadCloser) {
err := Content.Close()
if err != nil {
logger.Warnf("error closing IMDS metadata response body: %v", err)
}
}(metadata.Content)

content, err = io.ReadAll(metadata.Content)
if err != nil {
return nil, fmt.Errorf("error extracting metadata from the IMDS response: %w", err)
}

return content, nil
}

func fetchEC2ClusterNameTag(awsConfig awssdk.Config, instanceID string) (string, error) {
svc := NewEC2Client(awsConfig)
// clusterNameFromDescribeTag is a helper to extract EKS cluster name using DescribeTag.
func clusterNameFromDescribeTag(ctx context.Context, ec2Client EC2Client, instanceID string) (string, error) {
input := &ec2.DescribeTagsInput{
Filters: []types.Filter{
{
@@ -135,15 +228,13 @@ func fetchEC2ClusterNameTag(awsConfig awssdk.Config, instanceID string) (string,
},
},
{
Name: awssdk.String("key"),
Values: []string{
"eks:cluster-name",
},
Name: awssdk.String("key"),
Values: []string{eksClusterNameTagKey},
},
},
}

tagsResult, err := svc.DescribeTags(context.TODO(), input)
tagsResult, err := ec2Client.DescribeTags(ctx, input)
if err != nil {
return "", fmt.Errorf("error fetching EC2 Tags: %w", err)
}
330 changes: 233 additions & 97 deletions libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go
Original file line number Diff line number Diff line change
@@ -19,8 +19,11 @@ package add_cloud_metadata

import (
"context"
"errors"
"fmt"
"io"
"os"
"strings"
"testing"

awssdk "github.com/aws/aws-sdk-go-v2/aws"
@@ -43,66 +46,113 @@ func init() {
os.Setenv("AWS_EC2_METADATA_DISABLED", "true")
}

type getInstanceIDFunc func(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error)
type getMetaFunc func(ctx context.Context, input *imds.GetMetadataInput, f ...func(*imds.Options)) (*imds.GetMetadataOutput, error)
type getTagFunc func(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error)

type MockIMDSClient struct {
GetInstanceIdentityDocumentFunc func(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error)
GetInstanceIdentityDocumentFunc getInstanceIDFunc
GetMetadataFunc getMetaFunc
}

func (m *MockIMDSClient) GetMetadata(ctx context.Context, input *imds.GetMetadataInput, f ...func(*imds.Options)) (*imds.GetMetadataOutput, error) {
return m.GetMetadataFunc(ctx, input, f...)
}

func (m *MockIMDSClient) GetInstanceIdentityDocument(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error) {
return m.GetInstanceIdentityDocumentFunc(ctx, params, optFns...)
}

type MockEC2Client struct {
DescribeTagsFunc func(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error)
DescribeTagsFunc getTagFunc
}

func (e *MockEC2Client) DescribeTags(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error) {
return e.DescribeTagsFunc(ctx, params, optFns...)
}

var (
// not the best way to use a response template
// but this should serve until we need to test
// documents containing very different values
accountIDDoc1 = "111111111111111"
regionDoc1 = "us-east-1"
availabilityZoneDoc1 = "us-east-1c"
imageIDDoc1 = "ami-abcd1234"
instanceTypeDoc1 = "t2.medium"
instanceIDDoc2 = "i-22222222"
clusterNameKey = eksClusterNameTagKey
clusterNameValue = "test"
instanceIDDoc1 = "i-11111111"
customTagKey = "organization"
customTagValue = "orgName"
)

// generic getTagFunc implementation with IMDS disabled error to avoid IMDS response
var disabledIMDS getMetaFunc = func(ctx context.Context, input *imds.GetMetadataInput, f ...func(*imds.Options)) (*imds.GetMetadataOutput, error) {
return nil, errors.New("IMDS disabled mock error")
}

// set up a generic getTagFunc implementation with valid tags
var genericImdsGet getMetaFunc = func(ctx context.Context, input *imds.GetMetadataInput, f ...func(*imds.Options)) (*imds.GetMetadataOutput, error) {
tagKeys := fmt.Sprintf("%s\n%s", customTagKey, eksClusterNameTagKey)

if input.Path == tagsCategory {
// tag category request
return &imds.GetMetadataOutput{
Content: io.NopCloser(strings.NewReader(tagKeys)),
}, nil
}

// tag request
if strings.HasSuffix(input.Path, customTagKey) {
return &imds.GetMetadataOutput{
Content: io.NopCloser(strings.NewReader(customTagValue)),
}, nil
}

if strings.HasSuffix(input.Path, eksClusterNameTagKey) {
return &imds.GetMetadataOutput{
Content: io.NopCloser(strings.NewReader(clusterNameValue)),
}, nil
}
return nil, errors.New("invalid request")
}

// generic getInstanceIDFunc implementation with known response values and no error
var genericInstanceIDResponse getInstanceIDFunc = func(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error) {
return &imds.GetInstanceIdentityDocumentOutput{
InstanceIdentityDocument: imds.InstanceIdentityDocument{
AvailabilityZone: availabilityZoneDoc1,
Region: regionDoc1,
InstanceID: instanceIDDoc1,
InstanceType: instanceTypeDoc1,
AccountID: accountIDDoc1,
ImageID: imageIDDoc1,
},
}, nil
}

func TestMain(m *testing.M) {
logp.TestingSetup()
code := m.Run()
os.Exit(code)
}

func TestRetrieveAWSMetadataEC2(t *testing.T) {
var (
// not the best way to use a response template
// but this should serve until we need to test
// documents containing very different values
accountIDDoc1 = "111111111111111"
regionDoc1 = "us-east-1"
availabilityZoneDoc1 = "us-east-1c"
imageIDDoc1 = "ami-abcd1234"
instanceTypeDoc1 = "t2.medium"
instanceIDDoc2 = "i-22222222"
clusterNameKey = "eks:cluster-name"
clusterNameValue = "test"
instanceIDDoc1 = "i-11111111"
)

var tests = []struct {
testName string
mockGetInstanceIdentity func(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error)
mockEc2Tags func(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error)
mockGetInstanceIdentity getInstanceIDFunc
mockMetadata getMetaFunc
mockEc2Tags getTagFunc
processorOverwrite bool
previousEvent mapstr.M
expectedEvent mapstr.M
}{
{
testName: "valid instance identity document, no cluster tags",
mockGetInstanceIdentity: func(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error) {
return &imds.GetInstanceIdentityDocumentOutput{
InstanceIdentityDocument: imds.InstanceIdentityDocument{
AvailabilityZone: availabilityZoneDoc1,
Region: regionDoc1,
InstanceID: instanceIDDoc1,
InstanceType: instanceTypeDoc1,
AccountID: accountIDDoc1,
ImageID: imageIDDoc1,
},
}, nil
},
testName: "valid instance identity document, no cluster tags",
mockGetInstanceIdentity: genericInstanceIDResponse,
mockMetadata: disabledIMDS,
mockEc2Tags: func(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error) {
return &ec2.DescribeTagsOutput{
Tags: []types.TagDescription{},
@@ -124,19 +174,9 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
},
},
{
testName: "all fields from processor",
mockGetInstanceIdentity: func(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error) {
return &imds.GetInstanceIdentityDocumentOutput{
InstanceIdentityDocument: imds.InstanceIdentityDocument{
AvailabilityZone: availabilityZoneDoc1,
Region: regionDoc1,
InstanceID: instanceIDDoc1,
InstanceType: instanceTypeDoc1,
AccountID: accountIDDoc1,
ImageID: imageIDDoc1,
},
}, nil
},
testName: "all fields from processor",
mockGetInstanceIdentity: genericInstanceIDResponse,
mockMetadata: disabledIMDS,
mockEc2Tags: func(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error) {
return &ec2.DescribeTagsOutput{
Tags: []types.TagDescription{
@@ -168,22 +208,17 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
"id": fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", regionDoc1, accountIDDoc1, clusterNameValue),
},
},
"aws": mapstr.M{
"tags": mapstr.M{
eksClusterNameTagKey: clusterNameValue,
},
},
},
},
{
testName: "instanceId pre-informed, no overwrite",
mockGetInstanceIdentity: func(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error) {
return &imds.GetInstanceIdentityDocumentOutput{
InstanceIdentityDocument: imds.InstanceIdentityDocument{
AvailabilityZone: availabilityZoneDoc1,
Region: regionDoc1,
InstanceID: instanceIDDoc1,
InstanceType: instanceTypeDoc1,
AccountID: accountIDDoc1,
ImageID: imageIDDoc1,
},
}, nil
},
testName: "instanceId pre-informed, no overwrite",
mockGetInstanceIdentity: genericInstanceIDResponse,
mockMetadata: disabledIMDS,
mockEc2Tags: func(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error) {
return &ec2.DescribeTagsOutput{
Tags: []types.TagDescription{
@@ -212,25 +247,20 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
"id": fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", regionDoc1, accountIDDoc1, clusterNameValue),
},
},
"aws": mapstr.M{
"tags": mapstr.M{
eksClusterNameTagKey: clusterNameValue,
},
},
},
},
{
// NOTE: In this case, add_cloud_metadata will overwrite cloud fields because
// it won't detect cloud.provider as a cloud field. This is not the behavior we
// expect and will find a better solution later in issue 11697.
testName: "only cloud.provider pre-informed, no overwrite",
mockGetInstanceIdentity: func(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error) {
return &imds.GetInstanceIdentityDocumentOutput{
InstanceIdentityDocument: imds.InstanceIdentityDocument{
AvailabilityZone: availabilityZoneDoc1,
Region: regionDoc1,
InstanceID: instanceIDDoc1,
InstanceType: instanceTypeDoc1,
AccountID: accountIDDoc1,
ImageID: imageIDDoc1,
},
}, nil
},
testName: "only cloud.provider pre-informed, no overwrite",
mockGetInstanceIdentity: genericInstanceIDResponse,
mockMetadata: disabledIMDS,
mockEc2Tags: func(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error) {
return &ec2.DescribeTagsOutput{
Tags: []types.TagDescription{
@@ -265,22 +295,17 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
"id": fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", regionDoc1, accountIDDoc1, clusterNameValue),
},
},
"aws": mapstr.M{
"tags": mapstr.M{
eksClusterNameTagKey: clusterNameValue,
},
},
},
},
{
testName: "instanceId pre-informed, overwrite",
mockGetInstanceIdentity: func(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error) {
return &imds.GetInstanceIdentityDocumentOutput{
InstanceIdentityDocument: imds.InstanceIdentityDocument{
AvailabilityZone: availabilityZoneDoc1,
Region: regionDoc1,
InstanceID: instanceIDDoc1,
InstanceType: instanceTypeDoc1,
AccountID: accountIDDoc1,
ImageID: imageIDDoc1,
},
}, nil
},
testName: "instanceId pre-informed, overwrite",
mockGetInstanceIdentity: genericInstanceIDResponse,
mockMetadata: disabledIMDS,
mockEc2Tags: func(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error) {
return &ec2.DescribeTagsOutput{
Tags: []types.TagDescription{},
@@ -306,19 +331,9 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
},
},
{
testName: "only cloud.provider pre-informed, overwrite",
mockGetInstanceIdentity: func(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error) {
return &imds.GetInstanceIdentityDocumentOutput{
InstanceIdentityDocument: imds.InstanceIdentityDocument{
AvailabilityZone: availabilityZoneDoc1,
Region: regionDoc1,
InstanceID: instanceIDDoc1,
InstanceType: instanceTypeDoc1,
AccountID: accountIDDoc1,
ImageID: imageIDDoc1,
},
}, nil
},
testName: "only cloud.provider pre-informed, overwrite",
mockGetInstanceIdentity: genericInstanceIDResponse,
mockMetadata: disabledIMDS,
mockEc2Tags: func(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error) {
return &ec2.DescribeTagsOutput{
Tags: []types.TagDescription{},
@@ -342,6 +357,36 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
},
},
},
{
testName: "if enabled, extract tags from IMDS endpoint",
mockGetInstanceIdentity: genericInstanceIDResponse,
mockMetadata: genericImdsGet,
mockEc2Tags: nil, // could be nil as IMDS response fulfills tag
expectedEvent: mapstr.M{
"cloud": mapstr.M{
"provider": "aws",
"account": mapstr.M{"id": accountIDDoc1},
"instance": mapstr.M{"id": instanceIDDoc1},
"machine": mapstr.M{"type": instanceTypeDoc1},
"image": mapstr.M{"id": imageIDDoc1},
"region": regionDoc1,
"availability_zone": availabilityZoneDoc1,
"service": mapstr.M{"name": "EC2"},
},
"orchestrator": mapstr.M{
"cluster": mapstr.M{
"name": clusterNameValue,
"id": fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", regionDoc1, accountIDDoc1, clusterNameValue),
},
},
"aws": mapstr.M{
"tags": mapstr.M{
eksClusterNameTagKey: clusterNameValue,
customTagKey: customTagValue,
},
},
},
},
}

for _, tc := range tests {
@@ -350,6 +395,7 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
NewIMDSClient = func(cfg awssdk.Config) IMDSClient {
return &MockIMDSClient{
GetInstanceIdentityDocumentFunc: tc.mockGetInstanceIdentity,
GetMetadataFunc: tc.mockMetadata,
}
}
defer func() { NewIMDSClient = func(cfg awssdk.Config) IMDSClient { return imds.NewFromConfig(cfg) } }()
@@ -381,3 +427,93 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
})
}
}

func Test_getTags(t *testing.T) {
ctx := context.Background()
instanceId := "ami-abcd1234"
logger := logp.NewLogger("add_cloud_metadata test logger")

tests := []struct {
name string
imdsClient IMDSClient
ec2Client EC2Client
want map[string]string
}{
{
name: "tags extracted from IMDS if possible",
imdsClient: &MockIMDSClient{
GetMetadataFunc: genericImdsGet,
},
want: map[string]string{
customTagKey: customTagValue,
eksClusterNameTagKey: clusterNameValue,
},
},
{
name: "tag extraction fallback to DescribeTag if IMDS fetch results in an error",
imdsClient: &MockIMDSClient{
GetMetadataFunc: disabledIMDS,
},
ec2Client: &MockEC2Client{
DescribeTagsFunc: func(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error) {
return &ec2.DescribeTagsOutput{
Tags: []types.TagDescription{
{
Key: &clusterNameKey,
ResourceId: &instanceId,
ResourceType: "instance",
Value: &clusterNameValue,
},
},
}, nil
}},
want: map[string]string{
eksClusterNameTagKey: clusterNameValue,
},
},
{
name: "empty tags if all methods failed",
imdsClient: &MockIMDSClient{
GetMetadataFunc: disabledIMDS,
},
ec2Client: &MockEC2Client{
DescribeTagsFunc: func(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error) {
return nil, errors.New("some error from DescribeTag")
}},
want: map[string]string{},
},
{
name: "Empty tags values are ignored",
imdsClient: &MockIMDSClient{
GetMetadataFunc: func(ctx context.Context, input *imds.GetMetadataInput, f ...func(*imds.Options)) (*imds.GetMetadataOutput, error) {
if input.Path == tagsCategory {
// tag category request
return &imds.GetMetadataOutput{
Content: io.NopCloser(strings.NewReader(customTagKey)),
}, nil
}

// tag request
if strings.HasSuffix(input.Path, customTagKey) {
return &imds.GetMetadataOutput{
Content: io.NopCloser(strings.NewReader("")),
}, nil
}

return nil, errors.New("invalid request")
},
},
ec2Client: &MockEC2Client{
DescribeTagsFunc: func(ctx context.Context, params *ec2.DescribeTagsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeTagsOutput, error) {
return nil, errors.New("some error from DescribeTag")
}},
want: map[string]string{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tags := getTags(ctx, tt.imdsClient, tt.ec2Client, instanceId, logger)
assert.Equal(t, tags, tt.want)
})
}
}
2 changes: 1 addition & 1 deletion libbeat/processors/add_cloud_metadata/providers.go
Original file line number Diff line number Diff line change
@@ -187,7 +187,7 @@ func (p *addCloudMetadata) fetchMetadata() *result {
if result.err == nil && result.metadata != nil {
return &result
} else if result.err != nil {
p.logger.Errorf("add_cloud_metadata: received error %v", result.err)
p.logger.Errorf("add_cloud_metadata: received error for provider %s: %v", result.provider, result.err)
}
case <-ctx.Done():
p.logger.Debugf("add_cloud_metadata: timed-out waiting for all responses")

0 comments on commit c878397

Please sign in to comment.