Skip to content

Commit

Permalink
Add a test rule with expression
Browse files Browse the repository at this point in the history 
To test that routing rules work with datasets and namespaces, we add
another rule that uses the `{{cloud.account.id}}` for both if the
`cloud.region` is `eu-west-1`.
  • Loading branch information
@zmoog
zmoog committed Aug 23, 2023
1 parent 4a678ae commit edd774a
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 0 deletions.
38 changes: 38 additions & 0 deletions ...arallel/awsfirehose/data_stream/log/_dev/test/pipeline/test-cloudtrail-by-account-id.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"events": [
{
"message": "{\"eventVersion\":\"1.08\",\"userIdentity\":{\"type\":\"AWSService\",\"invokedBy\":\"cloudtrail.amazonaws.com\"},\"eventTime\":\"2023-07-17T21:02:26Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"sa-east-1\",\"sourceIPAddress\":\"cloudtrail.amazonaws.com\",\"userAgent\":\"cloudtrail.amazonaws.com\",\"requestParameters\":{\"roleArn\":\"arn:aws:iam::123456:role/service-role/aws-cloudtrail-logs-123456-b888baff_Role\",\"roleSessionName\":\"CLOUDWATCH_LOGS_DELIVERY_SESSION\"},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAZEDJODE3A5LVGLFB\",\"sessionToken\":\"IQoJb3JpZ2luX2VjEGUaCXNhLWVhc3QtMSJHMEUCIHgHmtcrhwDhosJlQVky+C2zsYDKuR99qVlNjGIp8FLWAiEAsJtTDQ3Arq8iXEOHwv0ImEQdGb5tbgc+fLpoK58Enb4q9AII3v//////////ARAEGgw2MjcyODYzNTAxMzQiDN5gNdfO4ZdSqDmmwSrIAicTBYZg+ZXjwiJTN/Bz2YsMWYU6psw5znG3/Gh3EJ1P3RCmB7d79X6XZzFVi2u2xdrnaY/sTKDfp1jdl8OoAsSKYwJiGbzjoQlv59bB6JqPbKfAKUPAmz6JEMWNFgWTtaQL9rNkdPz23u/1msoUSzxCcxR9f3A2dD4yqnVpNJe8ipuhxpBMzQ61vcGL4G5hQEDM/o8sORP2PXbK4O7QAuWOyuryYkHAPwY9RrL0WHfflGBEBQV6XlidGpsRCtIppZVn025n3DQOypDEaL3fKp0gUsMkDH+frFjxop4o4wRYC3CxXe3XRJ5/Te886rQry7RUfXlQtiCfojZO5ohcLB+z6Y/uCK0IHp3zrfl5shKsQIAFt7p0B8W7PK5yHE4W9HHRiktJ9wTtq1YCTaWECpnjW0bISNgumRmDOAJvVHAjSjfkr4yAlJkw4qm8pQY6vwGbBiuf98AfRFrXMy01hVdE3GNTBrIS68zxUJaOjBLgw8l0nEC00L+LPuqaASFWz65Dnq5JAjXaDD9E3iCi4klp4gZFAcj7uGgeBIPkP7Bpr4SvBfnnqCgE2oyFrWke3NnYtqkL5iHLJeGlOTrvI5ND2H4jurQv0KbiqwHt6DmGF3poZOrtf8R3piNcuCCDLU8RvhRVLHy5rKPzsWgNokBc9XXmgltwvB6rIgdZhBJzupzmy/NSoWZcOeH2ooEELw==\",\"expiration\":\"Jul 12, 2023, 10:02:26 PM\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAZEDJODE3NLJAH2FZC:CLOUDWATCH_LOGS_DELIVERY_SESSION\",\"arn\":\"arn:aws:sts::123456:assumed-role/aws-cloudtrail-logs-123456-b888baff_Role/CLOUDWATCH_LOGS_DELIVERY_SESSION\"}},\"requestID\":\"041c9e5f-a031-47d2-a4a0-011bc8d5352c\",\"eventID\":\"3096b662-7aa9-43e6-8bee-541a45686745\",\"readOnly\":true,\"resources\":[{\"accountId\":\"123456\",\"type\":\"AWS::IAM::Role\",\"ARN\":\"arn:aws:iam::123456:role/service-role/aws-cloudtrail-logs-123456-b888baff_Role\"}],\"eventType\":\"AwsApiCall\",\"managementEvent\":true,\"recipientAccountId\":\"123456\",\"sharedEventID\":\"a1c94275-884f-4c1f-b8dc-2e1bf4c94d29\",\"eventCategory\":\"Management\"}",
"@timestamp": "2023-07-25T21:04:35Z",
"aws": {
"firehose": {
"arn": "arn:aws:firehose:eu-west-1:123456:deliverystream/firehose-cloudtrail-logs-to-elastic",
"subscription_filters": "[cloudtrail-to-firehose]",
"request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55"
},
"kinesis": {
"name": "firehose-cloudtrail-logs-to-elastic",
"type": "deliverystream"
},
"cloudwatch": {
"log_group": "aws-cloudtrail-logs-123456-1c167310",
"log_stream": "123456_CloudTrail_eu-west-1_3"
}
},
"cloud": {
"account": {
"id": "123456"
},
"region": "eu-west-1",
"provider": "aws"
},
"data_stream": {
"dataset": "awsfirehose.log",
"namespace": "default",
"type": "logs"
},
"event": {
"id": "37670326805251200781477669690942747782212394134076063744"
}
}
]
}
41 changes: 41 additions & 0 deletions ...ehose/data_stream/log/_dev/test/pipeline/test-cloudtrail-by-account-id.json-expected.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
{
"expected": [
{
"@timestamp": "2023-07-25T21:04:35Z",
"aws": {
"cloudwatch": {
"log_group": "aws-cloudtrail-logs-123456-1c167310",
"log_stream": "123456_CloudTrail_eu-west-1_3"
},
"firehose": {
"arn": "arn:aws:firehose:eu-west-1:123456:deliverystream/firehose-cloudtrail-logs-to-elastic",
"request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55",
"subscription_filters": "[cloudtrail-to-firehose]"
},
"kinesis": {
"name": "firehose-cloudtrail-logs-to-elastic",
"type": "deliverystream"
}
},
"cloud": {
"account": {
"id": "123456"
},
"provider": "aws",
"region": "eu-west-1"
},
"data_stream": {
"dataset": "123456",
"namespace": "123456",
"type": "logs"
},
"ecs": {
"version": "8.0.0"
},
"event": {
"id": "37670326805251200781477669690942747782212394134076063744"
},
"message": "{\"eventVersion\":\"1.08\",\"userIdentity\":{\"type\":\"AWSService\",\"invokedBy\":\"cloudtrail.amazonaws.com\"},\"eventTime\":\"2023-07-17T21:02:26Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"sa-east-1\",\"sourceIPAddress\":\"cloudtrail.amazonaws.com\",\"userAgent\":\"cloudtrail.amazonaws.com\",\"requestParameters\":{\"roleArn\":\"arn:aws:iam::123456:role/service-role/aws-cloudtrail-logs-123456-b888baff_Role\",\"roleSessionName\":\"CLOUDWATCH_LOGS_DELIVERY_SESSION\"},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAZEDJODE3A5LVGLFB\",\"sessionToken\":\"IQoJb3JpZ2luX2VjEGUaCXNhLWVhc3QtMSJHMEUCIHgHmtcrhwDhosJlQVky+C2zsYDKuR99qVlNjGIp8FLWAiEAsJtTDQ3Arq8iXEOHwv0ImEQdGb5tbgc+fLpoK58Enb4q9AII3v//////////ARAEGgw2MjcyODYzNTAxMzQiDN5gNdfO4ZdSqDmmwSrIAicTBYZg+ZXjwiJTN/Bz2YsMWYU6psw5znG3/Gh3EJ1P3RCmB7d79X6XZzFVi2u2xdrnaY/sTKDfp1jdl8OoAsSKYwJiGbzjoQlv59bB6JqPbKfAKUPAmz6JEMWNFgWTtaQL9rNkdPz23u/1msoUSzxCcxR9f3A2dD4yqnVpNJe8ipuhxpBMzQ61vcGL4G5hQEDM/o8sORP2PXbK4O7QAuWOyuryYkHAPwY9RrL0WHfflGBEBQV6XlidGpsRCtIppZVn025n3DQOypDEaL3fKp0gUsMkDH+frFjxop4o4wRYC3CxXe3XRJ5/Te886rQry7RUfXlQtiCfojZO5ohcLB+z6Y/uCK0IHp3zrfl5shKsQIAFt7p0B8W7PK5yHE4W9HHRiktJ9wTtq1YCTaWECpnjW0bISNgumRmDOAJvVHAjSjfkr4yAlJkw4qm8pQY6vwGbBiuf98AfRFrXMy01hVdE3GNTBrIS68zxUJaOjBLgw8l0nEC00L+LPuqaASFWz65Dnq5JAjXaDD9E3iCi4klp4gZFAcj7uGgeBIPkP7Bpr4SvBfnnqCgE2oyFrWke3NnYtqkL5iHLJeGlOTrvI5ND2H4jurQv0KbiqwHt6DmGF3poZOrtf8R3piNcuCCDLU8RvhRVLHy5rKPzsWgNokBc9XXmgltwvB6rIgdZhBJzupzmy/NSoWZcOeH2ooEELw==\",\"expiration\":\"Jul 12, 2023, 10:02:26 PM\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAZEDJODE3NLJAH2FZC:CLOUDWATCH_LOGS_DELIVERY_SESSION\",\"arn\":\"arn:aws:sts::123456:assumed-role/aws-cloudtrail-logs-123456-b888baff_Role/CLOUDWATCH_LOGS_DELIVERY_SESSION\"}},\"requestID\":\"041c9e5f-a031-47d2-a4a0-011bc8d5352c\",\"eventID\":\"3096b662-7aa9-43e6-8bee-541a45686745\",\"readOnly\":true,\"resources\":[{\"accountId\":\"123456\",\"type\":\"AWS::IAM::Role\",\"ARN\":\"arn:aws:iam::123456:role/service-role/aws-cloudtrail-logs-123456-b888baff_Role\"}],\"eventType\":\"AwsApiCall\",\"managementEvent\":true,\"recipientAccountId\":\"123456\",\"sharedEventID\":\"a1c94275-884f-4c1f-b8dc-2e1bf4c94d29\",\"eventCategory\":\"Management\"}"
}
]
}
5 changes: 5 additions & 0 deletions test/packages/parallel/awsfirehose/data_stream/log/routing_rules.yml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
- source_dataset: awsfirehose.log
rules:
- target_dataset:
- "{{cloud.account.id}}"
namespace:
- "{{cloud.account.id}}"
if: ctx.cloud?.region == 'eu-west-1'
- target_dataset: aws.cloudtrail
if: ctx['aws.cloudwatch.log_stream'].contains('CloudTrail')
namespace:
Expand Down

0 comments on commit edd774a

Please sign in to comment.