Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[filebeat][httpjson]- Added min & max functions to the template engine #36036

Merged
merged 14 commits into from
Jul 24, 2023
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
- Add device support for Azure AD entity analytics. {pull}35807[35807]
- Improve CEL input performance. {pull}35915[35915]
- Adding filename details from zip to response for httpjson {issue}33952[33952] {pull}34044[34044]
- Added support for min/max template functions in httpjson input. {issue}36094[36094] {pull}36036[36036]
- Add `clean_session` configuration setting for MQTT input. {pull}35806[16204]
- Add fingerprint mode for the filestream scanner and new file identity based on it {issue}34419[34419] {pull}35734[35734]
- Add file system metadata to events ingested via filestream {issue}35801[35801] {pull}36065[36065]
Expand Down
6 changes: 6 additions & 0 deletions x-pack/filebeat/docs/inputs/input-httpjson.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,12 @@ Some built-in helper functions are provided to work with the input state inside
- `hmacBase64`: calculates the hmac signature of a list of strings concatenated together. Returns a base64 encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]`
- `hmac`: calculates the hmac signature of a list of strings concatenated together. Returns a hex encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]`
- `join`: joins a list using the specified separator. Example: `[[join .body.arr ","]]`
- `maxInt`: returns the maximum value of a list of signed integers where at least one value is required. If no values are provided, it returns an error.
- `minInt`: returns the minimum value of a list of signed integers where at least one value is required. If no values are provided, it returns an error.
- `maxUint`: returns the maximum value of a list of unsigned integers where at least one value is required. If no values are provided, it returns an error.
- `minUint`: returns the minimum value of a list of unsigned integers where at least one value is required. If no values are provided, it returns an error.
- `maxFloat`: returns the maximum value of a list of floating point numbers where at least one value is required. If no values are provided, it returns an error.
- `minFloat`: returns the minimum value of a list of floating point numbers where at least one value is required. If no values are provided, it returns an error.
- `mul`: multiplies two integers.
- `now`: returns the current `time.Time` object in UTC. Optionally, it can receive a `time.Duration` as a parameter. Example: `[[now (parseDuration "-1h")]]` returns the time at 1 hour before now.
- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`.
Expand Down
35 changes: 35 additions & 0 deletions x-pack/filebeat/input/httpjson/value_tpl.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ type valueTpl struct {
*template.Template
}

// number is a custom interface type that can be used to represent a number in a template.
type number interface{ ~int | ~uint | ~float64 }

func (t *valueTpl) Unpack(in string) error {
tpl, err := template.New("").
Option("missingkey=error").
Expand All @@ -66,6 +69,12 @@ func (t *valueTpl) Unpack(in string) error {
"hmacBase64": hmacStringBase64,
"join": join,
"toJSON": toJSON,
"maxInt": max[int],
"minInt": min[int],
"maxUint": max[uint],
"minUint": min[uint],
"maxFloat": max[float64],
"minFloat": min[float64],
"mul": mul,
"now": now,
"parseDate": parseDate,
Expand Down Expand Up @@ -295,6 +304,32 @@ func div(a, b int64) int64 {
return a / b
}

// min returns the minimum value of a list of numbers.
func min[T number](head T, tail ...T) T {
for _, t := range tail {
if t != t {
return t
}
if t < head {
head = t
}
}
return head
}

// max returns the maximum value of a list of numbers.
func max[T number](head T, tail ...T) T {
for _, t := range tail {
if t != t {
return t
}
if t > head {
head = t
}
}
return head
}

func base64Encode(values ...string) string {
data := strings.Join(values, "")
if data == "" {
Expand Down
140 changes: 140 additions & 0 deletions x-pack/filebeat/input/httpjson/value_tpl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,146 @@ func TestValueTpl(t *testing.T) {
paramTr: transformable{},
expectedVal: "4",
},
{
name: "func maxInt",
value: `[[maxInt 1 4]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "4",
},
{
name: "func maxInt with list",
value: `[[maxInt 1 4 5 3 2]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "5",
},
{
name: "func maxInt with year expression",
value: `[[maxInt (now.Year) 2023]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "2023",
},
{
name: "func maxInt with single value",
value: `[[maxInt 2023]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "2023",
},
{
name: "func maxInt with no arguments",
value: `[[maxInt]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedError: `template: :1:2: executing "" at <maxInt>: wrong number of args for maxInt: want at least 1 got 0`,
},
{
name: "func maxInt with unknown type in list",
value: `[[maxInt 1 "b" -2]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedError: `template: :1:11: executing "" at <"b">: expected integer; found "b"`,
},
{
name: "func maxUint",
value: `[[maxUint 18446744073709551615 18446744073709551614]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "18446744073709551615",
},
{
name: "func minUint",
value: `[[minUint 18446744073709551615 18446744073709551614]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "18446744073709551614",
},
{
name: "func minInt",
value: `[[minInt 1 4]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "1",
},
ShourieG marked this conversation as resolved.
Show resolved Hide resolved
{
name: "func minInt with list",
value: `[[minInt 4 6 2 1 3 7]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "1",
},
{
name: "func minInt with unknown type in list",
value: `[[minInt 1 "b" -2]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedError: `template: :1:11: executing "" at <"b">: expected integer; found "b"`,
},
{
name: "func minInt with year expression",
value: `[[ minInt (now.Year) 2023 ]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "2023",
},
{
name: "func minInt with single value",
value: `[[ minInt 2023 ]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "2023",
},
{
name: "func minInt with no arguments",
value: `[[minInt]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedError: `template: :1:2: executing "" at <minInt>: wrong number of args for minInt: want at least 1 got 0`,
},
{
name: "func maxFloat",
value: `[[maxFloat 1.23 4.666]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "4.666",
},
{
name: "func maxFloat with list",
value: `[[maxFloat 1.23 4.667 4.666]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "4.667",
},
{
name: "func maxFloat with full range",
value: `[[maxFloat 1.7976931348623157e+308 -1.7976931348623157e+308]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "1.7976931348623157e+308",
},
{
name: "func minFloat",
value: `[[minFloat 1.23 4.666]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "1.23",
},
{
name: "func minFloat with list",
value: `[[minFloat -2.23 1.23 4.667 4.666]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "-2.23",
},
{
name: "func minFloat with full range",
value: `[[minFloat 1.7976931348623157e+308 -1.7976931348623157e+308]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "-1.7976931348623157e+308",
},
{
name: "func sha1 hmac Hex",
value: `[[hmac "sha1" "secret" "string1" "string2"]]`,
Expand Down