diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index e94aea740924b..aa1068f502bac 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -425,6 +425,8 @@ userguide webhdfs winapi workarounds +XCHACHA +XSALSA yandex zeek zookeeper diff --git a/.github/actions/spelling/excludes.txt b/.github/actions/spelling/excludes.txt index fd124acd37b4d..d18d1c721531f 100644 --- a/.github/actions/spelling/excludes.txt +++ b/.github/actions/spelling/excludes.txt @@ -48,6 +48,7 @@ \.otf$ \.p12$ \.pattern$ +\.pb$ \.pdf$ \.pem$ \.png$ diff --git a/Cargo.lock b/Cargo.lock index e48ba38426549..14e805e09b788 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + [[package]] name = "aes" version = "0.8.2" @@ -160,6 +170,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle 1.0.0", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + [[package]] name = "anstyle" version = "0.3.1" @@ -172,6 +197,34 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle 1.0.0", + "windows-sys 0.48.0", +] + [[package]] name = "anyhow" version = "1.0.72" @@ -1205,9 +1258,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.19" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a1de45611fdb535bfde7b7de4fd54f4fd2b17b1737c0a59b69bf9b92074b8c" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", @@ -1251,11 +1304,12 @@ dependencies = [ [[package]] name = "azure_core" -version = "0.5.0" -source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=b4544d4920fa3064eb921340054cd9cc130b7664#b4544d4920fa3064eb921340054cd9cc130b7664" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b0f0eea648347e40f5f7f7e6bfea4553bcefad0fbf52044ea339e5ce3aba61" dependencies = [ "async-trait", - "base64 0.13.1", + "base64 0.21.2", "bytes 1.4.0", "dyn-clone", "futures 0.3.28", @@ -1264,11 +1318,11 @@ dependencies = [ "log", "paste", "pin-project", + "quick-xml 0.29.0", "rand 0.8.5", "reqwest", "rustc_version 0.4.0", "serde", - "serde-xml-rs", "serde_json", "time", "url", @@ -1277,17 +1331,18 @@ dependencies = [ [[package]] name = "azure_identity" -version = "0.6.0" -source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=b4544d4920fa3064eb921340054cd9cc130b7664#b4544d4920fa3064eb921340054cd9cc130b7664" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61712538f43d64b56725f335bc931d0eb42d2b082fb157056465bbadfdeb5dd3" dependencies = [ "async-lock", "async-trait", "azure_core", - "base64 0.13.1", "fix-hidden-lifetime-bug", "futures 0.3.28", "log", "oauth2", + "pin-project", "serde", "serde_json", "time", @@ -1297,20 +1352,19 @@ dependencies = [ [[package]] name = "azure_storage" -version = "0.6.0" -source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=b4544d4920fa3064eb921340054cd9cc130b7664#b4544d4920fa3064eb921340054cd9cc130b7664" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d9cfa13ed9acb51cd663e04f343bd550a92b455add96c90de387a9a6bc4dbc" dependencies = [ "RustyXML", "async-trait", "azure_core", - "base64 0.13.1", "bytes 1.4.0", "futures 0.3.28", "hmac", "log", "once_cell", "serde", - "serde-xml-rs", "serde_derive", "serde_json", "sha2 0.10.7", @@ -1321,19 +1375,17 @@ dependencies = [ [[package]] name = "azure_storage_blobs" -version = "0.6.0" -source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=b4544d4920fa3064eb921340054cd9cc130b7664#b4544d4920fa3064eb921340054cd9cc130b7664" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57cb0fe58af32a3fb49e560613cb1e4937f9f13161a2c1caf1bba0224435f2af" dependencies = [ "RustyXML", "azure_core", "azure_storage", - "base64 0.13.1", "bytes 1.4.0", "futures 0.3.28", "log", - "md5", "serde", - "serde-xml-rs", "serde_derive", "serde_json", "time", @@ -1801,11 +1853,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.77" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" +checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -1829,6 +1882,30 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chacha20" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fc89c7c5b9e7a02dfe45cd2367bae382f9ed31c61ca8debe5f827c420a2f08" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + [[package]] name = "charset" version = "0.1.3" @@ -1924,6 +2001,7 @@ checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e" dependencies = [ "crypto-common", "inout", + "zeroize", ] [[package]] @@ -1943,9 +2021,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.14" +version = "4.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "906f7fe1da4185b7a282b2bc90172a496f9def1aca4545fe7526810741591e14" +checksum = "c27cdf28c0f604ba3f512b0c9a409f8de8513e4816705deb0498b627e7c3a3fd" dependencies = [ "clap_builder", "clap_derive", @@ -1958,21 +2036,20 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eef05769009513df2eb1c3b4613e7fad873a14c600ff025b08f250f59fee7de" dependencies = [ - "clap 4.1.14", + "clap 4.3.21", "log", ] [[package]] name = "clap_builder" -version = "4.1.14" +version = "4.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "351f9ad9688141ed83dfd8f5fb998a06225ef444b48ff4dc43de6d409b7fd10b" +checksum = "08a9f1ab5e9f01a9b81f202e8562eb9a10de70abf9eaeac1be465c28b75aa4aa" dependencies = [ - "bitflags 1.3.2", + "anstream", + "anstyle 1.0.0", "clap_lex", - "is-terminal", "strsim 0.10.0", - "termcolor", "terminal_size 0.2.2", ] @@ -1982,14 +2059,14 @@ version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fc443334c81a804575546c5a8a79b4913b50e28d69232903604cada1de817ce" dependencies = [ - "clap 4.1.14", + "clap 4.3.21", ] [[package]] name = "clap_derive" -version = "4.1.14" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81d7dc0031c3a59a04fc2ba395c8e2dd463cba1859275f065d225f6122221b45" +checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" dependencies = [ "heck 0.4.1", "proc-macro2 1.0.66", @@ -1999,9 +2076,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] name = "clipboard-win" @@ -2046,7 +2123,7 @@ dependencies = [ "similar-asserts", "smallvec", "snafu", - "syslog_loose 0.19.0", + "syslog_loose", "tokio", "tokio-util", "tracing 0.1.37", @@ -2069,6 +2146,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "colored" version = "2.0.4" @@ -2281,7 +2364,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.1.14", + "clap 4.3.21", "criterion-plot", "futures 0.3.28", "is-terminal", @@ -2418,9 +2501,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] +[[package]] +name = "crypto_secretbox" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d6cf87adf719ddf43a805e92c6870a531aedda35ff640442cbaf8674e141e1" +dependencies = [ + "aead", + "cipher", + "generic-array", + "poly1305", + "salsa20", + "subtle", + "zeroize", +] + [[package]] name = "csv" version = "1.2.2" @@ -3473,12 +3572,13 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -4211,9 +4311,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.5" +version = "0.17.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ff8cc23a7393a397ed1d7f56e6365cba772aba9f9912ab968b03043c395d057" +checksum = "0b297dc40733f23a0e52728a58fa9489a5b7638a324932de16b41adc3ef80730" dependencies = [ "console", "instant", @@ -4966,12 +5066,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "md5" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" - [[package]] name = "memchr" version = "2.5.0" @@ -5691,7 +5785,7 @@ dependencies = [ "parking_lot", "percent-encoding", "pin-project", - "quick-xml", + "quick-xml 0.27.1", "reqwest", "serde", "serde_json", @@ -5725,9 +5819,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.55" +version = "0.10.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" +checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" dependencies = [ "bitflags 1.3.2", "cfg-if", @@ -5757,18 +5851,16 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.25.0+1.1.1t" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3173cd3626c43e3854b1b727422a276e568d9ec5fe8cec197822cf52cfb743d6" +version = "300.1.3+3.1.2" +source = "git+https://github.com/vectordotdev/openssl-src-rs.git?tag=release-300-force-engine+3.1.2#98b1172bcef15358ad7bbf4baa3a3aa59d831e81" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.90" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" +version = "0.9.91" +source = "git+https://github.com/vectordotdev/rust-openssl.git?tag=openssl-sys-v0.9.91+3.0.0#c3a8b494e0a8ab88db692c239d30c903353b56a3" dependencies = [ "cc", "libc", @@ -5962,9 +6054,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.5.6" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cbd939b234e95d72bc393d51788aec68aeeb5d51e748ca08ff3aad58cb722f7" +checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" dependencies = [ "thiserror", "ucd-trie", @@ -5972,9 +6064,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.5.6" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a81186863f3d0a27340815be8f2078dd8050b14cd71913db9fbda795e5f707d7" +checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853" dependencies = [ "pest", "pest_generator", @@ -5982,22 +6074,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.5.6" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a1ef20bf3193c15ac345acb32e26b3dc3223aff4d77ae4fc5359567683796b" +checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929" dependencies = [ "pest", "pest_meta", "proc-macro2 1.0.66", "quote 1.0.32", - "syn 1.0.109", + "syn 2.0.28", ] [[package]] name = "pest_meta" -version = "2.5.6" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e3b284b1f13a20dc5ebc90aff59a51b8d7137c221131b52a7260c08cbc1cc80" +checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48" dependencies = [ "once_cell", "pest", @@ -6063,18 +6155,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.32", @@ -6171,6 +6263,17 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.3.1" @@ -6588,6 +6691,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b9228215d82c7b61490fec1de287136b5de6f5700f6e58ea9ad61a7964ca51" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quickcheck" version = "1.0.3" @@ -7326,6 +7439,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -7469,9 +7591,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.180" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea67f183f058fe88a4e3ec6e2788e003840893b91bac4559cabedd00863b3ed" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] @@ -7506,18 +7628,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "serde-xml-rs" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3aa78ecda1ebc9ec9847d5d3aba7d618823446a049ba2491940506da6e2782" -dependencies = [ - "log", - "serde", - "thiserror", - "xml-rs", -] - [[package]] name = "serde_bytes" version = "0.11.12" @@ -7529,9 +7639,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.180" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.32", @@ -7649,17 +7759,18 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21e47d95bc83ed33b2ecf84f4187ad1ab9685d18ff28db000c99deac8ce180e3" +checksum = "1402f54f9a3b9e2efe71c1cea24e648acce55887983553eeb858cf3115acfd49" dependencies = [ "base64 0.21.2", "chrono", "hex", "indexmap 1.9.3", + "indexmap 2.0.0", "serde", "serde_json", - "serde_with_macros 3.1.0", + "serde_with_macros 3.2.0", "time", ] @@ -7677,9 +7788,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea3cee93715c2e266b9338b7544da68a9f24e227722ba482bd1c024367c77c65" +checksum = "9197f1ad0e3c173a0222d3c4404fb04c3afe87e962bcb327af73e8301fa203c7" dependencies = [ "darling 0.20.3", "proc-macro2 1.0.66", @@ -8201,16 +8312,6 @@ dependencies = [ "time", ] -[[package]] -name = "syslog_loose" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb75f176928530867b2a659e470f9c9ff71904695bab6556f7ad30f9039efd" -dependencies = [ - "chrono", - "nom", -] - [[package]] name = "syslog_loose" version = "0.19.0" @@ -9147,9 +9248,9 @@ dependencies = [ [[package]] name = "uaparser" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d705ae455d32248d299de9af5316a79ce9dc502c0b533aaeaf5f1c2fc02cc5" +checksum = "cf694e7b0434d4fad6c879e984e8fdc3a62f5533c3d421762244f9e9d03f6927" dependencies = [ "derive_more", "lazy_static", @@ -9225,6 +9326,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "unreachable" version = "1.0.0" @@ -9289,9 +9400,9 @@ checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1" [[package]] name = "utf8parse" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" @@ -9325,7 +9436,7 @@ dependencies = [ "atty", "cached", "chrono", - "clap 4.1.14", + "clap 4.3.21", "clap-verbosity-flag", "clap_complete", "confy", @@ -9400,7 +9511,7 @@ dependencies = [ "bytesize", "chrono", "cidr-utils", - "clap 4.1.14", + "clap 4.3.21", "codecs", "colored", "console-subscriber", @@ -9498,7 +9609,7 @@ dependencies = [ "serde-toml-merge", "serde_bytes", "serde_json", - "serde_with 3.1.0", + "serde_with 3.2.0", "serde_yaml 0.9.25", "sha2 0.10.7", "similar-asserts", @@ -9560,7 +9671,7 @@ dependencies = [ "anyhow", "async-trait", "chrono", - "clap 4.1.14", + "clap 4.3.21", "futures 0.3.28", "graphql_client", "indoc", @@ -9583,7 +9694,7 @@ dependencies = [ "async-trait", "bytecheck", "bytes 1.4.0", - "clap 4.1.14", + "clap 4.3.21", "crc32fast", "criterion", "crossbeam-queue", @@ -9666,7 +9777,7 @@ dependencies = [ "once_cell", "serde", "serde_json", - "serde_with 3.1.0", + "serde_with 3.2.0", "snafu", "toml 0.7.6", "tracing 0.1.37", @@ -9758,7 +9869,7 @@ dependencies = [ "security-framework", "serde", "serde_json", - "serde_with 3.1.0", + "serde_with 3.2.0", "similar-asserts", "smallvec", "snafu", @@ -9801,7 +9912,7 @@ dependencies = [ name = "vector-vrl-cli" version = "0.1.0" dependencies = [ - "clap 4.1.14", + "clap 4.3.21", "vector-vrl-functions", "vrl", ] @@ -9820,7 +9931,7 @@ dependencies = [ "ansi_term", "chrono", "chrono-tz", - "clap 4.1.14", + "clap 4.3.21", "enrichment", "glob", "prettydiff", @@ -9861,9 +9972,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee149a42aa313d18cf7a7e6b54e06df03b4a0e6e5dd42a2b8b6d5eeb98d582c1" +checksum = "bf1c43c0a6834959dd38eb762b462d156df043f3dfd86550ff62a96bd572466c" dependencies = [ "aes", "ansi_term", @@ -9875,12 +9986,14 @@ dependencies = [ "cbc", "cfb-mode", "cfg-if", + "chacha20poly1305", "charset", "chrono", "chrono-tz", "cidr-utils", - "clap 4.1.14", + "clap 4.3.21", "codespan-reporting", + "crypto_secretbox", "csv", "ctr", "data-encoding", @@ -9926,7 +10039,7 @@ dependencies = [ "sha3", "snafu", "strip-ansi-escapes", - "syslog_loose 0.18.0", + "syslog_loose", "termcolor", "thiserror", "tracing 0.1.37", @@ -10459,12 +10572,6 @@ dependencies = [ "tap", ] -[[package]] -name = "xml-rs" -version = "0.8.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52839dc911083a8ef63efa4d039d1f58b5e409f923e44c80828f206f66e5541c" - [[package]] name = "xmlparser" version = "0.13.5" diff --git a/Cargo.toml b/Cargo.toml index a0a130b577334..0669ae29d8122 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -118,7 +118,7 @@ members = [ ] [workspace.dependencies] -vrl = { version = "0.5.0", features = ["cli", "test", "test_framework", "arbitrary"] } +vrl = { version = "0.6.0", features = ["cli", "test", "test_framework", "arbitrary"] } [dependencies] vrl.workspace = true @@ -185,10 +185,10 @@ aws-smithy-http-tower = { git = "https://github.com/vectordotdev/aws-sdk-rust", aws-smithy-types = { git = "https://github.com/vectordotdev/aws-sdk-rust", rev = "3d6aefb7fcfced5fc2a7e761a87e4ddbda1ee670", default-features = false, optional = true } # Azure -azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest"], optional = true } -azure_identity = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest"], optional = true } -azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, optional = true } -azure_storage_blobs = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, optional = true } +azure_core = { version = "0.13", default-features = false, features = ["enable_reqwest"], optional = true } +azure_identity = { version = "0.13", default-features = false, features = ["enable_reqwest"], optional = true } +azure_storage = { version = "0.13", default-features = false, optional = true } +azure_storage_blobs = { version = "0.13", default-features = false, optional = true } # OpenDAL opendal = {version = "0.38", default-features = false, features = ["native-tls", "services-webhdfs"], optional = true} @@ -197,11 +197,11 @@ opendal = {version = "0.38", default-features = false, features = ["native-tls", tower = { version = "0.4.13", default-features = false, features = ["buffer", "limit", "retry", "timeout", "util", "balance", "discover"] } tower-http = { version = "0.4.3", default-features = false, features = ["decompression-gzip"]} # Serde -serde = { version = "1.0.180", default-features = false, features = ["derive"] } +serde = { version = "1.0.183", default-features = false, features = ["derive"] } serde-toml-merge = { version = "0.3.0", default-features = false } serde_bytes = { version = "0.11.12", default-features = false, features = ["std"], optional = true } serde_json = { version = "1.0.104", default-features = false, features = ["raw_value"] } -serde_with = { version = "3.1.0", default-features = false, features = ["macros", "std"] } +serde_with = { version = "3.2.0", default-features = false, features = ["macros", "std"] } serde_yaml = { version = "0.9.25", default-features = false } # Messagepack @@ -243,7 +243,7 @@ greptimedb-client = { git = "https://github.com/GreptimeTeam/greptimedb-client-r arc-swap = { version = "1.6", default-features = false, optional = true } async-compression = { version = "0.4.1", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } apache-avro = { version = "0.15.0", default-features = false, optional = true } -axum = { version = "0.6.19", default-features = false } +axum = { version = "0.6.20", default-features = false } base64 = { version = "0.21.2", default-features = false, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } bollard = { version = "0.14.0", default-features = false, features = ["ssl", "chrono"], optional = true } @@ -251,7 +251,7 @@ bytes = { version = "1.4.0", default-features = false, features = ["serde"] } bytesize = { version = "1.2.0", default-features = false } chrono = { version = "0.4.26", default-features = false, features = ["serde"] } cidr-utils = { version = "0.5.10", default-features = false } -clap = { version = "4.1.14", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } +clap = { version = "4.3.21", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } colored = { version = "2.0.4", default-features = false } csv = { version = "1.2", default-features = false } derivative = { version = "2.2.0", default-features = false } @@ -292,12 +292,12 @@ nkeys = { version = "0.3.1", default-features = false, optional = true } nom = { version = "7.1.3", default-features = false, optional = true } notify = { version = "6.0.1", default-features = false, features = ["macos_fsevent"] } once_cell = { version = "1.18", default-features = false } -openssl = { version = "0.10.55", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.56", default-features = false, features = ["vendored"] } openssl-probe = { version = "0.1.5", default-features = false } ordered-float = { version = "3.7.0", default-features = false } paste = "1.0.14" percent-encoding = { version = "2.3.0", default-features = false } -pin-project = { version = "1.1.2", default-features = false } +pin-project = { version = "1.1.3", default-features = false } postgres-openssl = { version = "0.5.0", default-features = false, features = ["runtime"], optional = true } pulsar = { version = "6.0.1", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true } rand = { version = "0.8.5", default-features = false, features = ["small_rng"] } @@ -345,15 +345,15 @@ nix = { version = "0.26.2", default-features = false, features = ["socket", "sig [build-dependencies] prost-build = { version = "0.11", default-features = false, optional = true } tonic-build = { version = "0.9", default-features = false, features = ["transport", "prost"], optional = true } -openssl-src = { version = "111", default-features = false, features = ["force-engine"] } +openssl-src = { version = "300", default-features = false, features = ["force-engine", "legacy"] } [dev-dependencies] approx = "0.5.1" assert_cmd = { version = "2.0.12", default-features = false } -azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest", "azurite_workaround"] } -azure_identity = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest"] } -azure_storage_blobs = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["azurite_workaround"] } -azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["azurite_workaround"] } +azure_core = { version = "0.13", default-features = false, features = ["enable_reqwest", "azurite_workaround"] } +azure_identity = { version = "0.13", default-features = false, features = ["enable_reqwest"] } +azure_storage_blobs = { version = "0.13", default-features = false, features = ["azurite_workaround"] } +azure_storage = { version = "0.13", default-features = false } base64 = "0.21.2" criterion = { version = "0.5.1", features = ["html_reports", "async_tokio"] } itertools = { version = "0.11.0", default-features = false, features = ["use_alloc"] } @@ -381,6 +381,11 @@ nix = { git = "https://github.com/vectordotdev/nix.git", branch = "memfd/gnu/mus # The `heim` crates depend on `ntapi` 0.3.7 on Windows, but that version has an # unaligned access bug fixed in the following revision. ntapi = { git = "https://github.com/MSxDOS/ntapi.git", rev = "24fc1e47677fc9f6e38e5f154e6011dc9b270da6" } +# The current `openssl-sys` crate will vendor the OpenSSL sources via +# `openssl-src` at version 1.1.1*, but we want version 3.1.*. Bring in forked +# version of that crate with the appropriate dependency patched in. +openssl-sys = { git = "https://github.com/vectordotdev/rust-openssl.git", tag = "openssl-sys-v0.9.91+3.0.0" } +openssl-src = { git = "https://github.com/vectordotdev/openssl-src-rs.git", tag = "release-300-force-engine+3.1.2"} [features] # Default features for *-unknown-linux-gnu and *-apple-darwin diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 7489d175b1fca..b6d5a876eae7e 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -4,6 +4,7 @@ RustyXML,https://github.com/Florob/RustyXML,MIT OR Apache-2.0,Florian Zeitz adler32,https://github.com/remram44/adler32-rs,Zlib,Remi Rampin +aead,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers aes,https://github.com/RustCrypto/block-ciphers,MIT OR Apache-2.0,RustCrypto Developers ahash,https://github.com/tkaitchuck/ahash,MIT OR Apache-2.0,Tom Kaitchuck aho-corasick,https://github.com/BurntSushi/aho-corasick,Unlicense OR MIT,Andrew Gallant @@ -11,6 +12,11 @@ amq-protocol,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine P android-tzdata,https://github.com/RumovZ/android-tzdata,MIT OR Apache-2.0,RumovZ android_system_properties,https://github.com/nical/android_system_properties,MIT OR Apache-2.0,Nicolas Silva ansi_term,https://github.com/ogham/rust-ansi-term,MIT,"ogham@bsago.me, Ryan Scheel (Havvy) , Josh Triplett " +anstream,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstream Authors +anstyle,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle Authors +anstyle-parse,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-parse Authors +anstyle-query,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-query Authors +anstyle-wincon,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-wincon Authors anyhow,https://github.com/dtolnay/anyhow,MIT OR Apache-2.0,David Tolnay anymap,https://github.com/chris-morgan/anymap,BlueOak-1.0.0 OR MIT OR Apache-2.0,Chris Morgan apache-avro,https://github.com/apache/avro,Apache-2.0,Apache Avro team @@ -108,6 +114,8 @@ cbc,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Devel cesu8,https://github.com/emk/cesu8-rs,Apache-2.0 OR MIT,Eric Kidd cfb-mode,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers cfg-if,https://github.com/alexcrichton/cfg-if,MIT OR Apache-2.0,Alex Crichton +chacha20,https://github.com/RustCrypto/stream-ciphers,Apache-2.0 OR MIT,RustCrypto Developers +chacha20poly1305,https://github.com/RustCrypto/AEADs/tree/master/chacha20poly1305,Apache-2.0 OR MIT,RustCrypto Developers charset,https://github.com/hsivonen/charset,MIT OR Apache-2.0,Henri Sivonen chrono,https://github.com/chronotope/chrono,MIT OR Apache-2.0,The chrono Authors chrono-tz,https://github.com/chronotope/chrono-tz,MIT OR Apache-2.0,The chrono-tz Authors @@ -120,6 +128,7 @@ clap_derive,https://github.com/clap-rs/clap/tree/master/clap_derive,MIT OR Apach clap_lex,https://github.com/clap-rs/clap/tree/master/clap_lex,MIT OR Apache-2.0,The clap_lex Authors clipboard-win,https://github.com/DoumanAsh/clipboard-win,BSL-1.0,Douman codespan-reporting,https://github.com/brendanzab/codespan,Apache-2.0,Brendan Zabarauskas +colorchoice,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The colorchoice Authors colored,https://github.com/mackwic/colored,MPL-2.0,Thomas Wickham combine,https://github.com/Marwes/combine,MIT,Markus Westerlind concurrent-queue,https://github.com/smol-rs/concurrent-queue,Apache-2.0 OR MIT,Stjepan Glavina @@ -141,6 +150,7 @@ crossbeam-utils,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossterm,https://github.com/crossterm-rs/crossterm,MIT,T. Post crossterm_winapi,https://github.com/crossterm-rs/crossterm-winapi,MIT,T. Post crypto-common,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers +crypto_secretbox,https://github.com/RustCrypto/nacl-compat/tree/master/crypto_secretbox,Apache-2.0 OR MIT,RustCrypto Developers csv,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant ctor,https://github.com/mmastrac/rust-ctor,Apache-2.0 OR MIT,Matt Mastracci ctr,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers @@ -311,7 +321,6 @@ matches,https://github.com/SimonSapin/rust-std-candidates,MIT,Simon Sapin maxminddb,https://github.com/oschwald/maxminddb-rust,ISC,Gregory J. Oschwald md-5,https://github.com/RustCrypto/hashes,MIT OR Apache-2.0,RustCrypto Developers -md5,https://github.com/stainless-steel/md5,Apache-2.0 OR MIT,"Ivan Ukhov , Kamal Ahmad , Konstantin Stepanov , Lukas Kalbertodt , Nathan Musoke , Scott Mabin , Tony Arcieri , Wim de With , Yosef Dinerstein " memchr,https://github.com/BurntSushi/memchr,Unlicense OR MIT,"Andrew Gallant , bluss" memmap2,https://github.com/RazrFalcon/memmap2-rs,MIT OR Apache-2.0,"Dan Burkert , Yevhenii Reizner " memoffset,https://github.com/Gilnaa/memoffset,MIT,Gilad Naaman @@ -383,6 +392,7 @@ pinky-swear,https://github.com/amqp-rs/pinky-swear,BSD-2-Clause,Marc-Antoine Per pkcs8,https://github.com/RustCrypto/formats/tree/master/pkcs8,Apache-2.0 OR MIT,RustCrypto Developers platforms,https://github.com/RustSec/platforms-crate,Apache-2.0 OR MIT,Tony Arcieri polling,https://github.com/smol-rs/polling,Apache-2.0 OR MIT,Stjepan Glavina +poly1305,https://github.com/RustCrypto/universal-hashes,Apache-2.0 OR MIT,RustCrypto Developers portable-atomic,https://github.com/taiki-e/portable-atomic,Apache-2.0 OR MIT,The portable-atomic Authors postgres-openssl,https://github.com/sfackler/rust-postgres,MIT OR Apache-2.0,Steven Fackler postgres-protocol,https://github.com/sfackler/rust-postgres,MIT OR Apache-2.0,Steven Fackler @@ -455,6 +465,7 @@ rustversion,https://github.com/dtolnay/rustversion,MIT OR Apache-2.0,David Tolna rusty-fork,https://github.com/altsysrq/rusty-fork,MIT OR Apache-2.0,Jason Lingle rustyline,https://github.com/kkawakam/rustyline,MIT,Katsu Kawakami ryu,https://github.com/dtolnay/ryu,Apache-2.0 OR BSL-1.0,David Tolnay +salsa20,https://github.com/RustCrypto/stream-ciphers,MIT OR Apache-2.0,RustCrypto Developers same-file,https://github.com/BurntSushi/same-file,Unlicense OR MIT,Andrew Gallant sasl2-sys,https://github.com/MaterializeInc/rust-sasl,Apache-2.0,"Materialize, Inc." scan_fmt,https://github.com/wlentz/scan_fmt,MIT,wlentz @@ -471,7 +482,6 @@ semver-parser,https://github.com/steveklabnik/semver-parser,MIT OR Apache-2.0,St serde,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde-toml-merge,https://github.com/jdrouet/serde-toml-merge,MIT,Jeremie Drouet serde-value,https://github.com/arcnmx/serde-value,MIT,arcnmx -serde-xml-rs,https://github.com/RReverser/serde-xml-rs,MIT,Ingvar Stepanyan serde_bytes,https://github.com/serde-rs/bytes,MIT OR Apache-2.0,David Tolnay serde_derive,https://github.com/serde-rs/serde,MIT OR Apache-2.0,David Tolnay serde_json,https://github.com/serde-rs/json,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " @@ -579,6 +589,7 @@ unicode-normalization,https://github.com/unicode-rs/unicode-normalization,MIT OR unicode-segmentation,https://github.com/unicode-rs/unicode-segmentation,MIT OR Apache-2.0,"kwantam , Manish Goregaokar " unicode-width,https://github.com/unicode-rs/unicode-width,MIT OR Apache-2.0,"kwantam , Manish Goregaokar " unicode-xid,https://github.com/unicode-rs/unicode-xid,MIT OR Apache-2.0,"erick.tryzelaar , kwantam , Manish Goregaokar " +universal-hash,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers unreachable,https://github.com/reem/rust-unreachable,MIT OR Apache-2.0,Jonathan Reem unsafe-libyaml,https://github.com/dtolnay/unsafe-libyaml,MIT,David Tolnay untrusted,https://github.com/briansmith/untrusted,ISC,Brian Smith @@ -587,7 +598,6 @@ url,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers urlencoding,https://github.com/kornelski/rust_urlencoding,MIT,"Kornel , Bertram Truong " utf-8,https://github.com/SimonSapin/rust-utf8,MIT OR Apache-2.0,Simon Sapin utf8-width,https://github.com/magiclen/utf8-width,MIT,Magic Len -utf8parse,https://github.com/jwilm/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix, Christopher Armstrong, Dylan DPC, Hunar Roop Kahlon" valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors vec_map,https://github.com/contain-rs/vec-map,MIT OR Apache-2.0,"Alex Crichton , Jorge Aparicio , Alexis Beingessner , Brian Anderson <>, tbu- <>, Manish Goregaokar <>, Aaron Turon , Adolfo Ochagavía <>, Niko Matsakis <>, Steven Fackler <>, Chase Southwood , Eduard Burtescu <>, Florian Wilkens <>, Félix Raimundo <>, Tibor Benke <>, Markus Siemens , Josh Branchaud , Huon Wilson , Corey Farwell , Aaron Liblong <>, Nick Cameron , Patrick Walton , Felix S Klock II <>, Andrew Paseltiner , Sean McArthur , Vadim Petrochenkov <>" @@ -631,7 +641,6 @@ winnow,https://github.com/winnow-rs/winnow,MIT,The winnow Authors winreg,https://github.com/gentoo90/winreg-rs,MIT,Igor Shaula woothee,https://github.com/woothee/woothee-rust,Apache-2.0,hhatto wyz,https://github.com/myrrlyn/wyz,MIT,myrrlyn -xml-rs,https://github.com/kornelski/xml-rs,MIT,Vladimir Matveev xmlparser,https://github.com/RazrFalcon/xmlparser,MIT OR Apache-2.0,Evgeniy Reizner yaml-rust,https://github.com/chyh1990/yaml-rust,MIT OR Apache-2.0,Yuheng Chen yansi,https://github.com/SergioBenitez/yansi,MIT OR Apache-2.0,Sergio Benitez diff --git a/benches/event.rs b/benches/event.rs index c28d0756a1a37..0f8322da087fd 100644 --- a/benches/event.rs +++ b/benches/event.rs @@ -1,6 +1,7 @@ use bytes::Bytes; use criterion::{criterion_group, BatchSize, Criterion}; use vector::event::LogEvent; +use vrl::event_path; fn benchmark_event_iterate(c: &mut Criterion) { let mut group = c.benchmark_group("event/iterate"); @@ -9,9 +10,9 @@ fn benchmark_event_iterate(c: &mut Criterion) { b.iter_batched_ref( || { let mut log = LogEvent::default(); - log.insert("key1", Bytes::from("value1")); - log.insert("key2", Bytes::from("value2")); - log.insert("key3", Bytes::from("value3")); + log.insert(event_path!("key1"), Bytes::from("value1")); + log.insert(event_path!("key2"), Bytes::from("value2")); + log.insert(event_path!("key3"), Bytes::from("value3")); log }, |e| e.all_fields().unwrap().count(), @@ -23,9 +24,15 @@ fn benchmark_event_iterate(c: &mut Criterion) { b.iter_batched_ref( || { let mut log = LogEvent::default(); - log.insert("key1.nested1.nested2", Bytes::from("value1")); - log.insert("key1.nested1.nested3", Bytes::from("value4")); - log.insert("key3", Bytes::from("value3")); + log.insert( + event_path!("key1", "nested1", "nested2"), + Bytes::from("value1"), + ); + log.insert( + event_path!("key1", "nested1", "nested3"), + Bytes::from("value4"), + ); + log.insert(event_path!("key3"), Bytes::from("value3")); log }, |e| e.all_fields().unwrap().count(), @@ -37,8 +44,8 @@ fn benchmark_event_iterate(c: &mut Criterion) { b.iter_batched_ref( || { let mut log = LogEvent::default(); - log.insert("key1.nested1[0]", Bytes::from("value1")); - log.insert("key1.nested1[1]", Bytes::from("value2")); + log.insert(event_path!("key1", "nested1", 0), Bytes::from("value1")); + log.insert(event_path!("key1", "nested1", 1), Bytes::from("value2")); log }, |e| e.all_fields().unwrap().count(), @@ -53,25 +60,31 @@ fn benchmark_event_create(c: &mut Criterion) { group.bench_function("single-level", |b| { b.iter(|| { let mut log = LogEvent::default(); - log.insert("key1", Bytes::from("value1")); - log.insert("key2", Bytes::from("value2")); - log.insert("key3", Bytes::from("value3")); + log.insert(event_path!("key1"), Bytes::from("value1")); + log.insert(event_path!("key2"), Bytes::from("value2")); + log.insert(event_path!("key3"), Bytes::from("value3")); }) }); group.bench_function("nested-keys", |b| { b.iter(|| { let mut log = LogEvent::default(); - log.insert("key1.nested1.nested2", Bytes::from("value1")); - log.insert("key1.nested1.nested3", Bytes::from("value4")); - log.insert("key3", Bytes::from("value3")); + log.insert( + event_path!("key1", "nested1", "nested2"), + Bytes::from("value1"), + ); + log.insert( + event_path!("key1", "nested1", "nested3"), + Bytes::from("value4"), + ); + log.insert(event_path!("key3"), Bytes::from("value3")); }) }); group.bench_function("array", |b| { b.iter(|| { let mut log = LogEvent::default(); - log.insert("key1.nested1[0]", Bytes::from("value1")); - log.insert("key1.nested1[1]", Bytes::from("value2")); + log.insert(event_path!("key1", "nested1", 0), Bytes::from("value1")); + log.insert(event_path!("key1", "nested1", 1), Bytes::from("value2")); }) }); } diff --git a/benches/lua.rs b/benches/lua.rs index 3eda8a9572d98..7aabba2996890 100644 --- a/benches/lua.rs +++ b/benches/lua.rs @@ -9,6 +9,7 @@ use vector::{ test_util::collect_ready, transforms::{self, OutputBuffer, Transform}, }; +use vrl::event_path; fn bench_add_fields(c: &mut Criterion) { let event = Event::from(LogEvent::default()); @@ -87,7 +88,7 @@ fn bench_field_filter(c: &mut Criterion) { let events = (0..num_events) .map(|i| { let mut event = LogEvent::default(); - event.insert("the_field", (i % 10).to_string()); + event.insert(event_path!("the_field"), (i % 10).to_string()); Event::from(event) }) .collect::>(); diff --git a/benches/remap.rs b/benches/remap.rs index 3523c541b09ed..5f703601be9a3 100644 --- a/benches/remap.rs +++ b/benches/remap.rs @@ -12,6 +12,7 @@ use vector::{ }, }; use vector_common::TimeZone; +use vrl::event_path; use vrl::prelude::*; criterion_group!( @@ -35,9 +36,18 @@ fn benchmark_remap(c: &mut Criterion) { let result = outputs.take_primary(); let output_1 = result.first().unwrap().as_log(); - debug_assert_eq!(output_1.get("foo").unwrap().to_string_lossy(), "bar"); - debug_assert_eq!(output_1.get("bar").unwrap().to_string_lossy(), "baz"); - debug_assert_eq!(output_1.get("copy").unwrap().to_string_lossy(), "buz"); + debug_assert_eq!( + output_1.get(event_path!("foo")).unwrap().to_string_lossy(), + "bar" + ); + debug_assert_eq!( + output_1.get(event_path!("bar")).unwrap().to_string_lossy(), + "baz" + ); + debug_assert_eq!( + output_1.get(event_path!("copy")).unwrap().to_string_lossy(), + "buz" + ); result }; @@ -67,7 +77,9 @@ fn benchmark_remap(c: &mut Criterion) { let event = { let mut event = Event::Log(LogEvent::from("augment me")); - event.as_mut_log().insert("copy_from", "buz".to_owned()); + event + .as_mut_log() + .insert(event_path!("copy_from"), "buz".to_owned()); event }; @@ -88,11 +100,11 @@ fn benchmark_remap(c: &mut Criterion) { let output_1 = result.first().unwrap().as_log(); debug_assert_eq!( - output_1.get("foo").unwrap().to_string_lossy(), + output_1.get(event_path!("foo")).unwrap().to_string_lossy(), r#"{"key": "value"}"# ); debug_assert_eq!( - output_1.get("bar").unwrap().to_string_lossy(), + output_1.get(event_path!("bar")).unwrap().to_string_lossy(), r#"{"key":"value"}"# ); @@ -141,10 +153,16 @@ fn benchmark_remap(c: &mut Criterion) { let result = outputs.take_primary(); let output_1 = result.first().unwrap().as_log(); - debug_assert_eq!(output_1.get("number").unwrap(), &Value::Integer(1234)); - debug_assert_eq!(output_1.get("bool").unwrap(), &Value::Boolean(true)); debug_assert_eq!( - output_1.get("timestamp").unwrap(), + output_1.get(event_path!("number")).unwrap(), + &Value::Integer(1234) + ); + debug_assert_eq!( + output_1.get(event_path!("bool")).unwrap(), + &Value::Boolean(true) + ); + debug_assert_eq!( + output_1.get(event_path!("timestamp")).unwrap(), &Value::Timestamp(timestamp), ); @@ -176,7 +194,7 @@ fn benchmark_remap(c: &mut Criterion) { ("bool", "yes"), ("timestamp", "19/06/2019:17:20:49 -0400"), ] { - event.as_mut_log().insert(key, value.to_owned()); + event.as_mut_log().insert(event_path!(key), value.to_owned()); } let timestamp = diff --git a/docs/tutorials/sinks/1_basic_sink.md b/docs/tutorials/sinks/1_basic_sink.md index 40553b8820992..fc188d5ac8e4d 100644 --- a/docs/tutorials/sinks/1_basic_sink.md +++ b/docs/tutorials/sinks/1_basic_sink.md @@ -33,7 +33,7 @@ is deserialized to the fields in this struct so the user can customise the sink's behaviour. ```rust -#[configurable_component(sink("basic", "Basic sink."))] +#[configurable_component(sink("basic"))] #[derive(Clone, Debug)] /// A basic sink that dumps its output to stdout. pub struct BasicConfig { diff --git a/docs/tutorials/sinks/2_http_sink.md b/docs/tutorials/sinks/2_http_sink.md index e92cb94bcac76..179ff4192c8cc 100644 --- a/docs/tutorials/sinks/2_http_sink.md +++ b/docs/tutorials/sinks/2_http_sink.md @@ -16,6 +16,7 @@ use crate::{ http::HttpClient, internal_events::SinkRequestBuildError, }; +use vector_core::config::telemetry; use bytes::Bytes; ``` @@ -81,12 +82,12 @@ struct BasicEncoder; The Encoder must implement the [`Encoder`][encoder] trait: ```rust -impl Encoder for BasicEncoder { +impl encoding::Encoder for BasicEncoder { fn encode_input( &self, input: Event, writer: &mut dyn std::io::Write, - ) -> std::io::Result { + ) -> std::io::Result<(usize, GroupedCountByteSize)> { } } ``` @@ -98,16 +99,25 @@ sending batches of events, or they may send a completely different type if each event is processed in some way prior to encoding. [`encode_input`][encoder_encode_input] serializes the event to a String and -writes these bytes: +writes these bytes. The function also creates a [`GroupedCountByteSize`] +[grouped_count_byte_size] object. This object tracks the size of the event +that is sent by the sink, optionally grouped by the source and service that +originated the event if Vector has been configured to do so. It is necessary to +calculate the sizes in this function since the encode function sometimes drops +fields from the event prior to encoding. We need the size to be calculated after +these fields have been dropped. ```rust fn encode_input( &self, input: Event, writer: &mut dyn std::io::Write, - ) -> std::io::Result { + ) -> std::io::Result<(usize, GroupedCountByteSize)> { + let mut byte_size = telemetry().create_request_count_byte_size(); + byte_size.add_event(&input, input.estimated_json_encoded_size_of()); + let event = serde_json::to_string(&input).unwrap(); - write_all(writer, 1, event.as_bytes()).map(|()| event.len()) + write_all(writer, 1, event.as_bytes()).map(|()| (event.len(), byte_size)) } ``` @@ -152,8 +162,12 @@ We need to implement a number of traits for the request to access these fields: ```rust impl MetaDescriptive for BasicRequest { - fn get_metadata(&self) -> RequestMetadata { - self.metadata + fn get_metadata(&self) -> &RequestMetadata { + &self.metadata + } + + fn metadata_mut(&mut self) -> &mut RequestMetadata { + &mut self.metadata } } @@ -249,7 +263,7 @@ when sending the event to an `amqp` server. mut input: Event, ) -> (Self::Metadata, RequestMetadataBuilder, Self::Events) { let finalizers = input.take_finalizers(); - let metadata_builder = RequestMetadataBuilder::from_events(&input); + let metadata_builder = RequestMetadataBuilder::from_event(&input); (finalizers, metadata_builder, input) } ``` @@ -338,7 +352,12 @@ that will be invoked to send the actual data. match client.call(req).await { Ok(response) => { if response.status().is_success() { - Ok(BasicResponse { byte_size }) + Ok(BasicResponse { + byte_size, + json_size: request + .metadata + .into_events_estimated_json_encoded_byte_size(), + }) } else { Err("received error response") } @@ -359,6 +378,7 @@ The return from our service must be an object that implements the ```rust struct BasicResponse { byte_size: usize, + json_size: GroupedCountByteSize, } impl DriverResponse for BasicResponse { @@ -366,11 +386,13 @@ impl DriverResponse for BasicResponse { EventStatus::Delivered } - fn events_sent(&self) -> RequestCountByteSize { - // (events count, byte size) - CountByteSize(1, self.byte_size).into() + fn events_sent(&self) -> &GroupedCountByteSize { + &self.json_size } -} + + fn bytes_sent(&self) -> Option { + Some(self.byte_size) + }} ``` Vector calls the methods in this trait to determine if the event was delivered successfully. @@ -492,3 +514,4 @@ BODY: [sinkbuilder_ext_into_driver]: https://rust-doc.vector.dev/vector/sinks/util/builder/trait.sinkbuilderext#method.into_driver [stream_filter_map]: https://docs.rs/futures/latest/futures/stream/trait.StreamExt.html#method.filter_map [driver]: https://rust-doc.vector.dev/vector_core/stream/struct.driver +[grouped_count_byte_size]: https://rust-doc.vector.dev/vector_common/request_metadata/enum.groupedcountbytesize diff --git a/lib/codecs/src/decoding/format/gelf.rs b/lib/codecs/src/decoding/format/gelf.rs index 7f9f17353b42a..88d7989c7e210 100644 --- a/lib/codecs/src/decoding/format/gelf.rs +++ b/lib/codecs/src/decoding/format/gelf.rs @@ -17,6 +17,7 @@ use vrl::value::kind::Collection; use vrl::value::{Kind, Value}; use super::{default_lossy, Deserializer}; +use crate::gelf::GELF_TARGET_PATHS; use crate::{gelf_fields::*, VALID_FIELD_REGEX}; /// On GELF decoding behavior: @@ -123,11 +124,11 @@ impl GelfDeserializer { .into()); } - log.insert(VERSION, parsed.version.to_string()); - log.insert(HOST, parsed.host.to_string()); + log.insert(&GELF_TARGET_PATHS.version, parsed.version.to_string()); + log.insert(&GELF_TARGET_PATHS.host, parsed.host.to_string()); if let Some(full_message) = &parsed.full_message { - log.insert(FULL_MESSAGE, full_message.to_string()); + log.insert(&GELF_TARGET_PATHS.full_message, full_message.to_string()); } if let Some(timestamp_key) = log_schema().timestamp_key_target_path() { @@ -145,19 +146,19 @@ impl GelfDeserializer { } if let Some(level) = parsed.level { - log.insert(LEVEL, level); + log.insert(&GELF_TARGET_PATHS.level, level); } if let Some(facility) = &parsed.facility { - log.insert(FACILITY, facility.to_string()); + log.insert(&GELF_TARGET_PATHS.facility, facility.to_string()); } if let Some(line) = parsed.line { log.insert( - LINE, + &GELF_TARGET_PATHS.line, Value::Float(ordered_float::NotNan::new(line).expect("JSON doesn't allow NaNs")), ); } if let Some(file) = &parsed.file { - log.insert(FILE, file.to_string()); + log.insert(&GELF_TARGET_PATHS.file, file.to_string()); } if let Some(add) = &parsed.additional_fields { diff --git a/lib/codecs/src/encoding/format/gelf.rs b/lib/codecs/src/encoding/format/gelf.rs index 7cce8cafc0d47..d35bad66decce 100644 --- a/lib/codecs/src/encoding/format/gelf.rs +++ b/lib/codecs/src/encoding/format/gelf.rs @@ -1,3 +1,4 @@ +use crate::gelf::GELF_TARGET_PATHS; use crate::{gelf_fields::*, VALID_FIELD_REGEX}; use bytes::{BufMut, BytesMut}; use lookup::event_path; @@ -12,7 +13,6 @@ use vector_core::{ event::Value, schema, }; -use vrl::path::PathPrefix; /// On GELF encoding behavior: /// Graylog has a relaxed parsing. They are much more lenient than the spec would @@ -131,20 +131,18 @@ fn coerce_required_fields(mut log: LogEvent) -> vector_common::Result } // add the VERSION if it does not exist - if !log.contains(VERSION) { - log.insert(VERSION, GELF_VERSION); + if !log.contains(&GELF_TARGET_PATHS.version) { + log.insert(&GELF_TARGET_PATHS.version, GELF_VERSION); } - if !log.contains(HOST) { + if !log.contains(&GELF_TARGET_PATHS.host) { err_missing_field(HOST)?; } - if !log.contains(SHORT_MESSAGE) { - if let Some(message_key) = log_schema().message_key() { - // rename the log_schema().message_key() to SHORT_MESSAGE - let target_path = (PathPrefix::Event, message_key); - if log.contains(target_path) { - log.rename_key(target_path, SHORT_MESSAGE); + if !log.contains(&GELF_TARGET_PATHS.short_message) { + if let Some(message_key) = log_schema().message_key_target_path() { + if log.contains(message_key) { + log.rename_key(message_key, &GELF_TARGET_PATHS.short_message); } else { err_missing_field(SHORT_MESSAGE)?; } diff --git a/lib/codecs/src/gelf.rs b/lib/codecs/src/gelf.rs index d059958f003a1..b78df6cb0e4d9 100644 --- a/lib/codecs/src/gelf.rs +++ b/lib/codecs/src/gelf.rs @@ -2,10 +2,11 @@ use once_cell::sync::Lazy; use regex::Regex; +use vrl::owned_value_path; +use vrl::path::OwnedTargetPath; /// GELF Message fields. Definitions from . pub mod gelf_fields { - /// (not a field) The latest version of the GELF specification. pub const GELF_VERSION: &str = "1.1"; @@ -40,6 +41,30 @@ pub mod gelf_fields { // < Every field with an underscore (_) prefix will be treated as an additional field. > } +/// GELF owned target paths. +pub(crate) struct GelfTargetPaths { + pub version: OwnedTargetPath, + pub host: OwnedTargetPath, + pub full_message: OwnedTargetPath, + pub level: OwnedTargetPath, + pub facility: OwnedTargetPath, + pub line: OwnedTargetPath, + pub file: OwnedTargetPath, + pub short_message: OwnedTargetPath, +} + +/// Lazily initialized singleton. +pub(crate) static GELF_TARGET_PATHS: Lazy = Lazy::new(|| GelfTargetPaths { + version: OwnedTargetPath::event(owned_value_path!(gelf_fields::VERSION)), + host: OwnedTargetPath::event(owned_value_path!(gelf_fields::HOST)), + full_message: OwnedTargetPath::event(owned_value_path!(gelf_fields::FULL_MESSAGE)), + level: OwnedTargetPath::event(owned_value_path!(gelf_fields::LEVEL)), + facility: OwnedTargetPath::event(owned_value_path!(gelf_fields::FACILITY)), + line: OwnedTargetPath::event(owned_value_path!(gelf_fields::LINE)), + file: OwnedTargetPath::event(owned_value_path!(gelf_fields::FILE)), + short_message: OwnedTargetPath::event(owned_value_path!(gelf_fields::SHORT_MESSAGE)), +}); + /// Regex for matching valid field names. Must contain only word chars, periods and dashes. /// Additional field names must also be prefixed with an `_` , however that is intentionally /// omitted from this regex to be checked separately to create a specific error message. diff --git a/lib/codecs/tests/data/native_encoding/schema.cue b/lib/codecs/tests/data/native_encoding/schema.cue index 34c1bec6adc3d..91ddb069b885b 100644 --- a/lib/codecs/tests/data/native_encoding/schema.cue +++ b/lib/codecs/tests/data/native_encoding/schema.cue @@ -5,8 +5,8 @@ #Trace: {...} #Metric: { - name: string - namespace?: string + name: string + namespace?: string tags?: {[string]: #TagValueSet} timestamp?: #Timestamp interval_ms?: int @@ -40,11 +40,11 @@ sum: number avg: number } - } +} } -#TagValueSet: { #TagValue | [...#TagValue] } +#TagValueSet: {#TagValue | [...#TagValue]} -#TagValue: { string | null } +#TagValue: {string | null} #Timestamp: =~"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(.\\d+)?Z" diff --git a/lib/enrichment/src/find_enrichment_table_records.rs b/lib/enrichment/src/find_enrichment_table_records.rs index 3ade47b7c55a6..7086709cd208a 100644 --- a/lib/enrichment/src/find_enrichment_table_records.rs +++ b/lib/enrichment/src/find_enrichment_table_records.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use vrl::prelude::*; +use crate::vrl_util::is_case_sensitive; use crate::{ vrl_util::{self, add_index, evaluate_condition}, Case, Condition, IndexHandle, TableRegistry, TableSearch, @@ -87,7 +88,7 @@ impl Function for FindEnrichmentTableRecords { fn compile( &self, - _state: &TypeState, + state: &TypeState, ctx: &mut FunctionCompileContext, arguments: ArgumentList, ) -> Compiled { @@ -102,7 +103,7 @@ impl Function for FindEnrichmentTableRecords { .collect::>(); let table = arguments - .required_enum("table", &tables)? + .required_enum("table", &tables, state)? .try_bytes_utf8_lossy() .expect("table is not valid utf8") .into_owned(); @@ -110,21 +111,7 @@ impl Function for FindEnrichmentTableRecords { let select = arguments.optional("select"); - let case_sensitive = arguments - .optional_literal("case_sensitive")? - .and_then(|literal| literal.resolve_constant()) - .map(|value| value.try_boolean()) - .transpose() - .expect("case_sensitive should be boolean") // This will have been caught by the type checker. - .map(|case_sensitive| { - if case_sensitive { - Case::Sensitive - } else { - Case::Insensitive - } - }) - .unwrap_or(Case::Sensitive); - + let case_sensitive = is_case_sensitive(&arguments, state)?; let index = Some( add_index(registry, &table, case_sensitive, &condition) .map_err(|err| Box::new(err) as Box<_>)?, diff --git a/lib/enrichment/src/get_enrichment_table_record.rs b/lib/enrichment/src/get_enrichment_table_record.rs index 3678d85e4fea0..a028894d89373 100644 --- a/lib/enrichment/src/get_enrichment_table_record.rs +++ b/lib/enrichment/src/get_enrichment_table_record.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use vrl::prelude::*; +use crate::vrl_util::is_case_sensitive; use crate::{ vrl_util::{self, add_index, evaluate_condition}, Case, Condition, IndexHandle, TableRegistry, TableSearch, @@ -79,7 +80,7 @@ impl Function for GetEnrichmentTableRecord { fn compile( &self, - _state: &TypeState, + state: &TypeState, ctx: &mut FunctionCompileContext, arguments: ArgumentList, ) -> Compiled { @@ -94,7 +95,7 @@ impl Function for GetEnrichmentTableRecord { .collect::>(); let table = arguments - .required_enum("table", &tables)? + .required_enum("table", &tables, state)? .try_bytes_utf8_lossy() .expect("table is not valid utf8") .into_owned(); @@ -102,21 +103,7 @@ impl Function for GetEnrichmentTableRecord { let select = arguments.optional("select"); - let case_sensitive = arguments - .optional_literal("case_sensitive")? - .and_then(|literal| literal.resolve_constant()) - .map(|value| value.try_boolean()) - .transpose() - .expect("case_sensitive should be boolean") // This will have been caught by the type checker. - .map(|case_sensitive| { - if case_sensitive { - Case::Sensitive - } else { - Case::Insensitive - } - }) - .unwrap_or(Case::Sensitive); - + let case_sensitive = is_case_sensitive(&arguments, state)?; let index = Some( add_index(registry, &table, case_sensitive, &condition) .map_err(|err| Box::new(err) as Box<_>)?, diff --git a/lib/enrichment/src/vrl_util.rs b/lib/enrichment/src/vrl_util.rs index f6aaa4dac02ce..ccdebd3cdfbb1 100644 --- a/lib/enrichment/src/vrl_util.rs +++ b/lib/enrichment/src/vrl_util.rs @@ -80,6 +80,26 @@ pub(crate) fn add_index( Ok(index) } +pub(crate) fn is_case_sensitive( + arguments: &ArgumentList, + state: &TypeState, +) -> Result { + Ok(arguments + .optional_literal("case_sensitive", state)? + .map(|value| { + let case_sensitive = value + .as_boolean() + .expect("case_sensitive should be boolean"); // This will have been caught by the type checker. + + if case_sensitive { + Case::Sensitive + } else { + Case::Insensitive + } + }) + .unwrap_or(Case::Sensitive)) +} + #[cfg(test)] mod tests { use std::sync::{Arc, Mutex}; diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 107904f46355e..b966793f97829 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -9,7 +9,7 @@ license = "MPL-2.0" [dependencies] # Serde -serde = { version = "1.0.180", default-features = false, features = ["derive"] } +serde = { version = "1.0.183", default-features = false, features = ["derive"] } serde_json = { version = "1.0.104", default-features = false, features = ["raw_value"] } # Error handling @@ -30,7 +30,7 @@ tokio-tungstenite = { version = "0.20.0", default-features = false, features = [ # External libs chrono = { version = "0.4.6", default-features = false, features = ["serde"] } -clap = { version = "4.1.14", default-features = false, features = ["derive"] } +clap = { version = "4.3.21", default-features = false, features = ["derive"] } url = { version = "2.4.0", default-features = false } uuid = { version = "1", default-features = false, features = ["serde", "v4"] } indoc = { version = "2.0.3", default-features = false } diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index a97b18d91c6cd..e416c82c4e61e 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -19,9 +19,9 @@ futures = { version = "0.3.28", default-features = false, features = ["std"] } memmap2 = { version = "0.7.1", default-features = false } metrics = "0.21.1" num-traits = { version = "0.2.16", default-features = false } -pin-project = { version = "1.1.2", default-features = false } +pin-project = { version = "1.1.3", default-features = false } rkyv = { version = "0.7.40", default-features = false, features = ["size_32", "std", "strict", "validation"] } -serde = { version = "1.0.180", default-features = false, features = ["derive"] } +serde = { version = "1.0.183", default-features = false, features = ["derive"] } snafu = { version = "0.7.5", default-features = false, features = ["std"] } tokio-util = { version = "0.7.0", default-features = false } tokio = { version = "1.29.1", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } @@ -32,7 +32,7 @@ vector-config-macros = { path = "../vector-config-macros", default-features = fa vector-common = { path = "../vector-common", default-features = false, features = ["byte_size_of", "serde"] } [dev-dependencies] -clap = "4.1.14" +clap = "4.3.21" criterion = { version = "0.5", features = ["html_reports", "async_tokio"] } crossbeam-queue = "0.3.8" hdrhistogram = "7.5.2" diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 320f3c43f1b3a..2797f6fa6d1e9 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -53,10 +53,10 @@ metrics = "0.21.1" nom = { version = "7", optional = true } ordered-float = { version = "3.7.0", default-features = false } paste = "1.0.14" -pin-project = { version = "1.1.2", default-features = false } +pin-project = { version = "1.1.3", default-features = false } ryu = { version = "1", default-features = false } serde_json = { version = "1.0.104", default-features = false, features = ["std", "raw_value"] } -serde = { version = "1.0.180", optional = true, features = ["derive"] } +serde = { version = "1.0.183", optional = true, features = ["derive"] } smallvec = { version = "1", default-features = false } snafu = { version = "0.7", optional = true } stream-cancel = { version = "0.8.1", default-features = false } diff --git a/lib/vector-config-macros/Cargo.toml b/lib/vector-config-macros/Cargo.toml index c9d38e761dd14..60c0e9b3523d6 100644 --- a/lib/vector-config-macros/Cargo.toml +++ b/lib/vector-config-macros/Cargo.toml @@ -16,5 +16,5 @@ syn = { version = "2.0", default-features = false, features = ["full", "extra-tr vector-config-common = { path = "../vector-config-common" } [dev-dependencies] -serde = { version = "1.0.180", default-features = false } +serde = { version = "1.0.183", default-features = false } vector-config = { path = "../vector-config" } diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index b73ec78d18629..83432b11e813e 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -21,7 +21,7 @@ num-traits = { version = "0.2.16", default-features = false } once_cell = { version = "1", default-features = false } serde = { version = "1.0", default-features = false } serde_json = { version = "1.0", default-features = false, features = ["std"] } -serde_with = { version = "3.1.0", default-features = false, features = ["std"] } +serde_with = { version = "3.2.0", default-features = false, features = ["std"] } snafu = { version = "0.7.5", default-features = false } toml = { version = "0.7.6", default-features = false } tracing = { version = "0.1.34", default-features = false } @@ -32,4 +32,4 @@ vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] assert-json-diff = { version = "2", default-features = false } -serde_with = { version = "3.1.0", default-features = false, features = ["std", "macros"] } +serde_with = { version = "3.2.0", default-features = false, features = ["std", "macros"] } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 43f7f880a6ce6..fb99ede58c4c7 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -31,18 +31,18 @@ mlua = { version = "0.8.9", default-features = false, features = ["lua54", "send no-proxy = { version = "0.3.3", default-features = false, features = ["serialize"] } once_cell = { version = "1.18", default-features = false } ordered-float = { version = "3.7.0", default-features = false } -openssl = { version = "0.10.55", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.56", default-features = false, features = ["vendored"] } parking_lot = { version = "0.12.1", default-features = false } -pin-project = { version = "1.1.2", default-features = false } +pin-project = { version = "1.1.3", default-features = false } proptest = { version = "1.2", optional = true } prost-types = { version = "0.11", default-features = false } prost = { version = "0.11", default-features = false, features = ["std"] } quanta = { version = "0.11.1", default-features = false } regex = { version = "1.9.1", default-features = false, features = ["std", "perf"] } ryu = { version = "1", default-features = false } -serde = { version = "1.0.180", default-features = false, features = ["derive", "rc"] } +serde = { version = "1.0.183", default-features = false, features = ["derive", "rc"] } serde_json = { version = "1.0.104", default-features = false } -serde_with = { version = "3.1.0", default-features = false, features = ["std", "macros"] } +serde_with = { version = "3.2.0", default-features = false, features = ["std", "macros"] } smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } snafu = { version = "0.7.5", default-features = false } socket2 = { version = "0.5.3", default-features = false } diff --git a/lib/vector-core/benches/event/log_event.rs b/lib/vector-core/benches/event/log_event.rs index 1b5ea41df44d5..7395ab98a60e4 100644 --- a/lib/vector-core/benches/event/log_event.rs +++ b/lib/vector-core/benches/event/log_event.rs @@ -6,6 +6,14 @@ use criterion::{ use lookup::event_path; use vector_core::event::LogEvent; +fn default_log_event() -> LogEvent { + let mut log_event = LogEvent::default(); + log_event.insert(event_path!("one"), 1); + log_event.insert(event_path!("two"), 2); + log_event.insert(event_path!("three"), 3); + log_event +} + fn rename_key_flat(c: &mut Criterion) { let mut group: BenchmarkGroup = c.benchmark_group("vector_core::event::log_event::LogEvent::rename_key_flat"); @@ -13,13 +21,7 @@ fn rename_key_flat(c: &mut Criterion) { group.bench_function("rename_flat_key (key is present)", move |b| { b.iter_batched( - || { - let mut log_event = LogEvent::default(); - log_event.insert("one", 1); - log_event.insert("two", 2); - log_event.insert("three", 3); - log_event - }, + default_log_event, |mut log_event| { log_event.rename_key(event_path!("one"), event_path!("1")); }, @@ -29,13 +31,7 @@ fn rename_key_flat(c: &mut Criterion) { group.bench_function("rename_flat_key (key is NOT present)", move |b| { b.iter_batched( - || { - let mut log_event = LogEvent::default(); - log_event.insert("one", 1); - log_event.insert("two", 2); - log_event.insert("three", 3); - log_event - }, + default_log_event, |mut log_event| { log_event.rename_key(event_path!("four"), event_path!("4")); }, diff --git a/lib/vector-core/src/event/discriminant.rs b/lib/vector-core/src/event/discriminant.rs index 7c1eb40863e1c..fcbd5d0fa818f 100644 --- a/lib/vector-core/src/event/discriminant.rs +++ b/lib/vector-core/src/event/discriminant.rs @@ -27,7 +27,13 @@ impl Discriminant { pub fn from_log_event(event: &LogEvent, discriminant_fields: &[impl AsRef]) -> Self { let values: Vec> = discriminant_fields .iter() - .map(|discriminant_field| event.get(discriminant_field.as_ref()).cloned()) + .map(|discriminant_field| { + event + .parse_path_and_get_value(discriminant_field.as_ref()) + .ok() + .flatten() + .cloned() + }) .collect(); Self { values } } diff --git a/lib/vector-core/src/event/log_event.rs b/lib/vector-core/src/event/log_event.rs index 4eee59b608bc5..112764eaf8d43 100644 --- a/lib/vector-core/src/event/log_event.rs +++ b/lib/vector-core/src/event/log_event.rs @@ -20,7 +20,7 @@ use vector_common::{ request_metadata::GetEventCountTags, EventDataEq, }; -use vrl::path::OwnedTargetPath; +use vrl::path::{parse_target_path, OwnedTargetPath, PathParseError}; use super::{ estimated_json_encoded_size_of::EstimatedJsonEncodedSizeOf, @@ -33,7 +33,7 @@ use crate::config::{log_schema, telemetry}; use crate::{event::MaybeAsLogMut, ByteSizeOf}; use lookup::{metadata_path, path}; use once_cell::sync::Lazy; -use vrl::owned_value_path; +use vrl::{event_path, owned_value_path}; static VECTOR_SOURCE_TYPE_PATH: Lazy> = Lazy::new(|| { Some(OwnedTargetPath::metadata(owned_value_path!( @@ -295,6 +295,16 @@ impl LogEvent { self.metadata.add_finalizer(finalizer); } + /// Parse the specified `path` and if there are no parsing errors, attempt to get a reference to a value. + /// # Errors + /// Will return an error if path parsing failed. + pub fn parse_path_and_get_value( + &self, + path: impl AsRef, + ) -> Result, PathParseError> { + parse_target_path(path.as_ref()).map(|path| self.get(&path)) + } + #[allow(clippy::needless_pass_by_value)] // TargetPath is always a reference pub fn get<'a>(&self, key: impl TargetPath<'a>) -> Option<&Value> { match key.prefix() { @@ -341,6 +351,19 @@ impl LogEvent { } } + /// Parse the specified `path` and if there are no parsing errors, attempt to insert the specified `value`. + /// + /// # Errors + /// Will return an error if path parsing failed. + pub fn parse_path_and_insert( + &mut self, + path: impl AsRef, + value: impl Into, + ) -> Result, PathParseError> { + let target_path = parse_target_path(path.as_ref())?; + Ok(self.insert(&target_path, value)) + } + #[allow(clippy::needless_pass_by_value)] // TargetPath is always a reference pub fn insert<'a>( &mut self, @@ -434,14 +457,16 @@ impl LogEvent { } /// Merge all fields specified at `fields` from `incoming` to `current`. + /// Note that `fields` containing dots and other special characters will be treated as a single segment. pub fn merge(&mut self, mut incoming: LogEvent, fields: &[impl AsRef]) { for field in fields { - let Some(incoming_val) = incoming.remove(field.as_ref()) else { + let field_path = event_path!(field.as_ref()); + let Some(incoming_val) = incoming.remove(field_path) else { continue }; - match self.get_mut(field.as_ref()) { + match self.get_mut(field_path) { None => { - self.insert(field.as_ref(), incoming_val); + self.insert(field_path, incoming_val); } Some(current_val) => current_val.merge(incoming_val), } @@ -635,6 +660,7 @@ impl TryInto for LogEvent { } } +#[cfg(any(test, feature = "test"))] impl std::ops::Index for LogEvent where T: AsRef, @@ -642,7 +668,9 @@ where type Output = Value; fn index(&self, key: T) -> &Value { - self.get(key.as_ref()) + self.parse_path_and_get_value(key.as_ref()) + .ok() + .flatten() .unwrap_or_else(|| panic!("Key is not found: {:?}", key.as_ref())) } } @@ -654,7 +682,9 @@ where { fn extend>(&mut self, iter: I) { for (k, v) in iter { - self.insert(k.as_ref(), v.into()); + if let Ok(path) = parse_target_path(k.as_ref()) { + self.insert(&path, v.into()); + } } } } @@ -677,6 +707,24 @@ impl Serialize for LogEvent { } } +// Tracing owned target paths used for tracing to log event conversions. +struct TracingTargetPaths { + pub(crate) timestamp: OwnedTargetPath, + pub(crate) kind: OwnedTargetPath, + pub(crate) module_path: OwnedTargetPath, + pub(crate) level: OwnedTargetPath, + pub(crate) target: OwnedTargetPath, +} + +/// Lazily initialized singleton. +static TRACING_TARGET_PATHS: Lazy = Lazy::new(|| TracingTargetPaths { + timestamp: OwnedTargetPath::event(owned_value_path!("timestamp")), + kind: OwnedTargetPath::event(owned_value_path!("metadata", "kind")), + level: OwnedTargetPath::event(owned_value_path!("metadata", "level")), + module_path: OwnedTargetPath::event(owned_value_path!("metadata", "module_path")), + target: OwnedTargetPath::event(owned_value_path!("metadata", "target")), +}); + impl From<&tracing::Event<'_>> for LogEvent { fn from(event: &tracing::Event<'_>) -> Self { let now = chrono::Utc::now(); @@ -684,11 +732,11 @@ impl From<&tracing::Event<'_>> for LogEvent { event.record(&mut maker); let mut log = maker; - log.insert("timestamp", now); + log.insert(&TRACING_TARGET_PATHS.timestamp, now); let meta = event.metadata(); log.insert( - "metadata.kind", + &TRACING_TARGET_PATHS.kind, if meta.is_event() { Value::Bytes("event".to_string().into()) } else if meta.is_span() { @@ -697,42 +745,42 @@ impl From<&tracing::Event<'_>> for LogEvent { Value::Null }, ); - log.insert("metadata.level", meta.level().to_string()); + log.insert(&TRACING_TARGET_PATHS.level, meta.level().to_string()); log.insert( - "metadata.module_path", + &TRACING_TARGET_PATHS.module_path, meta.module_path() .map_or(Value::Null, |mp| Value::Bytes(mp.to_string().into())), ); - log.insert("metadata.target", meta.target().to_string()); - + log.insert(&TRACING_TARGET_PATHS.target, meta.target().to_string()); log } } +/// Note that `tracing::field::Field` containing dots and other special characters will be treated as a single segment. impl tracing::field::Visit for LogEvent { fn record_str(&mut self, field: &tracing::field::Field, value: &str) { - self.insert(field.name(), value.to_string()); + self.insert(event_path!(field.name()), value.to_string()); } fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn Debug) { - self.insert(field.name(), format!("{value:?}")); + self.insert(event_path!(field.name()), format!("{value:?}")); } fn record_i64(&mut self, field: &tracing::field::Field, value: i64) { - self.insert(field.name(), value); + self.insert(event_path!(field.name()), value); } fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { - let field = field.name(); + let field_path = event_path!(field.name()); let converted: Result = value.try_into(); match converted { - Ok(value) => self.insert(field, value), - Err(_) => self.insert(field, value.to_string()), + Ok(value) => self.insert(field_path, value), + Err(_) => self.insert(field_path, value.to_string()), }; } fn record_bool(&mut self, field: &tracing::field::Field, value: bool) { - self.insert(field.name(), value); + self.insert(event_path!(field.name()), value); } } diff --git a/lib/vector-core/src/event/test/size_of.rs b/lib/vector-core/src/event/test/size_of.rs index 89770273dbadc..522b0f0520fe4 100644 --- a/lib/vector-core/src/event/test/size_of.rs +++ b/lib/vector-core/src/event/test/size_of.rs @@ -115,13 +115,12 @@ fn log_operation_maintains_size() { match action { Action::InsertFlat { key, value } => { let new_value_sz = value.size_of(); - let old_value_sz = log_event - .get((PathPrefix::Event, path!(key.as_str()))) - .map_or(0, ByteSizeOf::size_of); + let target_path = (PathPrefix::Event, path!(key.as_str())); + let old_value_sz = log_event.get(target_path).map_or(0, ByteSizeOf::size_of); if !log_event.contains(key.as_str()) { current_size += key.size_of(); } - log_event.insert((PathPrefix::Event, path!(&key)), value); + log_event.insert(target_path, value); current_size -= old_value_sz; current_size += new_value_sz; } diff --git a/lib/vector-core/src/event/trace.rs b/lib/vector-core/src/event/trace.rs index d5fc7856c9066..120c6c55f490b 100644 --- a/lib/vector-core/src/event/trace.rs +++ b/lib/vector-core/src/event/trace.rs @@ -7,6 +7,7 @@ use vector_common::{ internal_event::TaggedEventsSent, json_size::JsonSize, request_metadata::GetEventCountTags, EventDataEq, }; +use vrl::path::PathParseError; use super::{ BatchNotifier, EstimatedJsonEncodedSizeOf, EventFinalizer, EventFinalizers, EventMetadata, @@ -71,17 +72,27 @@ impl TraceEvent { self.0.as_map().expect("inner value must be a map") } + /// Parse the specified `path` and if there are no parsing errors, attempt to get a reference to a value. + /// # Errors + /// Will return an error if path parsing failed. + pub fn parse_path_and_get_value( + &self, + path: impl AsRef, + ) -> Result, PathParseError> { + self.0.parse_path_and_get_value(path) + } + #[allow(clippy::needless_pass_by_value)] // TargetPath is always a reference pub fn get<'a>(&self, key: impl TargetPath<'a>) -> Option<&Value> { self.0.get(key) } - pub fn get_mut(&mut self, key: impl AsRef) -> Option<&mut Value> { - self.0.get_mut(key.as_ref()) + pub fn get_mut<'a>(&mut self, key: impl TargetPath<'a>) -> Option<&mut Value> { + self.0.get_mut(key) } - pub fn contains(&self, key: impl AsRef) -> bool { - self.0.contains(key.as_ref()) + pub fn contains<'a>(&self, key: impl TargetPath<'a>) -> bool { + self.0.contains(key) } pub fn insert<'a>( diff --git a/lib/vector-core/src/tls/settings.rs b/lib/vector-core/src/tls/settings.rs index 4454cfc76026e..175660be97649 100644 --- a/lib/vector-core/src/tls/settings.rs +++ b/lib/vector-core/src/tls/settings.rs @@ -630,6 +630,7 @@ mod test { #[test] fn from_options_pkcs12() { + let _provider = openssl::provider::Provider::try_load(None, "legacy", true).unwrap(); let options = TlsConfig { crt_file: Some(TEST_PKCS12_PATH.into()), key_pass: Some("NOPASS".into()), diff --git a/lib/vector-lookup/Cargo.toml b/lib/vector-lookup/Cargo.toml index dc33d48355a60..986f0317a31d6 100644 --- a/lib/vector-lookup/Cargo.toml +++ b/lib/vector-lookup/Cargo.toml @@ -7,7 +7,7 @@ publish = false license = "MPL-2.0" [dependencies] -serde = { version = "1.0.180", default-features = false, features = ["derive", "alloc"] } +serde = { version = "1.0.183", default-features = false, features = ["derive", "alloc"] } vector-config = { path = "../vector-config" } vector-config-macros = { path = "../vector-config-macros" } vrl.workspace = true diff --git a/lib/vector-vrl/cli/Cargo.toml b/lib/vector-vrl/cli/Cargo.toml index 106e430355c56..f67b06f4e1db4 100644 --- a/lib/vector-vrl/cli/Cargo.toml +++ b/lib/vector-vrl/cli/Cargo.toml @@ -7,6 +7,6 @@ publish = false license = "MPL-2.0" [dependencies] -clap = { version = "4.1.14", features = ["derive"] } +clap = { version = "4.3.21", features = ["derive"] } vector-vrl-functions = { path = "../functions" } vrl.workspace = true diff --git a/lib/vector-vrl/functions/src/set_semantic_meaning.rs b/lib/vector-vrl/functions/src/set_semantic_meaning.rs index 26e61c1671f59..f1b53065d96ed 100644 --- a/lib/vector-vrl/functions/src/set_semantic_meaning.rs +++ b/lib/vector-vrl/functions/src/set_semantic_meaning.rs @@ -62,8 +62,7 @@ impl Function for SetSemanticMeaning { let query = arguments.required_query("target")?; let meaning = arguments - .required_literal("meaning")? - .to_value() + .required_literal("meaning", state)? .try_bytes_utf8_lossy() .expect("meaning not bytes") .into_owned(); diff --git a/lib/vector-vrl/tests/Cargo.toml b/lib/vector-vrl/tests/Cargo.toml index 96c8185f45ec9..0a0ddb25d56ef 100644 --- a/lib/vector-vrl/tests/Cargo.toml +++ b/lib/vector-vrl/tests/Cargo.toml @@ -13,7 +13,7 @@ vector-vrl-functions = { path = "../../vector-vrl/functions" } ansi_term = "0.12" chrono = "0.4" chrono-tz = "0.8" -clap = { version = "4.1.14", features = ["derive"] } +clap = { version = "4.3.21", features = ["derive"] } glob = "0.3" prettydiff = "0.6" regex = "1" diff --git a/scripts/cross/bootstrap-centos.sh b/scripts/cross/bootstrap-centos.sh index ce35c53283019..58b1cbba003c6 100755 --- a/scripts/cross/bootstrap-centos.sh +++ b/scripts/cross/bootstrap-centos.sh @@ -3,3 +3,7 @@ set -o errexit yum install -y unzip centos-release-scl yum install -y llvm-toolset-7 + +# needed to compile openssl +yum install -y perl-IPC-Cmd + diff --git a/src/api/schema/events/trace.rs b/src/api/schema/events/trace.rs index 9291e4db2888a..9bd52d3ce053e 100644 --- a/src/api/schema/events/trace.rs +++ b/src/api/schema/events/trace.rs @@ -1,5 +1,6 @@ use async_graphql::Object; use vector_common::encode_logfmt; +use vrl::event_path; use super::EventEncodingType; use crate::{event, topology::TapOutput}; @@ -48,7 +49,7 @@ impl Trace { /// Get JSON field data on the trace event, by field name async fn json(&self, field: String) -> Option { - self.event.get(field.as_str()).map(|field| { + self.event.get(event_path!(field.as_str())).map(|field| { serde_json::to_string(field) .expect("JSON serialization of log event field failed. Please report.") }) diff --git a/src/app.rs b/src/app.rs index 037743220bdf1..71cd76ea86c29 100644 --- a/src/app.rs +++ b/src/app.rs @@ -8,6 +8,7 @@ use futures::StreamExt; #[cfg(feature = "enterprise")] use futures_util::future::BoxFuture; use once_cell::race::OnceNonZeroUsize; +use openssl::provider::Provider; use tokio::{ runtime::{self, Runtime}, sync::mpsc, @@ -61,6 +62,7 @@ pub struct Application { pub require_healthy: Option, pub config: ApplicationConfig, pub signals: SignalPair, + pub openssl_legacy_provider: Option, } impl ApplicationConfig { @@ -189,6 +191,12 @@ impl Application { opts.root.internal_log_rate_limit, ); + let openssl_legacy_provider = opts + .root + .openssl_legacy_provider + .then(load_openssl_legacy_provider) + .flatten(); + let runtime = build_runtime(opts.root.threads, "vector-worker")?; // Signal handler for OS and provider messages. @@ -209,6 +217,7 @@ impl Application { require_healthy: opts.root.require_healthy, config, signals, + openssl_legacy_provider, }, )) } @@ -225,6 +234,7 @@ impl Application { require_healthy, config, signals, + openssl_legacy_provider, } = self; let topology_controller = SharedTopologyController::new(TopologyController { @@ -242,6 +252,7 @@ impl Application { graceful_crash_receiver: config.graceful_crash_receiver, signals, topology_controller, + openssl_legacy_provider, }) } } @@ -251,6 +262,7 @@ pub struct StartedApplication { pub graceful_crash_receiver: mpsc::UnboundedReceiver, pub signals: SignalPair, pub topology_controller: SharedTopologyController, + pub openssl_legacy_provider: Option, } impl StartedApplication { @@ -264,6 +276,7 @@ impl StartedApplication { graceful_crash_receiver, signals, topology_controller, + openssl_legacy_provider, } = self; let mut graceful_crash = UnboundedReceiverStream::new(graceful_crash_receiver); @@ -295,6 +308,7 @@ impl StartedApplication { signal, signal_rx, topology_controller, + openssl_legacy_provider, } } } @@ -350,6 +364,7 @@ pub struct FinishedApplication { pub signal: SignalTo, pub signal_rx: SignalRx, pub topology_controller: SharedTopologyController, + pub openssl_legacy_provider: Option, } impl FinishedApplication { @@ -358,6 +373,7 @@ impl FinishedApplication { signal, signal_rx, topology_controller, + openssl_legacy_provider, } = self; // At this point, we'll have the only reference to the shared topology controller and can @@ -367,11 +383,13 @@ impl FinishedApplication { .expect("fail to unwrap topology controller") .into_inner(); - match signal { + let status = match signal { SignalTo::Shutdown(_) => Self::stop(topology_controller, signal_rx).await, SignalTo::Quit => Self::quit(), _ => unreachable!(), - } + }; + drop(openssl_legacy_provider); + status } async fn stop(topology_controller: TopologyController, mut signal_rx: SignalRx) -> ExitStatus { @@ -542,3 +560,18 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) ); info!(message = "Log level is enabled.", level = ?level); } + +/// Load the legacy OpenSSL provider. +/// +/// The returned [Provider] must stay in scope for the entire lifetime of the application, as it +/// will be unloaded when it is dropped. +pub fn load_openssl_legacy_provider() -> Option { + warn!(message = "DEPRECATED The openssl legacy provider provides algorithms and key sizes no longer recommended for use."); + Provider::try_load(None, "legacy", true) + .map(|provider| { + info!(message = "Loaded openssl legacy provider."); + provider + }) + .map_err(|error| error!(message = "Failed to load openssl legacy provider.", %error)) + .ok() +} diff --git a/src/cli.rs b/src/cli.rs index 68d49db299061..1493e8db117e8 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -194,6 +194,10 @@ pub struct RootOpts { default_value = "5000" )] pub allocation_tracing_reporting_interval_ms: u64, + + /// Load the OpenSSL legacy provider. + #[arg(long, env = "VECTOR_OPENSSL_LEGACY_PROVIDER", default_value = "true")] + pub openssl_legacy_provider: bool, } impl RootOpts { diff --git a/src/codecs/encoding/transformer.rs b/src/codecs/encoding/transformer.rs index df711bd8f065b..425a0e22980d3 100644 --- a/src/codecs/encoding/transformer.rs +++ b/src/codecs/encoding/transformer.rs @@ -191,7 +191,7 @@ impl Transformer { } } for (k, v) in unix_timestamps { - log.insert(k.as_str(), v); + log.parse_path_and_insert(k, v).unwrap(); } } else { // root is not an object diff --git a/src/conditions/datadog_search.rs b/src/conditions/datadog_search.rs index 2e2e0e88f1344..90e9641567038 100644 --- a/src/conditions/datadog_search.rs +++ b/src/conditions/datadog_search.rs @@ -75,7 +75,12 @@ impl Filter for EventFilter { any_string_match("tags", move |value| value == field) } Field::Default(f) | Field::Facet(f) | Field::Reserved(f) => { - Run::boxed(move |log: &LogEvent| log.get(f.as_str()).is_some()) + Run::boxed(move |log: &LogEvent| { + log.parse_path_and_get_value(f.as_str()) + .ok() + .flatten() + .is_some() + }) } } } @@ -165,8 +170,11 @@ impl Filter for EventFilter { match field { // Facets are compared numerically if the value is numeric, or as strings otherwise. Field::Facet(f) => { - Run::boxed( - move |log: &LogEvent| match (log.get(f.as_str()), &comparison_value) { + Run::boxed(move |log: &LogEvent| { + match ( + log.parse_path_and_get_value(f.as_str()).ok().flatten(), + &comparison_value, + ) { // Integers. (Some(Value::Integer(lhs)), ComparisonValue::Integer(rhs)) => { match comparator { @@ -227,8 +235,8 @@ impl Filter for EventFilter { } } _ => false, - }, - ) + } + }) } // Tag values need extracting by "key:value" to be compared. Field::Tag(tag) => any_string_match("tags", move |value| match value.split_once(':') { @@ -266,9 +274,11 @@ where { let field = field.into(); - Run::boxed(move |log: &LogEvent| match log.get(field.as_str()) { - Some(Value::Bytes(v)) => func(String::from_utf8_lossy(v)), - _ => false, + Run::boxed(move |log: &LogEvent| { + match log.parse_path_and_get_value(field.as_str()).ok().flatten() { + Some(Value::Bytes(v)) => func(String::from_utf8_lossy(v)), + _ => false, + } }) } @@ -281,9 +291,11 @@ where { let field = field.into(); - Run::boxed(move |log: &LogEvent| match log.get(field.as_str()) { - Some(Value::Array(values)) => func(values), - _ => false, + Run::boxed(move |log: &LogEvent| { + match log.parse_path_and_get_value(field.as_str()).ok().flatten() { + Some(Value::Array(values)) => func(values), + _ => false, + } }) } diff --git a/src/conditions/vrl.rs b/src/conditions/vrl.rs index 5febe5ef1a222..6db16feb474eb 100644 --- a/src/conditions/vrl.rs +++ b/src/conditions/vrl.rs @@ -113,7 +113,7 @@ impl Conditional for Vrl { let result = result .map(|value| match value { Value::Boolean(boolean) => boolean, - _ => false, + _ => panic!("VRL condition did not return a boolean type"), }) .unwrap_or_else(|err| { emit!(VrlConditionExecutionError { diff --git a/src/config/unit_test/mod.rs b/src/config/unit_test/mod.rs index 3bfd52ad4b7ee..f61917d336835 100644 --- a/src/config/unit_test/mod.rs +++ b/src/config/unit_test/mod.rs @@ -572,7 +572,9 @@ fn build_input_event(input: &TestInput) -> Result { NotNan::new(*f).map_err(|_| "NaN value not supported".to_string())?, ), }; - event.insert(path.as_str(), value); + event + .parse_path_and_insert(path, value) + .map_err(|e| e.to_string())?; } Ok(event.into()) } else { diff --git a/src/sinks/aws_cloudwatch_logs/integration_tests.rs b/src/sinks/aws_cloudwatch_logs/integration_tests.rs index c0dbf0204260a..872a470216ba8 100644 --- a/src/sinks/aws_cloudwatch_logs/integration_tests.rs +++ b/src/sinks/aws_cloudwatch_logs/integration_tests.rs @@ -188,10 +188,7 @@ async fn cloudwatch_insert_out_of_range_timestamp() { let line = input_lines.next().unwrap(); let mut event = LogEvent::from(line.clone()); event.insert( - ( - lookup::PathPrefix::Event, - log_schema().timestamp_key().unwrap(), - ), + log_schema().timestamp_key_target_path().unwrap(), now + offset, ); events.push(Event::Log(event)); diff --git a/src/sinks/aws_cloudwatch_logs/request_builder.rs b/src/sinks/aws_cloudwatch_logs/request_builder.rs index edbf4a752233c..177a9ccc5a5e8 100644 --- a/src/sinks/aws_cloudwatch_logs/request_builder.rs +++ b/src/sinks/aws_cloudwatch_logs/request_builder.rs @@ -154,13 +154,7 @@ mod tests { let timestamp = Utc::now(); let message = "event message"; let mut event = LogEvent::from(message); - event.insert( - ( - lookup::PathPrefix::Event, - log_schema().timestamp_key().unwrap(), - ), - timestamp, - ); + event.insert(log_schema().timestamp_key_target_path().unwrap(), timestamp); let request = request_builder.build(event.into()).unwrap(); assert_eq!(request.timestamp, timestamp.timestamp_millis()); diff --git a/src/sinks/azure_blob/integration_tests.rs b/src/sinks/azure_blob/integration_tests.rs index ec3dc5ac1d1c8..9cddbef79f949 100644 --- a/src/sinks/azure_blob/integration_tests.rs +++ b/src/sinks/azure_blob/integration_tests.rs @@ -111,7 +111,11 @@ async fn azure_blob_insert_json_into_blob() { assert_eq!(blobs.len(), 1); assert!(blobs[0].clone().ends_with(".log")); let (blob, blob_lines) = config.get_blob(blobs[0].clone()).await; - assert_eq!(blob.properties.content_type, String::from("text/plain")); + assert_eq!(blob.properties.content_encoding, None); + assert_eq!( + blob.properties.content_type, + String::from("application/x-ndjson") + ); let expected = events .iter() .map(|event| serde_json::to_string(&event.as_log().all_fields().unwrap()).unwrap()) @@ -138,10 +142,8 @@ async fn azure_blob_insert_lines_into_blob_gzip() { assert_eq!(blobs.len(), 1); assert!(blobs[0].clone().ends_with(".log.gz")); let (blob, blob_lines) = config.get_blob(blobs[0].clone()).await; - assert_eq!( - blob.properties.content_type, - String::from("application/gzip") - ); + assert_eq!(blob.properties.content_encoding, Some(String::from("gzip"))); + assert_eq!(blob.properties.content_type, String::from("text/plain")); assert_eq!(lines, blob_lines); } @@ -170,9 +172,10 @@ async fn azure_blob_insert_json_into_blob_gzip() { assert_eq!(blobs.len(), 1); assert!(blobs[0].clone().ends_with(".log.gz")); let (blob, blob_lines) = config.get_blob(blobs[0].clone()).await; + assert_eq!(blob.properties.content_encoding, Some(String::from("gzip"))); assert_eq!( blob.properties.content_type, - String::from("application/gzip") + String::from("application/x-ndjson") ); let expected = events .iter() diff --git a/src/sinks/azure_blob/request_builder.rs b/src/sinks/azure_blob/request_builder.rs index ae8f8770381a5..0ae3fb848d922 100644 --- a/src/sinks/azure_blob/request_builder.rs +++ b/src/sinks/azure_blob/request_builder.rs @@ -93,20 +93,9 @@ impl RequestBuilder<(String, Vec)> for AzureBlobRequestOptions { AzureBlobRequest { blob_data, content_encoding: self.compression.content_encoding(), - content_type: self.compression.content_type(), + content_type: self.encoder.1.content_type(), metadata: azure_metadata, request_metadata, } } } - -impl Compression { - pub const fn content_type(self) -> &'static str { - match self { - Self::None => "text/plain", - Self::Gzip(_) => "application/gzip", - Self::Zlib(_) => "application/zlib", - Self::Zstd(_) => "application/zstd", - } - } -} diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index 0c4ad1f38dd53..d35606ac12938 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -129,7 +129,7 @@ fn azure_blob_build_request_with_compression() { assert_eq!(request.metadata.partition_key, "blob.log.gz".to_string()); assert_eq!(request.content_encoding, Some("gzip")); - assert_eq!(request.content_type, "application/gzip"); + assert_eq!(request.content_type, "text/plain"); } #[test] diff --git a/src/sinks/azure_monitor_logs.rs b/src/sinks/azure_monitor_logs.rs deleted file mode 100644 index 03ef6de56c267..0000000000000 --- a/src/sinks/azure_monitor_logs.rs +++ /dev/null @@ -1,693 +0,0 @@ -use bytes::Bytes; -use futures::{FutureExt, SinkExt}; -use http::{ - header, - header::{HeaderMap, HeaderName, HeaderValue}, - Request, StatusCode, Uri, -}; -use hyper::Body; -use lookup::lookup_v2::OptionalValuePath; -use lookup::{OwnedValuePath, PathPrefix}; -use once_cell::sync::Lazy; -use openssl::{base64, hash, pkey, sign}; -use regex::Regex; -use serde::{Deserialize, Serialize}; -use serde_json::Value as JsonValue; -use vector_common::sensitive_string::SensitiveString; -use vector_config::configurable_component; -use vector_core::schema; -use vrl::value::Kind; - -use crate::{ - codecs::Transformer, - config::{log_schema, AcknowledgementsConfig, Input, SinkConfig, SinkContext}, - event::{Event, Value}, - http::HttpClient, - sinks::{ - util::{ - http::{BatchedHttpSink, HttpEventEncoder, HttpSink}, - BatchConfig, BoxedRawValue, JsonArrayBuffer, RealtimeSizeBasedDefaultBatchSettings, - TowerRequestConfig, - }, - Healthcheck, VectorSink, - }, - tls::{TlsConfig, TlsSettings}, -}; - -fn default_host() -> String { - "ods.opinsights.azure.com".into() -} - -/// Configuration for the `azure_monitor_logs` sink. -#[configurable_component(sink( - "azure_monitor_logs", - "Publish log events to the Azure Monitor Logs service." -))] -#[derive(Clone, Debug)] -#[serde(deny_unknown_fields)] -pub struct AzureMonitorLogsConfig { - /// The [unique identifier][uniq_id] for the Log Analytics workspace. - /// - /// [uniq_id]: https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api#request-uri-parameters - #[configurable(metadata(docs::examples = "5ce893d9-2c32-4b6c-91a9-b0887c2de2d6"))] - #[configurable(metadata(docs::examples = "97ce69d9-b4be-4241-8dbd-d265edcf06c4"))] - pub customer_id: String, - - /// The [primary or the secondary key][shared_key] for the Log Analytics workspace. - /// - /// [shared_key]: https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api#authorization - #[configurable(metadata( - docs::examples = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" - ))] - #[configurable(metadata(docs::examples = "${AZURE_MONITOR_SHARED_KEY_ENV_VAR}"))] - pub shared_key: SensitiveString, - - /// The [record type][record_type] of the data that is being submitted. - /// - /// Can only contain letters, numbers, and underscores (_), and may not exceed 100 characters. - /// - /// [record_type]: https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api#request-headers - #[configurable(validation(pattern = "[a-zA-Z0-9_]{1,100}"))] - #[configurable(metadata(docs::examples = "MyTableName"))] - #[configurable(metadata(docs::examples = "MyRecordType"))] - pub log_type: String, - - /// The [Resource ID][resource_id] of the Azure resource the data should be associated with. - /// - /// [resource_id]: https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api#request-headers - #[configurable(metadata( - docs::examples = "/subscriptions/11111111-1111-1111-1111-111111111111/resourceGroups/otherResourceGroup/providers/Microsoft.Storage/storageAccounts/examplestorage" - ))] - #[configurable(metadata( - docs::examples = "/subscriptions/11111111-1111-1111-1111-111111111111/resourceGroups/examplegroup/providers/Microsoft.SQL/servers/serverName/databases/databaseName" - ))] - pub azure_resource_id: Option, - - /// [Alternative host][alt_host] for dedicated Azure regions. - /// - /// [alt_host]: https://docs.azure.cn/en-us/articles/guidance/developerdifferences#check-endpoints-in-azure - #[configurable(metadata(docs::examples = "ods.opinsights.azure.us"))] - #[configurable(metadata(docs::examples = "ods.opinsights.azure.cn"))] - #[serde(default = "default_host")] - pub(super) host: String, - - #[configurable(derived)] - #[serde( - default, - skip_serializing_if = "crate::serde::skip_serializing_if_default" - )] - pub encoding: Transformer, - - #[configurable(derived)] - #[serde(default)] - pub batch: BatchConfig, - - #[configurable(derived)] - #[serde(default)] - pub request: TowerRequestConfig, - - /// Use this option to customize the log field used as [`TimeGenerated`][1] in Azure. - /// - /// The setting of `log_schema.timestamp_key`, usually `timestamp`, is used here by default. - /// This field should be used in rare cases where `TimeGenerated` should point to a specific log - /// field. For example, use this field to set the log field `source_timestamp` as holding the - /// value that should be used as `TimeGenerated` on the Azure side. - /// - /// [1]: https://learn.microsoft.com/en-us/azure/azure-monitor/logs/log-standard-columns#timegenerated - #[configurable(metadata(docs::examples = "time_generated"))] - pub time_generated_key: Option, - - #[configurable(derived)] - pub tls: Option, - - #[configurable(derived)] - #[serde( - default, - deserialize_with = "crate::serde::bool_or_struct", - skip_serializing_if = "crate::serde::skip_serializing_if_default" - )] - acknowledgements: AcknowledgementsConfig, -} - -impl Default for AzureMonitorLogsConfig { - fn default() -> Self { - Self { - customer_id: "my-customer-id".to_string(), - shared_key: Default::default(), - log_type: "MyRecordType".to_string(), - azure_resource_id: None, - host: default_host(), - encoding: Default::default(), - batch: Default::default(), - request: Default::default(), - time_generated_key: None, - tls: None, - acknowledgements: Default::default(), - } - } -} - -#[derive(Deserialize, Serialize, Debug, Eq, PartialEq, Clone, Derivative)] -#[serde(rename_all = "snake_case")] -#[derivative(Default)] -pub enum Encoding { - #[derivative(Default)] - Default, -} - -static LOG_TYPE_REGEX: Lazy = Lazy::new(|| Regex::new(r"^\w+$").unwrap()); -static LOG_TYPE_HEADER: Lazy = Lazy::new(|| HeaderName::from_static("log-type")); -static X_MS_DATE_HEADER: Lazy = Lazy::new(|| HeaderName::from_static(X_MS_DATE)); -static X_MS_AZURE_RESOURCE_HEADER: Lazy = - Lazy::new(|| HeaderName::from_static("x-ms-azureresourceid")); -static TIME_GENERATED_FIELD_HEADER: Lazy = - Lazy::new(|| HeaderName::from_static("time-generated-field")); -static CONTENT_TYPE_VALUE: Lazy = Lazy::new(|| HeaderValue::from_static(CONTENT_TYPE)); - -impl_generate_config_from_default!(AzureMonitorLogsConfig); - -/// Max number of bytes in request body -const MAX_BATCH_SIZE: usize = 30 * 1024 * 1024; -/// API endpoint for submitting logs -const RESOURCE: &str = "/api/logs"; -/// JSON content type of logs -const CONTENT_TYPE: &str = "application/json"; -/// Custom header used for signing logs -const X_MS_DATE: &str = "x-ms-date"; -/// Shared key prefix -const SHARED_KEY: &str = "SharedKey"; -/// API version -const API_VERSION: &str = "2016-04-01"; - -#[async_trait::async_trait] -#[typetag::serde(name = "azure_monitor_logs")] -impl SinkConfig for AzureMonitorLogsConfig { - async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { - let batch_settings = self - .batch - .validate()? - .limit_max_bytes(MAX_BATCH_SIZE)? - .into_batch_settings()?; - - let time_generated_key = self.time_generated_key.clone().and_then(|k| k.path); - - let tls_settings = TlsSettings::from_options(&self.tls)?; - let client = HttpClient::new(Some(tls_settings), &cx.proxy)?; - - let sink = AzureMonitorLogsSink::new(self, time_generated_key)?; - let request_settings = self.request.unwrap_with(&TowerRequestConfig::default()); - - let healthcheck = healthcheck(sink.clone(), client.clone()).boxed(); - - let sink = BatchedHttpSink::new( - sink, - JsonArrayBuffer::new(batch_settings.size), - request_settings, - batch_settings.timeout, - client, - ) - .sink_map_err(|error| error!(message = "Fatal azure_monitor_logs sink error.", %error)); - - #[allow(deprecated)] - Ok((VectorSink::from_event_sink(sink), healthcheck)) - } - - fn input(&self) -> Input { - let requirements = - schema::Requirement::empty().optional_meaning("timestamp", Kind::timestamp()); - - Input::log().with_schema_requirement(requirements) - } - - fn acknowledgements(&self) -> &AcknowledgementsConfig { - &self.acknowledgements - } -} - -#[derive(Clone)] -struct AzureMonitorLogsSink { - uri: Uri, - customer_id: String, - time_generated_key: Option, - transformer: Transformer, - shared_key: pkey::PKey, - default_headers: HeaderMap, -} - -struct AzureMonitorLogsEventEncoder { - transformer: Transformer, - time_generated_key: Option, -} - -impl HttpEventEncoder for AzureMonitorLogsEventEncoder { - fn encode_event(&mut self, mut event: Event) -> Option { - self.transformer.transform(&mut event); - - // it seems like Azure Monitor doesn't support full 9-digit nanosecond precision - // adjust the timestamp format accordingly, keeping only milliseconds - let mut log = event.into_log(); - - // `.remove_timestamp()` will return the `timestamp` value regardless of location in Event or - // Metadata, the following `insert()` ensures it's encoded in the request. - let timestamp = if let Some(Value::Timestamp(ts)) = log.remove_timestamp() { - ts - } else { - chrono::Utc::now() - }; - - if let Some(timestamp_key) = &self.time_generated_key { - log.insert( - (PathPrefix::Event, timestamp_key), - JsonValue::String(timestamp.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)), - ); - } - - let entry = serde_json::json!(&log); - - Some(entry) - } -} - -#[async_trait::async_trait] -impl HttpSink for AzureMonitorLogsSink { - type Input = serde_json::Value; - type Output = Vec; - type Encoder = AzureMonitorLogsEventEncoder; - - fn build_encoder(&self) -> Self::Encoder { - AzureMonitorLogsEventEncoder { - transformer: self.transformer.clone(), - time_generated_key: self.time_generated_key.clone(), - } - } - - async fn build_request(&self, events: Self::Output) -> crate::Result> { - self.build_request_sync(events) - } -} - -impl AzureMonitorLogsSink { - fn new( - config: &AzureMonitorLogsConfig, - time_generated_key: Option, - ) -> crate::Result { - let url = format!( - "https://{}.{}{}?api-version={}", - config.customer_id, config.host, RESOURCE, API_VERSION - ); - let uri: Uri = url.parse()?; - - if config.shared_key.inner().is_empty() { - return Err("shared_key can't be an empty string".into()); - } - - let time_generated_key = - time_generated_key.or_else(|| log_schema().timestamp_key().cloned()); - - let shared_key_bytes = base64::decode_block(config.shared_key.inner())?; - let shared_key = pkey::PKey::hmac(&shared_key_bytes)?; - let mut default_headers = HeaderMap::with_capacity(3); - - if config.log_type.len() > 100 || !LOG_TYPE_REGEX.is_match(&config.log_type) { - return Err(format!( - "invalid log_type \"{}\": log type can only contain letters, numbers, and underscore (_), and may not exceed 100 characters", - config.log_type - ).into()); - } - - let log_type = HeaderValue::from_str(&config.log_type)?; - default_headers.insert(LOG_TYPE_HEADER.clone(), log_type); - - if let Some(timestamp_key) = &time_generated_key { - default_headers.insert( - TIME_GENERATED_FIELD_HEADER.clone(), - HeaderValue::try_from(timestamp_key.to_string())?, - ); - } - - if let Some(azure_resource_id) = &config.azure_resource_id { - if azure_resource_id.is_empty() { - return Err("azure_resource_id can't be an empty string".into()); - } - - default_headers.insert( - X_MS_AZURE_RESOURCE_HEADER.clone(), - HeaderValue::from_str(azure_resource_id)?, - ); - } - - default_headers.insert(header::CONTENT_TYPE, CONTENT_TYPE_VALUE.clone()); - - Ok(AzureMonitorLogsSink { - uri, - transformer: config.encoding.clone(), - customer_id: config.customer_id.clone(), - shared_key, - default_headers, - time_generated_key, - }) - } - - fn build_request_sync(&self, events: Vec) -> crate::Result> { - let body = crate::serde::json::to_bytes(&events)?.freeze(); - let len = body.len(); - - let mut request = Request::post(self.uri.clone()).body(body)?; - let rfc1123date = chrono::Utc::now() - .format("%a, %d %b %Y %H:%M:%S GMT") - .to_string(); - - let authorization = self.build_authorization_header_value(&rfc1123date, len)?; - - *request.headers_mut() = self.default_headers.clone(); - request - .headers_mut() - .insert(header::AUTHORIZATION, authorization.parse()?); - request - .headers_mut() - .insert(X_MS_DATE_HEADER.clone(), rfc1123date.parse()?); - - Ok(request) - } - - fn build_authorization_header_value( - &self, - rfc1123date: &str, - len: usize, - ) -> crate::Result { - let string_to_hash = format!( - "POST\n{}\n{}\n{}:{}\n{}", - len, CONTENT_TYPE, X_MS_DATE, rfc1123date, RESOURCE - ); - let mut signer = sign::Signer::new(hash::MessageDigest::sha256(), &self.shared_key)?; - signer.update(string_to_hash.as_bytes())?; - - let signature = signer.sign_to_vec()?; - let signature_base64 = base64::encode_block(&signature); - - Ok(format!( - "{} {}:{}", - SHARED_KEY, self.customer_id, signature_base64 - )) - } -} - -async fn healthcheck(sink: AzureMonitorLogsSink, client: HttpClient) -> crate::Result<()> { - let request = sink.build_request(vec![]).await?.map(Body::from); - - let res = client.send(request).await?; - - if res.status().is_server_error() { - return Err("Server returned a server error".into()); - } - - if res.status() == StatusCode::FORBIDDEN { - return Err("The service failed to authenticate the request. Verify that the workspace ID and connection key are valid".into()); - } - - if res.status() == StatusCode::NOT_FOUND { - return Err("Either the URL provided is incorrect, or the request is too large".into()); - } - - if res.status() == StatusCode::BAD_REQUEST { - return Err("The workspace has been closed or the request was invalid".into()); - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use futures::{future::ready, stream}; - use serde_json::value::RawValue; - - use super::*; - use crate::{ - event::LogEvent, - sinks::util::BatchSize, - test_util::{ - components::{run_and_assert_sink_compliance, SINK_TAGS}, - http::{always_200_response, spawn_blackhole_http_server}, - }, - }; - - #[test] - fn generate_config() { - crate::test_util::test_generate_config::(); - } - - #[tokio::test] - async fn component_spec_compliance() { - let mock_endpoint = spawn_blackhole_http_server(always_200_response).await; - - // This is just a dummy shared key. - let shared_key_bytes = base64::decode_block( - "ZnNkO2Zhc2RrbGZqYXNkaixmaG5tZXF3dWlsamtmYXNjZmouYXNkbmZrbHFhc2ZtYXNrbA==", - ) - .expect("should not fail to decode base64"); - let shared_key = - pkey::PKey::hmac(&shared_key_bytes).expect("should not fail to create HMAC key"); - - let sink = AzureMonitorLogsSink { - uri: mock_endpoint, - customer_id: "weee".to_string(), - time_generated_key: log_schema().timestamp_key().cloned(), - transformer: Default::default(), - shared_key, - default_headers: HeaderMap::new(), - }; - - let context = SinkContext::default(); - let client = - HttpClient::new(None, &context.proxy).expect("should not fail to create HTTP client"); - - let request_settings = - TowerRequestConfig::default().unwrap_with(&TowerRequestConfig::default()); - - let sink = BatchedHttpSink::new( - sink, - JsonArrayBuffer::new(BatchSize::const_default()), - request_settings, - Duration::from_secs(1), - client, - ) - .sink_map_err(|error| error!(message = "Fatal azure_monitor_logs sink error.", %error)); - - let event = Event::Log(LogEvent::from("simple message")); - #[allow(deprecated)] - run_and_assert_sink_compliance( - VectorSink::from_event_sink(sink), - stream::once(ready(event)), - &SINK_TAGS, - ) - .await; - } - - fn insert_timestamp_kv(log: &mut LogEvent) -> (String, String) { - let now = chrono::Utc::now(); - - let timestamp_key = log_schema().timestamp_key().unwrap(); - let timestamp_value = now.to_rfc3339_opts(chrono::SecondsFormat::Millis, true); - log.insert((PathPrefix::Event, timestamp_key), now); - - (timestamp_key.to_string(), timestamp_value) - } - - #[test] - fn encode_valid() { - let config: AzureMonitorLogsConfig = toml::from_str( - r#" - # random GUID and random 64 Base-64 encoded bytes - customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - shared_key = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" - log_type = "Vector" - "#, - ) - .unwrap(); - - let sink = AzureMonitorLogsSink::new(&config, None).unwrap(); - let mut log = [("message", "hello world")] - .iter() - .copied() - .collect::(); - let (timestamp_key, timestamp_value) = insert_timestamp_kv(&mut log); - - let event = Event::from(log); - let mut encoder = sink.build_encoder(); - let json = encoder.encode_event(event).unwrap(); - let expected_json = serde_json::json!({ - timestamp_key: timestamp_value, - "message": "hello world" - }); - assert_eq!(json, expected_json); - } - - #[test] - fn correct_request() { - let config: AzureMonitorLogsConfig = toml::from_str( - r#" - # random GUID and random 64 Base-64 encoded bytes - customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - shared_key = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" - log_type = "Vector" - azure_resource_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - "#, - ) - .unwrap(); - - let sink = AzureMonitorLogsSink::new(&config, None).unwrap(); - let mut encoder = sink.build_encoder(); - - let mut log1 = [("message", "hello")].iter().copied().collect::(); - let (timestamp_key1, timestamp_value1) = insert_timestamp_kv(&mut log1); - - let mut log2 = [("message", "world")].iter().copied().collect::(); - let (timestamp_key2, timestamp_value2) = insert_timestamp_kv(&mut log2); - - let event1 = encoder.encode_event(Event::from(log1)).unwrap(); - let event2 = encoder.encode_event(Event::from(log2)).unwrap(); - - let json1 = serde_json::to_string(&event1).unwrap(); - let json2 = serde_json::to_string(&event2).unwrap(); - let raw1 = RawValue::from_string(json1).unwrap(); - let raw2 = RawValue::from_string(json2).unwrap(); - - let events = vec![raw1, raw2]; - - let request = sink.build_request_sync(events); - - let (parts, body) = request.unwrap().into_parts(); - assert_eq!(&parts.method.to_string(), "POST"); - - let json: serde_json::Value = serde_json::from_slice(&body[..]).unwrap(); - let expected_json = serde_json::json!([ - { - timestamp_key1: timestamp_value1, - "message": "hello" - }, - { - timestamp_key2: timestamp_value2, - "message": "world" - } - ]); - assert_eq!(json, expected_json); - - let headers = parts.headers; - let rfc1123date = headers.get("x-ms-date").unwrap(); - - let auth_expected = sink - .build_authorization_header_value(rfc1123date.to_str().unwrap(), body.len()) - .unwrap(); - - let authorization = headers.get("authorization").unwrap(); - assert_eq!(authorization.to_str().unwrap(), &auth_expected); - - let log_type = headers.get("log-type").unwrap(); - assert_eq!(log_type.to_str().unwrap(), "Vector"); - - let time_generated_field = headers.get("time-generated-field").unwrap(); - let timestamp_key = log_schema().timestamp_key(); - assert_eq!( - time_generated_field.to_str().unwrap(), - timestamp_key.unwrap().to_string().as_str() - ); - - let azure_resource_id = headers.get("x-ms-azureresourceid").unwrap(); - assert_eq!( - azure_resource_id.to_str().unwrap(), - "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - ); - - assert_eq!( - &parts.uri.to_string(), - "https://97ce69d9-b4be-4241-8dbd-d265edcf06c4.ods.opinsights.azure.com/api/logs?api-version=2016-04-01" - ); - } - - #[tokio::test] - async fn fails_missing_creds() { - let config: AzureMonitorLogsConfig = toml::from_str( - r#" - customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - shared_key = "" - log_type = "Vector" - azure_resource_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - "#, - ) - .unwrap(); - if config.build(SinkContext::default()).await.is_ok() { - panic!("config.build failed to error"); - } - } - - #[test] - fn correct_host() { - let config_default = toml::from_str::( - r#" - customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - shared_key = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" - log_type = "Vector" - "#, - ) - .expect("Config parsing failed without custom host"); - assert_eq!(config_default.host, default_host()); - - let config_cn = toml::from_str::( - r#" - customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - shared_key = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" - log_type = "Vector" - host = "ods.opinsights.azure.cn" - "#, - ) - .expect("Config parsing failed with .cn custom host"); - assert_eq!(config_cn.host, "ods.opinsights.azure.cn"); - } - - #[tokio::test] - async fn fails_invalid_base64() { - let config: AzureMonitorLogsConfig = toml::from_str( - r#" - customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - shared_key = "1Qs77Vz40+iDMBBTRmROKJwnEX" - log_type = "Vector" - azure_resource_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - "#, - ) - .unwrap(); - if config.build(SinkContext::default()).await.is_ok() { - panic!("config.build failed to error"); - } - } - - #[test] - fn fails_config_missing_fields() { - toml::from_str::( - r#" - customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - shared_key = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" - azure_resource_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - "#, - ) - .expect_err("Config parsing failed to error with missing log_type"); - - toml::from_str::( - r#" - customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - log_type = "Vector" - azure_resource_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" - "#, - ) - .expect_err("Config parsing failed to error with missing shared_key"); - - toml::from_str::( - r#" - shared_key = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" - log_type = "Vector" - "#, - ) - .expect_err("Config parsing failed to error with missing customer_id"); - } -} diff --git a/src/sinks/azure_monitor_logs/config.rs b/src/sinks/azure_monitor_logs/config.rs new file mode 100644 index 0000000000000..7a48b88c7b82b --- /dev/null +++ b/src/sinks/azure_monitor_logs/config.rs @@ -0,0 +1,225 @@ +use lookup::{lookup_v2::OptionalValuePath, OwnedValuePath}; +use openssl::{base64, pkey}; + +use vector_common::sensitive_string::SensitiveString; +use vector_config::configurable_component; +use vector_core::{config::log_schema, schema}; +use vrl::value::Kind; + +use crate::{ + http::{get_http_scheme_from_uri, HttpClient}, + sinks::{ + prelude::*, + util::{http::HttpStatusRetryLogic, RealtimeSizeBasedDefaultBatchSettings, UriSerde}, + }, +}; + +use super::{ + service::{AzureMonitorLogsResponse, AzureMonitorLogsService}, + sink::AzureMonitorLogsSink, +}; + +/// Max number of bytes in request body +const MAX_BATCH_SIZE: usize = 30 * 1024 * 1024; + +pub(super) fn default_host() -> String { + "ods.opinsights.azure.com".into() +} + +/// Configuration for the `azure_monitor_logs` sink. +#[configurable_component(sink( + "azure_monitor_logs", + "Publish log events to the Azure Monitor Logs service." +))] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct AzureMonitorLogsConfig { + /// The [unique identifier][uniq_id] for the Log Analytics workspace. + /// + /// [uniq_id]: https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api#request-uri-parameters + #[configurable(metadata(docs::examples = "5ce893d9-2c32-4b6c-91a9-b0887c2de2d6"))] + #[configurable(metadata(docs::examples = "97ce69d9-b4be-4241-8dbd-d265edcf06c4"))] + pub customer_id: String, + + /// The [primary or the secondary key][shared_key] for the Log Analytics workspace. + /// + /// [shared_key]: https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api#authorization + #[configurable(metadata( + docs::examples = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" + ))] + #[configurable(metadata(docs::examples = "${AZURE_MONITOR_SHARED_KEY_ENV_VAR}"))] + pub shared_key: SensitiveString, + + /// The [record type][record_type] of the data that is being submitted. + /// + /// Can only contain letters, numbers, and underscores (_), and may not exceed 100 characters. + /// + /// [record_type]: https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api#request-headers + #[configurable(validation(pattern = "[a-zA-Z0-9_]{1,100}"))] + #[configurable(metadata(docs::examples = "MyTableName"))] + #[configurable(metadata(docs::examples = "MyRecordType"))] + pub log_type: String, + + /// The [Resource ID][resource_id] of the Azure resource the data should be associated with. + /// + /// [resource_id]: https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api#request-headers + #[configurable(metadata( + docs::examples = "/subscriptions/11111111-1111-1111-1111-111111111111/resourceGroups/otherResourceGroup/providers/Microsoft.Storage/storageAccounts/examplestorage" + ))] + #[configurable(metadata( + docs::examples = "/subscriptions/11111111-1111-1111-1111-111111111111/resourceGroups/examplegroup/providers/Microsoft.SQL/servers/serverName/databases/databaseName" + ))] + pub azure_resource_id: Option, + + /// [Alternative host][alt_host] for dedicated Azure regions. + /// + /// [alt_host]: https://docs.azure.cn/en-us/articles/guidance/developerdifferences#check-endpoints-in-azure + #[configurable(metadata(docs::examples = "ods.opinsights.azure.us"))] + #[configurable(metadata(docs::examples = "ods.opinsights.azure.cn"))] + #[serde(default = "default_host")] + pub(super) host: String, + + #[configurable(derived)] + #[serde( + default, + skip_serializing_if = "crate::serde::skip_serializing_if_default" + )] + pub encoding: Transformer, + + #[configurable(derived)] + #[serde(default)] + pub batch: BatchConfig, + + #[configurable(derived)] + #[serde(default)] + pub request: TowerRequestConfig, + + /// Use this option to customize the log field used as [`TimeGenerated`][1] in Azure. + /// + /// The setting of `log_schema.timestamp_key`, usually `timestamp`, is used here by default. + /// This field should be used in rare cases where `TimeGenerated` should point to a specific log + /// field. For example, use this field to set the log field `source_timestamp` as holding the + /// value that should be used as `TimeGenerated` on the Azure side. + /// + /// [1]: https://learn.microsoft.com/en-us/azure/azure-monitor/logs/log-standard-columns#timegenerated + #[configurable(metadata(docs::examples = "time_generated"))] + pub time_generated_key: Option, + + #[configurable(derived)] + pub tls: Option, + + #[configurable(derived)] + #[serde( + default, + deserialize_with = "crate::serde::bool_or_struct", + skip_serializing_if = "crate::serde::skip_serializing_if_default" + )] + pub acknowledgements: AcknowledgementsConfig, +} + +impl Default for AzureMonitorLogsConfig { + fn default() -> Self { + Self { + customer_id: "my-customer-id".to_string(), + shared_key: Default::default(), + log_type: "MyRecordType".to_string(), + azure_resource_id: None, + host: default_host(), + encoding: Default::default(), + batch: Default::default(), + request: Default::default(), + time_generated_key: None, + tls: None, + acknowledgements: Default::default(), + } + } +} + +impl AzureMonitorLogsConfig { + pub(super) fn build_shared_key(&self) -> crate::Result> { + if self.shared_key.inner().is_empty() { + return Err("shared_key cannot be an empty string".into()); + } + let shared_key_bytes = base64::decode_block(self.shared_key.inner())?; + let shared_key = pkey::PKey::hmac(&shared_key_bytes)?; + Ok(shared_key) + } + + fn get_time_generated_key(&self) -> Option { + self.time_generated_key + .clone() + .and_then(|k| k.path) + .or_else(|| log_schema().timestamp_key().cloned()) + } + + pub(super) async fn build_inner( + &self, + cx: SinkContext, + endpoint: UriSerde, + ) -> crate::Result<(VectorSink, Healthcheck)> { + let endpoint = endpoint.with_default_parts().uri; + let protocol = get_http_scheme_from_uri(&endpoint).to_string(); + + let batch_settings = self + .batch + .validate()? + .limit_max_bytes(MAX_BATCH_SIZE)? + .into_batcher_settings()?; + + let shared_key = self.build_shared_key()?; + let time_generated_key = self.get_time_generated_key(); + + let tls_settings = TlsSettings::from_options(&self.tls)?; + let client = HttpClient::new(Some(tls_settings), &cx.proxy)?; + + let service = AzureMonitorLogsService::new( + client, + endpoint, + self.customer_id.clone(), + self.azure_resource_id.as_deref(), + &self.log_type, + time_generated_key.clone(), + shared_key, + )?; + let healthcheck = service.healthcheck(); + + let retry_logic = + HttpStatusRetryLogic::new(|res: &AzureMonitorLogsResponse| res.http_status); + let request_settings = self.request.unwrap_with(&Default::default()); + let service = ServiceBuilder::new() + .settings(request_settings, retry_logic) + .service(service); + + let sink = AzureMonitorLogsSink::new( + batch_settings, + self.encoding.clone(), + service, + time_generated_key, + protocol, + ); + + Ok((VectorSink::from_event_streamsink(sink), healthcheck)) + } +} + +impl_generate_config_from_default!(AzureMonitorLogsConfig); + +#[async_trait::async_trait] +#[typetag::serde(name = "azure_monitor_logs")] +impl SinkConfig for AzureMonitorLogsConfig { + async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { + let endpoint = format!("https://{}.{}", self.customer_id, self.host).parse()?; + self.build_inner(cx, endpoint).await + } + + fn input(&self) -> Input { + let requirements = + schema::Requirement::empty().optional_meaning("timestamp", Kind::timestamp()); + + Input::log().with_schema_requirement(requirements) + } + + fn acknowledgements(&self) -> &AcknowledgementsConfig { + &self.acknowledgements + } +} diff --git a/src/sinks/azure_monitor_logs/mod.rs b/src/sinks/azure_monitor_logs/mod.rs new file mode 100644 index 0000000000000..e025b912f9b13 --- /dev/null +++ b/src/sinks/azure_monitor_logs/mod.rs @@ -0,0 +1,13 @@ +//! The Azure Monitor Logs [`vector_core::sink::VectorSink`] +//! +//! This module contains the [`vector_core::sink::VectorSink`] instance that is responsible for +//! taking a stream of [`vector_core::event::Event`] instances and forwarding them to the Azure +//! Monitor Logs service. + +mod config; +mod service; +mod sink; +#[cfg(test)] +mod tests; + +pub use config::AzureMonitorLogsConfig; diff --git a/src/sinks/azure_monitor_logs/service.rs b/src/sinks/azure_monitor_logs/service.rs new file mode 100644 index 0000000000000..285edd09e9902 --- /dev/null +++ b/src/sinks/azure_monitor_logs/service.rs @@ -0,0 +1,249 @@ +use bytes::Bytes; +use http::{ + header::{self, HeaderMap}, + HeaderName, HeaderValue, Request, StatusCode, Uri, +}; +use hyper::Body; +use lookup::lookup_v2::OwnedValuePath; +use once_cell::sync::Lazy; +use openssl::{base64, hash, pkey, sign}; +use regex::Regex; +use std::task::{Context, Poll}; +use tracing::Instrument; + +use crate::{http::HttpClient, sinks::prelude::*}; + +static LOG_TYPE_REGEX: Lazy = Lazy::new(|| Regex::new(r"^\w+$").unwrap()); +static LOG_TYPE_HEADER: Lazy = Lazy::new(|| HeaderName::from_static("log-type")); +static X_MS_DATE_HEADER: Lazy = Lazy::new(|| HeaderName::from_static(X_MS_DATE)); +static X_MS_AZURE_RESOURCE_HEADER: Lazy = + Lazy::new(|| HeaderName::from_static("x-ms-azureresourceid")); +static TIME_GENERATED_FIELD_HEADER: Lazy = + Lazy::new(|| HeaderName::from_static("time-generated-field")); +static CONTENT_TYPE_VALUE: Lazy = Lazy::new(|| HeaderValue::from_static(CONTENT_TYPE)); + +/// API endpoint for submitting logs +const RESOURCE: &str = "/api/logs"; +/// JSON content type of logs +const CONTENT_TYPE: &str = "application/json"; +/// Custom header used for signing logs +const X_MS_DATE: &str = "x-ms-date"; +/// Shared key prefix +const SHARED_KEY: &str = "SharedKey"; +/// API version +const API_VERSION: &str = "2016-04-01"; + +#[derive(Debug, Clone)] +pub struct AzureMonitorLogsRequest { + pub body: Bytes, + pub finalizers: EventFinalizers, + pub metadata: RequestMetadata, +} + +impl MetaDescriptive for AzureMonitorLogsRequest { + fn get_metadata(&self) -> &RequestMetadata { + &self.metadata + } + + fn metadata_mut(&mut self) -> &mut RequestMetadata { + &mut self.metadata + } +} + +impl Finalizable for AzureMonitorLogsRequest { + fn take_finalizers(&mut self) -> EventFinalizers { + self.finalizers.take_finalizers() + } +} + +pub struct AzureMonitorLogsResponse { + pub http_status: StatusCode, + pub events_byte_size: GroupedCountByteSize, + pub raw_byte_size: usize, +} + +impl DriverResponse for AzureMonitorLogsResponse { + fn event_status(&self) -> EventStatus { + match self.http_status.is_success() { + true => EventStatus::Delivered, + false => EventStatus::Rejected, + } + } + + fn events_sent(&self) -> &GroupedCountByteSize { + &self.events_byte_size + } + + fn bytes_sent(&self) -> Option { + Some(self.raw_byte_size) + } +} + +/// `AzureMonitorLogsService` is a `Tower` service used to send logs to Azure. +#[derive(Debug, Clone)] +pub struct AzureMonitorLogsService { + client: HttpClient, + endpoint: Uri, + customer_id: String, + shared_key: pkey::PKey, + default_headers: HeaderMap, +} + +impl AzureMonitorLogsService { + /// Creates a new `AzureMonitorLogsService`. + pub fn new( + client: HttpClient, + endpoint: Uri, + customer_id: String, + azure_resource_id: Option<&str>, + log_type: &str, + time_generated_key: Option, + shared_key: pkey::PKey, + ) -> crate::Result { + let mut parts = endpoint.into_parts(); + parts.path_and_query = Some( + format!("{RESOURCE}?api-version={API_VERSION}") + .parse() + .expect("path and query should never fail to parse"), + ); + let endpoint = Uri::from_parts(parts)?; + + let default_headers = { + let mut headers = HeaderMap::new(); + + if log_type.len() > 100 || !LOG_TYPE_REGEX.is_match(log_type) { + return Err(format!( + "invalid log_type \"{}\": log type can only contain letters, numbers, and underscore (_), and may not exceed 100 characters", + log_type + ).into()); + } + let log_type = HeaderValue::from_str(log_type)?; + headers.insert(LOG_TYPE_HEADER.clone(), log_type); + + if let Some(timestamp_key) = time_generated_key { + headers.insert( + TIME_GENERATED_FIELD_HEADER.clone(), + HeaderValue::try_from(timestamp_key.to_string())?, + ); + } + + if let Some(azure_resource_id) = azure_resource_id { + if azure_resource_id.is_empty() { + return Err("azure_resource_id can't be an empty string".into()); + } + headers.insert( + X_MS_AZURE_RESOURCE_HEADER.clone(), + HeaderValue::from_str(azure_resource_id)?, + ); + } + + headers.insert(header::CONTENT_TYPE, CONTENT_TYPE_VALUE.clone()); + headers + }; + + Ok(Self { + client, + endpoint, + customer_id, + shared_key, + default_headers, + }) + } + + fn build_authorization_header_value( + &self, + rfc1123date: &str, + len: usize, + ) -> crate::Result { + let string_to_hash = + format!("POST\n{len}\n{CONTENT_TYPE}\n{X_MS_DATE}:{rfc1123date}\n{RESOURCE}"); + let mut signer = sign::Signer::new(hash::MessageDigest::sha256(), &self.shared_key)?; + signer.update(string_to_hash.as_bytes())?; + + let signature = signer.sign_to_vec()?; + let signature_base64 = base64::encode_block(&signature); + + Ok(format!( + "{} {}:{}", + SHARED_KEY, self.customer_id, signature_base64 + )) + } + + fn build_request(&self, body: Bytes) -> crate::Result> { + let len = body.len(); + + let mut request = Request::post(&self.endpoint).body(Body::from(body))?; + + let rfc1123date = chrono::Utc::now() + .format("%a, %d %b %Y %H:%M:%S GMT") + .to_string(); + let authorization = self.build_authorization_header_value(&rfc1123date, len)?; + + *request.headers_mut() = self.default_headers.clone(); + request + .headers_mut() + .insert(header::AUTHORIZATION, authorization.parse()?); + request + .headers_mut() + .insert(X_MS_DATE_HEADER.clone(), rfc1123date.parse()?); + + Ok(request) + } + + pub fn healthcheck(&self) -> Healthcheck { + let mut client = self.client.clone(); + let request = self.build_request(Bytes::from("[]")); + Box::pin(async move { + let request = request?; + let res = client.call(request).in_current_span().await?; + + if res.status().is_server_error() { + return Err("Server returned a server error".into()); + } + + if res.status() == StatusCode::FORBIDDEN { + return Err("The service failed to authenticate the request. Verify that the workspace ID and connection key are valid".into()); + } + + if res.status() == StatusCode::NOT_FOUND { + return Err( + "Either the URL provided is incorrect, or the request is too large".into(), + ); + } + + if res.status() == StatusCode::BAD_REQUEST { + return Err("The workspace has been closed or the request was invalid".into()); + } + + Ok(()) + }) + } +} + +impl Service for AzureMonitorLogsService { + type Response = AzureMonitorLogsResponse; + type Error = crate::Error; + type Future = BoxFuture<'static, Result>; + + // Emission of Error internal event is handled upstream by the caller. + fn poll_ready(&mut self, _cx: &mut Context) -> Poll> { + Poll::Ready(Ok(())) + } + + // Emission of Error internal event is handled upstream by the caller. + fn call(&mut self, request: AzureMonitorLogsRequest) -> Self::Future { + let mut client = self.client.clone(); + let http_request = self.build_request(request.body); + Box::pin(async move { + let http_request = http_request?; + let response = client.call(http_request).in_current_span().await?; + Ok(AzureMonitorLogsResponse { + http_status: response.status(), + raw_byte_size: request.metadata.request_encoded_size(), + events_byte_size: request + .metadata + .into_events_estimated_json_encoded_byte_size(), + }) + }) + } +} diff --git a/src/sinks/azure_monitor_logs/sink.rs b/src/sinks/azure_monitor_logs/sink.rs new file mode 100644 index 0000000000000..7b610cedebaa9 --- /dev/null +++ b/src/sinks/azure_monitor_logs/sink.rs @@ -0,0 +1,176 @@ +use std::{fmt::Debug, io}; + +use bytes::Bytes; +use codecs::{encoding::Framer, CharacterDelimitedEncoder, JsonSerializerConfig}; +use lookup::{OwnedValuePath, PathPrefix}; + +use crate::sinks::prelude::*; + +use super::service::AzureMonitorLogsRequest; + +pub struct AzureMonitorLogsSink { + batch_settings: BatcherSettings, + encoding: JsonEncoding, + service: S, + protocol: String, +} + +impl AzureMonitorLogsSink +where + S: Service + Send + 'static, + S::Future: Send + 'static, + S::Response: DriverResponse + Send + 'static, + S::Error: Debug + Into + Send, +{ + pub fn new( + batch_settings: BatcherSettings, + transformer: Transformer, + service: S, + time_generated_key: Option, + protocol: String, + ) -> Self { + Self { + batch_settings, + encoding: JsonEncoding::new(transformer, time_generated_key), + service, + protocol, + } + } + + async fn run_inner(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { + input + .batched(self.batch_settings.into_byte_size_config()) + .request_builder( + None, + AzureMonitorLogsRequestBuilder { + encoding: self.encoding, + }, + ) + .filter_map(|request| async { + match request { + Err(error) => { + emit!(SinkRequestBuildError { error }); + None + } + Ok(req) => Some(req), + } + }) + .into_driver(self.service) + .protocol(self.protocol.clone()) + .run() + .await + } +} + +#[async_trait::async_trait] +impl StreamSink for AzureMonitorLogsSink +where + S: Service + Send + 'static, + S::Future: Send + 'static, + S::Response: DriverResponse + Send + 'static, + S::Error: Debug + Into + Send, +{ + async fn run( + self: Box, + input: futures_util::stream::BoxStream<'_, Event>, + ) -> Result<(), ()> { + self.run_inner(input).await + } +} + +/// Customized encoding specific to the Azure Monitor Logs sink, as the API does not support full +/// 9-digit nanosecond precision timestamps. +#[derive(Clone, Debug)] +pub(super) struct JsonEncoding { + time_generated_key: Option, + encoder: (Transformer, Encoder), +} + +impl JsonEncoding { + pub fn new(transformer: Transformer, time_generated_key: Option) -> Self { + Self { + time_generated_key, + encoder: ( + transformer, + Encoder::::new( + CharacterDelimitedEncoder::new(b',').into(), + JsonSerializerConfig::default().build().into(), + ), + ), + } + } +} + +impl crate::sinks::util::encoding::Encoder> for JsonEncoding { + fn encode_input( + &self, + mut input: Vec, + writer: &mut dyn io::Write, + ) -> io::Result<(usize, GroupedCountByteSize)> { + for event in input.iter_mut() { + let log = event.as_mut_log(); + + // `.remove_timestamp()` will return the `timestamp` value regardless of location in Event or + // Metadata, the following `insert()` ensures it's encoded in the request. + let timestamp = if let Some(Value::Timestamp(ts)) = log.remove_timestamp() { + ts + } else { + chrono::Utc::now() + }; + + if let Some(timestamp_key) = &self.time_generated_key { + log.insert( + (PathPrefix::Event, timestamp_key), + serde_json::Value::String( + timestamp.to_rfc3339_opts(chrono::SecondsFormat::Millis, true), + ), + ); + } + } + + self.encoder.encode_input(input, writer) + } +} + +struct AzureMonitorLogsRequestBuilder { + encoding: JsonEncoding, +} + +impl RequestBuilder> for AzureMonitorLogsRequestBuilder { + type Metadata = EventFinalizers; + type Events = Vec; + type Encoder = JsonEncoding; + type Payload = Bytes; + type Request = AzureMonitorLogsRequest; + type Error = std::io::Error; + + fn compression(&self) -> Compression { + Compression::None + } + + fn encoder(&self) -> &Self::Encoder { + &self.encoding + } + + fn split_input( + &self, + mut events: Vec, + ) -> (Self::Metadata, RequestMetadataBuilder, Self::Events) { + let finalizers = events.take_finalizers(); + let builder = RequestMetadataBuilder::from_events(&events); + (finalizers, builder, events) + } + + fn build_request( + &self, + finalizers: Self::Metadata, + request_metadata: RequestMetadata, + payload: EncodeResult, + ) -> Self::Request { + AzureMonitorLogsRequest { + body: payload.into_payload(), + finalizers, + metadata: request_metadata, + } + } +} diff --git a/src/sinks/azure_monitor_logs/tests.rs b/src/sinks/azure_monitor_logs/tests.rs new file mode 100644 index 0000000000000..a28e84e47b3be --- /dev/null +++ b/src/sinks/azure_monitor_logs/tests.rs @@ -0,0 +1,276 @@ +use std::time::Duration; + +use futures::{future::ready, stream}; +use http::Response; +use hyper::body; +use openssl::{base64, hash, pkey, sign}; +use tokio::time::timeout; +use vector_core::config::log_schema; + +use super::{ + config::{default_host, AzureMonitorLogsConfig}, + sink::JsonEncoding, +}; +use crate::{ + event::LogEvent, + sinks::{prelude::*, util::encoding::Encoder}, + test_util::{ + components::{run_and_assert_sink_compliance, SINK_TAGS}, + http::{always_200_response, spawn_blackhole_http_server}, + }, +}; + +#[test] +fn generate_config() { + crate::test_util::test_generate_config::(); +} + +#[tokio::test] +async fn component_spec_compliance() { + let mock_endpoint = spawn_blackhole_http_server(always_200_response).await; + + let config = AzureMonitorLogsConfig { + shared_key: "ZnNkO2Zhc2RrbGZqYXNkaixmaG5tZXF3dWlsamtmYXNjZmouYXNkbmZrbHFhc2ZtYXNrbA==" + .to_string() + .into(), + ..Default::default() + }; + + let context = SinkContext::default(); + let (sink, _healthcheck) = config + .build_inner(context, mock_endpoint.into()) + .await + .unwrap(); + + let event = Event::Log(LogEvent::from("simple message")); + run_and_assert_sink_compliance(sink, stream::once(ready(event)), &SINK_TAGS).await; +} + +#[tokio::test] +async fn fails_missing_creds() { + let config: AzureMonitorLogsConfig = toml::from_str( + r#" + customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + shared_key = "" + log_type = "Vector" + azure_resource_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + "#, + ) + .unwrap(); + if config.build(SinkContext::default()).await.is_ok() { + panic!("config.build failed to error"); + } +} + +#[test] +fn correct_host() { + let config_default = toml::from_str::( + r#" + customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + shared_key = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" + log_type = "Vector" + "#, + ) + .expect("Config parsing failed without custom host"); + assert_eq!(config_default.host, default_host()); + + let config_cn = toml::from_str::( + r#" + customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + shared_key = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" + log_type = "Vector" + host = "ods.opinsights.azure.cn" + "#, + ) + .expect("Config parsing failed with .cn custom host"); + assert_eq!(config_cn.host, "ods.opinsights.azure.cn"); +} + +#[tokio::test] +async fn fails_invalid_base64() { + let config: AzureMonitorLogsConfig = toml::from_str( + r#" + customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + shared_key = "1Qs77Vz40+iDMBBTRmROKJwnEX" + log_type = "Vector" + azure_resource_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + "#, + ) + .unwrap(); + if config.build(SinkContext::default()).await.is_ok() { + panic!("config.build failed to error"); + } +} + +#[test] +fn fails_config_missing_fields() { + toml::from_str::( + r#" + customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + shared_key = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" + azure_resource_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + "#, + ) + .expect_err("Config parsing failed to error with missing log_type"); + + toml::from_str::( + r#" + customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + log_type = "Vector" + azure_resource_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + "#, + ) + .expect_err("Config parsing failed to error with missing shared_key"); + + toml::from_str::( + r#" + shared_key = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" + log_type = "Vector" + "#, + ) + .expect_err("Config parsing failed to error with missing customer_id"); +} + +fn insert_timestamp_kv(log: &mut LogEvent) -> (String, String) { + let now = chrono::Utc::now(); + + let timestamp_value = now.to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + log.insert(log_schema().timestamp_key_target_path().unwrap(), now); + + ( + log_schema().timestamp_key().unwrap().to_string(), + timestamp_value, + ) +} + +fn build_authorization_header_value( + shared_key: &pkey::PKey, + customer_id: &str, + rfc1123date: &str, + len: usize, +) -> crate::Result { + let string_to_hash = + format!("POST\n{len}\napplication/json\nx-ms-date:{rfc1123date}\n/api/logs"); + let mut signer = sign::Signer::new(hash::MessageDigest::sha256(), shared_key)?; + signer.update(string_to_hash.as_bytes())?; + + let signature = signer.sign_to_vec()?; + let signature_base64 = base64::encode_block(&signature); + + Ok(format!("SharedKey {customer_id}:{signature_base64}")) +} + +#[tokio::test] +async fn correct_request() { + let config: AzureMonitorLogsConfig = toml::from_str( + r#" + # random GUID and random 64 Base-64 encoded bytes + customer_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + shared_key = "SERsIYhgMVlJB6uPsq49gCxNiruf6v0vhMYE+lfzbSGcXjdViZdV/e5pEMTYtw9f8SkVLf4LFlLCc2KxtRZfCA==" + log_type = "Vector" + azure_resource_id = "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + "#, + ) + .unwrap(); + + let mut log1 = [("message", "hello")].iter().copied().collect::(); + let (timestamp_key1, timestamp_value1) = insert_timestamp_kv(&mut log1); + + let mut log2 = [("message", "world")].iter().copied().collect::(); + let (timestamp_key2, timestamp_value2) = insert_timestamp_kv(&mut log2); + + let (tx, mut rx) = tokio::sync::mpsc::channel(1); + let mock_endpoint = spawn_blackhole_http_server(move |request| { + let tx = tx.clone(); + async move { + tx.send(request).await.unwrap(); + Ok(Response::new(hyper::Body::empty())) + } + }) + .await; + + let context = SinkContext::default(); + let (sink, _healthcheck) = config + .build_inner(context, mock_endpoint.into()) + .await + .unwrap(); + + run_and_assert_sink_compliance(sink, stream::iter(vec![log1, log2]), &SINK_TAGS).await; + + let request = timeout(Duration::from_millis(500), rx.recv()) + .await + .unwrap() + .unwrap(); + + let (parts, body) = request.into_parts(); + assert_eq!(&parts.method.to_string(), "POST"); + + let body = body::to_bytes(body).await.unwrap(); + let json: serde_json::Value = serde_json::from_slice(&body[..]).unwrap(); + let expected_json = serde_json::json!([ + { + timestamp_key1: timestamp_value1, + "message": "hello" + }, + { + timestamp_key2: timestamp_value2, + "message": "world" + } + ]); + assert_eq!(json, expected_json); + + let headers = parts.headers; + + let rfc1123date = headers.get("x-ms-date").unwrap(); + let shared_key = config.build_shared_key().unwrap(); + let auth_expected = build_authorization_header_value( + &shared_key, + &config.customer_id, + rfc1123date.to_str().unwrap(), + body.len(), + ) + .unwrap(); + let authorization = headers.get("authorization").unwrap(); + assert_eq!(authorization.to_str().unwrap(), &auth_expected); + + let log_type = headers.get("log-type").unwrap(); + assert_eq!(log_type.to_str().unwrap(), "Vector"); + + let time_generated_field = headers.get("time-generated-field").unwrap(); + let timestamp_key = log_schema().timestamp_key(); + assert_eq!( + time_generated_field.to_str().unwrap(), + timestamp_key.unwrap().to_string().as_str() + ); + + let azure_resource_id = headers.get("x-ms-azureresourceid").unwrap(); + assert_eq!( + azure_resource_id.to_str().unwrap(), + "97ce69d9-b4be-4241-8dbd-d265edcf06c4" + ); + + assert_eq!( + &parts.uri.path_and_query().unwrap().to_string(), + "/api/logs?api-version=2016-04-01" + ); +} + +#[test] +fn encode_valid() { + let mut log = [("message", "hello world")] + .iter() + .copied() + .collect::(); + let (timestamp_key, timestamp_value) = insert_timestamp_kv(&mut log); + + let event = Event::from(log); + let encoder = JsonEncoding::new(Default::default(), log_schema().timestamp_key().cloned()); + let mut encoded = vec![]; + encoder.encode_input(vec![event], &mut encoded).unwrap(); + let expected_json = serde_json::json!([{ + timestamp_key: timestamp_value, + "message": "hello world" + }]); + let json: serde_json::Value = serde_json::from_slice(&encoded).unwrap(); + assert_eq!(json, expected_json); +} diff --git a/src/sinks/datadog/traces/apm_stats/aggregation.rs b/src/sinks/datadog/traces/apm_stats/aggregation.rs index 9ae1ae6e50cc5..2c7d3a580e6ad 100644 --- a/src/sinks/datadog/traces/apm_stats/aggregation.rs +++ b/src/sinks/datadog/traces/apm_stats/aggregation.rs @@ -1,6 +1,7 @@ use std::{collections::BTreeMap, sync::Arc}; use chrono::Utc; +use vrl::event_path; use super::{ bucket::Bucket, ClientStatsBucket, ClientStatsPayload, PartitionKey, @@ -179,7 +180,7 @@ impl Aggregator { pub(crate) fn handle_trace(&mut self, partition_key: &PartitionKey, trace: &TraceEvent) { // Based on https://github.com/DataDog/datadog-agent/blob/cfa750c7412faa98e87a015f8ee670e5828bbe7f/pkg/trace/stats/concentrator.go#L148-L184 - let spans = match trace.get("spans") { + let spans = match trace.get(event_path!("spans")) { Some(Value::Array(v)) => v.iter().filter_map(|s| s.as_object()).collect(), _ => vec![], }; @@ -189,16 +190,16 @@ impl Aggregator { env: partition_key.env.clone().unwrap_or_default(), hostname: partition_key.hostname.clone().unwrap_or_default(), version: trace - .get("app_version") + .get(event_path!("app_version")) .map(|v| v.to_string_lossy().into_owned()) .unwrap_or_default(), container_id: trace - .get("container_id") + .get(event_path!("container_id")) .map(|v| v.to_string_lossy().into_owned()) .unwrap_or_default(), }; let synthetics = trace - .get("origin") + .get(event_path!("origin")) .map(|v| v.to_string_lossy().starts_with(TAG_SYNTHETICS)) .unwrap_or(false); diff --git a/src/sinks/datadog/traces/request_builder.rs b/src/sinks/datadog/traces/request_builder.rs index 68081627cc76a..64fa9f0d9c7ed 100644 --- a/src/sinks/datadog/traces/request_builder.rs +++ b/src/sinks/datadog/traces/request_builder.rs @@ -10,6 +10,7 @@ use prost::Message; use snafu::Snafu; use vector_common::request_metadata::RequestMetadata; use vector_core::event::{EventFinalizers, Finalizable}; +use vrl::event_path; use super::{ apm_stats::{compute_apm_stats, Aggregator}, @@ -283,7 +284,7 @@ impl DatadogTracesEncoder { fn vector_trace_into_dd_tracer_payload(trace: &TraceEvent) -> dd_proto::TracerPayload { let tags = trace - .get("tags") + .get(event_path!("tags")) .and_then(|m| m.as_object()) .map(|m| { m.iter() @@ -292,7 +293,7 @@ impl DatadogTracesEncoder { }) .unwrap_or_default(); - let spans = match trace.get("spans") { + let spans = match trace.get(event_path!("spans")) { Some(Value::Array(v)) => v .iter() .filter_map(|s| s.as_object().map(DatadogTracesEncoder::convert_span)) @@ -302,7 +303,7 @@ impl DatadogTracesEncoder { let chunk = dd_proto::TraceChunk { priority: trace - .get("priority") + .get(event_path!("priority")) .and_then(|v| v.as_integer().map(|v| v as i32)) // This should not happen for Datadog originated traces, but in case this field is not populated // we default to 1 (https://github.com/DataDog/datadog-agent/blob/eac2327/pkg/trace/sampler/sampler.go#L54-L55), @@ -310,11 +311,11 @@ impl DatadogTracesEncoder { // https://github.com/DataDog/datadog-agent/blob/3ea2eb4/pkg/trace/api/otlp.go#L309. .unwrap_or(1i32), origin: trace - .get("origin") + .get(event_path!("origin")) .map(|v| v.to_string_lossy().into_owned()) .unwrap_or_default(), dropped_trace: trace - .get("dropped") + .get(event_path!("dropped")) .and_then(|v| v.as_boolean()) .unwrap_or(false), spans, @@ -323,37 +324,37 @@ impl DatadogTracesEncoder { dd_proto::TracerPayload { container_id: trace - .get("container_id") + .get(event_path!("container_id")) .map(|v| v.to_string_lossy().into_owned()) .unwrap_or_default(), language_name: trace - .get("language_name") + .get(event_path!("language_name")) .map(|v| v.to_string_lossy().into_owned()) .unwrap_or_default(), language_version: trace - .get("language_version") + .get(event_path!("language_version")) .map(|v| v.to_string_lossy().into_owned()) .unwrap_or_default(), tracer_version: trace - .get("tracer_version") + .get(event_path!("tracer_version")) .map(|v| v.to_string_lossy().into_owned()) .unwrap_or_default(), runtime_id: trace - .get("runtime_id") + .get(event_path!("runtime_id")) .map(|v| v.to_string_lossy().into_owned()) .unwrap_or_default(), chunks: vec![chunk], tags, env: trace - .get("env") + .get(event_path!("env")) .map(|v| v.to_string_lossy().into_owned()) .unwrap_or_default(), hostname: trace - .get("hostname") + .get(event_path!("hostname")) .map(|v| v.to_string_lossy().into_owned()) .unwrap_or_default(), app_version: trace - .get("app_version") + .get(event_path!("app_version")) .map(|v| v.to_string_lossy().into_owned()) .unwrap_or_default(), } diff --git a/src/sinks/datadog/traces/sink.rs b/src/sinks/datadog/traces/sink.rs index fa1e2cc3f2280..4cf502a7eebae 100644 --- a/src/sinks/datadog/traces/sink.rs +++ b/src/sinks/datadog/traces/sink.rs @@ -7,6 +7,7 @@ use futures_util::{ }; use tokio::sync::oneshot::{channel, Sender}; use tower::Service; +use vrl::event_path; use vrl::path::PathPrefix; use vector_core::{ @@ -54,19 +55,21 @@ impl Partitioner for EventPartitioner { } Event::Trace(t) => PartitionKey { api_key: item.metadata().datadog_api_key(), - env: t.get("env").map(|s| s.to_string_lossy().into_owned()), + env: t + .get(event_path!("env")) + .map(|s| s.to_string_lossy().into_owned()), hostname: log_schema().host_key().and_then(|key| { t.get((PathPrefix::Event, key)) .map(|s| s.to_string_lossy().into_owned()) }), agent_version: t - .get("agent_version") + .get(event_path!("agent_version")) .map(|s| s.to_string_lossy().into_owned()), target_tps: t - .get("target_tps") + .get(event_path!("target_tps")) .and_then(|tps| tps.as_integer().map(Into::into)), error_tps: t - .get("error_tps") + .get(event_path!("error_tps")) .and_then(|tps| tps.as_integer().map(Into::into)), }, } diff --git a/src/sinks/elasticsearch/config.rs b/src/sinks/elasticsearch/config.rs index 9f50d79e44a2d..d84153fb2a899 100644 --- a/src/sinks/elasticsearch/config.rs +++ b/src/sinks/elasticsearch/config.rs @@ -443,7 +443,9 @@ impl DataStreamConfig { let (dtype, dataset, namespace) = if !self.auto_routing { (self.dtype(log)?, self.dataset(log)?, self.namespace(log)?) } else { - let data_stream = log.get("data_stream").and_then(|ds| ds.as_object()); + let data_stream = log + .get(event_path!("data_stream")) + .and_then(|ds| ds.as_object()); let dtype = data_stream .and_then(|ds| ds.get("type")) .map(|value| value.to_string_lossy().into_owned()) diff --git a/src/sinks/humio/logs.rs b/src/sinks/humio/logs.rs index a1d48088deddc..617bc9785ec89 100644 --- a/src/sinks/humio/logs.rs +++ b/src/sinks/humio/logs.rs @@ -223,15 +223,13 @@ mod tests { #[cfg(test)] #[cfg(feature = "humio-integration-tests")] mod integration_tests { - use std::{collections::HashMap, convert::TryFrom}; - use chrono::{TimeZone, Utc}; use futures::{future::ready, stream}; use indoc::indoc; use serde::Deserialize; use serde_json::{json, Value as JsonValue}; + use std::{collections::HashMap, convert::TryFrom}; use tokio::time::Duration; - use vrl::path::PathPrefix; use super::*; use crate::{ @@ -263,10 +261,7 @@ mod integration_tests { let message = random_string(100); let host = "192.168.1.1".to_string(); let mut event = LogEvent::from(message.clone()); - event.insert( - (PathPrefix::Event, log_schema().host_key().unwrap()), - host.clone(), - ); + event.insert(log_schema().host_key_target_path().unwrap(), host.clone()); let ts = Utc.timestamp_nanos(Utc::now().timestamp_millis() * 1_000_000 + 132_456); event.insert(log_schema().timestamp_key_target_path().unwrap(), ts); diff --git a/src/sinks/influxdb/logs.rs b/src/sinks/influxdb/logs.rs index 777f8b7d903d7..3f4ead23ca46b 100644 --- a/src/sinks/influxdb/logs.rs +++ b/src/sinks/influxdb/logs.rs @@ -4,6 +4,7 @@ use bytes::{Bytes, BytesMut}; use futures::SinkExt; use http::{Request, Uri}; use indoc::indoc; +use vrl::event_path; use vrl::path::OwnedValuePath; use vrl::value::Kind; @@ -280,7 +281,7 @@ impl HttpEventEncoder for InfluxDbLogsEncoder { } self.tags.replace("metric_type".to_string()); - log.insert("metric_type", "logs"); + log.insert(event_path!("metric_type"), "logs"); // Timestamp let timestamp = encode_timestamp(match log.remove_timestamp() { diff --git a/src/sinks/new_relic/model.rs b/src/sinks/new_relic/model.rs index c49337a13e3e4..27cf7292dcdbf 100644 --- a/src/sinks/new_relic/model.rs +++ b/src/sinks/new_relic/model.rs @@ -3,6 +3,7 @@ use std::{collections::HashMap, convert::TryFrom, fmt::Debug, time::SystemTime}; use chrono::{DateTime, Utc}; use ordered_float::NotNan; use serde::{Deserialize, Serialize}; +use vrl::event_path; use super::NewRelicSinkError; use crate::event::{Event, MetricValue, Value}; @@ -117,7 +118,7 @@ impl TryFrom> for EventsApiModel { event_model.insert(k, v.clone()); } - if let Some(message) = log.get("message") { + if let Some(message) = log.get(event_path!("message")) { let message = message.to_string_lossy().replace("\\\"", "\""); // If message contains a JSON string, parse it and insert all fields into self if let serde_json::Result::Ok(json_map) = @@ -189,7 +190,7 @@ impl TryFrom> for LogsApiModel { for (k, v) in log.convert_to_fields() { log_model.insert(k, v.clone()); } - if log.get("message").is_none() { + if log.get(event_path!("message")).is_none() { log_model.insert( "message".to_owned(), Value::from("log from vector".to_owned()), diff --git a/src/sinks/util/adaptive_concurrency/controller.rs b/src/sinks/util/adaptive_concurrency/controller.rs index 83250994bf752..19615f7ba2742 100644 --- a/src/sinks/util/adaptive_concurrency/controller.rs +++ b/src/sinks/util/adaptive_concurrency/controller.rs @@ -69,7 +69,7 @@ impl Controller { // current limit and the maximum, effectively bypassing all the // mechanisms. Otherwise, the current limit is set to 1 and the // maximum to MAX_CONCURRENCY. - let current_limit = concurrency.unwrap_or(1); + let current_limit = concurrency.unwrap_or(settings.initial_concurrency); Self { semaphore: Arc::new(ShrinkableSemaphore::new(current_limit)), concurrency, diff --git a/src/sinks/util/adaptive_concurrency/mod.rs b/src/sinks/util/adaptive_concurrency/mod.rs index a9795f70f2957..228d93d4e28b8 100644 --- a/src/sinks/util/adaptive_concurrency/mod.rs +++ b/src/sinks/util/adaptive_concurrency/mod.rs @@ -9,6 +9,8 @@ mod service; #[cfg(test)] pub mod tests; +// Make sure to update the max range of the `AdaptiveConcurrencySettings::initial_concurrency` when changing +// this constant. pub(super) const MAX_CONCURRENCY: usize = 200; pub(crate) use layer::AdaptiveConcurrencyLimitLayer; @@ -29,6 +31,15 @@ pub(self) fn instant_now() -> std::time::Instant { #[derive(Clone, Copy, Debug)] #[serde(deny_unknown_fields)] pub struct AdaptiveConcurrencySettings { + /// The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + /// + /// It is recommended to set this value to your service's average limit if you're seeing that it takes a + /// long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + /// `adaptive_concurrency_limit` metric. + #[configurable(validation(range(min = 1, max = 200)))] + #[serde(default = "default_initial_concurrency")] + pub(super) initial_concurrency: usize, + /// The fraction of the current value to set the new concurrency limit when decreasing the limit. /// /// Valid values are greater than `0` and less than `1`. Smaller values cause the algorithm to scale back rapidly @@ -63,6 +74,10 @@ pub struct AdaptiveConcurrencySettings { pub(super) rtt_deviation_scale: f64, } +const fn default_initial_concurrency() -> usize { + 1 +} + const fn default_decrease_ratio() -> f64 { 0.9 } @@ -84,6 +99,7 @@ impl AdaptiveConcurrencySettings { impl Default for AdaptiveConcurrencySettings { fn default() -> Self { Self { + initial_concurrency: default_initial_concurrency(), decrease_ratio: default_decrease_ratio(), ewma_alpha: default_ewma_alpha(), rtt_deviation_scale: default_rtt_deviation_scale(), diff --git a/src/sinks/util/buffer/compression.rs b/src/sinks/util/buffer/compression.rs index 8171e4a563238..11796b046104a 100644 --- a/src/sinks/util/buffer/compression.rs +++ b/src/sinks/util/buffer/compression.rs @@ -431,7 +431,21 @@ impl<'de> de::Deserialize<'de> for CompressionLevel { where E: de::Error, { - Ok(CompressionLevel::Val(v as u32)) + u32::try_from(v).map(CompressionLevel::Val).map_err(|err| { + de::Error::custom(format!( + "unsigned integer could not be converted to u32: {}", + err + )) + }) + } + + fn visit_i64(self, v: i64) -> Result + where + E: de::Error, + { + u32::try_from(v).map(CompressionLevel::Val).map_err(|err| { + de::Error::custom(format!("integer could not be converted to u32: {}", err)) + }) } } @@ -490,7 +504,7 @@ mod test { use super::{Compression, CompressionLevel}; #[test] - fn deserialization() { + fn deserialization_json() { let fixtures_valid = [ (r#""none""#, Compression::None), (r#""gzip""#, Compression::Gzip(CompressionLevel::default())), @@ -545,7 +559,7 @@ mod test { ), ( r#"{"algorithm": "gzip", "level": -1}"#, - r#"invalid type: integer `-1`, expected unsigned number or string at line 1 column 33"#, + r#"integer could not be converted to u32: out of range integral type conversion attempted at line 1 column 33"#, ), ( r#"{"algorithm": "gzip", "level": "good"}"#, @@ -575,6 +589,22 @@ mod test { } } + #[test] + fn deserialization_toml() { + let fixtures_valid = [ + // TOML differs from YAML and JSON by always parsing integers as signed + ( + r#"algorithm = "gzip" + level = 8"#, + Compression::Gzip(CompressionLevel::Val(8)), + ), + ]; + for (sources, result) in fixtures_valid.iter() { + let deserialized: Result = toml::from_str(sources); + assert_eq!(deserialized.expect("valid source"), *result); + } + } + #[test] fn from_and_to_value() { let fixtures_valid = [ diff --git a/src/sources/datadog_agent/traces.rs b/src/sources/datadog_agent/traces.rs index ec5a56636169f..bd9099c13a2ab 100644 --- a/src/sources/datadog_agent/traces.rs +++ b/src/sources/datadog_agent/traces.rs @@ -146,16 +146,16 @@ fn handle_dd_trace_payload_v1( &source.log_schema_source_type_key, Bytes::from("datadog_agent"), ); - trace_event.insert("payload_version", "v2".to_string()); + trace_event.insert(event_path!("payload_version"), "v2".to_string()); trace_event.insert(&source.log_schema_host_key, hostname.clone()); - trace_event.insert("env", env.clone()); - trace_event.insert("agent_version", agent_version.clone()); - trace_event.insert("target_tps", target_tps); - trace_event.insert("error_tps", error_tps); - if let Some(Value::Object(span_tags)) = trace_event.get_mut("tags") { + trace_event.insert(event_path!("env"), env.clone()); + trace_event.insert(event_path!("agent_version"), agent_version.clone()); + trace_event.insert(event_path!("target_tps"), target_tps); + trace_event.insert(event_path!("error_tps"), error_tps); + if let Some(Value::Object(span_tags)) = trace_event.get_mut(event_path!("tags")) { span_tags.extend(tags.clone()); } else { - trace_event.insert("tags", Value::from(tags.clone())); + trace_event.insert(event_path!("tags"), Value::from(tags.clone())); } Event::Trace(trace_event) }) diff --git a/src/sources/dnstap/mod.rs b/src/sources/dnstap/mod.rs index 529abc5c7134d..44d213847ce6a 100644 --- a/src/sources/dnstap/mod.rs +++ b/src/sources/dnstap/mod.rs @@ -7,6 +7,7 @@ use vector_common::internal_event::{ ByteSize, BytesReceived, InternalEventHandle as _, Protocol, Registered, }; use vector_config::configurable_component; +use vrl::event_path; use vrl::value::{kind::Collection, Kind}; use super::util::framestream::{build_framestream_unix_source, FrameHandler}; @@ -280,7 +281,7 @@ impl FrameHandler for DnstapFrameHandler { if self.raw_data_only { log_event.insert( - self.schema.dnstap_root_data_schema().raw_data(), + event_path!(self.schema.dnstap_root_data_schema().raw_data()), BASE64_STANDARD.encode(&frame), ); } else if let Err(err) = parse_dnstap_data(&self.schema, &mut log_event, frame) { diff --git a/src/sources/kafka.rs b/src/sources/kafka.rs index 1ccdb41bd3248..3bfbd0f09e366 100644 --- a/src/sources/kafka.rs +++ b/src/sources/kafka.rs @@ -23,7 +23,6 @@ use rdkafka::{ use serde_with::serde_as; use snafu::{ResultExt, Snafu}; use tokio_util::codec::FramedRead; -use vrl::path::PathPrefix; use vector_common::finalizer::OrderedFinalizer; use vector_config::configurable_component; @@ -604,11 +603,8 @@ impl ReceivedMessage { ); } LogNamespace::Legacy => { - if let Some(source_type_key) = log_schema().source_type_key() { - log.insert( - (PathPrefix::Event, source_type_key), - KafkaSourceConfig::NAME, - ); + if let Some(source_type_key) = log_schema().source_type_key_target_path() { + log.insert(source_type_key, KafkaSourceConfig::NAME); } } } @@ -924,7 +920,7 @@ mod integration_test { use tokio::time::sleep; use vector_buffers::topology::channel::BufferReceiver; use vector_core::event::EventStatus; - use vrl::value; + use vrl::{event_path, value}; use super::{test::*, *}; use crate::{ @@ -1132,7 +1128,7 @@ mod integration_test { let recv = BufferReceiver::new(recv.into()).into_stream(); let recv = recv.then(move |mut events| async move { events.iter_logs_mut().for_each(|log| { - log.insert("pipeline_id", id.to_string()); + log.insert(event_path!("pipeline_id"), id.to_string()); }); sleep(delay).await; events.iter_events_mut().for_each(|mut event| { diff --git a/src/sources/logstash.rs b/src/sources/logstash.rs index 1230884a4d951..ec3a038bd9739 100644 --- a/src/sources/logstash.rs +++ b/src/sources/logstash.rs @@ -9,7 +9,7 @@ use std::{ use bytes::{Buf, Bytes, BytesMut}; use codecs::{BytesDeserializerConfig, StreamDecodingError}; use flate2::read::ZlibDecoder; -use lookup::{event_path, metadata_path, owned_value_path, path, OwnedValuePath, PathPrefix}; +use lookup::{event_path, metadata_path, owned_value_path, path, OwnedValuePath}; use smallvec::{smallvec, SmallVec}; use snafu::{ResultExt, Snafu}; use tokio_util::codec::Decoder; @@ -232,9 +232,9 @@ impl TcpSource for LogstashSource { log.insert(metadata_path!("vector", "ingest_timestamp"), now); } LogNamespace::Legacy => { - if let Some(timestamp_key) = log_schema().timestamp_key() { + if let Some(timestamp_key) = log_schema().timestamp_key_target_path() { log.insert( - (PathPrefix::Event, timestamp_key), + timestamp_key, log_timestamp.unwrap_or_else(|| Value::from(now)), ); } diff --git a/src/sources/syslog.rs b/src/sources/syslog.rs index 0150da7f34466..c674939daef23 100644 --- a/src/sources/syslog.rs +++ b/src/sources/syslog.rs @@ -423,9 +423,7 @@ fn enrich_syslog_event( .get("timestamp") .and_then(|timestamp| timestamp.as_timestamp().cloned()) .unwrap_or_else(Utc::now); - if let Some(timestamp_key) = log_schema().timestamp_key_target_path() { - log.insert(timestamp_key, timestamp); - } + log.maybe_insert(log_schema().timestamp_key_target_path(), timestamp); } trace!( diff --git a/src/sources/util/framestream.rs b/src/sources/util/framestream.rs index 6208dc09425b5..287fb0e583585 100644 --- a/src/sources/util/framestream.rs +++ b/src/sources/util/framestream.rs @@ -666,7 +666,7 @@ mod test { let mut log_event = LogEvent::from(frame); log_event.insert( - log_schema().source_type_key().unwrap().to_string().as_str(), + log_schema().source_type_key_target_path().unwrap(), "framestream", ); if let Some(host) = received_from { diff --git a/src/sources/vector/mod.rs b/src/sources/vector/mod.rs index 4c4a746f5d886..7c680b4f9a49d 100644 --- a/src/sources/vector/mod.rs +++ b/src/sources/vector/mod.rs @@ -181,7 +181,9 @@ impl SourceConfig for VectorConfig { acknowledgements, log_namespace, }) - .accept_compressed(tonic::codec::CompressionEncoding::Gzip); + .accept_compressed(tonic::codec::CompressionEncoding::Gzip) + // Tonic added a default of 4MB in 0.9. This replaces the old behavior. + .max_decoding_message_size(usize::MAX); let source = run_grpc_server(self.address, tls_settings, service, cx.shutdown).map_err(|error| { @@ -273,16 +275,14 @@ mod test { #[cfg(feature = "sinks-vector")] #[cfg(test)] mod tests { - use vector_common::assert_event_data_eq; - use vector_core::config::log_schema; - use vrl::path::PathPrefix; - use super::*; use crate::{ config::{SinkConfig as _, SinkContext}, sinks::vector::VectorConfig as SinkConfig, test_util, SourceSender, }; + use vector_common::assert_event_data_eq; + use vector_core::config::log_schema; async fn run_test(vector_source_config_str: &str, addr: SocketAddr) { let config = format!(r#"address = "{}""#, addr); @@ -306,11 +306,11 @@ mod tests { let (mut events, stream) = test_util::random_events_with_stream(100, 100, None); sink.run(stream).await.unwrap(); - let source_type_key = log_schema().source_type_key().unwrap(); for event in &mut events { - event - .as_mut_log() - .insert((PathPrefix::Event, source_type_key), "vector"); + event.as_mut_log().insert( + log_schema().source_type_key_target_path().unwrap(), + "vector", + ); } let output = test_util::collect_ready(rx).await; diff --git a/src/template.rs b/src/template.rs index b4230f351b12e..570b27a55fa0a 100644 --- a/src/template.rs +++ b/src/template.rs @@ -167,13 +167,17 @@ impl Template { Part::Reference(key) => { out.push_str( &match event { - EventRef::Log(log) => log.get(&**key).map(Value::to_string_lossy), + EventRef::Log(log) => log + .parse_path_and_get_value(key) + .ok() + .and_then(|v| v.map(Value::to_string_lossy)), EventRef::Metric(metric) => { render_metric_field(key, metric).map(Cow::Borrowed) } - EventRef::Trace(trace) => { - trace.get(key.as_str()).map(Value::to_string_lossy) - } + EventRef::Trace(trace) => trace + .parse_path_and_get_value(key) + .ok() + .and_then(|v| v.map(Value::to_string_lossy)), } .unwrap_or_else(|| { missing_keys.push(key.to_owned()); @@ -499,13 +503,9 @@ mod tests { .expect("invalid timestamp"); let mut event = Event::Log(LogEvent::from("hello world")); - event.as_mut_log().insert( - ( - lookup::PathPrefix::Event, - log_schema().timestamp_key().unwrap(), - ), - ts, - ); + event + .as_mut_log() + .insert(log_schema().timestamp_key_target_path().unwrap(), ts); let template = Template::try_from("abcd-%F").unwrap(); @@ -520,13 +520,9 @@ mod tests { .expect("invalid timestamp"); let mut event = Event::Log(LogEvent::from("hello world")); - event.as_mut_log().insert( - ( - lookup::PathPrefix::Event, - log_schema().timestamp_key().unwrap(), - ), - ts, - ); + event + .as_mut_log() + .insert(log_schema().timestamp_key_target_path().unwrap(), ts); let template = Template::try_from("abcd-%F_%T").unwrap(); diff --git a/src/transforms/sample.rs b/src/transforms/sample.rs index 53ca4847f5723..a27b57f2cca9c 100644 --- a/src/transforms/sample.rs +++ b/src/transforms/sample.rs @@ -130,8 +130,14 @@ impl FunctionTransform for Sample { .key_field .as_ref() .and_then(|key_field| match &event { - Event::Log(event) => event.get(key_field.as_str()), - Event::Trace(event) => event.get(key_field.as_str()), + Event::Log(event) => event + .parse_path_and_get_value(key_field.as_str()) + .ok() + .flatten(), + Event::Trace(event) => event + .parse_path_and_get_value(key_field.as_str()) + .ok() + .flatten(), Event::Metric(_) => panic!("component can never receive metric events"), }) .map(|v| v.to_string_lossy()); diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index b2a6bb534194e..e9055c676dc1c 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -12,7 +12,7 @@ anyhow = "1.0.72" atty = "0.2.14" cached = "0.44.0" chrono = { version = "0.4.22", default-features = false, features = ["serde", "clock"] } -clap = { version = "4.1.14", features = ["derive"] } +clap = { version = "4.3.21", features = ["derive"] } clap-verbosity-flag = "2.0.1" clap_complete = "4.3.2" confy = "0.5.1" @@ -22,7 +22,7 @@ dunce = "1.0.4" glob = { version = "0.3.1", default-features = false } hex = "0.4.3" indexmap = { version = "2.0", default-features = false, features = ["serde", "std"] } -indicatif = { version = "0.17.5", features = ["improved_unicode"] } +indicatif = { version = "0.17.6", features = ["improved_unicode"] } itertools = "0.11.0" log = "0.4.19" once_cell = "1.18" diff --git a/website/cue/reference/components/sinks/base/appsignal.cue b/website/cue/reference/components/sinks/base/appsignal.cue index 79b5a26ff04cd..f3d81555a59d5 100644 --- a/website/cue/reference/components/sinks/base/appsignal.cue +++ b/website/cue/reference/components/sinks/base/appsignal.cue @@ -170,6 +170,17 @@ base: components: sinks: appsignal: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index 4cbe09f99f3b0..c0549fec49341 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -417,6 +417,17 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue index 1f54161e12839..c39bc5392be01 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue @@ -258,6 +258,17 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue index a43ed56875662..d5774f897043b 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue @@ -388,6 +388,17 @@ base: components: sinks: aws_kinesis_firehose: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue index 92ed82abd7849..77edb8cb46aa8 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue @@ -397,6 +397,17 @@ base: components: sinks: aws_kinesis_streams: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index 2f31fdbd5b3a6..1facd8fd4d40c 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -634,6 +634,17 @@ base: components: sinks: aws_s3: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/aws_sqs.cue b/website/cue/reference/components/sinks/base/aws_sqs.cue index 07bdf7373a655..ac6787c61fc41 100644 --- a/website/cue/reference/components/sinks/base/aws_sqs.cue +++ b/website/cue/reference/components/sinks/base/aws_sqs.cue @@ -350,6 +350,17 @@ base: components: sinks: aws_sqs: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/axiom.cue b/website/cue/reference/components/sinks/base/axiom.cue index bb619bc886098..82a857624b19f 100644 --- a/website/cue/reference/components/sinks/base/axiom.cue +++ b/website/cue/reference/components/sinks/base/axiom.cue @@ -108,6 +108,17 @@ base: components: sinks: axiom: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/azure_blob.cue b/website/cue/reference/components/sinks/base/azure_blob.cue index 5c2568145a597..46a1a6fbe5cf2 100644 --- a/website/cue/reference/components/sinks/base/azure_blob.cue +++ b/website/cue/reference/components/sinks/base/azure_blob.cue @@ -382,6 +382,17 @@ base: components: sinks: azure_blob: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/azure_monitor_logs.cue b/website/cue/reference/components/sinks/base/azure_monitor_logs.cue index 76de4da2dc5ac..aafbe16e318ab 100644 --- a/website/cue/reference/components/sinks/base/azure_monitor_logs.cue +++ b/website/cue/reference/components/sinks/base/azure_monitor_logs.cue @@ -166,6 +166,17 @@ base: components: sinks: azure_monitor_logs: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/clickhouse.cue b/website/cue/reference/components/sinks/base/clickhouse.cue index 6bccc5dfe1968..458cf35a21e81 100644 --- a/website/cue/reference/components/sinks/base/clickhouse.cue +++ b/website/cue/reference/components/sinks/base/clickhouse.cue @@ -219,6 +219,17 @@ base: components: sinks: clickhouse: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/databend.cue b/website/cue/reference/components/sinks/base/databend.cue index 069eb227207f7..2da07c23af456 100644 --- a/website/cue/reference/components/sinks/base/databend.cue +++ b/website/cue/reference/components/sinks/base/databend.cue @@ -258,6 +258,17 @@ base: components: sinks: databend: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/datadog_events.cue b/website/cue/reference/components/sinks/base/datadog_events.cue index 445c879d9d210..27eeab0f24410 100644 --- a/website/cue/reference/components/sinks/base/datadog_events.cue +++ b/website/cue/reference/components/sinks/base/datadog_events.cue @@ -103,6 +103,17 @@ base: components: sinks: datadog_events: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/datadog_logs.cue b/website/cue/reference/components/sinks/base/datadog_logs.cue index 852c972024102..71aa7a783d344 100644 --- a/website/cue/reference/components/sinks/base/datadog_logs.cue +++ b/website/cue/reference/components/sinks/base/datadog_logs.cue @@ -184,6 +184,17 @@ base: components: sinks: datadog_logs: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/datadog_metrics.cue b/website/cue/reference/components/sinks/base/datadog_metrics.cue index 2d95089686fa3..e79fc5d01943d 100644 --- a/website/cue/reference/components/sinks/base/datadog_metrics.cue +++ b/website/cue/reference/components/sinks/base/datadog_metrics.cue @@ -145,6 +145,17 @@ base: components: sinks: datadog_metrics: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/datadog_traces.cue b/website/cue/reference/components/sinks/base/datadog_traces.cue index 91ea597ae84be..8f8fd899d4121 100644 --- a/website/cue/reference/components/sinks/base/datadog_traces.cue +++ b/website/cue/reference/components/sinks/base/datadog_traces.cue @@ -154,6 +154,17 @@ base: components: sinks: datadog_traces: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/elasticsearch.cue b/website/cue/reference/components/sinks/base/elasticsearch.cue index 31ec23da07b00..56432ddfb10ad 100644 --- a/website/cue/reference/components/sinks/base/elasticsearch.cue +++ b/website/cue/reference/components/sinks/base/elasticsearch.cue @@ -571,6 +571,17 @@ base: components: sinks: elasticsearch: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue index 82d963fc5da3a..382162e99084c 100644 --- a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue +++ b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue @@ -306,6 +306,17 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue index 89b2f37613e0e..636acde2cdbea 100644 --- a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue +++ b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue @@ -465,6 +465,17 @@ base: components: sinks: gcp_cloud_storage: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/gcp_pubsub.cue b/website/cue/reference/components/sinks/base/gcp_pubsub.cue index 9c45e6195401c..e63e3e1afd92e 100644 --- a/website/cue/reference/components/sinks/base/gcp_pubsub.cue +++ b/website/cue/reference/components/sinks/base/gcp_pubsub.cue @@ -297,6 +297,17 @@ base: components: sinks: gcp_pubsub: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/gcp_stackdriver_logs.cue b/website/cue/reference/components/sinks/base/gcp_stackdriver_logs.cue index 646e9d3ec69a2..3da5fba5f8e65 100644 --- a/website/cue/reference/components/sinks/base/gcp_stackdriver_logs.cue +++ b/website/cue/reference/components/sinks/base/gcp_stackdriver_logs.cue @@ -212,6 +212,17 @@ base: components: sinks: gcp_stackdriver_logs: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/gcp_stackdriver_metrics.cue b/website/cue/reference/components/sinks/base/gcp_stackdriver_metrics.cue index 6f58d57ca46a9..49531983a3e51 100644 --- a/website/cue/reference/components/sinks/base/gcp_stackdriver_metrics.cue +++ b/website/cue/reference/components/sinks/base/gcp_stackdriver_metrics.cue @@ -154,6 +154,17 @@ base: components: sinks: gcp_stackdriver_metrics: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/greptimedb.cue b/website/cue/reference/components/sinks/base/greptimedb.cue index 6f25208e64bad..2334ba9cde8f8 100644 --- a/website/cue/reference/components/sinks/base/greptimedb.cue +++ b/website/cue/reference/components/sinks/base/greptimedb.cue @@ -142,6 +142,17 @@ base: components: sinks: greptimedb: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/honeycomb.cue b/website/cue/reference/components/sinks/base/honeycomb.cue index dbd1e0ab1f2a7..7fdcc3e59527a 100644 --- a/website/cue/reference/components/sinks/base/honeycomb.cue +++ b/website/cue/reference/components/sinks/base/honeycomb.cue @@ -135,6 +135,17 @@ base: components: sinks: honeycomb: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/http.cue b/website/cue/reference/components/sinks/base/http.cue index c737b39f58f72..36381b0119f58 100644 --- a/website/cue/reference/components/sinks/base/http.cue +++ b/website/cue/reference/components/sinks/base/http.cue @@ -403,6 +403,17 @@ base: components: sinks: http: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/humio_logs.cue b/website/cue/reference/components/sinks/base/humio_logs.cue index 8d8253417eb4b..6ad5779de9eb4 100644 --- a/website/cue/reference/components/sinks/base/humio_logs.cue +++ b/website/cue/reference/components/sinks/base/humio_logs.cue @@ -341,6 +341,17 @@ base: components: sinks: humio_logs: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/humio_metrics.cue b/website/cue/reference/components/sinks/base/humio_metrics.cue index b919e8c7b8f2e..9f7dc55bdd459 100644 --- a/website/cue/reference/components/sinks/base/humio_metrics.cue +++ b/website/cue/reference/components/sinks/base/humio_metrics.cue @@ -237,6 +237,17 @@ base: components: sinks: humio_metrics: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/influxdb_logs.cue b/website/cue/reference/components/sinks/base/influxdb_logs.cue index 21fc5d8ee2508..1d367fbd27f31 100644 --- a/website/cue/reference/components/sinks/base/influxdb_logs.cue +++ b/website/cue/reference/components/sinks/base/influxdb_logs.cue @@ -215,6 +215,17 @@ base: components: sinks: influxdb_logs: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/influxdb_metrics.cue b/website/cue/reference/components/sinks/base/influxdb_metrics.cue index 42e444f63c55f..b221c7e3db25e 100644 --- a/website/cue/reference/components/sinks/base/influxdb_metrics.cue +++ b/website/cue/reference/components/sinks/base/influxdb_metrics.cue @@ -173,6 +173,17 @@ base: components: sinks: influxdb_metrics: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/logdna.cue b/website/cue/reference/components/sinks/base/logdna.cue index 18b5ed6655acd..e2a03080cca13 100644 --- a/website/cue/reference/components/sinks/base/logdna.cue +++ b/website/cue/reference/components/sinks/base/logdna.cue @@ -178,6 +178,17 @@ base: components: sinks: logdna: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/loki.cue b/website/cue/reference/components/sinks/base/loki.cue index 0071c5b2ac985..dd02a5c16d792 100644 --- a/website/cue/reference/components/sinks/base/loki.cue +++ b/website/cue/reference/components/sinks/base/loki.cue @@ -407,6 +407,17 @@ base: components: sinks: loki: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/mezmo.cue b/website/cue/reference/components/sinks/base/mezmo.cue index 31907546e45e2..4ac6e3d290656 100644 --- a/website/cue/reference/components/sinks/base/mezmo.cue +++ b/website/cue/reference/components/sinks/base/mezmo.cue @@ -178,6 +178,17 @@ base: components: sinks: mezmo: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/new_relic.cue b/website/cue/reference/components/sinks/base/new_relic.cue index 9fcf952c07d8c..4d6cbc8d4ff6d 100644 --- a/website/cue/reference/components/sinks/base/new_relic.cue +++ b/website/cue/reference/components/sinks/base/new_relic.cue @@ -184,6 +184,17 @@ base: components: sinks: new_relic: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue index 1141e9e39dce4..78ff004317f56 100644 --- a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue +++ b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue @@ -322,6 +322,17 @@ base: components: sinks: prometheus_remote_write: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/redis.cue b/website/cue/reference/components/sinks/base/redis.cue index 774370ce58a00..f432ea7ff7fc3 100644 --- a/website/cue/reference/components/sinks/base/redis.cue +++ b/website/cue/reference/components/sinks/base/redis.cue @@ -301,6 +301,17 @@ base: components: sinks: redis: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/sematext_logs.cue b/website/cue/reference/components/sinks/base/sematext_logs.cue index 7d0290d630297..bc1f06127e696 100644 --- a/website/cue/reference/components/sinks/base/sematext_logs.cue +++ b/website/cue/reference/components/sinks/base/sematext_logs.cue @@ -145,6 +145,17 @@ base: components: sinks: sematext_logs: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/sematext_metrics.cue b/website/cue/reference/components/sinks/base/sematext_metrics.cue index 7befa370214de..1322b8bd26072 100644 --- a/website/cue/reference/components/sinks/base/sematext_metrics.cue +++ b/website/cue/reference/components/sinks/base/sematext_metrics.cue @@ -131,6 +131,17 @@ base: components: sinks: sematext_metrics: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue index 184db138643cc..ab946626c47ab 100644 --- a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue +++ b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue @@ -393,6 +393,17 @@ base: components: sinks: splunk_hec_logs: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/splunk_hec_metrics.cue b/website/cue/reference/components/sinks/base/splunk_hec_metrics.cue index ce147a1055624..f7f203a0f6944 100644 --- a/website/cue/reference/components/sinks/base/splunk_hec_metrics.cue +++ b/website/cue/reference/components/sinks/base/splunk_hec_metrics.cue @@ -211,6 +211,17 @@ base: components: sinks: splunk_hec_metrics: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/components/sinks/base/vector.cue b/website/cue/reference/components/sinks/base/vector.cue index b9b65a08b157e..9798d965b59d6 100644 --- a/website/cue/reference/components/sinks/base/vector.cue +++ b/website/cue/reference/components/sinks/base/vector.cue @@ -123,6 +123,17 @@ base: components: sinks: vector: configuration: { required: false type: float: default: 0.4 } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. diff --git a/website/cue/reference/remap/concepts/function.cue b/website/cue/reference/remap/concepts/function.cue index 60f026bb336e3..5885b48a2a857 100644 --- a/website/cue/reference/remap/concepts/function.cue +++ b/website/cue/reference/remap/concepts/function.cue @@ -19,5 +19,14 @@ remap: concepts: function: { is a defining characteristic of VRL and a primary source of its safety guarantees. """ } + deprecation: { + title: "Deprecation" + description: """ + VRL functions can be marked as "deprecated". When a function + is deprecated, a warning will be shown at runtime. + + Suggestions on how to update the VRL program can usually be found in the actual warning and the function documentation. + """ + } } } diff --git a/website/cue/reference/remap/expressions/function_call.cue b/website/cue/reference/remap/expressions/function_call.cue index a86a3fcff4e22..2226dd23652f9 100644 --- a/website/cue/reference/remap/expressions/function_call.cue +++ b/website/cue/reference/remap/expressions/function_call.cue @@ -134,6 +134,15 @@ remap: expressions: function_call: { failures. """ } + deprecation: { + title: "Deprecation" + description: """ + VRL functions can be marked as "deprecated". When a function + is deprecated, a warning will be shown at runtime. + + Suggestions on how to update the VRL program can usually be found in the actual warning and the function documentation. + """ + } } examples: [ diff --git a/website/cue/reference/remap/functions.cue b/website/cue/reference/remap/functions.cue index f8fc7df8b4292..151a61e5faabf 100644 --- a/website/cue/reference/remap/functions.cue +++ b/website/cue/reference/remap/functions.cue @@ -24,6 +24,7 @@ remap: { } internal_failure_reasons: [...string] examples?: [remap.#Example, ...remap.#Example] + deprecated: bool | *false } #FunctionCategory: "Array" | "Codec" | "Coerce" | "Convert" | "Debug" | "Enrichment" | "Enumerate" | "Event" | "Path" | "Cryptography" | "IP" | "Number" | "Object" | "Parse" | "Random" | "String" | "System" | "Timestamp" | "Type" diff --git a/website/cue/reference/remap/functions/encrypt.cue b/website/cue/reference/remap/functions/encrypt.cue index c37e68dfd45fc..4d91f4efeb372 100644 --- a/website/cue/reference/remap/functions/encrypt.cue +++ b/website/cue/reference/remap/functions/encrypt.cue @@ -13,9 +13,15 @@ remap: functions: encrypt: { * AES-256-OFB (key = 32 bytes, iv = 16 bytes) * AES-192-OFB (key = 24 bytes, iv = 16 bytes) * AES-128-OFB (key = 16 bytes, iv = 16 bytes) - * AES-256-CTR (key = 32 bytes, iv = 16 bytes) - * AES-192-CTR (key = 24 bytes, iv = 16 bytes) - * AES-128-CTR (key = 16 bytes, iv = 16 bytes) + * Deprecated - AES-256-CTR (key = 32 bytes, iv = 16 bytes) + * Deprecated - AES-192-CTR (key = 24 bytes, iv = 16 bytes) + * Deprecated - AES-128-CTR (key = 16 bytes, iv = 16 bytes) + * AES-256-CTR-LE (key = 32 bytes, iv = 16 bytes) + * AES-192-CTR-LE (key = 24 bytes, iv = 16 bytes) + * AES-128-CTR-LE (key = 16 bytes, iv = 16 bytes) + * AES-256-CTR-BE (key = 32 bytes, iv = 16 bytes) + * AES-192-CTR-BE (key = 24 bytes, iv = 16 bytes) + * AES-128-CTR-BE (key = 16 bytes, iv = 16 bytes) * AES-256-CBC-PKCS7 (key = 32 bytes, iv = 16 bytes) * AES-192-CBC-PKCS7 (key = 24 bytes, iv = 16 bytes) * AES-128-CBC-PKCS7 (key = 16 bytes, iv = 16 bytes) @@ -28,6 +34,9 @@ remap: functions: encrypt: { * AES-256-CBC-ISO10126 (key = 32 bytes, iv = 16 bytes) * AES-192-CBC-ISO10126 (key = 24 bytes, iv = 16 bytes) * AES-128-CBC-ISO10126 (key = 16 bytes, iv = 16 bytes) + * CHACHA20-POLY1305 (key = 32 bytes, iv = 12 bytes) + * XCHACHA20-POLY1305 (key = 32 bytes, iv = 24 bytes) + * XSALSA20-POLY1305 (key = 32 bytes, iv = 24 bytes) """ arguments: [ diff --git a/website/cue/reference/remap/functions/parse_nginx_log.cue b/website/cue/reference/remap/functions/parse_nginx_log.cue index 99f48be898b4d..0fd6c04f1a6f1 100644 --- a/website/cue/reference/remap/functions/parse_nginx_log.cue +++ b/website/cue/reference/remap/functions/parse_nginx_log.cue @@ -60,18 +60,15 @@ remap: functions: parse_nginx_log: { ) """# return: { + agent: "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36" client: "172.17.0.1" - user: "alice" - timestamp: "2021-04-01T12:02:31Z" + compression: "2.75" + referer: "http://localhost/somewhere" request: "POST /not-found HTTP/1.1" - method: "POST" - path: "/not-found" - protocol: "HTTP/1.1" - status: 404 size: 153 - referer: "http://localhost/somewhere" - agent: "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36" - compression: "2.75" + status: 404 + timestamp: "2021-04-01T12:02:31Z" + user: "alice" } }, { diff --git a/website/cue/reference/remap/functions/to_timestamp.cue b/website/cue/reference/remap/functions/to_timestamp.cue index e7329f19a8206..f46e3a37d23b1 100644 --- a/website/cue/reference/remap/functions/to_timestamp.cue +++ b/website/cue/reference/remap/functions/to_timestamp.cue @@ -3,10 +3,14 @@ package metadata remap: functions: to_timestamp: { category: "Coerce" description: """ + This function is deprecated. + For integer values, use `from_unix_timestamp`, otherwise use `parse_timestamp`. Coerces the `value` into a timestamp. """ notices: ["There is the possibility of precision loss due to float arithmetic when coercing floats."] + deprecated: true + arguments: [ { name: "value" @@ -47,35 +51,40 @@ remap: functions: to_timestamp: { source: """ to_timestamp!("2020-10-21T16:00:00Z") """ - return: "2020-10-21T16:00:00Z" + return: "2020-10-21T16:00:00Z" + skip_test: true }, { title: "Coerce a unix timestamp (integer) to a timestamp" source: """ to_timestamp!(1675968923) """ - return: "2023-02-09T18:55:23Z" + return: "2023-02-09T18:55:23Z" + skip_test: true }, { title: "Coerce a unix timestamp (float) to a timestamp" source: """ to_timestamp!(1675968923.567) """ - return: "2023-02-09T18:55:23.566999912Z" + return: "2023-02-09T18:55:23.566999912Z" + skip_test: true }, { title: "Coerce a unix timestamp, in milliseconds, to a timestamp" source: """ to_timestamp!(1676478566639, unit: "milliseconds") """ - return: "2023-02-15T16:29:26.639Z" + return: "2023-02-15T16:29:26.639Z" + skip_test: true }, { title: "Coerce a unix timestamp, in nanoseconds, to a timestamp" source: """ to_timestamp!(1675968923012312311, unit: "nanoseconds") """ - return: "2023-02-09T18:55:23.012312311Z" + return: "2023-02-09T18:55:23.012312311Z" + skip_test: true }, ] } diff --git a/website/layouts/partials/data.html b/website/layouts/partials/data.html index 3feefe90cd353..55f14ab1c44a5 100644 --- a/website/layouts/partials/data.html +++ b/website/layouts/partials/data.html @@ -688,6 +688,10 @@

{{ else }} {{ partial "badge.html" (dict "word" "fallible" "color" "yellow" "inline" true "href" $href) }} {{ end }} + {{/* Deprecation Badge */}} + {{ if $func.deprecated }} + {{ partial "badge.html" (dict "word" "deprecated" "color" "red" "inline" true "href" $href) }} + {{ end }}