feat(logs): Log noise filter

Cargo.Bazel.lock

@@ -1,5 +1,5 @@
 {
-  "checksum": "982c13a29a128a09b521a6a2a453103ab8aea1f631648c92e4ca6072ed60a065",
+  "checksum": "24114f0ea682762469e382f2de27cc6ac7c4cc2499f58e73623fb7837eea9e8b",
   "crates": {
     "actix-codec 0.5.2": {
       "name": "actix-codec",
@@ -28893,35 +28893,29 @@
         ],
         "crate_features": {
           "common": [
+            "elf",
+            "errno",
             "general",
             "ioctl",
             "no_std"
           ],
           "selects": {
             "aarch64-unknown-linux-gnu": [
-              "elf",
-              "errno",
               "prctl",
               "std",
               "system"
             ],
             "arm-unknown-linux-gnueabi": [
-              "elf",
-              "errno",
               "prctl",
               "std",
               "system"
             ],
             "armv7-unknown-linux-gnueabi": [
-              "elf",
-              "errno",
               "prctl",
               "std",
               "system"
             ],
             "i686-unknown-linux-gnu": [
-              "elf",
-              "errno",
               "prctl",
               "std",
               "system"
@@ -28937,8 +28931,6 @@
               "system"
             ],
             "x86_64-unknown-linux-gnu": [
-              "elf",
-              "errno",
               "prctl",
               "std",
               "system"
@@ -29192,6 +29184,122 @@
       },
       "license": null
     },
+    "log-noise-filter-backend 0.4.2": {
+      "name": "log-noise-filter-backend",
+      "version": "0.4.2",
+      "repository": null,
+      "targets": [],
+      "library_target_name": null,
+      "common_attrs": {
+        "compile_data_glob": [
+          "**"
+        ],
+        "deps": {
+          "common": [
+            {
+              "id": "axum 0.7.5",
+              "target": "axum"
+            },
+            {
+              "id": "clap 4.5.7",
+              "target": "clap"
+            },
+            {
+              "id": "regex 1.10.5",
+              "target": "regex"
+            },
+            {
+              "id": "serde 1.0.203",
+              "target": "serde"
+            },
+            {
+              "id": "serde_json 1.0.117",
+              "target": "serde_json"
+            },
+            {
+              "id": "slog 2.7.0",
+              "target": "slog"
+            },
+            {
+              "id": "slog-async 2.8.0",
+              "target": "slog_async"
+            },
+            {
+              "id": "slog-term 2.9.1",
+              "target": "slog_term"
+            },
+            {
+              "id": "tokio 1.38.0",
+              "target": "tokio"
+            }
+          ],
+          "selects": {}
+        },
+        "edition": "2021",
+        "version": "0.4.2"
+      },
+      "license": null
+    },
+    "log-noise-filter-downloader 0.4.2": {
+      "name": "log-noise-filter-downloader",
+      "version": "0.4.2",
+      "repository": null,
+      "targets": [],
+      "library_target_name": null,
+      "common_attrs": {
+        "compile_data_glob": [
+          "**"
+        ],
+        "deps": {
+          "common": [
+            {
+              "id": "anyhow 1.0.86",
+              "target": "anyhow"
+            },
+            {
+              "id": "clap 4.5.7",
+              "target": "clap"
+            },
+            {
+              "id": "reqwest 0.12.5",
+              "target": "reqwest"
+            },
+            {
+              "id": "serde 1.0.203",
+              "target": "serde"
+            },
+            {
+              "id": "serde_json 1.0.117",
+              "target": "serde_json"
+            },
+            {
+              "id": "slog 2.7.0",
+              "target": "slog"
+            },
+            {
+              "id": "slog-async 2.8.0",
+              "target": "slog_async"
+            },
+            {
+              "id": "slog-term 2.9.1",
+              "target": "slog_term"
+            },
+            {
+              "id": "tokio 1.38.0",
+              "target": "tokio"
+            },
+            {
+              "id": "url 2.5.2",
+              "target": "url"
+            }
+          ],
+          "selects": {}
+        },
+        "edition": "2021",
+        "version": "0.4.2"
+      },
+      "license": null
+    },
     "lzma-sys 0.1.20": {
       "name": "lzma-sys",
       "version": "0.1.20",
@@ -50794,6 +50902,8 @@
     "ic-management-backend 0.4.2": "rs/ic-management-backend",
     "ic-management-types 0.4.2": "rs/ic-management-types",
     "log-fetcher 0.4.2": "rs/log-fetcher",
+    "log-noise-filter-backend 0.4.2": "rs/ic-observability/log-noise-filter-backend",
+    "log-noise-filter-downloader 0.4.2": "rs/ic-observability/log-noise-filter-downloader",
     "multiservice-discovery 0.4.2": "rs/ic-observability/multiservice-discovery",
     "multiservice-discovery-downloader 0.4.2": "rs/ic-observability/multiservice-discovery-downloader",
     "multiservice-discovery-shared 0.4.2": "rs/ic-observability/multiservice-discovery-shared",

Cargo.toml

@@ -6,6 +6,8 @@ members = [
   "rs/ic-management-backend",
   "rs/ic-management-types",
   "rs/ic-observability/config-writer-common",
+  "rs/ic-observability/log-noise-filter-backend",
+  "rs/ic-observability/log-noise-filter-downloader",
   "rs/ic-observability/multiservice-discovery",
   "rs/ic-observability/multiservice-discovery-downloader",
   "rs/ic-observability/multiservice-discovery-shared",
@@ -18,7 +20,7 @@ members = [
   "rs/canister-log-fetcher",
   "rs/np-notifications",
   "rs/rollout-controller",
-  "rs/slack-notifications",
+  "rs/slack-notifications", 
 ]
 
 resolver = "2"

bazel/external_crates.bzl

@@ -51,6 +51,8 @@ def external_crates_repository():
             "//rs/ic-management-backend:Cargo.toml",
             "//rs/ic-management-types:Cargo.toml",
             "//rs/ic-observability/config-writer-common:Cargo.toml",
+            "//rs/ic-observability/log-noise-filter-backend:Cargo.toml",
+            "//rs/ic-observability/log-noise-filter-downloader:Cargo.toml",
             "//rs/ic-observability/multiservice-discovery:Cargo.toml",
             "//rs/ic-observability/multiservice-discovery-downloader:Cargo.toml",
             "//rs/ic-observability/multiservice-discovery-shared:Cargo.toml",

k8s/oci_images.bzl

@@ -39,7 +39,7 @@ def python_oci_image_rules(name, src, base_image = "@distroless_python3"):
         }
     )
 
-    tarball_name = "{}-tarball".format(binary.name)
+    tarball_name = "tarball".format(binary.name)
     oci_tarball(
         name = tarball_name,
         image = image_rule_name,

rs/ic-observability/log-noise-filter-backend/BUILD.bazel

@@ -0,0 +1,25 @@
+load("@crate_index_dre//:defs.bzl", "aliases", "all_crate_deps")
+load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test")
+load("@//rs:oci_images.bzl", "rust_binary_oci_image_rules")
+
+DEPS = []
+
+rust_binary(
+    name = "log-noise-filter-backend",
+    srcs = glob(["src/**/*.rs"]),
+    aliases = aliases(),
+    proc_macro_deps = all_crate_deps(
+        proc_macro = True,
+    ),
+    stamp = 1,
+    deps = all_crate_deps(
+        normal = True,
+    ) + DEPS,
+)
+
+rust_binary_oci_image_rules(
+    name = "oci_image",
+    src = ":log-noise-filter-backend",
+    base_image = "@debian-slim"
+)
+

rs/ic-observability/log-noise-filter-backend/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "log-noise-filter-backend"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+description.workspace = true
+documentation.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+clap = { workspace = true }
+axum = "0.7.4"
+slog = { workspace = true }
+slog-async = { workspace = true }
+slog-term = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+tokio = { workspace = true }
+regex = { workspace = true }

rs/ic-observability/log-noise-filter-backend/src/handlers/delete.rs

@@ -0,0 +1,15 @@
+use std::collections::BTreeMap;
+
+use axum::{extract::State, http::StatusCode, Json};
+
+use super::Server;
+
+pub async fn delete_criteria(
+    State(state): State<Server>,
+    Json(criteria): Json<Vec<u32>>,
+) -> Result<Json<BTreeMap<u32, String>>, (StatusCode, String)> {
+    match state.delete_criteria(criteria).await {
+        Ok(()) => Ok(Json(state.get_criteria_mapped().await)),
+        Err(missing) => Err((StatusCode::NOT_FOUND, format!("Missing indexes: {:?}", missing))),
+    }
+}

rs/ic-observability/log-noise-filter-backend/src/handlers/get.rs

@@ -0,0 +1,10 @@
+use std::collections::BTreeMap;
+
+use axum::http::StatusCode;
+use axum::{extract::State, Json};
+
+use super::Server;
+
+pub(crate) async fn get_criteria(State(state): State<Server>) -> Result<Json<BTreeMap<u32, String>>, (StatusCode, String)> {
+    Ok(Json(state.get_criteria_mapped().await))
+}