Skip to content

Commit

Permalink
feat(logs): Log noise filter (#348)
Browse files Browse the repository at this point in the history
  • Loading branch information
NikolaMilosa authored Jul 2, 2024
1 parent ea8bfe8 commit 68b109e
Show file tree
Hide file tree
Showing 25 changed files with 909 additions and 3 deletions.
130 changes: 129 additions & 1 deletion Cargo.Bazel.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"checksum": "d5d33d6454b8f860b5027bd8ef2ec6082f348be768b504ee6f55c78dd00704bc",
"checksum": "373e54054840c62d3438f63f1494707a66f3be25365a3abb2d53c837a692be7b",
"crates": {
"actix-codec 0.5.2": {
"name": "actix-codec",
Expand Down Expand Up @@ -29184,6 +29184,132 @@
},
"license": null
},
"log-noise-filter-backend 0.4.2": {
"name": "log-noise-filter-backend",
"version": "0.4.2",
"repository": null,
"targets": [
{
"Library": {
"crate_name": "log_noise_filter_backend",
"crate_root": "src/lib.rs",
"srcs": [
"**/*.rs"
]
}
}
],
"library_target_name": "log_noise_filter_backend",
"common_attrs": {
"compile_data_glob": [
"**"
],
"deps": {
"common": [
{
"id": "axum 0.7.5",
"target": "axum"
},
{
"id": "clap 4.5.7",
"target": "clap"
},
{
"id": "regex 1.10.5",
"target": "regex"
},
{
"id": "serde 1.0.203",
"target": "serde"
},
{
"id": "serde_json 1.0.117",
"target": "serde_json"
},
{
"id": "slog 2.7.0",
"target": "slog"
},
{
"id": "slog-async 2.8.0",
"target": "slog_async"
},
{
"id": "slog-term 2.9.1",
"target": "slog_term"
},
{
"id": "tokio 1.38.0",
"target": "tokio"
}
],
"selects": {}
},
"edition": "2021",
"version": "0.4.2"
},
"license": null
},
"log-noise-filter-downloader 0.4.2": {
"name": "log-noise-filter-downloader",
"version": "0.4.2",
"repository": null,
"targets": [],
"library_target_name": null,
"common_attrs": {
"compile_data_glob": [
"**"
],
"deps": {
"common": [
{
"id": "anyhow 1.0.86",
"target": "anyhow"
},
{
"id": "clap 4.5.7",
"target": "clap"
},
{
"id": "reqwest 0.12.5",
"target": "reqwest"
},
{
"id": "serde 1.0.203",
"target": "serde"
},
{
"id": "serde_json 1.0.117",
"target": "serde_json"
},
{
"id": "slog 2.7.0",
"target": "slog"
},
{
"id": "slog-async 2.8.0",
"target": "slog_async"
},
{
"id": "slog-term 2.9.1",
"target": "slog_term"
},
{
"id": "tokio 1.38.0",
"target": "tokio"
},
{
"id": "url 2.5.2",
"target": "url"
}
],
"selects": {}
},
"edition": "2021",
"version": "0.4.2"
},
"license": null
},
"lzma-sys 0.1.20": {
"name": "lzma-sys",
"version": "0.1.20",
Expand Down Expand Up @@ -50786,6 +50912,8 @@
"ic-management-backend 0.4.2": "rs/ic-management-backend",
"ic-management-types 0.4.2": "rs/ic-management-types",
"log-fetcher 0.4.2": "rs/log-fetcher",
"log-noise-filter-backend 0.4.2": "rs/ic-observability/log-noise-filter-backend",
"log-noise-filter-downloader 0.4.2": "rs/ic-observability/log-noise-filter-downloader",
"multiservice-discovery 0.4.2": "rs/ic-observability/multiservice-discovery",
"multiservice-discovery-downloader 0.4.2": "rs/ic-observability/multiservice-discovery-downloader",
"multiservice-discovery-shared 0.4.2": "rs/ic-observability/multiservice-discovery-shared",
Expand Down
32 changes: 32 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ members = [
"rs/ic-management-backend",
"rs/ic-management-types",
"rs/ic-observability/config-writer-common",
"rs/ic-observability/log-noise-filter-backend",
"rs/ic-observability/log-noise-filter-downloader",
"rs/ic-observability/multiservice-discovery",
"rs/ic-observability/multiservice-discovery-downloader",
"rs/ic-observability/multiservice-discovery-shared",
Expand All @@ -18,7 +20,7 @@ members = [
"rs/canister-log-fetcher",
"rs/np-notifications",
"rs/rollout-controller",
"rs/slack-notifications",
"rs/slack-notifications",
]

resolver = "2"
Expand All @@ -44,6 +46,7 @@ async-recursion = "1.1.1"
async-timer = "0.7.4"
async-trait = "0.1.80"
axum-otel-metrics = "0.8.1"
axum = "0.7.5"
backoff = { version = "0.4.0", features = ["tokio"] }
backon = "0.4.4"
candid = "0.10.9"
Expand Down
2 changes: 2 additions & 0 deletions bazel/external_crates.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ def external_crates_repository():
"//rs/ic-management-backend:Cargo.toml",
"//rs/ic-management-types:Cargo.toml",
"//rs/ic-observability/config-writer-common:Cargo.toml",
"//rs/ic-observability/log-noise-filter-backend:Cargo.toml",
"//rs/ic-observability/log-noise-filter-downloader:Cargo.toml",
"//rs/ic-observability/multiservice-discovery:Cargo.toml",
"//rs/ic-observability/multiservice-discovery-downloader:Cargo.toml",
"//rs/ic-observability/multiservice-discovery-shared:Cargo.toml",
Expand Down
59 changes: 59 additions & 0 deletions rs/ic-observability/log-noise-filter-backend/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
load("@crate_index_dre//:defs.bzl", "aliases", "all_crate_deps")
load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test", "rust_library")
load("@//rs:oci_images.bzl", "rust_binary_oci_image_rules")

package(default_visibility = ["//visibility:public"])

DEPS = []

rust_library(
name = "log-noise-filter-backend-lib",
srcs = glob(["src/**/*.rs"]),
aliases = aliases(),
crate_name = "log_noise_filter_backend",
proc_macro_deps = all_crate_deps(
proc_macro = True,
),
deps = all_crate_deps(
normal = True,
) + DEPS,
)

BINARY_DEPS = [
":log-noise-filter-backend-lib",
]

rust_binary(
name = "log-noise-filter-backend",
srcs = glob(["src/**/*.rs"]),
aliases = aliases(),
proc_macro_deps = all_crate_deps(
proc_macro = True,
),
stamp = 1,
deps = all_crate_deps(
normal = True,
) + DEPS + BINARY_DEPS,
)

rust_test(
name = "unit_test",
aliases = aliases(
normal_dev = True,
proc_macro_dev = True,
),
crate = ":log-noise-filter-backend-lib",
proc_macro_deps = all_crate_deps(
proc_macro_dev = True,
),
deps = all_crate_deps(
normal_dev = True,
) + DEPS,
)

rust_binary_oci_image_rules(
name = "oci_image",
src = ":log-noise-filter-backend",
base_image = "@debian-slim"
)

28 changes: 28 additions & 0 deletions rs/ic-observability/log-noise-filter-backend/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
[package]
name = "log-noise-filter-backend"
version.workspace = true
edition.workspace = true
authors.workspace = true
description.workspace = true
documentation.workspace = true

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
clap = { workspace = true }
axum = { workspace = true }
slog = { workspace = true }
slog-async = { workspace = true }
slog-term = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tokio = { workspace = true }
regex = { workspace = true }

[[bin]]
name = "log-noise-filter-backend"
path = "src/main.rs"

[lib]
name = "log_noise_filter_backend"
path = "src/lib.rs"
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
use std::collections::BTreeMap;

use axum::{extract::State, http::StatusCode, Json};
use slog::{info, warn};

use crate::handlers::Server;

pub async fn delete_criteria(
State(state): State<Server>,
Json(criteria): Json<Vec<u32>>,
) -> Result<Json<BTreeMap<u32, String>>, (StatusCode, String)> {
match state.delete_criteria(criteria.clone()).await {
Ok(()) => {
info!(state.logger, "Deleted criteria"; "indexes" => ?criteria);
Ok(Json(state.get_criteria_mapped().await))
}
Err(missing) => {
warn!(state.logger, "Failed to delete criteria"; "indexes" => ?missing);
Err((StatusCode::NOT_FOUND, format!("Missing indexes: {:?}", missing)))
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
use std::collections::BTreeMap;

use axum::http::StatusCode;
use axum::{extract::State, Json};

use crate::handlers::Server;

pub(crate) async fn get_criteria(State(state): State<Server>) -> Result<Json<BTreeMap<u32, String>>, (StatusCode, String)> {
Ok(Json(state.get_criteria_mapped().await))
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
pub mod delete_criteria;
pub mod get_criteria;
pub mod post_criteria;
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
use std::collections::BTreeMap;

use axum::{extract::State, http::StatusCode, Json};

use crate::handlers::Server;

pub async fn update(State(state): State<Server>, Json(criteria): Json<Vec<String>>) -> Result<Json<BTreeMap<u32, String>>, (StatusCode, String)> {
match state.update_criteria(criteria).await {
Ok(()) => Ok(Json(state.get_criteria_mapped().await)),
Err(v) => Err((StatusCode::BAD_REQUEST, format!("Invalid instances of regex: {:?}", v))),
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
use axum::{extract::State, http::StatusCode, Json};

use super::{Server, WholeState};

pub async fn get_all(State(server): State<Server>) -> Result<Json<WholeState>, (StatusCode, String)> {
let state = WholeState {
criteria: server.get_criteria_mapped().await,
rate: server.get_rate().await,
};

Ok(Json(state))
}
Loading

0 comments on commit 68b109e

Please sign in to comment.