From ece0b4a2f23510f15028558b14d5b86513cecaad Mon Sep 17 00:00:00 2001 From: Dirk Sammel Date: Fri, 29 Nov 2024 11:44:51 +0100 Subject: [PATCH 01/14] update rustls --- CHANGELOG.md | 4 +++- Cargo.lock | 22 ++-------------------- auditor-client/src/lib.rs | 2 +- 3 files changed, 6 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b8e7c16..1808357d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - pyauditor + Apel plugin + HTCondor collector: drop support for Python 3.8 ([@dirksammel](https://github.com/dirksammel)) ### Security +- [RUSTSEC-2024-0363]: Update sqlx from 0.7.4 to 0.8.2 (missed some occurrences) ([@dirksammel](https://github.com/dirksammel)) +- [RUSTSEC-2024-0399]: Update rustls from 0.23.16 to 0.23.19 ([@dirksammel](https://github.com/dirksammel)) ### Added - CI: Add workflow to test publishing to the PyPI test repo ([@dirksammel](https://github.com/dirksammel)) @@ -38,7 +40,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - CI: Add checks for import sorting and formatting to python linting workflow ([@dirksammel](https://github.com/dirksammel)) ### Changed -- Dependencies: Update sqlx from 0.7.4 to 0.8.2 ([@raghuvar-vijay](https://github.com/raghuvar-vijay)) - Dependencies: Update black from 24.4.2 to 24.10.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update build from 1.2.1 to 1.2.2.post1 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update crate-ci/typos from 1.23.3 to 1.26.8 ([@dirksammel](https://github.com/dirksammel)) @@ -55,6 +56,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Dependencies: Update serde_json from 1.0.121 to 1.0.132 ([@raghuvar-vijay](https://github.com/raghuvar-vijay)) - Dependencies: Update setuptools from 71.1.0 to 75.3.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update shalzz/zola-deploy-action from 0.19.1 to 0.19.2 ([@dirksammel](https://github.com/dirksammel)) +- Dependencies: Update sqlx from 0.7.4 to 0.8.2 ([@raghuvar-vijay](https://github.com/raghuvar-vijay)) - Dependencies: Update tarides/changelog-check-action from 2 to 3 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update types-pyyaml from 6.0.12.20240311 to 6.0.12.20240917 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update types-requests from 2.32.0.20240712 to 2.32.0.20241016 ([@dirksammel](https://github.com/dirksammel)) diff --git a/Cargo.lock b/Cargo.lock index dbaa4f69..0c19dc56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -361,7 +361,6 @@ dependencies = [ "once_cell", "opentelemetry 0.23.0", "opentelemetry-prometheus 0.16.0", - "opentelemetry_api", "opentelemetry_sdk 0.23.0", "prometheus", "quickcheck", @@ -434,7 +433,6 @@ dependencies = [ "num-traits", "opentelemetry 0.23.0", "opentelemetry-prometheus 0.16.0", - "opentelemetry_api", "opentelemetry_sdk 0.23.0", "prometheus", "serde", @@ -2053,22 +2051,6 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9ab5bd6c42fb9349dcf28af2ba9a0667f697f9bdcca045d39f2cec5543e2910" -[[package]] -name = "opentelemetry_api" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" -dependencies = [ - "futures-channel", - "futures-util", - "indexmap 1.9.3", - "js-sys", - "once_cell", - "pin-project-lite", - "thiserror", - "urlencoding", -] - [[package]] name = "opentelemetry_sdk" version = "0.22.1" @@ -2815,9 +2797,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.16" +version = "0.23.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" +checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" dependencies = [ "once_cell", "ring", diff --git a/auditor-client/src/lib.rs b/auditor-client/src/lib.rs index 47e4f203..26a136fd 100644 --- a/auditor-client/src/lib.rs +++ b/auditor-client/src/lib.rs @@ -892,7 +892,7 @@ impl Operator { } } -/// Implementations of conversion traits for the `Value` enum. +// Implementations of conversion traits for the `Value` enum. /// Conversion from chrono DateTime to Value::Datetime. impl From> for Value { From c3b6b6fad8a0d5db17ed57ddc8a861119d89497f Mon Sep 17 00:00:00 2001 From: Dirk Sammel Date: Fri, 29 Nov 2024 13:20:19 +0100 Subject: [PATCH 02/14] update pyo3 --- CHANGELOG.md | 2 ++ Cargo.toml | 4 +-- pyauditor/Cargo.toml | 2 +- pyauditor/src/blocking_client.rs | 10 ++++-- pyauditor/src/builder.rs | 4 +-- pyauditor/src/client.rs | 50 +++++++++++++++--------------- pyauditor/src/domain/record.rs | 4 +-- pyauditor/src/lib.rs | 2 +- pyauditor/src/queued_client.rs | 52 ++++++++++++++++---------------- 9 files changed, 69 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1808357d..c9b2dc5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Auditor Docker container: Switch from fixed to latest Rust version ([@dirksammel](https://github.com/dirksammel)) +- Dependencies: Switch from pyo3-asyncio 0.20.0 to pyo3-async-runtimes 0.22.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update crate-ci/typos from 1.26.8 to 1.27.3 ([@dirksammel](https://github.com/dirksammel)) +- Dependencies: Update pyo3 from 0.20.3 to 0.22.5 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update ruff from 0.7.1 to 0.7.3 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update setuptools from 75.3.0 to 75.4.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update sqlx from 0.7.4 to 0.8.2 (missed some occurrences) ([@dirksammel](https://github.com/dirksammel)) diff --git a/Cargo.toml b/Cargo.toml index 28a94822..b3538802 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,8 +29,8 @@ opentelemetry = "0.23.0" opentelemetry-prometheus = "0.16.0" opentelemetry_sdk = "0.23.0" prometheus = "0.13.4" -pyo3 = { version = "0.20.3", features = ["chrono", "extension-module", "anyhow"] } -pyo3-asyncio = { version = "0.20.0", features = ["attributes", "tokio-runtime"] } +pyo3 = { version = "0.22.5", features = ["chrono", "extension-module", "anyhow"] } +pyo3-async-runtimes = { version = "0.22.0", features = ["attributes", "tokio-runtime"] } quickcheck = "1.0.3" quickcheck_macros = "1.0.0" rand = "0.8.5" diff --git a/pyauditor/Cargo.toml b/pyauditor/Cargo.toml index 2aa25fd8..10985697 100644 --- a/pyauditor/Cargo.toml +++ b/pyauditor/Cargo.toml @@ -29,8 +29,8 @@ anyhow.workspace = true auditor-client.workspace = true auditor.workspace = true chrono.workspace = true -pyo3-asyncio.workspace = true pyo3.workspace = true +pyo3-async-runtimes.workspace = true serde.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/pyauditor/src/blocking_client.rs b/pyauditor/src/blocking_client.rs index 925dff18..8029b543 100644 --- a/pyauditor/src/blocking_client.rs +++ b/pyauditor/src/blocking_client.rs @@ -62,7 +62,10 @@ impl AuditorClientBlocking { /// /// records = client.get_stopped_since(start_since) /// - fn get_started_since(self_: PyRef<'_, Self>, timestamp: &PyDateTime) -> PyResult> { + fn get_started_since( + self_: PyRef<'_, Self>, + timestamp: &Bound<'_, PyDateTime>, + ) -> PyResult> { let since: DateTime = timestamp.extract()?; let query_string = auditor_client::QueryBuilder::new() .with_start_time(auditor_client::Operator::default().gte(since.into())) @@ -99,7 +102,10 @@ impl AuditorClientBlocking { /// /// records = client.get_stopped_since(stop_since) /// - fn get_stopped_since(self_: PyRef<'_, Self>, timestamp: &PyDateTime) -> PyResult> { + fn get_stopped_since( + self_: PyRef<'_, Self>, + timestamp: &Bound<'_, PyDateTime>, + ) -> PyResult> { let since: DateTime = timestamp.extract()?; let query_string = auditor_client::QueryBuilder::new() .with_stop_time(auditor_client::Operator::default().gte(since.into())) diff --git a/pyauditor/src/builder.rs b/pyauditor/src/builder.rs index e58d684d..c6bf0ca2 100644 --- a/pyauditor/src/builder.rs +++ b/pyauditor/src/builder.rs @@ -124,11 +124,11 @@ impl AuditorClientBuilder { } /// Build a ``QueuedAuditorClient`` from ``AuditorClientBuilder`` - pub fn build_queued<'a>(&'a self, py: Python<'a>) -> PyResult<&'a PyAny> { + pub fn build_queued<'a>(&'a self, py: Python<'a>) -> PyResult> { // Must clone here because `build` moves the builder, but python // does not allow that. Doesn't matter, Python is slow anyways. let builder = self.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(QueuedAuditorClient { inner: builder .build_queued() diff --git a/pyauditor/src/client.rs b/pyauditor/src/client.rs index 3567035c..c992a33d 100644 --- a/pyauditor/src/client.rs +++ b/pyauditor/src/client.rs @@ -58,7 +58,7 @@ impl Value { /// /// value = Value.set_datetime(start_time) #[staticmethod] - fn set_datetime(datetime: &PyDateTime) -> Result { + fn set_datetime(datetime: &Bound<'_, PyDateTime>) -> Result { let date_time: DateTime = datetime.extract()?; Ok(Value { inner: auditor_client::Value::Datetime(auditor_client::DateTimeUtcWrapper(date_time)), @@ -606,9 +606,9 @@ pub struct AuditorClient { impl AuditorClient { /// health_check() /// Returns ``true`` if the Auditor instance is healthy, ``false`` otherwise - fn health_check<'a>(self_: PyRef<'a, Self>, py: Python<'a>) -> PyResult<&'a PyAny> { + fn health_check<'a>(self_: PyRef<'a, Self>, py: Python<'a>) -> PyResult> { let inner = self_.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(inner.health_check().await) // Ok(Python::with_gil(|py| py.None())) }) @@ -616,9 +616,9 @@ impl AuditorClient { /// get() /// Gets all records from the Auditors database - fn get<'a>(self_: PyRef<'a, Self>, py: Python<'a>) -> PyResult<&'a PyAny> { + fn get<'a>(self_: PyRef<'a, Self>, py: Python<'a>) -> PyResult> { let inner = self_.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(inner .get() .await @@ -654,11 +654,11 @@ impl AuditorClient { /// fn get_started_since<'a>( self_: PyRef<'a, Self>, - timestamp: &PyDateTime, + timestamp: &Bound<'_, PyDateTime>, py: Python<'a>, - ) -> PyResult<&'a PyAny> { - let message = py.get_type::(); - PyErr::warn(py, message, "get_started_since is depreciated", 0)?; + ) -> PyResult> { + let message = py.get_type_bound::(); + PyErr::warn_bound(py, &message, "get_started_since is depreciated", 0)?; let since: DateTime = timestamp.extract()?; let inner = self_.inner.clone(); @@ -666,7 +666,7 @@ impl AuditorClient { .with_start_time(auditor_client::Operator::default().gte(since.into())) .build(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(inner .advanced_query(query_string.to_string()) .await @@ -702,18 +702,18 @@ impl AuditorClient { /// fn get_stopped_since<'a>( self_: PyRef<'a, Self>, - timestamp: &PyDateTime, + timestamp: &Bound<'_, PyDateTime>, py: Python<'a>, - ) -> PyResult<&'a PyAny> { - let message = py.get_type::(); - PyErr::warn(py, message, "get_stopped_since_since is depreciated", 0)?; + ) -> PyResult> { + let message = py.get_type_bound::(); + PyErr::warn_bound(py, &message, "get_stopped_since_since is depreciated", 0)?; let since: DateTime = timestamp.extract()?; let inner = self_.inner.clone(); let query_string = auditor_client::QueryBuilder::new() .with_stop_time(auditor_client::Operator::default().gte(since.into())) .build(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(inner .advanced_query(query_string.to_string()) .await @@ -744,9 +744,9 @@ impl AuditorClient { self_: PyRef<'a, Self>, query_string: String, py: Python<'a>, - ) -> PyResult<&'a PyAny> { + ) -> PyResult> { let inner = self_.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(inner .advanced_query(query_string) .await @@ -773,9 +773,9 @@ impl AuditorClient { self_: PyRef<'a, Self>, record_id: String, py: Python<'a>, - ) -> PyResult<&'a PyAny> { + ) -> PyResult> { let inner = self_.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { inner .get_single_record(record_id) .await @@ -786,9 +786,9 @@ impl AuditorClient { /// add(record: Record) /// Push a record to the Auditor instance - fn add<'a>(&self, record: Record, py: Python<'a>) -> PyResult<&'a PyAny> { + fn add<'a>(&self, record: Record, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { inner .add(&auditor::domain::RecordAdd::try_from(record.inner)?) .await @@ -798,7 +798,7 @@ impl AuditorClient { /// add(record: Record) /// Push a list of records to the Auditor instance - fn bulk_insert<'a>(&self, records: Vec, py: Python<'a>) -> PyResult<&'a PyAny> { + fn bulk_insert<'a>(&self, records: Vec, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); let bulk_insert_records: Result, anyhow::Error> = records @@ -806,7 +806,7 @@ impl AuditorClient { .map(|r| auditor::domain::RecordAdd::try_from(r.inner)) .collect(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { let bul = bulk_insert_records?; inner .bulk_insert(&bul) @@ -817,9 +817,9 @@ impl AuditorClient { /// update(record: Record) /// Update an existing record in the Auditor instance - fn update<'a>(&self, record: Record, py: Python<'a>) -> PyResult<&'a PyAny> { + fn update<'a>(&self, record: Record, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { inner .update(&auditor::domain::RecordUpdate::try_from(record.inner)?) .await diff --git a/pyauditor/src/domain/record.rs b/pyauditor/src/domain/record.rs index 59e7fc5d..9568462d 100644 --- a/pyauditor/src/domain/record.rs +++ b/pyauditor/src/domain/record.rs @@ -56,7 +56,7 @@ pub struct Record { #[pymethods] impl Record { #[new] - fn new(record_id: String, start_time: &PyDateTime) -> Result { + fn new(record_id: String, start_time: &Bound<'_, PyDateTime>) -> Result { let start_time: DateTime = start_time.extract()?; Ok(Record { inner: auditor::domain::Record { @@ -106,7 +106,7 @@ impl Record { /// :type stop_time: datetime.datetime fn with_stop_time<'a>( mut self_: PyRefMut<'a, Self>, - stop_time: &'a PyDateTime, + stop_time: &Bound<'a, PyDateTime>, ) -> Result, Error> { let stop_time: DateTime = stop_time.extract()?; self_.inner.stop_time = Some(stop_time); diff --git a/pyauditor/src/lib.rs b/pyauditor/src/lib.rs index 9d69367f..ac864755 100644 --- a/pyauditor/src/lib.rs +++ b/pyauditor/src/lib.rs @@ -17,7 +17,7 @@ mod queued_client; /// pyauditor is a client for interacting with an Auditor instance via Python. #[pymodule] -fn pyauditor(_py: Python, m: &PyModule) -> PyResult<()> { +fn pyauditor(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/pyauditor/src/queued_client.rs b/pyauditor/src/queued_client.rs index 30976b31..5ded4ef6 100644 --- a/pyauditor/src/queued_client.rs +++ b/pyauditor/src/queued_client.rs @@ -40,9 +40,9 @@ pub struct QueuedAuditorClient { impl QueuedAuditorClient { /// health_check() /// Returns ``true`` if the Auditor instance is healthy, ``false`` otherwise - fn health_check<'a>(self_: PyRef<'a, Self>, py: Python<'a>) -> PyResult<&'a PyAny> { + fn health_check<'a>(self_: PyRef<'a, Self>, py: Python<'a>) -> PyResult> { let inner = self_.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(inner.health_check().await) // Ok(Python::with_gil(|py| py.None())) }) @@ -50,9 +50,9 @@ impl QueuedAuditorClient { /// get() /// Gets all records from the Auditors database - fn get<'a>(self_: PyRef<'a, Self>, py: Python<'a>) -> PyResult<&'a PyAny> { + fn get<'a>(self_: PyRef<'a, Self>, py: Python<'a>) -> PyResult> { let inner = self_.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(inner .get() .await @@ -88,11 +88,11 @@ impl QueuedAuditorClient { /// fn get_started_since<'a>( self_: PyRef<'a, Self>, - timestamp: &PyDateTime, + timestamp: &Bound<'_, PyDateTime>, py: Python<'a>, - ) -> PyResult<&'a PyAny> { - let message = py.get_type::(); - PyErr::warn(py, message, "get_started_since is depreciated", 0)?; + ) -> PyResult> { + let message = py.get_type_bound::(); + PyErr::warn_bound(py, &message, "get_started_since is depreciated", 0)?; let since: DateTime = timestamp.extract()?; let inner = self_.inner.clone(); @@ -100,7 +100,7 @@ impl QueuedAuditorClient { .with_start_time(auditor_client::Operator::default().gte(since.into())) .build(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(inner .advanced_query(query_string.to_string()) .await @@ -136,18 +136,18 @@ impl QueuedAuditorClient { /// fn get_stopped_since<'a>( self_: PyRef<'a, Self>, - timestamp: &PyDateTime, + timestamp: &Bound<'_, PyDateTime>, py: Python<'a>, - ) -> PyResult<&'a PyAny> { - let message = py.get_type::(); - PyErr::warn(py, message, "get_stopped_since_since is depreciated", 0)?; + ) -> PyResult> { + let message = py.get_type_bound::(); + PyErr::warn_bound(py, &message, "get_stopped_since_since is depreciated", 0)?; let since: DateTime = timestamp.extract()?; let inner = self_.inner.clone(); let query_string = auditor_client::QueryBuilder::new() .with_stop_time(auditor_client::Operator::default().gte(since.into())) .build(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(inner .advanced_query(query_string.to_string()) .await @@ -178,9 +178,9 @@ impl QueuedAuditorClient { self_: PyRef<'a, Self>, query_string: String, py: Python<'a>, - ) -> PyResult<&'a PyAny> { + ) -> PyResult> { let inner = self_.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(inner .advanced_query(query_string) .await @@ -207,9 +207,9 @@ impl QueuedAuditorClient { self_: PyRef<'a, Self>, record_id: String, py: Python<'a>, - ) -> PyResult<&'a PyAny> { + ) -> PyResult> { let inner = self_.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { inner .get_single_record(record_id) .await @@ -220,9 +220,9 @@ impl QueuedAuditorClient { /// add(record: Record) /// Push a record to the Auditor instance - fn add<'a>(&self, record: Record, py: Python<'a>) -> PyResult<&'a PyAny> { + fn add<'a>(&self, record: Record, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { inner .add(&auditor::domain::RecordAdd::try_from(record.inner)?) .await @@ -232,7 +232,7 @@ impl QueuedAuditorClient { /// add(record: Record) /// Push a list of records to the Auditor instance - fn bulk_insert<'a>(&self, records: Vec, py: Python<'a>) -> PyResult<&'a PyAny> { + fn bulk_insert<'a>(&self, records: Vec, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); let bulk_insert_records: Result, anyhow::Error> = records @@ -240,7 +240,7 @@ impl QueuedAuditorClient { .map(|r| auditor::domain::RecordAdd::try_from(r.inner)) .collect(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { let bul = bulk_insert_records?; inner .bulk_insert(&bul) @@ -251,9 +251,9 @@ impl QueuedAuditorClient { /// update(record: Record) /// Update an existing record in the Auditor instance - fn update<'a>(&self, record: Record, py: Python<'a>) -> PyResult<&'a PyAny> { + fn update<'a>(&self, record: Record, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { inner .update(&auditor::domain::RecordUpdate::try_from(record.inner)?) .await @@ -262,9 +262,9 @@ impl QueuedAuditorClient { } /// Stops the background sync task - fn stop<'a>(&self, py: Python<'a>) -> PyResult<&'a PyAny> { + fn stop<'a>(&self, py: Python<'a>) -> PyResult> { let mut inner = self.inner.clone(); - pyo3_asyncio::tokio::future_into_py(py, async move { + pyo3_async_runtimes::tokio::future_into_py(py, async move { inner .stop() .await From d8ee13cfac691a91b027092a55443c84e0d1f3d2 Mon Sep 17 00:00:00 2001 From: Dirk Sammel Date: Thu, 5 Dec 2024 14:58:49 +0100 Subject: [PATCH 03/14] update hashbrown --- CHANGELOG.md | 1 + Cargo.lock | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9b2dc5d..6e470cc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security - [RUSTSEC-2024-0363]: Update sqlx from 0.7.4 to 0.8.2 (missed some occurrences) ([@dirksammel](https://github.com/dirksammel)) - [RUSTSEC-2024-0399]: Update rustls from 0.23.16 to 0.23.19 ([@dirksammel](https://github.com/dirksammel)) +- [RUSTSEC-2024-0402]: Update hashbrown from 0.15.0 to 0.15.2 ([@dirksammel](https://github.com/dirksammel)) ### Added - CI: Add workflow to test publishing to the PyPI test repo ([@dirksammel](https://github.com/dirksammel)) diff --git a/Cargo.lock b/Cargo.lock index 0c19dc56..52c4899e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "actix-codec" @@ -1386,9 +1386,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "hashlink" @@ -1640,7 +1640,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.15.2", "serde", ] From f39ee78d682b4211954dd562b1992cb7fac47c00 Mon Sep 17 00:00:00 2001 From: Dirk Sammel Date: Wed, 13 Nov 2024 15:09:35 +0100 Subject: [PATCH 04/14] add vo_mapping function --- CHANGELOG.md | 1 + media/website/content/_index.md | 11 +++- .../apel/src/auditor_apel_plugin/config.py | 21 +++++-- .../apel/src/auditor_apel_plugin/utility.py | 15 ++++- plugins/apel/tests/test_config.py | 57 ++++++++++++++++++- plugins/apel/tests/test_config.yml | 9 ++- plugins/apel/tests/test_utility.py | 27 ++++++++- 7 files changed, 126 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e470cc4..ec0f11b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [RUSTSEC-2024-0402]: Update hashbrown from 0.15.0 to 0.15.2 ([@dirksammel](https://github.com/dirksammel)) ### Added +- Apel plugin: Add function for user->VO mapping to config ([@dirksammel](https://github.com/dirksammel)) - CI: Add workflow to test publishing to the PyPI test repo ([@dirksammel](https://github.com/dirksammel)) ### Changed diff --git a/media/website/content/_index.md b/media/website/content/_index.md index e3a61aee..a7966cb3 100644 --- a/media/website/content/_index.md +++ b/media/website/content/_index.md @@ -682,8 +682,13 @@ summary_fields: GlobalUserName: !MetaField name: subject VO: !MetaField - name: voms - regex: (?<=%2F).*?\S(?=%2F) + name: user + function: + name: vo_mapping + parameters: + atlpr: atlas + atlsg: ops + ops: ops VOGroup: !MetaField name: voms regex: (?=%2F).*?\S(?=%2F) @@ -797,7 +802,7 @@ Different field types are available, depending on the source of the value that i `ComponentField` extracts the value from a `component` in the AUDITOR record. The mandatory parameter of this field is `name`, which gives the name of the component in the AUDITOR record. If the value needs to be modified, e.g. if it has another unit than the one expected by APEL, the optional parameter `divide_by` has to be used. -`MetaField` extracts the value from the `meta` information in the AUDITOR record. The mandatory parameter of this field is `name`, which gives the name of the component in the AUDITOR record. If the value needs to be modified, one of the optional parameters `regex` or `function` can be used. `regex` takes a regular expression, searches the value for this expression, and returns the complete match, `function` takes the name of a function and applies it to the value. The function has to be present in `config.py` and can be added via a pull request. +`MetaField` extracts the value from the `meta` information in the AUDITOR record. The mandatory parameter of this field is `name`, which gives the name of the component in the AUDITOR record. If the value needs to be modified, one of the optional parameters `regex` or `function` can be used. `regex` takes a regular expression, searches the value for this expression, and returns the complete match, `function` has the parameters `name` and `parameters`, where the latter is optional and can be used to provide additional parameters to the function. If you want to manipulate the value of the `Metafield` with a custom function, it has to be present in `utility.py` and can be added via a pull request. `ConstantField` has the mandatory parameter `value`, which is exactly what will be written in the message field. diff --git a/plugins/apel/src/auditor_apel_plugin/config.py b/plugins/apel/src/auditor_apel_plugin/config.py index f8f31115..6b646bbf 100644 --- a/plugins/apel/src/auditor_apel_plugin/config.py +++ b/plugins/apel/src/auditor_apel_plugin/config.py @@ -7,12 +7,14 @@ import re from datetime import datetime from enum import Enum -from typing import Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Union import yaml from pyauditor import Record from pydantic import BaseModel +from .utility import vo_mapping + logger = logging.getLogger("apel_plugin") @@ -28,6 +30,11 @@ def from_yaml(cls, loader: yaml.SafeLoader, node: yaml.nodes.MappingNode): return cls(**loader.construct_mapping(node, deep=True)) +class Function(Configurable): + name: str + parameters: Any = None + + class PluginConfig(Configurable): log_level: str time_json_path: str @@ -94,10 +101,12 @@ def get_value(self, record: Record) -> int: class MetaField(Field): name: str regex: Optional[str] = None - function: Optional[str] = None + function: Optional[Function] = None def get_value(self, record: Record) -> Union[str, int, float]: - function_dict: Dict[str, Callable[[str], Union[str, int, float]]] = {} + function_dict: Dict[str, Callable[[str, Any], Union[str, int, float]]] = { + "vo_mapping": vo_mapping + } try: value = record.meta.get(self.name)[0] @@ -116,14 +125,14 @@ def get_value(self, record: Record) -> Union[str, int, float]: return "None" elif self.function is not None: try: - function = function_dict[self.function] + function = function_dict[self.function.name] except KeyError: logger.critical( - f"Function {self.function} not found in dictionary of allowed " + f"Function {self.function.name} not found in dictionary of allowed " "functions" ) raise - value = function(value) + value = function(value, self.function.parameters) return value return value diff --git a/plugins/apel/src/auditor_apel_plugin/utility.py b/plugins/apel/src/auditor_apel_plugin/utility.py index 8d7294b7..235c8bb5 100644 --- a/plugins/apel/src/auditor_apel_plugin/utility.py +++ b/plugins/apel/src/auditor_apel_plugin/utility.py @@ -1,6 +1,9 @@ +import logging import pathlib from contextlib import contextmanager -from typing import IO, ContextManager, Literal, overload +from typing import IO, ContextManager, Dict, Literal, overload + +logger = logging.getLogger("apel_plugin") @overload @@ -29,3 +32,13 @@ def write_transaction( raise else: tmp_path.rename(path) + + +def vo_mapping(user: str, vo_dict: Dict[str, str]) -> str: + for k, v in vo_dict.items(): + if user.startswith(k): + return v + + logger.warning(f"No VO for user {user} found, will use None") + + return "None" diff --git a/plugins/apel/tests/test_config.py b/plugins/apel/tests/test_config.py index 93c29a39..f52b1dc8 100644 --- a/plugins/apel/tests/test_config.py +++ b/plugins/apel/tests/test_config.py @@ -11,6 +11,7 @@ Config, ConstantField, Field, + Function, MessageType, MetaField, NormalisedField, @@ -67,7 +68,7 @@ def test_plugin_config(self): value = all_fields["VO"].name - assert value == "voms" + assert value == "user" value = all_fields["CpuDuration"].name @@ -131,7 +132,9 @@ def test_get_value_meta_field_fail(self): assert value == "None" - meta_field = MetaField(name="meta_test", function="missing_function") + meta_field = MetaField( + name="meta_test", function=Function(name="missing_function") + ) with pytest.raises(Exception) as pytest_error: meta_field.get_value(record) @@ -336,3 +339,53 @@ def test_loaders(self): value = config.name assert value == "test_field" + + def test_vo_mapping(self): + record = pyauditor.Record( + "record_id", + datetime(1984, 3, 3, 0, 0, 0).astimezone(tz=timezone.utc), + ) + + vo_dict = {"atlpr": "atlas", "atlsg": "ops", "ops": "ops"} + + meta = pyauditor.Meta() + meta.insert("user", ["atlpr000"]) + record.with_meta(meta) + + meta_field = MetaField( + name="user", function=Function(name="vo_mapping", parameters=vo_dict) + ) + value = meta_field.get_value(record) + + assert value == "atlas" + + with open(Path.joinpath(test_dir, "test_config.yml"), "r") as f: + config: Config = yaml.load(f, Loader=get_loaders()) + + value = config.summary_fields.optional["VO"].get_value(record) + + assert value == "atlas" + + meta = pyauditor.Meta() + meta.insert("user", ["atlsg000"]) + record.with_meta(meta) + + value = config.summary_fields.optional["VO"].get_value(record) + + assert value == "ops" + + meta = pyauditor.Meta() + meta.insert("user", ["ops000"]) + record.with_meta(meta) + + value = config.summary_fields.optional["VO"].get_value(record) + + assert value == "ops" + + meta = pyauditor.Meta() + meta.insert("user", ["ilc000"]) + record.with_meta(meta) + + value = config.summary_fields.optional["VO"].get_value(record) + + assert value == "None" diff --git a/plugins/apel/tests/test_config.yml b/plugins/apel/tests/test_config.yml index 608746ce..cd34cdc7 100644 --- a/plugins/apel/tests/test_config.yml +++ b/plugins/apel/tests/test_config.yml @@ -45,8 +45,13 @@ summary_fields: GlobalUserName: !MetaField name: subject VO: !MetaField - name: voms - regex: (?<=%2F).*?\S(?=%2F) + name: user + function: + name: vo_mapping + parameters: + atlpr: atlas + atlsg: ops + ops: ops VOGroup: !MetaField name: voms regex: (?=%2F).*?\S(?=%2F) diff --git a/plugins/apel/tests/test_utility.py b/plugins/apel/tests/test_utility.py index 185b31a9..a3bf2965 100644 --- a/plugins/apel/tests/test_utility.py +++ b/plugins/apel/tests/test_utility.py @@ -3,7 +3,7 @@ import pytest -from auditor_apel_plugin.utility import write_transaction +from auditor_apel_plugin.utility import vo_mapping, write_transaction CONTENT = [ "Hello World!", @@ -51,3 +51,28 @@ def test_write_transaction_failure( assert file_path.read_text() == initial raise failure assert file_path.read_text() == final + + +class TestUtility: + def test_vo_mapping(self): + vo_dict = {"atlpr": "atlas", "atlsg": "ops", "ops": "ops"} + + user = "atlpr000" + value = vo_mapping(user, vo_dict) + + assert value == "atlas" + + user = "atlsg001" + value = vo_mapping(user, vo_dict) + + assert value == "ops" + + user = "ops" + value = vo_mapping(user, vo_dict) + + assert value == "ops" + + user = "ilc002" + value = vo_mapping(user, vo_dict) + + assert value == "None" From 09d17e6e77cdb161b459eee5f287f9975433d074 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 23:36:29 +0000 Subject: [PATCH 05/14] build(deps): bump ruff from 0.7.3 to 0.8.2 in /plugins/apel Bumps [ruff](https://github.com/astral-sh/ruff) from 0.7.3 to 0.8.2. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.7.3...0.8.2) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- CHANGELOG.md | 10 +++++----- plugins/apel/pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec0f11b8..e2031b06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Dependencies: Switch from pyo3-asyncio 0.20.0 to pyo3-async-runtimes 0.22.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update crate-ci/typos from 1.26.8 to 1.27.3 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update pyo3 from 0.20.3 to 0.22.5 ([@dirksammel](https://github.com/dirksammel)) -- Dependencies: Update ruff from 0.7.1 to 0.7.3 ([@dirksammel](https://github.com/dirksammel)) +- Dependencies: Update ruff from 0.7.1 to 0.8.2 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update setuptools from 75.3.0 to 75.4.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update sqlx from 0.7.4 to 0.8.2 (missed some occurrences) ([@dirksammel](https://github.com/dirksammel)) - Apel plugin: Update timestamp JSON atomically ([@maxfischer2781](https://github.com/maxfischer2781)) @@ -37,7 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - AUDITOR client: rename function `does_not_contains` to `does_not_contain` ### Security -- [RUSTSEC-2024-0363]: Update sqlx from 0.7.4 to 0.8.2 ([@raghuvar-vijay](https://github.com/raghuvar-vijay)) +- [RUSTSEC-2024-0363]: Update sqlx from 0.7.4 to 0.8.2 ([@raghuvar-vijay](https://github.com/raghuvar-vijay)) - [CVE-2024-45311]: Update quinn-proto from 0.11.3 to 0.11.8 ([@raghuvar-vijay](https://github.com/raghuvar-vijay)) ### Added @@ -66,7 +66,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Dependencies: Update types-requests from 2.32.0.20240712 to 2.32.0.20241016 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: update wiremock from 0.6.1 to 0.6.2 ([@raghuvar-vijay](https://github.com/raghuvar-vijay)) - HTCondor collector: Fix bug that the machine score was stored as an integer, not as a float ([@mschnepf](https://github.com/mschnepf)) -- Docs: Typo fixes in query documentation ([@dirksammel](https://github.com/dirksammel)) +- Docs: Typo fixes in query documentation ([@dirksammel](https://github.com/dirksammel)) - Docker files: Linting ([@dirksammel](https://github.com/dirksammel)) ### Removed @@ -276,7 +276,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Breaking changes ### Security - + ### Added - Docs: Add steps for creating a new release ([@QuantumDancer](https://github.com/QuantumDancer)) - CI: Add Python 3.12 to workflows ([@dirksammel](https://github.com/dirksammel)) @@ -294,7 +294,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.3.0] - 2023-11-17 ### Breaking changes -- Auditor: Standardize REST APIs. Routes have changed to single endpoint '/record' with methods such as 'GET', 'PUT', 'POST' (#465) ([@raghuvar-vijay](https://github.com/raghuvar-vijay)) +- Auditor: Standardize REST APIs. Routes have changed to single endpoint '/record' with methods such as 'GET', 'PUT', 'POST' (#465) ([@raghuvar-vijay](https://github.com/raghuvar-vijay)) - Priority plugin: 'auditor' configuration has to be present in the config file. 'prometheus' configuration is optional (#456) ([@raghuvar-vijay](https://github.com/raghuvar-vijay)) - Slurm collector: New filter options to filter slurm jobs are added. The `job_status` key in the config is moved to the `job_filter` section and is renamed to `status` (#472) ([@QuantumDancer](https://github.com/QuantumDancer)) diff --git a/plugins/apel/pyproject.toml b/plugins/apel/pyproject.toml index 9868f177..80ea8b38 100644 --- a/plugins/apel/pyproject.toml +++ b/plugins/apel/pyproject.toml @@ -19,7 +19,7 @@ readme = "README.md" [project.optional-dependencies] style = [ "black==24.10.0", - "ruff==0.7.3", + "ruff==0.8.2", ] tests = [ "pytest==8.3.3", From bc4ec8401fc509d51a696592fd62613d78ce87ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:56:56 +0000 Subject: [PATCH 06/14] build(deps): bump pydantic from 2.9.2 to 2.10.3 in /plugins/apel Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.9.2 to 2.10.3. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.9.2...v2.10.3) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/apel/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2031b06..202fa65e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Auditor Docker container: Switch from fixed to latest Rust version ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Switch from pyo3-asyncio 0.20.0 to pyo3-async-runtimes 0.22.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update crate-ci/typos from 1.26.8 to 1.27.3 ([@dirksammel](https://github.com/dirksammel)) +- Dependencies: Update pydantic from 2.9.2 to 2.10.3 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update pyo3 from 0.20.3 to 0.22.5 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update ruff from 0.7.1 to 0.8.2 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update setuptools from 75.3.0 to 75.4.0 ([@dirksammel](https://github.com/dirksammel)) diff --git a/plugins/apel/pyproject.toml b/plugins/apel/pyproject.toml index 80ea8b38..5885c96d 100644 --- a/plugins/apel/pyproject.toml +++ b/plugins/apel/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "requests==2.32.3", "cryptography==43.0.3", "pyyaml==6.0.2", - "pydantic==2.9.2", + "pydantic==2.10.3", ] description = "AUDITOR plugin for sending accounting data to APEL" readme = "README.md" From 348a5fffcce0a0b3b212f9844db03447f8b39931 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:57:01 +0000 Subject: [PATCH 07/14] build(deps): bump pytest from 8.3.3 to 8.3.4 in /plugins/apel Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.3.3 to 8.3.4. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.3...8.3.4) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/apel/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 202fa65e..b7945c08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Dependencies: Update crate-ci/typos from 1.26.8 to 1.27.3 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update pydantic from 2.9.2 to 2.10.3 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update pyo3 from 0.20.3 to 0.22.5 ([@dirksammel](https://github.com/dirksammel)) +- Dependencies: Update pytest from 8.3.3 to 8.3.4 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update ruff from 0.7.1 to 0.8.2 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update setuptools from 75.3.0 to 75.4.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update sqlx from 0.7.4 to 0.8.2 (missed some occurrences) ([@dirksammel](https://github.com/dirksammel)) diff --git a/plugins/apel/pyproject.toml b/plugins/apel/pyproject.toml index 5885c96d..4305ea8c 100644 --- a/plugins/apel/pyproject.toml +++ b/plugins/apel/pyproject.toml @@ -22,7 +22,7 @@ style = [ "ruff==0.8.2", ] tests = [ - "pytest==8.3.3", + "pytest==8.3.4", "pytest-cov==6.0.0", "mypy==1.13.0", "types-pyyaml==6.0.12.20240917", From 73a3ed49efb852a2fc0aaa8a69d196f0a877d353 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 23:51:22 +0000 Subject: [PATCH 08/14] build(deps): bump crate-ci/typos from 1.27.3 to 1.28.2 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.27.3 to 1.28.2. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/v1.27.3...v1.28.2) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/general.yml | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/general.yml b/.github/workflows/general.yml index bd24eb2e..017a6ded 100644 --- a/.github/workflows/general.yml +++ b/.github/workflows/general.yml @@ -179,7 +179,7 @@ jobs: uses: actions/checkout@v4 - name: Check spelling - uses: crate-ci/typos@v1.27.3 + uses: crate-ci/typos@v1.28.2 test_prometheus_exporter: name: Prometheus exporter tests diff --git a/CHANGELOG.md b/CHANGELOG.md index b7945c08..b5515131 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Auditor Docker container: Switch from fixed to latest Rust version ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Switch from pyo3-asyncio 0.20.0 to pyo3-async-runtimes 0.22.0 ([@dirksammel](https://github.com/dirksammel)) -- Dependencies: Update crate-ci/typos from 1.26.8 to 1.27.3 ([@dirksammel](https://github.com/dirksammel)) +- Dependencies: Update crate-ci/typos from 1.26.8 to 1.28.2 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update pydantic from 2.9.2 to 2.10.3 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update pyo3 from 0.20.3 to 0.22.5 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update pytest from 8.3.3 to 8.3.4 ([@dirksammel](https://github.com/dirksammel)) From ce7249af01c907ad82d93e85ba1c823d79ac8cea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:44:57 +0000 Subject: [PATCH 09/14] build(deps): bump cryptography from 43.0.3 to 44.0.0 in /plugins/apel Bumps [cryptography](https://github.com/pyca/cryptography) from 43.0.3 to 44.0.0. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/43.0.3...44.0.0) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/apel/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5515131..e1117d5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Auditor Docker container: Switch from fixed to latest Rust version ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Switch from pyo3-asyncio 0.20.0 to pyo3-async-runtimes 0.22.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update crate-ci/typos from 1.26.8 to 1.28.2 ([@dirksammel](https://github.com/dirksammel)) +- Dependencies: Update cryptography from 43.0.3 to 44.0.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update pydantic from 2.9.2 to 2.10.3 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update pyo3 from 0.20.3 to 0.22.5 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update pytest from 8.3.3 to 8.3.4 ([@dirksammel](https://github.com/dirksammel)) diff --git a/plugins/apel/pyproject.toml b/plugins/apel/pyproject.toml index 4305ea8c..f8cfd6bd 100644 --- a/plugins/apel/pyproject.toml +++ b/plugins/apel/pyproject.toml @@ -9,7 +9,7 @@ requires-python = ">=3.9" dependencies = [ "python-auditor==0.6.3", "requests==2.32.3", - "cryptography==43.0.3", + "cryptography==44.0.0", "pyyaml==6.0.2", "pydantic==2.10.3", ] From a296d2d085e96196e6e03ea9bcee8131fb508440 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 14:35:34 +0000 Subject: [PATCH 10/14] build(deps): bump setuptools from 75.4.0 to 75.6.0 in /plugins/apel Bumps [setuptools](https://github.com/pypa/setuptools) from 75.4.0 to 75.6.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v75.4.0...v75.6.0) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- CHANGELOG.md | 2 +- collectors/htcondor/pyproject.toml | 4 ++-- plugins/apel/pyproject.toml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1117d5f..c9a063ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Dependencies: Update pyo3 from 0.20.3 to 0.22.5 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update pytest from 8.3.3 to 8.3.4 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update ruff from 0.7.1 to 0.8.2 ([@dirksammel](https://github.com/dirksammel)) -- Dependencies: Update setuptools from 75.3.0 to 75.4.0 ([@dirksammel](https://github.com/dirksammel)) +- Dependencies: Update setuptools from 75.3.0 to 75.6.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update sqlx from 0.7.4 to 0.8.2 (missed some occurrences) ([@dirksammel](https://github.com/dirksammel)) - Apel plugin: Update timestamp JSON atomically ([@maxfischer2781](https://github.com/maxfischer2781)) diff --git a/collectors/htcondor/pyproject.toml b/collectors/htcondor/pyproject.toml index 3709ee05..d8e30f21 100644 --- a/collectors/htcondor/pyproject.toml +++ b/collectors/htcondor/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools==75.4.0"] +requires = ["setuptools==75.6.0"] build-backend = "setuptools.build_meta" [tool.setuptools] @@ -20,7 +20,7 @@ readme = "README.md" [project.optional-dependencies] build = [ "build==1.2.2.post1", - "setuptools==75.4.0", + "setuptools==75.6.0", ] [project.scripts] diff --git a/plugins/apel/pyproject.toml b/plugins/apel/pyproject.toml index f8cfd6bd..9a4cc1fd 100644 --- a/plugins/apel/pyproject.toml +++ b/plugins/apel/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools==75.4.0"] +requires = ["setuptools==75.6.0"] build-backend = "setuptools.build_meta" [project] @@ -30,7 +30,7 @@ tests = [ ] build = [ "build==1.2.2.post1", - "setuptools==75.4.0", + "setuptools==75.6.0", ] [project.scripts] From 2313c9dd83f89c36be1434ea5ad885c7547acf30 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 14:04:02 +0000 Subject: [PATCH 11/14] build(deps): bump codecov/codecov-action from 3 to 5 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3 to 5. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3...v5) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/coverage.yml | 2 +- .github/workflows/python_code_tests.yml | 2 +- CHANGELOG.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 44c63023..ff785793 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -73,7 +73,7 @@ jobs: run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 with: files: lcov.info fail_ci_if_error: true diff --git a/.github/workflows/python_code_tests.yml b/.github/workflows/python_code_tests.yml index f2ecede8..3054c6d5 100644 --- a/.github/workflows/python_code_tests.yml +++ b/.github/workflows/python_code_tests.yml @@ -33,7 +33,7 @@ jobs: - name: Run pytest run: pytest - name: Upload coverage - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 with: files: ./plugins/apel/lcov.info fail_ci_if_error: true diff --git a/CHANGELOG.md b/CHANGELOG.md index c9a063ea..b58055f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Auditor Docker container: Switch from fixed to latest Rust version ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Switch from pyo3-asyncio 0.20.0 to pyo3-async-runtimes 0.22.0 ([@dirksammel](https://github.com/dirksammel)) +- Dependencies: Update codecov/codecov-action from 3 to 5 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update crate-ci/typos from 1.26.8 to 1.28.2 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update cryptography from 43.0.3 to 44.0.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update pydantic from 2.9.2 to 2.10.3 ([@dirksammel](https://github.com/dirksammel)) From e305c06859e3a5732215ce2169c08aa5472cf9be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Raphael=20Kleinem=C3=BChl?= Date: Thu, 6 Jun 2024 13:10:28 +0200 Subject: [PATCH 12/14] Slurm collector fixes (#811,#812,#681,#178) --- CHANGELOG.md | 3 + collectors/slurm/src/sacctcaller.rs | 95 ++++++++++++++++++----------- 2 files changed, 62 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b58055f5..7b224253 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Dependencies: Update setuptools from 75.3.0 to 75.6.0 ([@dirksammel](https://github.com/dirksammel)) - Dependencies: Update sqlx from 0.7.4 to 0.8.2 (missed some occurrences) ([@dirksammel](https://github.com/dirksammel)) - Apel plugin: Update timestamp JSON atomically ([@maxfischer2781](https://github.com/maxfischer2781)) +- Slurm collector: Fix timezone offset of local timestamp `lastcheck` (#681, #178) ([@rkleinem](https://github.com/rkleinem)) +- Slurm collector: Ignore `.extern` steps instead of handling them as separate jobs (#812) ([@rkleinem](https://github.com/rkleinem)) +- Slurm collector: Ignore cancelled jobs which have never been started (#811) ([@rkleinem](https://github.com/rkleinem)) ### Removed - Dependencies: Remove opentelemetry_api (replaced by opentelemetry) ([@dirksammel](https://github.com/dirksammel)) diff --git a/collectors/slurm/src/sacctcaller.rs b/collectors/slurm/src/sacctcaller.rs index ff11d66d..5156a292 100644 --- a/collectors/slurm/src/sacctcaller.rs +++ b/collectors/slurm/src/sacctcaller.rs @@ -12,7 +12,7 @@ use auditor::{ constants::FORBIDDEN_CHARACTERS, domain::{Component, RecordAdd, Score}, }; -use chrono::{DateTime, FixedOffset, Local, Utc}; +use chrono::{DateTime, Local, Utc}; use color_eyre::eyre::{eyre, Result}; use itertools::Itertools; use once_cell::sync::Lazy; @@ -23,7 +23,7 @@ use crate::{ configuration::{AllowedTypes, ComponentConfig, KeyConfig, ParsableType, Settings}, database::Database, shutdown::Shutdown, - CONFIG, END, GROUP, JOBID, KEYS, START, USER, + CONFIG, END, GROUP, JOBID, KEYS, START, STATE, USER, }; type SacctRow = HashMap>; @@ -36,7 +36,12 @@ static BATCH_REGEX: Lazy = Lazy::new(|| { }); static SUB_REGEX: Lazy = Lazy::new(|| { - Regex::new(r"^[0-9]+\.[0-9]*$") + Regex::new(r"^[0-9_]+\.[0-9]*$") + .expect("Could not construct essential Regex for matching job ids.") +}); + +static EXTERN_REGEX: Lazy = Lazy::new(|| { + Regex::new(r"^[0-9_]+\.extern$") .expect("Could not construct essential Regex for matching job ids.") }); @@ -164,7 +169,6 @@ async fn get_job_info(database: &Database) -> Result> { let (nextcheck, rid) = if records.is_empty() { (lastcheck, last_record_id) } else { - let local_offset = Local::now().offset().utc_minus_local(); let (ts, rid) = records.iter().fold( (chrono::DateTime::::MIN_UTC, String::new()), |(acc, _acc_record_id), r| { @@ -175,10 +179,7 @@ async fn get_job_info(database: &Database) -> Result> { }, ); ( - DateTime::::from_naive_utc_and_offset( - ts.naive_utc(), - FixedOffset::east_opt(local_offset).unwrap(), - ), + DateTime::::from_naive_utc_and_offset(ts.naive_utc(), *Local::now().offset()), rid, ) }; @@ -227,37 +228,59 @@ fn tokenize_sacct_output(output: &str, keys: Vec) -> SacctRows { #[tracing::instrument(name = "Parse sacct rows", skip(sacct_rows, keys))] fn parse_sacct_rows(sacct_rows: SacctRows, keys: &[KeyConfig]) -> Result> { tracing::debug!("sacct_rows = {:?}", sacct_rows); - sacct_rows + let mut jobs = Vec::with_capacity(sacct_rows.len()); + for id in sacct_rows .keys() .filter(|k| !BATCH_REGEX.is_match(k)) .filter(|k| !SUB_REGEX.is_match(k)) - .map(|id| -> Result { - let map1 = sacct_rows.get(id).ok_or(eyre!("Cannot get map1"))?; - let map2 = sacct_rows.get(&format!("{id}.batch")); - Ok(keys.iter() - .cloned() - .filter_map(|KeyConfig {name: k, key_type: _, allow_empty: _}| { - let val = match map1.get(&k) { - Some(Some(v)) => Some(v.clone()), - _ => { - if let Some(map2) = map2 { - match map2.get(&k) { - Some(Some(v)) => Some(v.clone()), - _ => { - tracing::error!("Something went wrong during parsing (map1, id: {id}, key: {k}, value: {:?})", map2.get(&k)); - None - }, - } - } else { - tracing::error!("Something went wrong during parsing (map2, id: {id}, key: {k}, value: {:?})", map1.get(&k)); - None + .filter(|k| !EXTERN_REGEX.is_match(k)) + { + let map1 = sacct_rows.get(id).ok_or(eyre!("Cannot get map1"))?; + let map2 = sacct_rows.get(&format!("{id}.batch")); + // A state might look like "CANCELLED by 1000" + let cancelled = map1 + .get(STATE) + .and_then(Option::as_ref) + .ok_or(eyre!("Job {} has no state.", id))? + .extract_string()? + .starts_with("CANCELLED"); + if cancelled + && map1.get(START).and_then(Option::as_ref).is_none() + && map2.and_then(|m| m.get(START)).is_none() + { + tracing::debug!( + "Ignore Job {} since it was cancelled before it was started.", + id + ); + continue; + }; + let job = keys + .iter() + .cloned() + .filter_map(|KeyConfig {name: k, key_type: _, allow_empty: _}| { + let val = match map1.get(&k) { + Some(Some(v)) => Some(v.clone()), + _ => { + if let Some(map2) = map2 { + match map2.get(&k) { + Some(Some(v)) => Some(v.clone()), + _ => { + tracing::error!("Something went wrong during parsing (map1, id: {id}, key: {k}, value: {:?})", map2.get(&k)); + None + }, } - }, - }; - val.map(|val| (k, val)) - }) - .collect::()) - }).collect::>>() + } else { + tracing::error!("Something went wrong during parsing (map2, id: {id}, key: {k}, value: {:?})", map1.get(&k)); + None + } + }, + }; + val.map(|val| (k, val)) + }) + .collect::(); + jobs.push(job); + } + Ok(jobs) } #[tracing::instrument( @@ -450,7 +473,7 @@ fn construct_component_scores(job: &Job, component_config: &ComponentConfig) -> #[cfg(test)] mod tests { use auditor::domain::{ValidAmount, ValidName, ValidValue}; - use chrono::NaiveDateTime; + use chrono::{FixedOffset, NaiveDateTime}; use super::*; use crate::{ From 506966ae1b1125424ac54654d579b00e87bee667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Raphael=20Kleinem=C3=BChl?= Date: Thu, 6 Jun 2024 13:50:35 +0200 Subject: [PATCH 13/14] Slurm collector: Add tests for cancelled jobs --- collectors/slurm/src/sacctcaller.rs | 220 +++++++++++++++++++++++++++- 1 file changed, 219 insertions(+), 1 deletion(-) diff --git a/collectors/slurm/src/sacctcaller.rs b/collectors/slurm/src/sacctcaller.rs index 5156a292..b37009de 100644 --- a/collectors/slurm/src/sacctcaller.rs +++ b/collectors/slurm/src/sacctcaller.rs @@ -246,7 +246,10 @@ fn parse_sacct_rows(sacct_rows: SacctRows, keys: &[KeyConfig]) -> Result::from( + NaiveDateTime::parse_from_str( + "2023-11-07T10:14:01", + "%Y-%m-%dT%H:%M:%S", + ) + .unwrap() + .and_local_timezone( + FixedOffset::east_opt(Local::now().offset().local_minus_utc()) + .unwrap(), + ) + .unwrap(), + ))), + ), + ( + END.to_owned(), + Some(AllowedTypes::DateTime(DateTime::::from( + NaiveDateTime::parse_from_str( + "2023-11-07T11:39:09", + "%Y-%m-%dT%H:%M:%S", + ) + .unwrap() + .and_local_timezone( + FixedOffset::east_opt(Local::now().offset().local_minus_utc()) + .unwrap(), + ) + .unwrap(), + ))), + ), + ( + STATE.to_owned(), + Some(AllowedTypes::String("CANCELLED by 1000".to_owned())), + ), + ]), + ), + ( + "1234567.batch".to_owned(), + SacctRow::from([ + ( + JOBID.to_owned(), + Some(AllowedTypes::String("1234567.batch".to_owned())), + ), + ( + START.to_owned(), + Some(AllowedTypes::DateTime(DateTime::::from( + NaiveDateTime::parse_from_str( + "2023-11-07T10:14:01", + "%Y-%m-%dT%H:%M:%S", + ) + .unwrap() + .and_local_timezone( + FixedOffset::east_opt(Local::now().offset().local_minus_utc()) + .unwrap(), + ) + .unwrap(), + ))), + ), + ( + END.to_owned(), + Some(AllowedTypes::DateTime(DateTime::::from( + NaiveDateTime::parse_from_str( + "2023-11-07T11:39:09", + "%Y-%m-%dT%H:%M:%S", + ) + .unwrap() + .and_local_timezone( + FixedOffset::east_opt(Local::now().offset().local_minus_utc()) + .unwrap(), + ) + .unwrap(), + ))), + ), + ( + STATE.to_owned(), + Some(AllowedTypes::String("CANCELLED by 1000".to_owned())), + ), + ]), + ), + ]); + + let parsed_sacct_rows = parse_sacct_rows(sacct_rows, &keys).unwrap(); + + let expected = vec![Job::from([ + ( + "JobID".to_owned(), + AllowedTypes::String("1234567".to_owned()), + ), + ( + START.to_owned(), + AllowedTypes::DateTime(DateTime::::from( + NaiveDateTime::parse_from_str("2023-11-07T10:14:01", "%Y-%m-%dT%H:%M:%S") + .unwrap() + .and_local_timezone( + FixedOffset::east_opt(Local::now().offset().local_minus_utc()).unwrap(), + ) + .unwrap(), + )), + ), + ( + END.to_owned(), + AllowedTypes::DateTime(DateTime::::from( + NaiveDateTime::parse_from_str("2023-11-07T11:39:09", "%Y-%m-%dT%H:%M:%S") + .unwrap() + .and_local_timezone( + FixedOffset::east_opt(Local::now().offset().local_minus_utc()).unwrap(), + ) + .unwrap(), + )), + ), + ( + STATE.to_owned(), + AllowedTypes::String("CANCELLED by 1000".to_owned()), + ), + ])]; + assert_eq!(parsed_sacct_rows, expected); + } + #[test] fn construct_components_empty_config_succeeds() { let job = Job::from([( From a04358a139499651995a28af6ba48377d430322d Mon Sep 17 00:00:00 2001 From: raghuvar-vijay Date: Wed, 11 Dec 2024 16:04:15 +0100 Subject: [PATCH 14/14] build(deps): Update idna from 0.5.0 to 1.0.3 --- Cargo.lock | 326 +++++++++++++++++++++++++++++++++++++++++++++++------ deny.toml | 1 + 2 files changed, 293 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 52c4899e..6ba83af7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1033,6 +1033,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "dlv-list" version = "0.3.0" @@ -1399,12 +1410,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -1594,6 +1599,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1602,12 +1725,23 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -1770,6 +1904,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" + [[package]] name = "local-channel" version = "0.1.5" @@ -2360,9 +2500,9 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "pyo3" -version = "0.20.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" +checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" dependencies = [ "anyhow", "cfg-if", @@ -2370,7 +2510,7 @@ dependencies = [ "indoc", "libc", "memoffset", - "parking_lot", + "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -2379,35 +2519,35 @@ dependencies = [ ] [[package]] -name = "pyo3-asyncio" -version = "0.20.0" +name = "pyo3-async-runtimes" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea6b68e93db3622f3bb3bf363246cf948ed5375afe7abff98ccbdd50b184995" +checksum = "2529f0be73ffd2be0cc43c013a640796558aa12d7ca0aab5cc14f375b4733031" dependencies = [ "futures", "once_cell", "pin-project-lite", "pyo3", - "pyo3-asyncio-macros", + "pyo3-async-runtimes-macros", "tokio", ] [[package]] -name = "pyo3-asyncio-macros" -version = "0.20.0" +name = "pyo3-async-runtimes-macros" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c467178e1da6252c95c29ecf898b133f742e9181dca5def15dc24e19d45a39" +checksum = "22c26fd8e9fc19f53f0c1e00bf61471de6789f7eb263056f7f944a9cceb5823e" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.85", ] [[package]] name = "pyo3-build-config" -version = "0.20.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" +checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" dependencies = [ "once_cell", "target-lexicon", @@ -2415,9 +2555,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" +checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" dependencies = [ "libc", "pyo3-build-config", @@ -2425,9 +2565,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" +checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2437,11 +2577,11 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" +checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro2", "pyo3-build-config", "quote", @@ -2457,7 +2597,7 @@ dependencies = [ "auditor-client", "chrono", "pyo3", - "pyo3-asyncio", + "pyo3-async-runtimes", "serde", "serde_json", "tokio", @@ -3167,7 +3307,7 @@ checksum = "1804e8a7c7865599c9c79be146dc8a9fd8cc86935fa641d3ea58e5f0688abaa5" dependencies = [ "dotenvy", "either", - "heck 0.5.0", + "heck", "hex", "once_cell", "proc-macro2", @@ -3294,6 +3434,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "stringprep" version = "0.1.5" @@ -3348,6 +3494,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -3428,6 +3585,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -3717,9 +3884,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", @@ -3732,6 +3899,18 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" version = "1.11.0" @@ -4128,6 +4307,18 @@ dependencies = [ "url", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "yaml-rust" version = "0.4.5" @@ -4137,6 +4328,30 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -4158,12 +4373,55 @@ dependencies = [ "syn 2.0.85", ] +[[package]] +name = "zerofrom" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "zstd" version = "0.13.2" diff --git a/deny.toml b/deny.toml index eaf32f21..608c41d7 100644 --- a/deny.toml +++ b/deny.toml @@ -81,6 +81,7 @@ allow = [ "ISC", "OpenSSL", "CC0-1.0", + "Unicode-3.0" ] # The confidence threshold for detecting a license from license text. # The higher the value, the more closely the license text must be to the