feat(ourlogs): Allow log ingestion behind a flag (#4448)

This adds log ingestion (currently only working for the OTel log format)
behind a feature flag 'organizations:ourlogs-ingestion'.

This PR aims to be the minimum possible to support local and test-org
ingestion before we move to dogfooding.

![Screenshot 2025-01-15 at 1 26
49 PM](https://github.com/user-attachments/assets/45f0be20-05d8-45b4-988c-d00327fc2e9c)


Other notes:
- We need to add two `DataCategory`s because we need to track quantity
(for current discarded breadcrumb client outcome tracking) and also
bytes for total log bytes ingested, which is one of the quota
recommendations.
- Eventually we will convert Breadcrumbs into logs as well, very similar
to span extraction for spans on the event. How exactly that will work is
still being discussed with product and sdk folks.
- The name `ourlogs` is an internal name to disambiguate between 'our
log product' logs (corny, I know) and internally created logs. User
facing strings will be set to 'Log' to avoid exposing implementation
details.
- Depends on getsentry/sentry#81930 for the
ingest feature flag.

CHANGELOG.md

@@ -15,6 +15,7 @@
 **Internal**:
 
 - Updates performance score calculation on spans and events to also store cdf values as measurements. ([#4438](https://github.com/getsentry/relay/pull/4438))
+- Allow log ingestion behind a flag, only for internal use currently. ([#4448](https://github.com/getsentry/relay/pull/4448))
 
 ## 24.12.2
 

Cargo.toml

@@ -43,6 +43,7 @@ relay-kafka = { path = "relay-kafka" }
 relay-log = { path = "relay-log" }
 relay-metrics = { path = "relay-metrics" }
 relay-monitors = { path = "relay-monitors" }
+relay-ourlogs = { path = "relay-ourlogs" }
 relay-pattern = { path = "relay-pattern" }
 relay-pii = { path = "relay-pii" }
 relay-profiling = { path = "relay-profiling" }
@@ -150,7 +151,7 @@ reqwest = "0.12.9"
 rmp-serde = "1.3.0"
 sentry = "0.34.0"
 sentry-core = "0.34.0"
-sentry-kafka-schemas = { version = "0.1.122", default-features = false }
+sentry-kafka-schemas = { version = "0.1.129", default-features = false }
 sentry-release-parser = { version = "1.3.2", default-features = false }
 sentry-types = "0.34.0"
 semver = "1.0.23"

py/sentry_relay/consts.py

@@ -8,7 +8,7 @@
 
 
 class DataCategory(IntEnum):
-    # begin generated
+    # start generated
     DEFAULT = 0
     ERROR = 1
     TRANSACTION = 2

relay-cabi/include/relay.h

@@ -143,14 +143,14 @@ enum RelayDataCategory {
    */
   RELAY_DATA_CATEGORY_ATTACHMENT_ITEM = 22,
   /**
-   * LogCount
+   * LogItem
    *
    * This is the category for logs for which we store the count log events for users for measuring
    * missing breadcrumbs, and count of logs for rate limiting purposes.
    */
   RELAY_DATA_CATEGORY_LOG_ITEM = 23,
   /**
-   * LogBytes
+   * LogByte
    *
    * This is the category for logs for which we store log event total bytes for users.
    */

relay-cogs/src/lib.rs

@@ -117,6 +117,8 @@ pub enum AppFeature {
     Transactions,
     /// Errors.
     Errors,
+    /// Logs.
+    Logs,
     /// Spans.
     Spans,
     /// Sessions.
@@ -159,6 +161,7 @@ impl AppFeature {
             Self::Transactions => "transactions",
             Self::Errors => "errors",
             Self::Spans => "spans",
+            Self::Logs => "our_logs",
             Self::Sessions => "sessions",
             Self::ClientReports => "client_reports",
             Self::CheckIns => "check_ins",

relay-config/src/config.rs

@@ -617,6 +617,8 @@ pub struct Limits {
     /// The maximum payload size for a profile
     pub max_profile_size: ByteSize,
     /// The maximum payload size for a span.
+    pub max_log_size: ByteSize,
+    /// The maximum payload size for a span.
     pub max_span_size: ByteSize,
     /// The maximum payload size for a statsd metric.
     pub max_statsd_size: ByteSize,
@@ -683,6 +685,7 @@ impl Default for Limits {
             max_api_file_upload_size: ByteSize::mebibytes(40),
             max_api_chunk_upload_size: ByteSize::mebibytes(100),
             max_profile_size: ByteSize::mebibytes(50),
+            max_log_size: ByteSize::mebibytes(1),
             max_span_size: ByteSize::mebibytes(1),
             max_statsd_size: ByteSize::mebibytes(1),
             max_metric_buckets_size: ByteSize::mebibytes(1),
@@ -2213,6 +2216,11 @@ impl Config {
         self.values.limits.max_check_in_size.as_bytes()
     }
 
+    /// Returns the maximum payload size of a log in bytes.
+    pub fn max_log_size(&self) -> usize {
+        self.values.limits.max_log_size.as_bytes()
+    }
+
     /// Returns the maximum payload size of a span in bytes.
     pub fn max_span_size(&self) -> usize {
         self.values.limits.max_span_size.as_bytes()

relay-dynamic-config/src/feature.rs

@@ -102,7 +102,11 @@ pub enum Feature {
     /// Serialized as `organizations:ingest-spans-in-eap`
     #[serde(rename = "organizations:ingest-spans-in-eap")]
     IngestSpansInEap,
-
+    /// Enable log ingestion for our log product (this is not internal logging).
+    ///
+    /// Serialized as `organizations:ourlogs-ingestion`.
+    #[serde(rename = "organizations:ourlogs-ingestion")]
+    OurLogsIngestion,
     /// This feature has graduated and is hard-coded for external Relays.
     #[doc(hidden)]
     #[serde(rename = "projects:profiling-ingest-unsampled-profiles")]

relay-event-schema/src/processor/attrs.rs

@@ -46,6 +46,7 @@ pub enum ValueType {
     Message,
     Thread,
     Breadcrumb,
+    OurLog,
     Span,
     ClientSdkInfo,
 
@@ -84,6 +85,7 @@ relay_common::derive_fromstr_and_display!(ValueType, UnknownValueTypeError, {
     ValueType::Message => "message",
     ValueType::Thread => "thread",
     ValueType::Breadcrumb => "breadcrumb",
+    ValueType::OurLog => "ourlog",
     ValueType::Span => "span",
     ValueType::ClientSdkInfo => "sdk",
     ValueType::Minidump => "minidump",

relay-event-schema/src/processor/traits.rs

@@ -108,10 +108,12 @@ pub trait Processor: Sized {
     process_method!(process_breadcrumb, crate::protocol::Breadcrumb);
     process_method!(process_template_info, crate::protocol::TemplateInfo);
     process_method!(process_header_name, crate::protocol::HeaderName);
+    process_method!(process_ourlog, crate::protocol::OurLog);
     process_method!(process_span, crate::protocol::Span);
     process_method!(process_trace_context, crate::protocol::TraceContext);
     process_method!(process_native_image_path, crate::protocol::NativeImagePath);
     process_method!(process_contexts, crate::protocol::Contexts);
+    process_method!(process_attribute_value, crate::protocol::AttributeValue);
 
     fn process_other(
         &mut self,

relay-event-schema/src/protocol/mod.rs

@@ -18,6 +18,7 @@ mod mechanism;
 mod metrics;
 mod metrics_summary;
 mod nel;
+mod ourlog;
 mod relay_info;
 mod replay;
 mod request;
@@ -54,6 +55,7 @@ pub use self::mechanism::*;
 pub use self::metrics::*;
 pub use self::metrics_summary::*;
 pub use self::nel::*;
+pub use self::ourlog::*;
 pub use self::relay_info::*;
 pub use self::replay::*;
 pub use self::request::*;