Skip to content

Commit

Permalink
fix component docs
Browse files Browse the repository at this point in the history
  • Loading branch information
fuchsnj committed Apr 18, 2023
1 parent eaa7154 commit 21d76aa
Show file tree
Hide file tree
Showing 2 changed files with 141 additions and 1 deletion.
2 changes: 1 addition & 1 deletion lib/codecs/src/decoding/framing/newline_delimited.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ pub struct NewlineDelimitedDecoderOptions {
/// consider setting the maximum length to a reasonably large value as a safety net. This
/// ensures that processing is not actually unbounded.
#[serde(skip_serializing_if = "vector_core::serde::skip_serializing_if_default")]
max_length: Option<usize>,
pub max_length: Option<usize>,
}

impl NewlineDelimitedDecoderOptions {
Expand Down
140 changes: 140 additions & 0 deletions website/cue/reference/components/sources/base/aws_s3.cue
Original file line number Diff line number Diff line change
Expand Up @@ -135,11 +135,151 @@ base: components: sources: aws_s3: configuration: {
}
}
}
decoding: {
description: "Configures how events are decoded from raw bytes."
required: false
type: object: options: codec: {
description: "The codec to use for decoding events."
required: false
type: string: {
default: "bytes"
enum: {
bytes: "Uses the raw bytes as-is."
gelf: """
Decodes the raw bytes as a [GELF][gelf] message.
[gelf]: https://docs.graylog.org/docs/gelf
"""
json: """
Decodes the raw bytes as [JSON][json].
[json]: https://www.json.org/
"""
native: """
Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf].
This codec is **[experimental][experimental]**.
[vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto
[experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs
"""
native_json: """
Decodes the raw bytes as Vector’s [native JSON format][vector_native_json].
This codec is **[experimental][experimental]**.
[vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue
[experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs
"""
syslog: """
Decodes the raw bytes as a Syslog message.
Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the
[RFC 5424][rfc5424]-style format ("new" style, includes structured data).
[rfc3164]: https://www.ietf.org/rfc/rfc3164.txt
[rfc5424]: https://www.ietf.org/rfc/rfc5424.txt
"""
}
}
}
}
endpoint: {
description: "Custom endpoint for use with AWS-compatible services."
required: false
type: string: examples: ["http://127.0.0.0:5000/path/to/service"]
}
framing: {
description: """
Framing configuration.
Framing handles how events are separated when encoded in a raw byte form, where each event is
a frame that must be prefixed, or delimited, in a way that marks where an event begins and
ends within the byte stream.
"""
required: false
type: object: options: {
character_delimited: {
description: "Options for the character delimited decoder."
relevant_when: "method = \"character_delimited\""
required: true
type: object: options: {
delimiter: {
description: "The character that delimits byte sequences."
required: true
type: uint: {}
}
max_length: {
description: """
The maximum length of the byte buffer.
This length does *not* include the trailing delimiter.
By default, there is no maximum length enforced. If events are malformed, this can lead to
additional resource usage as events continue to be buffered in memory, and can potentially
lead to memory exhaustion in extreme cases.
If there is a risk of processing malformed data, such as logs with user-controlled input,
consider setting the maximum length to a reasonably large value as a safety net. This
ensures that processing is not actually unbounded.
"""
required: false
type: uint: {}
}
}
}
method: {
description: "The framing method."
required: false
type: string: {
default: "newline_delimited"
enum: {
bytes: "Byte frames are passed through as-is according to the underlying I/O boundaries (for example, split between messages or stream segments)."
character_delimited: "Byte frames which are delimited by a chosen character."
length_delimited: "Byte frames which are prefixed by an unsigned big-endian 32-bit integer indicating the length."
newline_delimited: "Byte frames which are delimited by a newline character."
octet_counting: """
Byte frames according to the [octet counting][octet_counting] format.
[octet_counting]: https://tools.ietf.org/html/rfc6587#section-3.4.1
"""
}
}
}
newline_delimited: {
description: "Options for the newline delimited decoder."
relevant_when: "method = \"newline_delimited\""
required: false
type: object: options: max_length: {
description: """
The maximum length of the byte buffer.
This length does *not* include the trailing delimiter.
By default, there is no maximum length enforced. If events are malformed, this can lead to
additional resource usage as events continue to be buffered in memory, and can potentially
lead to memory exhaustion in extreme cases.
If there is a risk of processing malformed data, such as logs with user-controlled input,
consider setting the maximum length to a reasonably large value as a safety net. This
ensures that processing is not actually unbounded.
"""
required: false
type: uint: {}
}
}
octet_counting: {
description: "Options for the octet counting decoder."
relevant_when: "method = \"octet_counting\""
required: false
type: object: options: max_length: {
description: "The maximum length of the byte buffer."
required: false
type: uint: {}
}
}
}
}
multiline: {
description: """
Multiline aggregation configuration.
Expand Down

0 comments on commit 21d76aa

Please sign in to comment.