feat(sources): New amqp (Rabbitmq) Source/Sink (#7120)

* Amqp Add amqp source and sink. Signed-off-by: dbcfd <[email protected]> * Changes following merge with master. Signed-off-by: Stephen Wakely <[email protected]> * Fixes following merge with master. Signed-off-by: Stephen Wakely <[email protected]> * Moved docs to website. Signed-off-by: Stephen Wakely <[email protected]> * Apply feedback. Signed-off-by: Stephen Wakely <[email protected]> * cue fmt Signed-off-by: Stephen Wakely <[email protected]> * Cue errors. Signed-off-by: Stephen Wakely <[email protected]> * Update to master Signed-off-by: Stephen Wakely <[email protected]> * Update integration tests. Signed-off-by: Stephen Wakely <[email protected]> * Clippy Signed-off-by: Stephen Wakely <[email protected]> * Udpate to stream sink. Signed-off-by: Stephen Wakely <[email protected]> * Update lapin. Signed-off-by: Stephen Wakely <[email protected]> * Feedback from Spencer. Signed-off-by: Stephen Wakely <[email protected]> * Fixed docs. Signed-off-by: Stephen Wakely <[email protected]> * Update integration tests to test emitted events. Signed-off-by: Stephen Wakely <[email protected]> * Separate into modules. Signed-off-by: Stephen Wakely <[email protected]> * Add namespacing for source. Signed-off-by: Stephen Wakely <[email protected]> * Fixed after master merge. Signed-off-by: Stephen Wakely <[email protected]> * Update sink tests. Signed-off-by: Stephen Wakely <[email protected]> * Fix docs key defaults. Signed-off-by: Stephen Wakely <[email protected]> * Source needs a valid connection to create ok. Signed-off-by: Stephen Wakely <[email protected]> * Tidy up code a little. Signed-off-by: Stephen Wakely <[email protected]> * Clippy Signed-off-by: Stephen Wakely <[email protected]> * Remove old integration file Signed-off-by: Stephen Wakely <[email protected]> * Update internal events. Signed-off-by: Stephen Wakely <[email protected]> * Added e2e acknowledgements. Signed-off-by: Stephen Wakely <[email protected]> * Clippy Signed-off-by: Stephen Wakely <[email protected]> * Formatting Signed-off-by: Stephen Wakely <[email protected]> * More formatting Signed-off-by: Stephen Wakely <[email protected]> * Fixed event errors. Signed-off-by: Stephen Wakely <[email protected]> * Removed nonexisting import. Signed-off-by: Stephen Wakely <[email protected]> * Configurable changes Signed-off-by: Stephen Wakely <[email protected]> * cue fmt Signed-off-by: Stephen Wakely <[email protected]> * cue fmt Signed-off-by: Stephen Wakely <[email protected]> * Fix internal events feature. Signed-off-by: Stephen Wakely <[email protected]> * Update sink configurable. Signed-off-by: Stephen Wakely <[email protected]> * Add sink to enum. Signed-off-by: Stephen Wakely <[email protected]> * Responding to feedback from Spencer, Pierre and Kyle. Signed-off-by: Stephen Wakely <[email protected]> * Fixes following merge with master. Signed-off-by: Stephen Wakely <[email protected]> * Further feedback from Spencer and Kyle. Signed-off-by: Stephen Wakely <[email protected]> * Add count to EventsReceived. Signed-off-by: Stephen Wakely <[email protected]> * Fix duplicate events CI error Signed-off-by: Stephen Wakely <[email protected]> * Rename routing_key to routing_key_field Signed-off-by: Stephen Wakely <[email protected]> * Fmt Signed-off-by: Stephen Wakely <[email protected]> * Updated tests. Signed-off-by: Stephen Wakely <[email protected]> * Cue fmt Signed-off-by: Stephen Wakely <[email protected]> * Update routing_key name Signed-off-by: Stephen Wakely <[email protected]> * Fix following merge. Signed-off-by: Stephen Wakely <[email protected]> * Use write_all utility fn. Signed-off-by: Stephen Wakely <[email protected]> * Renamed tests.rs to integration_tests.rs Signed-off-by: Stephen Wakely <[email protected]> * Remove inventory submit Signed-off-by: Stephen Wakely <[email protected]> Signed-off-by: dbcfd <[email protected]> Signed-off-by: Stephen Wakely <[email protected]> Co-authored-by: Stephen Wakely <[email protected]>
vectordotdev · Sep 20, 2022 · dec33dc · dec33dc
1 parent a3de633
commit dec33dc
Show file tree

Hide file tree

Showing 25 changed files with 2,077 additions and 2 deletions.
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -55,6 +55,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - test: 'amqp'
           - test: 'apex'
           - test: 'aws'
           - test: 'axiom'

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -198,6 +198,9 @@ prost-types = { version = "0.10.1", default-features = false, optional = true }
 goauth = { version = "0.13.1", optional = true }
 smpl_jwt = { version = "0.7.1", default-features = false, optional = true }
 
+# AMQP
+lapin = { version = "2.1.1", default-features = false, optional = true }
+
 # API
 async-graphql = { version = "4.0.13", default-features = false, optional = true, features = ["chrono"] }
 async-graphql-warp = { version = "4.0.13", default-features = false, optional = true }
@@ -442,6 +445,7 @@ enrichment-tables-geoip = ["dep:maxminddb"]
 # Sources
 sources = ["sources-logs", "sources-metrics"]
 sources-logs = [
+  "sources-amqp",
   "sources-aws_kinesis_firehose",
   "sources-aws_s3",
   "sources-aws_sqs",
@@ -484,6 +488,7 @@ sources-metrics = [
   "sources-vector",
 ]
 
+sources-amqp = ["lapin"]
 sources-apache_metrics = []
 sources-aws_ecs_metrics = []
 sources-aws_kinesis_firehose = ["dep:base64", "dep:infer"]
@@ -577,6 +582,7 @@ transforms-throttle = ["dep:governor"]
 # Sinks
 sinks = ["sinks-logs", "sinks-metrics"]
 sinks-logs = [
+  "sinks-amqp",
   "sinks-apex",
   "sinks-aws_cloudwatch_logs",
   "sinks-aws_kinesis_firehose",
@@ -631,6 +637,7 @@ sinks-metrics = [
   "sinks-splunk_hec"
 ]
 
+sinks-amqp = ["lapin"]
 sinks-apex = []
 sinks-aws_cloudwatch_logs = ["aws-core", "dep:aws-sdk-cloudwatchlogs"]
 sinks-aws_cloudwatch_metrics = ["aws-core", "dep:aws-sdk-cloudwatch"]
@@ -693,6 +700,7 @@ nightly = []
 
 # Testing-related features
 all-integration-tests = [
+  "amqp-integration-tests",
   "apex-integration-tests",
   "aws-integration-tests",
   "axiom-integration-tests",
@@ -728,6 +736,8 @@ all-integration-tests = [
   "dnstap-integration-tests",
 ]
 
+amqp-integration-tests = ["sources-amqp", "sinks-amqp"]
+
 aws-integration-tests = [
   "aws-cloudwatch-logs-integration-tests",
   "aws-cloudwatch-metrics-integration-tests",
@@ -877,4 +887,3 @@ name = "codecs"
 path = "benches/codecs/main.rs"
 harness = false
 required-features = ["codecs-benches"]
-
diff --git a/Makefile b/Makefile
@@ -334,7 +334,7 @@ test-enterprise: ## Runs enterprise related behavioral tests
 
 .PHONY: test-integration
 test-integration: ## Runs all integration tests
-test-integration: test-integration-apex test-integration-aws test-integration-axiom test-integration-azure test-integration-clickhouse test-integration-docker-logs test-integration-elasticsearch
+test-integration: test-integration-amqp test-integration-apex test-integration-aws test-integration-axiom test-integration-azure test-integration-clickhouse test-integration-docker-logs test-integration-elasticsearch
 test-integration: test-integration-eventstoredb test-integration-fluent test-integration-gcp test-integration-humio test-integration-http-scrape test-integration-influxdb
 test-integration: test-integration-kafka test-integration-logstash test-integration-loki test-integration-mongodb test-integration-nats
 test-integration: test-integration-nginx test-integration-opentelemetry test-integration-postgres test-integration-prometheus test-integration-pulsar

diff --git a/scripts/integration/docker-compose.amqp.yml b/scripts/integration/docker-compose.amqp.yml
@@ -0,0 +1,36 @@
+version: "3"
+
+services:
+  rabbitmq:
+    image: docker.io/rabbitmq:3.8
+    ports:
+      - 5672:5672
+  runner:
+    build:
+      context:  ${PWD}
+      dockerfile: scripts/integration/Dockerfile
+      args:
+        - RUST_VERSION=${RUST_VERSION}
+    working_dir: /code
+    command:
+      - "cargo"
+      - "nextest"
+      - "run"
+      - "--no-fail-fast"
+      - "--no-default-features"
+      - "--features"
+      - "amqp-integration-tests"
+      - "--lib"
+      - "${FILTER:-::amqp::}"
+    depends_on:
+      - rabbitmq
+    volumes:
+      - ${PWD}:/code
+      - target:/code/target
+      - cargogit:/usr/local/cargo/git
+      - cargoregistry:/usr/local/cargo/registry
+
+volumes:
+  target: {}
+  cargogit: {}
+  cargoregistry: {}