Add NATS JetStream support (#1866)

* Add NATS JetStream support Update shopify/sarama * Fix addresses * Don't change Addresses in Defaults * Update saramajetstream * Add missing error check Keep typing events for at least one minute * Use all configured NATS addresses * Update saramajetstream * Try setting up with NATS * Make sure NATS uses own persistent directory (TODO: make this configurable) * Update go.mod/go.sum * Jetstream package * Various other refactoring * Build fixes * Config tweaks, make random jetstream storage path for CI * Disable interest policies * Try to sane default on jetstream base path * Try to use in-memory for CI * Restore storage/retention * Update nats.go dependency * Adapt changes to config * Remove unneeded TopicFor * Dep update * Revert "Remove unneeded TopicFor" This reverts commit f5a4e4a. * Revert changes made to streams * Fix build problems * Update nats-server * Update go.mod/go.sum * Roomserver input API queuing using NATS * Fix topic naming * Prometheus metrics * More refactoring to remove saramajetstream * Add missing topic * Don't try to populate map that doesn't exist * Roomserver output topic * Update go.mod/go.sum * Message acknowledgements * Ack tweaks * Try to resume transaction re-sends * Try to resume transaction re-sends * Update to matrix-org/gomatrixserverlib@91dadfb * Remove internal.PartitionStorer from components that don't consume keychanges * Try to reduce re-allocations a bit in resolveConflictsV2 * Tweak delivery options on RS input * Publish send-to-device messages into correct JetStream subject * Async and sync roomserver input * Update dendrite-config.yaml * Remove roomserver tests for now (they need rewriting) * Remove roomserver test again (was merged back in) * Update documentation * Docker updates * More Docker updates * Update Docker readme again * Fix lint issues * Send final event in `processEvent` synchronously (since this might stop Sytest from being so upset) * Don't report event rejection errors via `/send`, since apparently this is upsetting tests that don't expect that * Go 1.16 instead of Go 1.13 for upgrade tests and Complement * Revert "Don't report event rejection errors via `/send`, since apparently this is upsetting tests that don't expect that" This reverts commit 3686752. * Don't report any errors on `/send` to see what fun that creates * Fix panics on closed channel sends * Enforce state key matches sender * Do the same for leave * Various tweaks to make tests happier Squashed commit of the following: commit 13f9028 Author: Neil Alexander <[email protected]> Date: Tue Jan 4 15:47:14 2022 +0000 Do the same for leave commit e6be7f0 Author: Neil Alexander <[email protected]> Date: Tue Jan 4 15:33:42 2022 +0000 Enforce state key matches sender commit 85ede6d Author: Neil Alexander <[email protected]> Date: Tue Jan 4 14:07:04 2022 +0000 Fix panics on closed channel sends commit 9755494 Author: Neil Alexander <[email protected]> Date: Tue Jan 4 13:38:22 2022 +0000 Don't report any errors on `/send` to see what fun that creates commit 3bb4f87 Author: Neil Alexander <[email protected]> Date: Tue Jan 4 13:00:26 2022 +0000 Revert "Don't report event rejection errors via `/send`, since apparently this is upsetting tests that don't expect that" This reverts commit 3686752. commit fe2673e Author: Neil Alexander <[email protected]> Date: Tue Jan 4 12:09:34 2022 +0000 Go 1.16 instead of Go 1.13 for upgrade tests and Complement commit 3686752 Author: Neil Alexander <[email protected]> Date: Tue Jan 4 11:51:45 2022 +0000 Don't report event rejection errors via `/send`, since apparently this is upsetting tests that don't expect that commit b028dfc Author: Neil Alexander <[email protected]> Date: Tue Jan 4 10:29:08 2022 +0000 Send final event in `processEvent` synchronously (since this might stop Sytest from being so upset) * Merge in NATS Server v2.6.6 and nats.go v1.13 into the in-process connection fork * Add `jetstream.WithJetStreamMessage` to make ack/nak-ing less messy, use process context in consumers * Fix consumer component name in federation API * Add comment explaining where streams are defined * Tweaks to roomserver input with comments * Finish that sentence that I apparently forgot to finish in INSTALL.md * Bump version number of config to 2 * Add comments around asynchronous sends to roomserver in processEventWithMissingState * More useful error message when the config version does not match * Set version in generate-config * Fix version in config.Defaults Co-authored-by: Neil Alexander <[email protected]>
matrix-org · Jan 5, 2022 · 161f145 · 161f145
1 parent a47b12d
commit 161f145
Show file tree

Hide file tree

Showing 75 changed files with 1,323 additions and 1,702 deletions.
diff --git a/appservice/appservice.go b/appservice/appservice.go
@@ -32,7 +32,7 @@ import (
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/kafka"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/sirupsen/logrus"
 )
@@ -58,7 +58,7 @@ func NewInternalAPI(
 			},
 		},
 	}
-	consumer, _ := kafka.SetupConsumerProducer(&base.Cfg.Global.Kafka)
+	js, _, _ := jetstream.Prepare(&base.Cfg.Global.JetStream)
 
 	// Create a connection to the appservice postgres DB
 	appserviceDB, err := storage.NewDatabase(&base.Cfg.AppServiceAPI.Database)
@@ -97,7 +97,7 @@ func NewInternalAPI(
 	// We can't add ASes at runtime so this is safe to do.
 	if len(workerStates) > 0 {
 		consumer := consumers.NewOutputRoomEventConsumer(
-			base.ProcessContext, base.Cfg, consumer, appserviceDB,
+			base.ProcessContext, base.Cfg, js, appserviceDB,
 			rsAPI, workerStates,
 		)
 		if err := consumer.Start(); err != nil {

diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
@@ -20,79 +20,81 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/storage"
 	"github.com/matrix-org/dendrite/appservice/types"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
 
-	"github.com/Shopify/sarama"
 	log "github.com/sirupsen/logrus"
 )
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
-	roomServerConsumer *internal.ContinualConsumer
-	asDB               storage.Database
-	rsAPI              api.RoomserverInternalAPI
-	serverName         string
-	workerStates       []types.ApplicationServiceWorkerState
+	ctx          context.Context
+	jetstream    nats.JetStreamContext
+	topic        string
+	asDB         storage.Database
+	rsAPI        api.RoomserverInternalAPI
+	serverName   string
+	workerStates []types.ApplicationServiceWorkerState
 }
 
 // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call
 // Start() to begin consuming from room servers.
 func NewOutputRoomEventConsumer(
 	process *process.ProcessContext,
 	cfg *config.Dendrite,
-	kafkaConsumer sarama.Consumer,
+	js nats.JetStreamContext,
 	appserviceDB storage.Database,
 	rsAPI api.RoomserverInternalAPI,
 	workerStates []types.ApplicationServiceWorkerState,
 ) *OutputRoomEventConsumer {
-	consumer := internal.ContinualConsumer{
-		Process:        process,
-		ComponentName:  "appservice/roomserver",
-		Topic:          cfg.Global.Kafka.TopicFor(config.TopicOutputRoomEvent),
-		Consumer:       kafkaConsumer,
-		PartitionStore: appserviceDB,
+	return &OutputRoomEventConsumer{
+		ctx:          process.Context(),
+		jetstream:    js,
+		topic:        cfg.Global.JetStream.TopicFor(jetstream.OutputRoomEvent),
+		asDB:         appserviceDB,
+		rsAPI:        rsAPI,
+		serverName:   string(cfg.Global.ServerName),
+		workerStates: workerStates,
 	}
-	s := &OutputRoomEventConsumer{
-		roomServerConsumer: &consumer,
-		asDB:               appserviceDB,
-		rsAPI:              rsAPI,
-		serverName:         string(cfg.Global.ServerName),
-		workerStates:       workerStates,
-	}
-	consumer.ProcessMessage = s.onMessage
-
-	return s
 }
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
-	return s.roomServerConsumer.Start()
+	_, err := s.jetstream.Subscribe(s.topic, s.onMessage)
+	return err
 }
 
 // onMessage is called when the appservice component receives a new event from
 // the room server output log.
-func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
-	// Parse out the event JSON
-	var output api.OutputEvent
-	if err := json.Unmarshal(msg.Value, &output); err != nil {
-		// If the message was invalid, log it and move on to the next message in the stream
-		log.WithError(err).Errorf("roomserver output log: message parse failure")
-		return nil
-	}
+func (s *OutputRoomEventConsumer) onMessage(msg *nats.Msg) {
+	jetstream.WithJetStreamMessage(msg, func(msg *nats.Msg) bool {
+		// Parse out the event JSON
+		var output api.OutputEvent
+		if err := json.Unmarshal(msg.Data, &output); err != nil {
+			// If the message was invalid, log it and move on to the next message in the stream
+			log.WithError(err).Errorf("roomserver output log: message parse failure")
+			return true
+		}
 
-	if output.Type != api.OutputTypeNewRoomEvent {
-		return nil
-	}
+		if output.Type != api.OutputTypeNewRoomEvent {
+			return true
+		}
+
+		events := []*gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event}
+		events = append(events, output.NewRoomEvent.AddStateEvents...)
 
-	events := []*gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event}
-	events = append(events, output.NewRoomEvent.AddStateEvents...)
+		// Send event to any relevant application services
+		if err := s.filterRoomserverEvents(context.TODO(), events); err != nil {
+			log.WithError(err).Errorf("roomserver output log: filter error")
+			return true
+		}
 
-	// Send event to any relevant application services
-	return s.filterRoomserverEvents(context.TODO(), events)
+		return true
+	})
 }
 
 // filterRoomserverEvents takes in events and decides whether any of them need

diff --git a/appservice/storage/interface.go b/appservice/storage/interface.go
@@ -17,12 +17,10 @@ package storage
 import (
 	"context"
 
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type Database interface {
-	internal.PartitionStorer
 	StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.HeaderedEvent) error
 	GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error)
 	CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error)

diff --git a/build/docker/README.md b/build/docker/README.md
@@ -19,7 +19,6 @@ not contain the Go toolchain etc.
 
 There are three sample `docker-compose` files:
 
-- `docker-compose.deps.yml` which runs the Postgres and Kafka prerequisites
 - `docker-compose.monolith.yml` which runs a monolith Dendrite deployment
 - `docker-compose.polylith.yml` which runs a polylith Dendrite deployment
 
@@ -28,7 +27,7 @@ There are three sample `docker-compose` files:
 The `docker-compose` files refer to the `/etc/dendrite` volume as where the
 runtime config should come from. The mounted folder must contain:
 
-- `dendrite.yaml` configuration file (based on the [`dendrite-config.yaml`](https://raw.githubusercontent.com/matrix-org/dendrite/master/dendrite-config.yaml)
+- `dendrite.yaml` configuration file (from the [Docker config folder](https://github.com/matrix-org/dendrite/tree/master/build/docker/config)
    sample in the `build/docker/config` folder of this repository.)
 - `matrix_key.pem` server key, as generated using `cmd/generate-keys`
 - `server.crt` certificate file
@@ -50,31 +49,19 @@ The key files will now exist in your current working directory, and can be mount
 
 ## Starting Dendrite as a monolith deployment
 
-Create your config based on the [`dendrite-config.yaml`](https://raw.githubusercontent.com/matrix-org/dendrite/master/dendrite-config.yaml) configuration file in the `build/docker/config` folder of this repository. And rename the config file to `dendrite.yml` (and put it in your `config` directory).
+Create your config based on the [`dendrite.yaml`](https://github.com/matrix-org/dendrite/tree/master/build/docker/config) configuration file in the `build/docker/config` folder of this repository.
 
-Once in place, start the PostgreSQL dependency:
-
-```
-docker-compose -f docker-compose.deps.yml up postgres
-```
-
-Wait a few seconds for PostgreSQL to finish starting up, and then start a monolith:
+Then start the deployment:
 
 ```
 docker-compose -f docker-compose.monolith.yml up
 ```
 
 ## Starting Dendrite as a polylith deployment
 
-Create your config based on the [`dendrite-config.yaml`](https://raw.githubusercontent.com/matrix-org/dendrite/master/dendrite-config.yaml) configuration file in the `build/docker/config` folder of this repository. And rename the config file to `dendrite.yml` (and put it in your `config` directory).
-
-Once in place, start all the dependencies:
-
-```
-docker-compose -f docker-compose.deps.yml up
-```
+Create your config based on the [`dendrite-config.yaml`](https://github.com/matrix-org/dendrite/tree/master/build/docker/config) configuration file in the `build/docker/config` folder of this repository.
 
-Wait a few seconds for PostgreSQL and Kafka to finish starting up, and then start a polylith:
+Then start the deployment:
 
 ```
 docker-compose -f docker-compose.polylith.yml up

diff --git a/build/docker/config/dendrite-config.yaml → build/docker/config/dendrite.yaml b/build/docker/config/dendrite-config.yaml → build/docker/config/dendrite.yaml
@@ -28,7 +28,7 @@
 # connection can be idle in seconds - a negative value is unlimited.
 
 # The version of the configuration file. 
-version: 1
+version: 2
 
 # Global Matrix configuration. This configuration applies to all components.
 global:
@@ -62,28 +62,27 @@ global:
   - matrix.org
   - vector.im
 
-  # Configuration for Kafka/Naffka.
-  kafka:
-    # List of Kafka broker addresses to connect to. This is not needed if using
-    # Naffka in monolith mode.
+  # Configuration for NATS JetStream
+  jetstream:
+    # A list of NATS Server addresses to connect to. If none are specified, an
+    # internal NATS server will be started automatically when running Dendrite
+    # in monolith mode. It is required to specify the address of at least one
+    # NATS Server node if running in polylith mode.
     addresses:
-      - kafka:9092
+      - jetstream:4222
 
-    # The prefix to use for Kafka topic names for this homeserver. Change this only if
-    # you are running more than one Dendrite homeserver on the same Kafka deployment.
-    topic_prefix: Dendrite
+    # Keep all NATS streams in memory, rather than persisting it to the storage
+    # path below. This option is present primarily for integration testing and
+    # should not be used on a real world Dendrite deployment.
+    in_memory: false
 
-    # Whether to use Naffka instead of Kafka. This is only available in monolith
-    # mode, but means that you can run a single-process server without requiring
-    # Kafka.
-    use_naffka: false
+    # Persistent directory to store JetStream streams in. This directory 
+    # should be preserved across Dendrite restarts.
+    storage_path: ./
 
-    # Naffka database options. Not required when using Kafka.
-    naffka_database:
-      connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable
-      max_open_conns: 10
-      max_idle_conns: 2
-      conn_max_lifetime: -1
+    # The prefix to use for stream names for this homeserver - really only
+    # useful if running more than one Dendrite on the same NATS deployment.
+    topic_prefix: Dendrite
 
   # Configuration for Prometheus metric collection.
   metrics:
@@ -266,6 +265,19 @@ media_api:
     height: 480
     method: scale
 
+# Configuration for experimental MSC's
+mscs:
+  # A list of enabled MSC's
+  # Currently valid values are:
+  # - msc2836    (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
+  # - msc2946    (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
+  mscs: []
+  database:
+    connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_mscs?sslmode=disable
+    max_open_conns: 5
+    max_idle_conns: 2
+    conn_max_lifetime: -1
+
 # Configuration for the Room Server.
 room_server:
   internal_api:

diff --git a/build/docker/docker-compose.deps.yml b/build/docker/docker-compose.deps.yml
diff --git a/build/docker/docker-compose.monolith.yml b/build/docker/docker-compose.monolith.yml
@@ -1,5 +1,25 @@
 version: "3.4"
 services:
+  postgres:
+    hostname: postgres
+    image: postgres:14
+    restart: always
+    volumes:
+      - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh
+      # To persist your PostgreSQL databases outside of the Docker image, 
+      # to prevent data loss, modify the following ./path_to path:
+      - ./path_to/postgresql:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: itsasecret
+      POSTGRES_USER: dendrite
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U dendrite"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    networks:
+      - internal
+
   monolith:
     hostname: monolith
     image: matrixdotorg/dendrite-monolith:latest
@@ -13,8 +33,11 @@ services:
     volumes:
       - ./config:/etc/dendrite
       - ./media:/var/dendrite/media
+    depends_on:
+      - postgres
     networks:
       - internal
+    restart: unless-stopped
 
 networks:
   internal: