[MM-53162] Implement global calls scoped lock to improve atomicity of…

… state operations (#460)

* Refactor cluster mutex

* Call locking utils

* Add/remove session

* Mute/Unmute,RaiseHand/LowerHand,ScreenSharing

* Call start post

* Call end

* Enable/disable calls

* Recording API

* Remove unused functions

* Mutex metrics

* Lint fixes

* Update .golangci.yml

* Writer DB

* Simplify call lock use

* Simplify call start case

* Fix KVGet semantic in case of no results

* Small tweaks

* Make Unlock() actually safe

* Simplify recording API handlers

* Move code around

* Prevent logging errors on duplicate session removals attempts

* Enforce maxPollInterval

* Add note to cleanCallState function

* Use channelID instead of callID

* Update new code

* [MM-53167] Persist participants list (#486)

* Persist list of participants on call end

* Exclude bot from participants list

* Fix potential nil access upon starting a recording (#495)

* fix linter errors

* Bump golangci-lint version

* Force input

* Revert "Force input"

This reverts commit 0b22839.

---------

Co-authored-by: Christopher Poile <[email protected]>

.github/workflows/ci.yml

@@ -13,4 +13,6 @@ permissions:
 jobs:
   plugin-ci:
     uses: mattermost/actions-workflows/.github/workflows/plugin-ci.yml@main
+    with:
+      golangci-lint-version: "v1.54.2"
     secrets: inherit

.golangci.yml

@@ -1,3 +1,6 @@
+service:
+  golangci-lint-version: 1.54.2
+
 run:
   timeout: 5m
   modules-download-mode: readonly
@@ -44,9 +47,3 @@ linters:
     - unused        # Checks Go code for unused constants, variables, functions and types
     - exportloopref # Checks for pointers to enclosing loop variables
     - errcheck      # Detects unchecked errors.
-
-issues:
-  exclude-rules:
-    - linters:
-      - unused
-      text: "getConfiguration"

go.mod

@@ -17,7 +17,9 @@ require (
 	github.com/mattermost/calls-recorder v0.4.0
 	github.com/mattermost/logr/v2 v2.0.16
 	github.com/mattermost/mattermost/server/public v0.0.0-20230613002302-62a3ee8adcb5
+	github.com/mattermost/mattermost/server/v8 v8.0.0-20230622213803-fece5d5dd276
 	github.com/mattermost/rtcd v0.11.1
+	github.com/mattermost/squirrel v0.2.0
 	github.com/pion/interceptor v0.1.12
 	github.com/pion/rtp v1.7.13
 	github.com/pkg/errors v0.9.1
@@ -39,15 +41,17 @@ require (
 	github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
-	github.com/go-sql-driver/mysql v1.7.0 // indirect
+	github.com/go-sql-driver/mysql v1.7.1 // indirect
 	github.com/gofrs/flock v0.8.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/graph-gophers/graphql-go v1.5.1-0.20230110080634-edea822f558a // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-plugin v1.4.9 // indirect
 	github.com/hashicorp/yamux v0.1.1 // indirect
-	github.com/lib/pq v1.10.7 // indirect
+	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
+	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
 	github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 // indirect
 	github.com/mattermost/ldap v3.0.4+incompatible // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -77,7 +81,7 @@ require (
 	github.com/pyroscope-io/godeltaprof v0.1.1 // indirect
 	github.com/segmentio/backo-go v1.0.1 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/tidwall/gjson v1.14.3 // indirect
+	github.com/tidwall/gjson v1.14.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tinylib/msgp v1.1.8 // indirect

go.sum

@@ -129,8 +129,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
-github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
+github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -279,8 +279,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
-github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
+github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
+github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
+github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
@@ -297,8 +301,12 @@ github.com/mattermost/logr/v2 v2.0.16 h1:jnePX4cPskC3WDFvUardh/xZfxNdsFXbEERJQ1k
 github.com/mattermost/logr/v2 v2.0.16/go.mod h1:1dm/YhTpozsqANXxo5Pi5zYLBsal2xY0pX+JZNbzYJY=
 github.com/mattermost/mattermost/server/public v0.0.0-20230613002302-62a3ee8adcb5 h1:X8ue+QQn0VJe6qR0q2kHGFhQCn5T4rqb7XS0t6R1r3A=
 github.com/mattermost/mattermost/server/public v0.0.0-20230613002302-62a3ee8adcb5/go.mod h1:TtECPYX/ftU43bCGqN5W3Ic2gPDN2+zeKhXfE2YPRvw=
+github.com/mattermost/mattermost/server/v8 v8.0.0-20230622213803-fece5d5dd276 h1:MTuN/jEH4XxCdhM2L+Xl0sDWVqm6nETzRZTczzpfzks=
+github.com/mattermost/mattermost/server/v8 v8.0.0-20230622213803-fece5d5dd276/go.mod h1:RAgzlRBZin+otiYdmrLHGdXkplEzVH5AfFzSKxdpT8A=
 github.com/mattermost/rtcd v0.11.1 h1:xaP/s0/WX8rDqHq05l8b4QLLJuRZXucr9Qh6cHTQSHk=
 github.com/mattermost/rtcd v0.11.1/go.mod h1:ketmoC7+9IOjynE5YJgR6GrFTG1b78byVlkVsLcCDa0=
+github.com/mattermost/squirrel v0.2.0 h1:8ZWeyf+MWQ2cL7hu9REZgLtz2IJi51qqZEovI3T3TT8=
+github.com/mattermost/squirrel v0.2.0/go.mod h1:NPPtk+CdpWre4GxMGoOpzEVFVc0ZoEFyJBZGCtn9nSU=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
@@ -515,8 +523,8 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
-github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
-github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
+github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=

lt/ws/client_test.go

@@ -67,12 +67,6 @@ func TestClose(t *testing.T) {
 		})
 		require.Nil(t, err)
 
-		go func() {
-			// Just drain the event channel
-			for range c.EventChannel {
-			}
-		}()
-
 		err = c.SendMessage("test_action", map[string]interface{}{"test": "data"})
 		assert.Nil(t, err)
 
@@ -95,12 +89,6 @@ func TestClose(t *testing.T) {
 		})
 		require.Nil(t, err)
 
-		go func() {
-			// Just drain the event channel
-			for range c.EventChannel {
-			}
-		}()
-
 		err = c.SendMessage("test_action", map[string]interface{}{"test": "data"})
 		assert.Nil(t, err)
 
@@ -119,12 +107,6 @@ func TestClose(t *testing.T) {
 		})
 		require.Nil(t, err)
 
-		go func() {
-			// Just drain the event channel
-			for range c.EventChannel {
-			}
-		}()
-
 		err = c.SendMessage("test_action", map[string]interface{}{"test": "data"})
 		assert.Nil(t, err)
 
@@ -165,12 +147,6 @@ func TestSendMessage(t *testing.T) {
 		})
 		require.Nil(t, err)
 
-		go func() {
-			// Just drain the event channel
-			for range c.EventChannel {
-			}
-		}()
-
 		err = c.SendMessage("test_action", map[string]interface{}{"test": "data"})
 		assert.Nil(t, err)
 
@@ -190,12 +166,6 @@ func TestSendMessage(t *testing.T) {
 		})
 		require.Nil(t, err)
 
-		go func() {
-			// Just drain the event channel
-			for range c.EventChannel {
-			}
-		}()
-
 		err = c.SendMessage("test_action", map[string]interface{}{"test": "data"})
 		assert.Nil(t, err)
 
@@ -214,12 +184,6 @@ func TestSendMessage(t *testing.T) {
 		})
 		require.Nil(t, err)
 
-		go func() {
-			// Just drain the event channel
-			for range c.EventChannel {
-			}
-		}()
-
 		err = c.SendMessage("test_action", map[string]interface{}{"test": "data"})
 		assert.Nil(t, err)
 
@@ -240,12 +204,6 @@ func TestSendMessage(t *testing.T) {
 		})
 		require.Nil(t, err)
 
-		go func() {
-			// Just drain the event channel
-			for range c.EventChannel {
-			}
-		}()
-
 		err = c.SendMessage("test_action", map[string]interface{}{"test": "data"})
 		assert.Nil(t, err)
 

server/activate.go

@@ -4,6 +4,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"time"
@@ -17,11 +18,15 @@ import (
 )
 
 func (p *Plugin) createBotSession() (*model.Session, error) {
-	m, err := cluster.NewMutex(p.API, "ensure_bot")
+	m, err := cluster.NewMutex(p.API, p.metrics, "ensure_bot", cluster.MutexConfig{})
 	if err != nil {
 		return nil, err
 	}
-	m.Lock()
+	lockCtx, cancelCtx := context.WithTimeout(context.Background(), lockTimeout)
+	defer cancelCtx()
+	if err := m.Lock(lockCtx); err != nil {
+		return nil, fmt.Errorf("failed to lock cluster mutex: %w", err)
+	}
 	defer m.Unlock()
 
 	botID, err := p.API.EnsureBotUser(&model.Bot{
@@ -53,6 +58,11 @@ func (p *Plugin) OnActivate() error {
 		return fmt.Errorf("disabled by environment flag")
 	}
 
+	if err := p.initDB(); err != nil {
+		p.LogError(err.Error())
+		return err
+	}
+
 	p.licenseChecker = enterprise.NewLicenseChecker(p.API)
 
 	if p.isSingleHandler() {
@@ -217,6 +227,12 @@ func (p *Plugin) OnDeactivate() error {
 	p.LogDebug("deactivate")
 	close(p.stopCh)
 
+	if p.wDB != nil {
+		if err := p.wDB.Close(); err != nil {
+			p.LogError(err.Error())
+		}
+	}
+
 	if p.rtcdManager != nil {
 		if err := p.rtcdManager.Close(); err != nil {
 			p.LogError(err.Error())