Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[MM-62410] Automatically cleanup stale calls in RTCD enabled deployments #940

Merged
merged 4 commits into from
Jan 13, 2025
Merged

[MM-62410] Automatically cleanup stale calls in RTCD enabled deployments #940

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d096a75
Update rtcd
streamer45 Jan 8, 2025
d36b3b9
Cleanup stale calls in RTCD enabled deployments
streamer45 Jan 8, 2025
72c10e1
Add comment
streamer45 Jan 13, 2025
7bc3551
Bump RTCD to v1.0.0
streamer45 Jan 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ require (
github.com/mattermost/mattermost-plugin-calls/server/public v0.0.3
github.com/mattermost/mattermost/server/public v0.1.9
github.com/mattermost/morph v1.1.0
github.com/mattermost/rtcd v0.18.1-0.20250107081358-290c5ce0a692
github.com/mattermost/rtcd v1.0.0
github.com/mattermost/squirrel v0.2.0
github.com/pkg/errors v0.9.1
github.com/rudderlabs/analytics-go v3.3.3+incompatible
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,8 +375,8 @@ github.com/mattermost/mattermost/server/public v0.1.9 h1:l/OKPRVuFeqL0yqRVC/Jpve
github.com/mattermost/mattermost/server/public v0.1.9/go.mod h1:SkTKbMul91Rq0v2dIxe8mqzUOY+3KwlwwLmAlxDfGCk=
github.com/mattermost/morph v1.1.0 h1:Q9vrJbeM3s2jfweGheq12EFIzdNp9a/6IovcbvOQ6Cw=
github.com/mattermost/morph v1.1.0/go.mod h1:gD+EaqX2UMyyuzmF4PFh4r33XneQ8Nzi+0E8nXjMa3A=
github.com/mattermost/rtcd v0.18.1-0.20250107081358-290c5ce0a692 h1:51Rxv6A5Esd11VW0PNHe0Vw3Zbye7J6iv3tXODTdknU=
github.com/mattermost/rtcd v0.18.1-0.20250107081358-290c5ce0a692/go.mod h1:hsRBk1e6dQZrZSK2FoqZAhO+oXg+1Je5zhF1ClLbbdA=
github.com/mattermost/rtcd v1.0.0 h1:7FjRcr7sinOOoSr+in/N/eeg/+BlowkY6m0FFBlM/6M=
github.com/mattermost/rtcd v1.0.0/go.mod h1:hsRBk1e6dQZrZSK2FoqZAhO+oXg+1Je5zhF1ClLbbdA=
github.com/mattermost/squirrel v0.2.0 h1:8ZWeyf+MWQ2cL7hu9REZgLtz2IJi51qqZEovI3T3TT8=
github.com/mattermost/squirrel v0.2.0/go.mod h1:NPPtk+CdpWre4GxMGoOpzEVFVc0ZoEFyJBZGCtn9nSU=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
Expand Down
2 changes: 1 addition & 1 deletion lt/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/aws/aws-sdk-go v1.50.3
github.com/hajimehoshi/go-mp3 v0.3.4
github.com/mattermost/mattermost/server/public v0.0.12
github.com/mattermost/rtcd v0.18.1-0.20250107081358-290c5ce0a692
github.com/mattermost/rtcd v1.0.0
github.com/pion/rtp v1.8.10
github.com/pion/webrtc/v4 v4.0.7
gopkg.in/hraban/opus.v2 v2.0.0-20230925203106-0188a62cb302
Expand Down
4 changes: 2 additions & 2 deletions lt/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,8 @@ github.com/mattermost/logr/v2 v2.0.21 h1:CMHsP+nrbRlEC4g7BwOk1GAnMtHkniFhlSQPXy5
github.com/mattermost/logr/v2 v2.0.21/go.mod h1:kZkB/zqKL9e+RY5gB3vGpsyenC+TpuiOenjMkvJJbzc=
github.com/mattermost/mattermost/server/public v0.0.12 h1:iunc9q4/XkArOrndEUn73uFw6v9TOEXEtp6Nm6Iv218=
github.com/mattermost/mattermost/server/public v0.0.12/go.mod h1:Bk+atJcELCIk9Yeq5FoqTr+gra9704+X4amrlwlTgSc=
github.com/mattermost/rtcd v0.18.1-0.20250107081358-290c5ce0a692 h1:51Rxv6A5Esd11VW0PNHe0Vw3Zbye7J6iv3tXODTdknU=
github.com/mattermost/rtcd v0.18.1-0.20250107081358-290c5ce0a692/go.mod h1:hsRBk1e6dQZrZSK2FoqZAhO+oXg+1Je5zhF1ClLbbdA=
github.com/mattermost/rtcd v1.0.0 h1:7FjRcr7sinOOoSr+in/N/eeg/+BlowkY6m0FFBlM/6M=
github.com/mattermost/rtcd v1.0.0/go.mod h1:hsRBk1e6dQZrZSK2FoqZAhO+oXg+1Je5zhF1ClLbbdA=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
Expand Down
4 changes: 4 additions & 0 deletions server/activate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,10 @@ func (p *Plugin) OnActivate() (retErr error) {
p.LogDebug("rtcd client manager initialized successfully")

p.rtcdManager = rtcdManager

if err := p.cleanUpState(); err != nil {
p.LogError("failed to cleanup state", "err", err.Error())
}
} else {
rtcServerConfig := rtc.ServerConfig{
ICEAddressUDP: cfg.UDPServerAddress,
Expand Down
2 changes: 2 additions & 0 deletions server/interfaces/interfaces.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,4 +57,6 @@ type RTCDClient interface {
Close() error
GetVersionInfo() (rtcd.VersionInfo, error)
GetSystemInfo() (rtcd.SystemInfo, error)
GetSession(callID, sessionID string) (rtc.SessionConfig, int, error)
GetSessions(callID string) ([]rtc.SessionConfig, int, error)
}
133 changes: 132 additions & 1 deletion .../mocks/github.com/mattermost/mattermost-plugin-calls/server/interfaces/mock_RTCDClient.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

53 changes: 53 additions & 0 deletions server/rtcd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"fmt"
"math/rand"
"net"
"net/http"
"net/url"
"os"
"strings"
Expand All @@ -21,6 +22,7 @@ import (
"github.com/mattermost/mattermost-plugin-calls/server/db"
"github.com/mattermost/mattermost-plugin-calls/server/interfaces"
"github.com/mattermost/mattermost-plugin-calls/server/license"
"github.com/mattermost/mattermost-plugin-calls/server/public"

rtcd "github.com/mattermost/rtcd/service"
"github.com/mattermost/rtcd/service/random"
Expand Down Expand Up @@ -706,3 +708,54 @@ func (h *rtcdHost) isFlagged() bool {
defer h.mut.RUnlock()
return h.flagged
}

// hasCallEnded checks if the call has ended by querying the RTCD host assigned to the call.
// Since this method is used to clean up the call state, it's important to be as conservative as possible
// and only return true if we are absolutely sure the call has ended.
func (m *rtcdClientManager) hasCallEnded(call *public.Call) bool {
m.ctx.LogDebug("RTCD host is set in call, checking...", "callID", call.ID, "rtcdHost", call.Props.RTCDHost)
host := m.getHost(call.Props.RTCDHost)
if host != nil {
m.ctx.LogDebug("RTCD host found", "callID", call.ID, "rtcdHost", call.Props.RTCDHost)

// Version compatibility check. We need to be talking to RTCD v1.0.0 or higher to be able to call GetSessions.
info, err := host.client.GetVersionInfo()
if err != nil {
m.ctx.LogDebug("failed to get version info", "err", err.Error(), "callID", call.ID, "rtcdHost", call.Props.RTCDHost)
return false
Comment on lines +723 to +725
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This defaults to call is still running if there's a problem getting the version info -- will that cause trouble? Should we return the err and have logic for it?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A stuck call is strictly better than killing a live call, so unless we get a guarantee from the RTCD side that the call has ended, we err on the side of caution.

}

// Always support dev builds.
if info.BuildVersion == "" || info.BuildVersion == "master" || strings.HasPrefix(info.BuildVersion, "dev") {
m.ctx.LogDebug("skipping version compatibility check", "buildVersion", info.BuildVersion, "callID", call.ID, "rtcdHost", call.Props.RTCDHost)
} else if err := checkMinVersion("v1.0.0", info.BuildVersion); err != nil {
m.ctx.LogDebug("RTCD host version is not compatible", "err", err.Error(), "callID", call.ID, "rtcdHost", call.Props.RTCDHost)
return false
}

cfgs, code, err := host.client.GetSessions(call.ID)
if err != nil {
m.ctx.LogDebug("failed to get sessions for call", "err", err.Error(), "callID", call.ID, "rtcdHost", call.Props.RTCDHost)
}

if code != http.StatusOK && code != http.StatusNotFound {
m.ctx.LogDebug("unexpected status code from RTCD", "code", code, "callID", call.ID, "rtcdHost", call.Props.RTCDHost)
// The request above could fail for various reasons, in which case we can't assume the call has ended.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah, this confirms the default behavior. Maybe add this comment to the above as well? (would just save some cognitive overhead)

return false
}

if len(cfgs) > 0 {
// The call is ongoing so we skip state cleanup.
m.ctx.LogDebug("call is still ongoing", "callID", call.ID, "rtcdHost", call.Props.RTCDHost)
return false
}

// no call ongoing
m.ctx.LogDebug("call was not found", "callID", call.ID, "rtcdHost", call.Props.RTCDHost)
} else {
// If the host is not found we can assume the RTCD node is gone for good so there's no way a call would be ongoing.
m.ctx.LogDebug("RTCD host not found", "callID", call.ID, "rtcdHost", call.Props.RTCDHost)
}

return true
}
9 changes: 9 additions & 0 deletions server/state.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,10 +368,19 @@ func (p *Plugin) cleanUpState() error {
p.LogError("failed to lock call", "err", err.Error())
continue
}

// If a call has a RTCD host assigned, we want to check with the RTCD side whether the call is still ongoing or not before
// cleaning up the state.
if p.rtcdManager != nil && call.Props.RTCDHost != "" && !p.rtcdManager.hasCallEnded(call) {
p.unlockCall(call.ChannelID)
continue
}

if err := p.cleanCallState(call); err != nil {
p.unlockCall(call.ChannelID)
return fmt.Errorf("failed to clean up state: %w", err)
}

p.unlockCall(call.ChannelID)
}

Expand Down
Loading
Loading