Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix race condition in plugin logs routine #2126

Merged
merged 2 commits into from
Jun 3, 2022
Merged

Fix race condition in plugin logs routine #2126

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1da097a
Fix race condition in plugin logs routine
carolynvs Jun 1, 2022
ee2055d
Move when we add to the wait group
carolynvs Jun 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions pkg/plugins/pluggable/connection.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"os/exec"
"strconv"
"strings"
"sync"
"time"

"get.porter.sh/porter/pkg/config"
Expand Down Expand Up @@ -55,6 +56,10 @@ type PluginConnection struct {
// cancelLogCtx is the cancellation function for our go-routine that collects the plugin logs
cancelLogCtx context.CancelFunc

// logsWaitGroup is used to ensure that any go routines spawned by the plugin connection
// complete when Close is called. Otherwise we can get into a race between us and when the logger is closed.
logsWaitGroup sync.WaitGroup

// logsWriter receives logs from the plugin's stdout.
logsWriter *io.PipeWriter

Expand Down Expand Up @@ -208,12 +213,14 @@ func (c *PluginConnection) Close(ctx context.Context) error {
c.client.HardKill()
}

// Stop processing logs from the plugin
// Stop processing logs from the plugin and wait for the log collection routine to complete
// This avoids a race where the log collector picks up a message but doesn't print it until
// after we close the logfile. This ensures that everything releated to the plugins is released
// when Close exits.
c.cancelLogCtx()

// Close the pipe between the plugin and porter
c.logsWriter.Close()
c.logsReader.Close()
c.logsWaitGroup.Wait()

c.client = nil
}
Expand Down Expand Up @@ -278,6 +285,7 @@ func (c *PluginConnection) setupLogCollector(ctx context.Context) {
c.logsReader, c.logsWriter = io.Pipe()
ctx, c.cancelLogCtx = context.WithCancel(ctx)

c.logsWaitGroup.Add(1)
go c.collectPluginLogs(ctx)
}

Expand All @@ -286,6 +294,8 @@ func (c *PluginConnection) setupLogCollector(ctx context.Context) {
// The best way to get that information is to instrument the plugin itself. This is mainly a fallback mechanism to
// collect logs from an uninstrumented plugin.
func (c *PluginConnection) collectPluginLogs(ctx context.Context) {
defer c.logsWaitGroup.Done()

ctx, span := tracing.StartSpan(ctx, attribute.String("plugin-key", c.key.String()))
defer span.EndSpan()

Expand All @@ -300,7 +310,7 @@ func (c *PluginConnection) collectPluginLogs(ctx context.Context) {
if err == io.EOF {
return
}
continue
return
}
if line == "" {
return
Expand Down