diff --git a/README.md b/README.md index bde76ec..9352a01 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ Set of lightweight tools, packages and modules that every open-source Go project always needs with almost no dependencies. -## NOTE: core module from this repository is now deprecated and move to standalone repo with higher compatibiltiy guarantees: https://github.com/efficientgo/core +## NOTE: core module from this repository is now deprecated and move to standalone repo with higher compatibility guarantees: https://github.com/efficientgo/core + ## Release model Since this is meant to be critical, tiny import, multi module toolset, there are currently no semver releases planned. It's designed to pin modules via git commits, all commits to master should be stable and properly tested, vetted and linted. @@ -202,6 +203,11 @@ This module provides the PathOrContent flag type which defines two flags to fetc // Also returns content of YAML file with substituted environment variables. // Follows K8s convention, i.e $(...), as mentioned here https://kubernetes.io/docs/tasks/inject-data-application/define-interdependent-environment-variables/. +// PathContentReloader is a helper that runs a given function every time a PathOrContent is changed. +// It is specially useful when paired with RegisterPathOrContent to reload configuration dynamically. +// It works based on a file-system watcher and has a debounce mechanism to avoid excessive reloads. +// You are still responsible to decide what to do with the new file inside the reload function. + // RegisterPathOrContent registers PathOrContent flag in kingpinCmdClause. // Content returns the content of the file when given or directly the content that has been passed to the flag. diff --git a/extkingpin/doc.go b/extkingpin/doc.go index f07082b..8fbde3c 100644 --- a/extkingpin/doc.go +++ b/extkingpin/doc.go @@ -7,6 +7,11 @@ package extkingpin // Also returns content of YAML file with substituted environment variables. // Follows K8s convention, i.e $(...), as mentioned here https://kubernetes.io/docs/tasks/inject-data-application/define-interdependent-environment-variables/. +// PathContentReloader is a helper that runs a given function every time a PathOrContent is changed. +// It is specially useful when paired with RegisterPathOrContent to reload configuration dynamically. +// It works based on a file-system watcher and has a debounce mechanism to avoid excessive reloads. +// You are still responsible to decide what to do with the new file inside the reload function. + // RegisterPathOrContent registers PathOrContent flag in kingpinCmdClause. // Content returns the content of the file when given or directly the content that has been passed to the flag. diff --git a/extkingpin/go.mod b/extkingpin/go.mod index 3a935b5..18af735 100644 --- a/extkingpin/go.mod +++ b/extkingpin/go.mod @@ -5,6 +5,8 @@ go 1.15 require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect + github.com/efficientgo/core v1.0.0-rc.2 + github.com/fsnotify/fsnotify v1.6.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.7.0 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 diff --git a/extkingpin/go.sum b/extkingpin/go.sum index 65a2924..073bad6 100644 --- a/extkingpin/go.sum +++ b/extkingpin/go.sum @@ -2,8 +2,15 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/efficientgo/core v1.0.0-rc.2 h1:7j62qHLnrZqO3V3UA0AqOGd5d5aXV3AX6m/NZBHp78I= +github.com/efficientgo/core v1.0.0-rc.2/go.mod h1:FfGdkzWarkuzOlY04VY+bGfb1lWrjaL6x/GLcQ4vJps= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -12,6 +19,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/extkingpin/pathorcontent.go b/extkingpin/pathorcontent.go index 0a4b5ff..beda0d7 100644 --- a/extkingpin/pathorcontent.go +++ b/extkingpin/pathorcontent.go @@ -29,6 +29,9 @@ type PathOrContent struct { content *string } +// PathOrContent has to implement the pathOrContent interface. +var _ pathOrContent = (*PathOrContent)(nil) + // Option is a functional option type for PathOrContent objects. type Option func(*PathOrContent) diff --git a/extkingpin/pathorcontent_reloader.go b/extkingpin/pathorcontent_reloader.go new file mode 100644 index 0000000..d4703d9 --- /dev/null +++ b/extkingpin/pathorcontent_reloader.go @@ -0,0 +1,139 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Taken from Thanos project. +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package extkingpin + +import ( + "context" + "fmt" + "io/ioutil" + "path" + "path/filepath" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/pkg/errors" +) + +// logger is an interface compatible with go-kit/logger. +type logger interface { + Log(keyvals ...interface{}) error +} + +// pathOrContent is an interface compatible with PathOrContent. +type pathOrContent interface { + Content() ([]byte, error) + Path() string +} + +// PathContentReloader starts a file watcher that monitors the file indicated by pathOrContent.Path() and runs +// reloadFunc whenever a change is detected. +// A debounce timer can be configured via function args to handle situations where many events that would trigger +// a reload are receive in a short period of time. Files will be effectively reloaded at the latest after 2 times +// the debounce timer. By default the debouncer timer is 1 second. +// To ensure renames and deletes are properly handled, the file watcher is put at the file's parent folder. See +// https://github.com/fsnotify/fsnotify/issues/214 for more details. +func PathContentReloader(ctx context.Context, fileContent pathOrContent, debugLogger logger, errorLogger logger, reloadFunc func(), debounceTime time.Duration) error { + filePath, err := filepath.Abs(fileContent.Path()) + if err != nil { + return errors.Wrap(err, "getting absolute file path") + } + + watcher, err := fsnotify.NewWatcher() + if filePath == "" { + _ = debugLogger.Log("msg", "no path detected for config reload") + } + if err != nil { + return errors.Wrap(err, "creating file watcher") + } + go func() { + var reloadTimer *time.Timer + if debounceTime != 0 { + reloadTimer = time.AfterFunc(debounceTime, func() { + reloadFunc() + _ = debugLogger.Log("msg", "configuration reloaded after debouncing") + }) + } + defer watcher.Close() + for { + select { + case <-ctx.Done(): + if reloadTimer != nil { + reloadTimer.Stop() + } + return + case event := <-watcher.Events: + // fsnotify sometimes sends a bunch of events without name or operation. + // It's unclear what they are and why they are sent - filter them out. + if event.Name == "" { + break + } + // We are watching the file's parent folder (more details on why this is done can be found below), but + // we are only interested in changes to the target file. Discard every other file as quickly as possible. + if event.Name != filePath { + break + } + // We only react to files being written or created. + // On "chmod" or "remove" we have nothing to do. + // On "rename" we have the old file name (not useful). A "create" event for the new file will come later. + if !event.Op.Has(fsnotify.Write) || !event.Op.Has(fsnotify.Create) { + break + } + _ = debugLogger.Log("msg", fmt.Sprintf("change detected for %s", filePath), "eventName", event.Name, "eventOp", event.Op) + if reloadTimer != nil { + reloadTimer.Reset(debounceTime) + } + case err := <-watcher.Errors: + _ = errorLogger.Log("msg", "watcher error", "error", err) + } + } + }() + // We watch the file's parent folder and not the file itself to better handle DELETE and RENAME events. Check + // https://github.com/fsnotify/fsnotify/issues/214 for more details. + if err := watcher.Add(path.Dir(filePath)); err != nil { + return errors.Wrapf(err, "adding path %s to file watcher", filePath) + } + return nil +} + +// StaticPathContent serves the contents of a given file through the pathOrContent interface. It's useful for tests +// that rely on such interface. +type StaticPathContent struct { + content []byte + path string +} + +var _ pathOrContent = (*StaticPathContent)(nil) + +// Content returns the static content. +func (t *StaticPathContent) Content() ([]byte, error) { + return t.content, nil +} + +// Path returns the path to the file that contains the content. +func (t *StaticPathContent) Path() string { + return t.path +} + +// NewStaticPathContent creates a new content that can be used to serve a static configuration. +func NewStaticPathContent(fromPath string) (*StaticPathContent, error) { + content, err := ioutil.ReadFile(fromPath) + + if err != nil { + return nil, errors.Wrapf(err, "could not load test content: %s", fromPath) + } + return &StaticPathContent{content, fromPath}, nil +} + +// Rewrite rewrites the file backing this StaticPathContent and swaps the local content cache. The file writing +// is needed to trigger the file system monitor. +func (t *StaticPathContent) Rewrite(newContent []byte) error { + t.content = newContent + // Write the file to ensure possible file watcher reloaders get triggered. + return ioutil.WriteFile(t.path, newContent, 0666) +} diff --git a/extkingpin/pathorcontent_reloader_test.go b/extkingpin/pathorcontent_reloader_test.go new file mode 100644 index 0000000..36f093f --- /dev/null +++ b/extkingpin/pathorcontent_reloader_test.go @@ -0,0 +1,124 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Taken from Thanos project. +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package extkingpin + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path" + "sync" + "testing" + "time" + + "github.com/efficientgo/core/testutil" +) + +func TestPathContentReloader(t *testing.T) { + type args struct { + runSteps func(t *testing.T, testFile string, pathContent *StaticPathContent) + } + tests := []struct { + name string + args args + wantReloads int + }{ + { + name: "Many operations, only rewrite triggers one reload", + args: args{ + runSteps: func(t *testing.T, testFile string, pathContent *StaticPathContent) { + testutil.Ok(t, os.Chmod(testFile, 0777)) + testutil.Ok(t, os.Remove(testFile)) + testutil.Ok(t, pathContent.Rewrite([]byte("test modified"))) + }, + }, + wantReloads: 1, + }, + { + name: "Many operations, only rename triggers one reload", + args: args{ + runSteps: func(t *testing.T, testFile string, pathContent *StaticPathContent) { + testutil.Ok(t, os.Chmod(testFile, 0777)) + testutil.Ok(t, os.Rename(testFile, testFile+".tmp")) + testutil.Ok(t, os.Rename(testFile+".tmp", testFile)) + }, + }, + wantReloads: 1, + }, + { + name: "Many operations, two rewrites trigger two reloads", + args: args{ + runSteps: func(t *testing.T, testFile string, pathContent *StaticPathContent) { + testutil.Ok(t, os.Chmod(testFile, 0777)) + testutil.Ok(t, os.Remove(testFile)) + testutil.Ok(t, pathContent.Rewrite([]byte("test modified"))) + time.Sleep(2 * time.Second) + testutil.Ok(t, pathContent.Rewrite([]byte("test modified again"))) + }, + }, + wantReloads: 1, + }, + { + name: "Chmod doesn't trigger reload", + args: args{ + runSteps: func(t *testing.T, testFile string, pathContent *StaticPathContent) { + testutil.Ok(t, os.Chmod(testFile, 0777)) + }, + }, + wantReloads: 0, + }, + { + name: "Remove doesn't trigger reload", + args: args{ + runSteps: func(t *testing.T, testFile string, pathContent *StaticPathContent) { + testutil.Ok(t, os.Remove(testFile)) + }, + }, + wantReloads: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testFile := path.Join(t.TempDir(), "test") + testutil.Ok(t, ioutil.WriteFile(testFile, []byte("test"), 0666)) + pathContent, err := NewStaticPathContent(testFile) + testutil.Ok(t, err) + + wg := &sync.WaitGroup{} + wg.Add(tt.wantReloads) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + reloadCount := 0 + err = PathContentReloader(ctx, pathContent, newTestLogger("debug"), newTestLogger("error"), func() { + reloadCount++ + wg.Done() + }, 100*time.Millisecond) + testutil.Ok(t, err) + + tt.args.runSteps(t, testFile, pathContent) + wg.Wait() + testutil.Equals(t, tt.wantReloads, reloadCount) + }) + } +} + +type testLogger struct { + prefix string +} + +func newTestLogger(prefix string) testLogger { + return testLogger{prefix: prefix} +} + +func (t testLogger) Log(keyvals ...interface{}) error { + _, err := fmt.Printf("[%s] %s", t.prefix, keyvals) + return err +}