-
Notifications
You must be signed in to change notification settings - Fork 173
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Moving leader election inside vault-k8s (#271)
Using operator-sdk's Become() for leader election. Retries Become() 10 times (with exp backoff) and then signals the caller to exit if it failed. command.Run() now watches an exitOnError channel for that case. Co-authored-by: Ben Ash <[email protected]>
- Loading branch information
Showing
9 changed files
with
414 additions
and
181 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,59 +1,65 @@ | ||
| package leader | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "io/ioutil" | ||
| "net/http" | ||
| "os" | ||
| ) | ||
| "context" | ||
| "sync/atomic" | ||
| "time" | ||
|
|
||
| const defaultURL = "http://localhost:4040/" | ||
| "github.com/cenkalti/backoff/v4" | ||
| "github.com/hashicorp/go-hclog" | ||
| operator_leader "github.com/operator-framework/operator-lib/leader" | ||
| "k8s.io/client-go/kubernetes" | ||
| ) | ||
|
|
||
| type LeaderElector struct { | ||
| URL string | ||
| type Elector interface { | ||
| // IsLeader returns whether this host is the leader | ||
| IsLeader() (bool, error) | ||
| } | ||
|
|
||
| type LeaderResponse struct { | ||
| Name string `json:"name"` | ||
| type LeaderForLife struct { | ||
| isLeader atomic.Value | ||
| } | ||
|
|
||
| // New returns a LeaderElector with the default service endpoint | ||
| func New() *LeaderElector { | ||
| return &LeaderElector{ | ||
| URL: defaultURL, | ||
| } | ||
| } | ||
| // New returns a Elector that uses the operator-sdk's leader for life elector | ||
| func New(ctx context.Context, logger hclog.Logger, clientset kubernetes.Interface, exitOnError chan error) *LeaderForLife { | ||
| le := &LeaderForLife{} | ||
| le.isLeader.Store(false) | ||
|
|
||
| go func() { | ||
| // The Become() function blocks until this replica becomes the "leader", | ||
| // by creating a ConfigMap with an OwnerReference. Another replica can | ||
| // become the leader when the current leader replica stops running, and | ||
| // the Kubernetes garbage collector deletes the vault-k8s-leader | ||
| // ConfigMap. | ||
|
|
||
| // New exponential backoff with 10 retries | ||
| expBo := backoff.NewExponentialBackOff() | ||
| expBo.MaxInterval = time.Second * 30 | ||
| bo := backoff.WithMaxRetries(expBo, 10) | ||
|
|
||
| err := backoff.Retry(func() error { | ||
| if err := operator_leader.Become(ctx, "vault-k8s-leader"); err != nil { | ||
| logger.Error("Trouble becoming leader", "error", err) | ||
| return err | ||
| } | ||
| return nil | ||
| }, bo) | ||
|
|
||
| if err != nil { | ||
| // Signal the caller to shutdown the injector server, since Become() | ||
| // failed all the retries | ||
| exitOnError <- err | ||
| return | ||
| } | ||
|
|
||
| le.isLeader.Store(true) | ||
| }() | ||
|
|
||
| // NewWithURL returns a LeaderElector with a custom service endpoint URL | ||
| func NewWithURL(URL string) *LeaderElector { | ||
| return &LeaderElector{ | ||
| URL: URL, | ||
| } | ||
| return le | ||
| } | ||
|
|
||
| // IsLeader returns whether this host is the leader | ||
| func (le *LeaderElector) IsLeader() (bool, error) { | ||
| resp, err := http.Get(le.URL) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| defer resp.Body.Close() | ||
| body, err := ioutil.ReadAll(resp.Body) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| leaderResp := &LeaderResponse{} | ||
| err = json.Unmarshal(body, leaderResp) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| hostname, err := os.Hostname() | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| if leaderResp.Name == hostname { | ||
| return true, nil | ||
| } | ||
|
|
||
| return false, nil | ||
| func (le *LeaderForLife) IsLeader() (bool, error) { | ||
| leaderStatus := le.isLeader.Load().(bool) | ||
| return leaderStatus, nil | ||
| } |
Oops, something went wrong.