Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Moving leader election inside vault-k8s #271

Merged
merged 24 commits into from
Aug 31, 2021
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
1060be0
using operator-sdk Become() for leader election
tvoran Jul 6, 2021
7eebe89
support v1 and v1beta1 admission api's
tvoran Jul 9, 2021
885d96b
set a leader label on injector replicas
tvoran Jul 10, 2021
e133800
retry with backoff for setting leader label
tvoran Jul 20, 2021
d12851b
warn about problems loading TLS keypair and leader
tvoran Jul 20, 2021
6d837ce
removed leader labeling
tvoran Jul 23, 2021
3736c60
updating k8s client-go
tvoran Jul 26, 2021
3303666
support v1 and v1beta1 admission api's
tvoran Jul 9, 2021
91474cf
more error handling for getAdminAPIVersion()
tvoran Jul 26, 2021
4b6837b
Merge branch 'VAULT-2403/update-client-go' into VAULT-2403/internal-l…
tvoran Jul 27, 2021
719ee63
updating webhook deployment yaml for v1
tvoran Jul 27, 2021
22ea7fe
Merge branch 'VAULT-2403/update-client-go' into VAULT-2403/internal-l…
tvoran Jul 27, 2021
fe858b7
v1 -> admissionv1
tvoran Jul 27, 2021
cf78476
Merge branch 'VAULT-2403/update-client-go' into VAULT-2403/internal-l…
tvoran Jul 27, 2021
31df5c8
Merge remote-tracking branch 'origin/master' into VAULT-2403/internal…
tvoran Jul 27, 2021
75258f7
added retries for the Become() call
tvoran Aug 10, 2021
d9ef2ff
operator-lib 0.6.0
tvoran Aug 10, 2021
30dc9cb
Merge remote-tracking branch 'origin/master' into VAULT-2403/internal…
tvoran Aug 19, 2021
2f0b6ee
defer stop ticker
tvoran Aug 19, 2021
e5522cc
Cleaning up deploy yaml
tvoran Aug 24, 2021
e6d0206
Merge remote-tracking branch 'origin/master' into VAULT-2403/internal…
tvoran Aug 26, 2021
0be17c3
Update leader/leader.go
tvoran Aug 27, 2021
b73527f
retry Become() 10 times then exit
tvoran Aug 31, 2021
a45b35e
Pass an actual error through exitOneError channel
tvoran Aug 31, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions agent-inject/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"github.com/hashicorp/vault-k8s/agent-inject/agent"
"github.com/hashicorp/vault/sdk/helper/strutil"
"github.com/mattbaird/jsonpatch"
"k8s.io/api/admission/v1beta1"
v1 "k8s.io/api/admission/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -86,8 +86,8 @@ func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
return
}

var admReq v1beta1.AdmissionReview
var admResp v1beta1.AdmissionReview
var admReq v1.AdmissionReview
var admResp v1.AdmissionReview
if _, _, err := deserializer().Decode(body, nil, &admReq); err != nil {
msg := fmt.Sprintf("error decoding admission request: %s", err)
http.Error(w, msg, http.StatusInternalServerError)
Expand All @@ -112,21 +112,21 @@ func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {

// Mutate takes an admission request and performs mutation if necessary,
// returning the final API response.
func (h *Handler) Mutate(req *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse {
func (h *Handler) Mutate(req *v1.AdmissionRequest) *v1.AdmissionResponse {
// Decode the pod from the request
var pod corev1.Pod
if err := json.Unmarshal(req.Object.Raw, &pod); err != nil {
h.Log.Error("could not unmarshal request to pod: %s", err)
h.Log.Debug("%s", req.Object.Raw)
return &v1beta1.AdmissionResponse{
return &v1.AdmissionResponse{
Result: &metav1.Status{
Message: err.Error(),
},
}
}

// Build the basic response
resp := &v1beta1.AdmissionResponse{
resp := &v1.AdmissionResponse{
Allowed: true,
UID: req.UID,
}
Expand Down Expand Up @@ -195,14 +195,14 @@ func (h *Handler) Mutate(req *v1beta1.AdmissionRequest) *v1beta1.AdmissionRespon
}

resp.Patch = patch
patchType := v1beta1.PatchTypeJSONPatch
patchType := v1.PatchTypeJSONPatch
resp.PatchType = &patchType

return resp
}

func admissionError(err error) *v1beta1.AdmissionResponse {
return &v1beta1.AdmissionResponse{
func admissionError(err error) *v1.AdmissionResponse {
return &v1.AdmissionResponse{
Result: &metav1.Status{
Message: err.Error(),
},
Expand Down
30 changes: 15 additions & 15 deletions agent-inject/handler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"github.com/hashicorp/vault-k8s/agent-inject/agent"
"github.com/mattbaird/jsonpatch"
"github.com/stretchr/testify/require"
"k8s.io/api/admission/v1beta1"
"k8s.io/api/admission/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -61,14 +61,14 @@ func TestHandlerHandle(t *testing.T) {
cases := []struct {
Name string
Handler Handler
Req v1beta1.AdmissionRequest
Req v1.AdmissionRequest
Err string // expected error string, not exact
Patches []jsonpatch.JsonPatchOperation
}{
{
"kube-system namespace",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Namespace: metav1.NamespaceSystem,
Object: encodeRaw(t, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -87,7 +87,7 @@ func TestHandlerHandle(t *testing.T) {
{
"already injected",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Object: encodeRaw(t, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
Expand All @@ -105,7 +105,7 @@ func TestHandlerHandle(t *testing.T) {
{
"no injection by default",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Object: encodeRaw(t, &corev1.Pod{
Spec: basicSpec,
}),
Expand All @@ -117,7 +117,7 @@ func TestHandlerHandle(t *testing.T) {
{
"injection disabled",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Namespace: "test",
Object: encodeRaw(t, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -136,7 +136,7 @@ func TestHandlerHandle(t *testing.T) {
{
"basic pod injection",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Namespace: "test",
Object: encodeRaw(t, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -188,7 +188,7 @@ func TestHandlerHandle(t *testing.T) {
{
"init first ",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Namespace: "test",
Object: encodeRaw(t, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -249,7 +249,7 @@ func TestHandlerHandle(t *testing.T) {
{
"configmap pod injection",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Namespace: "test",
Object: encodeRaw(t, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -305,7 +305,7 @@ func TestHandlerHandle(t *testing.T) {
{
"tls pod injection",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Namespace: "test",
Object: encodeRaw(t, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -366,7 +366,7 @@ func TestHandlerHandle(t *testing.T) {
{
"tls no configmap pod injection",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Namespace: "test",
Object: encodeRaw(t, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -423,7 +423,7 @@ func TestHandlerHandle(t *testing.T) {
{
"tls no configmap no init pod injection",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Namespace: "test",
Object: encodeRaw(t, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -469,7 +469,7 @@ func TestHandlerHandle(t *testing.T) {
{
"tls no configmap init only pod injection",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Namespace: "test",
Object: encodeRaw(t, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -519,7 +519,7 @@ func TestHandlerHandle(t *testing.T) {
{
"copy volume mounts pod injection",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Namespace: "test",
Object: encodeRaw(t, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -571,7 +571,7 @@ func TestHandlerHandle(t *testing.T) {
{
"invalid default template",
basicHandler(),
v1beta1.AdmissionRequest{
v1.AdmissionRequest{
Namespace: "foo",
Object: encodeRaw(t, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand Down
40 changes: 6 additions & 34 deletions deploy/injector-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,38 +20,6 @@ spec:
spec:
serviceAccountName: "vault-injector"
containers:
- name: leader-elector
image: k8s.gcr.io/leader-elector:0.4
args:
- --election=vault-agent-injector-leader
- --election-namespace=$(NAMESPACE)
- --http=0.0.0.0:4040
- --ttl=60s
env:
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
livenessProbe:
httpGet:
path: /
port: 4040
scheme: HTTP
failureThreshold: 2
initialDelaySeconds: 1
periodSeconds: 2
successThreshold: 1
timeoutSeconds: 5
readinessProbe:
httpGet:
path: /
port: 4040
scheme: HTTP
failureThreshold: 2
initialDelaySeconds: 2
periodSeconds: 2
successThreshold: 1
timeoutSeconds: 5
- name: sidecar-injector
image: "hashicorp/vault-k8s:0.10.2"
imagePullPolicy: IfNotPresent
Expand All @@ -60,6 +28,10 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: AGENT_INJECT_LISTEN
value: ":8080"
- name: AGENT_INJECT_LOG_LEVEL
Expand Down Expand Up @@ -95,7 +67,7 @@ spec:
port: 8080
scheme: HTTPS
failureThreshold: 2
initialDelaySeconds: 1
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this change on initial delay for readiness and liveness mean that leader election is a bit longer to establish through this method, or is it to make it less flaky in general?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I think it was just taking a little longer than a second while I was testing it locally to achieve leadership and generate the certs so that the liveness probe would pass.

initialDelaySeconds: 5
periodSeconds: 2
successThreshold: 1
timeoutSeconds: 5
Expand All @@ -105,7 +77,7 @@ spec:
port: 8080
scheme: HTTPS
failureThreshold: 2
initialDelaySeconds: 2
initialDelaySeconds: 5
periodSeconds: 2
successThreshold: 1
timeoutSeconds: 5
8 changes: 7 additions & 1 deletion deploy/injector-mutating-webhook.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
---
apiVersion: admissionregistration.k8s.io/v1beta1
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
name: vault-agent-injector-cfg
Expand All @@ -8,6 +8,10 @@ metadata:
app.kubernetes.io/instance: vault
webhooks:
- name: vault.hashicorp.com
sideEffects: None
admissionReviewVersions:
- "v1"
- "v1beta"
clientConfig:
service:
name: vault-agent-injector-svc
Expand All @@ -20,3 +24,5 @@ webhooks:
apiVersions: ["v1"]
resources: ["deployments", "jobs", "pods", "statefulsets"]
namespaceSelector: {}
objectSelector: {}
failurePolicy: Ignore
7 changes: 6 additions & 1 deletion deploy/injector-rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,18 @@ metadata:
app.kubernetes.io/instance: vault
rules:
- apiGroups: [""]
resources: ["endpoints", "secrets"]
resources: ["secrets", "configmaps"]
verbs:
- "create"
- "get"
- "watch"
- "list"
- "update"
- apiGroups: [""]
resources: ["pods"]
verbs:
- "get"
- "patch"
Comment on lines +61 to +62
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I haven't read the whole change set yet, but what uses these permissions?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's part of the Become() call; it looks like there's some checking and cleanup around situations where an old leader pod was evicted or the underlying node is NotReady: https://github.com/operator-framework/operator-lib/blob/v0.4.1/leader/leader.go#L166-L189

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
Expand Down
27 changes: 7 additions & 20 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,33 +4,20 @@ go 1.16

require (
github.com/armon/go-radix v1.0.0 // indirect
github.com/evanphx/json-patch v4.9.0+incompatible // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/googleapis/gnostic v0.2.0 // indirect
github.com/hashicorp/go-hclog v0.9.2
github.com/hashicorp/golang-lru v0.5.3 // indirect
github.com/hashicorp/vault/sdk v0.1.14-0.20191205220236-47cffd09f972
github.com/kelseyhightower/envconfig v1.4.0
github.com/kr/text v0.2.0
github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a
github.com/mattn/go-colorable v0.1.2 // indirect
github.com/mitchellh/cli v1.0.0
github.com/onsi/gomega v1.4.2 // indirect
github.com/pkg/errors v0.8.1
github.com/operator-framework/operator-lib v0.4.1
github.com/pkg/errors v0.9.1
github.com/posener/complete v1.2.1 // indirect
github.com/prometheus/client_golang v1.7.1
github.com/radovskyb/watcher v1.0.7
github.com/spf13/pflag v1.0.3 // indirect
github.com/stretchr/testify v1.4.0
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.2.8 // indirect
k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b
k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d
k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
k8s.io/klog v1.0.0 // indirect
k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 // indirect
k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d // indirect
sigs.k8s.io/yaml v1.1.0 // indirect
github.com/stretchr/testify v1.6.1
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
k8s.io/api v0.21.3
k8s.io/apimachinery v0.21.3
k8s.io/client-go v0.21.3
)
Loading