GPE-1250: Created a mutating webhook to ensure daemonsets don't sched…

…ule on fargate

.github/workflows/image_build_push_webhook.yaml

@@ -0,0 +1,22 @@
+name: Build Squid images
+
+on: 
+  push:
+    paths:
+      - .github/workflows/image_build_push_squid.yaml
+      - Docker/daemonset-webhook/**
+
+jobs:
+  webhook:
+    name: webhook image
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      DOCKERFILE_LOCATION: "./Docker/daemonset-webhook/Dockerfile"
+      DOCKERFILE_BUILD_CONTEXT: "./Docker/daemonset-webhook"
+      OVERRIDE_REPO_NAME: "node-affinity-daemonset"
+      USE_QUAY_ONLY: true
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}    

Docker/daemonset-webhook/Dockerfile

@@ -0,0 +1,18 @@
+FROM quay.io/cdis/golang:1.20-bullseye as build-deps
+
+ENV CGO_ENABLED=0
+ENV GOOS=linux
+ENV GOARCH=amd64
+
+WORKDIR $GOPATH/src/github.com/uc-cdis/node-affinity-daemonset/
+
+COPY node-affinity-daemonset/* .
+
+RUN go mod download
+
+RUN  go build -o /webhook
+
+FROM scratch
+COPY --from=build-deps /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build-deps /webhook /webhook
+CMD ["/webhook"]

Docker/daemonset-webhook/node-affinity-daemonset/go.mod

@@ -0,0 +1,67 @@
+
+go 1.18
+
+require (
+	k8s.io/api v0.27.2
+	k8s.io/apimachinery v0.27.2
+	sigs.k8s.io/controller-runtime v0.15.0
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/zapr v1.2.4 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.1 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.15.1 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/oauth2 v0.5.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/term v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.27.2 // indirect
+	k8s.io/client-go v0.27.2 // indirect
+	k8s.io/component-base v0.27.2 // indirect
+	k8s.io/klog/v2 v2.90.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
+	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)

Docker/daemonset-webhook/node-affinity-daemonset/main.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"os"
+
+	"github.com/go-logr/logr"
+	admissionv1 "k8s.io/api/admission/v1"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+	//jsonpatch "gomodules.xyz/jsonpatch/v2"
+)
+
+// DaemonSetMutator implements admission.Handler
+type DaemonSetMutator struct {
+	Decoder *admission.Decoder
+	Logger  logr.Logger
+}
+
+func (m *DaemonSetMutator) Handle(ctx context.Context, req admission.Request) admission.Response {
+	ds := &appsv1.DaemonSet{}
+	err := m.Decoder.Decode(req, ds)
+	if err != nil {
+		m.Logger.Error(err, "Failed to decode request")
+		return admission.Errored(http.StatusBadRequest, err)
+	}
+
+	// Add node affinity to avoid Fargate nodes
+	affinity := &corev1.Affinity{
+		NodeAffinity: &corev1.NodeAffinity{
+			RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
+				NodeSelectorTerms: []corev1.NodeSelectorTerm{
+					{
+						MatchExpressions: []corev1.NodeSelectorRequirement{
+							{
+								Key:      "eks.amazonaws.com/compute-type",
+								Operator: corev1.NodeSelectorOpNotIn,
+								Values:   []string{"fargate"},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	if ds.Spec.Template.Spec.Affinity == nil {
+		ds.Spec.Template.Spec.Affinity = affinity
+	} else {
+		ds.Spec.Template.Spec.Affinity.NodeAffinity = affinity.NodeAffinity
+	}
+
+	marshalledDS, err := json.Marshal(ds)
+	if err != nil {
+		m.Logger.Error(err, "Failed to marshal response")
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+
+	return admission.PatchResponseFromRaw(req.Object.Raw, marshalledDS)
+}
+
+func main() {
+	logger := zap.New(zap.UseDevMode(true))
+	ctrl.SetLogger(logger)
+	log := ctrl.Log.WithName("webhook")
+
+	scheme := runtime.NewScheme()
+	decoder := admission.NewDecoder(scheme)
+
+	m := &DaemonSetMutator{Decoder: decoder, Logger: log}
+
+	http.HandleFunc("/mutate", func(w http.ResponseWriter, r *http.Request) {
+		ctx := context.Background()
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			log.Error(err, "Could not read request body")
+			http.Error(w, "could not read request body", http.StatusBadRequest)
+			return
+		}
+		log.Info("Received request", "body", string(body))
+
+		review := &admissionv1.AdmissionReview{}
+		err = json.Unmarshal(body, review)
+		if err != nil {
+			log.Error(err, "Could not decode request body")
+			http.Error(w, "could not decode request body", http.StatusBadRequest)
+			return
+		}
+
+		req := admission.Request{
+			AdmissionRequest: *review.Request,
+		}
+
+		resp := m.Handle(ctx, req)
+
+		var patchBytes []byte
+		if resp.Patches != nil {
+			patchBytes, err = json.Marshal(resp.Patches)
+			if err != nil {
+				log.Error(err, "Could not marshal patches")
+				http.Error(w, "could not marshal patches", http.StatusInternalServerError)
+				return
+			}
+		}
+
+		review.Response = &admissionv1.AdmissionResponse{
+			UID:     review.Request.UID,
+			Allowed: resp.Allowed,
+			Result:  resp.Result,
+			Patch:   patchBytes,
+			PatchType: func() *admissionv1.PatchType {
+				if len(patchBytes) > 0 {
+					pt := admissionv1.PatchTypeJSONPatch
+					return &pt
+				}
+				return nil
+			}(),
+		}
+
+		respBytes, err := json.Marshal(review)
+		if err != nil {
+			log.Error(err, "Could not encode response")
+			http.Error(w, "could not encode response", http.StatusInternalServerError)
+			return
+		}
+
+		_, err = w.Write(respBytes)
+		if err != nil {
+			log.Error(err, "Could not write response")
+		}
+	})
+
+	log.Info("Starting webhook server")
+	if err := http.ListenAndServeTLS(":8443", "/etc/webhook/certs/tls.crt", "/etc/webhook/certs/tls.key", nil); err != nil {
+		log.Error(err, "Failed to start webhook server")
+		os.Exit(1)
+	}
+}

kube/services/node-affinity-daemonset/README.md

@@ -0,0 +1,44 @@
+# The webhook needs a valid cert to work, so we need to create one for the webhook and the deployment to use
+
+cat <<EOF > csr.conf
+[ req ]
+default_bits = 2048
+prompt = no
+default_md = sha256
+distinguished_name = dn
+
+[ dn ]
+CN = node-affinity-daemonset.kube-system.svc
+
+[ v3_ext ]
+subjectAltName = @alt_names
+
+[ alt_names ]
+DNS.1 = daemonset-node-affinity
+DNS.2 = node-affinity-daemonset.kube-system
+DNS.3 = node-affinity-daemonset.kube-system.svc
+DNS.4 = node-affinity-daemonset.kube-system.svc.cluster.local
+
+[ req_ext ]
+subjectAltName = @alt_names
+
+[ alt_names ]
+DNS.1 = daemonset-node-affinity
+DNS.2 = node-affinity-daemonset.kube-system
+DNS.3 = node-affinity-daemonset.kube-system.svc
+DNS.4 = node-affinity-daemonset.kube-system.svc.cluster.local
+EOF
+
+
+openssl req -new -nodes -x509 -newkey rsa:2048 -keyout server.key -out server.crt -days 365 -config csr.conf -extensions v3_ext
+
+kubectl create secret tls webhook-certs --cert=server.crt --key=server.key -n kube-system
+
+## This will make the base64 for your webhook.yaml file
+
+cat server.crt | base64 | tr -d '\n'
+
+
+### TODO
+
+Use cert-manager to create the cert for the webhook

kube/services/node-affinity-daemonset/deployment.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: node-affinity-daemonset
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: node-affinity-daemonset
+  template:
+    metadata:
+      labels:
+        app: node-affinity-daemonset
+    spec:
+      containers:
+        - name: node-affinity-daemonset
+          image: quay.io/cdis/node-affinity-daemonset:master
+          ports:
+            - containerPort: 8443
+          volumeMounts:
+            - name: webhook-certs
+              mountPath: /etc/webhook/certs
+              readOnly: true
+      volumes:
+        - name: webhook-certs
+          secret:
+            secretName: webhook-certs #pragma: allowlist secret

kube/services/node-affinity-daemonset/service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: node-affinity-daemonset
+  namespace: default
+spec:
+  ports:
+    - port: 443
+      targetPort: 8443
+  selector:
+    app: node-affinity-daemonset

kube/services/node-affinity-daemonset/webhook.yaml

@@ -0,0 +1,18 @@
+kind: MutatingWebhookConfiguration
+metadata:
+  name: node-affinity-daemonset
+webhooks:
+  - name: node-affinity-daemonset.k8s.io
+    clientConfig:
+      service:
+        name: node-affinity-daemonset-service
+        namespace: kube-system
+        path: "/mutate"
+      caBundle: <ca-bundle>
+    rules:
+      - operations: ["CREATE"]
+        apiGroups: ["apps"]
+        apiVersions: ["v1"]
+        resources: ["daemonsets"]
+    admissionReviewVersions: ["v1"]
+    sideEffects: None