From 0b4d1b3dd3236296bfd9bcfedbc8863b2f91f8f0 Mon Sep 17 00:00:00 2001 From: "liheng.zms" Date: Fri, 27 Dec 2024 16:05:59 +0800 Subject: [PATCH] podprobemarker support serverless pod Signed-off-by: liheng.zms --- Dockerfile | 2 +- Dockerfile_multiarch | 2 +- apis/apps/v1alpha1/node_pod_probe_types.go | 12 + apis/apps/v1alpha1/pod_probe_marker_types.go | 24 ++ .../pod_probe_marker_controller.go | 127 ++++++++-- .../pod_probe_marker_controller_test.go | 121 ++++++++++ .../podprobemarker_event_handler.go | 56 ++++- pkg/features/kruise_features.go | 5 + pkg/util/pods.go | 10 + .../pod/mutating/pod_create_update_handler.go | 8 + pkg/webhook/pod/mutating/pod_probe_marker.go | 101 ++++++++ .../pod/mutating/pod_probe_marker_test.go | 227 ++++++++++++++++++ 12 files changed, 674 insertions(+), 21 deletions(-) create mode 100644 pkg/webhook/pod/mutating/pod_probe_marker.go create mode 100644 pkg/webhook/pod/mutating/pod_probe_marker_test.go diff --git a/Dockerfile b/Dockerfile index b22371f68b..4128912298 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Build the manager and daemon binaries ARG BASE_IMAGE=alpine ARG BASE_IMAGE_VERSION=3.19@sha256:ae65dbf8749a7d4527648ccee1fa3deb6bfcae34cbc30fc67aa45c44dcaa90ee -FROM golang:1.20.14-alpine3.19@sha256:e47f121850f4e276b2b210c56df3fda9191278dd84a3a442bfe0b09934462a8f as builder +FROM golang:1.20.14-alpine3.19@sha256:e47f121850f4e276b2b210c56df3fda9191278dd84a3a442bfe0b09934462a8f AS builder WORKDIR /workspace # Copy the Go Modules manifests diff --git a/Dockerfile_multiarch b/Dockerfile_multiarch index ab9399ad87..8d4bf37b2b 100644 --- a/Dockerfile_multiarch +++ b/Dockerfile_multiarch @@ -2,7 +2,7 @@ ARG BASE_IMAGE=alpine ARG BASE_IMAGE_VERSION=3.19@sha256:ae65dbf8749a7d4527648ccee1fa3deb6bfcae34cbc30fc67aa45c44dcaa90ee ARG BUILD_BASE_IMAGE=golang:1.20.14-alpine3.19@sha256:e47f121850f4e276b2b210c56df3fda9191278dd84a3a442bfe0b09934462a8f -FROM --platform=$BUILDPLATFORM ${BUILD_BASE_IMAGE} as builder +FROM --platform=$BUILDPLATFORM ${BUILD_BASE_IMAGE} AS builder WORKDIR /workspace # Copy the Go Modules manifests diff --git a/apis/apps/v1alpha1/node_pod_probe_types.go b/apis/apps/v1alpha1/node_pod_probe_types.go index 772bc5f5f2..a86ca01a22 100644 --- a/apis/apps/v1alpha1/node_pod_probe_types.go +++ b/apis/apps/v1alpha1/node_pod_probe_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -87,6 +88,17 @@ const ( ProbeUnknown ProbeState = "Unknown" ) +func (p ProbeState) IsEqualPodConditionStatus(status corev1.ConditionStatus) bool { + switch status { + case corev1.ConditionTrue: + return p == ProbeSucceeded + case corev1.ConditionFalse: + return p == ProbeFailed + default: + return p == ProbeUnknown + } +} + // +genclient // +genclient:nonNamespaced // +k8s:openapi-gen=true diff --git a/apis/apps/v1alpha1/pod_probe_marker_types.go b/apis/apps/v1alpha1/pod_probe_marker_types.go index af74ccf559..883ed86f8d 100644 --- a/apis/apps/v1alpha1/pod_probe_marker_types.go +++ b/apis/apps/v1alpha1/pod_probe_marker_types.go @@ -21,6 +21,30 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + // PodProbeMarkerAnnotationKey records the Probe Spec, mainly used for serverless Pod scenarios, as follows: + // annotations: + // kruise.io/podprobe: | + // [ + // { + // "containerName": "minecraft", + // "name": "healthy", + // "podConditionType": "game.kruise.io/healthy", + // "probe": { + // "exec": { + // "command": [ + // "bash", + // "/data/probe.sh" + // ] + // } + // } + // } + // ] + PodProbeMarkerAnnotationKey = "kruise.io/podprobe" + // PodProbeMarkerListAnnotationKey records the injected PodProbeMarker Name List + PodProbeMarkerListAnnotationKey = "kruise.io/podprobemarker-list" +) + // PodProbeMarkerSpec defines the desired state of PodProbeMarker type PodProbeMarkerSpec struct { // Selector is a label query over pods that should exec custom probe diff --git a/pkg/controller/podprobemarker/pod_probe_marker_controller.go b/pkg/controller/podprobemarker/pod_probe_marker_controller.go index 89295b8779..d89dc91115 100644 --- a/pkg/controller/podprobemarker/pod_probe_marker_controller.go +++ b/pkg/controller/podprobemarker/pod_probe_marker_controller.go @@ -18,6 +18,7 @@ package podprobemarker import ( "context" + "encoding/json" "flag" "fmt" "reflect" @@ -25,8 +26,11 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" kubecontroller "k8s.io/kubernetes/pkg/controller" @@ -60,6 +64,8 @@ var ( const ( // PodProbeMarkerFinalizer is on PodProbeMarker, and used to remove podProbe from NodePodProbe.Spec PodProbeMarkerFinalizer = "kruise.io/node-pod-probe-cleanup" + + VirtualKubelet = "virtual-kubelet" ) /** @@ -145,14 +151,14 @@ func (r *ReconcilePodProbeMarker) syncPodProbeMarker(ns, name string) error { // Error reading the object - requeue the request. return err } - pods, err := r.getMatchingPods(ppm) + normalPods, serverlessPods, err := r.getMatchingPods(ppm) if err != nil { klog.ErrorS(err, "PodProbeMarker listed pods failed", "podProbeMarker", klog.KObj(ppm)) return err } // remove podProbe from NodePodProbe.Spec if !ppm.DeletionTimestamp.IsZero() { - return r.handlerPodProbeMarkerFinalizer(ppm, pods) + return r.handlerPodProbeMarkerFinalizer(ppm, normalPods) } // add finalizer if !controllerutil.ContainsFinalizer(ppm, PodProbeMarkerFinalizer) { @@ -164,9 +170,26 @@ func (r *ReconcilePodProbeMarker) syncPodProbeMarker(ns, name string) error { klog.V(3).InfoS("Added PodProbeMarker finalizer success", "podProbeMarker", klog.KObj(ppm)) } + // map[probe.PodConditionType] = probe.MarkerPolicy + markers := make(map[string][]appsv1alpha1.ProbeMarkerPolicy) + for _, probe := range ppm.Spec.Probes { + if probe.PodConditionType == "" || len(probe.MarkerPolicy) == 0 { + continue + } + markers[probe.PodConditionType] = probe.MarkerPolicy + } + if utilfeature.DefaultFeatureGate.Enabled(features.EnablePodProbeMarkerOnServerless) && len(markers) != 0 { + for _, pod := range serverlessPods { + if err = r.markerServerlessPod(pod, markers); err != nil { + klog.ErrorS(err, "Failed to marker serverless pod", "podProbeMarker", klog.KObj(ppm), "pod", klog.KObj(pod)) + return err + } + } + } + groupByNode := make(map[string][]*corev1.Pod) - for i, pod := range pods { - groupByNode[pod.Spec.NodeName] = append(groupByNode[pod.Spec.NodeName], pods[i]) + for i, pod := range normalPods { + groupByNode[pod.Spec.NodeName] = append(groupByNode[pod.Spec.NodeName], normalPods[i]) } // add podProbe in NodePodProbe @@ -182,11 +205,12 @@ func (r *ReconcilePodProbeMarker) syncPodProbeMarker(ns, name string) error { if err := r.Client.Get(context.TODO(), client.ObjectKey{Namespace: ppm.Namespace, Name: ppm.Name}, ppmClone); err != nil { klog.ErrorS(err, "Failed to get updated PodProbeMarker from client", "podProbeMarker", klog.KObj(ppm)) } - if ppmClone.Status.ObservedGeneration == ppmClone.Generation && int(ppmClone.Status.MatchedPods) == len(pods) { + matchedPods := len(normalPods) + len(serverlessPods) + if ppmClone.Status.ObservedGeneration == ppmClone.Generation && int(ppmClone.Status.MatchedPods) == matchedPods { return nil } ppmClone.Status.ObservedGeneration = ppmClone.Generation - ppmClone.Status.MatchedPods = int64(len(pods)) + ppmClone.Status.MatchedPods = int64(matchedPods) return r.Client.Status().Update(context.TODO(), ppmClone) }); err != nil { klog.ErrorS(err, "PodProbeMarker update status failed", "podProbeMarker", klog.KObj(ppm)) @@ -214,6 +238,59 @@ func (r *ReconcilePodProbeMarker) handlerPodProbeMarkerFinalizer(ppm *appsv1alph return nil } +// marker labels or annotations on Pod based on probing results +// markers is map[probe.PodConditionType] = probe.MarkerPolicy +func (r *ReconcilePodProbeMarker) markerServerlessPod(pod *corev1.Pod, markers map[string][]appsv1alpha1.ProbeMarkerPolicy) error { + newObjectMeta := *pod.ObjectMeta.DeepCopy() + if newObjectMeta.Annotations == nil { + newObjectMeta.Annotations = make(map[string]string) + } + if newObjectMeta.Labels == nil { + newObjectMeta.Labels = make(map[string]string) + } + + for cond, policy := range markers { + condition := util.GetCondition(pod, corev1.PodConditionType(cond)) + if condition == nil { + continue + } + + for _, obj := range policy { + if !obj.State.IsEqualPodConditionStatus(condition.Status) { + continue + } + for k, v := range obj.Annotations { + newObjectMeta.Annotations[k] = v + } + for k, v := range obj.Labels { + newObjectMeta.Labels[k] = v + } + } + } + + // no change + if reflect.DeepEqual(pod.ObjectMeta, newObjectMeta) { + return nil + } + + // patch change + oldBytes, _ := json.Marshal(corev1.Pod{ObjectMeta: pod.ObjectMeta}) + newBytes, _ := json.Marshal(corev1.Pod{ObjectMeta: newObjectMeta}) + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldBytes, newBytes, &corev1.Pod{}) + if err != nil { + return err + } + obj := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: pod.Namespace, Name: pod.Name}} + if err = r.Patch(context.TODO(), obj, client.RawPatch(types.StrategicMergePatchType, patchBytes)); err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + klog.V(3).InfoS("PodProbeMarker marker pod success", "pod", klog.KObj(pod), "patch", string(patchBytes)) + return nil +} + func (r *ReconcilePodProbeMarker) updateNodePodProbes(ppm *appsv1alpha1.PodProbeMarker, nodeName string, pods []*corev1.Pod) error { npp := &appsv1alpha1.NodePodProbe{} err := r.Get(context.TODO(), client.ObjectKey{Name: nodeName}, npp) @@ -355,28 +432,50 @@ func setPodContainerProbes(podProbe *appsv1alpha1.PodProbe, probe appsv1alpha1.P } // If you need update the pod object, you must DeepCopy it -func (r *ReconcilePodProbeMarker) getMatchingPods(ppm *appsv1alpha1.PodProbeMarker) ([]*corev1.Pod, error) { +// para1: normal pods, which is on normal node +// para2: serverless pods +func (r *ReconcilePodProbeMarker) getMatchingPods(ppm *appsv1alpha1.PodProbeMarker) ([]*corev1.Pod, []*corev1.Pod, error) { // get more faster selector selector, err := util.ValidatedLabelSelectorAsSelector(ppm.Spec.Selector) if err != nil { - return nil, err + return nil, nil, err } // DisableDeepCopy:true, indicates must be deep copy before update pod objection listOpts := &client.ListOptions{LabelSelector: selector, Namespace: ppm.Namespace} podList := &corev1.PodList{} if listErr := r.Client.List(context.TODO(), podList, listOpts, utilclient.DisableDeepCopy); listErr != nil { - return nil, err + return nil, nil, err } - pods := make([]*corev1.Pod, 0, len(podList.Items)) + normalPods := make([]*corev1.Pod, 0, len(podList.Items)) + serverlessPods := make([]*corev1.Pod, 0, len(podList.Items)) + nodes := map[string]*corev1.Node{} for i := range podList.Items { pod := &podList.Items[i] condition := util.GetCondition(pod, corev1.PodInitialized) - if kubecontroller.IsPodActive(pod) && pod.Spec.NodeName != "" && - condition != nil && condition.Status == corev1.ConditionTrue { - pods = append(pods, pod) + if !kubecontroller.IsPodActive(pod) || condition == nil || + condition.Status != corev1.ConditionTrue || pod.Spec.NodeName == "" { + continue + } + + node, ok := nodes[pod.Spec.NodeName] + if !ok { + node = &corev1.Node{} + if err = r.Get(context.TODO(), client.ObjectKey{Name: pod.Spec.NodeName}, node); err != nil { + if errors.IsNotFound(err) { + continue + } + return nil, nil, err + } + nodes[node.Name] = node + } + + if node.Labels["type"] == VirtualKubelet { + serverlessPods = append(serverlessPods, pod) + } else { + normalPods = append(normalPods, pod) } } - return pods, nil + return normalPods, serverlessPods, nil } func (r *ReconcilePodProbeMarker) removePodProbeFromNodePodProbe(ppmName, nppName string) error { diff --git a/pkg/controller/podprobemarker/pod_probe_marker_controller_test.go b/pkg/controller/podprobemarker/pod_probe_marker_controller_test.go index 47298c56fd..9a427b9adb 100644 --- a/pkg/controller/podprobemarker/pod_probe_marker_controller_test.go +++ b/pkg/controller/podprobemarker/pod_probe_marker_controller_test.go @@ -1961,3 +1961,124 @@ func TestUpdateNodePodProbes(t *testing.T) { }) } } + +func TestMarkerServerlessPod(t *testing.T) { + cases := []struct { + name string + getPod func() *corev1.Pod + markers map[string][]appsv1alpha1.ProbeMarkerPolicy + expectedLabels map[string]string + expectedAnnotations map[string]string + }{ + { + name: "probe success", + getPod: func() *corev1.Pod { + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-1", + Annotations: map[string]string{ + "app": "web", + }, + Labels: map[string]string{ + "app": "web", + }, + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodInitialized, + Status: corev1.ConditionTrue, + }, + { + Type: corev1.PodConditionType("game.io/idle"), + Status: corev1.ConditionTrue, + }, + { + Type: corev1.PodConditionType("game.io/healthy"), + Status: corev1.ConditionFalse, + }, + }, + }, + } + + return obj + }, + markers: map[string][]appsv1alpha1.ProbeMarkerPolicy{ + "game.io/idle": { + { + State: appsv1alpha1.ProbeSucceeded, + Labels: map[string]string{ + "gameserver-idle": "true", + }, + Annotations: map[string]string{ + "controller.kubernetes.io/pod-deletion-cost": "-10", + }, + }, + { + State: appsv1alpha1.ProbeFailed, + Labels: map[string]string{ + "gameserver-idle": "false", + }, + Annotations: map[string]string{ + "controller.kubernetes.io/pod-deletion-cost": "10", + }, + }, + }, + "game.io/healthy": { + { + State: appsv1alpha1.ProbeSucceeded, + Labels: map[string]string{ + "gameserver-healthy": "true", + }, + Annotations: map[string]string{ + "controller.kubernetes.io/ingress": "true", + }, + }, + { + State: appsv1alpha1.ProbeFailed, + Labels: map[string]string{ + "gameserver-healthy": "false", + }, + Annotations: map[string]string{ + "controller.kubernetes.io/ingress": "false", + }, + }, + }, + }, + expectedLabels: map[string]string{ + "gameserver-healthy": "false", + "gameserver-idle": "true", + "app": "web", + }, + expectedAnnotations: map[string]string{ + "controller.kubernetes.io/ingress": "false", + "controller.kubernetes.io/pod-deletion-cost": "-10", + "app": "web", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + pod := tc.getPod() + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(pod).Build() + r := &ReconcilePodProbeMarker{Client: fakeClient} + err := r.markerServerlessPod(tc.getPod(), tc.markers) + if err != nil { + t.Fatalf(err.Error()) + } + newPod := &corev1.Pod{} + err = fakeClient.Get(context.TODO(), client.ObjectKeyFromObject(pod), newPod) + if err != nil { + t.Fatalf(err.Error()) + } + if !reflect.DeepEqual(tc.expectedLabels, newPod.Labels) { + t.Errorf("expect: %v, but: %v", tc.expectedLabels, newPod.Labels) + } + if !reflect.DeepEqual(tc.expectedAnnotations, newPod.Annotations) { + t.Errorf("expect: %v, but: %v", tc.expectedAnnotations, newPod.Annotations) + } + }) + } + +} diff --git a/pkg/controller/podprobemarker/podprobemarker_event_handler.go b/pkg/controller/podprobemarker/podprobemarker_event_handler.go index 711e09418d..88d0d617d7 100644 --- a/pkg/controller/podprobemarker/podprobemarker_event_handler.go +++ b/pkg/controller/podprobemarker/podprobemarker_event_handler.go @@ -18,6 +18,8 @@ package podprobemarker import ( "context" + "reflect" + "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" @@ -32,8 +34,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" appsalphav1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/features" "github.com/openkruise/kruise/pkg/util" utilclient "github.com/openkruise/kruise/pkg/util/client" + utilfeature "github.com/openkruise/kruise/pkg/util/feature" ) var _ handler.EventHandler = &enqueueRequestForPodProbeMarker{} @@ -94,12 +98,12 @@ func (p *enqueueRequestForPod) Update(ctx context.Context, evt event.UpdateEvent // add pod probe to nodePodProbe.spec oldInitialCondition := util.GetCondition(old, corev1.PodInitialized) newInitialCondition := util.GetCondition(new, corev1.PodInitialized) - if newInitialCondition == nil { - return - } - if !kubecontroller.IsPodActive(new) { + if !kubecontroller.IsPodActive(new) || newInitialCondition == nil || + newInitialCondition.Status != corev1.ConditionTrue || new.Spec.NodeName == "" { return } + + // normal pod if ((oldInitialCondition == nil || oldInitialCondition.Status == corev1.ConditionFalse) && newInitialCondition.Status == corev1.ConditionTrue) || old.Status.PodIP != new.Status.PodIP { ppms, err := p.getPodProbeMarkerForPod(new) @@ -116,14 +120,56 @@ func (p *enqueueRequestForPod) Update(ctx context.Context, evt event.UpdateEvent }) } } + + // serverless pod + if utilfeature.DefaultFeatureGate.Enabled(features.EnablePodProbeMarkerOnServerless) { + if reflect.DeepEqual(old.Status.Conditions, new.Status.Conditions) { + return + } + node := &corev1.Node{} + if err := p.reader.Get(context.TODO(), client.ObjectKey{Name: new.Spec.NodeName}, node); err != nil { + klog.ErrorS(err, "Failed to get Node", "nodeName", new.Spec.NodeName) + return + } + if node.Labels["type"] != VirtualKubelet { + return + } + ppms, err := p.getPodProbeMarkerForPod(new) + if err != nil { + klog.ErrorS(err, "Failed to List PodProbeMarker") + return + } + for _, ppm := range ppms { + q.Add(reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: ppm.Namespace, + Name: ppm.Name, + }, + }) + } + } } func (p *enqueueRequestForPod) getPodProbeMarkerForPod(pod *corev1.Pod) ([]*appsalphav1.PodProbeMarker, error) { + var ppms []*appsalphav1.PodProbeMarker + // new pod have annotation kruise.io/podprobemarker-list + if str, ok := pod.Annotations[appsalphav1.PodProbeMarkerListAnnotationKey]; ok && str != "" { + names := strings.Split(str, ",") + for _, name := range names { + ppm := &appsalphav1.PodProbeMarker{} + if err := p.reader.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: name}, ppm); err != nil { + klog.ErrorS(err, "Failed to get PodProbeMarker", "name", name) + continue + } + ppms = append(ppms, ppm) + } + return ppms, nil + } + ppmList := &appsalphav1.PodProbeMarkerList{} if err := p.reader.List(context.TODO(), ppmList, &client.ListOptions{Namespace: pod.Namespace}, utilclient.DisableDeepCopy); err != nil { return nil, err } - var ppms []*appsalphav1.PodProbeMarker for i := range ppmList.Items { ppm := &ppmList.Items[i] // This error is irreversible, so continue diff --git a/pkg/features/kruise_features.go b/pkg/features/kruise_features.go index bfa086fb05..e8b1a4a741 100644 --- a/pkg/features/kruise_features.go +++ b/pkg/features/kruise_features.go @@ -138,6 +138,9 @@ const ( // InPlaceWorkloadVerticalScaling enable CloneSet/Advanced StatefulSet controller to support vertical scaling // of managed Pods. InPlaceWorkloadVerticalScaling featuregate.Feature = "InPlaceWorkloadVerticalScaling" + + // EnablePodProbeMarkerOnServerless enable PodProbeMarker on Serverless Pod + EnablePodProbeMarkerOnServerless featuregate.Feature = "EnablePodProbeMarkerOnServerless" ) var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ @@ -175,6 +178,7 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ StatefulSetAutoResizePVCGate: {Default: false, PreRelease: featuregate.Alpha}, ForceDeleteTimeoutExpectationFeatureGate: {Default: false, PreRelease: featuregate.Alpha}, InPlaceWorkloadVerticalScaling: {Default: false, PreRelease: featuregate.Alpha}, + EnablePodProbeMarkerOnServerless: {Default: false, PreRelease: featuregate.Alpha}, } func init() { @@ -202,6 +206,7 @@ func SetDefaultFeatureGates() { _ = utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=false", WorkloadSpread)) _ = utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=false", SidecarSetPatchPodMetadataDefaultsAllowed)) _ = utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=false", EnhancedLivenessProbeGate)) + _ = utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=false", EnablePodProbeMarkerOnServerless)) } if !utilfeature.DefaultFeatureGate.Enabled(KruiseDaemon) { _ = utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=false", PreDownloadImageForInPlaceUpdate)) diff --git a/pkg/util/pods.go b/pkg/util/pods.go index a17901018c..cbf79181c8 100644 --- a/pkg/util/pods.go +++ b/pkg/util/pods.go @@ -402,3 +402,13 @@ func GetPodContainerByName(cName string, pod *v1.Pod) *v1.Container { return nil } + +// IsRestartableInitContainer returns true if the initContainer has +// ContainerRestartPolicyAlways. +func IsRestartableInitContainer(initContainer *v1.Container) bool { + if initContainer.RestartPolicy == nil { + return false + } + + return *initContainer.RestartPolicy == v1.ContainerRestartPolicyAlways +} diff --git a/pkg/webhook/pod/mutating/pod_create_update_handler.go b/pkg/webhook/pod/mutating/pod_create_update_handler.go index c9cd6d6270..895330356b 100644 --- a/pkg/webhook/pod/mutating/pod_create_update_handler.go +++ b/pkg/webhook/pod/mutating/pod_create_update_handler.go @@ -109,6 +109,14 @@ func (h *PodCreateHandler) Handle(ctx context.Context, req admission.Request) ad } } + if utilfeature.DefaultFeatureGate.Enabled(features.EnablePodProbeMarkerOnServerless) { + if skip, err := h.podProbeMakerMutatingPod(ctx, req, obj); err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } else if !skip { + changed = true + } + } + if !changed { return admission.Allowed("") } diff --git a/pkg/webhook/pod/mutating/pod_probe_marker.go b/pkg/webhook/pod/mutating/pod_probe_marker.go new file mode 100644 index 0000000000..04e9a077f5 --- /dev/null +++ b/pkg/webhook/pod/mutating/pod_probe_marker.go @@ -0,0 +1,101 @@ +/* +Copyright 2024 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutating + +import ( + "context" + "fmt" + "strings" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/util" + utilclient "github.com/openkruise/kruise/pkg/util/client" + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog/v2" + "k8s.io/kube-openapi/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// mutating relate-pub annotation in pod +func (h *PodCreateHandler) podProbeMakerMutatingPod(ctx context.Context, req admission.Request, pod *corev1.Pod) (skip bool, err error) { + if len(req.AdmissionRequest.SubResource) > 0 || req.AdmissionRequest.Operation != admissionv1.Create || + req.AdmissionRequest.Resource.Resource != "pods" { + return true, nil + } + ppmList := &appsv1alpha1.PodProbeMarkerList{} + if err = h.Client.List(context.TODO(), ppmList, &client.ListOptions{Namespace: pod.Namespace}, utilclient.DisableDeepCopy); err != nil { + return false, err + } else if len(ppmList.Items) == 0 { + return true, nil + } + + containers := sets.NewString() + for _, c := range pod.Spec.Containers { + containers.Insert(c.Name) + } + for _, c := range pod.Spec.InitContainers { + if util.IsRestartableInitContainer(&c) { + containers.Insert(c.Name) + } + } + + matchedPodProbeMarkerName := sets.NewString() + matchedProbeKey := sets.NewString() + matchedConditions := sets.NewString() + matchedProbes := make([]appsv1alpha1.PodContainerProbe, 0) + for _, obj := range ppmList.Items { + // This error is irreversible, so continue + labelSelector, err := util.ValidatedLabelSelectorAsSelector(obj.Spec.Selector) + if err != nil { + continue + } + // If a PUB with a nil or empty selector creeps in, it should match nothing, not everything. + if labelSelector.Empty() || !labelSelector.Matches(labels.Set(pod.Labels)) { + continue + } + for i := range obj.Spec.Probes { + probe := obj.Spec.Probes[i] + key := fmt.Sprintf("%s/%s", probe.ContainerName, probe.Name) + if matchedConditions.Has(probe.PodConditionType) || matchedProbeKey.Has(key) || !containers.Has(probe.ContainerName) || probe.PodConditionType == "" { + continue + } + // No need to pass in marker related fields + probe.MarkerPolicy = nil + matchedProbes = append(matchedProbes, probe) + matchedProbeKey.Insert(key) + matchedConditions.Insert(probe.PodConditionType) + if !matchedPodProbeMarkerName.Has(obj.Name) { + matchedPodProbeMarkerName.Insert(obj.Name) + } + } + } + if len(matchedProbes) == 0 { + return true, nil + } + + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + body := util.DumpJSON(matchedProbes) + pod.Annotations[appsv1alpha1.PodProbeMarkerAnnotationKey] = body + pod.Annotations[appsv1alpha1.PodProbeMarkerListAnnotationKey] = strings.Join(matchedPodProbeMarkerName.List(), ",") + klog.V(3).InfoS("mutating add pod annotation", "namespace", pod.Namespace, "name", pod.Name, "key", appsv1alpha1.PodProbeMarkerAnnotationKey, "value", body) + return false, nil +} diff --git a/pkg/webhook/pod/mutating/pod_probe_marker_test.go b/pkg/webhook/pod/mutating/pod_probe_marker_test.go new file mode 100644 index 0000000000..f0f3a6513d --- /dev/null +++ b/pkg/webhook/pod/mutating/pod_probe_marker_test.go @@ -0,0 +1,227 @@ +/* +Copyright 2024 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutating + +import ( + "context" + "reflect" + "testing" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/util" + admissionv1 "k8s.io/api/admission/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func TestPodProbeMakerMutatingPod(t *testing.T) { + cases := []struct { + name string + getPod func() *v1.Pod + getPodProbeMarkers func() []*appsv1alpha1.PodProbeMarker + expected map[string]string + }{ + { + name: "podprobemarker, selector matched, but no conditionType", + getPod: func() *v1.Pod { + al := v1.ContainerRestartPolicyAlways + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "version": "test", + }, + Labels: map[string]string{ + "app": "web", + }, + Namespace: "test", + Name: "pod-1", + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "init-1", + }, + { + Name: "init-2", + RestartPolicy: &al, + }, + }, + Containers: []v1.Container{ + { + Name: "main", + }, + { + Name: "envoy", + }, + }, + }, + } + }, + getPodProbeMarkers: func() []*appsv1alpha1.PodProbeMarker { + obj1 := &appsv1alpha1.PodProbeMarker{ + ObjectMeta: metav1.ObjectMeta{ + Name: "healthy", + Namespace: "test", + }, + Spec: appsv1alpha1.PodProbeMarkerSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "web", + }, + }, + Probes: []appsv1alpha1.PodContainerProbe{ + { + ContainerName: "main", + Name: "healthy", + }, + { + ContainerName: "invalid", + Name: "healthy", + PodConditionType: "game.kruise.io/healthy", + }, + }, + }, + } + + return []*appsv1alpha1.PodProbeMarker{obj1} + }, + expected: map[string]string{ + "version": "test", + }, + }, + { + name: "podprobemarker, selector matched", + getPod: func() *v1.Pod { + al := v1.ContainerRestartPolicyAlways + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "version": "test", + }, + Labels: map[string]string{ + "app": "web", + }, + Namespace: "test", + Name: "pod-1", + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "init-1", + }, + { + Name: "init-2", + RestartPolicy: &al, + }, + }, + Containers: []v1.Container{ + { + Name: "main", + }, + { + Name: "envoy", + }, + }, + }, + } + }, + getPodProbeMarkers: func() []*appsv1alpha1.PodProbeMarker { + obj1 := &appsv1alpha1.PodProbeMarker{ + ObjectMeta: metav1.ObjectMeta{ + Name: "healthy", + Namespace: "test", + }, + Spec: appsv1alpha1.PodProbeMarkerSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "web", + }, + }, + Probes: []appsv1alpha1.PodContainerProbe{ + { + ContainerName: "main", + Name: "healthy", + PodConditionType: "game.kruise.io/healthy", + }, + { + ContainerName: "envoy", + Name: "healthy", + PodConditionType: "game.kruise.io/healthy", + }, + }, + }, + } + obj2 := &appsv1alpha1.PodProbeMarker{ + ObjectMeta: metav1.ObjectMeta{ + Name: "init", + Namespace: "test", + }, + Spec: appsv1alpha1.PodProbeMarkerSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "web", + }, + }, + Probes: []appsv1alpha1.PodContainerProbe{ + { + ContainerName: "init-1", + Name: "init", + PodConditionType: "game.kruise.io/init", + }, + { + ContainerName: "init-2", + Name: "init", + PodConditionType: "game.kruise.io/init", + }, + }, + }, + } + return []*appsv1alpha1.PodProbeMarker{obj1, obj2} + }, + expected: map[string]string{ + "version": "test", + appsv1alpha1.PodProbeMarkerAnnotationKey: `[{"name":"healthy","containerName":"main","probe":{},"podConditionType":"game.kruise.io/healthy"},{"name":"init","containerName":"init-2","probe":{},"podConditionType":"game.kruise.io/init"}]`, + appsv1alpha1.PodProbeMarkerListAnnotationKey: "healthy,init", + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + decoder := admission.NewDecoder(scheme.Scheme) + builder := fake.NewClientBuilder() + for i := range c.getPodProbeMarkers() { + obj := c.getPodProbeMarkers()[i] + builder.WithObjects(obj) + } + testClient := builder.Build() + podHandler := &PodCreateHandler{Decoder: decoder, Client: testClient} + req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") + pod := c.getPod() + if _, err := podHandler.podProbeMakerMutatingPod(context.Background(), req, pod); err != nil { + t.Fatalf("failed to mutating pod, err: %v", err) + } + if !reflect.DeepEqual(c.expected, pod.Annotations) { + t.Fatalf("expected: %s, got: %s", util.DumpJSON(c.expected), util.DumpJSON(pod.Annotations)) + } + }) + } +}