From ae3dd17cfc9e2e1cacf1fdd24a9cc036edf4e803 Mon Sep 17 00:00:00 2001 From: Nahshon Unna Tsameret <60659093+nunnatsa@users.noreply.github.com> Date: Tue, 27 Jun 2023 16:37:25 +0300 Subject: [PATCH] move the tuningpolicy_test.sh to golang (#2396) Signed-off-by: Nahshon Unna-Tsameret <nunnatsa@redhat.com> --- hack/check_tuningPolicy.sh | 69 ------------ hack/run-tests.sh | 3 - tests/func-tests/tuningpolicy_test.go | 102 ++++++++++++++++++ .../controllers/common/consts.go | 14 +++ .../controllers/common/hcoConditions.go | 66 ++++++++++++ .../controllers/common/hcoRequest.go | 45 ++++++++ tests/vendor/modules.txt | 2 + .../controller-runtime/pkg/reconcile/doc.go | 21 ++++ .../pkg/reconcile/reconcile.go | 102 ++++++++++++++++++ 9 files changed, 352 insertions(+), 72 deletions(-) delete mode 100755 hack/check_tuningPolicy.sh create mode 100644 tests/func-tests/tuningpolicy_test.go create mode 100644 tests/vendor/github.com/kubevirt/hyperconverged-cluster-operator/controllers/common/consts.go create mode 100644 tests/vendor/github.com/kubevirt/hyperconverged-cluster-operator/controllers/common/hcoConditions.go create mode 100644 tests/vendor/github.com/kubevirt/hyperconverged-cluster-operator/controllers/common/hcoRequest.go create mode 100644 tests/vendor/sigs.k8s.io/controller-runtime/pkg/reconcile/doc.go create mode 100644 tests/vendor/sigs.k8s.io/controller-runtime/pkg/reconcile/reconcile.go diff --git a/hack/check_tuningPolicy.sh b/hack/check_tuningPolicy.sh deleted file mode 100755 index 57dd9f389..000000000 --- a/hack/check_tuningPolicy.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -x -# -# This file is part of the KubeVirt project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Copyright 2022 Red Hat, Inc. -# -# This script checks the tuningPolicy configuration - -RATE_LIMITS=( - ".spec.configuration.apiConfiguration.restClient.rateLimiter.tokenBucketRateLimiter" - ".spec.configuration.controllerConfiguration.restClient.rateLimiter.tokenBucketRateLimiter" - ".spec.configuration.handlerConfiguration.restClient.rateLimiter.tokenBucketRateLimiter" - ".spec.configuration.webhookConfiguration.restClient.rateLimiter.tokenBucketRateLimiter" - -) - - -EXPECTED='{ - "burst": 200, - "qps": 100 -}' - -echo "Check that the TuningPolicy annotation is configuring the KV object as expected" - - -./hack/retry.sh 10 3 "(${KUBECTL_BINARY} annotate -n \"${INSTALLED_NAMESPACE}\" hco kubevirt-hyperconverged hco.kubevirt.io/tuningPolicy='{\"qps\":100,\"burst\":200}')" - -./hack/retry.sh 10 3 "(${KUBECTL_BINARY} patch -n \"${INSTALLED_NAMESPACE}\" hco kubevirt-hyperconverged --type=json -p='[{"op": "add", "path": "/spec/tuningPolicy", "value": "annotation"}]')" - -for jpath in "${RATE_LIMITS[@]}"; do - KUBECONFIG_OUT=$(${KUBECTL_BINARY} get kv -n "${INSTALLED_NAMESPACE}" kubevirt-kubevirt-hyperconverged -o json | jq "${jpath}") - if [[ $KUBECONFIG_OUT != $EXPECTED ]]; then - exit 1 - fi - sleep 2 -done - -./hack/retry.sh 10 3 "(${KUBECTL_BINARY} patch -n \"${INSTALLED_NAMESPACE}\" hco kubevirt-hyperconverged --type=json -p='[{"op": "remove", "path": "/spec/tuningPolicy", "value": "annotation"}]')" -./hack/retry.sh 10 3 "(${KUBECTL_BINARY} annotate -n \"${INSTALLED_NAMESPACE}\" hco kubevirt-hyperconverged hco.kubevirt.io/tuningPolicy-)" - -echo "Check that the TuningPolicy highBurst is configuring the KV object as expected" - -./hack/retry.sh 10 3 "(${KUBECTL_BINARY} patch -n \"${INSTALLED_NAMESPACE}\" hco kubevirt-hyperconverged --type=json -p='[{"op": "replace", "path": "/spec/tuningPolicy", "value": "highBurst"}]')" - -EXPECTED='{ - "burst": 400, - "qps": 200 -}' - -for jpath in "${RATE_LIMITS[@]}"; do - KUBECONFIG_OUT=$(${KUBECTL_BINARY} get kv -n "${INSTALLED_NAMESPACE}" kubevirt-kubevirt-hyperconverged -o json | jq "${jpath}") - if [[ $KUBECONFIG_OUT != $EXPECTED ]]; then - exit 1 - fi - sleep 2 -done - diff --git a/hack/run-tests.sh b/hack/run-tests.sh index 1bda13d70..bdd252bab 100755 --- a/hack/run-tests.sh +++ b/hack/run-tests.sh @@ -50,9 +50,6 @@ KUBECTL_BINARY=${KUBECTL_BINARY} ./hack/check_golden_images.sh # Check TLS profile on the webhook KUBECTL_BINARY=${KUBECTL_BINARY} ./hack/check_tlsprofile.sh -# Check tuning policy -KUBECTL_BINARY=${KUBECTL_BINARY} ./hack/check_tuningPolicy.sh - # check if HCO is able to correctly add back a label used as a label selector ${KUBECTL_BINARY} label priorityclass kubevirt-cluster-critical app- sleep 10 diff --git a/tests/func-tests/tuningpolicy_test.go b/tests/func-tests/tuningpolicy_test.go new file mode 100644 index 000000000..7e3bba154 --- /dev/null +++ b/tests/func-tests/tuningpolicy_test.go @@ -0,0 +1,102 @@ +package tests_test + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "kubevirt.io/client-go/kubecli" + + kvv1 "kubevirt.io/api/core/v1" + "kubevirt.io/kubevirt/tests/flags" + + "github.com/kubevirt/hyperconverged-cluster-operator/api/v1beta1" + "github.com/kubevirt/hyperconverged-cluster-operator/controllers/common" + tests "github.com/kubevirt/hyperconverged-cluster-operator/tests/func-tests" +) + +var _ = Describe("Check that the TuningPolicy annotation is configuring the KV object as expected", Serial, func() { + tests.FlagParse() + var ( + cli kubecli.KubevirtClient + ctx context.Context + ) + + BeforeEach(func() { + var err error + cli, err = kubecli.GetKubevirtClient() + Expect(cli).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + + ctx = context.Background() + }) + + AfterEach(func() { + hc := tests.GetHCO(ctx, cli) + + delete(hc.Annotations, common.TuningPolicyAnnotationName) + hc.Spec.TuningPolicy = "" + + tests.UpdateHCORetry(ctx, cli, hc) + }) + + It("should update KV with the tuningPolicy annotation", func() { + hc := tests.GetHCO(ctx, cli) + + if hc.Annotations == nil { + hc.Annotations = make(map[string]string) + } + hc.Annotations[common.TuningPolicyAnnotationName] = `{"qps":100,"burst":200}` + hc.Spec.TuningPolicy = v1beta1.HyperConvergedAnnotationTuningPolicy + + tests.UpdateHCORetry(ctx, cli, hc) + + expected := kvv1.TokenBucketRateLimiter{ + Burst: 200, + QPS: 100, + } + + checkTuningPolicy(cli, expected) + }) + + It("should update KV with the highBurst tuningPolicy", func() { + hc := tests.GetHCO(ctx, cli) + + delete(hc.Annotations, common.TuningPolicyAnnotationName) + hc.Spec.TuningPolicy = v1beta1.HyperConvergedHighBurstProfile + + tests.UpdateHCORetry(ctx, cli, hc) + + expected := kvv1.TokenBucketRateLimiter{ + Burst: 400, + QPS: 200, + } + + checkTuningPolicy(cli, expected) + }) +}) + +func checkTuningPolicy(cli kubecli.KubevirtClient, expected kvv1.TokenBucketRateLimiter) { + Eventually(func(g Gomega) { + kv, err := cli.KubeVirt(flags.KubeVirtInstallNamespace).Get("kubevirt-kubevirt-hyperconverged", &metav1.GetOptions{}) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(kv).ShouldNot(BeNil()) + g.Expect(kv.Spec.Configuration).ShouldNot(BeNil()) + + checkReloadableComponentConfiguration(g, kv.Spec.Configuration.APIConfiguration, expected) + checkReloadableComponentConfiguration(g, kv.Spec.Configuration.ControllerConfiguration, expected) + checkReloadableComponentConfiguration(g, kv.Spec.Configuration.HandlerConfiguration, expected) + checkReloadableComponentConfiguration(g, kv.Spec.Configuration.WebhookConfiguration, expected) + }).WithTimeout(time.Minute * 2).WithPolling(time.Second).Should(Succeed()) + +} + +func checkReloadableComponentConfiguration(g Gomega, actual *kvv1.ReloadableComponentConfiguration, expected kvv1.TokenBucketRateLimiter) { + g.ExpectWithOffset(1, actual).ShouldNot(BeNil()) + g.ExpectWithOffset(1, actual.RestClient).ShouldNot(BeNil()) + g.ExpectWithOffset(1, actual.RestClient.RateLimiter).ShouldNot(BeNil()) + g.ExpectWithOffset(1, actual.RestClient.RateLimiter.TokenBucketRateLimiter).Should(HaveValue(Equal(expected))) +} diff --git a/tests/vendor/github.com/kubevirt/hyperconverged-cluster-operator/controllers/common/consts.go b/tests/vendor/github.com/kubevirt/hyperconverged-cluster-operator/controllers/common/consts.go new file mode 100644 index 000000000..2dacc23f4 --- /dev/null +++ b/tests/vendor/github.com/kubevirt/hyperconverged-cluster-operator/controllers/common/consts.go @@ -0,0 +1,14 @@ +package common + +const ( + ReconcileCompleted = "ReconcileCompleted" + ReconcileCompletedMessage = "Reconcile completed successfully" + + // JSONPatch annotation names + JSONPatchKVAnnotationName = "kubevirt.kubevirt.io/jsonpatch" + JSONPatchCDIAnnotationName = "containerizeddataimporter.kubevirt.io/jsonpatch" + JSONPatchCNAOAnnotationName = "networkaddonsconfigs.kubevirt.io/jsonpatch" + JSONPatchSSPAnnotationName = "ssp.kubevirt.io/jsonpatch" + // Tuning Policy annotation name + TuningPolicyAnnotationName = "hco.kubevirt.io/tuningPolicy" +) diff --git a/tests/vendor/github.com/kubevirt/hyperconverged-cluster-operator/controllers/common/hcoConditions.go b/tests/vendor/github.com/kubevirt/hyperconverged-cluster-operator/controllers/common/hcoConditions.go new file mode 100644 index 000000000..e1df14950 --- /dev/null +++ b/tests/vendor/github.com/kubevirt/hyperconverged-cluster-operator/controllers/common/hcoConditions.go @@ -0,0 +1,66 @@ +package common + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + hcov1beta1 "github.com/kubevirt/hyperconverged-cluster-operator/api/v1beta1" +) + +var ( + HcoConditionTypes = []string{ + hcov1beta1.ConditionReconcileComplete, + hcov1beta1.ConditionAvailable, + hcov1beta1.ConditionProgressing, + hcov1beta1.ConditionDegraded, + hcov1beta1.ConditionUpgradeable, + } +) + +type HcoConditions map[string]metav1.Condition + +func NewHcoConditions() HcoConditions { + return HcoConditions{} +} + +func (hc HcoConditions) SetStatusCondition(newCondition metav1.Condition) { + existingCondition, exists := hc[newCondition.Type] + + if !exists { + hc[newCondition.Type] = newCondition + return + } + + if existingCondition.Status != newCondition.Status { + existingCondition.Status = newCondition.Status + } + + existingCondition.Reason = newCondition.Reason + existingCondition.Message = newCondition.Message + hc[newCondition.Type] = existingCondition +} + +func (hc HcoConditions) SetStatusConditionIfUnset(newCondition metav1.Condition) { + if !hc.HasCondition(newCondition.Type) { + hc.SetStatusCondition(newCondition) + } +} + +func (hc HcoConditions) IsEmpty() bool { + return len(hc) == 0 +} + +func (hc HcoConditions) HasCondition(conditionType string) bool { + _, exists := hc[conditionType] + + return exists +} + +func (hc HcoConditions) GetCondition(conditionType string) (metav1.Condition, bool) { + cond, found := hc[conditionType] + return cond, found +} + +func (hc HcoConditions) IsStatusConditionTrue(conditionType string) bool { + cond, found := hc[conditionType] + return found && cond.Status == metav1.ConditionTrue +} diff --git a/tests/vendor/github.com/kubevirt/hyperconverged-cluster-operator/controllers/common/hcoRequest.go b/tests/vendor/github.com/kubevirt/hyperconverged-cluster-operator/controllers/common/hcoRequest.go new file mode 100644 index 000000000..834a5c456 --- /dev/null +++ b/tests/vendor/github.com/kubevirt/hyperconverged-cluster-operator/controllers/common/hcoRequest.go @@ -0,0 +1,45 @@ +package common + +import ( + "context" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + hcov1beta1 "github.com/kubevirt/hyperconverged-cluster-operator/api/v1beta1" +) + +// hcoRequest - gather data for a specific request +type HcoRequest struct { + reconcile.Request // inheritance of operator request + Logger logr.Logger // request logger + Conditions HcoConditions // in-memory conditions + Ctx context.Context // context of this request, to be use for any other call + Instance *hcov1beta1.HyperConverged // the current state of the CR, as read from K8s + UpgradeMode bool // copy of the reconciler upgrade mode + ComponentUpgradeInProgress bool // if in upgrade mode, accumulate the component upgrade status + Dirty bool // is something was changed in the CR + StatusDirty bool // is something was changed in the CR's Status + HCOTriggered bool // if the request got triggered by a direct modification on HCO CR + Upgradeable bool // if all the operands are upgradeable +} + +func NewHcoRequest(ctx context.Context, request reconcile.Request, log logr.Logger, upgradeMode, hcoTriggered bool) *HcoRequest { + return &HcoRequest{ + Request: request, + Logger: log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name), + Conditions: NewHcoConditions(), + Ctx: ctx, + UpgradeMode: upgradeMode, + ComponentUpgradeInProgress: upgradeMode, + Dirty: false, + StatusDirty: false, + HCOTriggered: hcoTriggered, + Upgradeable: true, + } +} + +func (req *HcoRequest) SetUpgradeMode(upgradeMode bool) { + req.UpgradeMode = upgradeMode + req.ComponentUpgradeInProgress = upgradeMode +} diff --git a/tests/vendor/modules.txt b/tests/vendor/modules.txt index e0d6e30da..fc22dbc76 100644 --- a/tests/vendor/modules.txt +++ b/tests/vendor/modules.txt @@ -143,6 +143,7 @@ github.com/kubevirt/cluster-network-addons-operator/pkg/apis/networkaddonsoperat # github.com/kubevirt/hyperconverged-cluster-operator v1.8.0 => ../ ## explicit; go 1.19 github.com/kubevirt/hyperconverged-cluster-operator/api/v1beta1 +github.com/kubevirt/hyperconverged-cluster-operator/controllers/common github.com/kubevirt/hyperconverged-cluster-operator/pkg/util # github.com/mailru/easyjson v0.7.7 ## explicit; go 1.12 @@ -1000,6 +1001,7 @@ sigs.k8s.io/controller-runtime/pkg/healthz sigs.k8s.io/controller-runtime/pkg/internal/log sigs.k8s.io/controller-runtime/pkg/internal/objectutil sigs.k8s.io/controller-runtime/pkg/log +sigs.k8s.io/controller-runtime/pkg/reconcile sigs.k8s.io/controller-runtime/pkg/scheme # sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd ## explicit; go 1.18 diff --git a/tests/vendor/sigs.k8s.io/controller-runtime/pkg/reconcile/doc.go b/tests/vendor/sigs.k8s.io/controller-runtime/pkg/reconcile/doc.go new file mode 100644 index 000000000..d221dd7b3 --- /dev/null +++ b/tests/vendor/sigs.k8s.io/controller-runtime/pkg/reconcile/doc.go @@ -0,0 +1,21 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package reconcile defines the Reconciler interface to implement Kubernetes APIs. Reconciler is provided +to Controllers at creation time as the API implementation. +*/ +package reconcile diff --git a/tests/vendor/sigs.k8s.io/controller-runtime/pkg/reconcile/reconcile.go b/tests/vendor/sigs.k8s.io/controller-runtime/pkg/reconcile/reconcile.go new file mode 100644 index 000000000..8285e2ca9 --- /dev/null +++ b/tests/vendor/sigs.k8s.io/controller-runtime/pkg/reconcile/reconcile.go @@ -0,0 +1,102 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconcile + +import ( + "context" + "time" + + "k8s.io/apimachinery/pkg/types" +) + +// Result contains the result of a Reconciler invocation. +type Result struct { + // Requeue tells the Controller to requeue the reconcile key. Defaults to false. + Requeue bool + + // RequeueAfter if greater than 0, tells the Controller to requeue the reconcile key after the Duration. + // Implies that Requeue is true, there is no need to set Requeue to true at the same time as RequeueAfter. + RequeueAfter time.Duration +} + +// IsZero returns true if this result is empty. +func (r *Result) IsZero() bool { + if r == nil { + return true + } + return *r == Result{} +} + +// Request contains the information necessary to reconcile a Kubernetes object. This includes the +// information to uniquely identify the object - its Name and Namespace. It does NOT contain information about +// any specific Event or the object contents itself. +type Request struct { + // NamespacedName is the name and namespace of the object to reconcile. + types.NamespacedName +} + +/* +Reconciler implements a Kubernetes API for a specific Resource by Creating, Updating or Deleting Kubernetes +objects, or by making changes to systems external to the cluster (e.g. cloudproviders, github, etc). + +reconcile implementations compare the state specified in an object by a user against the actual cluster state, +and then perform operations to make the actual cluster state reflect the state specified by the user. + +Typically, reconcile is triggered by a Controller in response to cluster Events (e.g. Creating, Updating, +Deleting Kubernetes objects) or external Events (GitHub Webhooks, polling external sources, etc). + +Example reconcile Logic: + +* Read an object and all the Pods it owns. +* Observe that the object spec specifies 5 replicas but actual cluster contains only 1 Pod replica. +* Create 4 Pods and set their OwnerReferences to the object. + +reconcile may be implemented as either a type: + + type reconciler struct {} + + func (reconciler) Reconcile(ctx context.Context, o reconcile.Request) (reconcile.Result, error) { + // Implement business logic of reading and writing objects here + return reconcile.Result{}, nil + } + +Or as a function: + + reconcile.Func(func(ctx context.Context, o reconcile.Request) (reconcile.Result, error) { + // Implement business logic of reading and writing objects here + return reconcile.Result{}, nil + }) + +Reconciliation is level-based, meaning action isn't driven off changes in individual Events, but instead is +driven by actual cluster state read from the apiserver or a local cache. +For example if responding to a Pod Delete Event, the Request won't contain that a Pod was deleted, +instead the reconcile function observes this when reading the cluster state and seeing the Pod as missing. +*/ +type Reconciler interface { + // Reconcile performs a full reconciliation for the object referred to by the Request. + // The Controller will requeue the Request to be processed again if an error is non-nil or + // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. + Reconcile(context.Context, Request) (Result, error) +} + +// Func is a function that implements the reconcile interface. +type Func func(context.Context, Request) (Result, error) + +var _ Reconciler = Func(nil) + +// Reconcile implements Reconciler. +func (r Func) Reconcile(ctx context.Context, o Request) (Result, error) { return r(ctx, o) }