configurable qos resources for containers managed by cluster-manager …

…and klusterlet Signed-off-by: Dong Beiqing <[email protected]>
open-cluster-management-io · Jan 23, 2024 · fa5140e · fa5140e
1 parent 6cfce8c
commit fa5140e
Show file tree

Hide file tree

Showing 39 changed files with 863 additions and 1,283 deletions.
diff --git a/...-manager/config/crds/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml b/...-manager/config/crds/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml
@@ -259,6 +259,69 @@ spec:
                 description: RegistrationImagePullSpec represents the desired image
                   of registration controller/webhook installed on hub.
                 type: string
+              resourceRequirement:
+                description: ResourceRequirement specify QoS classes of deployments
+                  managed by clustermanager. It applies to all the containers in the
+                  deployments.
+                properties:
+                  resourceRequirements:
+                    description: ResourceRequirements defines resource requests and
+                      limits when Type is ResourceQosClassResourceRequirement
+                    properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable. It can only be
+                          set for containers."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. Requests cannot exceed
+                          Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
+                  type:
+                    default: Default
+                    enum:
+                    - Default
+                    - BestEffort
+                    - ResourceRequirement
+                    type: string
+                type: object
               workConfiguration:
                 description: WorkConfiguration contains the configuration of work
                 properties:

diff --git a/...-manager/olm-catalog/cluster-manager/manifests/cluster-manager.clusterserviceversion.yaml b/...-manager/olm-catalog/cluster-manager/manifests/cluster-manager.clusterserviceversion.yaml
@@ -59,7 +59,7 @@ metadata:
     categories: Integration & Delivery,OpenShift Optional
     certified: "false"
     containerImage: quay.io/open-cluster-management/registration-operator:latest
-    createdAt: "2024-01-15T16:05:25Z"
+    createdAt: "2024-01-23T02:00:17Z"
     description: Manages the installation and upgrade of the ClusterManager.
     operators.operatorframework.io/builder: operator-sdk-v1.32.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3

diff --git a/...atalog/cluster-manager/manifests/operator.open-cluster-management.io_clustermanagers.yaml b/...atalog/cluster-manager/manifests/operator.open-cluster-management.io_clustermanagers.yaml
@@ -259,6 +259,69 @@ spec:
                 description: RegistrationImagePullSpec represents the desired image
                   of registration controller/webhook installed on hub.
                 type: string
+              resourceRequirement:
+                description: ResourceRequirement specify QoS classes of deployments
+                  managed by clustermanager. It applies to all the containers in the
+                  deployments.
+                properties:
+                  resourceRequirements:
+                    description: ResourceRequirements defines resource requests and
+                      limits when Type is ResourceQosClassResourceRequirement
+                    properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable. It can only be
+                          set for containers."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. Requests cannot exceed
+                          Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
+                  type:
+                    default: Default
+                    enum:
+                    - Default
+                    - BestEffort
+                    - ResourceRequirement
+                    type: string
+                type: object
               workConfiguration:
                 description: WorkConfiguration contains the configuration of work
                 properties:

diff --git a/...y/klusterlet/config/crds/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml b/...y/klusterlet/config/crds/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml
@@ -244,14 +244,65 @@ spec:
                   will be used if unspecified.
                 type: string
               resourceRequirement:
-                description: ResourceRequirement specify QoS classes of klusterlet
-                  deployment
+                description: ResourceRequirement specify QoS classes of deployments
+                  managed by klusterlet. It applies to all the containers in the deployments.
                 properties:
+                  resourceRequirements:
+                    description: ResourceRequirements defines resource requests and
+                      limits when Type is ResourceQosClassResourceRequirement
+                    properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable. It can only be
+                          set for containers."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. Requests cannot exceed
+                          Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
                   type:
                     default: Default
                     enum:
                     - Default
                     - BestEffort
+                    - ResourceRequirement
                     type: string
                 type: object
               workConfiguration:

diff --git a/deploy/klusterlet/olm-catalog/klusterlet/manifests/klusterlet.clusterserviceversion.yaml b/deploy/klusterlet/olm-catalog/klusterlet/manifests/klusterlet.clusterserviceversion.yaml
@@ -31,7 +31,7 @@ metadata:
     categories: Integration & Delivery,OpenShift Optional
     certified: "false"
     containerImage: quay.io/open-cluster-management/registration-operator:latest
-    createdAt: "2024-01-15T16:05:25Z"
+    createdAt: "2024-01-23T02:00:17Z"
     description: Manages the installation and upgrade of the Klusterlet.
     operators.operatorframework.io/builder: operator-sdk-v1.32.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3

diff --git a/...let/olm-catalog/klusterlet/manifests/operator.open-cluster-management.io_klusterlets.yaml b/...let/olm-catalog/klusterlet/manifests/operator.open-cluster-management.io_klusterlets.yaml
@@ -244,14 +244,65 @@ spec:
                   will be used if unspecified.
                 type: string
               resourceRequirement:
-                description: ResourceRequirement specify QoS classes of klusterlet
-                  deployment
+                description: ResourceRequirement specify QoS classes of deployments
+                  managed by klusterlet. It applies to all the containers in the deployments.
                 properties:
+                  resourceRequirements:
+                    description: ResourceRequirements defines resource requests and
+                      limits when Type is ResourceQosClassResourceRequirement
+                    properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable. It can only be
+                          set for containers."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. Requests cannot exceed
+                          Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
                   type:
                     default: Default
                     enum:
                     - Default
                     - BestEffort
+                    - ResourceRequirement
                     type: string
                 type: object
               workConfiguration:

diff --git a/go.mod b/go.mod
@@ -34,7 +34,7 @@ require (
 	k8s.io/kube-aggregator v0.29.0
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	open-cluster-management.io/addon-framework v0.8.1-0.20240119025526-d2afcef1ff66
-	open-cluster-management.io/api v0.12.1-0.20240115071352-3d94ce8f3499
+	open-cluster-management.io/api v0.12.1-0.20240122084346-e7bd1bd9ea6a
 	open-cluster-management.io/sdk-go v0.0.0-20240118073603-6da392d39be0
 	sigs.k8s.io/controller-runtime v0.16.2
 	sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96

diff --git a/go.sum b/go.sum
@@ -452,8 +452,8 @@ k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCf
 k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 open-cluster-management.io/addon-framework v0.8.1-0.20240119025526-d2afcef1ff66 h1:d6vkhLLdam7/DLmZ4onPSbsRNHP3Rze0/0OkmjJgVjk=
 open-cluster-management.io/addon-framework v0.8.1-0.20240119025526-d2afcef1ff66/go.mod h1:aj97pgpGJ0/LpQzBVtU2oDFqqIiZLOPnsjLKG/sVkFw=
-open-cluster-management.io/api v0.12.1-0.20240115071352-3d94ce8f3499 h1:Odh200i57Z9V44eJak98VqdKMCAlj3m9dtnUp8285nE=
-open-cluster-management.io/api v0.12.1-0.20240115071352-3d94ce8f3499/go.mod h1:K3Rza3qN/W1+E1a+wbLtFatvdI8UlQWkSqBlpeRHMPw=
+open-cluster-management.io/api v0.12.1-0.20240122084346-e7bd1bd9ea6a h1:NjIU3aN4JSJjTotHiOkOCqYaPGG2tNtm7BY/o9uPb8M=
+open-cluster-management.io/api v0.12.1-0.20240122084346-e7bd1bd9ea6a/go.mod h1:vOz9InrJq1BDFEI51+OwAyq2M3tjYPY+1cnoQhMhIlE=
 open-cluster-management.io/sdk-go v0.0.0-20240118073603-6da392d39be0 h1:FCGRz0ZdCWCq6WebGJyr0XJfAsjEwYFJF6lXY4tBPFM=
 open-cluster-management.io/sdk-go v0.0.0-20240118073603-6da392d39be0/go.mod h1:LfUmo1hhhlQdui8CiNO8aUq050dbJTQ2nrE97DRFREs=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I=

diff --git a/manifests/cluster-manager/management/cluster-manager-placement-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-placement-deployment.yaml
@@ -74,10 +74,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp

diff --git a/manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml
@@ -82,10 +82,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp