Report Reserving queue metrics and stats. (#1206)

* [api/cq] Add flavor reservation values.

* [clusterqueue] Maintain different stats and metrics for Reserving and Admitted WL

* [api/lq] Add flavor reservation values.

* [localqueue] Maintain different stats for Reserving and Admitted WL

* Review Remarks

* Review Remarks

* Review remarks

apis/kueue/v1beta1/clusterqueue_types.go

@@ -186,19 +186,32 @@ type ResourceFlavorReference string
 
 // ClusterQueueStatus defines the observed state of ClusterQueue
 type ClusterQueueStatus struct {
-	// flavorsUsage are the used quotas, by flavor, currently in use by the
+	// flavorsReservation are the reserved quotas, by flavor, currently in use by the
 	// workloads assigned to this ClusterQueue.
 	// +listType=map
 	// +listMapKey=name
 	// +kubebuilder:validation:MaxItems=16
 	// +optional
+	FlavorsReservation []FlavorUsage `json:"flavorsReservation"`
+
+	// flavorsUsage are the used quotas, by flavor, currently in use by the
+	// workloads admitted in this ClusterQueue.
+	// +listType=map
+	// +listMapKey=name
+	// +kubebuilder:validation:MaxItems=16
+	// +optional
 	FlavorsUsage []FlavorUsage `json:"flavorsUsage"`
 
 	// pendingWorkloads is the number of workloads currently waiting to be
 	// admitted to this clusterQueue.
 	// +optional
 	PendingWorkloads int32 `json:"pendingWorkloads"`
 
+	// reservingWorkloads is the number of workloads currently reserving quota in this
+	// clusterQueue.
+	// +optional
+	ReservingWorkloads int32 `json:"reservingWorkloads"`
+
 	// admittedWorkloads is the number of workloads currently admitted to this
 	// clusterQueue and haven't finished yet.
 	// +optional

apis/kueue/v1beta1/localqueue_types.go

@@ -37,7 +37,12 @@ type LocalQueueStatus struct {
 	// +optional
 	PendingWorkloads int32 `json:"pendingWorkloads"`
 
-	// AdmittedWorkloads is the number of workloads in this LocalQueue
+	// reservingWorkloads is the number of workloads in this LocalQueue
+	// reserving quota in a ClusterQueue and that haven't finished yet.
+	// +optional
+	ReservingWorkloads int32 `json:"reservingWorkloads"`
+
+	// admittedWorkloads is the number of workloads in this LocalQueue
 	// admitted to a ClusterQueue and that haven't finished yet.
 	// +optional
 	AdmittedWorkloads int32 `json:"admittedWorkloads"`
@@ -51,7 +56,15 @@ type LocalQueueStatus struct {
 	// +patchMergeKey=type
 	Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
 
-	// flavorUsage are the used quotas, by flavor currently in use by the
+	// flavorsReservation are the reserved quotas, by flavor currently in use by the
+	// workloads assigned to this LocalQueue.
+	// +listType=map
+	// +listMapKey=name
+	// +kubebuilder:validation:MaxItems=16
+	// +optional
+	FlavorsReservation []LocalQueueFlavorUsage `json:"flavorsReservation"`
+
+	// flavorsUsage are the used quotas, by flavor currently in use by the
 	// workloads assigned to this LocalQueue.
 	// +listType=map
 	// +listMapKey=name

charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml

@@ -414,9 +414,59 @@ spec:
                 x-kubernetes-list-map-keys:
                 - type
                 x-kubernetes-list-type: map
+              flavorsReservation:
+                description: flavorsReservation are the reserved quotas, by flavor,
+                  currently in use by the workloads assigned to this ClusterQueue.
+                items:
+                  properties:
+                    name:
+                      description: name of the flavor.
+                      type: string
+                    resources:
+                      description: resources lists the quota usage for the resources
+                        in this flavor.
+                      items:
+                        properties:
+                          borrowed:
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            description: Borrowed is quantity of quota that is borrowed
+                              from the cohort. In other words, it's the used quota
+                              that is over the nominalQuota.
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
+                          name:
+                            description: name of the resource
+                            type: string
+                          total:
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            description: total is the total quantity of used quota,
+                              including the amount borrowed from the cohort.
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
+                        required:
+                        - name
+                        type: object
+                      maxItems: 16
+                      type: array
+                      x-kubernetes-list-map-keys:
+                      - name
+                      x-kubernetes-list-type: map
+                  required:
+                  - name
+                  - resources
+                  type: object
+                maxItems: 16
+                type: array
+                x-kubernetes-list-map-keys:
+                - name
+                x-kubernetes-list-type: map
               flavorsUsage:
                 description: flavorsUsage are the used quotas, by flavor, currently
-                  in use by the workloads assigned to this ClusterQueue.
+                  in use by the workloads admitted in this ClusterQueue.
                 items:
                   properties:
                     name:
@@ -501,6 +551,11 @@ spec:
                 required:
                 - lastChangeTime
                 type: object
+              reservingWorkloads:
+                description: reservingWorkloads is the number of workloads currently
+                  reserving quota in this clusterQueue.
+                format: int32
+                type: integer
             type: object
         type: object
     served: true

charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml

@@ -72,7 +72,7 @@ spec:
             description: LocalQueueStatus defines the observed state of LocalQueue
             properties:
               admittedWorkloads:
-                description: AdmittedWorkloads is the number of workloads in this
+                description: admittedWorkloads is the number of workloads in this
                   LocalQueue admitted to a ClusterQueue and that haven't finished
                   yet.
                 format: int32
@@ -151,7 +151,7 @@ spec:
                 - type
                 x-kubernetes-list-type: map
               flavorUsage:
-                description: flavorUsage are the used quotas, by flavor currently
+                description: flavorsUsage are the used quotas, by flavor currently
                   in use by the workloads assigned to this LocalQueue.
                 items:
                   properties:
@@ -190,11 +190,57 @@ spec:
                 x-kubernetes-list-map-keys:
                 - name
                 x-kubernetes-list-type: map
+              flavorsReservation:
+                description: flavorsReservation are the reserved quotas, by flavor
+                  currently in use by the workloads assigned to this LocalQueue.
+                items:
+                  properties:
+                    name:
+                      description: name of the flavor.
+                      type: string
+                    resources:
+                      description: resources lists the quota usage for the resources
+                        in this flavor.
+                      items:
+                        properties:
+                          name:
+                            description: name of the resource.
+                            type: string
+                          total:
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            description: total is the total quantity of used quota.
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
+                        required:
+                        - name
+                        type: object
+                      maxItems: 16
+                      type: array
+                      x-kubernetes-list-map-keys:
+                      - name
+                      x-kubernetes-list-type: map
+                  required:
+                  - name
+                  - resources
+                  type: object
+                maxItems: 16
+                type: array
+                x-kubernetes-list-map-keys:
+                - name
+                x-kubernetes-list-type: map
               pendingWorkloads:
                 description: PendingWorkloads is the number of Workloads in the LocalQueue
                   not yet admitted to a ClusterQueue
                 format: int32
                 type: integer
+              reservingWorkloads:
+                description: reservingWorkloads is the number of workloads in this
+                  LocalQueue reserving quota in a ClusterQueue and that haven't finished
+                  yet.
+                format: int32
+                type: integer
             type: object
         type: object
     served: true