Metadata Profile PoC

manifests/autotune/metadata-profiles/cluster-metadata-local-monitoring.json

@@ -0,0 +1,47 @@
+{
+  "apiVersion": "recommender.com/v1",
+  "kind": "KruizeMetadataProfile",
+  "metadata": {
+    "name": "cluster-metadata-local-monitoring"
+  },
+  "profile_version": 1,
+  "k8s_type": "openshift",
+  "query_variables": [
+    {
+      "name": "namespacesAcrossCluster",
+      "datasource": "prometheus",
+      "value_type": "double",
+      "kubernetes_object": "container",
+      "aggregation_functions": [
+        {
+          "function": "sum",
+          "query": "sum by (namespace) (avg_over_time(kube_namespace_status_phase{namespace!=\"\"}[$MEASUREMENT_DURATION_IN_MIN$m]))"
+        }
+      ]
+    },
+    {
+      "name": "workloadsAcrossCluster",
+      "datasource": "prometheus",
+      "value_type": "double",
+      "kubernetes_object": "container",
+      "aggregation_functions": [
+        {
+          "function": "sum",
+          "query": "sum by (namespace, workload, workload_type) (avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{workload!=\"\"}[$MEASUREMENT_DURATION_IN_MIN$m]))"
+        }
+      ]
+    },
+    {
+      "name": "containersAcrossCluster",
+      "datasource": "prometheus",
+      "value_type": "double",
+      "kubernetes_object": "container",
+      "aggregation_functions": [
+        {
+          "function": "sum",
+          "query": "sum by (container, image, workload, workload_type, namespace) (avg_over_time(kube_pod_container_info{container!=\"\"}[$MEASUREMENT_DURATION_IN_MIN$m]) * on (pod, namespace) group_left(workload, workload_type) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{workload!=\"\"}[$MEASUREMENT_DURATION_IN_MIN$m]))"
+        }
+      ]
+    }
+  ]
+}

manifests/autotune/metadata-profiles/kruize-metadata-profile-crd.yaml

@@ -0,0 +1,81 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  #name must match the spec fields below, and be in the form: <plural>.<group>
+  name: kruizemetadataprofiles.recommender.com
+spec:
+  # group name to use for REST API: /apis/<group>/<version>
+  group: "recommender.com"
+  names:
+    plural: kruizemetadataprofiles
+    singular: kruizemetadataprofile
+    #types can be identified with this tag
+    kind: KruizeMetadataProfile
+  scope: Namespaced
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/
+            community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/
+            community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            profile_version:
+              description: 'Version of the metadata profile'
+              type: number
+            k8s_type:
+              description: 'minikube or openshift'
+              type: string
+            query_variables:
+              description: 'Query variables to be used'
+              type: array
+              items:
+                type: object
+                properties:
+                  name:
+                    description: 'name of the variable'
+                    type: string
+                  datasource:
+                    description: 'datasource of the query'
+                    type: string
+                  value_type:
+                    description: 'can be double or integer'
+                    type: string
+                  kubernetes_object:
+                    description: 'k8s object that this query is tied to: "deployment", "pod" or "container"'
+                    type: string
+                  aggregation_functions:
+                    description: 'one of the query or aggregation_functions is mandatory'
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        function:
+                          description: 'aggregate functions associated with this variable'
+                          type: string
+                        query:
+                          description: 'query'
+                          type: string
+                        versions:
+                          description: 'Any specific versions that this query is tied to'
+                          type: string
+                      required:
+                        - function
+                        - query
+                required:
+                  - name
+                  - datasource
+          required:
+            - query_variables

manifests/autotune/metadata-profiles/kruize-metadata-profile-example.yaml

@@ -0,0 +1,82 @@
+apiVersion: "recommender.com/v1"
+kind: "KruizeMetadataProfile"
+metadata:
+  name: "cluster-metadata-local-monitoring"
+profile_version: 1.0
+k8s_type: openshift
+query_variables:
+
+- name: 'namespacesAcrossCluster'
+  datasource: prometheus
+  value_type: "double"
+  kubernetes_object: "container" # or namespace
+  aggregation_functions:
+  - function: sum
+    query: 'sum by (namespace) (avg_over_time(kube_namespace_status_phase{namespace!=""}[$MEASUREMENT_DURATION_IN_MIN$m]))'
+
+- name: 'namespacesForOrgAndClusterId'
+  datasource: prometheus
+  value_type: "double"
+  kubernetes_object: "container" # or namespace
+  aggregation_functions:
+  - function: sum
+    query: 'sum by (namespace) (avg_over_time(kube_namespace_status_phase{namespace!="", org_id="$ORG_ID$", cluster_id="$CLUSTER_ID$"}[$MEASUREMENT_DURATION_IN_MIN$m]))'
+
+- name: 'namespacesForAdditionalLabel' # or 'namespacesForCustomLabel'
+  datasource: prometheus
+  value_type: "double"
+  kubernetes_object: "container" # or namespace
+  aggregation_functions:
+  - function: sum
+    query: 'sum by (namespace) (avg_over_time(kube_namespace_status_phase{namespace!=""}[$MEASUREMENT_DURATION_IN_MIN$m]))'
+
+- name: 'workloadsAcrossCluster'
+  datasource: prometheus
+  value_type: "double"
+  kubernetes_object: "container"
+  aggregation_functions:
+  - function: sum
+    query: 'sum by (namespace, workload, workload_type) (avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{workload!=""}[$MEASUREMENT_DURATION_IN_MIN$m]))'
+
+- name: 'workloadsForOrgAndClusterId'
+  datasource: prometheus
+  value_type: "double"
+  kubernetes_object: "container"
+  aggregation_functions:
+  - function: sum
+    query: 'sum by (namespace, workload, workload_type) (avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{workload!="", org_id="$ORG_ID$", cluster_id="$CLUSTER_ID$"}[$MEASUREMENT_DURATION_IN_MIN$m]))'
+
+- name: 'workloadsForAdditionalLabel' # or 'workloadsForCustomLabel'
+  datasource: prometheus
+  value_type: "double"
+  kubernetes_object: "container"
+  aggregation_functions:
+  - function: sum
+    query: 'sum by (namespace, workload, workload_type) (avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{workload!="" ADDITIONAL_LABEL}[$MEASUREMENT_DURATION_IN_MIN$m]))'
+
+
+- name: 'containersAcrossCluster'
+  datasource: prometheus
+  value_type: "double"
+  kubernetes_object: "container"
+  aggregation_functions:
+  - function: sum
+    query: 'sum by (container, image, workload, workload_type, namespace) (avg_over_time(kube_pod_container_info{container!=""}[$MEASUREMENT_DURATION_IN_MIN$m]) * on (pod, namespace) group_left(workload, workload_type) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{workload!=""}[$MEASUREMENT_DURATION_IN_MIN$m]))'
+
+
+- name: 'containersForOrgAndClusterId'
+  datasource: prometheus
+  value_type: "double"
+  kubernetes_object: "container"
+  aggregation_functions:
+  - function: sum
+    query: 'sum by (container, image, workload, workload_type, namespace) (avg_over_time(kube_pod_container_info{container!="", org_id="$ORG_ID$", cluster_id="$CLUSTER_ID$"}[$MEASUREMENT_DURATION_IN_MIN$m]) * on (pod, namespace) group_left(workload, workload_type) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{workload!="", org_id="$ORG_ID$", cluster_id="$CLUSTER_ID$"}[$MEASUREMENT_DURATION_IN_MIN$m]))'
+
+# or 'containersForCustomLabel'
+- name: 'containersForAdditionalLabel'
+  datasource: prometheus
+  value_type: "double"
+  kubernetes_object: "container"
+  aggregation_functions:
+  - function: sum
+    query: 'sum by (container, image, workload, workload_type, namespace) (avg_over_time(kube_pod_container_info{container!="" ADDITIONAL_LABEL}[$MEASUREMENT_DURATION_IN_MIN$m]) * on (pod, namespace) group_left(workload, workload_type) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{workload!="" ADDITIONAL_LABEL}[$MEASUREMENT_DURATION_IN_MIN$m]))'

manifests/autotune/metadata-profiles/kruize-metadata-profile.yaml

@@ -0,0 +1,31 @@
+apiVersion: "recommender.com/v1"
+kind: "KruizeMetadataProfile"
+metadata:
+  name: "cluster-metadata-local-monitoring"
+profile_version: 1.0
+k8s_type: openshift
+query_variables:
+
+- name: 'namespacesAcrossCluster'
+  datasource: prometheus
+  value_type: "double"
+  kubernetes_object: "container" # or namespace
+  aggregation_functions:
+  - function: sum
+    query: 'sum by (namespace) (avg_over_time(kube_namespace_status_phase{namespace!=""}[$MEASUREMENT_DURATION_IN_MIN$d]))'
+
+- name: 'workloadsAcrossCluster'
+  datasource: prometheus
+  value_type: "double"
+  kubernetes_object: "container"
+  aggregation_functions:
+  - function: sum
+    query: 'sum by (namespace, workload, workload_type) (avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{workload!=""}[$MEASUREMENT_DURATION_IN_MIN$d]))'
+
+- name: 'containersAcrossCluster'
+  datasource: prometheus
+  value_type: "double"
+  kubernetes_object: "container"
+  aggregation_functions:
+  - function: sum
+    query: 'sum by (container, image, workload, workload_type, namespace) (avg_over_time(kube_pod_container_info{container!=""}[$MEASUREMENT_DURATION_IN_MIN$d]) * on (pod, namespace) group_left(workload, workload_type) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{workload!=""}[$MEASUREMENT_DURATION_IN_MIN$d]))'

migrations/kruize_local_ddl.sql

@@ -1,6 +1,7 @@
 create table IF NOT EXISTS kruize_authentication (id serial, authentication_type varchar(255), credentials jsonb, service_type varchar(255), primary key (id));
 create table IF NOT EXISTS kruize_datasources (version varchar(255), name varchar(255), provider varchar(255), serviceName varchar(255), namespace varchar(255), url varchar(255), authentication_id serial, FOREIGN KEY (authentication_id) REFERENCES kruize_authentication(id), primary key (name));
 create table IF NOT EXISTS kruize_dsmetadata (id serial, version varchar(255), datasource_name varchar(255), cluster_name varchar(255), namespace varchar(255), workload_type varchar(255), workload_name varchar(255), container_name varchar(255), container_image_name varchar(255), primary key (id));
-alter table kruize_experiments add column experiment_type varchar(255), add column metadata_id bigint references kruize_dsmetadata(id), alter column datasource type varchar(255);
+alter table kruize_experiments add column experiment_type varchar(255), add column metadata_id bigint references kruize_dsmetadata(id), alter column datasource type varchar(255), add column metadata_profile varchar(255);
 create table IF NOT EXISTS kruize_metric_profiles (api_version varchar(255), kind varchar(255), metadata jsonb, name varchar(255) not null, k8s_type varchar(255), profile_version float(53) not null, slo jsonb, primary key (name));
 alter table kruize_recommendations add column experiment_type varchar(255);
+create table IF NOT EXISTS kruize_metadata_profiles (api_version varchar(255), kind varchar(255), metadata jsonb, name varchar(255) not null, k8s_type varchar(255), profile_version float(53) not null, query_variables jsonb, primary key (name));

src/main/java/com/autotune/Autotune.java

@@ -20,6 +20,7 @@
 import com.autotune.analyzer.exceptions.KruizeErrorHandler;
 import com.autotune.analyzer.exceptions.MonitoringAgentNotFoundException;
 import com.autotune.analyzer.exceptions.MonitoringAgentNotSupportedException;
+import com.autotune.analyzer.metadataProfiles.MetadataProfileCollection;
 import com.autotune.analyzer.performanceProfiles.MetricProfileCollection;
 import com.autotune.analyzer.utils.AnalyzerConstants;
 import com.autotune.common.datasource.DataSourceCollection;
@@ -132,6 +133,8 @@ public static void main(String[] args) {
                 checkAvailableDataSources();
                 // load available metric profiles from db
                 loadMetricProfilesFromDB();
+                // load available metadata profiles from db
+                loadMetadataProfilesFromDB();
 
             }
             // close the existing session factory before recreating
@@ -221,6 +224,14 @@ private static void loadMetricProfilesFromDB() {
         metricProfileCollection.loadMetricProfilesFromDB();
     }
 
+    /**
+     * loads metadata profiles from database
+     */
+    private static void loadMetadataProfilesFromDB() {
+        MetadataProfileCollection metadataProfileCollection = MetadataProfileCollection.getInstance();
+        metadataProfileCollection.loadMetadataProfilesFromDB();
+    }
+
     private static void addAutotuneServlets(ServletContextHandler context) {
         context.addServlet(HealthService.class, HEALTH_SERVICE);
         // Start the Prometheus end point (/metrics) for Autotune

src/main/java/com/autotune/analyzer/Analyzer.java

@@ -56,6 +56,9 @@ public static void addServlets(ServletContextHandler context) {
         context.addServlet(MetricProfileService.class, ServerContext.CREATE_METRIC_PROFILE);
         context.addServlet(MetricProfileService.class, ServerContext.LIST_METRIC_PROFILES);
         context.addServlet(MetricProfileService.class, ServerContext.DELETE_METRIC_PROFILE);
+        context.addServlet(MetadataProfileService.class, ServerContext.CREATE_METADATA_PROFILE);
+        context.addServlet(MetadataProfileService.class, ServerContext.LIST_METADATA_PROFILES);
+        context.addServlet(MetadataProfileService.class, ServerContext.DELETE_METADATA_PROFILE);
         context.addServlet(ListDatasources.class, ServerContext.LIST_DATASOURCES);
         context.addServlet(DSMetadataService.class, ServerContext.DATASOURCE_METADATA);
         context.addServlet(BulkService.class, ServerContext.BULK_SERVICE);

src/main/java/com/autotune/analyzer/exceptions/MetadataProfileResponse.java

@@ -0,0 +1,48 @@
+package com.autotune.analyzer.exceptions;
+
+public class MetadataProfileResponse {
+    private String message;
+    private int httpcode;
+    private String documentationLink;
+    private String status;
+
+    public MetadataProfileResponse(String message, int httpcode, String documentationLink, String status) {
+        this.message = message;
+        this.httpcode = httpcode;
+        this.documentationLink = documentationLink;
+        this.status = status;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+
+    public int getHttpcode() {
+        return httpcode;
+    }
+
+    public void setHttpcode(int httpcode) {
+        this.httpcode = httpcode;
+    }
+
+    public String getDocumentationLink() {
+        return documentationLink;
+    }
+
+    public void setDocumentationLink(String documentationLink) {
+        this.documentationLink = documentationLink;
+    }
+
+    public String getStatus() {
+        return status;
+    }
+
+    public void setStatus(String status) {
+        this.status = status;
+    }
+}
+