add metadata feilds including labels, annotations, uid and image (#1120)

* add metadata feilds including labels, annotations, uid and image

* set the include_fields if customer set it otherwise go with default if enabled

build/common/installer/scripts/tomlparser.rb

@@ -25,6 +25,7 @@
 @adxDatabaseName = "containerinsights" # default for all configurations
 @logEnableMultiline = "false"
 @logEnableKubernetesMetadata = false
+@logKubernetesMetadataiIncludeFields = "podLabels,podAnnotations,podUid,image"
 if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
   @containerLogsRoute = "v1" # default is v1 for windows until windows agent integrates windows ama
   # This path format is necessary for fluent-bit in windows
@@ -211,6 +212,8 @@ def populateSettingValuesFromConfigMap(parsedConfig)
     begin
       if !parsedConfig[:log_collection_settings][:metadata_collection].nil? && !parsedConfig[:log_collection_settings][:metadata_collection][:enabled].nil?
         @logEnableKubernetesMetadata = parsedConfig[:log_collection_settings][:metadata_collection][:enabled]
+        if !parsedConfig[:log_collection_settings][:metadata_collection][:include_fields].nil?
+          @logKubernetesMetadataiIncludeFields = parsedConfig[:log_collection_settings][:metadata_collection][:include_fields]
         puts "config::Using config map setting for kubernetes metadata"
       end
     end
@@ -262,6 +265,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   file.write("export AZMON_ADX_DATABASE_NAME=#{@adxDatabaseName}\n")
   file.write("export AZMON_MULTILINE_ENABLED=#{@logEnableMultiline}\n")
   file.write("export AZMON_KUBERNETES_METADATA_ENABLED=#{@logEnableKubernetesMetadata}\n")
+  file.write("export AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS=#{@logKubernetesMetadataiIncludeFields}\n")
   # Close file after writing all environment variables
   file.close
   puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' "
@@ -328,6 +332,8 @@ def get_command_windows(env_variable_name, env_variable_value)
     file.write(commands)
     commands = get_command_windows("AZMON_KUBERNETES_METADATA_ENABLED ", @logEnableKubernetesMetadata)
     file.write(commands)
+    commands = get_command_windows("AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS", @logKubernetesMetadataiIncludeFields)
+    file.write(commands)
     # Close file after writing all environment variables
     file.close
     puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' "

kubernetes/container-azm-ms-agentconfig.yaml

@@ -54,7 +54,7 @@ data:
        #[log_collection_settings.metadata_collection]
           # if enabled will collect kubernetes metadata for ContainerLogv2 schema. Default false
           # enabled = false
-
+          # include_fields = ["podLabels","podAnnotations","podUid","image"]
 
   prometheus-data-collection-settings: |-
     # Custom Prometheus metrics data collection settings

source/plugins/ruby/CAdvisorMetricsAPIClient.rb

@@ -28,6 +28,7 @@ class CAdvisorMetricsAPIClient
   @clusterContainerLogSchemaVersion = ENV["AZMON_CONTAINER_LOG_SCHEMA_VERSION"]
   @clusterMultilineEnabled = ENV["AZMON_MULTILINE_ENABLED"]
   @clusterKubernetesMetadataEnabled = ENV["AZMON_KUBERNETES_METADATA_ENABLED"]
+  @clusterKubernetesMetadataiIncludeFields = ENV["AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS"]
 
   @dsPromInterval = ENV["TELEMETRY_DS_PROM_INTERVAL"]
   @dsPromFieldPassCount = ENV["TELEMETRY_DS_PROM_FIELDPASS_LENGTH"]
@@ -300,6 +301,9 @@ def getContainerCpuMetricItems(metricJSON, hostName, cpuMetricNameToCollect, met
                     if (!@clusterKubernetesMetadataEnabled.nil? && !@clusterKubernetesMetadataEnabled.empty?)
                       telemetryProps["metadataEnabled"] = @clusterKubernetesMetadataEnabled
                     end
+                    if (!@clusterKubernetesMetadataiIncludeFields.nil? && !@clusterKubernetesMetadataiIncludeFields.empty?)
+                      telemetryProps["metadataIncludeFields"] = @clusterKubernetesMetadataiIncludeFields
+                    end
                     ApplicationInsightsUtility.sendMetricTelemetry(metricNametoReturn, metricValue, telemetryProps)
                   end
                 end