make log filtering opt in explicit (#1140)

Co-authored-by: Amol Agrawal <[email protected]>

build/common/installer/scripts/fluent-bit-conf-customizer.rb

@@ -44,6 +44,7 @@ def substituteFluentBitPlaceHolders
     ignoreOlder = ENV["FBIT_TAIL_IGNORE_OLDER"]
     multilineLogging = ENV["AZMON_MULTILINE_ENABLED"]
     kubernetesMetadataCollection = ENV["AZMON_KUBERNETES_METADATA_ENABLED"]
+    annotationBasedLogFiltering = ENV["AZMON_ANNOTATION_BASED_LOG_FILTERING"]
 
     serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval
     serviceIntervalSetting = "Flush         " + serviceInterval
@@ -84,6 +85,12 @@ def substituteFluentBitPlaceHolders
       new_contents = new_contents.gsub("#${KubernetesFilterEnabled}", "")
     end
 
+    if !annotationBasedLogFiltering.nil? && annotationBasedLogFiltering.to_s.downcase == "true"
+      # enabled kubernetes fluent-bit plugin if not already enabled
+      new_contents = new_contents.gsub("#${KubernetesFilterEnabled}", "")
+      new_contents = new_contents.gsub("#${AnnotationBasedLogFilteringEnabled}", "")
+    end
+
     new_contents = substituteMultiline(multilineLogging, new_contents)
     File.open(@fluent_bit_conf_path, "w") { |file| file.puts new_contents }
     puts "config::Successfully substituted the placeholders in fluent-bit.conf file"

build/common/installer/scripts/tomlparser.rb

@@ -26,6 +26,7 @@
 @logEnableMultiline = "false"
 @logEnableKubernetesMetadata = false
 @logKubernetesMetadataiIncludeFields = "podLabels,podAnnotations,podUid,image"
+@annotationBasedLogFiltering = false
 if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
   @containerLogsRoute = "v1" # default is v1 for windows until windows agent integrates windows ama
   # This path format is necessary for fluent-bit in windows
@@ -219,6 +220,14 @@ def populateSettingValuesFromConfigMap(parsedConfig)
         end
       end
     end
+
+    #Get annotation based log filtering setting
+    begin
+      if !parsedConfig[:log_collection_settings][:filter_using_annotations].nil? && !parsedConfig[:log_collection_settings][:filter_using_annotations][:enabled].nil?
+        puts "config::Using config map setting for annotation based log filtering"
+        @annotationBasedLogFiltering = parsedConfig[:log_collection_settings][:filter_using_annotations][:enabled]
+      end
+    end
   end
 end
 
@@ -268,6 +277,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   file.write("export AZMON_MULTILINE_ENABLED=#{@logEnableMultiline}\n")
   file.write("export AZMON_KUBERNETES_METADATA_ENABLED=#{@logEnableKubernetesMetadata}\n")
   file.write("export AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS=#{@logKubernetesMetadataiIncludeFields}\n")
+  file.write("export AZMON_ANNOTATION_BASED_LOG_FILTERING=#{@annotationBasedLogFiltering}\n")
   # Close file after writing all environment variables
   file.close
   puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' "
@@ -336,6 +346,8 @@ def get_command_windows(env_variable_name, env_variable_value)
     file.write(commands)
     commands = get_command_windows("AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS", @logKubernetesMetadataiIncludeFields)
     file.write(commands)
+    commands = get_command_windows("AZMON_ANNOTATION_BASED_LOG_FILTERING", @annotationBasedLogFiltering)
+    file.write(commands)
     # Close file after writing all environment variables
     file.close
     puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' "

build/linux/installer/conf/fluent-bit.conf

@@ -54,5 +54,5 @@
 #${KubernetesFilterEnabled}    Kubelet_Port        10250
 #${KubernetesFilterEnabled}    Kubelet_Host        ${NODE_IP}
 #${KubernetesFilterEnabled}    tls.verify          Off
-#${KubernetesFilterEnabled}    K8S-Logging.Exclude On
+#${KubernetesFilterEnabled}#${AnnotationBasedLogFilteringEnabled}    K8S-Logging.Exclude On
 #${KubernetesFilterEnabled}    Kube_Tag_Prefix     oms.container.log.la.var.log.containers.

build/windows/installer/conf/fluent-bit.conf

@@ -53,5 +53,5 @@
 #${KubernetesFilterEnabled}    Kubelet_Port        10250
 #${KubernetesFilterEnabled}    Kubelet_Host        ${NODE_IP}
 #${KubernetesFilterEnabled}    tls.verify          Off
-#${KubernetesFilterEnabled}    K8S-Logging.Exclude On
+#${KubernetesFilterEnabled}#${AnnotationBasedLogFilteringEnabled}    K8S-Logging.Exclude On
 #${KubernetesFilterEnabled}    Kube_Tag_Prefix     oms.container.log.la.var.log.containers.

kubernetes/container-azm-ms-agentconfig.yaml

@@ -55,6 +55,9 @@ data:
           # if enabled will collect kubernetes metadata for ContainerLogv2 schema. Default false
           # enabled = false
           # include_fields = ["podLabels","podAnnotations","podUid","image"]
+       #[log_collection_settings.filter_using_annotations]
+          # if enabled will exclude logs from pods with annotations fluenbit.io/exclude: "true". Read more: https://docs.fluentbit.io/manual/pipeline/filters/kubernetes#kubernetes-annotations
+          # enabled = false
 
   prometheus-data-collection-settings: |-
     # Custom Prometheus metrics data collection settings