From 9833794d2680acc73ed00041cbb38caaf14b96cc Mon Sep 17 00:00:00 2001 From: longwan Date: Thu, 16 Nov 2023 16:20:48 -0800 Subject: [PATCH 01/25] add metadata feilds including labels, annotations, uid and image --- build/common/installer/scripts/tomlparser.rb | 5 +++++ kubernetes/container-azm-ms-agentconfig.yaml | 2 +- source/plugins/ruby/CAdvisorMetricsAPIClient.rb | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/build/common/installer/scripts/tomlparser.rb b/build/common/installer/scripts/tomlparser.rb index 894e2cd09..ffc96becb 100644 --- a/build/common/installer/scripts/tomlparser.rb +++ b/build/common/installer/scripts/tomlparser.rb @@ -25,6 +25,7 @@ @adxDatabaseName = "containerinsights" # default for all configurations @logEnableMultiline = "false" @logEnableKubernetesMetadata = false +@logKubernetesMetadataiIncludeFields = "podLabels,podAnnotations,poduid,image" if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 @containerLogsRoute = "v1" # default is v1 for windows until windows agent integrates windows ama # This path format is necessary for fluent-bit in windows @@ -211,6 +212,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) begin if !parsedConfig[:log_collection_settings][:metadata_collection].nil? && !parsedConfig[:log_collection_settings][:metadata_collection][:enabled].nil? @logEnableKubernetesMetadata = parsedConfig[:log_collection_settings][:metadata_collection][:enabled] + @logKubernetesMetadataiIncludeFields = parsedConfig[:log_collection_settings][:metadata_collection][:include_fields] puts "config::Using config map setting for kubernetes metadata" end end @@ -262,6 +264,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) file.write("export AZMON_ADX_DATABASE_NAME=#{@adxDatabaseName}\n") file.write("export AZMON_MULTILINE_ENABLED=#{@logEnableMultiline}\n") file.write("export AZMON_KUBERNETES_METADATA_ENABLED=#{@logEnableKubernetesMetadata}\n") + file.write("export AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS=#{@logKubernetesMetadataiIncludeFields}\n") # Close file after writing all environment variables file.close puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' " @@ -328,6 +331,8 @@ def get_command_windows(env_variable_name, env_variable_value) file.write(commands) commands = get_command_windows("AZMON_KUBERNETES_METADATA_ENABLED ", @logEnableKubernetesMetadata) file.write(commands) + commands = get_command_windows("AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS", @logKubernetesMetadataiIncludeFields) + file.write(commands) # Close file after writing all environment variables file.close puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' " diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index f879b4ac2..6777261a9 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -54,7 +54,7 @@ data: #[log_collection_settings.metadata_collection] # if enabled will collect kubernetes metadata for ContainerLogv2 schema. Default false # enabled = false - + # include_fields = ["podLabels","podAnnotations","podUID","image"] prometheus-data-collection-settings: |- # Custom Prometheus metrics data collection settings diff --git a/source/plugins/ruby/CAdvisorMetricsAPIClient.rb b/source/plugins/ruby/CAdvisorMetricsAPIClient.rb index 0f467ae4a..23ee556c4 100644 --- a/source/plugins/ruby/CAdvisorMetricsAPIClient.rb +++ b/source/plugins/ruby/CAdvisorMetricsAPIClient.rb @@ -28,6 +28,7 @@ class CAdvisorMetricsAPIClient @clusterContainerLogSchemaVersion = ENV["AZMON_CONTAINER_LOG_SCHEMA_VERSION"] @clusterMultilineEnabled = ENV["AZMON_MULTILINE_ENABLED"] @clusterKubernetesMetadataEnabled = ENV["AZMON_KUBERNETES_METADATA_ENABLED"] + @clusterKubernetesMetadataiIncludeFields = ENV["AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS"] @dsPromInterval = ENV["TELEMETRY_DS_PROM_INTERVAL"] @dsPromFieldPassCount = ENV["TELEMETRY_DS_PROM_FIELDPASS_LENGTH"] @@ -300,6 +301,9 @@ def getContainerCpuMetricItems(metricJSON, hostName, cpuMetricNameToCollect, met if (!@clusterKubernetesMetadataEnabled.nil? && !@clusterKubernetesMetadataEnabled.empty?) telemetryProps["metadataEnabled"] = @clusterKubernetesMetadataEnabled end + if (!@clusterKubernetesMetadataiIncludeFields.nil? && !@clusterKubernetesMetadataiIncludeFields.empty?) + telemetryProps["metadataIncludeFields"] = @clusterKubernetesMetadataiIncludeFields + end ApplicationInsightsUtility.sendMetricTelemetry(metricNametoReturn, metricValue, telemetryProps) end end From c7abdde00cd0800df4f5c61cb3c594f63b084a50 Mon Sep 17 00:00:00 2001 From: longwan Date: Fri, 17 Nov 2023 15:23:35 -0800 Subject: [PATCH 02/25] rename to podUid and add nil check --- build/common/installer/scripts/tomlparser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/common/installer/scripts/tomlparser.rb b/build/common/installer/scripts/tomlparser.rb index ffc96becb..fc86546d4 100644 --- a/build/common/installer/scripts/tomlparser.rb +++ b/build/common/installer/scripts/tomlparser.rb @@ -25,7 +25,7 @@ @adxDatabaseName = "containerinsights" # default for all configurations @logEnableMultiline = "false" @logEnableKubernetesMetadata = false -@logKubernetesMetadataiIncludeFields = "podLabels,podAnnotations,poduid,image" +@logKubernetesMetadataiIncludeFields = "podLabels,podAnnotations,podUid,image" if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 @containerLogsRoute = "v1" # default is v1 for windows until windows agent integrates windows ama # This path format is necessary for fluent-bit in windows @@ -210,7 +210,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) #Get Kubernetes Metadata setting begin - if !parsedConfig[:log_collection_settings][:metadata_collection].nil? && !parsedConfig[:log_collection_settings][:metadata_collection][:enabled].nil? + if !parsedConfig[:log_collection_settings][:metadata_collection].nil? && !parsedConfig[:log_collection_settings][:metadata_collection][:enabled].nil? && !parsedConfig[:log_collection_settings][:metadata_collection][:include_fields].nil? @logEnableKubernetesMetadata = parsedConfig[:log_collection_settings][:metadata_collection][:enabled] @logKubernetesMetadataiIncludeFields = parsedConfig[:log_collection_settings][:metadata_collection][:include_fields] puts "config::Using config map setting for kubernetes metadata" From 9dd959521f607260bc1c53856316ed4fc261b44d Mon Sep 17 00:00:00 2001 From: longwan Date: Fri, 17 Nov 2023 15:33:21 -0800 Subject: [PATCH 03/25] rename to podUid and add nil check --- kubernetes/container-azm-ms-agentconfig.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index 6777261a9..364e8faa5 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -54,7 +54,7 @@ data: #[log_collection_settings.metadata_collection] # if enabled will collect kubernetes metadata for ContainerLogv2 schema. Default false # enabled = false - # include_fields = ["podLabels","podAnnotations","podUID","image"] + # include_fields = ["podLabels","podAnnotations","podUid","image"] prometheus-data-collection-settings: |- # Custom Prometheus metrics data collection settings From a1d659ce609b027daf4db95464dd7a8c910a4739 Mon Sep 17 00:00:00 2001 From: longwan Date: Mon, 27 Nov 2023 15:28:30 -0800 Subject: [PATCH 04/25] set the include_fields if customer set it otherwise go with default if enabled --- build/common/installer/scripts/tomlparser.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build/common/installer/scripts/tomlparser.rb b/build/common/installer/scripts/tomlparser.rb index fc86546d4..d89f16df9 100644 --- a/build/common/installer/scripts/tomlparser.rb +++ b/build/common/installer/scripts/tomlparser.rb @@ -210,9 +210,10 @@ def populateSettingValuesFromConfigMap(parsedConfig) #Get Kubernetes Metadata setting begin - if !parsedConfig[:log_collection_settings][:metadata_collection].nil? && !parsedConfig[:log_collection_settings][:metadata_collection][:enabled].nil? && !parsedConfig[:log_collection_settings][:metadata_collection][:include_fields].nil? + if !parsedConfig[:log_collection_settings][:metadata_collection].nil? && !parsedConfig[:log_collection_settings][:metadata_collection][:enabled].nil? @logEnableKubernetesMetadata = parsedConfig[:log_collection_settings][:metadata_collection][:enabled] - @logKubernetesMetadataiIncludeFields = parsedConfig[:log_collection_settings][:metadata_collection][:include_fields] + if !parsedConfig[:log_collection_settings][:metadata_collection][:include_fields].nil? + @logKubernetesMetadataiIncludeFields = parsedConfig[:log_collection_settings][:metadata_collection][:include_fields] puts "config::Using config map setting for kubernetes metadata" end end From 822cdc35ce593d743b2bae56ea9f2cb2af2617e8 Mon Sep 17 00:00:00 2001 From: longwan Date: Tue, 28 Nov 2023 12:30:24 -0800 Subject: [PATCH 05/25] add end and puts log --- build/common/installer/scripts/tomlparser.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/common/installer/scripts/tomlparser.rb b/build/common/installer/scripts/tomlparser.rb index d89f16df9..575be3306 100644 --- a/build/common/installer/scripts/tomlparser.rb +++ b/build/common/installer/scripts/tomlparser.rb @@ -211,10 +211,12 @@ def populateSettingValuesFromConfigMap(parsedConfig) #Get Kubernetes Metadata setting begin if !parsedConfig[:log_collection_settings][:metadata_collection].nil? && !parsedConfig[:log_collection_settings][:metadata_collection][:enabled].nil? + puts "config::Using config map setting for kubernetes metadata" @logEnableKubernetesMetadata = parsedConfig[:log_collection_settings][:metadata_collection][:enabled] if !parsedConfig[:log_collection_settings][:metadata_collection][:include_fields].nil? + puts "config::Using config map setting for kubernetes metadata include fields" @logKubernetesMetadataiIncludeFields = parsedConfig[:log_collection_settings][:metadata_collection][:include_fields] - puts "config::Using config map setting for kubernetes metadata" + end end end end From 8b5aff84c33daca8046b9b420e52deb46bffdf46 Mon Sep 17 00:00:00 2001 From: longwan Date: Tue, 28 Nov 2023 14:09:36 -0800 Subject: [PATCH 06/25] add workload to testing clusters --- .../containerlogv2_linux-resource-app.yml | 35 +++++++++++++++++++ .../containerlogv2_windows-resource-app.yml | 34 ++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 test/scenario/containerlogv2_linux-resource-app.yml create mode 100644 test/scenario/containerlogv2_windows-resource-app.yml diff --git a/test/scenario/containerlogv2_linux-resource-app.yml b/test/scenario/containerlogv2_linux-resource-app.yml new file mode 100644 index 000000000..5c8afe573 --- /dev/null +++ b/test/scenario/containerlogv2_linux-resource-app.yml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: linux-resource-intensive-app + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: linux-resource-intensive-app + template: + metadata: + labels: + app: linux-resource-intensive-app + spec: + nodeSelector: + kubernetes.io/os: linux + containers: + - name: linux-resource-intensive-app + image: ubuntu + command: ["/bin/bash", "-c"] + args: + - | + counter=1 + while true; do + echo "Loop $counter" + # CPU and Memory intensive operations + result=1 + for i in {1..10000}; do + result=$(echo "$result + $i" | bc) + result=$(echo "scale=2; sqrt($result)" | bc) + done + counter=$((counter+1)) + sleep 1 + done diff --git a/test/scenario/containerlogv2_windows-resource-app.yml b/test/scenario/containerlogv2_windows-resource-app.yml new file mode 100644 index 000000000..a0cafd276 --- /dev/null +++ b/test/scenario/containerlogv2_windows-resource-app.yml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: windows-resource-intensive-app + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: windows-resource-intensive-app + template: + metadata: + labels: + app: windows-resource-intensive-app + spec: + nodeSelector: + kubernetes.io/os: windows + containers: + - name: windows-resource-intensive-app + image: mcr.microsoft.com/windows/servercore:ltsc2022 + command: ["powershell", "-c"] + args: + - | + $counter = 1 + while ($true) { + Write-Output "Loop $counter" + # CPU and Memory intensive operations + $result = 1 + for ($i = 1; $i -le 10000; $i++) { + $result = [Math]::Pow($result + $i, 2) + } + $counter++ + Start-Sleep -Seconds 1 + } From 03074d5a904dfce6fc95ac8be03c45ce7c099d9d Mon Sep 17 00:00:00 2001 From: longwan Date: Sun, 3 Dec 2023 18:32:23 -0800 Subject: [PATCH 07/25] containerlogv2 metadata backend change with feature flag on --- source/plugins/go/src/oms.go | 38 +++++++++++++++++++ .../containerlogv2_windows-resource-app.yml | 10 ++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index 0dedd6031..ae0b42ebe 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -161,6 +161,8 @@ var ( ContainerLogSchemaV2 bool // container log schema version from config map ContainerLogV2ConfigMap bool + // container log schema version from config map + KubernetesMetadataConfigMap bool //ADX Cluster URI AdxClusterUri string // ADX clientID @@ -272,6 +274,8 @@ type DataItemLAv2 struct { PodNamespace string `json:"PodNamespace"` LogMessage string `json:"LogMessage"` LogSource string `json:"LogSource"` + KubernetesMetadata string `json:"KubernetesMetadata"` + //LogLevel string `json:"LogLevel"` //PodLabels string `json:"PodLabels"` } @@ -285,6 +289,8 @@ type DataItemADX struct { PodNamespace string `json:"PodNamespace"` LogMessage string `json:"LogMessage"` LogSource string `json:"LogSource"` + KubernetesMetadata string `json:"KubernetesMetadata"` + //LogLevel string `json:"LogLevel"` //PodLabels string `json:"PodLabels"` AzureResourceId string `json:"AzureResourceId"` } @@ -1135,6 +1141,24 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { for _, record := range tailPluginRecords { containerID, k8sNamespace, k8sPodName, containerName := GetContainerIDK8sNamespacePodNameFromFileName(ToString(record["filepath"])) logEntrySource := ToString(record["stream"]) + kubernetesMetadata := "" + if KubernetesMetadataConfigMap { + if kubernetesMetadataJson, exists := record["kubernetes"]; exists { + kubernetesMetadataBytes, err := json.Marshal(kubernetesMetadataJson) + if err != nil { + message := fmt.Sprintf("Error while Marshalling kubernetesMetadataBytes to json bytes: %s", err.Error()) + Log(message) + SendException(message) + } + kubernetesMetadata = string(kubernetesMetadataBytes) + } else { + message := fmt.Sprintf("Error while fetching kubernetesMetadataJson") + Log(message) + continue + } + } + //include + if strings.EqualFold(logEntrySource, "stdout") { if containerID == "" || containsKey(StdoutIgnoreNsSet, k8sNamespace) { @@ -1160,6 +1184,9 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { } logEntry := ToString(record["log"]) + //filter loglevel and define here + //add another configmap flag + //logLevel := ToString(record["LogLevel"]) logEntryTimeStamp := ToString(record["time"]) if !ContainerLogV2ConfigMap && IsAADMSIAuthMode == true && !IsGenevaLogsIntegrationEnabled { @@ -1188,6 +1215,8 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { stringMap["LogMessage"] = logEntry stringMap["LogSource"] = logEntrySource stringMap["TimeGenerated"] = logEntryTimeStamp + stringMap["KubernetesMetadata"] = kubernetesMetadata + //stringMap["LogLevel"] = logLevel } else { stringMap["LogEntry"] = logEntry stringMap["LogEntrySource"] = logEntrySource @@ -1237,6 +1266,8 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { LogMessage: stringMap["LogMessage"], LogSource: stringMap["LogSource"], AzureResourceId: stringMap["AzureResourceId"], + KubernetesMetadata: stringMap["KubernetesMetadata"], + //LogLevel: stringMap["LogLevel"], } //ADX dataItemsADX = append(dataItemsADX, dataItemADX) @@ -1251,6 +1282,8 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { PodNamespace: stringMap["PodNamespace"], LogMessage: stringMap["LogMessage"], LogSource: stringMap["LogSource"], + KubernetesMetadata: stringMap["KubernetesMetadata"], + //LogLevel: stringMap["LogLevel"], } //ODS-v2 schema dataItemsLAv2 = append(dataItemsLAv2, dataItemLAv2) @@ -1870,6 +1903,11 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { ContainerLogSchemaV2 = false //default is v1 schema ContainerLogV2ConfigMap = (strings.Compare(ContainerLogSchemaVersion, ContainerLogV2SchemaVersion) == 0) + KubernetesMetadataConfigMap = false + KubernetesMetadataConfigMap = (strings.Compare(strings.ToLower(os.Getenv(AADMSIAuthMode)), "true") == 0) + //include + + if ContainerLogV2ConfigMap && ContainerLogsRouteADX != true { ContainerLogSchemaV2 = true Log("Container logs schema=%s", ContainerLogV2SchemaVersion) diff --git a/test/scenario/containerlogv2_windows-resource-app.yml b/test/scenario/containerlogv2_windows-resource-app.yml index a0cafd276..28dabb572 100644 --- a/test/scenario/containerlogv2_windows-resource-app.yml +++ b/test/scenario/containerlogv2_windows-resource-app.yml @@ -4,7 +4,7 @@ metadata: name: windows-resource-intensive-app namespace: kube-system spec: - replicas: 1 + replicas: 10 selector: matchLabels: app: windows-resource-intensive-app @@ -32,3 +32,11 @@ spec: $counter++ Start-Sleep -Seconds 1 } +## thound logs per second +## 10 pods in certain nodes combining logs each generate some logs scale performance podlevel +## multiple 100 pods + +## nodes level testing both windows linux if api then proabbly node level kubelet + + +## create a branch for current code branch for testing current stage \ No newline at end of file From dec7dd38c6d1c762be900d7cd841a3c91cb23917 Mon Sep 17 00:00:00 2001 From: longwan Date: Thu, 7 Dec 2023 14:47:32 -0800 Subject: [PATCH 08/25] include list change and data filter --- source/plugins/go/src/oms.go | 48 ++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index ae0b42ebe..bc0792865 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -161,8 +161,10 @@ var ( ContainerLogSchemaV2 bool // container log schema version from config map ContainerLogV2ConfigMap bool - // container log schema version from config map + // Kubernetes Metadata enabled through configmap flag KubernetesMetadataConfigMap bool + // Kubernetes Metadata enabled exclude list + KubernetesMetadataIncludeList []string //ADX Cluster URI AdxClusterUri string // ADX clientID @@ -1111,6 +1113,34 @@ func UpdateNumTelegrafMetricsSentTelemetry(numMetricsSent int, numSendErrors int ContainerLogTelemetryMutex.Unlock() } +func processIncludes(kubernetesMetadataMap map[string]interface{}, includesList []string) map[string]interface{} { + includedMetadata := make(map[string]interface{}) + for _, include := range includesList { + switch include { + case "podUid": + if val, ok := kubernetesMetadataMap["pod_id"]; ok { + includedMetadata["podUid"] = val + } + case "podLabels": + if val, ok := kubernetesMetadataMap["labels"]; ok { + includedMetadata["podLabels"] = val + } + case "podAnnotations": + if val, ok := kubernetesMetadataMap["annotations"]; ok { + includedMetadata["podAnnotations"] = val + } + case "image": + if hash, ok := kubernetesMetadataMap["container_hash"]; ok { + includedMetadata["image_hash"] = hash + } + if image, ok := kubernetesMetadataMap["container_image"]; ok { + includedMetadata["image"] = image + } + } + } + return includedMetadata +} + // PostDataHelper sends data to the ODS endpoint or oneagent or ADX func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { start := time.Now() @@ -1144,7 +1174,10 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { kubernetesMetadata := "" if KubernetesMetadataConfigMap { if kubernetesMetadataJson, exists := record["kubernetes"]; exists { - kubernetesMetadataBytes, err := json.Marshal(kubernetesMetadataJson) + kubernetesMetadataMap := kubernetesMetadataJson.(map[string]interface{}) + includedMetadata := processIncludes(kubernetesMetadataMap, KubernetesMetadataIncludeList) + + kubernetesMetadataBytes, err := json.Marshal(includedMetadata) if err != nil { message := fmt.Sprintf("Error while Marshalling kubernetesMetadataBytes to json bytes: %s", err.Error()) Log(message) @@ -1157,8 +1190,6 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { continue } } - //include - if strings.EqualFold(logEntrySource, "stdout") { if containerID == "" || containsKey(StdoutIgnoreNsSet, k8sNamespace) { @@ -1904,9 +1935,12 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { ContainerLogV2ConfigMap = (strings.Compare(ContainerLogSchemaVersion, ContainerLogV2SchemaVersion) == 0) KubernetesMetadataConfigMap = false - KubernetesMetadataConfigMap = (strings.Compare(strings.ToLower(os.Getenv(AADMSIAuthMode)), "true") == 0) - //include - + KubernetesMetadataConfigMap = (strings.Compare(strings.ToLower(os.Getenv("AZMON_KUBERNETES_METADATA_ENABLED")), "true") == 0) + //podLabels,podAnnotations,podUid,image + KubernetesMetadataIncludeList := os.Getenv("AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS") + if len(KubernetesMetadataIncludeList) == 0 { + KubernetesMetadataIncludeList = "podLabels,podAnnotations,podUid,image" + } if ContainerLogV2ConfigMap && ContainerLogsRouteADX != true { ContainerLogSchemaV2 = true From ffc4cf1f3cfdc5135f718d47d25a37d7330081e9 Mon Sep 17 00:00:00 2001 From: longwan Date: Tue, 12 Dec 2023 10:45:40 -0800 Subject: [PATCH 09/25] update script and fix tsl --- build/linux/installer/conf/fluent-bit.conf | 1 + .../containerlogv2_linux-resource-app.yml | 16 +++++--------- .../containerlogv2_windows-resource-app.yml | 21 +++++++------------ 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/build/linux/installer/conf/fluent-bit.conf b/build/linux/installer/conf/fluent-bit.conf index f05c21b1c..c1530dc5a 100644 --- a/build/linux/installer/conf/fluent-bit.conf +++ b/build/linux/installer/conf/fluent-bit.conf @@ -53,5 +53,6 @@ #${KubernetesFilterEnabled} Use_Kubelet true #${KubernetesFilterEnabled} Kubelet_Port 10250 #${KubernetesFilterEnabled} Kubelet_Host ${NODE_IP} +#${KubernetesFilterEnabled} tls.verify Off #${KubernetesFilterEnabled} K8S-Logging.Exclude On #${KubernetesFilterEnabled} Kube_Tag_Prefix oms.container.log.la.var.log.containers. diff --git a/test/scenario/containerlogv2_linux-resource-app.yml b/test/scenario/containerlogv2_linux-resource-app.yml index 5c8afe573..fa4d75ca9 100644 --- a/test/scenario/containerlogv2_linux-resource-app.yml +++ b/test/scenario/containerlogv2_linux-resource-app.yml @@ -4,7 +4,7 @@ metadata: name: linux-resource-intensive-app namespace: kube-system spec: - replicas: 1 + replicas: 10 selector: matchLabels: app: linux-resource-intensive-app @@ -21,15 +21,9 @@ spec: command: ["/bin/bash", "-c"] args: - | - counter=1 while true; do - echo "Loop $counter" - # CPU and Memory intensive operations - result=1 - for i in {1..10000}; do - result=$(echo "$result + $i" | bc) - result=$(echo "scale=2; sqrt($result)" | bc) - done - counter=$((counter+1)) - sleep 1 + timestamp=$(date "+%Y/%m/%d %H:%M:%S.%3N") + logEntry=$(printf 'Test-%.0s' {1..200}) + echo "$timestamp $logEntry" + sleep 0.001 # Sleep for 1 millisecond done diff --git a/test/scenario/containerlogv2_windows-resource-app.yml b/test/scenario/containerlogv2_windows-resource-app.yml index 28dabb572..9e7485d37 100644 --- a/test/scenario/containerlogv2_windows-resource-app.yml +++ b/test/scenario/containerlogv2_windows-resource-app.yml @@ -4,7 +4,7 @@ metadata: name: windows-resource-intensive-app namespace: kube-system spec: - replicas: 10 + replicas: 10 # 50 selector: matchLabels: app: windows-resource-intensive-app @@ -12,6 +12,7 @@ spec: metadata: labels: app: windows-resource-intensive-app + # more labels spec: nodeSelector: kubernetes.io/os: windows @@ -21,22 +22,16 @@ spec: command: ["powershell", "-c"] args: - | - $counter = 1 while ($true) { - Write-Output "Loop $counter" - # CPU and Memory intensive operations - $result = 1 - for ($i = 1; $i -le 10000; $i++) { - $result = [Math]::Pow($result + $i, 2) - } - $counter++ - Start-Sleep -Seconds 1 + $timestamp = Get-Date -Format "yyyy/MM/dd HH:mm:ss.fff" + $logEntry = 'Test-' * 200 # Adjusted to create a log entry of about 1000 bytes + Write-Output "$timestamp $logEntry" + Start-Sleep -Milliseconds 1 # Generates approximately 1000 logs per second } -## thound logs per second + +## thousand logs per second ## 10 pods in certain nodes combining logs each generate some logs scale performance podlevel ## multiple 100 pods ## nodes level testing both windows linux if api then proabbly node level kubelet - - ## create a branch for current code branch for testing current stage \ No newline at end of file From 92e13eae39299910ea3c9c7763e2d95dd51bcf2d Mon Sep 17 00:00:00 2001 From: longwan Date: Thu, 14 Dec 2023 14:22:39 -0800 Subject: [PATCH 10/25] adjust the marshal and log the output --- source/plugins/go/src/oms.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index bc0792865..29e5e13b6 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -1174,7 +1174,20 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { kubernetesMetadata := "" if KubernetesMetadataConfigMap { if kubernetesMetadataJson, exists := record["kubernetes"]; exists { - kubernetesMetadataMap := kubernetesMetadataJson.(map[string]interface{}) + n, err := fmt.Printf("Debug: kubernetesMetadataJson: %+v\n", kubernetesMetadataJson) + if err != nil { + Log(fmt.Sprintf("Error while printing kubernetesMetadataJson: %s", err)) + } + Log(fmt.Sprintf("Printed %d bytes", n)) + kubernetesMetadataMap := make(map[string]interface{}) + for k, v := range kubernetesMetadataJson.(map[interface{}]interface{}) { + if keyStr, ok := k.(string); ok { + kubernetesMetadataMap[keyStr] = v + } else { + Log("Error: Key in kubernetesMetadataJson is not a string") + continue + } + } includedMetadata := processIncludes(kubernetesMetadataMap, KubernetesMetadataIncludeList) kubernetesMetadataBytes, err := json.Marshal(includedMetadata) From 376a94ea4412d3833721c86267c1ae5ff27818d0 Mon Sep 17 00:00:00 2001 From: longwan Date: Thu, 14 Dec 2023 14:38:58 -0800 Subject: [PATCH 11/25] update trivy ignore --- .trivyignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.trivyignore b/.trivyignore index 4397bde78..173846cfe 100644 --- a/.trivyignore +++ b/.trivyignore @@ -3,6 +3,7 @@ CVE-2023-39325 #same as CVE-2023-44487 CVE-2023-3978 CVE-2023-44487 #false positive according to Mariner team GHSA-jq35-85cj-fj4p +CVE-2019-3826 #telegraf HIGH GHSA-m425-mq94-257g From a098502148c51b69bd3f82f73e2114c81082ae4e Mon Sep 17 00:00:00 2001 From: longwan Date: Thu, 14 Dec 2023 23:16:18 -0800 Subject: [PATCH 12/25] update debug --- source/plugins/go/src/oms.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index 29e5e13b6..dcaeb04e4 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -1174,11 +1174,6 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { kubernetesMetadata := "" if KubernetesMetadataConfigMap { if kubernetesMetadataJson, exists := record["kubernetes"]; exists { - n, err := fmt.Printf("Debug: kubernetesMetadataJson: %+v\n", kubernetesMetadataJson) - if err != nil { - Log(fmt.Sprintf("Error while printing kubernetesMetadataJson: %s", err)) - } - Log(fmt.Sprintf("Printed %d bytes", n)) kubernetesMetadataMap := make(map[string]interface{}) for k, v := range kubernetesMetadataJson.(map[interface{}]interface{}) { if keyStr, ok := k.(string); ok { @@ -1197,6 +1192,11 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { SendException(message) } kubernetesMetadata = string(kubernetesMetadataBytes) + n, err := fmt.Printf("Debug: kubernetesMetadata: %+v\n", kubernetesMetadata) + if err != nil { + Log(fmt.Sprintf("Error while printing kubernetesMetadata: %s", err)) + } + Log(fmt.Sprintf("Printed %d bytes", n)) } else { message := fmt.Sprintf("Error while fetching kubernetesMetadataJson") Log(message) From a09288c953bc5bf5fd05b1f2b9c3e036c111c34e Mon Sep 17 00:00:00 2001 From: longwan Date: Fri, 15 Dec 2023 11:57:39 -0800 Subject: [PATCH 13/25] add more debug info --- source/plugins/go/src/oms.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index dcaeb04e4..93eaba985 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -1174,15 +1174,24 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { kubernetesMetadata := "" if KubernetesMetadataConfigMap { if kubernetesMetadataJson, exists := record["kubernetes"]; exists { + n, err := fmt.Printf("Debug: kubernetesMetadataJson: %+v\n", kubernetesMetadataJson) + if err != nil { + Log(fmt.Sprintf("Error while printing kubernetesMetadataJson: %s", err)) + } + Log(fmt.Sprintf("Printed %d bytes", n)) + Log("Debug: kubernetesMetadataJson raw:", kubernetesMetadataJson) kubernetesMetadataMap := make(map[string]interface{}) for k, v := range kubernetesMetadataJson.(map[interface{}]interface{}) { if keyStr, ok := k.(string); ok { kubernetesMetadataMap[keyStr] = v + Log("Debug: kubernetesMetadataMap set succ") } else { Log("Error: Key in kubernetesMetadataJson is not a string") continue } } + Log("Debug: kubernetesMetadataMap:", kubernetesMetadataMap) + Log("Debug: KubernetesMetadataIncludeList: ", KubernetesMetadataIncludeList) includedMetadata := processIncludes(kubernetesMetadataMap, KubernetesMetadataIncludeList) kubernetesMetadataBytes, err := json.Marshal(includedMetadata) @@ -1192,11 +1201,11 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { SendException(message) } kubernetesMetadata = string(kubernetesMetadataBytes) - n, err := fmt.Printf("Debug: kubernetesMetadata: %+v\n", kubernetesMetadata) + l, err := fmt.Printf("Debug: kubernetesMetadata: %+v\n", kubernetesMetadata) if err != nil { Log(fmt.Sprintf("Error while printing kubernetesMetadata: %s", err)) } - Log(fmt.Sprintf("Printed %d bytes", n)) + Log(fmt.Sprintf("Printed %d bytes", l)) } else { message := fmt.Sprintf("Error while fetching kubernetesMetadataJson") Log(message) From 03d8f60f19d32ba443cc278ce9528447e219a5ee Mon Sep 17 00:00:00 2001 From: longwan Date: Fri, 15 Dec 2023 12:15:42 -0800 Subject: [PATCH 14/25] update ignore list --- .trivyignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.trivyignore b/.trivyignore index 173846cfe..8afe86b5f 100644 --- a/.trivyignore +++ b/.trivyignore @@ -10,5 +10,8 @@ GHSA-m425-mq94-257g CVE-2023-46129 CVE-2023-47090 -# ruby HIGH +#ruby HIGH CVE-2017-10784 + +#openssl +CVE-2023-5678 \ No newline at end of file From 3e2627ccc153885296f6af432440684662825dcd Mon Sep 17 00:00:00 2001 From: longwan Date: Fri, 15 Dec 2023 14:52:47 -0800 Subject: [PATCH 15/25] address the list split and logs --- source/plugins/go/src/oms.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index 93eaba985..b0d77c11b 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -1179,21 +1179,20 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { Log(fmt.Sprintf("Error while printing kubernetesMetadataJson: %s", err)) } Log(fmt.Sprintf("Printed %d bytes", n)) - Log("Debug: kubernetesMetadataJson raw:", kubernetesMetadataJson) + Log(fmt.Sprintf("Debug: kubernetesMetadataJson raw: %+v", kubernetesMetadataJson)) kubernetesMetadataMap := make(map[string]interface{}) for k, v := range kubernetesMetadataJson.(map[interface{}]interface{}) { if keyStr, ok := k.(string); ok { kubernetesMetadataMap[keyStr] = v - Log("Debug: kubernetesMetadataMap set succ") + Log(fmt.Sprintf("Debug: kubernetesMetadataMap set succ")) } else { - Log("Error: Key in kubernetesMetadataJson is not a string") + Log(fmt.Sprintf("Error: Key in kubernetesMetadataJson is not a string")) continue } } - Log("Debug: kubernetesMetadataMap:", kubernetesMetadataMap) - Log("Debug: KubernetesMetadataIncludeList: ", KubernetesMetadataIncludeList) + Log(fmt.Sprintf("Debug: kubernetesMetadataMap: %+v", kubernetesMetadataMap)) + Log(fmt.Sprintf("Debug: KubernetesMetadataIncludeList: %+v", KubernetesMetadataIncludeList)) includedMetadata := processIncludes(kubernetesMetadataMap, KubernetesMetadataIncludeList) - kubernetesMetadataBytes, err := json.Marshal(includedMetadata) if err != nil { message := fmt.Sprintf("Error while Marshalling kubernetesMetadataBytes to json bytes: %s", err.Error()) @@ -1958,10 +1957,11 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { KubernetesMetadataConfigMap = false KubernetesMetadataConfigMap = (strings.Compare(strings.ToLower(os.Getenv("AZMON_KUBERNETES_METADATA_ENABLED")), "true") == 0) - //podLabels,podAnnotations,podUid,image - KubernetesMetadataIncludeList := os.Getenv("AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS") - if len(KubernetesMetadataIncludeList) == 0 { - KubernetesMetadataIncludeList = "podLabels,podAnnotations,podUid,image" + metadataIncludeList := os.Getenv("AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS") + if KubernetesMetadataConfigMap && len(metadataIncludeList) > 0 { + KubernetesMetadataIncludeList = strings.Split(metadataIncludeList, ",") + } else { + KubernetesMetadataIncludeList = []string{"podLabels", "podAnnotations", "podUid", "image"} } if ContainerLogV2ConfigMap && ContainerLogsRouteADX != true { From dd4b716d525ac670b4b95f3c99ca74533b6b3421 Mon Sep 17 00:00:00 2001 From: longwan Date: Fri, 15 Dec 2023 17:20:26 -0800 Subject: [PATCH 16/25] logs --- source/plugins/go/src/oms.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index b0d77c11b..5415d87db 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -1174,12 +1174,15 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { kubernetesMetadata := "" if KubernetesMetadataConfigMap { if kubernetesMetadataJson, exists := record["kubernetes"]; exists { + Log("start kubernetesMetadataJson") n, err := fmt.Printf("Debug: kubernetesMetadataJson: %+v\n", kubernetesMetadataJson) if err != nil { Log(fmt.Sprintf("Error while printing kubernetesMetadataJson: %s", err)) } Log(fmt.Sprintf("Printed %d bytes", n)) + Log("LW 1") Log(fmt.Sprintf("Debug: kubernetesMetadataJson raw: %+v", kubernetesMetadataJson)) + Log("LW 2") kubernetesMetadataMap := make(map[string]interface{}) for k, v := range kubernetesMetadataJson.(map[interface{}]interface{}) { if keyStr, ok := k.(string); ok { @@ -1190,9 +1193,15 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { continue } } + Log("LW 3") Log(fmt.Sprintf("Debug: kubernetesMetadataMap: %+v", kubernetesMetadataMap)) - Log(fmt.Sprintf("Debug: KubernetesMetadataIncludeList: %+v", KubernetesMetadataIncludeList)) + k, err := fmt.Printf("Debug: KubernetesMetadataIncludeList: %+v\n", KubernetesMetadataIncludeList) + if err != nil { + Log(fmt.Sprintf("Error while printing KubernetesMetadataIncludeList: %s", err)) + } + Log(fmt.Sprintf("Printed %d bytes", k)) includedMetadata := processIncludes(kubernetesMetadataMap, KubernetesMetadataIncludeList) + Log("LW 4") kubernetesMetadataBytes, err := json.Marshal(includedMetadata) if err != nil { message := fmt.Sprintf("Error while Marshalling kubernetesMetadataBytes to json bytes: %s", err.Error()) From db4fd77be5b55a202a6f6ea2e6e21d6506bfea33 Mon Sep 17 00:00:00 2001 From: longwan Date: Sun, 17 Dec 2023 19:00:30 -0800 Subject: [PATCH 17/25] refactor code --- source/plugins/go/src/oms.go | 68 +++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index 5415d87db..ffd8bf83d 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -1141,6 +1141,31 @@ func processIncludes(kubernetesMetadataMap map[string]interface{}, includesList return includedMetadata } +func convertKubernetesMetadata(kubernetesMetadataJson interface{}) (map[string]interface{}, error) { + m, ok := kubernetesMetadataJson.(map[interface{}]interface{}) + if !ok { + return nil, fmt.Errorf("type assertion to map[interface{}]interface{} failed") + } + + strMap := make(map[string]interface{}) + for k, v := range m { + strKey, ok := k.(string) + if !ok { + continue + } + if subMap, isMap := v.(map[interface{}]interface{}); isMap { + convertedMap, err := convertKubernetesMetadata(subMap) + if err != nil { + return nil, err + } + strMap[strKey] = convertedMap + } else { + strMap[strKey] = v // Keep the original value + } + } + return strMap, nil +} + // PostDataHelper sends data to the ODS endpoint or oneagent or ADX func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { start := time.Now() @@ -1175,33 +1200,24 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { if KubernetesMetadataConfigMap { if kubernetesMetadataJson, exists := record["kubernetes"]; exists { Log("start kubernetesMetadataJson") - n, err := fmt.Printf("Debug: kubernetesMetadataJson: %+v\n", kubernetesMetadataJson) - if err != nil { - Log(fmt.Sprintf("Error while printing kubernetesMetadataJson: %s", err)) - } - Log(fmt.Sprintf("Printed %d bytes", n)) - Log("LW 1") Log(fmt.Sprintf("Debug: kubernetesMetadataJson raw: %+v", kubernetesMetadataJson)) - Log("LW 2") - kubernetesMetadataMap := make(map[string]interface{}) - for k, v := range kubernetesMetadataJson.(map[interface{}]interface{}) { - if keyStr, ok := k.(string); ok { - kubernetesMetadataMap[keyStr] = v - Log(fmt.Sprintf("Debug: kubernetesMetadataMap set succ")) - } else { - Log(fmt.Sprintf("Error: Key in kubernetesMetadataJson is not a string")) - continue - } - } - Log("LW 3") - Log(fmt.Sprintf("Debug: kubernetesMetadataMap: %+v", kubernetesMetadataMap)) - k, err := fmt.Printf("Debug: KubernetesMetadataIncludeList: %+v\n", KubernetesMetadataIncludeList) + // kubernetesMetadataMap := make(map[string]interface{}) + // for k, v := range kubernetesMetadataJson.(map[interface{}]interface{}) { + // if keyStr, ok := k.(string); ok { + // kubernetesMetadataMap[keyStr] = v + // Log(fmt.Sprintf("Debug: kubernetesMetadataMap set succ")) + // } else { + // Log(fmt.Sprintf("Error: Key in kubernetesMetadataJson is not a string")) + // continue + // } + // } + kubernetesMetadataMap, err := convertKubernetesMetadata(kubernetesMetadataJson) if err != nil { - Log(fmt.Sprintf("Error while printing KubernetesMetadataIncludeList: %s", err)) + Log(fmt.Sprintf("Error: %v", err)) } - Log(fmt.Sprintf("Printed %d bytes", k)) + Log(fmt.Sprintf("Debug: kubernetesMetadataMap: %+v", kubernetesMetadataMap)) + Log(fmt.Sprintf("Debug: KubernetesMetadataIncludeList: %+v\n", KubernetesMetadataIncludeList)) includedMetadata := processIncludes(kubernetesMetadataMap, KubernetesMetadataIncludeList) - Log("LW 4") kubernetesMetadataBytes, err := json.Marshal(includedMetadata) if err != nil { message := fmt.Sprintf("Error while Marshalling kubernetesMetadataBytes to json bytes: %s", err.Error()) @@ -1209,11 +1225,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { SendException(message) } kubernetesMetadata = string(kubernetesMetadataBytes) - l, err := fmt.Printf("Debug: kubernetesMetadata: %+v\n", kubernetesMetadata) - if err != nil { - Log(fmt.Sprintf("Error while printing kubernetesMetadata: %s", err)) - } - Log(fmt.Sprintf("Printed %d bytes", l)) + Log(fmt.Sprintf("Debug: kubernetesMetadata: %+v\n", kubernetesMetadata)) } else { message := fmt.Sprintf("Error while fetching kubernetesMetadataJson") Log(message) From 1fd45a9663b09185c6ce72e6d821c99b5add17e4 Mon Sep 17 00:00:00 2001 From: longwan Date: Sun, 17 Dec 2023 22:47:27 -0800 Subject: [PATCH 18/25] update include list format --- build/common/installer/scripts/tomlparser.rb | 2 +- source/plugins/go/src/oms.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build/common/installer/scripts/tomlparser.rb b/build/common/installer/scripts/tomlparser.rb index 575be3306..e1e9862d3 100644 --- a/build/common/installer/scripts/tomlparser.rb +++ b/build/common/installer/scripts/tomlparser.rb @@ -215,7 +215,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) @logEnableKubernetesMetadata = parsedConfig[:log_collection_settings][:metadata_collection][:enabled] if !parsedConfig[:log_collection_settings][:metadata_collection][:include_fields].nil? puts "config::Using config map setting for kubernetes metadata include fields" - @logKubernetesMetadataiIncludeFields = parsedConfig[:log_collection_settings][:metadata_collection][:include_fields] + @logKubernetesMetadataiIncludeFields = parsedConfig[:log_collection_settings][:metadata_collection][:include_fields].join(",") end end end diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index ffd8bf83d..443dad015 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -1979,6 +1979,8 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { KubernetesMetadataConfigMap = false KubernetesMetadataConfigMap = (strings.Compare(strings.ToLower(os.Getenv("AZMON_KUBERNETES_METADATA_ENABLED")), "true") == 0) metadataIncludeList := os.Getenv("AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS") + fmt.Sprintf("Debug: metadataIncludeList from Fetch: %+v\n", metadataIncludeList) + Log(fmt.Sprintf("Debug: metadataIncludeList from Fetch: %+v\n", metadataIncludeList)) if KubernetesMetadataConfigMap && len(metadataIncludeList) > 0 { KubernetesMetadataIncludeList = strings.Split(metadataIncludeList, ",") } else { From a5a3edc365cad129b2aa78cca7a779c2d9a9e17e Mon Sep 17 00:00:00 2001 From: longwan Date: Sun, 17 Dec 2023 23:42:53 -0800 Subject: [PATCH 19/25] clean up logs --- source/plugins/go/src/oms.go | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index 443dad015..601bb3545 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -1199,24 +1199,10 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { kubernetesMetadata := "" if KubernetesMetadataConfigMap { if kubernetesMetadataJson, exists := record["kubernetes"]; exists { - Log("start kubernetesMetadataJson") - Log(fmt.Sprintf("Debug: kubernetesMetadataJson raw: %+v", kubernetesMetadataJson)) - // kubernetesMetadataMap := make(map[string]interface{}) - // for k, v := range kubernetesMetadataJson.(map[interface{}]interface{}) { - // if keyStr, ok := k.(string); ok { - // kubernetesMetadataMap[keyStr] = v - // Log(fmt.Sprintf("Debug: kubernetesMetadataMap set succ")) - // } else { - // Log(fmt.Sprintf("Error: Key in kubernetesMetadataJson is not a string")) - // continue - // } - // } kubernetesMetadataMap, err := convertKubernetesMetadata(kubernetesMetadataJson) if err != nil { - Log(fmt.Sprintf("Error: %v", err)) + Log(fmt.Sprintf("Error convertKubernetesMetadata: %v", err)) } - Log(fmt.Sprintf("Debug: kubernetesMetadataMap: %+v", kubernetesMetadataMap)) - Log(fmt.Sprintf("Debug: KubernetesMetadataIncludeList: %+v\n", KubernetesMetadataIncludeList)) includedMetadata := processIncludes(kubernetesMetadataMap, KubernetesMetadataIncludeList) kubernetesMetadataBytes, err := json.Marshal(includedMetadata) if err != nil { @@ -1225,7 +1211,6 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { SendException(message) } kubernetesMetadata = string(kubernetesMetadataBytes) - Log(fmt.Sprintf("Debug: kubernetesMetadata: %+v\n", kubernetesMetadata)) } else { message := fmt.Sprintf("Error while fetching kubernetesMetadataJson") Log(message) @@ -1979,8 +1964,7 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { KubernetesMetadataConfigMap = false KubernetesMetadataConfigMap = (strings.Compare(strings.ToLower(os.Getenv("AZMON_KUBERNETES_METADATA_ENABLED")), "true") == 0) metadataIncludeList := os.Getenv("AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS") - fmt.Sprintf("Debug: metadataIncludeList from Fetch: %+v\n", metadataIncludeList) - Log(fmt.Sprintf("Debug: metadataIncludeList from Fetch: %+v\n", metadataIncludeList)) + Log(fmt.Sprintf("KubernetesMetadataIncludeList from configmap: %+v\n", metadataIncludeList)) if KubernetesMetadataConfigMap && len(metadataIncludeList) > 0 { KubernetesMetadataIncludeList = strings.Split(metadataIncludeList, ",") } else { From 4d1040ef307d43ef8b7abbd25ec294a20934b11b Mon Sep 17 00:00:00 2001 From: longwan Date: Mon, 18 Dec 2023 00:57:26 -0800 Subject: [PATCH 20/25] add convert bytes to string --- source/plugins/go/src/oms.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index 601bb3545..ac97c6dfd 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -1153,14 +1153,17 @@ func convertKubernetesMetadata(kubernetesMetadataJson interface{}) (map[string]i if !ok { continue } - if subMap, isMap := v.(map[interface{}]interface{}); isMap { - convertedMap, err := convertKubernetesMetadata(subMap) + switch val := v.(type) { + case map[interface{}]interface{}: + convertedMap, err := convertKubernetesMetadata(val) if err != nil { return nil, err } strMap[strKey] = convertedMap - } else { - strMap[strKey] = v // Keep the original value + case []byte: + strMap[strKey] = string(val) + default: + strMap[strKey] = val } } return strMap, nil From cc0216d90a8f3906aab6030b0529a10ce13e2973 Mon Sep 17 00:00:00 2001 From: longwan Date: Mon, 18 Dec 2023 04:43:20 -0800 Subject: [PATCH 21/25] clean up comment --- source/plugins/go/src/oms.go | 1 - 1 file changed, 1 deletion(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index ac97c6dfd..55c1ddf17 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -1246,7 +1246,6 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { logEntry := ToString(record["log"]) //filter loglevel and define here - //add another configmap flag //logLevel := ToString(record["LogLevel"]) logEntryTimeStamp := ToString(record["time"]) From d8cdee48c4fbd405220a090763aeaf4f95469089 Mon Sep 17 00:00:00 2001 From: longwan Date: Mon, 18 Dec 2023 08:17:14 -0800 Subject: [PATCH 22/25] address comments add check and scenario for empty list --- build/common/installer/scripts/tomlparser.rb | 5 ++++- source/plugins/go/src/oms.go | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/build/common/installer/scripts/tomlparser.rb b/build/common/installer/scripts/tomlparser.rb index e1e9862d3..b253617f2 100644 --- a/build/common/installer/scripts/tomlparser.rb +++ b/build/common/installer/scripts/tomlparser.rb @@ -215,7 +215,10 @@ def populateSettingValuesFromConfigMap(parsedConfig) @logEnableKubernetesMetadata = parsedConfig[:log_collection_settings][:metadata_collection][:enabled] if !parsedConfig[:log_collection_settings][:metadata_collection][:include_fields].nil? puts "config::Using config map setting for kubernetes metadata include fields" - @logKubernetesMetadataiIncludeFields = parsedConfig[:log_collection_settings][:metadata_collection][:include_fields].join(",") + include_fields = parsedConfig[:log_collection_settings][:metadata_collection][:include_fields] + if include_fields.kind_of?(Array) + @logKubernetesMetadataiIncludeFields = include_fields.join(",") + end end end end diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index 55c1ddf17..c0db47ca0 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -1967,10 +1967,11 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { KubernetesMetadataConfigMap = (strings.Compare(strings.ToLower(os.Getenv("AZMON_KUBERNETES_METADATA_ENABLED")), "true") == 0) metadataIncludeList := os.Getenv("AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS") Log(fmt.Sprintf("KubernetesMetadataIncludeList from configmap: %+v\n", metadataIncludeList)) + KubernetesMetadataIncludeList = []string{"podLabels", "podAnnotations", "podUid", "image"} if KubernetesMetadataConfigMap && len(metadataIncludeList) > 0 { KubernetesMetadataIncludeList = strings.Split(metadataIncludeList, ",") - } else { - KubernetesMetadataIncludeList = []string{"podLabels", "podAnnotations", "podUid", "image"} + } else if KubernetesMetadataConfigMap { + KubernetesMetadataIncludeList = []string{} } if ContainerLogV2ConfigMap && ContainerLogsRouteADX != true { From e02ac9bdd7d8b944538c4e03b1ff6fe1f77dd596 Mon Sep 17 00:00:00 2001 From: longwan Date: Mon, 18 Dec 2023 09:50:18 -0800 Subject: [PATCH 23/25] remove ADX support --- source/plugins/go/src/oms.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index c0db47ca0..dca01faf7 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -291,8 +291,6 @@ type DataItemADX struct { PodNamespace string `json:"PodNamespace"` LogMessage string `json:"LogMessage"` LogSource string `json:"LogSource"` - KubernetesMetadata string `json:"KubernetesMetadata"` - //LogLevel string `json:"LogLevel"` //PodLabels string `json:"PodLabels"` AzureResourceId string `json:"AzureResourceId"` } @@ -1266,7 +1264,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { } //ADX Schema & LAv2 schema are almost the same (except resourceId) - if ContainerLogSchemaV2 == true || ContainerLogsRouteADX == true { + if ContainerLogSchemaV2 == true { stringMap["Computer"] = Computer stringMap["ContainerId"] = containerID stringMap["ContainerName"] = containerName @@ -1277,6 +1275,15 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { stringMap["TimeGenerated"] = logEntryTimeStamp stringMap["KubernetesMetadata"] = kubernetesMetadata //stringMap["LogLevel"] = logLevel + } else if ContainerLogsRouteADX == true { + stringMap["Computer"] = Computer + stringMap["ContainerId"] = containerID + stringMap["ContainerName"] = containerName + stringMap["PodName"] = k8sPodName + stringMap["PodNamespace"] = k8sNamespace + stringMap["LogMessage"] = logEntry + stringMap["LogSource"] = logEntrySource + stringMap["TimeGenerated"] = logEntryTimeStamp } else { stringMap["LogEntry"] = logEntry stringMap["LogEntrySource"] = logEntrySource @@ -1326,8 +1333,6 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { LogMessage: stringMap["LogMessage"], LogSource: stringMap["LogSource"], AzureResourceId: stringMap["AzureResourceId"], - KubernetesMetadata: stringMap["KubernetesMetadata"], - //LogLevel: stringMap["LogLevel"], } //ADX dataItemsADX = append(dataItemsADX, dataItemADX) From 15402d802b00bec074454a5e0476680916ab4a30 Mon Sep 17 00:00:00 2001 From: longwan Date: Tue, 19 Dec 2023 09:38:58 -0800 Subject: [PATCH 24/25] rename to KubernetesMetadataEnabled --- source/plugins/go/src/oms.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index dca01faf7..a5465a062 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -162,7 +162,7 @@ var ( // container log schema version from config map ContainerLogV2ConfigMap bool // Kubernetes Metadata enabled through configmap flag - KubernetesMetadataConfigMap bool + KubernetesMetadataEnabled bool // Kubernetes Metadata enabled exclude list KubernetesMetadataIncludeList []string //ADX Cluster URI @@ -1198,7 +1198,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { containerID, k8sNamespace, k8sPodName, containerName := GetContainerIDK8sNamespacePodNameFromFileName(ToString(record["filepath"])) logEntrySource := ToString(record["stream"]) kubernetesMetadata := "" - if KubernetesMetadataConfigMap { + if KubernetesMetadataEnabled { if kubernetesMetadataJson, exists := record["kubernetes"]; exists { kubernetesMetadataMap, err := convertKubernetesMetadata(kubernetesMetadataJson) if err != nil { @@ -1968,14 +1968,14 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { ContainerLogSchemaV2 = false //default is v1 schema ContainerLogV2ConfigMap = (strings.Compare(ContainerLogSchemaVersion, ContainerLogV2SchemaVersion) == 0) - KubernetesMetadataConfigMap = false - KubernetesMetadataConfigMap = (strings.Compare(strings.ToLower(os.Getenv("AZMON_KUBERNETES_METADATA_ENABLED")), "true") == 0) + KubernetesMetadataEnabled = false + KubernetesMetadataEnabled = (strings.Compare(strings.ToLower(os.Getenv("AZMON_KUBERNETES_METADATA_ENABLED")), "true") == 0) metadataIncludeList := os.Getenv("AZMON_KUBERNETES_METADATA_INCLUDES_FIELDS") Log(fmt.Sprintf("KubernetesMetadataIncludeList from configmap: %+v\n", metadataIncludeList)) KubernetesMetadataIncludeList = []string{"podLabels", "podAnnotations", "podUid", "image"} - if KubernetesMetadataConfigMap && len(metadataIncludeList) > 0 { + if KubernetesMetadataEnabled && len(metadataIncludeList) > 0 { KubernetesMetadataIncludeList = strings.Split(metadataIncludeList, ",") - } else if KubernetesMetadataConfigMap { + } else if KubernetesMetadataEnabled { KubernetesMetadataIncludeList = []string{} } From 9c604a98f2d8ec62f0f245d4fe697e8f71106943 Mon Sep 17 00:00:00 2001 From: longwan Date: Tue, 19 Dec 2023 09:51:29 -0800 Subject: [PATCH 25/25] go fmt for better format --- source/plugins/go/src/oms.go | 84 ++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index a5465a062..eeffb66bd 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -268,15 +268,15 @@ type DataItemLAv1 struct { // DataItemLAv2 == ContainerLogV2 table in LA // Please keep the names same as destination column names, to avoid transforming one to another in the pipeline type DataItemLAv2 struct { - TimeGenerated string `json:"TimeGenerated"` - Computer string `json:"Computer"` - ContainerId string `json:"ContainerId"` - ContainerName string `json:"ContainerName"` - PodName string `json:"PodName"` - PodNamespace string `json:"PodNamespace"` - LogMessage string `json:"LogMessage"` - LogSource string `json:"LogSource"` - KubernetesMetadata string `json:"KubernetesMetadata"` + TimeGenerated string `json:"TimeGenerated"` + Computer string `json:"Computer"` + ContainerId string `json:"ContainerId"` + ContainerName string `json:"ContainerName"` + PodName string `json:"PodName"` + PodNamespace string `json:"PodNamespace"` + LogMessage string `json:"LogMessage"` + LogSource string `json:"LogSource"` + KubernetesMetadata string `json:"KubernetesMetadata"` //LogLevel string `json:"LogLevel"` //PodLabels string `json:"PodLabels"` } @@ -1140,31 +1140,31 @@ func processIncludes(kubernetesMetadataMap map[string]interface{}, includesList } func convertKubernetesMetadata(kubernetesMetadataJson interface{}) (map[string]interface{}, error) { - m, ok := kubernetesMetadataJson.(map[interface{}]interface{}) - if !ok { - return nil, fmt.Errorf("type assertion to map[interface{}]interface{} failed") - } - - strMap := make(map[string]interface{}) - for k, v := range m { - strKey, ok := k.(string) - if !ok { - continue - } - switch val := v.(type) { - case map[interface{}]interface{}: - convertedMap, err := convertKubernetesMetadata(val) - if err != nil { - return nil, err - } - strMap[strKey] = convertedMap - case []byte: - strMap[strKey] = string(val) - default: - strMap[strKey] = val - } - } - return strMap, nil + m, ok := kubernetesMetadataJson.(map[interface{}]interface{}) + if !ok { + return nil, fmt.Errorf("type assertion to map[interface{}]interface{} failed") + } + + strMap := make(map[string]interface{}) + for k, v := range m { + strKey, ok := k.(string) + if !ok { + continue + } + switch val := v.(type) { + case map[interface{}]interface{}: + convertedMap, err := convertKubernetesMetadata(val) + if err != nil { + return nil, err + } + strMap[strKey] = convertedMap + case []byte: + strMap[strKey] = string(val) + default: + strMap[strKey] = val + } + } + return strMap, nil } // PostDataHelper sends data to the ODS endpoint or oneagent or ADX @@ -1339,14 +1339,14 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { } else { if ContainerLogSchemaV2 == true { dataItemLAv2 = DataItemLAv2{ - TimeGenerated: stringMap["TimeGenerated"], - Computer: stringMap["Computer"], - ContainerId: stringMap["ContainerId"], - ContainerName: stringMap["ContainerName"], - PodName: stringMap["PodName"], - PodNamespace: stringMap["PodNamespace"], - LogMessage: stringMap["LogMessage"], - LogSource: stringMap["LogSource"], + TimeGenerated: stringMap["TimeGenerated"], + Computer: stringMap["Computer"], + ContainerId: stringMap["ContainerId"], + ContainerName: stringMap["ContainerName"], + PodName: stringMap["PodName"], + PodNamespace: stringMap["PodNamespace"], + LogMessage: stringMap["LogMessage"], + LogSource: stringMap["LogSource"], KubernetesMetadata: stringMap["KubernetesMetadata"], //LogLevel: stringMap["LogLevel"], }