Fix issues found in integrations after testing with logsdb

.buildkite/pipeline.yml

@@ -13,6 +13,10 @@ env:
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
 
+  # XXX: To be removed before merging.
+  STACK_VERSION: '8.16.0-SNAPSHOT'
+  STACK_LOGSDB_ENABLED: 'true'
+
   # Elastic package settings
   # Manage docker output/logs
   ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"

go.mod

@@ -217,3 +217,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace github.com/elastic/elastic-package => github.com/jsoriano/elastic-package v0.66.1-0.20240909114932-d115a79b285c

go.sum

@@ -108,8 +108,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0 h1:sx1lpZuTG5suJuvgix4FWQFCLFFbzkoOmPoHWYOPLCY=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0/go.mod h1:2/30n+2QRzRzus4TPVUV1T3U/j8g2ItUgvP0pcpjLGk=
-github.com/elastic/elastic-package v0.103.0 h1:iGfZCnt5jbBWvuwCAgqZ0aNCqgQhfrdaR5hwfoER0lQ=
-github.com/elastic/elastic-package v0.103.0/go.mod h1:X3pav1fywMMWSy+k5WsqxW4SItsCiYWC+kTymDnw+Cw=
 github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-licenser v0.4.2 h1:bPbGm8bUd8rxzSswFOqvQh1dAkKGkgAmrPxbUi+Y9+A=
@@ -278,6 +276,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jsoriano/elastic-package v0.66.1-0.20240909114932-d115a79b285c h1:Vi+dlvh1VjTajJiNYiGJq/MENnyZufT4aD+MMDnsYek=
+github.com/jsoriano/elastic-package v0.66.1-0.20240909114932-d115a79b285c/go.mod h1:X3pav1fywMMWSy+k5WsqxW4SItsCiYWC+kTymDnw+Cw=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=

packages/auditd_manager/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.17.2"
+  changes:
+    - description: Add missing definitions for fields containing values for arguments
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10919
 - version: "1.17.1"
   changes:
     - description: "Doc: Add doc for configuring Auditd Manager for Session View"

packages/auditd_manager/data_stream/auditd/fields/fields.yml

@@ -311,6 +311,9 @@
 - name: auditd.data.subj
   description: lspp subject's context string
   type: keyword
+- name: auditd.data.subj_user
+  description: lspp subject's context user name
+  type: keyword
 - name: auditd.data.cgroup
   description: path to cgroup in sysfs
   type: keyword
@@ -737,6 +740,8 @@
   type: keyword
 - name: auditd.data.result
   type: keyword
+- name: auditd.data.a*
+  type: keyword
 - name: auditd.data
   description: Auditd related data
   type: flattened

packages/auditd_manager/docs/README.md

@@ -275,6 +275,7 @@ An example event for `auditd` looks as following:
 |---|---|---|
 | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date |
 | auditd.data | Auditd related data | flattened |
+| auditd.data.a\* |  | keyword |
 | auditd.data.a0-N | the arguments to a syscall | keyword |
 | auditd.data.acct | a user's account name | keyword |
 | auditd.data.acl | access mode of resource assigned to vm | keyword |
@@ -466,6 +467,7 @@ An example event for `auditd` looks as following:
 | auditd.data.sport | local port number | long |
 | auditd.data.state | audit daemon configuration resulting state | keyword |
 | auditd.data.subj | lspp subject's context string | keyword |
+| auditd.data.subj_user | lspp subject's context user name | keyword |
 | auditd.data.success | whether the syscall was successful or not | keyword |
 | auditd.data.syscall | syscall number in effect when the event occurred | keyword |
 | auditd.data.table | netfilter table name | keyword |

packages/auditd_manager/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.0"
 name: auditd_manager
 title: "Auditd Manager"
-version: "1.17.1"
+version: "1.17.2"
 description: "The Auditd Manager Integration receives audit events from the Linux Audit Framework that is a part of the Linux kernel."
 type: integration
 categories:

packages/cisco_ios/data_stream/log/_dev/test/pipeline/test-asr920.log-config.yml

@@ -0,0 +1,7 @@
+skip:
+  reason: New value in network.community_id when running on newer versions
+  link: https://github.com/elastic/integrations/issues/10904
+
+# Dummy configuration because of https://github.com/elastic/elastic-package/issues/2051.
+fields:
+  tags: []

packages/cisco_ios/data_stream/log/_dev/test/pipeline/test-cisco-ios.log-config.yml

@@ -0,0 +1,7 @@
+skip:
+  reason: New value in network.community_id when running on newer versions
+  link: https://github.com/elastic/integrations/issues/10904
+
+# Dummy configuration because of https://github.com/elastic/elastic-package/issues/2051.
+fields:
+  tags: []

packages/kibana/data_stream/audit/fields/ecs.yml

@@ -52,3 +52,5 @@
   external: ecs
 - name: client.ip
   external: ecs
+- name: span.id
+  external: ecs

packages/kibana/docs/README.md

@@ -60,6 +60,7 @@ UI in Kibana.
 | process.pid | Process id. | long |
 | related.user | All the user names or other user identifiers seen on the event. | keyword |
 | service.node.roles | Roles of a service node. This allows for distinction between different running roles of the same service. In the case of Kibana, the `service.node.role` could be `ui` or `background_tasks` or both. In the case of Elasticsearch, the `service.node.role` could be `master` or `data` or both. Other services could use this to distinguish between a `web` and `worker` role running as part of the service. | keyword |
+| span.id | Unique identifier of the span within the scope of its trace. A span represents an operation within a transaction, such as a request to another service, or a database query. | keyword |
 | trace.id | Unique identifier of the trace. A trace groups multiple events like transactions that belong together. For example, a user request handled by multiple inter-connected services. | keyword |
 | transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword |
 | url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword |

packages/proofpoint_on_demand/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.1.1"
+  changes:
+    - description: Fix definition of subfields of nested objects
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10919
 - version: "0.1.0"
   changes:
     - description: Initial release.

packages/proofpoint_on_demand/data_stream/audit/fields/fields.yml

@@ -74,13 +74,12 @@
               description: The IP address of the service.
         - name: tags
           type: nested
-          fields:
-            - name: name
-              type: keyword
-              description: Tag name for the particular instance of event.
-            - name: value
-              type: keyword
-              description: The value associated with the tag name.
+        - name: tags.name
+          type: keyword
+          description: Tag name for the particular instance of event.
+        - name: tags.value
+          type: keyword
+          description: The value associated with the tag name.
         - name: ts
           type: date
           description: Timestamp of when the event to be audited occurred.

packages/proofpoint_on_demand/docs/README.md

@@ -226,6 +226,7 @@ An example event for `audit` looks as following:
 | proofpoint_on_demand.audit.service.customer_id | The customer id of the service. | keyword |
 | proofpoint_on_demand.audit.service.id | The IDM service id. | keyword |
 | proofpoint_on_demand.audit.service.ip_address | The IP address of the service. | ip |
+| proofpoint_on_demand.audit.tags |  | nested |
 | proofpoint_on_demand.audit.tags.name | Tag name for the particular instance of event. | keyword |
 | proofpoint_on_demand.audit.tags.value | The value associated with the tag name. | keyword |
 | proofpoint_on_demand.audit.ts | Timestamp of when the event to be audited occurred. | date |

packages/proofpoint_on_demand/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.1.4
 name: proofpoint_on_demand
 title: Proofpoint On Demand
-version: 0.1.0
+version: 0.1.1
 description: Collect logs from Proofpoint On Demand with Elastic Agent.
 type: integration
 categories:

packages/sophos/data_stream/xg/_dev/test/pipeline/test-sophos-18-5-firewall.log-config.yml

@@ -0,0 +1,7 @@
+skip:
+  reason: New value in network.community_id when running on newer versions
+  link: https://github.com/elastic/integrations/issues/10907
+
+# Dummy configuration because of https://github.com/elastic/elastic-package/issues/2051.
+fields:
+  tags: []

packages/sophos/data_stream/xg/_dev/test/pipeline/test-sophos-xg.log-config.yml

@@ -0,0 +1,7 @@
+skip:
+  reason: New value in network.community_id when running on newer versions
+  link: https://github.com/elastic/integrations/issues/10908
+
+# Dummy configuration because of https://github.com/elastic/elastic-package/issues/2051.
+fields:
+  tags: []