[Fleet] Create templates and pipelines when updating package of a sin…

…gle package policy from type integration to input. (#150199)

## Summary

Part of #149423 

When upgrading a single package policy with conflicts, we use the update
package API. When we update from an input package to an integration
package, we need to create the per-policy assets. This PR detects if an
update is changing from an integration package to an input package and
creates the assets as part of the update.

I've added an integration test for this scenario.

---------

Co-authored-by: Nicolas Chaulet <[email protected]>
Co-authored-by: Kibana Machine <[email protected]>

x-pack/plugins/fleet/server/services/package_policy.ts

@@ -513,8 +513,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
     esClient: ElasticsearchClient,
     id: string,
     packagePolicyUpdate: UpdatePackagePolicy,
-    options?: { user?: AuthenticatedUser; force?: boolean; skipUniqueNameVerification?: boolean },
-    currentVersion?: string
+    options?: { user?: AuthenticatedUser; force?: boolean; skipUniqueNameVerification?: boolean }
   ): Promise<PackagePolicy> {
     let enrichedPackagePolicy: UpdatePackagePolicy;
 
@@ -567,8 +566,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
 
     inputs = enforceFrozenInputs(oldPackagePolicy.inputs, inputs, options?.force);
     let elasticsearchPrivileges: NonNullable<PackagePolicy['elasticsearch']>['privileges'];
+    let pkgInfo;
     if (packagePolicy.package?.name) {
-      const pkgInfo = await getPackageInfo({
+      pkgInfo = await getPackageInfo({
         savedObjectsClient: soClient,
         pkgName: packagePolicy.package.name,
         pkgVersion: packagePolicy.package.version,
@@ -609,6 +609,34 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
 
     const newPolicy = (await this.get(soClient, id)) as PackagePolicy;
 
+    // if we have moved to an input package we need to create the index templates
+    // for the package policy as input packages create index templates per package policy
+    if (
+      pkgInfo &&
+      pkgInfo.type === 'input' &&
+      oldPackagePolicy.package &&
+      oldPackagePolicy.package?.version !== pkgInfo.version
+    ) {
+      if (oldPackagePolicy.package) {
+        const oldPackage = await getPackageInfo({
+          savedObjectsClient: soClient,
+          pkgName: oldPackagePolicy.package?.name,
+          pkgVersion: oldPackagePolicy.package?.version,
+          prerelease: true,
+        });
+
+        if (oldPackage.type === 'integration') {
+          await installAssetsForInputPackagePolicy({
+            logger: appContextService.getLogger(),
+            soClient,
+            esClient,
+            pkgInfo,
+            packagePolicy: newPolicy,
+            force: true,
+          });
+        }
+      }
+    }
     // Bump revision of associated agent policy
     const bumpPromise = agentPolicyService.bumpRevision(
       soClient,
@@ -1076,14 +1104,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
       ...options,
     };
 
-    await this.update(
-      soClient,
-      esClient,
-      id,
-      updatePackagePolicy,
-      updateOptions,
-      packagePolicy.package!.version
-    );
+    await this.update(soClient, esClient, id, updatePackagePolicy, updateOptions);
 
     // Persist any experimental feature opt-ins that come through the upgrade process to the Installation SO
     await updateDatastreamExperimentalFeatures(

x-pack/test/fleet_api_integration/apis/epm/get.ts

@@ -152,7 +152,7 @@ export default function (providerContext: FtrProviderContext) {
       // not from the package registry. This is because they contain a field the registry
       // does not support
       const res = await supertest
-        .get(`/api/fleet/epm/packages/integration_to_input/0.9.1?prerelease=true`)
+        .get(`/api/fleet/epm/packages/integration_to_input/2.0.0`)
         .expect(200);
 
       const packageInfo = res.body.item;

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "1.0.0"
+  changes:
+    - description: test
+      type: enhancement
+      link: http://some.url

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/agent/stream/stream.yml.hbs

@@ -0,0 +1,9 @@
+paths:
+{{#each paths}}
+  - {{this}}
+{{/each}}
+
+data_stream:
+  dataset: {{data_stream.dataset}}
+
+{{custom}}

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/manifest.yml

@@ -0,0 +1,28 @@
+title: Log Dataset
+type: logs
+streams:
+  - input: logfile
+    description: Collect your custom log files.
+    title: Collect log files
+    vars:
+      - name: paths
+        required: true
+        title: Log file path
+        description: Path to log files to be collected
+        type: text
+        multi: true
+      - name: data_stream.dataset
+        required: true
+        default: generic
+        title: Dataset name
+        description: >
+          Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html).
+
+        type: text
+      - name: custom
+        title: Custom configurations
+        description: >
+          Here YAML configuration options can be used to be added to your configuration. Be careful using this as it might break your configuration file.
+
+        type: yaml
+        default: ""

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/docs/README.md

@@ -0,0 +1,3 @@
+# Log Package
+
+The log package is used as a generic package based on which any log file can be tailed by adjusting the ingest pipeline.

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/manifest.yml

@@ -0,0 +1,24 @@
+format_version: 1.0.0
+name: integration_to_input
+title: Custom Logs
+description: >-
+  This package goes from an integration package to an input package
+type: integration
+version: 1.0.0
+release: ga
+license: basic
+categories:
+  - custom
+policy_templates:
+  - name: logs
+    title: Custom logs
+    description: Collect your custom log files.
+    inputs:
+      - type: logfile
+        title: Custom log file
+        description: Collect your custom log files.
+icons:
+  - src: "/img/icon.svg"
+    type: "image/svg+xml"
+owner:
+  github: elastic/elastic-agent-data-plane

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/agent/input/input.yml.hbs

@@ -0,0 +1,18 @@
+paths:
+{{#each paths}}
+  - {{this}}
+{{/each}}
+
+{{#if tags}}
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{/if}}
+
+{{#if pipeline}}
+pipeline: {{pipeline}}
+{{/if}}
+
+data_stream:
+  dataset: {{data_stream.dataset}}

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/changelog.yml

@@ -0,0 +1,11 @@
+# newer versions go on top
+- version: "2.0.0"
+  changes:
+    - description: Changed to input pkg
+      type: enhancement
+      link: http://some.url
+- version: "1.0.0"
+  changes:
+    - description: test
+      type: enhancement
+      link: http://some.url

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/docs/README.md

@@ -0,0 +1 @@
+# Custom Logs

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/fields/input.yml

@@ -0,0 +1,4 @@
+- name: input.name
+  type: constant_keyword
+  description: Sample field to be added.
+  value: logs

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/manifest.yml

@@ -0,0 +1,45 @@
+format_version: 1.0.0
+name: integration_to_input
+title: Custom Logs
+description: >-
+  This package goes from an integration package to an input package
+type: input
+version: 2.0.0
+license: basic
+categories:
+  - custom
+policy_templates:
+  - name: logs
+    type: logs
+    title: Custom log file
+    description: Collect your custom log files.
+    input: logfile
+    template_path: input.yml.hbs
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        required: true
+        show_user: true
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        required: true
+        show_user: false
+      - name: ignore_older
+        type: text
+        title: Ignore events older than
+        required: false
+        default: 72h
+icons:
+  - src: "/img/sample-logo.svg"
+    type: "image/svg+xml"
+screenshots:
+  - src: "/img/sample-screenshot.png"
+    title: "Sample screenshot"
+    size: "600x600"
+    type: "image/png"
+owner:
+  github: elastic/integrations

x-pack/test/fleet_api_integration/apis/package_policy/create.ts

@@ -455,7 +455,7 @@ export default function (providerContext: FtrProviderContext) {
             policy_id: agentPolicyId,
             package: {
               name: 'integration_to_input',
-              version: '0.9.1',
+              version: '2.0.0',
             },
             name: 'integration_to_input-1',
             description: '',
@@ -486,7 +486,7 @@ export default function (providerContext: FtrProviderContext) {
             policy_id: agentPolicyId,
             package: {
               name: 'integration_to_input',
-              version: '0.9.1',
+              version: '2.0.0',
             },
             name: 'integration_to_input-2',
             description: '',