From 93ec3cdd4016315942cd16b342b1587c399c1d20 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Thu, 9 Feb 2023 10:11:48 +0000 Subject: [PATCH] [Fleet] Create templates and pipelines when updating package of a single package policy from type integration to input. (#150199) ## Summary Part of #149423 When upgrading a single package policy with conflicts, we use the update package API. When we update from an input package to an integration package, we need to create the per-policy assets. This PR detects if an update is changing from an integration package to an input package and creates the assets as part of the update. I've added an integration test for this scenario. --------- Co-authored-by: Nicolas Chaulet Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../fleet/server/services/package_policy.ts | 43 +++++-- .../fleet_api_integration/apis/epm/get.ts | 2 +- .../integration_to_input/1.0.0/changelog.yml | 6 + .../log/agent/stream/stream.yml.hbs | 9 ++ .../data_stream/log/fields/agent.yml} | Bin 12295 -> 6381 bytes .../data_stream/log/fields/base-fields.yml | 12 ++ .../1.0.0/data_stream/log/manifest.yml | 28 +++++ .../integration_to_input/1.0.0/docs/README.md | 3 + .../integration_to_input/1.0.0/img/icon.svg | 4 + .../integration_to_input/1.0.0/manifest.yml | 24 ++++ .../2.0.0/agent/input/input.yml.hbs | 18 +++ .../integration_to_input/2.0.0/changelog.yml | 11 ++ .../integration_to_input/2.0.0/docs/README.md | 1 + .../2.0.0/fields/input.yml | 4 + .../2.0.0/img/sample-logo.svg | 1 + .../img/sample-screenshot.png} | Bin 30980 -> 18849 bytes .../integration_to_input/2.0.0/manifest.yml | 45 +++++++ .../apis/package_policy/create.ts | 4 +- .../apis/package_policy/update.ts | 111 ++++++++++++++++++ .../apis/package_policy/upgrade.ts | 4 +- 20 files changed, 314 insertions(+), 16 deletions(-) create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/changelog.yml create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/agent/stream/stream.yml.hbs rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/{integration_to_input-0.9.0.zip => 1.0.0/data_stream/log/fields/agent.yml} (51%) create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/fields/base-fields.yml create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/manifest.yml create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/docs/README.md create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/img/icon.svg create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/manifest.yml create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/agent/input/input.yml.hbs create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/changelog.yml create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/docs/README.md create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/fields/input.yml create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/img/sample-logo.svg rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/{integration_to_input-0.9.1.zip => 2.0.0/img/sample-screenshot.png} (60%) create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/manifest.yml diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index c665d9a6f6f97..5eacf8dd0c3a8 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -513,8 +513,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { esClient: ElasticsearchClient, id: string, packagePolicyUpdate: UpdatePackagePolicy, - options?: { user?: AuthenticatedUser; force?: boolean; skipUniqueNameVerification?: boolean }, - currentVersion?: string + options?: { user?: AuthenticatedUser; force?: boolean; skipUniqueNameVerification?: boolean } ): Promise { let enrichedPackagePolicy: UpdatePackagePolicy; @@ -567,8 +566,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient { inputs = enforceFrozenInputs(oldPackagePolicy.inputs, inputs, options?.force); let elasticsearchPrivileges: NonNullable['privileges']; + let pkgInfo; if (packagePolicy.package?.name) { - const pkgInfo = await getPackageInfo({ + pkgInfo = await getPackageInfo({ savedObjectsClient: soClient, pkgName: packagePolicy.package.name, pkgVersion: packagePolicy.package.version, @@ -609,6 +609,34 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const newPolicy = (await this.get(soClient, id)) as PackagePolicy; + // if we have moved to an input package we need to create the index templates + // for the package policy as input packages create index templates per package policy + if ( + pkgInfo && + pkgInfo.type === 'input' && + oldPackagePolicy.package && + oldPackagePolicy.package?.version !== pkgInfo.version + ) { + if (oldPackagePolicy.package) { + const oldPackage = await getPackageInfo({ + savedObjectsClient: soClient, + pkgName: oldPackagePolicy.package?.name, + pkgVersion: oldPackagePolicy.package?.version, + prerelease: true, + }); + + if (oldPackage.type === 'integration') { + await installAssetsForInputPackagePolicy({ + logger: appContextService.getLogger(), + soClient, + esClient, + pkgInfo, + packagePolicy: newPolicy, + force: true, + }); + } + } + } // Bump revision of associated agent policy const bumpPromise = agentPolicyService.bumpRevision( soClient, @@ -1076,14 +1104,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { ...options, }; - await this.update( - soClient, - esClient, - id, - updatePackagePolicy, - updateOptions, - packagePolicy.package!.version - ); + await this.update(soClient, esClient, id, updatePackagePolicy, updateOptions); // Persist any experimental feature opt-ins that come through the upgrade process to the Installation SO await updateDatastreamExperimentalFeatures( diff --git a/x-pack/test/fleet_api_integration/apis/epm/get.ts b/x-pack/test/fleet_api_integration/apis/epm/get.ts index 7d33b00f54c89..96bd1e01b69b1 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/get.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/get.ts @@ -152,7 +152,7 @@ export default function (providerContext: FtrProviderContext) { // not from the package registry. This is because they contain a field the registry // does not support const res = await supertest - .get(`/api/fleet/epm/packages/integration_to_input/0.9.1?prerelease=true`) + .get(`/api/fleet/epm/packages/integration_to_input/2.0.0`) .expect(200); const packageInfo = res.body.item; diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/changelog.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/changelog.yml new file mode 100644 index 0000000000000..0b37b5572dedf --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "1.0.0" + changes: + - description: test + type: enhancement + link: http://some.url diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/agent/stream/stream.yml.hbs b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000000..371933edee966 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/agent/stream/stream.yml.hbs @@ -0,0 +1,9 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} + +data_stream: + dataset: {{data_stream.dataset}} + +{{custom}} diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/integration_to_input-0.9.0.zip b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/fields/agent.yml similarity index 51% rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/integration_to_input-0.9.0.zip rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/fields/agent.yml index 4188e89d5b7581bdb89ae6d8d5ae70d897f60469..da4e652c53b8027c4302493bf093a76b6d1b301c 100644 GIT binary patch delta 9 QcmZoqcx$*>f#t0P02O8f;s5{u literal 12295 zcmcgyOK&7s74EzsG@C54K#CHsdJs;CyZYrfnFx@W>UQ^y>(NsW zw|lGzW)X=!3s9D5HtYZii50uBV+j%x2>1b!Sg~QlckZLQs$YI&oKAY&Rdwr}bI6&prQgi9Qc^SKgOD^gpkZzEKiE!n={3h%h)z!b1`C)1+Fr)~$N&{rfMJp6MJQN!jqiA=b zn)A?IyA0jwikRj)?|cdJiVHcF)oaI@WA*K%2RAuAy~b^)$K>TFpTIq;?DRD0i5M5B zr!{(rU*(+T$hYV+=E(-e=AKykHob6q>ZEZJ`uI)m^U7cU_W2QPOyAk=)4bR`JA!*| zJZ_(93jF@12ut%9!m-K%+ehR$UYNSjLn0Cnms|R>8-;0qgEdJ*`N(@m3d$nx*$HAsI#K9bAlwPVBnT65UATE-`+bkc z8|+qx-Qh_joHn-;CSn$)K|pUISJ(l&xz%L7Fi!B>WgVV4JswLNN~%nN6Fpwci3OQL zunrPiAnTC0WL##q!!*GqSom-a5pZjaIXK6XEvD;3PBV_-#aOs%dQyKgq_J7dqhsOl zY!U(q&*)3ZC@nt{Pmz-XlA7dbEuQ>s2o@A$Esz0aB9wL`-AX%|@w>;R0zKZYW?;=*eb&>LsEI*=+gfSVUn!8Qv;u=8TQ} zX7hu48B^gdvUkI<>v84JZ125y*)7~8X`ugzrGa>ua!4aZ5ImA9B%N^Oi5qNXWo31B zb!}~JeSKZ>GZAAS+rqQk!V}5x@KG4>E7Im>jt?U~q?JKWAje6sm0>XQ1B$4^1k$+5 z5sTG^QxG3PkWj6>IybnbIj~#XEbK_Pm9m;J8y8em8_m|z^2+Mkdfje2F7MQ*OH6L8 zUw+!Io5@T(W=m~+z)Xc=<%%S_qmq)0>)8V395!iMK6gdx7nIkJ!eh93bVX9$&8bV} zF2m3=!O4zoFGWmK1+WM4BC{W%HY~EP)0YwHimpg(FLbyaOgJ%KC`>2fT_C_Kn#_BW z_+`VaASbl&F?Vvgc3M^~B+tAi_1h##K94Ew7#|cqs`@C3&t!0m##2mokOs3V1Rgpc z0EUzE0}`lODSgJkg50fH&|qQ94ARv{T3lv(;^=b}U8=>AuS$U_z?{W>?ud?XMw}s^ zs{SNr9HtR!C{ju8bzKBdabj-x^|;pujYy#rRp#43ckYOAtZ;%I45=n30)4^xisyLI z1RE3i(=>m+g89BGLz%)+kfzy~%yLC8pZ7ZHP|7nhOAhRqwNv3G076C4Py)H9f^$BS z2-_41T7GqLI89}RXi(0ocog|wp)y7eOiZjSAgFplejN)t>S|^fY$||?%8W_|Sm`q8 z&}L}Bcw~EUMIX&JA=**oN#)$!x+mYXzGwV7?uDu6vNmV^u%9Z#JrGF`gaKwfP}LZg zE0%!B1|M)HaX0JP{XQ5g>?0B7XoL@VfJ{t~ijkcY`-le?qxs3ILom$oQUKka9k~PK z??na>mZX@4E`Z49+o5v=M3wn4C3s--+i9Ce0Y|6Kf)J)XXPDU$pm+j*fnY_Ke<}gD z*M|+H^5bEga33jZ#83HIuA-i-tsJf>q-%1c8~SJ?ijqA&{G5+Q#fh@s$YK&k!Lfji zxexv3knlT5u^PLgss`)>5xC)i`W=i1?pJ<9>6C~pEW~WnNyIT{+aiLx!w5KB&W%9w zU7p;ywRg|lk$O@lPNUKt0S~W9Co)i_gRJqD9;uE>ek-f`+N8H^lb!-4ZwdW zJC_aCvAtM%fKffkJsJE@#Vr=d=ZSV6Xyuf9D}fWEZh79&IC>1TphS^E9~j9VP;mR? z{Sk))tM1XL+=PtoG|a57q^>OJ!a{<{nxb{g*rw`_d0%$yXJ7`Y{l33$MfA5c$k5$^}%TId%;YQn0* z3@B6yG8+?pX1i|0;}}^+k}y*nO{cda(PCjYrRTO+1RbY^5qIRVHdfyoxu@^OmcCm`rYY- z$vAP}b|_RYPC9#gM10`~wB&**`~A_f3q&9Wg^Ay!T_beddaxx;`0Lv5{&D-6Qi;B^ zO}Kh769TV`4n!&#PPh8IVV7-d<*NB(cQ!67gUF&+g@>b^-(Yyq5uVCvg_&%Ho`(iY zp=Py{NQ2SiW4}ukrfiW-jm(}xDGWd=%F3&W9l6ITpVEkkKTOfYx!4zvQj=A)<%(U+ zOKyy2?-WGb#||n%wTv-cAS*gb^ep!nCn%J02?GkOjg;hCdXL=fr)KJP-{f*NNy-Wct>%P>mgwPlnqdf4KSQ+h-@bT zG?7C6=%$Bl8sWs879&6fr+0?sCOGvN9nnqI*%7n+Hx|5k z8|+|bkA1j#XSet!3*}J`EKR5a3?0hdWOyB#IrUpwtH94Dy0hh%fGb#{WF}3pzmyTeS7{q zg+FopW!f=_GmhsF6#>_=;^S_ntG950_;!=t^-8N*VLeU`qkbO?KDZH{T(8twowb_y zzd{bZM)(#rWJB`(YM(d+H%_Y`Y!<}Y*r@3^{So`SC8f&)bcXOF; zw>~nmU8^@%w^v)0TAn!(Z8of>2CMHZZ?&w}Qme6ySv6Th|HkwVn(H`8(W+Z3%PZ^5 z48p{hG4-A1>Q-}2KcZ2ifgrPfux2fZ+u(gx9U*6o~){>IV%>AG3v=n7)3ixvJiVNRs+U6PV@dQiC+{ z7MbI1a>)OF_}TBj09K&yY`k!WD^0}*)Fax-q3(ciBEqU$^)hAt4J7c%vvp-t;#Cqw z%*N$n*{UMn-lyKeL}xd|T8dn;M~2MN%mz?dq9=eK=3ZIl^xWujCZar8SH{*DR{bGm z*DiZFnG(=%Bm$IL2QaJ~MmT;!$407HzN1r5<$j29!$T~VQ?qIws;MGfs1i_|UUs`M z#d825Ihh>V$uL(2t#VjkOX58Rkoar)%%OS>B}7H;j*)A6{arilRxF`IC-xX48D0JialX;`9@o^8}_KQXdldy9l^B0+Ujv z9G;x54mv(LqMn?eI-Db4duMZ?iGkTbllQ>o;?2KOR!U1Ck|iE^9ecwIhref?m}}_-Sq5P=G%EX6KOtA9{=%DjHGT0 zJ)oJI(QCf@q-6s0dGh$xS1&WQS&IhaeDf%jdRy~(^7vn`U5M}0N@lnb)$udXglyG^ zf_nejd~Rbnx0s6Z*pQH}`8;|2i*t~j#J%&xCWJW>`zPO-EA|-PEJ~dR5TguNDm{7p o`Poz_@TZa4I8?qE6sfJ7U3KO+!DqyPW_ diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/fields/base-fields.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/fields/base-fields.yml new file mode 100644 index 0000000000000..7c798f4534ca5 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/manifest.yml new file mode 100644 index 0000000000000..c3d93e36b7cf3 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/data_stream/log/manifest.yml @@ -0,0 +1,28 @@ +title: Log Dataset +type: logs +streams: + - input: logfile + description: Collect your custom log files. + title: Collect log files + vars: + - name: paths + required: true + title: Log file path + description: Path to log files to be collected + type: text + multi: true + - name: data_stream.dataset + required: true + default: generic + title: Dataset name + description: > + Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). + + type: text + - name: custom + title: Custom configurations + description: > + Here YAML configuration options can be used to be added to your configuration. Be careful using this as it might break your configuration file. + + type: yaml + default: "" diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/docs/README.md b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/docs/README.md new file mode 100644 index 0000000000000..ddc05a316388b --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/docs/README.md @@ -0,0 +1,3 @@ +# Log Package + +The log package is used as a generic package based on which any log file can be tailed by adjusting the ingest pipeline. diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/img/icon.svg b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/img/icon.svg new file mode 100644 index 0000000000000..173fdec5072e9 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/img/icon.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/manifest.yml new file mode 100644 index 0000000000000..4cee11c7c996b --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/1.0.0/manifest.yml @@ -0,0 +1,24 @@ +format_version: 1.0.0 +name: integration_to_input +title: Custom Logs +description: >- + This package goes from an integration package to an input package +type: integration +version: 1.0.0 +release: ga +license: basic +categories: + - custom +policy_templates: + - name: logs + title: Custom logs + description: Collect your custom log files. + inputs: + - type: logfile + title: Custom log file + description: Collect your custom log files. +icons: + - src: "/img/icon.svg" + type: "image/svg+xml" +owner: + github: elastic/elastic-agent-data-plane diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/agent/input/input.yml.hbs b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/agent/input/input.yml.hbs new file mode 100644 index 0000000000000..1ba86fa98a2f8 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/agent/input/input.yml.hbs @@ -0,0 +1,18 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} + +{{#if tags}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{/if}} + +{{#if pipeline}} +pipeline: {{pipeline}} +{{/if}} + +data_stream: + dataset: {{data_stream.dataset}} \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/changelog.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/changelog.yml new file mode 100644 index 0000000000000..ef016b6ede8fd --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/changelog.yml @@ -0,0 +1,11 @@ +# newer versions go on top +- version: "2.0.0" + changes: + - description: Changed to input pkg + type: enhancement + link: http://some.url +- version: "1.0.0" + changes: + - description: test + type: enhancement + link: http://some.url diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/docs/README.md b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/docs/README.md new file mode 100644 index 0000000000000..9f29c89e0f5ef --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/docs/README.md @@ -0,0 +1 @@ +# Custom Logs \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/fields/input.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/fields/input.yml new file mode 100644 index 0000000000000..f5851c64b6b3a --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/fields/input.yml @@ -0,0 +1,4 @@ +- name: input.name + type: constant_keyword + description: Sample field to be added. + value: logs \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/img/sample-logo.svg b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/img/sample-logo.svg new file mode 100644 index 0000000000000..6268dd88f3b3d --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/img/sample-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/integration_to_input-0.9.1.zip b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/img/sample-screenshot.png similarity index 60% rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/integration_to_input-0.9.1.zip rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/integration_to_input/2.0.0/img/sample-screenshot.png index 560755894bfa4f3846812c13baed4c7725ff1345..d7a56a3ecc078c38636698cefba33f86291dd178 100644 GIT binary patch delta 11 ScmZqq#JF%WKdf~WJ8`5aiDWj}v$s9do@u3LdYJBs zoy}SuOGum$LYxp>B8;?0B>n&nAgx$#oY1bsffIW`2*hnA;PQ~kM^e46N|Ff2D+&*>sqC)A{ckMUCk34%tIj@BMH0TWdG>rS3X}lTs z2cxv9a*u0EZ`?nl)PBpav?PafGdO%>F$aF*-ez_$jfX+W3ruQjqqe3f7x0sKQu)$( zWHS=c={l5N%4?`|P4Gygj&`pV_XbmKv#I;zzCDq@2yq&}Gm5WtltAe#(_{F-B()dc zxUm(*Tc|y^b^e{pfB)|9R;OFEr(0xGO$Obnn~;3vw9-~$rKKcOF7(+dtt$KnUoC}~ zeS7g(bd{bJ%x4IGS=m-XSRSFxv~aR=f?ieN$Cpb5rrL~^$lD0}CmPLAjZ-Q!}MADez5jJ;ro0|Beyt8KO?`p8?7-R(&$?$1|MLxzEV* z1?Ha&`(L2JQ>;$xp918VZ%^=lk$}@W!EjM|Eia$QQY<0J#MS`iX;dQtf82;T4<(~2 zl-Kg|i7ZirRFi#C%D%OH?@^Q8QP1*r*Yfg-EM<44CPafzF1kir7aISX5Ra1K2B~&U z6vvXC_}{U>JXgpG{&S&OKsY677bqTBpMO5EfI~q#!M{M_<0((@|9EgOq<%W^ubsYe z@5UNz${)3bb!AD9UWV&UT$HHY6DnbU*#|$wqZy6q(fkEmBB7qq50uyP@`)@_gY>@y z1_J+?kpVIO{|5%2j`0`6@Lbu6{m%v43+wa(@ZY$7_T1Jx{zs=VVd>!HptZJS0k4XGdj#i)J$`ug{8_(*l!-4onXoz^&;NgO3K$J13&F1%k{^P zF9m+9%f#*8UY)Tfd;B= zGrBGsfsX2~uiaA@RI=Z;!^8 z-+J#|I_{@y{`;nHFUGOe_4}P5iaUf~j$L-8b&2(ZonXiw2EzpJxf0fiS=?u7Jg7H| z%4OCNZ@n78RI&kN45`W5LDCwA1A0Za%&rR&ZFFX0q}bp=N3>%seKZJ`Seq-roWgbj^q3YZFFfhu-Uk zz0Og_kWa9g$t4l0H=$n4lZTxWj@NL*cEWbrU1?;7-eBDz>~zzW2L0X+qiChkkNZJ` zJq&}LwRm@>p)!>jI{q7Mw-@!3l}0{Yc6N67j=|$$XGvF8bqSmstQGkQngpZp7SLsR z*=+9a-tmVW@ZFAEqr~5e0?6O?qXbpMnC@VEgZabIZ+1g~++S&=!%@(<`WmgMxoVeg z-n`jh?Ulw|Xr$TPaeSK_hRI|ORn?}(HM41O+tg62dv2Z!n!#1QsdIGpxawuDC|nqb ze%y78ro}zmS_RP}k|ij*&Mk;CZ68&tLj>fa51M;slbf1HwT7S}7c`T51~a&6+RWsR z>*aag&vBbPQ=L_lyM_T0M29-Uo+hmz5EJH4OL3_huP~TX!xv9f2WJFP!W^rBA z7{YAIi9lFbOmn$M++dH)i3Dbu;So1ogFFIJ=0X(LY^HHX7ckhuF65G>qdUw(2ZLwAwYuG~~pNa1z68Xap1nN9KZ-`^{T$tZa!~Ys$gauUncaeN4l^yG%1MdQJp2 zRLladYZ}9E+it?UG{bkeYniB_oCI*q(&2-a<)G3ti9|V(sM>788C221%czO#mHr}p z)3rUS6{M2l89G;8n|O&!a5K0^3c(p8a(aMdb4`Q7+`w4rppua@Y*f{?n`=gcWi*lG zNa`Bbb$1o~ZJ{x8k^#5ZneK3l?DW7}=0q|%tk9vTw1sr`L?8IQg>i;iXbj(yIdvh0 zZ49((M9|l^GeD@pXhO@vwDer731|&80m>k{E_tkOx^oPBw#TyoXma8{8K3B1IFwyoI!YHO?BO-ss!EMTOm zP}%gUqlw8uxsU-5@-P8miB5AyCnDv7m_aw1#hR$5Nkc}+K}SbSS3SZA7>2+}15Ua@ zIN25)Uxx>x!m{RI;~CdIbzSPHtF*Y$K5z(vmZL+9mQ&6oO(nTD zRk~V{llI)SGz^sKnh{*uMI%kOkkcw?foRUrOLea2N@*E)O_xDA%Qhl19YE=Z3ly;! ziHronUY&fzGhC_^YfrHhsTw(prrJc&1jNV%xM(kYO#?}XIWiX{otqA+nqgu_z}AB& z+gN2Fo?P1YDyq{{?JC7PyX)`$^jl{Yg|7Kg@H2K3^nA~s%m`dB20=gR#wj24#}DXV zJ-vnR!p@%i{m(xC{?l8ELf3qG&)5w0{C>EN*t8OKUukW}!=9gR<{|eotX8>NFQO!T zbCykgbXHHpG(xOXi2&;panjW$0;2oa>R|WNdVrlA!uTLz+e5^QejlM%`5{}`0MV47 z;G4Fht}dgpu%Dr_UaBWz|9TX*02Y+k@{_PtZ~0&u55qug1)2;o1%lK1AV#}Kn`wZ! z4J~q_G>sxPs96Mmf<=_j@-=YP2M$A9514qY3!YcE|qHT4Z@ zaV|t3p~`eY`P(Y}s<2w{NvdR7oVHYthv=i!A|33n&Ec5kU%ApZs*|6n&Qv#NNC!+Z zk+x597_~s#L#Itu7_!N4acPYmGyd=>jKyZNx|5rPq^onC+Sz!$oJLvd zE!mfQ-HGF<0-^%`>fr*N(G=-mfA33kAj;%N#25rs@N1RuOIhreV$ zePA($D5FEV>iGhlSrqADzwzZ~^F4fku}E(^r5Nk|qinH&!)F#_)k*W}Q2?D<6zO39 zPe)>m@y7#j;*q^bc{)=VEC2JaECgrjfMHVcBE-ok#-THYA|34i<=JfKoj4S_7Y- + This package goes from an integration package to an input package +type: input +version: 2.0.0 +license: basic +categories: + - custom +policy_templates: + - name: logs + type: logs + title: Custom log file + description: Collect your custom log files. + input: logfile + template_path: input.yml.hbs + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + - name: ignore_older + type: text + title: Ignore events older than + required: false + default: 72h +icons: + - src: "/img/sample-logo.svg" + type: "image/svg+xml" +screenshots: + - src: "/img/sample-screenshot.png" + title: "Sample screenshot" + size: "600x600" + type: "image/png" +owner: + github: elastic/integrations \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts index ed5122d02eb4f..1783a1fc53d09 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts @@ -455,7 +455,7 @@ export default function (providerContext: FtrProviderContext) { policy_id: agentPolicyId, package: { name: 'integration_to_input', - version: '0.9.1', + version: '2.0.0', }, name: 'integration_to_input-1', description: '', @@ -486,7 +486,7 @@ export default function (providerContext: FtrProviderContext) { policy_id: agentPolicyId, package: { name: 'integration_to_input', - version: '0.9.1', + version: '2.0.0', }, name: 'integration_to_input-2', description: '', diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/update.ts b/x-pack/test/fleet_api_integration/apis/package_policy/update.ts index 9c4b35030ced3..2206527667a41 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/update.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/update.ts @@ -7,6 +7,7 @@ import expect from '@kbn/expect'; import { policyFactory } from '@kbn/security-solution-plugin/common/endpoint/models/policy_config'; import type { NewPackagePolicy } from '@kbn/fleet-plugin/common'; +import { sortBy } from 'lodash'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; import { testUsers } from '../test_users'; @@ -17,12 +18,36 @@ export default function (providerContext: FtrProviderContext) { const dockerServers = getService('dockerServers'); const kibanaServer = getService('kibanaServer'); const esArchiver = getService('esArchiver'); + const es = getService('es'); + + const expectIdArraysEqual = (arr1: any[], arr2: any[]) => { + expect(sortBy(arr1, 'id')).to.eql(sortBy(arr2, 'id')); + }; + + const getInstallationSavedObject = async (name: string, version: string) => { + const res = await supertest.get(`/api/fleet/epm/packages/${name}-${version}`).expect(200); + return res.body.item.savedObject.attributes; + }; const getPackagePolicyById = async (id: string) => { const { body } = await supertest.get(`/api/fleet/package_policies/${id}`); return body; }; + const getComponentTemplate = async (name: string) => { + try { + const { component_templates: templates } = await es.cluster.getComponentTemplate({ name }); + + return templates?.[0] || null; + } catch (e) { + if (e.statusCode === 404) { + return null; + } + + throw e; + } + }; + const server = dockerServers.get('registry'); // use function () {} and not () => {} here // because `this` has to point to the Mocha context @@ -504,5 +529,91 @@ export default function (providerContext: FtrProviderContext) { ); }); }); + + describe('Input Packages', () => { + it('should install index templates when upgrading from input package to integration package', async () => { + const { body: packagePolicyResponse } = await supertest + .post(`/api/fleet/package_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + policy_id: agentPolicyId, + force: true, + package: { + name: 'integration_to_input', + version: '1.0.0', + }, + name: 'integration_to_input-1', + description: '', + namespace: 'default', + inputs: { + 'logs-logfile': { + enabled: true, + streams: { + 'integration_to_input.log': { + enabled: true, + vars: { + paths: ['/tmp/test.log'], + 'data_stream.dataset': 'somedataset', + custom: '', + }, + }, + }, + }, + }, + }); + + const inputPackagePolicyId = packagePolicyResponse.item.id; + + await supertest + .put(`/api/fleet/package_policies/${inputPackagePolicyId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + package: { + name: 'integration_to_input', + version: '2.0.0', + experimental_data_stream_features: [], + }, + name: 'integration_to_input-1', + namespace: 'default', + description: '', + policy_id: agentPolicyId, + vars: {}, + inputs: { + 'logs-logfile': { + enabled: true, + streams: { + 'integration_to_input.logs': { + enabled: true, + vars: { + paths: ['/tmp/test.log'], + 'data_stream.dataset': 'somedataset', + tags: ['tag1'], + ignore_older: '72h', + }, + }, + }, + }, + }, + }) + .expect(200); + + const installation = await getInstallationSavedObject('integration_to_input', '2.0.0'); + expectIdArraysEqual(installation.installed_es, [ + // assets from version 1.0.0 + { id: 'logs-integration_to_input.log', type: 'index_template' }, + { id: 'logs-integration_to_input.log-1.0.0', type: 'ingest_pipeline' }, + { id: 'logs-integration_to_input.log@custom', type: 'component_template' }, + { id: 'logs-integration_to_input.log@package', type: 'component_template' }, + // assets from version 2.0.0 for new package policy + { id: 'logs-somedataset-2.0.0', type: 'ingest_pipeline' }, + { id: 'logs-somedataset', type: 'index_template' }, + { id: 'logs-somedataset@package', type: 'component_template' }, + { id: 'logs-somedataset@custom', type: 'component_template' }, + ]); + + const dataset3PkgComponentTemplate = await getComponentTemplate('logs-somedataset@package'); + expect(dataset3PkgComponentTemplate).not.to.be(null); + }); + }); }); } diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts b/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts index 361f75fc9d369..bad75e9746e6a 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts @@ -1141,7 +1141,7 @@ export default function (providerContext: FtrProviderContext) { }); describe('when upgrading from an integration package to an input package where a required variable has been added', function () { - withTestPackage('integration_to_input', '0.9.1'); + withTestPackage('integration_to_input', '2.0.0'); beforeEach(async function () { const { body: agentPolicyResponse } = await supertest @@ -1162,7 +1162,7 @@ export default function (providerContext: FtrProviderContext) { policy_id: agentPolicyId, package: { name: 'integration_to_input', - version: '0.9.0', + version: '1.0.0', }, name: 'integration_to_input-1', description: '',