From e3c05d6503cec405cd0d5eb650e937e863d3566b Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 6 Nov 2023 13:22:07 -0600 Subject: [PATCH] [build] Create CDN assets (#169707) Closes https://github.com/elastic/kibana/issues/169427 Adds a new build step `createCdnAssets` that will create an archive `kibana--cdn-assets.tar.gz` with static assets organized using the request structure of the kibana client. - By default CDN assets are created - Adding the flag `node scripts/build --skip-cdn-assets` will skip creation - `ci:build-cdn-assets` can be used to create and upload the archive for testing Testing: see https://github.com/elastic/kibana/pull/169408. Builds are available in the artifacts tab on the `Build Distribution` step. 1) Extract builds 2) ``` python3 -m http.server -b localhost -d kibana-8.12.0-SNAPSHOT-cdn-assets 8000 ``` 3) ``` echo 'server.cdn.url: http://localhost:8000' >> kibana-8.12.0-SNAPSHOT/config/kibana.yml ``` 4) ``` kibana-8.12.0-SNAPSHOT/bin/kibana ``` --- .buildkite/scripts/build_kibana.sh | 2 + .../scripts/steps/artifacts/docker_image.sh | 3 +- .../scripts/steps/package_testing/build.sh | 2 +- src/dev/build/args.test.ts | 7 ++ src/dev/build/args.ts | 2 + src/dev/build/build_distributables.ts | 5 + src/dev/build/tasks/create_cdn_assets_task.ts | 102 ++++++++++++++++++ src/dev/build/tasks/index.ts | 1 + 8 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 src/dev/build/tasks/create_cdn_assets_task.ts diff --git a/.buildkite/scripts/build_kibana.sh b/.buildkite/scripts/build_kibana.sh index 01810d26758c2..66e53f80bdf5a 100755 --- a/.buildkite/scripts/build_kibana.sh +++ b/.buildkite/scripts/build_kibana.sh @@ -14,6 +14,7 @@ is_pr_with_label "ci:build-docker-cross-compile" && BUILD_ARGS+=("--docker-cross is_pr_with_label "ci:build-os-packages" || BUILD_ARGS+=("--skip-os-packages") is_pr_with_label "ci:build-canvas-shareable-runtime" || BUILD_ARGS+=("--skip-canvas-shareable-runtime") is_pr_with_label "ci:build-docker-contexts" || BUILD_ARGS+=("--skip-docker-contexts") +is_pr_with_label "ci:build-cdn-assets" || BUILD_ARGS+=("--skip-cdn-assets") echo "> node scripts/build" "${BUILD_ARGS[@]}" node scripts/build "${BUILD_ARGS[@]}" @@ -24,6 +25,7 @@ if is_pr_with_label "ci:build-cloud-image"; then --skip-initialize \ --skip-generic-folders \ --skip-platform-folders \ + --skip-cdn-assets \ --skip-archives \ --docker-images \ --docker-tag-qualifier="$GIT_COMMIT" \ diff --git a/.buildkite/scripts/steps/artifacts/docker_image.sh b/.buildkite/scripts/steps/artifacts/docker_image.sh index 9b4edc16024f0..0518e17f6524e 100755 --- a/.buildkite/scripts/steps/artifacts/docker_image.sh +++ b/.buildkite/scripts/steps/artifacts/docker_image.sh @@ -37,7 +37,8 @@ node scripts/build \ --skip-docker-ubuntu \ --skip-docker-ubi \ --skip-docker-cloud \ - --skip-docker-contexts + --skip-docker-contexts \ + --skip-cdn-assets echo "--- Tag images" docker rmi "$KIBANA_IMAGE" diff --git a/.buildkite/scripts/steps/package_testing/build.sh b/.buildkite/scripts/steps/package_testing/build.sh index 3cd7b3d154989..892b8bf312d51 100755 --- a/.buildkite/scripts/steps/package_testing/build.sh +++ b/.buildkite/scripts/steps/package_testing/build.sh @@ -4,7 +4,7 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh -node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-serverless --skip-docker-ubi --skip-docker-contexts +node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-serverless --skip-docker-ubi --skip-docker-contexts --skip-cdn-assets DOCKER_FILE="kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz" diff --git a/src/dev/build/args.test.ts b/src/dev/build/args.test.ts index 4f351bba980b2..e379ae02ceba3 100644 --- a/src/dev/build/args.test.ts +++ b/src/dev/build/args.test.ts @@ -29,6 +29,7 @@ it('build default and oss dist for current platform, without packages, by defaul "buildOptions": Object { "buildCanvasShareableRuntime": true, "createArchives": true, + "createCdnAssets": true, "createDebPackage": false, "createDockerCloud": false, "createDockerContexts": true, @@ -67,6 +68,7 @@ it('builds packages if --all-platforms is passed', () => { "buildOptions": Object { "buildCanvasShareableRuntime": true, "createArchives": true, + "createCdnAssets": true, "createDebPackage": true, "createDockerCloud": true, "createDockerContexts": true, @@ -105,6 +107,7 @@ it('limits packages if --rpm passed with --all-platforms', () => { "buildOptions": Object { "buildCanvasShareableRuntime": true, "createArchives": true, + "createCdnAssets": true, "createDebPackage": false, "createDockerCloud": false, "createDockerContexts": true, @@ -143,6 +146,7 @@ it('limits packages if --deb passed with --all-platforms', () => { "buildOptions": Object { "buildCanvasShareableRuntime": true, "createArchives": true, + "createCdnAssets": true, "createDebPackage": true, "createDockerCloud": false, "createDockerContexts": true, @@ -182,6 +186,7 @@ it('limits packages if --docker passed with --all-platforms', () => { "buildOptions": Object { "buildCanvasShareableRuntime": true, "createArchives": true, + "createCdnAssets": true, "createDebPackage": false, "createDockerCloud": true, "createDockerContexts": true, @@ -228,6 +233,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform "buildOptions": Object { "buildCanvasShareableRuntime": true, "createArchives": true, + "createCdnAssets": true, "createDebPackage": false, "createDockerCloud": true, "createDockerContexts": true, @@ -267,6 +273,7 @@ it('limits packages if --all-platforms passed with --skip-docker-ubuntu', () => "buildOptions": Object { "buildCanvasShareableRuntime": true, "createArchives": true, + "createCdnAssets": true, "createDebPackage": true, "createDockerCloud": true, "createDockerContexts": true, diff --git a/src/dev/build/args.ts b/src/dev/build/args.ts index 68e111912722f..8f16d5ce571c0 100644 --- a/src/dev/build/args.ts +++ b/src/dev/build/args.ts @@ -16,6 +16,7 @@ export function readCliArgs(argv: string[]) { const flags = getopts(argv, { boolean: [ 'skip-archives', + 'skip-cdn-assets', 'skip-initialize', 'skip-generic-folders', 'skip-platform-folders', @@ -132,6 +133,7 @@ export function readCliArgs(argv: string[]) { createGenericFolders: !Boolean(flags['skip-generic-folders']), createPlatformFolders: !Boolean(flags['skip-platform-folders']), createArchives: !Boolean(flags['skip-archives']), + createCdnAssets: !Boolean(flags['skip-cdn-assets']), createRpmPackage: isOsPackageDesired('rpm'), createDebPackage: isOsPackageDesired('deb'), createDockerUbuntu: diff --git a/src/dev/build/build_distributables.ts b/src/dev/build/build_distributables.ts index 23203491e0d13..b324780e15672 100644 --- a/src/dev/build/build_distributables.ts +++ b/src/dev/build/build_distributables.ts @@ -26,6 +26,7 @@ export interface BuildOptions { createGenericFolders: boolean; createPlatformFolders: boolean; createArchives: boolean; + createCdnAssets: boolean; createRpmPackage: boolean; createDebPackage: boolean; createDockerUBI: boolean; @@ -113,6 +114,10 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions await run(Tasks.AssertPathLength); await run(Tasks.AssertNoUUID); } + // control w/ --skip-cdn-assets + if (options.createCdnAssets) { + await run(Tasks.CreateCdnAssets); + } /** * package platform-specific builds into archives diff --git a/src/dev/build/tasks/create_cdn_assets_task.ts b/src/dev/build/tasks/create_cdn_assets_task.ts new file mode 100644 index 0000000000000..e03e1f6775672 --- /dev/null +++ b/src/dev/build/tasks/create_cdn_assets_task.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { readFileSync } from 'fs'; +import { access } from 'fs/promises'; + +import { resolve, dirname } from 'path'; +import { asyncForEach } from '@kbn/std'; +import { Jsonc } from '@kbn/repo-packages'; + +import del from 'del'; +import globby from 'globby'; + +import { mkdirp, compressTar, Task, copyAll } from '../lib'; + +export const CreateCdnAssets: Task = { + description: 'Creating CDN assets', + + async run(config, log, build) { + const buildSource = build.resolvePath(); + const buildNum = config.getBuildNumber(); + const buildVersion = config.getBuildVersion(); + const assets = config.resolveFromRepo('build', 'cdn-assets'); + const bundles = resolve(assets, String(buildNum), 'bundles'); + + await del(assets); + await mkdirp(assets); + + // Plugins + + const plugins = globby.sync([`${buildSource}/node_modules/@kbn/**/*/kibana.jsonc`]); + await asyncForEach(plugins, async (path) => { + const manifest = Jsonc.parse(readFileSync(path, 'utf8')) as any; + if (manifest?.plugin?.id) { + const pluginRoot = resolve(dirname(path)); + + try { + // packages/core/plugins/core-plugins-server-internal/src/plugins_service.ts + const assetsSource = resolve(pluginRoot, 'assets'); + const assetsDest = resolve('plugins', manifest.plugin.id, 'assets'); + await access(assetsSource); + await mkdirp(assetsDest); + await copyAll(assetsSource, assetsDest); + } catch (e) { + // assets are optional + if (!(e.code === 'ENOENT' && e.syscall === 'access')) throw e; + } + + try { + // packages/core/apps/core-apps-server-internal/src/bundle_routes/register_bundle_routes.ts + const bundlesSource = resolve(pluginRoot, 'target', 'public'); + const bundlesDest = resolve(bundles, 'plugin', manifest.plugin.id, '1.0.0'); + await access(bundlesSource); + await mkdirp(bundlesDest); + await copyAll(bundlesSource, bundlesDest); + } catch (e) { + // bundles are optional + if (!(e.code === 'ENOENT' && e.syscall === 'access')) throw e; + } + } + }); + + // packages/core/apps/core-apps-server-internal/src/bundle_routes/register_bundle_routes.ts + await copyAll( + resolve(buildSource, 'node_modules/@kbn/ui-shared-deps-npm/shared_built_assets'), + resolve(bundles, 'kbn-ui-shared-deps-npm') + ); + await copyAll( + resolve(buildSource, 'node_modules/@kbn/ui-shared-deps-src/shared_built_assets'), + resolve(bundles, 'kbn-ui-shared-deps-src') + ); + await copyAll( + resolve(buildSource, 'node_modules/@kbn/core/target/public'), + resolve(bundles, 'core') + ); + await copyAll(resolve(buildSource, 'node_modules/@kbn/monaco'), resolve(bundles, 'kbn-monaco')); + + // packages/core/apps/core-apps-server-internal/src/core_app.ts + await copyAll( + resolve(buildSource, 'node_modules/@kbn/core-apps-server-internal/assets'), + resolve(assets, 'ui') + ); + + await compressTar({ + source: assets, + destination: config.resolveFromTarget(`kibana-${buildVersion}-cdn-assets.tar.gz`), + archiverOptions: { + gzip: true, + gzipOptions: { + level: 9, + }, + }, + createRootDirectory: true, + rootDirectoryName: `kibana-${buildVersion}-cdn-assets`, + }); + }, +}; diff --git a/src/dev/build/tasks/index.ts b/src/dev/build/tasks/index.ts index 9abd34a2d9f51..e62c09e637d6a 100644 --- a/src/dev/build/tasks/index.ts +++ b/src/dev/build/tasks/index.ts @@ -14,6 +14,7 @@ export * from './clean_tasks'; export * from './copy_legacy_source_task'; export * from './create_archives_sources_task'; export * from './create_archives_task'; +export * from './create_cdn_assets_task'; export * from './create_empty_dirs_and_files_task'; export * from './create_readme_task'; export * from './download_cloud_dependencies';