Skip to content

Commit

Permalink
[build] Create CDN assets (#169707)
Browse files Browse the repository at this point in the history
Closes #169427

Adds a new build step `createCdnAssets` that will create an archive
`kibana-<version>-cdn-assets.tar.gz` with static assets organized using
the request structure of the kibana client.

- By default CDN assets are created
- Adding the flag `node scripts/build --skip-cdn-assets` will skip
creation
- `ci:build-cdn-assets` can be used to create and upload the archive for
testing

Testing: see #169408. Builds are
available in the artifacts tab on the `Build Distribution` step.

1) Extract builds
2) ```
python3 -m http.server -b localhost -d kibana-8.12.0-SNAPSHOT-cdn-assets
8000
    ```
3) ```
echo 'server.cdn.url: http://localhost:8000' >>
kibana-8.12.0-SNAPSHOT/config/kibana.yml
    ```
4) ```
   kibana-8.12.0-SNAPSHOT/bin/kibana
   ```
  • Loading branch information
jbudz authored Nov 6, 2023
1 parent f3f85b4 commit e3c05d6
Show file tree
Hide file tree
Showing 8 changed files with 122 additions and 2 deletions.
2 changes: 2 additions & 0 deletions .buildkite/scripts/build_kibana.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ is_pr_with_label "ci:build-docker-cross-compile" && BUILD_ARGS+=("--docker-cross
is_pr_with_label "ci:build-os-packages" || BUILD_ARGS+=("--skip-os-packages")
is_pr_with_label "ci:build-canvas-shareable-runtime" || BUILD_ARGS+=("--skip-canvas-shareable-runtime")
is_pr_with_label "ci:build-docker-contexts" || BUILD_ARGS+=("--skip-docker-contexts")
is_pr_with_label "ci:build-cdn-assets" || BUILD_ARGS+=("--skip-cdn-assets")

echo "> node scripts/build" "${BUILD_ARGS[@]}"
node scripts/build "${BUILD_ARGS[@]}"
Expand All @@ -24,6 +25,7 @@ if is_pr_with_label "ci:build-cloud-image"; then
--skip-initialize \
--skip-generic-folders \
--skip-platform-folders \
--skip-cdn-assets \
--skip-archives \
--docker-images \
--docker-tag-qualifier="$GIT_COMMIT" \
Expand Down
3 changes: 2 additions & 1 deletion .buildkite/scripts/steps/artifacts/docker_image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ node scripts/build \
--skip-docker-ubuntu \
--skip-docker-ubi \
--skip-docker-cloud \
--skip-docker-contexts
--skip-docker-contexts \
--skip-cdn-assets

echo "--- Tag images"
docker rmi "$KIBANA_IMAGE"
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/scripts/steps/package_testing/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ set -euo pipefail

.buildkite/scripts/bootstrap.sh

node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-serverless --skip-docker-ubi --skip-docker-contexts
node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-serverless --skip-docker-ubi --skip-docker-contexts --skip-cdn-assets

DOCKER_FILE="kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz"

Expand Down
7 changes: 7 additions & 0 deletions src/dev/build/args.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ it('build default and oss dist for current platform, without packages, by defaul
"buildOptions": Object {
"buildCanvasShareableRuntime": true,
"createArchives": true,
"createCdnAssets": true,
"createDebPackage": false,
"createDockerCloud": false,
"createDockerContexts": true,
Expand Down Expand Up @@ -67,6 +68,7 @@ it('builds packages if --all-platforms is passed', () => {
"buildOptions": Object {
"buildCanvasShareableRuntime": true,
"createArchives": true,
"createCdnAssets": true,
"createDebPackage": true,
"createDockerCloud": true,
"createDockerContexts": true,
Expand Down Expand Up @@ -105,6 +107,7 @@ it('limits packages if --rpm passed with --all-platforms', () => {
"buildOptions": Object {
"buildCanvasShareableRuntime": true,
"createArchives": true,
"createCdnAssets": true,
"createDebPackage": false,
"createDockerCloud": false,
"createDockerContexts": true,
Expand Down Expand Up @@ -143,6 +146,7 @@ it('limits packages if --deb passed with --all-platforms', () => {
"buildOptions": Object {
"buildCanvasShareableRuntime": true,
"createArchives": true,
"createCdnAssets": true,
"createDebPackage": true,
"createDockerCloud": false,
"createDockerContexts": true,
Expand Down Expand Up @@ -182,6 +186,7 @@ it('limits packages if --docker passed with --all-platforms', () => {
"buildOptions": Object {
"buildCanvasShareableRuntime": true,
"createArchives": true,
"createCdnAssets": true,
"createDebPackage": false,
"createDockerCloud": true,
"createDockerContexts": true,
Expand Down Expand Up @@ -228,6 +233,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform
"buildOptions": Object {
"buildCanvasShareableRuntime": true,
"createArchives": true,
"createCdnAssets": true,
"createDebPackage": false,
"createDockerCloud": true,
"createDockerContexts": true,
Expand Down Expand Up @@ -267,6 +273,7 @@ it('limits packages if --all-platforms passed with --skip-docker-ubuntu', () =>
"buildOptions": Object {
"buildCanvasShareableRuntime": true,
"createArchives": true,
"createCdnAssets": true,
"createDebPackage": true,
"createDockerCloud": true,
"createDockerContexts": true,
Expand Down
2 changes: 2 additions & 0 deletions src/dev/build/args.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export function readCliArgs(argv: string[]) {
const flags = getopts(argv, {
boolean: [
'skip-archives',
'skip-cdn-assets',
'skip-initialize',
'skip-generic-folders',
'skip-platform-folders',
Expand Down Expand Up @@ -132,6 +133,7 @@ export function readCliArgs(argv: string[]) {
createGenericFolders: !Boolean(flags['skip-generic-folders']),
createPlatformFolders: !Boolean(flags['skip-platform-folders']),
createArchives: !Boolean(flags['skip-archives']),
createCdnAssets: !Boolean(flags['skip-cdn-assets']),
createRpmPackage: isOsPackageDesired('rpm'),
createDebPackage: isOsPackageDesired('deb'),
createDockerUbuntu:
Expand Down
5 changes: 5 additions & 0 deletions src/dev/build/build_distributables.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ export interface BuildOptions {
createGenericFolders: boolean;
createPlatformFolders: boolean;
createArchives: boolean;
createCdnAssets: boolean;
createRpmPackage: boolean;
createDebPackage: boolean;
createDockerUBI: boolean;
Expand Down Expand Up @@ -113,6 +114,10 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions
await run(Tasks.AssertPathLength);
await run(Tasks.AssertNoUUID);
}
// control w/ --skip-cdn-assets
if (options.createCdnAssets) {
await run(Tasks.CreateCdnAssets);
}

/**
* package platform-specific builds into archives
Expand Down
102 changes: 102 additions & 0 deletions src/dev/build/tasks/create_cdn_assets_task.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import { readFileSync } from 'fs';
import { access } from 'fs/promises';

import { resolve, dirname } from 'path';
import { asyncForEach } from '@kbn/std';
import { Jsonc } from '@kbn/repo-packages';

import del from 'del';
import globby from 'globby';

import { mkdirp, compressTar, Task, copyAll } from '../lib';

export const CreateCdnAssets: Task = {
description: 'Creating CDN assets',

async run(config, log, build) {
const buildSource = build.resolvePath();
const buildNum = config.getBuildNumber();
const buildVersion = config.getBuildVersion();
const assets = config.resolveFromRepo('build', 'cdn-assets');
const bundles = resolve(assets, String(buildNum), 'bundles');

await del(assets);
await mkdirp(assets);

// Plugins

const plugins = globby.sync([`${buildSource}/node_modules/@kbn/**/*/kibana.jsonc`]);
await asyncForEach(plugins, async (path) => {
const manifest = Jsonc.parse(readFileSync(path, 'utf8')) as any;
if (manifest?.plugin?.id) {
const pluginRoot = resolve(dirname(path));

try {
// packages/core/plugins/core-plugins-server-internal/src/plugins_service.ts
const assetsSource = resolve(pluginRoot, 'assets');
const assetsDest = resolve('plugins', manifest.plugin.id, 'assets');
await access(assetsSource);
await mkdirp(assetsDest);
await copyAll(assetsSource, assetsDest);
} catch (e) {
// assets are optional
if (!(e.code === 'ENOENT' && e.syscall === 'access')) throw e;
}

try {
// packages/core/apps/core-apps-server-internal/src/bundle_routes/register_bundle_routes.ts
const bundlesSource = resolve(pluginRoot, 'target', 'public');
const bundlesDest = resolve(bundles, 'plugin', manifest.plugin.id, '1.0.0');
await access(bundlesSource);
await mkdirp(bundlesDest);
await copyAll(bundlesSource, bundlesDest);
} catch (e) {
// bundles are optional
if (!(e.code === 'ENOENT' && e.syscall === 'access')) throw e;
}
}
});

// packages/core/apps/core-apps-server-internal/src/bundle_routes/register_bundle_routes.ts
await copyAll(
resolve(buildSource, 'node_modules/@kbn/ui-shared-deps-npm/shared_built_assets'),
resolve(bundles, 'kbn-ui-shared-deps-npm')
);
await copyAll(
resolve(buildSource, 'node_modules/@kbn/ui-shared-deps-src/shared_built_assets'),
resolve(bundles, 'kbn-ui-shared-deps-src')
);
await copyAll(
resolve(buildSource, 'node_modules/@kbn/core/target/public'),
resolve(bundles, 'core')
);
await copyAll(resolve(buildSource, 'node_modules/@kbn/monaco'), resolve(bundles, 'kbn-monaco'));

// packages/core/apps/core-apps-server-internal/src/core_app.ts
await copyAll(
resolve(buildSource, 'node_modules/@kbn/core-apps-server-internal/assets'),
resolve(assets, 'ui')
);

await compressTar({
source: assets,
destination: config.resolveFromTarget(`kibana-${buildVersion}-cdn-assets.tar.gz`),
archiverOptions: {
gzip: true,
gzipOptions: {
level: 9,
},
},
createRootDirectory: true,
rootDirectoryName: `kibana-${buildVersion}-cdn-assets`,
});
},
};
1 change: 1 addition & 0 deletions src/dev/build/tasks/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ export * from './clean_tasks';
export * from './copy_legacy_source_task';
export * from './create_archives_sources_task';
export * from './create_archives_task';
export * from './create_cdn_assets_task';
export * from './create_empty_dirs_and_files_task';
export * from './create_readme_task';
export * from './download_cloud_dependencies';
Expand Down

0 comments on commit e3c05d6

Please sign in to comment.