From 043ff023ada77fcac9e35f95134349591ce123a1 Mon Sep 17 00:00:00 2001 From: ruflin Date: Tue, 17 Dec 2019 09:37:21 +0100 Subject: [PATCH] [EPM] Implement getConfig for dataset This implements a getConfig method on a dataset object. This allows to build the configuration for each dataset in a package. After the configuration for each dataset is built, this config must be stored in the datasource object. The end goal is store this as the streams in the datasource object. As at the moment we just have a strings I added a `config` field to the datasource which can be used to ship it the agent. This should be cleaned up later again. Configs is an array of strings at the moment as each entry is the input part of the stream. At the moment, it is not stored to the datasource object yet. --- x-pack/legacy/plugins/epm/common/types.ts | 13 ++++++-- .../plugins/epm/server/datasources/create.ts | 32 +++++++++++++++++++ .../epm/server/lib/agent/agent.test.ts | 2 +- .../plugins/epm/server/lib/agent/agent.ts | 15 +++------ .../plugins/ingest/server/libs/types.ts | 1 + 5 files changed, 49 insertions(+), 14 deletions(-) diff --git a/x-pack/legacy/plugins/epm/common/types.ts b/x-pack/legacy/plugins/epm/common/types.ts index 070fc81deef27..ccb165787bc4c 100644 --- a/x-pack/legacy/plugins/epm/common/types.ts +++ b/x-pack/legacy/plugins/epm/common/types.ts @@ -16,7 +16,7 @@ export enum InstallationStatus { } export type ServiceName = 'kibana' | 'elasticsearch'; -export type AssetType = KibanaAssetType | ElasticsearchAssetType; +export type AssetType = KibanaAssetType | ElasticsearchAssetType | AgentAssetType; export enum KibanaAssetType { dashboard = 'dashboard', @@ -31,6 +31,10 @@ export enum ElasticsearchAssetType { ilmPolicy = 'ilm-policy', } +export enum AgentAssetType { + input = 'input', +} + // from /package/{name} // type Package struct at https://github.com/elastic/package-registry/blob/master/util/package.go // https://github.com/elastic/package-registry/blob/master/docs/api/package.json @@ -120,12 +124,17 @@ export interface Dataset { name: string; release: string; ingeset_pipeline: string; - vars: object[]; + vars: VarsEntry[]; type: string; // This is for convenience and not in the output from the registry. When creating a dataset, this info should be added. package: string; } +export interface VarsEntry { + name: string; + default: string; +} + // some properties are optional in Registry responses but required in EPM // internal until we need them interface PackageAdditions { diff --git a/x-pack/legacy/plugins/epm/server/datasources/create.ts b/x-pack/legacy/plugins/epm/server/datasources/create.ts index e11edb960de97..7cc1204bf38d6 100644 --- a/x-pack/legacy/plugins/epm/server/datasources/create.ts +++ b/x-pack/legacy/plugins/epm/server/datasources/create.ts @@ -16,6 +16,7 @@ import { installTemplates } from '../lib/elasticsearch/template/install'; import { getPackageInfo, PackageNotInstalledError } from '../packages'; import * as Registry from '../registry'; import { Request } from '../types'; +import { createInput } from '../lib/agent/agent'; export async function createDatasource(options: { savedObjectsClient: SavedObjectsClientContract; @@ -38,6 +39,14 @@ export async function createDatasource(options: { await baseSetup(callCluster); const pkg = await Registry.fetchInfo(pkgkey); + // Collect the config for each dataset + const configs: string[] = []; + if (pkg.datasets) { + for (const dataset of pkg.datasets) { + configs.push(await getConfig(pkgkey, dataset)); + } + } + await Promise.all([ installTemplates(pkg, callCluster), saveDatasourceReferences({ @@ -184,3 +193,26 @@ async function ingestDatasourceCreate({ }, }).then(response => response.json()); } + +async function getConfig(pkgkey: string, dataset: Dataset): Promise { + const vars = dataset.vars; + + // This searches for the /agent/input.yml file + const paths = await Registry.getArchiveInfo(pkgkey, (entry: Registry.ArchiveEntry) => + isDatasetInput(entry, dataset.name) + ); + + if (paths.length === 1) { + const buffer = Registry.getAsset(paths[0]); + // Load input template from path + return createInput(vars, buffer.toString()); + } + return ''; +} + +const isDatasetInput = ({ path }: Registry.ArchiveEntry, datasetName: string) => { + const pathParts = Registry.pathParts(path); + return !isDirectory({ path }) && pathParts.type === 'input' && pathParts.dataset === datasetName; +}; + +const isDirectory = ({ path }: Registry.ArchiveEntry) => path.endsWith('/'); diff --git a/x-pack/legacy/plugins/epm/server/lib/agent/agent.test.ts b/x-pack/legacy/plugins/epm/server/lib/agent/agent.test.ts index 123f0360ebdad..4f75ba0332418 100644 --- a/x-pack/legacy/plugins/epm/server/lib/agent/agent.test.ts +++ b/x-pack/legacy/plugins/epm/server/lib/agent/agent.test.ts @@ -15,7 +15,7 @@ test('test converting input and manifest into template', () => { ); const inputTemplate = fs.readFileSync(path.join(__dirname, 'tests/input.yml'), 'utf8'); - const output = createInput(manifest, inputTemplate); + const output = createInput(manifest.vars, inputTemplate); // Golden file path const generatedFile = path.join(__dirname, './tests/input.generated.yaml'); diff --git a/x-pack/legacy/plugins/epm/server/lib/agent/agent.ts b/x-pack/legacy/plugins/epm/server/lib/agent/agent.ts index 88d2507df8f21..561a9f43c3fc7 100644 --- a/x-pack/legacy/plugins/epm/server/lib/agent/agent.ts +++ b/x-pack/legacy/plugins/epm/server/lib/agent/agent.ts @@ -5,23 +5,16 @@ */ import Handlebars from 'handlebars'; +import { VarsEntry } from '../../../common/types'; -interface Manifest { - vars: VarsEntry[]; -} - -interface VarsEntry { - name: string; - default: string; -} /** - * This takes a manifest object as input and merges it with the input template. + * This takes a dataset object as input and merges it with the input template. * It returns the resolved template as a string. */ -export function createInput(manifest: Manifest, inputTemplate: string): string { +export function createInput(vars: VarsEntry[], inputTemplate: string): string { const view: Record = {}; - for (const v of manifest.vars) { + for (const v of vars) { view[v.name] = v.default; } diff --git a/x-pack/legacy/plugins/ingest/server/libs/types.ts b/x-pack/legacy/plugins/ingest/server/libs/types.ts index c706804a51d37..e35f306f3f4e1 100644 --- a/x-pack/legacy/plugins/ingest/server/libs/types.ts +++ b/x-pack/legacy/plugins/ingest/server/libs/types.ts @@ -50,6 +50,7 @@ export interface Datasource extends SavedObjectAttributes { package: Package; read_alias?: string; streams: Stream[]; + config: string[]; } /**