[EPM] Implement getConfig for dataset

This implements a getConfig method on a dataset object. This allows to build the configuration for each dataset in a package.

After the configuration for each dataset is built, this config must be stored in the datasource object. The end goal is store this as the streams in the datasource object. As at the moment we just have a strings I added a `config` field to the datasource which can be used to ship it the agent. This should be cleaned up later again.

Configs is an array of strings at the moment as each entry is the input part of the stream.

At the moment, it is not stored to the datasource object yet.

x-pack/legacy/plugins/epm/common/types.ts

@@ -16,7 +16,7 @@ export enum InstallationStatus {
 }
 
 export type ServiceName = 'kibana' | 'elasticsearch';
-export type AssetType = KibanaAssetType | ElasticsearchAssetType;
+export type AssetType = KibanaAssetType | ElasticsearchAssetType | AgentAssetType;
 
 export enum KibanaAssetType {
   dashboard = 'dashboard',
@@ -31,6 +31,10 @@ export enum ElasticsearchAssetType {
   ilmPolicy = 'ilm-policy',
 }
 
+export enum AgentAssetType {
+  input = 'input',
+}
+
 // from /package/{name}
 // type Package struct at https://github.com/elastic/package-registry/blob/master/util/package.go
 // https://github.com/elastic/package-registry/blob/master/docs/api/package.json
@@ -120,12 +124,17 @@ export interface Dataset {
   name: string;
   release: string;
   ingeset_pipeline: string;
-  vars: object[];
+  vars: VarsEntry[];
   type: string;
   // This is for convenience and not in the output from the registry. When creating a dataset, this info should be added.
   package: string;
 }
 
+export interface VarsEntry {
+  name: string;
+  default: string;
+}
+
 // some properties are optional in Registry responses but required in EPM
 // internal until we need them
 interface PackageAdditions {

x-pack/legacy/plugins/epm/server/datasources/create.ts

@@ -16,6 +16,7 @@ import { installTemplates } from '../lib/elasticsearch/template/install';
 import { getPackageInfo, PackageNotInstalledError } from '../packages';
 import * as Registry from '../registry';
 import { Request } from '../types';
+import { createInput } from '../lib/agent/agent';
 
 export async function createDatasource(options: {
   savedObjectsClient: SavedObjectsClientContract;
@@ -38,6 +39,14 @@ export async function createDatasource(options: {
   await baseSetup(callCluster);
   const pkg = await Registry.fetchInfo(pkgkey);
 
+  // Collect the config for each dataset
+  const configs: string[] = [];
+  if (pkg.datasets) {
+    for (const dataset of pkg.datasets) {
+      configs.push(await getConfig(pkgkey, dataset));
+    }
+  }
+
   await Promise.all([
     installTemplates(pkg, callCluster),
     saveDatasourceReferences({
@@ -184,3 +193,26 @@ async function ingestDatasourceCreate({
     },
   }).then(response => response.json());
 }
+
+async function getConfig(pkgkey: string, dataset: Dataset): Promise<string> {
+  const vars = dataset.vars;
+
+  // This searches for the /agent/input.yml file
+  const paths = await Registry.getArchiveInfo(pkgkey, (entry: Registry.ArchiveEntry) =>
+    isDatasetInput(entry, dataset.name)
+  );
+
+  if (paths.length === 1) {
+    const buffer = Registry.getAsset(paths[0]);
+    // Load input template from path
+    return createInput(vars, buffer.toString());
+  }
+  return '';
+}
+
+const isDatasetInput = ({ path }: Registry.ArchiveEntry, datasetName: string) => {
+  const pathParts = Registry.pathParts(path);
+  return !isDirectory({ path }) && pathParts.type === 'input' && pathParts.dataset === datasetName;
+};
+
+const isDirectory = ({ path }: Registry.ArchiveEntry) => path.endsWith('/');

x-pack/legacy/plugins/epm/server/lib/agent/agent.test.ts

@@ -15,7 +15,7 @@ test('test converting input and manifest into template', () => {
   );
 
   const inputTemplate = fs.readFileSync(path.join(__dirname, 'tests/input.yml'), 'utf8');
-  const output = createInput(manifest, inputTemplate);
+  const output = createInput(manifest.vars, inputTemplate);
 
   // Golden file path
   const generatedFile = path.join(__dirname, './tests/input.generated.yaml');

x-pack/legacy/plugins/epm/server/lib/agent/agent.ts

@@ -5,23 +5,16 @@
  */
 
 import Handlebars from 'handlebars';
+import { VarsEntry } from '../../../common/types';
 
-interface Manifest {
-  vars: VarsEntry[];
-}
-
-interface VarsEntry {
-  name: string;
-  default: string;
-}
 /**
- * This takes a manifest object as input and merges it with the input template.
+ * This takes a dataset object as input and merges it with the input template.
  * It returns the resolved template as a string.
  */
-export function createInput(manifest: Manifest, inputTemplate: string): string {
+export function createInput(vars: VarsEntry[], inputTemplate: string): string {
   const view: Record<VarsEntry['name'], VarsEntry['default']> = {};
 
-  for (const v of manifest.vars) {
+  for (const v of vars) {
     view[v.name] = v.default;
   }
 

x-pack/legacy/plugins/ingest/server/libs/types.ts

@@ -50,6 +50,7 @@ export interface Datasource extends SavedObjectAttributes {
   package: Package;
   read_alias?: string;
   streams: Stream[];
+  config: string[];
 }
 
 /**