[EPM] Implement getConfig for dataset

x-pack/legacy/plugins/epm/common/types.ts

@@ -16,7 +16,7 @@ export enum InstallationStatus {
 }
 
 export type ServiceName = 'kibana' | 'elasticsearch';
-export type AssetType = KibanaAssetType | ElasticsearchAssetType;
+export type AssetType = KibanaAssetType | ElasticsearchAssetType | AgentAssetType;
 
 export enum KibanaAssetType {
   dashboard = 'dashboard',
@@ -31,6 +31,10 @@ export enum ElasticsearchAssetType {
   ilmPolicy = 'ilm-policy',
 }
 
+export enum AgentAssetType {
+  input = 'input',
+}
+
 // from /package/{name}
 // type Package struct at https://github.com/elastic/package-registry/blob/master/util/package.go
 // https://github.com/elastic/package-registry/blob/master/docs/api/package.json
@@ -120,12 +124,17 @@ export interface Dataset {
   name: string;
   release: string;
   ingeset_pipeline: string;
-  vars: object[];
+  vars: VarsEntry[];
   type: string;
   // This is for convenience and not in the output from the registry. When creating a dataset, this info should be added.
   package: string;
 }
 
+export interface VarsEntry {
+  name: string;
+  default: string;
+}
+
 // some properties are optional in Registry responses but required in EPM
 // internal until we need them
 interface PackageAdditions {

x-pack/legacy/plugins/epm/server/datasources/create.ts

@@ -16,6 +16,7 @@ import { installTemplates } from '../lib/elasticsearch/template/install';
 import { getPackageInfo, PackageNotInstalledError } from '../packages';
 import * as Registry from '../registry';
 import { Request } from '../types';
+import { createInput } from '../lib/agent/agent';
 
 export async function createDatasource(options: {
   savedObjectsClient: SavedObjectsClientContract;
@@ -38,6 +39,14 @@ export async function createDatasource(options: {
   await baseSetup(callCluster);
   const pkg = await Registry.fetchInfo(pkgkey);
 
+  // Collect the config for each dataset
+  const inputs: string[] = [];
+  if (pkg.datasets) {
+    for (const dataset of pkg.datasets) {
+      inputs.push(await getConfig(pkgkey, dataset));
+    }
+  }
+
   await Promise.all([
     installTemplates(pkg, callCluster),
     saveDatasourceReferences({
@@ -47,6 +56,7 @@ export async function createDatasource(options: {
       datasets,
       toSave,
       request,
+      inputs,
     }),
   ]);
 
@@ -73,8 +83,9 @@ async function saveDatasourceReferences(options: {
   datasourceName: string;
   toSave: AssetReference[];
   request: Request;
+  inputs: string[];
 }) {
-  const { savedObjectsClient, pkg, toSave, datasets, datasourceName, request } = options;
+  const { savedObjectsClient, pkg, toSave, datasets, datasourceName, request, inputs } = options;
   const savedDatasource = await getDatasource({ savedObjectsClient, pkg });
   const savedAssets = savedDatasource?.package.assets;
   const assetsReducer = (current: Asset[] = [], pending: Asset) => {
@@ -90,6 +101,8 @@ async function saveDatasourceReferences(options: {
     datasets,
     assets: toInstall,
   });
+
+  datasource.inputs = inputs;
   // ideally we'd call .create from /x-pack/legacy/plugins/ingest/server/libs/datasources.ts#L22
   // or something similar, but it's a class not an object so many pieces are missing
   // we'd still need `user` from the request object, but that's not terrible
@@ -184,3 +197,26 @@ async function ingestDatasourceCreate({
     },
   }).then(response => response.json());
 }
+
+async function getConfig(pkgkey: string, dataset: Dataset): Promise<string> {
+  const vars = dataset.vars;
+
+  // This searches for the /agent/input.yml file
+  const paths = await Registry.getArchiveInfo(pkgkey, (entry: Registry.ArchiveEntry) =>
+    isDatasetInput(entry, dataset.name)
+  );
+
+  if (paths.length === 1) {
+    const buffer = Registry.getAsset(paths[0]);
+    // Load input template from path
+    return createInput(vars, buffer.toString());
+  }
+  return '';
+}
+
+const isDatasetInput = ({ path }: Registry.ArchiveEntry, datasetName: string) => {
+  const pathParts = Registry.pathParts(path);
+  return !isDirectory({ path }) && pathParts.type === 'input' && pathParts.dataset === datasetName;
+};
+
+const isDirectory = ({ path }: Registry.ArchiveEntry) => path.endsWith('/');

x-pack/legacy/plugins/epm/server/lib/agent/agent.test.ts

@@ -15,7 +15,7 @@ test('test converting input and manifest into template', () => {
   );
 
   const inputTemplate = fs.readFileSync(path.join(__dirname, 'tests/input.yml'), 'utf8');
-  const output = createInput(manifest, inputTemplate);
+  const output = createInput(manifest.vars, inputTemplate);
 
   // Golden file path
   const generatedFile = path.join(__dirname, './tests/input.generated.yaml');

x-pack/legacy/plugins/epm/server/lib/agent/agent.ts

@@ -5,23 +5,16 @@
  */
 
 import Handlebars from 'handlebars';
+import { VarsEntry } from '../../../common/types';
 
-interface Manifest {
-  vars: VarsEntry[];
-}
-
-interface VarsEntry {
-  name: string;
-  default: string;
-}
 /**
- * This takes a manifest object as input and merges it with the input template.
+ * This takes a dataset object as input and merges it with the input template.
  * It returns the resolved template as a string.
  */
-export function createInput(manifest: Manifest, inputTemplate: string): string {
+export function createInput(vars: VarsEntry[], inputTemplate: string): string {
   const view: Record<VarsEntry['name'], VarsEntry['default']> = {};
 
-  for (const v of manifest.vars) {
+  for (const v of vars) {
     view[v.name] = v.default;
   }
 

x-pack/legacy/plugins/ingest/server/libs/types.ts

@@ -50,6 +50,8 @@ export interface Datasource extends SavedObjectAttributes {
   package: Package;
   read_alias?: string;
   streams: Stream[];
+  // TODO: These are the input parts of the streams. To be remove again when we have the streams
+  inputs: string[];
 }
 
 /**