Skip to content

Commit

Permalink
Merge branch 'main' into vis_editors_gauge-custom_color
Browse files Browse the repository at this point in the history
  • Loading branch information
kibanamachine authored Mar 8, 2022
2 parents 4f57e71 + ed4c19c commit 673e451
Show file tree
Hide file tree
Showing 259 changed files with 4,308 additions and 75,868 deletions.
30 changes: 14 additions & 16 deletions .eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@
* Side Public License, v 1.
*/

const Path = require('path');
const Fs = require('fs');

const globby = require('globby');

const APACHE_2_0_LICENSE_HEADER = `
/*
* Licensed to Elasticsearch B.V. under one or more contributor
Expand Down Expand Up @@ -89,23 +94,16 @@ const SAFER_LODASH_SET_DEFINITELYTYPED_HEADER = `
*/
`;

const packagePkgJsons = globby.sync('*/package.json', {
cwd: Path.resolve(__dirname, 'packages'),
absolute: true,
});

/** Packages which should not be included within production code. */
const DEV_PACKAGES = [
'kbn-babel-code-parser',
'kbn-dev-utils',
'kbn-cli-dev-mode',
'kbn-docs-utils',
'kbn-es*',
'kbn-eslint*',
'kbn-optimizer',
'kbn-plugin-generator',
'kbn-plugin-helpers',
'kbn-pm',
'kbn-storybook',
'kbn-telemetry-tools',
'kbn-test',
'kbn-type-summarizer',
];
const DEV_PACKAGES = packagePkgJsons.flatMap((path) => {
const pkg = JSON.parse(Fs.readFileSync(path, 'utf8'));
return pkg.kibana && pkg.kibana.devOnly ? Path.dirname(Path.basename(path)) : [];
});

/** Directories (at any depth) which include dev-only code. */
const DEV_DIRECTORIES = [
Expand Down
14 changes: 7 additions & 7 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -424,24 +424,24 @@
},
"devDependencies": {
"@apidevtools/swagger-parser": "^10.0.3",
"@babel/cli": "^7.17.0",
"@babel/core": "^7.17.2",
"@babel/cli": "^7.17.6",
"@babel/core": "^7.17.5",
"@babel/eslint-parser": "^7.17.0",
"@babel/eslint-plugin": "^7.16.5",
"@babel/generator": "^7.17.0",
"@babel/parser": "^7.17.0",
"@babel/generator": "^7.17.3",
"@babel/parser": "^7.17.3",
"@babel/plugin-proposal-class-properties": "^7.16.7",
"@babel/plugin-proposal-export-namespace-from": "^7.16.7",
"@babel/plugin-proposal-nullish-coalescing-operator": "^7.16.7",
"@babel/plugin-proposal-object-rest-spread": "^7.16.7",
"@babel/plugin-proposal-object-rest-spread": "^7.17.3",
"@babel/plugin-proposal-optional-chaining": "^7.16.7",
"@babel/plugin-proposal-private-methods": "^7.16.11",
"@babel/plugin-transform-runtime": "^7.17.0",
"@babel/preset-env": "^7.16.11",
"@babel/preset-react": "^7.16.7",
"@babel/preset-typescript": "^7.16.7",
"@babel/register": "^7.17.0",
"@babel/traverse": "^7.17.0",
"@babel/traverse": "^7.17.3",
"@babel/types": "^7.17.0",
"@bazel/ibazel": "^0.15.10",
"@bazel/typescript": "4.0.0",
Expand Down Expand Up @@ -731,7 +731,7 @@
"babel-plugin-add-module-exports": "^1.0.4",
"babel-plugin-istanbul": "^6.1.1",
"babel-plugin-require-context-hook": "^1.0.0",
"babel-plugin-styled-components": "^2.0.2",
"babel-plugin-styled-components": "^2.0.6",
"babel-plugin-transform-react-remove-prop-types": "^0.4.24",
"backport": "^7.3.1",
"callsites": "^3.1.0",
Expand Down
2 changes: 2 additions & 0 deletions packages/elastic-apm-synthtrace/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ RUNTIME_DEPS = [
"@npm//object-hash",
"@npm//p-limit",
"@npm//yargs",
"@npm//node-fetch",
]

TYPES_DEPS = [
Expand All @@ -43,6 +44,7 @@ TYPES_DEPS = [
"@npm//@types/object-hash",
"@npm//moment",
"@npm//p-limit",
"@npm//@types/node-fetch",
]

jsts_transpiler(
Expand Down
1 change: 1 addition & 0 deletions packages/elastic-apm-synthtrace/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,4 @@ export { createLogger, LogLevel } from './lib/utils/create_logger';

export type { Fields } from './lib/entity';
export type { ApmException, ApmSynthtraceEsClient } from './lib/apm';
export type { SpanIterable } from './lib/span_iterable';
8 changes: 6 additions & 2 deletions packages/elastic-apm-synthtrace/src/lib/apm/apm_fields.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,14 @@ export type ApmUserAgentFields = Partial<{
export interface ApmException {
message: string;
}
export interface Observer {
version: string;
version_major: number;
}

export type ApmFields = Fields &
Partial<{
'timestamp.us'?: number;
'agent.name': string;
'agent.version': string;
'container.id': string;
Expand All @@ -47,8 +52,7 @@ export type ApmFields = Fields &
'host.name': string;
'kubernetes.pod.uid': string;
'metricset.name': string;
'observer.version': string;
'observer.version_major': number;
observer: Observer;
'parent.id': string;
'processor.event': string;
'processor.name': string;
Expand Down
10 changes: 5 additions & 5 deletions packages/elastic-apm-synthtrace/src/lib/apm/base_span.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export class BaseSpan extends Serializable<ApmFields> {
});
}

parent(span: BaseSpan) {
parent(span: BaseSpan): this {
this.fields['trace.id'] = span.fields['trace.id'];
this.fields['parent.id'] = span.isSpan()
? span.fields['span.id']
Expand All @@ -40,7 +40,7 @@ export class BaseSpan extends Serializable<ApmFields> {
return this;
}

children(...children: BaseSpan[]) {
children(...children: BaseSpan[]): this {
children.forEach((child) => {
child.parent(this);
});
Expand All @@ -50,17 +50,17 @@ export class BaseSpan extends Serializable<ApmFields> {
return this;
}

success() {
success(): this {
this.fields['event.outcome'] = 'success';
return this;
}

failure() {
failure(): this {
this.fields['event.outcome'] = 'failure';
return this;
}

outcome(outcome: 'success' | 'failure' | 'unknown') {
outcome(outcome: 'success' | 'failure' | 'unknown'): this {
this.fields['event.outcome'] = outcome;
return this;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,56 +7,115 @@
*/

import { Client } from '@elastic/elasticsearch';
import { uploadEvents } from '../../../scripts/utils/upload_events';
import { Fields } from '../../entity';
import { cleanWriteTargets } from '../../utils/clean_write_targets';
import { getBreakdownMetrics } from '../utils/get_breakdown_metrics';
import { getSpanDestinationMetrics } from '../utils/get_span_destination_metrics';
import { getTransactionMetrics } from '../utils/get_transaction_metrics';
import { getApmWriteTargets } from '../utils/get_apm_write_targets';
import { Logger } from '../../utils/create_logger';
import { apmEventsToElasticsearchOutput } from '../utils/apm_events_to_elasticsearch_output';
import { ApmFields } from '../apm_fields';
import { SpanIterable } from '../../span_iterable';
import { StreamProcessor } from '../../stream_processor';
import { SpanGeneratorsUnion } from '../../span_generators_union';

export interface StreamToBulkOptions {
concurrency?: number;
maxDocs?: number;
mapToIndex?: (document: Record<string, any>) => string;
}

export class ApmSynthtraceEsClient {
constructor(private readonly client: Client, private readonly logger: Logger) {}
constructor(
private readonly client: Client,
private readonly logger: Logger,
private readonly forceDataStreams: boolean
) {}

private getWriteTargets() {
return getApmWriteTargets({ client: this.client });
return getApmWriteTargets({ client: this.client, forceDataStreams: this.forceDataStreams });
}

clean() {
return this.getWriteTargets().then((writeTargets) =>
cleanWriteTargets({
return this.getWriteTargets().then(async (writeTargets) => {
const indices = Object.values(writeTargets);
this.logger.info(`Attempting to clean: ${indices}`);
if (this.forceDataStreams) {
for (const name of indices) {
const dataStream = await this.client.indices.getDataStream({ name }, { ignore: [404] });
if (dataStream.data_streams && dataStream.data_streams.length > 0) {
this.logger.debug(`Deleting datastream: ${name}`);
await this.client.indices.deleteDataStream({ name });
}
}
return;
}

return cleanWriteTargets({
client: this.client,
targets: Object.values(writeTargets),
targets: indices,
logger: this.logger,
})
);
});
});
}

async index(events: Fields[]) {
const writeTargets = await this.getWriteTargets();
async updateComponentTemplates(numberOfPrimaryShards: number) {
const response = await this.client.cluster.getComponentTemplate({ name: '*apm*@custom' });
for (const componentTemplate of response.component_templates) {
if (componentTemplate.component_template._meta?.package?.name !== 'apm') continue;

const eventsToIndex = apmEventsToElasticsearchOutput({
events: [
...events,
...getTransactionMetrics(events),
...getSpanDestinationMetrics(events),
...getBreakdownMetrics(events),
],
writeTargets,
});
componentTemplate.component_template.template.settings = {
index: {
number_of_shards: numberOfPrimaryShards,
},
};

await uploadEvents({
batchSize: 1000,
client: this.client,
clientWorkers: 5,
events: eventsToIndex,
logger: this.logger,
const putTemplate = await this.client.cluster.putComponentTemplate({
name: componentTemplate.name,
...componentTemplate.component_template,
});
this.logger.info(
`- updated component template ${componentTemplate.name}, acknowledged: ${putTemplate.acknowledged}`
);
}
}

async index(events: SpanIterable | SpanIterable[], options?: StreamToBulkOptions) {
const dataStream = Array.isArray(events) ? new SpanGeneratorsUnion(events) : events;

const writeTargets = await this.getWriteTargets();
// TODO logger.perf
await this.client.helpers.bulk<ApmFields>({
concurrency: options?.concurrency ?? 10,
refresh: false,
refreshOnCompletion: false,
datasource: new StreamProcessor({
processors: StreamProcessor.apmProcessors,
maxSourceEvents: options?.maxDocs,
logger: this.logger,
})
// TODO https://github.com/elastic/elasticsearch-js/issues/1610
// having to map here is awkward, it'd be better to map just before serialization.
.streamToDocumentAsync(StreamProcessor.toDocument, dataStream),
onDrop: (doc) => {
this.logger.info(doc);
},
// TODO bug in client not passing generic to BulkHelperOptions<>
// https://github.com/elastic/elasticsearch-js/issues/1611
onDocument: (doc: unknown) => {
const d = doc as Record<string, any>;
const index = options?.mapToIndex
? options?.mapToIndex(d)
: this.forceDataStreams
? StreamProcessor.getDataStreamForEvent(d, writeTargets)
: StreamProcessor.getIndexForEvent(d, writeTargets);
return { create: { _index: index } };
},
});

const indices = Object.values(writeTargets);
this.logger.info(`Indexed all data attempting to refresh: ${indices}`);

return this.client.indices.refresh({
index: Object.values(writeTargets),
index: indices,
allow_no_indices: true,
ignore_unavailable: true,
});
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import fetch from 'node-fetch';
import { Logger } from '../../utils/create_logger';

export class ApmSynthtraceKibanaClient {
constructor(private readonly logger: Logger) {}

async migrateCloudToManagedApm(cloudId: string, username: string, password: string) {
await this.logger.perf('migrate_apm_on_cloud', async () => {
this.logger.info('attempting to migrate cloud instance over to managed APM');
const cloudUrls = Buffer.from(cloudId.split(':')[1], 'base64').toString().split('$');
const kibanaCloudUrl = `https://${cloudUrls[2]}.${cloudUrls[0]}`;
const response = await fetch(
kibanaCloudUrl + '/internal/apm/fleet/cloud_apm_package_policy',
{
method: 'POST', // *GET, POST, PUT, DELETE, etc.
headers: {
Authorization: 'Basic ' + Buffer.from(username + ':' + password).toString('base64'),
Accept: 'application/json',
'Content-Type': 'application/json',
'kbn-xsrf': 'kibana',
},
}
);
const responseJson = await response.json();
if (responseJson.message) {
this.logger.info(`Cloud Instance already migrated to managed APM: ${responseJson.message}`);
}
if (responseJson.cloudApmPackagePolicy) {
this.logger.info(
`Cloud Instance migrated to managed APM: ${responseJson.cloudApmPackagePolicy.package.version}`
);
}
});
}
}

This file was deleted.

Loading

0 comments on commit 673e451

Please sign in to comment.