diff --git a/x-pack/legacy/plugins/apm/public/context/ChartsSyncContext.tsx b/x-pack/legacy/plugins/apm/public/context/ChartsSyncContext.tsx index afce0811b48f6..065e0b8733122 100644 --- a/x-pack/legacy/plugins/apm/public/context/ChartsSyncContext.tsx +++ b/x-pack/legacy/plugins/apm/public/context/ChartsSyncContext.tsx @@ -28,7 +28,7 @@ const ChartsSyncContextProvider: React.FC = ({ children }) => { callApmApi => { if (start && end && serviceName) { return callApmApi({ - pathname: '/api/apm/services/{serviceName}/annotations', + pathname: '/api/apm/services/{serviceName}/annotation/search', params: { path: { serviceName diff --git a/x-pack/legacy/plugins/apm/scripts/optimize-tsconfig/tsconfig.json b/x-pack/legacy/plugins/apm/scripts/optimize-tsconfig/tsconfig.json index 8f6b0f35e4b52..45be372694948 100644 --- a/x-pack/legacy/plugins/apm/scripts/optimize-tsconfig/tsconfig.json +++ b/x-pack/legacy/plugins/apm/scripts/optimize-tsconfig/tsconfig.json @@ -1,6 +1,7 @@ { "include": [ "./plugins/apm/**/*", + "./plugins/observability/**/*", "./legacy/plugins/apm/**/*", "./typings/**/*" ], diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json index 7ffdb676c740f..4cb3e01c5a211 100644 --- a/x-pack/plugins/apm/kibana.json +++ b/x-pack/plugins/apm/kibana.json @@ -9,5 +9,5 @@ ], "ui": false, "requiredPlugins": ["apm_oss", "data", "home", "licensing"], - "optionalPlugins": ["cloud", "usageCollection", "taskManager","actions", "alerting"] + "optionalPlugins": ["cloud", "usageCollection", "taskManager","actions", "alerting", "observability"] } diff --git a/x-pack/plugins/apm/server/lib/helpers/create_or_update_index.ts b/x-pack/plugins/apm/server/lib/helpers/create_or_update_index.ts deleted file mode 100644 index 4df02786b1fb5..0000000000000 --- a/x-pack/plugins/apm/server/lib/helpers/create_or_update_index.ts +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import pRetry from 'p-retry'; -import { IClusterClient, Logger } from 'src/core/server'; -import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; - -export type Mappings = - | { - dynamic?: boolean | 'strict'; - properties: Record; - dynamic_templates?: any[]; - } - | { - type: string; - ignore_above?: number; - scaling_factor?: number; - ignore_malformed?: boolean; - coerce?: boolean; - fields?: Record; - }; - -export async function createOrUpdateIndex({ - index, - mappings, - esClient, - logger -}: { - index: string; - mappings: Mappings; - esClient: IClusterClient; - logger: Logger; -}) { - try { - /* - * In some cases we could be trying to create an index before ES is ready. - * When this happens, we retry creating the index with exponential backoff. - * We use retry's default formula, meaning that the first retry happens after 2s, - * the 5th after 32s, and the final attempt after around 17m. If the final attempt fails, - * the error is logged to the console. - * See https://github.com/sindresorhus/p-retry and https://github.com/tim-kos/node-retry. - */ - await pRetry(async () => { - const { callAsInternalUser } = esClient; - const indexExists = await callAsInternalUser('indices.exists', { index }); - const result = indexExists - ? await updateExistingIndex({ - index, - callAsInternalUser, - mappings - }) - : await createNewIndex({ - index, - callAsInternalUser, - mappings - }); - - if (!result.acknowledged) { - const resultError = - result && result.error && JSON.stringify(result.error); - throw new Error(resultError); - } - }); - } catch (e) { - logger.error( - `Could not create APM index: '${index}'. Error: ${e.message}.` - ); - } -} - -function createNewIndex({ - index, - callAsInternalUser, - mappings -}: { - index: string; - callAsInternalUser: CallCluster; - mappings: Mappings; -}) { - return callAsInternalUser('indices.create', { - index, - body: { - // auto_expand_replicas: Allows cluster to not have replicas for this index - settings: { 'index.auto_expand_replicas': '0-1' }, - mappings - } - }); -} - -function updateExistingIndex({ - index, - callAsInternalUser, - mappings -}: { - index: string; - callAsInternalUser: CallCluster; - mappings: Mappings; -}) { - return callAsInternalUser('indices.putMapping', { - index, - body: mappings - }); -} diff --git a/x-pack/plugins/apm/server/lib/helpers/es_client.ts b/x-pack/plugins/apm/server/lib/helpers/es_client.ts index c22084dbb7168..667d194aeb821 100644 --- a/x-pack/plugins/apm/server/lib/helpers/es_client.ts +++ b/x-pack/plugins/apm/server/lib/helpers/es_client.ts @@ -7,10 +7,10 @@ /* eslint-disable no-console */ import { IndexDocumentParams, - IndicesDeleteParams, SearchParams, IndicesCreateParams, - DeleteDocumentResponse + DeleteDocumentResponse, + DeleteDocumentParams } from 'elasticsearch'; import { cloneDeep, isString, merge } from 'lodash'; import { KibanaRequest } from 'src/core/server'; @@ -205,7 +205,9 @@ export function getESClient( index: (params: APMIndexDocumentParams) => { return callEs('index', params); }, - delete: (params: IndicesDeleteParams): Promise => { + delete: ( + params: Omit + ): Promise => { return callEs('delete', params); }, indicesCreate: (params: IndicesCreateParams) => { diff --git a/x-pack/plugins/apm/server/lib/services/annotations/get_derived_service_annotations.ts b/x-pack/plugins/apm/server/lib/services/annotations/get_derived_service_annotations.ts new file mode 100644 index 0000000000000..b6ed2c47b7e1c --- /dev/null +++ b/x-pack/plugins/apm/server/lib/services/annotations/get_derived_service_annotations.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { isNumber } from 'lodash'; +import { Annotation, AnnotationType } from '../../../../common/annotations'; +import { SetupTimeRange, Setup } from '../../helpers/setup_request'; +import { ESFilter } from '../../../../typings/elasticsearch'; +import { rangeFilter } from '../../helpers/range_filter'; +import { + PROCESSOR_EVENT, + SERVICE_NAME, + SERVICE_ENVIRONMENT, + SERVICE_VERSION +} from '../../../../common/elasticsearch_fieldnames'; + +export async function getDerivedServiceAnnotations({ + setup, + serviceName, + environment +}: { + serviceName: string; + environment?: string; + setup: Setup & SetupTimeRange; +}) { + const { start, end, client, indices } = setup; + + const filter: ESFilter[] = [ + { term: { [PROCESSOR_EVENT]: 'transaction' } }, + { range: rangeFilter(start, end) }, + { term: { [SERVICE_NAME]: serviceName } } + ]; + + if (environment) { + filter.push({ term: { [SERVICE_ENVIRONMENT]: environment } }); + } + + const versions = + ( + await client.search({ + index: indices['apm_oss.transactionIndices'], + body: { + size: 0, + track_total_hits: false, + query: { + bool: { + filter + } + }, + aggs: { + versions: { + terms: { + field: SERVICE_VERSION + } + } + } + } + }) + ).aggregations?.versions.buckets.map(bucket => bucket.key) ?? []; + + if (versions.length > 1) { + const annotations = await Promise.all( + versions.map(async version => { + const response = await client.search({ + index: indices['apm_oss.transactionIndices'], + body: { + size: 0, + query: { + bool: { + filter: filter + .filter(esFilter => !Object.keys(esFilter).includes('range')) + .concat({ + term: { + [SERVICE_VERSION]: version + } + }) + } + }, + aggs: { + first_seen: { + min: { + field: '@timestamp' + } + } + }, + track_total_hits: false + } + }); + + const firstSeen = response.aggregations?.first_seen.value; + + if (!isNumber(firstSeen)) { + throw new Error( + 'First seen for version was unexpectedly undefined or null.' + ); + } + + if (firstSeen < start || firstSeen > end) { + return null; + } + + return { + type: AnnotationType.VERSION, + id: version, + time: firstSeen, + text: version + }; + }) + ); + return annotations.filter(Boolean) as Annotation[]; + } + return []; +} diff --git a/x-pack/plugins/apm/server/lib/services/annotations/index.test.ts b/x-pack/plugins/apm/server/lib/services/annotations/index.test.ts index 0e52982c6de28..abce49d1e2379 100644 --- a/x-pack/plugins/apm/server/lib/services/annotations/index.test.ts +++ b/x-pack/plugins/apm/server/lib/services/annotations/index.test.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { getServiceAnnotations } from '.'; +import { getDerivedServiceAnnotations } from './get_derived_service_annotations'; import { SearchParamsMock, inspectSearchParams @@ -24,7 +24,7 @@ describe('getServiceAnnotations', () => { it('returns no annotations', async () => { mock = await inspectSearchParams( setup => - getServiceAnnotations({ + getDerivedServiceAnnotations({ setup, serviceName: 'foo', environment: 'bar' @@ -34,7 +34,7 @@ describe('getServiceAnnotations', () => { } ); - expect(mock.response).toEqual({ annotations: [] }); + expect(mock.response).toEqual([]); }); }); @@ -42,7 +42,7 @@ describe('getServiceAnnotations', () => { it('returns no annotations', async () => { mock = await inspectSearchParams( setup => - getServiceAnnotations({ + getDerivedServiceAnnotations({ setup, serviceName: 'foo', environment: 'bar' @@ -52,7 +52,7 @@ describe('getServiceAnnotations', () => { } ); - expect(mock.response).toEqual({ annotations: [] }); + expect(mock.response).toEqual([]); }); }); @@ -65,7 +65,7 @@ describe('getServiceAnnotations', () => { ]; mock = await inspectSearchParams( setup => - getServiceAnnotations({ + getDerivedServiceAnnotations({ setup, serviceName: 'foo', environment: 'bar' @@ -77,22 +77,20 @@ describe('getServiceAnnotations', () => { expect(mock.spy.mock.calls.length).toBe(3); - expect(mock.response).toEqual({ - annotations: [ - { - id: '8.0.0', - text: '8.0.0', - time: 1.5281138e12, - type: 'version' - }, - { - id: '7.5.0', - text: '7.5.0', - time: 1.5281138e12, - type: 'version' - } - ] - }); + expect(mock.response).toEqual([ + { + id: '8.0.0', + text: '8.0.0', + time: 1.5281138e12, + type: 'version' + }, + { + id: '7.5.0', + text: '7.5.0', + time: 1.5281138e12, + type: 'version' + } + ]); }); }); }); diff --git a/x-pack/plugins/apm/server/lib/services/annotations/index.ts b/x-pack/plugins/apm/server/lib/services/annotations/index.ts index c03746ca220ee..e8af94adabf3d 100644 --- a/x-pack/plugins/apm/server/lib/services/annotations/index.ts +++ b/x-pack/plugins/apm/server/lib/services/annotations/index.ts @@ -3,112 +3,66 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { isNumber } from 'lodash'; -import { Annotation, AnnotationType } from '../../../../common/annotations'; -import { ESFilter } from '../../../../typings/elasticsearch'; -import { - SERVICE_NAME, - SERVICE_ENVIRONMENT, - PROCESSOR_EVENT -} from '../../../../common/elasticsearch_fieldnames'; +import { SERVICE_NAME } from '../../../../common/elasticsearch_fieldnames'; +import { ScopedAnnotationsClient } from '../../../../../observability/server'; +import { getDerivedServiceAnnotations } from './get_derived_service_annotations'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -import { rangeFilter } from '../../helpers/range_filter'; -import { SERVICE_VERSION } from '../../../../common/elasticsearch_fieldnames'; +import { Annotation, AnnotationType } from '../../../../common/annotations'; export async function getServiceAnnotations({ setup, serviceName, - environment + environment, + annotationsClient }: { serviceName: string; environment?: string; setup: Setup & SetupTimeRange; + annotationsClient?: ScopedAnnotationsClient; }) { - const { start, end, client, indices } = setup; - - const filter: ESFilter[] = [ - { term: { [PROCESSOR_EVENT]: 'transaction' } }, - { range: rangeFilter(start, end) }, - { term: { [SERVICE_NAME]: serviceName } } - ]; + async function getStoredAnnotations(): Promise { + if (!annotationsClient) { + return []; + } - if (environment) { - filter.push({ term: { [SERVICE_ENVIRONMENT]: environment } }); - } - - const versions = - ( - await client.search({ - index: indices['apm_oss.transactionIndices'], - body: { - size: 0, - track_total_hits: false, - query: { - bool: { - filter - } - }, - aggs: { - versions: { - terms: { - field: SERVICE_VERSION - } - } - } + const response = await annotationsClient.search({ + start: setup.start, + end: setup.end, + annotation: { + type: 'deployment' + }, + tags: ['apm'], + filter: { + term: { + [SERVICE_NAME]: serviceName } - }) - ).aggregations?.versions.buckets.map(bucket => bucket.key) ?? []; - - if (versions.length > 1) { - const annotations = await Promise.all( - versions.map(async version => { - const response = await client.search({ - index: indices['apm_oss.transactionIndices'], - body: { - size: 0, - query: { - bool: { - filter: filter - .filter(esFilter => !Object.keys(esFilter).includes('range')) - .concat({ - term: { - [SERVICE_VERSION]: version - } - }) - } - }, - aggs: { - first_seen: { - min: { - field: '@timestamp' - } - } - }, - track_total_hits: false - } - }); + } + }); - const firstSeen = response.aggregations?.first_seen.value; + return response.hits.hits.map(hit => { + return { + type: AnnotationType.VERSION, + id: hit._id, + time: new Date(hit._source['@timestamp']).getTime(), + text: hit._source.message + }; + }); + } - if (!isNumber(firstSeen)) { - throw new Error( - 'First seen for version was unexpectedly undefined or null.' - ); - } + const [derivedAnnotations, storedAnnotations] = await Promise.all([ + getDerivedServiceAnnotations({ + setup, + serviceName, + environment + }), + getStoredAnnotations() + ]); - if (firstSeen < start || firstSeen > end) { - return null; - } - - return { - type: AnnotationType.VERSION, - id: version, - time: firstSeen, - text: version - }; - }) - ); - return { annotations: annotations.filter(Boolean) as Annotation[] }; + if (storedAnnotations.length) { + return { annotations: storedAnnotations }; } - return { annotations: [] }; + + return { + annotations: derivedAnnotations + }; } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/create_agent_config_index.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/create_agent_config_index.ts index b2dc22ceb2918..356863c5f6e1e 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/create_agent_config_index.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/create_agent_config_index.ts @@ -5,11 +5,11 @@ */ import { IClusterClient, Logger } from 'src/core/server'; -import { APMConfig } from '../../..'; import { createOrUpdateIndex, - Mappings -} from '../../helpers/create_or_update_index'; + MappingsDefinition +} from '../../../../../observability/server'; +import { APMConfig } from '../../..'; import { getApmIndicesConfig } from '../apm_indices/get_apm_indices'; export async function createApmAgentConfigurationIndex({ @@ -22,10 +22,15 @@ export async function createApmAgentConfigurationIndex({ logger: Logger; }) { const index = getApmIndicesConfig(config).apmAgentConfigurationIndex; - return createOrUpdateIndex({ index, esClient, logger, mappings }); + return createOrUpdateIndex({ + index, + apiCaller: esClient.callAsInternalUser, + logger, + mappings + }); } -const mappings: Mappings = { +const mappings: MappingsDefinition = { dynamic: 'strict', dynamic_templates: [ { diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/delete_configuration.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/delete_configuration.ts index 293c01d4b61d5..be5f9f342557d 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/delete_configuration.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/delete_configuration.ts @@ -16,7 +16,7 @@ export async function deleteConfiguration({ const { internalClient, indices } = setup; const params = { - refresh: 'wait_for', + refresh: 'wait_for' as const, index: indices.apmAgentConfigurationIndex, id: configurationId }; diff --git a/x-pack/plugins/apm/server/lib/settings/custom_link/create_custom_link_index.ts b/x-pack/plugins/apm/server/lib/settings/custom_link/create_custom_link_index.ts index 42b99b34beea7..bc0af3b0bb254 100644 --- a/x-pack/plugins/apm/server/lib/settings/custom_link/create_custom_link_index.ts +++ b/x-pack/plugins/apm/server/lib/settings/custom_link/create_custom_link_index.ts @@ -5,11 +5,11 @@ */ import { IClusterClient, Logger } from 'src/core/server'; -import { APMConfig } from '../../..'; import { createOrUpdateIndex, - Mappings -} from '../../helpers/create_or_update_index'; + MappingsDefinition +} from '../../../../../observability/server'; +import { APMConfig } from '../../..'; import { getApmIndicesConfig } from '../apm_indices/get_apm_indices'; export const createApmCustomLinkIndex = async ({ @@ -22,10 +22,15 @@ export const createApmCustomLinkIndex = async ({ logger: Logger; }) => { const index = getApmIndicesConfig(config).apmCustomLinkIndex; - return createOrUpdateIndex({ index, esClient, logger, mappings }); + return createOrUpdateIndex({ + index, + apiCaller: esClient.callAsInternalUser, + logger, + mappings + }); }; -const mappings: Mappings = { +const mappings: MappingsDefinition = { dynamic: 'strict', properties: { '@timestamp': { diff --git a/x-pack/plugins/apm/server/lib/settings/custom_link/delete_custom_link.ts b/x-pack/plugins/apm/server/lib/settings/custom_link/delete_custom_link.ts index 2f3ea0940cb26..215c30b9581ff 100644 --- a/x-pack/plugins/apm/server/lib/settings/custom_link/delete_custom_link.ts +++ b/x-pack/plugins/apm/server/lib/settings/custom_link/delete_custom_link.ts @@ -16,7 +16,7 @@ export async function deleteCustomLink({ const { internalClient, indices } = setup; const params = { - refresh: 'wait_for', + refresh: 'wait_for' as const, index: indices.apmCustomLinkIndex, id: customLinkId }; diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index b434d41982f4c..9691c3e0a5a6c 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -14,6 +14,7 @@ import { Observable, combineLatest, AsyncSubject } from 'rxjs'; import { map, take } from 'rxjs/operators'; import { Server } from 'hapi'; import { once } from 'lodash'; +import { ObservabilityPluginSetup } from '../../observability/server'; import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; import { TaskManagerSetupContract } from '../../task_manager/server'; import { AlertingPlugin } from '../../alerting/server'; @@ -62,6 +63,7 @@ export class APMPlugin implements Plugin { taskManager?: TaskManagerSetupContract; alerting?: AlertingPlugin['setup']; actions?: ActionsPlugin['setup']; + observability?: ObservabilityPluginSetup; } ) { this.logger = this.initContext.logger.get(); @@ -82,6 +84,9 @@ export class APMPlugin implements Plugin { createApmApi().init(core, { config$: mergedConfig$, logger: this.logger!, + plugins: { + observability: plugins.observability + }, __LEGACY }); }); diff --git a/x-pack/plugins/apm/server/routes/create_api/index.test.ts b/x-pack/plugins/apm/server/routes/create_api/index.test.ts index 312dae1d1f9d2..9b83a49405d05 100644 --- a/x-pack/plugins/apm/server/routes/create_api/index.test.ts +++ b/x-pack/plugins/apm/server/routes/create_api/index.test.ts @@ -41,7 +41,8 @@ const getCoreMock = () => { logger: ({ error: jest.fn() } as unknown) as Logger, - __LEGACY: {} as LegacySetup + __LEGACY: {} as LegacySetup, + plugins: {} } }; }; diff --git a/x-pack/plugins/apm/server/routes/create_api/index.ts b/x-pack/plugins/apm/server/routes/create_api/index.ts index e216574f8a02e..b480558e4bc5c 100644 --- a/x-pack/plugins/apm/server/routes/create_api/index.ts +++ b/x-pack/plugins/apm/server/routes/create_api/index.ts @@ -30,7 +30,7 @@ export function createApi() { factoryFns.push(fn); return this as any; }, - init(core, { config$, logger, __LEGACY }) { + init(core, { config$, logger, __LEGACY, plugins }) { const router = core.http.createRouter(); let config = {} as APMConfig; @@ -137,6 +137,7 @@ export function createApi() { context: { ...context, __LEGACY, + plugins, // Only return values for parameters that have runtime types, // but always include query as _debug is always set even if // it's not defined in the route. diff --git a/x-pack/plugins/apm/server/routes/create_apm_api.ts b/x-pack/plugins/apm/server/routes/create_apm_api.ts index 57b3f282852c4..524e86d9b8f73 100644 --- a/x-pack/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/plugins/apm/server/routes/create_apm_api.ts @@ -18,7 +18,8 @@ import { serviceTransactionTypesRoute, servicesRoute, serviceNodeMetadataRoute, - serviceAnnotationsRoute + serviceAnnotationsRoute, + serviceAnnotationsCreateRoute } from './services'; import { agentConfigurationRoute, @@ -85,6 +86,7 @@ const createApmApi = () => { .add(servicesRoute) .add(serviceNodeMetadataRoute) .add(serviceAnnotationsRoute) + .add(serviceAnnotationsCreateRoute) // Agent configuration .add(getSingleAgentConfigurationRoute) diff --git a/x-pack/plugins/apm/server/routes/services.ts b/x-pack/plugins/apm/server/routes/services.ts index 1c6561ee24c93..b47542273cc5b 100644 --- a/x-pack/plugins/apm/server/routes/services.ts +++ b/x-pack/plugins/apm/server/routes/services.ts @@ -5,6 +5,9 @@ */ import * as t from 'io-ts'; +import Boom from 'boom'; +import { unique } from 'lodash'; +import { ScopedAnnotationsClient } from '../../../observability/server'; import { setupRequest } from '../lib/helpers/setup_request'; import { getServiceAgentName } from '../lib/services/get_service_agent_name'; import { getServices } from '../lib/services/get_services'; @@ -13,6 +16,7 @@ import { getServiceNodeMetadata } from '../lib/services/get_service_node_metadat import { createRoute } from './create_route'; import { uiFiltersRt, rangeRt } from './default_api_types'; import { getServiceAnnotations } from '../lib/services/annotations'; +import { dateAsStringRt } from '../../common/runtime_types/date_as_string_rt'; export const servicesRoute = createRoute(core => ({ path: '/api/apm/services', @@ -74,7 +78,7 @@ export const serviceNodeMetadataRoute = createRoute(() => ({ })); export const serviceAnnotationsRoute = createRoute(() => ({ - path: '/api/apm/services/{serviceName}/annotations', + path: '/api/apm/services/{serviceName}/annotation/search', params: { path: t.type({ serviceName: t.string @@ -91,10 +95,77 @@ export const serviceAnnotationsRoute = createRoute(() => ({ const { serviceName } = context.params.path; const { environment } = context.params.query; + let annotationsClient: ScopedAnnotationsClient | undefined; + + if (context.plugins.observability) { + annotationsClient = await context.plugins.observability.getScopedAnnotationsClient( + request + ); + } + return getServiceAnnotations({ setup, serviceName, - environment + environment, + annotationsClient + }); + } +})); + +export const serviceAnnotationsCreateRoute = createRoute(() => ({ + path: '/api/apm/services/{serviceName}/annotation', + method: 'POST', + options: { + tags: ['access:apm', 'access:apm_write'] + }, + params: { + path: t.type({ + serviceName: t.string + }), + body: t.intersection([ + t.type({ + '@timestamp': dateAsStringRt, + service: t.intersection([ + t.type({ + version: t.string + }), + t.partial({ + environment: t.string + }) + ]) + }), + t.partial({ + message: t.string, + tags: t.array(t.string) + }) + ]) + }, + handler: async ({ request, context }) => { + let annotationsClient: ScopedAnnotationsClient | undefined; + + if (context.plugins.observability) { + annotationsClient = await context.plugins.observability.getScopedAnnotationsClient( + request + ); + } + + if (!annotationsClient) { + throw Boom.notFound(); + } + + const { body, path } = context.params; + + return annotationsClient.create({ + message: body.service.version, + ...body, + annotation: { + type: 'deployment' + }, + service: { + ...body.service, + name: path.serviceName + }, + tags: unique(['apm'].concat(body.tags ?? [])) }); } })); diff --git a/x-pack/plugins/apm/server/routes/typings.ts b/x-pack/plugins/apm/server/routes/typings.ts index 3dc485630c180..33e6331bade08 100644 --- a/x-pack/plugins/apm/server/routes/typings.ts +++ b/x-pack/plugins/apm/server/routes/typings.ts @@ -14,6 +14,7 @@ import { import { PickByValue, Optional } from 'utility-types'; import { Observable } from 'rxjs'; import { Server } from 'hapi'; +import { ObservabilityPluginSetup } from '../../../observability/server'; import { FetchOptions } from '../../../../legacy/plugins/apm/public/services/rest/callApi'; import { APMConfig } from '..'; @@ -61,6 +62,9 @@ export type APMRequestHandlerContext< params: { query: { _debug: boolean } } & TDecodedParams; config: APMConfig; logger: Logger; + plugins: { + observability?: ObservabilityPluginSetup; + }; __LEGACY: { server: APMLegacyServer; }; @@ -107,6 +111,9 @@ export interface ServerAPI { context: { config$: Observable; logger: Logger; + plugins: { + observability?: ObservabilityPluginSetup; + }; __LEGACY: { server: Server }; } ) => void; diff --git a/x-pack/plugins/observability/common/annotations.ts b/x-pack/plugins/observability/common/annotations.ts new file mode 100644 index 0000000000000..4f2b548053565 --- /dev/null +++ b/x-pack/plugins/observability/common/annotations.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { JsonObject } from '../../../../src/plugins/kibana_utils/common'; +import { fromKueryExpression, toElasticsearchQuery } from '../../../../src/plugins/data/common'; +import { dateAsStringRt } from '../../apm/common/runtime_types/date_as_string_rt'; + +const toEsQueryRt = new t.Type( + 'elasticsearchQuery', + (input): input is JsonObject => true, + (input, context) => { + if (!input) { + return t.failure(input, context, 'input is empty'); + } + try { + const ast = fromKueryExpression(input); + const filter = toElasticsearchQuery(ast); + return t.success(filter); + } catch (err) { + return t.failure(input, context, err.message); + } + }, + () => { + throw new Error('Cannot encode a decoded kuery expression'); + } +); + +export const searchAnnotationsRt = t.intersection([ + t.type({ + start: t.number, + end: t.number, + }), + t.partial({ + size: t.number, + annotation: t.type({ + type: t.string, + }), + tags: t.array(t.string), + filter: toEsQueryRt, + }), +]); + +export const createAnnotationRt = t.intersection([ + t.type({ + annotation: t.type({ + type: t.string, + }), + '@timestamp': dateAsStringRt, + message: t.string, + }), + t.partial({ + tags: t.array(t.string), + service: t.partial({ + name: t.string, + environment: t.string, + version: t.string, + }), + }), +]); + +export const deleteAnnotationRt = t.type({ + id: t.string, +}); + +export const getAnnotationByIdRt = t.type({ + id: t.string, +}); + +export interface Annotation { + annotation: { + type: string; + }; + tags?: string[]; + message: string; + service?: { + name?: string; + environment?: string; + version?: string; + }; + event: { + created: string; + }; + '@timestamp': string; +} diff --git a/x-pack/plugins/observability/kibana.json b/x-pack/plugins/observability/kibana.json index 438b9ddea4734..8e2cfe980039c 100644 --- a/x-pack/plugins/observability/kibana.json +++ b/x-pack/plugins/observability/kibana.json @@ -2,6 +2,10 @@ "id": "observability", "version": "8.0.0", "kibanaVersion": "kibana", - "configPath": ["xpack", "observability"], - "ui": true + "configPath": [ + "xpack", + "observability" + ], + "ui": true, + "server": true } diff --git a/x-pack/plugins/observability/server/index.ts b/x-pack/plugins/observability/server/index.ts new file mode 100644 index 0000000000000..78550b781b411 --- /dev/null +++ b/x-pack/plugins/observability/server/index.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; +import { PluginInitializerContext } from 'src/core/server'; +import { ObservabilityPlugin, ObservabilityPluginSetup } from './plugin'; +import { createOrUpdateIndex, MappingsDefinition } from './utils/create_or_update_index'; +import { ScopedAnnotationsClient } from './lib/annotations/bootstrap_annotations'; + +export const config = { + schema: schema.object({ + enabled: schema.boolean({ defaultValue: true }), + annotations: schema.object({ + enabled: schema.boolean({ defaultValue: true }), + index: schema.string({ defaultValue: 'observability-annotations' }), + }), + }), +}; + +export type ObservabilityConfig = TypeOf; + +export const plugin = (initContext: PluginInitializerContext) => + new ObservabilityPlugin(initContext); + +export { + createOrUpdateIndex, + MappingsDefinition, + ObservabilityPluginSetup, + ScopedAnnotationsClient, +}; diff --git a/x-pack/plugins/observability/server/lib/annotations/bootstrap_annotations.ts b/x-pack/plugins/observability/server/lib/annotations/bootstrap_annotations.ts new file mode 100644 index 0000000000000..1814899e5f28b --- /dev/null +++ b/x-pack/plugins/observability/server/lib/annotations/bootstrap_annotations.ts @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import * as t from 'io-ts'; +import { CoreSetup, PluginInitializerContext, KibanaRequest, RequestHandler } from 'kibana/server'; +import { schema } from '@kbn/config-schema'; +import { PathReporter } from 'io-ts/lib/PathReporter'; +import { isLeft } from 'fp-ts/lib/Either'; +import { + getAnnotationByIdRt, + createAnnotationRt, + deleteAnnotationRt, +} from '../../../common/annotations'; +import { PromiseReturnType } from '../../../../apm/typings/common'; +import { createAnnotationsClient } from './create_annotations_client'; + +interface Params { + index: string; + core: CoreSetup; + context: PluginInitializerContext; +} + +export type ScopedAnnotationsClientFactory = PromiseReturnType< + typeof bootstrapAnnotations +>['getScopedAnnotationsClient']; + +export type ScopedAnnotationsClient = ReturnType; +export type AnnotationsAPI = PromiseReturnType; + +const unknowns = schema.object({}, { unknowns: 'allow' }); + +export async function bootstrapAnnotations({ index, core, context }: Params) { + const logger = context.logger.get('annotations'); + + function wrapRouteHandler>( + types: TType, + handler: (params: { data: t.TypeOf; client: ScopedAnnotationsClient }) => Promise + ): RequestHandler { + return async (...args: Parameters) => { + const [, request, response] = args; + + const rt = types; + + const data = { + body: request.body, + query: request.query, + params: request.params, + }; + + const validation = rt.decode(data); + + if (isLeft(validation)) { + return response.badRequest({ + body: PathReporter.report(validation).join(', '), + }); + } + + const apiCaller = core.elasticsearch.dataClient.asScoped(request).callAsCurrentUser; + + const client = createAnnotationsClient({ + index, + apiCaller, + logger, + }); + + const res = await handler({ + data: validation.right as any, + client, + }); + + return response.ok({ + body: res, + }); + }; + } + + const router = core.http.createRouter(); + + router.post( + { + path: '/api/observability/annotation', + validate: { + body: unknowns, + }, + }, + wrapRouteHandler(t.type({ body: createAnnotationRt }), ({ data, client }) => { + return client.create(data.body); + }) + ); + + router.delete( + { + path: '/api/observability/annotation/{id}', + validate: { + params: unknowns, + }, + }, + wrapRouteHandler(t.type({ params: deleteAnnotationRt }), ({ data, client }) => { + return client.delete(data.params); + }) + ); + + router.get( + { + path: '/api/observability/annotation/{id}', + validate: { + params: unknowns, + }, + }, + wrapRouteHandler(t.type({ params: getAnnotationByIdRt }), ({ data, client }) => { + return client.getById(data.params); + }) + ); + + return { + getScopedAnnotationsClient: (request: KibanaRequest) => { + return createAnnotationsClient({ + index, + apiCaller: core.elasticsearch.dataClient.asScoped(request).callAsCurrentUser, + logger, + }); + }, + }; +} diff --git a/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts b/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts new file mode 100644 index 0000000000000..bd838e47db7a8 --- /dev/null +++ b/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { APICaller, Logger } from 'kibana/server'; +import * as t from 'io-ts'; +import { SearchResponse, Client } from 'elasticsearch'; +import { + createAnnotationRt, + searchAnnotationsRt, + deleteAnnotationRt, + Annotation, + getAnnotationByIdRt, +} from '../../../common/annotations'; +import { PromiseReturnType } from '../../../../apm/typings/common'; +import { createOrUpdateIndex } from '../../utils/create_or_update_index'; + +type SearchParams = t.TypeOf; +type CreateParams = t.TypeOf; +type DeleteParams = t.TypeOf; +type GetByIdParams = t.TypeOf; + +interface IndexDocumentResponse { + _shards: { + total: number; + failed: number; + successful: number; + }; + _index: string; + _type: string; + _id: string; + _version: number; + _seq_no: number; + _primary_term: number; + result: string; +} + +export function createAnnotationsClient(params: { + index: string; + apiCaller: APICaller; + logger: Logger; +}) { + const { index, apiCaller, logger } = params; + + const initIndex = () => + createOrUpdateIndex({ + index, + mappings: { + dynamic: 'strict', + properties: { + annotation: { + properties: { + type: { + type: 'keyword', + }, + }, + }, + message: { + type: 'text', + }, + tags: { + type: 'keyword', + }, + '@timestamp': { + type: 'date', + }, + event: { + properties: { + created: { + type: 'date', + }, + }, + }, + service: { + properties: { + name: { + type: 'keyword', + }, + environment: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + }, + }, + }, + }, + apiCaller, + logger, + }); + + return { + search: async (searchParams: SearchParams): Promise> => { + const { start, end, size, annotation, tags, filter } = searchParams; + + try { + return await apiCaller('search', { + index, + body: { + size: size ?? 50, + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: start, + lt: end, + }, + }, + }, + ...(annotation?.type ? [{ term: { 'annotation.type': annotation.type } }] : []), + ...(tags ? [{ terms: { tags } }] : []), + ...(filter ? [filter] : []), + ], + }, + }, + }, + }); + } catch (err) { + if (err.body?.error?.type !== 'index_not_found_exception') { + throw err; + } + return { + hits: { + total: 0, + max_score: 0, + hits: [], + }, + took: 0, + _shards: { + failed: 0, + skipped: 0, + successful: 0, + total: 0, + }, + timed_out: false, + }; + } + }, + create: async ( + createParams: CreateParams + ): Promise<{ _id: string; _index: string; _source: Annotation }> => { + await initIndex(); + + const annotation = { + ...createParams, + event: { + created: new Date().toISOString(), + }, + }; + + const response = (await apiCaller('index', { + index, + body: annotation, + refresh: 'wait_for', + })) as IndexDocumentResponse; + + return { + _id: response._id, + _index: response._index, + _source: annotation, + }; + }, + getById: async (getByIdParams: GetByIdParams) => { + const { id } = getByIdParams; + + return apiCaller('get', { + id, + index, + }); + }, + delete: async (deleteParams: DeleteParams) => { + const { id } = deleteParams; + + const response = (await apiCaller('delete', { + index, + id, + refresh: 'wait_for', + })) as PromiseReturnType; + return response; + }, + }; +} diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts new file mode 100644 index 0000000000000..340feec15bb0c --- /dev/null +++ b/x-pack/plugins/observability/server/plugin.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { PluginInitializerContext, Plugin, CoreSetup } from 'src/core/server'; +import { take } from 'rxjs/operators'; +import { ObservabilityConfig } from '.'; +import { + bootstrapAnnotations, + ScopedAnnotationsClient, + ScopedAnnotationsClientFactory, + AnnotationsAPI, +} from './lib/annotations/bootstrap_annotations'; + +type LazyScopedAnnotationsClientFactory = ( + ...args: Parameters +) => Promise; + +export interface ObservabilityPluginSetup { + getScopedAnnotationsClient: LazyScopedAnnotationsClientFactory; +} + +export class ObservabilityPlugin implements Plugin { + constructor(private readonly initContext: PluginInitializerContext) { + this.initContext = initContext; + } + + public async setup(core: CoreSetup, plugins: {}): Promise { + const config$ = this.initContext.config.create(); + + const config = await config$.pipe(take(1)).toPromise(); + + let annotationsApiPromise: Promise | undefined; + + if (config.annotations.enabled) { + annotationsApiPromise = bootstrapAnnotations({ + core, + index: config.annotations.index, + context: this.initContext, + }).catch(err => { + this.initContext.logger.get('annotations').warn(err); + throw err; + }); + } + + return { + getScopedAnnotationsClient: async (...args) => { + return annotationsApiPromise + ? (await annotationsApiPromise).getScopedAnnotationsClient(...args) + : undefined; + }, + }; + } + + public start() {} + + public stop() {} +} diff --git a/x-pack/plugins/observability/server/utils/create_or_update_index.ts b/x-pack/plugins/observability/server/utils/create_or_update_index.ts new file mode 100644 index 0000000000000..2c6f3dbefdeb1 --- /dev/null +++ b/x-pack/plugins/observability/server/utils/create_or_update_index.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import pRetry from 'p-retry'; +import { Logger, APICaller } from 'src/core/server'; + +export interface MappingsObject { + type: string; + ignore_above?: number; + scaling_factor?: number; + ignore_malformed?: boolean; + coerce?: boolean; + fields?: Record; +} + +export interface MappingsDefinition { + dynamic?: boolean | 'strict'; + properties: Record; + dynamic_templates?: any[]; +} + +export async function createOrUpdateIndex({ + index, + mappings, + apiCaller, + logger, +}: { + index: string; + mappings: MappingsDefinition; + apiCaller: APICaller; + logger: Logger; +}) { + try { + /* + * In some cases we could be trying to create an index before ES is ready. + * When this happens, we retry creating the index with exponential backoff. + * We use retry's default formula, meaning that the first retry happens after 2s, + * the 5th after 32s, and the final attempt after around 17m. If the final attempt fails, + * the error is logged to the console. + * See https://github.com/sindresorhus/p-retry and https://github.com/tim-kos/node-retry. + */ + await pRetry( + async () => { + const indexExists = await apiCaller('indices.exists', { index }); + const result = indexExists + ? await updateExistingIndex({ + index, + apiCaller, + mappings, + }) + : await createNewIndex({ + index, + apiCaller, + mappings, + }); + + if (!result.acknowledged) { + const resultError = result && result.error && JSON.stringify(result.error); + throw new Error(resultError); + } + }, + { + onFailedAttempt: e => { + logger.warn(`Could not create index: '${index}'. Retrying...`); + logger.warn(e); + }, + } + ); + } catch (e) { + logger.error(`Could not create index: '${index}'. Error: ${e.message}.`); + } +} + +function createNewIndex({ + index, + apiCaller, + mappings, +}: { + index: string; + apiCaller: APICaller; + mappings: MappingsDefinition; +}) { + return apiCaller('indices.create', { + index, + body: { + // auto_expand_replicas: Allows cluster to not have replicas for this index + settings: { 'index.auto_expand_replicas': '0-1' }, + mappings, + }, + }); +} + +function updateExistingIndex({ + index, + apiCaller, + mappings, +}: { + index: string; + apiCaller: APICaller; + mappings: MappingsDefinition; +}) { + return apiCaller('indices.putMapping', { + index, + body: mappings, + }); +} diff --git a/x-pack/test/api_integration/apis/apm/annotations.ts b/x-pack/test/api_integration/apis/apm/annotations.ts new file mode 100644 index 0000000000000..6935ab7372eae --- /dev/null +++ b/x-pack/test/api_integration/apis/apm/annotations.ts @@ -0,0 +1,324 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { merge, cloneDeep, isPlainObject } from 'lodash'; +import { JsonObject } from 'src/plugins/kibana_utils/common'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +const DEFAULT_INDEX_NAME = 'observability-annotations'; + +export default function annotationApiTests({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const es = getService('es'); + + function expectContainsObj(source: JsonObject, expected: JsonObject) { + expect(source).to.eql( + merge(cloneDeep(source), expected, (a, b) => { + if (isPlainObject(a) && isPlainObject(b)) { + return undefined; + } + return b; + }) + ); + } + + function request({ method, url, data }: { method: string; url: string; data?: JsonObject }) { + switch (method.toLowerCase()) { + case 'get': + return supertest.get(url).set('kbn-xsrf', 'foo'); + + case 'post': + return supertest + .post(url) + .send(data) + .set('kbn-xsrf', 'foo'); + + default: + throw new Error(`Unsupported methoed ${method}`); + } + } + + describe('APM annotations', () => { + describe('when creating an annotation', () => { + afterEach(async () => { + const indexExists = (await es.indices.exists({ index: DEFAULT_INDEX_NAME })).body; + if (indexExists) { + await es.indices.delete({ + index: DEFAULT_INDEX_NAME, + }); + } + }); + + it('fails with a 400 bad request if data is missing', async () => { + const response = await request({ + url: '/api/apm/services/opbeans-java/annotation', + method: 'POST', + }); + + expect(response.status).to.be(400); + }); + + it('fails with a 400 bad request if data is invalid', async () => { + const invalidTimestampResponse = await request({ + url: '/api/apm/services/opbeans-java/annotation', + method: 'POST', + data: { + '@timestamp': 'foo', + message: 'foo', + }, + }); + + expect(invalidTimestampResponse.status).to.be(400); + + const missingServiceVersionResponse = await request({ + url: '/api/apm/services/opbeans-java/annotation', + method: 'POST', + data: { + '@timestamp': new Date().toISOString(), + message: 'New deployment', + }, + }); + + expect(missingServiceVersionResponse.status).to.be(400); + }); + + it('completes with a 200 and the created annotation if data is complete and valid', async () => { + const timestamp = new Date().toISOString(); + + const response = await request({ + url: '/api/apm/services/opbeans-java/annotation', + method: 'POST', + data: { + '@timestamp': timestamp, + message: 'New deployment', + tags: ['foo'], + service: { + version: '1.1', + environment: 'production', + }, + }, + }); + + expect(response.status).to.be(200); + + expectContainsObj(response.body, { + _source: { + annotation: { + type: 'deployment', + }, + tags: ['apm', 'foo'], + message: 'New deployment', + '@timestamp': timestamp, + service: { + name: 'opbeans-java', + version: '1.1', + environment: 'production', + }, + }, + }); + }); + + it('prefills `message` and `tags`', async () => { + const timestamp = new Date().toISOString(); + + const response = await request({ + url: '/api/apm/services/opbeans-java/annotation', + method: 'POST', + data: { + '@timestamp': timestamp, + service: { + version: '1.1', + }, + }, + }); + + expectContainsObj(response.body, { + _source: { + annotation: { + type: 'deployment', + }, + tags: ['apm'], + message: '1.1', + '@timestamp': timestamp, + service: { + name: 'opbeans-java', + version: '1.1', + }, + }, + }); + }); + }); + + describe('when mixing stored and derived annotations', () => { + const transactionIndexName = 'apm-8.0.0-transaction'; + const serviceName = 'opbeans-java'; + + beforeEach(async () => { + await es.indices.create({ + index: transactionIndexName, + body: { + mappings: { + properties: { + service: { + properties: { + name: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + }, + }, + '@timestamp': { + type: 'date', + }, + observer: { + properties: { + version_major: { + type: 'long', + }, + }, + }, + }, + }, + }, + }); + + await es.index({ + index: transactionIndexName, + body: { + '@timestamp': new Date(2020, 4, 2, 18, 30).toISOString(), + processor: { + event: 'transaction', + }, + service: { + name: serviceName, + version: '1.1', + }, + observer: { + version_major: 8, + }, + }, + refresh: 'wait_for', + }); + + await es.index({ + index: transactionIndexName, + body: { + '@timestamp': new Date(2020, 4, 2, 19, 30).toISOString(), + processor: { + event: 'transaction', + }, + service: { + name: serviceName, + version: '1.2', + }, + observer: { + version_major: 8, + }, + }, + refresh: 'wait_for', + }); + }); + + afterEach(async () => { + await es.indices.delete({ + index: transactionIndexName, + }); + + const annotationIndexExists = ( + await es.indices.exists({ + index: DEFAULT_INDEX_NAME, + }) + ).body; + + if (annotationIndexExists) { + await es.indices.delete({ + index: DEFAULT_INDEX_NAME, + }); + } + }); + + it('returns the derived annotations if there are no stored annotations', async () => { + const range = { + start: new Date(2020, 4, 2, 18).toISOString(), + end: new Date(2020, 4, 2, 20).toISOString(), + }; + + const response = await request({ + url: `/api/apm/services/${serviceName}/annotation/search?start=${range.start}&end=${range.end}`, + method: 'GET', + }); + + expect(response.status).to.be(200); + + expect(response.body.annotations.length).to.be(2); + expect(response.body.annotations[0].text).to.be('1.1'); + expect(response.body.annotations[1].text).to.be('1.2'); + }); + + it('returns the stored annotations only if there are any', async () => { + const range = { + start: new Date(2020, 4, 2, 18).toISOString(), + end: new Date(2020, 4, 2, 23).toISOString(), + }; + + expect( + ( + await request({ + url: `/api/apm/services/${serviceName}/annotation`, + method: 'POST', + data: { + service: { + version: '1.3', + }, + '@timestamp': new Date(2020, 4, 2, 21, 30).toISOString(), + }, + }) + ).status + ).to.be(200); + + const response = await request({ + url: `/api/apm/services/${serviceName}/annotation/search?start=${range.start}&end=${range.end}`, + method: 'GET', + }); + + expect(response.body.annotations.length).to.be(1); + expect(response.body.annotations[0].text).to.be('1.3'); + + const earlierRange = { + start: new Date(2020, 4, 2, 18).toISOString(), + end: new Date(2020, 4, 2, 20).toISOString(), + }; + + expect( + ( + await request({ + url: `/api/apm/services/${serviceName}/annotation`, + method: 'POST', + data: { + service: { + version: '1.3', + }, + '@timestamp': new Date(2020, 4, 2, 21, 30).toISOString(), + }, + }) + ).status + ).to.be(200); + + const responseFromEarlierRange = await request({ + url: `/api/apm/services/${serviceName}/annotation/search?start=${earlierRange.start}&end=${earlierRange.end}`, + method: 'GET', + }); + + expect(responseFromEarlierRange.body.annotations.length).to.be(2); + expect(responseFromEarlierRange.body.annotations[0].text).to.be('1.1'); + expect(responseFromEarlierRange.body.annotations[1].text).to.be('1.2'); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/apis/apm/feature_controls.ts b/x-pack/test/api_integration/apis/apm/feature_controls.ts index 9f76941935bb7..5f61c963a69aa 100644 --- a/x-pack/test/api_integration/apis/apm/feature_controls.ts +++ b/x-pack/test/api_integration/apis/apm/feature_controls.ts @@ -42,7 +42,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext) { // this doubles as a smoke test for the _debug query parameter req: { - url: `/api/apm/services/foo/errors?start=${start}&end=${end}&uiFilters=%7B%7D_debug=true`, + url: `/api/apm/services/foo/errors?start=${start}&end=${end}&uiFilters=%7B%7D&_debug=true`, }, expectForbidden: expect404, expectResponse: expect200, diff --git a/x-pack/test/api_integration/apis/apm/index.ts b/x-pack/test/api_integration/apis/apm/index.ts index 4a4265cfd0739..de076e8c46729 100644 --- a/x-pack/test/api_integration/apis/apm/index.ts +++ b/x-pack/test/api_integration/apis/apm/index.ts @@ -8,6 +8,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function apmApiIntegrationTests({ loadTestFile }: FtrProviderContext) { describe('APM specs', () => { + loadTestFile(require.resolve('./annotations')); loadTestFile(require.resolve('./feature_controls')); loadTestFile(require.resolve('./agent_configuration')); loadTestFile(require.resolve('./custom_link')); diff --git a/x-pack/test/api_integration/apis/index.js b/x-pack/test/api_integration/apis/index.js index 0a87dcb4b5bb0..75fa90bb4c3fe 100644 --- a/x-pack/test/api_integration/apis/index.js +++ b/x-pack/test/api_integration/apis/index.js @@ -31,5 +31,6 @@ export default function({ loadTestFile }) { loadTestFile(require.resolve('./ingest')); loadTestFile(require.resolve('./endpoint')); loadTestFile(require.resolve('./ml')); + loadTestFile(require.resolve('./observability')); }); } diff --git a/x-pack/test/api_integration/apis/observability/annotations.ts b/x-pack/test/api_integration/apis/observability/annotations.ts new file mode 100644 index 0000000000000..047924865d1d6 --- /dev/null +++ b/x-pack/test/api_integration/apis/observability/annotations.ts @@ -0,0 +1,286 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { JsonObject } from 'src/plugins/kibana_utils/common'; +import { Annotation } from '../../../../plugins/observability/common/annotations'; +import { ESSearchHit } from '../../../../plugins/apm/typings/elasticsearch'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +const DEFAULT_INDEX_NAME = 'observability-annotations'; + +export default function annotationApiTests({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const es = getService('es'); + + function request({ method, url, data }: { method: string; url: string; data?: JsonObject }) { + switch (method.toLowerCase()) { + case 'get': + return supertest.get(url).set('kbn-xsrf', 'foo'); + + case 'post': + return supertest + .post(url) + .send(data) + .set('kbn-xsrf', 'foo'); + + case 'delete': + return supertest + .delete(url) + .send(data) + .set('kbn-xsrf', 'foo'); + + default: + throw new Error(`Unsupported methoed ${method}`); + } + } + + describe('Observability annotations', () => { + describe('when creating an annotation', () => { + afterEach(async () => { + const indexExists = (await es.indices.exists({ index: DEFAULT_INDEX_NAME })).body; + if (indexExists) { + await es.indices.delete({ + index: DEFAULT_INDEX_NAME, + }); + } + }); + + it('fails with a 400 bad request if data is missing', async () => { + const response = await request({ + url: '/api/observability/annotation', + method: 'POST', + }); + + expect(response.status).to.be(400); + }); + + it('fails with a 400 bad request if data is invalid', async () => { + const invalidTimestampResponse = await request({ + url: '/api/observability/annotation', + method: 'POST', + data: { + annotation: { + type: 'deployment', + }, + '@timestamp': 'foo', + message: 'foo', + }, + }); + + expect(invalidTimestampResponse.status).to.be(400); + + const missingMessageResponse = await request({ + url: '/api/observability/annotation', + method: 'POST', + data: { + annotation: { + type: 'deployment', + }, + '@timestamp': new Date().toISOString(), + }, + }); + + expect(missingMessageResponse.status).to.be(400); + }); + + it('completes with a 200 and the created annotation if data is complete and valid', async () => { + const timestamp = new Date().toISOString(); + + const response = await request({ + url: '/api/observability/annotation', + method: 'POST', + data: { + annotation: { + type: 'deployment', + }, + '@timestamp': timestamp, + message: 'test message', + tags: ['apm'], + }, + }); + + expect(response.status).to.be(200); + + const { _source, _id, _index } = response.body; + + expect(response.body).to.eql({ + _index, + _id, + _source: { + annotation: { + type: 'deployment', + }, + '@timestamp': timestamp, + message: 'test message', + tags: ['apm'], + event: { + created: _source.event.created, + }, + }, + }); + + expect(_id).to.be.a('string'); + + expect(_source.event.created).to.be.a('string'); + + const created = new Date(_source.event.created).getTime(); + expect(created).to.be.greaterThan(0); + expect(_index).to.be(DEFAULT_INDEX_NAME); + }); + + it('indexes the annotation', async () => { + const response = await request({ + url: '/api/observability/annotation', + method: 'POST', + data: { + annotation: { + type: 'deployment', + }, + '@timestamp': new Date().toISOString(), + message: 'test message', + tags: ['apm'], + }, + }); + + expect(response.status).to.be(200); + + const search = await es.search({ + index: DEFAULT_INDEX_NAME, + track_total_hits: true, + }); + + expect(search.body.hits.total.value).to.be(1); + + expect(search.body.hits.hits[0]._source).to.eql(response.body._source); + expect(search.body.hits.hits[0]._id).to.eql(response.body._id); + }); + + it('returns the annotation', async () => { + const { _id: id1 } = ( + await request({ + url: '/api/observability/annotation', + method: 'POST', + data: { + annotation: { + type: 'deployment', + }, + '@timestamp': new Date().toISOString(), + message: '1', + tags: ['apm'], + }, + }) + ).body; + + const { _id: id2 } = ( + await request({ + url: '/api/observability/annotation', + method: 'POST', + data: { + annotation: { + type: 'deployment', + }, + '@timestamp': new Date().toISOString(), + message: '2', + tags: ['apm'], + }, + }) + ).body; + + expect( + ( + await request({ + url: `/api/observability/annotation/${id1}`, + method: 'GET', + }) + ).body._source.message + ).to.be('1'); + + expect( + ( + await request({ + url: `/api/observability/annotation/${id2}`, + method: 'GET', + }) + ).body._source.message + ).to.be('2'); + }); + + it('deletes the annotation', async () => { + await request({ + url: '/api/observability/annotation', + method: 'POST', + data: { + annotation: { + type: 'deployment', + }, + '@timestamp': new Date().toISOString(), + message: 'test message', + tags: ['apm'], + }, + }); + + await request({ + url: '/api/observability/annotation', + method: 'POST', + data: { + annotation: { + type: 'deployment', + }, + '@timestamp': new Date().toISOString(), + message: 'test message 2', + tags: ['apm'], + }, + }); + + const initialSearch = await es.search({ + index: DEFAULT_INDEX_NAME, + track_total_hits: true, + }); + + expect(initialSearch.body.hits.total.value).to.be(2); + + const [id1, id2] = initialSearch.body.hits.hits.map( + (hit: ESSearchHit) => hit._id + ); + + expect( + ( + await request({ + url: `/api/observability/annotation/${id1}`, + method: 'DELETE', + }) + ).status + ).to.be(200); + + const searchAfterFirstDelete = await es.search({ + index: DEFAULT_INDEX_NAME, + track_total_hits: true, + }); + + expect(searchAfterFirstDelete.body.hits.total.value).to.be(1); + + expect(searchAfterFirstDelete.body.hits.hits[0]._id).to.be(id2); + + expect( + ( + await request({ + url: `/api/observability/annotation/${id2}`, + method: 'DELETE', + }) + ).status + ).to.be(200); + + const searchAfterSecondDelete = await es.search({ + index: DEFAULT_INDEX_NAME, + track_total_hits: true, + }); + + expect(searchAfterSecondDelete.body.hits.total.value).to.be(0); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/apis/observability/index.ts b/x-pack/test/api_integration/apis/observability/index.ts new file mode 100644 index 0000000000000..25e3e0693c3c0 --- /dev/null +++ b/x-pack/test/api_integration/apis/observability/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function observabilityApiIntegrationTests({ loadTestFile }: FtrProviderContext) { + describe('Observability specs', () => { + loadTestFile(require.resolve('./annotations')); + }); +}