Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ML] Fix custom index name settings, functional tests for APM Latency Correlation. #105200

Merged
merged 16 commits into from
Jul 19, 2021
Merged

[ML] Fix custom index name settings, functional tests for APM Latency Correlation. #105200

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
48ceb8f
apm correlations functional tests
walterra Jul 12, 2021
0bc79de
fix types.
walterra Jul 12, 2021
9a39121
fix passing on custom apm index names.
walterra Jul 13, 2021
a39dcf8
fix types
walterra Jul 13, 2021
dada021
use 8.0.0 dataset instead of ml_8.0.0
walterra Jul 14, 2021
2b2872c
fix variable names
walterra Jul 15, 2021
c439dbd
Merge branch 'master' into ml-apm-correlations-functional-tests
walterra Jul 15, 2021
ecfd6e4
fix cross-cluster search error handling.
walterra Jul 16, 2021
60bbed5
fix passing on cssWarning.
walterra Jul 16, 2021
b4dc37e
fix types.
walterra Jul 16, 2021
7592578
fix translation.
walterra Jul 16, 2021
42a7cbd
text fixes
walterra Jul 19, 2021
7d04ca3
lint fix
walterra Jul 19, 2021
1a63809
use observability:enableInspectEsQueries instead of debug=true
walterra Jul 19, 2021
06b5b8f
Merge branch 'master' into ml-apm-correlations-functional-tests
walterra Jul 19, 2021
5922e84
add retry to avoid flakyness.
walterra Jul 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion x-pack/plugins/apm/common/search_strategies/correlations/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ export interface ResponseHit {
}

export interface SearchServiceParams {
index: string;
environment?: string;
kuery?: string;
serviceName?: string;
Expand All @@ -31,6 +30,10 @@ export interface SearchServiceParams {
percentileThresholdValue?: number;
}

export interface SearchServiceFetchParams extends SearchServiceParams {
index: string;
}

export interface SearchServiceValue {
histogram: HistogramItem[];
value: string;
Expand Down
11 changes: 10 additions & 1 deletion x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,11 @@ const chartTheme: PartialTheme = {
},
};

// Log based axis cannot start a 0. Use a small positive number instead.
const yAxisDomain = {
min: 0.00001,
};

interface CorrelationsChartProps {
field?: string;
value?: string;
Expand Down Expand Up @@ -140,7 +145,10 @@ export function CorrelationsChart({
const histogram = replaceHistogramDotsWithBars(originalHistogram);

return (
<div style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}>
<div
data-test-subj="apmCorrelationsChart"
style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}
>
<Chart
size={{
height: '250px',
Expand Down Expand Up @@ -168,6 +176,7 @@ export function CorrelationsChart({
/>
<Axis
id="y-axis"
domain={yAxisDomain}
title={i18n.translate(
'xpack.apm.correlations.latency.chart.numberOfTransactionsLabel',
{ defaultMessage: '# transactions' }
Expand Down
7 changes: 6 additions & 1 deletion x-pack/plugins/apm/public/components/app/correlations/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ export function Correlations() {
return (
<>
<EuiButton
data-test-subj="apmViewCorrelationsButton"
fill
onClick={() => {
setIsFlyoutVisible(true);
Expand All @@ -147,13 +148,17 @@ export function Correlations() {
{isFlyoutVisible && (
<EuiPortal>
<EuiFlyout
data-test-subj="apmCorrelationsFlyout"
size="l"
ownFocus
onClose={() => setIsFlyoutVisible(false)}
>
<EuiFlyoutHeader hasBorder aria-labelledby="correlations-flyout">
<EuiTitle>
<h2 id="correlations-flyout">
<h2
data-test-subj="apmCorrelationsFlyoutHeader"
id="correlations-flyout"
>
{CORRELATIONS_TITLE}
&nbsp;
<EuiBetaBadge
Expand Down
121 changes: 91 additions & 30 deletions x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
import React, { useEffect, useMemo, useState } from 'react';
import { useHistory } from 'react-router-dom';
import {
EuiCallOut,
EuiCode,
EuiAccordion,
EuiPanel,
EuiIcon,
EuiBasicTableColumn,
EuiButton,
Expand All @@ -34,7 +38,10 @@ import {
} from './correlations_table';
import { useCorrelations } from './use_correlations';
import { push } from '../../shared/Links/url_helpers';
import { useUiTracker } from '../../../../../observability/public';
import {
enableInspectEsQueries,
useUiTracker,
} from '../../../../../observability/public';
import { asPreciseDecimal } from '../../../../common/utils/formatters';
import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context';
import { LatencyCorrelationsHelpPopover } from './ml_latency_correlations_help_popover';
Expand All @@ -58,15 +65,19 @@ interface MlCorrelationsTerms {

export function MlLatencyCorrelations({ onClose }: Props) {
const {
core: { notifications },
core: { notifications, uiSettings },
} = useApmPluginContext();

const { serviceName, transactionType } = useApmServiceContext();
const { urlParams } = useUrlParams();

const { environment, kuery, transactionName, start, end } = urlParams;

const displayLog = uiSettings.get<boolean>(enableInspectEsQueries);

const {
ccsWarning,
log,
error,
histograms,
percentileThresholdValue,
Expand All @@ -76,7 +87,6 @@ export function MlLatencyCorrelations({ onClose }: Props) {
cancelFetch,
overallHistogram: originalOverallHistogram,
} = useCorrelations({
index: 'apm-*',
...{
...{
environment,
Expand Down Expand Up @@ -286,9 +296,10 @@ export function MlLatencyCorrelations({ onClose }: Props) {
</EuiFlexItem>
<EuiFlexItem>
<EuiFlexGroup direction="column" gutterSize="none">
<EuiFlexItem>
<EuiFlexItem data-test-subj="apmCorrelationsLatencyCorrelationsProgressTitle">
<EuiText size="xs" color="subdued">
<FormattedMessage
data-test-subj="apmCorrelationsLatencyCorrelationsProgressTitle"
id="xpack.apm.correlations.latencyCorrelations.progressTitle"
defaultMessage="Progress: {progress}%"
values={{ progress: Math.round(progress * 100) }}
Expand All @@ -313,11 +324,35 @@ export function MlLatencyCorrelations({ onClose }: Props) {
</EuiFlexItem>
</EuiFlexGroup>

{ccsWarning && (
<>
<EuiSpacer size="m" />
<EuiCallOut
title={i18n.translate(
'xpack.apm.correlations.latencyCorrelations.ccsWarningCalloutTitle',
{
defaultMessage: 'Cross-cluster search compatibility',
}
)}
color="warning"
>
<p>
{i18n.translate(
'xpack.apm.correlations.latencyCorrelations.ccsWarningCalloutBody',
{
defaultMessage:
'Data for the correlation analysis could not be fully retrieved. This feature is supported only for 7.14 and later versions.',
}
)}
</p>
</EuiCallOut>
</>
)}
<EuiSpacer size="m" />
{overallHistogram !== undefined ? (
<>
<EuiTitle size="xxs">
<h4>
<h4 data-test-subj="apmCorrelationsLatencyCorrelationsChartTitle">
{i18n.translate(
'xpack.apm.correlations.latencyCorrelations.chartTitle',
{
Expand All @@ -341,32 +376,58 @@ export function MlLatencyCorrelations({ onClose }: Props) {
</>
) : null}

{histograms.length > 0 && selectedHistogram !== undefined && (
<CorrelationsTable
// @ts-ignore correlations don't have the same column format other tables have
columns={mlCorrelationColumns}
// @ts-expect-error correlations don't have the same significant term other tables have
significantTerms={histogramTerms}
status={FETCH_STATUS.SUCCESS}
setSelectedSignificantTerm={setSelectedSignificantTerm}
selectedTerm={{
fieldName: selectedHistogram.field,
fieldValue: selectedHistogram.value,
}}
onFilter={onClose}
/>
<div data-test-subj="apmCorrelationsTable">
{histograms.length > 0 && selectedHistogram !== undefined && (
<CorrelationsTable
// @ts-ignore correlations don't have the same column format other tables have
columns={mlCorrelationColumns}
// @ts-expect-error correlations don't have the same significant term other tables have
significantTerms={histogramTerms}
status={FETCH_STATUS.SUCCESS}
setSelectedSignificantTerm={setSelectedSignificantTerm}
selectedTerm={{
fieldName: selectedHistogram.field,
fieldValue: selectedHistogram.value,
}}
onFilter={onClose}
/>
)}
{histograms.length < 1 && progress > 0.99 ? (
<>
<EuiSpacer size="m" />
<EuiText textAlign="center">
<FormattedMessage
id="xpack.apm.correlations.latencyCorrelations.noCorrelationsText"
defaultMessage="No significant correlations found"
/>
</EuiText>
</>
) : null}
</div>
{log.length > 0 && displayLog && (
<EuiAccordion
id="accordion1"
buttonContent={i18n.translate(
'xpack.apm.correlations.latencyCorrelations.logButtonContent',
{
defaultMessage: 'Log',
}
)}
>
<EuiPanel color="subdued">
{log.map((d, i) => {
const splitItem = d.split(': ');
return (
<p key={i}>
<small>
<EuiCode>{splitItem[0]}</EuiCode> {splitItem[1]}
</small>
</p>
);
})}
</EuiPanel>
</EuiAccordion>
)}
{histograms.length < 1 && progress > 0.99 ? (
<>
<EuiSpacer size="m" />
<EuiText textAlign="center">
<FormattedMessage
id="xpack.apm.correlations.latencyCorrelations.noCorrelationsText"
defaultMessage="No significant correlations found"
/>
</EuiText>
</>
) : null}
</>
);
}
4 changes: 3 additions & 1 deletion x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import { useKibana } from '../../../../../../../src/plugins/kibana_react/public'
import { ApmPluginStartDeps } from '../../../plugin';

interface CorrelationsOptions {
index: string;
environment?: string;
kuery?: string;
serviceName?: string;
Expand All @@ -37,6 +36,7 @@ interface RawResponse {
values: SearchServiceValue[];
overallHistogram: HistogramItem[];
log: string[];
ccsWarning: boolean;
}

export const useCorrelations = (params: CorrelationsOptions) => {
Expand Down Expand Up @@ -106,6 +106,8 @@ export const useCorrelations = (params: CorrelationsOptions) => {
};

return {
ccsWarning: rawResponse?.ccsWarning ?? false,
log: rawResponse?.log ?? [],
error,
histograms: rawResponse?.values ?? [],
percentileThresholdValue:
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ export function getServiceColumns({
)}
<EuiFlexItem className="apmServiceList__serviceNameContainer">
<AppLink
data-test-subj="apmServiceListAppLink"
serviceName={serviceName}
transactionType={transactionType}
className="eui-textTruncate"
Expand Down
4 changes: 3 additions & 1 deletion x-pack/plugins/apm/public/components/routing/templates/apm_service_template/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,9 @@ function TemplateWithContext({
<EuiFlexGroup alignItems="center">
<EuiFlexItem grow={false}>
<EuiTitle size="l">
<>{serviceName}</>
<h1 data-test-subj="apmMainTemplateHeaderServiceName">
{serviceName}
</h1>
</EuiTitle>
</EuiFlexItem>
<EuiFlexItem grow={false}>
Expand Down
29 changes: 26 additions & 3 deletions x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import { shuffle, range } from 'lodash';
import type { ElasticsearchClient } from 'src/core/server';
import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
import { fetchTransactionDurationFieldCandidates } from './query_field_candidates';
import { fetchTransactionDurationFieldValuePairs } from './query_field_value_pairs';
import { fetchTransactionDurationPercentiles } from './query_percentiles';
Expand All @@ -16,6 +17,7 @@ import { fetchTransactionDurationRanges, HistogramItem } from './query_ranges';
import type {
AsyncSearchProviderProgress,
SearchServiceParams,
SearchServiceFetchParams,
SearchServiceValue,
} from '../../../../common/search_strategies/correlations/types';
import { computeExpectationsAndRanges } from './utils/aggregation_utils';
Expand All @@ -28,11 +30,14 @@ const currentTimeAsString = () => new Date().toISOString();

export const asyncSearchServiceProvider = (
esClient: ElasticsearchClient,
params: SearchServiceParams
getApmIndices: () => Promise<ApmIndicesConfig>,
searchServiceParams: SearchServiceParams,
includeFrozen: boolean
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is includeFrozen being used here?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just re-read the post description and my question is redundant here :) Sorry the noise

) => {
let isCancelled = false;
let isRunning = true;
let error: Error;
let ccsWarning = false;
const log: string[] = [];
const logMessage = (message: string) =>
log.push(`${currentTimeAsString()}: ${message}`);
Expand Down Expand Up @@ -63,7 +68,15 @@ export const asyncSearchServiceProvider = (
};

const fetchCorrelations = async () => {
let params: SearchServiceFetchParams | undefined;

try {
const indices = await getApmIndices();
params = {
...searchServiceParams,
index: indices['apm_oss.transactionIndices'],
};

// 95th percentile to be displayed as a marker in the log log chart
const {
totalDocs,
Expand Down Expand Up @@ -172,7 +185,7 @@ export const asyncSearchServiceProvider = (

async function* fetchTransactionDurationHistograms() {
for (const item of shuffle(fieldValuePairs)) {
if (item === undefined || isCancelled) {
if (params === undefined || item === undefined || isCancelled) {
isRunning = false;
return;
}
Expand Down Expand Up @@ -222,10 +235,15 @@ export const asyncSearchServiceProvider = (
yield undefined;
}
} catch (e) {
// don't fail the whole process for individual correlation queries, just add the error to the internal log.
// don't fail the whole process for individual correlation queries,
// just add the error to the internal log and check if we'd want to set the
// cross-cluster search compatibility warning to true.
logMessage(
`Failed to fetch correlation/kstest for '${item.field}/${item.value}'`
);
if (params?.index.includes(':')) {
ccsWarning = true;
}
yield undefined;
}
}
Expand All @@ -247,6 +265,10 @@ export const asyncSearchServiceProvider = (
error = e;
}

if (error !== undefined && params?.index.includes(':')) {
ccsWarning = true;
}

isRunning = false;
};

Expand All @@ -256,6 +278,7 @@ export const asyncSearchServiceProvider = (
const sortedValues = values.sort((a, b) => b.correlation - a.correlation);

return {
ccsWarning,
error,
log,
isRunning,
Expand Down
Loading