From 92817469143a222e6d1be8b3369151d55c7fe47d Mon Sep 17 00:00:00 2001 From: Galen Kistler <109082771+gtk-grafana@users.noreply.github.com> Date: Tue, 8 Nov 2022 08:37:11 -0600 Subject: [PATCH] Prometheus: remove /series endpoint calls in query builder label names and values for supported clients (#58087) * add other filter variables to match param for label values query against filter values, in order to resolve bug in which filter value options would display that aren't relevant in the current query editor context, i.e. options would display that upon select would display no data * expanding current unit test coverage to cover calls to new API * interpolate the label name string instead of the match promql expression --- .../prometheus/language_provider.mock.ts | 1 + .../prometheus/language_provider.ts | 59 +++++++++++++- .../prometheus/metric_find_query.test.ts | 80 +++++++++++++++++-- .../querybuilder/components/MetricSelect.tsx | 10 +-- .../components/PromQueryBuilder.test.tsx | 56 ++++++++++++- .../components/PromQueryBuilder.tsx | 61 ++++++++++++-- 6 files changed, 245 insertions(+), 22 deletions(-) diff --git a/public/app/plugins/datasource/prometheus/language_provider.mock.ts b/public/app/plugins/datasource/prometheus/language_provider.mock.ts index 25924a65e2b6c..ed2944ba942d1 100644 --- a/public/app/plugins/datasource/prometheus/language_provider.mock.ts +++ b/public/app/plugins/datasource/prometheus/language_provider.mock.ts @@ -11,6 +11,7 @@ export class EmptyLanguageProviderMock { getSeries = jest.fn().mockReturnValue({ __name__: [] }); fetchSeries = jest.fn().mockReturnValue([]); fetchSeriesLabels = jest.fn().mockReturnValue([]); + fetchSeriesLabelsMatch = jest.fn().mockReturnValue([]); fetchLabels = jest.fn(); loadMetricsMetadata = jest.fn(); } diff --git a/public/app/plugins/datasource/prometheus/language_provider.ts b/public/app/plugins/datasource/prometheus/language_provider.ts index 8700b8909b421..d7a286fc6dbfe 100644 --- a/public/app/plugins/datasource/prometheus/language_provider.ts +++ b/public/app/plugins/datasource/prometheus/language_provider.ts @@ -1,4 +1,4 @@ -import { once, chain, difference } from 'lodash'; +import { chain, difference, once } from 'lodash'; import LRU from 'lru-cache'; import Prism from 'prismjs'; import { Value } from 'slate'; @@ -471,6 +471,10 @@ export default class PromQlLanguageProvider extends LanguageProvider { } } + /** + * @todo cache + * @param key + */ fetchLabelValues = async (key: string): Promise => { const params = this.datasource.getTimeRangeParams(); const url = `/api/v1/label/${this.datasource.interpolateString(key)}/values`; @@ -498,7 +502,22 @@ export default class PromQlLanguageProvider extends LanguageProvider { } /** - * Fetch labels for a series. This is cached by its args but also by the global timeRange currently selected as + * Fetches all values for a label, with optional match[] + * @param name + * @param match + */ + fetchSeriesValues = async (name: string, match?: string): Promise => { + const interpolatedName = name ? this.datasource.interpolateString(name) : null; + const range = this.datasource.getTimeRangeParams(); + const urlParams = { + ...range, + ...(interpolatedName && { 'match[]': match }), + }; + return await this.request(`/api/v1/label/${interpolatedName}/values`, [], urlParams); + }; + + /** + * Fetch labels for a series using /series endpoint. This is cached by its args but also by the global timeRange currently selected as * they can change over requested time. * @param name * @param withName @@ -533,6 +552,42 @@ export default class PromQlLanguageProvider extends LanguageProvider { return value; }; + /** + * Fetch labels for a series using /labels endpoint. This is cached by its args but also by the global timeRange currently selected as + * they can change over requested time. + * @param name + * @param withName + */ + fetchSeriesLabelsMatch = async (name: string, withName?: boolean): Promise> => { + const interpolatedName = this.datasource.interpolateString(name); + const range = this.datasource.getTimeRangeParams(); + const urlParams = { + ...range, + 'match[]': interpolatedName, + }; + const url = `/api/v1/labels`; + // Cache key is a bit different here. We add the `withName` param and also round up to a minute the intervals. + // The rounding may seem strange but makes relative intervals like now-1h less prone to need separate request every + // millisecond while still actually getting all the keys for the correct interval. This still can create problems + // when user does not the newest values for a minute if already cached. + const cacheParams = new URLSearchParams({ + 'match[]': interpolatedName, + start: roundSecToMin(parseInt(range.start, 10)).toString(), + end: roundSecToMin(parseInt(range.end, 10)).toString(), + withName: withName ? 'true' : 'false', + }); + + const cacheKey = `${url}?${cacheParams.toString()}`; + let value = this.labelsCache.get(cacheKey); + if (!value) { + const data: string[] = await this.request(url, [], urlParams); + // Convert string array to Record + value = data.reduce((ac, a) => ({ ...ac, [a]: '' }), {}); + this.labelsCache.set(cacheKey, value); + } + return value; + }; + /** * Fetch series for a selector. Use this for raw results. Use fetchSeriesLabels() to get labels. * @param match diff --git a/public/app/plugins/datasource/prometheus/metric_find_query.test.ts b/public/app/plugins/datasource/prometheus/metric_find_query.test.ts index 6becb8d7cb9d1..e2adf5530b7f7 100644 --- a/public/app/plugins/datasource/prometheus/metric_find_query.test.ts +++ b/public/app/plugins/datasource/prometheus/metric_find_query.test.ts @@ -5,6 +5,8 @@ import { DataSourceInstanceSettings, toUtc } from '@grafana/data'; import { FetchResponse } from '@grafana/runtime'; import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__ +import { PromApplication } from '../../../types/unified-alerting-dto'; + import { PrometheusDatasource } from './datasource'; import PrometheusMetricFindQuery from './metric_find_query'; import { PromOptions } from './types'; @@ -23,7 +25,7 @@ const instanceSettings = { user: 'test', password: 'mupp', jsonData: { httpMethod: 'GET' }, -} as unknown as DataSourceInstanceSettings; +} as Partial> as DataSourceInstanceSettings; const raw = { from: toUtc('2018-04-25 10:00'), to: toUtc('2018-04-25 11:00'), @@ -52,14 +54,22 @@ beforeEach(() => { }); describe('PrometheusMetricFindQuery', () => { - let ds: PrometheusDatasource; + let legacyPrometheusDatasource: PrometheusDatasource; + let prometheusDatasource: PrometheusDatasource; beforeEach(() => { - ds = new PrometheusDatasource(instanceSettings, templateSrvStub); + legacyPrometheusDatasource = new PrometheusDatasource(instanceSettings, templateSrvStub); + prometheusDatasource = new PrometheusDatasource( + { + ...instanceSettings, + jsonData: { ...instanceSettings.jsonData, prometheusVersion: '2.2.0', prometheusType: PromApplication.Mimir }, + }, + templateSrvStub + ); }); - const setupMetricFindQuery = (data: any) => { + const setupMetricFindQuery = (data: any, datasource?: PrometheusDatasource) => { fetchMock.mockImplementation(() => of({ status: 'success', data: data.response } as unknown as FetchResponse)); - return new PrometheusMetricFindQuery(ds, data.query); + return new PrometheusMetricFindQuery(datasource ?? legacyPrometheusDatasource, data.query); }; describe('When performing metricFindQuery', () => { @@ -102,6 +112,7 @@ describe('PrometheusMetricFindQuery', () => { }); }); + // it('label_values(metric, resource) should generate series query with correct time', async () => { const query = setupMetricFindQuery({ query: 'label_values(metric, resource)', @@ -179,6 +190,7 @@ describe('PrometheusMetricFindQuery', () => { headers: {}, }); }); + // it('metrics(metric.*) should generate metric name query', async () => { const query = setupMetricFindQuery({ @@ -277,5 +289,63 @@ describe('PrometheusMetricFindQuery', () => { headers: {}, }); }); + + // + it('label_values(metric, resource) should generate label values query with correct time', async () => { + const metricName = 'metricName'; + const resourceName = 'resourceName'; + const query = setupMetricFindQuery( + { + query: `label_values(${metricName}, ${resourceName})`, + response: { + data: [ + { __name__: `${metricName}`, resourceName: 'value1' }, + { __name__: `${metricName}`, resourceName: 'value2' }, + { __name__: `${metricName}`, resourceName: 'value3' }, + ], + }, + }, + prometheusDatasource + ); + const results = await query.process(); + + expect(results).toHaveLength(3); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/1/resources/api/v1/label/${resourceName}/values?match${encodeURIComponent( + '[]' + )}=${metricName}&start=${raw.from.unix()}&end=${raw.to.unix()}`, + hideFromInspector: true, + headers: {}, + }); + }); + + it('label_values(metric{label1="foo", label2="bar", label3="baz"}, resource) should generate label values query with correct time', async () => { + const metricName = 'metricName'; + const resourceName = 'resourceName'; + const label1Name = 'label1'; + const label1Value = 'label1Value'; + const query = setupMetricFindQuery( + { + query: `label_values(${metricName}{${label1Name}="${label1Value}"}, ${resourceName})`, + response: { + data: [{ __name__: metricName, resourceName: label1Value }], + }, + }, + prometheusDatasource + ); + const results = await query.process(); + + expect(results).toHaveLength(1); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/1/resources/api/v1/label/${resourceName}/values?match%5B%5D=${metricName}%7B${label1Name}%3D%22${label1Value}%22%7D&start=1524650400&end=1524654000`, + hideFromInspector: true, + headers: {}, + }); + }); + // }); }); diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx b/public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx index db1636ed70583..ce42fe854d475 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx @@ -22,7 +22,7 @@ export interface Props { labelsFilters: QueryBuilderLabelFilter[]; } -const MAX_NUMBER_OF_RESULTS = 1000; +export const PROMETHEUS_QUERY_BUILDER_MAX_RESULTS = 1000; export function MetricSelect({ datasource, query, onChange, onGetMetrics, labelsFilters }: Props) { const styles = useStyles2(getStyles); @@ -109,8 +109,8 @@ export function MetricSelect({ datasource, query, onChange, onGetMetrics, labels // Since some customers can have millions of metrics, whenever the user changes the autocomplete text we want to call the backend and request all metrics that match the current query string const results = datasource.metricFindQuery(formatKeyValueStringsForLabelValuesQuery(query, labelsFilters)); return results.then((results) => { - if (results.length > MAX_NUMBER_OF_RESULTS) { - results.splice(0, results.length - MAX_NUMBER_OF_RESULTS); + if (results.length > PROMETHEUS_QUERY_BUILDER_MAX_RESULTS) { + results.splice(0, results.length - PROMETHEUS_QUERY_BUILDER_MAX_RESULTS); } return results.map((result) => { return { @@ -137,8 +137,8 @@ export function MetricSelect({ datasource, query, onChange, onGetMetrics, labels onOpenMenu={async () => { setState({ isLoading: true }); const metrics = await onGetMetrics(); - if (metrics.length > MAX_NUMBER_OF_RESULTS) { - metrics.splice(0, metrics.length - MAX_NUMBER_OF_RESULTS); + if (metrics.length > PROMETHEUS_QUERY_BUILDER_MAX_RESULTS) { + metrics.splice(0, metrics.length - PROMETHEUS_QUERY_BUILDER_MAX_RESULTS); } setState({ metrics, isLoading: undefined }); }} diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.test.tsx b/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.test.tsx index 855aa243ff881..2d006875657d5 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.test.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.test.tsx @@ -11,9 +11,11 @@ import { TimeRange, } from '@grafana/data'; +import { PromApplication } from '../../../../../types/unified-alerting-dto'; import { PrometheusDatasource } from '../../datasource'; import PromQlLanguageProvider from '../../language_provider'; import { EmptyLanguageProviderMock } from '../../language_provider.mock'; +import { PromOptions } from '../../types'; import { getLabelSelects } from '../testUtils'; import { PromVisualQuery } from '../types'; @@ -101,6 +103,7 @@ describe('PromQueryBuilder', () => { await waitFor(() => expect(datasource.getVariables).toBeCalled()); }); + // it('tries to load labels when metric selected', async () => { const { languageProvider } = setup(); await openLabelNameSelect(); @@ -127,6 +130,7 @@ describe('PromQueryBuilder', () => { expect(languageProvider.fetchSeriesLabels).toBeCalledWith('{label_name="label_value", __name__="random_metric"}') ); }); + // it('tries to load labels when metric is not selected', async () => { const { languageProvider } = setup({ @@ -224,16 +228,56 @@ describe('PromQueryBuilder', () => { ); expect(await screen.queryByText(EXPLAIN_LABEL_FILTER_CONTENT)).not.toBeInTheDocument(); }); + + // + it('tries to load labels when metric selected modern prom', async () => { + const { languageProvider } = setup(undefined, undefined, { + jsonData: { prometheusVersion: '2.38.1', prometheusType: PromApplication.Prometheus }, + }); + await openLabelNameSelect(); + await waitFor(() => expect(languageProvider.fetchSeriesLabelsMatch).toBeCalledWith('{__name__="random_metric"}')); + }); + + it('tries to load variables in label field modern prom', async () => { + const { datasource } = setup(undefined, undefined, { + jsonData: { prometheusVersion: '2.38.1', prometheusType: PromApplication.Prometheus }, + }); + datasource.getVariables = jest.fn().mockReturnValue([]); + await openLabelNameSelect(); + await waitFor(() => expect(datasource.getVariables).toBeCalled()); + }); + + it('tries to load labels when metric selected and other labels are already present modern prom', async () => { + const { languageProvider } = setup( + { + ...defaultQuery, + labels: [ + { label: 'label_name', op: '=', value: 'label_value' }, + { label: 'foo', op: '=', value: 'bar' }, + ], + }, + undefined, + { jsonData: { prometheusVersion: '2.38.1', prometheusType: PromApplication.Prometheus } } + ); + await openLabelNameSelect(1); + await waitFor(() => + expect(languageProvider.fetchSeriesLabelsMatch).toBeCalledWith( + '{label_name="label_value", __name__="random_metric"}' + ) + ); + }); + // }); -function createDatasource() { +function createDatasource(options?: Partial>) { const languageProvider = new EmptyLanguageProviderMock() as unknown as PromQlLanguageProvider; const datasource = new PrometheusDatasource( { url: '', jsonData: {}, meta: {} as DataSourcePluginMeta, - } as DataSourceInstanceSettings, + ...options, + } as DataSourceInstanceSettings, undefined, undefined, languageProvider @@ -251,8 +295,12 @@ function createProps(datasource: PrometheusDatasource, data?: PanelData) { }; } -function setup(query: PromVisualQuery = defaultQuery, data?: PanelData) { - const { datasource, languageProvider } = createDatasource(); +function setup( + query: PromVisualQuery = defaultQuery, + data?: PanelData, + datasourceOptionsOverride?: Partial> +) { + const { datasource, languageProvider } = createDatasource(datasourceOptionsOverride); const props = createProps(datasource, data); const { container } = render(); return { languageProvider, datasource, container }; diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx b/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx index af28adbe0d217..f6e081cb0a013 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx @@ -53,6 +53,11 @@ export const PromQueryBuilder = React.memo((props) => { [datasource] ); + /** + * Function kicked off when user interacts with label in label filters. + * Formats a promQL expression and passes that off to helper functions depending on API support + * @param forLabel + */ const onGetLabelNames = async (forLabel: Partial): Promise> => { // If no metric we need to use a different method if (!query.metric) { @@ -64,7 +69,13 @@ export const PromQueryBuilder = React.memo((props) => { const labelsToConsider = query.labels.filter((x) => x !== forLabel); labelsToConsider.push({ label: '__name__', op: '=', value: query.metric }); const expr = promQueryModeller.renderLabels(labelsToConsider); - const labelsIndex = await datasource.languageProvider.fetchSeriesLabels(expr); + + let labelsIndex; + if (datasource.hasLabelsMatchAPISupport()) { + labelsIndex = await datasource.languageProvider.fetchSeriesLabelsMatch(expr); + } else { + labelsIndex = await datasource.languageProvider.fetchSeriesLabels(expr); + } // filter out already used labels return Object.keys(labelsIndex) @@ -72,12 +83,47 @@ export const PromQueryBuilder = React.memo((props) => { .map((k) => ({ value: k })); }; - const onGetLabelValues = async (forLabel: Partial) => { + /** + * Helper function to fetch and format label value results from legacy API + * @param forLabel + * @param promQLExpression + */ + const getLabelValuesFromSeriesAPI = async (forLabel: Partial, promQLExpression: string) => { if (!forLabel.label) { return []; } + const result = await datasource.languageProvider.fetchSeriesLabels(promQLExpression); + const forLabelInterpolated = datasource.interpolateString(forLabel.label); + return result[forLabelInterpolated].map((v) => ({ value: v })) ?? []; + }; - // If no metric we need to use a different method + /** + * Helper function to fetch label values from a promql string expression and a label + * @param forLabel + * @param promQLExpression + */ + const getLabelValuesFromLabelValuesAPI = async ( + forLabel: Partial, + promQLExpression: string + ) => { + if (!forLabel.label) { + return []; + } + return (await datasource.languageProvider.fetchSeriesValues(forLabel.label, promQLExpression)).map((v) => ({ + value: v, + })); + }; + + /** + * Function kicked off when users interact with the value of the label filters + * Formats a promQL expression and passes that into helper functions depending on API support + * @param forLabel + */ + const onGetLabelValues = async (forLabel: Partial) => { + if (!forLabel.label) { + return []; + } + // If no metric is selected, we can get the raw list of labels if (!query.metric) { return (await datasource.languageProvider.getLabelValues(forLabel.label)).map((v) => ({ value: v })); } @@ -85,9 +131,12 @@ export const PromQueryBuilder = React.memo((props) => { const labelsToConsider = query.labels.filter((x) => x !== forLabel); labelsToConsider.push({ label: '__name__', op: '=', value: query.metric }); const expr = promQueryModeller.renderLabels(labelsToConsider); - const result = await datasource.languageProvider.fetchSeriesLabels(expr); - const forLabelInterpolated = datasource.interpolateString(forLabel.label); - return result[forLabelInterpolated].map((v) => ({ value: v })) ?? []; + + if (datasource.hasLabelsMatchAPISupport()) { + return getLabelValuesFromLabelValuesAPI(forLabel, expr); + } else { + return getLabelValuesFromSeriesAPI(forLabel, expr); + } }; const onGetMetrics = useCallback(() => {