From c449b2aea5e0c6996bfc4d21d800ad87a9ae44f6 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 14 Feb 2023 09:41:43 -0500 Subject: [PATCH] [8.7] [Fleet] Fix agent policy selection and creation when there are a large number of agent policies ( > 1k) (#151119) (#151131) # Backport This will backport the following commits from `main` to `8.7`: - [[Fleet] Fix agent policy selection and creation when there are a large number of agent policies ( > 1k) (#151119)](https://github.com/elastic/kibana/pull/151119) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Mark Hopkin --- .../plugins/fleet/common/openapi/bundled.json | 8 ++ .../plugins/fleet/common/openapi/bundled.yaml | 9 ++ .../common/openapi/paths/agent_policies.yaml | 5 + .../common/types/rest_spec/agent_policy.ts | 1 + .../steps/step_select_agent_policy.tsx | 100 +++++++++++------- .../components/steps/step_select_hosts.tsx | 3 +- .../server/routes/agent_policy/handlers.ts | 12 ++- .../server/types/rest_spec/agent_policy.ts | 1 + 8 files changed, 97 insertions(+), 42 deletions(-) diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index cd84715414a3b..b711393256d3b 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -2511,6 +2511,14 @@ "in": "query", "name": "full", "description": "When set to true, retrieve the related package policies for each agent policy." + }, + { + "schema": { + "type": "boolean" + }, + "in": "query", + "name": "noAgentCount", + "description": "When set to true, do not count how many agents are in the agent policy, this can improve performance if you are searching over a large number of agent policies. The \"agents\" property will always be 0 if set to true." } ], "description": "" diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index 7effd0216dd4c..b5f6de994f365 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -1551,6 +1551,15 @@ paths: description: >- When set to true, retrieve the related package policies for each agent policy. + - schema: + type: boolean + in: query + name: noAgentCount + description: >- + When set to true, do not count how many agents are in the agent + policy, this can improve performance if you are searching over a + large number of agent policies. The "agents" property will always be + 0 if set to true. description: '' post: summary: Agent policy - Create diff --git a/x-pack/plugins/fleet/common/openapi/paths/agent_policies.yaml b/x-pack/plugins/fleet/common/openapi/paths/agent_policies.yaml index d431024445656..eb701057c2953 100644 --- a/x-pack/plugins/fleet/common/openapi/paths/agent_policies.yaml +++ b/x-pack/plugins/fleet/common/openapi/paths/agent_policies.yaml @@ -34,6 +34,11 @@ get: in: query name: full description: When set to true, retrieve the related package policies for each agent policy. + - schema: + type: boolean + in: query + name: noAgentCount + description: When set to true, do not count how many agents are in the agent policy, this can improve performance if you are searching over a large number of agent policies. The "agents" property will always be 0 if set to true. description: '' post: diff --git a/x-pack/plugins/fleet/common/types/rest_spec/agent_policy.ts b/x-pack/plugins/fleet/common/types/rest_spec/agent_policy.ts index a62b1a1b9e4e6..738a8201b14a1 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/agent_policy.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/agent_policy.ts @@ -11,6 +11,7 @@ import type { ListResult, ListWithKuery, BulkGetResult } from './common'; export interface GetAgentPoliciesRequest { query: ListWithKuery & { + noAgentCount?: boolean; full?: boolean; }; } diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_select_agent_policy.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_select_agent_policy.tsx index cef02bfdc84da..718392ca8e2b1 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_select_agent_policy.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_select_agent_policy.tsx @@ -22,23 +22,20 @@ import { } from '@elastic/eui'; import { Error } from '../../../../../components'; -import type { - AgentPolicy, - Output, - PackageInfo, - GetAgentPoliciesResponseItem, -} from '../../../../../types'; +import type { AgentPolicy, Output, PackageInfo } from '../../../../../types'; import { isPackageLimited, doesAgentPolicyAlreadyIncludePackage } from '../../../../../services'; import { useGetAgentPolicies, useGetOutputs, sendGetOneAgentPolicy, useFleetStatus, + useGetPackagePolicies, } from '../../../../../hooks'; import { FLEET_APM_PACKAGE, SO_SEARCH_LIMIT, outputType, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, } from '../../../../../../../../common/constants'; const AgentPolicyFormRow = styled(EuiFormRow)` @@ -58,22 +55,36 @@ function useAgentPoliciesOptions(packageInfo?: PackageInfo) { perPage: SO_SEARCH_LIMIT, sortField: 'name', sortOrder: 'asc', - full: true, + noAgentCount: true, // agentPolicy.agents will always be 0 + full: false, // package_policies will always be empty }); const agentPolicies = useMemo( () => agentPoliciesData?.items.filter((policy) => !policy.is_managed) || [], [agentPoliciesData?.items] ); - const agentPoliciesById = useMemo(() => { - return agentPolicies.reduce((acc: { [key: string]: GetAgentPoliciesResponseItem }, policy) => { - acc[policy.id] = policy; - return acc; - }, {}); - }, [agentPolicies]); - const { data: outputsData, isLoading: isOutputLoading } = useGetOutputs(); + // get all package policies with apm integration or the current integration + const { data: packagePoliciesForThisPackage, isLoading: isLoadingPackagePolicies } = + useGetPackagePolicies({ + page: 1, + perPage: SO_SEARCH_LIMIT, + kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name: ${packageInfo?.name}`, + }); + + const packagePoliciesForThisPackageByAgentPolicyId = useMemo( + () => + packagePoliciesForThisPackage?.items.reduce( + (acc: { [key: string]: boolean }, packagePolicy) => { + acc[packagePolicy.policy_id] = true; + return acc; + }, + {} + ), + [packagePoliciesForThisPackage?.items] + ); + const { getDataOutputForPolicy } = useMemo(() => { const defaultOutput = (outputsData?.items ?? []).find((output) => output.is_default); const outputsById = (outputsData?.items ?? []).reduce( @@ -85,7 +96,7 @@ function useAgentPoliciesOptions(packageInfo?: PackageInfo) { ); return { - getDataOutputForPolicy: (policy: AgentPolicy) => { + getDataOutputForPolicy: (policy: Pick) => { return policy.data_output_id ? outputsById[policy.data_output_id] : defaultOutput; }, }; @@ -94,20 +105,19 @@ function useAgentPoliciesOptions(packageInfo?: PackageInfo) { const agentPolicyOptions: Array> = useMemo( () => packageInfo - ? agentPolicies.map((agentConf) => { - const isLimitedPackageAlreadyInPolicy = doesAgentPolicyHaveLimitedPackage( - agentConf, - packageInfo - ); + ? agentPolicies.map((policy) => { + const isLimitedPackageAlreadyInPolicy = + isPackageLimited(packageInfo) && + packagePoliciesForThisPackageByAgentPolicyId?.[policy.id]; const isAPMPackageAndDataOutputIsLogstash = - packageInfo.name === FLEET_APM_PACKAGE && - getDataOutputForPolicy(agentConf)?.type === outputType.Logstash; + packageInfo?.name === FLEET_APM_PACKAGE && + getDataOutputForPolicy(policy)?.type === outputType.Logstash; return { inputDisplay: ( <> - {agentConf.name} + {policy.name} {isAPMPackageAndDataOutputIsLogstash && ( <> @@ -121,21 +131,25 @@ function useAgentPoliciesOptions(packageInfo?: PackageInfo) { )} ), - value: agentConf.id, + value: policy.id, disabled: isLimitedPackageAlreadyInPolicy || isAPMPackageAndDataOutputIsLogstash, 'data-test-subj': 'agentPolicyItem', }; }) : [], - [agentPolicies, packageInfo, getDataOutputForPolicy] + [ + packageInfo, + agentPolicies, + packagePoliciesForThisPackageByAgentPolicyId, + getDataOutputForPolicy, + ] ); return { agentPoliciesError, - isLoading: isOutputLoading || isAgentPoliciesLoading, - agentPolicies, - agentPoliciesById, + isLoading: isOutputLoading || isAgentPoliciesLoading || isLoadingPackagePolicies, agentPolicyOptions, + agentPolicies, }; } @@ -154,7 +168,7 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{ }> = ({ packageInfo, agentPolicy, - updateAgentPolicy, + updateAgentPolicy: updateSelectedAgentPolicy, setHasAgentPolicyError, selectedAgentPolicyId, }) => { @@ -162,7 +176,7 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{ const [selectedAgentPolicyError, setSelectedAgentPolicyError] = useState(); - const { agentPolicies, agentPoliciesById, isLoading, agentPoliciesError, agentPolicyOptions } = + const { isLoading, agentPoliciesError, agentPolicyOptions, agentPolicies } = useAgentPoliciesOptions(packageInfo); // Selected agent policy state const [selectedPolicyId, setSelectedPolicyId] = useState( @@ -170,10 +184,23 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{ (selectedAgentPolicyId || (agentPolicies.length === 1 ? agentPolicies[0].id : undefined)) ); + const [isLoadingSelectedAgentPolicy, setIsLoadingSelectedAgentPolicy] = useState(false); + const [selectedAgentPolicy, setSelectedAgentPolicy] = useState( + agentPolicy + ); + + const updateAgentPolicy = useCallback( + (selectedPolicy: AgentPolicy | undefined) => { + setSelectedAgentPolicy(selectedPolicy); + updateSelectedAgentPolicy(selectedPolicy); + }, + [updateSelectedAgentPolicy] + ); // Update parent selected agent policy state useEffect(() => { const fetchAgentPolicyInfo = async () => { if (selectedPolicyId) { + setIsLoadingSelectedAgentPolicy(true); const { data, error } = await sendGetOneAgentPolicy(selectedPolicyId); if (error) { setSelectedAgentPolicyError(error); @@ -182,6 +209,7 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{ setSelectedAgentPolicyError(undefined); updateAgentPolicy(data.item); } + setIsLoadingSelectedAgentPolicy(false); } else { setSelectedAgentPolicyError(undefined); updateAgentPolicy(undefined); @@ -268,12 +296,12 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{ } helpText={ - isFleetReady && selectedPolicyId ? ( + isFleetReady && selectedPolicyId && !isLoadingSelectedAgentPolicy ? ( ) : null @@ -281,10 +309,8 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{ isInvalid={Boolean( !selectedPolicyId || !packageInfo || - doesAgentPolicyHaveLimitedPackage( - agentPoliciesById[selectedPolicyId], - packageInfo - ) + (selectedAgentPolicy && + doesAgentPolicyHaveLimitedPackage(selectedAgentPolicy, packageInfo)) )} error={ !selectedPolicyId ? ( @@ -308,7 +334,7 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{ } )} fullWidth - isLoading={isLoading || !packageInfo} + isLoading={isLoading || !packageInfo || isLoadingSelectedAgentPolicy} options={agentPolicyOptions} valueOfSelected={selectedPolicyId} onChange={onChange} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_select_hosts.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_select_hosts.tsx index 992501e012a8b..87ef9ac864bfd 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_select_hosts.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_select_hosts.tsx @@ -64,7 +64,8 @@ export const StepSelectHosts: React.FunctionComponent = ({ perPage: SO_SEARCH_LIMIT, sortField: 'name', sortOrder: 'asc', - full: true, + full: false, // package_policies will always be empty + noAgentCount: true, // agentPolicy.agents will always be 0 }); if (err) { // eslint-disable-next-line no-console diff --git a/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts index 9da77a9bed210..505f5e1b89b0a 100644 --- a/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts @@ -75,7 +75,7 @@ export const getAgentPoliciesHandler: FleetRequestHandler< const fleetContext = await context.fleet; const soClient = fleetContext.internalSoClient; const esClient = coreContext.elasticsearch.client.asInternalUser; - const { full: withPackagePolicies = false, ...restOfQuery } = request.query; + const { full: withPackagePolicies = false, noAgentCount = false, ...restOfQuery } = request.query; try { const { items, total, page, perPage } = await agentPolicyService.list(soClient, { withPackagePolicies, @@ -88,9 +88,11 @@ export const getAgentPoliciesHandler: FleetRequestHandler< page, perPage, }; - - await populateAssignedAgentsCount(esClient, soClient, items); - + if (!noAgentCount) { + await populateAssignedAgentsCount(esClient, soClient, items); + } else { + items.forEach((item) => (item.agents = 0)); + } return response.ok({ body }); } catch (error) { return defaultFleetErrorHandler({ error, response }); @@ -136,10 +138,12 @@ export const getOneAgentPolicyHandler: RequestHandler< TypeOf > = async (context, request, response) => { const coreContext = await context.core; + const esClient = coreContext.elasticsearch.client.asInternalUser; const soClient = coreContext.savedObjects.client; try { const agentPolicy = await agentPolicyService.get(soClient, request.params.agentPolicyId); if (agentPolicy) { + await populateAssignedAgentsCount(esClient, soClient, [agentPolicy]); const body: GetOneAgentPolicyResponse = { item: agentPolicy, }; diff --git a/x-pack/plugins/fleet/server/types/rest_spec/agent_policy.ts b/x-pack/plugins/fleet/server/types/rest_spec/agent_policy.ts index c2599d79b3364..25275ed998c58 100644 --- a/x-pack/plugins/fleet/server/types/rest_spec/agent_policy.ts +++ b/x-pack/plugins/fleet/server/types/rest_spec/agent_policy.ts @@ -13,6 +13,7 @@ import { ListWithKuerySchema, BulkRequestBodySchema } from './common'; export const GetAgentPoliciesRequestSchema = { query: ListWithKuerySchema.extends({ + noAgentCount: schema.maybe(schema.boolean()), full: schema.maybe(schema.boolean()), }), };