[8.7] [Fleet] Fix agent policy selection and creation when there are …

…a large number of agent policies ( > 1k) (#151119) (#151131) # Backport This will backport the following commits from `main` to `8.7`: - [[Fleet] Fix agent policy selection and creation when there are a large number of agent policies ( > 1k) (#151119)](#151119)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Mark Hopkin <[email protected]>
elastic · Feb 14, 2023 · c449b2a · c449b2a
1 parent d335ebd
commit c449b2a
Show file tree

Hide file tree

Showing 8 changed files with 97 additions and 42 deletions.
diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json
@@ -2511,6 +2511,14 @@
             "in": "query",
             "name": "full",
             "description": "When set to true, retrieve the related package policies for each agent policy."
+          },
+          {
+            "schema": {
+              "type": "boolean"
+            },
+            "in": "query",
+            "name": "noAgentCount",
+            "description": "When set to true, do not count how many agents are in the agent policy, this can improve performance if you are searching over a large number of agent policies. The \"agents\" property will always be 0 if set to true."
           }
         ],
         "description": ""

diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml
@@ -1551,6 +1551,15 @@ paths:
           description: >-
             When set to true, retrieve the related package policies for each
             agent policy.
+        - schema:
+            type: boolean
+          in: query
+          name: noAgentCount
+          description: >-
+            When set to true, do not count how many agents are in the agent
+            policy, this can improve performance if you are searching over a
+            large number of agent policies. The "agents" property will always be
+            0 if set to true.
       description: ''
     post:
       summary: Agent policy - Create

diff --git a/x-pack/plugins/fleet/common/openapi/paths/agent_policies.yaml b/x-pack/plugins/fleet/common/openapi/paths/agent_policies.yaml
@@ -34,6 +34,11 @@ get:
       in: query
       name: full
       description: When set to true, retrieve the related package policies for each agent policy.
+    - schema:
+        type: boolean
+      in: query
+      name: noAgentCount
+      description: When set to true, do not count how many agents are in the agent policy, this can improve performance if you are searching over a large number of agent policies. The "agents" property will always be 0 if set to true. 
 
   description: ''
 post:

diff --git a/x-pack/plugins/fleet/common/types/rest_spec/agent_policy.ts b/x-pack/plugins/fleet/common/types/rest_spec/agent_policy.ts
@@ -11,6 +11,7 @@ import type { ListResult, ListWithKuery, BulkGetResult } from './common';
 
 export interface GetAgentPoliciesRequest {
   query: ListWithKuery & {
+    noAgentCount?: boolean;
     full?: boolean;
   };
 }

diff --git a/...ons/agent_policy/create_package_policy_page/components/steps/step_select_agent_policy.tsx b/...ons/agent_policy/create_package_policy_page/components/steps/step_select_agent_policy.tsx
@@ -22,23 +22,20 @@ import {
 } from '@elastic/eui';
 
 import { Error } from '../../../../../components';
-import type {
-  AgentPolicy,
-  Output,
-  PackageInfo,
-  GetAgentPoliciesResponseItem,
-} from '../../../../../types';
+import type { AgentPolicy, Output, PackageInfo } from '../../../../../types';
 import { isPackageLimited, doesAgentPolicyAlreadyIncludePackage } from '../../../../../services';
 import {
   useGetAgentPolicies,
   useGetOutputs,
   sendGetOneAgentPolicy,
   useFleetStatus,
+  useGetPackagePolicies,
 } from '../../../../../hooks';
 import {
   FLEET_APM_PACKAGE,
   SO_SEARCH_LIMIT,
   outputType,
+  PACKAGE_POLICY_SAVED_OBJECT_TYPE,
 } from '../../../../../../../../common/constants';
 
 const AgentPolicyFormRow = styled(EuiFormRow)`
@@ -58,22 +55,36 @@ function useAgentPoliciesOptions(packageInfo?: PackageInfo) {
     perPage: SO_SEARCH_LIMIT,
     sortField: 'name',
     sortOrder: 'asc',
-    full: true,
+    noAgentCount: true, // agentPolicy.agents will always be 0
+    full: false, // package_policies will always be empty
   });
   const agentPolicies = useMemo(
     () => agentPoliciesData?.items.filter((policy) => !policy.is_managed) || [],
     [agentPoliciesData?.items]
   );
 
-  const agentPoliciesById = useMemo(() => {
-    return agentPolicies.reduce((acc: { [key: string]: GetAgentPoliciesResponseItem }, policy) => {
-      acc[policy.id] = policy;
-      return acc;
-    }, {});
-  }, [agentPolicies]);
-
   const { data: outputsData, isLoading: isOutputLoading } = useGetOutputs();
 
+  // get all package policies with apm integration or the current integration
+  const { data: packagePoliciesForThisPackage, isLoading: isLoadingPackagePolicies } =
+    useGetPackagePolicies({
+      page: 1,
+      perPage: SO_SEARCH_LIMIT,
+      kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name: ${packageInfo?.name}`,
+    });
+
+  const packagePoliciesForThisPackageByAgentPolicyId = useMemo(
+    () =>
+      packagePoliciesForThisPackage?.items.reduce(
+        (acc: { [key: string]: boolean }, packagePolicy) => {
+          acc[packagePolicy.policy_id] = true;
+          return acc;
+        },
+        {}
+      ),
+    [packagePoliciesForThisPackage?.items]
+  );
+
   const { getDataOutputForPolicy } = useMemo(() => {
     const defaultOutput = (outputsData?.items ?? []).find((output) => output.is_default);
     const outputsById = (outputsData?.items ?? []).reduce(
@@ -85,7 +96,7 @@ function useAgentPoliciesOptions(packageInfo?: PackageInfo) {
     );
 
     return {
-      getDataOutputForPolicy: (policy: AgentPolicy) => {
+      getDataOutputForPolicy: (policy: Pick<AgentPolicy, 'data_output_id'>) => {
         return policy.data_output_id ? outputsById[policy.data_output_id] : defaultOutput;
       },
     };
@@ -94,20 +105,19 @@ function useAgentPoliciesOptions(packageInfo?: PackageInfo) {
   const agentPolicyOptions: Array<EuiSuperSelectOption<string>> = useMemo(
     () =>
       packageInfo
-        ? agentPolicies.map((agentConf) => {
-            const isLimitedPackageAlreadyInPolicy = doesAgentPolicyHaveLimitedPackage(
-              agentConf,
-              packageInfo
-            );
+        ? agentPolicies.map((policy) => {
+            const isLimitedPackageAlreadyInPolicy =
+              isPackageLimited(packageInfo) &&
+              packagePoliciesForThisPackageByAgentPolicyId?.[policy.id];
 
             const isAPMPackageAndDataOutputIsLogstash =
-              packageInfo.name === FLEET_APM_PACKAGE &&
-              getDataOutputForPolicy(agentConf)?.type === outputType.Logstash;
+              packageInfo?.name === FLEET_APM_PACKAGE &&
+              getDataOutputForPolicy(policy)?.type === outputType.Logstash;
 
             return {
               inputDisplay: (
                 <>
-                  <EuiText size="s">{agentConf.name}</EuiText>
+                  <EuiText size="s">{policy.name}</EuiText>
                   {isAPMPackageAndDataOutputIsLogstash && (
                     <>
                       <EuiSpacer size="xs" />
@@ -121,21 +131,25 @@ function useAgentPoliciesOptions(packageInfo?: PackageInfo) {
                   )}
                 </>
               ),
-              value: agentConf.id,
+              value: policy.id,
               disabled: isLimitedPackageAlreadyInPolicy || isAPMPackageAndDataOutputIsLogstash,
               'data-test-subj': 'agentPolicyItem',
             };
           })
         : [],
-    [agentPolicies, packageInfo, getDataOutputForPolicy]
+    [
+      packageInfo,
+      agentPolicies,
+      packagePoliciesForThisPackageByAgentPolicyId,
+      getDataOutputForPolicy,
+    ]
   );
 
   return {
     agentPoliciesError,
-    isLoading: isOutputLoading || isAgentPoliciesLoading,
-    agentPolicies,
-    agentPoliciesById,
+    isLoading: isOutputLoading || isAgentPoliciesLoading || isLoadingPackagePolicies,
     agentPolicyOptions,
+    agentPolicies,
   };
 }
 
@@ -154,26 +168,39 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{
 }> = ({
   packageInfo,
   agentPolicy,
-  updateAgentPolicy,
+  updateAgentPolicy: updateSelectedAgentPolicy,
   setHasAgentPolicyError,
   selectedAgentPolicyId,
 }) => {
   const { isReady: isFleetReady } = useFleetStatus();
 
   const [selectedAgentPolicyError, setSelectedAgentPolicyError] = useState<Error>();
 
-  const { agentPolicies, agentPoliciesById, isLoading, agentPoliciesError, agentPolicyOptions } =
+  const { isLoading, agentPoliciesError, agentPolicyOptions, agentPolicies } =
     useAgentPoliciesOptions(packageInfo);
   // Selected agent policy state
   const [selectedPolicyId, setSelectedPolicyId] = useState<string | undefined>(
     agentPolicy?.id ??
       (selectedAgentPolicyId || (agentPolicies.length === 1 ? agentPolicies[0].id : undefined))
   );
 
+  const [isLoadingSelectedAgentPolicy, setIsLoadingSelectedAgentPolicy] = useState<boolean>(false);
+  const [selectedAgentPolicy, setSelectedAgentPolicy] = useState<AgentPolicy | undefined>(
+    agentPolicy
+  );
+
+  const updateAgentPolicy = useCallback(
+    (selectedPolicy: AgentPolicy | undefined) => {
+      setSelectedAgentPolicy(selectedPolicy);
+      updateSelectedAgentPolicy(selectedPolicy);
+    },
+    [updateSelectedAgentPolicy]
+  );
   // Update parent selected agent policy state
   useEffect(() => {
     const fetchAgentPolicyInfo = async () => {
       if (selectedPolicyId) {
+        setIsLoadingSelectedAgentPolicy(true);
         const { data, error } = await sendGetOneAgentPolicy(selectedPolicyId);
         if (error) {
           setSelectedAgentPolicyError(error);
@@ -182,6 +209,7 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{
           setSelectedAgentPolicyError(undefined);
           updateAgentPolicy(data.item);
         }
+        setIsLoadingSelectedAgentPolicy(false);
       } else {
         setSelectedAgentPolicyError(undefined);
         updateAgentPolicy(undefined);
@@ -268,23 +296,21 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{
                 </EuiFlexGroup>
               }
               helpText={
-                isFleetReady && selectedPolicyId ? (
+                isFleetReady && selectedPolicyId && !isLoadingSelectedAgentPolicy ? (
                   <FormattedMessage
                     id="xpack.fleet.createPackagePolicy.StepSelectPolicy.agentPolicyAgentsDescriptionText"
                     defaultMessage="{count, plural, one {# agent is} other {# agents are}} enrolled with the selected agent policy."
                     values={{
-                      count: agentPoliciesById[selectedPolicyId]?.agents ?? 0,
+                      count: selectedAgentPolicy?.agents ?? 0,
                     }}
                   />
                 ) : null
               }
               isInvalid={Boolean(
                 !selectedPolicyId ||
                   !packageInfo ||
-                  doesAgentPolicyHaveLimitedPackage(
-                    agentPoliciesById[selectedPolicyId],
-                    packageInfo
-                  )
+                  (selectedAgentPolicy &&
+                    doesAgentPolicyHaveLimitedPackage(selectedAgentPolicy, packageInfo))
               )}
               error={
                 !selectedPolicyId ? (
@@ -308,7 +334,7 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{
                   }
                 )}
                 fullWidth
-                isLoading={isLoading || !packageInfo}
+                isLoading={isLoading || !packageInfo || isLoadingSelectedAgentPolicy}
                 options={agentPolicyOptions}
                 valueOfSelected={selectedPolicyId}
                 onChange={onChange}

diff --git a/...t/sections/agent_policy/create_package_policy_page/components/steps/step_select_hosts.tsx b/...t/sections/agent_policy/create_package_policy_page/components/steps/step_select_hosts.tsx
@@ -64,7 +64,8 @@ export const StepSelectHosts: React.FunctionComponent<Props> = ({
     perPage: SO_SEARCH_LIMIT,
     sortField: 'name',
     sortOrder: 'asc',
-    full: true,
+    full: false, // package_policies will always be empty
+    noAgentCount: true, // agentPolicy.agents will always be 0
   });
   if (err) {
     // eslint-disable-next-line no-console

diff --git a/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts
@@ -75,7 +75,7 @@ export const getAgentPoliciesHandler: FleetRequestHandler<
   const fleetContext = await context.fleet;
   const soClient = fleetContext.internalSoClient;
   const esClient = coreContext.elasticsearch.client.asInternalUser;
-  const { full: withPackagePolicies = false, ...restOfQuery } = request.query;
+  const { full: withPackagePolicies = false, noAgentCount = false, ...restOfQuery } = request.query;
   try {
     const { items, total, page, perPage } = await agentPolicyService.list(soClient, {
       withPackagePolicies,
@@ -88,9 +88,11 @@ export const getAgentPoliciesHandler: FleetRequestHandler<
       page,
       perPage,
     };
-
-    await populateAssignedAgentsCount(esClient, soClient, items);
-
+    if (!noAgentCount) {
+      await populateAssignedAgentsCount(esClient, soClient, items);
+    } else {
+      items.forEach((item) => (item.agents = 0));
+    }
     return response.ok({ body });
   } catch (error) {
     return defaultFleetErrorHandler({ error, response });
@@ -136,10 +138,12 @@ export const getOneAgentPolicyHandler: RequestHandler<
   TypeOf<typeof GetOneAgentPolicyRequestSchema.params>
 > = async (context, request, response) => {
   const coreContext = await context.core;
+  const esClient = coreContext.elasticsearch.client.asInternalUser;
   const soClient = coreContext.savedObjects.client;
   try {
     const agentPolicy = await agentPolicyService.get(soClient, request.params.agentPolicyId);
     if (agentPolicy) {
+      await populateAssignedAgentsCount(esClient, soClient, [agentPolicy]);
       const body: GetOneAgentPolicyResponse = {
         item: agentPolicy,
       };

diff --git a/x-pack/plugins/fleet/server/types/rest_spec/agent_policy.ts b/x-pack/plugins/fleet/server/types/rest_spec/agent_policy.ts
@@ -13,6 +13,7 @@ import { ListWithKuerySchema, BulkRequestBodySchema } from './common';
 
 export const GetAgentPoliciesRequestSchema = {
   query: ListWithKuerySchema.extends({
+    noAgentCount: schema.maybe(schema.boolean()),
     full: schema.maybe(schema.boolean()),
   }),
 };