Merge pull request #157 from JupiterOne/INT-7921-ingest-applications

INT-7921: ingest applications

docs/jupiterone.md

@@ -102,6 +102,7 @@ The following entities are created:
 | Account               | `crowdstrike_account`               | `Account`       |
 | Application           | `crowdstrike_detected_application`  | `Application`   |
 | Device Sensor Agent   | `crowdstrike_sensor`                | `HostAgent`     |
+| Discover Application  | `crowdstrike_discover_application`  | `Application`   |
 | Prevention Policy     | `crowdstrike_prevention_policy`     | `ControlPolicy` |
 | Service               | `crowdstrike_endpoint_protection`   | `Service`       |
 | Vulnerability         | `crowdstrike_vulnerability`         | `Finding`       |
@@ -118,6 +119,7 @@ The following relationships are created:
 | `crowdstrike_detected_application` | **HAS**               | `crowdstrike_vulnerability`         |
 | `crowdstrike_prevention_policy`    | **ENFORCES**          | `crowdstrike_endpoint_protection`   |
 | `crowdstrike_sensor`               | **ASSIGNED**          | `crowdstrike_prevention_policy`     |
+| `crowdstrike_sensor`               | **HAS**               | `crowdstrike_discover_application`  |
 | `crowdstrike_sensor`               | **HAS**               | `crowdstrike_zero_trust_assessment` |
 | `crowdstrike_vulnerability`        | **EXPLOITS**          | `crowdstrike_sensor`                |
 

src/crowdstrike/CrowdStrikeApiGateway.ts

@@ -14,7 +14,7 @@ import {
   RateLimitState,
   ResourcesResponse,
 } from './types';
-import { RequestInit } from 'node-fetch';
+import fetch, { RequestInit } from 'node-fetch';
 import { FalconAPIResourceIterationCallback } from './FalconAPIClient';
 import { ICrowdStrikeApiClientQueryBuilder } from './CrowdStrikeApiClientQueryBuilder';
 import { Total } from './Total';
@@ -24,7 +24,7 @@ function getUnixTimeNow() {
   return Date.now() / 1000;
 }
 
-async function sleep(ms) {
+async function sleep(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
@@ -56,13 +56,13 @@ export class CrowdStrikeApiGateway {
   private readonly rateLimitConfig: RateLimitConfig = DEFAULT_RATE_LIMIT_CONFIG;
   private total: Total;
   private queryBuilder: ICrowdStrikeApiClientQueryBuilder;
-  private fetcher;
+  private fetcher: typeof fetch;
 
   constructor(
     credentials: OAuth2ClientCredentials,
     logger: IntegrationLogger,
     queryBuilder: ICrowdStrikeApiClientQueryBuilder,
-    fetcher,
+    fetcher: typeof fetch,
     attemptOptions?: AttemptOptions,
   ) {
     this.queryBuilder = queryBuilder;
@@ -361,8 +361,11 @@ export class CrowdStrikeApiGateway {
         'pagination response details',
       );
 
-      paginationParams = response.meta.pagination as PaginationMeta;
       seen += response.resources.length;
+      paginationParams = {
+        ...response.meta.pagination,
+        offset: seen,
+      } as PaginationMeta;
 
       const baseUrl = this.queryBuilder.buildResourcePathUrl(
         availabilityZone,

src/crowdstrike/FalconAPIClient.ts

@@ -1,6 +1,8 @@
 import { URLSearchParams } from 'url';
 
 import {
+  DiscoverApplication,
+  DiscoverApplicationIdentifier,
   Device,
   DeviceIdentifier,
   OAuth2Token,
@@ -139,6 +141,31 @@ export class FalconAPIClient {
       resourcePath: '/zero-trust-assessment/queries/assessments/v1',
     });
   }
+
+  /**
+   * Iterates the known device applications, providing pages
+   * of the collection based on the provided query to the provided callback.
+   *
+   * @param input
+   * @returns Promise
+   */
+  public async iterateApplications(input: {
+    callback: FalconAPIResourceIterationCallback<DiscoverApplication>;
+    query?: QueryParams;
+  }): Promise<void> {
+    return this.crowdStrikeApiGateway.paginateResources<DiscoverApplicationIdentifier>(
+      {
+        callback: async (appsIds) => {
+          if (appsIds.length) {
+            return input.callback(await this.fetchApplications(appsIds));
+          }
+        },
+        query: input.query,
+        resourcePath: '/discover/queries/applications/v1',
+      },
+    );
+  }
+
   /**
    * Iterates prevention policies using the "combined" API, providing pages of
    * the collection to the provided callback.
@@ -195,6 +222,7 @@ export class FalconAPIClient {
 
     return response.resources;
   }
+
   /***
    * ZTA details
    * https://falconpy.io/Service-Collections/Zero-Trust-Assessment.html#getassessmentv1
@@ -204,11 +232,11 @@ export class FalconAPIClient {
     for (const id of ids) {
       searchParams.append('ids', id);
     }
-
     const availabilityZone = this.crowdStrikeApiGateway.getAvailabilityZone();
+
     const response =
       await this.crowdStrikeApiGateway.executeAPIRequestWithRetries<
-        ResourcesResponse<any>
+        ResourcesResponse<ZeroTrustAssessment>
       >(
         `https://api.${availabilityZone}crowdstrike.com/zero-trust-assessment/entities/assessments/v1?` +
           searchParams,
@@ -222,4 +250,29 @@ export class FalconAPIClient {
 
     return response.resources;
   }
+
+  /**
+   * Discover Service - applications.
+   * Swagger: https://assets.falcon.us-2.crowdstrike.com/support/api/swagger-us2.html#/discover/get-applications
+   */
+  private async fetchApplications(
+    ids: string[],
+  ): Promise<DiscoverApplication[]> {
+    const availabilityZone = this.crowdStrikeApiGateway.getAvailabilityZone();
+    const queryParams = ids.map((id) => `ids=${id}`).join('&');
+    const response =
+      await this.crowdStrikeApiGateway.executeAPIRequestWithRetries<
+        ResourcesResponse<DiscoverApplication>
+      >(
+        `https://api.${availabilityZone}crowdstrike.com/discover/entities/applications/v1?${queryParams}`,
+        {
+          method: 'GET',
+          headers: {
+            'Content-Type': 'application/json',
+            accept: 'application/json',
+          },
+        },
+      );
+    return response.resources;
+  }
 }

src/crowdstrike/types.ts

@@ -136,6 +136,7 @@ export type ResourcesResponse<T> = {
  * The identifier of a discovered device within the CrowdStrike Falcon platform.
  */
 export type DeviceIdentifier = string;
+export type DiscoverApplicationIdentifier = string;
 
 export type Device = {
   device_id: DeviceIdentifier;
@@ -187,10 +188,10 @@ export type Vulnerability = {
     site_name: string;
     system_manufacturer: string;
   };
-  apps?: Application[];
+  apps?: DetectedApplication[];
 };
 
-export type Application = {
+export type DetectedApplication = {
   product_name_version: string;
   sub_status: string;
   remediation?: {
@@ -201,6 +202,49 @@ export type Application = {
   };
 };
 
+export interface DiscoverApplication {
+  id: DiscoverApplicationIdentifier;
+  category?: string;
+  cid: string;
+  name?: string;
+  vendor?: string;
+  version?: string;
+  name_vendor?: string;
+  name_vendor_version?: string;
+  versioning_scheme?: string;
+  architectures?: string[];
+  groups?: string[];
+  installation_paths?: string[];
+  installation_timestamp?: string;
+  first_seen_timestamp?: string;
+  last_updated_timestamp?: string;
+  last_used_timestamp?: string;
+  is_suspicious?: boolean;
+  is_normalized?: boolean;
+  host?: Host;
+  last_used_file_hash?: string;
+  last_used_file_name?: string;
+  last_used_user_name?: string;
+  last_used_user_sid?: string;
+}
+
+export interface Host {
+  id: string;
+  aid?: string;
+  country?: string;
+  platform_name?: string;
+  os_version?: string;
+  kernel_version?: string;
+  product_type_desc?: string;
+  system_manufacturer?: string;
+  agent_version?: string;
+  external_ip?: string;
+  hostname?: string;
+  current_mac_address?: string;
+  current_network_prefix?: string;
+  internet_exposure?: string;
+}
+
 export type ZTA_Score = {
   aid: string;
   score: number;

src/jupiterone/converters.test.ts

@@ -7,7 +7,7 @@ import {
 import { createMockExecutionContext } from '@jupiterone/integration-sdk-testing';
 
 import {
-  Application,
+  DetectedApplication,
   Device,
   PreventionPolicy,
   Vulnerability,
@@ -18,7 +18,7 @@ import {
   createPreventionPolicyEntity,
   createProtectionServiceEntity,
   createVulnerabilityEntity,
-  createApplicationEntity,
+  createDetectedApplicationEntity,
 } from './converters';
 
 describe('createAccountEntity', () => {
@@ -398,9 +398,9 @@ describe('createVulnerabilityEntity', () => {
   });
 });
 
-describe('createApplicationEntity', () => {
+describe('createDetectedApplicationEntity', () => {
   test('properties transferred', () => {
-    const source: Application = {
+    const source: DetectedApplication = {
       product_name_version: 'Windows Server 2016 1607',
       sub_status: 'open',
       remediation: {
@@ -411,7 +411,7 @@ describe('createApplicationEntity', () => {
       },
     };
 
-    expect(createApplicationEntity(source)).toEqual({
+    expect(createDetectedApplicationEntity(source)).toEqual({
       _class: ['Application'],
       _key: 'windows-server-2016-1607',
       _rawData: [

src/jupiterone/converters.ts

@@ -6,7 +6,8 @@ import {
 import { Entities } from '../steps/constants';
 
 import {
-  Application,
+  DiscoverApplication,
+  DetectedApplication,
   Device,
   PreventionPolicy,
   Vulnerability,
@@ -93,6 +94,10 @@ export function buildEc2InstanceArn(source: Device): string | undefined {
   return `arn:aws:ec2:${region}:${serviceProviderAccountId}:instance/${instanceId}`;
 }
 
+export function createSensorAgentKey(deviceId: string): string {
+  return deviceId;
+}
+
 export function createSensorAgentEntity(source: Device) {
   return createIntegrationEntity({
     entityData: {
@@ -106,7 +111,7 @@ export function createSensorAgentEntity(source: Device) {
         // version is upgraded.  This is listed in their API documentation
         // and it notes that this means that it's a valid case for a Host to
         // potentially have multiple sensors protecting it.
-        _key: source.device_id,
+        _key: createSensorAgentKey(source.device_id),
         name: source.hostname,
         function: ['anti-malware', 'activity-monitor'],
         firstSeenOn: parseTimePropertyValue(source.first_seen),
@@ -216,13 +221,13 @@ export function createVulnerabilityEntity(source: Vulnerability) {
   });
 }
 
-export function createApplicationEntity(source: Application) {
+export function createDetectedApplicationEntity(source: DetectedApplication) {
   return createIntegrationEntity({
     entityData: {
       source,
       assign: {
-        _class: Entities.APPLICATION._class,
-        _type: Entities.APPLICATION._type,
+        _class: Entities.DETECTED_APPLICATION._class,
+        _type: Entities.DETECTED_APPLICATION._type,
         _key: source.product_name_version.toLowerCase().replace(/\s/g, '-'),
         name: source.product_name_version,
         open: source.sub_status.toLowerCase() === 'open',
@@ -232,6 +237,33 @@ export function createApplicationEntity(source: Application) {
     },
   });
 }
+
+export function createDiscoverApplicationEntity(source: DiscoverApplication) {
+  return createIntegrationEntity({
+    entityData: {
+      source,
+      assign: {
+        _class: Entities.DISCOVER_APPLICATION._class,
+        _type: Entities.DISCOVER_APPLICATION._type,
+        _key: source.id,
+        name: source.name || source.id,
+        id: source.id,
+        vendor: source.vendor,
+        version: source.version,
+        versioningScheme: source.versioning_scheme,
+        architectures: source.architectures,
+        isSuspicious: source.is_suspicious,
+        isNormalized: source.is_normalized,
+        installationPaths: source.installation_paths,
+        installedOn: parseTimePropertyValue(source.installation_timestamp),
+        firstSeenOn: parseTimePropertyValue(source.first_seen_timestamp),
+        lastUpdatedOn: parseTimePropertyValue(source.last_updated_timestamp),
+        lastUsedOn: parseTimePropertyValue(source.last_used_timestamp),
+      },
+    },
+  });
+}
+
 export function createZeroTrustAssessmentEntity(source: ZeroTrustAssessment) {
   const unmet_os_signals = source.assessment_items.os_signals.filter(
     (s) => s.meets_criteria == 'no',